Refine seller store implementation
This commit is contained in:
+16
-1
@@ -20,7 +20,14 @@ from pydantic import BaseModel, Field
|
||||
|
||||
from api.proxy import guess_referer
|
||||
from config import get_settings
|
||||
from services.storage import ResourceSizeLimitError, download_bytes, local_path, read_local_bytes
|
||||
from services.storage import (
|
||||
ResourceDownloadTimeoutError,
|
||||
ResourceSizeLimitError,
|
||||
UnsafeRemoteUrlError,
|
||||
download_bytes,
|
||||
local_path,
|
||||
read_local_bytes,
|
||||
)
|
||||
|
||||
router = APIRouter(prefix="/api", tags=["export"])
|
||||
|
||||
@@ -111,8 +118,12 @@ async def export_images(req: ExportImagesRequest):
|
||||
for img in req.images:
|
||||
try:
|
||||
data, ctype = await _download(img.url)
|
||||
except UnsafeRemoteUrlError as exc:
|
||||
raise HTTPException(status_code=400, detail=f"图片地址不允许导出:{exc}") from exc
|
||||
except ResourceSizeLimitError as exc:
|
||||
raise HTTPException(status_code=413, detail=f"无法导出「{img.key or '图片'}」:{exc}") from exc
|
||||
except ResourceDownloadTimeoutError as exc:
|
||||
raise HTTPException(status_code=504, detail=str(exc)) from exc
|
||||
except Exception: # noqa: BLE001
|
||||
continue # 单张下载失败不中断整包
|
||||
if not _is_image(img.url, ctype):
|
||||
@@ -161,8 +172,12 @@ async def export_image(img: ExportImageItem):
|
||||
"""代理下载单张图片,兼容源站防盗链与本地媒体路径。"""
|
||||
try:
|
||||
data, ctype = await _download(img.url)
|
||||
except UnsafeRemoteUrlError as exc:
|
||||
raise HTTPException(status_code=400, detail=f"图片地址不允许下载:{exc}") from exc
|
||||
except ResourceSizeLimitError as exc:
|
||||
raise HTTPException(status_code=413, detail=f"图片文件过大,无法下载:{exc}") from exc
|
||||
except ResourceDownloadTimeoutError as exc:
|
||||
raise HTTPException(status_code=504, detail=str(exc)) from exc
|
||||
except Exception as exc: # noqa: BLE001
|
||||
raise HTTPException(status_code=422, detail="图片下载失败") from exc
|
||||
if not _is_image(img.url, ctype):
|
||||
|
||||
+22
-2
@@ -16,6 +16,7 @@ from models.enums import AssetStatus, Stage
|
||||
from schemas.collection import MaterialsRequest, MaterialsResponse, TextMaterial
|
||||
from schemas.suite import SuiteOut, WatermarkOptions
|
||||
from services.image_assets import WatermarkUpdate, get_image_asset_service
|
||||
from services.storage import ImageValidationError, ResourceSizeLimitError, normalize_image_bytes
|
||||
from services.jobs import create_image_job, get_job, list_product_jobs, suite_view
|
||||
from config import get_settings
|
||||
|
||||
@@ -258,6 +259,15 @@ async def upload_material_bytes(
|
||||
raise HTTPException(status_code=413, detail=f"图片文件过大,单张最多 {limit // 1024 // 1024} MB")
|
||||
chunks.append(chunk)
|
||||
data = b"".join(chunks)
|
||||
if type != "img":
|
||||
raise HTTPException(status_code=400, detail="该接口目前仅支持上传图片素材")
|
||||
try:
|
||||
normalized = normalize_image_bytes(data, max_bytes=limit)
|
||||
except ResourceSizeLimitError as exc:
|
||||
raise HTTPException(status_code=413, detail=str(exc)) from exc
|
||||
except ImageValidationError as exc:
|
||||
raise HTTPException(status_code=422, detail=str(exc)) from exc
|
||||
|
||||
asset = ProductAsset(
|
||||
product_id=product.id,
|
||||
group_key=group_key,
|
||||
@@ -272,13 +282,23 @@ async def upload_material_bytes(
|
||||
# 直接转存为素材唯一原图(current 与 original 初始指向同一对象)
|
||||
image_assets = get_image_asset_service()
|
||||
try:
|
||||
await image_assets.save_original(asset, data, file.content_type or "")
|
||||
await image_assets.save_original(
|
||||
asset,
|
||||
normalized.data,
|
||||
normalized.content_type,
|
||||
max_bytes=limit,
|
||||
)
|
||||
asset.status = AssetStatus.uploaded
|
||||
except Exception as exc: # noqa: BLE001
|
||||
asset.status = AssetStatus.failed
|
||||
asset.error = str(exc)[:500]
|
||||
await db.commit()
|
||||
return {"asset_id": str(asset.id), "status": asset.status.value}
|
||||
return {
|
||||
"asset_id": str(asset.id),
|
||||
"status": asset.status.value,
|
||||
"width": asset.width,
|
||||
"height": asset.height,
|
||||
}
|
||||
|
||||
|
||||
@router.get("/products/{product_id}/fingerprints")
|
||||
|
||||
+10
-1
@@ -8,7 +8,12 @@ from urllib.parse import urlparse
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Query, Response
|
||||
|
||||
from services.storage import ResourceSizeLimitError, download_bytes
|
||||
from services.storage import (
|
||||
ResourceDownloadTimeoutError,
|
||||
ResourceSizeLimitError,
|
||||
UnsafeRemoteUrlError,
|
||||
download_bytes,
|
||||
)
|
||||
|
||||
router = APIRouter(prefix="/api", tags=["proxy"])
|
||||
|
||||
@@ -40,8 +45,12 @@ async def proxy_image(url: str = Query(..., description="源站图片 URL")):
|
||||
raise HTTPException(status_code=400, detail="仅支持 http/https URL")
|
||||
try:
|
||||
data, ctype = await download_bytes(url, referer=guess_referer(url))
|
||||
except UnsafeRemoteUrlError as exc:
|
||||
raise HTTPException(status_code=400, detail=f"图片地址不允许代理:{exc}") from exc
|
||||
except ResourceSizeLimitError as exc:
|
||||
raise HTTPException(status_code=413, detail=f"图片文件过大,无法代理:{exc}") from exc
|
||||
except ResourceDownloadTimeoutError as exc:
|
||||
raise HTTPException(status_code=504, detail=str(exc)) from exc
|
||||
except Exception as exc: # noqa: BLE001
|
||||
raise HTTPException(status_code=502, detail=f"图片拉取失败: {exc}") from exc
|
||||
if not ctype.startswith("image/"):
|
||||
|
||||
@@ -0,0 +1,140 @@
|
||||
"""商品图片轮播视频:复用后台任务状态机,FFmpeg 在独立 Worker 中执行。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import APIRouter, HTTPException
|
||||
from fastapi.responses import FileResponse, RedirectResponse
|
||||
from sqlalchemy import select
|
||||
|
||||
from db import get_session_factory
|
||||
from models import Product, ProductAsset
|
||||
from models.enums import AssetStatus
|
||||
from schemas.video import VideoCreateRequest, VideoCreateResponse, VideoDeleteResponse, VideoJobOut
|
||||
from services.jobs import create_image_job, delete_job, get_job, list_product_jobs, retry_failed_items, video_job_view
|
||||
from services.storage import get_storage
|
||||
from services.storage import local_path
|
||||
from services.video_renderer import slide_seconds_for
|
||||
|
||||
router = APIRouter(prefix="/api", tags=["video"])
|
||||
VIDEO_JOB_KIND = "slideshow_video"
|
||||
|
||||
|
||||
def _job_out(result) -> VideoJobOut:
|
||||
return VideoJobOut(**video_job_view(*result))
|
||||
|
||||
|
||||
@router.post("/products/{product_id}/video-jobs", response_model=VideoCreateResponse)
|
||||
async def create_video_job(product_id: str, req: VideoCreateRequest) -> VideoCreateResponse:
|
||||
try:
|
||||
product_uuid = uuid.UUID(product_id)
|
||||
asset_ids = [uuid.UUID(asset_id) for asset_id in req.asset_ids]
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=400, detail="商品或图片 ID 不合法") from exc
|
||||
|
||||
async with get_session_factory()() as db:
|
||||
product = await db.get(Product, product_uuid)
|
||||
if product is None:
|
||||
raise HTTPException(status_code=404, detail="商品不存在")
|
||||
assets = (await db.scalars(
|
||||
select(ProductAsset).where(
|
||||
ProductAsset.product_id == product_uuid,
|
||||
ProductAsset.id.in_(asset_ids),
|
||||
ProductAsset.type == "img",
|
||||
ProductAsset.status == AssetStatus.uploaded,
|
||||
)
|
||||
)).all()
|
||||
by_id = {asset.id: asset for asset in assets}
|
||||
ordered = [by_id.get(asset_id) for asset_id in asset_ids]
|
||||
if any(asset is None or not asset.stored_url for asset in ordered):
|
||||
raise HTTPException(status_code=400, detail="所选图片必须属于当前商品且已完成转存")
|
||||
|
||||
seconds = slide_seconds_for(len(ordered))
|
||||
task = await create_image_job(
|
||||
product_id=product_id,
|
||||
kind=VIDEO_JOB_KIND,
|
||||
priority=5,
|
||||
provider="ffmpeg",
|
||||
model=None,
|
||||
platform="wb_ozon",
|
||||
lang="ru",
|
||||
ratio="3:4",
|
||||
style_set=0,
|
||||
request_payload={
|
||||
"preset": "wb_ozon",
|
||||
"slide_seconds": seconds,
|
||||
"sources": [
|
||||
{"asset_id": str(asset.id), "url": asset.stored_url, "storage_key": asset.current_storage_key}
|
||||
for asset in ordered if asset is not None
|
||||
],
|
||||
},
|
||||
item_payloads=[{"title": "商品轮播视频", "kind": VIDEO_JOB_KIND}],
|
||||
)
|
||||
return VideoCreateResponse(job_id=str(task.id))
|
||||
|
||||
|
||||
@router.get("/products/{product_id}/video-jobs", response_model=list[VideoJobOut])
|
||||
async def list_video_jobs(product_id: str, active_only: bool = False):
|
||||
jobs = await list_product_jobs(product_id, active_only=active_only, kinds=(VIDEO_JOB_KIND,))
|
||||
views: list[VideoJobOut] = []
|
||||
for job in jobs:
|
||||
result = await get_job(str(job.id))
|
||||
if result is not None:
|
||||
views.append(_job_out(result))
|
||||
return views
|
||||
|
||||
|
||||
@router.get("/video-jobs/{job_id}", response_model=VideoJobOut)
|
||||
async def get_video_job(job_id: str):
|
||||
result = await get_job(job_id)
|
||||
if result is None or result[0].kind != VIDEO_JOB_KIND:
|
||||
raise HTTPException(status_code=404, detail="视频任务不存在")
|
||||
return _job_out(result)
|
||||
|
||||
|
||||
@router.post("/video-jobs/{job_id}/cancel", response_model=VideoDeleteResponse)
|
||||
async def cancel_video_job(job_id: str) -> VideoDeleteResponse:
|
||||
# 视频任务不像生图任务需要保留诊断历史:取消后立即从页面和数据库消失。
|
||||
return await _purge_video_job(job_id)
|
||||
|
||||
|
||||
@router.post("/video-jobs/{job_id}/retry", response_model=VideoJobOut)
|
||||
async def retry_video_job(job_id: str):
|
||||
job = await retry_failed_items(job_id)
|
||||
result = await get_job(str(job.id)) if job and job.kind == VIDEO_JOB_KIND else None
|
||||
if result is None:
|
||||
raise HTTPException(status_code=404, detail="视频任务不存在")
|
||||
return _job_out(result)
|
||||
|
||||
|
||||
async def _purge_video_job(job_id: str) -> VideoDeleteResponse:
|
||||
result = await get_job(job_id)
|
||||
if result is None or result[0].kind != VIDEO_JOB_KIND:
|
||||
raise HTTPException(status_code=404, detail="视频任务不存在")
|
||||
job, items = result
|
||||
# 先删该任务专属目录;即使 FFmpeg 正在运行,Worker 下一次检查也会发现任务不存在并终止。
|
||||
if job.product_id:
|
||||
await get_storage().delete_prefix(f"products/{job.product_id}/videos/{job.id}/")
|
||||
if not await delete_job(job_id):
|
||||
raise HTTPException(status_code=404, detail="视频任务不存在")
|
||||
return VideoDeleteResponse()
|
||||
|
||||
|
||||
@router.delete("/video-jobs/{job_id}", response_model=VideoDeleteResponse)
|
||||
async def delete_video_job(job_id: str) -> VideoDeleteResponse:
|
||||
"""删除完成/失败视频及其 MP4;任务记录不保留。"""
|
||||
return await _purge_video_job(job_id)
|
||||
|
||||
|
||||
@router.get("/video-jobs/{job_id}/download")
|
||||
async def download_video(job_id: str):
|
||||
result = await get_job(job_id)
|
||||
if result is None or result[0].kind != VIDEO_JOB_KIND or not result[1] or not result[1][0].result_url:
|
||||
raise HTTPException(status_code=404, detail="视频文件不存在")
|
||||
url = result[1][0].result_url
|
||||
path = local_path(url)
|
||||
filename = f"product-video-{job_id}.mp4"
|
||||
if path is not None and Path(path).is_file():
|
||||
return FileResponse(path, media_type="video/mp4", filename=filename)
|
||||
return RedirectResponse(url, status_code=307)
|
||||
@@ -53,12 +53,35 @@ class Settings(BaseSettings):
|
||||
# 单张图片在后端需要解码/加水印,限制单文件可避免一次请求耗尽内存。
|
||||
image_upload_max_bytes: int = 20 * 1024 * 1024
|
||||
image_download_max_bytes: int = 20 * 1024 * 1024
|
||||
# 图片解码前依据真实宽高限制像素,避免超大图耗尽 Pillow/水印处理内存。
|
||||
image_max_pixels: int = 40_000_000
|
||||
# 远程图片下载防护:连接/读写/总耗时分别受控;重定向逐跳校验目标地址。
|
||||
image_download_connect_timeout: float = 10.0
|
||||
image_download_read_timeout: float = 30.0
|
||||
image_download_total_timeout: float = 60.0
|
||||
image_download_max_redirects: int = 3
|
||||
collection_max_images: int = 100
|
||||
export_max_images: int = 50
|
||||
export_zip_max_bytes: int = 200 * 1024 * 1024
|
||||
# ZIP 超过该值自动落临时磁盘,不继续占用进程内存。
|
||||
export_zip_memory_threshold_bytes: int = 2 * 1024 * 1024
|
||||
|
||||
# ── 商品轮播视频(FFmpeg)──
|
||||
# Ubuntu 默认安装包会提供同名命令;容器或特殊路径可在 .env 覆盖。
|
||||
video_ffmpeg_path: str = "ffmpeg"
|
||||
video_ffprobe_path: str = "ffprobe"
|
||||
video_width: int = 810
|
||||
video_height: int = 1080
|
||||
video_fps: int = 30
|
||||
video_transition_seconds: float = 0.35
|
||||
video_crf: int = 22
|
||||
video_max_rate: str = "3M"
|
||||
video_buffer_size: str = "6M"
|
||||
video_max_bytes: int = 45 * 1024 * 1024
|
||||
video_max_duration_seconds: int = 180
|
||||
video_render_timeout_seconds: int = 120
|
||||
background_job_lease_seconds: int = 300
|
||||
|
||||
# ── V2.1 套图生图(provider 路由与 image-suite-studio 一致)──
|
||||
image_provider: str = "doubao" # 未知模型时的兜底 provider:doubao | tongyi | rightapi
|
||||
request_timeout: int = 300 # 单张生图请求超时(秒)
|
||||
|
||||
+2
-1
@@ -5,7 +5,7 @@ from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
from sqlalchemy import text
|
||||
|
||||
from api import ai, export, fx, image, materials, products, proxy, suite
|
||||
from api import ai, export, fx, image, materials, products, proxy, suite, video
|
||||
from config import get_settings
|
||||
from db import get_engine
|
||||
|
||||
@@ -36,6 +36,7 @@ app.include_router(image.router)
|
||||
app.include_router(suite.router)
|
||||
app.include_router(export.router)
|
||||
app.include_router(proxy.router)
|
||||
app.include_router(video.router)
|
||||
|
||||
# ⚠️ 冻结链路(Ozon API 直传):代码移至 legacy/,接口保留但不再投入
|
||||
from legacy.api import categories, ozon, publish, shops # noqa: E402
|
||||
|
||||
@@ -19,7 +19,14 @@ depends_on = None
|
||||
def upgrade() -> None:
|
||||
# 开发态 FastAPI 会在 startup 以 Base.metadata.create_all 兜底;若它先于 Alembic
|
||||
# 启动,表已完整存在,此处只补迁移版本记录,避免 create_table 冲突。
|
||||
if "background_jobs" in sa.inspect(op.get_bind()).get_table_names():
|
||||
inspector = sa.inspect(op.get_bind())
|
||||
if "background_jobs" in inspector.get_table_names():
|
||||
# 开发态若先由 metadata.create_all 建表,仍补上后来新增的索引。
|
||||
existing_indexes = {
|
||||
index["name"] for index in inspector.get_indexes("background_job_items")
|
||||
}
|
||||
if "ix_background_job_items_updated_at" not in existing_indexes:
|
||||
op.create_index("ix_background_job_items_updated_at", "background_job_items", ["updated_at"])
|
||||
return
|
||||
op.create_table(
|
||||
"background_jobs",
|
||||
@@ -85,8 +92,10 @@ def upgrade() -> None:
|
||||
op.create_index("ix_background_job_items_provider_task_id", "background_job_items", ["provider_task_id"])
|
||||
op.create_index("ix_background_job_items_next_run_at", "background_job_items", ["next_run_at"])
|
||||
op.create_index("ix_background_job_items_lease_expires_at", "background_job_items", ["lease_expires_at"])
|
||||
op.create_index("ix_background_job_items_updated_at", "background_job_items", ["updated_at"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index("ix_background_job_items_updated_at", table_name="background_job_items")
|
||||
op.drop_table("background_job_items")
|
||||
op.drop_table("background_jobs")
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
"""轮播商品视频的请求与任务响应契约。"""
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from pydantic import BaseModel, Field, field_validator
|
||||
|
||||
|
||||
class VideoCreateRequest(BaseModel):
|
||||
"""按传入顺序把 2–5 张当前商品图片合成为无音轨轮播视频。"""
|
||||
|
||||
asset_ids: list[str] = Field(min_length=2, max_length=5)
|
||||
|
||||
@field_validator("asset_ids")
|
||||
@classmethod
|
||||
def asset_ids_must_be_unique(cls, values: list[str]) -> list[str]:
|
||||
if len(set(values)) != len(values):
|
||||
raise ValueError("合成视频不能重复选择同一张图片")
|
||||
return values
|
||||
|
||||
|
||||
class VideoCreateResponse(BaseModel):
|
||||
job_id: str
|
||||
|
||||
|
||||
class VideoDeleteResponse(BaseModel):
|
||||
deleted: bool = True
|
||||
|
||||
|
||||
class VideoResultOut(BaseModel):
|
||||
item_id: str
|
||||
status: str
|
||||
url: str | None = None
|
||||
error: str | None = None
|
||||
|
||||
|
||||
class VideoJobOut(BaseModel):
|
||||
id: str
|
||||
kind: str = "slideshow_video"
|
||||
status: str
|
||||
total: int = 1
|
||||
completed: int = 0
|
||||
failed: int = 0
|
||||
cancel_requested: bool = False
|
||||
preset: str = "wb_ozon"
|
||||
source_asset_ids: list[str] = Field(default_factory=list)
|
||||
slide_seconds: float = 0.0
|
||||
duration_seconds: float | None = None
|
||||
width: int = 810
|
||||
height: int = 1080
|
||||
file_size_bytes: int | None = None
|
||||
created_at: datetime | None = None
|
||||
started_at: datetime | None = None
|
||||
completed_at: datetime | None = None
|
||||
result: VideoResultOut | None = None
|
||||
error: str | None = None
|
||||
@@ -0,0 +1,69 @@
|
||||
"""为历史图片素材补齐真实宽高,不改写已有图片文件。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import mimetypes
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import or_, select
|
||||
|
||||
SERVER_DIR = Path(__file__).resolve().parents[1]
|
||||
if str(SERVER_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(SERVER_DIR))
|
||||
|
||||
from db import get_session_factory
|
||||
from config import get_settings
|
||||
from models import ProductAsset
|
||||
from services.storage import local_path, normalize_image_bytes, read_local_bytes
|
||||
|
||||
|
||||
async def backfill(dry_run: bool = False) -> tuple[int, list[str]]:
|
||||
"""返回成功回填数和最多十条失败摘要。"""
|
||||
updated = 0
|
||||
failures: list[str] = []
|
||||
async with get_session_factory()() as db:
|
||||
assets = (await db.scalars(
|
||||
select(ProductAsset).where(
|
||||
ProductAsset.type == "img",
|
||||
or_(ProductAsset.width.is_(None), ProductAsset.height.is_(None)),
|
||||
)
|
||||
)).all()
|
||||
for asset in assets:
|
||||
url = asset.original_url or asset.stored_url
|
||||
path = local_path(url or "")
|
||||
if path is None:
|
||||
if len(failures) < 10:
|
||||
failures.append(f"{asset.id}: 本地原图不存在,跳过")
|
||||
continue
|
||||
try:
|
||||
data = read_local_bytes(path)
|
||||
image = normalize_image_bytes(
|
||||
data,
|
||||
max_bytes=max(len(data), get_settings().image_download_max_bytes),
|
||||
)
|
||||
asset.width = image.width
|
||||
asset.height = image.height
|
||||
updated += 1
|
||||
except Exception as exc: # noqa: BLE001
|
||||
if len(failures) < 10:
|
||||
failures.append(f"{asset.id}: {exc}")
|
||||
if not dry_run:
|
||||
await db.commit()
|
||||
return updated, failures
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--dry-run", action="store_true", help="只检查,不写数据库")
|
||||
args = parser.parse_args()
|
||||
updated, failures = asyncio.run(backfill(dry_run=args.dry_run))
|
||||
mode = "将回填" if args.dry_run else "已回填"
|
||||
print(f"{mode} {updated} 条图片素材的宽高")
|
||||
for failure in failures:
|
||||
print(f"跳过:{failure}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -11,8 +11,17 @@ import uuid
|
||||
from dataclasses import dataclass
|
||||
|
||||
from api.proxy import guess_referer
|
||||
from config import get_settings
|
||||
from models import ProductAsset
|
||||
from services.storage import download_bytes, get_storage, image_extension, local_path, read_local_bytes, storage_key_from_url
|
||||
from services.storage import (
|
||||
download_bytes,
|
||||
get_storage,
|
||||
image_extension,
|
||||
local_path,
|
||||
normalize_image_bytes,
|
||||
read_local_bytes,
|
||||
storage_key_from_url,
|
||||
)
|
||||
from services.watermark import apply_watermark
|
||||
|
||||
|
||||
@@ -46,8 +55,20 @@ class ImageAssetService:
|
||||
return read_local_bytes(path), mimetypes.guess_type(path.name)[0] or "image/jpeg"
|
||||
return await download_bytes(url, referer=guess_referer(url))
|
||||
|
||||
async def save_original(self, asset: ProductAsset, data: bytes, content_type: str = "") -> str:
|
||||
async def save_original(
|
||||
self,
|
||||
asset: ProductAsset,
|
||||
data: bytes,
|
||||
content_type: str = "",
|
||||
max_bytes: int | None = None,
|
||||
) -> str:
|
||||
"""写入该素材唯一的无水印原图,并将 current 同步到原图。"""
|
||||
if asset.type == "img":
|
||||
normalized = normalize_image_bytes(data, max_bytes=max_bytes)
|
||||
data = normalized.data
|
||||
content_type = normalized.content_type
|
||||
asset.width = normalized.width
|
||||
asset.height = normalized.height
|
||||
key = self._key(asset, "original", data, content_type)
|
||||
url = await self.storage.save_bytes(data, key, content_type)
|
||||
asset.original_url = url
|
||||
@@ -60,7 +81,12 @@ class ImageAssetService:
|
||||
|
||||
async def save_original_from_url(self, asset: ProductAsset, url: str) -> str:
|
||||
data, content_type = await self._read(url)
|
||||
return await self.save_original(asset, data, content_type)
|
||||
return await self.save_original(
|
||||
asset,
|
||||
data,
|
||||
content_type,
|
||||
max_bytes=get_settings().image_download_max_bytes,
|
||||
)
|
||||
|
||||
async def ensure_original(self, asset: ProductAsset) -> tuple[bytes, str]:
|
||||
"""为旧记录补齐原图字段;首次触碰旧素材时迁入新的商品/素材路径。"""
|
||||
|
||||
@@ -26,6 +26,7 @@ from services.storage import get_storage
|
||||
from services.suite_service import append_generated_asset
|
||||
from services.tasks import TaskImage
|
||||
from services.watermark import apply_watermark
|
||||
from services.video_renderer import VideoRenderCancelled, render_slideshow
|
||||
|
||||
log = logging.getLogger("background.worker")
|
||||
|
||||
@@ -113,11 +114,52 @@ async def _process_watermark_item(claim: ClaimedItem) -> None:
|
||||
await mark_item_failed(claim.item_id, str(exc))
|
||||
|
||||
|
||||
async def _process_slideshow_video_item(claim: ClaimedItem) -> None:
|
||||
"""一个视频任务只有一个子项;输入 URL 在提交时冻结,避免后续选择状态影响成片。"""
|
||||
payload = claim.request_payload
|
||||
sources = list(payload.get("sources") or [])
|
||||
source_urls = [str(source.get("url") or "") for source in sources]
|
||||
if not 2 <= len(source_urls) <= 5 or any(not url for url in source_urls):
|
||||
await mark_item_failed(claim.item_id, "视频任务缺少 2–5 张可用图片")
|
||||
return
|
||||
try:
|
||||
async def should_cancel() -> bool:
|
||||
return not await job_can_write(claim.job_id)
|
||||
|
||||
rendered = await render_slideshow(source_urls, should_cancel=should_cancel)
|
||||
if not await job_can_write(claim.job_id):
|
||||
await mark_item_cancelled(claim.item_id)
|
||||
return
|
||||
key = f"products/{claim.product_id}/videos/{claim.job_id}/result.mp4"
|
||||
result_url = await get_storage().save_bytes(rendered.data, key, "video/mp4")
|
||||
async with get_session_factory()() as db:
|
||||
from models import BackgroundJobItem
|
||||
item = await db.get(BackgroundJobItem, uuid.UUID(claim.item_id))
|
||||
if item is not None:
|
||||
item.payload = {
|
||||
**dict(item.payload),
|
||||
"duration_seconds": rendered.duration_seconds,
|
||||
"width": rendered.width,
|
||||
"height": rendered.height,
|
||||
"file_size_bytes": len(rendered.data),
|
||||
}
|
||||
await db.commit()
|
||||
await mark_item_done(claim.item_id, result_url=result_url, source_url=None, watermark_config=None, generated_asset_id=None)
|
||||
except VideoRenderCancelled:
|
||||
await mark_item_cancelled(claim.item_id)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
log.exception("后台轮播视频任务 %s/%s 执行失败", claim.job_id, claim.item_id)
|
||||
await mark_item_failed(claim.item_id, str(exc))
|
||||
|
||||
|
||||
async def process_item(claim: ClaimedItem) -> None:
|
||||
"""推进一次子任务。外部模型未完成时只写 provider id 并退出,让出 Worker。"""
|
||||
if claim.job_kind == "watermark_batch":
|
||||
await _process_watermark_item(claim)
|
||||
return
|
||||
if claim.job_kind == "slideshow_video":
|
||||
await _process_slideshow_video_item(claim)
|
||||
return
|
||||
if claim.job_kind not in {"image_suite", "image_edit"}:
|
||||
await mark_item_failed(claim.item_id, f"Worker 暂不支持任务类型: {claim.job_kind}")
|
||||
return
|
||||
@@ -201,7 +243,7 @@ async def run_worker(*, once: bool = False, idle_seconds: float = 0.75) -> None:
|
||||
log.warning("恢复了 %d 个过期后台任务租约", recovered)
|
||||
log.info("后台 Worker 已启动:%s", owner)
|
||||
while True:
|
||||
claim = await claim_next_item(owner)
|
||||
claim = await claim_next_item(owner, lease_seconds=get_settings().background_job_lease_seconds)
|
||||
if claim is None:
|
||||
if once:
|
||||
return
|
||||
|
||||
@@ -310,6 +310,21 @@ async def cancel_job(job_id: str) -> BackgroundJob | None:
|
||||
return job
|
||||
|
||||
|
||||
async def delete_job(job_id: str) -> bool:
|
||||
"""硬删除一个后台任务及其子项;调用方须先处理对应的存储对象。"""
|
||||
try:
|
||||
parsed = uuid.UUID(job_id)
|
||||
except ValueError:
|
||||
return False
|
||||
async with get_session_factory()() as db:
|
||||
job = await db.get(BackgroundJob, parsed)
|
||||
if job is None:
|
||||
return False
|
||||
await db.delete(job)
|
||||
await db.commit()
|
||||
return True
|
||||
|
||||
|
||||
async def cancel_product_jobs(product_id: str) -> int:
|
||||
"""硬删除商品前取消其未完成任务,Worker 随后只会丢弃结果而不会回写素材。"""
|
||||
try:
|
||||
@@ -468,3 +483,25 @@ def suite_view(job: BackgroundJob, items: list[BackgroundJobItem]) -> dict[str,
|
||||
for item in items
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def video_job_view(job: BackgroundJob, items: list[BackgroundJobItem]) -> dict[str, Any]:
|
||||
"""轮播视频的轻量任务视图;仍复用 BackgroundJob 的恢复/取消/重试状态机。"""
|
||||
payload = dict(job.request_payload or {})
|
||||
item = items[0] if items else None
|
||||
item_payload = dict(item.payload or {}) if item else {}
|
||||
return {
|
||||
"id": str(job.id), "kind": job.kind, "status": job.status,
|
||||
"total": job.total, "completed": job.completed, "failed": job.failed,
|
||||
"cancel_requested": job.cancel_requested, "preset": payload.get("preset", "wb_ozon"),
|
||||
"source_asset_ids": [str(source.get("asset_id")) for source in payload.get("sources", [])],
|
||||
"slide_seconds": payload.get("slide_seconds", 0),
|
||||
"duration_seconds": item_payload.get("duration_seconds"),
|
||||
"width": item_payload.get("width", 810), "height": item_payload.get("height", 1080),
|
||||
"file_size_bytes": item_payload.get("file_size_bytes"),
|
||||
"created_at": job.created_at, "started_at": job.started_at, "completed_at": job.completed_at,
|
||||
"result": None if item is None else {
|
||||
"item_id": str(item.id), "status": item.status, "url": item.result_url, "error": item.error,
|
||||
},
|
||||
"error": job.error,
|
||||
}
|
||||
|
||||
+220
-24
@@ -1,12 +1,20 @@
|
||||
"""图片/文件存储抽象:本地文件系统(开发兜底)+ 七牛(生产)。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import io
|
||||
import ipaddress
|
||||
import mimetypes
|
||||
import socket
|
||||
import shutil
|
||||
import uuid
|
||||
import warnings
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from urllib.parse import urljoin, urlparse
|
||||
|
||||
import httpx
|
||||
from PIL import Image, ImageOps, UnidentifiedImageError
|
||||
|
||||
from config import get_settings
|
||||
|
||||
@@ -26,6 +34,157 @@ class ResourceSizeLimitError(ValueError):
|
||||
super().__init__(detail)
|
||||
|
||||
|
||||
class UnsafeRemoteUrlError(ValueError):
|
||||
"""远程资源 URL 指向回环、内网或其他保留网络,不能由服务端代请求。"""
|
||||
|
||||
|
||||
class ResourceDownloadTimeoutError(TimeoutError):
|
||||
"""远程资源下载超过连接、读取或总耗时限制。"""
|
||||
|
||||
|
||||
class ImageValidationError(ValueError):
|
||||
"""图片字节无法解码,或不符合格式与像素限制。"""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class NormalizedImage:
|
||||
"""经真实格式校验后的图片字节及可落库元数据。"""
|
||||
|
||||
data: bytes
|
||||
content_type: str
|
||||
width: int
|
||||
height: int
|
||||
image_format: str
|
||||
orientation_normalized: bool
|
||||
|
||||
|
||||
_IMAGE_CONTENT_TYPES = {
|
||||
"JPEG": "image/jpeg",
|
||||
"PNG": "image/png",
|
||||
"WEBP": "image/webp",
|
||||
"GIF": "image/gif",
|
||||
"BMP": "image/bmp",
|
||||
}
|
||||
|
||||
|
||||
def normalize_image_bytes(
|
||||
data: bytes,
|
||||
max_bytes: int | None = None,
|
||||
max_pixels: int | None = None,
|
||||
) -> NormalizedImage:
|
||||
"""按真实图片内容校验、纠正 EXIF 方向,并返回可信 MIME 与宽高。
|
||||
|
||||
不盲从 URL 后缀或客户端 Content-Type。为避免无意义的质量损耗,仅当 EXIF
|
||||
方向不是默认值时才重编码;动画图不重编码,以免丢失帧。
|
||||
"""
|
||||
settings = get_settings()
|
||||
byte_limit = max_bytes if max_bytes is not None else settings.image_upload_max_bytes
|
||||
pixel_limit = max_pixels if max_pixels is not None else settings.image_max_pixels
|
||||
if byte_limit <= 0 or pixel_limit <= 0:
|
||||
raise ValueError("图片大小和像素上限必须大于 0")
|
||||
if not data:
|
||||
raise ImageValidationError("图片内容为空")
|
||||
if len(data) > byte_limit:
|
||||
raise ResourceSizeLimitError(byte_limit, len(data))
|
||||
|
||||
try:
|
||||
# Pillow 的默认 bomb 保护不能替代项目自定义像素上限;把 warning 升级为错误。
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("error", Image.DecompressionBombWarning)
|
||||
with Image.open(io.BytesIO(data)) as image:
|
||||
image_format = (image.format or "").upper()
|
||||
if image_format not in _IMAGE_CONTENT_TYPES:
|
||||
raise ImageValidationError(f"不支持的图片格式:{image_format or '未知'}")
|
||||
|
||||
width, height = image.size
|
||||
if width <= 0 or height <= 0 or width * height > pixel_limit:
|
||||
raise ImageValidationError(
|
||||
f"图片像素超过上限(最多 {pixel_limit:,},当前 {width:,} × {height:,})"
|
||||
)
|
||||
|
||||
# 强制实际解码,拒绝截断/伪造图片而不是只相信文件头。
|
||||
image.load()
|
||||
frame_count = getattr(image, "n_frames", 1)
|
||||
orientation = image.getexif().get(274, 1) if frame_count == 1 else 1
|
||||
normalized = orientation not in (None, 1)
|
||||
output = data
|
||||
if normalized:
|
||||
corrected = ImageOps.exif_transpose(image)
|
||||
width, height = corrected.size
|
||||
if image_format == "JPEG" and corrected.mode not in {"RGB", "L"}:
|
||||
corrected = corrected.convert("RGB")
|
||||
buffer = io.BytesIO()
|
||||
corrected.save(buffer, format=image_format)
|
||||
output = buffer.getvalue()
|
||||
except (UnidentifiedImageError, OSError, Image.DecompressionBombError) as exc:
|
||||
raise ImageValidationError("文件不是可用的图片,或图片数据已损坏") from exc
|
||||
except Image.DecompressionBombWarning as exc:
|
||||
raise ImageValidationError("图片像素过大,已拒绝解码") from exc
|
||||
|
||||
if len(output) > byte_limit:
|
||||
raise ResourceSizeLimitError(byte_limit, len(output))
|
||||
return NormalizedImage(
|
||||
data=output,
|
||||
content_type=_IMAGE_CONTENT_TYPES[image_format],
|
||||
width=width,
|
||||
height=height,
|
||||
image_format=image_format,
|
||||
orientation_normalized=normalized,
|
||||
)
|
||||
|
||||
|
||||
def _is_public_ip(address: ipaddress.IPv4Address | ipaddress.IPv6Address) -> bool:
|
||||
"""仅允许可公开路由的地址;IPv4-mapped IPv6 需按其 IPv4 实际地址判断。"""
|
||||
if isinstance(address, ipaddress.IPv6Address) and address.ipv4_mapped:
|
||||
return _is_public_ip(address.ipv4_mapped)
|
||||
return address.is_global
|
||||
|
||||
|
||||
async def validate_remote_url(url: str) -> None:
|
||||
"""校验服务端允许代请求的公网 HTTP(S) URL。
|
||||
|
||||
该校验用于代理、采集转存、AI 参考图和导出。项目自己的 `/media/` 文件
|
||||
在调用方会先直读磁盘;其他回环或私网 URL 一律不能通过网络访问。
|
||||
"""
|
||||
try:
|
||||
parsed = urlparse(url)
|
||||
port = parsed.port
|
||||
except ValueError as exc:
|
||||
raise UnsafeRemoteUrlError("图片地址端口无效") from exc
|
||||
|
||||
if parsed.scheme.lower() not in {"http", "https"}:
|
||||
raise UnsafeRemoteUrlError("仅允许 http/https 图片地址")
|
||||
if not parsed.hostname:
|
||||
raise UnsafeRemoteUrlError("图片地址缺少主机名")
|
||||
if parsed.username is not None or parsed.password is not None:
|
||||
raise UnsafeRemoteUrlError("图片地址不能包含用户名或密码")
|
||||
|
||||
host = parsed.hostname.rstrip(".")
|
||||
if host.lower() in {"localhost", "localhost.localdomain"}:
|
||||
raise UnsafeRemoteUrlError("不允许请求 localhost 或本机网络地址")
|
||||
|
||||
try:
|
||||
addresses = {ipaddress.ip_address(host)}
|
||||
except ValueError:
|
||||
try:
|
||||
infos = await asyncio.get_running_loop().getaddrinfo(
|
||||
host,
|
||||
port or (443 if parsed.scheme.lower() == "https" else 80),
|
||||
type=socket.SOCK_STREAM,
|
||||
)
|
||||
except socket.gaierror as exc:
|
||||
raise UnsafeRemoteUrlError("图片地址域名无法解析") from exc
|
||||
addresses = {ipaddress.ip_address(info[4][0]) for info in infos}
|
||||
|
||||
if not addresses:
|
||||
raise UnsafeRemoteUrlError("图片地址域名未解析到有效地址")
|
||||
private_addresses = [str(address) for address in addresses if not _is_public_ip(address)]
|
||||
if private_addresses:
|
||||
raise UnsafeRemoteUrlError(
|
||||
f"不允许请求非公网地址({', '.join(sorted(private_addresses))})"
|
||||
)
|
||||
|
||||
|
||||
def _ext_from_content_type(content_type: str) -> str:
|
||||
ctype = (content_type or "").split(";")[0].strip().lower()
|
||||
mapping = {
|
||||
@@ -48,41 +207,78 @@ def image_extension(data: bytes, content_type: str = "") -> str:
|
||||
return ".webp"
|
||||
if data[:3] == b"GIF":
|
||||
return ".gif"
|
||||
if data[:2] == b"\xff\xd8":
|
||||
return ".jpg"
|
||||
if data[:2] == b"BM":
|
||||
return ".bmp"
|
||||
return _ext_from_content_type(content_type)
|
||||
|
||||
|
||||
async def download_bytes(
|
||||
url: str,
|
||||
referer: str | None = None,
|
||||
timeout: float = 60.0,
|
||||
timeout: float | None = None,
|
||||
max_bytes: int | None = None,
|
||||
) -> tuple[bytes, str]:
|
||||
"""流式下载远程图片并限制体积。返回 (bytes, content_type)。"""
|
||||
limit = max_bytes or get_settings().image_download_max_bytes
|
||||
"""安全地流式下载公网资源,并限制地址、重定向、体积和耗时。"""
|
||||
settings = get_settings()
|
||||
limit = max_bytes or settings.image_download_max_bytes
|
||||
if limit <= 0:
|
||||
raise ValueError("图片下载大小上限必须大于 0")
|
||||
headers = {"Referer": referer} if referer else {}
|
||||
async with httpx.AsyncClient(timeout=timeout, follow_redirects=True) as client:
|
||||
async with client.stream("GET", url, headers=headers) as resp:
|
||||
resp.raise_for_status()
|
||||
length = resp.headers.get("content-length")
|
||||
if length:
|
||||
try:
|
||||
declared_size = int(length)
|
||||
except ValueError:
|
||||
declared_size = None
|
||||
if declared_size is not None and declared_size > limit:
|
||||
raise ResourceSizeLimitError(limit, declared_size)
|
||||
total_timeout = timeout if timeout is not None else settings.image_download_total_timeout
|
||||
if total_timeout <= 0:
|
||||
raise ValueError("图片下载总超时必须大于 0")
|
||||
if settings.image_download_connect_timeout <= 0 or settings.image_download_read_timeout <= 0:
|
||||
raise ValueError("图片下载连接和读取超时必须大于 0")
|
||||
if settings.image_download_max_redirects < 0:
|
||||
raise ValueError("图片下载重定向上限不能小于 0")
|
||||
|
||||
chunks: list[bytes] = []
|
||||
total = 0
|
||||
async for chunk in resp.aiter_bytes():
|
||||
total += len(chunk)
|
||||
if total > limit:
|
||||
raise ResourceSizeLimitError(limit, total)
|
||||
chunks.append(chunk)
|
||||
ctype = (resp.headers.get("content-type") or "application/octet-stream").split(";")[0].strip()
|
||||
return b"".join(chunks), ctype
|
||||
headers = {"Referer": referer} if referer else {}
|
||||
http_timeout = httpx.Timeout(
|
||||
connect=settings.image_download_connect_timeout,
|
||||
read=settings.image_download_read_timeout,
|
||||
write=settings.image_download_read_timeout,
|
||||
pool=settings.image_download_connect_timeout,
|
||||
)
|
||||
current_url = url
|
||||
try:
|
||||
async with asyncio.timeout(total_timeout):
|
||||
async with httpx.AsyncClient(timeout=http_timeout, follow_redirects=False) as client:
|
||||
for redirect_count in range(settings.image_download_max_redirects + 1):
|
||||
await validate_remote_url(current_url)
|
||||
async with client.stream("GET", current_url, headers=headers) as resp:
|
||||
if resp.is_redirect:
|
||||
location = resp.headers.get("location")
|
||||
if not location:
|
||||
raise httpx.RemoteProtocolError("重定向响应缺少 Location")
|
||||
if redirect_count >= settings.image_download_max_redirects:
|
||||
raise httpx.TooManyRedirects("图片重定向次数超过上限")
|
||||
current_url = urljoin(str(resp.url), location)
|
||||
continue
|
||||
|
||||
resp.raise_for_status()
|
||||
length = resp.headers.get("content-length")
|
||||
if length:
|
||||
try:
|
||||
declared_size = int(length)
|
||||
except ValueError:
|
||||
declared_size = None
|
||||
if declared_size is not None and declared_size > limit:
|
||||
raise ResourceSizeLimitError(limit, declared_size)
|
||||
|
||||
chunks: list[bytes] = []
|
||||
total = 0
|
||||
async for chunk in resp.aiter_bytes():
|
||||
total += len(chunk)
|
||||
if total > limit:
|
||||
raise ResourceSizeLimitError(limit, total)
|
||||
chunks.append(chunk)
|
||||
ctype = (resp.headers.get("content-type") or "application/octet-stream").split(";")[0].strip()
|
||||
return b"".join(chunks), ctype
|
||||
except (TimeoutError, httpx.TimeoutException) as exc:
|
||||
raise ResourceDownloadTimeoutError(f"图片下载超时(最多 {total_timeout:g} 秒)") from exc
|
||||
|
||||
raise RuntimeError("图片下载未返回有效响应")
|
||||
|
||||
|
||||
def read_local_bytes(path: Path, max_bytes: int | None = None) -> bytes:
|
||||
|
||||
@@ -0,0 +1,187 @@
|
||||
"""用 FFmpeg 合成平台兼容的无音轨商品轮播视频。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import tempfile
|
||||
from collections.abc import Awaitable, Callable
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from config import get_settings
|
||||
from services.storage import (
|
||||
ResourceSizeLimitError,
|
||||
download_bytes,
|
||||
image_extension,
|
||||
local_path,
|
||||
normalize_image_bytes,
|
||||
read_local_bytes,
|
||||
)
|
||||
|
||||
|
||||
class VideoRenderError(RuntimeError):
|
||||
"""FFmpeg 无法生成符合预设的商品视频。"""
|
||||
|
||||
|
||||
class VideoRenderCancelled(VideoRenderError):
|
||||
"""任务在 FFmpeg 运行期间被用户取消。"""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RenderedVideo:
|
||||
data: bytes
|
||||
duration_seconds: float
|
||||
width: int
|
||||
height: int
|
||||
|
||||
|
||||
def slide_seconds_for(count: int) -> float:
|
||||
"""按图片数量维持约 9–13 秒成片,并确保两图时也不会过短。"""
|
||||
by_count = {2: 4.5, 3: 3.5, 4: 3.0, 5: 2.8}
|
||||
try:
|
||||
return by_count[count]
|
||||
except KeyError as exc:
|
||||
raise VideoRenderError("轮播视频只支持 2–5 张图片") from exc
|
||||
|
||||
|
||||
async def _read_image(url: str) -> tuple[bytes, str]:
|
||||
path = local_path(url)
|
||||
if path is not None:
|
||||
return read_local_bytes(path), ""
|
||||
return await download_bytes(url, max_bytes=get_settings().image_download_max_bytes)
|
||||
|
||||
|
||||
async def _run_process(
|
||||
args: list[str],
|
||||
*,
|
||||
should_cancel: Callable[[], Awaitable[bool]],
|
||||
timeout_seconds: int,
|
||||
) -> None:
|
||||
process = await asyncio.create_subprocess_exec(
|
||||
*args,
|
||||
stdout=asyncio.subprocess.DEVNULL,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
)
|
||||
try:
|
||||
waited = 0.0
|
||||
while process.returncode is None:
|
||||
if await should_cancel():
|
||||
process.terminate()
|
||||
try:
|
||||
await asyncio.wait_for(process.wait(), timeout=5)
|
||||
except TimeoutError:
|
||||
process.kill()
|
||||
await process.wait()
|
||||
raise VideoRenderCancelled("视频任务已取消")
|
||||
await asyncio.sleep(0.25)
|
||||
waited += 0.25
|
||||
if waited > timeout_seconds:
|
||||
process.kill()
|
||||
await process.wait()
|
||||
raise VideoRenderError(f"视频渲染超过 {timeout_seconds} 秒,请减少图片后重试")
|
||||
stderr = (await process.stderr.read()).decode("utf-8", errors="replace") if process.stderr else ""
|
||||
if process.returncode != 0:
|
||||
raise VideoRenderError(f"FFmpeg 渲染失败:{stderr[-800:] or '未知错误'}")
|
||||
finally:
|
||||
if process.returncode is None:
|
||||
process.kill()
|
||||
await process.wait()
|
||||
|
||||
|
||||
async def _probe(path: Path) -> dict:
|
||||
settings = get_settings()
|
||||
process = await asyncio.create_subprocess_exec(
|
||||
settings.video_ffprobe_path,
|
||||
"-v", "error", "-show_entries", "format=duration,size:stream=codec_name,width,height,pix_fmt",
|
||||
"-of", "json", str(path),
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
)
|
||||
stdout, stderr = await process.communicate()
|
||||
if process.returncode != 0:
|
||||
raise VideoRenderError(f"无法校验视频输出:{stderr.decode('utf-8', errors='replace')[-400:]}")
|
||||
try:
|
||||
return json.loads(stdout)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise VideoRenderError("FFprobe 未返回可解析的视频信息") from exc
|
||||
|
||||
|
||||
async def render_slideshow(
|
||||
source_urls: list[str],
|
||||
*,
|
||||
should_cancel: Callable[[], Awaitable[bool]],
|
||||
) -> RenderedVideo:
|
||||
"""将已入库图片缩放留白后串为 MP4,严禁接收调用方任意命令参数。"""
|
||||
settings = get_settings()
|
||||
if not settings.video_ffmpeg_path or not settings.video_ffprobe_path:
|
||||
raise VideoRenderError("未配置 FFmpeg 或 FFprobe 可执行路径")
|
||||
count = len(source_urls)
|
||||
seconds = slide_seconds_for(count)
|
||||
transition = settings.video_transition_seconds
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="osk-video-") as temp_dir:
|
||||
workdir = Path(temp_dir)
|
||||
inputs: list[Path] = []
|
||||
for index, url in enumerate(source_urls):
|
||||
if await should_cancel():
|
||||
raise VideoRenderCancelled("视频任务已取消")
|
||||
data, content_type = await _read_image(url)
|
||||
normalized = normalize_image_bytes(data, max_bytes=settings.image_download_max_bytes)
|
||||
suffix = image_extension(normalized.data, content_type or normalized.content_type)
|
||||
image_path = workdir / f"input-{index}{suffix}"
|
||||
image_path.write_bytes(normalized.data)
|
||||
inputs.append(image_path)
|
||||
|
||||
output_path = workdir / "result.mp4"
|
||||
args = [settings.video_ffmpeg_path, "-y", "-hide_banner", "-loglevel", "error"]
|
||||
for image_path in inputs:
|
||||
args.extend(["-loop", "1", "-framerate", str(settings.video_fps), "-t", str(seconds), "-i", str(image_path)])
|
||||
|
||||
filters: list[str] = []
|
||||
for index in range(count):
|
||||
filters.append(
|
||||
f"[{index}:v]scale={settings.video_width}:{settings.video_height}:"
|
||||
"force_original_aspect_ratio=decrease,"
|
||||
f"pad={settings.video_width}:{settings.video_height}:(ow-iw)/2:(oh-ih):color=white,"
|
||||
"setsar=1,format=yuv420p"
|
||||
f"[slide{index}]"
|
||||
)
|
||||
previous = "slide0"
|
||||
accumulated = seconds
|
||||
for index in range(1, count):
|
||||
output = f"fade{index}"
|
||||
filters.append(
|
||||
f"[{previous}][slide{index}]xfade=transition=fade:duration={transition}:"
|
||||
f"offset={accumulated - transition:.3f}[{output}]"
|
||||
)
|
||||
previous = output
|
||||
accumulated += seconds - transition
|
||||
filters.append(f"[{previous}]fps={settings.video_fps},format=yuv420p[video]")
|
||||
args.extend([
|
||||
"-filter_complex", ";".join(filters), "-map", "[video]",
|
||||
"-an", "-c:v", "libx264", "-profile:v", "high", "-pix_fmt", "yuv420p",
|
||||
"-preset", "medium", "-crf", str(settings.video_crf),
|
||||
"-maxrate", settings.video_max_rate, "-bufsize", settings.video_buffer_size,
|
||||
"-movflags", "+faststart", str(output_path),
|
||||
])
|
||||
await _run_process(args, should_cancel=should_cancel, timeout_seconds=settings.video_render_timeout_seconds)
|
||||
if not output_path.is_file():
|
||||
raise VideoRenderError("FFmpeg 未生成视频文件")
|
||||
file_size = output_path.stat().st_size
|
||||
if file_size > settings.video_max_bytes:
|
||||
raise ResourceSizeLimitError(settings.video_max_bytes, file_size)
|
||||
probe = await _probe(output_path)
|
||||
stream = next((item for item in probe.get("streams", []) if item.get("codec_name") == "h264"), None)
|
||||
if stream is None:
|
||||
raise VideoRenderError("输出视频不是 H.264 编码")
|
||||
width, height = int(stream.get("width", 0)), int(stream.get("height", 0))
|
||||
duration = float((probe.get("format") or {}).get("duration", 0))
|
||||
if (width, height) != (settings.video_width, settings.video_height):
|
||||
raise VideoRenderError(f"输出分辨率异常:{width} × {height}")
|
||||
if not 0 < duration <= settings.video_max_duration_seconds:
|
||||
raise VideoRenderError(f"输出视频时长异常:{duration:.1f} 秒")
|
||||
data = output_path.read_bytes()
|
||||
if len(data) != file_size:
|
||||
raise VideoRenderError("读取视频输出失败")
|
||||
return RenderedVideo(data=data, duration_seconds=duration, width=width, height=height)
|
||||
@@ -0,0 +1,105 @@
|
||||
"""远程图片下载的 SSRF 防护回归测试。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
from pathlib import Path
|
||||
from io import BytesIO
|
||||
|
||||
import httpx
|
||||
from PIL import Image
|
||||
|
||||
SERVER_DIR = Path(__file__).resolve().parents[1]
|
||||
if str(SERVER_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(SERVER_DIR))
|
||||
|
||||
from services.storage import (
|
||||
ResourceDownloadTimeoutError,
|
||||
ImageValidationError,
|
||||
UnsafeRemoteUrlError,
|
||||
download_bytes,
|
||||
normalize_image_bytes,
|
||||
validate_remote_url,
|
||||
)
|
||||
|
||||
|
||||
class RemoteUrlValidationTests(unittest.IsolatedAsyncioTestCase):
|
||||
async def test_allows_public_literal_ip(self) -> None:
|
||||
await validate_remote_url("https://1.1.1.1/image.jpg")
|
||||
|
||||
async def test_rejects_local_and_private_targets(self) -> None:
|
||||
for url in (
|
||||
"http://localhost:8800/image.jpg",
|
||||
"http://127.0.0.1/image.jpg",
|
||||
"http://10.0.0.1/image.jpg",
|
||||
"http://169.254.169.254/latest/meta-data/",
|
||||
"http://[::1]/image.jpg",
|
||||
):
|
||||
with self.subTest(url=url), self.assertRaises(UnsafeRemoteUrlError):
|
||||
await validate_remote_url(url)
|
||||
|
||||
async def test_download_rejects_before_opening_a_local_connection(self) -> None:
|
||||
with self.assertRaises(UnsafeRemoteUrlError):
|
||||
await download_bytes("http://127.0.0.1:1/image.jpg")
|
||||
|
||||
async def test_redirect_target_is_validated_again(self) -> None:
|
||||
async def handler(request: httpx.Request) -> httpx.Response:
|
||||
self.assertEqual(request.url.host, "1.1.1.1")
|
||||
return httpx.Response(
|
||||
302,
|
||||
headers={"location": "http://127.0.0.1:9000/private.jpg"},
|
||||
request=request,
|
||||
)
|
||||
|
||||
real_async_client = httpx.AsyncClient
|
||||
with patch(
|
||||
"services.storage.httpx.AsyncClient",
|
||||
side_effect=lambda **kwargs: real_async_client(
|
||||
transport=httpx.MockTransport(handler), **kwargs
|
||||
),
|
||||
), self.assertRaises(UnsafeRemoteUrlError):
|
||||
await download_bytes("https://1.1.1.1/redirect.jpg")
|
||||
|
||||
async def test_http_client_timeout_is_normalized(self) -> None:
|
||||
async def handler(request: httpx.Request) -> httpx.Response:
|
||||
raise httpx.ReadTimeout("slow response", request=request)
|
||||
|
||||
real_async_client = httpx.AsyncClient
|
||||
with patch(
|
||||
"services.storage.httpx.AsyncClient",
|
||||
side_effect=lambda **kwargs: real_async_client(
|
||||
transport=httpx.MockTransport(handler), **kwargs
|
||||
),
|
||||
), self.assertRaises(ResourceDownloadTimeoutError):
|
||||
await download_bytes("https://1.1.1.1/slow.jpg")
|
||||
|
||||
|
||||
class ImageNormalizationTests(unittest.TestCase):
|
||||
def test_uses_real_format_and_corrects_exif_orientation(self) -> None:
|
||||
source = Image.new("RGB", (30, 10), "red")
|
||||
exif = Image.Exif()
|
||||
exif[274] = 6 # 顺时针旋转 90°
|
||||
raw = BytesIO()
|
||||
source.save(raw, format="JPEG", exif=exif)
|
||||
|
||||
image = normalize_image_bytes(raw.getvalue(), max_pixels=1_000)
|
||||
|
||||
self.assertEqual(image.content_type, "image/jpeg")
|
||||
self.assertEqual((image.width, image.height), (10, 30))
|
||||
self.assertTrue(image.orientation_normalized)
|
||||
with Image.open(BytesIO(image.data)) as stored:
|
||||
self.assertNotEqual(stored.getexif().get(274), 6)
|
||||
|
||||
def test_rejects_invalid_content_and_pixel_bomb(self) -> None:
|
||||
with self.assertRaises(ImageValidationError):
|
||||
normalize_image_bytes(b"not an image")
|
||||
|
||||
raw = BytesIO()
|
||||
Image.new("RGB", (11, 10), "blue").save(raw, format="PNG")
|
||||
with self.assertRaises(ImageValidationError):
|
||||
normalize_image_bytes(raw.getvalue(), max_pixels=100)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,26 @@
|
||||
"""轮播视频的无 FFmpeg 依赖单测:时长策略是 API/前端说明的共同契约。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
SERVER_DIR = Path(__file__).resolve().parents[1]
|
||||
if str(SERVER_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(SERVER_DIR))
|
||||
|
||||
from services.video_renderer import VideoRenderError, slide_seconds_for
|
||||
|
||||
|
||||
class SlideDurationTests(unittest.TestCase):
|
||||
def test_duration_strategy_for_two_to_five_images(self) -> None:
|
||||
self.assertEqual([slide_seconds_for(count) for count in range(2, 6)], [4.5, 3.5, 3.0, 2.8])
|
||||
|
||||
def test_rejects_unsupported_count(self) -> None:
|
||||
for count in (0, 1, 6):
|
||||
with self.subTest(count=count), self.assertRaises(VideoRenderError):
|
||||
slide_seconds_for(count)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user