Refine seller store implementation

This commit is contained in:
R524809
2026-08-31 14:37:28 +08:00
parent 957289f717
commit 2a667b4f3b
27 changed files with 1650 additions and 624 deletions
+16 -1
View File
@@ -20,7 +20,14 @@ from pydantic import BaseModel, Field
from api.proxy import guess_referer
from config import get_settings
from services.storage import ResourceSizeLimitError, download_bytes, local_path, read_local_bytes
from services.storage import (
ResourceDownloadTimeoutError,
ResourceSizeLimitError,
UnsafeRemoteUrlError,
download_bytes,
local_path,
read_local_bytes,
)
router = APIRouter(prefix="/api", tags=["export"])
@@ -111,8 +118,12 @@ async def export_images(req: ExportImagesRequest):
for img in req.images:
try:
data, ctype = await _download(img.url)
except UnsafeRemoteUrlError as exc:
raise HTTPException(status_code=400, detail=f"图片地址不允许导出:{exc}") from exc
except ResourceSizeLimitError as exc:
raise HTTPException(status_code=413, detail=f"无法导出「{img.key or '图片'}」:{exc}") from exc
except ResourceDownloadTimeoutError as exc:
raise HTTPException(status_code=504, detail=str(exc)) from exc
except Exception: # noqa: BLE001
continue # 单张下载失败不中断整包
if not _is_image(img.url, ctype):
@@ -161,8 +172,12 @@ async def export_image(img: ExportImageItem):
"""代理下载单张图片,兼容源站防盗链与本地媒体路径。"""
try:
data, ctype = await _download(img.url)
except UnsafeRemoteUrlError as exc:
raise HTTPException(status_code=400, detail=f"图片地址不允许下载:{exc}") from exc
except ResourceSizeLimitError as exc:
raise HTTPException(status_code=413, detail=f"图片文件过大,无法下载:{exc}") from exc
except ResourceDownloadTimeoutError as exc:
raise HTTPException(status_code=504, detail=str(exc)) from exc
except Exception as exc: # noqa: BLE001
raise HTTPException(status_code=422, detail="图片下载失败") from exc
if not _is_image(img.url, ctype):
+22 -2
View File
@@ -16,6 +16,7 @@ from models.enums import AssetStatus, Stage
from schemas.collection import MaterialsRequest, MaterialsResponse, TextMaterial
from schemas.suite import SuiteOut, WatermarkOptions
from services.image_assets import WatermarkUpdate, get_image_asset_service
from services.storage import ImageValidationError, ResourceSizeLimitError, normalize_image_bytes
from services.jobs import create_image_job, get_job, list_product_jobs, suite_view
from config import get_settings
@@ -258,6 +259,15 @@ async def upload_material_bytes(
raise HTTPException(status_code=413, detail=f"图片文件过大,单张最多 {limit // 1024 // 1024} MB")
chunks.append(chunk)
data = b"".join(chunks)
if type != "img":
raise HTTPException(status_code=400, detail="该接口目前仅支持上传图片素材")
try:
normalized = normalize_image_bytes(data, max_bytes=limit)
except ResourceSizeLimitError as exc:
raise HTTPException(status_code=413, detail=str(exc)) from exc
except ImageValidationError as exc:
raise HTTPException(status_code=422, detail=str(exc)) from exc
asset = ProductAsset(
product_id=product.id,
group_key=group_key,
@@ -272,13 +282,23 @@ async def upload_material_bytes(
# 直接转存为素材唯一原图(current 与 original 初始指向同一对象)
image_assets = get_image_asset_service()
try:
await image_assets.save_original(asset, data, file.content_type or "")
await image_assets.save_original(
asset,
normalized.data,
normalized.content_type,
max_bytes=limit,
)
asset.status = AssetStatus.uploaded
except Exception as exc: # noqa: BLE001
asset.status = AssetStatus.failed
asset.error = str(exc)[:500]
await db.commit()
return {"asset_id": str(asset.id), "status": asset.status.value}
return {
"asset_id": str(asset.id),
"status": asset.status.value,
"width": asset.width,
"height": asset.height,
}
@router.get("/products/{product_id}/fingerprints")
+10 -1
View File
@@ -8,7 +8,12 @@ from urllib.parse import urlparse
from fastapi import APIRouter, HTTPException, Query, Response
from services.storage import ResourceSizeLimitError, download_bytes
from services.storage import (
ResourceDownloadTimeoutError,
ResourceSizeLimitError,
UnsafeRemoteUrlError,
download_bytes,
)
router = APIRouter(prefix="/api", tags=["proxy"])
@@ -40,8 +45,12 @@ async def proxy_image(url: str = Query(..., description="源站图片 URL")):
raise HTTPException(status_code=400, detail="仅支持 http/https URL")
try:
data, ctype = await download_bytes(url, referer=guess_referer(url))
except UnsafeRemoteUrlError as exc:
raise HTTPException(status_code=400, detail=f"图片地址不允许代理:{exc}") from exc
except ResourceSizeLimitError as exc:
raise HTTPException(status_code=413, detail=f"图片文件过大,无法代理:{exc}") from exc
except ResourceDownloadTimeoutError as exc:
raise HTTPException(status_code=504, detail=str(exc)) from exc
except Exception as exc: # noqa: BLE001
raise HTTPException(status_code=502, detail=f"图片拉取失败: {exc}") from exc
if not ctype.startswith("image/"):
+140
View File
@@ -0,0 +1,140 @@
"""商品图片轮播视频:复用后台任务状态机,FFmpeg 在独立 Worker 中执行。"""
from __future__ import annotations
import uuid
from pathlib import Path
from fastapi import APIRouter, HTTPException
from fastapi.responses import FileResponse, RedirectResponse
from sqlalchemy import select
from db import get_session_factory
from models import Product, ProductAsset
from models.enums import AssetStatus
from schemas.video import VideoCreateRequest, VideoCreateResponse, VideoDeleteResponse, VideoJobOut
from services.jobs import create_image_job, delete_job, get_job, list_product_jobs, retry_failed_items, video_job_view
from services.storage import get_storage
from services.storage import local_path
from services.video_renderer import slide_seconds_for
router = APIRouter(prefix="/api", tags=["video"])
VIDEO_JOB_KIND = "slideshow_video"
def _job_out(result) -> VideoJobOut:
return VideoJobOut(**video_job_view(*result))
@router.post("/products/{product_id}/video-jobs", response_model=VideoCreateResponse)
async def create_video_job(product_id: str, req: VideoCreateRequest) -> VideoCreateResponse:
try:
product_uuid = uuid.UUID(product_id)
asset_ids = [uuid.UUID(asset_id) for asset_id in req.asset_ids]
except ValueError as exc:
raise HTTPException(status_code=400, detail="商品或图片 ID 不合法") from exc
async with get_session_factory()() as db:
product = await db.get(Product, product_uuid)
if product is None:
raise HTTPException(status_code=404, detail="商品不存在")
assets = (await db.scalars(
select(ProductAsset).where(
ProductAsset.product_id == product_uuid,
ProductAsset.id.in_(asset_ids),
ProductAsset.type == "img",
ProductAsset.status == AssetStatus.uploaded,
)
)).all()
by_id = {asset.id: asset for asset in assets}
ordered = [by_id.get(asset_id) for asset_id in asset_ids]
if any(asset is None or not asset.stored_url for asset in ordered):
raise HTTPException(status_code=400, detail="所选图片必须属于当前商品且已完成转存")
seconds = slide_seconds_for(len(ordered))
task = await create_image_job(
product_id=product_id,
kind=VIDEO_JOB_KIND,
priority=5,
provider="ffmpeg",
model=None,
platform="wb_ozon",
lang="ru",
ratio="3:4",
style_set=0,
request_payload={
"preset": "wb_ozon",
"slide_seconds": seconds,
"sources": [
{"asset_id": str(asset.id), "url": asset.stored_url, "storage_key": asset.current_storage_key}
for asset in ordered if asset is not None
],
},
item_payloads=[{"title": "商品轮播视频", "kind": VIDEO_JOB_KIND}],
)
return VideoCreateResponse(job_id=str(task.id))
@router.get("/products/{product_id}/video-jobs", response_model=list[VideoJobOut])
async def list_video_jobs(product_id: str, active_only: bool = False):
jobs = await list_product_jobs(product_id, active_only=active_only, kinds=(VIDEO_JOB_KIND,))
views: list[VideoJobOut] = []
for job in jobs:
result = await get_job(str(job.id))
if result is not None:
views.append(_job_out(result))
return views
@router.get("/video-jobs/{job_id}", response_model=VideoJobOut)
async def get_video_job(job_id: str):
result = await get_job(job_id)
if result is None or result[0].kind != VIDEO_JOB_KIND:
raise HTTPException(status_code=404, detail="视频任务不存在")
return _job_out(result)
@router.post("/video-jobs/{job_id}/cancel", response_model=VideoDeleteResponse)
async def cancel_video_job(job_id: str) -> VideoDeleteResponse:
# 视频任务不像生图任务需要保留诊断历史:取消后立即从页面和数据库消失。
return await _purge_video_job(job_id)
@router.post("/video-jobs/{job_id}/retry", response_model=VideoJobOut)
async def retry_video_job(job_id: str):
job = await retry_failed_items(job_id)
result = await get_job(str(job.id)) if job and job.kind == VIDEO_JOB_KIND else None
if result is None:
raise HTTPException(status_code=404, detail="视频任务不存在")
return _job_out(result)
async def _purge_video_job(job_id: str) -> VideoDeleteResponse:
result = await get_job(job_id)
if result is None or result[0].kind != VIDEO_JOB_KIND:
raise HTTPException(status_code=404, detail="视频任务不存在")
job, items = result
# 先删该任务专属目录;即使 FFmpeg 正在运行,Worker 下一次检查也会发现任务不存在并终止。
if job.product_id:
await get_storage().delete_prefix(f"products/{job.product_id}/videos/{job.id}/")
if not await delete_job(job_id):
raise HTTPException(status_code=404, detail="视频任务不存在")
return VideoDeleteResponse()
@router.delete("/video-jobs/{job_id}", response_model=VideoDeleteResponse)
async def delete_video_job(job_id: str) -> VideoDeleteResponse:
"""删除完成/失败视频及其 MP4;任务记录不保留。"""
return await _purge_video_job(job_id)
@router.get("/video-jobs/{job_id}/download")
async def download_video(job_id: str):
result = await get_job(job_id)
if result is None or result[0].kind != VIDEO_JOB_KIND or not result[1] or not result[1][0].result_url:
raise HTTPException(status_code=404, detail="视频文件不存在")
url = result[1][0].result_url
path = local_path(url)
filename = f"product-video-{job_id}.mp4"
if path is not None and Path(path).is_file():
return FileResponse(path, media_type="video/mp4", filename=filename)
return RedirectResponse(url, status_code=307)
+23
View File
@@ -53,12 +53,35 @@ class Settings(BaseSettings):
# 单张图片在后端需要解码/加水印,限制单文件可避免一次请求耗尽内存。
image_upload_max_bytes: int = 20 * 1024 * 1024
image_download_max_bytes: int = 20 * 1024 * 1024
# 图片解码前依据真实宽高限制像素,避免超大图耗尽 Pillow/水印处理内存。
image_max_pixels: int = 40_000_000
# 远程图片下载防护:连接/读写/总耗时分别受控;重定向逐跳校验目标地址。
image_download_connect_timeout: float = 10.0
image_download_read_timeout: float = 30.0
image_download_total_timeout: float = 60.0
image_download_max_redirects: int = 3
collection_max_images: int = 100
export_max_images: int = 50
export_zip_max_bytes: int = 200 * 1024 * 1024
# ZIP 超过该值自动落临时磁盘,不继续占用进程内存。
export_zip_memory_threshold_bytes: int = 2 * 1024 * 1024
# ── 商品轮播视频(FFmpeg)──
# Ubuntu 默认安装包会提供同名命令;容器或特殊路径可在 .env 覆盖。
video_ffmpeg_path: str = "ffmpeg"
video_ffprobe_path: str = "ffprobe"
video_width: int = 810
video_height: int = 1080
video_fps: int = 30
video_transition_seconds: float = 0.35
video_crf: int = 22
video_max_rate: str = "3M"
video_buffer_size: str = "6M"
video_max_bytes: int = 45 * 1024 * 1024
video_max_duration_seconds: int = 180
video_render_timeout_seconds: int = 120
background_job_lease_seconds: int = 300
# ── V2.1 套图生图(provider 路由与 image-suite-studio 一致)──
image_provider: str = "doubao" # 未知模型时的兜底 provider:doubao | tongyi | rightapi
request_timeout: int = 300 # 单张生图请求超时(秒)
+2 -1
View File
@@ -5,7 +5,7 @@ from fastapi.middleware.cors import CORSMiddleware
from fastapi.staticfiles import StaticFiles
from sqlalchemy import text
from api import ai, export, fx, image, materials, products, proxy, suite
from api import ai, export, fx, image, materials, products, proxy, suite, video
from config import get_settings
from db import get_engine
@@ -36,6 +36,7 @@ app.include_router(image.router)
app.include_router(suite.router)
app.include_router(export.router)
app.include_router(proxy.router)
app.include_router(video.router)
# ⚠️ 冻结链路(Ozon API 直传):代码移至 legacy/,接口保留但不再投入
from legacy.api import categories, ozon, publish, shops # noqa: E402
@@ -19,7 +19,14 @@ depends_on = None
def upgrade() -> None:
# 开发态 FastAPI 会在 startup 以 Base.metadata.create_all 兜底;若它先于 Alembic
# 启动,表已完整存在,此处只补迁移版本记录,避免 create_table 冲突。
if "background_jobs" in sa.inspect(op.get_bind()).get_table_names():
inspector = sa.inspect(op.get_bind())
if "background_jobs" in inspector.get_table_names():
# 开发态若先由 metadata.create_all 建表,仍补上后来新增的索引。
existing_indexes = {
index["name"] for index in inspector.get_indexes("background_job_items")
}
if "ix_background_job_items_updated_at" not in existing_indexes:
op.create_index("ix_background_job_items_updated_at", "background_job_items", ["updated_at"])
return
op.create_table(
"background_jobs",
@@ -85,8 +92,10 @@ def upgrade() -> None:
op.create_index("ix_background_job_items_provider_task_id", "background_job_items", ["provider_task_id"])
op.create_index("ix_background_job_items_next_run_at", "background_job_items", ["next_run_at"])
op.create_index("ix_background_job_items_lease_expires_at", "background_job_items", ["lease_expires_at"])
op.create_index("ix_background_job_items_updated_at", "background_job_items", ["updated_at"])
def downgrade() -> None:
op.drop_index("ix_background_job_items_updated_at", table_name="background_job_items")
op.drop_table("background_job_items")
op.drop_table("background_jobs")
+56
View File
@@ -0,0 +1,56 @@
"""轮播商品视频的请求与任务响应契约。"""
from __future__ import annotations
from datetime import datetime
from pydantic import BaseModel, Field, field_validator
class VideoCreateRequest(BaseModel):
"""按传入顺序把 2–5 张当前商品图片合成为无音轨轮播视频。"""
asset_ids: list[str] = Field(min_length=2, max_length=5)
@field_validator("asset_ids")
@classmethod
def asset_ids_must_be_unique(cls, values: list[str]) -> list[str]:
if len(set(values)) != len(values):
raise ValueError("合成视频不能重复选择同一张图片")
return values
class VideoCreateResponse(BaseModel):
job_id: str
class VideoDeleteResponse(BaseModel):
deleted: bool = True
class VideoResultOut(BaseModel):
item_id: str
status: str
url: str | None = None
error: str | None = None
class VideoJobOut(BaseModel):
id: str
kind: str = "slideshow_video"
status: str
total: int = 1
completed: int = 0
failed: int = 0
cancel_requested: bool = False
preset: str = "wb_ozon"
source_asset_ids: list[str] = Field(default_factory=list)
slide_seconds: float = 0.0
duration_seconds: float | None = None
width: int = 810
height: int = 1080
file_size_bytes: int | None = None
created_at: datetime | None = None
started_at: datetime | None = None
completed_at: datetime | None = None
result: VideoResultOut | None = None
error: str | None = None
@@ -0,0 +1,69 @@
"""为历史图片素材补齐真实宽高,不改写已有图片文件。"""
from __future__ import annotations
import argparse
import asyncio
import mimetypes
import sys
from pathlib import Path
from sqlalchemy import or_, select
SERVER_DIR = Path(__file__).resolve().parents[1]
if str(SERVER_DIR) not in sys.path:
sys.path.insert(0, str(SERVER_DIR))
from db import get_session_factory
from config import get_settings
from models import ProductAsset
from services.storage import local_path, normalize_image_bytes, read_local_bytes
async def backfill(dry_run: bool = False) -> tuple[int, list[str]]:
"""返回成功回填数和最多十条失败摘要。"""
updated = 0
failures: list[str] = []
async with get_session_factory()() as db:
assets = (await db.scalars(
select(ProductAsset).where(
ProductAsset.type == "img",
or_(ProductAsset.width.is_(None), ProductAsset.height.is_(None)),
)
)).all()
for asset in assets:
url = asset.original_url or asset.stored_url
path = local_path(url or "")
if path is None:
if len(failures) < 10:
failures.append(f"{asset.id}: 本地原图不存在,跳过")
continue
try:
data = read_local_bytes(path)
image = normalize_image_bytes(
data,
max_bytes=max(len(data), get_settings().image_download_max_bytes),
)
asset.width = image.width
asset.height = image.height
updated += 1
except Exception as exc: # noqa: BLE001
if len(failures) < 10:
failures.append(f"{asset.id}: {exc}")
if not dry_run:
await db.commit()
return updated, failures
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--dry-run", action="store_true", help="只检查,不写数据库")
args = parser.parse_args()
updated, failures = asyncio.run(backfill(dry_run=args.dry_run))
mode = "将回填" if args.dry_run else "已回填"
print(f"{mode} {updated} 条图片素材的宽高")
for failure in failures:
print(f"跳过:{failure}")
if __name__ == "__main__":
main()
+29 -3
View File
@@ -11,8 +11,17 @@ import uuid
from dataclasses import dataclass
from api.proxy import guess_referer
from config import get_settings
from models import ProductAsset
from services.storage import download_bytes, get_storage, image_extension, local_path, read_local_bytes, storage_key_from_url
from services.storage import (
download_bytes,
get_storage,
image_extension,
local_path,
normalize_image_bytes,
read_local_bytes,
storage_key_from_url,
)
from services.watermark import apply_watermark
@@ -46,8 +55,20 @@ class ImageAssetService:
return read_local_bytes(path), mimetypes.guess_type(path.name)[0] or "image/jpeg"
return await download_bytes(url, referer=guess_referer(url))
async def save_original(self, asset: ProductAsset, data: bytes, content_type: str = "") -> str:
async def save_original(
self,
asset: ProductAsset,
data: bytes,
content_type: str = "",
max_bytes: int | None = None,
) -> str:
"""写入该素材唯一的无水印原图,并将 current 同步到原图。"""
if asset.type == "img":
normalized = normalize_image_bytes(data, max_bytes=max_bytes)
data = normalized.data
content_type = normalized.content_type
asset.width = normalized.width
asset.height = normalized.height
key = self._key(asset, "original", data, content_type)
url = await self.storage.save_bytes(data, key, content_type)
asset.original_url = url
@@ -60,7 +81,12 @@ class ImageAssetService:
async def save_original_from_url(self, asset: ProductAsset, url: str) -> str:
data, content_type = await self._read(url)
return await self.save_original(asset, data, content_type)
return await self.save_original(
asset,
data,
content_type,
max_bytes=get_settings().image_download_max_bytes,
)
async def ensure_original(self, asset: ProductAsset) -> tuple[bytes, str]:
"""为旧记录补齐原图字段;首次触碰旧素材时迁入新的商品/素材路径。"""
+43 -1
View File
@@ -26,6 +26,7 @@ from services.storage import get_storage
from services.suite_service import append_generated_asset
from services.tasks import TaskImage
from services.watermark import apply_watermark
from services.video_renderer import VideoRenderCancelled, render_slideshow
log = logging.getLogger("background.worker")
@@ -113,11 +114,52 @@ async def _process_watermark_item(claim: ClaimedItem) -> None:
await mark_item_failed(claim.item_id, str(exc))
async def _process_slideshow_video_item(claim: ClaimedItem) -> None:
"""一个视频任务只有一个子项;输入 URL 在提交时冻结,避免后续选择状态影响成片。"""
payload = claim.request_payload
sources = list(payload.get("sources") or [])
source_urls = [str(source.get("url") or "") for source in sources]
if not 2 <= len(source_urls) <= 5 or any(not url for url in source_urls):
await mark_item_failed(claim.item_id, "视频任务缺少 2–5 张可用图片")
return
try:
async def should_cancel() -> bool:
return not await job_can_write(claim.job_id)
rendered = await render_slideshow(source_urls, should_cancel=should_cancel)
if not await job_can_write(claim.job_id):
await mark_item_cancelled(claim.item_id)
return
key = f"products/{claim.product_id}/videos/{claim.job_id}/result.mp4"
result_url = await get_storage().save_bytes(rendered.data, key, "video/mp4")
async with get_session_factory()() as db:
from models import BackgroundJobItem
item = await db.get(BackgroundJobItem, uuid.UUID(claim.item_id))
if item is not None:
item.payload = {
**dict(item.payload),
"duration_seconds": rendered.duration_seconds,
"width": rendered.width,
"height": rendered.height,
"file_size_bytes": len(rendered.data),
}
await db.commit()
await mark_item_done(claim.item_id, result_url=result_url, source_url=None, watermark_config=None, generated_asset_id=None)
except VideoRenderCancelled:
await mark_item_cancelled(claim.item_id)
except Exception as exc: # noqa: BLE001
log.exception("后台轮播视频任务 %s/%s 执行失败", claim.job_id, claim.item_id)
await mark_item_failed(claim.item_id, str(exc))
async def process_item(claim: ClaimedItem) -> None:
"""推进一次子任务。外部模型未完成时只写 provider id 并退出,让出 Worker。"""
if claim.job_kind == "watermark_batch":
await _process_watermark_item(claim)
return
if claim.job_kind == "slideshow_video":
await _process_slideshow_video_item(claim)
return
if claim.job_kind not in {"image_suite", "image_edit"}:
await mark_item_failed(claim.item_id, f"Worker 暂不支持任务类型: {claim.job_kind}")
return
@@ -201,7 +243,7 @@ async def run_worker(*, once: bool = False, idle_seconds: float = 0.75) -> None:
log.warning("恢复了 %d 个过期后台任务租约", recovered)
log.info("后台 Worker 已启动:%s", owner)
while True:
claim = await claim_next_item(owner)
claim = await claim_next_item(owner, lease_seconds=get_settings().background_job_lease_seconds)
if claim is None:
if once:
return
+37
View File
@@ -310,6 +310,21 @@ async def cancel_job(job_id: str) -> BackgroundJob | None:
return job
async def delete_job(job_id: str) -> bool:
"""硬删除一个后台任务及其子项;调用方须先处理对应的存储对象。"""
try:
parsed = uuid.UUID(job_id)
except ValueError:
return False
async with get_session_factory()() as db:
job = await db.get(BackgroundJob, parsed)
if job is None:
return False
await db.delete(job)
await db.commit()
return True
async def cancel_product_jobs(product_id: str) -> int:
"""硬删除商品前取消其未完成任务,Worker 随后只会丢弃结果而不会回写素材。"""
try:
@@ -468,3 +483,25 @@ def suite_view(job: BackgroundJob, items: list[BackgroundJobItem]) -> dict[str,
for item in items
],
}
def video_job_view(job: BackgroundJob, items: list[BackgroundJobItem]) -> dict[str, Any]:
"""轮播视频的轻量任务视图;仍复用 BackgroundJob 的恢复/取消/重试状态机。"""
payload = dict(job.request_payload or {})
item = items[0] if items else None
item_payload = dict(item.payload or {}) if item else {}
return {
"id": str(job.id), "kind": job.kind, "status": job.status,
"total": job.total, "completed": job.completed, "failed": job.failed,
"cancel_requested": job.cancel_requested, "preset": payload.get("preset", "wb_ozon"),
"source_asset_ids": [str(source.get("asset_id")) for source in payload.get("sources", [])],
"slide_seconds": payload.get("slide_seconds", 0),
"duration_seconds": item_payload.get("duration_seconds"),
"width": item_payload.get("width", 810), "height": item_payload.get("height", 1080),
"file_size_bytes": item_payload.get("file_size_bytes"),
"created_at": job.created_at, "started_at": job.started_at, "completed_at": job.completed_at,
"result": None if item is None else {
"item_id": str(item.id), "status": item.status, "url": item.result_url, "error": item.error,
},
"error": job.error,
}
+220 -24
View File
@@ -1,12 +1,20 @@
"""图片/文件存储抽象:本地文件系统(开发兜底)+ 七牛(生产)。"""
from __future__ import annotations
import asyncio
import io
import ipaddress
import mimetypes
import socket
import shutil
import uuid
import warnings
from dataclasses import dataclass
from pathlib import Path
from urllib.parse import urljoin, urlparse
import httpx
from PIL import Image, ImageOps, UnidentifiedImageError
from config import get_settings
@@ -26,6 +34,157 @@ class ResourceSizeLimitError(ValueError):
super().__init__(detail)
class UnsafeRemoteUrlError(ValueError):
"""远程资源 URL 指向回环、内网或其他保留网络,不能由服务端代请求。"""
class ResourceDownloadTimeoutError(TimeoutError):
"""远程资源下载超过连接、读取或总耗时限制。"""
class ImageValidationError(ValueError):
"""图片字节无法解码,或不符合格式与像素限制。"""
@dataclass(frozen=True)
class NormalizedImage:
"""经真实格式校验后的图片字节及可落库元数据。"""
data: bytes
content_type: str
width: int
height: int
image_format: str
orientation_normalized: bool
_IMAGE_CONTENT_TYPES = {
"JPEG": "image/jpeg",
"PNG": "image/png",
"WEBP": "image/webp",
"GIF": "image/gif",
"BMP": "image/bmp",
}
def normalize_image_bytes(
data: bytes,
max_bytes: int | None = None,
max_pixels: int | None = None,
) -> NormalizedImage:
"""按真实图片内容校验、纠正 EXIF 方向,并返回可信 MIME 与宽高。
不盲从 URL 后缀或客户端 Content-Type。为避免无意义的质量损耗,仅当 EXIF
方向不是默认值时才重编码;动画图不重编码,以免丢失帧。
"""
settings = get_settings()
byte_limit = max_bytes if max_bytes is not None else settings.image_upload_max_bytes
pixel_limit = max_pixels if max_pixels is not None else settings.image_max_pixels
if byte_limit <= 0 or pixel_limit <= 0:
raise ValueError("图片大小和像素上限必须大于 0")
if not data:
raise ImageValidationError("图片内容为空")
if len(data) > byte_limit:
raise ResourceSizeLimitError(byte_limit, len(data))
try:
# Pillow 的默认 bomb 保护不能替代项目自定义像素上限;把 warning 升级为错误。
with warnings.catch_warnings():
warnings.simplefilter("error", Image.DecompressionBombWarning)
with Image.open(io.BytesIO(data)) as image:
image_format = (image.format or "").upper()
if image_format not in _IMAGE_CONTENT_TYPES:
raise ImageValidationError(f"不支持的图片格式:{image_format or '未知'}")
width, height = image.size
if width <= 0 or height <= 0 or width * height > pixel_limit:
raise ImageValidationError(
f"图片像素超过上限(最多 {pixel_limit:,},当前 {width:,} × {height:,})"
)
# 强制实际解码,拒绝截断/伪造图片而不是只相信文件头。
image.load()
frame_count = getattr(image, "n_frames", 1)
orientation = image.getexif().get(274, 1) if frame_count == 1 else 1
normalized = orientation not in (None, 1)
output = data
if normalized:
corrected = ImageOps.exif_transpose(image)
width, height = corrected.size
if image_format == "JPEG" and corrected.mode not in {"RGB", "L"}:
corrected = corrected.convert("RGB")
buffer = io.BytesIO()
corrected.save(buffer, format=image_format)
output = buffer.getvalue()
except (UnidentifiedImageError, OSError, Image.DecompressionBombError) as exc:
raise ImageValidationError("文件不是可用的图片,或图片数据已损坏") from exc
except Image.DecompressionBombWarning as exc:
raise ImageValidationError("图片像素过大,已拒绝解码") from exc
if len(output) > byte_limit:
raise ResourceSizeLimitError(byte_limit, len(output))
return NormalizedImage(
data=output,
content_type=_IMAGE_CONTENT_TYPES[image_format],
width=width,
height=height,
image_format=image_format,
orientation_normalized=normalized,
)
def _is_public_ip(address: ipaddress.IPv4Address | ipaddress.IPv6Address) -> bool:
"""仅允许可公开路由的地址;IPv4-mapped IPv6 需按其 IPv4 实际地址判断。"""
if isinstance(address, ipaddress.IPv6Address) and address.ipv4_mapped:
return _is_public_ip(address.ipv4_mapped)
return address.is_global
async def validate_remote_url(url: str) -> None:
"""校验服务端允许代请求的公网 HTTP(S) URL。
该校验用于代理、采集转存、AI 参考图和导出。项目自己的 `/media/` 文件
在调用方会先直读磁盘;其他回环或私网 URL 一律不能通过网络访问。
"""
try:
parsed = urlparse(url)
port = parsed.port
except ValueError as exc:
raise UnsafeRemoteUrlError("图片地址端口无效") from exc
if parsed.scheme.lower() not in {"http", "https"}:
raise UnsafeRemoteUrlError("仅允许 http/https 图片地址")
if not parsed.hostname:
raise UnsafeRemoteUrlError("图片地址缺少主机名")
if parsed.username is not None or parsed.password is not None:
raise UnsafeRemoteUrlError("图片地址不能包含用户名或密码")
host = parsed.hostname.rstrip(".")
if host.lower() in {"localhost", "localhost.localdomain"}:
raise UnsafeRemoteUrlError("不允许请求 localhost 或本机网络地址")
try:
addresses = {ipaddress.ip_address(host)}
except ValueError:
try:
infos = await asyncio.get_running_loop().getaddrinfo(
host,
port or (443 if parsed.scheme.lower() == "https" else 80),
type=socket.SOCK_STREAM,
)
except socket.gaierror as exc:
raise UnsafeRemoteUrlError("图片地址域名无法解析") from exc
addresses = {ipaddress.ip_address(info[4][0]) for info in infos}
if not addresses:
raise UnsafeRemoteUrlError("图片地址域名未解析到有效地址")
private_addresses = [str(address) for address in addresses if not _is_public_ip(address)]
if private_addresses:
raise UnsafeRemoteUrlError(
f"不允许请求非公网地址({', '.join(sorted(private_addresses))})"
)
def _ext_from_content_type(content_type: str) -> str:
ctype = (content_type or "").split(";")[0].strip().lower()
mapping = {
@@ -48,41 +207,78 @@ def image_extension(data: bytes, content_type: str = "") -> str:
return ".webp"
if data[:3] == b"GIF":
return ".gif"
if data[:2] == b"\xff\xd8":
return ".jpg"
if data[:2] == b"BM":
return ".bmp"
return _ext_from_content_type(content_type)
async def download_bytes(
url: str,
referer: str | None = None,
timeout: float = 60.0,
timeout: float | None = None,
max_bytes: int | None = None,
) -> tuple[bytes, str]:
"""流式下载远程图片并限制体积。返回 (bytes, content_type)。"""
limit = max_bytes or get_settings().image_download_max_bytes
"""安全地流式下载公网资源,并限制地址、重定向、体积和耗时。"""
settings = get_settings()
limit = max_bytes or settings.image_download_max_bytes
if limit <= 0:
raise ValueError("图片下载大小上限必须大于 0")
headers = {"Referer": referer} if referer else {}
async with httpx.AsyncClient(timeout=timeout, follow_redirects=True) as client:
async with client.stream("GET", url, headers=headers) as resp:
resp.raise_for_status()
length = resp.headers.get("content-length")
if length:
try:
declared_size = int(length)
except ValueError:
declared_size = None
if declared_size is not None and declared_size > limit:
raise ResourceSizeLimitError(limit, declared_size)
total_timeout = timeout if timeout is not None else settings.image_download_total_timeout
if total_timeout <= 0:
raise ValueError("图片下载总超时必须大于 0")
if settings.image_download_connect_timeout <= 0 or settings.image_download_read_timeout <= 0:
raise ValueError("图片下载连接和读取超时必须大于 0")
if settings.image_download_max_redirects < 0:
raise ValueError("图片下载重定向上限不能小于 0")
chunks: list[bytes] = []
total = 0
async for chunk in resp.aiter_bytes():
total += len(chunk)
if total > limit:
raise ResourceSizeLimitError(limit, total)
chunks.append(chunk)
ctype = (resp.headers.get("content-type") or "application/octet-stream").split(";")[0].strip()
return b"".join(chunks), ctype
headers = {"Referer": referer} if referer else {}
http_timeout = httpx.Timeout(
connect=settings.image_download_connect_timeout,
read=settings.image_download_read_timeout,
write=settings.image_download_read_timeout,
pool=settings.image_download_connect_timeout,
)
current_url = url
try:
async with asyncio.timeout(total_timeout):
async with httpx.AsyncClient(timeout=http_timeout, follow_redirects=False) as client:
for redirect_count in range(settings.image_download_max_redirects + 1):
await validate_remote_url(current_url)
async with client.stream("GET", current_url, headers=headers) as resp:
if resp.is_redirect:
location = resp.headers.get("location")
if not location:
raise httpx.RemoteProtocolError("重定向响应缺少 Location")
if redirect_count >= settings.image_download_max_redirects:
raise httpx.TooManyRedirects("图片重定向次数超过上限")
current_url = urljoin(str(resp.url), location)
continue
resp.raise_for_status()
length = resp.headers.get("content-length")
if length:
try:
declared_size = int(length)
except ValueError:
declared_size = None
if declared_size is not None and declared_size > limit:
raise ResourceSizeLimitError(limit, declared_size)
chunks: list[bytes] = []
total = 0
async for chunk in resp.aiter_bytes():
total += len(chunk)
if total > limit:
raise ResourceSizeLimitError(limit, total)
chunks.append(chunk)
ctype = (resp.headers.get("content-type") or "application/octet-stream").split(";")[0].strip()
return b"".join(chunks), ctype
except (TimeoutError, httpx.TimeoutException) as exc:
raise ResourceDownloadTimeoutError(f"图片下载超时(最多 {total_timeout:g} 秒)") from exc
raise RuntimeError("图片下载未返回有效响应")
def read_local_bytes(path: Path, max_bytes: int | None = None) -> bytes:
+187
View File
@@ -0,0 +1,187 @@
"""用 FFmpeg 合成平台兼容的无音轨商品轮播视频。"""
from __future__ import annotations
import asyncio
import json
import os
import tempfile
from collections.abc import Awaitable, Callable
from dataclasses import dataclass
from pathlib import Path
from config import get_settings
from services.storage import (
ResourceSizeLimitError,
download_bytes,
image_extension,
local_path,
normalize_image_bytes,
read_local_bytes,
)
class VideoRenderError(RuntimeError):
"""FFmpeg 无法生成符合预设的商品视频。"""
class VideoRenderCancelled(VideoRenderError):
"""任务在 FFmpeg 运行期间被用户取消。"""
@dataclass(frozen=True)
class RenderedVideo:
data: bytes
duration_seconds: float
width: int
height: int
def slide_seconds_for(count: int) -> float:
"""按图片数量维持约 9–13 秒成片,并确保两图时也不会过短。"""
by_count = {2: 4.5, 3: 3.5, 4: 3.0, 5: 2.8}
try:
return by_count[count]
except KeyError as exc:
raise VideoRenderError("轮播视频只支持 2–5 张图片") from exc
async def _read_image(url: str) -> tuple[bytes, str]:
path = local_path(url)
if path is not None:
return read_local_bytes(path), ""
return await download_bytes(url, max_bytes=get_settings().image_download_max_bytes)
async def _run_process(
args: list[str],
*,
should_cancel: Callable[[], Awaitable[bool]],
timeout_seconds: int,
) -> None:
process = await asyncio.create_subprocess_exec(
*args,
stdout=asyncio.subprocess.DEVNULL,
stderr=asyncio.subprocess.PIPE,
)
try:
waited = 0.0
while process.returncode is None:
if await should_cancel():
process.terminate()
try:
await asyncio.wait_for(process.wait(), timeout=5)
except TimeoutError:
process.kill()
await process.wait()
raise VideoRenderCancelled("视频任务已取消")
await asyncio.sleep(0.25)
waited += 0.25
if waited > timeout_seconds:
process.kill()
await process.wait()
raise VideoRenderError(f"视频渲染超过 {timeout_seconds} 秒,请减少图片后重试")
stderr = (await process.stderr.read()).decode("utf-8", errors="replace") if process.stderr else ""
if process.returncode != 0:
raise VideoRenderError(f"FFmpeg 渲染失败:{stderr[-800:] or '未知错误'}")
finally:
if process.returncode is None:
process.kill()
await process.wait()
async def _probe(path: Path) -> dict:
settings = get_settings()
process = await asyncio.create_subprocess_exec(
settings.video_ffprobe_path,
"-v", "error", "-show_entries", "format=duration,size:stream=codec_name,width,height,pix_fmt",
"-of", "json", str(path),
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
stdout, stderr = await process.communicate()
if process.returncode != 0:
raise VideoRenderError(f"无法校验视频输出:{stderr.decode('utf-8', errors='replace')[-400:]}")
try:
return json.loads(stdout)
except json.JSONDecodeError as exc:
raise VideoRenderError("FFprobe 未返回可解析的视频信息") from exc
async def render_slideshow(
source_urls: list[str],
*,
should_cancel: Callable[[], Awaitable[bool]],
) -> RenderedVideo:
"""将已入库图片缩放留白后串为 MP4,严禁接收调用方任意命令参数。"""
settings = get_settings()
if not settings.video_ffmpeg_path or not settings.video_ffprobe_path:
raise VideoRenderError("未配置 FFmpeg 或 FFprobe 可执行路径")
count = len(source_urls)
seconds = slide_seconds_for(count)
transition = settings.video_transition_seconds
with tempfile.TemporaryDirectory(prefix="osk-video-") as temp_dir:
workdir = Path(temp_dir)
inputs: list[Path] = []
for index, url in enumerate(source_urls):
if await should_cancel():
raise VideoRenderCancelled("视频任务已取消")
data, content_type = await _read_image(url)
normalized = normalize_image_bytes(data, max_bytes=settings.image_download_max_bytes)
suffix = image_extension(normalized.data, content_type or normalized.content_type)
image_path = workdir / f"input-{index}{suffix}"
image_path.write_bytes(normalized.data)
inputs.append(image_path)
output_path = workdir / "result.mp4"
args = [settings.video_ffmpeg_path, "-y", "-hide_banner", "-loglevel", "error"]
for image_path in inputs:
args.extend(["-loop", "1", "-framerate", str(settings.video_fps), "-t", str(seconds), "-i", str(image_path)])
filters: list[str] = []
for index in range(count):
filters.append(
f"[{index}:v]scale={settings.video_width}:{settings.video_height}:"
"force_original_aspect_ratio=decrease,"
f"pad={settings.video_width}:{settings.video_height}:(ow-iw)/2:(oh-ih):color=white,"
"setsar=1,format=yuv420p"
f"[slide{index}]"
)
previous = "slide0"
accumulated = seconds
for index in range(1, count):
output = f"fade{index}"
filters.append(
f"[{previous}][slide{index}]xfade=transition=fade:duration={transition}:"
f"offset={accumulated - transition:.3f}[{output}]"
)
previous = output
accumulated += seconds - transition
filters.append(f"[{previous}]fps={settings.video_fps},format=yuv420p[video]")
args.extend([
"-filter_complex", ";".join(filters), "-map", "[video]",
"-an", "-c:v", "libx264", "-profile:v", "high", "-pix_fmt", "yuv420p",
"-preset", "medium", "-crf", str(settings.video_crf),
"-maxrate", settings.video_max_rate, "-bufsize", settings.video_buffer_size,
"-movflags", "+faststart", str(output_path),
])
await _run_process(args, should_cancel=should_cancel, timeout_seconds=settings.video_render_timeout_seconds)
if not output_path.is_file():
raise VideoRenderError("FFmpeg 未生成视频文件")
file_size = output_path.stat().st_size
if file_size > settings.video_max_bytes:
raise ResourceSizeLimitError(settings.video_max_bytes, file_size)
probe = await _probe(output_path)
stream = next((item for item in probe.get("streams", []) if item.get("codec_name") == "h264"), None)
if stream is None:
raise VideoRenderError("输出视频不是 H.264 编码")
width, height = int(stream.get("width", 0)), int(stream.get("height", 0))
duration = float((probe.get("format") or {}).get("duration", 0))
if (width, height) != (settings.video_width, settings.video_height):
raise VideoRenderError(f"输出分辨率异常:{width} × {height}")
if not 0 < duration <= settings.video_max_duration_seconds:
raise VideoRenderError(f"输出视频时长异常:{duration:.1f} 秒")
data = output_path.read_bytes()
if len(data) != file_size:
raise VideoRenderError("读取视频输出失败")
return RenderedVideo(data=data, duration_seconds=duration, width=width, height=height)
+105
View File
@@ -0,0 +1,105 @@
"""远程图片下载的 SSRF 防护回归测试。"""
from __future__ import annotations
import sys
import unittest
from unittest.mock import patch
from pathlib import Path
from io import BytesIO
import httpx
from PIL import Image
SERVER_DIR = Path(__file__).resolve().parents[1]
if str(SERVER_DIR) not in sys.path:
sys.path.insert(0, str(SERVER_DIR))
from services.storage import (
ResourceDownloadTimeoutError,
ImageValidationError,
UnsafeRemoteUrlError,
download_bytes,
normalize_image_bytes,
validate_remote_url,
)
class RemoteUrlValidationTests(unittest.IsolatedAsyncioTestCase):
async def test_allows_public_literal_ip(self) -> None:
await validate_remote_url("https://1.1.1.1/image.jpg")
async def test_rejects_local_and_private_targets(self) -> None:
for url in (
"http://localhost:8800/image.jpg",
"http://127.0.0.1/image.jpg",
"http://10.0.0.1/image.jpg",
"http://169.254.169.254/latest/meta-data/",
"http://[::1]/image.jpg",
):
with self.subTest(url=url), self.assertRaises(UnsafeRemoteUrlError):
await validate_remote_url(url)
async def test_download_rejects_before_opening_a_local_connection(self) -> None:
with self.assertRaises(UnsafeRemoteUrlError):
await download_bytes("http://127.0.0.1:1/image.jpg")
async def test_redirect_target_is_validated_again(self) -> None:
async def handler(request: httpx.Request) -> httpx.Response:
self.assertEqual(request.url.host, "1.1.1.1")
return httpx.Response(
302,
headers={"location": "http://127.0.0.1:9000/private.jpg"},
request=request,
)
real_async_client = httpx.AsyncClient
with patch(
"services.storage.httpx.AsyncClient",
side_effect=lambda **kwargs: real_async_client(
transport=httpx.MockTransport(handler), **kwargs
),
), self.assertRaises(UnsafeRemoteUrlError):
await download_bytes("https://1.1.1.1/redirect.jpg")
async def test_http_client_timeout_is_normalized(self) -> None:
async def handler(request: httpx.Request) -> httpx.Response:
raise httpx.ReadTimeout("slow response", request=request)
real_async_client = httpx.AsyncClient
with patch(
"services.storage.httpx.AsyncClient",
side_effect=lambda **kwargs: real_async_client(
transport=httpx.MockTransport(handler), **kwargs
),
), self.assertRaises(ResourceDownloadTimeoutError):
await download_bytes("https://1.1.1.1/slow.jpg")
class ImageNormalizationTests(unittest.TestCase):
def test_uses_real_format_and_corrects_exif_orientation(self) -> None:
source = Image.new("RGB", (30, 10), "red")
exif = Image.Exif()
exif[274] = 6 # 顺时针旋转 90°
raw = BytesIO()
source.save(raw, format="JPEG", exif=exif)
image = normalize_image_bytes(raw.getvalue(), max_pixels=1_000)
self.assertEqual(image.content_type, "image/jpeg")
self.assertEqual((image.width, image.height), (10, 30))
self.assertTrue(image.orientation_normalized)
with Image.open(BytesIO(image.data)) as stored:
self.assertNotEqual(stored.getexif().get(274), 6)
def test_rejects_invalid_content_and_pixel_bomb(self) -> None:
with self.assertRaises(ImageValidationError):
normalize_image_bytes(b"not an image")
raw = BytesIO()
Image.new("RGB", (11, 10), "blue").save(raw, format="PNG")
with self.assertRaises(ImageValidationError):
normalize_image_bytes(raw.getvalue(), max_pixels=100)
if __name__ == "__main__":
unittest.main()
+26
View File
@@ -0,0 +1,26 @@
"""轮播视频的无 FFmpeg 依赖单测:时长策略是 API/前端说明的共同契约。"""
from __future__ import annotations
import sys
import unittest
from pathlib import Path
SERVER_DIR = Path(__file__).resolve().parents[1]
if str(SERVER_DIR) not in sys.path:
sys.path.insert(0, str(SERVER_DIR))
from services.video_renderer import VideoRenderError, slide_seconds_for
class SlideDurationTests(unittest.TestCase):
def test_duration_strategy_for_two_to_five_images(self) -> None:
self.assertEqual([slide_seconds_for(count) for count in range(2, 6)], [4.5, 3.5, 3.0, 2.8])
def test_rejects_unsupported_count(self) -> None:
for count in (0, 1, 6):
with self.subTest(count=count), self.assertRaises(VideoRenderError):
slide_seconds_for(count)
if __name__ == "__main__":
unittest.main()