feat(api-watermark): 水印接口优化

2026-09-04 15:58:53 +08:00
parent 009082d0f4
commit f635359216
8 changed files with 7350 additions and 5436 deletions
@@ -2,7 +2,11 @@ import { existsSync } from 'node:fs';
 import { resolve } from 'node:path';
 import { loadEnvFile } from 'node:process';

-for (const directory of [process.cwd(), resolve(process.cwd(), '..'), resolve(process.cwd(), '../..')]) {
+for (const directory of [
+    process.cwd(),
+    resolve(process.cwd(), '..'),
+    resolve(process.cwd(), '../..'),
+]) {
    const path = resolve(directory, '.env');
    if (existsSync(path)) {
        loadEnvFile(path);
@@ -16,8 +20,27 @@ function required(name: string): string {
    return value;
 }

+function objectStorageCredential(
+    accessKeyName: string,
+    localFallbackName: string,
+): string {
+    return process.env[accessKeyName] || required(localFallbackName);
+}
+
 export const workerConfig = {
    databaseUrl: required('DATABASE_URL'),
    redisUrl: required('REDIS_URL'),
    storeSecretEncryptionKey: required('STORE_SECRET_ENCRYPTION_KEY'),
+    objectStorageEndpoint:
+        process.env.OBJECT_STORAGE_ENDPOINT ?? 'http://localhost:9000',
+    objectStorageForcePathStyle:
+        (process.env.OBJECT_STORAGE_FORCE_PATH_STYLE ?? 'true') === 'true',
+    objectStorageAccessKey: objectStorageCredential(
+        'OBJECT_STORAGE_ACCESS_KEY',
+        'MINIO_ROOT_USER',
+    ),
+    objectStorageSecretKey: objectStorageCredential(
+        'OBJECT_STORAGE_SECRET_KEY',
+        'MINIO_ROOT_PASSWORD',
+    ),
 };
@@ -1,16 +1,38 @@
 import { randomUUID } from 'node:crypto';
+import { S3Client } from '@aws-sdk/client-s3';
 import { Queue, Worker } from 'bullmq';
 import { Redis } from 'ioredis';
 import { createApplicationDatabase, type Database } from '@atlas/database';
 import { workerConfig } from './config.js';
 import { OutboxDispatcher, type OutboxJob } from './outbox-dispatcher.js';
-import { OzonCredentialError, OzonRequestError, verifyOzonCredentials } from './ozon-client.js';
+import {
+    OzonCredentialError,
+    OzonRequestError,
+    verifyOzonCredentials,
+} from './ozon-client.js';
 import { decryptStoreSecret } from './store-secret.crypto.js';
+import { MediaUploadCleanup } from './media-upload-cleanup.js';

-const connection = new Redis(workerConfig.redisUrl, { maxRetriesPerRequest: null });
+const connection = new Redis(workerConfig.redisUrl, {
+    maxRetriesPerRequest: null,
+});
 const database = createApplicationDatabase(workerConfig.databaseUrl);
 const queue = new Queue<OutboxJob>('atlas-dispatch', { connection });
-const dispatcher = new OutboxDispatcher(database, queue, `worker_${randomUUID()}`);
+const dispatcher = new OutboxDispatcher(
+    database,
+    queue,
+    `worker_${randomUUID()}`,
+);
+const objectStorage = new S3Client({
+    endpoint: workerConfig.objectStorageEndpoint,
+    region: 'us-east-1',
+    forcePathStyle: workerConfig.objectStorageForcePathStyle,
+    credentials: {
+        accessKeyId: workerConfig.objectStorageAccessKey,
+        secretAccessKey: workerConfig.objectStorageSecretKey,
+    },
+});
+const mediaUploadCleanup = new MediaUploadCleanup(database, objectStorage);

 const worker = new Worker<OutboxJob>(
    'atlas-dispatch',
@@ -28,28 +50,39 @@ worker.on('failed', (job, error) => {
 });

 async function verifyStoreCredentials(database: Database, job: OutboxJob) {
-    const source = await database.workspaceTransaction(job.workspaceId, async (tx) => {
-        await tx.query(
-            `UPDATE atlas.operations SET status = 'running', started_at = COALESCE(started_at, now()),
+    const source = await database.workspaceTransaction(
+        job.workspaceId,
+        async (tx) => {
+            await tx.query(
+                `UPDATE atlas.operations SET status = 'running', started_at = COALESCE(started_at, now()),
             attempt_count = attempt_count + 1 WHERE id = $1 AND status IN ('queued', 'running')`,
-            [job.payload.operation_id],
-        );
-        const result = await tx.query<{
-            platform_code: string; platform_client_id: string; secret_ciphertext: Buffer; secret_nonce: Buffer;
-        }>(
-            `SELECT p.code AS platform_code, s.platform_client_id, secret.secret_ciphertext, secret.secret_nonce
+                [job.payload.operation_id],
+            );
+            const result = await tx.query<{
+                platform_code: string;
+                platform_client_id: string;
+                secret_ciphertext: Buffer;
+                secret_nonce: Buffer;
+            }>(
+                `SELECT p.code AS platform_code, s.platform_client_id, secret.secret_ciphertext, secret.secret_nonce
             FROM atlas.operations operation JOIN atlas.stores s ON s.id = operation.resource_id AND s.deleted_at IS NULL
             JOIN atlas.platforms p ON p.id = s.platform_id JOIN atlas.store_secrets secret ON secret.id = $2
             WHERE operation.id = $1 AND operation.kind = 'store_credential_verification'
               AND secret.store_id = s.id AND secret.kind = 'seller_api_key' AND secret.revoked_at IS NULL`,
-            [job.payload.operation_id, job.payload.secret_id ?? ''],
-        );
-        return result.rows[0] ?? null;
-    });
+                [job.payload.operation_id, job.payload.secret_id ?? ''],
+            );
+            return result.rows[0] ?? null;
+        },
+    );
    if (!source) return cancelStaleOperation(database, job);
-    const apiKey = decryptStoreSecret(source.secret_ciphertext, source.secret_nonce, workerConfig.storeSecretEncryptionKey);
+    const apiKey = decryptStoreSecret(
+        source.secret_ciphertext,
+        source.secret_nonce,
+        workerConfig.storeSecretEncryptionKey,
+    );
    try {
-        if (source.platform_code.toUpperCase() !== 'OZON') throw new OzonRequestError(422);
+        if (source.platform_code.toUpperCase() !== 'OZON')
+            throw new OzonRequestError(422);
        await verifyOzonCredentials(source.platform_client_id, apiKey);
        await completeVerification(database, job, true);
    } catch (error) {
@@ -66,12 +99,14 @@ async function verifyStoreCredentials(database: Database, job: OutboxJob) {
 }

 async function cancelStaleOperation(database: Database, job: OutboxJob) {
-    await database.workspaceTransaction(job.workspaceId, (tx) => tx.query(
-        `UPDATE atlas.operations SET status = 'cancelled', completed_at = now(),
+    await database.workspaceTransaction(job.workspaceId, (tx) =>
+        tx.query(
+            `UPDATE atlas.operations SET status = 'cancelled', completed_at = now(),
         error_code = 'CREDENTIAL_REPLACED', error_message = '凭据已更新，旧验证任务已取消'
         WHERE id = $1 AND status IN ('queued', 'running')`,
-        [job.payload.operation_id],
-    ));
+            [job.payload.operation_id],
+        ),
+    );
 }

 async function completeVerification(
@@ -103,13 +138,17 @@ async function completeVerification(
                   credential_checked_at = now(), last_error_code = $2,
                   last_error_message = $3, last_error_at = now(),
                   revision = revision + 1, updated_at = now() WHERE id = $1`,
-            valid ? [job.payload.store_id] : [
-                job.payload.store_id,
-                rejectedStatus === 403 ? 'OZON_CREDENTIAL_FORBIDDEN' : 'OZON_CREDENTIAL_UNAUTHORIZED',
-                rejectedStatus === 403
-                    ? 'Ozon 拒绝访问：请检查 Client ID 与 API Key 是否属于同一账户，及密钥是否具备读取权限'
-                    : 'Ozon 未接受该 Client ID 或 API Key',
-            ],
+            valid
+                ? [job.payload.store_id]
+                : [
+                      job.payload.store_id,
+                      rejectedStatus === 403
+                          ? 'OZON_CREDENTIAL_FORBIDDEN'
+                          : 'OZON_CREDENTIAL_UNAUTHORIZED',
+                      rejectedStatus === 403
+                          ? 'Ozon 拒绝访问：请检查 Client ID 与 API Key 是否属于同一账户，及密钥是否具备读取权限'
+                          : 'Ozon 未接受该 Client ID 或 API Key',
+                  ],
        );
        await tx.query(
            valid
@@ -119,18 +158,26 @@ async function completeVerification(
                : `UPDATE atlas.operations SET status = 'failed', completed_at = now(),
                   error_code = $2, error_message = $3
                   WHERE id = $1 AND status IN ('queued', 'running')`,
-            valid ? [job.payload.operation_id] : [
-                job.payload.operation_id,
-                rejectedStatus === 403 ? 'OZON_CREDENTIAL_FORBIDDEN' : 'OZON_CREDENTIAL_UNAUTHORIZED',
-                rejectedStatus === 403
-                    ? 'Ozon 拒绝访问：请检查 Client ID 与 API Key 是否属于同一账户，及密钥是否具备读取权限'
-                    : 'Ozon 未接受该 Client ID 或 API Key',
-            ],
+            valid
+                ? [job.payload.operation_id]
+                : [
+                      job.payload.operation_id,
+                      rejectedStatus === 403
+                          ? 'OZON_CREDENTIAL_FORBIDDEN'
+                          : 'OZON_CREDENTIAL_UNAUTHORIZED',
+                      rejectedStatus === 403
+                          ? 'Ozon 拒绝访问：请检查 Client ID 与 API Key 是否属于同一账户，及密钥是否具备读取权限'
+                          : 'Ozon 未接受该 Client ID 或 API Key',
+                  ],
        );
    });
 }

-async function markRequestRejected(database: Database, job: OutboxJob, status: number) {
+async function markRequestRejected(
+    database: Database,
+    job: OutboxJob,
+    status: number,
+) {
    await database.workspaceTransaction(job.workspaceId, async (tx) => {
        const current = await tx.query<{ id: string }>(
            `SELECT s.id FROM atlas.stores s JOIN atlas.store_secrets secret ON secret.id = $2
@@ -156,7 +203,11 @@ async function markRequestRejected(database: Database, job: OutboxJob, status: n
    });
 }

-async function markUnavailable(database: Database, job: OutboxJob, _error: Error) {
+async function markUnavailable(
+    database: Database,
+    job: OutboxJob,
+    _error: Error,
+) {
    await database.workspaceTransaction(job.workspaceId, async (tx) => {
        const current = await tx.query<{ id: string }>(
            `SELECT s.id FROM atlas.stores s JOIN atlas.store_secrets secret ON secret.id = $2
@@ -180,23 +231,57 @@ async function markUnavailable(database: Database, job: OutboxJob, _error: Error
    });
 }

-let timer: ReturnType<typeof setInterval> | undefined;
+let dispatcherTimer: ReturnType<typeof setInterval> | undefined;
+let mediaCleanupTimer: ReturnType<typeof setInterval> | undefined;
 async function bootstrap() {
    await queue.waitUntilReady();
    await worker.waitUntilReady();
    await dispatcher.dispatchBatch();
-    timer = setInterval(() => void dispatcher.dispatchBatch().catch((error: unknown) => {
-        console.error('outbox_dispatch_failed', error instanceof Error ? error.message : error);
-    }), 2_000);
-    console.info('Atlas Worker is ready; consuming transactional-outbox events.');
+    dispatcherTimer = setInterval(
+        () =>
+            void dispatcher.dispatchBatch().catch((error: unknown) => {
+                console.error(
+                    'outbox_dispatch_failed',
+                    error instanceof Error ? error.message : error,
+                );
+            }),
+        2_000,
+    );
+    void mediaUploadCleanup.run().catch((error: unknown) => {
+        console.error(
+            'media_upload_cleanup_startup_failed',
+            error instanceof Error ? error.message : error,
+        );
+    });
+    mediaCleanupTimer = setInterval(
+        () =>
+            void mediaUploadCleanup.run().catch((error: unknown) => {
+                console.error(
+                    'media_upload_cleanup_failed',
+                    error instanceof Error ? error.message : error,
+                );
+            }),
+        5 * 60_000,
+    );
+    console.info(
+        'Atlas Worker is ready; consuming transactional-outbox events.',
+    );
 }
 void bootstrap().catch((error: unknown) => {
-    console.error('worker_startup_failed', error instanceof Error ? error.message : error);
+    console.error(
+        'worker_startup_failed',
+        error instanceof Error ? error.message : error,
+    );
    process.exitCode = 1;
 });
 async function shutdown() {
-    if (timer) clearInterval(timer);
-    await worker.close(); await queue.close(); await database.close(); await connection.quit();
+    if (dispatcherTimer) clearInterval(dispatcherTimer);
+    if (mediaCleanupTimer) clearInterval(mediaCleanupTimer);
+    await worker.close();
+    await queue.close();
+    objectStorage.destroy();
+    await database.close();
+    await connection.quit();
 }
 process.once('SIGINT', () => void shutdown());
 process.once('SIGTERM', () => void shutdown());
@@ -0,0 +1,107 @@
+import { DeleteObjectCommand, type S3Client } from '@aws-sdk/client-s3';
+import type { Database } from '@atlas/database';
+
+type CleanupCandidate = {
+    id: string;
+    bucket: string;
+    object_key: string;
+};
+
+export class MediaUploadCleanup {
+    private running: Promise<void> | null = null;
+
+    public constructor(
+        private readonly database: Database,
+        private readonly s3: S3Client,
+    ) {}
+
+    /**
+     * 上传 URL 到期并不代表浏览器一定上传过对象；DeleteObject 对不存在对象也安全，
+     * 所以终态意图统一尝试删除。只在删除成功后写 object_cleaned_at，失败下轮重试。
+     */
+    public async run() {
+        if (this.running) return this.running;
+        this.running = this.runOnce().finally(() => {
+            this.running = null;
+        });
+        return this.running;
+    }
+
+    private async runOnce() {
+        const workspaces = await this.database.query<{ id: string }>(
+            'SELECT id FROM atlas.workspaces',
+        );
+        let expired = 0;
+        let cleaned = 0;
+        for (const workspace of workspaces.rows) {
+            expired += await this.expirePending(workspace.id);
+            const candidates = await this.claimCandidates(workspace.id);
+            for (const candidate of candidates) {
+                try {
+                    await this.s3.send(
+                        new DeleteObjectCommand({
+                            Bucket: candidate.bucket,
+                            Key: candidate.object_key,
+                        }),
+                    );
+                    await this.markCleaned(workspace.id, candidate.id);
+                    cleaned += 1;
+                } catch (error) {
+                    console.warn(
+                        'media_upload_cleanup_failed',
+                        JSON.stringify({
+                            workspace_id: workspace.id,
+                            upload_id: candidate.id,
+                            reason:
+                                error instanceof Error
+                                    ? error.message
+                                    : 'unknown_error',
+                        }),
+                    );
+                }
+            }
+        }
+        if (expired || cleaned) {
+            console.info(
+                'media_upload_cleanup_completed',
+                JSON.stringify({ expired, cleaned }),
+            );
+        }
+    }
+
+    private async expirePending(workspaceId: string): Promise<number> {
+        return this.database.workspaceTransaction(workspaceId, async (tx) => {
+            const result = await tx.query(
+                `UPDATE atlas.media_upload_intents
+                 SET status = 'expired', error_code = 'UPLOAD_EXPIRED',
+                     error_message = '上传意图已过期', completed_at = now()
+                 WHERE status = 'pending' AND expires_at <= now()`,
+            );
+            return result.rowCount ?? 0;
+        });
+    }
+
+    private async claimCandidates(workspaceId: string) {
+        return this.database.workspaceTransaction(workspaceId, async (tx) => {
+            const candidates = await tx.query<CleanupCandidate>(
+                `SELECT id, bucket, object_key
+                 FROM atlas.media_upload_intents
+                 WHERE status IN ('failed', 'expired') AND object_cleaned_at IS NULL
+                 ORDER BY created_at
+                 LIMIT 50`,
+            );
+            return candidates.rows;
+        });
+    }
+
+    private async markCleaned(workspaceId: string, intentId: string) {
+        await this.database.workspaceTransaction(workspaceId, (tx) =>
+            tx.query(
+                `UPDATE atlas.media_upload_intents SET object_cleaned_at = now()
+                 WHERE id = $1 AND status IN ('failed', 'expired')
+                   AND object_cleaned_at IS NULL`,
+                [intentId],
+            ),
+        );
+    }
+}