feat: 完成分类、worksheet 接口开发，资源迁移到七牛

2026-10-16 17:32:23 +08:00
parent e6ded00a4c
commit 8827110f2f
107 changed files with 12775 additions and 125 deletions
@@ -0,0 +1,104 @@
+import { randomUUID } from 'node:crypto';
+import { mkdir, open, realpath } from 'node:fs/promises';
+import {
+    basename,
+    dirname,
+    isAbsolute,
+    relative,
+    resolve,
+    sep,
+} from 'node:path';
+import { parseArgs } from 'node:util';
+import { loadConfig, packageRoot, SyncError } from './config.mjs';
+import { buildPlan } from './files.mjs';
+import { safeFailure, syncPlan } from './sync.mjs';
+
+/** 报告以独占、仅当前用户可读写方式创建；拒绝写入源备份目录，避免污染下次清单。 */
+async function reserveReport(path, sourceDir) {
+    const check = (target) => {
+        const inside = relative(sourceDir, target);
+        if (
+            !inside ||
+            (!isAbsolute(inside) &&
+                inside !== '..' &&
+                !inside.startsWith(`..${sep}`))
+        )
+            throw new SyncError('REPORT_INSIDE_SOURCE');
+    };
+    check(path);
+    await mkdir(dirname(path), { recursive: true, mode: 0o700 });
+    const canonicalPath = resolve(
+        await realpath(dirname(path)),
+        basename(path),
+    );
+    check(canonicalPath);
+    return open(canonicalPath, 'wx', 0o600);
+}
+
+async function main() {
+    const { values } = parseArgs({
+        options: {
+            apply: { type: 'boolean' },
+            'dry-run': { type: 'boolean' },
+            only: { type: 'string', multiple: true },
+            report: { type: 'string' },
+            env: { type: 'string' },
+            help: { type: 'boolean' },
+        },
+    });
+    if (values.help) {
+        console.log(
+            '资源同步七牛：默认预演，不修改云端或数据库。\n\npnpm assets:sync [--only assets/previews] [--report /绝对路径/report.json]\npnpm assets:sync --apply [--only assets/previews]\n\n配置：packages/asset-sync/.env；可用 --env 指定其他配置文件。AK/SK 只通过配置/环境变量提供。',
+        );
+        return;
+    }
+    if (values.apply && values['dry-run'])
+        throw new SyncError('CONFLICTING_OPTIONS');
+    const config = await loadConfig(
+        values.env ? { envFile: resolve(values.env) } : {},
+    );
+    const plan = await buildPlan(config, values.only ?? []);
+    const reportPath = values.report
+        ? resolve(values.report)
+        : resolve(packageRoot, 'reports', `${Date.now()}-${randomUUID()}.json`);
+    const output = await reserveReport(reportPath, plan.sourceDir);
+    try {
+        // 预演不加载 SDK，不要求七牛凭据，也不调用任何远程接口。
+        const client = values.apply
+            ? (await import('./qiniu-client.mjs')).createQiniuClient(config)
+            : undefined;
+        const report = await syncPlan(config, plan, {
+            apply: Boolean(values.apply),
+            client,
+            progress: (value) => console.log(JSON.stringify(value)),
+        });
+        await output.writeFile(`${JSON.stringify(report, null, 4)}\n`, 'utf8');
+        console.log(
+            JSON.stringify({
+                mode: report.mode,
+                ok: report.ok,
+                summary: report.summary,
+                ...(report.error ? { error: report.error } : {}),
+                report: reportPath,
+            }),
+        );
+        if (!report.ok) process.exitCode = 1;
+    } finally {
+        await output.close();
+    }
+}
+
+main().catch((error) => {
+    // 不打印异常对象或 stack；配置文件及 SDK 异常可能包含凭据或签名地址。
+    console.error(
+        JSON.stringify({
+            ok: false,
+            error: safeFailure(error),
+            message:
+                error instanceof SyncError
+                    ? error.message
+                    : '同步未完成，请检查配置、源目录和报告路径',
+        }),
+    );
+    process.exitCode = 1;
+});
@@ -0,0 +1,126 @@
+import { readFile } from 'node:fs/promises';
+import { dirname, isAbsolute, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { parseEnv } from 'node:util';
+
+export const packageRoot = fileURLToPath(new URL('../', import.meta.url));
+
+/** 有限错误码与固定提示；禁止把 SDK 原始异常、签名或配置值写入日志。 */
+export class SyncError extends Error {
+    constructor(code, message = code, statusCode = 0, retryable = false) {
+        super(message);
+        this.code = code;
+        this.statusCode = statusCode;
+        this.retryable = retryable;
+    }
+}
+
+/** 本地文件配置与环境变量合并，环境变量优先；相对源路径以配置文件所在目录为准。 */
+export async function loadConfig({
+    env = process.env,
+    envFile = resolve(packageRoot, '.env'),
+} = {}) {
+    let values = {};
+    try {
+        values = parseEnv(await readFile(envFile, 'utf8'));
+    } catch (error) {
+        if (error.code !== 'ENOENT')
+            throw new SyncError('CONFIG_READ_FAILED', '无法读取配置文件');
+    }
+    const merged = { ...values, ...env };
+    const number = (key, fallback, min, max) => {
+        const raw = merged[key] ?? String(fallback);
+        if (
+            !/^\d+$/.test(raw) ||
+            !Number.isSafeInteger(Number(raw)) ||
+            Number(raw) < min ||
+            Number(raw) > max
+        )
+            throw new SyncError('INVALID_CONFIG', `${key} 配置不合法`);
+        return Number(raw);
+    };
+    const source = merged.SOURCE_DIR ?? '../../../doodle-assets/cloud_backup';
+    if (!source.trim())
+        throw new SyncError('INVALID_CONFIG', 'SOURCE_DIR 不能为空');
+    const prefix = normalizeRelativePath(
+        merged.QINIU_KEY_PREFIX ?? 'doodle/',
+        true,
+    );
+    const bucket = (key) => {
+        const value = (merged[key] ?? '').trim();
+        if (value && !/^[a-z0-9][a-z0-9-]{1,62}$/.test(value))
+            throw new SyncError('INVALID_CONFIG', `${key} 配置不合法`);
+        return value;
+    };
+    const domain = (merged.QINIU_PUBLIC_DOMAIN ?? '').trim();
+    if (domain) {
+        try {
+            const url = new URL(domain);
+            if (
+                url.protocol !== 'https:' ||
+                url.username ||
+                url.password ||
+                url.pathname !== '/' ||
+                url.search ||
+                url.hash
+            )
+                throw new Error();
+        } catch {
+            throw new SyncError(
+                'INVALID_CONFIG',
+                'QINIU_PUBLIC_DOMAIN 必须为 HTTPS origin',
+            );
+        }
+    }
+    const region = (merged.QINIU_REGION ?? '').trim();
+    if (region && !/^[a-z0-9][a-z0-9-]{0,30}$/.test(region))
+        throw new SyncError('INVALID_CONFIG', 'QINIU_REGION 配置不合法');
+    return {
+        sourceDir: isAbsolute(source)
+            ? resolve(source)
+            : resolve(dirname(envFile), source),
+        ak: (merged.QINIU_AK ?? '').trim(),
+        sk: (merged.QINIU_SK ?? '').trim(),
+        publicBucket: bucket('QINIU_BUCKET'),
+        region,
+        prefix: prefix ? `${prefix}/` : '',
+        publicDomain: domain.replace(/\/$/, ''),
+        concurrency: number('SYNC_CONCURRENCY', 3, 1, 10),
+        retries: number('SYNC_RETRIES', 2, 0, 5),
+        maxFileBytes: number('SYNC_MAX_FILE_BYTES', 67108864, 1, 1073741824),
+    };
+}
+
+/** 文件 key 与筛选前缀使用 POSIX 相对路径，拒绝越界、反斜线或空路径段。 */
+export function normalizeRelativePath(value, allowEmpty = false) {
+    if (
+        typeof value !== 'string' ||
+        value.includes('\\') ||
+        [...value].some(
+            (char) => char.charCodeAt(0) < 32 || char.charCodeAt(0) === 127,
+        )
+    )
+        throw new SyncError('INVALID_PATH', '相对路径配置不合法');
+    const text = value.replace(/\/+$/, '');
+    if (!text && allowEmpty && !value.startsWith('/')) return '';
+    if (
+        !text ||
+        text.startsWith('/') ||
+        text.split('/').some((part) => !part || part === '.' || part === '..')
+    )
+        throw new SyncError('INVALID_PATH', '路径必须位于备份目录内');
+    return text;
+}
+
+/** 执行前一次检查全部目标；头像与其他资源共用 QINIU_BUCKET 公开空间。 */
+export function validateApplyConfig(config, files) {
+    if (!config.ak || !config.sk || /\s/.test(config.ak + config.sk))
+        throw new SyncError(
+            'CREDENTIALS_REQUIRED',
+            '请配置 QINIU_AK 和 QINIU_SK',
+        );
+    for (const file of files) {
+        if (!file.bucket)
+            throw new SyncError('BUCKET_REQUIRED', '请配置 QINIU_BUCKET');
+    }
+}
@@ -0,0 +1,160 @@
+import { createHash } from 'node:crypto';
+import { constants } from 'node:fs';
+import { lstat, open, readdir, realpath } from 'node:fs/promises';
+import { join, relative, sep } from 'node:path';
+import { normalizeRelativePath, SyncError } from './config.mjs';
+
+/** 七牛 ETag：4 MiB 分块 SHA-1，单块前缀 0x16，多块摘要前缀 0x96。 */
+export function qiniuEtag(buffer) {
+    const blockSize = 4 * 1024 * 1024;
+    const sha1 = (data) => createHash('sha1').update(data).digest();
+    if (buffer.length <= blockSize)
+        return Buffer.concat([Buffer.from([0x16]), sha1(buffer)]).toString(
+            'base64url',
+        );
+    const blocks = [];
+    for (let offset = 0; offset < buffer.length; offset += blockSize)
+        blocks.push(sha1(buffer.subarray(offset, offset + blockSize)));
+    return Buffer.concat([
+        Buffer.from([0x96]),
+        sha1(Buffer.concat(blocks)),
+    ]).toString('base64url');
+}
+
+/** 只读普通文件，不跟随文件或父目录软链接；限制单文件内存占用。 */
+export async function readSource(config, relativePath) {
+    relativePath = normalizeRelativePath(relativePath);
+    let parent = config.sourceDir;
+    for (const part of relativePath.split('/').slice(0, -1)) {
+        parent = join(parent, part);
+        if ((await lstat(parent)).isSymbolicLink())
+            throw new SyncError('SYMLINK_NOT_ALLOWED', '源目录包含软链接');
+    }
+    const path = join(config.sourceDir, relativePath);
+    const actual = await realpath(path);
+    if (!actual.startsWith(`${config.sourceDir}${sep}`))
+        throw new SyncError('SOURCE_OUTSIDE_ROOT');
+    const handle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW);
+    try {
+        const stat = await handle.stat();
+        if (!stat.isFile() || stat.size > config.maxFileBytes)
+            throw new SyncError(
+                'FILE_TOO_LARGE_OR_INVALID',
+                '文件超限或不是普通文件',
+            );
+        const buffer = await handle.readFile();
+        if (buffer.length > config.maxFileBytes)
+            throw new SyncError('FILE_TOO_LARGE_OR_INVALID');
+        return {
+            buffer,
+            size: buffer.length,
+            sha256: createHash('sha256').update(buffer).digest('hex'),
+            etag: qiniuEtag(buffer),
+        };
+    } finally {
+        await handle.close();
+    }
+}
+
+function mimeType(path, buffer) {
+    const head = buffer.subarray(0, 16);
+    if (
+        head
+            .subarray(0, 8)
+            .equals(Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]))
+    )
+        return 'image/png';
+    if (head[0] === 255 && head[1] === 216 && head[2] === 255)
+        return 'image/jpeg';
+    if (
+        head.toString('ascii', 0, 4) === 'RIFF' &&
+        head.toString('ascii', 8, 12) === 'WEBP'
+    )
+        return 'image/webp';
+    if (/^GIF8[79]a/.test(head.toString('ascii'))) return 'image/gif';
+    if (head.toString('ascii', 0, 4) === 'wOF2') return 'font/woff2';
+    if (head.toString('ascii', 0, 4) === 'wOFF') return 'font/woff';
+    if (head.length >= 4 && head.readUInt32BE(0) === 0x00010000)
+        return 'font/ttf';
+    if (head.toString('ascii', 0, 4) === 'OTTO') return 'font/otf';
+    if (path.toLowerCase().endsWith('.json')) {
+        try {
+            JSON.parse(buffer.toString('utf8'));
+            return 'application/json';
+        } catch {
+            throw new SyncError('INVALID_JSON', 'JSON 文件损坏');
+        }
+    }
+    if (
+        path.toLowerCase().endsWith('.svg') &&
+        /^\s*(?:<\?xml[^>]*>\s*)?<svg[\s>]/.test(buffer.toString('utf8'))
+    )
+        return 'image/svg+xml';
+    return 'application/octet-stream';
+}
+
+/** 预演同样读取文件并生成摘要；头像与其他资源统一上传到公开空间。 */
+export async function buildPlan(config, only = []) {
+    const root = await lstat(config.sourceDir);
+    if (!root.isDirectory() || root.isSymbolicLink())
+        throw new SyncError('INVALID_SOURCE_DIR', '备份路径必须为真实目录');
+    config = { ...config, sourceDir: await realpath(config.sourceDir) };
+    const filters = only.map((value) => normalizeRelativePath(value));
+    const selected = (path) =>
+        !filters.length ||
+        filters.some(
+            (prefix) => path === prefix || path.startsWith(`${prefix}/`),
+        );
+    const paths = [];
+    async function walk(dir) {
+        for (const entry of (await readdir(dir, { withFileTypes: true })).sort(
+            (a, b) => a.name.localeCompare(b.name, 'en'),
+        )) {
+            if (entry.name.startsWith('.')) continue;
+            const path = join(dir, entry.name);
+            const rel = relative(config.sourceDir, path).split(sep).join('/');
+            if (entry.isSymbolicLink())
+                throw new SyncError(
+                    'SYMLINK_NOT_ALLOWED',
+                    '备份目录包含软链接，请移除后重试',
+                );
+            if (entry.isDirectory()) await walk(path);
+            else if (entry.isFile() && selected(rel))
+                paths.push(normalizeRelativePath(rel));
+            else if (!entry.isFile())
+                throw new SyncError('UNSUPPORTED_FILE', '备份包含非普通文件');
+        }
+    }
+    await walk(config.sourceDir);
+    if (!paths.length) throw new SyncError('NO_FILES', '没有找到匹配文件');
+    const files = [];
+    for (const path of paths) {
+        const { buffer, ...hashes } = await readSource(config, path);
+        const visibility = 'public';
+        const key = `${config.prefix}${path}`;
+        if (Buffer.byteLength(key) > 750)
+            throw new SyncError('KEY_TOO_LONG', '七牛对象路径超过 750 字节');
+        const mime = mimeType(path, buffer);
+        if (
+            (path.startsWith('assets/previews/') ||
+                path.startsWith('user-avatars/')) &&
+            !mime.startsWith('image/')
+        )
+            throw new SyncError(
+                'INVALID_IMAGE',
+                '预览或头像文件不是支持的图片格式',
+            );
+        files.push({
+            relativePath: path,
+            key,
+            bucket: config.publicBucket || null,
+            visibility,
+            mimeType: mime,
+            ...hashes,
+            publicUrl: config.publicDomain
+                ? `${config.publicDomain}/${key.split('/').map(encodeURIComponent).join('/')}`
+                : null,
+        });
+    }
+    return { sourceDir: config.sourceDir, files };
+}
@@ -0,0 +1,87 @@
+import qiniu from 'qiniu';
+import { SyncError } from './config.mjs';
+
+const networkCodes = new Set([
+    'ECONNRESET',
+    'ETIMEDOUT',
+    'ECONNREFUSED',
+    'ENOTFOUND',
+    'EAI_AGAIN',
+    'UND_ERR_CONNECT_TIMEOUT',
+]);
+
+/** 仅以状态码和有限网络错误识别重试；不保留 SDK 返回体或异常信息。 */
+async function sdkCall(operation, allowed = []) {
+    let result;
+    try {
+        result = await operation();
+    } catch (error) {
+        const status =
+            Number(error?.statusCode ?? error?.resp?.statusCode) || 0;
+        if (allowed.includes(status)) return { status, data: null };
+        throw new SyncError(
+            'QINIU_REQUEST_FAILED',
+            '七牛请求失败',
+            status,
+            status === 429 ||
+                (status >= 500 && status <= 599) ||
+                networkCodes.has(error?.code),
+        );
+    }
+    const status = result?.resp?.statusCode ?? 0;
+    if (status < 200 || status >= 300) {
+        if (allowed.includes(status)) return { status, data: result.data };
+        throw new SyncError(
+            'QINIU_REQUEST_FAILED',
+            '七牛请求失败',
+            status,
+            status === 429 || (status >= 500 && status <= 599),
+        );
+    }
+    return { status, data: result.data };
+}
+
+/** 七牛官方 SDK 适配：强制 HTTPS、逐对象上传凭证、仅新增；不创建或修改空间权限。 */
+export function createQiniuClient(config, sdk = qiniu) {
+    const sdkConfig = new sdk.conf.Config({ useHttpsDomain: true });
+    if (config.region)
+        sdkConfig.regionsProvider = sdk.httpc.Region.fromRegionId(
+            config.region,
+        );
+    const mac = new sdk.auth.digest.Mac(config.ak, config.sk);
+    const manager = new sdk.rs.BucketManager(mac, sdkConfig);
+    const uploader = new sdk.form_up.FormUploader(sdkConfig);
+    return {
+        async bucketInfo(bucket) {
+            return (await sdkCall(() => manager.getBucketInfo(bucket))).data;
+        },
+        async stat(bucket, key) {
+            const result = await sdkCall(
+                () => manager.stat(bucket, key),
+                [612],
+            );
+            return result.status === 612 ? null : result.data;
+        },
+        async upload(file, buffer) {
+            const policy = new sdk.rs.PutPolicy({
+                scope: `${file.bucket}:${file.key}`,
+                insertOnly: 1,
+                expires: 3600,
+                fsizeLimit: file.size,
+            });
+            const token = policy.uploadToken(mac);
+            const extra = new sdk.form_up.PutExtra();
+            extra.fname = file.relativePath.split('/').at(-1);
+            extra.mimeType = file.mimeType;
+            extra.checkCrc = true;
+            const result = await sdkCall(() =>
+                uploader.put(token, file.key, buffer, extra),
+            );
+            if (
+                result.data?.key !== file.key ||
+                result.data?.hash !== file.etag
+            )
+                throw new SyncError('UPLOAD_RESPONSE_MISMATCH');
+        },
+    };
+}
@@ -0,0 +1,152 @@
+import { readSource } from './files.mjs';
+import { SyncError, validateApplyConfig } from './config.mjs';
+
+/** SDK 错误不透传；本地源码可用固定的错误码定位失败环节。 */
+export function safeFailure(error) {
+    return error instanceof SyncError
+        ? { code: error.code, statusCode: error.statusCode || null }
+        : { code: 'LOCAL_OPERATION_FAILED', statusCode: null };
+}
+
+/** 只重试瞬时网络/限流/服务异常；同名冲突、授权失败不做盲目覆盖或重试。 */
+export async function retry(
+    operation,
+    retries,
+    sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
+) {
+    for (let attempt = 0; ; attempt++) {
+        try {
+            return await operation();
+        } catch (error) {
+            if (
+                !(error instanceof SyncError) ||
+                !error.retryable ||
+                attempt >= retries
+            )
+                throw error;
+            await sleep(Math.min(200 * 2 ** attempt, 3000));
+        }
+    }
+}
+
+function matches(remote, file) {
+    return (
+        remote &&
+        remote.hash === file.etag &&
+        Number.isSafeInteger(remote.fsize) &&
+        remote.fsize === file.size
+    );
+}
+
+/** 默认只预演；正式同步先检查所有空间权限，之后有限并发逐项校验，失败不影响报告落盘。 */
+export async function syncPlan(
+    config,
+    plan,
+    { apply = false, client, progress = () => {}, sleep } = {},
+) {
+    const report = {
+        schemaVersion: 1,
+        mode: apply ? 'apply' : 'dry-run',
+        startedAt: new Date().toISOString(),
+        sourceDir: plan.sourceDir,
+        files: plan.files.map((file) => ({
+            ...file,
+            status: apply ? 'not_attempted' : 'planned',
+        })),
+    };
+    const run = (operation) => retry(operation, config.retries, sleep);
+    if (apply) {
+        try {
+            validateApplyConfig(config, plan.files);
+            const destinations = new Set(plan.files.map((file) => file.bucket));
+            // 包含头像在内的资源共用公开空间，执行前核对实际访问权限。
+            for (const bucket of destinations) {
+                const info = await run(() => client.bucketInfo(bucket));
+                if (info.private !== 0)
+                    throw new SyncError('BUCKET_VISIBILITY_MISMATCH');
+            }
+            let cursor = 0;
+            const worker = async () => {
+                for (;;) {
+                    const index = cursor++;
+                    const file = report.files[index];
+                    if (!file) return;
+                    try {
+                        const source = await readSource(
+                            { ...config, sourceDir: plan.sourceDir },
+                            file.relativePath,
+                        );
+                        if (
+                            source.sha256 !== file.sha256 ||
+                            source.size !== file.size
+                        )
+                            throw new SyncError('SOURCE_CHANGED');
+                        const remote = await run(() =>
+                            client.stat(file.bucket, file.key),
+                        );
+                        if (remote) {
+                            if (!matches(remote, file))
+                                throw new SyncError('REMOTE_CONFLICT');
+                            file.status = 'skipped';
+                        } else {
+                            try {
+                                await run(() =>
+                                    client.upload(file, source.buffer),
+                                );
+                            } catch (error) {
+                                // 614 表示另一任务已创建对象；按摘要复核，不覆盖同名文件。
+                                if (
+                                    !(error instanceof SyncError) ||
+                                    error.statusCode !== 614
+                                )
+                                    throw error;
+                            }
+                            const uploaded = await run(() =>
+                                client.stat(file.bucket, file.key),
+                            );
+                            if (!matches(uploaded, file))
+                                throw new SyncError('VERIFY_FAILED');
+                            file.status = 'uploaded';
+                        }
+                        file.verifiedBy = 'qiniu-etag-and-size';
+                    } catch (error) {
+                        file.status =
+                            error instanceof SyncError &&
+                            error.code === 'REMOTE_CONFLICT'
+                                ? 'conflict'
+                                : 'failed';
+                        file.error = safeFailure(error);
+                    }
+                    progress({
+                        completed: report.files.filter(
+                            (entry) => entry.status !== 'not_attempted',
+                        ).length,
+                        total: report.files.length,
+                        status: file.status,
+                    });
+                }
+            };
+            await Promise.all(
+                Array.from(
+                    { length: Math.min(config.concurrency, plan.files.length) },
+                    worker,
+                ),
+            );
+        } catch (error) {
+            report.error = safeFailure(error);
+        }
+    }
+    report.finishedAt = new Date().toISOString();
+    report.summary = {
+        total: report.files.length,
+        bytes: report.files.reduce((sum, file) => sum + file.size, 0),
+    };
+    for (const file of report.files)
+        report.summary[file.status] = (report.summary[file.status] ?? 0) + 1;
+    report.ok =
+        !report.error &&
+        !report.summary.failed &&
+        !report.summary.conflict &&
+        !report.summary.not_attempted;
+    return report;
+}