doodle-store/packages/asset-sync/test/sync.test.mjs

import assert from 'node:assert/strict';
import { execFileSync } from 'node:child_process';
import {
    mkdtemp,
    mkdir,
    readFile,
    rm,
    stat,
    symlink,
    writeFile,
} from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join, resolve } from 'node:path';
import { test } from 'node:test';
import qiniu from 'qiniu';
import {
    loadConfig,
    normalizeRelativePath,
    packageRoot,
    SyncError,
} from '../src/config.mjs';
import { buildPlan, qiniuEtag } from '../src/files.mjs';
import { safeFailure, syncPlan } from '../src/sync.mjs';
import { createQiniuClient } from '../src/qiniu-client.mjs';

const jpeg = Buffer.from([
    255, 216, 255, 224, 0, 16, 74, 70, 73, 70, 0, 1, 0, 0, 0, 0,
]);
async function fixture(context) {
    const dir = await mkdtemp(join(tmpdir(), 'doodle-sync-'));
    context.after(() => rm(dir, { recursive: true, force: true }));
    const source = join(dir, 'source');
    await mkdir(join(source, 'assets/previews/math'), { recursive: true });
    await mkdir(join(source, 'user-avatars'));
    await mkdir(join(source, 'content'));
    await writeFile(join(source, 'assets/previews/math/中文 图.jpg'), jpeg);
    await writeFile(
        join(source, 'user-avatars/u.png'),
        Buffer.from([137, 80, 78, 71, 13, 10, 26, 10, 0, 0, 0, 0]),
    );
    await writeFile(join(source, 'content/page-config.json'), '{"home":null}');
    await writeFile(join(source, '.DS_Store'), 'ignored');
    const envFile = join(dir, '.env');
    await writeFile(
        envFile,
        `SOURCE_DIR=./source\nQINIU_AK=test-ak-never-log\nQINIU_SK=test-sk-never-log\nQINIU_BUCKET=test-public\nQINIU_PUBLIC_DOMAIN=https://cdn.example.test\n`,
    );
    const config = await loadConfig({ env: {}, envFile });
    const plan = await buildPlan(config);
    return { dir, source, envFile, config, plan };
}
function fakeClient() {
    const objects = new Map();
    const calls = [];
    return {
        calls,
        objects,
        async bucketInfo(bucket) {
            calls.push(['info', bucket]);
            return { private: bucket === 'test-private' ? 1 : 0 };
        },
        async stat(bucket, key) {
            calls.push(['stat', bucket, key]);
            return objects.get(`${bucket}/${key}`) ?? null;
        },
        async upload(file, buffer) {
            calls.push(['upload', file.key]);
            const id = `${file.bucket}/${file.key}`;
            if (objects.has(id))
                throw new SyncError('QINIU_REQUEST_FAILED', 'conflict', 614);
            objects.set(id, { hash: qiniuEtag(buffer), fsize: buffer.length });
        },
    };
}
const noSleep = async () => {};

test('配置：相对源路径按 env 文件定位，环境变量覆盖，拒绝非法前缀/域名/数值', async (context) => {
    const fixtureData = await fixture(context);
    assert.equal(fixtureData.config.sourceDir, fixtureData.source);
    assert.equal(fixtureData.config.prefix, 'doodle/');
    const override = await loadConfig({
        env: { QINIU_KEY_PREFIX: '' },
        envFile: fixtureData.envFile,
    });
    assert.equal(override.prefix, '');
    for (const env of [
        { SYNC_CONCURRENCY: '0' },
        { SYNC_RETRIES: '-1' },
        { QINIU_PUBLIC_DOMAIN: 'https://user:secret@host.test' },
        { QINIU_KEY_PREFIX: '../escape' },
        { QINIU_KEY_PREFIX: '/absolute' },
    ])
        await assert.rejects(
            loadConfig({ env, envFile: fixtureData.envFile }),
            SyncError,
        );
    for (const path of ['../x', 'a/../../b', '/x', 'a\\b', 'a//b', 'x\0'])
        assert.throws(() => normalizeRelativePath(path), SyncError);
});

test('ETag：固定向量覆盖空文件、普通内容与 4 MiB 分块边界', () => {
    assert.equal(qiniuEtag(Buffer.alloc(0)), 'Fto5o-5ea0sNMlW_75VgGJCv2AcJ');
    assert.equal(
        qiniuEtag(Buffer.from('hello')),
        'Fqr0xh3cxeii2r7eDztILNmuqUNN',
    );
    // 固定多块期望值由独立 SHA-1 实现生成，避免只用同一函数自我验证。
    assert.equal(
        qiniuEtag(Buffer.alloc(4194304)),
        'FivMvS848VwT631aif2dhfWV4jvD',
    );
    assert.equal(
        qiniuEtag(Buffer.alloc(4194305)),
        'lhCFgki5yzon0rjN9uJusf6qtsF6',
    );
});

test('预演：不调用七牛，不包含 AK/SK，保留目录/摘要头像与预览共用公开空间', async (context) => {
    const { config, plan } = await fixture(context);
    const report = await syncPlan(config, plan, {
        client: new Proxy(
            {},
            {
                get() {
                    throw new Error('禁止网络');
                },
            },
        ),
    });
    assert.equal(report.ok, true);
    assert.equal(report.summary.planned, 3);
    const avatar = report.files.find((file) =>
        file.relativePath.startsWith('user-avatars/'),
    );
    assert.equal(avatar.visibility, 'public');
    assert.equal(avatar.bucket, 'test-public');
    assert.equal(
        avatar.publicUrl,
        'https://cdn.example.test/doodle/user-avatars/u.png',
    );
    assert.ok(report.files.every((file) => file.bucket === avatar.bucket));
    const preview = report.files.find((file) =>
        file.relativePath.startsWith('assets/'),
    );
    assert.equal(preview.key, 'doodle/assets/previews/math/中文 图.jpg');
    assert.match(preview.publicUrl, /%E4%B8%AD%E6%96%87%20%E5%9B%BE/);
    assert.equal(preview.mimeType, 'image/jpeg');
    assert.equal(preview.sha256.length, 64);
    assert.ok(!JSON.stringify(report).includes(config.ak));
    assert.ok(!JSON.stringify(report).includes(config.sk));
    assert.equal(
        (await buildPlan(config, ['assets/previews'])).files.length,
        1,
    );
});

test('同步：上传后逐对象校验，重跑跳过，相同 key 的不同内容绝不覆盖', async (context) => {
    const { config, plan } = await fixture(context);
    const client = fakeClient();
    const first = await syncPlan(config, plan, { apply: true, client });
    assert.equal(first.summary.uploaded, 3);
    assert.equal(first.ok, true);
    assert.equal(client.objects.size, 3);
    const second = await syncPlan(config, plan, { apply: true, client });
    assert.equal(second.summary.skipped, 3);
    const item = plan.files[0];
    client.objects.set(`${item.bucket}/${item.key}`, {
        hash: 'different',
        fsize: item.size,
    });
    const third = await syncPlan(config, plan, { apply: true, client });
    assert.equal(third.summary.conflict, 1);
    assert.equal(third.summary.skipped, 2);
    assert.equal(third.ok, false);
    assert.equal(
        client.objects.get(`${item.bucket}/${item.key}`).hash,
        'different',
    );
    assert.equal(client.calls.filter(([op]) => op === 'upload').length, 3);
});

test('全批预检查：缺少凭据/空间、目标空间实际私有时不上传任何文件', async (context) => {
    const { config, plan } = await fixture(context);
    for (const changed of [
        { ...config, ak: '' },
        { ...config, publicBucket: '' },
    ]) {
        const client = fakeClient();
        const changedPlan = await buildPlan(changed);
        const result = await syncPlan(changed, changedPlan, {
            apply: true,
            client,
        });
        assert.equal(result.ok, false);
        assert.equal(client.objects.size, 0);
    }
    const missing = {
        ...plan,
        files: plan.files.map((file) =>
            file.relativePath.startsWith('user-avatars/')
                ? { ...file, bucket: null }
                : file,
        ),
    };
    const client = fakeClient();
    assert.equal(
        (await syncPlan(config, missing, { apply: true, client })).error.code,
        'BUCKET_REQUIRED',
    );
    client.bucketInfo = async () => ({ private: 1 });
    assert.equal(
        (await syncPlan(config, plan, { apply: true, client })).error.code,
        'BUCKET_VISIBILITY_MISMATCH',
    );
    assert.equal(client.objects.size, 0);
});

test('瞬时错误重试：上传成功但响应丢失时按 614 和远端摘要恢复，不覆盖', async (context) => {
    const { config, plan } = await fixture(context);
    const client = fakeClient();
    const normalUpload = client.upload;
    let lost = false;
    client.upload = async (file, buffer) => {
        await normalUpload(file, buffer);
        if (!lost) {
            lost = true;
            throw new SyncError('QINIU_REQUEST_FAILED', 'lost', 0, true);
        }
    };
    const result = await syncPlan(config, plan, {
        apply: true,
        client,
        sleep: noSleep,
    });
    assert.equal(result.ok, true);
    assert.equal(result.summary.uploaded, 3);
    assert.equal(client.objects.size, 3);
});

test('源文件变化、远端摘要错误会失败；普通异常和 SDK 凭据不会写进报告', async (context) => {
    const { config, plan, source } = await fixture(context);
    await writeFile(
        join(source, plan.files[0].relativePath),
        Buffer.concat([jpeg, Buffer.from('changed')]),
    );
    const client = fakeClient();
    const result = await syncPlan(config, plan, { apply: true, client });
    assert.equal(result.summary.failed, 1);
    assert.equal(result.files[0].error.code, 'SOURCE_CHANGED');
    const other = fakeClient();
    other.stat = async () => null;
    const verify = await syncPlan(
        config,
        { ...plan, files: [plan.files[1]] },
        { apply: true, client: other },
    );
    assert.equal(verify.files[0].error.code, 'VERIFY_FAILED');
    assert.deepEqual(safeFailure(new Error(`${config.ak} ${config.sk}`)), {
        code: 'LOCAL_OPERATION_FAILED',
        statusCode: null,
    });
});

test('扫描：拒绝软链接、超限和损坏图片，保留空普通文件', async (context) => {
    const { config, source } = await fixture(context);
    await assert.rejects(buildPlan({ ...config, maxFileBytes: 1 }), SyncError);
    await writeFile(join(source, 'empty.txt'), '');
    assert.ok(
        (await buildPlan(config)).files.some(
            (file) => file.relativePath === 'empty.txt' && file.size === 0,
        ),
    );
    await writeFile(
        join(source, 'assets/previews/math/bad.jpg'),
        '<html>denied</html>',
    );
    await assert.rejects(buildPlan(config), { code: 'INVALID_IMAGE' });
    await rm(join(source, 'assets/previews/math/bad.jpg'));
    await symlink(join(source, 'content'), join(source, 'link'));
    await assert.rejects(buildPlan(config), { code: 'SYMLINK_NOT_ALLOWED' });
});

test('官方 SDK 适配：HTTPS、明确 key 的 insertOnly token、MIME/CRC 和安全错误映射', async (context) => {
    const { config, plan } = await fixture(context);
    const file = plan.files[0];
    let token, policy, extraValue;
    const sdk = {
        ...qiniu,
        rs: {
            ...qiniu.rs,
            BucketManager: class {
                constructor(_mac, conf) {
                    assert.equal(conf.useHttpsDomain, true);
                }
                async getBucketInfo() {
                    return { resp: { statusCode: 200 }, data: { private: 0 } };
                }
                async stat() {
                    return {
                        resp: { statusCode: 612 },
                        data: { error: 'ignored' },
                    };
                }
            },
        },
        form_up: {
            ...qiniu.form_up,
            FormUploader: class {
                async put(value, key, _buffer, extra) {
                    token = value;
                    extraValue = extra;
                    return {
                        resp: { statusCode: 200 },
                        data: { key, hash: file.etag },
                    };
                }
            },
        },
    };
    const client = createQiniuClient(config, sdk);
    assert.equal(await client.stat(file.bucket, file.key), null);
    assert.equal((await client.bucketInfo(file.bucket)).private, 0);
    await client.upload(file, jpeg);
    policy = JSON.parse(
        Buffer.from(token.split(':')[2], 'base64url').toString(),
    );
    assert.equal(policy.insertOnly, 1);
    assert.equal(policy.scope, `${file.bucket}:${file.key}`);
    assert.equal(extraValue.mimeType, file.mimeType);
    assert.equal(extraValue.checkCrc, true);
    sdk.rs.BucketManager = class {
        async stat() {
            throw Object.assign(new Error(config.sk), {
                resp: { statusCode: 401 },
            });
        }
    };
    await assert.rejects(
        createQiniuClient(config, sdk).stat(file.bucket, file.key),
        (error) =>
            error.statusCode === 401 &&
            !error.retryable &&
            !error.message.includes(config.sk),
    );
});

test('CLI：默认只预演，报告独占且权限 0600；禁止报告写入源目录', async (context) => {
    const { dir, envFile, source } = await fixture(context);
    const reportPath = join(dir, 'report.json');
    const cli = resolve(packageRoot, 'src/cli.mjs');
    const result = execFileSync(
        process.execPath,
        [cli, '--env', envFile, '--report', reportPath],
        { encoding: 'utf8', env: { PATH: process.env.PATH } },
    );
    assert.equal(JSON.parse(result).mode, 'dry-run');
    assert.equal(
        JSON.parse(await readFile(reportPath, 'utf8')).summary.planned,
        3,
    );
    assert.equal((await stat(reportPath)).mode & 0o777, 0o600);
    const deniedReport = join(dir, 'no-credentials.json');
    assert.throws(() =>
        execFileSync(
            process.execPath,
            [cli, '--env', envFile, '--apply', '--report', deniedReport],
            {
                stdio: 'pipe',
                env: { PATH: process.env.PATH, QINIU_AK: '', QINIU_SK: '' },
            },
        ),
    );
    const denied = JSON.parse(await readFile(deniedReport, 'utf8'));
    assert.equal(denied.error.code, 'CREDENTIALS_REQUIRED');
    assert.equal(denied.summary.not_attempted, 3);

    assert.throws(() =>
        execFileSync(
            process.execPath,
            [cli, '--env', envFile, '--report', reportPath],
            { stdio: 'pipe' },
        ),
    );
    assert.throws(() =>
        execFileSync(
            process.execPath,
            [cli, '--env', envFile, '--report', join(source, 'report.json')],
            { stdio: 'pipe' },
        ),
    );
});