feat: 完成分类、worksheet 接口开发，资源迁移到七牛
This commit is contained in:
R524809
@@ -0,0 +1,391 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import {
|
||||
mkdtemp,
|
||||
mkdir,
|
||||
readFile,
|
||||
rm,
|
||||
stat,
|
||||
symlink,
|
||||
writeFile,
|
||||
} from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join, resolve } from 'node:path';
|
||||
import { test } from 'node:test';
|
||||
import qiniu from 'qiniu';
|
||||
import {
|
||||
loadConfig,
|
||||
normalizeRelativePath,
|
||||
packageRoot,
|
||||
SyncError,
|
||||
} from '../src/config.mjs';
|
||||
import { buildPlan, qiniuEtag } from '../src/files.mjs';
|
||||
import { safeFailure, syncPlan } from '../src/sync.mjs';
|
||||
import { createQiniuClient } from '../src/qiniu-client.mjs';
|
||||
|
||||
const jpeg = Buffer.from([
|
||||
255, 216, 255, 224, 0, 16, 74, 70, 73, 70, 0, 1, 0, 0, 0, 0,
|
||||
]);
|
||||
async function fixture(context) {
|
||||
const dir = await mkdtemp(join(tmpdir(), 'doodle-sync-'));
|
||||
context.after(() => rm(dir, { recursive: true, force: true }));
|
||||
const source = join(dir, 'source');
|
||||
await mkdir(join(source, 'assets/previews/math'), { recursive: true });
|
||||
await mkdir(join(source, 'user-avatars'));
|
||||
await mkdir(join(source, 'content'));
|
||||
await writeFile(join(source, 'assets/previews/math/中文 图.jpg'), jpeg);
|
||||
await writeFile(
|
||||
join(source, 'user-avatars/u.png'),
|
||||
Buffer.from([137, 80, 78, 71, 13, 10, 26, 10, 0, 0, 0, 0]),
|
||||
);
|
||||
await writeFile(join(source, 'content/page-config.json'), '{"home":null}');
|
||||
await writeFile(join(source, '.DS_Store'), 'ignored');
|
||||
const envFile = join(dir, '.env');
|
||||
await writeFile(
|
||||
envFile,
|
||||
`SOURCE_DIR=./source\nQINIU_AK=test-ak-never-log\nQINIU_SK=test-sk-never-log\nQINIU_BUCKET=test-public\nQINIU_PUBLIC_DOMAIN=https://cdn.example.test\n`,
|
||||
);
|
||||
const config = await loadConfig({ env: {}, envFile });
|
||||
const plan = await buildPlan(config);
|
||||
return { dir, source, envFile, config, plan };
|
||||
}
|
||||
function fakeClient() {
|
||||
const objects = new Map();
|
||||
const calls = [];
|
||||
return {
|
||||
calls,
|
||||
objects,
|
||||
async bucketInfo(bucket) {
|
||||
calls.push(['info', bucket]);
|
||||
return { private: bucket === 'test-private' ? 1 : 0 };
|
||||
},
|
||||
async stat(bucket, key) {
|
||||
calls.push(['stat', bucket, key]);
|
||||
return objects.get(`${bucket}/${key}`) ?? null;
|
||||
},
|
||||
async upload(file, buffer) {
|
||||
calls.push(['upload', file.key]);
|
||||
const id = `${file.bucket}/${file.key}`;
|
||||
if (objects.has(id))
|
||||
throw new SyncError('QINIU_REQUEST_FAILED', 'conflict', 614);
|
||||
objects.set(id, { hash: qiniuEtag(buffer), fsize: buffer.length });
|
||||
},
|
||||
};
|
||||
}
|
||||
const noSleep = async () => {};
|
||||
|
||||
test('配置：相对源路径按 env 文件定位，环境变量覆盖，拒绝非法前缀/域名/数值', async (context) => {
|
||||
const fixtureData = await fixture(context);
|
||||
assert.equal(fixtureData.config.sourceDir, fixtureData.source);
|
||||
assert.equal(fixtureData.config.prefix, 'doodle/');
|
||||
const override = await loadConfig({
|
||||
env: { QINIU_KEY_PREFIX: '' },
|
||||
envFile: fixtureData.envFile,
|
||||
});
|
||||
assert.equal(override.prefix, '');
|
||||
for (const env of [
|
||||
{ SYNC_CONCURRENCY: '0' },
|
||||
{ SYNC_RETRIES: '-1' },
|
||||
{ QINIU_PUBLIC_DOMAIN: 'https://user:secret@host.test' },
|
||||
{ QINIU_KEY_PREFIX: '../escape' },
|
||||
{ QINIU_KEY_PREFIX: '/absolute' },
|
||||
])
|
||||
await assert.rejects(
|
||||
loadConfig({ env, envFile: fixtureData.envFile }),
|
||||
SyncError,
|
||||
);
|
||||
for (const path of ['../x', 'a/../../b', '/x', 'a\\b', 'a//b', 'x\0'])
|
||||
assert.throws(() => normalizeRelativePath(path), SyncError);
|
||||
});
|
||||
|
||||
test('ETag：固定向量覆盖空文件、普通内容与 4 MiB 分块边界', () => {
|
||||
assert.equal(qiniuEtag(Buffer.alloc(0)), 'Fto5o-5ea0sNMlW_75VgGJCv2AcJ');
|
||||
assert.equal(
|
||||
qiniuEtag(Buffer.from('hello')),
|
||||
'Fqr0xh3cxeii2r7eDztILNmuqUNN',
|
||||
);
|
||||
// 固定多块期望值由独立 SHA-1 实现生成，避免只用同一函数自我验证。
|
||||
assert.equal(
|
||||
qiniuEtag(Buffer.alloc(4194304)),
|
||||
'FivMvS848VwT631aif2dhfWV4jvD',
|
||||
);
|
||||
assert.equal(
|
||||
qiniuEtag(Buffer.alloc(4194305)),
|
||||
'lhCFgki5yzon0rjN9uJusf6qtsF6',
|
||||
);
|
||||
});
|
||||
|
||||
test('预演：不调用七牛，不包含 AK/SK，保留目录/摘要头像与预览共用公开空间', async (context) => {
|
||||
const { config, plan } = await fixture(context);
|
||||
const report = await syncPlan(config, plan, {
|
||||
client: new Proxy(
|
||||
{},
|
||||
{
|
||||
get() {
|
||||
throw new Error('禁止网络');
|
||||
},
|
||||
},
|
||||
),
|
||||
});
|
||||
assert.equal(report.ok, true);
|
||||
assert.equal(report.summary.planned, 3);
|
||||
const avatar = report.files.find((file) =>
|
||||
file.relativePath.startsWith('user-avatars/'),
|
||||
);
|
||||
assert.equal(avatar.visibility, 'public');
|
||||
assert.equal(avatar.bucket, 'test-public');
|
||||
assert.equal(
|
||||
avatar.publicUrl,
|
||||
'https://cdn.example.test/doodle/user-avatars/u.png',
|
||||
);
|
||||
assert.ok(report.files.every((file) => file.bucket === avatar.bucket));
|
||||
const preview = report.files.find((file) =>
|
||||
file.relativePath.startsWith('assets/'),
|
||||
);
|
||||
assert.equal(preview.key, 'doodle/assets/previews/math/中文 图.jpg');
|
||||
assert.match(preview.publicUrl, /%E4%B8%AD%E6%96%87%20%E5%9B%BE/);
|
||||
assert.equal(preview.mimeType, 'image/jpeg');
|
||||
assert.equal(preview.sha256.length, 64);
|
||||
assert.ok(!JSON.stringify(report).includes(config.ak));
|
||||
assert.ok(!JSON.stringify(report).includes(config.sk));
|
||||
assert.equal(
|
||||
(await buildPlan(config, ['assets/previews'])).files.length,
|
||||
1,
|
||||
);
|
||||
});
|
||||
|
||||
test('同步：上传后逐对象校验，重跑跳过，相同 key 的不同内容绝不覆盖', async (context) => {
|
||||
const { config, plan } = await fixture(context);
|
||||
const client = fakeClient();
|
||||
const first = await syncPlan(config, plan, { apply: true, client });
|
||||
assert.equal(first.summary.uploaded, 3);
|
||||
assert.equal(first.ok, true);
|
||||
assert.equal(client.objects.size, 3);
|
||||
const second = await syncPlan(config, plan, { apply: true, client });
|
||||
assert.equal(second.summary.skipped, 3);
|
||||
const item = plan.files[0];
|
||||
client.objects.set(`${item.bucket}/${item.key}`, {
|
||||
hash: 'different',
|
||||
fsize: item.size,
|
||||
});
|
||||
const third = await syncPlan(config, plan, { apply: true, client });
|
||||
assert.equal(third.summary.conflict, 1);
|
||||
assert.equal(third.summary.skipped, 2);
|
||||
assert.equal(third.ok, false);
|
||||
assert.equal(
|
||||
client.objects.get(`${item.bucket}/${item.key}`).hash,
|
||||
'different',
|
||||
);
|
||||
assert.equal(client.calls.filter(([op]) => op === 'upload').length, 3);
|
||||
});
|
||||
|
||||
test('全批预检查：缺少凭据/空间、目标空间实际私有时不上传任何文件', async (context) => {
|
||||
const { config, plan } = await fixture(context);
|
||||
for (const changed of [
|
||||
{ ...config, ak: '' },
|
||||
{ ...config, publicBucket: '' },
|
||||
]) {
|
||||
const client = fakeClient();
|
||||
const changedPlan = await buildPlan(changed);
|
||||
const result = await syncPlan(changed, changedPlan, {
|
||||
apply: true,
|
||||
client,
|
||||
});
|
||||
assert.equal(result.ok, false);
|
||||
assert.equal(client.objects.size, 0);
|
||||
}
|
||||
const missing = {
|
||||
...plan,
|
||||
files: plan.files.map((file) =>
|
||||
file.relativePath.startsWith('user-avatars/')
|
||||
? { ...file, bucket: null }
|
||||
: file,
|
||||
),
|
||||
};
|
||||
const client = fakeClient();
|
||||
assert.equal(
|
||||
(await syncPlan(config, missing, { apply: true, client })).error.code,
|
||||
'BUCKET_REQUIRED',
|
||||
);
|
||||
client.bucketInfo = async () => ({ private: 1 });
|
||||
assert.equal(
|
||||
(await syncPlan(config, plan, { apply: true, client })).error.code,
|
||||
'BUCKET_VISIBILITY_MISMATCH',
|
||||
);
|
||||
assert.equal(client.objects.size, 0);
|
||||
});
|
||||
|
||||
test('瞬时错误重试：上传成功但响应丢失时按 614 和远端摘要恢复，不覆盖', async (context) => {
|
||||
const { config, plan } = await fixture(context);
|
||||
const client = fakeClient();
|
||||
const normalUpload = client.upload;
|
||||
let lost = false;
|
||||
client.upload = async (file, buffer) => {
|
||||
await normalUpload(file, buffer);
|
||||
if (!lost) {
|
||||
lost = true;
|
||||
throw new SyncError('QINIU_REQUEST_FAILED', 'lost', 0, true);
|
||||
}
|
||||
};
|
||||
const result = await syncPlan(config, plan, {
|
||||
apply: true,
|
||||
client,
|
||||
sleep: noSleep,
|
||||
});
|
||||
assert.equal(result.ok, true);
|
||||
assert.equal(result.summary.uploaded, 3);
|
||||
assert.equal(client.objects.size, 3);
|
||||
});
|
||||
|
||||
test('源文件变化、远端摘要错误会失败；普通异常和 SDK 凭据不会写进报告', async (context) => {
|
||||
const { config, plan, source } = await fixture(context);
|
||||
await writeFile(
|
||||
join(source, plan.files[0].relativePath),
|
||||
Buffer.concat([jpeg, Buffer.from('changed')]),
|
||||
);
|
||||
const client = fakeClient();
|
||||
const result = await syncPlan(config, plan, { apply: true, client });
|
||||
assert.equal(result.summary.failed, 1);
|
||||
assert.equal(result.files[0].error.code, 'SOURCE_CHANGED');
|
||||
const other = fakeClient();
|
||||
other.stat = async () => null;
|
||||
const verify = await syncPlan(
|
||||
config,
|
||||
{ ...plan, files: [plan.files[1]] },
|
||||
{ apply: true, client: other },
|
||||
);
|
||||
assert.equal(verify.files[0].error.code, 'VERIFY_FAILED');
|
||||
assert.deepEqual(safeFailure(new Error(`${config.ak} ${config.sk}`)), {
|
||||
code: 'LOCAL_OPERATION_FAILED',
|
||||
statusCode: null,
|
||||
});
|
||||
});
|
||||
|
||||
test('扫描：拒绝软链接、超限和损坏图片，保留空普通文件', async (context) => {
|
||||
const { config, source } = await fixture(context);
|
||||
await assert.rejects(buildPlan({ ...config, maxFileBytes: 1 }), SyncError);
|
||||
await writeFile(join(source, 'empty.txt'), '');
|
||||
assert.ok(
|
||||
(await buildPlan(config)).files.some(
|
||||
(file) => file.relativePath === 'empty.txt' && file.size === 0,
|
||||
),
|
||||
);
|
||||
await writeFile(
|
||||
join(source, 'assets/previews/math/bad.jpg'),
|
||||
'<html>denied</html>',
|
||||
);
|
||||
await assert.rejects(buildPlan(config), { code: 'INVALID_IMAGE' });
|
||||
await rm(join(source, 'assets/previews/math/bad.jpg'));
|
||||
await symlink(join(source, 'content'), join(source, 'link'));
|
||||
await assert.rejects(buildPlan(config), { code: 'SYMLINK_NOT_ALLOWED' });
|
||||
});
|
||||
|
||||
test('官方 SDK 适配：HTTPS、明确 key 的 insertOnly token、MIME/CRC 和安全错误映射', async (context) => {
|
||||
const { config, plan } = await fixture(context);
|
||||
const file = plan.files[0];
|
||||
let token, policy, extraValue;
|
||||
const sdk = {
|
||||
...qiniu,
|
||||
rs: {
|
||||
...qiniu.rs,
|
||||
BucketManager: class {
|
||||
constructor(_mac, conf) {
|
||||
assert.equal(conf.useHttpsDomain, true);
|
||||
}
|
||||
async getBucketInfo() {
|
||||
return { resp: { statusCode: 200 }, data: { private: 0 } };
|
||||
}
|
||||
async stat() {
|
||||
return {
|
||||
resp: { statusCode: 612 },
|
||||
data: { error: 'ignored' },
|
||||
};
|
||||
}
|
||||
},
|
||||
},
|
||||
form_up: {
|
||||
...qiniu.form_up,
|
||||
FormUploader: class {
|
||||
async put(value, key, _buffer, extra) {
|
||||
token = value;
|
||||
extraValue = extra;
|
||||
return {
|
||||
resp: { statusCode: 200 },
|
||||
data: { key, hash: file.etag },
|
||||
};
|
||||
}
|
||||
},
|
||||
},
|
||||
};
|
||||
const client = createQiniuClient(config, sdk);
|
||||
assert.equal(await client.stat(file.bucket, file.key), null);
|
||||
assert.equal((await client.bucketInfo(file.bucket)).private, 0);
|
||||
await client.upload(file, jpeg);
|
||||
policy = JSON.parse(
|
||||
Buffer.from(token.split(':')[2], 'base64url').toString(),
|
||||
);
|
||||
assert.equal(policy.insertOnly, 1);
|
||||
assert.equal(policy.scope, `${file.bucket}:${file.key}`);
|
||||
assert.equal(extraValue.mimeType, file.mimeType);
|
||||
assert.equal(extraValue.checkCrc, true);
|
||||
sdk.rs.BucketManager = class {
|
||||
async stat() {
|
||||
throw Object.assign(new Error(config.sk), {
|
||||
resp: { statusCode: 401 },
|
||||
});
|
||||
}
|
||||
};
|
||||
await assert.rejects(
|
||||
createQiniuClient(config, sdk).stat(file.bucket, file.key),
|
||||
(error) =>
|
||||
error.statusCode === 401 &&
|
||||
!error.retryable &&
|
||||
!error.message.includes(config.sk),
|
||||
);
|
||||
});
|
||||
|
||||
test('CLI：默认只预演，报告独占且权限 0600；禁止报告写入源目录', async (context) => {
|
||||
const { dir, envFile, source } = await fixture(context);
|
||||
const reportPath = join(dir, 'report.json');
|
||||
const cli = resolve(packageRoot, 'src/cli.mjs');
|
||||
const result = execFileSync(
|
||||
process.execPath,
|
||||
[cli, '--env', envFile, '--report', reportPath],
|
||||
{ encoding: 'utf8', env: { PATH: process.env.PATH } },
|
||||
);
|
||||
assert.equal(JSON.parse(result).mode, 'dry-run');
|
||||
assert.equal(
|
||||
JSON.parse(await readFile(reportPath, 'utf8')).summary.planned,
|
||||
3,
|
||||
);
|
||||
assert.equal((await stat(reportPath)).mode & 0o777, 0o600);
|
||||
const deniedReport = join(dir, 'no-credentials.json');
|
||||
assert.throws(() =>
|
||||
execFileSync(
|
||||
process.execPath,
|
||||
[cli, '--env', envFile, '--apply', '--report', deniedReport],
|
||||
{
|
||||
stdio: 'pipe',
|
||||
env: { PATH: process.env.PATH, QINIU_AK: '', QINIU_SK: '' },
|
||||
},
|
||||
),
|
||||
);
|
||||
const denied = JSON.parse(await readFile(deniedReport, 'utf8'));
|
||||
assert.equal(denied.error.code, 'CREDENTIALS_REQUIRED');
|
||||
assert.equal(denied.summary.not_attempted, 3);
|
||||
|
||||
assert.throws(() =>
|
||||
execFileSync(
|
||||
process.execPath,
|
||||
[cli, '--env', envFile, '--report', reportPath],
|
||||
{ stdio: 'pipe' },
|
||||
),
|
||||
);
|
||||
assert.throws(() =>
|
||||
execFileSync(
|
||||
process.execPath,
|
||||
[cli, '--env', envFile, '--report', join(source, 'report.json')],
|
||||
{ stdio: 'pipe' },
|
||||
),
|
||||
);
|
||||
});
|
||||
Reference in New Issue
Block a user