feat(category): 类目树、类目属性、属性字典维护系统构建完成
This commit is contained in:
+15
-1
@@ -28,7 +28,7 @@ const connection = new Redis(workerConfig.redisUrl, {
|
||||
});
|
||||
const database = createApplicationDatabase(workerConfig.databaseUrl);
|
||||
const queue = new Queue<OutboxJob>('atlas-dispatch', { connection });
|
||||
const platformQueue = new Queue<PlatformCatalogJob>('atlas-dispatch', { connection });
|
||||
const platformQueue = new Queue<PlatformCatalogJob>('atlas-platform-catalog', { connection });
|
||||
const dispatcher = new OutboxDispatcher(
|
||||
database,
|
||||
queue,
|
||||
@@ -84,6 +84,13 @@ const worker = new Worker<OutboxJob | PlatformCatalogJob>(
|
||||
{ connection, concurrency: 5 },
|
||||
);
|
||||
|
||||
// 平台目录使用独立队列,交互型高优先级刷新不会与素材任务共用等待队列。
|
||||
const platformWorker = new Worker<PlatformCatalogJob>(
|
||||
'atlas-platform-catalog',
|
||||
async (job) => syncPlatformCatalog(database, job.data),
|
||||
{ connection, concurrency: 5 },
|
||||
);
|
||||
|
||||
async function cleanupCollection(database: Database, job: OutboxJob & { payload: { operation_id: string; collection_item_ids: string[] } }) {
|
||||
await database.workspaceTransaction(job.workspaceId, async (tx) => {
|
||||
await tx.query(`UPDATE atlas.operations SET status = 'running', started_at = coalesce(started_at, now()), attempt_count = attempt_count + 1 WHERE id = $1 AND status = 'queued'`, [job.payload.operation_id]);
|
||||
@@ -106,6 +113,10 @@ worker.on('failed', (job, error) => {
|
||||
void markCatalogSyncUnavailable(database, job.data as CatalogSyncJob, error);
|
||||
}
|
||||
});
|
||||
platformWorker.on('failed', (job) => {
|
||||
if (!job || job.attemptsMade < (job.opts.attempts ?? 1)) return;
|
||||
void markPlatformCatalogUnavailable(database, job.data);
|
||||
});
|
||||
|
||||
async function verifyStoreCredentials(database: Database, job: StoreVerificationJob) {
|
||||
const source = await database.workspaceTransaction(
|
||||
@@ -300,7 +311,9 @@ let mediaCleanupTimer: ReturnType<typeof setInterval> | undefined;
|
||||
let platformCatalogTimer: ReturnType<typeof setInterval> | undefined;
|
||||
async function bootstrap() {
|
||||
await queue.waitUntilReady();
|
||||
await platformQueue.waitUntilReady();
|
||||
await worker.waitUntilReady();
|
||||
await platformWorker.waitUntilReady();
|
||||
await dispatcher.dispatchBatch();
|
||||
await platformDispatcher.dispatchBatch();
|
||||
await platformScheduler.run();
|
||||
@@ -359,6 +372,7 @@ async function shutdown() {
|
||||
if (mediaCleanupTimer) clearInterval(mediaCleanupTimer);
|
||||
if (platformCatalogTimer) clearInterval(platformCatalogTimer);
|
||||
await worker.close();
|
||||
await platformWorker.close();
|
||||
await queue.close();
|
||||
await platformQueue.close();
|
||||
objectStorage.destroy();
|
||||
|
||||
@@ -15,20 +15,25 @@ export class PlatformCatalogDispatcher {
|
||||
) {}
|
||||
|
||||
public async dispatchBatch() {
|
||||
await this.promoteDispatchedJobs();
|
||||
const events = await this.database.transaction((tx) => tx.query<{
|
||||
id: string; run_id: string; platform_id: string;
|
||||
id: string; run_id: string; platform_id: string; priority: 'high' | 'low';
|
||||
}>(
|
||||
`WITH candidates AS (
|
||||
SELECT id FROM atlas.platform_catalog_outbox_events
|
||||
WHERE (status = 'pending' AND next_attempt_at <= now())
|
||||
OR (status = 'dispatching' AND locked_until < now())
|
||||
ORDER BY created_at FOR UPDATE SKIP LOCKED LIMIT 20
|
||||
SELECT event.id, run.priority
|
||||
FROM atlas.platform_catalog_outbox_events event
|
||||
JOIN atlas.platform_catalog_sync_runs run ON run.id = event.run_id
|
||||
WHERE (event.status = 'pending' AND event.next_attempt_at <= now())
|
||||
OR (event.status = 'dispatching' AND event.locked_until < now())
|
||||
ORDER BY CASE run.priority WHEN 'high' THEN 0 ELSE 1 END,
|
||||
event.created_at
|
||||
FOR UPDATE OF event SKIP LOCKED LIMIT 20
|
||||
)
|
||||
UPDATE atlas.platform_catalog_outbox_events event
|
||||
SET status = 'dispatching', locked_by = $1,
|
||||
locked_until = now() + interval '2 minutes', updated_at = now()
|
||||
FROM candidates WHERE event.id = candidates.id
|
||||
RETURNING event.id, event.run_id, event.platform_id`,
|
||||
RETURNING event.id, event.run_id, event.platform_id, candidates.priority`,
|
||||
[this.workerId],
|
||||
));
|
||||
for (const event of events.rows) {
|
||||
@@ -39,6 +44,7 @@ export class PlatformCatalogDispatcher {
|
||||
platformId: event.platform_id,
|
||||
}, {
|
||||
jobId: event.id,
|
||||
priority: event.priority === 'high' ? 1 : 10,
|
||||
attempts: 5,
|
||||
backoff: { type: 'exponential', delay: 5_000 },
|
||||
removeOnComplete: 1000,
|
||||
@@ -65,4 +71,29 @@ export class PlatformCatalogDispatcher {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async promoteDispatchedJobs() {
|
||||
const promoted = await this.database.query<{
|
||||
id: string; promoted_at: Date;
|
||||
}>(
|
||||
`SELECT event.id, run.promoted_at
|
||||
FROM atlas.platform_catalog_outbox_events event
|
||||
JOIN atlas.platform_catalog_sync_runs run ON run.id = event.run_id
|
||||
WHERE event.status = 'dispatched' AND run.status = 'queued'
|
||||
AND run.priority = 'high' AND run.promoted_at IS NOT NULL
|
||||
AND event.updated_at < run.promoted_at
|
||||
ORDER BY run.promoted_at LIMIT 20`,
|
||||
);
|
||||
for (const event of promoted.rows) {
|
||||
const job = await this.queue.getJob(event.id);
|
||||
// BullMQ 的有优先级任务位于 prioritized 集合,不能用 isWaiting 判断。
|
||||
// changePriority 会在任务已开始时保持当前执行,不会创建重复任务。
|
||||
if (job) await job.changePriority({ priority: 1 });
|
||||
await this.database.query(
|
||||
`UPDATE atlas.platform_catalog_outbox_events
|
||||
SET updated_at = now() WHERE id = $1 AND updated_at < $2`,
|
||||
[event.id, event.promoted_at],
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,9 +1,16 @@
|
||||
import type { Database, DatabaseTransaction } from '@atlas/database';
|
||||
|
||||
type Candidate = { scope: 'tree' | 'attributes' | 'dictionary'; scope_key: string;
|
||||
category_id: string | null; attribute_id: string | null };
|
||||
type SourceRow = { revision: number };
|
||||
type BatchItem = {
|
||||
batch_id: string;
|
||||
category_id: string;
|
||||
source_revision: number;
|
||||
priority: 'high' | 'low';
|
||||
};
|
||||
|
||||
/** 定时任务逐平台限量排队,避免启动时一次扫完全部类目和大字典。 */
|
||||
const platformActiveRunLimit = 6;
|
||||
|
||||
/** 每周只自动刷新树;类型属性仅从管理员显式创建的批次中限量领取。 */
|
||||
export class PlatformCatalogScheduler {
|
||||
public constructor(private readonly database: Database) {}
|
||||
|
||||
@@ -20,84 +27,201 @@ export class PlatformCatalogScheduler {
|
||||
|
||||
private async schedulePlatform(platformId: string) {
|
||||
await this.database.transaction(async (tx) => {
|
||||
const source = await tx.query<{ revision: number }>(
|
||||
const source = await tx.query<SourceRow>(
|
||||
`SELECT revision FROM atlas.platform_catalog_sources
|
||||
WHERE platform_id = $1 FOR UPDATE`,
|
||||
[platformId],
|
||||
);
|
||||
if (!source.rows[0]) return;
|
||||
await refreshBatchStatuses(tx, platformId, source.rows[0].revision);
|
||||
|
||||
const pending = await tx.query<{ count: string }>(
|
||||
`SELECT count(*)::text AS count FROM atlas.platform_catalog_sync_runs
|
||||
WHERE platform_id = $1 AND status IN ('queued', 'running')`,
|
||||
[platformId],
|
||||
);
|
||||
if (Number(pending.rows[0]?.count ?? 0) >= 6) return;
|
||||
const candidates = await tx.query<Candidate>(
|
||||
`WITH possible AS (
|
||||
SELECT 'tree'::text AS scope, 'tree'::text AS scope_key,
|
||||
NULL::uuid AS category_id, NULL::uuid AS attribute_id,
|
||||
interval '24 hours' AS refresh_interval, 0 AS priority
|
||||
UNION ALL
|
||||
SELECT 'attributes', 'attributes:' || category.id, category.id, NULL::uuid,
|
||||
interval '24 hours', 1
|
||||
FROM atlas.platform_product_categories category
|
||||
WHERE category.platform_id = $1 AND category.is_active
|
||||
AND category.external_type_id IS NOT NULL AND category.external_type_id <> '0'
|
||||
UNION ALL
|
||||
SELECT 'dictionary', 'dictionary:' || attribute.id,
|
||||
attribute.platform_product_category_id, attribute.id,
|
||||
interval '7 days', 2
|
||||
FROM atlas.platform_product_category_attributes attribute
|
||||
JOIN atlas.platform_product_categories category
|
||||
ON category.id = attribute.platform_product_category_id
|
||||
WHERE attribute.platform_id = $1 AND attribute.is_active
|
||||
AND attribute.dictionary_id IS NOT NULL AND category.is_active
|
||||
)
|
||||
SELECT possible.scope, possible.scope_key, possible.category_id, possible.attribute_id
|
||||
FROM possible
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1 FROM atlas.platform_catalog_sync_runs run
|
||||
WHERE run.platform_id = $1 AND run.scope_key = possible.scope_key
|
||||
AND run.status = 'succeeded'
|
||||
AND run.completed_at > now() - possible.refresh_interval
|
||||
) AND NOT EXISTS (
|
||||
SELECT 1 FROM atlas.platform_catalog_sync_runs run
|
||||
WHERE run.platform_id = $1 AND run.scope_key = possible.scope_key
|
||||
AND run.created_at > now() - interval '1 hour'
|
||||
) AND NOT EXISTS (
|
||||
SELECT 1 FROM atlas.platform_catalog_sync_runs run
|
||||
WHERE run.platform_id = $1 AND run.scope_key = possible.scope_key
|
||||
AND run.status IN ('queued', 'running')
|
||||
)
|
||||
ORDER BY possible.priority, possible.scope_key LIMIT 6`,
|
||||
[platformId],
|
||||
);
|
||||
const capacity = 6 - Number(pending.rows[0]?.count ?? 0);
|
||||
for (const candidate of candidates.rows.slice(0, capacity)) {
|
||||
await enqueueScheduled(tx, platformId, source.rows[0].revision, candidate);
|
||||
let capacity = platformActiveRunLimit - Number(pending.rows[0]?.count ?? 0);
|
||||
if (capacity <= 0) return;
|
||||
|
||||
if (await treeNeedsRefresh(tx, platformId)) {
|
||||
await enqueueRun(tx, {
|
||||
platformId,
|
||||
scopeKey: 'tree',
|
||||
sourceRevision: source.rows[0].revision,
|
||||
trigger: 'weekly',
|
||||
priority: 'low',
|
||||
});
|
||||
capacity -= 1;
|
||||
}
|
||||
if (capacity <= 0) return;
|
||||
|
||||
const items = await pendingBatchItems(tx, platformId, capacity);
|
||||
for (const item of items) {
|
||||
const runId = await enqueueRun(tx, {
|
||||
platformId,
|
||||
scopeKey: `attributes:${item.category_id}`,
|
||||
categoryId: item.category_id,
|
||||
sourceRevision: item.source_revision,
|
||||
trigger: 'admin_batch',
|
||||
priority: item.priority,
|
||||
});
|
||||
await tx.query(
|
||||
`UPDATE atlas.platform_catalog_attribute_batch_items
|
||||
SET run_id = $3, linked_at = now()
|
||||
WHERE batch_id = $1 AND category_id = $2 AND run_id IS NULL`,
|
||||
[item.batch_id, item.category_id, runId],
|
||||
);
|
||||
}
|
||||
if (items.length) {
|
||||
await tx.query(
|
||||
`UPDATE atlas.platform_catalog_attribute_batches
|
||||
SET status = 'running', started_at = COALESCE(started_at, now())
|
||||
WHERE id = $1 AND status = 'queued'`,
|
||||
[items[0].batch_id],
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async function enqueueScheduled(
|
||||
async function treeNeedsRefresh(tx: DatabaseTransaction, platformId: string) {
|
||||
const result = await tx.query<{ due: boolean }>(
|
||||
`SELECT
|
||||
(state.last_success_at IS NULL
|
||||
OR state.last_success_at <= now() - interval '7 days')
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM atlas.platform_catalog_sync_runs run
|
||||
WHERE run.platform_id = $1 AND run.scope_key = 'tree'
|
||||
AND run.status IN ('queued', 'running')
|
||||
)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM atlas.platform_catalog_sync_runs run
|
||||
WHERE run.platform_id = $1 AND run.scope_key = 'tree'
|
||||
AND run.created_at > now() - interval '1 hour'
|
||||
) AS due
|
||||
FROM (SELECT 1) marker
|
||||
LEFT JOIN atlas.platform_catalog_resource_states state
|
||||
ON state.platform_id = $1 AND state.scope_key = 'tree'`,
|
||||
[platformId],
|
||||
);
|
||||
return result.rows[0]?.due === true;
|
||||
}
|
||||
|
||||
async function pendingBatchItems(
|
||||
tx: DatabaseTransaction,
|
||||
platformId: string,
|
||||
limit: number,
|
||||
): Promise<BatchItem[]> {
|
||||
const result = await tx.query<BatchItem>(
|
||||
`SELECT item.batch_id, item.category_id, batch.source_revision,
|
||||
CASE WHEN state.last_success_at IS NULL
|
||||
OR state.last_success_at < now() -
|
||||
make_interval(days => batch.attribute_expire_days)
|
||||
THEN 'high' ELSE 'low' END AS priority
|
||||
FROM atlas.platform_catalog_attribute_batches batch
|
||||
JOIN atlas.platform_catalog_attribute_batch_items item
|
||||
ON item.batch_id = batch.id AND item.platform_id = batch.platform_id
|
||||
LEFT JOIN atlas.platform_catalog_resource_states state
|
||||
ON state.platform_id = batch.platform_id
|
||||
AND state.scope_key = 'attributes:' || item.category_id
|
||||
WHERE batch.platform_id = $1 AND batch.status IN ('queued', 'running')
|
||||
AND item.run_id IS NULL
|
||||
ORDER BY CASE WHEN state.last_success_at IS NULL
|
||||
OR state.last_success_at < now() -
|
||||
make_interval(days => batch.attribute_expire_days)
|
||||
THEN 0 ELSE 1 END,
|
||||
item.created_at, item.category_id
|
||||
LIMIT $2
|
||||
FOR UPDATE OF item SKIP LOCKED`,
|
||||
[platformId, limit],
|
||||
);
|
||||
return result.rows;
|
||||
}
|
||||
|
||||
async function refreshBatchStatuses(
|
||||
tx: DatabaseTransaction,
|
||||
platformId: string,
|
||||
sourceRevision: number,
|
||||
candidate: Candidate,
|
||||
) {
|
||||
const run = await tx.query<{ id: string }>(
|
||||
await tx.query(
|
||||
`UPDATE atlas.platform_catalog_attribute_batches
|
||||
SET status = 'cancelled', completed_at = now()
|
||||
WHERE platform_id = $1 AND status IN ('queued', 'running')
|
||||
AND source_revision <> $2`,
|
||||
[platformId, sourceRevision],
|
||||
);
|
||||
await tx.query(
|
||||
`WITH progress AS (
|
||||
SELECT batch.id,
|
||||
count(*) FILTER (WHERE item.run_id IS NULL) AS unlinked_count,
|
||||
count(*) FILTER (WHERE run.status IN ('queued', 'running')) AS active_count,
|
||||
count(*) FILTER (WHERE run.status = 'failed') AS failed_count,
|
||||
count(*) FILTER (WHERE run.status = 'cancelled') AS cancelled_count
|
||||
FROM atlas.platform_catalog_attribute_batches batch
|
||||
JOIN atlas.platform_catalog_attribute_batch_items item ON item.batch_id = batch.id
|
||||
LEFT JOIN atlas.platform_catalog_sync_runs run ON run.id = item.run_id
|
||||
WHERE batch.platform_id = $1 AND batch.status IN ('queued', 'running')
|
||||
GROUP BY batch.id
|
||||
)
|
||||
UPDATE atlas.platform_catalog_attribute_batches batch
|
||||
SET status = CASE
|
||||
WHEN progress.unlinked_count = 0 AND progress.active_count = 0
|
||||
THEN CASE WHEN progress.failed_count > 0 THEN 'failed'::atlas.operation_status
|
||||
WHEN progress.cancelled_count > 0 THEN 'cancelled'::atlas.operation_status
|
||||
ELSE 'succeeded'::atlas.operation_status END
|
||||
ELSE batch.status
|
||||
END,
|
||||
completed_at = CASE
|
||||
WHEN progress.unlinked_count = 0 AND progress.active_count = 0
|
||||
THEN COALESCE(batch.completed_at, now())
|
||||
ELSE batch.completed_at
|
||||
END
|
||||
FROM progress WHERE batch.id = progress.id`,
|
||||
[platformId],
|
||||
);
|
||||
}
|
||||
|
||||
async function enqueueRun(
|
||||
tx: DatabaseTransaction,
|
||||
input: {
|
||||
platformId: string;
|
||||
scopeKey: string;
|
||||
categoryId?: string;
|
||||
sourceRevision: number;
|
||||
trigger: 'weekly' | 'admin_batch';
|
||||
priority: 'high' | 'low';
|
||||
},
|
||||
) {
|
||||
const active = await tx.query<{ id: string; priority: 'high' | 'low' }>(
|
||||
`SELECT id, priority FROM atlas.platform_catalog_sync_runs
|
||||
WHERE platform_id = $1 AND scope_key = $2 AND status IN ('queued', 'running')
|
||||
LIMIT 1`,
|
||||
[input.platformId, input.scopeKey],
|
||||
);
|
||||
if (active.rows[0]) {
|
||||
if (input.priority === 'high' && active.rows[0].priority === 'low') {
|
||||
await tx.query(
|
||||
`UPDATE atlas.platform_catalog_sync_runs
|
||||
SET priority = 'high', promoted_at = now()
|
||||
WHERE id = $1 AND status = 'queued'`,
|
||||
[active.rows[0].id],
|
||||
);
|
||||
}
|
||||
return active.rows[0].id;
|
||||
}
|
||||
const inserted = await tx.query<{ id: string }>(
|
||||
`INSERT INTO atlas.platform_catalog_sync_runs
|
||||
(platform_id, scope, scope_key, category_id, attribute_id,
|
||||
trigger_kind, source_revision)
|
||||
VALUES ($1, $2, $3, $4, $5, 'scheduled', $6)
|
||||
(platform_id, scope, scope_key, category_id, trigger_kind,
|
||||
priority, source_revision)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7)
|
||||
RETURNING id`,
|
||||
[platformId, candidate.scope, candidate.scope_key,
|
||||
candidate.category_id, candidate.attribute_id, sourceRevision],
|
||||
[input.platformId, input.categoryId ? 'attributes' : 'tree', input.scopeKey,
|
||||
input.categoryId ?? null, input.trigger, input.priority, input.sourceRevision],
|
||||
);
|
||||
await tx.query(
|
||||
`INSERT INTO atlas.platform_catalog_outbox_events (run_id, platform_id)
|
||||
VALUES ($1, $2)`,
|
||||
[run.rows[0].id, platformId],
|
||||
[inserted.rows[0].id, input.platformId],
|
||||
);
|
||||
return inserted.rows[0].id;
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import type { Database, DatabaseTransaction } from '@atlas/database';
|
||||
import {
|
||||
fetchOzonCategoryAttributes, fetchOzonCategoryTree, fetchOzonDictionaryBatch,
|
||||
fetchOzonDictionaryValues,
|
||||
OzonCredentialError, OzonRequestError, type OzonCategoryNode,
|
||||
type OzonDictionaryValue,
|
||||
} from './ozon-client.js';
|
||||
@@ -10,7 +11,10 @@ import { decryptStoreSecret } from './store-secret.crypto.js';
|
||||
import { workerConfig } from './config.js';
|
||||
|
||||
type RunSource = {
|
||||
id: string; platform_id: string; scope: 'tree' | 'attributes' | 'dictionary';
|
||||
id: string; platform_id: string;
|
||||
scope: 'tree' | 'attributes' | 'dictionary' | 'dictionary_search';
|
||||
scope_key: string;
|
||||
search_key: string | null;
|
||||
category_id: string | null; attribute_id: string | null;
|
||||
source_revision: number; platform_code: string;
|
||||
client_id: string; secret_ciphertext: Buffer; secret_nonce: Buffer;
|
||||
@@ -22,7 +26,7 @@ type RunSource = {
|
||||
type SyncPayload =
|
||||
| { scope: 'tree'; nodes: OzonCategoryNode[] }
|
||||
| { scope: 'attributes'; attributes: Awaited<ReturnType<typeof fetchOzonCategoryAttributes>> }
|
||||
| { scope: 'dictionary'; batch: Awaited<ReturnType<typeof fetchOzonDictionaryBatch>> };
|
||||
| { scope: 'dictionary' | 'dictionary_search'; batch: Awaited<ReturnType<typeof fetchOzonDictionaryBatch>> };
|
||||
|
||||
export async function syncPlatformCatalog(database: Database, job: PlatformCatalogJob) {
|
||||
const source = await claimRun(database, job);
|
||||
@@ -60,14 +64,31 @@ export async function syncPlatformCatalog(database: Database, job: PlatformCatal
|
||||
await finishRun(database, job.runId, 'failed', 'ATTRIBUTE_NOT_FOUND', '字典属性已失效');
|
||||
return;
|
||||
}
|
||||
payload = {
|
||||
scope: 'dictionary',
|
||||
batch: await fetchOzonDictionaryBatch(source.client_id, apiKey, {
|
||||
attributeId: source.platform_attribute_id,
|
||||
categoryId: source.external_category_id,
|
||||
typeId: source.external_type_id,
|
||||
}),
|
||||
const dictionaryInput = {
|
||||
attributeId: source.platform_attribute_id,
|
||||
categoryId: source.external_category_id,
|
||||
typeId: source.external_type_id,
|
||||
};
|
||||
payload = source.scope === 'dictionary_search'
|
||||
? {
|
||||
scope: 'dictionary_search',
|
||||
batch: {
|
||||
values: await fetchOzonDictionaryValues(
|
||||
source.client_id,
|
||||
apiKey,
|
||||
{ ...dictionaryInput, query: source.search_key ?? '' },
|
||||
),
|
||||
isComplete: false,
|
||||
},
|
||||
}
|
||||
: {
|
||||
scope: 'dictionary',
|
||||
batch: await fetchOzonDictionaryBatch(
|
||||
source.client_id,
|
||||
apiKey,
|
||||
dictionaryInput,
|
||||
),
|
||||
};
|
||||
}
|
||||
}
|
||||
await persistAndComplete(database, source, payload);
|
||||
@@ -100,7 +121,8 @@ export async function markPlatformCatalogUnavailable(database: Database, job: Pl
|
||||
async function claimRun(database: Database, job: PlatformCatalogJob): Promise<RunSource | null> {
|
||||
return database.transaction(async (tx) => {
|
||||
const result = await tx.query<RunSource>(
|
||||
`SELECT run.id, run.platform_id, run.scope, run.category_id, run.attribute_id,
|
||||
`SELECT run.id, run.platform_id, run.scope, run.scope_key, run.search_key,
|
||||
run.category_id, run.attribute_id,
|
||||
run.source_revision, platform.code AS platform_code,
|
||||
source.client_id, source.secret_ciphertext, source.secret_nonce,
|
||||
source.revision AS current_revision,
|
||||
@@ -159,6 +181,13 @@ async function persistAndComplete(database: Database, source: RunSource, payload
|
||||
source.attribute_id ?? '', payload.batch.values, payload.batch.isComplete,
|
||||
);
|
||||
}
|
||||
await recordResourceSuccess(
|
||||
tx,
|
||||
source,
|
||||
payload.scope === 'dictionary' || payload.scope === 'dictionary_search'
|
||||
? payload.batch.isComplete
|
||||
: true,
|
||||
);
|
||||
await updateRun(tx, source.id, 'succeeded', result);
|
||||
});
|
||||
}
|
||||
@@ -318,7 +347,62 @@ async function finishRun(
|
||||
database: Database, runId: string,
|
||||
status: 'failed' | 'cancelled', code: string, message: string,
|
||||
) {
|
||||
await database.transaction((tx) => updateRun(tx, runId, status, null, code, message));
|
||||
await database.transaction(async (tx) => {
|
||||
if (status === 'failed') {
|
||||
await recordResourceFailure(tx, runId, code, message);
|
||||
}
|
||||
await updateRun(tx, runId, status, null, code, message);
|
||||
});
|
||||
}
|
||||
|
||||
async function recordResourceSuccess(
|
||||
tx: DatabaseTransaction,
|
||||
source: RunSource,
|
||||
isComplete: boolean,
|
||||
) {
|
||||
await tx.query(
|
||||
`INSERT INTO atlas.platform_catalog_resource_states
|
||||
(platform_id, scope, scope_key, category_id, attribute_id, search_key,
|
||||
last_success_at, last_success_run_id, is_complete)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, now(), $7, $8)
|
||||
ON CONFLICT (platform_id, scope_key) DO UPDATE
|
||||
SET scope = EXCLUDED.scope,
|
||||
category_id = EXCLUDED.category_id,
|
||||
attribute_id = EXCLUDED.attribute_id,
|
||||
search_key = EXCLUDED.search_key,
|
||||
last_success_at = EXCLUDED.last_success_at,
|
||||
last_success_run_id = EXCLUDED.last_success_run_id,
|
||||
is_complete = EXCLUDED.is_complete,
|
||||
last_error_code = NULL,
|
||||
last_error_message = NULL,
|
||||
last_error_at = NULL,
|
||||
updated_at = now()`,
|
||||
[source.platform_id, source.scope, source.scope_key,
|
||||
source.category_id, source.attribute_id, source.search_key,
|
||||
source.id, isComplete],
|
||||
);
|
||||
}
|
||||
|
||||
async function recordResourceFailure(
|
||||
tx: DatabaseTransaction,
|
||||
runId: string,
|
||||
code: string,
|
||||
message: string,
|
||||
) {
|
||||
await tx.query(
|
||||
`INSERT INTO atlas.platform_catalog_resource_states
|
||||
(platform_id, scope, scope_key, category_id, attribute_id, search_key,
|
||||
is_complete, last_error_code, last_error_message, last_error_at)
|
||||
SELECT platform_id, scope, scope_key, category_id, attribute_id, search_key,
|
||||
false, $2, $3, now()
|
||||
FROM atlas.platform_catalog_sync_runs WHERE id = $1
|
||||
ON CONFLICT (platform_id, scope_key) DO UPDATE
|
||||
SET last_error_code = EXCLUDED.last_error_code,
|
||||
last_error_message = EXCLUDED.last_error_message,
|
||||
last_error_at = EXCLUDED.last_error_at,
|
||||
updated_at = now()`,
|
||||
[runId, code, message],
|
||||
);
|
||||
}
|
||||
|
||||
async function updateRun(
|
||||
|
||||
@@ -0,0 +1,120 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { test } from 'node:test';
|
||||
import { PlatformCatalogScheduler } from '../src/platform-catalog-scheduler.ts';
|
||||
|
||||
function schedulerDatabase(handler) {
|
||||
const statements = [];
|
||||
const query = async (sql, parameters = []) => {
|
||||
statements.push({ sql, parameters });
|
||||
return handler(sql, parameters);
|
||||
};
|
||||
return {
|
||||
statements,
|
||||
database: {
|
||||
query: async (sql, parameters = []) => {
|
||||
statements.push({ sql, parameters });
|
||||
if (sql.includes('SELECT source.platform_id')) {
|
||||
return { rows: [{ platform_id: 'platform-1' }], rowCount: 1 };
|
||||
}
|
||||
return handler(sql, parameters);
|
||||
},
|
||||
transaction: async (work) => work({ query }),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
test('没有管理员批次时只检查每周类目树,不自动创建属性或字典任务', async () => {
|
||||
const { database, statements } = schedulerDatabase(async (sql) => {
|
||||
if (sql.includes('SELECT revision FROM atlas.platform_catalog_sources')) {
|
||||
return { rows: [{ revision: 1 }], rowCount: 1 };
|
||||
}
|
||||
if (sql.includes("count(*)::text AS count FROM atlas.platform_catalog_sync_runs")) {
|
||||
return { rows: [{ count: '0' }], rowCount: 1 };
|
||||
}
|
||||
if (sql.includes('AS due')) return { rows: [{ due: false }], rowCount: 1 };
|
||||
if (sql.includes('SELECT item.batch_id')) return { rows: [], rowCount: 0 };
|
||||
return { rows: [], rowCount: 0 };
|
||||
});
|
||||
|
||||
await new PlatformCatalogScheduler(database).run();
|
||||
|
||||
assert.equal(
|
||||
statements.some(({ sql }) => sql.includes('INSERT INTO atlas.platform_catalog_sync_runs')),
|
||||
false,
|
||||
);
|
||||
assert.equal(
|
||||
statements.some(({ sql }) => sql.includes("scope = 'dictionary'")),
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
test('到期类目树创建每周低优先级任务', async () => {
|
||||
const { database, statements } = schedulerDatabase(async (sql) => {
|
||||
if (sql.includes('SELECT revision FROM atlas.platform_catalog_sources')) {
|
||||
return { rows: [{ revision: 3 }], rowCount: 1 };
|
||||
}
|
||||
if (sql.includes("count(*)::text AS count FROM atlas.platform_catalog_sync_runs")) {
|
||||
return { rows: [{ count: '0' }], rowCount: 1 };
|
||||
}
|
||||
if (sql.includes('AS due')) return { rows: [{ due: true }], rowCount: 1 };
|
||||
if (sql.includes('SELECT id, priority FROM atlas.platform_catalog_sync_runs')) {
|
||||
return { rows: [], rowCount: 0 };
|
||||
}
|
||||
if (sql.includes('INSERT INTO atlas.platform_catalog_sync_runs')) {
|
||||
return { rows: [{ id: 'run-tree' }], rowCount: 1 };
|
||||
}
|
||||
if (sql.includes('SELECT item.batch_id')) return { rows: [], rowCount: 0 };
|
||||
return { rows: [], rowCount: 0 };
|
||||
});
|
||||
|
||||
await new PlatformCatalogScheduler(database).run();
|
||||
|
||||
const insert = statements.find(({ sql }) =>
|
||||
sql.includes('INSERT INTO atlas.platform_catalog_sync_runs'));
|
||||
assert.ok(insert);
|
||||
assert.deepEqual(insert.parameters, [
|
||||
'platform-1', 'tree', 'tree', null, 'weekly', 'low', 3,
|
||||
]);
|
||||
});
|
||||
|
||||
test('管理员批次按资源年龄创建高优先级类型属性任务', async () => {
|
||||
const { database, statements } = schedulerDatabase(async (sql) => {
|
||||
if (sql.includes('SELECT revision FROM atlas.platform_catalog_sources')) {
|
||||
return { rows: [{ revision: 4 }], rowCount: 1 };
|
||||
}
|
||||
if (sql.includes("count(*)::text AS count FROM atlas.platform_catalog_sync_runs")) {
|
||||
return { rows: [{ count: '0' }], rowCount: 1 };
|
||||
}
|
||||
if (sql.includes('AS due')) return { rows: [{ due: false }], rowCount: 1 };
|
||||
if (sql.includes('SELECT item.batch_id')) {
|
||||
return {
|
||||
rows: [{
|
||||
batch_id: 'batch-1',
|
||||
category_id: 'category-1',
|
||||
source_revision: 4,
|
||||
priority: 'high',
|
||||
}],
|
||||
rowCount: 1,
|
||||
};
|
||||
}
|
||||
if (sql.includes('SELECT id, priority FROM atlas.platform_catalog_sync_runs')) {
|
||||
return { rows: [], rowCount: 0 };
|
||||
}
|
||||
if (sql.includes('INSERT INTO atlas.platform_catalog_sync_runs')) {
|
||||
return { rows: [{ id: 'run-attribute' }], rowCount: 1 };
|
||||
}
|
||||
return { rows: [], rowCount: 0 };
|
||||
});
|
||||
|
||||
await new PlatformCatalogScheduler(database).run();
|
||||
|
||||
const insert = statements.find(({ sql }) =>
|
||||
sql.includes('INSERT INTO atlas.platform_catalog_sync_runs'));
|
||||
assert.ok(insert);
|
||||
assert.deepEqual(insert.parameters, [
|
||||
'platform-1', 'attributes', 'attributes:category-1', 'category-1',
|
||||
'admin_batch', 'high', 4,
|
||||
]);
|
||||
assert.ok(statements.some(({ sql }) =>
|
||||
sql.includes('UPDATE atlas.platform_catalog_attribute_batch_items')));
|
||||
});
|
||||
Reference in New Issue
Block a user