feat(category): 类目树、属性、属性字典维护,新增系统管理员管理类目数据

This commit is contained in:
R524809
2026-09-18 18:55:16 +08:00
parent fcddb11da0
commit 388429dbd0
58 changed files with 4347 additions and 640 deletions
+36 -6
View File
@@ -19,17 +19,25 @@ import {
type CatalogSyncScope,
} from './ozon-catalog-sync.js';
import { scheduleStoreCategoryTreeSync } from './store-category-tree-scheduler.js';
import { PlatformCatalogDispatcher, type PlatformCatalogJob } from './platform-catalog-dispatcher.js';
import { PlatformCatalogScheduler } from './platform-catalog-scheduler.js';
import { markPlatformCatalogUnavailable, syncPlatformCatalog } from './platform-catalog-sync.js';
const connection = new Redis(workerConfig.redisUrl, {
maxRetriesPerRequest: null,
});
const database = createApplicationDatabase(workerConfig.databaseUrl);
const queue = new Queue<OutboxJob>('atlas-dispatch', { connection });
const platformQueue = new Queue<PlatformCatalogJob>('atlas-dispatch', { connection });
const dispatcher = new OutboxDispatcher(
database,
queue,
`worker_${randomUUID()}`,
);
const platformDispatcher = new PlatformCatalogDispatcher(
database, platformQueue, `platform_worker_${randomUUID()}`,
);
const platformScheduler = new PlatformCatalogScheduler(database);
const objectStorage = new S3Client({
endpoint: workerConfig.objectStorageEndpoint,
region: 'us-east-1',
@@ -54,15 +62,17 @@ type CatalogSyncJob = OutboxJob & {
};
};
const worker = new Worker<OutboxJob>(
const worker = new Worker<OutboxJob | PlatformCatalogJob>(
'atlas-dispatch',
async (job) => {
if (job.name === 'store_credential_verification.requested') {
if (job.name === 'platform_catalog_sync.requested') {
await syncPlatformCatalog(database, job.data as PlatformCatalogJob);
} else if (job.name === 'store_credential_verification.requested') {
await verifyStoreCredentials(database, job.data as OutboxJob & { payload: { store_id: string; operation_id: string; secret_id?: string } });
} else if (job.name === 'collection.cleanup.requested') {
await cleanupCollection(database, job.data as OutboxJob & { payload: { operation_id: string; collection_item_ids: string[] } });
} else if (job.name === 'collection.assets.materialize.requested' || job.name === 'collection.assets.retry.requested') {
await materializeCollectionAssets(database, objectStorage, workerConfig.objectStorageBucket, job.data);
await materializeCollectionAssets(database, objectStorage, workerConfig.objectStorageBucket, job.data as OutboxJob);
} else if (
job.name === 'store_category_tree_sync.requested' ||
job.name === 'store_category_attribute_sync.requested' ||
@@ -84,7 +94,9 @@ async function cleanupCollection(database: Database, job: OutboxJob & { payload:
worker.on('failed', (job, error) => {
if (!job || job.attemptsMade < (job.opts.attempts ?? 1)) return;
if (job.name === 'store_credential_verification.requested') {
if (job.name === 'platform_catalog_sync.requested') {
void markPlatformCatalogUnavailable(database, job.data as PlatformCatalogJob);
} else if (job.name === 'store_credential_verification.requested') {
void markUnavailable(database, job.data as StoreVerificationJob, error);
} else if (
job.name === 'store_category_tree_sync.requested' ||
@@ -285,20 +297,36 @@ async function markUnavailable(
let dispatcherTimer: ReturnType<typeof setInterval> | undefined;
let mediaCleanupTimer: ReturnType<typeof setInterval> | undefined;
let platformCatalogTimer: ReturnType<typeof setInterval> | undefined;
async function bootstrap() {
await queue.waitUntilReady();
await worker.waitUntilReady();
await dispatcher.dispatchBatch();
await platformDispatcher.dispatchBatch();
await platformScheduler.run();
dispatcherTimer = setInterval(
() =>
() => {
void dispatcher.dispatchBatch().catch((error: unknown) => {
console.error(
'outbox_dispatch_failed',
error instanceof Error ? error.message : error,
);
}),
});
void platformDispatcher.dispatchBatch().catch((error: unknown) => {
console.error(
'platform_outbox_dispatch_failed',
error instanceof Error ? error.message : error,
);
});
},
2_000,
);
platformCatalogTimer = setInterval(
() => void platformScheduler.run().catch((error: unknown) => {
console.error('platform_catalog_schedule_failed', error instanceof Error ? error.message : error);
}),
60_000,
);
void mediaUploadCleanup.run().catch((error: unknown) => {
console.error(
'media_upload_cleanup_startup_failed',
@@ -329,8 +357,10 @@ void bootstrap().catch((error: unknown) => {
async function shutdown() {
if (dispatcherTimer) clearInterval(dispatcherTimer);
if (mediaCleanupTimer) clearInterval(mediaCleanupTimer);
if (platformCatalogTimer) clearInterval(platformCatalogTimer);
await worker.close();
await queue.close();
await platformQueue.close();
objectStorage.destroy();
await database.close();
await connection.quit();
+75 -8
View File
@@ -87,18 +87,85 @@ export async function fetchOzonDictionaryValues(
},
) {
const query = input.query?.trim() ?? '';
const path = query
? '/v1/description-category/attribute/values/search'
: '/v1/description-category/attribute/values';
const body = await requestOzon(clientId, apiKey, path, {
const request = {
attribute_id: numberOrText(input.attributeId),
description_category_id: numberOrText(input.categoryId),
type_id: numberOrText(input.typeId),
language: 'ZH_HANS',
limit: query ? 100 : 1_000,
...(query ? { value: query } : { last_value_id: 0 }),
});
return readResultArray<OzonDictionaryValue>(body, '属性字典');
};
if (query) {
const body = await requestOzon(
clientId, apiKey, '/v1/description-category/attribute/values/search',
{ ...request, limit: 100, value: query },
);
return readResultArray<OzonDictionaryValue>(body, '属性字典');
}
const values: OzonDictionaryValue[] = [];
const seen = new Set<string>();
let lastValueId: number | string = 0;
// 大字典仅预取首批 1000 项;后续值由关键词搜索补充,重复游标不得造成无限循环。
for (let pageIndex = 0; pageIndex < 20 && values.length < 1_000; pageIndex += 1) {
const body = await requestOzon(
clientId, apiKey, '/v1/description-category/attribute/values',
{ ...request, limit: 1_000 - values.length, last_value_id: lastValueId },
);
const page = readResultArray<OzonDictionaryValue>(body, '属性字典');
let added = 0;
for (const value of page) {
const id = String(value.id ?? '');
if (!id || seen.has(id)) continue;
seen.add(id);
values.push(value);
added += 1;
}
const nextId = page.at(-1)?.id;
if (
(body as { has_next?: unknown }).has_next !== true || !page.length ||
!added || nextId == null || String(nextId) === String(lastValueId)
) break;
lastValueId = typeof nextId === 'string' ? numberOrText(nextId) : nextId;
}
return values;
}
/** 平台级字典按页获取,明确返回完整性;大字典不会伪装成已全量同步。 */
export async function fetchOzonDictionaryBatch(
clientId: string,
apiKey: string,
input: { attributeId: string; categoryId: string; typeId: string },
): Promise<{ values: OzonDictionaryValue[]; isComplete: boolean }> {
const values: OzonDictionaryValue[] = [];
const seen = new Set<string>();
let lastValueId: number | string = 0;
for (let pageIndex = 0; pageIndex < 5; pageIndex += 1) {
const body = await requestOzon(
clientId, apiKey, '/v1/description-category/attribute/values',
{
attribute_id: numberOrText(input.attributeId),
description_category_id: numberOrText(input.categoryId),
type_id: numberOrText(input.typeId),
language: 'ZH_HANS',
limit: 2000,
last_value_id: lastValueId,
},
);
const page = readResultArray<OzonDictionaryValue>(body, '属性字典');
for (const value of page) {
const id = String(value.id ?? '').trim();
if (id && !seen.has(id)) {
seen.add(id);
values.push(value);
}
}
const hasNext = (body as { has_next?: unknown }).has_next === true;
if (!hasNext) return { values, isComplete: true };
const nextId = page.at(-1)?.id;
if (!page.length || nextId == null || String(nextId) === String(lastValueId)) {
throw new OzonRetryableError('Ozon 字典分页游标没有前进');
}
lastValueId = nextId;
}
return { values, isComplete: false };
}
async function requestOzon(
@@ -0,0 +1,68 @@
import type { Queue } from 'bullmq';
import type { Database } from '@atlas/database';
export type PlatformCatalogJob = {
eventId: string;
runId: string;
platformId: string;
};
export class PlatformCatalogDispatcher {
public constructor(
private readonly database: Database,
private readonly queue: Queue<PlatformCatalogJob>,
private readonly workerId: string,
) {}
public async dispatchBatch() {
const events = await this.database.transaction((tx) => tx.query<{
id: string; run_id: string; platform_id: string;
}>(
`WITH candidates AS (
SELECT id FROM atlas.platform_catalog_outbox_events
WHERE (status = 'pending' AND next_attempt_at <= now())
OR (status = 'dispatching' AND locked_until < now())
ORDER BY created_at FOR UPDATE SKIP LOCKED LIMIT 20
)
UPDATE atlas.platform_catalog_outbox_events event
SET status = 'dispatching', locked_by = $1,
locked_until = now() + interval '2 minutes', updated_at = now()
FROM candidates WHERE event.id = candidates.id
RETURNING event.id, event.run_id, event.platform_id`,
[this.workerId],
));
for (const event of events.rows) {
try {
await this.queue.add('platform_catalog_sync.requested', {
eventId: event.id,
runId: event.run_id,
platformId: event.platform_id,
}, {
jobId: event.id,
attempts: 5,
backoff: { type: 'exponential', delay: 5_000 },
removeOnComplete: 1000,
removeOnFail: 1000,
});
await this.database.query(
`UPDATE atlas.platform_catalog_outbox_events
SET status = 'dispatched', dispatched_at = now(),
locked_by = NULL, locked_until = NULL, updated_at = now()
WHERE id = $1 AND status = 'dispatching' AND locked_by = $2`,
[event.id, this.workerId],
);
} catch (error) {
await this.database.query(
`UPDATE atlas.platform_catalog_outbox_events
SET status = 'pending', attempt_count = attempt_count + 1,
next_attempt_at = now() + interval '30 seconds',
locked_by = NULL, locked_until = NULL,
last_error_code = 'QUEUE_UNAVAILABLE', updated_at = now()
WHERE id = $1 AND status = 'dispatching' AND locked_by = $2`,
[event.id, this.workerId],
);
throw error;
}
}
}
}
+107
View File
@@ -0,0 +1,107 @@
import type { OzonCategoryAttribute, OzonCategoryNode } from './ozon-client.js';
export type PlatformCategoryRecord = {
external_category_id: string;
external_type_id: string;
parent_key: string | null;
name: string;
path: string;
is_active: boolean;
source_payload: OzonCategoryNode;
};
/** Ozon 类型叶子可能省略类目 ID,需要继承最近的父类目。 */
export function flattenPlatformCategories(
nodes: OzonCategoryNode[],
path: string[] = [],
parentKey: string | null = null,
inheritedCategory?: { id: string; name: string },
): PlatformCategoryRecord[] {
const records: PlatformCategoryRecord[] = [];
for (const node of nodes) {
const explicitId = String(node.description_category_id ?? '').trim();
const explicitName = node.category_name?.trim() ?? '';
const categoryId = explicitId || inheritedCategory?.id || '';
const categoryName = explicitName || inheritedCategory?.name || '';
const typeId = String(node.type_id ?? 0);
const name = explicitName || node.type_name?.trim() || categoryName;
if (!categoryId || !name) continue;
const nextPath = [...path, name];
const key = `${categoryId}:${typeId}`;
records.push({
external_category_id: categoryId,
external_type_id: typeId,
parent_key: parentKey,
name,
path: nextPath.join(' › '),
is_active: node.disabled !== true && typeId !== '0',
source_payload: node,
});
records.push(...flattenPlatformCategories(
node.children ?? [],
nextPath,
key,
explicitId && explicitName ? { id: explicitId, name: explicitName } : inheritedCategory,
));
}
return records;
}
export type PlatformAttributeRecord = {
platform_attribute_id: string;
complex_id: string;
name: string;
description: string | null;
group_id: string | null;
group_name: string | null;
value_type: 'text' | 'integer' | 'decimal' | 'boolean' | 'dictionary';
is_required: boolean;
is_collection: boolean;
max_value_count: number;
dictionary_id: string | null;
is_aspect: boolean;
category_dependent: boolean;
source_payload: OzonCategoryAttribute;
};
export function mapPlatformAttributes(values: OzonCategoryAttribute[]): PlatformAttributeRecord[] {
const byKey = new Map<string, PlatformAttributeRecord>();
for (const value of values) {
const attributeId = String(value.id ?? '').trim();
const name = value.name?.trim();
if (!attributeId || !name) continue;
const complexId = String(value.attribute_complex_id ?? value.complex_id ?? 0);
const dictionaryId = String(value.dictionary_id ?? '').trim();
const hasDictionary = !!dictionaryId && dictionaryId !== '0';
const isCollection = value.is_collection === true;
const requestedMax = Number(value.max_value_count ?? (isCollection ? 100 : 1));
const type = String(value.type ?? '').toLowerCase();
const record: PlatformAttributeRecord = {
platform_attribute_id: attributeId,
complex_id: complexId,
name,
description: stringOrNull(value.description),
group_id: value.group_id == null ? null : String(value.group_id),
group_name: stringOrNull(value.group_name),
value_type: hasDictionary ? 'dictionary'
: type === 'integer' ? 'integer'
: type === 'decimal' || type === 'number' ? 'decimal'
: type === 'boolean' ? 'boolean' : 'text',
is_required: value.is_required === true,
is_collection: isCollection,
max_value_count: isCollection
? Math.max(1, Math.min(100, Number.isFinite(requestedMax) ? requestedMax : 100))
: 1,
dictionary_id: hasDictionary ? dictionaryId : null,
is_aspect: value.is_aspect === true,
category_dependent: value.category_dependent === true,
source_payload: value,
};
byKey.set(`${attributeId}:${complexId}`, record);
}
return [...byKey.values()];
}
function stringOrNull(value: unknown): string | null {
return typeof value === 'string' && value.trim() ? value.trim() : null;
}
@@ -0,0 +1,103 @@
import type { Database, DatabaseTransaction } from '@atlas/database';
type Candidate = { scope: 'tree' | 'attributes' | 'dictionary'; scope_key: string;
category_id: string | null; attribute_id: string | null };
/** 定时任务逐平台限量排队,避免启动时一次扫完全部类目和大字典。 */
export class PlatformCatalogScheduler {
public constructor(private readonly database: Database) {}
public async run() {
const sources = await this.database.query<{ platform_id: string }>(
`SELECT source.platform_id FROM atlas.platform_catalog_sources source
JOIN atlas.platforms platform ON platform.id = source.platform_id
WHERE platform.status = 'active' AND upper(platform.code) = 'OZON'`,
);
for (const source of sources.rows) {
await this.schedulePlatform(source.platform_id);
}
}
private async schedulePlatform(platformId: string) {
await this.database.transaction(async (tx) => {
const source = await tx.query<{ revision: number }>(
`SELECT revision FROM atlas.platform_catalog_sources
WHERE platform_id = $1 FOR UPDATE`,
[platformId],
);
if (!source.rows[0]) return;
const pending = await tx.query<{ count: string }>(
`SELECT count(*)::text AS count FROM atlas.platform_catalog_sync_runs
WHERE platform_id = $1 AND status IN ('queued', 'running')`,
[platformId],
);
if (Number(pending.rows[0]?.count ?? 0) >= 6) return;
const candidates = await tx.query<Candidate>(
`WITH possible AS (
SELECT 'tree'::text AS scope, 'tree'::text AS scope_key,
NULL::uuid AS category_id, NULL::uuid AS attribute_id,
interval '24 hours' AS refresh_interval, 0 AS priority
UNION ALL
SELECT 'attributes', 'attributes:' || category.id, category.id, NULL::uuid,
interval '24 hours', 1
FROM atlas.platform_product_categories category
WHERE category.platform_id = $1 AND category.is_active
AND category.external_type_id IS NOT NULL AND category.external_type_id <> '0'
UNION ALL
SELECT 'dictionary', 'dictionary:' || attribute.id,
attribute.platform_product_category_id, attribute.id,
interval '7 days', 2
FROM atlas.platform_product_category_attributes attribute
JOIN atlas.platform_product_categories category
ON category.id = attribute.platform_product_category_id
WHERE attribute.platform_id = $1 AND attribute.is_active
AND attribute.dictionary_id IS NOT NULL AND category.is_active
)
SELECT possible.scope, possible.scope_key, possible.category_id, possible.attribute_id
FROM possible
WHERE NOT EXISTS (
SELECT 1 FROM atlas.platform_catalog_sync_runs run
WHERE run.platform_id = $1 AND run.scope_key = possible.scope_key
AND run.status = 'succeeded'
AND run.completed_at > now() - possible.refresh_interval
) AND NOT EXISTS (
SELECT 1 FROM atlas.platform_catalog_sync_runs run
WHERE run.platform_id = $1 AND run.scope_key = possible.scope_key
AND run.created_at > now() - interval '1 hour'
) AND NOT EXISTS (
SELECT 1 FROM atlas.platform_catalog_sync_runs run
WHERE run.platform_id = $1 AND run.scope_key = possible.scope_key
AND run.status IN ('queued', 'running')
)
ORDER BY possible.priority, possible.scope_key LIMIT 6`,
[platformId],
);
const capacity = 6 - Number(pending.rows[0]?.count ?? 0);
for (const candidate of candidates.rows.slice(0, capacity)) {
await enqueueScheduled(tx, platformId, source.rows[0].revision, candidate);
}
});
}
}
async function enqueueScheduled(
tx: DatabaseTransaction,
platformId: string,
sourceRevision: number,
candidate: Candidate,
) {
const run = await tx.query<{ id: string }>(
`INSERT INTO atlas.platform_catalog_sync_runs
(platform_id, scope, scope_key, category_id, attribute_id,
trigger_kind, source_revision)
VALUES ($1, $2, $3, $4, $5, 'scheduled', $6)
RETURNING id`,
[platformId, candidate.scope, candidate.scope_key,
candidate.category_id, candidate.attribute_id, sourceRevision],
);
await tx.query(
`INSERT INTO atlas.platform_catalog_outbox_events (run_id, platform_id)
VALUES ($1, $2)`,
[run.rows[0].id, platformId],
);
}
+337
View File
@@ -0,0 +1,337 @@
import type { Database, DatabaseTransaction } from '@atlas/database';
import {
fetchOzonCategoryAttributes, fetchOzonCategoryTree, fetchOzonDictionaryBatch,
OzonCredentialError, OzonRequestError, type OzonCategoryNode,
type OzonDictionaryValue,
} from './ozon-client.js';
import { flattenPlatformCategories, mapPlatformAttributes } from './platform-catalog-mappers.js';
import type { PlatformCatalogJob } from './platform-catalog-dispatcher.js';
import { decryptStoreSecret } from './store-secret.crypto.js';
import { workerConfig } from './config.js';
type RunSource = {
id: string; platform_id: string; scope: 'tree' | 'attributes' | 'dictionary';
category_id: string | null; attribute_id: string | null;
source_revision: number; platform_code: string;
client_id: string; secret_ciphertext: Buffer; secret_nonce: Buffer;
current_revision: number;
external_category_id: string | null; external_type_id: string | null;
platform_attribute_id: string | null;
};
type SyncPayload =
| { scope: 'tree'; nodes: OzonCategoryNode[] }
| { scope: 'attributes'; attributes: Awaited<ReturnType<typeof fetchOzonCategoryAttributes>> }
| { scope: 'dictionary'; batch: Awaited<ReturnType<typeof fetchOzonDictionaryBatch>> };
export async function syncPlatformCatalog(database: Database, job: PlatformCatalogJob) {
const source = await claimRun(database, job);
if (!source) return;
if (source.source_revision !== source.current_revision) {
await finishRun(database, job.runId, 'cancelled', 'CATALOG_SOURCE_REPLACED', '同步来源已更新');
return;
}
if (source.platform_code.toUpperCase() !== 'OZON') {
await finishRun(database, job.runId, 'failed', 'PLATFORM_CATALOG_UNSUPPORTED', '该平台尚未接入目录同步');
return;
}
const apiKey = decryptStoreSecret(
source.secret_ciphertext, source.secret_nonce, workerConfig.storeSecretEncryptionKey,
);
try {
let payload: SyncPayload;
if (source.scope === 'tree') {
payload = { scope: 'tree', nodes: await fetchOzonCategoryTree(source.client_id, apiKey) };
} else {
if (!source.external_category_id || !source.external_type_id || source.external_type_id === '0') {
await finishRun(database, job.runId, 'failed', 'CATEGORY_TYPE_UNAVAILABLE', '商品类型不存在或已停用');
return;
}
if (source.scope === 'attributes') {
payload = {
scope: 'attributes',
attributes: await fetchOzonCategoryAttributes(
source.client_id, apiKey,
source.external_category_id, source.external_type_id,
),
};
} else {
if (!source.platform_attribute_id) {
await finishRun(database, job.runId, 'failed', 'ATTRIBUTE_NOT_FOUND', '字典属性已失效');
return;
}
payload = {
scope: 'dictionary',
batch: await fetchOzonDictionaryBatch(source.client_id, apiKey, {
attributeId: source.platform_attribute_id,
categoryId: source.external_category_id,
typeId: source.external_type_id,
}),
};
}
}
await persistAndComplete(database, source, payload);
} catch (error) {
if (error instanceof OzonCredentialError) {
await finishRun(
database, job.runId, 'failed', 'OZON_CATALOG_CREDENTIAL_INVALID',
error.status === 403 ? 'Ozon 拒绝读取目录,请检查密钥权限' : 'Ozon 未接受同步来源凭据',
);
return;
}
if (error instanceof OzonRequestError) {
await finishRun(
database, job.runId, 'failed', 'OZON_CATALOG_REQUEST_REJECTED',
`Ozon 拒绝目录同步请求(HTTP ${error.status})`,
);
return;
}
throw error;
}
}
export async function markPlatformCatalogUnavailable(database: Database, job: PlatformCatalogJob) {
await finishRun(
database, job.runId, 'failed', 'PLATFORM_CATALOG_SYNC_UNAVAILABLE',
'暂时无法同步平台目录,请稍后重试',
);
}
async function claimRun(database: Database, job: PlatformCatalogJob): Promise<RunSource | null> {
return database.transaction(async (tx) => {
const result = await tx.query<RunSource>(
`SELECT run.id, run.platform_id, run.scope, run.category_id, run.attribute_id,
run.source_revision, platform.code AS platform_code,
source.client_id, source.secret_ciphertext, source.secret_nonce,
source.revision AS current_revision,
category.external_category_id, category.external_type_id,
attribute.platform_attribute_id
FROM atlas.platform_catalog_sync_runs run
JOIN atlas.platforms platform ON platform.id = run.platform_id
JOIN atlas.platform_catalog_sources source ON source.platform_id = run.platform_id
LEFT JOIN atlas.platform_product_categories category
ON category.id = run.category_id AND category.platform_id = run.platform_id
LEFT JOIN atlas.platform_product_category_attributes attribute
ON attribute.id = run.attribute_id AND attribute.platform_id = run.platform_id
WHERE run.id = $1 AND run.platform_id = $2
AND run.status IN ('queued', 'running')
FOR UPDATE OF run`,
[job.runId, job.platformId],
);
const source = result.rows[0];
if (!source) return null;
await tx.query(
`UPDATE atlas.platform_catalog_sync_runs
SET status = 'running', started_at = coalesce(started_at, now()),
attempt_count = attempt_count + 1
WHERE id = $1`,
[job.runId],
);
return source;
});
}
async function persistAndComplete(database: Database, source: RunSource, payload: SyncPayload) {
await database.transaction(async (tx) => {
// 凭据更新与结果发布共用来源行锁,旧任务不能覆盖新来源的目录。
const revision = await tx.query<{ revision: number }>(
`SELECT revision FROM atlas.platform_catalog_sources
WHERE platform_id = $1 FOR UPDATE`,
[source.platform_id],
);
const run = await tx.query<{ status: string }>(
`SELECT status FROM atlas.platform_catalog_sync_runs WHERE id = $1 FOR UPDATE`,
[source.id],
);
if (run.rows[0]?.status !== 'running') return;
if (revision.rows[0]?.revision !== source.source_revision) {
await updateRun(tx, source.id, 'cancelled', null, 'CATALOG_SOURCE_REPLACED', '同步来源已更新');
return;
}
let result: Record<string, unknown>;
if (payload.scope === 'tree') {
result = await persistTree(tx, source.platform_id, payload.nodes);
} else if (payload.scope === 'attributes') {
result = await persistAttributes(tx, source.platform_id, source.category_id ?? '', payload.attributes);
} else {
result = await persistDictionary(
tx, source.platform_id, source.category_id ?? '',
source.attribute_id ?? '', payload.batch.values, payload.batch.isComplete,
);
}
await updateRun(tx, source.id, 'succeeded', result);
});
}
async function persistTree(tx: DatabaseTransaction, platformId: string, nodes: OzonCategoryNode[]) {
const records = [...new Map(flattenPlatformCategories(nodes).map((node) => [
`${node.external_category_id}:${node.external_type_id}`, node,
])).values()];
if (!records.length) throw new Error('Ozon 返回了空类目树');
const syncTime = new Date();
await tx.query(
`INSERT INTO atlas.platform_product_categories
(platform_id, external_category_id, external_type_id, name, path,
is_active, source_payload, synced_at)
SELECT $1, external_category_id, external_type_id, name, path,
is_active, source_payload, $3::timestamptz
FROM jsonb_to_recordset($2::jsonb) AS row(
external_category_id text, external_type_id text, parent_key text,
name text, path text, is_active boolean, source_payload jsonb)
ON CONFLICT (platform_id, external_category_id, external_type_id) DO UPDATE
SET name = EXCLUDED.name, path = EXCLUDED.path, is_active = EXCLUDED.is_active,
source_payload = EXCLUDED.source_payload, synced_at = EXCLUDED.synced_at,
updated_at = now()`,
[platformId, JSON.stringify(records), syncTime],
);
await tx.query(
`WITH incoming AS (
SELECT * FROM jsonb_to_recordset($2::jsonb) AS row(
external_category_id text, external_type_id text, parent_key text,
name text, path text, is_active boolean, source_payload jsonb)
), resolved AS (
SELECT child.id AS child_id, parent.id AS parent_id
FROM incoming
JOIN atlas.platform_product_categories child
ON child.platform_id = $1
AND child.external_category_id = incoming.external_category_id
AND child.external_type_id = incoming.external_type_id
LEFT JOIN atlas.platform_product_categories parent
ON parent.platform_id = $1
AND parent.external_category_id || ':' || parent.external_type_id = incoming.parent_key
)
UPDATE atlas.platform_product_categories category
SET parent_id = resolved.parent_id, updated_at = now()
FROM resolved WHERE category.id = resolved.child_id`,
[platformId, JSON.stringify(records)],
);
await tx.query(
`UPDATE atlas.platform_product_categories
SET is_active = false, updated_at = now()
WHERE platform_id = $1 AND synced_at IS DISTINCT FROM $2::timestamptz`,
[platformId, syncTime],
);
return { category_count: records.length };
}
async function persistAttributes(
tx: DatabaseTransaction, platformId: string, categoryId: string,
attributes: Awaited<ReturnType<typeof fetchOzonCategoryAttributes>>,
) {
const records = mapPlatformAttributes(attributes);
if (!records.length) throw new Error('Ozon 返回了空属性定义');
const syncTime = new Date();
await tx.query(
`INSERT INTO atlas.platform_product_category_attributes
(platform_id, platform_product_category_id, platform_attribute_id, complex_id,
name, description, group_id, group_name, value_type, is_required,
is_collection, max_value_count, dictionary_id, is_aspect,
category_dependent, is_active, source_payload, synced_at)
SELECT $1, $2, platform_attribute_id, complex_id,
name, description, group_id, group_name, value_type, is_required,
is_collection, max_value_count, dictionary_id, is_aspect,
category_dependent, true, source_payload, $4::timestamptz
FROM jsonb_to_recordset($3::jsonb) AS row(
platform_attribute_id text, complex_id text, name text, description text,
group_id text, group_name text, value_type text, is_required boolean,
is_collection boolean, max_value_count integer, dictionary_id text,
is_aspect boolean, category_dependent boolean, source_payload jsonb)
ON CONFLICT (platform_id, platform_product_category_id, platform_attribute_id, complex_id)
DO UPDATE SET name = EXCLUDED.name, description = EXCLUDED.description,
group_id = EXCLUDED.group_id, group_name = EXCLUDED.group_name,
value_type = EXCLUDED.value_type, is_required = EXCLUDED.is_required,
is_collection = EXCLUDED.is_collection, max_value_count = EXCLUDED.max_value_count,
dictionary_id = EXCLUDED.dictionary_id, is_aspect = EXCLUDED.is_aspect,
category_dependent = EXCLUDED.category_dependent, is_active = true,
source_payload = EXCLUDED.source_payload, synced_at = EXCLUDED.synced_at,
updated_at = now()`,
[platformId, categoryId, JSON.stringify(records), syncTime],
);
await tx.query(
`UPDATE atlas.platform_product_category_attributes
SET is_active = false, updated_at = now()
WHERE platform_id = $1 AND platform_product_category_id = $2
AND synced_at IS DISTINCT FROM $3::timestamptz`,
[platformId, categoryId, syncTime],
);
return { attribute_count: records.length };
}
async function persistDictionary(
tx: DatabaseTransaction, platformId: string, categoryId: string,
attributeId: string, values: OzonDictionaryValue[], isComplete: boolean,
) {
const attribute = await tx.query<{ dictionary_id: string }>(
`SELECT dictionary_id FROM atlas.platform_product_category_attributes
WHERE id = $1 AND platform_id = $2 AND platform_product_category_id = $3
AND is_active AND dictionary_id IS NOT NULL`,
[attributeId, platformId, categoryId],
);
if (!attribute.rows[0]) throw new Error('字典属性已失效');
const byId = new Map<string, {
external_value_id: string; value: string; info: string | null;
picture: string | null; source_payload: OzonDictionaryValue;
}>();
for (const value of values) {
const id = String(value.id ?? '').trim();
const label = value.value?.trim() ?? '';
if (!id || !label) continue;
byId.set(id, {
external_value_id: id,
value: label,
info: typeof value.info === 'string' ? value.info : null,
picture: typeof value.picture === 'string' ? value.picture : null,
source_payload: value,
});
}
const records = [...byId.values()];
const syncTime = new Date();
if (records.length) {
await tx.query(
`INSERT INTO atlas.platform_catalog_dictionary_values
(platform_id, platform_product_category_id, attribute_id, dictionary_id,
external_value_id, value, info, picture, source_payload, synced_at)
SELECT $1, $2, $3, $4, external_value_id, value, info, picture,
source_payload, $6::timestamptz
FROM jsonb_to_recordset($5::jsonb) AS row(
external_value_id text, value text, info text, picture text, source_payload jsonb)
ON CONFLICT (attribute_id, external_value_id) DO UPDATE
SET value = EXCLUDED.value, info = EXCLUDED.info,
picture = EXCLUDED.picture, source_payload = EXCLUDED.source_payload,
is_active = true, synced_at = EXCLUDED.synced_at, updated_at = now()`,
[platformId, categoryId, attributeId, attribute.rows[0].dictionary_id,
JSON.stringify(records), syncTime],
);
}
if (isComplete) {
await tx.query(
`UPDATE atlas.platform_catalog_dictionary_values
SET is_active = false, updated_at = now()
WHERE attribute_id = $1 AND synced_at IS DISTINCT FROM $2::timestamptz`,
[attributeId, syncTime],
);
}
return { dictionary_value_count: records.length, is_complete: isComplete };
}
async function finishRun(
database: Database, runId: string,
status: 'failed' | 'cancelled', code: string, message: string,
) {
await database.transaction((tx) => updateRun(tx, runId, status, null, code, message));
}
async function updateRun(
tx: DatabaseTransaction, runId: string,
status: 'succeeded' | 'failed' | 'cancelled',
result: Record<string, unknown> | null,
code?: string, message?: string,
) {
await tx.query(
`UPDATE atlas.platform_catalog_sync_runs
SET status = $2::atlas.operation_status, completed_at = now(),
result = $3::jsonb, error_code = $4, error_message = $5
WHERE id = $1 AND status IN ('queued', 'running')`,
[runId, status, JSON.stringify(result ?? {}), code ?? null, message ?? null],
);
}