Update seller store experience
This commit is contained in:
R524809
@@ -1,6 +1,6 @@
|
||||
import { Body, Controller, Get, Headers, Inject, Param, Patch, Post, Query, Req, Res } from '@nestjs/common';
|
||||
import { z } from 'zod';
|
||||
import { bulkDeleteCollectionItemsRequestSchema, clearCollectionItemsRequestSchema, createCollectionItemRequestSchema, listCollectionItemsQuerySchema, retryCollectionAssetsRequestSchema, updateCollectionItemRequestSchema, type Workspace } from '@atlas/contracts';
|
||||
import { bulkDeleteCollectionItemsRequestSchema, clearCollectionItemsRequestSchema, createCollectionItemRequestSchema, listCollectionItemsQuerySchema, materializeCollectionAssetsRequestSchema, retryCollectionAssetsRequestSchema, updateCollectionItemRequestSchema, type Workspace } from '@atlas/contracts';
|
||||
import { ApiError } from '../common/api-error.js';
|
||||
import type { AuthenticatedRequest } from '../auth/auth.guard.js';
|
||||
import { CurrentWorkspace } from '../workspace/current-workspace.decorator.js';
|
||||
@@ -32,6 +32,9 @@ export class CollectionController {
|
||||
@Post(':id/asset-retries') async retry(@Param('id') id: string, @Body() body: unknown, @Headers('idempotency-key') key: string | undefined, @Req() request: AuthenticatedRequest, @CurrentWorkspace() workspace: Workspace, @Res({ passthrough: true }) response: Response) {
|
||||
const result = await this.collection.retryAssets(workspace.id, request.auth.userId, requireUuid('collection_item_id', id), requireKey(key), parse(retryCollectionAssetsRequestSchema, body)); response.status(result.status); return result.body;
|
||||
}
|
||||
@Post(':id/asset-materializations') async materialize(@Param('id') id: string, @Body() body: unknown, @Headers('idempotency-key') key: string | undefined, @Req() request: AuthenticatedRequest, @CurrentWorkspace() workspace: Workspace, @Res({ passthrough: true }) response: Response) {
|
||||
const result = await this.collection.materializeAssets(workspace.id, request.auth.userId, requireUuid('collection_item_id', id), requireKey(key), parse(materializeCollectionAssetsRequestSchema, body)); response.status(result.status); return result.body;
|
||||
}
|
||||
}
|
||||
function parse<T extends { safeParse: (input: unknown) => { success: boolean; data?: unknown; error?: { issues: Array<{ path: PropertyKey[]; message: string }> } } }>(schema: T, input: unknown) { const result = schema.safeParse(input); if (result.success) return result.data as never; throw new ApiError(422, 'VALIDATION_FAILED', '请求参数不符合要求', result.error?.issues.map((x) => ({ field: x.path.join('.'), code: 'INVALID_FIELD', message: x.message }))); }
|
||||
function requireUuid(field: string, value: string) { if (z.uuid().safeParse(value).success) return value; throw new ApiError(422, 'VALIDATION_FAILED', '请求参数不符合要求', [{ field, code: 'INVALID_FIELD', message: `${field} 必须是 UUID` }]); }
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { Inject, Injectable } from '@nestjs/common';
|
||||
import type { CreateCollectionItemRequest, ListCollectionItemsQuery, UpdateCollectionItemRequest } from '@atlas/contracts';
|
||||
import type { CreateCollectionItemRequest, ListCollectionItemsQuery, MaterializeCollectionAssetsRequest, UpdateCollectionItemRequest } from '@atlas/contracts';
|
||||
import type { Database, DatabaseTransaction } from '@atlas/database';
|
||||
import { ApiError } from '../common/api-error.js';
|
||||
import { IdempotencyService } from '../common/idempotency.service.js';
|
||||
@@ -37,7 +37,7 @@ export class CollectionService {
|
||||
const item = await this.requireItem(tx, id);
|
||||
const source = await tx.query<{ normalized_payload: unknown; raw_evidence: unknown; parser_version: string | null; collected_at: Date | string | null; source_snapshot_revision: string | number }>('SELECT normalized_payload, raw_evidence, parser_version, collected_at, source_snapshot_revision FROM atlas.collection_items WHERE id = $1 AND deleted_at IS NULL', [id]);
|
||||
const variants = await tx.query<{ id: string; source_variant_key: string | null; source_sku: string | null; display_name: string | null; sort_order: number }>('SELECT id, source_variant_key, source_sku, display_name, sort_order FROM atlas.product_variants WHERE product_id = $1 AND deleted_at IS NULL ORDER BY sort_order, id', [item.product_id]);
|
||||
const transfers = await tx.query<{ id: string; source_variant_id: string | null; role: string; source_url: string; status: string; asset_id: string | null; error_payload: unknown; created_at: Date | string; completed_at: Date | string | null }>('SELECT id, source_variant_id, role::text AS role, source_url, status::text AS status, asset_id, error_payload, created_at, completed_at FROM atlas.collection_asset_transfers WHERE collection_item_id = $1 AND source_snapshot_revision = $2 ORDER BY created_at, id', [id, source.rows[0]?.source_snapshot_revision ?? 1]);
|
||||
const transfers = await tx.query<{ id: string; source_variant_id: string | null; media_type: 'image' | 'video'; poster_url: string | null; role: string; source_url: string; status: string; asset_id: string | null; error_payload: unknown; created_at: Date | string; completed_at: Date | string | null }>('SELECT id, source_variant_id, media_type::text AS media_type, poster_url, role::text AS role, source_url, status::text AS status, asset_id, error_payload, created_at, completed_at FROM atlas.collection_asset_transfers WHERE collection_item_id = $1 AND source_snapshot_revision = $2 ORDER BY created_at, id', [id, source.rows[0]?.source_snapshot_revision ?? 1]);
|
||||
return { ...item, normalized_payload: source.rows[0]?.normalized_payload ?? {}, raw_evidence: source.rows[0]?.raw_evidence ?? {}, parser_version: source.rows[0]?.parser_version ?? null, collected_at: source.rows[0]?.collected_at ? iso(source.rows[0].collected_at) : null, variants: variants.rows, asset_transfers: transfers.rows.map((transfer) => ({ ...transfer, created_at: iso(transfer.created_at), completed_at: transfer.completed_at ? iso(transfer.completed_at) : null })) };
|
||||
});
|
||||
}
|
||||
@@ -51,6 +51,7 @@ export class CollectionService {
|
||||
async bulkDelete(workspaceId: string, userId: string, key: string, ids: { ids: string[] }) { return this.removeMany(workspaceId, userId, key, '/api/collection-items/bulk-delete', ids, false); }
|
||||
async clear(workspaceId: string, userId: string, key: string, input: { confirm: true; expected_active_count: number }) { return this.removeMany(workspaceId, userId, key, '/api/collection-items/clear', input, true); }
|
||||
async retryAssets(workspaceId: string, userId: string, id: string, key: string, input: { transfer_ids: string[] }) { return this.idempotency.run({ workspaceId, userId, method: 'POST', path: `/api/collection-items/${id}/asset-retries`, key, request: input }, async (tx) => { await this.requireItem(tx, id); const result = await tx.query<{ source_snapshot_revision: string | number }>(`UPDATE atlas.collection_asset_transfers t SET status = 'queued', error_payload = NULL, updated_at = now() FROM atlas.collection_items c WHERE t.collection_item_id = c.id AND t.collection_item_id = $1 AND t.id = ANY($2::uuid[]) AND t.source_snapshot_revision = c.source_snapshot_revision AND t.status IN ('failed', 'cancelled') RETURNING c.source_snapshot_revision`, [id, input.transfer_ids]); if (result.rowCount !== input.transfer_ids.length) throw new ApiError(422, 'ASSET_RETRY_INVALID', '只能重试当前采集快照中失败或已取消的素材任务'); const operation = await this.operation(tx, userId, 'collection_asset_retry', id); await this.event(tx, id, 'collection.assets.retry.requested', { operation_id: operation.id, collection_item_id: id, source_snapshot_revision: Number(result.rows[0].source_snapshot_revision) }); return { status: 202, body: { data: { operation } } }; }); }
|
||||
async materializeAssets(workspaceId: string, userId: string, id: string, key: string, input: MaterializeCollectionAssetsRequest) { return this.idempotency.run({ workspaceId, userId, method: 'POST', path: `/api/collection-items/${id}/asset-materializations`, key, request: input }, async (tx) => { await this.requireItem(tx, id); const result = await tx.query<{ source_snapshot_revision: string | number }>(`UPDATE atlas.collection_asset_transfers t SET status = 'queued', error_payload = NULL, completed_at = NULL, updated_at = now() FROM atlas.collection_items c WHERE t.collection_item_id = c.id AND t.collection_item_id = $1 AND t.id = ANY($2::uuid[]) AND t.source_snapshot_revision = c.source_snapshot_revision AND t.status = 'pending_selection' RETURNING c.source_snapshot_revision`, [id, input.transfer_ids]); if (result.rowCount !== input.transfer_ids.length) throw new ApiError(422, 'ASSET_MATERIALIZATION_INVALID', '只能转存当前采集快照中尚未转存的来源媒体'); const operation = await this.operation(tx, userId, 'collection_asset_materialization', id); await this.event(tx, operation.id, 'collection.assets.materialize.requested', { operation_id: operation.id, collection_item_id: id, source_snapshot_revision: Number(result.rows[0].source_snapshot_revision) }); return { status: 202, body: { data: { operation } } }; }); }
|
||||
private async removeMany(workspaceId: string, userId: string, key: string, path: string, input: unknown, clear: boolean) { return this.idempotency.run({ workspaceId, userId, method: 'POST', path, key, request: input }, async (tx) => { const requested = input as { ids?: string[]; expected_active_count?: number }; const count = await tx.query<{ count: string }>('SELECT count(*)::text AS count FROM atlas.collection_items WHERE deleted_at IS NULL'); if (clear && Number(count.rows[0].count) !== requested.expected_active_count) throw new ApiError(409, 'COLLECTION_COUNT_CHANGED', '采集箱数量已变化，请重新确认'); const deleted = await tx.query<{ id: string }>(clear ? `UPDATE atlas.collection_items SET status = 'deleted', deleted_at = now(), revision = revision + 1, updated_at = now() WHERE deleted_at IS NULL RETURNING id` : `UPDATE atlas.collection_items SET status = 'deleted', deleted_at = now(), revision = revision + 1, updated_at = now() WHERE id = ANY($1::uuid[]) AND deleted_at IS NULL RETURNING id`, clear ? [] : [requested.ids]); if (!clear && deleted.rows.length !== requested.ids?.length) throw new ApiError(404, 'RESOURCE_NOT_FOUND', '部分采集记录不存在'); const operation = await this.operation(tx, userId, 'collection_cleanup', deleted.rows[0]?.id ?? null); await this.event(tx, operation.id, 'collection.cleanup.requested', { operation_id: operation.id, collection_item_ids: deleted.rows.map((row) => row.id) }); return { status: 202, body: { data: { deleted_count: deleted.rows.length, operation } } }; }); }
|
||||
private async operation(tx: DatabaseTransaction, userId: string, kind: string, resourceId: string | null) { const row = await tx.query<{ id: string; status: string; created_at: Date | string }>(`INSERT INTO atlas.operations (workspace_id, kind, resource_type, resource_id, created_by_user_id) VALUES (current_setting('app.workspace_id')::uuid, $1, 'collection_item', $2, $3) RETURNING id, status, created_at`, [kind, resourceId, userId]); return { id: row.rows[0].id, kind, status: row.rows[0].status, created_at: iso(row.rows[0].created_at), completed_at: null, error: null }; }
|
||||
private async event(tx: DatabaseTransaction, aggregateId: string, type: string, payload: unknown) { await tx.query(`INSERT INTO atlas.outbox_events (workspace_id, aggregate_type, aggregate_id, event_type, event_payload, deduplication_key) VALUES (current_setting('app.workspace_id')::uuid, 'collection_item', $1, $2, $3, $4)`, [aggregateId, type, payload, `${type}:${aggregateId}`]); }
|
||||
|
||||
@@ -9,39 +9,110 @@ import { DATABASE } from '../database/database.module.js';
|
||||
export class ExtensionService {
|
||||
public constructor(
|
||||
@Inject(DATABASE) private readonly database: Database,
|
||||
@Inject(IdempotencyService) private readonly idempotency: IdempotencyService,
|
||||
@Inject(IdempotencyService)
|
||||
private readonly idempotency: IdempotencyService,
|
||||
) {}
|
||||
|
||||
public async ingest(workspaceId: string, userId: string, key: string, payload: CollectedProductPayload) {
|
||||
return this.idempotency.run({ workspaceId, userId, method: 'POST', path: '/api/extension/materials', key, request: payload }, async (tx) => {
|
||||
const platform = await tx.query<{ id: string; code: string }>(`SELECT id, code FROM atlas.platforms
|
||||
WHERE code = $1 AND status = 'active' AND coalesce((capabilities->>'can_be_source')::boolean, false)`, [payload.source.platform.toUpperCase()]);
|
||||
if (!platform.rows[0]) throw new ApiError(422, 'SOURCE_PLATFORM_UNAVAILABLE', '该来源平台当前不可采集');
|
||||
const existing = await this.findExistingSourceItem(
|
||||
tx,
|
||||
public async ingest(
|
||||
workspaceId: string,
|
||||
userId: string,
|
||||
key: string,
|
||||
payload: CollectedProductPayload,
|
||||
) {
|
||||
return this.idempotency.run(
|
||||
{
|
||||
workspaceId,
|
||||
platform.rows[0].id,
|
||||
payload,
|
||||
);
|
||||
if (existing) {
|
||||
await this.replaceSourceSnapshot(
|
||||
userId,
|
||||
method: 'POST',
|
||||
path: '/api/extension/materials',
|
||||
key,
|
||||
request: payload,
|
||||
},
|
||||
async (tx) => {
|
||||
const platform = await tx.query<{ id: string; code: string }>(
|
||||
`SELECT id, code FROM atlas.platforms
|
||||
WHERE code = $1 AND status = 'active' AND coalesce((capabilities->>'can_be_source')::boolean, false)`,
|
||||
[payload.source.platform.toUpperCase()],
|
||||
);
|
||||
if (!platform.rows[0])
|
||||
throw new ApiError(
|
||||
422,
|
||||
'SOURCE_PLATFORM_UNAVAILABLE',
|
||||
'该来源平台当前不可采集',
|
||||
);
|
||||
const existing = await this.findExistingSourceItem(
|
||||
tx,
|
||||
existing.id,
|
||||
existing.product_id,
|
||||
workspaceId,
|
||||
platform.rows[0].id,
|
||||
payload,
|
||||
);
|
||||
return { status: 200, body: { data: await this.materialResponse(tx, existing.id, existing.product_id, true, true) } };
|
||||
}
|
||||
const { status, sourcePrice } = sourceState(payload);
|
||||
const item = await tx.query<{ id: string }>(`INSERT INTO atlas.collection_items
|
||||
if (existing) {
|
||||
await this.replaceSourceSnapshot(
|
||||
tx,
|
||||
workspaceId,
|
||||
existing.id,
|
||||
existing.product_id,
|
||||
payload,
|
||||
);
|
||||
return {
|
||||
status: 200,
|
||||
body: {
|
||||
data: await this.materialResponse(
|
||||
tx,
|
||||
existing.id,
|
||||
existing.product_id,
|
||||
true,
|
||||
true,
|
||||
),
|
||||
},
|
||||
};
|
||||
}
|
||||
const { status, sourcePrice } = sourceState(payload);
|
||||
const item = await tx.query<{ id: string }>(
|
||||
`INSERT INTO atlas.collection_items
|
||||
(workspace_id, source_kind, source_platform_id, source_item_key, source_url, source_title, source_currency_code, source_min_price, source_max_price, status, raw_evidence, normalized_payload, parser_version, collected_at)
|
||||
VALUES (current_setting('app.workspace_id')::uuid, 'platform', $1, $2, $3, $4, $5, $6, $6, $7::atlas.collection_item_status, $8, $9, $10, $11) RETURNING id`,
|
||||
[platform.rows[0].id, payload.source.itemKey, payload.source.canonicalUrl, payload.common.title?.value ?? null, sourcePrice?.currencyCode ?? null, sourcePrice?.amount ?? null, status, JSON.stringify(payload.evidence), JSON.stringify(payload), payload.source.parserVersion, payload.source.collectedAt]);
|
||||
const product = await tx.query<{ id: string }>(`INSERT INTO atlas.products (workspace_id, collection_item_id, source_title)
|
||||
VALUES (current_setting('app.workspace_id')::uuid, $1, $2) RETURNING id`, [item.rows[0].id, payload.common.title?.value ?? null]);
|
||||
await this.writeSourceSnapshotRows(tx, item.rows[0].id, product.rows[0].id, 1, payload);
|
||||
return { status: 201, body: { data: await this.materialResponse(tx, item.rows[0].id, product.rows[0].id, false, false) } };
|
||||
});
|
||||
[
|
||||
platform.rows[0].id,
|
||||
payload.source.itemKey,
|
||||
payload.source.canonicalUrl,
|
||||
payload.common.title?.value ?? null,
|
||||
sourcePrice?.currencyCode ?? null,
|
||||
sourcePrice?.amount ?? null,
|
||||
status,
|
||||
JSON.stringify(payload.evidence),
|
||||
JSON.stringify(payload),
|
||||
payload.source.parserVersion,
|
||||
payload.source.collectedAt,
|
||||
],
|
||||
);
|
||||
const product = await tx.query<{ id: string }>(
|
||||
`INSERT INTO atlas.products (workspace_id, collection_item_id, source_title)
|
||||
VALUES (current_setting('app.workspace_id')::uuid, $1, $2) RETURNING id`,
|
||||
[item.rows[0].id, payload.common.title?.value ?? null],
|
||||
);
|
||||
await this.writeSourceSnapshotRows(
|
||||
tx,
|
||||
workspaceId,
|
||||
item.rows[0].id,
|
||||
product.rows[0].id,
|
||||
1,
|
||||
payload,
|
||||
);
|
||||
return {
|
||||
status: 201,
|
||||
body: {
|
||||
data: await this.materialResponse(
|
||||
tx,
|
||||
item.rows[0].id,
|
||||
product.rows[0].id,
|
||||
false,
|
||||
false,
|
||||
),
|
||||
},
|
||||
};
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -70,7 +141,10 @@ export class ExtensionService {
|
||||
const sourceSkus = [
|
||||
...new Set(
|
||||
payload.variants
|
||||
.map((variant) => variant.sourceSku ?? variant.sourceVariantKey)
|
||||
.map(
|
||||
(variant) =>
|
||||
variant.sourceSku ?? variant.sourceVariantKey,
|
||||
)
|
||||
.filter((sku) => sku.trim().length > 0),
|
||||
),
|
||||
];
|
||||
@@ -109,55 +183,201 @@ export class ExtensionService {
|
||||
return byVariantOverlap.rows[0] ?? null;
|
||||
}
|
||||
|
||||
private async replaceSourceSnapshot(tx: { query: Database['query'] }, collectionItemId: string, productId: string, payload: CollectedProductPayload) {
|
||||
private async replaceSourceSnapshot(
|
||||
tx: { query: Database['query'] },
|
||||
workspaceId: string,
|
||||
collectionItemId: string,
|
||||
productId: string,
|
||||
payload: CollectedProductPayload,
|
||||
) {
|
||||
const { status, sourcePrice } = sourceState(payload);
|
||||
// 不删除旧任务记录：运行中的 Worker 可能已拿到它。取消并保留审计记录，
|
||||
// 不删除旧候选/任务记录：运行中的 Worker 可能已拿到它。取消并保留审计记录，
|
||||
// 新快照在查询和 Worker 中均按 source_snapshot_revision 隔离。
|
||||
await tx.query(`UPDATE atlas.collection_asset_transfers SET status = 'cancelled', completed_at = now(), updated_at = now()
|
||||
WHERE collection_item_id = $1 AND status IN ('queued', 'running')`, [collectionItemId]);
|
||||
const item = await tx.query<{ source_snapshot_revision: string | number }>(`UPDATE atlas.collection_items
|
||||
await tx.query(
|
||||
`UPDATE atlas.collection_asset_transfers SET status = 'cancelled', completed_at = now(), updated_at = now()
|
||||
WHERE collection_item_id = $1 AND status IN ('pending_selection', 'queued', 'running')`,
|
||||
[collectionItemId],
|
||||
);
|
||||
const item = await tx.query<{
|
||||
source_snapshot_revision: string | number;
|
||||
}>(
|
||||
`UPDATE atlas.collection_items
|
||||
SET source_url = $1, source_title = $2, source_currency_code = $3, source_min_price = $4, source_max_price = $4,
|
||||
status = $5::atlas.collection_item_status, raw_evidence = $6, normalized_payload = $7, parser_version = $8,
|
||||
collected_at = $9, source_snapshot_revision = source_snapshot_revision + 1, revision = revision + 1, updated_at = now()
|
||||
WHERE id = $10 AND deleted_at IS NULL RETURNING source_snapshot_revision`,
|
||||
[payload.source.canonicalUrl, payload.common.title?.value ?? null, sourcePrice?.currencyCode ?? null, sourcePrice?.amount ?? null, status, JSON.stringify(payload.evidence), JSON.stringify(payload), payload.source.parserVersion, payload.source.collectedAt, collectionItemId]);
|
||||
[
|
||||
payload.source.canonicalUrl,
|
||||
payload.common.title?.value ?? null,
|
||||
sourcePrice?.currencyCode ?? null,
|
||||
sourcePrice?.amount ?? null,
|
||||
status,
|
||||
JSON.stringify(payload.evidence),
|
||||
JSON.stringify(payload),
|
||||
payload.source.parserVersion,
|
||||
payload.source.collectedAt,
|
||||
collectionItemId,
|
||||
],
|
||||
);
|
||||
const snapshotRevision = Number(item.rows[0]?.source_snapshot_revision);
|
||||
if (!Number.isSafeInteger(snapshotRevision) || snapshotRevision < 1) throw new Error('采集来源快照版本更新失败');
|
||||
await tx.query(`UPDATE atlas.products SET source_title = $1, updated_at = now(), version = version + 1 WHERE id = $2`, [payload.common.title?.value ?? null, productId]);
|
||||
await tx.query('DELETE FROM atlas.product_variants WHERE product_id = $1', [productId]);
|
||||
await this.writeSourceSnapshotRows(tx, collectionItemId, productId, snapshotRevision, payload);
|
||||
if (!Number.isSafeInteger(snapshotRevision) || snapshotRevision < 1)
|
||||
throw new Error('采集来源快照版本更新失败');
|
||||
await tx.query(
|
||||
`UPDATE atlas.products SET source_title = $1, updated_at = now(), version = version + 1 WHERE id = $2`,
|
||||
[payload.common.title?.value ?? null, productId],
|
||||
);
|
||||
// The historical task rows must remain auditable after replacing the
|
||||
// source snapshot. Clear only their optional variant pointer before
|
||||
// deleting old variants. The original composite FK used ON DELETE SET
|
||||
// NULL and PostgreSQL tried to null both key columns, including the
|
||||
// required workspace_id.
|
||||
await tx.query(
|
||||
`UPDATE atlas.collection_asset_transfers
|
||||
SET source_variant_id = NULL, updated_at = now()
|
||||
WHERE product_id = $1 AND source_variant_id IS NOT NULL`,
|
||||
[productId],
|
||||
);
|
||||
await tx.query(
|
||||
'DELETE FROM atlas.product_variants WHERE product_id = $1',
|
||||
[productId],
|
||||
);
|
||||
await this.writeSourceSnapshotRows(
|
||||
tx,
|
||||
workspaceId,
|
||||
collectionItemId,
|
||||
productId,
|
||||
snapshotRevision,
|
||||
payload,
|
||||
);
|
||||
return snapshotRevision;
|
||||
}
|
||||
|
||||
private async writeSourceSnapshotRows(tx: { query: Database['query'] }, collectionItemId: string, productId: string, snapshotRevision: number, payload: CollectedProductPayload) {
|
||||
const verifiedVariants = payload.variants.filter((variant) => variant.sourceVariantKey.trim().length > 0);
|
||||
const variantRows = verifiedVariants.length ? verifiedVariants : [{ sourceVariantKey: null, sourceSku: undefined, displayName: '默认 SKU' }];
|
||||
private async writeSourceSnapshotRows(
|
||||
tx: { query: Database['query'] },
|
||||
workspaceId: string,
|
||||
collectionItemId: string,
|
||||
productId: string,
|
||||
snapshotRevision: number,
|
||||
payload: CollectedProductPayload,
|
||||
) {
|
||||
const verifiedVariants = payload.variants.filter(
|
||||
(variant) => variant.sourceVariantKey.trim().length > 0,
|
||||
);
|
||||
const variantRows = verifiedVariants.length
|
||||
? verifiedVariants
|
||||
: [
|
||||
{
|
||||
sourceVariantKey: null,
|
||||
sourceSku: undefined,
|
||||
displayName: '默认 SKU',
|
||||
},
|
||||
];
|
||||
const variants = new Map<string, string>();
|
||||
for (let index = 0; index < variantRows.length; index += 1) {
|
||||
const variant = variantRows[index];
|
||||
const inserted = await tx.query<{ id: string }>(`INSERT INTO atlas.product_variants (workspace_id, product_id, source_variant_key, source_sku, display_name, sort_order)
|
||||
VALUES (current_setting('app.workspace_id')::uuid, $1, $2, $3, $4, $5) RETURNING id`, [productId, variant.sourceVariantKey, variant.sourceSku ?? null, variant.displayName, index]);
|
||||
if (variant.sourceVariantKey) variants.set(variant.sourceVariantKey, inserted.rows[0].id);
|
||||
const inserted = await tx.query<{ id: string }>(
|
||||
`INSERT INTO atlas.product_variants (workspace_id, product_id, source_variant_key, source_sku, display_name, sort_order)
|
||||
VALUES ($1, $2, $3, $4, $5, $6) RETURNING id`,
|
||||
[
|
||||
workspaceId,
|
||||
productId,
|
||||
variant.sourceVariantKey,
|
||||
variant.sourceSku ?? null,
|
||||
variant.displayName,
|
||||
index,
|
||||
],
|
||||
);
|
||||
if (variant.sourceVariantKey)
|
||||
variants.set(variant.sourceVariantKey, inserted.rows[0].id);
|
||||
}
|
||||
for (const asset of payload.assets) {
|
||||
await tx.query(`INSERT INTO atlas.collection_asset_transfers (workspace_id, collection_item_id, product_id, source_variant_id, role, source_url, status, error_payload, source_snapshot_revision)
|
||||
VALUES (current_setting('app.workspace_id')::uuid, $1, $2, $3, $4::atlas.media_candidate_role, $5, 'queued', $6, $7)`,
|
||||
[collectionItemId, productId, asset.variantSourceKey ? variants.get(asset.variantSourceKey) ?? null : null, asset.role, asset.url, JSON.stringify({ relation: asset.relation, referer_origin: asset.refererOrigin ?? null, evidence_ref: asset.evidenceRef ?? null, sort_order: asset.sortOrder }), snapshotRevision]);
|
||||
await tx.query(
|
||||
`INSERT INTO atlas.collection_asset_transfers (workspace_id, collection_item_id, product_id, source_variant_id, media_type, poster_url, role, source_url, status, error_payload, source_snapshot_revision)
|
||||
VALUES ($1, $2, $3, $4, $5::atlas.collection_media_type, $6, $7::atlas.media_candidate_role, $8, 'pending_selection', $9, $10)`,
|
||||
[
|
||||
workspaceId,
|
||||
collectionItemId,
|
||||
productId,
|
||||
asset.variantSourceKey
|
||||
? (variants.get(asset.variantSourceKey) ?? null)
|
||||
: null,
|
||||
asset.mediaType,
|
||||
asset.posterUrl ?? null,
|
||||
asset.role,
|
||||
asset.url,
|
||||
JSON.stringify({
|
||||
relation: asset.relation,
|
||||
referer_origin: asset.refererOrigin ?? null,
|
||||
evidence_ref: asset.evidenceRef ?? null,
|
||||
sort_order: asset.sortOrder,
|
||||
}),
|
||||
snapshotRevision,
|
||||
],
|
||||
);
|
||||
}
|
||||
if (payload.assets.length) await tx.query(`INSERT INTO atlas.outbox_events (workspace_id, aggregate_type, aggregate_id, event_type, event_payload, deduplication_key)
|
||||
VALUES (current_setting('app.workspace_id')::uuid, 'collection_item', $1, 'collection.assets.materialize.requested', jsonb_build_object('collection_item_id', $1::uuid, 'source_snapshot_revision', $2::bigint), $3)`, [collectionItemId, snapshotRevision, `collection.assets.materialize:${collectionItemId}:${snapshotRevision}`]);
|
||||
else await tx.query(`UPDATE atlas.collection_items SET status = CASE WHEN status = 'needs_review' THEN status ELSE 'ready_for_edit' END, updated_at = now() WHERE id = $1 AND source_snapshot_revision = $2`, [collectionItemId, snapshotRevision]);
|
||||
// Collecting records source-media candidates only. A later explicit
|
||||
// selection queues the media and creates the materialisation event.
|
||||
await tx.query(
|
||||
`UPDATE atlas.collection_items SET status = CASE WHEN status = 'needs_review' THEN status ELSE 'ready_for_edit' END, updated_at = now() WHERE id = $1 AND source_snapshot_revision = $2`,
|
||||
[collectionItemId, snapshotRevision],
|
||||
);
|
||||
}
|
||||
|
||||
private async materialResponse(tx: { query: Database['query'] }, collectionItemId: string, productId: string, reused: boolean, refreshed: boolean) {
|
||||
const count = await tx.query<{ total: string; queued: string; succeeded: string; failed: string }>(`SELECT count(*)::text AS total, count(*) FILTER (WHERE t.status = 'queued')::text AS queued, count(*) FILTER (WHERE t.status = 'succeeded')::text AS succeeded, count(*) FILTER (WHERE t.status = 'failed')::text AS failed FROM atlas.collection_asset_transfers t JOIN atlas.collection_items c ON c.id = t.collection_item_id WHERE t.collection_item_id = $1 AND t.source_snapshot_revision = c.source_snapshot_revision`, [collectionItemId]);
|
||||
const status = await tx.query<{ status: string; source_title: string | null }>('SELECT status, source_title FROM atlas.collection_items WHERE id = $1', [collectionItemId]);
|
||||
return { collection_item: { id: collectionItemId, status: status.rows[0]?.status ?? 'received', source_title: status.rows[0]?.source_title ?? null, variant_count: 0 }, product_id: productId, reused, refreshed, editor_entry_path: `/collection-test/${collectionItemId}`, materialization: { total: Number(count.rows[0]?.total ?? 0), queued: Number(count.rows[0]?.queued ?? 0), succeeded: Number(count.rows[0]?.succeeded ?? 0), failed: Number(count.rows[0]?.failed ?? 0) } };
|
||||
private async materialResponse(
|
||||
tx: { query: Database['query'] },
|
||||
collectionItemId: string,
|
||||
productId: string,
|
||||
reused: boolean,
|
||||
refreshed: boolean,
|
||||
) {
|
||||
const count = await tx.query<{
|
||||
total: string;
|
||||
pending_selection: string;
|
||||
queued: string;
|
||||
succeeded: string;
|
||||
failed: string;
|
||||
}>(
|
||||
`SELECT count(*)::text AS total, count(*) FILTER (WHERE t.status = 'pending_selection')::text AS pending_selection, count(*) FILTER (WHERE t.status = 'queued')::text AS queued, count(*) FILTER (WHERE t.status = 'succeeded')::text AS succeeded, count(*) FILTER (WHERE t.status = 'failed')::text AS failed FROM atlas.collection_asset_transfers t JOIN atlas.collection_items c ON c.id = t.collection_item_id WHERE t.collection_item_id = $1 AND t.source_snapshot_revision = c.source_snapshot_revision`,
|
||||
[collectionItemId],
|
||||
);
|
||||
const status = await tx.query<{
|
||||
status: string;
|
||||
source_title: string | null;
|
||||
}>(
|
||||
'SELECT status, source_title FROM atlas.collection_items WHERE id = $1',
|
||||
[collectionItemId],
|
||||
);
|
||||
return {
|
||||
collection_item: {
|
||||
id: collectionItemId,
|
||||
status: status.rows[0]?.status ?? 'received',
|
||||
source_title: status.rows[0]?.source_title ?? null,
|
||||
variant_count: 0,
|
||||
},
|
||||
product_id: productId,
|
||||
reused,
|
||||
refreshed,
|
||||
editor_entry_path: `/collection-test/${collectionItemId}`,
|
||||
materialization: {
|
||||
total: Number(count.rows[0]?.total ?? 0),
|
||||
pending_selection: Number(count.rows[0]?.pending_selection ?? 0),
|
||||
queued: Number(count.rows[0]?.queued ?? 0),
|
||||
succeeded: Number(count.rows[0]?.succeeded ?? 0),
|
||||
failed: Number(count.rows[0]?.failed ?? 0),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
function sourceState(payload: CollectedProductPayload) {
|
||||
const verifiedVariants = payload.variants.filter((variant) => variant.sourceVariantKey.trim().length > 0);
|
||||
return { status: verifiedVariants.length || !payload.variantAxes.length ? 'materializing' : 'needs_review', sourcePrice: payload.common.sourcePrice };
|
||||
const verifiedVariants = payload.variants.filter(
|
||||
(variant) => variant.sourceVariantKey.trim().length > 0,
|
||||
);
|
||||
return {
|
||||
status:
|
||||
verifiedVariants.length || !payload.variantAxes.length
|
||||
? 'ready_for_edit'
|
||||
: 'needs_review',
|
||||
sourcePrice: payload.common.sourcePrice,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -7,30 +7,326 @@ import { afterAll, beforeAll, describe, it } from 'vitest';
|
||||
import { DATABASE } from '../src/database/database.module.js';
|
||||
import { createApplication } from '../src/main.js';
|
||||
|
||||
let app: NestExpressApplication; let database: Database; let baseUrl = ''; let workspaceId = ''; let userId = '';
|
||||
async function request(path: string, init: RequestInit = {}) { const response = await fetch(`${baseUrl}${path}`, { redirect: 'manual', ...init, headers: { 'Content-Type': 'application/json', ...init.headers } }); return { response, body: response.status === 204 || response.status === 302 ? undefined : await response.json() as any }; }
|
||||
let app: NestExpressApplication;
|
||||
let database: Database;
|
||||
let baseUrl = '';
|
||||
let workspaceId = '';
|
||||
let userId = '';
|
||||
async function request(path: string, init: RequestInit = {}) {
|
||||
const response = await fetch(`${baseUrl}${path}`, {
|
||||
redirect: 'manual',
|
||||
...init,
|
||||
headers: { 'Content-Type': 'application/json', ...init.headers },
|
||||
});
|
||||
return {
|
||||
response,
|
||||
body:
|
||||
response.status === 204 || response.status === 302
|
||||
? undefined
|
||||
: ((await response.json()) as any),
|
||||
};
|
||||
}
|
||||
|
||||
describe('扩展授权与采集 HTTP 集成测试', () => {
|
||||
beforeAll(async () => { Object.assign(process.env, { NODE_ENV: 'test', SWAGGER_ENABLED: 'false', AUTH_REGISTER_IP_LIMIT: '100', AUTH_REGISTER_EMAIL_LIMIT: '100' }); const created = await createApplication(); app = created.app; database = app.get<Database>(DATABASE); await app.listen(0, '127.0.0.1'); baseUrl = `http://127.0.0.1:${(app.getHttpServer().address() as AddressInfo).port}`; });
|
||||
afterAll(async () => { await database.workspaceTransaction(workspaceId, async (tx) => { await tx.query('DELETE FROM atlas.idempotency_requests WHERE workspace_id = $1', [workspaceId]); await tx.query('DELETE FROM atlas.outbox_events WHERE workspace_id = $1', [workspaceId]); await tx.query('DELETE FROM atlas.collection_asset_transfers WHERE workspace_id = $1', [workspaceId]); await tx.query('DELETE FROM atlas.product_variants WHERE workspace_id = $1', [workspaceId]); await tx.query('DELETE FROM atlas.products WHERE workspace_id = $1', [workspaceId]); await tx.query('DELETE FROM atlas.collection_items WHERE workspace_id = $1', [workspaceId]); await tx.query('DELETE FROM atlas.extension_authorization_codes WHERE workspace_id = $1', [workspaceId]); await tx.query('DELETE FROM atlas.extension_sessions WHERE workspace_id = $1', [workspaceId]); }); await database.query('DELETE FROM atlas.workspace_memberships WHERE user_id = $1', [userId]); await database.query('DELETE FROM atlas.workspaces WHERE id = $1', [workspaceId]); await database.query('DELETE FROM atlas.users WHERE id = $1', [userId]); await app.close(); });
|
||||
beforeAll(async () => {
|
||||
Object.assign(process.env, {
|
||||
NODE_ENV: 'test',
|
||||
SWAGGER_ENABLED: 'false',
|
||||
AUTH_REGISTER_IP_LIMIT: '100',
|
||||
AUTH_REGISTER_EMAIL_LIMIT: '100',
|
||||
});
|
||||
const created = await createApplication();
|
||||
app = created.app;
|
||||
database = app.get<Database>(DATABASE);
|
||||
await app.listen(0, '127.0.0.1');
|
||||
baseUrl = `http://127.0.0.1:${(app.getHttpServer().address() as AddressInfo).port}`;
|
||||
});
|
||||
afterAll(async () => {
|
||||
await database.workspaceTransaction(workspaceId, async (tx) => {
|
||||
await tx.query(
|
||||
'DELETE FROM atlas.idempotency_requests WHERE workspace_id = $1',
|
||||
[workspaceId],
|
||||
);
|
||||
await tx.query(
|
||||
'DELETE FROM atlas.outbox_events WHERE workspace_id = $1',
|
||||
[workspaceId],
|
||||
);
|
||||
await tx.query(
|
||||
'DELETE FROM atlas.operations WHERE workspace_id = $1',
|
||||
[workspaceId],
|
||||
);
|
||||
await tx.query(
|
||||
'DELETE FROM atlas.collection_asset_transfers WHERE workspace_id = $1',
|
||||
[workspaceId],
|
||||
);
|
||||
await tx.query(
|
||||
'DELETE FROM atlas.product_variants WHERE workspace_id = $1',
|
||||
[workspaceId],
|
||||
);
|
||||
await tx.query(
|
||||
'DELETE FROM atlas.products WHERE workspace_id = $1',
|
||||
[workspaceId],
|
||||
);
|
||||
await tx.query(
|
||||
'DELETE FROM atlas.collection_items WHERE workspace_id = $1',
|
||||
[workspaceId],
|
||||
);
|
||||
await tx.query(
|
||||
'DELETE FROM atlas.extension_authorization_codes WHERE workspace_id = $1',
|
||||
[workspaceId],
|
||||
);
|
||||
await tx.query(
|
||||
'DELETE FROM atlas.extension_sessions WHERE workspace_id = $1',
|
||||
[workspaceId],
|
||||
);
|
||||
});
|
||||
await database.query(
|
||||
'DELETE FROM atlas.workspace_memberships WHERE user_id = $1',
|
||||
[userId],
|
||||
);
|
||||
await database.query('DELETE FROM atlas.workspaces WHERE id = $1', [
|
||||
workspaceId,
|
||||
]);
|
||||
await database.query('DELETE FROM atlas.users WHERE id = $1', [userId]);
|
||||
await app.close();
|
||||
});
|
||||
|
||||
it('未登录令牌返回可操作的扩展会话错误', async () => {
|
||||
const session = await request('/api/extension/session');
|
||||
assert.equal(session.response.status, 401);
|
||||
assert.equal(session.body.error.code, 'SESSION_EXPIRED');
|
||||
assert.equal(session.body.error.message, 'Atlas ERP 登录已过期，请重新登录');
|
||||
assert.equal(
|
||||
session.body.error.message,
|
||||
'Atlas ERP 登录已过期，请重新登录',
|
||||
);
|
||||
});
|
||||
|
||||
it('通过 Atlas Web access token 写入来源商品，并以新快照覆盖重复采集', async () => {
|
||||
const registration = await request('/api/auth/register', { method: 'POST', body: JSON.stringify({ email: `extension-${randomUUID()}@example.test`, password: 'correct-horse-battery-staple-123', password_confirmation: 'correct-horse-battery-staple-123' }) });
|
||||
assert.equal(registration.response.status, 201); userId = registration.body.data.user.id; workspaceId = registration.body.data.workspace.id; const webAccessToken = registration.body.data.access_token as string;
|
||||
const session = await request('/api/extension/session', { headers: { Authorization: `Bearer ${webAccessToken}` } }); assert.equal(session.response.status, 200); assert.equal(session.body.data.workspace.id, workspaceId);
|
||||
const payload = { source: { platform: 'ozon', itemKey: '123456789', canonicalUrl: 'https://www.ozon.ru/product/demo-123456789/', collectedAt: new Date().toISOString(), parserVersion: 'test-1' }, common: { title: { value: '来源商品', confidence: 'verified' }, sourcePrice: { amount: '99.90', currencyCode: 'RUB' }, attributes: [], categories: [] }, variantAxes: [], variants: [{ sourceVariantKey: '123456789', sourceSku: '123456789', displayName: '默认 SKU', selections: [], confidence: 'verified', evidenceRefs: ['page'] }], assets: [{ url: 'https://images.example.test/product.jpg', role: 'source_main', relation: 'product_candidate', sortOrder: 0, evidenceRef: 'page' }], evidence: [{ ref: 'page', kind: 'page', path: 'test' }] };
|
||||
const first = await request('/api/extension/materials', { method: 'POST', headers: { Authorization: `Bearer ${webAccessToken}`, 'Idempotency-Key': randomUUID() }, body: JSON.stringify(payload) }); assert.equal(first.response.status, 201); assert.equal(first.body.data.reused, false); assert.equal(first.body.data.materialization.queued, 1);
|
||||
const registration = await request('/api/auth/register', {
|
||||
method: 'POST',
|
||||
body: JSON.stringify({
|
||||
email: `extension-${randomUUID()}@example.test`,
|
||||
password: 'correct-horse-battery-staple-123',
|
||||
password_confirmation: 'correct-horse-battery-staple-123',
|
||||
}),
|
||||
});
|
||||
assert.equal(registration.response.status, 201);
|
||||
userId = registration.body.data.user.id;
|
||||
workspaceId = registration.body.data.workspace.id;
|
||||
const webAccessToken = registration.body.data.access_token as string;
|
||||
const session = await request('/api/extension/session', {
|
||||
headers: { Authorization: `Bearer ${webAccessToken}` },
|
||||
});
|
||||
assert.equal(session.response.status, 200);
|
||||
assert.equal(session.body.data.workspace.id, workspaceId);
|
||||
const payload = {
|
||||
source: {
|
||||
platform: 'ozon',
|
||||
itemKey: '123456789',
|
||||
canonicalUrl: 'https://www.ozon.ru/product/demo-123456789/',
|
||||
collectedAt: new Date().toISOString(),
|
||||
parserVersion: 'test-1',
|
||||
},
|
||||
common: {
|
||||
title: { value: '来源商品', confidence: 'verified' },
|
||||
sourcePrice: { amount: '99.90', currencyCode: 'RUB' },
|
||||
attributes: [],
|
||||
categories: [],
|
||||
},
|
||||
variantAxes: [],
|
||||
variants: [
|
||||
{
|
||||
sourceVariantKey: '123456789',
|
||||
sourceSku: '123456789',
|
||||
displayName: '默认 SKU',
|
||||
selections: [],
|
||||
confidence: 'verified',
|
||||
evidenceRefs: ['page'],
|
||||
},
|
||||
],
|
||||
assets: [
|
||||
{
|
||||
url: 'https://images.example.test/product.jpg',
|
||||
role: 'source_main',
|
||||
relation: 'product_candidate',
|
||||
sortOrder: 0,
|
||||
evidenceRef: 'page',
|
||||
},
|
||||
],
|
||||
evidence: [{ ref: 'page', kind: 'page', path: 'test' }],
|
||||
};
|
||||
const first = await request('/api/extension/materials', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
Authorization: `Bearer ${webAccessToken}`,
|
||||
'Idempotency-Key': randomUUID(),
|
||||
},
|
||||
body: JSON.stringify(payload),
|
||||
});
|
||||
assert.equal(first.response.status, 201);
|
||||
assert.equal(first.body.data.reused, false);
|
||||
assert.equal(first.body.data.collection_item.status, 'ready_for_edit');
|
||||
assert.equal(first.body.data.materialization.pending_selection, 1);
|
||||
assert.equal(first.body.data.materialization.queued, 0);
|
||||
// A sibling Ozon URL has a different entered SKU. It must still reuse
|
||||
// the item because the submitted real-SKU sets overlap at 123456789.
|
||||
const refreshedPayload = { ...payload, source: { ...payload.source, itemKey: '987654321', canonicalUrl: 'https://www.ozon.ru/product/demo-987654321/', collectedAt: new Date(Date.now() + 1_000).toISOString(), parserVersion: 'test-2' }, common: { ...payload.common, title: { value: '第二次采集的中文标题', confidence: 'verified' }, sourcePrice: { amount: '109.90', currencyCode: 'RUB' } }, variants: [{ sourceVariantKey: '123456789', sourceSku: '123456789', displayName: '第一 SKU', selections: [], confidence: 'verified', evidenceRefs: ['page'] }, { sourceVariantKey: '987654321', sourceSku: '987654321', displayName: '第二次采集 SKU', selections: [], confidence: 'verified', evidenceRefs: ['page'] }], assets: [{ url: 'https://images.example.test/refreshed-product.jpg', role: 'source_main', relation: 'product_candidate', sortOrder: 0, evidenceRef: 'page' }] };
|
||||
const second = await request('/api/extension/materials', { method: 'POST', headers: { Authorization: `Bearer ${webAccessToken}`, 'Idempotency-Key': randomUUID() }, body: JSON.stringify(refreshedPayload) }); assert.equal(second.response.status, 200); assert.equal(second.body.data.reused, true); assert.equal(second.body.data.refreshed, true); assert.equal(second.body.data.collection_item.id, first.body.data.collection_item.id); assert.equal(second.body.data.materialization.queued, 1);
|
||||
const inspection = await request(`/api/collection-items/${first.body.data.collection_item.id}`, { headers: { Authorization: `Bearer ${webAccessToken}` } }); assert.equal(inspection.response.status, 200); assert.equal(inspection.body.data.title, '第二次采集的中文标题'); assert.equal(inspection.body.data.normalized_payload.source.parserVersion, 'test-2'); assert.equal(inspection.body.data.variants.length, 2); assert.deepEqual(inspection.body.data.variants.map((variant: { source_variant_key: string }) => variant.source_variant_key), ['123456789', '987654321']); assert.equal(inspection.body.data.asset_transfers.length, 1); assert.equal(inspection.body.data.asset_transfers[0].source_url, 'https://images.example.test/refreshed-product.jpg');
|
||||
const refreshedPayload = {
|
||||
...payload,
|
||||
source: {
|
||||
...payload.source,
|
||||
itemKey: '987654321',
|
||||
canonicalUrl: 'https://www.ozon.ru/product/demo-987654321/',
|
||||
collectedAt: new Date(Date.now() + 1_000).toISOString(),
|
||||
parserVersion: 'test-2',
|
||||
},
|
||||
common: {
|
||||
...payload.common,
|
||||
title: {
|
||||
value: '第二次采集的中文标题',
|
||||
confidence: 'verified',
|
||||
},
|
||||
sourcePrice: { amount: '109.90', currencyCode: 'RUB' },
|
||||
},
|
||||
variants: [
|
||||
{
|
||||
sourceVariantKey: '123456789',
|
||||
sourceSku: '123456789',
|
||||
displayName: '第一 SKU',
|
||||
selections: [],
|
||||
confidence: 'verified',
|
||||
evidenceRefs: ['page'],
|
||||
},
|
||||
{
|
||||
sourceVariantKey: '987654321',
|
||||
sourceSku: '987654321',
|
||||
displayName: '第二次采集 SKU',
|
||||
selections: [],
|
||||
confidence: 'verified',
|
||||
evidenceRefs: ['page'],
|
||||
},
|
||||
],
|
||||
assets: [
|
||||
{
|
||||
url: 'https://images.example.test/refreshed-product.jpg',
|
||||
role: 'source_main',
|
||||
relation: 'product_candidate',
|
||||
sortOrder: 0,
|
||||
evidenceRef: 'page',
|
||||
},
|
||||
{
|
||||
url: 'https://videos.example.test/refreshed-product.mp4',
|
||||
mediaType: 'video',
|
||||
posterUrl: 'https://images.example.test/refreshed-video-poster.jpg',
|
||||
role: 'source_main',
|
||||
relation: 'product_candidate',
|
||||
sortOrder: 1,
|
||||
evidenceRef: 'page',
|
||||
},
|
||||
],
|
||||
};
|
||||
const second = await request('/api/extension/materials', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
Authorization: `Bearer ${webAccessToken}`,
|
||||
'Idempotency-Key': randomUUID(),
|
||||
},
|
||||
body: JSON.stringify(refreshedPayload),
|
||||
});
|
||||
assert.equal(second.response.status, 200);
|
||||
assert.equal(second.body.data.reused, true);
|
||||
assert.equal(second.body.data.refreshed, true);
|
||||
assert.equal(
|
||||
second.body.data.collection_item.id,
|
||||
first.body.data.collection_item.id,
|
||||
);
|
||||
assert.equal(second.body.data.materialization.pending_selection, 2);
|
||||
assert.equal(second.body.data.materialization.queued, 0);
|
||||
const inspection = await request(
|
||||
`/api/collection-items/${first.body.data.collection_item.id}`,
|
||||
{ headers: { Authorization: `Bearer ${webAccessToken}` } },
|
||||
);
|
||||
assert.equal(inspection.response.status, 200);
|
||||
assert.equal(inspection.body.data.title, '第二次采集的中文标题');
|
||||
assert.equal(
|
||||
inspection.body.data.normalized_payload.source.parserVersion,
|
||||
'test-2',
|
||||
);
|
||||
assert.equal(inspection.body.data.variants.length, 2);
|
||||
assert.deepEqual(
|
||||
inspection.body.data.variants.map(
|
||||
(variant: { source_variant_key: string }) =>
|
||||
variant.source_variant_key,
|
||||
),
|
||||
['123456789', '987654321'],
|
||||
);
|
||||
assert.equal(inspection.body.data.asset_transfers.length, 2);
|
||||
assert.equal(
|
||||
inspection.body.data.asset_transfers[0].source_url,
|
||||
'https://images.example.test/refreshed-product.jpg',
|
||||
);
|
||||
assert.equal(
|
||||
inspection.body.data.asset_transfers[0].status,
|
||||
'pending_selection',
|
||||
);
|
||||
const videoTransfer = inspection.body.data.asset_transfers.find(
|
||||
(transfer: { media_type: string }) => transfer.media_type === 'video',
|
||||
);
|
||||
assert.equal(videoTransfer?.source_url, 'https://videos.example.test/refreshed-product.mp4');
|
||||
assert.equal(videoTransfer?.poster_url, 'https://images.example.test/refreshed-video-poster.jpg');
|
||||
const eventsBeforeSelection = await database.workspaceTransaction(
|
||||
workspaceId,
|
||||
async (tx) =>
|
||||
tx.query<{ count: string }>(
|
||||
`SELECT count(*)::text AS count FROM atlas.outbox_events
|
||||
WHERE event_type = 'collection.assets.materialize.requested'
|
||||
AND event_payload->>'collection_item_id' = $1`,
|
||||
[first.body.data.collection_item.id],
|
||||
),
|
||||
);
|
||||
assert.equal(eventsBeforeSelection.rows[0]?.count, '0');
|
||||
const materialization = await request(
|
||||
`/api/collection-items/${first.body.data.collection_item.id}/asset-materializations`,
|
||||
{
|
||||
method: 'POST',
|
||||
headers: {
|
||||
Authorization: `Bearer ${webAccessToken}`,
|
||||
'Idempotency-Key': randomUUID(),
|
||||
},
|
||||
body: JSON.stringify({
|
||||
transfer_ids: [inspection.body.data.asset_transfers[0].id],
|
||||
}),
|
||||
},
|
||||
);
|
||||
assert.equal(materialization.response.status, 202);
|
||||
const afterSelection = await request(
|
||||
`/api/collection-items/${first.body.data.collection_item.id}`,
|
||||
{ headers: { Authorization: `Bearer ${webAccessToken}` } },
|
||||
);
|
||||
assert.equal(afterSelection.response.status, 200);
|
||||
assert.ok(
|
||||
['queued', 'running', 'failed', 'succeeded'].includes(
|
||||
afterSelection.body.data.asset_transfers[0].status,
|
||||
),
|
||||
);
|
||||
const eventsAfterSelection = await database.workspaceTransaction(
|
||||
workspaceId,
|
||||
async (tx) =>
|
||||
tx.query<{ count: string }>(
|
||||
`SELECT count(*)::text AS count FROM atlas.outbox_events
|
||||
WHERE event_type = 'collection.assets.materialize.requested'
|
||||
AND event_payload->>'collection_item_id' = $1`,
|
||||
[first.body.data.collection_item.id],
|
||||
),
|
||||
);
|
||||
assert.equal(eventsAfterSelection.rows[0]?.count, '1');
|
||||
const transferWorkspace = await database.workspaceTransaction(
|
||||
workspaceId,
|
||||
async (tx) =>
|
||||
tx.query<{ workspace_id: string }>(
|
||||
'SELECT workspace_id FROM atlas.collection_asset_transfers WHERE id = $1',
|
||||
[inspection.body.data.asset_transfers[0].id],
|
||||
),
|
||||
);
|
||||
assert.equal(transferWorkspace.rows[0]?.workspace_id, workspaceId);
|
||||
});
|
||||
});
|
||||
|
||||
Vendored
+1
@@ -8,6 +8,7 @@ declare module "wxt/browser" {
|
||||
| "/background.js"
|
||||
| "/content-scripts/collector.js"
|
||||
| "/content-scripts/ozon-page-json.js"
|
||||
| "/content-scripts/source-page-state.js"
|
||||
type HtmlPublicPath = Extract<PublicPath, `${string}.html`>
|
||||
export interface WxtRuntime {
|
||||
getURL(path: PublicPath): string;
|
||||
|
||||
@@ -3,6 +3,7 @@ import {
|
||||
supportedProductPlatform,
|
||||
} from '../lib/source-parsers.js';
|
||||
import type { OzonPageJsonFetcher } from '../lib/ozon-parser.js';
|
||||
import type { MarketplacePageStateFetcher } from '../lib/source-parsers.js';
|
||||
|
||||
export default defineContentScript({
|
||||
matches: [
|
||||
@@ -72,6 +73,9 @@ export default defineContentScript({
|
||||
supportedProductPlatform() === 'ozon'
|
||||
? requestOzonPageJson
|
||||
: undefined,
|
||||
supportedProductPlatform() && supportedProductPlatform() !== 'ozon'
|
||||
? requestMarketplacePageState
|
||||
: undefined,
|
||||
);
|
||||
const response = await browser.runtime.sendMessage({
|
||||
type: 'atlas.collect',
|
||||
@@ -128,6 +132,23 @@ const requestOzonPageJson: OzonPageJsonFetcher = ({ kind, sku }) =>
|
||||
);
|
||||
});
|
||||
|
||||
const requestMarketplacePageState: MarketplacePageStateFetcher = (platform) =>
|
||||
new Promise((resolve) => {
|
||||
const requestId = crypto.randomUUID();
|
||||
const timeout = window.setTimeout(() => {
|
||||
window.removeEventListener('message', receive);
|
||||
resolve({});
|
||||
}, 3_000);
|
||||
const receive = (event: MessageEvent<unknown>) => {
|
||||
if (event.source !== window || !isMarketplaceStateResponse(event.data, requestId)) return;
|
||||
window.clearTimeout(timeout);
|
||||
window.removeEventListener('message', receive);
|
||||
resolve(event.data.data);
|
||||
};
|
||||
window.addEventListener('message', receive);
|
||||
window.postMessage({ source: 'atlas-collector', type: 'source-page-state.request', requestId, platform }, location.origin);
|
||||
});
|
||||
|
||||
function isBridgeResponse(
|
||||
value: unknown,
|
||||
requestId: string,
|
||||
@@ -149,3 +170,10 @@ function isBridgeResponse(
|
||||
(message.ok || typeof message.error === 'string')
|
||||
);
|
||||
}
|
||||
|
||||
function isMarketplaceStateResponse(value: unknown, requestId: string): value is { source: 'atlas-collector'; type: 'source-page-state.response'; requestId: string; data: unknown } {
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value)) return false;
|
||||
const message = value as Record<string, unknown>;
|
||||
return message.source === 'atlas-collector' && message.type === 'source-page-state.response'
|
||||
&& message.requestId === requestId && !!message.data && typeof message.data === 'object';
|
||||
}
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
/**
|
||||
* A deliberately narrow MAIN-world bridge for the mature 1688/Taobao/Tmall
|
||||
* structured collectors. It never exposes cookies, headers, arbitrary window
|
||||
* keys, or a generic fetch primitive; only public product-state projections.
|
||||
*/
|
||||
export default defineContentScript({
|
||||
matches: [
|
||||
'*://detail.1688.com/*',
|
||||
'*://item.taobao.com/*',
|
||||
'*://detail.tmall.com/*',
|
||||
],
|
||||
world: 'MAIN',
|
||||
runAt: 'document_idle',
|
||||
main() {
|
||||
window.addEventListener('message', (event) => {
|
||||
if (event.source !== window || !isRequest(event.data)) return;
|
||||
const data = event.data.platform === '1688'
|
||||
? project1688((window as unknown as Record<string, unknown>).context)
|
||||
: projectTaobao(window as unknown as Record<string, unknown>);
|
||||
window.postMessage({
|
||||
source: 'atlas-collector',
|
||||
type: 'source-page-state.response',
|
||||
requestId: event.data.requestId,
|
||||
data,
|
||||
}, location.origin);
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
type Request = { source: 'atlas-collector'; type: 'source-page-state.request'; requestId: string; platform: '1688' | 'taobao' | 'tmall' };
|
||||
function isRequest(value: unknown): value is Request {
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value)) return false;
|
||||
const data = value as Record<string, unknown>;
|
||||
return data.source === 'atlas-collector' && data.type === 'source-page-state.request'
|
||||
&& typeof data.requestId === 'string' && /^[a-zA-Z0-9-]{16,120}$/.test(data.requestId)
|
||||
&& (data.platform === '1688' || data.platform === 'taobao' || data.platform === 'tmall');
|
||||
}
|
||||
|
||||
const safeClone = (value: unknown) => { try { return JSON.parse(JSON.stringify(value)); } catch { return undefined; } };
|
||||
function find(value: unknown, key: string): unknown {
|
||||
const pending: unknown[] = [value];
|
||||
const seen = new Set<object>();
|
||||
let visited = 0;
|
||||
while (pending.length && visited < 300_000) {
|
||||
const current = pending.shift();
|
||||
if (!current || typeof current !== 'object' || seen.has(current)) continue;
|
||||
seen.add(current);
|
||||
visited += 1;
|
||||
const record = current as Record<string, unknown>;
|
||||
if (key in record) return record[key];
|
||||
pending.push(...Object.values(record));
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
function project1688(context: unknown) {
|
||||
const projected: Record<string, unknown> = {};
|
||||
for (const key of ['gallery', 'tempModel', 'skuModel', 'tradeModel', 'skuMapOriginal', 'pieceWeightScaleInfo', 'productAttributes']) {
|
||||
const value = find(context, key);
|
||||
if (value !== undefined) projected[key] = safeClone(value);
|
||||
}
|
||||
return projected;
|
||||
}
|
||||
function projectTaobao(windowState: Record<string, unknown>) {
|
||||
const ice = windowState.__ICE_APP_CONTEXT__ as Record<string, unknown> | undefined;
|
||||
const result = ice?.loaderData && typeof ice.loaderData === 'object'
|
||||
? (((ice.loaderData as Record<string, unknown>).home as Record<string, unknown> | undefined)?.data as Record<string, unknown> | undefined)?.res
|
||||
: undefined;
|
||||
return { product: safeClone(result) };
|
||||
}
|
||||
@@ -61,6 +61,7 @@ export async function collectOzonProduct(
|
||||
const localStates = readLocalWidgetStates(documentRoot);
|
||||
let attributes = extractOzonCharacteristics(localStates);
|
||||
let description = extractOzonDescription(localStates);
|
||||
const jsonLdDescription = extractOzonJsonLdDescription(documentRoot);
|
||||
const heading = firstState(localStates, 'webProductHeading');
|
||||
const priceState = firstState(localStates, 'webPrice');
|
||||
const title =
|
||||
@@ -101,6 +102,14 @@ export async function collectOzonProduct(
|
||||
'ozon:gallery:page',
|
||||
canonical,
|
||||
);
|
||||
addGalleryVideos(
|
||||
localStates,
|
||||
currentSku,
|
||||
assets,
|
||||
assetKeys,
|
||||
'ozon:gallery:page',
|
||||
canonical,
|
||||
);
|
||||
if (attributes.length) {
|
||||
evidence.push({
|
||||
ref: 'ozon:characteristics:page',
|
||||
@@ -109,6 +118,14 @@ export async function collectOzonProduct(
|
||||
raw_value: `发现 ${attributes.length} 项商品特征`,
|
||||
});
|
||||
}
|
||||
if (jsonLdDescription) {
|
||||
evidence.push({
|
||||
ref: 'ozon:json-ld:product.description',
|
||||
kind: 'json_ld',
|
||||
path: 'script[type="application/ld+json"] Product.description',
|
||||
raw_value: `JSON-LD 商品描述 ${jsonLdDescription.value.length} 个字符`,
|
||||
});
|
||||
}
|
||||
|
||||
const categories = collectCategories(documentRoot, canonical);
|
||||
if (categories.length) {
|
||||
@@ -162,12 +179,22 @@ export async function collectOzonProduct(
|
||||
canonical,
|
||||
);
|
||||
description = description ?? extractOzonDescription(detailStates);
|
||||
const detailAttributes = extractOzonCharacteristics(detailStates);
|
||||
attributes = mergeOzonCharacteristics(attributes, detailAttributes);
|
||||
evidence.push({
|
||||
ref: 'ozon:detail:page',
|
||||
kind: 'state',
|
||||
path: 'webDescription|webRichContent|webFeatures image fields',
|
||||
raw_value: `发现 ${added} 张商品详情图`,
|
||||
});
|
||||
if (detailAttributes.length) {
|
||||
evidence.push({
|
||||
ref: 'ozon:characteristics:detail',
|
||||
kind: 'state',
|
||||
path: 'webShortCharacteristics|webDetailedCharacteristics|webCharacteristics.characteristics[]',
|
||||
raw_value: `详情段补全 ${detailAttributes.length} 项商品特征`,
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
evidence.push(
|
||||
fetchFailureEvidence('ozon:detail:page', currentSku, error),
|
||||
@@ -230,6 +257,14 @@ export async function collectOzonProduct(
|
||||
`ozon:gallery:${result.sku}`,
|
||||
canonical,
|
||||
);
|
||||
addGalleryVideos(
|
||||
result.states,
|
||||
result.sku,
|
||||
assets,
|
||||
assetKeys,
|
||||
`ozon:gallery:${result.sku}`,
|
||||
canonical,
|
||||
);
|
||||
if (result.sku === currentSku) {
|
||||
const detailedAttributes = extractOzonCharacteristics(
|
||||
result.states,
|
||||
@@ -270,6 +305,11 @@ export async function collectOzonProduct(
|
||||
});
|
||||
}
|
||||
|
||||
// Ozon's rich widget is preferred because it is the dedicated description
|
||||
// segment. JSON-LD is the proven old-collector fallback for pages where
|
||||
// Ozon only places the seller description in the initial document.
|
||||
description = description ?? jsonLdDescription;
|
||||
|
||||
addAspectCoverAssets(axes, assets, assetKeys, canonical);
|
||||
|
||||
const variants = [...candidates.values()]
|
||||
@@ -301,7 +341,7 @@ export async function collectOzonProduct(
|
||||
itemKey: currentSku,
|
||||
canonicalUrl: canonical,
|
||||
collectedAt: new Date().toISOString(),
|
||||
parserVersion: 'ozon-public-2.2.0',
|
||||
parserVersion: 'ozon-public-2.3.0',
|
||||
},
|
||||
common: {
|
||||
...(title
|
||||
@@ -452,6 +492,7 @@ function addAspectCoverAssets(
|
||||
assetKeys.add(key);
|
||||
assets.push({
|
||||
url,
|
||||
mediaType: 'image',
|
||||
role: 'source_sku',
|
||||
relation: 'axis_value',
|
||||
sortOrder: assets.length,
|
||||
@@ -491,6 +532,7 @@ function addDetailAssets(
|
||||
assetKeys.add(key);
|
||||
assets.push({
|
||||
url,
|
||||
mediaType: 'image',
|
||||
role: 'source_detail',
|
||||
relation: 'product_candidate',
|
||||
sortOrder: added++,
|
||||
@@ -621,6 +663,55 @@ export function extractOzonDescription(
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/** Reads only schema.org Product.description as a fallback, never arbitrary
|
||||
* JSON-LD page copy. This mirrors the mature collector's JSON-LD path. */
|
||||
export function extractOzonJsonLdDescription(
|
||||
root: Document,
|
||||
): OzonDescription | undefined {
|
||||
for (const script of Array.from(root.scripts)) {
|
||||
if (script.type !== 'application/ld+json' || !script.textContent?.trim())
|
||||
continue;
|
||||
try {
|
||||
const product = findJsonLdProduct(JSON.parse(script.textContent));
|
||||
const raw = product && typeof product.description === 'string'
|
||||
? product.description.trim()
|
||||
: '';
|
||||
if (!raw) continue;
|
||||
return {
|
||||
value: raw.replace(/\s+\n/g, '\n').replace(/\n{3,}/g, '\n\n').slice(0, 100_000),
|
||||
rawValue: raw.slice(0, 120_000),
|
||||
};
|
||||
} catch {
|
||||
// A malformed analytics JSON-LD block is not product evidence.
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function findJsonLdProduct(
|
||||
value: unknown,
|
||||
visited = new Set<unknown>(),
|
||||
): JsonRecord | undefined {
|
||||
if (!value || typeof value !== 'object' || visited.has(value)) return undefined;
|
||||
visited.add(value);
|
||||
if (Array.isArray(value)) {
|
||||
for (const child of value) {
|
||||
const product = findJsonLdProduct(child, visited);
|
||||
if (product) return product;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
const record = value as JsonRecord;
|
||||
const type = record['@type'];
|
||||
if (type === 'Product' || (Array.isArray(type) && type.includes('Product')))
|
||||
return record;
|
||||
if (Array.isArray(record['@graph'])) {
|
||||
const product = findJsonLdProduct(record['@graph'], visited);
|
||||
if (product) return product;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function findRichAnnotationJson(
|
||||
value: unknown,
|
||||
visited = new Set<unknown>(),
|
||||
@@ -716,6 +807,7 @@ function addGalleryAssets(
|
||||
assetKeys.add(key);
|
||||
assets.push({
|
||||
url,
|
||||
mediaType: 'image',
|
||||
// A gallery is product photography for this real SKU. It is not
|
||||
// a SKU-option thumbnail merely because it is the second image.
|
||||
role: 'source_main',
|
||||
@@ -727,6 +819,103 @@ function addGalleryAssets(
|
||||
}
|
||||
}
|
||||
|
||||
/** Adds only Ozon's named product-gallery videos. Page-wide <video> nodes can
|
||||
* belong to reviews or recommendations and are intentionally never considered. */
|
||||
function addGalleryVideos(
|
||||
states: JsonRecord[],
|
||||
sku: string,
|
||||
assets: Asset[],
|
||||
assetKeys: Set<string>,
|
||||
evidenceRef: string,
|
||||
pageUrl: string,
|
||||
) {
|
||||
const galleries = states.filter((state) => {
|
||||
const widget = String(state.__atlasWidgetName ?? '').replace(/^state-/, '');
|
||||
return /^web(?:Gallery|Video)-/.test(widget);
|
||||
});
|
||||
for (const gallery of galleries) {
|
||||
for (const video of extractOzonGalleryVideos(gallery, pageUrl)) {
|
||||
if (assets.length >= 500) return;
|
||||
const key = `video:${sku}:${video.url}`;
|
||||
if (assetKeys.has(key)) continue;
|
||||
assetKeys.add(key);
|
||||
assets.push({
|
||||
url: video.url,
|
||||
mediaType: 'video',
|
||||
...(video.posterUrl ? { posterUrl: video.posterUrl } : {}),
|
||||
role: 'source_main',
|
||||
variantSourceKey: sku,
|
||||
relation: 'variant_direct',
|
||||
sortOrder: assets.length,
|
||||
evidenceRef,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Pure helper for the strictly named Ozon gallery/video state. HLS, blobs,
|
||||
* and non-HTTPS URLs stay out of the collection contract in this first phase. */
|
||||
export function extractOzonGalleryVideos(gallery: JsonRecord, pageUrl: string) {
|
||||
const videos: Array<{ url: string; posterUrl?: string }> = [];
|
||||
const seen = new Set<string>();
|
||||
// Ozon changes whether the direct file lives in videos[], video, or a
|
||||
// nested gallery payload. The old collector walked this *one allowlisted
|
||||
// widget* for direct video extensions; retaining that scope is critical.
|
||||
for (const { value, parent } of nestedVideoValues(gallery)) {
|
||||
const url = directVideoUrl(
|
||||
typeof value === 'string'
|
||||
? value
|
||||
: stringAt(value, ['url', 'videoUrl', 'src']),
|
||||
pageUrl,
|
||||
);
|
||||
if (!url) continue;
|
||||
const posterUrl = ozonImageUrl(
|
||||
(typeof value === 'string'
|
||||
? undefined
|
||||
: stringAt(value, ['coverImage', 'coverUrl', 'poster', 'previewImage']))
|
||||
?? stringAt(parent, ['coverImage', 'coverUrl', 'poster', 'previewImage']),
|
||||
pageUrl,
|
||||
);
|
||||
if (seen.has(url)) {
|
||||
const existing = videos.find((video) => video.url === url);
|
||||
if (existing && !existing.posterUrl && posterUrl)
|
||||
existing.posterUrl = posterUrl;
|
||||
continue;
|
||||
}
|
||||
seen.add(url);
|
||||
videos.push({ url, ...(posterUrl ? { posterUrl } : {}) });
|
||||
}
|
||||
return videos;
|
||||
}
|
||||
|
||||
function nestedVideoValues(root: JsonRecord) {
|
||||
const values: Array<{ value: string | JsonRecord; parent?: JsonRecord }> = [];
|
||||
const pending: Array<{ value: unknown; parent?: JsonRecord }> = [{ value: root }];
|
||||
const seen = new Set<object>();
|
||||
while (pending.length) {
|
||||
const current = pending.pop()!;
|
||||
if (typeof current.value === 'string') {
|
||||
if (/\.(mp4|webm)(?:$|[?#])/i.test(current.value))
|
||||
values.push({ value: current.value, parent: current.parent });
|
||||
continue;
|
||||
}
|
||||
if (!current.value || typeof current.value !== 'object') continue;
|
||||
if (seen.has(current.value)) continue;
|
||||
seen.add(current.value);
|
||||
if (Array.isArray(current.value)) {
|
||||
for (const child of current.value)
|
||||
pending.push({ value: child, parent: current.parent });
|
||||
continue;
|
||||
}
|
||||
const record = current.value as JsonRecord;
|
||||
if (stringAt(record, ['url', 'videoUrl', 'src']))
|
||||
values.push({ value: record, parent: current.parent });
|
||||
for (const child of Object.values(record))
|
||||
pending.push({ value: child, parent: record });
|
||||
}
|
||||
return values;
|
||||
}
|
||||
|
||||
/** Pure helper used by fixture tests and the structured Ozon gallery reader. */
|
||||
export function extractOzonGalleryImageUrls(
|
||||
gallery: JsonRecord,
|
||||
@@ -777,7 +966,7 @@ function widgetStatesFromPageJson(value: unknown): JsonRecord[] {
|
||||
const states: JsonRecord[] = [];
|
||||
for (const [widgetName, raw] of Object.entries(value.widgetStates)) {
|
||||
if (
|
||||
!/^web(Aspects|AspectsModal|ProductHeading|Price|Gallery|Description|RichContent|Features|ShortCharacteristics|DetailedCharacteristics|Characteristics)-/.test(
|
||||
!/^web(Aspects|AspectsModal|ProductHeading|Price|Gallery|Video|Description|RichContent|Features|ShortCharacteristics|DetailedCharacteristics|Characteristics)-/.test(
|
||||
widgetName,
|
||||
)
|
||||
)
|
||||
@@ -997,3 +1186,15 @@ function ozonImageUrl(value: string | undefined, pageUrl: string) {
|
||||
const url = normaliseImageUrl(value, pageUrl, 'ozon');
|
||||
return url?.startsWith('https://') ? url : undefined;
|
||||
}
|
||||
|
||||
function directVideoUrl(value: string | undefined, pageUrl: string) {
|
||||
if (!value) return undefined;
|
||||
try {
|
||||
const url = new URL(value, pageUrl);
|
||||
if (url.protocol !== 'https:' || /\.m3u8(?:$|[?#])/i.test(url.pathname))
|
||||
return undefined;
|
||||
return /\.(mp4|webm)(?:$|[?#])/i.test(url.pathname) ? url.href : undefined;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -180,6 +180,7 @@ export function collectPlatformDomAssets(
|
||||
seen.add(key);
|
||||
assets.push({
|
||||
url,
|
||||
mediaType: 'image',
|
||||
role: group.role,
|
||||
relation: group.relation,
|
||||
sortOrder: assets.length,
|
||||
|
||||
@@ -22,22 +22,32 @@ export function supportedProductPlatform(
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Returns only the public, allowlisted product-state projection supplied by
|
||||
* the platform-specific MAIN-world bridge. */
|
||||
export type MarketplacePageStateFetcher = (
|
||||
platform: Exclude<SourcePlatform, 'ozon'>,
|
||||
) => Promise<unknown>;
|
||||
|
||||
export async function collectCurrentProduct(
|
||||
root: Document = document,
|
||||
pageUrl = location.href,
|
||||
fetchOzonPageJson?: OzonPageJsonFetcher,
|
||||
fetchMarketplacePageState?: MarketplacePageStateFetcher,
|
||||
): Promise<CollectedProductPayload> {
|
||||
const platform = supportedProductPlatform(pageUrl);
|
||||
if (platform === 'ozon')
|
||||
return collectOzonProduct(root, pageUrl, fetchOzonPageJson);
|
||||
if (!platform) throw new Error('当前页面不是支持的商品详情页');
|
||||
const pageState = fetchMarketplacePageState
|
||||
? await fetchMarketplacePageState(platform)
|
||||
: undefined;
|
||||
return platform === '1688'
|
||||
? collect1688(root, pageUrl)
|
||||
: collectTaobao(root, pageUrl, platform);
|
||||
? collect1688(root, pageUrl, pageState)
|
||||
: collectTaobao(root, pageUrl, platform, pageState);
|
||||
}
|
||||
|
||||
function collect1688(root: Document, pageUrl: string): CollectedProductPayload {
|
||||
const state = pageState(root);
|
||||
function collect1688(root: Document, pageUrl: string, bridgedState?: unknown): CollectedProductPayload {
|
||||
const state = [bridgedState, ...pageState(root)];
|
||||
const skuMap = findFirst(state, 'skuMapOriginal');
|
||||
const skuProps = findFirst(state, 'skuProps');
|
||||
const axes = parse1688Axes(skuProps);
|
||||
@@ -50,14 +60,19 @@ function collect1688(root: Document, pageUrl: string): CollectedProductPayload {
|
||||
},
|
||||
];
|
||||
const variants: Variant[] = [];
|
||||
if (isRecord(skuMap)) {
|
||||
const skuRows = Array.isArray(skuMap)
|
||||
? skuMap.map((row) => [stringAt(isRecord(row) ? row : {}, ['specAttrs', 'propPath', 'skuId']) ?? '', row] as const)
|
||||
: isRecord(skuMap)
|
||||
? Object.entries(skuMap)
|
||||
: [];
|
||||
if (skuRows.length) {
|
||||
evidence.push({
|
||||
ref: '1688:sku-map',
|
||||
kind: 'state',
|
||||
path: 'skuMapOriginal',
|
||||
raw_value: `组合记录 ${Object.keys(skuMap).length} 条`,
|
||||
raw_value: `组合记录 ${skuRows.length} 条`,
|
||||
});
|
||||
for (const [combination, row] of Object.entries(skuMap)) {
|
||||
for (const [combination, row] of skuRows) {
|
||||
if (!isRecord(row)) continue;
|
||||
const sourceSku = stringAt(row, ['skuId', 'sku_id', 'skuCode']);
|
||||
// A skuMapOriginal entry itself is a source-provided sellable-combination
|
||||
@@ -97,7 +112,8 @@ function collect1688(root: Document, pageUrl: string): CollectedProductPayload {
|
||||
axes,
|
||||
uniqueVariants(variants),
|
||||
evidence,
|
||||
'1688-state-1.0.0',
|
||||
'1688-structured-2.0.0',
|
||||
state,
|
||||
);
|
||||
}
|
||||
|
||||
@@ -105,8 +121,9 @@ function collectTaobao(
|
||||
root: Document,
|
||||
pageUrl: string,
|
||||
platform: 'taobao' | 'tmall',
|
||||
bridgedState?: unknown,
|
||||
): CollectedProductPayload {
|
||||
const state = pageState(root);
|
||||
const state = [bridgedState, ...pageState(root)];
|
||||
const skuBase = findFirst(state, 'skuBase');
|
||||
const skuCore = findFirst(state, 'skuCore');
|
||||
const axes = parseTaobaoAxes(isRecord(skuBase) ? skuBase.props : undefined);
|
||||
@@ -170,7 +187,8 @@ function collectTaobao(
|
||||
axes,
|
||||
uniqueVariants(variants),
|
||||
evidence,
|
||||
`${platform}-state-1.0.0`,
|
||||
`${platform}-structured-2.0.0`,
|
||||
state,
|
||||
);
|
||||
}
|
||||
|
||||
@@ -183,15 +201,18 @@ function basePayload(
|
||||
variants: Variant[],
|
||||
evidence: CollectedProductPayload['evidence'],
|
||||
parserVersion: string,
|
||||
states: unknown[],
|
||||
): CollectedProductPayload {
|
||||
const canonical =
|
||||
root.querySelector<HTMLLinkElement>('link[rel="canonical"]')?.href ??
|
||||
pageUrl.split('#')[0];
|
||||
const title =
|
||||
const facts = extractMarketplaceFacts(root, states, platform);
|
||||
const title = facts.title ?? (
|
||||
root.querySelector('h1')?.textContent?.replace(/\s+/g, ' ').trim() ||
|
||||
root
|
||||
.querySelector<HTMLMetaElement>('meta[property="og:title"]')
|
||||
?.content?.trim();
|
||||
?.content?.trim()
|
||||
);
|
||||
const assets = collectPlatformDomAssets(platform, root, canonical);
|
||||
evidence.push({
|
||||
ref: `${platform}:dom:image-profile`,
|
||||
@@ -199,6 +220,15 @@ function basePayload(
|
||||
path: 'platform-scoped gallery / sku / detail selectors',
|
||||
raw_value: `采集 ${assets.length} 张受限商品区域图片`,
|
||||
});
|
||||
const videoCount = addMarketplaceVideos(platform, states, assets, canonical);
|
||||
if (videoCount) evidence.push({
|
||||
ref: `${platform}:structured:videos`,
|
||||
kind: 'state',
|
||||
path: platform === '1688' ? 'gallery.fields.video|videos' : 'item.videos',
|
||||
raw_value: `采集 ${videoCount} 个结构化商品视频`,
|
||||
});
|
||||
if (facts.attributes.length) evidence.push({ ref: `${platform}:structured:attributes`, kind: 'state', path: 'allowlisted product state parameter lists', raw_value: `采集 ${facts.attributes.length} 项结构化特征` });
|
||||
if (facts.description) evidence.push({ ref: `${platform}:json-ld:description`, kind: 'json_ld', path: 'Product.description', raw_value: `采集 ${facts.description.length} 个字符描述` });
|
||||
return {
|
||||
source: {
|
||||
platform,
|
||||
@@ -217,8 +247,10 @@ function basePayload(
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
attributes: [],
|
||||
categories: [],
|
||||
...(facts.price ? { sourcePrice: { amount: facts.price, currencyCode: 'CNY', raw_value: facts.price } } : {}),
|
||||
...(facts.description ? { description: { value: facts.description, raw_value: facts.description, confidence: 'verified' as const } } : {}),
|
||||
attributes: facts.attributes,
|
||||
categories: facts.categories,
|
||||
},
|
||||
variantAxes: axes,
|
||||
variants,
|
||||
@@ -227,6 +259,137 @@ function basePayload(
|
||||
};
|
||||
}
|
||||
|
||||
/** The old collector's proven video sources are structured product state. Do
|
||||
* not broaden this to page video elements: they also contain feeds and reviews. */
|
||||
function addMarketplaceVideos(
|
||||
platform: Exclude<SourcePlatform, 'ozon'>,
|
||||
states: unknown[],
|
||||
assets: CollectedProductPayload['assets'],
|
||||
pageUrl: string,
|
||||
) {
|
||||
const gallery = findFirst(states, 'gallery');
|
||||
const item = findFirst(states, 'item');
|
||||
const galleryFields = isRecord(gallery) && isRecord(gallery.fields)
|
||||
? gallery.fields
|
||||
: gallery;
|
||||
const raw = platform === '1688'
|
||||
? [
|
||||
...(isRecord(galleryFields) && Array.isArray(galleryFields.videos) ? galleryFields.videos : []),
|
||||
...(isRecord(galleryFields) && galleryFields.video !== undefined ? [galleryFields.video] : []),
|
||||
]
|
||||
: isRecord(item) && Array.isArray(item.videos)
|
||||
? item.videos
|
||||
: [];
|
||||
let added = 0;
|
||||
const seen = new Set(assets.map((asset) => `video:${asset.url}`));
|
||||
for (const entry of raw) {
|
||||
if (assets.length >= 500) break;
|
||||
const record = isRecord(entry) ? entry : undefined;
|
||||
const url = directVideoUrl(
|
||||
typeof entry === 'string' ? entry : stringAt(record ?? {}, ['url', 'videoUrl', 'src']),
|
||||
pageUrl,
|
||||
);
|
||||
if (!url || seen.has(`video:${url}`)) continue;
|
||||
seen.add(`video:${url}`);
|
||||
const posterUrl = imageUrl(
|
||||
stringAt(record ?? {}, ['videoThumbnailURL', 'thumbnailUrl', 'coverUrl', 'coverImage', 'poster']),
|
||||
pageUrl,
|
||||
);
|
||||
assets.push({
|
||||
url,
|
||||
mediaType: 'video',
|
||||
...(posterUrl ? { posterUrl } : {}),
|
||||
role: 'source_main',
|
||||
relation: 'product_candidate',
|
||||
sortOrder: assets.length,
|
||||
evidenceRef: `${platform}:structured:videos`,
|
||||
});
|
||||
added += 1;
|
||||
}
|
||||
return added;
|
||||
}
|
||||
|
||||
function directVideoUrl(value: string | undefined, pageUrl: string) {
|
||||
if (!value) return undefined;
|
||||
try {
|
||||
const url = new URL(value, pageUrl);
|
||||
if (url.protocol !== 'https:' || /\.m3u8(?:$|[?#])/i.test(url.pathname)) return undefined;
|
||||
return /\.(mp4|webm)(?:$|[?#])/i.test(url.pathname) ? url.href : undefined;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
function imageUrl(value: string | undefined, pageUrl: string) {
|
||||
if (!value) return undefined;
|
||||
try {
|
||||
const url = new URL(value, pageUrl);
|
||||
return url.protocol === 'https:' ? url.href : undefined;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
function extractMarketplaceFacts(root: Document, states: unknown[], platform: Exclude<SourcePlatform, 'ozon'>) {
|
||||
const attributes: CollectedProductPayload['common']['attributes'] = [];
|
||||
const seen = new Set<string>();
|
||||
for (const key of ['basicParamList', 'enhanceParamList', 'attributes', 'props']) {
|
||||
const list = findFirst(states, key);
|
||||
if (!Array.isArray(list)) continue;
|
||||
for (const [index, entry] of list.entries()) {
|
||||
if (!isRecord(entry)) continue;
|
||||
const name = stringAt(entry, ['propertyName', 'name', 'key', 'title']);
|
||||
const value = stringAt(entry, ['valueName', 'value', 'text']);
|
||||
if (!name || !value || seen.has(`${name}\u0000${value}`)) continue;
|
||||
seen.add(`${name}\u0000${value}`);
|
||||
attributes.push({ key: `${platform}:${stringAt(entry, ['propertyId', 'id', 'key']) ?? index}`.slice(0, 200), name: name.slice(0, 300), value: value.slice(0, 2_000), raw_value: value.slice(0, 4_000) });
|
||||
}
|
||||
}
|
||||
const product = jsonLdProduct(root);
|
||||
const item = findFirst(states, 'item');
|
||||
const temp = findFirst(states, 'tempModel');
|
||||
const tradeModel = findFirst(states, 'tradeModel');
|
||||
const priceVO = findFirst(states, 'priceVO');
|
||||
const price = moneyAt(isRecord(tradeModel) ? tradeModel : {}, ['minPrice', 'price'])
|
||||
?? moneyAt(isRecord(priceVO) ? priceVO : {}, ['priceText', 'price', 'extraPrice'])
|
||||
?? decimal(product?.offers && isRecord(product.offers) ? product.offers.price : undefined);
|
||||
const categoryNames = Array.from(root.querySelectorAll<HTMLAnchorElement>('[class*="breadcrumb"] a, [class*="BreadCrumb"] a, nav a[href*="category"]')).map((link) => link.textContent?.replace(/\s+/g, ' ').trim() ?? '').filter(Boolean).slice(0, 50);
|
||||
return {
|
||||
title: (isRecord(item) ? stringAt(item, ['title', 'name']) : undefined) ?? (isRecord(temp) ? stringAt(temp, ['offerTitle']) : undefined) ?? (product ? stringAt(product, ['name']) : undefined),
|
||||
price,
|
||||
description: (product ? stringAt(product, ['description']) : undefined) ?? scopedDescription(root, platform),
|
||||
attributes,
|
||||
categories: [...new Set(categoryNames)].map((name) => ({ key: `${platform}:category:${name.toLowerCase()}`.slice(0, 300), name })),
|
||||
};
|
||||
}
|
||||
|
||||
function scopedDescription(root: Document, platform: Exclude<SourcePlatform, 'ozon'>) {
|
||||
const selectors = platform === '1688'
|
||||
? ['.de-description-detail', '#detailContentContainer', '.html-description']
|
||||
: ['[class*="tabDetailWrap--"]', '[class*="detailInfo--"]'];
|
||||
for (const selector of selectors) {
|
||||
const value = root.querySelector(selector)?.textContent?.replace(/\s+/g, ' ').trim();
|
||||
if (value && value.length >= 10) return value.slice(0, 100_000);
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function jsonLdProduct(root: Document): Json | undefined {
|
||||
for (const script of Array.from(root.scripts)) {
|
||||
if (script.type !== 'application/ld+json' || !script.textContent?.trim()) continue;
|
||||
try { const found = findJsonLdProduct(JSON.parse(script.textContent)); if (found) return found; } catch { /* ignored */ }
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
function findJsonLdProduct(value: unknown): Json | undefined {
|
||||
if (Array.isArray(value)) { for (const child of value) { const found = findJsonLdProduct(child); if (found) return found; } return undefined; }
|
||||
if (!isRecord(value)) return undefined;
|
||||
const type = value['@type'];
|
||||
if (type === 'Product' || (Array.isArray(type) && type.includes('Product'))) return value;
|
||||
return Array.isArray(value['@graph']) ? findJsonLdProduct(value['@graph']) : undefined;
|
||||
}
|
||||
function decimal(value: unknown) { const raw = typeof value === 'string' || typeof value === 'number' ? String(value).replace(',', '.') : ''; return /^\d+(\.\d{1,4})?$/.test(raw) ? raw : undefined; }
|
||||
|
||||
function pageState(root: Document) {
|
||||
const all: unknown[] = [];
|
||||
for (const script of Array.from(root.scripts)) {
|
||||
|
||||
@@ -5,7 +5,9 @@ import test from 'node:test';
|
||||
import {
|
||||
extractOzonCharacteristics,
|
||||
extractOzonDescription,
|
||||
extractOzonJsonLdDescription,
|
||||
extractOzonGalleryImageUrls,
|
||||
extractOzonGalleryVideos,
|
||||
} from '../lib/ozon-parser.ts';
|
||||
import { dedupeImageUrl, normaliseImageUrl } from '../lib/product-media.ts';
|
||||
|
||||
@@ -73,6 +75,41 @@ test('Ozon rich annotation converts to description text without images', () => {
|
||||
assert.equal(description?.value, '耐用的背包\n适合旅行');
|
||||
});
|
||||
|
||||
test('Ozon JSON-LD Product description is the fallback when rich content is absent', () => {
|
||||
const root = {
|
||||
scripts: [{
|
||||
type: 'application/ld+json',
|
||||
textContent: JSON.stringify({
|
||||
'@context': 'http://schema.org',
|
||||
'@type': 'Product',
|
||||
name: '喷雾器',
|
||||
description: '来自 Ozon JSON-LD 的完整商品描述',
|
||||
}),
|
||||
}],
|
||||
} as unknown as Document;
|
||||
assert.equal(
|
||||
extractOzonJsonLdDescription(root)?.value,
|
||||
'来自 Ozon JSON-LD 的完整商品描述',
|
||||
);
|
||||
});
|
||||
|
||||
test('Ozon full characteristics merge with the five short header characteristics', () => {
|
||||
const attributes = extractOzonCharacteristics([
|
||||
{
|
||||
__atlasWidgetName: 'webShortCharacteristics-fixture',
|
||||
characteristics: [{ id: 'type', title: { textRs: [{ content: '类型' }] }, values: [{ text: '喷雾器' }] }],
|
||||
},
|
||||
{
|
||||
__atlasWidgetName: 'webCharacteristics-fixture',
|
||||
characteristics: [{ id: 'weight', title: { textRs: [{ content: '商品重量，克' }] }, values: [{ text: '385' }] }],
|
||||
},
|
||||
]);
|
||||
assert.deepEqual(attributes.map(({ name, value }) => ({ name, value })), [
|
||||
{ name: '类型', value: '喷雾器' },
|
||||
{ name: '商品重量，克', value: '385' },
|
||||
]);
|
||||
});
|
||||
|
||||
test('Ozon thumbnail variants normalise to one original URL', () => {
|
||||
const original = 'https://ir.ozone.ru/s3/multimedia-1-9/13597160877.jpg';
|
||||
const thumbnail =
|
||||
@@ -84,6 +121,29 @@ test('Ozon thumbnail variants normalise to one original URL', () => {
|
||||
);
|
||||
});
|
||||
|
||||
test('Ozon gallery only accepts direct product MP4/WebM videos and retains poster', () => {
|
||||
assert.deepEqual(
|
||||
extractOzonGalleryVideos(
|
||||
{
|
||||
videos: [
|
||||
{
|
||||
url: 'https://cdn.ozon.ru/product/demo.mp4',
|
||||
coverImage: 'https://ir.ozone.ru/s3/multimedia-1/demo.jpg',
|
||||
},
|
||||
{ videoUrl: 'https://cdn.ozon.ru/product/demo.mp4' },
|
||||
{ url: 'https://cdn.ozon.ru/product/stream.m3u8' },
|
||||
{ url: 'blob:https://www.ozon.ru/opaque' },
|
||||
],
|
||||
},
|
||||
'https://www.ozon.ru/product/demo-1/',
|
||||
),
|
||||
[{
|
||||
url: 'https://cdn.ozon.ru/product/demo.mp4',
|
||||
posterUrl: 'https://ir.ozone.ru/s3/multimedia-1/demo.jpg',
|
||||
}],
|
||||
);
|
||||
});
|
||||
|
||||
test('Alibaba transformed image URLs normalise before de-duplication', () => {
|
||||
const original = 'https://cbu01.alicdn.com/img/ibank/O1CN01.jpg';
|
||||
const transformed = `${original}_400x400.jpg?width=400`;
|
||||
|
||||
@@ -49,6 +49,7 @@ export default function CollectionInspectionPage() {
|
||||
() => new Map((record?.variants ?? []).map((variant) => [variant.id, variant])),
|
||||
[record]
|
||||
);
|
||||
const payload = normalisedPayload(record?.normalized_payload);
|
||||
|
||||
if (error)
|
||||
return <Alert type="error" message="无法读取采集数据" description={error} showIcon />;
|
||||
@@ -65,7 +66,7 @@ export default function CollectionInspectionPage() {
|
||||
type="info"
|
||||
showIcon
|
||||
style={{ marginBottom: 16 }}
|
||||
message="这里展示扩展实际提交、服务端实际保存的数据，不代表已确认的商品编辑字段。"
|
||||
message="这里展示扩展实际提交、服务端实际保存的数据；未转存图片或视频直接使用采集来源预览，不代表已确认的商品编辑字段。"
|
||||
/>
|
||||
<Descriptions
|
||||
bordered
|
||||
@@ -81,9 +82,71 @@ export default function CollectionInspectionPage() {
|
||||
{ key: 'status', label: '采集状态', children: record.collection_status },
|
||||
{ key: 'parser', label: '解析器版本', children: record.parser_version ?? '—' },
|
||||
{ key: 'sku', label: '已创建 SKU', children: record.variants.length },
|
||||
{ key: 'assets', label: '素材任务', children: record.asset_transfers.length },
|
||||
{ key: 'assets', label: '素材候选', children: record.asset_transfers.length },
|
||||
{ key: 'attributes', label: '已采集特征', children: payload.attributes.length },
|
||||
{ key: 'categories', label: '已采集类目', children: payload.categories.length },
|
||||
]}
|
||||
/>
|
||||
<Typography.Title level={5} style={{ marginTop: 24 }}>
|
||||
商品描述
|
||||
</Typography.Title>
|
||||
<Card size="small">
|
||||
{payload.description ? (
|
||||
<Typography.Paragraph style={{ whiteSpace: 'pre-wrap', marginBottom: 0 }}>
|
||||
{payload.description}
|
||||
</Typography.Paragraph>
|
||||
) : (
|
||||
<Typography.Text type="secondary">本次采集未取得商品描述。</Typography.Text>
|
||||
)}
|
||||
</Card>
|
||||
<Typography.Title level={5} style={{ marginTop: 24 }}>
|
||||
类目与特征
|
||||
</Typography.Title>
|
||||
<Descriptions
|
||||
bordered
|
||||
size="small"
|
||||
column={1}
|
||||
items={[
|
||||
{
|
||||
key: 'category-path',
|
||||
label: '类目路径',
|
||||
children: payload.categories.length
|
||||
? payload.categories.map((category) => category.name).join(' › ')
|
||||
: '未采集',
|
||||
},
|
||||
]}
|
||||
/>
|
||||
<Table
|
||||
size="small"
|
||||
pagination={false}
|
||||
rowKey={(attribute) => `${attribute.key}:${attribute.value}`}
|
||||
dataSource={payload.attributes}
|
||||
style={{ marginTop: 12 }}
|
||||
columns={[
|
||||
{ title: '特征', dataIndex: 'name' },
|
||||
{ title: '值', dataIndex: 'value' },
|
||||
{ title: '来源键', dataIndex: 'key', ellipsis: true },
|
||||
]}
|
||||
locale={{ emptyText: '本次采集未取得商品特征。' }}
|
||||
/>
|
||||
<Typography.Title level={5} style={{ marginTop: 24 }}>
|
||||
规格轴
|
||||
</Typography.Title>
|
||||
<Table
|
||||
size="small"
|
||||
pagination={false}
|
||||
rowKey="key"
|
||||
dataSource={payload.variantAxes}
|
||||
columns={[
|
||||
{ title: '规格名称', dataIndex: 'name' },
|
||||
{ title: '来源键', dataIndex: 'key', ellipsis: true },
|
||||
{
|
||||
title: '规格值',
|
||||
render: (_, axis) => axis.values.map((value) => value.name).join(' / ') || '—',
|
||||
},
|
||||
]}
|
||||
locale={{ emptyText: '本次采集未取得规格轴。' }}
|
||||
/>
|
||||
<Typography.Title level={5} style={{ marginTop: 24 }}>
|
||||
SKU 草稿
|
||||
</Typography.Title>
|
||||
@@ -111,7 +174,7 @@ export default function CollectionInspectionPage() {
|
||||
]}
|
||||
/>
|
||||
<Typography.Title level={5} style={{ marginTop: 24 }}>
|
||||
素材转存任务
|
||||
素材候选与转存状态
|
||||
</Typography.Title>
|
||||
<Table
|
||||
size="small"
|
||||
@@ -124,26 +187,50 @@ export default function CollectionInspectionPage() {
|
||||
width: 108,
|
||||
render: (_, transfer) => {
|
||||
const url = transfer.asset_id
|
||||
? previewUrls[transfer.asset_id]
|
||||
: undefined;
|
||||
? previewUrls[transfer.asset_id] ?? transfer.source_url
|
||||
: transfer.source_url;
|
||||
if (transfer.media_type === 'video')
|
||||
return url ? (
|
||||
<video
|
||||
controls
|
||||
preload="none"
|
||||
poster={transfer.poster_url ?? undefined}
|
||||
src={url}
|
||||
style={{ width: 96, height: 72, objectFit: 'cover', background: '#000' }}
|
||||
>
|
||||
当前浏览器不支持视频预览。
|
||||
</video>
|
||||
) : (
|
||||
<Typography.Text type="secondary">
|
||||
暂无来源视频
|
||||
</Typography.Text>
|
||||
);
|
||||
if (url)
|
||||
return (
|
||||
<Image
|
||||
width={72}
|
||||
height={72}
|
||||
src={url}
|
||||
alt="已转存采集素材"
|
||||
alt={
|
||||
transfer.asset_id
|
||||
? '已转存采集素材'
|
||||
: '采集来源图片'
|
||||
}
|
||||
style={{ objectFit: 'cover' }}
|
||||
/>
|
||||
);
|
||||
return (
|
||||
<Typography.Text type="secondary">
|
||||
{transfer.status === 'succeeded' ? '预览加载中…' : '待转存'}
|
||||
暂无来源图片
|
||||
</Typography.Text>
|
||||
);
|
||||
},
|
||||
},
|
||||
{ title: '类型', dataIndex: 'role' },
|
||||
{
|
||||
title: '类型',
|
||||
render: (_, transfer) =>
|
||||
`${transfer.media_type === 'video' ? '视频' : '图片'} · ${transfer.role}`,
|
||||
},
|
||||
{
|
||||
title: '关联 SKU',
|
||||
render: (_, transfer) => {
|
||||
@@ -153,7 +240,20 @@ export default function CollectionInspectionPage() {
|
||||
return variant?.source_sku ?? variant?.source_variant_key ?? '商品级';
|
||||
},
|
||||
},
|
||||
{ title: '状态', dataIndex: 'status' },
|
||||
{
|
||||
title: '状态',
|
||||
dataIndex: 'status',
|
||||
render: (status) => {
|
||||
if (status === 'pending_selection')
|
||||
return <Tag color="default">候选，未转存</Tag>;
|
||||
if (status === 'queued') return <Tag color="processing">等待转存</Tag>;
|
||||
if (status === 'running') return <Tag color="processing">转存中</Tag>;
|
||||
if (status === 'succeeded') return <Tag color="success">已转存</Tag>;
|
||||
if (status === 'failed') return <Tag color="error">转存失败</Tag>;
|
||||
if (status === 'cancelled') return <Tag>已取消</Tag>;
|
||||
return status;
|
||||
},
|
||||
},
|
||||
{
|
||||
title: '来源 URL',
|
||||
dataIndex: 'source_url',
|
||||
@@ -229,3 +329,52 @@ function Json({ value }: { value: unknown }) {
|
||||
</pre>
|
||||
);
|
||||
}
|
||||
|
||||
type InspectionAttribute = { key: string; name: string; value: string };
|
||||
type InspectionCategory = { key: string; name: string };
|
||||
type InspectionAxis = { key: string; name: string; values: Array<{ key: string; name: string }> };
|
||||
|
||||
function normalisedPayload(value: unknown): {
|
||||
description: string | null;
|
||||
attributes: InspectionAttribute[];
|
||||
categories: InspectionCategory[];
|
||||
variantAxes: InspectionAxis[];
|
||||
} {
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value))
|
||||
return { description: null, attributes: [], categories: [], variantAxes: [] };
|
||||
const root = value as Record<string, unknown>;
|
||||
const common = record(root.common);
|
||||
const description = record(common?.description)?.value;
|
||||
return {
|
||||
description: typeof description === 'string' ? description : null,
|
||||
attributes: records(common?.attributes).flatMap((item) =>
|
||||
typeof item.key === 'string' && typeof item.name === 'string' && typeof item.value === 'string'
|
||||
? [{ key: item.key, name: item.name, value: item.value }]
|
||||
: []),
|
||||
categories: records(common?.categories).flatMap((item) =>
|
||||
typeof item.key === 'string' && typeof item.name === 'string'
|
||||
? [{ key: item.key, name: item.name }]
|
||||
: []),
|
||||
variantAxes: records(root.variantAxes).flatMap((item) => {
|
||||
if (typeof item.key !== 'string' || typeof item.name !== 'string') return [];
|
||||
return [{
|
||||
key: item.key,
|
||||
name: item.name,
|
||||
values: records(item.values).flatMap((option) =>
|
||||
typeof option.key === 'string' && typeof option.name === 'string'
|
||||
? [{ key: option.key, name: option.name }]
|
||||
: []),
|
||||
}];
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
function record(value: unknown): Record<string, unknown> | undefined {
|
||||
return value && typeof value === 'object' && !Array.isArray(value)
|
||||
? value as Record<string, unknown>
|
||||
: undefined;
|
||||
}
|
||||
|
||||
function records(value: unknown): Record<string, unknown>[] {
|
||||
return Array.isArray(value) ? value.filter((item): item is Record<string, unknown> => !!record(item)) : [];
|
||||
}
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
import type { CollectionItem, CollectionItemInspection, CreateCollectionItemRequest, CursorPaginationMeta, ListCollectionItemsQuery, UpdateCollectionItemRequest } from '@atlas/contracts';
|
||||
import type { CollectionItem, CollectionItemInspection, CreateCollectionItemRequest, CursorPaginationMeta, ListCollectionItemsQuery, MaterializeCollectionAssetsRequest, UpdateCollectionItemRequest } from '@atlas/contracts';
|
||||
import { api } from './api';
|
||||
export const collectionApi = {
|
||||
list: (platformId: string, query: Partial<ListCollectionItemsQuery> = {}) => api.getList<CollectionItem, CursorPaginationMeta & { active_count: number; collection_limit: number }>('/collection-items', { headers: { 'X-Atlas-Platform-Id': platformId }, params: { limit: 20, ...query } }),
|
||||
create: (payload: CreateCollectionItemRequest) => api.post<{ collection_item: CollectionItem; product_id: string; editor_entry_path: string }>('/collection-items', payload, { headers: { 'Idempotency-Key': crypto.randomUUID() } }),
|
||||
inspection: (id: string) => api.get<CollectionItemInspection>(`/collection-items/${id}`),
|
||||
materializeAssets: (id: string, payload: MaterializeCollectionAssetsRequest) => api.post<{ operation: { id: string; kind: string; status: string } }>(`/collection-items/${id}/asset-materializations`, payload, { headers: { 'Idempotency-Key': crypto.randomUUID() } }),
|
||||
update: (id: string, payload: UpdateCollectionItemRequest, revision: number) => api.patch<CollectionItem>(`/collection-items/${id}`, payload, { headers: { 'Idempotency-Key': crypto.randomUUID(), 'If-Match': `"${revision}"` } }),
|
||||
bulkDelete: (ids: string[]) => api.post<{ deleted_count: number }>('/collection-items/bulk-delete', { ids }, { headers: { 'Idempotency-Key': crypto.randomUUID() } }),
|
||||
clear: (expected_active_count: number) => api.post<{ deleted_count: number }>('/collection-items/clear', { confirm: true, expected_active_count }, { headers: { 'Idempotency-Key': crypto.randomUUID() } }),
|
||||
|
||||
@@ -5,11 +5,12 @@ import { CreateBucketCommand, HeadBucketCommand, PutObjectCommand, type S3Client
|
||||
import type { Database } from '@atlas/database';
|
||||
import type { OutboxJob } from './outbox-dispatcher.js';
|
||||
|
||||
const MAX_BYTES = 20 * 1024 * 1024;
|
||||
type Transfer = { id: string; source_url: string; source_variant_id: string | null; source_snapshot_revision: string | number };
|
||||
const MAX_IMAGE_BYTES = 20 * 1024 * 1024;
|
||||
const MAX_VIDEO_BYTES = 200 * 1024 * 1024;
|
||||
type Transfer = { id: string; source_url: string; source_variant_id: string | null; source_snapshot_revision: string | number; media_type: 'image' | 'video' };
|
||||
|
||||
/** Downloads source images only after a public-address check. A failed image is
|
||||
* terminal for this transfer, not for the product collection transaction. */
|
||||
/** Downloads selected source media only after a public-address check. A failed
|
||||
* transfer is terminal for that candidate, not for the product collection transaction. */
|
||||
export async function materializeCollectionAssets(database: Database, storage: S3Client, bucket: string, job: OutboxJob) {
|
||||
await ensureBucket(storage, bucket);
|
||||
const snapshotRevision = Number(job.payload.source_snapshot_revision ?? 1);
|
||||
@@ -17,7 +18,7 @@ export async function materializeCollectionAssets(database: Database, storage: S
|
||||
const transfers = await database.workspaceTransaction(job.workspaceId, async (tx) => {
|
||||
const rows = await tx.query<Transfer>(`UPDATE atlas.collection_asset_transfers SET status = 'running', updated_at = now()
|
||||
WHERE id IN (SELECT id FROM atlas.collection_asset_transfers WHERE collection_item_id = $1 AND source_snapshot_revision = $2 AND status = 'queued' ORDER BY created_at FOR UPDATE SKIP LOCKED)
|
||||
RETURNING id, source_url, source_variant_id, source_snapshot_revision`, [String(job.payload.collection_item_id), snapshotRevision]);
|
||||
RETURNING id, source_url, source_variant_id, source_snapshot_revision, media_type::text AS media_type`, [String(job.payload.collection_item_id), snapshotRevision]);
|
||||
return rows.rows;
|
||||
});
|
||||
for (const transfer of transfers) await materializeOne(database, storage, bucket, job.workspaceId, transfer);
|
||||
@@ -29,7 +30,7 @@ export async function materializeCollectionAssets(database: Database, storage: S
|
||||
async function materializeOne(database: Database, storage: S3Client, bucket: string, workspaceId: string, transfer: Transfer) {
|
||||
const snapshotRevision = Number(transfer.source_snapshot_revision);
|
||||
try {
|
||||
const downloaded = await downloadImage(transfer.source_url);
|
||||
const downloaded = await downloadMedia(transfer.source_url, transfer.media_type);
|
||||
const current = await database.workspaceTransaction(workspaceId, async (tx) => (await tx.query<{ id: string }>(`SELECT t.id FROM atlas.collection_asset_transfers t JOIN atlas.collection_items c ON c.id = t.collection_item_id
|
||||
WHERE t.id = $1 AND t.status = 'running' AND t.source_snapshot_revision = $2 AND c.source_snapshot_revision = $2`, [transfer.id, snapshotRevision])).rows[0]);
|
||||
if (!current) return;
|
||||
@@ -52,19 +53,24 @@ async function materializeOne(database: Database, storage: S3Client, bucket: str
|
||||
}
|
||||
}
|
||||
|
||||
async function downloadImage(source: string) {
|
||||
async function downloadMedia(source: string, mediaType: Transfer['media_type']) {
|
||||
let url = new URL(source);
|
||||
if (mediaType === 'video' && /\.m3u8(?:$|[?#])/i.test(url.pathname))
|
||||
throw new Error('暂不支持 HLS 视频转存');
|
||||
for (let redirect = 0; redirect < 4; redirect += 1) {
|
||||
await assertPublicUrl(url);
|
||||
const response = await fetch(url, { redirect: 'manual', signal: AbortSignal.timeout(20_000), headers: { Accept: 'image/avif,image/webp,image/png,image/jpeg,*/*;q=0.8', 'User-Agent': 'Atlas-Collector/0.1' } });
|
||||
const response = await fetch(url, { redirect: 'manual', signal: AbortSignal.timeout(60_000), headers: { Accept: mediaType === 'video' ? 'video/mp4,video/webm,*/*;q=0.8' : 'image/avif,image/webp,image/png,image/jpeg,*/*;q=0.8', 'User-Agent': 'Atlas-Collector/0.1' } });
|
||||
if ([301, 302, 303, 307, 308].includes(response.status)) {
|
||||
const location = response.headers.get('location'); if (!location) throw new Error('来源素材重定向缺少地址'); url = new URL(location, url); continue;
|
||||
}
|
||||
if (!response.ok || !response.body) throw new Error(`来源素材请求失败（HTTP ${response.status}）`);
|
||||
const chunks: Buffer[] = []; let total = 0;
|
||||
for await (const chunk of response.body as AsyncIterable<Uint8Array>) { total += chunk.byteLength; if (total > MAX_BYTES) throw new Error('来源素材超过 20MB'); chunks.push(Buffer.from(chunk)); }
|
||||
const bytes = Buffer.concat(chunks, total); const image = detectImage(bytes); if (!image) throw new Error('来源素材不是支持的 PNG、JPEG 或 WebP 图片');
|
||||
return { bytes, ...image };
|
||||
const limit = mediaType === 'video' ? MAX_VIDEO_BYTES : MAX_IMAGE_BYTES;
|
||||
for await (const chunk of response.body as AsyncIterable<Uint8Array>) { total += chunk.byteLength; if (total > limit) throw new Error(mediaType === 'video' ? '来源视频超过 200MB' : '来源图片超过 20MB'); chunks.push(Buffer.from(chunk)); }
|
||||
const bytes = Buffer.concat(chunks, total);
|
||||
const media = mediaType === 'video' ? detectVideo(bytes) : detectImage(bytes);
|
||||
if (!media) throw new Error(mediaType === 'video' ? '来源素材不是支持的 MP4 或 WebM 视频' : '来源素材不是支持的 PNG、JPEG 或 WebP 图片');
|
||||
return { bytes, ...media };
|
||||
}
|
||||
throw new Error('来源素材重定向次数过多');
|
||||
}
|
||||
@@ -84,4 +90,9 @@ function detectImage(bytes: Buffer): { contentType: 'image/png' | 'image/jpeg' |
|
||||
if (bytes.length > 9 && bytes[0] === 0xff && bytes[1] === 0xd8) for (let offset = 2; offset + 9 < bytes.length;) { if (bytes[offset] !== 0xff) return null; const marker = bytes[offset + 1], length = bytes.readUInt16BE(offset + 2); if (length < 2 || offset + 2 + length > bytes.length) return null; if ([0xc0, 0xc1, 0xc2, 0xc3, 0xc5, 0xc6, 0xc7, 0xc9, 0xca, 0xcb, 0xcd, 0xce, 0xcf].includes(marker)) return dimensions('image/jpeg', bytes.readUInt16BE(offset + 7), bytes.readUInt16BE(offset + 5)); offset += length + 2; }
|
||||
return null;
|
||||
}
|
||||
function detectVideo(bytes: Buffer): { contentType: 'video/mp4' | 'video/webm'; width: null; height: null } | null {
|
||||
if (bytes.length >= 12 && bytes.subarray(4, 8).toString('ascii') === 'ftyp') return { contentType: 'video/mp4', width: null, height: null };
|
||||
if (bytes.length >= 4 && bytes.subarray(0, 4).equals(Buffer.from([0x1a, 0x45, 0xdf, 0xa3]))) return { contentType: 'video/webm', width: null, height: null };
|
||||
return null;
|
||||
}
|
||||
function dimensions(contentType: 'image/png' | 'image/jpeg' | 'image/webp', width: number, height: number) { return width >= 1 && height >= 1 && width <= 8192 && height <= 8192 ? { contentType, width, height } : null; }
|
||||
|
||||
Reference in New Issue
Block a user