Update seller store experience
This commit is contained in:
R524809
@@ -1,6 +1,6 @@
|
||||
import { Body, Controller, Get, Headers, Inject, Param, Patch, Post, Query, Req, Res } from '@nestjs/common';
|
||||
import { z } from 'zod';
|
||||
import { bulkDeleteCollectionItemsRequestSchema, clearCollectionItemsRequestSchema, createCollectionItemRequestSchema, listCollectionItemsQuerySchema, retryCollectionAssetsRequestSchema, updateCollectionItemRequestSchema, type Workspace } from '@atlas/contracts';
|
||||
import { bulkDeleteCollectionItemsRequestSchema, clearCollectionItemsRequestSchema, createCollectionItemRequestSchema, listCollectionItemsQuerySchema, materializeCollectionAssetsRequestSchema, retryCollectionAssetsRequestSchema, updateCollectionItemRequestSchema, type Workspace } from '@atlas/contracts';
|
||||
import { ApiError } from '../common/api-error.js';
|
||||
import type { AuthenticatedRequest } from '../auth/auth.guard.js';
|
||||
import { CurrentWorkspace } from '../workspace/current-workspace.decorator.js';
|
||||
@@ -32,6 +32,9 @@ export class CollectionController {
|
||||
@Post(':id/asset-retries') async retry(@Param('id') id: string, @Body() body: unknown, @Headers('idempotency-key') key: string | undefined, @Req() request: AuthenticatedRequest, @CurrentWorkspace() workspace: Workspace, @Res({ passthrough: true }) response: Response) {
|
||||
const result = await this.collection.retryAssets(workspace.id, request.auth.userId, requireUuid('collection_item_id', id), requireKey(key), parse(retryCollectionAssetsRequestSchema, body)); response.status(result.status); return result.body;
|
||||
}
|
||||
@Post(':id/asset-materializations') async materialize(@Param('id') id: string, @Body() body: unknown, @Headers('idempotency-key') key: string | undefined, @Req() request: AuthenticatedRequest, @CurrentWorkspace() workspace: Workspace, @Res({ passthrough: true }) response: Response) {
|
||||
const result = await this.collection.materializeAssets(workspace.id, request.auth.userId, requireUuid('collection_item_id', id), requireKey(key), parse(materializeCollectionAssetsRequestSchema, body)); response.status(result.status); return result.body;
|
||||
}
|
||||
}
|
||||
function parse<T extends { safeParse: (input: unknown) => { success: boolean; data?: unknown; error?: { issues: Array<{ path: PropertyKey[]; message: string }> } } }>(schema: T, input: unknown) { const result = schema.safeParse(input); if (result.success) return result.data as never; throw new ApiError(422, 'VALIDATION_FAILED', '请求参数不符合要求', result.error?.issues.map((x) => ({ field: x.path.join('.'), code: 'INVALID_FIELD', message: x.message }))); }
|
||||
function requireUuid(field: string, value: string) { if (z.uuid().safeParse(value).success) return value; throw new ApiError(422, 'VALIDATION_FAILED', '请求参数不符合要求', [{ field, code: 'INVALID_FIELD', message: `${field} 必须是 UUID` }]); }
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { Inject, Injectable } from '@nestjs/common';
|
||||
import type { CreateCollectionItemRequest, ListCollectionItemsQuery, UpdateCollectionItemRequest } from '@atlas/contracts';
|
||||
import type { CreateCollectionItemRequest, ListCollectionItemsQuery, MaterializeCollectionAssetsRequest, UpdateCollectionItemRequest } from '@atlas/contracts';
|
||||
import type { Database, DatabaseTransaction } from '@atlas/database';
|
||||
import { ApiError } from '../common/api-error.js';
|
||||
import { IdempotencyService } from '../common/idempotency.service.js';
|
||||
@@ -37,7 +37,7 @@ export class CollectionService {
|
||||
const item = await this.requireItem(tx, id);
|
||||
const source = await tx.query<{ normalized_payload: unknown; raw_evidence: unknown; parser_version: string | null; collected_at: Date | string | null; source_snapshot_revision: string | number }>('SELECT normalized_payload, raw_evidence, parser_version, collected_at, source_snapshot_revision FROM atlas.collection_items WHERE id = $1 AND deleted_at IS NULL', [id]);
|
||||
const variants = await tx.query<{ id: string; source_variant_key: string | null; source_sku: string | null; display_name: string | null; sort_order: number }>('SELECT id, source_variant_key, source_sku, display_name, sort_order FROM atlas.product_variants WHERE product_id = $1 AND deleted_at IS NULL ORDER BY sort_order, id', [item.product_id]);
|
||||
const transfers = await tx.query<{ id: string; source_variant_id: string | null; role: string; source_url: string; status: string; asset_id: string | null; error_payload: unknown; created_at: Date | string; completed_at: Date | string | null }>('SELECT id, source_variant_id, role::text AS role, source_url, status::text AS status, asset_id, error_payload, created_at, completed_at FROM atlas.collection_asset_transfers WHERE collection_item_id = $1 AND source_snapshot_revision = $2 ORDER BY created_at, id', [id, source.rows[0]?.source_snapshot_revision ?? 1]);
|
||||
const transfers = await tx.query<{ id: string; source_variant_id: string | null; media_type: 'image' | 'video'; poster_url: string | null; role: string; source_url: string; status: string; asset_id: string | null; error_payload: unknown; created_at: Date | string; completed_at: Date | string | null }>('SELECT id, source_variant_id, media_type::text AS media_type, poster_url, role::text AS role, source_url, status::text AS status, asset_id, error_payload, created_at, completed_at FROM atlas.collection_asset_transfers WHERE collection_item_id = $1 AND source_snapshot_revision = $2 ORDER BY created_at, id', [id, source.rows[0]?.source_snapshot_revision ?? 1]);
|
||||
return { ...item, normalized_payload: source.rows[0]?.normalized_payload ?? {}, raw_evidence: source.rows[0]?.raw_evidence ?? {}, parser_version: source.rows[0]?.parser_version ?? null, collected_at: source.rows[0]?.collected_at ? iso(source.rows[0].collected_at) : null, variants: variants.rows, asset_transfers: transfers.rows.map((transfer) => ({ ...transfer, created_at: iso(transfer.created_at), completed_at: transfer.completed_at ? iso(transfer.completed_at) : null })) };
|
||||
});
|
||||
}
|
||||
@@ -51,6 +51,7 @@ export class CollectionService {
|
||||
async bulkDelete(workspaceId: string, userId: string, key: string, ids: { ids: string[] }) { return this.removeMany(workspaceId, userId, key, '/api/collection-items/bulk-delete', ids, false); }
|
||||
async clear(workspaceId: string, userId: string, key: string, input: { confirm: true; expected_active_count: number }) { return this.removeMany(workspaceId, userId, key, '/api/collection-items/clear', input, true); }
|
||||
async retryAssets(workspaceId: string, userId: string, id: string, key: string, input: { transfer_ids: string[] }) { return this.idempotency.run({ workspaceId, userId, method: 'POST', path: `/api/collection-items/${id}/asset-retries`, key, request: input }, async (tx) => { await this.requireItem(tx, id); const result = await tx.query<{ source_snapshot_revision: string | number }>(`UPDATE atlas.collection_asset_transfers t SET status = 'queued', error_payload = NULL, updated_at = now() FROM atlas.collection_items c WHERE t.collection_item_id = c.id AND t.collection_item_id = $1 AND t.id = ANY($2::uuid[]) AND t.source_snapshot_revision = c.source_snapshot_revision AND t.status IN ('failed', 'cancelled') RETURNING c.source_snapshot_revision`, [id, input.transfer_ids]); if (result.rowCount !== input.transfer_ids.length) throw new ApiError(422, 'ASSET_RETRY_INVALID', '只能重试当前采集快照中失败或已取消的素材任务'); const operation = await this.operation(tx, userId, 'collection_asset_retry', id); await this.event(tx, id, 'collection.assets.retry.requested', { operation_id: operation.id, collection_item_id: id, source_snapshot_revision: Number(result.rows[0].source_snapshot_revision) }); return { status: 202, body: { data: { operation } } }; }); }
|
||||
async materializeAssets(workspaceId: string, userId: string, id: string, key: string, input: MaterializeCollectionAssetsRequest) { return this.idempotency.run({ workspaceId, userId, method: 'POST', path: `/api/collection-items/${id}/asset-materializations`, key, request: input }, async (tx) => { await this.requireItem(tx, id); const result = await tx.query<{ source_snapshot_revision: string | number }>(`UPDATE atlas.collection_asset_transfers t SET status = 'queued', error_payload = NULL, completed_at = NULL, updated_at = now() FROM atlas.collection_items c WHERE t.collection_item_id = c.id AND t.collection_item_id = $1 AND t.id = ANY($2::uuid[]) AND t.source_snapshot_revision = c.source_snapshot_revision AND t.status = 'pending_selection' RETURNING c.source_snapshot_revision`, [id, input.transfer_ids]); if (result.rowCount !== input.transfer_ids.length) throw new ApiError(422, 'ASSET_MATERIALIZATION_INVALID', '只能转存当前采集快照中尚未转存的来源媒体'); const operation = await this.operation(tx, userId, 'collection_asset_materialization', id); await this.event(tx, operation.id, 'collection.assets.materialize.requested', { operation_id: operation.id, collection_item_id: id, source_snapshot_revision: Number(result.rows[0].source_snapshot_revision) }); return { status: 202, body: { data: { operation } } }; }); }
|
||||
private async removeMany(workspaceId: string, userId: string, key: string, path: string, input: unknown, clear: boolean) { return this.idempotency.run({ workspaceId, userId, method: 'POST', path, key, request: input }, async (tx) => { const requested = input as { ids?: string[]; expected_active_count?: number }; const count = await tx.query<{ count: string }>('SELECT count(*)::text AS count FROM atlas.collection_items WHERE deleted_at IS NULL'); if (clear && Number(count.rows[0].count) !== requested.expected_active_count) throw new ApiError(409, 'COLLECTION_COUNT_CHANGED', '采集箱数量已变化，请重新确认'); const deleted = await tx.query<{ id: string }>(clear ? `UPDATE atlas.collection_items SET status = 'deleted', deleted_at = now(), revision = revision + 1, updated_at = now() WHERE deleted_at IS NULL RETURNING id` : `UPDATE atlas.collection_items SET status = 'deleted', deleted_at = now(), revision = revision + 1, updated_at = now() WHERE id = ANY($1::uuid[]) AND deleted_at IS NULL RETURNING id`, clear ? [] : [requested.ids]); if (!clear && deleted.rows.length !== requested.ids?.length) throw new ApiError(404, 'RESOURCE_NOT_FOUND', '部分采集记录不存在'); const operation = await this.operation(tx, userId, 'collection_cleanup', deleted.rows[0]?.id ?? null); await this.event(tx, operation.id, 'collection.cleanup.requested', { operation_id: operation.id, collection_item_ids: deleted.rows.map((row) => row.id) }); return { status: 202, body: { data: { deleted_count: deleted.rows.length, operation } } }; }); }
|
||||
private async operation(tx: DatabaseTransaction, userId: string, kind: string, resourceId: string | null) { const row = await tx.query<{ id: string; status: string; created_at: Date | string }>(`INSERT INTO atlas.operations (workspace_id, kind, resource_type, resource_id, created_by_user_id) VALUES (current_setting('app.workspace_id')::uuid, $1, 'collection_item', $2, $3) RETURNING id, status, created_at`, [kind, resourceId, userId]); return { id: row.rows[0].id, kind, status: row.rows[0].status, created_at: iso(row.rows[0].created_at), completed_at: null, error: null }; }
|
||||
private async event(tx: DatabaseTransaction, aggregateId: string, type: string, payload: unknown) { await tx.query(`INSERT INTO atlas.outbox_events (workspace_id, aggregate_type, aggregate_id, event_type, event_payload, deduplication_key) VALUES (current_setting('app.workspace_id')::uuid, 'collection_item', $1, $2, $3, $4)`, [aggregateId, type, payload, `${type}:${aggregateId}`]); }
|
||||
|
||||
@@ -9,39 +9,110 @@ import { DATABASE } from '../database/database.module.js';
|
||||
export class ExtensionService {
|
||||
public constructor(
|
||||
@Inject(DATABASE) private readonly database: Database,
|
||||
@Inject(IdempotencyService) private readonly idempotency: IdempotencyService,
|
||||
@Inject(IdempotencyService)
|
||||
private readonly idempotency: IdempotencyService,
|
||||
) {}
|
||||
|
||||
public async ingest(workspaceId: string, userId: string, key: string, payload: CollectedProductPayload) {
|
||||
return this.idempotency.run({ workspaceId, userId, method: 'POST', path: '/api/extension/materials', key, request: payload }, async (tx) => {
|
||||
const platform = await tx.query<{ id: string; code: string }>(`SELECT id, code FROM atlas.platforms
|
||||
WHERE code = $1 AND status = 'active' AND coalesce((capabilities->>'can_be_source')::boolean, false)`, [payload.source.platform.toUpperCase()]);
|
||||
if (!platform.rows[0]) throw new ApiError(422, 'SOURCE_PLATFORM_UNAVAILABLE', '该来源平台当前不可采集');
|
||||
const existing = await this.findExistingSourceItem(
|
||||
tx,
|
||||
public async ingest(
|
||||
workspaceId: string,
|
||||
userId: string,
|
||||
key: string,
|
||||
payload: CollectedProductPayload,
|
||||
) {
|
||||
return this.idempotency.run(
|
||||
{
|
||||
workspaceId,
|
||||
platform.rows[0].id,
|
||||
payload,
|
||||
);
|
||||
if (existing) {
|
||||
await this.replaceSourceSnapshot(
|
||||
userId,
|
||||
method: 'POST',
|
||||
path: '/api/extension/materials',
|
||||
key,
|
||||
request: payload,
|
||||
},
|
||||
async (tx) => {
|
||||
const platform = await tx.query<{ id: string; code: string }>(
|
||||
`SELECT id, code FROM atlas.platforms
|
||||
WHERE code = $1 AND status = 'active' AND coalesce((capabilities->>'can_be_source')::boolean, false)`,
|
||||
[payload.source.platform.toUpperCase()],
|
||||
);
|
||||
if (!platform.rows[0])
|
||||
throw new ApiError(
|
||||
422,
|
||||
'SOURCE_PLATFORM_UNAVAILABLE',
|
||||
'该来源平台当前不可采集',
|
||||
);
|
||||
const existing = await this.findExistingSourceItem(
|
||||
tx,
|
||||
existing.id,
|
||||
existing.product_id,
|
||||
workspaceId,
|
||||
platform.rows[0].id,
|
||||
payload,
|
||||
);
|
||||
return { status: 200, body: { data: await this.materialResponse(tx, existing.id, existing.product_id, true, true) } };
|
||||
}
|
||||
const { status, sourcePrice } = sourceState(payload);
|
||||
const item = await tx.query<{ id: string }>(`INSERT INTO atlas.collection_items
|
||||
if (existing) {
|
||||
await this.replaceSourceSnapshot(
|
||||
tx,
|
||||
workspaceId,
|
||||
existing.id,
|
||||
existing.product_id,
|
||||
payload,
|
||||
);
|
||||
return {
|
||||
status: 200,
|
||||
body: {
|
||||
data: await this.materialResponse(
|
||||
tx,
|
||||
existing.id,
|
||||
existing.product_id,
|
||||
true,
|
||||
true,
|
||||
),
|
||||
},
|
||||
};
|
||||
}
|
||||
const { status, sourcePrice } = sourceState(payload);
|
||||
const item = await tx.query<{ id: string }>(
|
||||
`INSERT INTO atlas.collection_items
|
||||
(workspace_id, source_kind, source_platform_id, source_item_key, source_url, source_title, source_currency_code, source_min_price, source_max_price, status, raw_evidence, normalized_payload, parser_version, collected_at)
|
||||
VALUES (current_setting('app.workspace_id')::uuid, 'platform', $1, $2, $3, $4, $5, $6, $6, $7::atlas.collection_item_status, $8, $9, $10, $11) RETURNING id`,
|
||||
[platform.rows[0].id, payload.source.itemKey, payload.source.canonicalUrl, payload.common.title?.value ?? null, sourcePrice?.currencyCode ?? null, sourcePrice?.amount ?? null, status, JSON.stringify(payload.evidence), JSON.stringify(payload), payload.source.parserVersion, payload.source.collectedAt]);
|
||||
const product = await tx.query<{ id: string }>(`INSERT INTO atlas.products (workspace_id, collection_item_id, source_title)
|
||||
VALUES (current_setting('app.workspace_id')::uuid, $1, $2) RETURNING id`, [item.rows[0].id, payload.common.title?.value ?? null]);
|
||||
await this.writeSourceSnapshotRows(tx, item.rows[0].id, product.rows[0].id, 1, payload);
|
||||
return { status: 201, body: { data: await this.materialResponse(tx, item.rows[0].id, product.rows[0].id, false, false) } };
|
||||
});
|
||||
[
|
||||
platform.rows[0].id,
|
||||
payload.source.itemKey,
|
||||
payload.source.canonicalUrl,
|
||||
payload.common.title?.value ?? null,
|
||||
sourcePrice?.currencyCode ?? null,
|
||||
sourcePrice?.amount ?? null,
|
||||
status,
|
||||
JSON.stringify(payload.evidence),
|
||||
JSON.stringify(payload),
|
||||
payload.source.parserVersion,
|
||||
payload.source.collectedAt,
|
||||
],
|
||||
);
|
||||
const product = await tx.query<{ id: string }>(
|
||||
`INSERT INTO atlas.products (workspace_id, collection_item_id, source_title)
|
||||
VALUES (current_setting('app.workspace_id')::uuid, $1, $2) RETURNING id`,
|
||||
[item.rows[0].id, payload.common.title?.value ?? null],
|
||||
);
|
||||
await this.writeSourceSnapshotRows(
|
||||
tx,
|
||||
workspaceId,
|
||||
item.rows[0].id,
|
||||
product.rows[0].id,
|
||||
1,
|
||||
payload,
|
||||
);
|
||||
return {
|
||||
status: 201,
|
||||
body: {
|
||||
data: await this.materialResponse(
|
||||
tx,
|
||||
item.rows[0].id,
|
||||
product.rows[0].id,
|
||||
false,
|
||||
false,
|
||||
),
|
||||
},
|
||||
};
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -70,7 +141,10 @@ export class ExtensionService {
|
||||
const sourceSkus = [
|
||||
...new Set(
|
||||
payload.variants
|
||||
.map((variant) => variant.sourceSku ?? variant.sourceVariantKey)
|
||||
.map(
|
||||
(variant) =>
|
||||
variant.sourceSku ?? variant.sourceVariantKey,
|
||||
)
|
||||
.filter((sku) => sku.trim().length > 0),
|
||||
),
|
||||
];
|
||||
@@ -109,55 +183,201 @@ export class ExtensionService {
|
||||
return byVariantOverlap.rows[0] ?? null;
|
||||
}
|
||||
|
||||
private async replaceSourceSnapshot(tx: { query: Database['query'] }, collectionItemId: string, productId: string, payload: CollectedProductPayload) {
|
||||
private async replaceSourceSnapshot(
|
||||
tx: { query: Database['query'] },
|
||||
workspaceId: string,
|
||||
collectionItemId: string,
|
||||
productId: string,
|
||||
payload: CollectedProductPayload,
|
||||
) {
|
||||
const { status, sourcePrice } = sourceState(payload);
|
||||
// 不删除旧任务记录：运行中的 Worker 可能已拿到它。取消并保留审计记录，
|
||||
// 不删除旧候选/任务记录：运行中的 Worker 可能已拿到它。取消并保留审计记录，
|
||||
// 新快照在查询和 Worker 中均按 source_snapshot_revision 隔离。
|
||||
await tx.query(`UPDATE atlas.collection_asset_transfers SET status = 'cancelled', completed_at = now(), updated_at = now()
|
||||
WHERE collection_item_id = $1 AND status IN ('queued', 'running')`, [collectionItemId]);
|
||||
const item = await tx.query<{ source_snapshot_revision: string | number }>(`UPDATE atlas.collection_items
|
||||
await tx.query(
|
||||
`UPDATE atlas.collection_asset_transfers SET status = 'cancelled', completed_at = now(), updated_at = now()
|
||||
WHERE collection_item_id = $1 AND status IN ('pending_selection', 'queued', 'running')`,
|
||||
[collectionItemId],
|
||||
);
|
||||
const item = await tx.query<{
|
||||
source_snapshot_revision: string | number;
|
||||
}>(
|
||||
`UPDATE atlas.collection_items
|
||||
SET source_url = $1, source_title = $2, source_currency_code = $3, source_min_price = $4, source_max_price = $4,
|
||||
status = $5::atlas.collection_item_status, raw_evidence = $6, normalized_payload = $7, parser_version = $8,
|
||||
collected_at = $9, source_snapshot_revision = source_snapshot_revision + 1, revision = revision + 1, updated_at = now()
|
||||
WHERE id = $10 AND deleted_at IS NULL RETURNING source_snapshot_revision`,
|
||||
[payload.source.canonicalUrl, payload.common.title?.value ?? null, sourcePrice?.currencyCode ?? null, sourcePrice?.amount ?? null, status, JSON.stringify(payload.evidence), JSON.stringify(payload), payload.source.parserVersion, payload.source.collectedAt, collectionItemId]);
|
||||
[
|
||||
payload.source.canonicalUrl,
|
||||
payload.common.title?.value ?? null,
|
||||
sourcePrice?.currencyCode ?? null,
|
||||
sourcePrice?.amount ?? null,
|
||||
status,
|
||||
JSON.stringify(payload.evidence),
|
||||
JSON.stringify(payload),
|
||||
payload.source.parserVersion,
|
||||
payload.source.collectedAt,
|
||||
collectionItemId,
|
||||
],
|
||||
);
|
||||
const snapshotRevision = Number(item.rows[0]?.source_snapshot_revision);
|
||||
if (!Number.isSafeInteger(snapshotRevision) || snapshotRevision < 1) throw new Error('采集来源快照版本更新失败');
|
||||
await tx.query(`UPDATE atlas.products SET source_title = $1, updated_at = now(), version = version + 1 WHERE id = $2`, [payload.common.title?.value ?? null, productId]);
|
||||
await tx.query('DELETE FROM atlas.product_variants WHERE product_id = $1', [productId]);
|
||||
await this.writeSourceSnapshotRows(tx, collectionItemId, productId, snapshotRevision, payload);
|
||||
if (!Number.isSafeInteger(snapshotRevision) || snapshotRevision < 1)
|
||||
throw new Error('采集来源快照版本更新失败');
|
||||
await tx.query(
|
||||
`UPDATE atlas.products SET source_title = $1, updated_at = now(), version = version + 1 WHERE id = $2`,
|
||||
[payload.common.title?.value ?? null, productId],
|
||||
);
|
||||
// The historical task rows must remain auditable after replacing the
|
||||
// source snapshot. Clear only their optional variant pointer before
|
||||
// deleting old variants. The original composite FK used ON DELETE SET
|
||||
// NULL and PostgreSQL tried to null both key columns, including the
|
||||
// required workspace_id.
|
||||
await tx.query(
|
||||
`UPDATE atlas.collection_asset_transfers
|
||||
SET source_variant_id = NULL, updated_at = now()
|
||||
WHERE product_id = $1 AND source_variant_id IS NOT NULL`,
|
||||
[productId],
|
||||
);
|
||||
await tx.query(
|
||||
'DELETE FROM atlas.product_variants WHERE product_id = $1',
|
||||
[productId],
|
||||
);
|
||||
await this.writeSourceSnapshotRows(
|
||||
tx,
|
||||
workspaceId,
|
||||
collectionItemId,
|
||||
productId,
|
||||
snapshotRevision,
|
||||
payload,
|
||||
);
|
||||
return snapshotRevision;
|
||||
}
|
||||
|
||||
private async writeSourceSnapshotRows(tx: { query: Database['query'] }, collectionItemId: string, productId: string, snapshotRevision: number, payload: CollectedProductPayload) {
|
||||
const verifiedVariants = payload.variants.filter((variant) => variant.sourceVariantKey.trim().length > 0);
|
||||
const variantRows = verifiedVariants.length ? verifiedVariants : [{ sourceVariantKey: null, sourceSku: undefined, displayName: '默认 SKU' }];
|
||||
private async writeSourceSnapshotRows(
|
||||
tx: { query: Database['query'] },
|
||||
workspaceId: string,
|
||||
collectionItemId: string,
|
||||
productId: string,
|
||||
snapshotRevision: number,
|
||||
payload: CollectedProductPayload,
|
||||
) {
|
||||
const verifiedVariants = payload.variants.filter(
|
||||
(variant) => variant.sourceVariantKey.trim().length > 0,
|
||||
);
|
||||
const variantRows = verifiedVariants.length
|
||||
? verifiedVariants
|
||||
: [
|
||||
{
|
||||
sourceVariantKey: null,
|
||||
sourceSku: undefined,
|
||||
displayName: '默认 SKU',
|
||||
},
|
||||
];
|
||||
const variants = new Map<string, string>();
|
||||
for (let index = 0; index < variantRows.length; index += 1) {
|
||||
const variant = variantRows[index];
|
||||
const inserted = await tx.query<{ id: string }>(`INSERT INTO atlas.product_variants (workspace_id, product_id, source_variant_key, source_sku, display_name, sort_order)
|
||||
VALUES (current_setting('app.workspace_id')::uuid, $1, $2, $3, $4, $5) RETURNING id`, [productId, variant.sourceVariantKey, variant.sourceSku ?? null, variant.displayName, index]);
|
||||
if (variant.sourceVariantKey) variants.set(variant.sourceVariantKey, inserted.rows[0].id);
|
||||
const inserted = await tx.query<{ id: string }>(
|
||||
`INSERT INTO atlas.product_variants (workspace_id, product_id, source_variant_key, source_sku, display_name, sort_order)
|
||||
VALUES ($1, $2, $3, $4, $5, $6) RETURNING id`,
|
||||
[
|
||||
workspaceId,
|
||||
productId,
|
||||
variant.sourceVariantKey,
|
||||
variant.sourceSku ?? null,
|
||||
variant.displayName,
|
||||
index,
|
||||
],
|
||||
);
|
||||
if (variant.sourceVariantKey)
|
||||
variants.set(variant.sourceVariantKey, inserted.rows[0].id);
|
||||
}
|
||||
for (const asset of payload.assets) {
|
||||
await tx.query(`INSERT INTO atlas.collection_asset_transfers (workspace_id, collection_item_id, product_id, source_variant_id, role, source_url, status, error_payload, source_snapshot_revision)
|
||||
VALUES (current_setting('app.workspace_id')::uuid, $1, $2, $3, $4::atlas.media_candidate_role, $5, 'queued', $6, $7)`,
|
||||
[collectionItemId, productId, asset.variantSourceKey ? variants.get(asset.variantSourceKey) ?? null : null, asset.role, asset.url, JSON.stringify({ relation: asset.relation, referer_origin: asset.refererOrigin ?? null, evidence_ref: asset.evidenceRef ?? null, sort_order: asset.sortOrder }), snapshotRevision]);
|
||||
await tx.query(
|
||||
`INSERT INTO atlas.collection_asset_transfers (workspace_id, collection_item_id, product_id, source_variant_id, media_type, poster_url, role, source_url, status, error_payload, source_snapshot_revision)
|
||||
VALUES ($1, $2, $3, $4, $5::atlas.collection_media_type, $6, $7::atlas.media_candidate_role, $8, 'pending_selection', $9, $10)`,
|
||||
[
|
||||
workspaceId,
|
||||
collectionItemId,
|
||||
productId,
|
||||
asset.variantSourceKey
|
||||
? (variants.get(asset.variantSourceKey) ?? null)
|
||||
: null,
|
||||
asset.mediaType,
|
||||
asset.posterUrl ?? null,
|
||||
asset.role,
|
||||
asset.url,
|
||||
JSON.stringify({
|
||||
relation: asset.relation,
|
||||
referer_origin: asset.refererOrigin ?? null,
|
||||
evidence_ref: asset.evidenceRef ?? null,
|
||||
sort_order: asset.sortOrder,
|
||||
}),
|
||||
snapshotRevision,
|
||||
],
|
||||
);
|
||||
}
|
||||
if (payload.assets.length) await tx.query(`INSERT INTO atlas.outbox_events (workspace_id, aggregate_type, aggregate_id, event_type, event_payload, deduplication_key)
|
||||
VALUES (current_setting('app.workspace_id')::uuid, 'collection_item', $1, 'collection.assets.materialize.requested', jsonb_build_object('collection_item_id', $1::uuid, 'source_snapshot_revision', $2::bigint), $3)`, [collectionItemId, snapshotRevision, `collection.assets.materialize:${collectionItemId}:${snapshotRevision}`]);
|
||||
else await tx.query(`UPDATE atlas.collection_items SET status = CASE WHEN status = 'needs_review' THEN status ELSE 'ready_for_edit' END, updated_at = now() WHERE id = $1 AND source_snapshot_revision = $2`, [collectionItemId, snapshotRevision]);
|
||||
// Collecting records source-media candidates only. A later explicit
|
||||
// selection queues the media and creates the materialisation event.
|
||||
await tx.query(
|
||||
`UPDATE atlas.collection_items SET status = CASE WHEN status = 'needs_review' THEN status ELSE 'ready_for_edit' END, updated_at = now() WHERE id = $1 AND source_snapshot_revision = $2`,
|
||||
[collectionItemId, snapshotRevision],
|
||||
);
|
||||
}
|
||||
|
||||
private async materialResponse(tx: { query: Database['query'] }, collectionItemId: string, productId: string, reused: boolean, refreshed: boolean) {
|
||||
const count = await tx.query<{ total: string; queued: string; succeeded: string; failed: string }>(`SELECT count(*)::text AS total, count(*) FILTER (WHERE t.status = 'queued')::text AS queued, count(*) FILTER (WHERE t.status = 'succeeded')::text AS succeeded, count(*) FILTER (WHERE t.status = 'failed')::text AS failed FROM atlas.collection_asset_transfers t JOIN atlas.collection_items c ON c.id = t.collection_item_id WHERE t.collection_item_id = $1 AND t.source_snapshot_revision = c.source_snapshot_revision`, [collectionItemId]);
|
||||
const status = await tx.query<{ status: string; source_title: string | null }>('SELECT status, source_title FROM atlas.collection_items WHERE id = $1', [collectionItemId]);
|
||||
return { collection_item: { id: collectionItemId, status: status.rows[0]?.status ?? 'received', source_title: status.rows[0]?.source_title ?? null, variant_count: 0 }, product_id: productId, reused, refreshed, editor_entry_path: `/collection-test/${collectionItemId}`, materialization: { total: Number(count.rows[0]?.total ?? 0), queued: Number(count.rows[0]?.queued ?? 0), succeeded: Number(count.rows[0]?.succeeded ?? 0), failed: Number(count.rows[0]?.failed ?? 0) } };
|
||||
private async materialResponse(
|
||||
tx: { query: Database['query'] },
|
||||
collectionItemId: string,
|
||||
productId: string,
|
||||
reused: boolean,
|
||||
refreshed: boolean,
|
||||
) {
|
||||
const count = await tx.query<{
|
||||
total: string;
|
||||
pending_selection: string;
|
||||
queued: string;
|
||||
succeeded: string;
|
||||
failed: string;
|
||||
}>(
|
||||
`SELECT count(*)::text AS total, count(*) FILTER (WHERE t.status = 'pending_selection')::text AS pending_selection, count(*) FILTER (WHERE t.status = 'queued')::text AS queued, count(*) FILTER (WHERE t.status = 'succeeded')::text AS succeeded, count(*) FILTER (WHERE t.status = 'failed')::text AS failed FROM atlas.collection_asset_transfers t JOIN atlas.collection_items c ON c.id = t.collection_item_id WHERE t.collection_item_id = $1 AND t.source_snapshot_revision = c.source_snapshot_revision`,
|
||||
[collectionItemId],
|
||||
);
|
||||
const status = await tx.query<{
|
||||
status: string;
|
||||
source_title: string | null;
|
||||
}>(
|
||||
'SELECT status, source_title FROM atlas.collection_items WHERE id = $1',
|
||||
[collectionItemId],
|
||||
);
|
||||
return {
|
||||
collection_item: {
|
||||
id: collectionItemId,
|
||||
status: status.rows[0]?.status ?? 'received',
|
||||
source_title: status.rows[0]?.source_title ?? null,
|
||||
variant_count: 0,
|
||||
},
|
||||
product_id: productId,
|
||||
reused,
|
||||
refreshed,
|
||||
editor_entry_path: `/collection-test/${collectionItemId}`,
|
||||
materialization: {
|
||||
total: Number(count.rows[0]?.total ?? 0),
|
||||
pending_selection: Number(count.rows[0]?.pending_selection ?? 0),
|
||||
queued: Number(count.rows[0]?.queued ?? 0),
|
||||
succeeded: Number(count.rows[0]?.succeeded ?? 0),
|
||||
failed: Number(count.rows[0]?.failed ?? 0),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
function sourceState(payload: CollectedProductPayload) {
|
||||
const verifiedVariants = payload.variants.filter((variant) => variant.sourceVariantKey.trim().length > 0);
|
||||
return { status: verifiedVariants.length || !payload.variantAxes.length ? 'materializing' : 'needs_review', sourcePrice: payload.common.sourcePrice };
|
||||
const verifiedVariants = payload.variants.filter(
|
||||
(variant) => variant.sourceVariantKey.trim().length > 0,
|
||||
);
|
||||
return {
|
||||
status:
|
||||
verifiedVariants.length || !payload.variantAxes.length
|
||||
? 'ready_for_edit'
|
||||
: 'needs_review',
|
||||
sourcePrice: payload.common.sourcePrice,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -7,30 +7,326 @@ import { afterAll, beforeAll, describe, it } from 'vitest';
|
||||
import { DATABASE } from '../src/database/database.module.js';
|
||||
import { createApplication } from '../src/main.js';
|
||||
|
||||
let app: NestExpressApplication; let database: Database; let baseUrl = ''; let workspaceId = ''; let userId = '';
|
||||
async function request(path: string, init: RequestInit = {}) { const response = await fetch(`${baseUrl}${path}`, { redirect: 'manual', ...init, headers: { 'Content-Type': 'application/json', ...init.headers } }); return { response, body: response.status === 204 || response.status === 302 ? undefined : await response.json() as any }; }
|
||||
let app: NestExpressApplication;
|
||||
let database: Database;
|
||||
let baseUrl = '';
|
||||
let workspaceId = '';
|
||||
let userId = '';
|
||||
async function request(path: string, init: RequestInit = {}) {
|
||||
const response = await fetch(`${baseUrl}${path}`, {
|
||||
redirect: 'manual',
|
||||
...init,
|
||||
headers: { 'Content-Type': 'application/json', ...init.headers },
|
||||
});
|
||||
return {
|
||||
response,
|
||||
body:
|
||||
response.status === 204 || response.status === 302
|
||||
? undefined
|
||||
: ((await response.json()) as any),
|
||||
};
|
||||
}
|
||||
|
||||
describe('扩展授权与采集 HTTP 集成测试', () => {
|
||||
beforeAll(async () => { Object.assign(process.env, { NODE_ENV: 'test', SWAGGER_ENABLED: 'false', AUTH_REGISTER_IP_LIMIT: '100', AUTH_REGISTER_EMAIL_LIMIT: '100' }); const created = await createApplication(); app = created.app; database = app.get<Database>(DATABASE); await app.listen(0, '127.0.0.1'); baseUrl = `http://127.0.0.1:${(app.getHttpServer().address() as AddressInfo).port}`; });
|
||||
afterAll(async () => { await database.workspaceTransaction(workspaceId, async (tx) => { await tx.query('DELETE FROM atlas.idempotency_requests WHERE workspace_id = $1', [workspaceId]); await tx.query('DELETE FROM atlas.outbox_events WHERE workspace_id = $1', [workspaceId]); await tx.query('DELETE FROM atlas.collection_asset_transfers WHERE workspace_id = $1', [workspaceId]); await tx.query('DELETE FROM atlas.product_variants WHERE workspace_id = $1', [workspaceId]); await tx.query('DELETE FROM atlas.products WHERE workspace_id = $1', [workspaceId]); await tx.query('DELETE FROM atlas.collection_items WHERE workspace_id = $1', [workspaceId]); await tx.query('DELETE FROM atlas.extension_authorization_codes WHERE workspace_id = $1', [workspaceId]); await tx.query('DELETE FROM atlas.extension_sessions WHERE workspace_id = $1', [workspaceId]); }); await database.query('DELETE FROM atlas.workspace_memberships WHERE user_id = $1', [userId]); await database.query('DELETE FROM atlas.workspaces WHERE id = $1', [workspaceId]); await database.query('DELETE FROM atlas.users WHERE id = $1', [userId]); await app.close(); });
|
||||
beforeAll(async () => {
|
||||
Object.assign(process.env, {
|
||||
NODE_ENV: 'test',
|
||||
SWAGGER_ENABLED: 'false',
|
||||
AUTH_REGISTER_IP_LIMIT: '100',
|
||||
AUTH_REGISTER_EMAIL_LIMIT: '100',
|
||||
});
|
||||
const created = await createApplication();
|
||||
app = created.app;
|
||||
database = app.get<Database>(DATABASE);
|
||||
await app.listen(0, '127.0.0.1');
|
||||
baseUrl = `http://127.0.0.1:${(app.getHttpServer().address() as AddressInfo).port}`;
|
||||
});
|
||||
afterAll(async () => {
|
||||
await database.workspaceTransaction(workspaceId, async (tx) => {
|
||||
await tx.query(
|
||||
'DELETE FROM atlas.idempotency_requests WHERE workspace_id = $1',
|
||||
[workspaceId],
|
||||
);
|
||||
await tx.query(
|
||||
'DELETE FROM atlas.outbox_events WHERE workspace_id = $1',
|
||||
[workspaceId],
|
||||
);
|
||||
await tx.query(
|
||||
'DELETE FROM atlas.operations WHERE workspace_id = $1',
|
||||
[workspaceId],
|
||||
);
|
||||
await tx.query(
|
||||
'DELETE FROM atlas.collection_asset_transfers WHERE workspace_id = $1',
|
||||
[workspaceId],
|
||||
);
|
||||
await tx.query(
|
||||
'DELETE FROM atlas.product_variants WHERE workspace_id = $1',
|
||||
[workspaceId],
|
||||
);
|
||||
await tx.query(
|
||||
'DELETE FROM atlas.products WHERE workspace_id = $1',
|
||||
[workspaceId],
|
||||
);
|
||||
await tx.query(
|
||||
'DELETE FROM atlas.collection_items WHERE workspace_id = $1',
|
||||
[workspaceId],
|
||||
);
|
||||
await tx.query(
|
||||
'DELETE FROM atlas.extension_authorization_codes WHERE workspace_id = $1',
|
||||
[workspaceId],
|
||||
);
|
||||
await tx.query(
|
||||
'DELETE FROM atlas.extension_sessions WHERE workspace_id = $1',
|
||||
[workspaceId],
|
||||
);
|
||||
});
|
||||
await database.query(
|
||||
'DELETE FROM atlas.workspace_memberships WHERE user_id = $1',
|
||||
[userId],
|
||||
);
|
||||
await database.query('DELETE FROM atlas.workspaces WHERE id = $1', [
|
||||
workspaceId,
|
||||
]);
|
||||
await database.query('DELETE FROM atlas.users WHERE id = $1', [userId]);
|
||||
await app.close();
|
||||
});
|
||||
|
||||
it('未登录令牌返回可操作的扩展会话错误', async () => {
|
||||
const session = await request('/api/extension/session');
|
||||
assert.equal(session.response.status, 401);
|
||||
assert.equal(session.body.error.code, 'SESSION_EXPIRED');
|
||||
assert.equal(session.body.error.message, 'Atlas ERP 登录已过期，请重新登录');
|
||||
assert.equal(
|
||||
session.body.error.message,
|
||||
'Atlas ERP 登录已过期，请重新登录',
|
||||
);
|
||||
});
|
||||
|
||||
it('通过 Atlas Web access token 写入来源商品，并以新快照覆盖重复采集', async () => {
|
||||
const registration = await request('/api/auth/register', { method: 'POST', body: JSON.stringify({ email: `extension-${randomUUID()}@example.test`, password: 'correct-horse-battery-staple-123', password_confirmation: 'correct-horse-battery-staple-123' }) });
|
||||
assert.equal(registration.response.status, 201); userId = registration.body.data.user.id; workspaceId = registration.body.data.workspace.id; const webAccessToken = registration.body.data.access_token as string;
|
||||
const session = await request('/api/extension/session', { headers: { Authorization: `Bearer ${webAccessToken}` } }); assert.equal(session.response.status, 200); assert.equal(session.body.data.workspace.id, workspaceId);
|
||||
const payload = { source: { platform: 'ozon', itemKey: '123456789', canonicalUrl: 'https://www.ozon.ru/product/demo-123456789/', collectedAt: new Date().toISOString(), parserVersion: 'test-1' }, common: { title: { value: '来源商品', confidence: 'verified' }, sourcePrice: { amount: '99.90', currencyCode: 'RUB' }, attributes: [], categories: [] }, variantAxes: [], variants: [{ sourceVariantKey: '123456789', sourceSku: '123456789', displayName: '默认 SKU', selections: [], confidence: 'verified', evidenceRefs: ['page'] }], assets: [{ url: 'https://images.example.test/product.jpg', role: 'source_main', relation: 'product_candidate', sortOrder: 0, evidenceRef: 'page' }], evidence: [{ ref: 'page', kind: 'page', path: 'test' }] };
|
||||
const first = await request('/api/extension/materials', { method: 'POST', headers: { Authorization: `Bearer ${webAccessToken}`, 'Idempotency-Key': randomUUID() }, body: JSON.stringify(payload) }); assert.equal(first.response.status, 201); assert.equal(first.body.data.reused, false); assert.equal(first.body.data.materialization.queued, 1);
|
||||
const registration = await request('/api/auth/register', {
|
||||
method: 'POST',
|
||||
body: JSON.stringify({
|
||||
email: `extension-${randomUUID()}@example.test`,
|
||||
password: 'correct-horse-battery-staple-123',
|
||||
password_confirmation: 'correct-horse-battery-staple-123',
|
||||
}),
|
||||
});
|
||||
assert.equal(registration.response.status, 201);
|
||||
userId = registration.body.data.user.id;
|
||||
workspaceId = registration.body.data.workspace.id;
|
||||
const webAccessToken = registration.body.data.access_token as string;
|
||||
const session = await request('/api/extension/session', {
|
||||
headers: { Authorization: `Bearer ${webAccessToken}` },
|
||||
});
|
||||
assert.equal(session.response.status, 200);
|
||||
assert.equal(session.body.data.workspace.id, workspaceId);
|
||||
const payload = {
|
||||
source: {
|
||||
platform: 'ozon',
|
||||
itemKey: '123456789',
|
||||
canonicalUrl: 'https://www.ozon.ru/product/demo-123456789/',
|
||||
collectedAt: new Date().toISOString(),
|
||||
parserVersion: 'test-1',
|
||||
},
|
||||
common: {
|
||||
title: { value: '来源商品', confidence: 'verified' },
|
||||
sourcePrice: { amount: '99.90', currencyCode: 'RUB' },
|
||||
attributes: [],
|
||||
categories: [],
|
||||
},
|
||||
variantAxes: [],
|
||||
variants: [
|
||||
{
|
||||
sourceVariantKey: '123456789',
|
||||
sourceSku: '123456789',
|
||||
displayName: '默认 SKU',
|
||||
selections: [],
|
||||
confidence: 'verified',
|
||||
evidenceRefs: ['page'],
|
||||
},
|
||||
],
|
||||
assets: [
|
||||
{
|
||||
url: 'https://images.example.test/product.jpg',
|
||||
role: 'source_main',
|
||||
relation: 'product_candidate',
|
||||
sortOrder: 0,
|
||||
evidenceRef: 'page',
|
||||
},
|
||||
],
|
||||
evidence: [{ ref: 'page', kind: 'page', path: 'test' }],
|
||||
};
|
||||
const first = await request('/api/extension/materials', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
Authorization: `Bearer ${webAccessToken}`,
|
||||
'Idempotency-Key': randomUUID(),
|
||||
},
|
||||
body: JSON.stringify(payload),
|
||||
});
|
||||
assert.equal(first.response.status, 201);
|
||||
assert.equal(first.body.data.reused, false);
|
||||
assert.equal(first.body.data.collection_item.status, 'ready_for_edit');
|
||||
assert.equal(first.body.data.materialization.pending_selection, 1);
|
||||
assert.equal(first.body.data.materialization.queued, 0);
|
||||
// A sibling Ozon URL has a different entered SKU. It must still reuse
|
||||
// the item because the submitted real-SKU sets overlap at 123456789.
|
||||
const refreshedPayload = { ...payload, source: { ...payload.source, itemKey: '987654321', canonicalUrl: 'https://www.ozon.ru/product/demo-987654321/', collectedAt: new Date(Date.now() + 1_000).toISOString(), parserVersion: 'test-2' }, common: { ...payload.common, title: { value: '第二次采集的中文标题', confidence: 'verified' }, sourcePrice: { amount: '109.90', currencyCode: 'RUB' } }, variants: [{ sourceVariantKey: '123456789', sourceSku: '123456789', displayName: '第一 SKU', selections: [], confidence: 'verified', evidenceRefs: ['page'] }, { sourceVariantKey: '987654321', sourceSku: '987654321', displayName: '第二次采集 SKU', selections: [], confidence: 'verified', evidenceRefs: ['page'] }], assets: [{ url: 'https://images.example.test/refreshed-product.jpg', role: 'source_main', relation: 'product_candidate', sortOrder: 0, evidenceRef: 'page' }] };
|
||||
const second = await request('/api/extension/materials', { method: 'POST', headers: { Authorization: `Bearer ${webAccessToken}`, 'Idempotency-Key': randomUUID() }, body: JSON.stringify(refreshedPayload) }); assert.equal(second.response.status, 200); assert.equal(second.body.data.reused, true); assert.equal(second.body.data.refreshed, true); assert.equal(second.body.data.collection_item.id, first.body.data.collection_item.id); assert.equal(second.body.data.materialization.queued, 1);
|
||||
const inspection = await request(`/api/collection-items/${first.body.data.collection_item.id}`, { headers: { Authorization: `Bearer ${webAccessToken}` } }); assert.equal(inspection.response.status, 200); assert.equal(inspection.body.data.title, '第二次采集的中文标题'); assert.equal(inspection.body.data.normalized_payload.source.parserVersion, 'test-2'); assert.equal(inspection.body.data.variants.length, 2); assert.deepEqual(inspection.body.data.variants.map((variant: { source_variant_key: string }) => variant.source_variant_key), ['123456789', '987654321']); assert.equal(inspection.body.data.asset_transfers.length, 1); assert.equal(inspection.body.data.asset_transfers[0].source_url, 'https://images.example.test/refreshed-product.jpg');
|
||||
const refreshedPayload = {
|
||||
...payload,
|
||||
source: {
|
||||
...payload.source,
|
||||
itemKey: '987654321',
|
||||
canonicalUrl: 'https://www.ozon.ru/product/demo-987654321/',
|
||||
collectedAt: new Date(Date.now() + 1_000).toISOString(),
|
||||
parserVersion: 'test-2',
|
||||
},
|
||||
common: {
|
||||
...payload.common,
|
||||
title: {
|
||||
value: '第二次采集的中文标题',
|
||||
confidence: 'verified',
|
||||
},
|
||||
sourcePrice: { amount: '109.90', currencyCode: 'RUB' },
|
||||
},
|
||||
variants: [
|
||||
{
|
||||
sourceVariantKey: '123456789',
|
||||
sourceSku: '123456789',
|
||||
displayName: '第一 SKU',
|
||||
selections: [],
|
||||
confidence: 'verified',
|
||||
evidenceRefs: ['page'],
|
||||
},
|
||||
{
|
||||
sourceVariantKey: '987654321',
|
||||
sourceSku: '987654321',
|
||||
displayName: '第二次采集 SKU',
|
||||
selections: [],
|
||||
confidence: 'verified',
|
||||
evidenceRefs: ['page'],
|
||||
},
|
||||
],
|
||||
assets: [
|
||||
{
|
||||
url: 'https://images.example.test/refreshed-product.jpg',
|
||||
role: 'source_main',
|
||||
relation: 'product_candidate',
|
||||
sortOrder: 0,
|
||||
evidenceRef: 'page',
|
||||
},
|
||||
{
|
||||
url: 'https://videos.example.test/refreshed-product.mp4',
|
||||
mediaType: 'video',
|
||||
posterUrl: 'https://images.example.test/refreshed-video-poster.jpg',
|
||||
role: 'source_main',
|
||||
relation: 'product_candidate',
|
||||
sortOrder: 1,
|
||||
evidenceRef: 'page',
|
||||
},
|
||||
],
|
||||
};
|
||||
const second = await request('/api/extension/materials', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
Authorization: `Bearer ${webAccessToken}`,
|
||||
'Idempotency-Key': randomUUID(),
|
||||
},
|
||||
body: JSON.stringify(refreshedPayload),
|
||||
});
|
||||
assert.equal(second.response.status, 200);
|
||||
assert.equal(second.body.data.reused, true);
|
||||
assert.equal(second.body.data.refreshed, true);
|
||||
assert.equal(
|
||||
second.body.data.collection_item.id,
|
||||
first.body.data.collection_item.id,
|
||||
);
|
||||
assert.equal(second.body.data.materialization.pending_selection, 2);
|
||||
assert.equal(second.body.data.materialization.queued, 0);
|
||||
const inspection = await request(
|
||||
`/api/collection-items/${first.body.data.collection_item.id}`,
|
||||
{ headers: { Authorization: `Bearer ${webAccessToken}` } },
|
||||
);
|
||||
assert.equal(inspection.response.status, 200);
|
||||
assert.equal(inspection.body.data.title, '第二次采集的中文标题');
|
||||
assert.equal(
|
||||
inspection.body.data.normalized_payload.source.parserVersion,
|
||||
'test-2',
|
||||
);
|
||||
assert.equal(inspection.body.data.variants.length, 2);
|
||||
assert.deepEqual(
|
||||
inspection.body.data.variants.map(
|
||||
(variant: { source_variant_key: string }) =>
|
||||
variant.source_variant_key,
|
||||
),
|
||||
['123456789', '987654321'],
|
||||
);
|
||||
assert.equal(inspection.body.data.asset_transfers.length, 2);
|
||||
assert.equal(
|
||||
inspection.body.data.asset_transfers[0].source_url,
|
||||
'https://images.example.test/refreshed-product.jpg',
|
||||
);
|
||||
assert.equal(
|
||||
inspection.body.data.asset_transfers[0].status,
|
||||
'pending_selection',
|
||||
);
|
||||
const videoTransfer = inspection.body.data.asset_transfers.find(
|
||||
(transfer: { media_type: string }) => transfer.media_type === 'video',
|
||||
);
|
||||
assert.equal(videoTransfer?.source_url, 'https://videos.example.test/refreshed-product.mp4');
|
||||
assert.equal(videoTransfer?.poster_url, 'https://images.example.test/refreshed-video-poster.jpg');
|
||||
const eventsBeforeSelection = await database.workspaceTransaction(
|
||||
workspaceId,
|
||||
async (tx) =>
|
||||
tx.query<{ count: string }>(
|
||||
`SELECT count(*)::text AS count FROM atlas.outbox_events
|
||||
WHERE event_type = 'collection.assets.materialize.requested'
|
||||
AND event_payload->>'collection_item_id' = $1`,
|
||||
[first.body.data.collection_item.id],
|
||||
),
|
||||
);
|
||||
assert.equal(eventsBeforeSelection.rows[0]?.count, '0');
|
||||
const materialization = await request(
|
||||
`/api/collection-items/${first.body.data.collection_item.id}/asset-materializations`,
|
||||
{
|
||||
method: 'POST',
|
||||
headers: {
|
||||
Authorization: `Bearer ${webAccessToken}`,
|
||||
'Idempotency-Key': randomUUID(),
|
||||
},
|
||||
body: JSON.stringify({
|
||||
transfer_ids: [inspection.body.data.asset_transfers[0].id],
|
||||
}),
|
||||
},
|
||||
);
|
||||
assert.equal(materialization.response.status, 202);
|
||||
const afterSelection = await request(
|
||||
`/api/collection-items/${first.body.data.collection_item.id}`,
|
||||
{ headers: { Authorization: `Bearer ${webAccessToken}` } },
|
||||
);
|
||||
assert.equal(afterSelection.response.status, 200);
|
||||
assert.ok(
|
||||
['queued', 'running', 'failed', 'succeeded'].includes(
|
||||
afterSelection.body.data.asset_transfers[0].status,
|
||||
),
|
||||
);
|
||||
const eventsAfterSelection = await database.workspaceTransaction(
|
||||
workspaceId,
|
||||
async (tx) =>
|
||||
tx.query<{ count: string }>(
|
||||
`SELECT count(*)::text AS count FROM atlas.outbox_events
|
||||
WHERE event_type = 'collection.assets.materialize.requested'
|
||||
AND event_payload->>'collection_item_id' = $1`,
|
||||
[first.body.data.collection_item.id],
|
||||
),
|
||||
);
|
||||
assert.equal(eventsAfterSelection.rows[0]?.count, '1');
|
||||
const transferWorkspace = await database.workspaceTransaction(
|
||||
workspaceId,
|
||||
async (tx) =>
|
||||
tx.query<{ workspace_id: string }>(
|
||||
'SELECT workspace_id FROM atlas.collection_asset_transfers WHERE id = $1',
|
||||
[inspection.body.data.asset_transfers[0].id],
|
||||
),
|
||||
);
|
||||
assert.equal(transferWorkspace.rows[0]?.workspace_id, workspaceId);
|
||||
});
|
||||
});
|
||||
|
||||
Vendored
+1
@@ -8,6 +8,7 @@ declare module "wxt/browser" {
|
||||
| "/background.js"
|
||||
| "/content-scripts/collector.js"
|
||||
| "/content-scripts/ozon-page-json.js"
|
||||
| "/content-scripts/source-page-state.js"
|
||||
type HtmlPublicPath = Extract<PublicPath, `${string}.html`>
|
||||
export interface WxtRuntime {
|
||||
getURL(path: PublicPath): string;
|
||||
|
||||
@@ -3,6 +3,7 @@ import {
|
||||
supportedProductPlatform,
|
||||
} from '../lib/source-parsers.js';
|
||||
import type { OzonPageJsonFetcher } from '../lib/ozon-parser.js';
|
||||
import type { MarketplacePageStateFetcher } from '../lib/source-parsers.js';
|
||||
|
||||
export default defineContentScript({
|
||||
matches: [
|
||||
@@ -72,6 +73,9 @@ export default defineContentScript({
|
||||
supportedProductPlatform() === 'ozon'
|
||||
? requestOzonPageJson
|
||||
: undefined,
|
||||
supportedProductPlatform() && supportedProductPlatform() !== 'ozon'
|
||||
? requestMarketplacePageState
|
||||
: undefined,
|
||||
);
|
||||
const response = await browser.runtime.sendMessage({
|
||||
type: 'atlas.collect',
|
||||
@@ -128,6 +132,23 @@ const requestOzonPageJson: OzonPageJsonFetcher = ({ kind, sku }) =>
|
||||
);
|
||||
});
|
||||
|
||||
const requestMarketplacePageState: MarketplacePageStateFetcher = (platform) =>
|
||||
new Promise((resolve) => {
|
||||
const requestId = crypto.randomUUID();
|
||||
const timeout = window.setTimeout(() => {
|
||||
window.removeEventListener('message', receive);
|
||||
resolve({});
|
||||
}, 3_000);
|
||||
const receive = (event: MessageEvent<unknown>) => {
|
||||
if (event.source !== window || !isMarketplaceStateResponse(event.data, requestId)) return;
|
||||
window.clearTimeout(timeout);
|
||||
window.removeEventListener('message', receive);
|
||||
resolve(event.data.data);
|
||||
};
|
||||
window.addEventListener('message', receive);
|
||||
window.postMessage({ source: 'atlas-collector', type: 'source-page-state.request', requestId, platform }, location.origin);
|
||||
});
|
||||
|
||||
function isBridgeResponse(
|
||||
value: unknown,
|
||||
requestId: string,
|
||||
@@ -149,3 +170,10 @@ function isBridgeResponse(
|
||||
(message.ok || typeof message.error === 'string')
|
||||
);
|
||||
}
|
||||
|
||||
function isMarketplaceStateResponse(value: unknown, requestId: string): value is { source: 'atlas-collector'; type: 'source-page-state.response'; requestId: string; data: unknown } {
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value)) return false;
|
||||
const message = value as Record<string, unknown>;
|
||||
return message.source === 'atlas-collector' && message.type === 'source-page-state.response'
|
||||
&& message.requestId === requestId && !!message.data && typeof message.data === 'object';
|
||||
}
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
/**
|
||||
* A deliberately narrow MAIN-world bridge for the mature 1688/Taobao/Tmall
|
||||
* structured collectors. It never exposes cookies, headers, arbitrary window
|
||||
* keys, or a generic fetch primitive; only public product-state projections.
|
||||
*/
|
||||
export default defineContentScript({
|
||||
matches: [
|
||||
'*://detail.1688.com/*',
|
||||
'*://item.taobao.com/*',
|
||||
'*://detail.tmall.com/*',
|
||||
],
|
||||
world: 'MAIN',
|
||||
runAt: 'document_idle',
|
||||
main() {
|
||||
window.addEventListener('message', (event) => {
|
||||
if (event.source !== window || !isRequest(event.data)) return;
|
||||
const data = event.data.platform === '1688'
|
||||
? project1688((window as unknown as Record<string, unknown>).context)
|
||||
: projectTaobao(window as unknown as Record<string, unknown>);
|
||||
window.postMessage({
|
||||
source: 'atlas-collector',
|
||||
type: 'source-page-state.response',
|
||||
requestId: event.data.requestId,
|
||||
data,
|
||||
}, location.origin);
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
type Request = { source: 'atlas-collector'; type: 'source-page-state.request'; requestId: string; platform: '1688' | 'taobao' | 'tmall' };
|
||||
function isRequest(value: unknown): value is Request {
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value)) return false;
|
||||
const data = value as Record<string, unknown>;
|
||||
return data.source === 'atlas-collector' && data.type === 'source-page-state.request'
|
||||
&& typeof data.requestId === 'string' && /^[a-zA-Z0-9-]{16,120}$/.test(data.requestId)
|
||||
&& (data.platform === '1688' || data.platform === 'taobao' || data.platform === 'tmall');
|
||||
}
|
||||
|
||||
const safeClone = (value: unknown) => { try { return JSON.parse(JSON.stringify(value)); } catch { return undefined; } };
|
||||
function find(value: unknown, key: string): unknown {
|
||||
const pending: unknown[] = [value];
|
||||
const seen = new Set<object>();
|
||||
let visited = 0;
|
||||
while (pending.length && visited < 300_000) {
|
||||
const current = pending.shift();
|
||||
if (!current || typeof current !== 'object' || seen.has(current)) continue;
|
||||
seen.add(current);
|
||||
visited += 1;
|
||||
const record = current as Record<string, unknown>;
|
||||
if (key in record) return record[key];
|
||||
pending.push(...Object.values(record));
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
function project1688(context: unknown) {
|
||||
const projected: Record<string, unknown> = {};
|
||||
for (const key of ['gallery', 'tempModel', 'skuModel', 'tradeModel', 'skuMapOriginal', 'pieceWeightScaleInfo', 'productAttributes']) {
|
||||
const value = find(context, key);
|
||||
if (value !== undefined) projected[key] = safeClone(value);
|
||||
}
|
||||
return projected;
|
||||
}
|
||||
function projectTaobao(windowState: Record<string, unknown>) {
|
||||
const ice = windowState.__ICE_APP_CONTEXT__ as Record<string, unknown> | undefined;
|
||||
const result = ice?.loaderData && typeof ice.loaderData === 'object'
|
||||
? (((ice.loaderData as Record<string, unknown>).home as Record<string, unknown> | undefined)?.data as Record<string, unknown> | undefined)?.res
|
||||
: undefined;
|
||||
return { product: safeClone(result) };
|
||||
}
|
||||
@@ -61,6 +61,7 @@ export async function collectOzonProduct(
|
||||
const localStates = readLocalWidgetStates(documentRoot);
|
||||
let attributes = extractOzonCharacteristics(localStates);
|
||||
let description = extractOzonDescription(localStates);
|
||||
const jsonLdDescription = extractOzonJsonLdDescription(documentRoot);
|
||||
const heading = firstState(localStates, 'webProductHeading');
|
||||
const priceState = firstState(localStates, 'webPrice');
|
||||
const title =
|
||||
@@ -101,6 +102,14 @@ export async function collectOzonProduct(
|
||||
'ozon:gallery:page',
|
||||
canonical,
|
||||
);
|
||||
addGalleryVideos(
|
||||
localStates,
|
||||
currentSku,
|
||||
assets,
|
||||
assetKeys,
|
||||
'ozon:gallery:page',
|
||||
canonical,
|
||||
);
|
||||
if (attributes.length) {
|
||||
evidence.push({
|
||||
ref: 'ozon:characteristics:page',
|
||||
@@ -109,6 +118,14 @@ export async function collectOzonProduct(
|
||||
raw_value: `发现 ${attributes.length} 项商品特征`,
|
||||
});
|
||||
}
|
||||
if (jsonLdDescription) {
|
||||
evidence.push({
|
||||
ref: 'ozon:json-ld:product.description',
|
||||
kind: 'json_ld',
|
||||
path: 'script[type="application/ld+json"] Product.description',
|
||||
raw_value: `JSON-LD 商品描述 ${jsonLdDescription.value.length} 个字符`,
|
||||
});
|
||||
}
|
||||
|
||||
const categories = collectCategories(documentRoot, canonical);
|
||||
if (categories.length) {
|
||||
@@ -162,12 +179,22 @@ export async function collectOzonProduct(
|
||||
canonical,
|
||||
);
|
||||
description = description ?? extractOzonDescription(detailStates);
|
||||
const detailAttributes = extractOzonCharacteristics(detailStates);
|
||||
attributes = mergeOzonCharacteristics(attributes, detailAttributes);
|
||||
evidence.push({
|
||||
ref: 'ozon:detail:page',
|
||||
kind: 'state',
|
||||
path: 'webDescription|webRichContent|webFeatures image fields',
|
||||
raw_value: `发现 ${added} 张商品详情图`,
|
||||
});
|
||||
if (detailAttributes.length) {
|
||||
evidence.push({
|
||||
ref: 'ozon:characteristics:detail',
|
||||
kind: 'state',
|
||||
path: 'webShortCharacteristics|webDetailedCharacteristics|webCharacteristics.characteristics[]',
|
||||
raw_value: `详情段补全 ${detailAttributes.length} 项商品特征`,
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
evidence.push(
|
||||
fetchFailureEvidence('ozon:detail:page', currentSku, error),
|
||||
@@ -230,6 +257,14 @@ export async function collectOzonProduct(
|
||||
`ozon:gallery:${result.sku}`,
|
||||
canonical,
|
||||
);
|
||||
addGalleryVideos(
|
||||
result.states,
|
||||
result.sku,
|
||||
assets,
|
||||
assetKeys,
|
||||
`ozon:gallery:${result.sku}`,
|
||||
canonical,
|
||||
);
|
||||
if (result.sku === currentSku) {
|
||||
const detailedAttributes = extractOzonCharacteristics(
|
||||
result.states,
|
||||
@@ -270,6 +305,11 @@ export async function collectOzonProduct(
|
||||
});
|
||||
}
|
||||
|
||||
// Ozon's rich widget is preferred because it is the dedicated description
|
||||
// segment. JSON-LD is the proven old-collector fallback for pages where
|
||||
// Ozon only places the seller description in the initial document.
|
||||
description = description ?? jsonLdDescription;
|
||||
|
||||
addAspectCoverAssets(axes, assets, assetKeys, canonical);
|
||||
|
||||
const variants = [...candidates.values()]
|
||||
@@ -301,7 +341,7 @@ export async function collectOzonProduct(
|
||||
itemKey: currentSku,
|
||||
canonicalUrl: canonical,
|
||||
collectedAt: new Date().toISOString(),
|
||||
parserVersion: 'ozon-public-2.2.0',
|
||||
parserVersion: 'ozon-public-2.3.0',
|
||||
},
|
||||
common: {
|
||||
...(title
|
||||
@@ -452,6 +492,7 @@ function addAspectCoverAssets(
|
||||
assetKeys.add(key);
|
||||
assets.push({
|
||||
url,
|
||||
mediaType: 'image',
|
||||
role: 'source_sku',
|
||||
relation: 'axis_value',
|
||||
sortOrder: assets.length,
|
||||
@@ -491,6 +532,7 @@ function addDetailAssets(
|
||||
assetKeys.add(key);
|
||||
assets.push({
|
||||
url,
|
||||
mediaType: 'image',
|
||||
role: 'source_detail',
|
||||
relation: 'product_candidate',
|
||||
sortOrder: added++,
|
||||
@@ -621,6 +663,55 @@ export function extractOzonDescription(
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/** Reads only schema.org Product.description as a fallback, never arbitrary
|
||||
* JSON-LD page copy. This mirrors the mature collector's JSON-LD path. */
|
||||
export function extractOzonJsonLdDescription(
|
||||
root: Document,
|
||||
): OzonDescription | undefined {
|
||||
for (const script of Array.from(root.scripts)) {
|
||||
if (script.type !== 'application/ld+json' || !script.textContent?.trim())
|
||||
continue;
|
||||
try {
|
||||
const product = findJsonLdProduct(JSON.parse(script.textContent));
|
||||
const raw = product && typeof product.description === 'string'
|
||||
? product.description.trim()
|
||||
: '';
|
||||
if (!raw) continue;
|
||||
return {
|
||||
value: raw.replace(/\s+\n/g, '\n').replace(/\n{3,}/g, '\n\n').slice(0, 100_000),
|
||||
rawValue: raw.slice(0, 120_000),
|
||||
};
|
||||
} catch {
|
||||
// A malformed analytics JSON-LD block is not product evidence.
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function findJsonLdProduct(
|
||||
value: unknown,
|
||||
visited = new Set<unknown>(),
|
||||
): JsonRecord | undefined {
|
||||
if (!value || typeof value !== 'object' || visited.has(value)) return undefined;
|
||||
visited.add(value);
|
||||
if (Array.isArray(value)) {
|
||||
for (const child of value) {
|
||||
const product = findJsonLdProduct(child, visited);
|
||||
if (product) return product;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
const record = value as JsonRecord;
|
||||
const type = record['@type'];
|
||||
if (type === 'Product' || (Array.isArray(type) && type.includes('Product')))
|
||||
return record;
|
||||
if (Array.isArray(record['@graph'])) {
|
||||
const product = findJsonLdProduct(record['@graph'], visited);
|
||||
if (product) return product;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function findRichAnnotationJson(
|
||||
value: unknown,
|
||||
visited = new Set<unknown>(),
|
||||
@@ -716,6 +807,7 @@ function addGalleryAssets(
|
||||
assetKeys.add(key);
|
||||
assets.push({
|
||||
url,
|
||||
mediaType: 'image',
|
||||
// A gallery is product photography for this real SKU. It is not
|
||||
// a SKU-option thumbnail merely because it is the second image.
|
||||
role: 'source_main',
|
||||
@@ -727,6 +819,103 @@ function addGalleryAssets(
|
||||
}
|
||||
}
|
||||
|
||||
/** Adds only Ozon's named product-gallery videos. Page-wide <video> nodes can
|
||||
* belong to reviews or recommendations and are intentionally never considered. */
|
||||
function addGalleryVideos(
|
||||
states: JsonRecord[],
|
||||
sku: string,
|
||||
assets: Asset[],
|
||||
assetKeys: Set<string>,
|
||||
evidenceRef: string,
|
||||
pageUrl: string,
|
||||
) {
|
||||
const galleries = states.filter((state) => {
|
||||
const widget = String(state.__atlasWidgetName ?? '').replace(/^state-/, '');
|
||||
return /^web(?:Gallery|Video)-/.test(widget);
|
||||
});
|
||||
for (const gallery of galleries) {
|
||||
for (const video of extractOzonGalleryVideos(gallery, pageUrl)) {
|
||||
if (assets.length >= 500) return;
|
||||
const key = `video:${sku}:${video.url}`;
|
||||
if (assetKeys.has(key)) continue;
|
||||
assetKeys.add(key);
|
||||
assets.push({
|
||||
url: video.url,
|
||||
mediaType: 'video',
|
||||
...(video.posterUrl ? { posterUrl: video.posterUrl } : {}),
|
||||
role: 'source_main',
|
||||
variantSourceKey: sku,
|
||||
relation: 'variant_direct',
|
||||
sortOrder: assets.length,
|
||||
evidenceRef,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Pure helper for the strictly named Ozon gallery/video state. HLS, blobs,
|
||||
* and non-HTTPS URLs stay out of the collection contract in this first phase. */
|
||||
export function extractOzonGalleryVideos(gallery: JsonRecord, pageUrl: string) {
|
||||
const videos: Array<{ url: string; posterUrl?: string }> = [];
|
||||
const seen = new Set<string>();
|
||||
// Ozon changes whether the direct file lives in videos[], video, or a
|
||||
// nested gallery payload. The old collector walked this *one allowlisted
|
||||
// widget* for direct video extensions; retaining that scope is critical.
|
||||
for (const { value, parent } of nestedVideoValues(gallery)) {
|
||||
const url = directVideoUrl(
|
||||
typeof value === 'string'
|
||||
? value
|
||||
: stringAt(value, ['url', 'videoUrl', 'src']),
|
||||
pageUrl,
|
||||
);
|
||||
if (!url) continue;
|
||||
const posterUrl = ozonImageUrl(
|
||||
(typeof value === 'string'
|
||||
? undefined
|
||||
: stringAt(value, ['coverImage', 'coverUrl', 'poster', 'previewImage']))
|
||||
?? stringAt(parent, ['coverImage', 'coverUrl', 'poster', 'previewImage']),
|
||||
pageUrl,
|
||||
);
|
||||
if (seen.has(url)) {
|
||||
const existing = videos.find((video) => video.url === url);
|
||||
if (existing && !existing.posterUrl && posterUrl)
|
||||
existing.posterUrl = posterUrl;
|
||||
continue;
|
||||
}
|
||||
seen.add(url);
|
||||
videos.push({ url, ...(posterUrl ? { posterUrl } : {}) });
|
||||
}
|
||||
return videos;
|
||||
}
|
||||
|
||||
function nestedVideoValues(root: JsonRecord) {
|
||||
const values: Array<{ value: string | JsonRecord; parent?: JsonRecord }> = [];
|
||||
const pending: Array<{ value: unknown; parent?: JsonRecord }> = [{ value: root }];
|
||||
const seen = new Set<object>();
|
||||
while (pending.length) {
|
||||
const current = pending.pop()!;
|
||||
if (typeof current.value === 'string') {
|
||||
if (/\.(mp4|webm)(?:$|[?#])/i.test(current.value))
|
||||
values.push({ value: current.value, parent: current.parent });
|
||||
continue;
|
||||
}
|
||||
if (!current.value || typeof current.value !== 'object') continue;
|
||||
if (seen.has(current.value)) continue;
|
||||
seen.add(current.value);
|
||||
if (Array.isArray(current.value)) {
|
||||
for (const child of current.value)
|
||||
pending.push({ value: child, parent: current.parent });
|
||||
continue;
|
||||
}
|
||||
const record = current.value as JsonRecord;
|
||||
if (stringAt(record, ['url', 'videoUrl', 'src']))
|
||||
values.push({ value: record, parent: current.parent });
|
||||
for (const child of Object.values(record))
|
||||
pending.push({ value: child, parent: record });
|
||||
}
|
||||
return values;
|
||||
}
|
||||
|
||||
/** Pure helper used by fixture tests and the structured Ozon gallery reader. */
|
||||
export function extractOzonGalleryImageUrls(
|
||||
gallery: JsonRecord,
|
||||
@@ -777,7 +966,7 @@ function widgetStatesFromPageJson(value: unknown): JsonRecord[] {
|
||||
const states: JsonRecord[] = [];
|
||||
for (const [widgetName, raw] of Object.entries(value.widgetStates)) {
|
||||
if (
|
||||
!/^web(Aspects|AspectsModal|ProductHeading|Price|Gallery|Description|RichContent|Features|ShortCharacteristics|DetailedCharacteristics|Characteristics)-/.test(
|
||||
!/^web(Aspects|AspectsModal|ProductHeading|Price|Gallery|Video|Description|RichContent|Features|ShortCharacteristics|DetailedCharacteristics|Characteristics)-/.test(
|
||||
widgetName,
|
||||
)
|
||||
)
|
||||
@@ -997,3 +1186,15 @@ function ozonImageUrl(value: string | undefined, pageUrl: string) {
|
||||
const url = normaliseImageUrl(value, pageUrl, 'ozon');
|
||||
return url?.startsWith('https://') ? url : undefined;
|
||||
}
|
||||
|
||||
function directVideoUrl(value: string | undefined, pageUrl: string) {
|
||||
if (!value) return undefined;
|
||||
try {
|
||||
const url = new URL(value, pageUrl);
|
||||
if (url.protocol !== 'https:' || /\.m3u8(?:$|[?#])/i.test(url.pathname))
|
||||
return undefined;
|
||||
return /\.(mp4|webm)(?:$|[?#])/i.test(url.pathname) ? url.href : undefined;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -180,6 +180,7 @@ export function collectPlatformDomAssets(
|
||||
seen.add(key);
|
||||
assets.push({
|
||||
url,
|
||||
mediaType: 'image',
|
||||
role: group.role,
|
||||
relation: group.relation,
|
||||
sortOrder: assets.length,
|
||||
|
||||
@@ -22,22 +22,32 @@ export function supportedProductPlatform(
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Returns only the public, allowlisted product-state projection supplied by
|
||||
* the platform-specific MAIN-world bridge. */
|
||||
export type MarketplacePageStateFetcher = (
|
||||
platform: Exclude<SourcePlatform, 'ozon'>,
|
||||
) => Promise<unknown>;
|
||||
|
||||
export async function collectCurrentProduct(
|
||||
root: Document = document,
|
||||
pageUrl = location.href,
|
||||
fetchOzonPageJson?: OzonPageJsonFetcher,
|
||||
fetchMarketplacePageState?: MarketplacePageStateFetcher,
|
||||
): Promise<CollectedProductPayload> {
|
||||
const platform = supportedProductPlatform(pageUrl);
|
||||
if (platform === 'ozon')
|
||||
return collectOzonProduct(root, pageUrl, fetchOzonPageJson);
|
||||
if (!platform) throw new Error('当前页面不是支持的商品详情页');
|
||||
const pageState = fetchMarketplacePageState
|
||||
? await fetchMarketplacePageState(platform)
|
||||
: undefined;
|
||||
return platform === '1688'
|
||||
? collect1688(root, pageUrl)
|
||||
: collectTaobao(root, pageUrl, platform);
|
||||
? collect1688(root, pageUrl, pageState)
|
||||
: collectTaobao(root, pageUrl, platform, pageState);
|
||||
}
|
||||
|
||||
function collect1688(root: Document, pageUrl: string): CollectedProductPayload {
|
||||
const state = pageState(root);
|
||||
function collect1688(root: Document, pageUrl: string, bridgedState?: unknown): CollectedProductPayload {
|
||||
const state = [bridgedState, ...pageState(root)];
|
||||
const skuMap = findFirst(state, 'skuMapOriginal');
|
||||
const skuProps = findFirst(state, 'skuProps');
|
||||
const axes = parse1688Axes(skuProps);
|
||||
@@ -50,14 +60,19 @@ function collect1688(root: Document, pageUrl: string): CollectedProductPayload {
|
||||
},
|
||||
];
|
||||
const variants: Variant[] = [];
|
||||
if (isRecord(skuMap)) {
|
||||
const skuRows = Array.isArray(skuMap)
|
||||
? skuMap.map((row) => [stringAt(isRecord(row) ? row : {}, ['specAttrs', 'propPath', 'skuId']) ?? '', row] as const)
|
||||
: isRecord(skuMap)
|
||||
? Object.entries(skuMap)
|
||||
: [];
|
||||
if (skuRows.length) {
|
||||
evidence.push({
|
||||
ref: '1688:sku-map',
|
||||
kind: 'state',
|
||||
path: 'skuMapOriginal',
|
||||
raw_value: `组合记录 ${Object.keys(skuMap).length} 条`,
|
||||
raw_value: `组合记录 ${skuRows.length} 条`,
|
||||
});
|
||||
for (const [combination, row] of Object.entries(skuMap)) {
|
||||
for (const [combination, row] of skuRows) {
|
||||
if (!isRecord(row)) continue;
|
||||
const sourceSku = stringAt(row, ['skuId', 'sku_id', 'skuCode']);
|
||||
// A skuMapOriginal entry itself is a source-provided sellable-combination
|
||||
@@ -97,7 +112,8 @@ function collect1688(root: Document, pageUrl: string): CollectedProductPayload {
|
||||
axes,
|
||||
uniqueVariants(variants),
|
||||
evidence,
|
||||
'1688-state-1.0.0',
|
||||
'1688-structured-2.0.0',
|
||||
state,
|
||||
);
|
||||
}
|
||||
|
||||
@@ -105,8 +121,9 @@ function collectTaobao(
|
||||
root: Document,
|
||||
pageUrl: string,
|
||||
platform: 'taobao' | 'tmall',
|
||||
bridgedState?: unknown,
|
||||
): CollectedProductPayload {
|
||||
const state = pageState(root);
|
||||
const state = [bridgedState, ...pageState(root)];
|
||||
const skuBase = findFirst(state, 'skuBase');
|
||||
const skuCore = findFirst(state, 'skuCore');
|
||||
const axes = parseTaobaoAxes(isRecord(skuBase) ? skuBase.props : undefined);
|
||||
@@ -170,7 +187,8 @@ function collectTaobao(
|
||||
axes,
|
||||
uniqueVariants(variants),
|
||||
evidence,
|
||||
`${platform}-state-1.0.0`,
|
||||
`${platform}-structured-2.0.0`,
|
||||
state,
|
||||
);
|
||||
}
|
||||
|
||||
@@ -183,15 +201,18 @@ function basePayload(
|
||||
variants: Variant[],
|
||||
evidence: CollectedProductPayload['evidence'],
|
||||
parserVersion: string,
|
||||
states: unknown[],
|
||||
): CollectedProductPayload {
|
||||
const canonical =
|
||||
root.querySelector<HTMLLinkElement>('link[rel="canonical"]')?.href ??
|
||||
pageUrl.split('#')[0];
|
||||
const title =
|
||||
const facts = extractMarketplaceFacts(root, states, platform);
|
||||
const title = facts.title ?? (
|
||||
root.querySelector('h1')?.textContent?.replace(/\s+/g, ' ').trim() ||
|
||||
root
|
||||
.querySelector<HTMLMetaElement>('meta[property="og:title"]')
|
||||
?.content?.trim();
|
||||
?.content?.trim()
|
||||
);
|
||||
const assets = collectPlatformDomAssets(platform, root, canonical);
|
||||
evidence.push({
|
||||
ref: `${platform}:dom:image-profile`,
|
||||
@@ -199,6 +220,15 @@ function basePayload(
|
||||
path: 'platform-scoped gallery / sku / detail selectors',
|
||||
raw_value: `采集 ${assets.length} 张受限商品区域图片`,
|
||||
});
|
||||
const videoCount = addMarketplaceVideos(platform, states, assets, canonical);
|
||||
if (videoCount) evidence.push({
|
||||
ref: `${platform}:structured:videos`,
|
||||
kind: 'state',
|
||||
path: platform === '1688' ? 'gallery.fields.video|videos' : 'item.videos',
|
||||
raw_value: `采集 ${videoCount} 个结构化商品视频`,
|
||||
});
|
||||
if (facts.attributes.length) evidence.push({ ref: `${platform}:structured:attributes`, kind: 'state', path: 'allowlisted product state parameter lists', raw_value: `采集 ${facts.attributes.length} 项结构化特征` });
|
||||
if (facts.description) evidence.push({ ref: `${platform}:json-ld:description`, kind: 'json_ld', path: 'Product.description', raw_value: `采集 ${facts.description.length} 个字符描述` });
|
||||
return {
|
||||
source: {
|
||||
platform,
|
||||
@@ -217,8 +247,10 @@ function basePayload(
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
attributes: [],
|
||||
categories: [],
|
||||
...(facts.price ? { sourcePrice: { amount: facts.price, currencyCode: 'CNY', raw_value: facts.price } } : {}),
|
||||
...(facts.description ? { description: { value: facts.description, raw_value: facts.description, confidence: 'verified' as const } } : {}),
|
||||
attributes: facts.attributes,
|
||||
categories: facts.categories,
|
||||
},
|
||||
variantAxes: axes,
|
||||
variants,
|
||||
@@ -227,6 +259,137 @@ function basePayload(
|
||||
};
|
||||
}
|
||||
|
||||
/** The old collector's proven video sources are structured product state. Do
|
||||
* not broaden this to page video elements: they also contain feeds and reviews. */
|
||||
function addMarketplaceVideos(
|
||||
platform: Exclude<SourcePlatform, 'ozon'>,
|
||||
states: unknown[],
|
||||
assets: CollectedProductPayload['assets'],
|
||||
pageUrl: string,
|
||||
) {
|
||||
const gallery = findFirst(states, 'gallery');
|
||||
const item = findFirst(states, 'item');
|
||||
const galleryFields = isRecord(gallery) && isRecord(gallery.fields)
|
||||
? gallery.fields
|
||||
: gallery;
|
||||
const raw = platform === '1688'
|
||||
? [
|
||||
...(isRecord(galleryFields) && Array.isArray(galleryFields.videos) ? galleryFields.videos : []),
|
||||
...(isRecord(galleryFields) && galleryFields.video !== undefined ? [galleryFields.video] : []),
|
||||
]
|
||||
: isRecord(item) && Array.isArray(item.videos)
|
||||
? item.videos
|
||||
: [];
|
||||
let added = 0;
|
||||
const seen = new Set(assets.map((asset) => `video:${asset.url}`));
|
||||
for (const entry of raw) {
|
||||
if (assets.length >= 500) break;
|
||||
const record = isRecord(entry) ? entry : undefined;
|
||||
const url = directVideoUrl(
|
||||
typeof entry === 'string' ? entry : stringAt(record ?? {}, ['url', 'videoUrl', 'src']),
|
||||
pageUrl,
|
||||
);
|
||||
if (!url || seen.has(`video:${url}`)) continue;
|
||||
seen.add(`video:${url}`);
|
||||
const posterUrl = imageUrl(
|
||||
stringAt(record ?? {}, ['videoThumbnailURL', 'thumbnailUrl', 'coverUrl', 'coverImage', 'poster']),
|
||||
pageUrl,
|
||||
);
|
||||
assets.push({
|
||||
url,
|
||||
mediaType: 'video',
|
||||
...(posterUrl ? { posterUrl } : {}),
|
||||
role: 'source_main',
|
||||
relation: 'product_candidate',
|
||||
sortOrder: assets.length,
|
||||
evidenceRef: `${platform}:structured:videos`,
|
||||
});
|
||||
added += 1;
|
||||
}
|
||||
return added;
|
||||
}
|
||||
|
||||
function directVideoUrl(value: string | undefined, pageUrl: string) {
|
||||
if (!value) return undefined;
|
||||
try {
|
||||
const url = new URL(value, pageUrl);
|
||||
if (url.protocol !== 'https:' || /\.m3u8(?:$|[?#])/i.test(url.pathname)) return undefined;
|
||||
return /\.(mp4|webm)(?:$|[?#])/i.test(url.pathname) ? url.href : undefined;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
function imageUrl(value: string | undefined, pageUrl: string) {
|
||||
if (!value) return undefined;
|
||||
try {
|
||||
const url = new URL(value, pageUrl);
|
||||
return url.protocol === 'https:' ? url.href : undefined;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
function extractMarketplaceFacts(root: Document, states: unknown[], platform: Exclude<SourcePlatform, 'ozon'>) {
|
||||
const attributes: CollectedProductPayload['common']['attributes'] = [];
|
||||
const seen = new Set<string>();
|
||||
for (const key of ['basicParamList', 'enhanceParamList', 'attributes', 'props']) {
|
||||
const list = findFirst(states, key);
|
||||
if (!Array.isArray(list)) continue;
|
||||
for (const [index, entry] of list.entries()) {
|
||||
if (!isRecord(entry)) continue;
|
||||
const name = stringAt(entry, ['propertyName', 'name', 'key', 'title']);
|
||||
const value = stringAt(entry, ['valueName', 'value', 'text']);
|
||||
if (!name || !value || seen.has(`${name}\u0000${value}`)) continue;
|
||||
seen.add(`${name}\u0000${value}`);
|
||||
attributes.push({ key: `${platform}:${stringAt(entry, ['propertyId', 'id', 'key']) ?? index}`.slice(0, 200), name: name.slice(0, 300), value: value.slice(0, 2_000), raw_value: value.slice(0, 4_000) });
|
||||
}
|
||||
}
|
||||
const product = jsonLdProduct(root);
|
||||
const item = findFirst(states, 'item');
|
||||
const temp = findFirst(states, 'tempModel');
|
||||
const tradeModel = findFirst(states, 'tradeModel');
|
||||
const priceVO = findFirst(states, 'priceVO');
|
||||
const price = moneyAt(isRecord(tradeModel) ? tradeModel : {}, ['minPrice', 'price'])
|
||||
?? moneyAt(isRecord(priceVO) ? priceVO : {}, ['priceText', 'price', 'extraPrice'])
|
||||
?? decimal(product?.offers && isRecord(product.offers) ? product.offers.price : undefined);
|
||||
const categoryNames = Array.from(root.querySelectorAll<HTMLAnchorElement>('[class*="breadcrumb"] a, [class*="BreadCrumb"] a, nav a[href*="category"]')).map((link) => link.textContent?.replace(/\s+/g, ' ').trim() ?? '').filter(Boolean).slice(0, 50);
|
||||
return {
|
||||
title: (isRecord(item) ? stringAt(item, ['title', 'name']) : undefined) ?? (isRecord(temp) ? stringAt(temp, ['offerTitle']) : undefined) ?? (product ? stringAt(product, ['name']) : undefined),
|
||||
price,
|
||||
description: (product ? stringAt(product, ['description']) : undefined) ?? scopedDescription(root, platform),
|
||||
attributes,
|
||||
categories: [...new Set(categoryNames)].map((name) => ({ key: `${platform}:category:${name.toLowerCase()}`.slice(0, 300), name })),
|
||||
};
|
||||
}
|
||||
|
||||
function scopedDescription(root: Document, platform: Exclude<SourcePlatform, 'ozon'>) {
|
||||
const selectors = platform === '1688'
|
||||
? ['.de-description-detail', '#detailContentContainer', '.html-description']
|
||||
: ['[class*="tabDetailWrap--"]', '[class*="detailInfo--"]'];
|
||||
for (const selector of selectors) {
|
||||
const value = root.querySelector(selector)?.textContent?.replace(/\s+/g, ' ').trim();
|
||||
if (value && value.length >= 10) return value.slice(0, 100_000);
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function jsonLdProduct(root: Document): Json | undefined {
|
||||
for (const script of Array.from(root.scripts)) {
|
||||
if (script.type !== 'application/ld+json' || !script.textContent?.trim()) continue;
|
||||
try { const found = findJsonLdProduct(JSON.parse(script.textContent)); if (found) return found; } catch { /* ignored */ }
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
function findJsonLdProduct(value: unknown): Json | undefined {
|
||||
if (Array.isArray(value)) { for (const child of value) { const found = findJsonLdProduct(child); if (found) return found; } return undefined; }
|
||||
if (!isRecord(value)) return undefined;
|
||||
const type = value['@type'];
|
||||
if (type === 'Product' || (Array.isArray(type) && type.includes('Product'))) return value;
|
||||
return Array.isArray(value['@graph']) ? findJsonLdProduct(value['@graph']) : undefined;
|
||||
}
|
||||
function decimal(value: unknown) { const raw = typeof value === 'string' || typeof value === 'number' ? String(value).replace(',', '.') : ''; return /^\d+(\.\d{1,4})?$/.test(raw) ? raw : undefined; }
|
||||
|
||||
function pageState(root: Document) {
|
||||
const all: unknown[] = [];
|
||||
for (const script of Array.from(root.scripts)) {
|
||||
|
||||
@@ -5,7 +5,9 @@ import test from 'node:test';
|
||||
import {
|
||||
extractOzonCharacteristics,
|
||||
extractOzonDescription,
|
||||
extractOzonJsonLdDescription,
|
||||
extractOzonGalleryImageUrls,
|
||||
extractOzonGalleryVideos,
|
||||
} from '../lib/ozon-parser.ts';
|
||||
import { dedupeImageUrl, normaliseImageUrl } from '../lib/product-media.ts';
|
||||
|
||||
@@ -73,6 +75,41 @@ test('Ozon rich annotation converts to description text without images', () => {
|
||||
assert.equal(description?.value, '耐用的背包\n适合旅行');
|
||||
});
|
||||
|
||||
test('Ozon JSON-LD Product description is the fallback when rich content is absent', () => {
|
||||
const root = {
|
||||
scripts: [{
|
||||
type: 'application/ld+json',
|
||||
textContent: JSON.stringify({
|
||||
'@context': 'http://schema.org',
|
||||
'@type': 'Product',
|
||||
name: '喷雾器',
|
||||
description: '来自 Ozon JSON-LD 的完整商品描述',
|
||||
}),
|
||||
}],
|
||||
} as unknown as Document;
|
||||
assert.equal(
|
||||
extractOzonJsonLdDescription(root)?.value,
|
||||
'来自 Ozon JSON-LD 的完整商品描述',
|
||||
);
|
||||
});
|
||||
|
||||
test('Ozon full characteristics merge with the five short header characteristics', () => {
|
||||
const attributes = extractOzonCharacteristics([
|
||||
{
|
||||
__atlasWidgetName: 'webShortCharacteristics-fixture',
|
||||
characteristics: [{ id: 'type', title: { textRs: [{ content: '类型' }] }, values: [{ text: '喷雾器' }] }],
|
||||
},
|
||||
{
|
||||
__atlasWidgetName: 'webCharacteristics-fixture',
|
||||
characteristics: [{ id: 'weight', title: { textRs: [{ content: '商品重量，克' }] }, values: [{ text: '385' }] }],
|
||||
},
|
||||
]);
|
||||
assert.deepEqual(attributes.map(({ name, value }) => ({ name, value })), [
|
||||
{ name: '类型', value: '喷雾器' },
|
||||
{ name: '商品重量，克', value: '385' },
|
||||
]);
|
||||
});
|
||||
|
||||
test('Ozon thumbnail variants normalise to one original URL', () => {
|
||||
const original = 'https://ir.ozone.ru/s3/multimedia-1-9/13597160877.jpg';
|
||||
const thumbnail =
|
||||
@@ -84,6 +121,29 @@ test('Ozon thumbnail variants normalise to one original URL', () => {
|
||||
);
|
||||
});
|
||||
|
||||
test('Ozon gallery only accepts direct product MP4/WebM videos and retains poster', () => {
|
||||
assert.deepEqual(
|
||||
extractOzonGalleryVideos(
|
||||
{
|
||||
videos: [
|
||||
{
|
||||
url: 'https://cdn.ozon.ru/product/demo.mp4',
|
||||
coverImage: 'https://ir.ozone.ru/s3/multimedia-1/demo.jpg',
|
||||
},
|
||||
{ videoUrl: 'https://cdn.ozon.ru/product/demo.mp4' },
|
||||
{ url: 'https://cdn.ozon.ru/product/stream.m3u8' },
|
||||
{ url: 'blob:https://www.ozon.ru/opaque' },
|
||||
],
|
||||
},
|
||||
'https://www.ozon.ru/product/demo-1/',
|
||||
),
|
||||
[{
|
||||
url: 'https://cdn.ozon.ru/product/demo.mp4',
|
||||
posterUrl: 'https://ir.ozone.ru/s3/multimedia-1/demo.jpg',
|
||||
}],
|
||||
);
|
||||
});
|
||||
|
||||
test('Alibaba transformed image URLs normalise before de-duplication', () => {
|
||||
const original = 'https://cbu01.alicdn.com/img/ibank/O1CN01.jpg';
|
||||
const transformed = `${original}_400x400.jpg?width=400`;
|
||||
|
||||
@@ -49,6 +49,7 @@ export default function CollectionInspectionPage() {
|
||||
() => new Map((record?.variants ?? []).map((variant) => [variant.id, variant])),
|
||||
[record]
|
||||
);
|
||||
const payload = normalisedPayload(record?.normalized_payload);
|
||||
|
||||
if (error)
|
||||
return <Alert type="error" message="无法读取采集数据" description={error} showIcon />;
|
||||
@@ -65,7 +66,7 @@ export default function CollectionInspectionPage() {
|
||||
type="info"
|
||||
showIcon
|
||||
style={{ marginBottom: 16 }}
|
||||
message="这里展示扩展实际提交、服务端实际保存的数据，不代表已确认的商品编辑字段。"
|
||||
message="这里展示扩展实际提交、服务端实际保存的数据；未转存图片或视频直接使用采集来源预览，不代表已确认的商品编辑字段。"
|
||||
/>
|
||||
<Descriptions
|
||||
bordered
|
||||
@@ -81,9 +82,71 @@ export default function CollectionInspectionPage() {
|
||||
{ key: 'status', label: '采集状态', children: record.collection_status },
|
||||
{ key: 'parser', label: '解析器版本', children: record.parser_version ?? '—' },
|
||||
{ key: 'sku', label: '已创建 SKU', children: record.variants.length },
|
||||
{ key: 'assets', label: '素材任务', children: record.asset_transfers.length },
|
||||
{ key: 'assets', label: '素材候选', children: record.asset_transfers.length },
|
||||
{ key: 'attributes', label: '已采集特征', children: payload.attributes.length },
|
||||
{ key: 'categories', label: '已采集类目', children: payload.categories.length },
|
||||
]}
|
||||
/>
|
||||
<Typography.Title level={5} style={{ marginTop: 24 }}>
|
||||
商品描述
|
||||
</Typography.Title>
|
||||
<Card size="small">
|
||||
{payload.description ? (
|
||||
<Typography.Paragraph style={{ whiteSpace: 'pre-wrap', marginBottom: 0 }}>
|
||||
{payload.description}
|
||||
</Typography.Paragraph>
|
||||
) : (
|
||||
<Typography.Text type="secondary">本次采集未取得商品描述。</Typography.Text>
|
||||
)}
|
||||
</Card>
|
||||
<Typography.Title level={5} style={{ marginTop: 24 }}>
|
||||
类目与特征
|
||||
</Typography.Title>
|
||||
<Descriptions
|
||||
bordered
|
||||
size="small"
|
||||
column={1}
|
||||
items={[
|
||||
{
|
||||
key: 'category-path',
|
||||
label: '类目路径',
|
||||
children: payload.categories.length
|
||||
? payload.categories.map((category) => category.name).join(' › ')
|
||||
: '未采集',
|
||||
},
|
||||
]}
|
||||
/>
|
||||
<Table
|
||||
size="small"
|
||||
pagination={false}
|
||||
rowKey={(attribute) => `${attribute.key}:${attribute.value}`}
|
||||
dataSource={payload.attributes}
|
||||
style={{ marginTop: 12 }}
|
||||
columns={[
|
||||
{ title: '特征', dataIndex: 'name' },
|
||||
{ title: '值', dataIndex: 'value' },
|
||||
{ title: '来源键', dataIndex: 'key', ellipsis: true },
|
||||
]}
|
||||
locale={{ emptyText: '本次采集未取得商品特征。' }}
|
||||
/>
|
||||
<Typography.Title level={5} style={{ marginTop: 24 }}>
|
||||
规格轴
|
||||
</Typography.Title>
|
||||
<Table
|
||||
size="small"
|
||||
pagination={false}
|
||||
rowKey="key"
|
||||
dataSource={payload.variantAxes}
|
||||
columns={[
|
||||
{ title: '规格名称', dataIndex: 'name' },
|
||||
{ title: '来源键', dataIndex: 'key', ellipsis: true },
|
||||
{
|
||||
title: '规格值',
|
||||
render: (_, axis) => axis.values.map((value) => value.name).join(' / ') || '—',
|
||||
},
|
||||
]}
|
||||
locale={{ emptyText: '本次采集未取得规格轴。' }}
|
||||
/>
|
||||
<Typography.Title level={5} style={{ marginTop: 24 }}>
|
||||
SKU 草稿
|
||||
</Typography.Title>
|
||||
@@ -111,7 +174,7 @@ export default function CollectionInspectionPage() {
|
||||
]}
|
||||
/>
|
||||
<Typography.Title level={5} style={{ marginTop: 24 }}>
|
||||
素材转存任务
|
||||
素材候选与转存状态
|
||||
</Typography.Title>
|
||||
<Table
|
||||
size="small"
|
||||
@@ -124,26 +187,50 @@ export default function CollectionInspectionPage() {
|
||||
width: 108,
|
||||
render: (_, transfer) => {
|
||||
const url = transfer.asset_id
|
||||
? previewUrls[transfer.asset_id]
|
||||
: undefined;
|
||||
? previewUrls[transfer.asset_id] ?? transfer.source_url
|
||||
: transfer.source_url;
|
||||
if (transfer.media_type === 'video')
|
||||
return url ? (
|
||||
<video
|
||||
controls
|
||||
preload="none"
|
||||
poster={transfer.poster_url ?? undefined}
|
||||
src={url}
|
||||
style={{ width: 96, height: 72, objectFit: 'cover', background: '#000' }}
|
||||
>
|
||||
当前浏览器不支持视频预览。
|
||||
</video>
|
||||
) : (
|
||||
<Typography.Text type="secondary">
|
||||
暂无来源视频
|
||||
</Typography.Text>
|
||||
);
|
||||
if (url)
|
||||
return (
|
||||
<Image
|
||||
width={72}
|
||||
height={72}
|
||||
src={url}
|
||||
alt="已转存采集素材"
|
||||
alt={
|
||||
transfer.asset_id
|
||||
? '已转存采集素材'
|
||||
: '采集来源图片'
|
||||
}
|
||||
style={{ objectFit: 'cover' }}
|
||||
/>
|
||||
);
|
||||
return (
|
||||
<Typography.Text type="secondary">
|
||||
{transfer.status === 'succeeded' ? '预览加载中…' : '待转存'}
|
||||
暂无来源图片
|
||||
</Typography.Text>
|
||||
);
|
||||
},
|
||||
},
|
||||
{ title: '类型', dataIndex: 'role' },
|
||||
{
|
||||
title: '类型',
|
||||
render: (_, transfer) =>
|
||||
`${transfer.media_type === 'video' ? '视频' : '图片'} · ${transfer.role}`,
|
||||
},
|
||||
{
|
||||
title: '关联 SKU',
|
||||
render: (_, transfer) => {
|
||||
@@ -153,7 +240,20 @@ export default function CollectionInspectionPage() {
|
||||
return variant?.source_sku ?? variant?.source_variant_key ?? '商品级';
|
||||
},
|
||||
},
|
||||
{ title: '状态', dataIndex: 'status' },
|
||||
{
|
||||
title: '状态',
|
||||
dataIndex: 'status',
|
||||
render: (status) => {
|
||||
if (status === 'pending_selection')
|
||||
return <Tag color="default">候选，未转存</Tag>;
|
||||
if (status === 'queued') return <Tag color="processing">等待转存</Tag>;
|
||||
if (status === 'running') return <Tag color="processing">转存中</Tag>;
|
||||
if (status === 'succeeded') return <Tag color="success">已转存</Tag>;
|
||||
if (status === 'failed') return <Tag color="error">转存失败</Tag>;
|
||||
if (status === 'cancelled') return <Tag>已取消</Tag>;
|
||||
return status;
|
||||
},
|
||||
},
|
||||
{
|
||||
title: '来源 URL',
|
||||
dataIndex: 'source_url',
|
||||
@@ -229,3 +329,52 @@ function Json({ value }: { value: unknown }) {
|
||||
</pre>
|
||||
);
|
||||
}
|
||||
|
||||
type InspectionAttribute = { key: string; name: string; value: string };
|
||||
type InspectionCategory = { key: string; name: string };
|
||||
type InspectionAxis = { key: string; name: string; values: Array<{ key: string; name: string }> };
|
||||
|
||||
function normalisedPayload(value: unknown): {
|
||||
description: string | null;
|
||||
attributes: InspectionAttribute[];
|
||||
categories: InspectionCategory[];
|
||||
variantAxes: InspectionAxis[];
|
||||
} {
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value))
|
||||
return { description: null, attributes: [], categories: [], variantAxes: [] };
|
||||
const root = value as Record<string, unknown>;
|
||||
const common = record(root.common);
|
||||
const description = record(common?.description)?.value;
|
||||
return {
|
||||
description: typeof description === 'string' ? description : null,
|
||||
attributes: records(common?.attributes).flatMap((item) =>
|
||||
typeof item.key === 'string' && typeof item.name === 'string' && typeof item.value === 'string'
|
||||
? [{ key: item.key, name: item.name, value: item.value }]
|
||||
: []),
|
||||
categories: records(common?.categories).flatMap((item) =>
|
||||
typeof item.key === 'string' && typeof item.name === 'string'
|
||||
? [{ key: item.key, name: item.name }]
|
||||
: []),
|
||||
variantAxes: records(root.variantAxes).flatMap((item) => {
|
||||
if (typeof item.key !== 'string' || typeof item.name !== 'string') return [];
|
||||
return [{
|
||||
key: item.key,
|
||||
name: item.name,
|
||||
values: records(item.values).flatMap((option) =>
|
||||
typeof option.key === 'string' && typeof option.name === 'string'
|
||||
? [{ key: option.key, name: option.name }]
|
||||
: []),
|
||||
}];
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
function record(value: unknown): Record<string, unknown> | undefined {
|
||||
return value && typeof value === 'object' && !Array.isArray(value)
|
||||
? value as Record<string, unknown>
|
||||
: undefined;
|
||||
}
|
||||
|
||||
function records(value: unknown): Record<string, unknown>[] {
|
||||
return Array.isArray(value) ? value.filter((item): item is Record<string, unknown> => !!record(item)) : [];
|
||||
}
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
import type { CollectionItem, CollectionItemInspection, CreateCollectionItemRequest, CursorPaginationMeta, ListCollectionItemsQuery, UpdateCollectionItemRequest } from '@atlas/contracts';
|
||||
import type { CollectionItem, CollectionItemInspection, CreateCollectionItemRequest, CursorPaginationMeta, ListCollectionItemsQuery, MaterializeCollectionAssetsRequest, UpdateCollectionItemRequest } from '@atlas/contracts';
|
||||
import { api } from './api';
|
||||
export const collectionApi = {
|
||||
list: (platformId: string, query: Partial<ListCollectionItemsQuery> = {}) => api.getList<CollectionItem, CursorPaginationMeta & { active_count: number; collection_limit: number }>('/collection-items', { headers: { 'X-Atlas-Platform-Id': platformId }, params: { limit: 20, ...query } }),
|
||||
create: (payload: CreateCollectionItemRequest) => api.post<{ collection_item: CollectionItem; product_id: string; editor_entry_path: string }>('/collection-items', payload, { headers: { 'Idempotency-Key': crypto.randomUUID() } }),
|
||||
inspection: (id: string) => api.get<CollectionItemInspection>(`/collection-items/${id}`),
|
||||
materializeAssets: (id: string, payload: MaterializeCollectionAssetsRequest) => api.post<{ operation: { id: string; kind: string; status: string } }>(`/collection-items/${id}/asset-materializations`, payload, { headers: { 'Idempotency-Key': crypto.randomUUID() } }),
|
||||
update: (id: string, payload: UpdateCollectionItemRequest, revision: number) => api.patch<CollectionItem>(`/collection-items/${id}`, payload, { headers: { 'Idempotency-Key': crypto.randomUUID(), 'If-Match': `"${revision}"` } }),
|
||||
bulkDelete: (ids: string[]) => api.post<{ deleted_count: number }>('/collection-items/bulk-delete', { ids }, { headers: { 'Idempotency-Key': crypto.randomUUID() } }),
|
||||
clear: (expected_active_count: number) => api.post<{ deleted_count: number }>('/collection-items/clear', { confirm: true, expected_active_count }, { headers: { 'Idempotency-Key': crypto.randomUUID() } }),
|
||||
|
||||
@@ -5,11 +5,12 @@ import { CreateBucketCommand, HeadBucketCommand, PutObjectCommand, type S3Client
|
||||
import type { Database } from '@atlas/database';
|
||||
import type { OutboxJob } from './outbox-dispatcher.js';
|
||||
|
||||
const MAX_BYTES = 20 * 1024 * 1024;
|
||||
type Transfer = { id: string; source_url: string; source_variant_id: string | null; source_snapshot_revision: string | number };
|
||||
const MAX_IMAGE_BYTES = 20 * 1024 * 1024;
|
||||
const MAX_VIDEO_BYTES = 200 * 1024 * 1024;
|
||||
type Transfer = { id: string; source_url: string; source_variant_id: string | null; source_snapshot_revision: string | number; media_type: 'image' | 'video' };
|
||||
|
||||
/** Downloads source images only after a public-address check. A failed image is
|
||||
* terminal for this transfer, not for the product collection transaction. */
|
||||
/** Downloads selected source media only after a public-address check. A failed
|
||||
* transfer is terminal for that candidate, not for the product collection transaction. */
|
||||
export async function materializeCollectionAssets(database: Database, storage: S3Client, bucket: string, job: OutboxJob) {
|
||||
await ensureBucket(storage, bucket);
|
||||
const snapshotRevision = Number(job.payload.source_snapshot_revision ?? 1);
|
||||
@@ -17,7 +18,7 @@ export async function materializeCollectionAssets(database: Database, storage: S
|
||||
const transfers = await database.workspaceTransaction(job.workspaceId, async (tx) => {
|
||||
const rows = await tx.query<Transfer>(`UPDATE atlas.collection_asset_transfers SET status = 'running', updated_at = now()
|
||||
WHERE id IN (SELECT id FROM atlas.collection_asset_transfers WHERE collection_item_id = $1 AND source_snapshot_revision = $2 AND status = 'queued' ORDER BY created_at FOR UPDATE SKIP LOCKED)
|
||||
RETURNING id, source_url, source_variant_id, source_snapshot_revision`, [String(job.payload.collection_item_id), snapshotRevision]);
|
||||
RETURNING id, source_url, source_variant_id, source_snapshot_revision, media_type::text AS media_type`, [String(job.payload.collection_item_id), snapshotRevision]);
|
||||
return rows.rows;
|
||||
});
|
||||
for (const transfer of transfers) await materializeOne(database, storage, bucket, job.workspaceId, transfer);
|
||||
@@ -29,7 +30,7 @@ export async function materializeCollectionAssets(database: Database, storage: S
|
||||
async function materializeOne(database: Database, storage: S3Client, bucket: string, workspaceId: string, transfer: Transfer) {
|
||||
const snapshotRevision = Number(transfer.source_snapshot_revision);
|
||||
try {
|
||||
const downloaded = await downloadImage(transfer.source_url);
|
||||
const downloaded = await downloadMedia(transfer.source_url, transfer.media_type);
|
||||
const current = await database.workspaceTransaction(workspaceId, async (tx) => (await tx.query<{ id: string }>(`SELECT t.id FROM atlas.collection_asset_transfers t JOIN atlas.collection_items c ON c.id = t.collection_item_id
|
||||
WHERE t.id = $1 AND t.status = 'running' AND t.source_snapshot_revision = $2 AND c.source_snapshot_revision = $2`, [transfer.id, snapshotRevision])).rows[0]);
|
||||
if (!current) return;
|
||||
@@ -52,19 +53,24 @@ async function materializeOne(database: Database, storage: S3Client, bucket: str
|
||||
}
|
||||
}
|
||||
|
||||
async function downloadImage(source: string) {
|
||||
async function downloadMedia(source: string, mediaType: Transfer['media_type']) {
|
||||
let url = new URL(source);
|
||||
if (mediaType === 'video' && /\.m3u8(?:$|[?#])/i.test(url.pathname))
|
||||
throw new Error('暂不支持 HLS 视频转存');
|
||||
for (let redirect = 0; redirect < 4; redirect += 1) {
|
||||
await assertPublicUrl(url);
|
||||
const response = await fetch(url, { redirect: 'manual', signal: AbortSignal.timeout(20_000), headers: { Accept: 'image/avif,image/webp,image/png,image/jpeg,*/*;q=0.8', 'User-Agent': 'Atlas-Collector/0.1' } });
|
||||
const response = await fetch(url, { redirect: 'manual', signal: AbortSignal.timeout(60_000), headers: { Accept: mediaType === 'video' ? 'video/mp4,video/webm,*/*;q=0.8' : 'image/avif,image/webp,image/png,image/jpeg,*/*;q=0.8', 'User-Agent': 'Atlas-Collector/0.1' } });
|
||||
if ([301, 302, 303, 307, 308].includes(response.status)) {
|
||||
const location = response.headers.get('location'); if (!location) throw new Error('来源素材重定向缺少地址'); url = new URL(location, url); continue;
|
||||
}
|
||||
if (!response.ok || !response.body) throw new Error(`来源素材请求失败（HTTP ${response.status}）`);
|
||||
const chunks: Buffer[] = []; let total = 0;
|
||||
for await (const chunk of response.body as AsyncIterable<Uint8Array>) { total += chunk.byteLength; if (total > MAX_BYTES) throw new Error('来源素材超过 20MB'); chunks.push(Buffer.from(chunk)); }
|
||||
const bytes = Buffer.concat(chunks, total); const image = detectImage(bytes); if (!image) throw new Error('来源素材不是支持的 PNG、JPEG 或 WebP 图片');
|
||||
return { bytes, ...image };
|
||||
const limit = mediaType === 'video' ? MAX_VIDEO_BYTES : MAX_IMAGE_BYTES;
|
||||
for await (const chunk of response.body as AsyncIterable<Uint8Array>) { total += chunk.byteLength; if (total > limit) throw new Error(mediaType === 'video' ? '来源视频超过 200MB' : '来源图片超过 20MB'); chunks.push(Buffer.from(chunk)); }
|
||||
const bytes = Buffer.concat(chunks, total);
|
||||
const media = mediaType === 'video' ? detectVideo(bytes) : detectImage(bytes);
|
||||
if (!media) throw new Error(mediaType === 'video' ? '来源素材不是支持的 MP4 或 WebM 视频' : '来源素材不是支持的 PNG、JPEG 或 WebP 图片');
|
||||
return { bytes, ...media };
|
||||
}
|
||||
throw new Error('来源素材重定向次数过多');
|
||||
}
|
||||
@@ -84,4 +90,9 @@ function detectImage(bytes: Buffer): { contentType: 'image/png' | 'image/jpeg' |
|
||||
if (bytes.length > 9 && bytes[0] === 0xff && bytes[1] === 0xd8) for (let offset = 2; offset + 9 < bytes.length;) { if (bytes[offset] !== 0xff) return null; const marker = bytes[offset + 1], length = bytes.readUInt16BE(offset + 2); if (length < 2 || offset + 2 + length > bytes.length) return null; if ([0xc0, 0xc1, 0xc2, 0xc3, 0xc5, 0xc6, 0xc7, 0xc9, 0xca, 0xcb, 0xcd, 0xce, 0xcf].includes(marker)) return dimensions('image/jpeg', bytes.readUInt16BE(offset + 7), bytes.readUInt16BE(offset + 5)); offset += length + 2; }
|
||||
return null;
|
||||
}
|
||||
function detectVideo(bytes: Buffer): { contentType: 'video/mp4' | 'video/webm'; width: null; height: null } | null {
|
||||
if (bytes.length >= 12 && bytes.subarray(4, 8).toString('ascii') === 'ftyp') return { contentType: 'video/mp4', width: null, height: null };
|
||||
if (bytes.length >= 4 && bytes.subarray(0, 4).equals(Buffer.from([0x1a, 0x45, 0xdf, 0xa3]))) return { contentType: 'video/webm', width: null, height: null };
|
||||
return null;
|
||||
}
|
||||
function dimensions(contentType: 'image/png' | 'image/jpeg' | 'image/webp', width: number, height: number) { return width >= 1 && height >= 1 && width <= 8192 && height <= 8192 ? { contentType, width, height } : null; }
|
||||
|
||||
+8
-6
@@ -9,6 +9,7 @@
|
||||
| 项目 | [`project/`](./project/) | 产品定位、交接、协作约定与已确认的长期约束 |
|
||||
| 产品需求 | [`requirements/`](./requirements/) | 页面、流程、字段和待确认事项 |
|
||||
| 基础架构 | [`architecture/`](./architecture/) | 多租户、平台、门店、水印、物流和定价的数据设计及 DDL |
|
||||
| 图片存储策略 | [`architecture/media-storage-strategy.md`](./architecture/media-storage-strategy.md) | 采集候选图片的按需物化、AI/水印处理与发布素材边界。 |
|
||||
| 商品领域设计 | [`product/`](./product/) | 已有的多 SKU 商品编辑页、领域模型与 API 草案 |
|
||||
| 项目结构 | [`architecture/project-structure.md`](./architecture/project-structure.md) | pnpm monorepo 的目录、依赖关系与本地运行边界。 |
|
||||
| 外部平台资料 | [`integrations/`](./integrations/) | Ozon Seller API 的整理资料与适配依据 |
|
||||
@@ -32,9 +33,10 @@
|
||||
12. [`architecture/database-security.md`](./architecture/database-security.md)：迁移/应用数据库角色、RLS 强制策略与部署职责。
|
||||
13. [`architecture/transactional-outbox-and-idempotency.md`](./architecture/transactional-outbox-and-idempotency.md)：关键写请求去重、事务 Outbox、BullMQ 投递与验收规则。
|
||||
14. [`guides/object-storage-object-key.md`](./guides/object-storage-object-key.md)：对象存储、`object_key`、来源 URL 与预签名 URL 的使用边界。
|
||||
15. [`guides/browser-extension-background-service-worker.md`](./guides/browser-extension-background-service-worker.md)：浏览器扩展后台 Service Worker 的运行模型、消息调用链与调试方法。
|
||||
16. [`guides/minio-local-deployment.md`](./guides/minio-local-deployment.md)：MinIO 简介、本地 Docker 部署与应用侧使用方法。
|
||||
17. [`guides/postgresql-local-development.md`](./guides/postgresql-local-development.md)：原生 PostgreSQL / Docker Compose 本地数据库、角色、迁移与建表。
|
||||
18. [`api/README.md`](./api/README.md)：HTTP API 统一约定及第一批账户、门店、水印接口。
|
||||
19. [`reviews/implementation-readiness-open-items.md`](./reviews/implementation-readiness-open-items.md)：实现前必须闭环的数据库/API 事项与已确认约束。
|
||||
19. 后续按模块补全定价、采集、商品编辑、发布和在线商品接口。
|
||||
15. [`architecture/media-storage-strategy.md`](./architecture/media-storage-strategy.md)：**采集图片不自动转存、按实际使用再物化的强制设计规则。**
|
||||
16. [`guides/browser-extension-background-service-worker.md`](./guides/browser-extension-background-service-worker.md)：浏览器扩展后台 Service Worker 的运行模型、消息调用链与调试方法。
|
||||
17. [`guides/minio-local-deployment.md`](./guides/minio-local-deployment.md)：MinIO 简介、本地 Docker 部署与应用侧使用方法。
|
||||
18. [`guides/postgresql-local-development.md`](./guides/postgresql-local-development.md)：原生 PostgreSQL / Docker Compose 本地数据库、角色、迁移与建表。
|
||||
19. [`api/README.md`](./api/README.md)：HTTP API 统一约定及第一批账户、门店、水印接口。
|
||||
20. [`reviews/implementation-readiness-open-items.md`](./reviews/implementation-readiness-open-items.md)：实现前必须闭环的数据库/API 事项与已确认约束。
|
||||
21. 后续按模块补全定价、采集、商品编辑、发布和在线商品接口。
|
||||
|
||||
@@ -22,24 +22,24 @@
|
||||
|
||||
`POST /api/extension/materials`
|
||||
|
||||
Headers：扩展 Bearer token 与 `Idempotency-Key` 必填。请求体为已定义的 `CollectedProductPayload`，其 `source` 至少包含 `platform`、`itemKey`、`canonicalUrl`、`collectedAt`、`parserVersion`；`variants` 仅列来源有稳定证据的真实组合，媒体只传来源 URL/关系和证据，不传图片二进制。
|
||||
Headers：扩展 Bearer token 与 `Idempotency-Key` 必填。请求体为已定义的 `CollectedProductPayload`，其 `source` 至少包含 `platform`、`itemKey`、`canonicalUrl`、`collectedAt`、`parserVersion`；`variants` 仅列来源有稳定证据的真实组合，媒体只传来源 URL、`mediaType`（`image` / `video`）、可选 `posterUrl`、关系和证据，不传二进制。
|
||||
|
||||
请求体不得包含或传入 `workspace_id`、`target_platform_id`、`platform_id`、`store_id`、`api_key`、Cookie、授权头或来源站会话字段。服务端从扩展令牌解析工作空间，并将来源 `platform` 映射为系统已启用的来源平台。
|
||||
|
||||
服务端在一个事务中：检查工作空间采集箱上限 → 按 `workspace + source_platform + source_item_key` 查找活动记录 → 创建记录，或以本次页面快照覆盖已有记录的来源 URL、标题、价格、原始证据、规范化载荷、SKU 草稿与候选素材 → 再经 outbox 投递素材 Worker。重复上报不会增加采集箱配额，且会保留原有 `collection_item` 与 `product` ID；旧素材任务被取消，新素材任务使用新的来源快照版本，旧 Worker 不得回写新快照。
|
||||
服务端在一个事务中：检查工作空间采集箱上限 → 按 `workspace + source_platform + source_item_key` 查找活动记录 → 创建记录，或以本次页面快照覆盖已有记录的来源 URL、标题、价格、原始证据、规范化载荷、SKU 草稿与候选素材。采集阶段的图片和视频只保存为 `pending_selection` 候选记录，**不下载媒体、不投递素材 Worker**；用户在编辑流程中选择媒体后，才调用按需转存接口。重复上报不会增加采集箱配额，且会保留原有 `collection_item` 与 `product` ID；旧快照中尚未完成的候选或任务会被取消，新快照使用新的来源快照版本，旧 Worker 不得回写新快照。
|
||||
|
||||
成功返回 `201`（新建）或 `200`（复用）：
|
||||
|
||||
```json
|
||||
{ "data": {
|
||||
"collection_item": { "id": "uuid", "status": "materializing", "source_title": "商品标题", "variant_count": 3 },
|
||||
"collection_item": { "id": "uuid", "status": "ready_for_edit", "source_title": "商品标题", "variant_count": 3 },
|
||||
"product_id": "uuid", "reused": false, "refreshed": false,
|
||||
"editor_entry_path": "/products/uuid",
|
||||
"materialization": { "total": 12, "queued": 12, "succeeded": 0, "failed": 0 }
|
||||
"materialization": { "total": 12, "pending_selection": 12, "queued": 0, "succeeded": 0, "failed": 0 }
|
||||
} }
|
||||
```
|
||||
|
||||
达到工作空间上限时返回 `422 COLLECTION_LIMIT_REACHED`，并附 `{current_count, limit}`；不能因重复点击或并发请求绕过 1000 条限制。单个来源素材失败不使采集事务失败，关键商品/SKU 解析失败才使记录进入 `failed` 并返回可展示的脱敏摘要。
|
||||
达到工作空间上限时返回 `422 COLLECTION_LIMIT_REACHED`，并附 `{current_count, limit}`；不能因重复点击或并发请求绕过 1000 条限制。采集阶段不会因下载来源媒体失败而失败；关键商品/SKU 解析失败才使记录进入 `failed` 并返回可展示的脱敏摘要。
|
||||
|
||||
## 3. Ozon Seller 网页本地观察
|
||||
|
||||
@@ -80,7 +80,7 @@ Headers：扩展 Bearer token 与 `Idempotency-Key` 必填。请求体为已定
|
||||
}], "meta": { "next_cursor": null, "has_more": false, "active_count": 12, "collection_limit": 1000 } }
|
||||
```
|
||||
|
||||
`listing_summary.status` 映射 UI 的未上架/已部分上架/已上架；`collection_status` 是采集/素材处理状态，二者绝不混用。首张素材转存未完成时 thumbnail 为占位状态；商品未命名时 `title` 返回 `null`，前端展示“未命名商品”。
|
||||
`listing_summary.status` 映射 UI 的未上架/已部分上架/已上架；`collection_status` 是采集/素材处理状态，二者绝不混用。采集测试页应直接使用候选的 `source_url` 预览图片，视频则用 `preload=none` 的播放器，不等待转存、也不为预览创建存储对象；商品未命名时 `title` 返回 `null`，前端展示“未命名商品”。
|
||||
|
||||
`GET /api/collection-items/{collection_item_id}`
|
||||
|
||||
@@ -90,7 +90,7 @@ Headers：扩展 Bearer token 与 `Idempotency-Key` 必填。请求体为已定
|
||||
|
||||
Headers：`If-Match`、`Idempotency-Key` 必填。首期仅允许修改 `note`；标题、SKU 和素材在商品编辑 API 中修改。响应返回更新后的采集记录。`If-Match` 对应记录的 `revision`，包括采集 Worker 状态更新在内的每次修改都递增；不得覆盖 Worker 的状态更新。
|
||||
|
||||
## 5. 手动创建、删除与素材重试
|
||||
## 5. 手动创建、按需转存、删除与素材重试
|
||||
|
||||
`POST /api/collection-items`
|
||||
|
||||
@@ -102,6 +102,10 @@ Headers：`Idempotency-Key` 必填。
|
||||
|
||||
创建 `source_kind=manual` 的采集记录、空商品草稿和默认 SKU，返回 `201`、`collection_item`、`product_id` 与 `editor_entry_path`。它仍不绑定目标平台/店铺，也不绕过之后的类目、定价和首次发布校验。
|
||||
|
||||
`POST /api/collection-items/{collection_item_id}/asset-materializations`
|
||||
|
||||
Headers：`Idempotency-Key` 必填。请求 `{ "transfer_ids": ["uuid"] }`；只能选择当前来源快照中状态为 `pending_selection` 的候选图片或视频。服务端将所选候选改为 `queued`，创建 operation，并通过 Outbox 投递 `collection.assets.materialize.requested` 给 Worker；返回 `202` 和 operation。该接口是商品编辑、加水印、AI 处理和发布流程共用的按需转存原语，不应在采集测试页自动调用。
|
||||
|
||||
`POST /api/collection-items/{collection_item_id}/asset-retries`
|
||||
|
||||
Headers：`Idempotency-Key` 必填。请求 `{ "transfer_ids": ["uuid"] }`；只能重试该采集记录中 `failed` 或 `cancelled` 的转存任务。返回 `202` 和 operation。不能用它重试来源商品解析或已成功素材。
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
# 商品采集媒体（图片与视频）存储策略
|
||||
|
||||
> 状态：采集阶段的按需物化已落地；正式商品编辑、水印、AI 图片处理和发布尚未接入选择动作。后续任何相关开发必须以本文为准；接口、数据模型和 Worker 改动须先在同一任务中完成相应契约更新。
|
||||
|
||||
## 1. 目标与基本原则
|
||||
|
||||
采集阶段的绝大多数图片和视频最终不会被用户选中、编辑或发布。首期因此采用**按需物化**：先保存可审查的来源候选信息，只有用户实际使用媒体时才下载、校验并写入 Atlas 对象存储。
|
||||
|
||||
1. `source_url` 是来源证据与即时预览地址，不是 Atlas 控制的正式素材，也不是可靠的长期地址。
|
||||
2. `media_asset` 才是经过大小、格式、安全校验并由 Atlas 控制的正式素材；编辑、发布和需要稳定重试的处理链路只能引用它。
|
||||
3. 浏览器可显示来源图片不代表后端或第三方 AI 服务可稳定下载它。来源可能有防盗链、临时 URL、限流、地区限制、图片替换或失效。
|
||||
4. Worker 不携带来源站 Cookie，不能为了下载图片绕过来源站的会话或防盗链机制。
|
||||
5. 首期不做“已选择但未发布素材”的自动生命周期删除。自动清理需要定义引用、编辑中状态、恢复和误删处理，复杂度与风险超过当前收益。
|
||||
|
||||
## 2. 场景决策表
|
||||
|
||||
| 场景 | 是否需要先转存原图 | 原因 |
|
||||
|---|---:|---|
|
||||
| 采集测试页预览 | 否 | 直接显示来源 URL；视频使用 `preload=none`，最快、零存储成本 |
|
||||
| 采集箱暂存 | 否 | 只保存候选 URL、媒体类型、角色、SKU 关联、视频封面与证据 |
|
||||
| 用户选图/视频进入编辑 | 建议是 | 转存选中的媒体，保证后续可用、可重试 |
|
||||
| 加水印 | 不一定 | 可在任务中临时下载来源图→加水印→只保存水印结果；但来源失效后无法重试 |
|
||||
| AI 生图 / AI 改图 | 通常要先转存或至少临时下载 | AI 服务需要实际图片字节，不能可靠地只依赖来源 URL |
|
||||
| 正式发布 | 是 | 必须使用 Atlas 控制的已验证素材，不能依赖第三方 URL |
|
||||
|
||||
## 3. 推荐流程
|
||||
|
||||
```text
|
||||
来源商品页
|
||||
→ 采集候选 source_url、媒体类型、图片角色、SKU/规格关联、排序、视频封面和证据
|
||||
→ 采集箱 / 测试页直接加载 source_url 供人工核对
|
||||
→ 用户选择图片、请求水印/AI 处理或发起发布
|
||||
→ 按需下载 → HTTPS/格式/大小/内容校验 → 创建或复用 media_asset
|
||||
→ 编辑、派生任务和发布仅引用 media_asset
|
||||
```
|
||||
|
||||
### 3.1 采集与即时预览
|
||||
|
||||
- 采集 API 只持久化来源候选信息；不得在采集完成后自动创建“下载全部候选图”的素材任务或 Outbox 事件。
|
||||
- 测试页和后续采集箱详情优先以 `source_url` 作为 `<img>` 或 `<video preload="none">` 预览。跨域媒体通常可以由浏览器显示，但来源拒绝加载时只显示明确的失败状态与“查看来源”链接。
|
||||
- 来源预览不得被误写入 `media_assets`、商品正式媒体关系、发布请求或长期缓存。
|
||||
|
||||
### 3.2 选择图片与正式编辑
|
||||
|
||||
- 用户首次把候选图设为商品主图、SKU 图、详情图，或进入依赖稳定原图的编辑操作时，服务端创建按需转存任务。
|
||||
- 转存完成后，界面和后续操作切换到该 `media_asset` 的短时访问 URL；来源 URL 仍保留为采集证据。
|
||||
- 转存失败或来源图已不可访问时，应提示用户重新采集/重新选择，而不是尝试携带来源 Cookie 或伪造 Referer 绕过限制。
|
||||
|
||||
### 3.3 水印与 AI 图片处理
|
||||
|
||||
- 水印任务可以临时下载来源图并只存储水印派生结果，以节约原图存储；但该模式无法在来源失效后可靠重试。需要稳定重试、再次编辑或保留原图时，应先转存原图。
|
||||
- AI 生图不一定需要来源图；AI 改图、图生图、抠图等需要图片输入的能力，必须获得实际图片字节。实现可选择先物化 `media_asset`，或在单次任务内临时下载后提交给 AI 提供商。
|
||||
- 向 AI 提供商发送来源图片字节或可访问 URL 属于对第三方的数据传输。产品界面、隐私说明和任务审计必须在该能力立项时明确告知；不得默认将所有采集候选图发送给 AI。
|
||||
- 所有水印和 AI 的成功输出都必须保存为 Atlas 控制的 `media_asset`，并记录输入素材/来源、模板或模型版本与任务证据。
|
||||
|
||||
### 3.4 发布
|
||||
|
||||
- 发布前必须验证每张已选图片是当前工作空间内已校验的 `media_asset`。
|
||||
- 禁止将来源站 URL 直接提交给目标销售平台；发布任务只能使用 Atlas 对象存储中的受控素材或由平台要求的上传结果。
|
||||
|
||||
## 4. 首期明确不做的能力
|
||||
|
||||
不实现“已选择但未发布素材在 7～30 天未使用即自动删除”。实现这一策略需要追踪编辑草稿、引用关系、处理中任务、发布快照、恢复窗口和误删申诉；在没有真实成本数据前，首期保留用户已按需转存的素材。
|
||||
|
||||
未来如存储成本成为问题，优先考虑只清理**从未被商品媒体、派生任务或发布快照引用**的素材，并在独立需求、数据模型、Worker 与用户通知方案评审后实施。
|
||||
|
||||
## 5. 已落地的按需物化边界
|
||||
|
||||
`POST /api/extension/materials` 已将图片和视频保存为 `collection_asset_transfers` 的 `pending_selection` 候选，不会创建素材 Worker 的 Outbox 事件。候选记录带 `media_type`，视频可另带 `poster_url`。重复采集会取消旧快照中尚未完成的候选或转存任务，并以新快照候选覆盖它们。
|
||||
|
||||
`POST /api/collection-items/{id}/asset-materializations` 是当前唯一的按需转存入口：请求必须包含当前快照中的一个或多个 `pending_selection` 候选 ID，并携带 `Idempotency-Key`。服务端将所选行标为 `queued`、创建 operation 并写入 `collection.assets.materialize.requested` Outbox 事件；既有 Worker 继续执行安全下载、校验、去重与 `media_asset` 创建。
|
||||
|
||||
`/collection-test/{id}` 对图片候选直接显示 `source_url`，对视频候选使用按需播放的来源 URL；已转存的历史素材则优先显示 Atlas 短时预览 URL。该测试页只核对数据，不能触发转存。
|
||||
|
||||
### 5.1 首期视频格式边界
|
||||
|
||||
- 当前采集器只接受来源商品结构化状态中明确给出的 HTTPS `MP4` / `WebM` 直链；Ozon 读取 `webGallery` / `webVideo`，1688 读取 `gallery.fields.video|videos`，淘宝/天猫读取 `item.videos`。绝不扫描整页 `video` 元素。
|
||||
- `blob:` URL、`m3u8` / HLS、DRM 和无法识别的扩展不进入候选；转存 Worker 再以文件魔数确认 MP4/WebM。MP4/WebM 单文件上限为 200MB、最长下载等待 60 秒；图片保持 20MB 限制。
|
||||
- 首期不做视频转码、封面抽帧、HLS 下载、DRM 处理、视频水印或发布。视频尺寸和时长也暂不解析；`media_asset.content_type` 是后续处理时的媒体判断依据。
|
||||
|
||||
下一阶段必须在正式商品编辑页实现选图后调用上述接口，并补齐来源失效、防盗链、重复点击、跨工作空间访问、AI/水印输入和发布前素材校验的集成测试；不得绕过该接口直接把来源 URL 写入正式商品媒体或发布请求。
|
||||
|
||||
## 6. 待优化 P1：来源站下载限流与退避
|
||||
|
||||
### 6.1 当前风险边界
|
||||
|
||||
按需物化已避免“采集一次下载全部图片”，但当前 Worker 尚未按来源平台或域名限流：单个转存任务内部逐张下载，Worker 总并发为 5；多个商品同时选图、多个 Worker 实例，或用户在测试页一次打开大量候选图片时，都可能在短时间向同一来源 CDN 发出多次请求。
|
||||
|
||||
- 测试页的候选图片预览由用户浏览器直接请求 `source_url`，不经过 Atlas Worker；它不会消耗 Atlas 存储，但多张图片同屏会形成来源站请求突发。
|
||||
- Worker 下载会限制 HTTPS 公网地址、最多 3 次重定向、60 秒超时；图片上限 20MB、视频上限 200MB。这些是安全和资源保护，**不是来源站限流**。
|
||||
- 当前 `429`、`503` 等下载失败会把候选标记为失败；不会读取 `Retry-After` 后延迟重试，因此不容易持续猛撞来源站，但会降低成功率。
|
||||
- 素材仅在下载后按内容 SHA-256 去重；相同的来源 URL 被多个候选选中时，当前仍可能被重复下载。
|
||||
|
||||
### 6.2 正式编辑页接入前必须完成
|
||||
|
||||
1. 建立基于 Redis 的分布式下载限流器，限流键至少覆盖来源平台和规范化域名；初始建议每域名最多 2 个并发、每秒 2～3 个请求、短时突发不超过 4。实际阈值需按来源平台压测和运行数据调整。
|
||||
2. 将 `429`、带 `Retry-After` 的响应和临时 `503` 转为可调度的延迟重试；遵守 `Retry-After`，使用指数退避与抖动，不能立即重试。
|
||||
3. 对归一化后的来源 URL 建立下载任务去重：同一图片仅下载一次，完成后让多个候选关联同一 `media_asset`，而不是依赖下载后的哈希去重。
|
||||
4. 对连续限流或临时不可用的来源建立熔断/降速窗口，并在操作状态中给出可理解的“来源暂时限流，稍后重试”反馈。
|
||||
5. 测试页和正式采集箱详情采用懒加载、分页或虚拟列表，避免首次进入即请求全部来源预览图；来源图加载失败时保留“查看来源”链接，不能尝试携带 Cookie 或伪造 Referer 绕过防盗链。
|
||||
|
||||
在以上能力落地前，不允许将按需转存用于自动批量选图、全量素材预热或无人值守的大规模转存。
|
||||
+36
-1
@@ -6,7 +6,7 @@
|
||||
|
||||
这是一个全新的云端产品，不是 `ozon-seller-kit` 的本地版重构。目标用户可拥有多个门店，每个门店只能绑定一个平台；平台自身可支持多个市场、国家、语言和币种。因此语言、币种、类目和发布规则不能写死为 Ozon 或中文/俄文。
|
||||
|
||||
当前已完成 pnpm monorepo 初始化：包含 Web、NestJS API、BullMQ Worker、WXT 浏览器扩展、共享 API 契约、数据库包、本地基础设施 Compose 与依赖锁文件。API 已实现健康/readiness、集中配置与安全基础、认证、个人工作空间上下文、平台目录/上下文、门店、水印/素材、采集箱与扩展采集入库；Worker 已具备 Ozon 凭据验证、采集记录清理和来源素材转存消费链路，仓库同步尚未接入。Web 已具备完整前端框架（布局、路由、主题）、登录/注册页、页头平台选择器、店铺管理、水印管理、真实采集箱页面和采集数据测试页；浏览器扩展已提供毛子 ERP 风格的网页登录态同步、悬浮窗及 Ozon/1688/淘宝/天猫首版页面状态解析。不要从旧项目复制 SQLite 模型、无鉴权 API、旧 `/trial/:id` 或商品编辑页组件。可选择性复用已经验证过的纯计价公式、采集扩展思路和 AI 任务经验，但必须先适配多租户和云端边界。
|
||||
当前已完成 pnpm monorepo 初始化：包含 Web、NestJS API、BullMQ Worker、WXT 浏览器扩展、共享 API 契约、数据库包、本地基础设施 Compose 与依赖锁文件。API 已实现健康/readiness、集中配置与安全基础、认证、个人工作空间上下文、平台目录/上下文、门店、水印/素材、采集箱与扩展采集入库；Worker 已具备 Ozon 凭据验证、采集记录清理和按需来源素材转存消费链路，仓库同步尚未接入。Web 已具备完整前端框架（布局、路由、主题）、登录/注册页、页头平台选择器、店铺管理、水印管理、真实采集箱页面和采集数据测试页；浏览器扩展已提供毛子 ERP 风格的网页登录态同步、悬浮窗及 Ozon/1688/淘宝/天猫首版页面状态解析。不要从旧项目复制 SQLite 模型、无鉴权 API、旧 `/trial/:id` 或商品编辑页组件。可选择性复用已经验证过的纯计价公式、采集扩展思路和 AI 任务经验，但必须先适配多租户和云端边界。
|
||||
|
||||
当前运行时基线：Node 24.20.0、pnpm 11.25.0、PostgreSQL 18.6、Redis 7.4.11、MinIO `RELEASE.2025-09-07T16-13-09Z`；Web 默认端口为 `8100`，API 默认端口为 `8200`，扩展 dev server 默认端口为 `8110`（仅本地开发用），Worker 无 HTTP 端口。完整目录、依赖边界与版本表见 [`architecture/project-structure.md`](../architecture/project-structure.md) 和 [`architecture/technology-and-deployment.md`](../architecture/technology-and-deployment.md)。
|
||||
|
||||
@@ -127,6 +127,40 @@
|
||||
- **兼容性提醒：** 已有采集记录不会自动重新解析历史 `normalized_payload`。例如上面的现有记录仍标记为 `ozon-public-2.1.0`，只能验证新素材预览；需重新在来源商品页执行“编辑上架”，系统会覆盖同一采集快照，届时在规范化载荷查看新版属性与描述。
|
||||
- 已通过扩展 `wxt prepare + tsc + build`、Web `vite build`、`tsx --test test/*.test.mts`（5 项，包括 Ozon 页面快照中“类型/原产国/材料”等 5 个特征和富文本描述样本）以及浏览器测试页实测。
|
||||
|
||||
### 1.14 开发进度（2026-09-08）：重采集素材任务外键修复
|
||||
|
||||
- 修复重复“编辑上架”覆盖来源快照时 `collection_asset_transfers.workspace_id` 为空导致的数据库非空约束错误。根因不在新任务插入：旧的复合外键 `(workspace_id, source_variant_id) ... ON DELETE SET NULL` 在删除旧 SKU 时，会按 PostgreSQL 规则同时清空两个引用列，误将不可为空的 `workspace_id` 设为 `NULL`。
|
||||
- 覆盖流程现在先将历史素材任务的可空 `source_variant_id` 显式置空，再删除旧 SKU。迁移 `014_collection_asset_variant_fk.sql` 同时把外键改为 `ON DELETE SET NULL (source_variant_id)`，从 schema 层杜绝同类删除操作再次触发该错误。已在本地数据库实际执行并核验最终约束定义。
|
||||
- 该错误发生在同一数据库事务内，因此报错的旧请求会完整回滚，不会留下半成品变体、转存任务或 Outbox 事件。扩展可在 API 开发进程热重载后直接再次点击“编辑上架”。
|
||||
- 已通过 API TypeScript 检查、`apps/api/test/extension.e2e.test.ts`（首次采集 → 同组 SKU 重采集 → 快照覆盖）和新增的转存任务 `workspace_id` 断言。
|
||||
|
||||
### 1.15 已确认设计（2026-09-08）：采集图片按需物化
|
||||
|
||||
- **新规则取代“采集即转存”：** 采集阶段仅保存候选来源 URL、图片角色、SKU/规格关联、排序与证据；测试页直接加载来源 URL，以最快速度核对且不产生对象存储和下载费用。`media_asset` 只代表 Atlas 已验证、可稳定编辑与发布的正式素材。
|
||||
- 用户选图进入编辑时建议转存；加水印可在单次任务中临时下载来源图、仅存水印结果（来源失效后不可重试）；AI 改图等需要图片字节，通常先转存或在受控任务中临时下载；AI 成功输出和正式发布图片必须是 Atlas 控制的 `media_asset`。不得把来源 URL 直接提交到目标平台，也不得为下载图片读取来源 Cookie 或绕过防盗链。
|
||||
- 首期**不做自动删除已选择但未发布素材**。该能力涉及引用、编辑草稿、处理中任务、恢复和误删风险，当前不值得引入。以后如需控成本，只能先考虑清理从未被任何商品、派生任务或发布快照引用的素材，并单独立项。
|
||||
- 完整决策、场景表、AI/水印边界与按需物化要求见 [`media-storage-strategy.md`](../architecture/media-storage-strategy.md)。后续采集、编辑、AI、水印和发布任务开始前必须先读此文；按需物化已通过 `pending_selection` 候选状态和 `POST /api/collection-items/{id}/asset-materializations` 落地，正式商品编辑页尚未接入选图动作。
|
||||
|
||||
### 1.16 开发进度（2026-09-08）：采集图片按需转存与来源预览
|
||||
|
||||
- 已新增迁移 `015_collection_asset_pending_selection.sql`、`016_collection_asset_pending_selection_default.sql`：为素材处理枚举增加 `pending_selection`，并将新候选的默认状态改为该值。枚举变更和使用它的默认值/索引刻意拆为两个迁移事务，避免 PostgreSQL 新枚举值在同一事务中立即使用的限制。
|
||||
- `POST /api/extension/materials` 现在只保存来源图片候选，绝不自动创建 `collection.assets.materialize.requested` Outbox 事件或下载图片；快照覆盖会取消旧快照中 `pending_selection`、`queued`、`running` 的候选/任务。返回的 `materialization` 新增 `pending_selection` 计数。
|
||||
- 已新增受 `Idempotency-Key` 保护的 `POST /api/collection-items/{id}/asset-materializations`：只能选当前快照的 `pending_selection` 候选，转为 `queued` 后创建 operation 并投递现有来源素材 Worker。商品编辑、AI、水印和发布后续均复用此入口；采集测试页不会调用它。
|
||||
- `/collection-test/{collectionItemId}` 现在直接显示候选的来源 URL；已转存素材仍通过短时受控 URL 预览。页面明确区分“候选，未转存”“等待转存”“转存中”“已转存”等状态，方便核对而无需等待或消耗存储。
|
||||
|
||||
### 1.17 开发进度（2026-09-08）：采集候选视频（首期）
|
||||
|
||||
- 已新增并在本地开发数据库执行 `017_collection_video_candidates.sql`：`collection_asset_transfers` 新增 `media_type`（`image` / `video`）和可选 `poster_url`。共享采集契约的素材项新增 `mediaType`（默认为 `image`）及 `posterUrl`；检查读模型和测试页可区分媒体类型。
|
||||
- Ozon 仅从固定 `webGallery` / `webVideo` 状态读取视频；1688 仅从旧采集器验证过的 `gallery.fields.video|videos` 状态读取；淘宝、天猫仅从旧采集器验证过的 `item.videos` 状态读取。三者都不扫描页面 `video` 标签，因此不会将评价、推荐或广告视频误采为商品视频。
|
||||
- 首期候选仅接受 HTTPS 的直接 MP4/WebM URL，并保留可用封面 URL。`blob:`、HLS (`m3u8`)、DRM 与未知格式不采集；Worker 按需转存时以 MP4/WebM 文件魔数复核，图片仍限制 20MB，视频限制 200MB。暂不实现视频转码、封面抽帧、时长/尺寸解析、视频水印或发布。
|
||||
- `/collection-test/{collectionItemId}` 对未转存视频显示 `preload=none` 原生播放器，对已转存视频使用 Atlas 短时预览 URL；与图片一样，采集本身不下载也不产生存储成本。完整规则见 [`media-storage-strategy.md`](../architecture/media-storage-strategy.md)。
|
||||
|
||||
### 1.18 开发进度（2026-09-08）：旧采集器结构化路径迁移
|
||||
|
||||
- Ozon 解析器已对齐旧采集器的结构化合并方案：`webDescription/webRichContent` 富文本优先，`schema.org Product.description` 为兜底；首屏 `webShortCharacteristics`、默认商品页和详情页返回的 `webCharacteristics/webDetailedCharacteristics` 都会合并去重。`ozon-public-2.3.0` 是对应版本；旧快照必须重新采集。
|
||||
- 1688、淘宝、天猫已加入受限 MAIN-world 商品状态桥：1688 仅投影公开商品的画廊、SKU、价格、参数等固定字段；淘宝/天猫仅投影 `__ICE_APP_CONTEXT__` 中的公开商品结果。桥不提供任意 window 读取、Cookie、请求头或通用 fetch。新采集器将这些结构化数据、JSON-LD、受限详情 DOM 与既有图片 profile 映射为 Atlas 新 `CollectedProductPayload`。
|
||||
- `/collection-test/{collectionItemId}` 现单独展示商品描述、类目路径、全部特征表、规格轴、SKU 草稿、素材候选，并保留规范化载荷和原始证据，供解析核对。扩展解析器测试覆盖 JSON-LD 描述兜底、完整特征合并和图片白名单。
|
||||
|
||||
## 2. 已确认产品决策
|
||||
|
||||
| 主题 | 决策 |
|
||||
@@ -204,6 +238,7 @@ User → Workspace → Membership / Role
|
||||
- [商品领域数据模型与 DDL](../architecture/product-domain-data-model.md)
|
||||
- [幂等与 Transactional Outbox](../architecture/transactional-outbox-and-idempotency.md)
|
||||
- [数据库角色与 RLS](../architecture/database-security.md)
|
||||
- [商品采集图片与存储策略](../architecture/media-storage-strategy.md)
|
||||
- [实现前待决与待落地事项](../reviews/implementation-readiness-open-items.md)
|
||||
- [浏览器采集插件需求与技术方案](../requirements/collector-browser-extension.md)
|
||||
- [M2.2 / M2.3 扩展采集交接](./m2.2-m2.3-extension-collection.md)
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
# M2.2 / M2.3：扩展授权、素材转存与 Ozon 公开页采集
|
||||
# M2.2 / M2.3：扩展授权、素材候选与 Ozon 公开页采集
|
||||
|
||||
> 状态：已完成首个可运行闭环（2026-09-05）。Seller 网页四接口观察实验不属于本批次，仍待独立任务实施。
|
||||
|
||||
@@ -7,14 +7,14 @@
|
||||
1. 扩展按毛子 ERP 风格复用 Atlas Web 登录态：点击登录会打开 Atlas Web；Web 登录/恢复后把当前短期 access token 写入本站 localStorage，扩展每 2 秒从该 Atlas 标签读取并保存到 extension storage。未登录时用户在该页完成登录，标签不会被扩展自动关闭。
|
||||
2. 扩展以同步到的 Web Bearer token 调用会话检查和采集接口；服务端同时验证 JWT 与未撤销的 Web session，并从当前用户成员关系解析工作空间。Token 失效时插件清除缓存并重新要求从 Atlas Web 登录同步。
|
||||
3. 扩展上报仅携带来源、规范化 SKU/规格轴、候选素材 URL 和证据；`POST /api/extension/materials` 从 Web Bearer 身份解析工作空间，不接受目标平台、门店、Cookie 或 API Key。非 Ozon 来源按 item key 复用；Ozon 额外按已确认 `source_sku` 集合相交复用同一采集记录。
|
||||
4. 上报事务创建来源采集记录、SPU、来源确认 SKU 和素材转存任务；同一来源商品再次采集时保留记录与商品 ID，并用本次来源快照替换来源字段、SKU 与候选素材。素材异步事件经 Transactional Outbox 进入 Worker，任务带快照版本；Worker 仅处理当前版本，旧任务不得回写。Worker 仅下载 HTTPS 公网地址，限制重定向、私网地址、20MB 和 PNG/JPEG/WebP，按 SHA-256 去重后写入对象存储。单张失败不会使商品草稿丢失。
|
||||
4. 上报事务创建来源采集记录、SPU、来源确认 SKU 和媒体候选（`pending_selection`）；同一来源商品再次采集时保留记录与商品 ID，并用本次来源快照替换来源字段、SKU 与候选素材。采集本身不下载媒体也不写素材 Outbox。编辑流程选中候选后，`POST /api/collection-items/{id}/asset-materializations` 才将指定候选置为 `queued` 并经 Transactional Outbox 进入 Worker；任务带快照版本，Worker 仅处理当前版本，旧任务不得回写。Worker 仅下载 HTTPS 公网地址，限制重定向、私网地址；图片限制 20MB 和 PNG/JPEG/WebP，视频限制 200MB 和 MP4/WebM，均按 SHA-256 去重后写入对象存储。单张失败不会使商品草稿丢失。
|
||||
5. Ozon (`ru/kz/by`) 商品详情页显示 Shadow DOM 小浮窗。未登录只显示“登录 Atlas ERP”；已登录显示“编辑上架 / 采集商品 / 进入 ERP”。商品编辑模块尚未完成时，“编辑上架”成功后打开 `/collection-test/{collectionItemId}` 供审查采集结果；采集商品只提示进入采集箱。
|
||||
6. Ozon 解析器保留当前公开 SKU 作为入口证据，优先解析 `state-webAspects-*.data-state.aspects[].variants[]` 的真实变体；不会扫描整页 SKU 或把规格轴做笛卡尔积。来源未给可确认组合时，仍保留当前 SKU。
|
||||
7. Ozon 多 SKU 补全通过受限 MAIN world bridge 调用同源公开页面 JSON：先读取 `modal/aspectsNew` 的 `webAspectsModal`，再以并发 3、上限 100 的队列补全 `/product/<sku>/` 中的标题、价格、画廊和关联 SKU。每个已补全 SKU 的画廊图片使用 `variant_direct` 关联；单个请求失败保留已发现 SKU 与失败证据。
|
||||
8. Ozon 公开响应没有可依赖 SPU/组 ID。服务端在同一来源平台下按新旧 `source_sku` 集合相交覆盖快照；若命中多条历史记录返回 `SOURCE_VARIANT_GROUP_AMBIGUOUS`，不静默合并。
|
||||
9. 图片采集沿用旧采集插件已验证的分组 profile：1688、淘宝、天猫只从主图画廊、规格选择器和商品详情容器读取图片，绝不扫描全页 `document.images`。淘宝/天猫详情图排除评论和推荐卡片；URL 会在入库前还原 CDN 原图并按归一化 URL 去重。
|
||||
10. Ozon 只从 `webGallery` 读取商品画廊，`coverImage` 与 `images[].src` 归为该真实 SKU 的 `source_main + variant_direct`，不会将第二张及以后的画廊图误标为 SKU 图；`webAspects.data.coverImage` 作为 `source_sku + axis_value`。受限 bridge 额外读取固定的富文本页段来获取 `webDescription` / `webRichContent` / `webFeatures` 内的详情图，仍不接触评价或推荐组件。
|
||||
11. 采集数据测试页会为转存成功且已生成 `asset_id` 的素材按 4 路并发申请短时预览 URL，直接显示缩略图，并展示其关联来源 SKU；预览 URL 不保存到业务数据。Ozon 解析器 `ozon-public-2.2.0` 现额外读取短/完整 `web*Characteristics.characteristics[]` 为公共属性，并将 `webDescription.richAnnotationJson` 转为公共描述文本、原 JSON 放入 `raw_value`。该桥接仍只允许固定 Ozon 页面段和 widget 白名单；已有旧版载荷必须重采集才会拥有这些字段。
|
||||
11. 采集数据测试页对未转存候选直接加载 `source_url`；图片展示缩略图，视频采用不预加载的原生播放器，最快核对且不产生对象存储/Worker 下载费用；已经生成 `asset_id` 的历史素材才按 4 路并发申请短时预览 URL。页面会展示关联来源 SKU；预览 URL 不保存到业务数据。Ozon 解析器 `ozon-public-2.3.0` 现额外读取短/完整 `web*Characteristics.characteristics[]` 为公共属性，并将 `webDescription.richAnnotationJson` 转为公共描述文本、原 JSON 放入 `raw_value`。该桥接仍只允许固定 Ozon 页面段和 widget 白名单；已有旧版载荷必须重采集才会拥有这些字段。
|
||||
|
||||
## 配置与安全前提
|
||||
|
||||
|
||||
@@ -18,9 +18,12 @@ export const listCollectionItemsQuerySchema = z.object({
|
||||
export const bulkDeleteCollectionItemsRequestSchema = z.object({ ids: z.array(uuid).min(1).max(100) }).strict();
|
||||
export const clearCollectionItemsRequestSchema = z.object({ confirm: z.literal(true), expected_active_count: z.number().int().min(0) }).strict();
|
||||
export const retryCollectionAssetsRequestSchema = z.object({ transfer_ids: z.array(uuid).min(1).max(100) }).strict();
|
||||
/** 将已采集的来源图片候选按需转存为 Atlas 正式素材。 */
|
||||
export const materializeCollectionAssetsRequestSchema = z.object({ transfer_ids: z.array(uuid).min(1).max(100) }).strict();
|
||||
export type CreateCollectionItemRequest = z.infer<typeof createCollectionItemRequestSchema>;
|
||||
export type UpdateCollectionItemRequest = z.infer<typeof updateCollectionItemRequestSchema>;
|
||||
export type ListCollectionItemsQuery = z.infer<typeof listCollectionItemsQuerySchema>;
|
||||
export type MaterializeCollectionAssetsRequest = z.infer<typeof materializeCollectionAssetsRequestSchema>;
|
||||
|
||||
export type CollectionItem = {
|
||||
id: string; product_id: string; source: { kind: 'platform' | 'manual'; platform_code: string | null; url: string | null };
|
||||
@@ -40,7 +43,7 @@ export type CollectionItemInspection = CollectionItem & {
|
||||
parser_version: string | null;
|
||||
collected_at: string | null;
|
||||
variants: Array<{ id: string; source_variant_key: string | null; source_sku: string | null; display_name: string | null; sort_order: number }>;
|
||||
asset_transfers: Array<{ id: string; source_variant_id: string | null; role: string; source_url: string; status: string; asset_id: string | null; error_payload: unknown; created_at: string; completed_at: string | null }>;
|
||||
asset_transfers: Array<{ id: string; source_variant_id: string | null; media_type: 'image' | 'video'; poster_url: string | null; role: string; source_url: string; status: string; asset_id: string | null; error_payload: unknown; created_at: string; completed_at: string | null }>;
|
||||
};
|
||||
|
||||
/**
|
||||
@@ -70,7 +73,7 @@ export const collectedProductPayloadSchema = z.object({
|
||||
confidence: z.enum(['verified', 'partial']), evidenceRefs: z.array(z.string().min(1).max(300)).max(100),
|
||||
}).strict()).max(100),
|
||||
assets: z.array(z.object({
|
||||
url: z.url(), role: z.enum(['source_main', 'source_sku', 'source_detail']), variantSourceKey: z.string().min(1).max(500).optional(),
|
||||
url: z.url(), mediaType: z.enum(['image', 'video']).default('image'), posterUrl: z.url().optional(), role: z.enum(['source_main', 'source_sku', 'source_detail']), variantSourceKey: z.string().min(1).max(500).optional(),
|
||||
relation: z.enum(['variant_direct', 'axis_value', 'product_candidate', 'inherited_product_default']), sortOrder: z.number().int().min(0).max(10_000).default(0), refererOrigin: z.url().optional(), evidenceRef: z.string().max(300).optional(),
|
||||
}).strict()).max(500),
|
||||
evidence: z.array(z.object({ ref: z.string().min(1).max(300), kind: z.enum(['page', 'state', 'json_ld', 'dom']), path: z.string().max(1_000).optional(), raw_value: z.string().max(20_000).optional() }).strict()).max(500),
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
SET search_path TO atlas, public;
|
||||
|
||||
-- `workspace_id` is the tenant boundary and is never nullable. PostgreSQL's
|
||||
-- unqualified ON DELETE SET NULL on a composite FK clears every referencing
|
||||
-- column, including workspace_id. Only the optional variant pointer may be
|
||||
-- cleared when a re-collected source snapshot replaces its variants.
|
||||
ALTER TABLE collection_asset_transfers
|
||||
DROP CONSTRAINT collection_asset_transfers_workspace_id_source_variant_id_fkey;
|
||||
|
||||
ALTER TABLE collection_asset_transfers
|
||||
ADD CONSTRAINT collection_asset_transfers_workspace_id_source_variant_id_fkey
|
||||
FOREIGN KEY (workspace_id, source_variant_id)
|
||||
REFERENCES product_variants(workspace_id, id)
|
||||
ON DELETE SET NULL (source_variant_id);
|
||||
@@ -0,0 +1,5 @@
|
||||
SET search_path TO atlas, public;
|
||||
|
||||
-- Source-image candidates are recorded during collection but are not fetched
|
||||
-- until the user actually selects one for editing, processing or publishing.
|
||||
ALTER TYPE processing_status ADD VALUE IF NOT EXISTS 'pending_selection';
|
||||
@@ -0,0 +1,8 @@
|
||||
SET search_path TO atlas, public;
|
||||
|
||||
ALTER TABLE collection_asset_transfers
|
||||
ALTER COLUMN status SET DEFAULT 'pending_selection';
|
||||
|
||||
CREATE INDEX ix_collection_asset_transfers_pending_selection
|
||||
ON collection_asset_transfers (workspace_id, collection_item_id, source_snapshot_revision, created_at)
|
||||
WHERE status = 'pending_selection';
|
||||
@@ -0,0 +1,10 @@
|
||||
SET search_path TO atlas, public;
|
||||
|
||||
CREATE TYPE collection_media_type AS ENUM ('image', 'video');
|
||||
|
||||
ALTER TABLE collection_asset_transfers
|
||||
ADD COLUMN media_type collection_media_type NOT NULL DEFAULT 'image',
|
||||
ADD COLUMN poster_url TEXT;
|
||||
|
||||
CREATE INDEX collection_asset_transfers_media_status_idx
|
||||
ON collection_asset_transfers (workspace_id, media_type, status, created_at);
|
||||
File diff suppressed because one or more lines are too long
Reference in New Issue
Block a user