From ceb644a3f9ee4f75a0c7d12e24b4ecd3feaa5cc9 Mon Sep 17 00:00:00 2001 From: R524809 Date: Mon, 7 Sep 2026 17:51:11 +0800 Subject: [PATCH] Expand seller store functionality --- .env.example | 3 - Atlas Local - Migrator.session.sql | 38 + apps/api/src/collection/collection.service.ts | 8 +- apps/api/src/config/app-config.module.ts | 4 - .../api/src/extension/extension-auth.guard.ts | 28 +- .../api/src/extension/extension.controller.ts | 22 +- apps/api/src/extension/extension.service.ts | 239 +- apps/api/test/extension.e2e.test.ts | 32 +- apps/extension/.wxt/types/paths.d.ts | 1 + .../entrypoints/collector.content.ts | 164 +- .../entrypoints/ozon-page-json.content.ts | 105 + apps/extension/lib/api.ts | 84 +- apps/extension/lib/ozon-parser.ts | 1019 +- apps/extension/lib/product-media.ts | 336 + apps/extension/lib/source-parsers.ts | 449 +- apps/extension/package.json | 5 +- apps/extension/test/product-media.test.mts | 91 + apps/extension/wxt.config.ts | 9 +- .../collection/CollectionInspectionPage.tsx | 252 +- apps/web/src/services/authToken.ts | 9 +- apps/worker/src/collection-assets.ts | 20 +- docs/README.md | 9 +- docs/api/collection-and-extension.md | 35 +- ...ser-extension-background-service-worker.md | 73 + docs/project/HANDOFF.md | 38 +- .../development-collaboration-contract.md | 2 +- docs/project/extension-login-alternatives.md | 31 + .../project/m2.2-m2.3-extension-collection.md | 30 +- docs/requirements/account-store-watermark.md | 13 +- .../collector-browser-extension.md | 25 +- .../011_extension_link_requests.sql | 24 + .../012_remove_extension_link_requests.sql | 5 + ...013_collection_source_snapshot_refresh.sql | 14 + packages/reference/ozon1.html | 30027 ++++++++++++++++ pnpm-lock.yaml | 3 + 35 files changed, 32883 insertions(+), 364 deletions(-) create mode 100644 apps/extension/entrypoints/ozon-page-json.content.ts create mode 100644 apps/extension/lib/product-media.ts create mode 100644 apps/extension/test/product-media.test.mts create mode 100644 docs/guides/browser-extension-background-service-worker.md create mode 100644 docs/project/extension-login-alternatives.md create mode 100644 packages/database/migrations/011_extension_link_requests.sql create mode 100644 packages/database/migrations/012_remove_extension_link_requests.sql create mode 100644 packages/database/migrations/013_collection_source_snapshot_refresh.sql create mode 100644 packages/reference/ozon1.html diff --git a/.env.example b/.env.example index 8d4a8a0..36c6827 100644 --- a/.env.example +++ b/.env.example @@ -19,9 +19,6 @@ NODE_ENV=development # Browser origin allowed to send credentialed requests to this API. WEB_ORIGIN=http://localhost:8100 API_PUBLIC_ORIGIN=http://localhost:8200 -# Must exactly equal chrome.identity.getRedirectURL('callback') for the installed collector build. -EXTENSION_COLLECTOR_CLIENT_ID=atlas-collector -EXTENSION_COLLECTOR_REDIRECT_URI=https://atlas-collector.chromiumapp.org/callback # HTTP listener port for the NestJS API. API_PORT=8200 diff --git a/Atlas Local - Migrator.session.sql b/Atlas Local - Migrator.session.sql index e69de29..c29306c 100644 --- a/Atlas Local - Migrator.session.sql +++ b/Atlas Local - Migrator.session.sql @@ -0,0 +1,38 @@ +SELECT + u.id, + u.email, + u.display_name, + u.status, + u.created_at, + u.last_login_at, + w.id AS workspace_id, + w.name AS workspace_name, + m.role AS workspace_role +FROM atlas.users u +LEFT JOIN atlas.workspace_memberships m ON m.user_id = u.id +LEFT JOIN atlas.workspaces w ON w.id = m.workspace_id +ORDER BY u.created_at DESC; + +SELECT code, name, capabilities +FROM atlas.platforms +WHERE code = 'OZON'; + +BEGIN; + +SET search_path TO atlas, public; + +INSERT INTO platform_currency_support (platform_id, currency_code) +SELECT id, 'CNY'::CHAR(3) +FROM platforms +WHERE code = 'OZON' +ON CONFLICT (platform_id, currency_code) DO UPDATE +SET is_enabled = true; + +UPDATE platforms +SET capabilities = capabilities || jsonb_build_object( + 'default_currency_code', 'CNY' + ), + updated_at = now() +WHERE code = 'OZON'; + +COMMIT; \ No newline at end of file diff --git a/apps/api/src/collection/collection.service.ts b/apps/api/src/collection/collection.service.ts index 7ba758f..cb09168 100644 --- a/apps/api/src/collection/collection.service.ts +++ b/apps/api/src/collection/collection.service.ts @@ -35,9 +35,9 @@ export class CollectionService { async detail(workspaceId: string, id: string) { return this.database.workspaceTransaction(workspaceId, async (tx) => { const item = await this.requireItem(tx, id); - const source = await tx.query<{ normalized_payload: unknown; raw_evidence: unknown; parser_version: string | null; collected_at: Date | string | null }>('SELECT normalized_payload, raw_evidence, parser_version, collected_at FROM atlas.collection_items WHERE id = $1 AND deleted_at IS NULL', [id]); + const source = await tx.query<{ normalized_payload: unknown; raw_evidence: unknown; parser_version: string | null; collected_at: Date | string | null; source_snapshot_revision: string | number }>('SELECT normalized_payload, raw_evidence, parser_version, collected_at, source_snapshot_revision FROM atlas.collection_items WHERE id = $1 AND deleted_at IS NULL', [id]); const variants = await tx.query<{ id: string; source_variant_key: string | null; source_sku: string | null; display_name: string | null; sort_order: number }>('SELECT id, source_variant_key, source_sku, display_name, sort_order FROM atlas.product_variants WHERE product_id = $1 AND deleted_at IS NULL ORDER BY sort_order, id', [item.product_id]); - const transfers = await tx.query<{ id: string; source_variant_id: string | null; role: string; source_url: string; status: string; asset_id: string | null; error_payload: unknown; created_at: Date | string; completed_at: Date | string | null }>('SELECT id, source_variant_id, role::text AS role, source_url, status::text AS status, asset_id, error_payload, created_at, completed_at FROM atlas.collection_asset_transfers WHERE collection_item_id = $1 ORDER BY created_at, id', [id]); + const transfers = await tx.query<{ id: string; source_variant_id: string | null; role: string; source_url: string; status: string; asset_id: string | null; error_payload: unknown; created_at: Date | string; completed_at: Date | string | null }>('SELECT id, source_variant_id, role::text AS role, source_url, status::text AS status, asset_id, error_payload, created_at, completed_at FROM atlas.collection_asset_transfers WHERE collection_item_id = $1 AND source_snapshot_revision = $2 ORDER BY created_at, id', [id, source.rows[0]?.source_snapshot_revision ?? 1]); return { ...item, normalized_payload: source.rows[0]?.normalized_payload ?? {}, raw_evidence: source.rows[0]?.raw_evidence ?? {}, parser_version: source.rows[0]?.parser_version ?? null, collected_at: source.rows[0]?.collected_at ? iso(source.rows[0].collected_at) : null, variants: variants.rows, asset_transfers: transfers.rows.map((transfer) => ({ ...transfer, created_at: iso(transfer.created_at), completed_at: transfer.completed_at ? iso(transfer.completed_at) : null })) }; }); } @@ -50,7 +50,7 @@ export class CollectionService { } async bulkDelete(workspaceId: string, userId: string, key: string, ids: { ids: string[] }) { return this.removeMany(workspaceId, userId, key, '/api/collection-items/bulk-delete', ids, false); } async clear(workspaceId: string, userId: string, key: string, input: { confirm: true; expected_active_count: number }) { return this.removeMany(workspaceId, userId, key, '/api/collection-items/clear', input, true); } - async retryAssets(workspaceId: string, userId: string, id: string, key: string, input: { transfer_ids: string[] }) { return this.idempotency.run({ workspaceId, userId, method: 'POST', path: `/api/collection-items/${id}/asset-retries`, key, request: input }, async (tx) => { await this.requireItem(tx, id); const result = await tx.query(`UPDATE atlas.collection_asset_transfers SET status = 'queued', error_payload = NULL, updated_at = now() WHERE collection_item_id = $1 AND id = ANY($2::uuid[]) AND status IN ('failed', 'cancelled')`, [id, input.transfer_ids]); if (result.rowCount !== input.transfer_ids.length) throw new ApiError(422, 'ASSET_RETRY_INVALID', '只能重试当前采集记录中失败或已取消的素材任务'); const operation = await this.operation(tx, userId, 'collection_asset_retry', id); await this.event(tx, id, 'collection.assets.retry.requested', { operation_id: operation.id, collection_item_id: id }); return { status: 202, body: { data: { operation } } }; }); } + async retryAssets(workspaceId: string, userId: string, id: string, key: string, input: { transfer_ids: string[] }) { return this.idempotency.run({ workspaceId, userId, method: 'POST', path: `/api/collection-items/${id}/asset-retries`, key, request: input }, async (tx) => { await this.requireItem(tx, id); const result = await tx.query<{ source_snapshot_revision: string | number }>(`UPDATE atlas.collection_asset_transfers t SET status = 'queued', error_payload = NULL, updated_at = now() FROM atlas.collection_items c WHERE t.collection_item_id = c.id AND t.collection_item_id = $1 AND t.id = ANY($2::uuid[]) AND t.source_snapshot_revision = c.source_snapshot_revision AND t.status IN ('failed', 'cancelled') RETURNING c.source_snapshot_revision`, [id, input.transfer_ids]); if (result.rowCount !== input.transfer_ids.length) throw new ApiError(422, 'ASSET_RETRY_INVALID', '只能重试当前采集快照中失败或已取消的素材任务'); const operation = await this.operation(tx, userId, 'collection_asset_retry', id); await this.event(tx, id, 'collection.assets.retry.requested', { operation_id: operation.id, collection_item_id: id, source_snapshot_revision: Number(result.rows[0].source_snapshot_revision) }); return { status: 202, body: { data: { operation } } }; }); } private async removeMany(workspaceId: string, userId: string, key: string, path: string, input: unknown, clear: boolean) { return this.idempotency.run({ workspaceId, userId, method: 'POST', path, key, request: input }, async (tx) => { const requested = input as { ids?: string[]; expected_active_count?: number }; const count = await tx.query<{ count: string }>('SELECT count(*)::text AS count FROM atlas.collection_items WHERE deleted_at IS NULL'); if (clear && Number(count.rows[0].count) !== requested.expected_active_count) throw new ApiError(409, 'COLLECTION_COUNT_CHANGED', '采集箱数量已变化，请重新确认'); const deleted = await tx.query<{ id: string }>(clear ? `UPDATE atlas.collection_items SET status = 'deleted', deleted_at = now(), revision = revision + 1, updated_at = now() WHERE deleted_at IS NULL RETURNING id` : `UPDATE atlas.collection_items SET status = 'deleted', deleted_at = now(), revision = revision + 1, updated_at = now() WHERE id = ANY($1::uuid[]) AND deleted_at IS NULL RETURNING id`, clear ? [] : [requested.ids]); if (!clear && deleted.rows.length !== requested.ids?.length) throw new ApiError(404, 'RESOURCE_NOT_FOUND', '部分采集记录不存在'); const operation = await this.operation(tx, userId, 'collection_cleanup', deleted.rows[0]?.id ?? null); await this.event(tx, operation.id, 'collection.cleanup.requested', { operation_id: operation.id, collection_item_ids: deleted.rows.map((row) => row.id) }); return { status: 202, body: { data: { deleted_count: deleted.rows.length, operation } } }; }); } private async operation(tx: DatabaseTransaction, userId: string, kind: string, resourceId: string | null) { const row = await tx.query<{ id: string; status: string; created_at: Date | string }>(`INSERT INTO atlas.operations (workspace_id, kind, resource_type, resource_id, created_by_user_id) VALUES (current_setting('app.workspace_id')::uuid, $1, 'collection_item', $2, $3) RETURNING id, status, created_at`, [kind, resourceId, userId]); return { id: row.rows[0].id, kind, status: row.rows[0].status, created_at: iso(row.rows[0].created_at), completed_at: null, error: null }; } private async event(tx: DatabaseTransaction, aggregateId: string, type: string, payload: unknown) { await tx.query(`INSERT INTO atlas.outbox_events (workspace_id, aggregate_type, aggregate_id, event_type, event_payload, deduplication_key) VALUES (current_setting('app.workspace_id')::uuid, 'collection_item', $1, $2, $3, $4)`, [aggregateId, type, payload, `${type}:${aggregateId}`]); } @@ -58,7 +58,7 @@ export class CollectionService { private async requireItem(tx: DatabaseTransaction, id: string) { const row = await tx.query(`${selectRow} WHERE c.id = $1 AND c.deleted_at IS NULL ${collectionGroupBy}`, [id]); if (!row.rows[0]) throw new ApiError(404, 'RESOURCE_NOT_FOUND', '采集记录不存在'); return toItem(row.rows[0]); } private async versionConflict(tx: DatabaseTransaction, id: string, expected: number) { const row = await tx.query<{ revision: number }>('SELECT revision FROM atlas.collection_items WHERE id = $1 AND deleted_at IS NULL', [id]); if (!row.rows[0]) throw new ApiError(404, 'RESOURCE_NOT_FOUND', '采集记录不存在'); if (row.rows[0].revision !== expected) throw new ApiError(409, 'VERSION_CONFLICT', '采集记录已被更新', [{ field: 'If-Match', code: 'VERSION_CONFLICT', message: `当前版本为 ${row.rows[0].revision}`, current_revision: row.rows[0].revision }]); } } -const selectRow = `SELECT c.id, product.id AS product_id, c.source_kind, p.code AS platform_code, c.source_url, c.source_title, c.source_min_price::text, c.source_max_price::text, c.source_currency_code, c.note, c.status, c.created_at, c.updated_at, c.revision, count(DISTINCT v.id)::text AS variant_count, min(t.status) AS transfer_status FROM atlas.collection_items c JOIN atlas.products product ON product.collection_item_id = c.id LEFT JOIN atlas.platforms p ON p.id = c.source_platform_id LEFT JOIN atlas.product_variants v ON v.product_id = product.id AND v.deleted_at IS NULL LEFT JOIN atlas.collection_asset_transfers t ON t.collection_item_id = c.id`; +const selectRow = `SELECT c.id, product.id AS product_id, c.source_kind, p.code AS platform_code, c.source_url, c.source_title, c.source_min_price::text, c.source_max_price::text, c.source_currency_code, c.note, c.status, c.created_at, c.updated_at, c.revision, count(DISTINCT v.id)::text AS variant_count, min(t.status) AS transfer_status FROM atlas.collection_items c JOIN atlas.products product ON product.collection_item_id = c.id LEFT JOIN atlas.platforms p ON p.id = c.source_platform_id LEFT JOIN atlas.product_variants v ON v.product_id = product.id AND v.deleted_at IS NULL LEFT JOIN atlas.collection_asset_transfers t ON t.collection_item_id = c.id AND t.source_snapshot_revision = c.source_snapshot_revision`; const collectionGroupBy = 'GROUP BY c.id, product.id, p.code'; function toItem(row: Row) { return { id: row.id, product_id: row.product_id, source: { kind: row.source_kind, platform_code: row.platform_code, url: row.source_url }, title: row.source_title, thumbnail: { asset_id: null, preview_url: null, status: row.transfer_status ?? 'none' }, source_price: { min: row.source_min_price, max: row.source_max_price, currency_code: row.source_currency_code }, variant_summary: { count: Number(row.variant_count), needs_review: row.status === 'needs_review' }, note: row.note, collection_status: row.status, listing_summary: { status: 'not_listed' as const, detail: '等待首次上架', last_error: null }, created_at: iso(row.created_at), updated_at: iso(row.updated_at), revision: row.revision }; } function iso(value: Date | string) { return value instanceof Date ? value.toISOString() : value; } diff --git a/apps/api/src/config/app-config.module.ts b/apps/api/src/config/app-config.module.ts index 4619447..745a933 100644 --- a/apps/api/src/config/app-config.module.ts +++ b/apps/api/src/config/app-config.module.ts @@ -27,10 +27,6 @@ const environmentSchema = z.object({ API_PORT: z.coerce.number().int().min(1).max(65_535).default(8200), API_PUBLIC_ORIGIN: z.url().default('http://localhost:8200'), WEB_ORIGIN: z.url().default('http://localhost:8100'), - // The redirect URI is deliberately exact. A wildcard would allow another - // installed extension to exchange a code created from the user's Web session. - EXTENSION_COLLECTOR_CLIENT_ID: z.string().min(1).max(128).default('atlas-collector'), - EXTENSION_COLLECTOR_REDIRECT_URI: z.url().default('https://atlas-collector.chromiumapp.org/callback'), DATABASE_URL: z.url(), REDIS_URL: z.url(), OBJECT_STORAGE_ENDPOINT: z.url().default('http://localhost:9000'), diff --git a/apps/api/src/extension/extension-auth.guard.ts b/apps/api/src/extension/extension-auth.guard.ts index 586d4f9..b243a0a 100644 --- a/apps/api/src/extension/extension-auth.guard.ts +++ b/apps/api/src/extension/extension-auth.guard.ts @@ -1,5 +1,8 @@ -import { CanActivate, ExecutionContext, Inject, Injectable } from '@nestjs/common'; -import { ExtensionService } from './extension.service.js'; +import { CanActivate, ExecutionContext, Inject, Injectable, UnauthorizedException } from '@nestjs/common'; +import type { Database } from '@atlas/database'; +import { AuthService } from '../auth/auth.service.js'; +import { ApiError } from '../common/api-error.js'; +import { DATABASE } from '../database/database.module.js'; export type ExtensionAuthenticatedRequest = { header: (name: string) => string | undefined; @@ -8,10 +11,27 @@ export type ExtensionAuthenticatedRequest = { @Injectable() export class ExtensionAccessGuard implements CanActivate { - public constructor(@Inject(ExtensionService) private readonly extension: ExtensionService) {} + public constructor(@Inject(AuthService) private readonly auth: AuthService, @Inject(DATABASE) private readonly database: Database) {} public async canActivate(context: ExecutionContext) { const request = context.switchToHttp().getRequest(); - request.extensionAuth = await this.extension.authenticate(request.header('authorization')); + // 毛子 ERP 风格：插件携带从 Atlas Web 同步而来的短期 Web access token。 + // AuthService 同时核对 JWT 和未撤销的 Web session，不能仅信任 token 声明。 + let authenticated: { userId: string; sessionId: string }; + try { + authenticated = await this.auth.authenticate(request.header('authorization')); + } catch (error) { + // AuthService 的通用认证守卫会抛 Nest UnauthorizedException；扩展接口 + // 应返回稳定、可操作的错误信息，不能在 UI 中退化成“请求无法处理”。 + if (error instanceof UnauthorizedException) + throw new ApiError(401, 'SESSION_EXPIRED', 'Atlas ERP 登录已过期，请重新登录'); + throw error; + } + const workspace = await this.database.query<{ id: string }>(`SELECT w.id FROM atlas.workspace_memberships m + JOIN atlas.workspaces w ON w.id = m.workspace_id + WHERE m.user_id = $1 AND w.status = 'active' + ORDER BY m.joined_at LIMIT 1`, [authenticated.userId]); + if (!workspace.rows[0]) throw new ApiError(403, 'WORKSPACE_MEMBERSHIP_REQUIRED', '当前账户没有可用工作空间'); + request.extensionAuth = { userId: authenticated.userId, workspaceId: workspace.rows[0].id, sessionId: authenticated.sessionId, clientId: 'atlas-web' }; return true; } } diff --git a/apps/api/src/extension/extension.controller.ts b/apps/api/src/extension/extension.controller.ts index 6661ab6..1e8bff8 100644 --- a/apps/api/src/extension/extension.controller.ts +++ b/apps/api/src/extension/extension.controller.ts @@ -1,4 +1,4 @@ -import { Body, Controller, Get, Headers, Inject, Post, Query, Req, Res, UseGuards } from '@nestjs/common'; +import { Body, Controller, Get, Headers, Inject, Post, Req, Res, UseGuards } from '@nestjs/common'; import { z } from 'zod'; import { collectedProductPayloadSchema } from '@atlas/contracts'; import { ApiError } from '../common/api-error.js'; @@ -6,28 +6,11 @@ import { Public } from '../common/public.decorator.js'; import { ExtensionAccessGuard, type ExtensionAuthenticatedRequest } from './extension-auth.guard.js'; import { ExtensionService } from './extension.service.js'; -type RequestLike = { header: (name: string) => string | undefined }; -type ResponseLike = { redirect: (status: number, url: string) => void; status: (code: number) => ResponseLike }; -const authorizeSchema = z.object({ client_id: z.string().min(1).max(128), redirect_uri: z.url(), state: z.string().min(16).max(1_000), code_challenge: z.string().regex(/^[A-Za-z0-9_-]{43,128}$/), code_challenge_method: z.literal('S256') }).strict(); -const exchangeSchema = z.object({ client_id: z.string().min(1).max(128), code: z.string().min(32).max(1_000), redirect_uri: z.url(), code_verifier: z.string().min(43).max(128) }).strict(); -const refreshSchema = z.object({ refresh_token: z.string().min(32).max(1_000) }).strict(); +type ResponseLike = { status: (code: number) => ResponseLike }; @Controller('extension') export class ExtensionController { public constructor(@Inject(ExtensionService) private readonly extension: ExtensionService) {} - @Get('authorize') @Public() - public async authorize(@Query() query: unknown, @Req() request: RequestLike, @Res() response: ResponseLike) { - const result = await this.extension.authorize(parse(authorizeSchema, query) as z.infer, cookie(request.header('cookie'), 'atlas_refresh')); - const target = 'redirect_url' in result ? result.redirect_url : result.login_url; - if (!target) throw new ApiError(500, 'EXTENSION_AUTHORIZATION_REDIRECT_MISSING', '扩展授权跳转地址缺失'); - response.redirect(302, target); - } - @Post('auth/exchange') @Public() - public async exchange(@Body() body: unknown, @Res({ passthrough: true }) response: ResponseLike) { const result = await this.extension.exchange(parse(exchangeSchema, body) as z.infer); response.status(result.status); return result.body; } - @Post('auth/refresh') @Public() - public async refresh(@Body() body: unknown, @Res({ passthrough: true }) response: ResponseLike) { const input = parse(refreshSchema, body) as z.infer; const result = await this.extension.refresh(input.refresh_token); response.status(result.status); return result.body; } - @Post('auth/revoke') @Public() @UseGuards(ExtensionAccessGuard) - public async revoke(@Req() request: ExtensionAuthenticatedRequest, @Res({ passthrough: true }) response: ResponseLike) { await this.extension.revoke(request.extensionAuth.workspaceId, request.extensionAuth.sessionId); response.status(204); } @Get('session') @Public() @UseGuards(ExtensionAccessGuard) public session(@Req() request: ExtensionAuthenticatedRequest) { return { data: { authenticated: true, workspace: { id: request.extensionAuth.workspaceId, available: true }, can_collect: true } }; } @Post('materials') @Public() @UseGuards(ExtensionAccessGuard) @@ -37,4 +20,3 @@ export class ExtensionController { } } function parse { success: boolean; data?: unknown; error?: { issues: Array<{ path: PropertyKey[]; message: string }> } } }>(schema: T, value: unknown) { const result = schema.safeParse(value); if (result.success) return result.data as never; throw new ApiError(422, 'VALIDATION_FAILED', '请求参数不符合要求', result.error?.issues.map((item) => ({ field: item.path.join('.'), code: 'INVALID_FIELD', message: item.message }))); } -function cookie(header: string | undefined, name: string) { return header?.split(';').map((part) => part.trim()).find((part) => part.startsWith(`${name}=`))?.slice(name.length + 1); } diff --git a/apps/api/src/extension/extension.service.ts b/apps/api/src/extension/extension.service.ts index 913c8d3..607ca99 100644 --- a/apps/api/src/extension/extension.service.ts +++ b/apps/api/src/extension/extension.service.ts @@ -1,140 +1,163 @@ -import { createHash, randomUUID } from 'node:crypto'; import { Inject, Injectable } from '@nestjs/common'; import type { CollectedProductPayload } from '@atlas/contracts'; import type { Database } from '@atlas/database'; -import { createOpaqueToken, signExtensionAccessToken, tokenHash, verifyExtensionAccessToken } from '../auth/auth.crypto.js'; -import { AuthService } from '../auth/auth.service.js'; import { ApiError } from '../common/api-error.js'; import { IdempotencyService } from '../common/idempotency.service.js'; -import { APP_CONFIG, type AppConfig } from '../config/app-config.module.js'; import { DATABASE } from '../database/database.module.js'; -type AuthorizeInput = { client_id: string; redirect_uri: string; state: string; code_challenge: string; code_challenge_method: 'S256' }; -type ExchangeInput = { client_id: string; code: string; redirect_uri: string; code_verifier: string }; -type ExtensionSession = { id: string; workspace_id: string; user_id: string; client_id: string; expires_at: Date | string; revoked_at: Date | string | null }; - @Injectable() export class ExtensionService { public constructor( @Inject(DATABASE) private readonly database: Database, - @Inject(AuthService) private readonly auth: AuthService, @Inject(IdempotencyService) private readonly idempotency: IdempotencyService, - @Inject(APP_CONFIG) private readonly config: AppConfig, ) {} - public async authorize(input: AuthorizeInput, refreshToken: string | undefined) { - this.assertClient(input.client_id, input.redirect_uri); - const session = await this.auth.resolveWebRefreshSession(refreshToken); - if (!session) return { login_url: `${this.config.WEB_ORIGIN}/login?continue=${encodeURIComponent(this.authorizationUrl(input))}` }; - // Prefixing a high-entropy secret with the workspace UUID lets the server - // establish RLS before looking up the code; it is not an authorization secret. - const rawCode = `${session.workspace.id}.${createOpaqueToken()}`; - await this.database.workspaceTransaction(session.workspace.id, async (tx) => { - await tx.query(`INSERT INTO atlas.extension_authorization_codes - (workspace_id, user_id, client_id, redirect_uri, code_hash, code_challenge, expires_at) - VALUES (current_setting('app.workspace_id')::uuid, $1, $2, $3, $4, $5, now() + interval '60 seconds')`, - [session.userId, input.client_id, input.redirect_uri, tokenHash(rawCode), input.code_challenge]); - }); - const callback = new URL(input.redirect_uri); - callback.searchParams.set('code', rawCode); - callback.searchParams.set('state', input.state); - return { redirect_url: callback.toString() }; - } - - public async exchange(input: ExchangeInput) { - this.assertClient(input.client_id, input.redirect_uri); - const hash = tokenHash(input.code); const workspaceId = workspacePrefix(input.code); - if (!workspaceId) throw new ApiError(401, 'EXTENSION_AUTHORIZATION_INVALID', '扩展授权码无效或已过期'); - const code = await this.database.workspaceTransaction(workspaceId, async (tx) => (await tx.query<{ workspace_id: string; user_id: string; client_id: string; redirect_uri: string; code_challenge: string }>( - `SELECT workspace_id, user_id, client_id, redirect_uri, code_challenge FROM atlas.extension_authorization_codes WHERE code_hash = $1 AND consumed_at IS NULL AND expires_at > now()`, [hash])).rows[0]); - if (!code || code.client_id !== input.client_id || code.redirect_uri !== input.redirect_uri || !constantEquals(code.code_challenge, pkceChallenge(input.code_verifier))) { - throw new ApiError(401, 'EXTENSION_AUTHORIZATION_INVALID', '扩展授权码无效或已过期'); - } - return this.database.workspaceTransaction(code.workspace_id, async (tx) => { - const consumed = await tx.query(`UPDATE atlas.extension_authorization_codes SET consumed_at = now() - WHERE code_hash = $1 AND consumed_at IS NULL AND expires_at > now()`, [hash]); - if (!consumed.rowCount) throw new ApiError(401, 'EXTENSION_AUTHORIZATION_INVALID', '扩展授权码无效或已过期'); - return this.createSession(tx, code.workspace_id, code.user_id, code.client_id); - }); - } - - public async refresh(refreshToken: string) { - const workspaceId = workspacePrefix(refreshToken); if (!workspaceId) throw new ApiError(401, 'EXTENSION_SESSION_EXPIRED', '扩展登录已过期'); - return this.database.workspaceTransaction(workspaceId, async (tx) => { - const result = await tx.query(`SELECT id, workspace_id, user_id, client_id, expires_at, revoked_at FROM atlas.extension_sessions WHERE refresh_token_hash = $1 AND revoked_at IS NULL AND expires_at > now() FOR UPDATE`, [tokenHash(refreshToken)]); - const old = result.rows[0]; - if (!old) throw new ApiError(401, 'EXTENSION_SESSION_EXPIRED', '扩展登录已过期'); - await tx.query('UPDATE atlas.extension_sessions SET revoked_at = now(), last_used_at = now() WHERE id = $1', [old.id]); - return this.createSession(tx, old.workspace_id, old.user_id, old.client_id); - }); - } - - public async revoke(workspaceId: string, sessionId: string) { - await this.database.workspaceTransaction(workspaceId, (tx) => tx.query('UPDATE atlas.extension_sessions SET revoked_at = now() WHERE id = $1 AND revoked_at IS NULL', [sessionId])); - } - - public async authenticate(header: string | undefined) { - const token = header?.replace(/^Bearer\s+/i, ''); - const claims = token ? verifyExtensionAccessToken(token, this.config.AUTH_JWT_SECRET) : null; - if (!claims) throw new ApiError(401, 'EXTENSION_SESSION_EXPIRED', '扩展登录已过期'); - const session = await this.database.workspaceTransaction(claims.wid, async (tx) => (await tx.query(`SELECT id, workspace_id, user_id, client_id, expires_at, revoked_at FROM atlas.extension_sessions WHERE id = $1 AND user_id = $2 AND revoked_at IS NULL AND expires_at > now()`, [claims.sid, claims.sub])).rows[0]); - if (!session) throw new ApiError(401, 'EXTENSION_SESSION_EXPIRED', '扩展登录已过期'); - return { userId: session.user_id, workspaceId: session.workspace_id, sessionId: session.id, clientId: session.client_id }; - } - public async ingest(workspaceId: string, userId: string, key: string, payload: CollectedProductPayload) { return this.idempotency.run({ workspaceId, userId, method: 'POST', path: '/api/extension/materials', key, request: payload }, async (tx) => { const platform = await tx.query<{ id: string; code: string }>(`SELECT id, code FROM atlas.platforms WHERE code = $1 AND status = 'active' AND coalesce((capabilities->>'can_be_source')::boolean, false)`, [payload.source.platform.toUpperCase()]); if (!platform.rows[0]) throw new ApiError(422, 'SOURCE_PLATFORM_UNAVAILABLE', '该来源平台当前不可采集'); - const existing = await tx.query<{ id: string; product_id: string }>(`SELECT c.id, p.id AS product_id FROM atlas.collection_items c JOIN atlas.products p ON p.collection_item_id = c.id - WHERE c.source_platform_id = $1 AND c.source_item_key = $2 AND c.deleted_at IS NULL`, [platform.rows[0].id, payload.source.itemKey]); - if (existing.rows[0]) return { status: 200, body: { data: await this.materialResponse(tx, existing.rows[0].id, existing.rows[0].product_id, true) } }; - const verifiedVariants = payload.variants.filter((variant) => variant.sourceVariantKey.trim().length > 0); - const status = verifiedVariants.length || !payload.variantAxes.length ? 'materializing' : 'needs_review'; - const sourcePrice = payload.common.sourcePrice; + const existing = await this.findExistingSourceItem( + tx, + workspaceId, + platform.rows[0].id, + payload, + ); + if (existing) { + await this.replaceSourceSnapshot( + tx, + existing.id, + existing.product_id, + payload, + ); + return { status: 200, body: { data: await this.materialResponse(tx, existing.id, existing.product_id, true, true) } }; + } + const { status, sourcePrice } = sourceState(payload); const item = await tx.query<{ id: string }>(`INSERT INTO atlas.collection_items (workspace_id, source_kind, source_platform_id, source_item_key, source_url, source_title, source_currency_code, source_min_price, source_max_price, status, raw_evidence, normalized_payload, parser_version, collected_at) VALUES (current_setting('app.workspace_id')::uuid, 'platform', $1, $2, $3, $4, $5, $6, $6, $7::atlas.collection_item_status, $8, $9, $10, $11) RETURNING id`, [platform.rows[0].id, payload.source.itemKey, payload.source.canonicalUrl, payload.common.title?.value ?? null, sourcePrice?.currencyCode ?? null, sourcePrice?.amount ?? null, status, JSON.stringify(payload.evidence), JSON.stringify(payload), payload.source.parserVersion, payload.source.collectedAt]); const product = await tx.query<{ id: string }>(`INSERT INTO atlas.products (workspace_id, collection_item_id, source_title) VALUES (current_setting('app.workspace_id')::uuid, $1, $2) RETURNING id`, [item.rows[0].id, payload.common.title?.value ?? null]); - const variantRows = verifiedVariants.length ? verifiedVariants : [{ sourceVariantKey: null, sourceSku: undefined, displayName: '默认 SKU' }]; - const variants = new Map(); - for (let index = 0; index < variantRows.length; index += 1) { - const variant = variantRows[index]; - const inserted = await tx.query<{ id: string }>(`INSERT INTO atlas.product_variants (workspace_id, product_id, source_variant_key, source_sku, display_name, sort_order) - VALUES (current_setting('app.workspace_id')::uuid, $1, $2, $3, $4, $5) RETURNING id`, [product.rows[0].id, variant.sourceVariantKey, variant.sourceSku ?? null, variant.displayName, index]); - if (variant.sourceVariantKey) variants.set(variant.sourceVariantKey, inserted.rows[0].id); - } - for (const asset of payload.assets) { - await tx.query(`INSERT INTO atlas.collection_asset_transfers (workspace_id, collection_item_id, product_id, source_variant_id, role, source_url, status, error_payload) - VALUES (current_setting('app.workspace_id')::uuid, $1, $2, $3, $4::atlas.media_candidate_role, $5, 'queued', $6)`, - [item.rows[0].id, product.rows[0].id, asset.variantSourceKey ? variants.get(asset.variantSourceKey) ?? null : null, asset.role, asset.url, JSON.stringify({ relation: asset.relation, referer_origin: asset.refererOrigin ?? null, evidence_ref: asset.evidenceRef ?? null, sort_order: asset.sortOrder })]); - } - if (payload.assets.length) await tx.query(`INSERT INTO atlas.outbox_events (workspace_id, aggregate_type, aggregate_id, event_type, event_payload, deduplication_key) - VALUES (current_setting('app.workspace_id')::uuid, 'collection_item', $1, 'collection.assets.materialize.requested', jsonb_build_object('collection_item_id', $1), $2)`, [item.rows[0].id, `collection.assets.materialize:${item.rows[0].id}`]); - else await tx.query(`UPDATE atlas.collection_items SET status = CASE WHEN status = 'needs_review' THEN status ELSE 'ready_for_edit' END, updated_at = now() WHERE id = $1`, [item.rows[0].id]); - return { status: 201, body: { data: await this.materialResponse(tx, item.rows[0].id, product.rows[0].id, false) } }; + await this.writeSourceSnapshotRows(tx, item.rows[0].id, product.rows[0].id, 1, payload); + return { status: 201, body: { data: await this.materialResponse(tx, item.rows[0].id, product.rows[0].id, false, false) } }; }); } - private async materialResponse(tx: { query: Database['query'] }, collectionItemId: string, productId: string, reused: boolean) { - const count = await tx.query<{ total: string; queued: string; succeeded: string; failed: string }>(`SELECT count(*)::text AS total, count(*) FILTER (WHERE status = 'queued')::text AS queued, count(*) FILTER (WHERE status = 'succeeded')::text AS succeeded, count(*) FILTER (WHERE status = 'failed')::text AS failed FROM atlas.collection_asset_transfers WHERE collection_item_id = $1`, [collectionItemId]); - const status = await tx.query<{ status: string; source_title: string | null }>('SELECT status, source_title FROM atlas.collection_items WHERE id = $1', [collectionItemId]); - return { collection_item: { id: collectionItemId, status: status.rows[0]?.status ?? 'received', source_title: status.rows[0]?.source_title ?? null, variant_count: 0 }, product_id: productId, reused, editor_entry_path: `/collection-test/${collectionItemId}`, materialization: { total: Number(count.rows[0]?.total ?? 0), queued: Number(count.rows[0]?.queued ?? 0), succeeded: Number(count.rows[0]?.succeeded ?? 0), failed: Number(count.rows[0]?.failed ?? 0) } }; + /** + * Ozon has no public, stable SPU ID. The entered URL SKU remains the source + * identity for audit purposes, while a re-collection from any sibling URL + * reuses the same item when its real source SKU set intersects this one. + */ + private async findExistingSourceItem( + tx: { query: Database['query'] }, + workspaceId: string, + platformId: string, + payload: CollectedProductPayload, + ): Promise<{ id: string; product_id: string } | null> { + const bySourceKey = await tx.query<{ id: string; product_id: string }>( + `SELECT c.id, p.id AS product_id + FROM atlas.collection_items c + JOIN atlas.products p ON p.collection_item_id = c.id + WHERE c.source_platform_id = $1 + AND c.source_item_key = $2 + AND c.deleted_at IS NULL`, + [platformId, payload.source.itemKey], + ); + if (bySourceKey.rows[0]) return bySourceKey.rows[0]; + if (payload.source.platform !== 'ozon') return null; + + const sourceSkus = [ + ...new Set( + payload.variants + .map((variant) => variant.sourceSku ?? variant.sourceVariantKey) + .filter((sku) => sku.trim().length > 0), + ), + ]; + if (!sourceSkus.length) return null; + // Different sibling URLs have different source_item_key values. Locking + // each real SKU in sorted order makes two concurrent first collections + // of the same public variant group observe one another's snapshot. + for (const sourceSku of sourceSkus.sort()) { + await tx.query( + `SELECT pg_advisory_xact_lock(hashtextextended($1::text, 0))`, + [`${workspaceId}:${platformId}:${sourceSku}`], + ); + } + const byVariantOverlap = await tx.query<{ + id: string; + product_id: string; + }>( + `SELECT DISTINCT c.id, p.id AS product_id + FROM atlas.collection_items c + JOIN atlas.products p ON p.collection_item_id = c.id + JOIN atlas.product_variants v ON v.product_id = p.id + WHERE c.source_platform_id = $1 + AND c.deleted_at IS NULL + AND v.deleted_at IS NULL + AND coalesce(v.source_sku, v.source_variant_key) = ANY($2::text[]) + LIMIT 2`, + [platformId, sourceSkus], + ); + if (byVariantOverlap.rows.length > 1) { + throw new ApiError( + 409, + 'SOURCE_VARIANT_GROUP_AMBIGUOUS', + '发现多条历史采集记录包含相同 Ozon SKU，请先在采集箱处理重复记录', + ); + } + return byVariantOverlap.rows[0] ?? null; } - private async createSession(tx: { query: Database['query'] }, workspaceId: string, userId: string, clientId: string) { - const refreshToken = `${workspaceId}.${createOpaqueToken()}`; const id = randomUUID(); - await tx.query(`INSERT INTO atlas.extension_sessions (id, workspace_id, user_id, client_id, refresh_token_hash, expires_at) - VALUES ($1, current_setting('app.workspace_id')::uuid, $2, $3, $4, now() + interval '30 days')`, [id, userId, clientId, tokenHash(refreshToken)]); - const accessToken = signExtensionAccessToken({ sub: userId, sid: id, wid: workspaceId, typ: 'extension', iat: Math.floor(Date.now() / 1000), exp: Math.floor(Date.now() / 1000) + 15 * 60 }, this.config.AUTH_JWT_SECRET); - return { status: 200, body: { data: { access_token: accessToken, expires_in: 900, refresh_token: refreshToken, refresh_token_expires_in: 2_592_000, token_type: 'Bearer' } } }; + private async replaceSourceSnapshot(tx: { query: Database['query'] }, collectionItemId: string, productId: string, payload: CollectedProductPayload) { + const { status, sourcePrice } = sourceState(payload); + // 不删除旧任务记录：运行中的 Worker 可能已拿到它。取消并保留审计记录， + // 新快照在查询和 Worker 中均按 source_snapshot_revision 隔离。 + await tx.query(`UPDATE atlas.collection_asset_transfers SET status = 'cancelled', completed_at = now(), updated_at = now() + WHERE collection_item_id = $1 AND status IN ('queued', 'running')`, [collectionItemId]); + const item = await tx.query<{ source_snapshot_revision: string | number }>(`UPDATE atlas.collection_items + SET source_url = $1, source_title = $2, source_currency_code = $3, source_min_price = $4, source_max_price = $4, + status = $5::atlas.collection_item_status, raw_evidence = $6, normalized_payload = $7, parser_version = $8, + collected_at = $9, source_snapshot_revision = source_snapshot_revision + 1, revision = revision + 1, updated_at = now() + WHERE id = $10 AND deleted_at IS NULL RETURNING source_snapshot_revision`, + [payload.source.canonicalUrl, payload.common.title?.value ?? null, sourcePrice?.currencyCode ?? null, sourcePrice?.amount ?? null, status, JSON.stringify(payload.evidence), JSON.stringify(payload), payload.source.parserVersion, payload.source.collectedAt, collectionItemId]); + const snapshotRevision = Number(item.rows[0]?.source_snapshot_revision); + if (!Number.isSafeInteger(snapshotRevision) || snapshotRevision < 1) throw new Error('采集来源快照版本更新失败'); + await tx.query(`UPDATE atlas.products SET source_title = $1, updated_at = now(), version = version + 1 WHERE id = $2`, [payload.common.title?.value ?? null, productId]); + await tx.query('DELETE FROM atlas.product_variants WHERE product_id = $1', [productId]); + await this.writeSourceSnapshotRows(tx, collectionItemId, productId, snapshotRevision, payload); + return snapshotRevision; } - private assertClient(clientId: string, redirectUri: string) { if (clientId !== this.config.EXTENSION_COLLECTOR_CLIENT_ID || redirectUri !== this.config.EXTENSION_COLLECTOR_REDIRECT_URI) throw new ApiError(422, 'EXTENSION_CLIENT_INVALID', '扩展客户端或回调地址未登记'); } - private authorizationUrl(input: AuthorizeInput) { const url = new URL(`${this.config.API_PUBLIC_ORIGIN}/api/extension/authorize`); for (const [key, value] of Object.entries(input)) url.searchParams.set(key, value); return url.toString(); } + + private async writeSourceSnapshotRows(tx: { query: Database['query'] }, collectionItemId: string, productId: string, snapshotRevision: number, payload: CollectedProductPayload) { + const verifiedVariants = payload.variants.filter((variant) => variant.sourceVariantKey.trim().length > 0); + const variantRows = verifiedVariants.length ? verifiedVariants : [{ sourceVariantKey: null, sourceSku: undefined, displayName: '默认 SKU' }]; + const variants = new Map(); + for (let index = 0; index < variantRows.length; index += 1) { + const variant = variantRows[index]; + const inserted = await tx.query<{ id: string }>(`INSERT INTO atlas.product_variants (workspace_id, product_id, source_variant_key, source_sku, display_name, sort_order) + VALUES (current_setting('app.workspace_id')::uuid, $1, $2, $3, $4, $5) RETURNING id`, [productId, variant.sourceVariantKey, variant.sourceSku ?? null, variant.displayName, index]); + if (variant.sourceVariantKey) variants.set(variant.sourceVariantKey, inserted.rows[0].id); + } + for (const asset of payload.assets) { + await tx.query(`INSERT INTO atlas.collection_asset_transfers (workspace_id, collection_item_id, product_id, source_variant_id, role, source_url, status, error_payload, source_snapshot_revision) + VALUES (current_setting('app.workspace_id')::uuid, $1, $2, $3, $4::atlas.media_candidate_role, $5, 'queued', $6, $7)`, + [collectionItemId, productId, asset.variantSourceKey ? variants.get(asset.variantSourceKey) ?? null : null, asset.role, asset.url, JSON.stringify({ relation: asset.relation, referer_origin: asset.refererOrigin ?? null, evidence_ref: asset.evidenceRef ?? null, sort_order: asset.sortOrder }), snapshotRevision]); + } + if (payload.assets.length) await tx.query(`INSERT INTO atlas.outbox_events (workspace_id, aggregate_type, aggregate_id, event_type, event_payload, deduplication_key) + VALUES (current_setting('app.workspace_id')::uuid, 'collection_item', $1, 'collection.assets.materialize.requested', jsonb_build_object('collection_item_id', $1::uuid, 'source_snapshot_revision', $2::bigint), $3)`, [collectionItemId, snapshotRevision, `collection.assets.materialize:${collectionItemId}:${snapshotRevision}`]); + else await tx.query(`UPDATE atlas.collection_items SET status = CASE WHEN status = 'needs_review' THEN status ELSE 'ready_for_edit' END, updated_at = now() WHERE id = $1 AND source_snapshot_revision = $2`, [collectionItemId, snapshotRevision]); + } + + private async materialResponse(tx: { query: Database['query'] }, collectionItemId: string, productId: string, reused: boolean, refreshed: boolean) { + const count = await tx.query<{ total: string; queued: string; succeeded: string; failed: string }>(`SELECT count(*)::text AS total, count(*) FILTER (WHERE t.status = 'queued')::text AS queued, count(*) FILTER (WHERE t.status = 'succeeded')::text AS succeeded, count(*) FILTER (WHERE t.status = 'failed')::text AS failed FROM atlas.collection_asset_transfers t JOIN atlas.collection_items c ON c.id = t.collection_item_id WHERE t.collection_item_id = $1 AND t.source_snapshot_revision = c.source_snapshot_revision`, [collectionItemId]); + const status = await tx.query<{ status: string; source_title: string | null }>('SELECT status, source_title FROM atlas.collection_items WHERE id = $1', [collectionItemId]); + return { collection_item: { id: collectionItemId, status: status.rows[0]?.status ?? 'received', source_title: status.rows[0]?.source_title ?? null, variant_count: 0 }, product_id: productId, reused, refreshed, editor_entry_path: `/collection-test/${collectionItemId}`, materialization: { total: Number(count.rows[0]?.total ?? 0), queued: Number(count.rows[0]?.queued ?? 0), succeeded: Number(count.rows[0]?.succeeded ?? 0), failed: Number(count.rows[0]?.failed ?? 0) } }; + } + +} + +function sourceState(payload: CollectedProductPayload) { + const verifiedVariants = payload.variants.filter((variant) => variant.sourceVariantKey.trim().length > 0); + return { status: verifiedVariants.length || !payload.variantAxes.length ? 'materializing' : 'needs_review', sourcePrice: payload.common.sourcePrice }; } -function pkceChallenge(verifier: string) { return createHash('sha256').update(verifier).digest('base64url'); } -function constantEquals(left: string, right: string) { return left.length === right.length && left.split('').every((value, index) => value === right[index]); } -function workspacePrefix(value: string) { const prefix = value.split('.', 1)[0]; return /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(prefix) ? prefix : null; } diff --git a/apps/api/test/extension.e2e.test.ts b/apps/api/test/extension.e2e.test.ts index cb15417..530b97f 100644 --- a/apps/api/test/extension.e2e.test.ts +++ b/apps/api/test/extension.e2e.test.ts @@ -1,5 +1,5 @@ import assert from 'node:assert/strict'; -import { createHash, randomUUID } from 'node:crypto'; +import { randomUUID } from 'node:crypto'; import type { AddressInfo } from 'node:net'; import type { Database } from '@atlas/database'; import type { NestExpressApplication } from '@nestjs/platform-express'; @@ -8,27 +8,29 @@ import { DATABASE } from '../src/database/database.module.js'; import { createApplication } from '../src/main.js'; let app: NestExpressApplication; let database: Database; let baseUrl = ''; let workspaceId = ''; let userId = ''; -const redirectUri = 'https://atlas-collector.chromiumapp.org/callback'; async function request(path: string, init: RequestInit = {}) { const response = await fetch(`${baseUrl}${path}`, { redirect: 'manual', ...init, headers: { 'Content-Type': 'application/json', ...init.headers } }); return { response, body: response.status === 204 || response.status === 302 ? undefined : await response.json() as any }; } -function challenge(verifier: string) { return createHash('sha256').update(verifier).digest('base64url'); } describe('扩展授权与采集 HTTP 集成测试', () => { beforeAll(async () => { Object.assign(process.env, { NODE_ENV: 'test', SWAGGER_ENABLED: 'false', AUTH_REGISTER_IP_LIMIT: '100', AUTH_REGISTER_EMAIL_LIMIT: '100' }); const created = await createApplication(); app = created.app; database = app.get(DATABASE); await app.listen(0, '127.0.0.1'); baseUrl = `http://127.0.0.1:${(app.getHttpServer().address() as AddressInfo).port}`; }); afterAll(async () => { await database.workspaceTransaction(workspaceId, async (tx) => { await tx.query('DELETE FROM atlas.idempotency_requests WHERE workspace_id = $1', [workspaceId]); await tx.query('DELETE FROM atlas.outbox_events WHERE workspace_id = $1', [workspaceId]); await tx.query('DELETE FROM atlas.collection_asset_transfers WHERE workspace_id = $1', [workspaceId]); await tx.query('DELETE FROM atlas.product_variants WHERE workspace_id = $1', [workspaceId]); await tx.query('DELETE FROM atlas.products WHERE workspace_id = $1', [workspaceId]); await tx.query('DELETE FROM atlas.collection_items WHERE workspace_id = $1', [workspaceId]); await tx.query('DELETE FROM atlas.extension_authorization_codes WHERE workspace_id = $1', [workspaceId]); await tx.query('DELETE FROM atlas.extension_sessions WHERE workspace_id = $1', [workspaceId]); }); await database.query('DELETE FROM atlas.workspace_memberships WHERE user_id = $1', [userId]); await database.query('DELETE FROM atlas.workspaces WHERE id = $1', [workspaceId]); await database.query('DELETE FROM atlas.users WHERE id = $1', [userId]); await app.close(); }); - it('通过已有 Web 会话完成 PKCE 并幂等写入来源商品', async () => { + it('未登录令牌返回可操作的扩展会话错误', async () => { + const session = await request('/api/extension/session'); + assert.equal(session.response.status, 401); + assert.equal(session.body.error.code, 'SESSION_EXPIRED'); + assert.equal(session.body.error.message, 'Atlas ERP 登录已过期，请重新登录'); + }); + + it('通过 Atlas Web access token 写入来源商品，并以新快照覆盖重复采集', async () => { const registration = await request('/api/auth/register', { method: 'POST', body: JSON.stringify({ email: `extension-${randomUUID()}@example.test`, password: 'correct-horse-battery-staple-123', password_confirmation: 'correct-horse-battery-staple-123' }) }); assert.equal(registration.response.status, 201); userId = registration.body.data.user.id; workspaceId = registration.body.data.workspace.id; const webAccessToken = registration.body.data.access_token as string; - const cookie = registration.response.headers.get('set-cookie')?.match(/atlas_refresh=[^;]+/)?.[0]; assert.ok(cookie); - const verifier = 'a'.repeat(64); const state = 'b'.repeat(32); - const authorize = await request(`/api/extension/authorize?${new URLSearchParams({ client_id: 'atlas-collector', redirect_uri: redirectUri, state, code_challenge: challenge(verifier), code_challenge_method: 'S256' })}`, { headers: { Cookie: cookie } }); - assert.equal(authorize.response.status, 302); const callback = new URL(authorize.response.headers.get('location')!); assert.equal(callback.origin, 'https://atlas-collector.chromiumapp.org'); assert.equal(callback.searchParams.get('state'), state); - const exchange = await request('/api/extension/auth/exchange', { method: 'POST', body: JSON.stringify({ client_id: 'atlas-collector', redirect_uri: redirectUri, code: callback.searchParams.get('code'), code_verifier: verifier }) }); - assert.equal(exchange.response.status, 200); const accessToken = exchange.body.data.access_token as string; assert.ok(exchange.body.data.refresh_token); - const session = await request('/api/extension/session', { headers: { Authorization: `Bearer ${accessToken}` } }); assert.equal(session.response.status, 200); assert.equal(session.body.data.workspace.id, workspaceId); - const payload = { source: { platform: 'ozon', itemKey: '123456789', canonicalUrl: 'https://www.ozon.ru/product/demo-123456789/', collectedAt: new Date().toISOString(), parserVersion: 'test-1' }, common: { title: { value: '来源商品', confidence: 'verified' }, attributes: [], categories: [] }, variantAxes: [], variants: [{ sourceVariantKey: '123456789', sourceSku: '123456789', displayName: '默认 SKU', selections: [], confidence: 'verified', evidenceRefs: ['page'] }], assets: [], evidence: [{ ref: 'page', kind: 'page', path: 'test' }] }; - const first = await request('/api/extension/materials', { method: 'POST', headers: { Authorization: `Bearer ${accessToken}`, 'Idempotency-Key': randomUUID() }, body: JSON.stringify(payload) }); assert.equal(first.response.status, 201); assert.equal(first.body.data.reused, false); - const second = await request('/api/extension/materials', { method: 'POST', headers: { Authorization: `Bearer ${accessToken}`, 'Idempotency-Key': randomUUID() }, body: JSON.stringify(payload) }); assert.equal(second.response.status, 200); assert.equal(second.body.data.reused, true); assert.equal(second.body.data.collection_item.id, first.body.data.collection_item.id); - const inspection = await request(`/api/collection-items/${first.body.data.collection_item.id}`, { headers: { Authorization: `Bearer ${webAccessToken}` } }); assert.equal(inspection.response.status, 200); assert.equal(inspection.body.data.variants[0].source_variant_key, '123456789'); assert.equal(inspection.body.data.normalized_payload.source.platform, 'ozon'); + const session = await request('/api/extension/session', { headers: { Authorization: `Bearer ${webAccessToken}` } }); assert.equal(session.response.status, 200); assert.equal(session.body.data.workspace.id, workspaceId); + const payload = { source: { platform: 'ozon', itemKey: '123456789', canonicalUrl: 'https://www.ozon.ru/product/demo-123456789/', collectedAt: new Date().toISOString(), parserVersion: 'test-1' }, common: { title: { value: '来源商品', confidence: 'verified' }, sourcePrice: { amount: '99.90', currencyCode: 'RUB' }, attributes: [], categories: [] }, variantAxes: [], variants: [{ sourceVariantKey: '123456789', sourceSku: '123456789', displayName: '默认 SKU', selections: [], confidence: 'verified', evidenceRefs: ['page'] }], assets: [{ url: 'https://images.example.test/product.jpg', role: 'source_main', relation: 'product_candidate', sortOrder: 0, evidenceRef: 'page' }], evidence: [{ ref: 'page', kind: 'page', path: 'test' }] }; + const first = await request('/api/extension/materials', { method: 'POST', headers: { Authorization: `Bearer ${webAccessToken}`, 'Idempotency-Key': randomUUID() }, body: JSON.stringify(payload) }); assert.equal(first.response.status, 201); assert.equal(first.body.data.reused, false); assert.equal(first.body.data.materialization.queued, 1); + // A sibling Ozon URL has a different entered SKU. It must still reuse + // the item because the submitted real-SKU sets overlap at 123456789. + const refreshedPayload = { ...payload, source: { ...payload.source, itemKey: '987654321', canonicalUrl: 'https://www.ozon.ru/product/demo-987654321/', collectedAt: new Date(Date.now() + 1_000).toISOString(), parserVersion: 'test-2' }, common: { ...payload.common, title: { value: '第二次采集的中文标题', confidence: 'verified' }, sourcePrice: { amount: '109.90', currencyCode: 'RUB' } }, variants: [{ sourceVariantKey: '123456789', sourceSku: '123456789', displayName: '第一 SKU', selections: [], confidence: 'verified', evidenceRefs: ['page'] }, { sourceVariantKey: '987654321', sourceSku: '987654321', displayName: '第二次采集 SKU', selections: [], confidence: 'verified', evidenceRefs: ['page'] }], assets: [{ url: 'https://images.example.test/refreshed-product.jpg', role: 'source_main', relation: 'product_candidate', sortOrder: 0, evidenceRef: 'page' }] }; + const second = await request('/api/extension/materials', { method: 'POST', headers: { Authorization: `Bearer ${webAccessToken}`, 'Idempotency-Key': randomUUID() }, body: JSON.stringify(refreshedPayload) }); assert.equal(second.response.status, 200); assert.equal(second.body.data.reused, true); assert.equal(second.body.data.refreshed, true); assert.equal(second.body.data.collection_item.id, first.body.data.collection_item.id); assert.equal(second.body.data.materialization.queued, 1); + const inspection = await request(`/api/collection-items/${first.body.data.collection_item.id}`, { headers: { Authorization: `Bearer ${webAccessToken}` } }); assert.equal(inspection.response.status, 200); assert.equal(inspection.body.data.title, '第二次采集的中文标题'); assert.equal(inspection.body.data.normalized_payload.source.parserVersion, 'test-2'); assert.equal(inspection.body.data.variants.length, 2); assert.deepEqual(inspection.body.data.variants.map((variant: { source_variant_key: string }) => variant.source_variant_key), ['123456789', '987654321']); assert.equal(inspection.body.data.asset_transfers.length, 1); assert.equal(inspection.body.data.asset_transfers[0].source_url, 'https://images.example.test/refreshed-product.jpg'); }); }); diff --git a/apps/extension/.wxt/types/paths.d.ts b/apps/extension/.wxt/types/paths.d.ts index f5f88c9..ceafbc1 100644 --- a/apps/extension/.wxt/types/paths.d.ts +++ b/apps/extension/.wxt/types/paths.d.ts @@ -7,6 +7,7 @@ declare module "wxt/browser" { | "/" | "/background.js" | "/content-scripts/collector.js" + | "/content-scripts/ozon-page-json.js" type HtmlPublicPath = Extract export interface WxtRuntime { getURL(path: PublicPath): string; diff --git a/apps/extension/entrypoints/collector.content.ts b/apps/extension/entrypoints/collector.content.ts index 2d3701f..9c3183f 100644 --- a/apps/extension/entrypoints/collector.content.ts +++ b/apps/extension/entrypoints/collector.content.ts @@ -1,15 +1,151 @@ -import { collectCurrentProduct, supportedProductPlatform } from '../lib/source-parsers.js'; +import { + collectCurrentProduct, + supportedProductPlatform, +} from '../lib/source-parsers.js'; +import type { OzonPageJsonFetcher } from '../lib/ozon-parser.js'; -export default defineContentScript({ matches: ['*://www.ozon.ru/*', '*://www.ozon.kz/*', '*://www.ozon.by/*', '*://detail.1688.com/*', '*://item.taobao.com/*', '*://detail.tmall.com/*'], runAt: 'document_idle', main() { - let mount: HTMLDivElement | null = null; let shadow: ShadowRoot | null = null; let observedUrl = location.href; - const render = async () => { - if (!supportedProductPlatform()) { mount?.remove(); mount = null; shadow = null; return; } - if (!mount) { mount = document.createElement('div'); mount.style.cssText = 'position:fixed;right:22px;bottom:80px;z-index:2147483647'; document.documentElement.append(mount); shadow = mount.attachShadow({ mode: 'open' }); } - const root = shadow!; const session = await browser.runtime.sendMessage({ type: 'atlas.session' }).then((value: { session?: unknown }) => value.session).catch(() => null); - root.innerHTML = `
Atlas ERP
${session ? '' : ''}
`; - root.querySelector('.dot')?.addEventListener('click', () => { mount?.remove(); mount = null; shadow = null; }); - root.querySelectorAll('[data-action]').forEach((button) => button.addEventListener('click', () => void action(button.dataset.action ?? '', root))); - }; - const action = async (kind: string, shadow: ShadowRoot) => { const hint = shadow.querySelector('[data-hint]')!; try { if (kind === 'login') { hint.textContent = '正在登录…'; await browser.runtime.sendMessage({ type: 'atlas.login' }); await render(); return; } if (kind === 'erp') { await browser.runtime.sendMessage({ type: 'atlas.open', path: '/' }); return; } hint.textContent = '正在采集…'; const payload = collectCurrentProduct(); const response = await browser.runtime.sendMessage({ type: 'atlas.collect', payload }); if (kind === 'edit') await browser.runtime.sendMessage({ type: 'atlas.open', path: response.result.data.editor_entry_path }); hint.textContent = kind === 'edit' ? '已打开采集数据测试页' : '已进入采集箱'; } catch (error) { hint.textContent = error instanceof Error ? error.message : '采集失败，请重试'; } }; - void render(); setInterval(() => { if (location.href !== observedUrl) { observedUrl = location.href; void render(); } }, 700); -} }); +export default defineContentScript({ + matches: [ + '*://www.ozon.ru/*', + '*://www.ozon.kz/*', + '*://www.ozon.by/*', + '*://detail.1688.com/*', + '*://item.taobao.com/*', + '*://detail.tmall.com/*', + ], + runAt: 'document_idle', + main() { + let mount: HTMLDivElement | null = null; + let shadow: ShadowRoot | null = null; + let observedUrl = location.href; + + const render = async () => { + if (!supportedProductPlatform()) { + mount?.remove(); + mount = null; + shadow = null; + return; + } + if (!mount) { + mount = document.createElement('div'); + mount.style.cssText = + 'position:fixed;right:22px;bottom:80px;z-index:2147483647'; + document.documentElement.append(mount); + shadow = mount.attachShadow({ mode: 'open' }); + } + const root = shadow!; + const session = await browser.runtime + .sendMessage({ type: 'atlas.session' }) + .then((value: { session?: unknown }) => value.session) + .catch(() => null); + root.innerHTML = `
Atlas ERP
${session ? '' : ''}
`; + root.querySelector('.dot')?.addEventListener('click', () => { + mount?.remove(); + mount = null; + shadow = null; + }); + root.querySelectorAll('[data-action]').forEach( + (button) => + button.addEventListener('click', () => + void action(button.dataset.action ?? '', root), + ), + ); + }; + + const action = async (kind: string, root: ShadowRoot) => { + const hint = root.querySelector('[data-hint]')!; + try { + if (kind === 'login') { + hint.textContent = '正在登录…'; + await browser.runtime.sendMessage({ type: 'atlas.login' }); + await render(); + return; + } + if (kind === 'erp') { + await browser.runtime.sendMessage({ type: 'atlas.open', path: '/' }); + return; + } + hint.textContent = '正在采集多 SKU…'; + const payload = await collectCurrentProduct( + document, + location.href, + supportedProductPlatform() === 'ozon' + ? requestOzonPageJson + : undefined, + ); + const response = await browser.runtime.sendMessage({ + type: 'atlas.collect', + payload, + }); + if (kind === 'edit') { + await browser.runtime.sendMessage({ + type: 'atlas.open', + path: response.result.data.editor_entry_path, + }); + } + hint.textContent = + kind === 'edit' ? '已打开采集数据测试页' : '已进入采集箱'; + } catch (error) { + hint.textContent = + error instanceof Error ? error.message : '采集失败，请重试'; + } + }; + + void render(); + setInterval(() => { + if (location.href !== observedUrl) { + observedUrl = location.href; + void render(); + } + }, 700); + }, +}); + +const requestOzonPageJson: OzonPageJsonFetcher = ({ kind, sku }) => + new Promise((resolve, reject) => { + const requestId = crypto.randomUUID(); + const timeout = window.setTimeout(() => { + window.removeEventListener('message', receive); + reject(new Error('Ozon 页面状态读取超时')); + }, 15_000); + const receive = (event: MessageEvent) => { + if (event.source !== window || !isBridgeResponse(event.data, requestId)) return; + window.clearTimeout(timeout); + window.removeEventListener('message', receive); + if (event.data.ok) resolve(event.data.data); + else reject(new Error(event.data.error)); + }; + window.addEventListener('message', receive); + window.postMessage( + { + source: 'atlas-collector', + type: 'ozon-page-json.request', + requestId, + kind, + sku, + }, + location.origin, + ); + }); + +function isBridgeResponse( + value: unknown, + requestId: string, +): value is { + source: 'atlas-collector'; + type: 'ozon-page-json.response'; + requestId: string; + ok: boolean; + data?: unknown; + error: string; +} { + if (!value || typeof value !== 'object' || Array.isArray(value)) return false; + const message = value as Record; + return ( + message.source === 'atlas-collector' && + message.type === 'ozon-page-json.response' && + message.requestId === requestId && + typeof message.ok === 'boolean' && + (message.ok || typeof message.error === 'string') + ); +} diff --git a/apps/extension/entrypoints/ozon-page-json.content.ts b/apps/extension/entrypoints/ozon-page-json.content.ts new file mode 100644 index 0000000..ea4b0dd --- /dev/null +++ b/apps/extension/entrypoints/ozon-page-json.content.ts @@ -0,0 +1,105 @@ +/** + * Runs in Ozon's MAIN world so the request has exactly the same origin and + * session context as the page. It accepts only Atlas' two fixed, numeric-SKU + * routes and returns a small widget-state allowlist; it never exposes cookies, + * request headers or arbitrary fetch capability to the isolated script. + */ +export default defineContentScript({ + matches: ['*://www.ozon.ru/*', '*://www.ozon.kz/*', '*://www.ozon.by/*'], + world: 'MAIN', + runAt: 'document_idle', + main() { + window.addEventListener('message', (event) => { + if (event.source !== window || !isRequest(event.data)) return; + void respond(event.data); + }); + }, +}); + +type Request = { + source: 'atlas-collector'; + type: 'ozon-page-json.request'; + requestId: string; + kind: 'modal-aspects' | 'product' | 'product-detail'; + sku: string; +}; + +function isRequest(value: unknown): value is Request { + if (!value || typeof value !== 'object' || Array.isArray(value)) + return false; + const data = value as Record; + return ( + data.source === 'atlas-collector' && + data.type === 'ozon-page-json.request' && + typeof data.requestId === 'string' && + /^[a-zA-Z0-9-]{16,120}$/.test(data.requestId) && + (data.kind === 'modal-aspects' || + data.kind === 'product' || + data.kind === 'product-detail') && + typeof data.sku === 'string' && + /^\d{4,}$/.test(data.sku) + ); +} + +async function respond(request: Request) { + try { + const pagePath = + request.kind === 'modal-aspects' + ? `/modal/aspectsNew?product_id=${request.sku}` + : request.kind === 'product-detail' + ? `/product/${request.sku}/?layout_container=pdpPage2column&layout_page_index=2` + : `/product/${request.sku}/`; + const response = await fetch( + `/api/entrypoint-api.bx/page/json/v2?url=${encodeURIComponent(pagePath)}`, + { credentials: 'same-origin' }, + ); + if (!response.ok) + throw new Error(`Ozon 页面状态请求失败（HTTP ${response.status}）`); + const value = await response.json(); + window.postMessage( + { + source: 'atlas-collector', + type: 'ozon-page-json.response', + requestId: request.requestId, + ok: true, + data: allowlistedPageState(value), + }, + location.origin, + ); + } catch (error) { + window.postMessage( + { + source: 'atlas-collector', + type: 'ozon-page-json.response', + requestId: request.requestId, + ok: false, + error: + error instanceof Error + ? error.message + : 'Ozon 页面状态请求失败', + }, + location.origin, + ); + } +} + +function allowlistedPageState(value: unknown) { + if (!value || typeof value !== 'object' || Array.isArray(value)) + return { widgetStates: {} }; + const states = (value as { widgetStates?: unknown }).widgetStates; + if (!states || typeof states !== 'object' || Array.isArray(states)) + return { widgetStates: {} }; + const widgetStates: Record = {}; + for (const [name, state] of Object.entries( + states as Record, + )) { + if ( + /^web(Aspects|AspectsModal|ProductHeading|Price|Gallery|Description|RichContent|Features|ShortCharacteristics|DetailedCharacteristics|Characteristics)-/.test( + name, + ) + ) { + widgetStates[name] = state; + } + } + return { widgetStates }; +} diff --git a/apps/extension/lib/api.ts b/apps/extension/lib/api.ts index c23a845..4a280e9 100644 --- a/apps/extension/lib/api.ts +++ b/apps/extension/lib/api.ts @@ -2,13 +2,11 @@ import type { CollectedProductPayload } from '@atlas/contracts'; const apiOrigin = (import.meta.env.ATLAS_API_ORIGIN as string | undefined) ?? 'http://localhost:8200'; const webOrigin = (import.meta.env.ATLAS_WEB_ORIGIN as string | undefined) ?? 'http://localhost:8100'; -const clientId = (import.meta.env.ATLAS_EXTENSION_CLIENT_ID as string | undefined) ?? 'atlas-collector'; -type Tokens = { access_token: string; expires_in: number; refresh_token: string; refresh_token_expires_in: number; token_type: 'Bearer' }; type Session = { authenticated: true; workspace: { id: string; available: boolean }; can_collect: boolean }; -const storageKey = 'atlas.extension.tokens.v1'; +const storageKey = 'atlas.extension.web-token.v1'; -async function loadTokens() { return (await browser.storage.local.get(storageKey))[storageKey] as Tokens | undefined; } -async function saveTokens(tokens: Tokens) { await browser.storage.local.set({ [storageKey]: tokens }); } +async function loadToken() { return (await browser.storage.local.get(storageKey))[storageKey] as string | undefined; } +async function saveToken(token: string) { await browser.storage.local.set({ [storageKey]: token }); } async function clearTokens() { await browser.storage.local.remove(storageKey); } async function request(path: string, options: RequestInit = {}, token?: string) { const response = await fetch(`${apiOrigin}/api${path}`, { ...options, headers: { Accept: 'application/json', ...(options.headers ?? {}), ...(token ? { Authorization: `Bearer ${token}` } : {}) } }); @@ -16,24 +14,70 @@ async function request(path: string, options: RequestInit = {}, token?: string) return response.status === 204 ? undefined : response.json() as Promise<{ data: unknown }>; } async function activeAccessToken() { - let tokens = await loadTokens(); if (!tokens) return null; - try { await request('/extension/session', {}, tokens.access_token); return tokens.access_token; } - catch (error) { - if ((error as { status?: number }).status !== 401) throw error; - try { const refreshed = await request('/extension/auth/refresh', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ refresh_token: tokens.refresh_token }) }) as { data: Tokens }; tokens = refreshed.data; await saveTokens(tokens); return tokens.access_token; } - catch { await clearTokens(); return null; } - } + const cached = await loadToken(); + const usableCached = await validateAccessToken(cached); + if (usableCached) return usableCached; + + // 用户可能先在 Atlas Web 登录、再打开来源商品页。此时插件还没有自己的 + // 缓存，仍应直接从已打开的 Atlas 标签页同步，而不是错误地展示“登录”。 + const synchronized = await syncAccessTokenFromWebTabs(); + return validateAccessToken(synchronized); } export async function getSession(): Promise { const token = await activeAccessToken(); if (!token) return null; return (await request('/extension/session', {}, token) as { data: Session }).data; } export async function login() { - const verifier = crypto.getRandomValues(new Uint8Array(64)); const verifierText = base64url(verifier); const challenge = base64url(new Uint8Array(await crypto.subtle.digest('SHA-256', new TextEncoder().encode(verifierText)))); - const state = base64url(crypto.getRandomValues(new Uint8Array(32))); const redirectUri = browser.identity.getRedirectURL('callback'); - const authorize = new URL(`${apiOrigin}/api/extension/authorize`); authorize.search = new URLSearchParams({ client_id: clientId, redirect_uri: redirectUri, state, code_challenge: challenge, code_challenge_method: 'S256' }).toString(); - const callback = await browser.identity.launchWebAuthFlow({ url: authorize.toString(), interactive: true }); - if (!callback) throw new Error('未完成 Atlas ERP 登录'); const result = new URL(callback); if (result.searchParams.get('state') !== state || !result.searchParams.get('code')) throw new Error('扩展授权状态校验失败'); - const exchanged = await request('/extension/auth/exchange', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ client_id: clientId, code: result.searchParams.get('code'), redirect_uri: redirectUri, code_verifier: verifierText }) }) as { data: Tokens }; - await saveTokens(exchanged.data); return getSession(); + const currentSession = await getSession(); + if (currentSession) return currentSession; + + const tab = await browser.tabs.create({ url: webOrigin, active: true }); + if (tab.id === undefined) throw new Error('无法打开 Atlas ERP 登录页'); + const deadline = Date.now() + 3 * 60 * 1_000; + while (Date.now() < deadline) { + const token = await readWebAccessToken(tab.id); + if (token) { + await saveToken(token); + const session = await getSession(); + if (session) return session; + await clearTokens(); + } + await delay(2_000); + } + throw new Error('登录超时，请在 Atlas ERP 网页完成登录后重试'); } export async function submit(payload: CollectedProductPayload) { const token = await activeAccessToken(); if (!token) throw new Error('请先登录 Atlas ERP'); return request('/extension/materials', { method: 'POST', headers: { 'Content-Type': 'application/json', 'Idempotency-Key': crypto.randomUUID() }, body: JSON.stringify(payload) }, token) as Promise<{ data: { editor_entry_path: string; product_id: string; collection_item: { id: string }; reused: boolean } }>; } export async function openErp(path = '/') { await browser.tabs.create({ url: new URL(path, webOrigin).toString() }); } -function base64url(bytes: Uint8Array) { let value = ''; for (const byte of bytes) value += String.fromCharCode(byte); return btoa(value).replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/, ''); } +async function readWebAccessToken(tabId: number) { + try { + const result = await browser.scripting.executeScript({ target: { tabId }, func: () => localStorage.getItem('atlas.extension.web-access') }); + const raw = result[0]?.result; + if (typeof raw !== 'string') return null; + const parsed = JSON.parse(raw) as { accessToken?: unknown }; + return typeof parsed.accessToken === 'string' && parsed.accessToken.length > 0 ? parsed.accessToken : null; + } catch { return null; } +} +async function syncAccessTokenFromWebTabs() { + const tabs = await browser.tabs.query({ url: `${webOrigin}/*` }); + // tabs.query 返回的顺序未承诺。倒序优先尝试最近的标签页（Chrome 通常最后返回它）。 + for (const tab of tabs.reverse()) { + if (tab.id === undefined) continue; + const token = await readWebAccessToken(tab.id); + if (token) { + await saveToken(token); + return token; + } + } + return null; +} +async function validateAccessToken(token: string | undefined | null) { + if (!token) return null; + try { + await request('/extension/session', {}, token); + return token; + } catch (error) { + if ((error as { status?: number }).status === 401) { + await clearTokens(); + return null; + } + throw error; + } +} +function delay(milliseconds: number) { return new Promise((resolve) => setTimeout(resolve, milliseconds)); } diff --git a/apps/extension/lib/ozon-parser.ts b/apps/extension/lib/ozon-parser.ts index 1c61d3c..ebc12c6 100644 --- a/apps/extension/lib/ozon-parser.ts +++ b/apps/extension/lib/ozon-parser.ts @@ -1,32 +1,999 @@ import type { CollectedProductPayload } from '@atlas/contracts'; +import { dedupeImageUrl, normaliseImageUrl } from './product-media.js'; -export function isOzonProductPage(url = location.href) { return /^https:\/\/www\.ozon\.(ru|kz|by)\/product\//.test(url); } -export function collectOzonProduct(documentRoot: Document = document, pageUrl = location.href): CollectedProductPayload { - const canonical = documentRoot.querySelector('link[rel="canonical"]')?.href ?? pageUrl.split('?')[0]; - const currentSku = skuFromUrl(canonical); if (!currentSku) throw new Error('未识别到 Ozon 商品 SKU'); - const title = text(documentRoot.querySelector('h1')) ?? documentRoot.querySelector('meta[property="og:title"]')?.content?.trim(); - const images = Array.from(documentRoot.images).map((image) => image.currentSrc || image.src).filter((url) => /^https:\/\//.test(url)).filter(unique).slice(0, 30); - const evidence: CollectedProductPayload['evidence'] = [{ ref: 'page:url', kind: 'page', path: 'location.href', raw_value: canonical }]; - const variants = new Map(); - variants.set(currentSku, { sourceVariantKey: currentSku, sourceSku: currentSku, displayName: title ?? `Ozon SKU ${currentSku}`, selections: [], confidence: 'verified', evidenceRefs: ['page:url'] }); - for (const script of Array.from(documentRoot.scripts)) { - const source = script.textContent ?? ''; - // Only inspect Ozon's variant-related state; never use generic page-wide - // SKU hits because recommendation cards would create false variants. - if (!/webProducts|webAspects|aspects/i.test(source)) continue; - const matches = source.matchAll(/["']sku["']\s*[:=]\s*["']?(\d{4,})/g); - let count = 0; - for (const match of matches) { if (count++ >= 100) break; const sku = match[1]; if (!variants.has(sku)) variants.set(sku, { sourceVariantKey: sku, sourceSku: sku, displayName: `Ozon SKU ${sku}`, selections: [], confidence: 'partial', evidenceRefs: ['state:variant-sku'] }); } - if (count) evidence.push({ ref: 'state:variant-sku', kind: 'state', path: 'webProducts/webAspects', raw_value: `发现 ${count} 个公开 SKU` }); +export type OzonPageJsonRequest = { + kind: 'modal-aspects' | 'product' | 'product-detail'; + sku: string; +}; + +export type OzonPageJsonFetcher = ( + request: OzonPageJsonRequest, +) => Promise; + +type Category = CollectedProductPayload['common']['categories'][number]; +type Axis = CollectedProductPayload['variantAxes'][number]; +type Variant = CollectedProductPayload['variants'][number]; +type Asset = CollectedProductPayload['assets'][number]; +type JsonRecord = Record; + +type Candidate = { + sku: string; + title?: string; + price?: string; + selections: Variant['selections']; + evidenceRefs: Set; + hydrated: boolean; +}; + +export function isOzonProductPage(url = location.href) { + return /^https:\/\/www\.ozon\.(ru|kz|by)\/product\//.test(url); +} + +/** + * Ozon does not expose a stable public SPU / variant-group identifier. The + * public contract is the set of real variant SKUs in webAspects. We retain the + * entered SKU as provenance and let the API merge re-collections by SKU-set + * overlap instead of inventing a group ID. + */ +export async function collectOzonProduct( + documentRoot: Document = document, + pageUrl = location.href, + fetchPageJson?: OzonPageJsonFetcher, +): Promise { + const canonical = + documentRoot.querySelector('link[rel="canonical"]') + ?.href ?? pageUrl.split('?')[0]; + const currentSku = skuFromUrl(canonical); + if (!currentSku) throw new Error('未识别到 Ozon 商品 SKU'); + + const evidence: CollectedProductPayload['evidence'] = [ + { + ref: 'page:url', + kind: 'page', + path: 'location.href', + raw_value: canonical, + }, + ]; + const candidates = new Map(); + const axes = new Map(); + const assets: Asset[] = []; + const assetKeys = new Set(); + const localStates = readLocalWidgetStates(documentRoot); + let attributes = extractOzonCharacteristics(localStates); + let description = extractOzonDescription(localStates); + const heading = firstState(localStates, 'webProductHeading'); + const priceState = firstState(localStates, 'webPrice'); + const title = + stringAt(heading, ['title']) ?? + text(documentRoot.querySelector('h1')) ?? + documentRoot + .querySelector('meta[property="og:title"]') + ?.content?.trim(); + const commonPrice = + money(stringAt(priceState, ['price'])) ?? parsePrice(documentRoot); + const currency = currencyFor(documentRoot, stringAt(priceState, ['price'])); + + addCandidate(candidates, currentSku, { + title, + price: commonPrice, + evidenceRef: 'page:url', + }); + const pageVariants = discoverVariants( + localStates, + candidates, + axes, + 'ozon:aspects:page', + canonical, + ); + if (pageVariants) { + evidence.push({ + ref: 'ozon:aspects:page', + kind: 'state', + path: 'state-webAspects-*.data-state.aspects[].variants[]', + raw_value: `发现 ${pageVariants} 个不同的公开 SKU`, + }); } - const amount = parsePrice(documentRoot); const currency = documentRoot.querySelector('meta[property="product:price:currency"]')?.content?.toUpperCase() ?? 'RUB'; + addGalleryAssets( + localStates, + currentSku, + assets, + assetKeys, + 'ozon:gallery:page', + canonical, + ); + if (attributes.length) { + evidence.push({ + ref: 'ozon:characteristics:page', + kind: 'state', + path: 'webShortCharacteristics|webDetailedCharacteristics|webCharacteristics.characteristics[]', + raw_value: `发现 ${attributes.length} 项商品特征`, + }); + } + + const categories = collectCategories(documentRoot, canonical); + if (categories.length) { + evidence.push({ + ref: 'ozon:breadcrumb', + kind: 'json_ld', + path: 'BreadcrumbList/itemListElement', + raw_value: categories.map((category) => category.name).join(' > '), + }); + } + + if (fetchPageJson) { + try { + const modalStates = widgetStatesFromPageJson( + await fetchPageJson({ kind: 'modal-aspects', sku: currentSku }), + ); + const modalVariants = discoverVariants( + modalStates, + candidates, + axes, + 'ozon:aspects:modal', + canonical, + ); + evidence.push({ + ref: 'ozon:aspects:modal', + kind: 'state', + path: 'webAspectsModal-*.aspects[].variants[]', + raw_value: `发现 ${modalVariants} 个不同的公开 SKU`, + }); + } catch (error) { + evidence.push( + fetchFailureEvidence('ozon:aspects:modal', currentSku, error), + ); + } + + // Rich content is a separate, fixed Ozon page segment. Ask for it + // once for the entered SKU; never scrape arbitrary images after + // scrolling through recommendations or buyer reviews. + try { + const detailStates = widgetStatesFromPageJson( + await fetchPageJson({ + kind: 'product-detail', + sku: currentSku, + }), + ); + const added = addDetailAssets( + detailStates, + assets, + assetKeys, + 'ozon:detail:page', + canonical, + ); + description = description ?? extractOzonDescription(detailStates); + evidence.push({ + ref: 'ozon:detail:page', + kind: 'state', + path: 'webDescription|webRichContent|webFeatures image fields', + raw_value: `发现 ${added} 张商品详情图`, + }); + } catch (error) { + evidence.push( + fetchFailureEvidence('ozon:detail:page', currentSku, error), + ); + } + + // A detail page can expose a SKU omitted by the initial modal. Append + // new real SKUs to this bounded queue; never construct combinations. + const pending = [...candidates.keys()]; + const queued = new Set(pending); + const hydrated = new Set(); + while (pending.length && hydrated.size < 100) { + const batch = pending.splice(0, 3); + const results = await Promise.all( + batch.map(async (sku) => { + try { + return { + sku, + states: widgetStatesFromPageJson( + await fetchPageJson({ kind: 'product', sku }), + ), + }; + } catch (error) { + return { sku, error }; + } + }), + ); + for (const result of results) { + hydrated.add(result.sku); + if ('error' in result) { + evidence.push( + fetchFailureEvidence( + `ozon:hydrate:${result.sku}`, + result.sku, + result.error, + ), + ); + continue; + } + const detail = firstState(result.states, 'webProductHeading'); + const detailPrice = firstState(result.states, 'webPrice'); + addCandidate(candidates, result.sku, { + title: stringAt(detail, ['title']), + price: money(stringAt(detailPrice, ['price'])), + evidenceRef: `ozon:hydrate:${result.sku}`, + hydrated: true, + }); + discoverVariants( + result.states, + candidates, + axes, + `ozon:hydrate:${result.sku}`, + canonical, + ); + addGalleryAssets( + result.states, + result.sku, + assets, + assetKeys, + `ozon:gallery:${result.sku}`, + canonical, + ); + if (result.sku === currentSku) { + const detailedAttributes = extractOzonCharacteristics( + result.states, + ); + attributes = mergeOzonCharacteristics( + attributes, + detailedAttributes, + ); + if (detailedAttributes.length) { + evidence.push({ + ref: 'ozon:characteristics:current', + kind: 'state', + path: 'webCharacteristics-*.characteristics[]', + raw_value: `当前 SKU 补全 ${detailedAttributes.length} 项商品特征`, + }); + } + } + for (const sku of candidates.keys()) { + if (!queued.has(sku) && queued.size < 100) { + queued.add(sku); + pending.push(sku); + } + } + } + } + evidence.push({ + ref: 'ozon:variant-hydration', + kind: 'state', + path: 'page/json/v2 product queue', + raw_value: `发现 ${candidates.size} 个 SKU，已尝试补全 ${hydrated.size} 个`, + }); + } else { + evidence.push({ + ref: 'ozon:variant-hydration', + kind: 'state', + path: 'page/json/v2 product queue', + raw_value: '未连接 Ozon 页面桥接；仅保存当前页面已渲染的变体状态', + }); + } + + addAspectCoverAssets(axes, assets, assetKeys, canonical); + + const variants = [...candidates.values()] + .slice(0, 100) + .map((candidate) => ({ + sourceVariantKey: candidate.sku, + sourceSku: candidate.sku, + displayName: candidate.title ?? `Ozon SKU ${candidate.sku}`, + selections: candidate.selections, + ...(candidate.price + ? { + sourcePrice: { + amount: candidate.price, + currencyCode: currency, + }, + } + : {}), + confidence: + candidate.hydrated || candidate.sku === currentSku + ? 'verified' + : 'partial', + evidenceRefs: [...candidate.evidenceRefs], + })); + return { - source: { platform: 'ozon', itemKey: currentSku, canonicalUrl: canonical, collectedAt: new Date().toISOString(), parserVersion: 'ozon-public-1.0.0' }, - common: { ...(title ? { title: { value: title, raw_value: title, confidence: 'verified' as const } } : {}), ...(amount ? { sourcePrice: { amount, currencyCode: currency, raw_value: amount } } : {}), attributes: [], categories: [] }, - variantAxes: [], variants: [...variants.values()], assets: images.map((url, index) => ({ url, role: 'source_main' as const, relation: 'product_candidate' as const, sortOrder: index, evidenceRef: 'page:gallery' })), evidence, + source: { + platform: 'ozon', + // Entered SKU remains audit provenance. The API reuses by sourceSku overlap. + itemKey: currentSku, + canonicalUrl: canonical, + collectedAt: new Date().toISOString(), + parserVersion: 'ozon-public-2.2.0', + }, + common: { + ...(title + ? { + title: { + value: title, + raw_value: title, + confidence: 'verified' as const, + }, + } + : {}), + ...(commonPrice + ? { + sourcePrice: { + amount: commonPrice, + currencyCode: currency, + raw_value: commonPrice, + }, + } + : {}), + ...(description + ? { + description: { + value: description.value, + raw_value: description.rawValue, + confidence: 'verified' as const, + }, + } + : {}), + attributes, + categories, + }, + variantAxes: [...axes.values()].slice(0, 20).map((axis) => ({ + ...axis, + values: axis.values.slice(0, 200), + })), + variants, + assets, + evidence, }; } -function skuFromUrl(url: string) { return url.match(/-(\d+)(?:\/|$|\?)/)?.[1] ?? null; } -function text(element: Element | null) { const value = element?.textContent?.replace(/\s+/g, ' ').trim(); return value || undefined; } -function parsePrice(root: Document) { const meta = root.querySelector('meta[property="product:price:amount"]')?.content; if (meta && /^\d+(\.\d+)?$/.test(meta)) return meta; const candidate = Array.from(root.querySelectorAll('[data-widget*="webPrice"], [data-widget*="price"]')).map((node) => node.textContent ?? '').find((value) => /\d/.test(value)); const digits = candidate?.replace(/[^\d,\.]/g, '').replace(',', '.'); return digits && /^\d+(\.\d+)?$/.test(digits) ? digits : undefined; } -function unique(value: string, index: number, array: string[]) { return array.indexOf(value) === index; } + +function addCandidate( + candidates: Map, + sku: string, + input: { + title?: string; + price?: string; + selection?: Variant['selections'][number]; + evidenceRef: string; + hydrated?: boolean; + }, +) { + const candidate = candidates.get(sku) ?? { + sku, + selections: [], + evidenceRefs: new Set(), + hydrated: false, + }; + candidate.title = input.title ?? candidate.title; + candidate.price = input.price ?? candidate.price; + candidate.hydrated = candidate.hydrated || input.hydrated === true; + candidate.evidenceRefs.add(input.evidenceRef); + if ( + input.selection && + !candidate.selections.some( + (selection) => + selection.axisKey === input.selection?.axisKey && + selection.valueKey === input.selection?.valueKey, + ) + ) { + candidate.selections.push(input.selection); + } + candidates.set(sku, candidate); +} + +function discoverVariants( + states: JsonRecord[], + candidates: Map, + axes: Map, + evidenceRef: string, + pageUrl: string, +) { + const discovered = new Set(); + for (const state of states) { + if (!Array.isArray(state.aspects)) continue; + state.aspects.forEach((rawAspect, index) => { + if (!isRecord(rawAspect) || !Array.isArray(rawAspect.variants)) + return; + const axisKey = + stringAt(rawAspect, ['aspectKey', 'key', 'id']) ?? + `ozon:aspect:${index}`; + const axisName = + stringAt(rawAspect, ['aspectName', 'name']) ?? axisKey; + const axis = axes.get(axisKey) ?? { + key: axisKey, + name: axisName, + values: [], + }; + for (const rawVariant of rawAspect.variants) { + if (!isRecord(rawVariant)) continue; + const sku = stringAt(rawVariant, ['sku']); + if (!sku) continue; + discovered.add(sku); + if (!candidates.has(sku) && candidates.size >= 100) continue; + const data = isRecord(rawVariant.data) ? rawVariant.data : {}; + const valueName = + stringAt(data, ['searchableText', 'title']) ?? `SKU ${sku}`; + const valueKey = `${axisKey}:${valueName.toLocaleLowerCase()}`; + if (!axis.values.some((value) => value.key === valueKey)) { + const cover = ozonImageUrl( + stringAt(data, ['coverImage']), + pageUrl, + ); + axis.values.push({ + key: valueKey, + name: valueName, + ...(cover ? { image_url: cover } : {}), + }); + } + addCandidate(candidates, sku, { + title: stringAt(data, ['title', 'searchableText']), + price: money(stringAt(data, ['price', 'cardPrice'])), + selection: { axisKey, valueKey, valueName }, + evidenceRef, + }); + } + if (axis.values.length) axes.set(axisKey, axis); + }); + } + return discovered.size; +} + +/** Adds variant-axis covers without pretending that they identify one SKU. */ +function addAspectCoverAssets( + axes: Map, + assets: Asset[], + assetKeys: Set, + pageUrl: string, +) { + for (const axis of axes.values()) { + for (const value of axis.values) { + const url = ozonImageUrl(value.image_url, pageUrl); + if (!url || assets.length >= 500) continue; + value.image_url = url; + const key = `sku:${axis.key}:${value.key}:${dedupeImageUrl(url, 'ozon')}`; + if (assetKeys.has(key)) continue; + assetKeys.add(key); + assets.push({ + url, + role: 'source_sku', + relation: 'axis_value', + sortOrder: assets.length, + evidenceRef: `ozon:aspect-cover:${axis.key}:${value.key}`, + }); + } + } +} + +/** + * Detail widgets are allowlisted by the bridge. Reading image-shaped fields + * from only these widgets is safe; scanning page-wide image nodes is not. + */ +function addDetailAssets( + states: JsonRecord[], + assets: Asset[], + assetKeys: Set, + evidenceRef: string, + pageUrl: string, +) { + const urls = new Set(); + for (const state of states) { + const widget = String(state.__atlasWidgetName ?? '').replace( + /^state-/, + '', + ); + if (!/^web(?:Description|RichContent|Features)-/.test(widget)) continue; + collectDetailImageUrls(state, urls); + } + let added = 0; + for (const rawUrl of urls) { + if (assets.length >= 500) break; + const url = ozonImageUrl(rawUrl, pageUrl); + if (!url) continue; + const key = `detail:${dedupeImageUrl(url, 'ozon')}`; + if (assetKeys.has(key)) continue; + assetKeys.add(key); + assets.push({ + url, + role: 'source_detail', + relation: 'product_candidate', + sortOrder: added++, + evidenceRef, + }); + } + return added; +} + +function collectDetailImageUrls( + value: unknown, + urls: Set, + visited = new Set(), +) { + if (!value || typeof value !== 'object' || visited.has(value)) return; + visited.add(value); + if (Array.isArray(value)) { + value.forEach((entry) => collectDetailImageUrls(entry, urls, visited)); + return; + } + for (const [key, child] of Object.entries(value as JsonRecord)) { + if ( + typeof child === 'string' && + /^(src|url|image|imageurl|coverimage)$/i.test(key) && + /^https?:\/\//.test(child) + ) { + urls.add(child); + } else if (child && typeof child === 'object') { + collectDetailImageUrls(child, urls, visited); + } + } +} + +type OzonCharacteristic = + CollectedProductPayload['common']['attributes'][number]; + +/** + * Ozon renders a short parameter list in the initial document and exposes the + * complete list in the normal product page state. Keep this reader limited to + * those named widgets so arbitrary page copy never turns into product params. + */ +export function extractOzonCharacteristics( + states: JsonRecord[], +): OzonCharacteristic[] { + const attributes: OzonCharacteristic[] = []; + for (const state of states) { + const widget = String(state.__atlasWidgetName ?? '').replace( + /^state-/, + '', + ); + if ( + !/^web(?:ShortCharacteristics|DetailedCharacteristics|Characteristics)-/.test( + widget, + ) || + !Array.isArray(state.characteristics) + ) + continue; + for (const [index, entry] of state.characteristics.entries()) { + if (!isRecord(entry)) continue; + const name = richText(entry.title); + const value = Array.isArray(entry.values) + ? entry.values.map(richText).filter(Boolean).join(', ') + : richText(entry.value); + if (!name || !value) continue; + const rawId = stringAt(entry, ['id']); + attributes.push({ + key: `ozon:${rawId ?? `characteristic:${normaliseKey(name, index)}`}`.slice( + 0, + 200, + ), + name: name.slice(0, 300), + value: value.slice(0, 2_000), + raw_value: value.slice(0, 4_000), + }); + } + } + return mergeOzonCharacteristics([], attributes); +} + +function mergeOzonCharacteristics( + base: OzonCharacteristic[], + additions: OzonCharacteristic[], +) { + const merged = [...base]; + const seen = new Set( + base.map((attribute) => `${attribute.name}\u0000${attribute.value}`), + ); + for (const attribute of additions) { + if (merged.length >= 500) break; + const identity = `${attribute.name}\u0000${attribute.value}`; + if (seen.has(identity)) continue; + seen.add(identity); + merged.push(attribute); + } + return merged; +} + +type OzonDescription = { value: string; rawValue: string }; + +/** Extracts only Ozon's rich-description payload, converting it to reviewable text. */ +export function extractOzonDescription( + states: JsonRecord[], +): OzonDescription | undefined { + for (const state of states) { + const widget = String(state.__atlasWidgetName ?? '').replace( + /^state-/, + '', + ); + if (!/^web(?:Description|RichContent)-/.test(widget)) continue; + const raw = findRichAnnotationJson(state); + if (!raw) continue; + let value = raw; + try { + value = richDescriptionText(JSON.parse(raw)); + } catch { + // Some older page responses already contain plain text here. + } + value = value + .replace(/\s+\n/g, '\n') + .replace(/\n{3,}/g, '\n\n') + .trim(); + if (!value) continue; + return { + value: value.slice(0, 100_000), + rawValue: raw.slice(0, 120_000), + }; + } + return undefined; +} + +function findRichAnnotationJson( + value: unknown, + visited = new Set(), +): string | undefined { + if (!value || typeof value !== 'object' || visited.has(value)) + return undefined; + visited.add(value); + if (Array.isArray(value)) { + for (const child of value) { + const found = findRichAnnotationJson(child, visited); + if (found) return found; + } + return undefined; + } + const record = value as JsonRecord; + if ( + typeof record.richAnnotationJson === 'string' && + record.richAnnotationJson.trim() + ) + return record.richAnnotationJson; + for (const child of Object.values(record)) { + const found = findRichAnnotationJson(child, visited); + if (found) return found; + } + return undefined; +} + +function richDescriptionText( + value: unknown, + visited = new Set(), +): string { + if (typeof value === 'string') return value.trim(); + if (!value || typeof value !== 'object' || visited.has(value)) return ''; + visited.add(value); + if (Array.isArray(value)) + return value + .map((child) => richDescriptionText(child, visited)) + .filter(Boolean) + .join('\n'); + const record = value as JsonRecord; + const direct = + typeof record.text === 'string' + ? record.text + : typeof record.content === 'string' + ? record.content + : ''; + const children = Object.entries(record) + .filter( + ([key]) => + !['text', 'content', 'type', 'src', 'url', 'image'].includes( + key, + ), + ) + .map(([, child]) => richDescriptionText(child, visited)) + .filter(Boolean); + return [direct, ...children].filter(Boolean).join('\n'); +} + +function richText(value: unknown): string { + if (typeof value === 'string' || typeof value === 'number') + return String(value).replace(/\s+/g, ' ').trim(); + if (Array.isArray(value)) + return value.map(richText).filter(Boolean).join(' ').trim(); + if (!isRecord(value)) return ''; + return richText(value.textRs ?? value.text ?? value.content); +} + +function normaliseKey(value: string, index: number) { + return ( + value + .toLocaleLowerCase() + .replace(/[^\p{L}\p{N}]+/gu, '-') + .replace(/^-+|-+$/g, '') + .slice(0, 150) || String(index) + ); +} + +function addGalleryAssets( + states: JsonRecord[], + sku: string, + assets: Asset[], + assetKeys: Set, + evidenceRef: string, + pageUrl: string, +) { + const gallery = firstState(states, 'webGallery'); + if (!gallery) return; + const urls = extractOzonGalleryImageUrls(gallery, pageUrl); + for (const [index, url] of urls.entries()) { + if (assets.length >= 500) return; + const key = `main:${sku}:${url ? dedupeImageUrl(url, 'ozon') : ''}`; + if (assetKeys.has(key)) continue; + assetKeys.add(key); + assets.push({ + url, + // A gallery is product photography for this real SKU. It is not + // a SKU-option thumbnail merely because it is the second image. + role: 'source_main', + variantSourceKey: sku, + relation: 'variant_direct', + sortOrder: index, + evidenceRef, + }); + } +} + +/** Pure helper used by fixture tests and the structured Ozon gallery reader. */ +export function extractOzonGalleryImageUrls( + gallery: JsonRecord, + pageUrl: string, +) { + const urls: string[] = []; + const seen = new Set(); + for (const image of [ + gallery.coverImage, + ...(Array.isArray(gallery.images) ? gallery.images : []), + ]) { + const url = ozonImageUrl( + typeof image === 'string' + ? image + : isRecord(image) + ? stringAt(image, ['src', 'url']) + : undefined, + pageUrl, + ); + if (!url) continue; + const key = dedupeImageUrl(url, 'ozon'); + if (seen.has(key)) continue; + seen.add(key); + urls.push(url); + } + return urls; +} + +function readLocalWidgetStates(root: Document): JsonRecord[] { + const states: JsonRecord[] = []; + for (const element of Array.from( + root.querySelectorAll('[id^="state-web"][data-state]'), + )) { + const raw = element.getAttribute('data-state'); + if (!raw?.trim()) continue; + try { + const state = JSON.parse(raw); + if (isRecord(state)) states.push(withWidgetName(state, element.id)); + } catch { + // Partially updated state must not fail the whole collection. + } + } + return states; +} + +function widgetStatesFromPageJson(value: unknown): JsonRecord[] { + if (!isRecord(value) || !isRecord(value.widgetStates)) return []; + const states: JsonRecord[] = []; + for (const [widgetName, raw] of Object.entries(value.widgetStates)) { + if ( + !/^web(Aspects|AspectsModal|ProductHeading|Price|Gallery|Description|RichContent|Features|ShortCharacteristics|DetailedCharacteristics|Characteristics)-/.test( + widgetName, + ) + ) + continue; + try { + const state = typeof raw === 'string' ? JSON.parse(raw) : raw; + if (isRecord(state)) states.push(withWidgetName(state, widgetName)); + } catch { + // One malformed Ozon widget only loses that evidence. + } + } + return states; +} + +function withWidgetName(state: JsonRecord, widgetName: string): JsonRecord { + return { ...state, __atlasWidgetName: widgetName }; +} + +function firstState( + states: JsonRecord[], + prefix: string, +): JsonRecord | undefined { + return states.find((state) => + String(state.__atlasWidgetName ?? '') + .replace(/^state-/, '') + .startsWith(prefix), + ); +} + +function fetchFailureEvidence(ref: string, sku: string, error: unknown) { + const message = error instanceof Error ? error.message : '请求失败'; + return { + ref, + kind: 'state' as const, + path: `page/json/v2 product ${sku}`, + raw_value: `SKU ${sku} 补全失败：${message.slice(0, 500)}`, + }; +} + +/** JSON-LD 面包屑优先；Ozon 不输出时读取可见面包屑。 */ +function collectCategories(root: Document, pageUrl: string): Category[] { + for (const script of Array.from(root.scripts)) { + if ( + script.type !== 'application/ld+json' || + !script.textContent?.trim() + ) + continue; + try { + const categories = categoriesFromJsonLd( + JSON.parse(script.textContent), + pageUrl, + ); + if (categories.length) return categories; + } catch { + // Invalid dynamic JSON-LD does not affect collection. + } + } + const links = Array.from( + root.querySelectorAll( + '[data-widget*="breadCrumbs"] a, [data-widget*="breadcrumb"] a, nav[aria-label*="хлеб"] a, nav[aria-label*="bread"] a', + ), + ); + return uniqueCategories( + links.map((link) => + category(link.textContent, link.getAttribute('href'), pageUrl), + ), + ); +} + +function categoriesFromJsonLd(value: unknown, pageUrl: string): Category[] { + const list = findBreadcrumbList(value); + if (!list || !Array.isArray(list.itemListElement)) return []; + return uniqueCategories( + list.itemListElement.map((entry) => { + if (!isRecord(entry)) return null; + const item = entry.item; + const itemUrl = + typeof item === 'string' + ? item + : isRecord(item) + ? stringAt(item, ['@id', 'url']) + : undefined; + return category( + stringAt(entry, ['name']) ?? + (isRecord(item) ? stringAt(item, ['name']) : undefined), + itemUrl, + pageUrl, + ); + }), + ); +} + +function findBreadcrumbList( + value: unknown, + visited = new Set(), +): JsonRecord | null { + if (!isRecord(value) || visited.has(value)) return null; + visited.add(value); + const type = value['@type']; + if ( + (typeof type === 'string' && type === 'BreadcrumbList') || + (Array.isArray(type) && type.includes('BreadcrumbList')) + ) + return value; + for (const child of Object.values(value)) { + if (Array.isArray(child)) { + for (const entry of child) { + const found = findBreadcrumbList(entry, visited); + if (found) return found; + } + } else { + const found = findBreadcrumbList(child, visited); + if (found) return found; + } + } + return null; +} + +function category( + rawName: unknown, + rawUrl: unknown, + pageUrl: string, +): Category | null { + const name = + typeof rawName === 'string' ? rawName.replace(/\s+/g, ' ').trim() : ''; + if ( + !name || + /^(ozon|главная|home|首页|全部分类|все категории)$/i.test(name) + ) + return null; + let key = name.toLowerCase(); + if (typeof rawUrl === 'string' && rawUrl) { + try { + const url = new URL(rawUrl, pageUrl); + key = `${url.hostname}${url.pathname.replace(/\/$/, '')}${url.search}`; + } catch { + // Use the normalised name as a stable fallback key. + } + } + return { key: `ozon:${key}`, name }; +} + +function uniqueCategories(items: Array): Category[] { + const seen = new Set(); + return items.filter( + (item): item is Category => + !!item && !seen.has(item.key) && !!seen.add(item.key), + ); +} + +function isRecord(value: unknown): value is JsonRecord { + return !!value && typeof value === 'object' && !Array.isArray(value); +} + +function stringAt(value: JsonRecord | undefined, keys: string[]) { + if (!value) return undefined; + for (const key of keys) { + const item = value[key]; + if ( + (typeof item === 'string' || typeof item === 'number') && + String(item).trim() + ) + return String(item).trim(); + } + return undefined; +} + +function skuFromUrl(url: string) { + return url.match(/-(\d+)(?:\/|$|\?)/)?.[1] ?? null; +} + +function text(element: Element | null) { + const value = element?.textContent?.replace(/\s+/g, ' ').trim(); + return value || undefined; +} + +function money(raw: string | undefined) { + if (!raw) return undefined; + const amount = raw.replace(/[^\d,\.]/g, '').replace(',', '.'); + return /^\d+(\.\d{1,4})?$/.test(amount) ? amount : undefined; +} + +function parsePrice(root: Document) { + const meta = root.querySelector( + 'meta[property="product:price:amount"]', + )?.content; + if (meta && /^\d+(\.\d+)?$/.test(meta)) return meta; + const candidate = Array.from( + root.querySelectorAll( + '[data-widget*="webPrice"], [data-widget*="price"]', + ), + ) + .map((node) => node.textContent ?? '') + .find((value) => /\d/.test(value)); + return money(candidate); +} + +function currencyFor(root: Document, price: string | undefined) { + const meta = root + .querySelector( + 'meta[property="product:price:currency"]', + ) + ?.content?.toUpperCase(); + if (meta && /^[A-Z]{3}$/.test(meta)) return meta; + if (price?.includes('¥')) return 'CNY'; + if (price?.includes('₸')) return 'KZT'; + if (price?.includes('Br')) return 'BYN'; + return 'RUB'; +} + +function httpsUrl(value: string | undefined) { + return value && /^https:\/\//.test(value) ? value : undefined; +} + +function ozonImageUrl(value: string | undefined, pageUrl: string) { + if (!value) return undefined; + const url = normaliseImageUrl(value, pageUrl, 'ozon'); + return url?.startsWith('https://') ? url : undefined; +} diff --git a/apps/extension/lib/product-media.ts b/apps/extension/lib/product-media.ts new file mode 100644 index 0000000..49cb0e4 --- /dev/null +++ b/apps/extension/lib/product-media.ts @@ -0,0 +1,336 @@ +import type { CollectedProductPayload } from '@atlas/contracts'; + +type SupportedMediaPlatform = 'ozon' | '1688' | 'taobao' | 'tmall'; +type DomMediaPlatform = Exclude; +type Asset = CollectedProductPayload['assets'][number]; + +type ImageGroup = { + role: Asset['role']; + relation: Asset['relation']; + selectors: string[]; + sourceProps?: Array< + | 'data-lazyload-src' + | 'data-src' + | 'srcset' + | 'currentSrc' + | 'src' + | 'backgroundImage' + >; + excludeWithin?: string[]; + minWidth?: number; + minHeight?: number; +}; + +/** + * These profiles are ported from the proven collector extension. The crucial + * invariant is that images always come from a product widget, never from a + * page-wide `document.images` scan (which also includes logos, ads and + * recommendation cards). + */ +const DOM_IMAGE_PROFILES: Record = { + '1688': [ + { + role: 'source_main', + relation: 'inherited_product_default', + selectors: [ + '[data-module="od_picture_gallery"] img', + '.module-od-picture-gallery img', + '#recyclerview .detail-gallery-turn-wrapper .detail-gallery-img', + '#screen .od-gallery-turn-item-wrapper .od-gallery-img', + '#content .od-scroller-item .v-image-cover', + '#content .od-picture-gallery-list .v-image-cover', + '#dt-tab img', + '.detail-gallery-turn img.detail-gallery-img', + '.img-list-wrapper img.od-gallery-img', + ], + minWidth: 200, + minHeight: 200, + }, + { + role: 'source_sku', + relation: 'axis_value', + selectors: [ + '[data-module="od_sku_selection"] img', + '.module-od-sku-selection img', + '.pc-sku-wrapper .prop-item-inner-wrapper', + '.sku-item-wrapper', + '.specification-cell', + '.sku-filter-button', + '.expand-view-item', + '.feature-item img', + ], + sourceProps: ['backgroundImage'], + minWidth: 20, + minHeight: 20, + }, + { + role: 'source_detail', + relation: 'product_candidate', + selectors: [ + '#detail img', + '.de-description-detail img', + '#detailContentContainer img', + '.html-description img', + ], + minWidth: 300, + minHeight: 100, + }, + ], + taobao: [ + { + role: 'source_main', + relation: 'inherited_product_default', + selectors: [ + '[class*="picGallery--"] img', + '#picGalleryEle img', + '[class*="thumbnailPic--"]', + ], + }, + { + role: 'source_sku', + relation: 'axis_value', + selectors: [ + '[class*="valueItem--"]', + '[class*="valueItemImgWrap--"]', + ], + minWidth: 20, + minHeight: 20, + }, + taobaoDetailProfile(), + ], + tmall: [ + { + role: 'source_main', + relation: 'inherited_product_default', + selectors: [ + '[class*="picGallery--"] img', + '#picGalleryEle img', + '[class*="thumbnailPic--"]', + ], + }, + { + role: 'source_sku', + relation: 'axis_value', + selectors: [ + '[class*="valueItem--"]', + '[class*="valueItemImgWrap--"]', + ], + minWidth: 20, + minHeight: 20, + }, + taobaoDetailProfile(), + ], +}; + +function taobaoDetailProfile(): ImageGroup { + return { + role: 'source_detail', + relation: 'product_candidate', + selectors: [ + '[class*="tabDetailWrap--"] img', + '[class*="detailInfo--"] img', + ], + // The old collector verified that review photos and recommendation + // cards can be large enough to pass a size-only filter. + excludeWithin: [ + '[class*="Comment--"]', + '[class*="comments--"]', + '[class*="userInfo--"]', + '[class*="rate"]', + '[class*="RecommendInfo--"]', + '[data-spm="recommends"]', + '[class*="recommend--"]', + '[class*="cardPic--"]', + ], + minWidth: 300, + minHeight: 100, + }; +} + +export function collectPlatformDomAssets( + platform: DomMediaPlatform, + root: Document, + pageUrl: string, +): Asset[] { + const assets: Asset[] = []; + const seen = new Set(); + for (const group of DOM_IMAGE_PROFILES[platform]) { + for (const selector of group.selectors) { + for (const element of Array.from( + root.querySelectorAll(selector), + )) { + if ( + group.excludeWithin?.some((excluded) => + element.closest(excluded), + ) + ) + continue; + if (!passesDimensions(element, group)) continue; + const url = imageUrlFromElement( + element, + group.sourceProps, + pageUrl, + platform, + ); + if (!url) continue; + // Keep the old collector's group-level de-duplication: an + // aspect cover may legitimately also be a gallery image. + const key = `${group.role}:${dedupeImageUrl(url, platform)}`; + if (seen.has(key)) continue; + seen.add(key); + assets.push({ + url, + role: group.role, + relation: group.relation, + sortOrder: assets.length, + evidenceRef: `${platform}:dom:${group.role}`, + }); + } + } + } + return assets; +} + +/** Restores the original CDN URL before payload storage and de-duplication. */ +export function normaliseImageUrl( + raw: string, + pageUrl: string, + platform: SupportedMediaPlatform, +): string | undefined { + let url: URL; + try { + url = new URL(raw, pageUrl); + } catch { + return undefined; + } + if (url.protocol !== 'https:' && url.protocol !== 'http:') return undefined; + + if (platform === 'ozon') { + // Ozon thumbnails use /wc50/, /wc100/, /c600/, etc. The same media + // without this segment is its original CDN resource. + url.pathname = url.pathname + .replace(/\/(?:wc|c)\d+\//i, '/') + .replace(/\/{2,}/g, '/'); + } else { + // Alibaba image URLs commonly append a transform after the file + // extension (e.g. image.jpg_400x400.jpg). Keep the original image. + const original = url.pathname.match( + /^(.+?\.(?:jpe?g|png|gif|bmp|heic|webp|avif))_/i, + ); + if (original) url.pathname = original[1]; + } + for (const parameter of ['width', 'height', 'size', 'quality', 'w', 'h']) { + url.searchParams.delete(parameter); + } + return url.toString(); +} + +export function dedupeImageUrl(url: string, platform: SupportedMediaPlatform) { + const normalised = normaliseImageUrl(url, url, platform) ?? url; + try { + const parsed = new URL(normalised); + parsed.search = ''; + parsed.hash = ''; + return parsed.toString(); + } catch { + return normalised; + } +} + +function imageUrlFromElement( + element: HTMLElement, + sourceProps: ImageGroup['sourceProps'] = [ + 'data-lazyload-src', + 'data-src', + 'srcset', + 'currentSrc', + 'src', + 'backgroundImage', + ], + pageUrl: string, + platform: SupportedMediaPlatform, +) { + const nestedTarget = element.matches('img, source') + ? null + : element.querySelector('img, source'); + const imageTarget = element.matches('img, source') ? element : nestedTarget; + for (const prop of sourceProps) { + // 1688 uses background images on the selected SKU option container. + // For every other property, prefer the actual nested image/source. + const target = prop === 'backgroundImage' ? element : imageTarget; + if (!target) continue; + let raw = ''; + if (prop === 'srcset') + raw = bestSrcset(target.getAttribute('srcset') ?? ''); + else if (prop === 'currentSrc') + raw = (target as HTMLImageElement).currentSrc ?? ''; + else if (prop === 'backgroundImage') { + const style = + target.ownerDocument.defaultView?.getComputedStyle(target) + .backgroundImage ?? target.style.backgroundImage; + raw = style.match(/url\(["']?(.*?)["']?\)/i)?.[1] ?? ''; + // Some 1688 SKU selectors directly match an despite the + // group being background-image-first. + if (!raw && target.tagName === 'IMG') { + raw = + (target as HTMLImageElement).currentSrc || + target.getAttribute('src') || + ''; + } + if (!raw && nestedTarget && nestedTarget !== target) { + const nestedStyle = + nestedTarget.ownerDocument.defaultView?.getComputedStyle( + nestedTarget, + ).backgroundImage ?? nestedTarget.style.backgroundImage; + raw = + nestedStyle.match(/url\(["']?(.*?)["']?\)/i)?.[1] ?? + ((nestedTarget as HTMLImageElement).currentSrc || + nestedTarget.getAttribute('src') || + ''); + } + } else raw = target.getAttribute(prop) ?? ''; + const url = raw && normaliseImageUrl(raw, pageUrl, platform); + if (url) return url; + } + return undefined; +} + +function passesDimensions(element: HTMLElement, group: ImageGroup) { + if (!group.minWidth && !group.minHeight) return true; + const image = element.matches('img') + ? (element as HTMLImageElement) + : element.querySelector('img'); + const measured = image ?? element; + const width = image?.naturalWidth || measured.offsetWidth || 0; + const height = image?.naturalHeight || measured.offsetHeight || 0; + // Browser snapshots and unrendered lazy images often report 0. As in the + // old collector, only reject when a real measured size proves it is tiny. + return !( + width > 0 && + height > 0 && + (width < (group.minWidth ?? 0) || height < (group.minHeight ?? 0)) + ); +} + +function bestSrcset(srcset: string) { + let best = ''; + let bestSize = -1; + for (const part of srcset + .split(',') + .map((value) => value.trim()) + .filter(Boolean)) { + const [url, descriptor = ''] = part.split(/\s+/, 2); + const width = descriptor.match(/^(\d+)w$/)?.[1]; + const density = descriptor.match(/^(\d+(?:\.\d+)?)x$/)?.[1]; + const size = width + ? Number(width) + : density + ? Number(density) * 1_000 + : 0; + if (size >= bestSize) { + best = url; + bestSize = size; + } + } + return best; +} diff --git a/apps/extension/lib/source-parsers.ts b/apps/extension/lib/source-parsers.ts index b0b10a0..063d8c7 100644 --- a/apps/extension/lib/source-parsers.ts +++ b/apps/extension/lib/source-parsers.ts @@ -1,32 +1,62 @@ import type { CollectedProductPayload } from '@atlas/contracts'; -import { collectOzonProduct, isOzonProductPage } from './ozon-parser.js'; +import { + collectOzonProduct, + isOzonProductPage, + type OzonPageJsonFetcher, +} from './ozon-parser.js'; +import { collectPlatformDomAssets } from './product-media.js'; type SourcePlatform = '1688' | 'taobao' | 'tmall' | 'ozon'; type Json = Record; type Axis = CollectedProductPayload['variantAxes'][number]; type Variant = CollectedProductPayload['variants'][number]; -export function supportedProductPlatform(url = location.href): SourcePlatform | null { +export function supportedProductPlatform( + url = location.href, +): SourcePlatform | null { if (isOzonProductPage(url)) return 'ozon'; - if (/^https:\/\/detail\.1688\.com\/offer\/\d+\.html/.test(url)) return '1688'; + if (/^https:\/\/detail\.1688\.com\/offer\/\d+\.html/.test(url)) + return '1688'; if (/^https:\/\/item\.taobao\.com\/item\.htm/.test(url)) return 'taobao'; if (/^https:\/\/detail\.tmall\.com\/item\.htm/.test(url)) return 'tmall'; return null; } -export function collectCurrentProduct(root: Document = document, pageUrl = location.href): CollectedProductPayload { +export async function collectCurrentProduct( + root: Document = document, + pageUrl = location.href, + fetchOzonPageJson?: OzonPageJsonFetcher, +): Promise { const platform = supportedProductPlatform(pageUrl); - if (platform === 'ozon') return collectOzonProduct(root, pageUrl); + if (platform === 'ozon') + return collectOzonProduct(root, pageUrl, fetchOzonPageJson); if (!platform) throw new Error('当前页面不是支持的商品详情页'); - return platform === '1688' ? collect1688(root, pageUrl) : collectTaobao(root, pageUrl, platform); + return platform === '1688' + ? collect1688(root, pageUrl) + : collectTaobao(root, pageUrl, platform); } function collect1688(root: Document, pageUrl: string): CollectedProductPayload { - const state = pageState(root); const skuMap = findFirst(state, 'skuMapOriginal'); const skuProps = findFirst(state, 'skuProps'); - const axes = parse1688Axes(skuProps); const evidence: CollectedProductPayload['evidence'] = [{ ref: 'page:url', kind: 'page', path: 'location.href', raw_value: pageUrl }]; + const state = pageState(root); + const skuMap = findFirst(state, 'skuMapOriginal'); + const skuProps = findFirst(state, 'skuProps'); + const axes = parse1688Axes(skuProps); + const evidence: CollectedProductPayload['evidence'] = [ + { + ref: 'page:url', + kind: 'page', + path: 'location.href', + raw_value: pageUrl, + }, + ]; const variants: Variant[] = []; if (isRecord(skuMap)) { - evidence.push({ ref: '1688:sku-map', kind: 'state', path: 'skuMapOriginal', raw_value: `组合记录 ${Object.keys(skuMap).length} 条` }); + evidence.push({ + ref: '1688:sku-map', + kind: 'state', + path: 'skuMapOriginal', + raw_value: `组合记录 ${Object.keys(skuMap).length} 条`, + }); for (const [combination, row] of Object.entries(skuMap)) { if (!isRecord(row)) continue; const sourceSku = stringAt(row, ['skuId', 'sku_id', 'skuCode']); @@ -34,50 +64,391 @@ function collect1688(root: Document, pageUrl: string): CollectedProductPayload { // record. Never expand it with the Cartesian product of skuProps. const sourceVariantKey = sourceSku ?? `1688:${combination}`; if (!combination || combination === 'undefined') continue; - variants.push({ sourceVariantKey, ...(sourceSku ? { sourceSku } : {}), displayName: selectionsForCombination(combination, axes).map((item) => item.valueName).join(' / ') || `1688 SKU ${sourceSku ?? combination}`, selections: selectionsForCombination(combination, axes), ...(moneyAt(row, ['discountPrice', 'price', 'salePrice']) ? { sourcePrice: { amount: moneyAt(row, ['discountPrice', 'price', 'salePrice'])!, currencyCode: 'CNY' } } : {}), confidence: 'verified', evidenceRefs: ['1688:sku-map'] }); + variants.push({ + sourceVariantKey, + ...(sourceSku ? { sourceSku } : {}), + displayName: + selectionsForCombination(combination, axes) + .map((item) => item.valueName) + .join(' / ') || `1688 SKU ${sourceSku ?? combination}`, + selections: selectionsForCombination(combination, axes), + ...(moneyAt(row, ['discountPrice', 'price', 'salePrice']) + ? { + sourcePrice: { + amount: moneyAt(row, [ + 'discountPrice', + 'price', + 'salePrice', + ])!, + currencyCode: 'CNY', + }, + } + : {}), + confidence: 'verified', + evidenceRefs: ['1688:sku-map'], + }); } } - return basePayload('1688', offerId(pageUrl), root, pageUrl, axes, uniqueVariants(variants), evidence, '1688-state-1.0.0'); + return basePayload( + '1688', + offerId(pageUrl), + root, + pageUrl, + axes, + uniqueVariants(variants), + evidence, + '1688-state-1.0.0', + ); } -function collectTaobao(root: Document, pageUrl: string, platform: 'taobao' | 'tmall'): CollectedProductPayload { - const state = pageState(root); const skuBase = findFirst(state, 'skuBase'); const skuCore = findFirst(state, 'skuCore'); - const axes = parseTaobaoAxes(isRecord(skuBase) ? skuBase.props : undefined); const evidence: CollectedProductPayload['evidence'] = [{ ref: 'page:url', kind: 'page', path: 'location.href', raw_value: pageUrl }]; +function collectTaobao( + root: Document, + pageUrl: string, + platform: 'taobao' | 'tmall', +): CollectedProductPayload { + const state = pageState(root); + const skuBase = findFirst(state, 'skuBase'); + const skuCore = findFirst(state, 'skuCore'); + const axes = parseTaobaoAxes(isRecord(skuBase) ? skuBase.props : undefined); + const evidence: CollectedProductPayload['evidence'] = [ + { + ref: 'page:url', + kind: 'page', + path: 'location.href', + raw_value: pageUrl, + }, + ]; const variants: Variant[] = []; - const skus = isRecord(skuBase) && Array.isArray(skuBase.skus) ? skuBase.skus : []; - const sku2info = isRecord(skuCore) && isRecord(skuCore.sku2info) ? skuCore.sku2info : {}; - if (skus.length) evidence.push({ ref: 'taobao:sku-base', kind: 'state', path: 'skuBase.skus', raw_value: `组合记录 ${skus.length} 条` }); + const skus = + isRecord(skuBase) && Array.isArray(skuBase.skus) ? skuBase.skus : []; + const sku2info = + isRecord(skuCore) && isRecord(skuCore.sku2info) ? skuCore.sku2info : {}; + if (skus.length) + evidence.push({ + ref: 'taobao:sku-base', + kind: 'state', + path: 'skuBase.skus', + raw_value: `组合记录 ${skus.length} 条`, + }); for (const sku of skus) { if (!isRecord(sku)) continue; - const skuId = stringAt(sku, ['skuId', 'sku_id']); const propPath = stringAt(sku, ['propPath', 'prop_path']); + const skuId = stringAt(sku, ['skuId', 'sku_id']); + const propPath = stringAt(sku, ['propPath', 'prop_path']); // skuId + propPath is the required combination proof. Props alone only // become axes and cause a single default SKU on the server. if (!skuId || !propPath) continue; const info = isRecord(sku2info[skuId]) ? sku2info[skuId] : {}; const selections = selectionsForPropPath(propPath, axes); - variants.push({ sourceVariantKey: skuId, sourceSku: skuId, displayName: selections.map((item) => item.valueName).join(' / ') || `SKU ${skuId}`, selections, ...(moneyAt(info, ['price', 'promotionPrice', 'discountPrice']) ? { sourcePrice: { amount: moneyAt(info, ['price', 'promotionPrice', 'discountPrice'])!, currencyCode: 'CNY' } } : {}), confidence: 'verified', evidenceRefs: ['taobao:sku-base'] }); + variants.push({ + sourceVariantKey: skuId, + sourceSku: skuId, + displayName: + selections.map((item) => item.valueName).join(' / ') || + `SKU ${skuId}`, + selections, + ...(moneyAt(info, ['price', 'promotionPrice', 'discountPrice']) + ? { + sourcePrice: { + amount: moneyAt(info, [ + 'price', + 'promotionPrice', + 'discountPrice', + ])!, + currencyCode: 'CNY', + }, + } + : {}), + confidence: 'verified', + evidenceRefs: ['taobao:sku-base'], + }); } - return basePayload(platform, itemId(pageUrl), root, pageUrl, axes, uniqueVariants(variants), evidence, `${platform}-state-1.0.0`); + return basePayload( + platform, + itemId(pageUrl), + root, + pageUrl, + axes, + uniqueVariants(variants), + evidence, + `${platform}-state-1.0.0`, + ); } -function basePayload(platform: Exclude, itemKey: string, root: Document, pageUrl: string, axes: Axis[], variants: Variant[], evidence: CollectedProductPayload['evidence'], parserVersion: string): CollectedProductPayload { - const canonical = root.querySelector('link[rel="canonical"]')?.href ?? pageUrl.split('#')[0]; const title = root.querySelector('h1')?.textContent?.replace(/\s+/g, ' ').trim() || root.querySelector('meta[property="og:title"]')?.content?.trim(); - const assets = Array.from(root.images).map((image) => image.currentSrc || image.src).filter((url) => /^https:\/\//.test(url)).filter((url, index, all) => all.indexOf(url) === index).slice(0, 40).map((url, sortOrder) => ({ url, role: 'source_main' as const, relation: 'product_candidate' as const, sortOrder, evidenceRef: 'page:gallery' })); - return { source: { platform, itemKey, canonicalUrl: canonical, collectedAt: new Date().toISOString(), parserVersion }, common: { ...(title ? { title: { value: title, raw_value: title, confidence: 'verified' as const } } : {}), attributes: [], categories: [] }, variantAxes: axes, variants, assets, evidence }; +function basePayload( + platform: Exclude, + itemKey: string, + root: Document, + pageUrl: string, + axes: Axis[], + variants: Variant[], + evidence: CollectedProductPayload['evidence'], + parserVersion: string, +): CollectedProductPayload { + const canonical = + root.querySelector('link[rel="canonical"]')?.href ?? + pageUrl.split('#')[0]; + const title = + root.querySelector('h1')?.textContent?.replace(/\s+/g, ' ').trim() || + root + .querySelector('meta[property="og:title"]') + ?.content?.trim(); + const assets = collectPlatformDomAssets(platform, root, canonical); + evidence.push({ + ref: `${platform}:dom:image-profile`, + kind: 'dom', + path: 'platform-scoped gallery / sku / detail selectors', + raw_value: `采集 ${assets.length} 张受限商品区域图片`, + }); + return { + source: { + platform, + itemKey, + canonicalUrl: canonical, + collectedAt: new Date().toISOString(), + parserVersion, + }, + common: { + ...(title + ? { + title: { + value: title, + raw_value: title, + confidence: 'verified' as const, + }, + } + : {}), + attributes: [], + categories: [], + }, + variantAxes: axes, + variants, + assets, + evidence, + }; } -function pageState(root: Document) { const all: unknown[] = []; for (const script of Array.from(root.scripts)) { const text = script.textContent?.trim() ?? ''; if (text.length < 2 || text.length > 5_000_000) continue; if (script.type === 'application/ld+json') { try { all.push(JSON.parse(text)); } catch { /* ignored evidence */ } } for (const marker of ['skuMapOriginal', 'skuBase', 'skuCore', 'skuProps']) { const object = objectAfterMarker(text, marker); if (object) all.push(object); } } return all; } -function objectAfterMarker(text: string, marker: string): unknown | null { const position = text.indexOf(marker); if (position < 0) return null; const start = text.slice(position).search(/[\[{]/); if (start < 0) return null; const first = position + start; const open = text[first]; const close = open === '{' ? '}' : ']'; let depth = 0; let quote = ''; let escaped = false; for (let index = first; index < text.length && index - first < 2_000_000; index += 1) { const char = text[index]; if (quote) { if (escaped) escaped = false; else if (char === '\\') escaped = true; else if (char === quote) quote = ''; continue; } if (char === '"' || char === "'") { quote = char; continue; } if (char === open) depth += 1; if (char === close && --depth === 0) { try { return JSON.parse(text.slice(first, index + 1)); } catch { return null; } } } return null; } -function findFirst(states: unknown[], key: string): unknown { for (const state of states) { const found = findKey(state, key); if (found !== undefined) return found; } return undefined; } -function findKey(value: unknown, key: string, seen = new Set()): unknown { if (!value || typeof value !== 'object' || seen.has(value)) return undefined; seen.add(value); if (isRecord(value) && key in value) return value[key]; for (const child of Array.isArray(value) ? value : Object.values(value)) { const found = findKey(child, key, seen); if (found !== undefined) return found; } return undefined; } -function parse1688Axes(value: unknown): Axis[] { const entries = Array.isArray(value) ? value : isRecord(value) && Array.isArray(value.skuProps) ? value.skuProps : []; return entries.flatMap((entry, axisIndex) => { if (!isRecord(entry)) return []; const axisKey = stringAt(entry, ['prop', 'propId', 'id']) ?? `axis:${axisIndex}`; const name = stringAt(entry, ['name', 'propName']) ?? axisKey; const values = (Array.isArray(entry.value) ? entry.value : Array.isArray(entry.values) ? entry.values : []).flatMap((item, valueIndex) => isRecord(item) ? [{ key: stringAt(item, ['valueId', 'vid', 'id']) ?? `${axisKey}:${valueIndex}`, name: stringAt(item, ['name', 'valueName']) ?? `${valueIndex + 1}`, ...(urlAt(item, ['imageUrl', 'image', 'imageUrlLarge']) ? { image_url: urlAt(item, ['imageUrl', 'image', 'imageUrlLarge'])! } : {}) }] : []); return values.length ? [{ key: axisKey, name, values }] : []; }); } -function parseTaobaoAxes(value: unknown): Axis[] { return (Array.isArray(value) ? value : []).flatMap((entry, index) => { if (!isRecord(entry)) return []; const key = stringAt(entry, ['pid', 'propId']) ?? `axis:${index}`; const name = stringAt(entry, ['name', 'propName']) ?? key; const values = (Array.isArray(entry.values) ? entry.values : []).flatMap((item, valueIndex) => isRecord(item) ? [{ key: stringAt(item, ['vid', 'valueId']) ?? `${key}:${valueIndex}`, name: stringAt(item, ['name', 'valueName']) ?? `${valueIndex + 1}`, ...(urlAt(item, ['image', 'imageUrl']) ? { image_url: urlAt(item, ['image', 'imageUrl'])! } : {}) }] : []); return values.length ? [{ key, name, values }] : []; }); } -function selectionsForPropPath(path: string, axes: Axis[]) { return path.split(/[;,]/).flatMap((pair) => { const [axisKey, valueKey] = pair.split(':'); const axis = axes.find((item) => item.key === axisKey); const value = axis?.values.find((item) => item.key === valueKey); return axis && value ? [{ axisKey, valueKey, valueName: value.name }] : []; }); } -function selectionsForCombination(combination: string, axes: Axis[]) { const ids = combination.split(/[;,&]/).map((value) => value.trim()).filter(Boolean); return axes.flatMap((axis) => { const value = axis.values.find((item) => ids.includes(item.key)); return value ? [{ axisKey: axis.key, valueKey: value.key, valueName: value.name }] : []; }); } -function stringAt(value: Json, keys: string[]) { for (const key of keys) if (typeof value[key] === 'string' || typeof value[key] === 'number') return String(value[key]); return undefined; } -function moneyAt(value: Json, keys: string[]) { const raw = stringAt(value, keys)?.replace(/,/g, '.'); return raw && /^\d+(\.\d{1,4})?$/.test(raw) ? raw : undefined; } -function urlAt(value: Json, keys: string[]) { const raw = stringAt(value, keys); return raw && /^https:\/\//.test(raw) ? raw : undefined; } -function uniqueVariants(variants: Variant[]) { const seen = new Set(); return variants.filter((variant) => !seen.has(variant.sourceVariantKey) && !!seen.add(variant.sourceVariantKey)); } -function offerId(url: string) { return url.match(/\/offer\/(\d+)\.html/)?.[1] ?? `1688:${new URL(url).pathname}`; } -function itemId(url: string) { return new URL(url).searchParams.get('id') ?? new URL(url).pathname; } -function isRecord(value: unknown): value is Json { return !!value && typeof value === 'object' && !Array.isArray(value); } +function pageState(root: Document) { + const all: unknown[] = []; + for (const script of Array.from(root.scripts)) { + const text = script.textContent?.trim() ?? ''; + if (text.length < 2 || text.length > 5_000_000) continue; + if (script.type === 'application/ld+json') { + try { + all.push(JSON.parse(text)); + } catch { + /* ignored evidence */ + } + } + for (const marker of [ + 'skuMapOriginal', + 'skuBase', + 'skuCore', + 'skuProps', + ]) { + const object = objectAfterMarker(text, marker); + if (object) all.push(object); + } + } + return all; +} +function objectAfterMarker(text: string, marker: string): unknown | null { + const position = text.indexOf(marker); + if (position < 0) return null; + const start = text.slice(position).search(/[\[{]/); + if (start < 0) return null; + const first = position + start; + const open = text[first]; + const close = open === '{' ? '}' : ']'; + let depth = 0; + let quote = ''; + let escaped = false; + for ( + let index = first; + index < text.length && index - first < 2_000_000; + index += 1 + ) { + const char = text[index]; + if (quote) { + if (escaped) escaped = false; + else if (char === '\\') escaped = true; + else if (char === quote) quote = ''; + continue; + } + if (char === '"' || char === "'") { + quote = char; + continue; + } + if (char === open) depth += 1; + if (char === close && --depth === 0) { + try { + return JSON.parse(text.slice(first, index + 1)); + } catch { + return null; + } + } + } + return null; +} +function findFirst(states: unknown[], key: string): unknown { + for (const state of states) { + const found = findKey(state, key); + if (found !== undefined) return found; + } + return undefined; +} +function findKey( + value: unknown, + key: string, + seen = new Set(), +): unknown { + if (!value || typeof value !== 'object' || seen.has(value)) + return undefined; + seen.add(value); + if (isRecord(value) && key in value) return value[key]; + for (const child of Array.isArray(value) ? value : Object.values(value)) { + const found = findKey(child, key, seen); + if (found !== undefined) return found; + } + return undefined; +} +function parse1688Axes(value: unknown): Axis[] { + const entries = Array.isArray(value) + ? value + : isRecord(value) && Array.isArray(value.skuProps) + ? value.skuProps + : []; + return entries.flatMap((entry, axisIndex) => { + if (!isRecord(entry)) return []; + const axisKey = + stringAt(entry, ['prop', 'propId', 'id']) ?? `axis:${axisIndex}`; + const name = stringAt(entry, ['name', 'propName']) ?? axisKey; + const values = ( + Array.isArray(entry.value) + ? entry.value + : Array.isArray(entry.values) + ? entry.values + : [] + ).flatMap((item, valueIndex) => + isRecord(item) + ? [ + { + key: + stringAt(item, ['valueId', 'vid', 'id']) ?? + `${axisKey}:${valueIndex}`, + name: + stringAt(item, ['name', 'valueName']) ?? + `${valueIndex + 1}`, + ...(urlAt(item, [ + 'imageUrl', + 'image', + 'imageUrlLarge', + ]) + ? { + image_url: urlAt(item, [ + 'imageUrl', + 'image', + 'imageUrlLarge', + ])!, + } + : {}), + }, + ] + : [], + ); + return values.length ? [{ key: axisKey, name, values }] : []; + }); +} +function parseTaobaoAxes(value: unknown): Axis[] { + return (Array.isArray(value) ? value : []).flatMap((entry, index) => { + if (!isRecord(entry)) return []; + const key = stringAt(entry, ['pid', 'propId']) ?? `axis:${index}`; + const name = stringAt(entry, ['name', 'propName']) ?? key; + const values = ( + Array.isArray(entry.values) ? entry.values : [] + ).flatMap((item, valueIndex) => + isRecord(item) + ? [ + { + key: + stringAt(item, ['vid', 'valueId']) ?? + `${key}:${valueIndex}`, + name: + stringAt(item, ['name', 'valueName']) ?? + `${valueIndex + 1}`, + ...(urlAt(item, ['image', 'imageUrl']) + ? { + image_url: urlAt(item, [ + 'image', + 'imageUrl', + ])!, + } + : {}), + }, + ] + : [], + ); + return values.length ? [{ key, name, values }] : []; + }); +} +function selectionsForPropPath(path: string, axes: Axis[]) { + return path.split(/[;,]/).flatMap((pair) => { + const [axisKey, valueKey] = pair.split(':'); + const axis = axes.find((item) => item.key === axisKey); + const value = axis?.values.find((item) => item.key === valueKey); + return axis && value + ? [{ axisKey, valueKey, valueName: value.name }] + : []; + }); +} +function selectionsForCombination(combination: string, axes: Axis[]) { + const ids = combination + .split(/[;,&]/) + .map((value) => value.trim()) + .filter(Boolean); + return axes.flatMap((axis) => { + const value = axis.values.find((item) => ids.includes(item.key)); + return value + ? [ + { + axisKey: axis.key, + valueKey: value.key, + valueName: value.name, + }, + ] + : []; + }); +} +function stringAt(value: Json, keys: string[]) { + for (const key of keys) + if (typeof value[key] === 'string' || typeof value[key] === 'number') + return String(value[key]); + return undefined; +} +function moneyAt(value: Json, keys: string[]) { + const raw = stringAt(value, keys)?.replace(/,/g, '.'); + return raw && /^\d+(\.\d{1,4})?$/.test(raw) ? raw : undefined; +} +function urlAt(value: Json, keys: string[]) { + const raw = stringAt(value, keys); + return raw && /^https:\/\//.test(raw) ? raw : undefined; +} +function uniqueVariants(variants: Variant[]) { + const seen = new Set(); + return variants.filter( + (variant) => + !seen.has(variant.sourceVariantKey) && + !!seen.add(variant.sourceVariantKey), + ); +} +function offerId(url: string) { + return ( + url.match(/\/offer\/(\d+)\.html/)?.[1] ?? + `1688:${new URL(url).pathname}` + ); +} +function itemId(url: string) { + return new URL(url).searchParams.get('id') ?? new URL(url).pathname; +} +function isRecord(value: unknown): value is Json { + return !!value && typeof value === 'object' && !Array.isArray(value); +} diff --git a/apps/extension/package.json b/apps/extension/package.json index 3a26551..f310b13 100644 --- a/apps/extension/package.json +++ b/apps/extension/package.json @@ -6,7 +6,9 @@ "scripts": { "build": "wxt build", "dev": "wxt", - "typecheck": "wxt prepare && tsc -p tsconfig.json --noEmit" + "typecheck": "wxt prepare && tsc -p tsconfig.json --noEmit", + "test": "tsx --test test/*.test.mts", + "test:parsers": "tsx --test test/*.test.mts" }, "dependencies": { "@atlas/contracts": "workspace:*", @@ -17,6 +19,7 @@ "devDependencies": { "@types/react": "19.2.18", "@types/react-dom": "19.2.5", + "tsx": "4.23.13", "typescript": "7.0.2", "wxt": "0.21.4" } diff --git a/apps/extension/test/product-media.test.mts b/apps/extension/test/product-media.test.mts new file mode 100644 index 0000000..7f45a19 --- /dev/null +++ b/apps/extension/test/product-media.test.mts @@ -0,0 +1,91 @@ +import assert from 'node:assert/strict'; +import { readFile } from 'node:fs/promises'; +import test from 'node:test'; + +import { + extractOzonCharacteristics, + extractOzonDescription, + extractOzonGalleryImageUrls, +} from '../lib/ozon-parser.ts'; +import { dedupeImageUrl, normaliseImageUrl } from '../lib/product-media.ts'; + +test('Ozon gallery fixture only returns product-gallery images', async () => { + const fixture = await readFile( + new URL('../../../packages/reference/ozon1.html', import.meta.url), + 'utf8', + ); + const state = fixture.match( + /id="state-webGallery-[^"]+"\s+data-state='([^']+)'/, + )?.[1]; + assert.ok(state, 'Ozon gallery widget state must exist in the fixture'); + + const urls = extractOzonGalleryImageUrls( + JSON.parse(state), + 'https://www.ozon.ru/product/ryukzak-5387005639/', + ); + assert.equal(urls.length, 11); + assert.ok(urls.every((url) => /\/s3\/multimedia-/.test(url))); + assert.ok( + urls.every( + (url) => !/logo_ozon|payments-cdn|recommend|review/i.test(url), + ), + ); +}); + +test('Ozon short characteristics fixture returns its product parameters', async () => { + const fixture = await readFile( + new URL('../../../packages/reference/ozon1.html', import.meta.url), + 'utf8', + ); + const state = fixture.match( + /id="state-webShortCharacteristics-[^"]+"\s+data-state='([^']+)'/, + )?.[1]; + assert.ok(state, 'Ozon short characteristics widget state must exist'); + + const attributes = extractOzonCharacteristics([ + { + ...JSON.parse(state), + __atlasWidgetName: 'webShortCharacteristics-fixture', + }, + ]); + assert.equal(attributes.length, 5); + assert.deepEqual( + attributes.slice(0, 3).map(({ name, value }) => ({ name, value })), + [ + { name: '类型', value: '背包' }, + { name: '原产国', value: '中国' }, + { name: '材料', value: 'Cordura 1000D' }, + ], + ); +}); + +test('Ozon rich annotation converts to description text without images', () => { + const description = extractOzonDescription([ + { + __atlasWidgetName: 'webDescription-fixture', + richAnnotationJson: JSON.stringify([ + { type: 'text', content: '耐用的背包' }, + { type: 'image', src: 'https://ir.ozone.ru/s3/example.jpg' }, + { type: 'text', text: '适合旅行' }, + ]), + }, + ]); + assert.equal(description?.value, '耐用的背包\n适合旅行'); +}); + +test('Ozon thumbnail variants normalise to one original URL', () => { + const original = 'https://ir.ozone.ru/s3/multimedia-1-9/13597160877.jpg'; + const thumbnail = + 'https://ir.ozone.ru/s3/multimedia-1-9/wc100/13597160877.jpg?width=100&quality=80'; + assert.equal(normaliseImageUrl(thumbnail, original, 'ozon'), original); + assert.equal( + dedupeImageUrl(thumbnail, 'ozon'), + dedupeImageUrl(original, 'ozon'), + ); +}); + +test('Alibaba transformed image URLs normalise before de-duplication', () => { + const original = 'https://cbu01.alicdn.com/img/ibank/O1CN01.jpg'; + const transformed = `${original}_400x400.jpg?width=400`; + assert.equal(normaliseImageUrl(transformed, original, '1688'), original); +}); diff --git a/apps/extension/wxt.config.ts b/apps/extension/wxt.config.ts index 2cd811c..18d345e 100644 --- a/apps/extension/wxt.config.ts +++ b/apps/extension/wxt.config.ts @@ -1,5 +1,7 @@ import { defineConfig } from 'wxt'; +const atlasWebOrigin = new URL(process.env.ATLAS_WEB_ORIGIN ?? 'http://localhost:8100').origin; + export default defineConfig({ modules: ['@wxt-dev/module-react'], // 扩展 dev server 固定 8110：默认取 3000–3010 第一个空闲端口，会漂移且易与常见服务冲突 @@ -12,9 +14,14 @@ export default defineConfig({ manifest: { name: 'Atlas Seller Kit Collector', description: '将已确认的来源商品采集到 Atlas 工作空间。', - permissions: ['storage', 'identity'], + // Only the public key is committed. Chrome derives a stable extension ID + // from it for unpacked and self-hosted builds; never put a private key here. + key: 'MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAsPGP9lPew7Xf+xWAc7yIAp+eLjC5SxkqzMXV/9dyn/HtMsskSbVGuYbmg0MWJdi7g3fLe/sa16Q+xp59C+yt11vKGO3a9vho6SfKDnXzkiLBvxKILTBvymnNtk4U/O5LMPWU838TnOZNRIxICJdvp1rF0EO2V8mgOY6cBenXNha1WMYpjfl17ehaKxgrKTFOT63ynTgT6NVbf3sFwi0hFCDH59TfTu5w8iQbYpm+2C+N9cmWfCcnmxtz3TxL+PCtW+RQhfEH1AFCxKZe8JkWqe2XrxLvQHjBlmM4dSvrazA454NhvGkQD41hQqLkfNAooSxLby33t2LWkY7dJmVJDQIDAQAB', + // `tabs` 只用于查询已打开的 Atlas 标签页以同步登录态；不读取其他网页内容。 + permissions: ['storage', 'scripting', 'tabs'], host_permissions: [ 'http://localhost:8200/*', + `${atlasWebOrigin}/*`, '*://*.ozon.ru/*', '*://*.ozon.kz/*', '*://*.ozon.by/*', diff --git a/apps/web/src/pages/collection/CollectionInspectionPage.tsx b/apps/web/src/pages/collection/CollectionInspectionPage.tsx index b2144cf..7c809da 100644 --- a/apps/web/src/pages/collection/CollectionInspectionPage.tsx +++ b/apps/web/src/pages/collection/CollectionInspectionPage.tsx @@ -1,9 +1,10 @@ -import { useEffect, useState } from 'react'; +import { useEffect, useMemo, useState } from 'react'; import { useParams } from 'react-router-dom'; -import { Alert, Card, Collapse, Descriptions, Spin, Table, Tag, Typography } from 'antd'; +import { Alert, Card, Collapse, Descriptions, Image, Spin, Table, Tag, Typography } from 'antd'; import type { CollectionItemInspection } from '@atlas/contracts'; import { collectionApi } from '@/services/collection'; import { apiErrorMessage } from '@/services/api'; +import { mediaApi } from '@/services/media'; /** Temporary parser inspection surface. This is deliberately read-only: it lets * product and parser work be checked before these values become editor fields. */ @@ -11,35 +12,220 @@ export default function CollectionInspectionPage() { const { id } = useParams<{ id: string }>(); const [record, setRecord] = useState(); const [error, setError] = useState(); - useEffect(() => { if (!id) return; void collectionApi.inspection(id).then(setRecord).catch((reason) => setError(apiErrorMessage(reason))); }, [id]); - if (error) return ; - if (!record) return
; - return 只读测试}> - - - SKU 草稿 - value ?? '默认 SKU（待确认）' }, - { title: '来源货号', dataIndex: 'source_sku', render: (value) => value ?? '—' }, - { title: '名称', dataIndex: 'display_name', render: (value) => value ?? '—' }, - ]} /> - 素材转存任务 -
查看来源 }, - { title: '错误', dataIndex: 'error_payload', render: (value) => value ? {JSON.stringify(value)} : '—' }, - ]} /> - }, - { key: 'evidence', label: '原始证据（raw_evidence）', children: }, - ]} /> - ; + const [previewUrls, setPreviewUrls] = useState>({}); + + useEffect(() => { + if (!id) return; + void collectionApi + .inspection(id) + .then(setRecord) + .catch((reason) => setError(apiErrorMessage(reason))); + }, [id]); + + const assetIds = useMemo( + () => [ + ...new Set( + (record?.asset_transfers ?? []) + .filter((transfer) => transfer.status === 'succeeded' && !!transfer.asset_id) + .map((transfer) => transfer.asset_id as string) + ), + ], + [record] + ); + + useEffect(() => { + let disposed = false; + setPreviewUrls({}); + if (!assetIds.length) return; + void loadPreviewUrls(assetIds).then((urls) => { + if (!disposed) setPreviewUrls(urls); + }); + return () => { + disposed = true; + }; + }, [assetIds]); + + const variantsById = useMemo( + () => new Map((record?.variants ?? []).map((variant) => [variant.id, variant])), + [record] + ); + + if (error) + return ; + if (!record) + return ( +
+ +
+ ); + + return ( + 只读测试}> + + + + SKU 草稿 + +
value ?? '默认 SKU（待确认）', + }, + { + title: '来源货号', + dataIndex: 'source_sku', + render: (value) => value ?? '—', + }, + { + title: '名称', + dataIndex: 'display_name', + render: (value) => value ?? '—', + }, + ]} + /> + + 素材转存任务 + +
{ + const url = transfer.asset_id + ? previewUrls[transfer.asset_id] + : undefined; + if (url) + return ( + + ); + return ( + + {transfer.status === 'succeeded' ? '预览加载中…' : '待转存'} + + ); + }, + }, + { title: '类型', dataIndex: 'role' }, + { + title: '关联 SKU', + render: (_, transfer) => { + const variant = transfer.source_variant_id + ? variantsById.get(transfer.source_variant_id) + : undefined; + return variant?.source_sku ?? variant?.source_variant_key ?? '商品级'; + }, + }, + { title: '状态', dataIndex: 'status' }, + { + title: '来源 URL', + dataIndex: 'source_url', + ellipsis: true, + render: (value) => ( + + 查看来源 + + ), + }, + { + title: '错误', + dataIndex: 'error_payload', + render: (value) => + value ? ( + + {JSON.stringify(value)} + + ) : ( + '—' + ), + }, + ]} + /> + , + }, + { + key: 'evidence', + label: '原始证据（raw_evidence）', + children: , + }, + ]} + /> + + ); +} + +async function loadPreviewUrls(assetIds: string[]) { + const urls: Record = {}; + let cursor = 0; + const worker = async () => { + while (cursor < assetIds.length) { + const assetId = assetIds[cursor++]; + try { + urls[assetId] = (await mediaApi.createPreviewUrl(assetId)).url; + } catch { + // The task row remains inspectable through its source URL and error state. + } + } + }; + await Promise.all(Array.from({ length: Math.min(4, assetIds.length) }, () => worker())); + return urls; +} + +function Json({ value }: { value: unknown }) { + return ( + 
+            {JSON.stringify(value, null, 2)}
+
+ ); } -function Json({ value }: { value: unknown }) { return 
{JSON.stringify(value, null, 2)}
; } diff --git a/apps/web/src/services/authToken.ts b/apps/web/src/services/authToken.ts index c8afc8b..16070b2 100644 --- a/apps/web/src/services/authToken.ts +++ b/apps/web/src/services/authToken.ts @@ -1,16 +1,19 @@ /** - * access token 仅在内存中短暂持有（docs/api/conventions.md §1）， - * 不得写入 localStorage 等持久化存储；页面刷新后由 /auth/refresh 通过 - * HttpOnly cookie 恢复会话（接入会话恢复逻辑时实现）。 + * 正常 Web 调用仍只从内存读取 token。为自托管采集插件互操作，维护者 + * 明确批准将当前短期 token 镜像到本站 localStorage；见 + * docs/project/extension-login-alternatives.md，不得扩大到其他调用方。 */ let accessToken: string | null = null; +export const extensionWebAccessStorageKey = 'atlas.extension.web-access'; export const authToken = { get: () => accessToken, set: (token: string) => { accessToken = token; + localStorage.setItem(extensionWebAccessStorageKey, JSON.stringify({ accessToken: token })); }, clear: () => { accessToken = null; + localStorage.removeItem(extensionWebAccessStorageKey); }, }; diff --git a/apps/worker/src/collection-assets.ts b/apps/worker/src/collection-assets.ts index 6c37503..2396a58 100644 --- a/apps/worker/src/collection-assets.ts +++ b/apps/worker/src/collection-assets.ts @@ -6,27 +6,33 @@ import type { Database } from '@atlas/database'; import type { OutboxJob } from './outbox-dispatcher.js'; const MAX_BYTES = 20 * 1024 * 1024; -type Transfer = { id: string; source_url: string; source_variant_id: string | null }; +type Transfer = { id: string; source_url: string; source_variant_id: string | null; source_snapshot_revision: string | number }; /** Downloads source images only after a public-address check. A failed image is * terminal for this transfer, not for the product collection transaction. */ export async function materializeCollectionAssets(database: Database, storage: S3Client, bucket: string, job: OutboxJob) { await ensureBucket(storage, bucket); + const snapshotRevision = Number(job.payload.source_snapshot_revision ?? 1); + if (!Number.isSafeInteger(snapshotRevision) || snapshotRevision < 1) return; const transfers = await database.workspaceTransaction(job.workspaceId, async (tx) => { const rows = await tx.query(`UPDATE atlas.collection_asset_transfers SET status = 'running', updated_at = now() - WHERE id IN (SELECT id FROM atlas.collection_asset_transfers WHERE collection_item_id = $1 AND status = 'queued' ORDER BY created_at FOR UPDATE SKIP LOCKED) - RETURNING id, source_url, source_variant_id`, [String(job.payload.collection_item_id)]); + WHERE id IN (SELECT id FROM atlas.collection_asset_transfers WHERE collection_item_id = $1 AND source_snapshot_revision = $2 AND status = 'queued' ORDER BY created_at FOR UPDATE SKIP LOCKED) + RETURNING id, source_url, source_variant_id, source_snapshot_revision`, [String(job.payload.collection_item_id), snapshotRevision]); return rows.rows; }); for (const transfer of transfers) await materializeOne(database, storage, bucket, job.workspaceId, transfer); await database.workspaceTransaction(job.workspaceId, async (tx) => { - await tx.query(`UPDATE atlas.collection_items SET status = CASE WHEN EXISTS (SELECT 1 FROM atlas.collection_asset_transfers WHERE collection_item_id = $1 AND status = 'running') THEN 'materializing'::atlas.collection_item_status ELSE 'ready_for_edit'::atlas.collection_item_status END, updated_at = now(), revision = revision + 1 WHERE id = $1 AND deleted_at IS NULL`, [String(job.payload.collection_item_id)]); + await tx.query(`UPDATE atlas.collection_items SET status = CASE WHEN EXISTS (SELECT 1 FROM atlas.collection_asset_transfers WHERE collection_item_id = $1 AND source_snapshot_revision = $2 AND status = 'running') THEN 'materializing'::atlas.collection_item_status ELSE 'ready_for_edit'::atlas.collection_item_status END, updated_at = now(), revision = revision + 1 WHERE id = $1 AND source_snapshot_revision = $2 AND deleted_at IS NULL`, [String(job.payload.collection_item_id), snapshotRevision]); }); } async function materializeOne(database: Database, storage: S3Client, bucket: string, workspaceId: string, transfer: Transfer) { + const snapshotRevision = Number(transfer.source_snapshot_revision); try { const downloaded = await downloadImage(transfer.source_url); + const current = await database.workspaceTransaction(workspaceId, async (tx) => (await tx.query<{ id: string }>(`SELECT t.id FROM atlas.collection_asset_transfers t JOIN atlas.collection_items c ON c.id = t.collection_item_id + WHERE t.id = $1 AND t.status = 'running' AND t.source_snapshot_revision = $2 AND c.source_snapshot_revision = $2`, [transfer.id, snapshotRevision])).rows[0]); + if (!current) return; const checksum = createHash('sha256').update(downloaded.bytes).digest('hex'); const objectKey = `collections/${workspaceId}/${checksum}`; await storage.send(new PutObjectCommand({ Bucket: bucket, Key: objectKey, Body: downloaded.bytes, ContentType: downloaded.contentType })); @@ -34,15 +40,15 @@ async function materializeOne(database: Database, storage: S3Client, bucket: str const existing = await tx.query<{ id: string }>('SELECT id FROM atlas.media_assets WHERE checksum_sha256 = $1', [checksum]); const assetId = existing.rows[0]?.id; if (assetId) { - await tx.query(`UPDATE atlas.collection_asset_transfers SET status = 'succeeded', asset_id = $2, completed_at = now(), updated_at = now(), error_payload = NULL WHERE id = $1 AND status = 'running'`, [transfer.id, assetId]); + await tx.query(`UPDATE atlas.collection_asset_transfers SET status = 'succeeded', asset_id = $2, completed_at = now(), updated_at = now(), error_payload = NULL WHERE id = $1 AND status = 'running' AND source_snapshot_revision = $3`, [transfer.id, assetId, snapshotRevision]); return; } const asset = await tx.query<{ id: string }>(`INSERT INTO atlas.media_assets (workspace_id, storage_provider, bucket, object_key, content_type, byte_size, checksum_sha256, width_px, height_px) VALUES (current_setting('app.workspace_id')::uuid, 'local_minio', $1, $2, $3, $4, $5, $6, $7) RETURNING id`, [bucket, objectKey, downloaded.contentType, downloaded.bytes.byteLength, checksum, downloaded.width, downloaded.height]); - await tx.query(`UPDATE atlas.collection_asset_transfers SET status = 'succeeded', asset_id = $2, completed_at = now(), updated_at = now(), error_payload = NULL WHERE id = $1 AND status = 'running'`, [transfer.id, asset.rows[0].id]); + await tx.query(`UPDATE atlas.collection_asset_transfers SET status = 'succeeded', asset_id = $2, completed_at = now(), updated_at = now(), error_payload = NULL WHERE id = $1 AND status = 'running' AND source_snapshot_revision = $3`, [transfer.id, asset.rows[0].id, snapshotRevision]); }); } catch (error) { - await database.workspaceTransaction(workspaceId, (tx) => tx.query(`UPDATE atlas.collection_asset_transfers SET status = 'failed', completed_at = now(), updated_at = now(), error_payload = jsonb_build_object('code', 'SOURCE_ASSET_DOWNLOAD_FAILED', 'message', $2) WHERE id = $1 AND status = 'running'`, [transfer.id, error instanceof Error ? error.message.slice(0, 300) : '素材下载失败'])); + await database.workspaceTransaction(workspaceId, (tx) => tx.query(`UPDATE atlas.collection_asset_transfers SET status = 'failed', completed_at = now(), updated_at = now(), error_payload = jsonb_build_object('code', 'SOURCE_ASSET_DOWNLOAD_FAILED', 'message', $2) WHERE id = $1 AND status = 'running' AND source_snapshot_revision = $3`, [transfer.id, error instanceof Error ? error.message.slice(0, 300) : '素材下载失败', snapshotRevision])); } } diff --git a/docs/README.md b/docs/README.md index 45109dd..1db480d 100644 --- a/docs/README.md +++ b/docs/README.md @@ -32,8 +32,9 @@ 12. [`architecture/database-security.md`](./architecture/database-security.md)：迁移/应用数据库角色、RLS 强制策略与部署职责。 13. [`architecture/transactional-outbox-and-idempotency.md`](./architecture/transactional-outbox-and-idempotency.md)：关键写请求去重、事务 Outbox、BullMQ 投递与验收规则。 14. [`guides/object-storage-object-key.md`](./guides/object-storage-object-key.md)：对象存储、`object_key`、来源 URL 与预签名 URL 的使用边界。 -15. [`guides/minio-local-deployment.md`](./guides/minio-local-deployment.md)：MinIO 简介、本地 Docker 部署与应用侧使用方法。 -16. [`guides/postgresql-local-development.md`](./guides/postgresql-local-development.md)：原生 PostgreSQL / Docker Compose 本地数据库、角色、迁移与建表。 -17. [`api/README.md`](./api/README.md)：HTTP API 统一约定及第一批账户、门店、水印接口。 -18. [`reviews/implementation-readiness-open-items.md`](./reviews/implementation-readiness-open-items.md)：实现前必须闭环的数据库/API 事项与已确认约束。 +15. [`guides/browser-extension-background-service-worker.md`](./guides/browser-extension-background-service-worker.md)：浏览器扩展后台 Service Worker 的运行模型、消息调用链与调试方法。 +16. [`guides/minio-local-deployment.md`](./guides/minio-local-deployment.md)：MinIO 简介、本地 Docker 部署与应用侧使用方法。 +17. [`guides/postgresql-local-development.md`](./guides/postgresql-local-development.md)：原生 PostgreSQL / Docker Compose 本地数据库、角色、迁移与建表。 +18. [`api/README.md`](./api/README.md)：HTTP API 统一约定及第一批账户、门店、水印接口。 +19. [`reviews/implementation-readiness-open-items.md`](./reviews/implementation-readiness-open-items.md)：实现前必须闭环的数据库/API 事项与已确认约束。 19. 后续按模块补全定价、采集、商品编辑、发布和在线商品接口。 diff --git a/docs/api/collection-and-extension.md b/docs/api/collection-and-extension.md index bec1245..860a493 100644 --- a/docs/api/collection-and-extension.md +++ b/docs/api/collection-and-extension.md @@ -4,42 +4,19 @@ ## 1. 浏览器扩展授权 -扩展不读取 ERP 网页 localStorage，也不用 Web 端 refresh cookie。每个发布扩展在服务端登记 `client_id` 和允许的 `chrome-extension://{extension-id}/…` 回调地址，扩展使用 PKCE 授权码流程。 +首期按毛子 ERP 风格复用 Atlas Web 登录态。用户从插件点击登录后，扩展打开固定 Atlas Web 页面；用户在该页完成登录或会话恢复后，Web 将当前短期 access token 镜像到本站 `localStorage["atlas.extension.web-access"]`。扩展每 2 秒仅从该 Atlas 标签读取这个值，保存到 Chrome extension storage，并以 Bearer token 调用下述接口。 -1. 扩展打开：`GET /api/extension/authorize?client_id={id}&redirect_uri={uri}&state={random}&code_challenge={S256}&code_challenge_method=S256`。 -2. 用户在 Atlas Web 完成登录；首期不额外展示工作空间选择/授权确认页。 -3. 服务端只向已登记的 `redirect_uri` 回跳 `?code={one-time}&state={original}`。授权码 60 秒有效、只能交换一次，且绑定 client、回调地址和 PKCE challenge。 -4. 扩展调用 `POST /api/extension/auth/exchange`： - -```json -{ - "client_id": "atlas-collector", - "code": "one-time-code", - "redirect_uri": "chrome-extension://extension-id/callback", - "code_verifier": "original-pkce-verifier" -} -``` - -成功返回扩展专用短期 access token 与可轮换 refresh token；令牌只存 Chrome 扩展专属存储： - -```json -{ "data": { - "access_token": "eyJ…", "expires_in": 900, "refresh_token": "opaque", "refresh_token_expires_in": 2592000, - "token_type": "Bearer" -} } -``` - -`POST /api/extension/auth/refresh` 接收 `{ "refresh_token": "opaque" }`，轮换并返回同形状令牌；`POST /api/extension/auth/revoke` 撤销当前扩展 refresh token，返回 `204`。令牌续期、撤销和错误响应不得泄露 Web 会话或其他工作空间信息。 +这是维护者批准的自托管插件例外：扩展不读取 refresh cookie、Cookie、来源站 localStorage 或来源站授权信息；Web token 不得进入日志、数据库、Outbox 或采集载荷。Token 失效返回 `401` 后，插件清除自己的缓存并要求用户再次从 Atlas Web 登录同步。完整取舍见 [`../project/extension-login-alternatives.md`](../project/extension-login-alternatives.md)。 `GET /api/extension/session` -使用扩展 access token，返回最小化状态： +使用从 Atlas Web 同步的 Bearer token，返回最小化状态： ```json { "data": { "authenticated": true, "workspace": { "id": "uuid", "available": true }, "can_collect": true } } ``` -不返回平台、默认店铺、Cookie、Ozon 凭据或 Web refresh token。令牌失效返回 `401`，插件只展示“登录 Atlas ERP”。 +不返回平台、默认店铺、Cookie、Ozon 凭据或 Web refresh token。令牌失效返回 `401`，插件清除缓存并只展示“登录 Atlas ERP”。 ## 2. 插件上报采集结果 @@ -49,14 +26,14 @@ Headers：扩展 Bearer token 与 `Idempotency-Key` 必填。请求体为已定 请求体不得包含或传入 `workspace_id`、`target_platform_id`、`platform_id`、`store_id`、`api_key`、Cookie、授权头或来源站会话字段。服务端从扩展令牌解析工作空间，并将来源 `platform` 映射为系统已启用的来源平台。 -服务端在一个事务中：检查工作空间采集箱上限 → 按 `workspace + source_platform + source_item_key` 查找活动记录 → 创建或复用 `collection_item`、`product`、来源确认的 `product_variants` 和持久化 `collection_asset_transfers` → 再经 outbox 投递素材 Worker。重复上报不会增加采集箱配额，也不覆盖用户已编辑草稿。 +服务端在一个事务中：检查工作空间采集箱上限 → 按 `workspace + source_platform + source_item_key` 查找活动记录 → 创建记录，或以本次页面快照覆盖已有记录的来源 URL、标题、价格、原始证据、规范化载荷、SKU 草稿与候选素材 → 再经 outbox 投递素材 Worker。重复上报不会增加采集箱配额，且会保留原有 `collection_item` 与 `product` ID；旧素材任务被取消，新素材任务使用新的来源快照版本，旧 Worker 不得回写新快照。 成功返回 `201`（新建）或 `200`（复用）： ```json { "data": { "collection_item": { "id": "uuid", "status": "materializing", "source_title": "商品标题", "variant_count": 3 }, - "product_id": "uuid", "reused": false, + "product_id": "uuid", "reused": false, "refreshed": false, "editor_entry_path": "/products/uuid", "materialization": { "total": 12, "queued": 12, "succeeded": 0, "failed": 0 } } } diff --git a/docs/guides/browser-extension-background-service-worker.md b/docs/guides/browser-extension-background-service-worker.md new file mode 100644 index 0000000..7f1180a --- /dev/null +++ b/docs/guides/browser-extension-background-service-worker.md @@ -0,0 +1,73 @@ +# 浏览器扩展的 Background Service Worker + +> 本文以 Atlas 采集插件为例，说明 Chrome Manifest V3 扩展的后台运行环境、消息调用链与调试方法。 + +## 1. 它是什么 + +`Background Service Worker` 是 Chrome Manifest V3 扩展的后台运行环境。它不属于任何网页，也没有固定 UI；Chrome 在收到消息、标签页事件等需要处理的事件时唤醒它，空闲后停止它以节省资源。 + +因此，后台代码不能假设进程始终存活，也不能只用模块变量保存登录态或业务状态。需要跨唤醒保留的数据应写入 `chrome.storage`；本项目使用 `chrome.storage.local` 保存插件同步得到的短期登录令牌。 + +## 2. Atlas 采集插件中的调用链 + +```text +商品网页（Content Script） + 点击“编辑上架” + │ browser.runtime.sendMessage(...) + ▼ +扩展 Background Service Worker + 读取 chrome.storage 中的登录令牌 + fetch Atlas API，并附加 Authorization + │ + ▼ +Atlas API + │ + ▼ +Background 将结果回传给 Content Script + 打开采集测试页 / 展示错误 +``` + +项目中三个环境彼此隔离： + +| 环境 | 运行位置 | 负责内容 | +|---|---|---| +| 商品网页 | Ozon、1688、淘宝、天猫等页面 | 原网站 DOM 与原网站 Network 请求 | +| Content Script | 注入商品网页的扩展脚本 | 悬浮窗、商品解析、向后台发送消息 | +| Background Service Worker | `chrome-extension://…` 扩展上下文 | 登录令牌、跨站 API 请求、打开标签页 | + +采集请求由 Background 发出，所以它不会出现在商品网页 DevTools 的 Network 面板。商品页 DevTools 只能看到来源网站自身的网络请求。 + +## 3. 为什么采用后台请求 + +- 登录令牌保留在扩展存储中，不必暴露给来源商品页的 JavaScript。 +- Content Script 仅负责页面交互和净化后的商品载荷；API 鉴权、请求和跳转集中在后台处理。 +- 后台拥有扩展声明的 `host_permissions`，可访问 Atlas API 与受支持来源站点；页面本身的跨域限制不会决定插件后台请求的权限。 + +后台权限仍应遵循最小化原则：本项目只允许访问 Atlas API、Atlas Web 与明确支持的来源站点，绝不读取来源站 Cookie 或 localStorage。 + +## 4. 前端如何调试 + +### 4.1 调试后台 API 请求 + +1. 打开 `chrome://extensions`，开启“开发者模式”。 +2. 找到 Atlas 采集插件，点击卡片中的 **Service Worker** 或“检查视图”。 +3. 在打开的 DevTools 中查看： + - **Console**：后台异常、消息处理失败； + - **Network**：`/api/extension/session`、`/api/extension/materials` 等 Atlas API 请求、状态码与响应体； + - **Application → Storage → Extension storage**：插件保存的登录态数据。 + +这里才是排查 `Authorization` 请求头、API 4xx/5xx，以及采集写入失败的正确位置。 + +### 4.2 调试悬浮窗和商品解析 + +在商品详情页按 F12，使用该页面的 DevTools： + +- **Console**：查看 Content Script 的页面解析、悬浮窗和 `sendMessage` 调用错误； +- **Sources**：寻找 `chrome-extension://…/content-scripts/collector.js`，可对商品解析或点击处理函数下断点； +- **Network**：只用于分析 Ozon、1688、淘宝、天猫页面自身的请求与页面数据来源，不应期待看到 Atlas API 请求。 + +### 4.3 Service Worker 生命周期注意事项 + +Service Worker 空闲后会被 Chrome 停止；再次刷新商品页、点击悬浮窗或收到扩展事件时才会重新唤醒。调试时建议在 Service Worker DevTools 勾选“保留日志”，避免后台停止后丢失 Console 输出。 + +不要依赖 Service Worker 模块变量存储登录状态或待处理任务。需要持久化的数据应使用 `chrome.storage`、后端数据库或其他可恢复的存储；异步操作应能安全地在下一次后台唤醒后继续或重试。 diff --git a/docs/project/HANDOFF.md b/docs/project/HANDOFF.md index 6619dce..da6cde5 100644 --- a/docs/project/HANDOFF.md +++ b/docs/project/HANDOFF.md @@ -1,12 +1,12 @@ # Atlas Seller Kit 交接说明 -> 更新：2026-09-06。此文是新 AI 对话、架构讨论与开发工作的首读入口。开始开发前，必须先阅读并遵守 [`development-collaboration-contract.md`](./development-collaboration-contract.md)。 +> 更新：2026-09-07。此文是新 AI 对话、架构讨论与开发工作的首读入口。开始开发前，必须先阅读并遵守 [`development-collaboration-contract.md`](./development-collaboration-contract.md)。 ## 1. 项目定位 这是一个全新的云端产品，不是 `ozon-seller-kit` 的本地版重构。目标用户可拥有多个门店，每个门店只能绑定一个平台；平台自身可支持多个市场、国家、语言和币种。因此语言、币种、类目和发布规则不能写死为 Ozon 或中文/俄文。 -当前已完成 pnpm monorepo 初始化：包含 Web、NestJS API、BullMQ Worker、WXT 浏览器扩展、共享 API 契约、数据库包、本地基础设施 Compose 与依赖锁文件。API 已实现健康/readiness、集中配置与安全基础、认证、个人工作空间上下文、平台目录/上下文、门店、水印/素材、采集箱与扩展采集入库；Worker 已具备 Ozon 凭据验证、采集记录清理和来源素材转存消费链路，仓库同步尚未接入。Web 已具备完整前端框架（布局、路由、主题）、登录/注册页、页头平台选择器、店铺管理、水印管理、真实采集箱页面和采集数据测试页；浏览器扩展已提供 PKCE 授权、悬浮窗及 Ozon/1688/淘宝/天猫首版页面状态解析。不要从旧项目复制 SQLite 模型、无鉴权 API、旧 `/trial/:id` 或商品编辑页组件。可选择性复用已经验证过的纯计价公式、采集扩展思路和 AI 任务经验，但必须先适配多租户和云端边界。 +当前已完成 pnpm monorepo 初始化：包含 Web、NestJS API、BullMQ Worker、WXT 浏览器扩展、共享 API 契约、数据库包、本地基础设施 Compose 与依赖锁文件。API 已实现健康/readiness、集中配置与安全基础、认证、个人工作空间上下文、平台目录/上下文、门店、水印/素材、采集箱与扩展采集入库；Worker 已具备 Ozon 凭据验证、采集记录清理和来源素材转存消费链路，仓库同步尚未接入。Web 已具备完整前端框架（布局、路由、主题）、登录/注册页、页头平台选择器、店铺管理、水印管理、真实采集箱页面和采集数据测试页；浏览器扩展已提供毛子 ERP 风格的网页登录态同步、悬浮窗及 Ozon/1688/淘宝/天猫首版页面状态解析。不要从旧项目复制 SQLite 模型、无鉴权 API、旧 `/trial/:id` 或商品编辑页组件。可选择性复用已经验证过的纯计价公式、采集扩展思路和 AI 任务经验，但必须先适配多租户和云端边界。 当前运行时基线：Node 24.20.0、pnpm 11.25.0、PostgreSQL 18.6、Redis 7.4.11、MinIO `RELEASE.2025-09-07T16-13-09Z`；Web 默认端口为 `8100`，API 默认端口为 `8200`，扩展 dev server 默认端口为 `8110`（仅本地开发用），Worker 无 HTTP 端口。完整目录、依赖边界与版本表见 [`architecture/project-structure.md`](../architecture/project-structure.md) 和 [`architecture/technology-and-deployment.md`](../architecture/technology-and-deployment.md)。 @@ -89,7 +89,7 @@ ### 1.9 开发进度（2026-09-05）：M2.2 / M2.3 扩展采集首个闭环 -- 已实现扩展 PKCE 授权码、扩展专用 access/refresh token 轮换、现有 Atlas Web 会话的静默授权分支，以及扩展会话检查/撤销。扩展不读取 Web token 或 refresh Cookie；回调 URI 必须按已安装扩展 ID 精确登记。 +- 已实现毛子 ERP 风格的 Atlas Web 登录态同步：扩展打开 Atlas Web 标签，轮询读取本站 localStorage 中的短期 Web access token，保存到 extension storage 并用 Bearer token 检查会话/采集。扩展不读取 refresh Cookie、来源站 Cookie 或来源站 localStorage。 - 已实现 `POST /api/extension/materials`：从扩展身份推导工作空间，创建或复用未绑定目标平台/门店的采集记录、SPU、SKU 草稿和来源素材任务。采集载荷严格限制为来源证据、规范化字段和素材 URL，不能携带店铺、Cookie、API Key 或目标平台。 - Worker 已消费素材转存 outbox 事件，执行 HTTPS 公网地址校验、大小/图片格式限制、哈希去重与 MinIO/Kodo 转存；单个素材失败不回滚采集箱商品。 - 浏览器扩展已实现 Ozon 公开商品页的小型 Shadow DOM 悬浮窗及未登录/已登录状态。Ozon 首版解析器以当前公开 SKU 和公开变体状态为证据，不生成笛卡尔积假 SKU。Ozon Seller 四接口本地观察、Ozon 变体详情逐页补全及 1688/淘宝/天猫解析仍未开始。 @@ -100,7 +100,32 @@ - 扩展浮窗现覆盖 1688 商品详情、淘宝商品详情和天猫商品详情。1688 仅遍历 `skuMapOriginal` 中实际存在的组合；淘宝/天猫仅在同时取得 `skuBase.skus[].skuId + propPath` 时创建多 SKU，并以 `skuCore.sku2info` 补充参考价格。规格轴本身从不展开为笛卡尔积。 - 缺少真实组合证据时，扩展只保存规格轴，服务端创建一个默认 SKU 并标记为待确认；颜色等轴值图片仍作为候选素材，绝不伪造来源 SKU 图片绑定。 - “编辑上架”暂时不进入未完成的商品编辑模块，改为打开 `/collection-test/{collectionItemId}`。该只读测试页展示实际保存的规范化载荷、原始证据、SKU 草稿和素材转存任务，用于验证每个平台的采集结果。`GET /api/collection-items/{id}` 已提供该工作空间隔离的检查读模型。 -- 已通过 API/Worker TypeScript 检查、Web 与扩展生产构建；`apps/api/test/extension.e2e.test.ts` 已覆盖 PKCE 授权、扩展会话、采集幂等复用和测试页所用检查读模型。尚未在真实来源商品页执行样本回归，接手者应按第 6 节先完成该验证。 +- 已通过 API/Worker TypeScript 检查、Web 与扩展生产构建；`apps/api/test/extension.e2e.test.ts` 已覆盖扩展会话、采集幂等复用和测试页所用检查读模型。尚未在真实来源商品页执行样本回归，接手者应按第 6 节先完成该验证。 + +### 1.11 开发进度（2026-09-07）：扩展网页登录态同步与稳定 ID + +- 扩展登录采用毛子 ERP 模式：点击后打开并保留 Atlas Web 标签；Web 登录或恢复会话后写入本站 localStorage，扩展每 2 秒读取并验证该短期 Web Bearer token。没有有效网页会话时，用户在同一标签登录后即可完成插件登录。 +- Atlas Web access token 写入 localStorage 是维护者批准的窄范围例外；扩展不读取 refresh Cookie、来源站 Cookie 或来源站 localStorage。Token 失效后插件清除缓存并重新要求网页登录。 +- 不再需要 `EXTENSION_COLLECTOR_REDIRECT_URI` 或 `identity` 权限；方案 A 的网页确认 + PKCE 配对设计单独记录在 [`extension-login-alternatives.md`](./extension-login-alternatives.md)，当前不作为插件默认路径。 +- `apps/extension/wxt.config.ts` 的 manifest 带有仅公开的 RSA `key`，Chrome 为解压加载和自托管包保持稳定扩展 ID；私钥未写入仓库。`wxt build` 已确认最终 `manifest.json` 含该 key。 +- 已通过 contracts、API（含测试 TS）、Web、扩展的类型/生产构建，以及 `apps/api/test/extension.e2e.test.ts`：Web Bearer 会话检查 → 工作空间解析 → 采集幂等上报。 +- 同一工作空间内的同一来源商品再次采集时，系统保留 `collection_item` / `product` ID，但用最新页面快照覆盖来源标题、价格、证据、SKU 与候选素材。迁移 `013_collection_source_snapshot_refresh.sql` 为记录和素材任务增加快照版本；旧转存任务取消，Worker 仅处理当前版本，不能将旧页面内容回写到新快照。 +- Ozon 公开商品页解析器现从 JSON-LD `BreadcrumbList` 提取来源类目（可见面包屑为回退），保存当前页面语言的类目路径及对应面包屑证据；解析器版本为 `ozon-public-1.1.0`。 + +### 1.12 开发进度（2026-09-07）：Ozon 公开多 SKU 采集与归并 + +- 解析器改为优先读取 Ozon 已渲染的 `state-webAspects-*.data-state.aspects[].variants[]`，不再用脚本文本正则猜测 SKU。保存的三 SKU 页面快照 `packages/reference/ozon1.html` 已确认能直接得到 `5387004839`、`5387005639`、`5387005221`。 +- 增加只限 Ozon MAIN world 的页面 JSON bridge：只接受数字 SKU 和固定的 `modal/aspectsNew` / `product/` 路径，白名单返回 `webAspects`、`webProductHeading`、`webPrice`、`webGallery` 状态；不暴露 Cookie、请求头或任意请求能力。扩展以并发 3、最多 100 SKU 队列补全实际变体详情；个别失败保留已发现变体和证据。 +- Ozon 公开页面没有稳定 SPU/组 ID；当前 URL SKU 仅保存为入口证据。API 对 Ozon 用本次 `source_sku` 集合与已有 `product_variants.source_sku` 的交集复用并覆盖同一采集快照，允许用户从同组任一 SKU 页面重复采集。多个既有记录命中时返回 `SOURCE_VARIANT_GROUP_AMBIGUOUS`，不静默合并。 + +### 1.13 开发进度（2026-09-07）：Ozon 商品特征、描述与素材预览 + +- `/collection-test/{collectionItemId}` 的“素材转存任务”现在直接展示转存成功素材的缩略图，并显示关联来源 SKU。页面为每个 `succeeded + asset_id` 的任务按 4 路并发申请素材短时预览 URL；URL 只保存在浏览器内存，不能写回采集记录、产品或对象存储路径。没有 SKU 直接关联的规格轴图片会明确显示为“商品级”。已在现有三 SKU 采集记录 `7172fa38-0e4d-485e-9d53-1bd92aced79a` 实测确认 37 张素材可显示。 +- Ozon 解析器升级为 `ozon-public-2.2.0`。本地页面优先读取 `webShortCharacteristics`、`webDetailedCharacteristics`、`webCharacteristics` 的 `characteristics[]`；当前入口 SKU 的 `/product//` 补全响应会再读取完整 `webCharacteristics`，去重后写入 `common.attributes`。这沿用旧插件已验证的数据形状，不通过页面文本或全页 DOM 猜测参数。 +- 固定的 Ozon 详情页段仍只读取 `webDescription` / `webRichContent`。其中 `webDescription.richAnnotationJson` 被转换为纯文本写入 `common.description`，原 JSON 同时保留在 `raw_value` 供检查；富文本中的图片只走既有详情素材白名单，不会混入描述文本。 +- MAIN world bridge 的返回白名单已经增加上述三个 Characteristics widget，仍然只接受固定 Ozon 同源路径和数字 SKU，不暴露 Cookie、请求头或任意请求能力。 +- **兼容性提醒：** 已有采集记录不会自动重新解析历史 `normalized_payload`。例如上面的现有记录仍标记为 `ozon-public-2.1.0`，只能验证新素材预览；需重新在来源商品页执行“编辑上架”，系统会覆盖同一采集快照，届时在规范化载荷查看新版属性与描述。 +- 已通过扩展 `wxt prepare + tsc + build`、Web `vite build`、`tsx --test test/*.test.mts`（5 项，包括 Ozon 页面快照中“类型/原产国/材料”等 5 个特征和富文本描述样本）以及浏览器测试页实测。 ## 2. 已确认产品决策 @@ -162,10 +187,11 @@ User → Workspace → Membership / Role 下一步建议按以下顺序推进： -1. 用允许保存的单 SKU、单规格轴和颜色×尺码页面样本，在 Ozon、1688、淘宝、天猫逐项验证采集结果；页面状态未暴露组合数据时补 MAIN world bridge 或站点适配器，但绝不以规格轴猜测 SKU。 -2. 完成 Ozon 公开变体队列的逐 SKU hydrate；再独立实施 Seller 网页四接口本地观察实验，默认关闭、仅保存净化结果且不影响标准采集。 +1. 在真实 Ozon 多 SKU 页面加载已构建扩展，确认 MAIN world bridge 未被页面策略阻断、三 SKU 都写入采集测试页、SKU 级画廊关系正确，并在新版规范化载荷确认完整特征与描述；再以任一兄弟 SKU 页面重采集，确认服务端覆盖同一记录。 +2. 用允许保存的单 SKU、单规格轴和颜色×尺码页面样本，在 1688、淘宝、天猫逐项验证采集结果；页面状态未暴露组合数据时补站点适配器，但绝不以规格轴猜测 SKU。之后独立实施 Seller 网页四接口本地观察实验，默认关闭、仅保存净化结果且不影响标准采集。 3. 将当前 `/collection-test/{collectionItemId}` 只读检查面替换为正式商品编辑入口：按页头平台创建发布配置、预选同平台默认门店，并接入 SKU/媒体/类目编辑。 4. 进入商品编辑/在线同步前，关闭评审中的 P1：SKU 与平台发布配置同商品归属、未映射属性持久化、统一乐观锁 Repository 规则；平台能力 Schema 已在 M2.1 按固定键落地。在线商品归档策略和 Client ID 编辑边界仍待确认。 +5. **账户认证待开发：忘记密码。** 登录页提供入口；请求接口始终返回同一结果以防邮箱枚举，使用短时有效的一次性不透明邮件令牌（数据库仅存哈希），确认重置后撤销该用户全部 session。实施前先补充认证 API 契约、邮件投递配置和迁移；交付 API、Web、邮件链接与限流/令牌失效回归测试。详情见 [`requirements/account-store-watermark.md`](../requirements/account-store-watermark.md) §2.3。 账户资料的已确认补充：`users.avatar_asset_id` 只保存素材引用，绝不保存公开 URL；`users.revision` 用于 `PATCH /api/auth/me` 的 `If-Match`。资料更新已实现幂等与版本冲突检测；修改密码已实现旧密码校验、按 IP/用户限流、撤销旧会话并轮换新会话。头像上传必须复用素材模块的预签名直传与校验流程，不引入 base64 直传；素材模块尚未实现时头像只能为 `null`。 diff --git a/docs/project/development-collaboration-contract.md b/docs/project/development-collaboration-contract.md index 5b94a3a..f14c17a 100644 --- a/docs/project/development-collaboration-contract.md +++ b/docs/project/development-collaboration-contract.md @@ -42,7 +42,7 @@ 1. PostgreSQL 是业务事实来源。所有业务数据必须有 `workspace_id`，或能经强制外键安全追溯到工作空间；查询必须在事务中设置 `SET LOCAL app.workspace_id`。 2. 数据库约束、RLS 与迁移是安全边界；不得只依赖 API 服务层校验。迁移必须可重复执行、可在全新本地环境验证。 3. 关键可重试写入必须实现幂等；触发异步工作的事务必须同时写入 Transactional Outbox。Redis/BullMQ 不得成为业务事实的唯一来源，Worker 必须幂等。 -4. access token 不得写入浏览器持久化存储；Cookie、API Key、预签名 URL 等秘密不得写进日志、Outbox 载荷、fixture 或提交进仓库。 +4. access token 不得写入浏览器持久化存储；Cookie、API Key、预签名 URL 等秘密不得写进日志、Outbox 载荷、fixture 或提交进仓库。**维护者已于 2026-09-07 批准唯一例外：**为自托管采集插件的网页登录态同步，Atlas Web 可在本站 `localStorage["atlas.extension.web-access"]` 写入当前短期 access token，扩展仅可从 Atlas Web 标签读取并保存到 extension storage；不得读取/写入 refresh Cookie，生产 host permission 必须限制为 Atlas Web 域。完整取舍见 [`extension-login-alternatives.md`](./extension-login-alternatives.md)。 5. 不得从旧 `ozon-seller-kit` 复制 SQLite、多店铺无隔离、无鉴权 API 或旧商品编辑实现。复用逻辑前必须适配多租户与云端边界。 ## 5. 工作方式与集成规则 diff --git a/docs/project/extension-login-alternatives.md b/docs/project/extension-login-alternatives.md new file mode 100644 index 0000000..0f22c02 --- /dev/null +++ b/docs/project/extension-login-alternatives.md @@ -0,0 +1,31 @@ +# 浏览器采集插件登录方案备忘 + +> 状态：2026-09-07 记录。本文保留备选方案，不代表当前默认实现。 + +## 方案 A：网页确认 + PKCE 配对（备选，暂不启用） + +1. 插件创建 3 分钟有效的连接请求，生成本地 PKCE verifier/challenge。 +2. 插件打开固定的 Atlas `/extension-connect` 网页。 +3. 网页复用 HttpOnly Web 会话确认连接；未登录时登录后继续。 +4. 插件轮询请求状态，凭 verifier 换取独立 access/refresh token。 + +优点：Web refresh cookie 和 Web access token 永不交给扩展；扩展令牌可独立撤销。缺点：需要配对表、轮询端点、确认页、状态机和失败回收，当前阶段对“登录后采集”的简单需求而言复杂度过高。 + +## 方案 B：毛子 ERP 风格的网页登录态同步（当前采用） + +1. 用户在插件点击登录。 +2. 插件打开固定 Atlas ERP 网页；未登录时用户在该网页完成登录。 +3. Web 登录或刷新会话后，把当前 access token 写到本站 `localStorage["atlas.extension.web-access"]`。 +4. 插件每 2 秒从该 Atlas 网页标签读取该值；读取到后写入自己的 `chrome.storage.local`。 +5. 插件用该 Bearer token 调用扩展会话校验接口；校验成功即显示采集操作。 +6. 后续扩展 API 请求经后台统一附加 `Authorization: Bearer `。 + +该流程不依赖 `chrome.identity`、`chromiumapp.org` 回调 URL 或用户配置的扩展 ID；扩展清单的公开 `key` 仅用于稳定扩展 ID。 + +## 安全取舍与边界 + +- 这是对项目“Web access token 不落持久化存储”规则的**维护者批准的、窄范围例外**，仅为自托管采集插件互操作。 +- Token 仅写入 Atlas Web 自己的 origin，扩展只通过 `scripting.executeScript` 从其被打开的 Atlas 标签读取；不会读取 Cookie、refresh token、来源站点账号或来源站点 localStorage。 +- 插件仍只取得短期 Web access token；收到 401 时清除插件缓存，用户需再次从 Atlas 网页同步新 Token。 +- 当前阶段不应把该 Token 写入日志、数据库、Outbox、采集载荷或页面 DOM；生产部署必须将扩展 host permission 限定为正式 Atlas Web 域。 +- 方案 A 保留为未来需要独立可撤销插件会话时的升级路径。 diff --git a/docs/project/m2.2-m2.3-extension-collection.md b/docs/project/m2.2-m2.3-extension-collection.md index 976a65f..ecdc952 100644 --- a/docs/project/m2.2-m2.3-extension-collection.md +++ b/docs/project/m2.2-m2.3-extension-collection.md @@ -4,24 +4,29 @@ ## 已交付 -1. 扩展使用 PKCE 授权码登录，而不是读取 Atlas Web 的 localStorage 或 refresh Cookie。`GET /api/extension/authorize` 会优先复用用户浏览器已有的 Atlas Web 会话：会话有效则无感签发一次性 code；没有会话才跳转登录页，成功登录后回到原授权请求。 -2. 服务端保存 code 与扩展 refresh token 的哈希，code 60 秒且一次性使用，扩展 access token 15 分钟、refresh token 30 天并轮换。令牌含扩展专用 audience，不能被普通 Web/API 身份守卫接受。 -3. 扩展上报仅携带来源、规范化 SKU/规格轴、候选素材 URL 和证据；`POST /api/extension/materials` 从扩展令牌解析工作空间，不接受目标平台、门店、Cookie 或 API Key。相同工作空间、来源平台和来源 item key 复用同一采集记录。 -4. 上报事务创建来源采集记录、SPU、来源确认 SKU 和素材转存任务；素材异步事件经 Transactional Outbox 进入 Worker。Worker 仅下载 HTTPS 公网地址，限制重定向、私网地址、20MB 和 PNG/JPEG/WebP，按 SHA-256 去重后写入对象存储。单张失败不会使商品草稿丢失。 -5. Ozon (`ru/kz/by`) 商品详情页显示 Shadow DOM 小浮窗。未登录只显示“登录 Atlas ERP”；已登录显示“编辑上架 / 采集商品 / 进入 ERP”。编辑上架成功后打开现有 `/products/{productId}` ERP 编辑入口；采集商品只提示进入采集箱。 -6. Ozon 解析器保留当前公开 SKU，并只从带 `webProducts`、`webAspects` 或 `aspects` 标识的页面状态发现关联 SKU；不会扫描整页 SKU 或把规格轴做笛卡尔积。来源未给可确认组合时，仍保留当前 SKU。主图作为商品候选素材上报，不伪造 SKU 图片关系。 +1. 扩展按毛子 ERP 风格复用 Atlas Web 登录态：点击登录会打开 Atlas Web；Web 登录/恢复后把当前短期 access token 写入本站 localStorage，扩展每 2 秒从该 Atlas 标签读取并保存到 extension storage。未登录时用户在该页完成登录，标签不会被扩展自动关闭。 +2. 扩展以同步到的 Web Bearer token 调用会话检查和采集接口；服务端同时验证 JWT 与未撤销的 Web session，并从当前用户成员关系解析工作空间。Token 失效时插件清除缓存并重新要求从 Atlas Web 登录同步。 +3. 扩展上报仅携带来源、规范化 SKU/规格轴、候选素材 URL 和证据；`POST /api/extension/materials` 从 Web Bearer 身份解析工作空间，不接受目标平台、门店、Cookie 或 API Key。非 Ozon 来源按 item key 复用；Ozon 额外按已确认 `source_sku` 集合相交复用同一采集记录。 +4. 上报事务创建来源采集记录、SPU、来源确认 SKU 和素材转存任务；同一来源商品再次采集时保留记录与商品 ID，并用本次来源快照替换来源字段、SKU 与候选素材。素材异步事件经 Transactional Outbox 进入 Worker，任务带快照版本；Worker 仅处理当前版本，旧任务不得回写。Worker 仅下载 HTTPS 公网地址，限制重定向、私网地址、20MB 和 PNG/JPEG/WebP，按 SHA-256 去重后写入对象存储。单张失败不会使商品草稿丢失。 +5. Ozon (`ru/kz/by`) 商品详情页显示 Shadow DOM 小浮窗。未登录只显示“登录 Atlas ERP”；已登录显示“编辑上架 / 采集商品 / 进入 ERP”。商品编辑模块尚未完成时，“编辑上架”成功后打开 `/collection-test/{collectionItemId}` 供审查采集结果；采集商品只提示进入采集箱。 +6. Ozon 解析器保留当前公开 SKU 作为入口证据，优先解析 `state-webAspects-*.data-state.aspects[].variants[]` 的真实变体；不会扫描整页 SKU 或把规格轴做笛卡尔积。来源未给可确认组合时，仍保留当前 SKU。 +7. Ozon 多 SKU 补全通过受限 MAIN world bridge 调用同源公开页面 JSON：先读取 `modal/aspectsNew` 的 `webAspectsModal`，再以并发 3、上限 100 的队列补全 `/product//` 中的标题、价格、画廊和关联 SKU。每个已补全 SKU 的画廊图片使用 `variant_direct` 关联；单个请求失败保留已发现 SKU 与失败证据。 +8. Ozon 公开响应没有可依赖 SPU/组 ID。服务端在同一来源平台下按新旧 `source_sku` 集合相交覆盖快照；若命中多条历史记录返回 `SOURCE_VARIANT_GROUP_AMBIGUOUS`，不静默合并。 +9. 图片采集沿用旧采集插件已验证的分组 profile：1688、淘宝、天猫只从主图画廊、规格选择器和商品详情容器读取图片，绝不扫描全页 `document.images`。淘宝/天猫详情图排除评论和推荐卡片；URL 会在入库前还原 CDN 原图并按归一化 URL 去重。 +10. Ozon 只从 `webGallery` 读取商品画廊，`coverImage` 与 `images[].src` 归为该真实 SKU 的 `source_main + variant_direct`，不会将第二张及以后的画廊图误标为 SKU 图；`webAspects.data.coverImage` 作为 `source_sku + axis_value`。受限 bridge 额外读取固定的富文本页段来获取 `webDescription` / `webRichContent` / `webFeatures` 内的详情图，仍不接触评价或推荐组件。 +11. 采集数据测试页会为转存成功且已生成 `asset_id` 的素材按 4 路并发申请短时预览 URL，直接显示缩略图，并展示其关联来源 SKU；预览 URL 不保存到业务数据。Ozon 解析器 `ozon-public-2.2.0` 现额外读取短/完整 `web*Characteristics.characteristics[]` 为公共属性，并将 `webDescription.richAnnotationJson` 转为公共描述文本、原 JSON 放入 `raw_value`。该桥接仍只允许固定 Ozon 页面段和 widget 白名单；已有旧版载荷必须重采集才会拥有这些字段。 ## 配置与安全前提 -- `EXTENSION_COLLECTOR_REDIRECT_URI` 必须精确等于已安装扩展的 `chrome.identity.getRedirectURL('callback')`；不可用通配符代替。 -- 本地默认 API 为 `http://localhost:8200`。生产构建应为 Atlas API 域配置扩展 host permission 与 `ATLAS_API_ORIGIN`，并登记对应扩展 ID 的回调 URI。 -- 扩展 refresh token 仅存 Chrome extension storage；Web refresh Cookie 从不返回给扩展。Worker 不接触浏览器 Cookie。 +- 扩展清单带有仅公开的 RSA `key`，Chrome 因此为自托管和解压加载生成稳定 ID；该 ID 不参与登录授权，严禁提交私钥。 +- 本地默认 API 为 `http://localhost:8200`。生产构建应为 Atlas API 域配置扩展 host permission 与 `ATLAS_API_ORIGIN`；不需要按用户登记回调 URI。 +- Web access token 镜像到 Atlas Web 自己的 localStorage 是维护者批准的自托管插件例外；扩展不读取 refresh Cookie、来源站 Cookie 或来源站 localStorage。Worker 不接触浏览器 Cookie。安全取舍见 [`extension-login-alternatives.md`](./extension-login-alternatives.md)。 ## 尚未实现 -- Ozon 变体队列逐页 hydrate、SKU 级公开详情补全；当前只消费已加载公开页面状态。 -- 1688、淘宝、天猫解析器，以及 Ozon Seller 网页四接口本地观察实验。 -- 采集箱详情页的素材关联可视化、重试 UI 和商品编辑/发布闭环。 +- Ozon 多 SKU 的真实来源页人工回归（`packages/reference/ozon1.html` 已作为三 SKU 快照）；Ozon Seller 网页四接口观察实验仍独立于此标准公开采集链路。 +- Ozon Seller 网页四接口本地观察实验；1688、淘宝、天猫基础页面状态解析已在 M2.4 完成，但仍需要真实来源页面回归。 +- 正式采集箱详情页的素材关联可视化、重试 UI 和商品编辑/发布闭环（当前仅 `/collection-test/{id}` 提供只读素材预览与关联 SKU）。 ## 验收命令 @@ -30,5 +35,6 @@ ./apps/api/node_modules/.bin/tsc -p apps/api/tsconfig.json --noEmit ./apps/worker/node_modules/.bin/tsc -p apps/worker/tsconfig.json --noEmit (cd apps/extension && ./node_modules/.bin/wxt prepare && ./node_modules/.bin/tsc -p tsconfig.json --noEmit && ./node_modules/.bin/wxt build) +(cd apps/extension && ./node_modules/.bin/tsx --test test/*.test.mts) (cd apps/web && ./node_modules/.bin/vite build) ``` diff --git a/docs/requirements/account-store-watermark.md b/docs/requirements/account-store-watermark.md index 0fb4eb8..6a78824 100644 --- a/docs/requirements/account-store-watermark.md +++ b/docs/requirements/account-store-watermark.md @@ -25,9 +25,18 @@ ### 2.2 登录与会话 -用户用邮箱和密码登录。登录成功进入默认工作空间与首页；若后续一个账户拥有多个工作空间，再增加切换入口。首期需要提供退出登录；忘记密码、短信登录、第三方登录、设备管理和多因素认证均不在本轮范围。 +用户用邮箱和密码登录。登录成功进入默认工作空间与首页；若后续一个账户拥有多个工作空间，再增加切换入口。首期需要提供退出登录；短信登录、第三方登录、设备管理和多因素认证均不在本轮范围。 -### 2.3 全局平台与当前店铺上下文 +### 2.3 忘记密码（待开发任务） + +该功能已列入下一轮账户认证开发，尚未实现。登录页增加“忘记密码”入口，用户可提交邮箱并通过邮件中的一次性链接设置新密码。 + +- 请求重置时，无论邮箱是否已注册，页面均显示相同的成功提示，避免枚举账户；服务端须按 IP 与邮箱摘要限流，且不得记录明文密码、重置令牌或完整重置链接。 +- 重置链接使用随机、不透明、一次性令牌；数据库只保存其哈希与过期时间。令牌须短时有效，使用、过期或再次申请后不得重复使用。 +- 用户在重置页输入新密码与确认密码。成功后更新密码哈希、撤销该账户所有已有 session，并引导用户用新密码登录；旧 access token 与插件缓存随之失效。 +- 开发前须补充 `docs/api/auth.md`、数据库迁移、邮件投递配置与失败/限流错误契约；交付须包含 API、Web 页面与邮件链接的集成测试，且测试不能输出令牌或密码。 + +### 2.4 全局平台与当前店铺上下文 平台和当前店铺是全站工作上下文，不在每个业务页面重复选择。页头在用户信息左侧放置“平台选择器”；用户切换平台后，菜单、类目、物流商、佣金规则和页面数据均切换到该平台范围。 diff --git a/docs/requirements/collector-browser-extension.md b/docs/requirements/collector-browser-extension.md index f24e86c..188239b 100644 --- a/docs/requirements/collector-browser-extension.md +++ b/docs/requirements/collector-browser-extension.md @@ -1,6 +1,6 @@ # 浏览器采集插件：需求与技术方案 -> 状态：已确认核心方向，待实施。本文是 Atlas Seller Kit 从来源商品详情页进入采集箱、商品编辑与后续 Ozon 上线的采集入口规范。 +> 状态：核心闭环已实施；Ozon 公开多 SKU 解析与归并方案于 2026-09-07 落地，仍需用真实页面回归验证。本文是 Atlas Seller Kit 从来源商品详情页进入采集箱、商品编辑与后续 Ozon 上线的采集入口规范。 > > 本文取代旧 `ozon-seller-kit` 采集插件的产品定位，但只复用经验证的页面采集思路，不复用旧项目的本地数据库、无鉴权接口、`/trial/:id` 页面、AI 套图或生图流程。 @@ -43,7 +43,7 @@ 首期不在悬浮窗中选择平台或店铺。服务端从插件授权身份只解析工作空间，创建未绑定目标平台/门店的采集记录和商品基础草稿。用户进入 Atlas ERP 编辑页时，页面使用页头当前选中的平台；首期页头默认选择 Ozon，服务端取得该平台发布配置并预选默认店铺。若当前平台没有可用门店，允许保留采集箱记录，但不能进入该平台的商品编辑/发布流程，应引导用户在店铺管理中添加门店。 -推荐采用扩展专用的短期授权码流程：插件打开 Atlas 登录/授权页，登录页完成后向 `chrome-extension://{extension-id}/...` 回传一次性 code，插件交换短期 access token 与可轮换 refresh token。令牌只放 Chrome 扩展专属存储，后台请求统一携带 `Authorization: Bearer`。不得像参考插件一样直接读取 ERP 网页 localStorage 中的长期 Token。 +首期按毛子 ERP 风格复用 Atlas Web 登录态：插件打开 Atlas Web 登录页，登录/会话恢复后从 Atlas Web 标签读取本站 localStorage 中的短期 access token，保存到 Chrome extension storage，后台请求统一携带 `Authorization: Bearer`。不得读取 refresh Cookie、来源站 Cookie 或来源站 localStorage；该受限例外及后续独立令牌升级方案见 [`../project/extension-login-alternatives.md`](../project/extension-login-alternatives.md)。 ### 2.3 三个操作 @@ -84,7 +84,7 @@ | `failed` | 采集请求或关键记录创建失败；保留可诊断错误。 | | `archived` | 用户不再处理的采集记录。 | -同一工作空间内，以 `source_platform + source_item_key` 进行幂等。重复“采集商品”应返回已有采集记录；重复“编辑上架”打开同一未绑定目标的商品草稿或其已绑定的编辑入口。只有用户在 ERP 中明确选择“重新采集并合并/创建副本”时，才允许覆盖或复制。 +同一工作空间内，以 `source_platform + source_item_key` 识别同一来源商品。重复“采集商品”或“编辑上架”保留已有采集记录和商品草稿 ID，但以本次页面重新采集的来源信息覆盖上一份来源快照：标题、价格、原始证据、规范化载荷、来源 SKU 与候选素材均以最新采集为准。旧素材任务取消，新素材任务必须带新的快照版本，旧 Worker 不得回写最新内容。用户希望保留旧版本或创建副本的需求，留待正式商品编辑模块提供明确操作后再设计。 ### 3.1 采集箱与目标店铺的绑定时点 @@ -161,7 +161,7 @@ interface CollectedVariant { 1. 创建多个 SKU 的前提是来源提供稳定组合记录、SKU ID、变体 SKU、`propPath` 等可验证证据；单独的“红色”“M 码”选项不够。 2. `source_variant_key` 优先使用来源 SKU ID。不存在来源 ID 时，使用来源提供的属性 ID/值 ID 的有序组合，并记录构造规则与风险；不得只拼本地化展示名称。 -3. 同一 `source_variant_key` 重采集应更新其采集记录和候选素材，不应再创建一个 SKU。 +3. 同一 `source_variant_key` 重采集应更新其采集记录和候选素材，不应再创建一个 SKU。Ozon 没有可依赖的公开 SPU/变体组 ID：同一来源站点下，本次真实 `source_sku` 集合与既有采集记录任一 SKU 相交时，应复用该记录并覆盖来源快照；若相交到多条历史记录，必须报重复冲突，不得静默合并或删除。 4. 只识别出规格轴但没有组合时，创建一个默认 SKU，状态为 `needs_review`；规格轴仍保留给编辑页辅助人工拆分。 5. 不使用笛卡尔积构造 SKU。来源若只返回“颜色选项”和“尺码选项”而没有组合可售表，无法证明每种组合都存在。 6. 来源价格、库存、重量、尺寸只作采集参考。发布前必须由用户确认有效包装规格、定价快照、目标店铺类目和目标语言内容。 @@ -176,11 +176,20 @@ interface CollectedVariant { #### 采集步骤 -1. 从当前商品页的白名单状态组件读取标题、价格、画廊、视频、参数、类目和当前来源 SKU。 -2. 从包含产品变体的页面状态（参考实现中的 `webProductsModal` / `aspects[].variants` 一类组件）提取每一条 `sku`、标题、价格、划线价、封面、规格文本和关联链接。旧采集插件只保留了 `webAspects` 的名称与图片，丢弃了 `sku`，新实现不得如此。 -3. 将当前 SKU 和发现到的关联 SKU 放入有界队列；按 `sku` 去重。对每个 SKU 加载该 SKU 的公开详情页面/页面 JSON，补齐图片、视频、描述与可能继续出现的关联 SKU。 +1. 优先从当前商品页的白名单状态组件读取标题、价格、画廊、参数、类目和当前来源 SKU。`[id^="state-webAspects-"][data-state]` 的 `aspects[].variants[]` 是首选的已渲染真实变体表；`webShortCharacteristics` / `webDetailedCharacteristics` / `webCharacteristics` 的 `characteristics[]` 转为公共商品属性。不得扫描脚本文本或整页 SKU 字符串。画廊只读取 `webGallery.coverImage` 和 `webGallery.images[].src`，先将 Ozon `/wcN/` / `/cN/` 缩略 URL 还原为原图，再去重。 +2. 若当前页只渲染了部分变体，使用只限 Ozon 域名、只接受数字 SKU 的 MAIN world bridge 请求 `/api/entrypoint-api.bx/page/json/v2?url=/modal/aspectsNew?product_id=`，从 `webAspectsModal-*.aspects[].variants[]` 提取每一条 `sku`、标题、价格、封面和规格文本。桥接只返回这些白名单 widget 状态，绝不暴露 Cookie、请求头或任意 URL 请求能力。详情图只允许从固定的第二页段 `webDescription`、`webRichContent`、`webFeatures` 中取图片字段；`webDescription.richAnnotationJson` 仅转换为描述纯文本，不采集图片 URL 以外的页面组件内容。 +3. 将当前 SKU 和发现到的关联 SKU 放入有界队列；按真实 `sku` 去重。对每个 SKU 通过同一 bridge 请求 `/api/entrypoint-api.bx/page/json/v2?url=/product//`，从 `webProductHeading`、`webPrice`、`webGallery`、`webAspects` 与当前 SKU 的完整 `webCharacteristics` 补齐详情，并将新发现的真实 SKU 继续入队。 4. 队列达到固定上限（建议 100）或无新 SKU 时停止。记录 `discovered_count`、`hydrated_count`、失败项与页面证据。 -5. 每个成功补全的来源 SKU 创建一个 `CollectedVariant`；没有专属图片时设置 `inherited_product_default`，不丢弃变体。 +5. 每个发现的来源 SKU 创建一个 `CollectedVariant`；补全成功时记录为 `verified` 并把 `webGallery` 的每一张图片以 `source_main + variant_direct` 绑定到该 SKU。规格 `data.coverImage` 以 `source_sku + axis_value` 记录，不能因同一画廊的第二张图片而虚构 SKU 图片关系。没有专属图片时设置 `inherited_product_default`，不丢弃变体。不得用 `document.images`，以免把 Ozon Logo、支付图和推荐商品误采为商品素材。 + +#### 组身份与重复采集 + +Ozon 当前公开页面没有可依赖的 `group_id`、`model_id` 或 SPU ID。`/modal/aspectsNew` 的 `product_id` 参数以及链接的 `from_sku` 都是当前页面 SKU，页面 widget ID 也只是易变的前端实例 ID。因此： + +1. `source.itemKey` 保存用户实际打开的 SKU，作为采集入口证据；它不是 SPU 主键。 +2. 服务端以同一工作空间、同一来源平台的 `product_variants.source_sku` 与本次完整 SKU 集合求交集。唯一命中时保留既有 `collection_item` / `product` ID，用最新页面资料、SKU 与候选素材覆盖旧快照。 +3. 排序 SKU 集合的哈希仅可作为诊断辅助，不能作为永久身份键；Ozon 随时可能增加、隐藏或拆分变体。 +4. 多个既有采集记录同时命中时返回 `SOURCE_VARIANT_GROUP_AMBIGUOUS`，等待显式的人工合并能力，避免错误地合并不同商品。 #### 完整性边界 diff --git a/packages/database/migrations/011_extension_link_requests.sql b/packages/database/migrations/011_extension_link_requests.sql new file mode 100644 index 0000000..cb8036d --- /dev/null +++ b/packages/database/migrations/011_extension_link_requests.sql @@ -0,0 +1,24 @@ +SET search_path TO atlas, public; + +-- A link request exists before it belongs to a workspace, so it cannot use the +-- normal workspace RLS policy. Its UUID is high entropy and the final token +-- exchange additionally requires the PKCE verifier stored with the request. +CREATE TABLE extension_link_requests ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + client_id TEXT NOT NULL, + code_challenge TEXT NOT NULL, + workspace_id UUID REFERENCES workspaces(id) ON DELETE RESTRICT, + user_id UUID REFERENCES users(id) ON DELETE RESTRICT, + approved_at TIMESTAMPTZ, + consumed_at TIMESTAMPTZ, + expires_at TIMESTAMPTZ NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CHECK (expires_at > created_at), + CHECK ((workspace_id IS NULL AND user_id IS NULL AND approved_at IS NULL) OR (workspace_id IS NOT NULL AND user_id IS NOT NULL AND approved_at IS NOT NULL)) +); + +CREATE INDEX ix_extension_link_requests_pending ON extension_link_requests (expires_at) + WHERE approved_at IS NULL AND consumed_at IS NULL; + +COMMENT ON TABLE extension_link_requests IS + 'Short-lived browser-to-extension pairing requests. Access is protected by unguessable IDs and PKCE, not workspace RLS, because the request is created before Web-session approval.'; diff --git a/packages/database/migrations/012_remove_extension_link_requests.sql b/packages/database/migrations/012_remove_extension_link_requests.sql new file mode 100644 index 0000000..aef9d5f --- /dev/null +++ b/packages/database/migrations/012_remove_extension_link_requests.sql @@ -0,0 +1,5 @@ +SET search_path TO atlas, public; + +-- The PKCE pairing prototype was replaced before release by the documented +-- Web-login-state synchronization flow. Remove its unreferenced bootstrap table. +DROP TABLE IF EXISTS extension_link_requests; diff --git a/packages/database/migrations/013_collection_source_snapshot_refresh.sql b/packages/database/migrations/013_collection_source_snapshot_refresh.sql new file mode 100644 index 0000000..27d5bf1 --- /dev/null +++ b/packages/database/migrations/013_collection_source_snapshot_refresh.sql @@ -0,0 +1,14 @@ +SET search_path TO atlas, public; + +-- 同一来源商品重新采集时保留 collection_item / product 的稳定 ID，但使用新的 +-- 来源快照替换可再采集数据。素材任务携带快照版本，避免旧 Worker 任务回写旧素材。 +ALTER TABLE collection_items + ADD COLUMN source_snapshot_revision BIGINT NOT NULL DEFAULT 1 + CHECK (source_snapshot_revision > 0); + +ALTER TABLE collection_asset_transfers + ADD COLUMN source_snapshot_revision BIGINT NOT NULL DEFAULT 1 + CHECK (source_snapshot_revision > 0); + +CREATE INDEX ix_collection_asset_transfers_snapshot_queue + ON collection_asset_transfers (collection_item_id, source_snapshot_revision, status, created_at); diff --git a/packages/reference/ozon1.html b/packages/reference/ozon1.html new file mode 100644 index 0000000..887daa7 --- /dev/null +++ b/packages/reference/ozon1.html @@ -0,0 +1,30027 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 背包——在OZON低价购买 (5387005639) + + + + + + + + + + + + + + + + + + + + + + + + +
+
+
+
+ +
+
+
+ +
+
+
+
+
+
+
+
+
+
+
•
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
    +
  1. 配饰 + + + +
    2. +
  2. 男士配饰 + + + +
    3. +
  3. 包袋和背包 + + + +
    4. +
  4. 背包 + +
    5. +
+
+
+
+ +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ +
+
+
+
+
+
+
+
背包 #1
+
+
+
+
+
+
相似商品 +
+
+
+
+
+
+
+
+
+
+
+
+
+
+

+ 背包 +

+
+
+
+
+
+
+
+
+
颜色: 卡其色
+
+
+
+ +
+
+
+
+ +
+ +
+
+
+
+
+ +
+
+
+
+
+
+
+
关于商品 +
+
+
前往描述 +
+
+ + +
+
+
+
+
+
类型 +
+
背包 +
+
+
+
原产国 +
+
+
中国 + +
+
+
+
+
材料 +
+
Cordura + 1000D
+
+
+
高度，厘米 +
+
44 +
+
+
+
宽度，厘米 +
+
24 +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ +
+
+ + +
+
+
+
+
本周折扣
+
+ +
+
+
+
50
+
个单位剩余
+
+
+ +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
219,02 ¥ + 430,12 ¥ +
+
与其他银行 +
+
+
+
+ +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 背包 + +
+
+
+
+
+
+
+ + + + + + + + + +
+
+
+
+
+
+
+
+
+
Панда Маркет +
+
+
Панда Маркет
+
+
去商店
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ +
+
+
+
+ +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ +
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 49aaf6f..cd04948 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -103,6 +103,9 @@ importers: '@types/react-dom': specifier: 19.2.5 version: 19.2.5(@types/react@19.2.18) + tsx: + specifier: 4.23.13 + version: 4.23.13 typescript: specifier: 7.0.2 version: 7.0.2