From 4c0f3a4a798111ace79616c6c102c7bb91ac61be Mon Sep 17 00:00:00 2001 From: joey-home Date: Sun, 6 Sep 2026 22:32:39 +0800 Subject: [PATCH] =?UTF-8?q?feat(extension):=20=E9=87=87=E9=9B=86=E6=8F=92?= =?UTF-8?q?=E4=BB=B6=E5=8A=9F=E8=83=BD=E5=BC=80=E5=8F=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.example | 4 + apps/api/src/app.module.ts | 2 + apps/api/src/auth/auth.crypto.ts | 19 +++ apps/api/src/auth/auth.module.ts | 1 + apps/api/src/auth/auth.service.ts | 20 +++ .../src/collection/collection.controller.ts | 3 + apps/api/src/collection/collection.service.ts | 11 +- apps/api/src/config/app-config.module.ts | 5 + .../api/src/extension/extension-auth.guard.ts | 17 +++ .../api/src/extension/extension.controller.ts | 40 +++++ apps/api/src/extension/extension.module.ts | 7 + apps/api/src/extension/extension.service.ts | 140 ++++++++++++++++++ apps/api/src/main.ts | 10 +- apps/api/test/extension.e2e.test.ts | 34 +++++ apps/extension/.wxt/types/paths.d.ts | 1 + apps/extension/entrypoints/background.ts | 9 +- .../entrypoints/collector.content.ts | 15 ++ apps/extension/lib/api.ts | 39 +++++ apps/extension/lib/ozon-parser.ts | 32 ++++ apps/extension/lib/source-parsers.ts | 83 +++++++++++ apps/extension/wxt.config.ts | 1 + apps/web/src/pages/auth/LoginPage.tsx | 8 + .../collection/CollectionInspectionPage.tsx | 45 ++++++ apps/web/src/router/index.tsx | 2 + apps/web/src/services/collection.ts | 3 +- apps/worker/src/collection-assets.ts | 81 ++++++++++ apps/worker/src/config.ts | 1 + apps/worker/src/main.ts | 3 + docs/api/collection-and-extension.md | 2 +- docs/project/HANDOFF.md | 31 +++- .../project/m2.2-m2.3-extension-collection.md | 34 +++++ .../m2.4-source-parsers-and-inspection.md | 27 ++++ .../collector-browser-extension.md | 2 +- packages/contracts/src/collection.ts | 38 ++++- .../010_extension_collection_auth.sql | 40 +++++ 35 files changed, 790 insertions(+), 20 deletions(-) create mode 100644 apps/api/src/extension/extension-auth.guard.ts create mode 100644 apps/api/src/extension/extension.controller.ts create mode 100644 apps/api/src/extension/extension.module.ts create mode 100644 apps/api/src/extension/extension.service.ts create mode 100644 apps/api/test/extension.e2e.test.ts create mode 100644 apps/extension/entrypoints/collector.content.ts create mode 100644 apps/extension/lib/api.ts create mode 100644 apps/extension/lib/ozon-parser.ts create mode 100644 apps/extension/lib/source-parsers.ts create mode 100644 apps/web/src/pages/collection/CollectionInspectionPage.tsx create mode 100644 apps/worker/src/collection-assets.ts create mode 100644 docs/project/m2.2-m2.3-extension-collection.md create mode 100644 docs/project/m2.4-source-parsers-and-inspection.md create mode 100644 packages/database/migrations/010_extension_collection_auth.sql diff --git a/.env.example b/.env.example index cc5b42e..8d4a8a0 100644 --- a/.env.example +++ b/.env.example @@ -18,6 +18,10 @@ MIGRATOR_DATABASE_URL=postgresql://atlas_migrator:change-me-local-only@localhost NODE_ENV=development # Browser origin allowed to send credentialed requests to this API. WEB_ORIGIN=http://localhost:8100 +API_PUBLIC_ORIGIN=http://localhost:8200 +# Must exactly equal chrome.identity.getRedirectURL('callback') for the installed collector build. +EXTENSION_COLLECTOR_CLIENT_ID=atlas-collector +EXTENSION_COLLECTOR_REDIRECT_URI=https://atlas-collector.chromiumapp.org/callback # HTTP listener port for the NestJS API. API_PORT=8200 diff --git a/apps/api/src/app.module.ts b/apps/api/src/app.module.ts index 8f4474b..1363aa0 100644 --- a/apps/api/src/app.module.ts +++ b/apps/api/src/app.module.ts @@ -10,6 +10,7 @@ import { StoresModule } from './stores/stores.module.js'; import { WatermarksModule } from './watermarks/watermarks.module.js'; import { MediaModule } from './media/media.module.js'; import { CollectionModule } from './collection/collection.module.js'; +import { ExtensionModule } from './extension/extension.module.js'; @Controller() class HealthController { @@ -47,6 +48,7 @@ class HealthController { WatermarksModule, MediaModule, CollectionModule, + ExtensionModule, ], controllers: [HealthController], }) diff --git a/apps/api/src/auth/auth.crypto.ts b/apps/api/src/auth/auth.crypto.ts index cfb998f..3719199 100644 --- a/apps/api/src/auth/auth.crypto.ts +++ b/apps/api/src/auth/auth.crypto.ts @@ -51,6 +51,8 @@ export type AccessTokenClaims = { exp: number; iat: number; }; + +export type ExtensionAccessTokenClaims = AccessTokenClaims & { typ: 'extension'; wid: string }; export function signAccessToken( claims: AccessTokenClaims, secret: string, @@ -92,3 +94,20 @@ export function verifyAccessToken( return null; } } + +/** Extension tokens have an explicit audience so they can never be accepted by + * the normal Web/API guard, even though both are signed by the same key. */ +export function signExtensionAccessToken( + claims: ExtensionAccessTokenClaims, + secret: string, +): string { + const header = Buffer.from(JSON.stringify({ alg: 'HS256', typ: 'JWT' })).toString('base64url'); + const payload = Buffer.from(JSON.stringify(claims)).toString('base64url'); + const signature = createHmac('sha256', secret).update(`${header}.${payload}`).digest('base64url'); + return `${header}.${payload}.${signature}`; +} + +export function verifyExtensionAccessToken(token: string, secret: string): ExtensionAccessTokenClaims | null { + const claims = verifyAccessToken(token, secret) as Partial | null; + return claims?.typ === 'extension' ? claims as ExtensionAccessTokenClaims : null; +} diff --git a/apps/api/src/auth/auth.module.ts b/apps/api/src/auth/auth.module.ts index ee68230..df3d4c7 100644 --- a/apps/api/src/auth/auth.module.ts +++ b/apps/api/src/auth/auth.module.ts @@ -16,5 +16,6 @@ import { IdempotencyService } from '../common/idempotency.service.js'; { provide: APP_GUARD, useExisting: AccessTokenGuard }, { provide: APP_GUARD, useExisting: WorkspaceContextGuard }, ], + exports: [AuthService, IdempotencyService], }) export class AuthModule {} diff --git a/apps/api/src/auth/auth.service.ts b/apps/api/src/auth/auth.service.ts index 52ac2ca..9b507cc 100644 --- a/apps/api/src/auth/auth.service.ts +++ b/apps/api/src/auth/auth.service.ts @@ -176,6 +176,26 @@ export class AuthService { }); } + /** Resolves an existing Web refresh cookie without rotating it. This is used + * only to complete an interactive extension authorization request in the + * user's browser; the cookie itself is never returned to the extension. */ + public async resolveWebRefreshSession(refreshToken: string | undefined) { + if (!refreshToken) return null; + return this.database.transaction(async (transaction) => { + const sessions = await transaction.query( + `SELECT u.id, u.email, u.display_name, u.avatar_asset_id, u.email_verified_at, u.created_at, + u.revision, u.status, u.password_hash, s.id AS session_id + FROM atlas.user_sessions s JOIN atlas.users u ON u.id = s.user_id + WHERE s.refresh_token_hash = $1 AND s.revoked_at IS NULL AND s.expires_at > now()`, + [tokenHash(refreshToken)], + ); + const user = sessions.rows[0]; + if (!user || user.status !== 'active') return null; + const workspace = await findWorkspaceInTransaction(transaction, user.id); + return { userId: user.id, workspace }; + }); + } + public async logout(sessionId: string) { await this.database.query( 'UPDATE atlas.user_sessions SET revoked_at = now() WHERE id = $1 AND revoked_at IS NULL', diff --git a/apps/api/src/collection/collection.controller.ts b/apps/api/src/collection/collection.controller.ts index 8b878a0..ba43f2e 100644 --- a/apps/api/src/collection/collection.controller.ts +++ b/apps/api/src/collection/collection.controller.ts @@ -14,6 +14,9 @@ export class CollectionController { if (!platformId) throw new ApiError(400, 'PLATFORM_CONTEXT_REQUIRED', '请求缺少 X-Atlas-Platform-Id'); return this.collection.list(workspace.id, requireUuid('X-Atlas-Platform-Id', platformId), parse(listCollectionItemsQuerySchema, query)); } + @Get(':id') async detail(@Param('id') id: string, @CurrentWorkspace() workspace: Workspace) { + return { data: await this.collection.detail(workspace.id, requireUuid('collection_item_id', id)) }; + } @Post() async create(@Body() body: unknown, @Headers('idempotency-key') key: string | undefined, @Req() request: AuthenticatedRequest, @CurrentWorkspace() workspace: Workspace, @Res({ passthrough: true }) response: Response) { const result = await this.collection.create(workspace.id, request.auth.userId, requireKey(key), parse(createCollectionItemRequestSchema, body)); response.status(result.status); return result.body; } diff --git a/apps/api/src/collection/collection.service.ts b/apps/api/src/collection/collection.service.ts index e049189..7ba758f 100644 --- a/apps/api/src/collection/collection.service.ts +++ b/apps/api/src/collection/collection.service.ts @@ -29,7 +29,16 @@ export class CollectionService { const item = await tx.query<{ id: string }>(`INSERT INTO atlas.collection_items (workspace_id, source_kind, source_item_key, source_title, note, status) VALUES (current_setting('app.workspace_id')::uuid, 'manual', $1, $2, $3, 'ready_for_edit') RETURNING id`, [randomUUID(), input.title ?? null, input.note ?? null]); const product = await tx.query<{ id: string }>(`INSERT INTO atlas.products (workspace_id, collection_item_id, source_title) VALUES (current_setting('app.workspace_id')::uuid, $1, $2) RETURNING id`, [item.rows[0].id, input.title ?? null]); await tx.query(`INSERT INTO atlas.product_variants (workspace_id, product_id, display_name, sort_order) VALUES (current_setting('app.workspace_id')::uuid, $1, '默认 SKU', 0)`, [product.rows[0].id]); - return { status: 201, body: { data: { collection_item: await this.requireItem(tx, item.rows[0].id), product_id: product.rows[0].id, editor_entry_path: `/collection-items/${item.rows[0].id}/edit` } } }; + return { status: 201, body: { data: { collection_item: await this.requireItem(tx, item.rows[0].id), product_id: product.rows[0].id, editor_entry_path: `/products/${product.rows[0].id}` } } }; + }); + } + async detail(workspaceId: string, id: string) { + return this.database.workspaceTransaction(workspaceId, async (tx) => { + const item = await this.requireItem(tx, id); + const source = await tx.query<{ normalized_payload: unknown; raw_evidence: unknown; parser_version: string | null; collected_at: Date | string | null }>('SELECT normalized_payload, raw_evidence, parser_version, collected_at FROM atlas.collection_items WHERE id = $1 AND deleted_at IS NULL', [id]); + const variants = await tx.query<{ id: string; source_variant_key: string | null; source_sku: string | null; display_name: string | null; sort_order: number }>('SELECT id, source_variant_key, source_sku, display_name, sort_order FROM atlas.product_variants WHERE product_id = $1 AND deleted_at IS NULL ORDER BY sort_order, id', [item.product_id]); + const transfers = await tx.query<{ id: string; source_variant_id: string | null; role: string; source_url: string; status: string; asset_id: string | null; error_payload: unknown; created_at: Date | string; completed_at: Date | string | null }>('SELECT id, source_variant_id, role::text AS role, source_url, status::text AS status, asset_id, error_payload, created_at, completed_at FROM atlas.collection_asset_transfers WHERE collection_item_id = $1 ORDER BY created_at, id', [id]); + return { ...item, normalized_payload: source.rows[0]?.normalized_payload ?? {}, raw_evidence: source.rows[0]?.raw_evidence ?? {}, parser_version: source.rows[0]?.parser_version ?? null, collected_at: source.rows[0]?.collected_at ? iso(source.rows[0].collected_at) : null, variants: variants.rows, asset_transfers: transfers.rows.map((transfer) => ({ ...transfer, created_at: iso(transfer.created_at), completed_at: transfer.completed_at ? iso(transfer.completed_at) : null })) }; }); } async update(workspaceId: string, userId: string, id: string, key: string, expected: number, input: UpdateCollectionItemRequest) { diff --git a/apps/api/src/config/app-config.module.ts b/apps/api/src/config/app-config.module.ts index e1d162c..4619447 100644 --- a/apps/api/src/config/app-config.module.ts +++ b/apps/api/src/config/app-config.module.ts @@ -25,7 +25,12 @@ const environmentSchema = z.object({ .enum(['development', 'test', 'production']) .default('development'), API_PORT: z.coerce.number().int().min(1).max(65_535).default(8200), + API_PUBLIC_ORIGIN: z.url().default('http://localhost:8200'), WEB_ORIGIN: z.url().default('http://localhost:8100'), + // The redirect URI is deliberately exact. A wildcard would allow another + // installed extension to exchange a code created from the user's Web session. + EXTENSION_COLLECTOR_CLIENT_ID: z.string().min(1).max(128).default('atlas-collector'), + EXTENSION_COLLECTOR_REDIRECT_URI: z.url().default('https://atlas-collector.chromiumapp.org/callback'), DATABASE_URL: z.url(), REDIS_URL: z.url(), OBJECT_STORAGE_ENDPOINT: z.url().default('http://localhost:9000'), diff --git a/apps/api/src/extension/extension-auth.guard.ts b/apps/api/src/extension/extension-auth.guard.ts new file mode 100644 index 0000000..586d4f9 --- /dev/null +++ b/apps/api/src/extension/extension-auth.guard.ts @@ -0,0 +1,17 @@ +import { CanActivate, ExecutionContext, Inject, Injectable } from '@nestjs/common'; +import { ExtensionService } from './extension.service.js'; + +export type ExtensionAuthenticatedRequest = { + header: (name: string) => string | undefined; + extensionAuth: { userId: string; workspaceId: string; sessionId: string; clientId: string }; +}; + +@Injectable() +export class ExtensionAccessGuard implements CanActivate { + public constructor(@Inject(ExtensionService) private readonly extension: ExtensionService) {} + public async canActivate(context: ExecutionContext) { + const request = context.switchToHttp().getRequest(); + request.extensionAuth = await this.extension.authenticate(request.header('authorization')); + return true; + } +} diff --git a/apps/api/src/extension/extension.controller.ts b/apps/api/src/extension/extension.controller.ts new file mode 100644 index 0000000..6661ab6 --- /dev/null +++ b/apps/api/src/extension/extension.controller.ts @@ -0,0 +1,40 @@ +import { Body, Controller, Get, Headers, Inject, Post, Query, Req, Res, UseGuards } from '@nestjs/common'; +import { z } from 'zod'; +import { collectedProductPayloadSchema } from '@atlas/contracts'; +import { ApiError } from '../common/api-error.js'; +import { Public } from '../common/public.decorator.js'; +import { ExtensionAccessGuard, type ExtensionAuthenticatedRequest } from './extension-auth.guard.js'; +import { ExtensionService } from './extension.service.js'; + +type RequestLike = { header: (name: string) => string | undefined }; +type ResponseLike = { redirect: (status: number, url: string) => void; status: (code: number) => ResponseLike }; +const authorizeSchema = z.object({ client_id: z.string().min(1).max(128), redirect_uri: z.url(), state: z.string().min(16).max(1_000), code_challenge: z.string().regex(/^[A-Za-z0-9_-]{43,128}$/), code_challenge_method: z.literal('S256') }).strict(); +const exchangeSchema = z.object({ client_id: z.string().min(1).max(128), code: z.string().min(32).max(1_000), redirect_uri: z.url(), code_verifier: z.string().min(43).max(128) }).strict(); +const refreshSchema = z.object({ refresh_token: z.string().min(32).max(1_000) }).strict(); + +@Controller('extension') +export class ExtensionController { + public constructor(@Inject(ExtensionService) private readonly extension: ExtensionService) {} + @Get('authorize') @Public() + public async authorize(@Query() query: unknown, @Req() request: RequestLike, @Res() response: ResponseLike) { + const result = await this.extension.authorize(parse(authorizeSchema, query) as z.infer, cookie(request.header('cookie'), 'atlas_refresh')); + const target = 'redirect_url' in result ? result.redirect_url : result.login_url; + if (!target) throw new ApiError(500, 'EXTENSION_AUTHORIZATION_REDIRECT_MISSING', '扩展授权跳转地址缺失'); + response.redirect(302, target); + } + @Post('auth/exchange') @Public() + public async exchange(@Body() body: unknown, @Res({ passthrough: true }) response: ResponseLike) { const result = await this.extension.exchange(parse(exchangeSchema, body) as z.infer); response.status(result.status); return result.body; } + @Post('auth/refresh') @Public() + public async refresh(@Body() body: unknown, @Res({ passthrough: true }) response: ResponseLike) { const input = parse(refreshSchema, body) as z.infer; const result = await this.extension.refresh(input.refresh_token); response.status(result.status); return result.body; } + @Post('auth/revoke') @Public() @UseGuards(ExtensionAccessGuard) + public async revoke(@Req() request: ExtensionAuthenticatedRequest, @Res({ passthrough: true }) response: ResponseLike) { await this.extension.revoke(request.extensionAuth.workspaceId, request.extensionAuth.sessionId); response.status(204); } + @Get('session') @Public() @UseGuards(ExtensionAccessGuard) + public session(@Req() request: ExtensionAuthenticatedRequest) { return { data: { authenticated: true, workspace: { id: request.extensionAuth.workspaceId, available: true }, can_collect: true } }; } + @Post('materials') @Public() @UseGuards(ExtensionAccessGuard) + public async materials(@Req() request: ExtensionAuthenticatedRequest, @Headers('idempotency-key') key: string | undefined, @Body() body: unknown, @Res({ passthrough: true }) response: ResponseLike) { + if (!key || !z.uuid().safeParse(key).success) throw new ApiError(400, 'IDEMPOTENCY_KEY_REQUIRED', '请求缺少或格式错误的 Idempotency-Key'); + const result = await this.extension.ingest(request.extensionAuth.workspaceId, request.extensionAuth.userId, key, parse(collectedProductPayloadSchema, body)); response.status(result.status); return result.body; + } +} +function parse { success: boolean; data?: unknown; error?: { issues: Array<{ path: PropertyKey[]; message: string }> } } }>(schema: T, value: unknown) { const result = schema.safeParse(value); if (result.success) return result.data as never; throw new ApiError(422, 'VALIDATION_FAILED', '请求参数不符合要求', result.error?.issues.map((item) => ({ field: item.path.join('.'), code: 'INVALID_FIELD', message: item.message }))); } +function cookie(header: string | undefined, name: string) { return header?.split(';').map((part) => part.trim()).find((part) => part.startsWith(`${name}=`))?.slice(name.length + 1); } diff --git a/apps/api/src/extension/extension.module.ts b/apps/api/src/extension/extension.module.ts new file mode 100644 index 0000000..766e230 --- /dev/null +++ b/apps/api/src/extension/extension.module.ts @@ -0,0 +1,7 @@ +import { Module } from '@nestjs/common'; +import { AuthModule } from '../auth/auth.module.js'; +import { ExtensionAccessGuard } from './extension-auth.guard.js'; +import { ExtensionController } from './extension.controller.js'; +import { ExtensionService } from './extension.service.js'; +@Module({ imports: [AuthModule], controllers: [ExtensionController], providers: [ExtensionService, ExtensionAccessGuard] }) +export class ExtensionModule {} diff --git a/apps/api/src/extension/extension.service.ts b/apps/api/src/extension/extension.service.ts new file mode 100644 index 0000000..913c8d3 --- /dev/null +++ b/apps/api/src/extension/extension.service.ts @@ -0,0 +1,140 @@ +import { createHash, randomUUID } from 'node:crypto'; +import { Inject, Injectable } from '@nestjs/common'; +import type { CollectedProductPayload } from '@atlas/contracts'; +import type { Database } from '@atlas/database'; +import { createOpaqueToken, signExtensionAccessToken, tokenHash, verifyExtensionAccessToken } from '../auth/auth.crypto.js'; +import { AuthService } from '../auth/auth.service.js'; +import { ApiError } from '../common/api-error.js'; +import { IdempotencyService } from '../common/idempotency.service.js'; +import { APP_CONFIG, type AppConfig } from '../config/app-config.module.js'; +import { DATABASE } from '../database/database.module.js'; + +type AuthorizeInput = { client_id: string; redirect_uri: string; state: string; code_challenge: string; code_challenge_method: 'S256' }; +type ExchangeInput = { client_id: string; code: string; redirect_uri: string; code_verifier: string }; +type ExtensionSession = { id: string; workspace_id: string; user_id: string; client_id: string; expires_at: Date | string; revoked_at: Date | string | null }; + +@Injectable() +export class ExtensionService { + public constructor( + @Inject(DATABASE) private readonly database: Database, + @Inject(AuthService) private readonly auth: AuthService, + @Inject(IdempotencyService) private readonly idempotency: IdempotencyService, + @Inject(APP_CONFIG) private readonly config: AppConfig, + ) {} + + public async authorize(input: AuthorizeInput, refreshToken: string | undefined) { + this.assertClient(input.client_id, input.redirect_uri); + const session = await this.auth.resolveWebRefreshSession(refreshToken); + if (!session) return { login_url: `${this.config.WEB_ORIGIN}/login?continue=${encodeURIComponent(this.authorizationUrl(input))}` }; + // Prefixing a high-entropy secret with the workspace UUID lets the server + // establish RLS before looking up the code; it is not an authorization secret. + const rawCode = `${session.workspace.id}.${createOpaqueToken()}`; + await this.database.workspaceTransaction(session.workspace.id, async (tx) => { + await tx.query(`INSERT INTO atlas.extension_authorization_codes + (workspace_id, user_id, client_id, redirect_uri, code_hash, code_challenge, expires_at) + VALUES (current_setting('app.workspace_id')::uuid, $1, $2, $3, $4, $5, now() + interval '60 seconds')`, + [session.userId, input.client_id, input.redirect_uri, tokenHash(rawCode), input.code_challenge]); + }); + const callback = new URL(input.redirect_uri); + callback.searchParams.set('code', rawCode); + callback.searchParams.set('state', input.state); + return { redirect_url: callback.toString() }; + } + + public async exchange(input: ExchangeInput) { + this.assertClient(input.client_id, input.redirect_uri); + const hash = tokenHash(input.code); const workspaceId = workspacePrefix(input.code); + if (!workspaceId) throw new ApiError(401, 'EXTENSION_AUTHORIZATION_INVALID', '扩展授权码无效或已过期'); + const code = await this.database.workspaceTransaction(workspaceId, async (tx) => (await tx.query<{ workspace_id: string; user_id: string; client_id: string; redirect_uri: string; code_challenge: string }>( + `SELECT workspace_id, user_id, client_id, redirect_uri, code_challenge FROM atlas.extension_authorization_codes WHERE code_hash = $1 AND consumed_at IS NULL AND expires_at > now()`, [hash])).rows[0]); + if (!code || code.client_id !== input.client_id || code.redirect_uri !== input.redirect_uri || !constantEquals(code.code_challenge, pkceChallenge(input.code_verifier))) { + throw new ApiError(401, 'EXTENSION_AUTHORIZATION_INVALID', '扩展授权码无效或已过期'); + } + return this.database.workspaceTransaction(code.workspace_id, async (tx) => { + const consumed = await tx.query(`UPDATE atlas.extension_authorization_codes SET consumed_at = now() + WHERE code_hash = $1 AND consumed_at IS NULL AND expires_at > now()`, [hash]); + if (!consumed.rowCount) throw new ApiError(401, 'EXTENSION_AUTHORIZATION_INVALID', '扩展授权码无效或已过期'); + return this.createSession(tx, code.workspace_id, code.user_id, code.client_id); + }); + } + + public async refresh(refreshToken: string) { + const workspaceId = workspacePrefix(refreshToken); if (!workspaceId) throw new ApiError(401, 'EXTENSION_SESSION_EXPIRED', '扩展登录已过期'); + return this.database.workspaceTransaction(workspaceId, async (tx) => { + const result = await tx.query(`SELECT id, workspace_id, user_id, client_id, expires_at, revoked_at FROM atlas.extension_sessions WHERE refresh_token_hash = $1 AND revoked_at IS NULL AND expires_at > now() FOR UPDATE`, [tokenHash(refreshToken)]); + const old = result.rows[0]; + if (!old) throw new ApiError(401, 'EXTENSION_SESSION_EXPIRED', '扩展登录已过期'); + await tx.query('UPDATE atlas.extension_sessions SET revoked_at = now(), last_used_at = now() WHERE id = $1', [old.id]); + return this.createSession(tx, old.workspace_id, old.user_id, old.client_id); + }); + } + + public async revoke(workspaceId: string, sessionId: string) { + await this.database.workspaceTransaction(workspaceId, (tx) => tx.query('UPDATE atlas.extension_sessions SET revoked_at = now() WHERE id = $1 AND revoked_at IS NULL', [sessionId])); + } + + public async authenticate(header: string | undefined) { + const token = header?.replace(/^Bearer\s+/i, ''); + const claims = token ? verifyExtensionAccessToken(token, this.config.AUTH_JWT_SECRET) : null; + if (!claims) throw new ApiError(401, 'EXTENSION_SESSION_EXPIRED', '扩展登录已过期'); + const session = await this.database.workspaceTransaction(claims.wid, async (tx) => (await tx.query(`SELECT id, workspace_id, user_id, client_id, expires_at, revoked_at FROM atlas.extension_sessions WHERE id = $1 AND user_id = $2 AND revoked_at IS NULL AND expires_at > now()`, [claims.sid, claims.sub])).rows[0]); + if (!session) throw new ApiError(401, 'EXTENSION_SESSION_EXPIRED', '扩展登录已过期'); + return { userId: session.user_id, workspaceId: session.workspace_id, sessionId: session.id, clientId: session.client_id }; + } + + public async ingest(workspaceId: string, userId: string, key: string, payload: CollectedProductPayload) { + return this.idempotency.run({ workspaceId, userId, method: 'POST', path: '/api/extension/materials', key, request: payload }, async (tx) => { + const platform = await tx.query<{ id: string; code: string }>(`SELECT id, code FROM atlas.platforms + WHERE code = $1 AND status = 'active' AND coalesce((capabilities->>'can_be_source')::boolean, false)`, [payload.source.platform.toUpperCase()]); + if (!platform.rows[0]) throw new ApiError(422, 'SOURCE_PLATFORM_UNAVAILABLE', '该来源平台当前不可采集'); + const existing = await tx.query<{ id: string; product_id: string }>(`SELECT c.id, p.id AS product_id FROM atlas.collection_items c JOIN atlas.products p ON p.collection_item_id = c.id + WHERE c.source_platform_id = $1 AND c.source_item_key = $2 AND c.deleted_at IS NULL`, [platform.rows[0].id, payload.source.itemKey]); + if (existing.rows[0]) return { status: 200, body: { data: await this.materialResponse(tx, existing.rows[0].id, existing.rows[0].product_id, true) } }; + const verifiedVariants = payload.variants.filter((variant) => variant.sourceVariantKey.trim().length > 0); + const status = verifiedVariants.length || !payload.variantAxes.length ? 'materializing' : 'needs_review'; + const sourcePrice = payload.common.sourcePrice; + const item = await tx.query<{ id: string }>(`INSERT INTO atlas.collection_items + (workspace_id, source_kind, source_platform_id, source_item_key, source_url, source_title, source_currency_code, source_min_price, source_max_price, status, raw_evidence, normalized_payload, parser_version, collected_at) + VALUES (current_setting('app.workspace_id')::uuid, 'platform', $1, $2, $3, $4, $5, $6, $6, $7::atlas.collection_item_status, $8, $9, $10, $11) RETURNING id`, + [platform.rows[0].id, payload.source.itemKey, payload.source.canonicalUrl, payload.common.title?.value ?? null, sourcePrice?.currencyCode ?? null, sourcePrice?.amount ?? null, status, JSON.stringify(payload.evidence), JSON.stringify(payload), payload.source.parserVersion, payload.source.collectedAt]); + const product = await tx.query<{ id: string }>(`INSERT INTO atlas.products (workspace_id, collection_item_id, source_title) + VALUES (current_setting('app.workspace_id')::uuid, $1, $2) RETURNING id`, [item.rows[0].id, payload.common.title?.value ?? null]); + const variantRows = verifiedVariants.length ? verifiedVariants : [{ sourceVariantKey: null, sourceSku: undefined, displayName: '默认 SKU' }]; + const variants = new Map(); + for (let index = 0; index < variantRows.length; index += 1) { + const variant = variantRows[index]; + const inserted = await tx.query<{ id: string }>(`INSERT INTO atlas.product_variants (workspace_id, product_id, source_variant_key, source_sku, display_name, sort_order) + VALUES (current_setting('app.workspace_id')::uuid, $1, $2, $3, $4, $5) RETURNING id`, [product.rows[0].id, variant.sourceVariantKey, variant.sourceSku ?? null, variant.displayName, index]); + if (variant.sourceVariantKey) variants.set(variant.sourceVariantKey, inserted.rows[0].id); + } + for (const asset of payload.assets) { + await tx.query(`INSERT INTO atlas.collection_asset_transfers (workspace_id, collection_item_id, product_id, source_variant_id, role, source_url, status, error_payload) + VALUES (current_setting('app.workspace_id')::uuid, $1, $2, $3, $4::atlas.media_candidate_role, $5, 'queued', $6)`, + [item.rows[0].id, product.rows[0].id, asset.variantSourceKey ? variants.get(asset.variantSourceKey) ?? null : null, asset.role, asset.url, JSON.stringify({ relation: asset.relation, referer_origin: asset.refererOrigin ?? null, evidence_ref: asset.evidenceRef ?? null, sort_order: asset.sortOrder })]); + } + if (payload.assets.length) await tx.query(`INSERT INTO atlas.outbox_events (workspace_id, aggregate_type, aggregate_id, event_type, event_payload, deduplication_key) + VALUES (current_setting('app.workspace_id')::uuid, 'collection_item', $1, 'collection.assets.materialize.requested', jsonb_build_object('collection_item_id', $1), $2)`, [item.rows[0].id, `collection.assets.materialize:${item.rows[0].id}`]); + else await tx.query(`UPDATE atlas.collection_items SET status = CASE WHEN status = 'needs_review' THEN status ELSE 'ready_for_edit' END, updated_at = now() WHERE id = $1`, [item.rows[0].id]); + return { status: 201, body: { data: await this.materialResponse(tx, item.rows[0].id, product.rows[0].id, false) } }; + }); + } + + private async materialResponse(tx: { query: Database['query'] }, collectionItemId: string, productId: string, reused: boolean) { + const count = await tx.query<{ total: string; queued: string; succeeded: string; failed: string }>(`SELECT count(*)::text AS total, count(*) FILTER (WHERE status = 'queued')::text AS queued, count(*) FILTER (WHERE status = 'succeeded')::text AS succeeded, count(*) FILTER (WHERE status = 'failed')::text AS failed FROM atlas.collection_asset_transfers WHERE collection_item_id = $1`, [collectionItemId]); + const status = await tx.query<{ status: string; source_title: string | null }>('SELECT status, source_title FROM atlas.collection_items WHERE id = $1', [collectionItemId]); + return { collection_item: { id: collectionItemId, status: status.rows[0]?.status ?? 'received', source_title: status.rows[0]?.source_title ?? null, variant_count: 0 }, product_id: productId, reused, editor_entry_path: `/collection-test/${collectionItemId}`, materialization: { total: Number(count.rows[0]?.total ?? 0), queued: Number(count.rows[0]?.queued ?? 0), succeeded: Number(count.rows[0]?.succeeded ?? 0), failed: Number(count.rows[0]?.failed ?? 0) } }; + } + + private async createSession(tx: { query: Database['query'] }, workspaceId: string, userId: string, clientId: string) { + const refreshToken = `${workspaceId}.${createOpaqueToken()}`; const id = randomUUID(); + await tx.query(`INSERT INTO atlas.extension_sessions (id, workspace_id, user_id, client_id, refresh_token_hash, expires_at) + VALUES ($1, current_setting('app.workspace_id')::uuid, $2, $3, $4, now() + interval '30 days')`, [id, userId, clientId, tokenHash(refreshToken)]); + const accessToken = signExtensionAccessToken({ sub: userId, sid: id, wid: workspaceId, typ: 'extension', iat: Math.floor(Date.now() / 1000), exp: Math.floor(Date.now() / 1000) + 15 * 60 }, this.config.AUTH_JWT_SECRET); + return { status: 200, body: { data: { access_token: accessToken, expires_in: 900, refresh_token: refreshToken, refresh_token_expires_in: 2_592_000, token_type: 'Bearer' } } }; + } + private assertClient(clientId: string, redirectUri: string) { if (clientId !== this.config.EXTENSION_COLLECTOR_CLIENT_ID || redirectUri !== this.config.EXTENSION_COLLECTOR_REDIRECT_URI) throw new ApiError(422, 'EXTENSION_CLIENT_INVALID', '扩展客户端或回调地址未登记'); } + private authorizationUrl(input: AuthorizeInput) { const url = new URL(`${this.config.API_PUBLIC_ORIGIN}/api/extension/authorize`); for (const [key, value] of Object.entries(input)) url.searchParams.set(key, value); return url.toString(); } +} +function pkceChallenge(verifier: string) { return createHash('sha256').update(verifier).digest('base64url'); } +function constantEquals(left: string, right: string) { return left.length === right.length && left.split('').every((value, index) => value === right[index]); } +function workspacePrefix(value: string) { const prefix = value.split('.', 1)[0]; return /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(prefix) ? prefix : null; } diff --git a/apps/api/src/main.ts b/apps/api/src/main.ts index d10dd1a..5ec30c3 100644 --- a/apps/api/src/main.ts +++ b/apps/api/src/main.ts @@ -35,7 +35,15 @@ export async function createApplication() { } app.enableCors({ - origin: config.WEB_ORIGIN, + origin: (origin, callback) => { + // Extension API calls are bearer-token authenticated. Allow only Chrome + // extension origins in addition to the configured Web application. + if (!origin || origin === config.WEB_ORIGIN || /^chrome-extension:\/\/[a-p]{32}$/.test(origin)) { + callback(null, true); + return; + } + callback(null, false); + }, credentials: true, methods: ['GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'OPTIONS'], allowedHeaders: [ diff --git a/apps/api/test/extension.e2e.test.ts b/apps/api/test/extension.e2e.test.ts new file mode 100644 index 0000000..cb15417 --- /dev/null +++ b/apps/api/test/extension.e2e.test.ts @@ -0,0 +1,34 @@ +import assert from 'node:assert/strict'; +import { createHash, randomUUID } from 'node:crypto'; +import type { AddressInfo } from 'node:net'; +import type { Database } from '@atlas/database'; +import type { NestExpressApplication } from '@nestjs/platform-express'; +import { afterAll, beforeAll, describe, it } from 'vitest'; +import { DATABASE } from '../src/database/database.module.js'; +import { createApplication } from '../src/main.js'; + +let app: NestExpressApplication; let database: Database; let baseUrl = ''; let workspaceId = ''; let userId = ''; +const redirectUri = 'https://atlas-collector.chromiumapp.org/callback'; +async function request(path: string, init: RequestInit = {}) { const response = await fetch(`${baseUrl}${path}`, { redirect: 'manual', ...init, headers: { 'Content-Type': 'application/json', ...init.headers } }); return { response, body: response.status === 204 || response.status === 302 ? undefined : await response.json() as any }; } +function challenge(verifier: string) { return createHash('sha256').update(verifier).digest('base64url'); } + +describe('扩展授权与采集 HTTP 集成测试', () => { + beforeAll(async () => { Object.assign(process.env, { NODE_ENV: 'test', SWAGGER_ENABLED: 'false', AUTH_REGISTER_IP_LIMIT: '100', AUTH_REGISTER_EMAIL_LIMIT: '100' }); const created = await createApplication(); app = created.app; database = app.get(DATABASE); await app.listen(0, '127.0.0.1'); baseUrl = `http://127.0.0.1:${(app.getHttpServer().address() as AddressInfo).port}`; }); + afterAll(async () => { await database.workspaceTransaction(workspaceId, async (tx) => { await tx.query('DELETE FROM atlas.idempotency_requests WHERE workspace_id = $1', [workspaceId]); await tx.query('DELETE FROM atlas.outbox_events WHERE workspace_id = $1', [workspaceId]); await tx.query('DELETE FROM atlas.collection_asset_transfers WHERE workspace_id = $1', [workspaceId]); await tx.query('DELETE FROM atlas.product_variants WHERE workspace_id = $1', [workspaceId]); await tx.query('DELETE FROM atlas.products WHERE workspace_id = $1', [workspaceId]); await tx.query('DELETE FROM atlas.collection_items WHERE workspace_id = $1', [workspaceId]); await tx.query('DELETE FROM atlas.extension_authorization_codes WHERE workspace_id = $1', [workspaceId]); await tx.query('DELETE FROM atlas.extension_sessions WHERE workspace_id = $1', [workspaceId]); }); await database.query('DELETE FROM atlas.workspace_memberships WHERE user_id = $1', [userId]); await database.query('DELETE FROM atlas.workspaces WHERE id = $1', [workspaceId]); await database.query('DELETE FROM atlas.users WHERE id = $1', [userId]); await app.close(); }); + + it('通过已有 Web 会话完成 PKCE 并幂等写入来源商品', async () => { + const registration = await request('/api/auth/register', { method: 'POST', body: JSON.stringify({ email: `extension-${randomUUID()}@example.test`, password: 'correct-horse-battery-staple-123', password_confirmation: 'correct-horse-battery-staple-123' }) }); + assert.equal(registration.response.status, 201); userId = registration.body.data.user.id; workspaceId = registration.body.data.workspace.id; const webAccessToken = registration.body.data.access_token as string; + const cookie = registration.response.headers.get('set-cookie')?.match(/atlas_refresh=[^;]+/)?.[0]; assert.ok(cookie); + const verifier = 'a'.repeat(64); const state = 'b'.repeat(32); + const authorize = await request(`/api/extension/authorize?${new URLSearchParams({ client_id: 'atlas-collector', redirect_uri: redirectUri, state, code_challenge: challenge(verifier), code_challenge_method: 'S256' })}`, { headers: { Cookie: cookie } }); + assert.equal(authorize.response.status, 302); const callback = new URL(authorize.response.headers.get('location')!); assert.equal(callback.origin, 'https://atlas-collector.chromiumapp.org'); assert.equal(callback.searchParams.get('state'), state); + const exchange = await request('/api/extension/auth/exchange', { method: 'POST', body: JSON.stringify({ client_id: 'atlas-collector', redirect_uri: redirectUri, code: callback.searchParams.get('code'), code_verifier: verifier }) }); + assert.equal(exchange.response.status, 200); const accessToken = exchange.body.data.access_token as string; assert.ok(exchange.body.data.refresh_token); + const session = await request('/api/extension/session', { headers: { Authorization: `Bearer ${accessToken}` } }); assert.equal(session.response.status, 200); assert.equal(session.body.data.workspace.id, workspaceId); + const payload = { source: { platform: 'ozon', itemKey: '123456789', canonicalUrl: 'https://www.ozon.ru/product/demo-123456789/', collectedAt: new Date().toISOString(), parserVersion: 'test-1' }, common: { title: { value: '来源商品', confidence: 'verified' }, attributes: [], categories: [] }, variantAxes: [], variants: [{ sourceVariantKey: '123456789', sourceSku: '123456789', displayName: '默认 SKU', selections: [], confidence: 'verified', evidenceRefs: ['page'] }], assets: [], evidence: [{ ref: 'page', kind: 'page', path: 'test' }] }; + const first = await request('/api/extension/materials', { method: 'POST', headers: { Authorization: `Bearer ${accessToken}`, 'Idempotency-Key': randomUUID() }, body: JSON.stringify(payload) }); assert.equal(first.response.status, 201); assert.equal(first.body.data.reused, false); + const second = await request('/api/extension/materials', { method: 'POST', headers: { Authorization: `Bearer ${accessToken}`, 'Idempotency-Key': randomUUID() }, body: JSON.stringify(payload) }); assert.equal(second.response.status, 200); assert.equal(second.body.data.reused, true); assert.equal(second.body.data.collection_item.id, first.body.data.collection_item.id); + const inspection = await request(`/api/collection-items/${first.body.data.collection_item.id}`, { headers: { Authorization: `Bearer ${webAccessToken}` } }); assert.equal(inspection.response.status, 200); assert.equal(inspection.body.data.variants[0].source_variant_key, '123456789'); assert.equal(inspection.body.data.normalized_payload.source.platform, 'ozon'); + }); +}); diff --git a/apps/extension/.wxt/types/paths.d.ts b/apps/extension/.wxt/types/paths.d.ts index 5acf274..f5f88c9 100644 --- a/apps/extension/.wxt/types/paths.d.ts +++ b/apps/extension/.wxt/types/paths.d.ts @@ -6,6 +6,7 @@ declare module "wxt/browser" { | "" | "/" | "/background.js" + | "/content-scripts/collector.js" type HtmlPublicPath = Extract export interface WxtRuntime { getURL(path: PublicPath): string; diff --git a/apps/extension/entrypoints/background.ts b/apps/extension/entrypoints/background.ts index ea87781..1c495f1 100644 --- a/apps/extension/entrypoints/background.ts +++ b/apps/extension/entrypoints/background.ts @@ -1,3 +1,10 @@ +import { getSession, login, openErp, submit } from '../lib/api.js'; export default defineBackground(() => { - console.info('Atlas collector background started.'); + browser.runtime.onMessage.addListener(async (message: { type?: string; payload?: unknown; path?: string }) => { + if (message?.type === 'atlas.session') return { session: await getSession() }; + if (message?.type === 'atlas.login') return { session: await login() }; + if (message?.type === 'atlas.collect') return { result: await submit(message.payload as import('@atlas/contracts').CollectedProductPayload) }; + if (message?.type === 'atlas.open') { await openErp(message.path); return { ok: true }; } + return undefined; + }); }); diff --git a/apps/extension/entrypoints/collector.content.ts b/apps/extension/entrypoints/collector.content.ts new file mode 100644 index 0000000..2d3701f --- /dev/null +++ b/apps/extension/entrypoints/collector.content.ts @@ -0,0 +1,15 @@ +import { collectCurrentProduct, supportedProductPlatform } from '../lib/source-parsers.js'; + +export default defineContentScript({ matches: ['*://www.ozon.ru/*', '*://www.ozon.kz/*', '*://www.ozon.by/*', '*://detail.1688.com/*', '*://item.taobao.com/*', '*://detail.tmall.com/*'], runAt: 'document_idle', main() { + let mount: HTMLDivElement | null = null; let shadow: ShadowRoot | null = null; let observedUrl = location.href; + const render = async () => { + if (!supportedProductPlatform()) { mount?.remove(); mount = null; shadow = null; return; } + if (!mount) { mount = document.createElement('div'); mount.style.cssText = 'position:fixed;right:22px;bottom:80px;z-index:2147483647'; document.documentElement.append(mount); shadow = mount.attachShadow({ mode: 'open' }); } + const root = shadow!; const session = await browser.runtime.sendMessage({ type: 'atlas.session' }).then((value: { session?: unknown }) => value.session).catch(() => null); + root.innerHTML = `
Atlas ERP
${session ? '' : ''}
`; + root.querySelector('.dot')?.addEventListener('click', () => { mount?.remove(); mount = null; shadow = null; }); + root.querySelectorAll('[data-action]').forEach((button) => button.addEventListener('click', () => void action(button.dataset.action ?? '', root))); + }; + const action = async (kind: string, shadow: ShadowRoot) => { const hint = shadow.querySelector('[data-hint]')!; try { if (kind === 'login') { hint.textContent = '正在登录…'; await browser.runtime.sendMessage({ type: 'atlas.login' }); await render(); return; } if (kind === 'erp') { await browser.runtime.sendMessage({ type: 'atlas.open', path: '/' }); return; } hint.textContent = '正在采集…'; const payload = collectCurrentProduct(); const response = await browser.runtime.sendMessage({ type: 'atlas.collect', payload }); if (kind === 'edit') await browser.runtime.sendMessage({ type: 'atlas.open', path: response.result.data.editor_entry_path }); hint.textContent = kind === 'edit' ? '已打开采集数据测试页' : '已进入采集箱'; } catch (error) { hint.textContent = error instanceof Error ? error.message : '采集失败，请重试'; } }; + void render(); setInterval(() => { if (location.href !== observedUrl) { observedUrl = location.href; void render(); } }, 700); +} }); diff --git a/apps/extension/lib/api.ts b/apps/extension/lib/api.ts new file mode 100644 index 0000000..c23a845 --- /dev/null +++ b/apps/extension/lib/api.ts @@ -0,0 +1,39 @@ +import type { CollectedProductPayload } from '@atlas/contracts'; + +const apiOrigin = (import.meta.env.ATLAS_API_ORIGIN as string | undefined) ?? 'http://localhost:8200'; +const webOrigin = (import.meta.env.ATLAS_WEB_ORIGIN as string | undefined) ?? 'http://localhost:8100'; +const clientId = (import.meta.env.ATLAS_EXTENSION_CLIENT_ID as string | undefined) ?? 'atlas-collector'; +type Tokens = { access_token: string; expires_in: number; refresh_token: string; refresh_token_expires_in: number; token_type: 'Bearer' }; +type Session = { authenticated: true; workspace: { id: string; available: boolean }; can_collect: boolean }; +const storageKey = 'atlas.extension.tokens.v1'; + +async function loadTokens() { return (await browser.storage.local.get(storageKey))[storageKey] as Tokens | undefined; } +async function saveTokens(tokens: Tokens) { await browser.storage.local.set({ [storageKey]: tokens }); } +async function clearTokens() { await browser.storage.local.remove(storageKey); } +async function request(path: string, options: RequestInit = {}, token?: string) { + const response = await fetch(`${apiOrigin}/api${path}`, { ...options, headers: { Accept: 'application/json', ...(options.headers ?? {}), ...(token ? { Authorization: `Bearer ${token}` } : {}) } }); + if (!response.ok) { const body = await response.json().catch(() => null) as { error?: { message?: string } } | null; throw Object.assign(new Error(body?.error?.message ?? `请求失败（HTTP ${response.status}）`), { status: response.status }); } + return response.status === 204 ? undefined : response.json() as Promise<{ data: unknown }>; +} +async function activeAccessToken() { + let tokens = await loadTokens(); if (!tokens) return null; + try { await request('/extension/session', {}, tokens.access_token); return tokens.access_token; } + catch (error) { + if ((error as { status?: number }).status !== 401) throw error; + try { const refreshed = await request('/extension/auth/refresh', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ refresh_token: tokens.refresh_token }) }) as { data: Tokens }; tokens = refreshed.data; await saveTokens(tokens); return tokens.access_token; } + catch { await clearTokens(); return null; } + } +} +export async function getSession(): Promise { const token = await activeAccessToken(); if (!token) return null; return (await request('/extension/session', {}, token) as { data: Session }).data; } +export async function login() { + const verifier = crypto.getRandomValues(new Uint8Array(64)); const verifierText = base64url(verifier); const challenge = base64url(new Uint8Array(await crypto.subtle.digest('SHA-256', new TextEncoder().encode(verifierText)))); + const state = base64url(crypto.getRandomValues(new Uint8Array(32))); const redirectUri = browser.identity.getRedirectURL('callback'); + const authorize = new URL(`${apiOrigin}/api/extension/authorize`); authorize.search = new URLSearchParams({ client_id: clientId, redirect_uri: redirectUri, state, code_challenge: challenge, code_challenge_method: 'S256' }).toString(); + const callback = await browser.identity.launchWebAuthFlow({ url: authorize.toString(), interactive: true }); + if (!callback) throw new Error('未完成 Atlas ERP 登录'); const result = new URL(callback); if (result.searchParams.get('state') !== state || !result.searchParams.get('code')) throw new Error('扩展授权状态校验失败'); + const exchanged = await request('/extension/auth/exchange', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ client_id: clientId, code: result.searchParams.get('code'), redirect_uri: redirectUri, code_verifier: verifierText }) }) as { data: Tokens }; + await saveTokens(exchanged.data); return getSession(); +} +export async function submit(payload: CollectedProductPayload) { const token = await activeAccessToken(); if (!token) throw new Error('请先登录 Atlas ERP'); return request('/extension/materials', { method: 'POST', headers: { 'Content-Type': 'application/json', 'Idempotency-Key': crypto.randomUUID() }, body: JSON.stringify(payload) }, token) as Promise<{ data: { editor_entry_path: string; product_id: string; collection_item: { id: string }; reused: boolean } }>; } +export async function openErp(path = '/') { await browser.tabs.create({ url: new URL(path, webOrigin).toString() }); } +function base64url(bytes: Uint8Array) { let value = ''; for (const byte of bytes) value += String.fromCharCode(byte); return btoa(value).replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/, ''); } diff --git a/apps/extension/lib/ozon-parser.ts b/apps/extension/lib/ozon-parser.ts new file mode 100644 index 0000000..1c61d3c --- /dev/null +++ b/apps/extension/lib/ozon-parser.ts @@ -0,0 +1,32 @@ +import type { CollectedProductPayload } from '@atlas/contracts'; + +export function isOzonProductPage(url = location.href) { return /^https:\/\/www\.ozon\.(ru|kz|by)\/product\//.test(url); } +export function collectOzonProduct(documentRoot: Document = document, pageUrl = location.href): CollectedProductPayload { + const canonical = documentRoot.querySelector('link[rel="canonical"]')?.href ?? pageUrl.split('?')[0]; + const currentSku = skuFromUrl(canonical); if (!currentSku) throw new Error('未识别到 Ozon 商品 SKU'); + const title = text(documentRoot.querySelector('h1')) ?? documentRoot.querySelector('meta[property="og:title"]')?.content?.trim(); + const images = Array.from(documentRoot.images).map((image) => image.currentSrc || image.src).filter((url) => /^https:\/\//.test(url)).filter(unique).slice(0, 30); + const evidence: CollectedProductPayload['evidence'] = [{ ref: 'page:url', kind: 'page', path: 'location.href', raw_value: canonical }]; + const variants = new Map(); + variants.set(currentSku, { sourceVariantKey: currentSku, sourceSku: currentSku, displayName: title ?? `Ozon SKU ${currentSku}`, selections: [], confidence: 'verified', evidenceRefs: ['page:url'] }); + for (const script of Array.from(documentRoot.scripts)) { + const source = script.textContent ?? ''; + // Only inspect Ozon's variant-related state; never use generic page-wide + // SKU hits because recommendation cards would create false variants. + if (!/webProducts|webAspects|aspects/i.test(source)) continue; + const matches = source.matchAll(/["']sku["']\s*[:=]\s*["']?(\d{4,})/g); + let count = 0; + for (const match of matches) { if (count++ >= 100) break; const sku = match[1]; if (!variants.has(sku)) variants.set(sku, { sourceVariantKey: sku, sourceSku: sku, displayName: `Ozon SKU ${sku}`, selections: [], confidence: 'partial', evidenceRefs: ['state:variant-sku'] }); } + if (count) evidence.push({ ref: 'state:variant-sku', kind: 'state', path: 'webProducts/webAspects', raw_value: `发现 ${count} 个公开 SKU` }); + } + const amount = parsePrice(documentRoot); const currency = documentRoot.querySelector('meta[property="product:price:currency"]')?.content?.toUpperCase() ?? 'RUB'; + return { + source: { platform: 'ozon', itemKey: currentSku, canonicalUrl: canonical, collectedAt: new Date().toISOString(), parserVersion: 'ozon-public-1.0.0' }, + common: { ...(title ? { title: { value: title, raw_value: title, confidence: 'verified' as const } } : {}), ...(amount ? { sourcePrice: { amount, currencyCode: currency, raw_value: amount } } : {}), attributes: [], categories: [] }, + variantAxes: [], variants: [...variants.values()], assets: images.map((url, index) => ({ url, role: 'source_main' as const, relation: 'product_candidate' as const, sortOrder: index, evidenceRef: 'page:gallery' })), evidence, + }; +} +function skuFromUrl(url: string) { return url.match(/-(\d+)(?:\/|$|\?)/)?.[1] ?? null; } +function text(element: Element | null) { const value = element?.textContent?.replace(/\s+/g, ' ').trim(); return value || undefined; } +function parsePrice(root: Document) { const meta = root.querySelector('meta[property="product:price:amount"]')?.content; if (meta && /^\d+(\.\d+)?$/.test(meta)) return meta; const candidate = Array.from(root.querySelectorAll('[data-widget*="webPrice"], [data-widget*="price"]')).map((node) => node.textContent ?? '').find((value) => /\d/.test(value)); const digits = candidate?.replace(/[^\d,\.]/g, '').replace(',', '.'); return digits && /^\d+(\.\d+)?$/.test(digits) ? digits : undefined; } +function unique(value: string, index: number, array: string[]) { return array.indexOf(value) === index; } diff --git a/apps/extension/lib/source-parsers.ts b/apps/extension/lib/source-parsers.ts new file mode 100644 index 0000000..b0b10a0 --- /dev/null +++ b/apps/extension/lib/source-parsers.ts @@ -0,0 +1,83 @@ +import type { CollectedProductPayload } from '@atlas/contracts'; +import { collectOzonProduct, isOzonProductPage } from './ozon-parser.js'; + +type SourcePlatform = '1688' | 'taobao' | 'tmall' | 'ozon'; +type Json = Record; +type Axis = CollectedProductPayload['variantAxes'][number]; +type Variant = CollectedProductPayload['variants'][number]; + +export function supportedProductPlatform(url = location.href): SourcePlatform | null { + if (isOzonProductPage(url)) return 'ozon'; + if (/^https:\/\/detail\.1688\.com\/offer\/\d+\.html/.test(url)) return '1688'; + if (/^https:\/\/item\.taobao\.com\/item\.htm/.test(url)) return 'taobao'; + if (/^https:\/\/detail\.tmall\.com\/item\.htm/.test(url)) return 'tmall'; + return null; +} + +export function collectCurrentProduct(root: Document = document, pageUrl = location.href): CollectedProductPayload { + const platform = supportedProductPlatform(pageUrl); + if (platform === 'ozon') return collectOzonProduct(root, pageUrl); + if (!platform) throw new Error('当前页面不是支持的商品详情页'); + return platform === '1688' ? collect1688(root, pageUrl) : collectTaobao(root, pageUrl, platform); +} + +function collect1688(root: Document, pageUrl: string): CollectedProductPayload { + const state = pageState(root); const skuMap = findFirst(state, 'skuMapOriginal'); const skuProps = findFirst(state, 'skuProps'); + const axes = parse1688Axes(skuProps); const evidence: CollectedProductPayload['evidence'] = [{ ref: 'page:url', kind: 'page', path: 'location.href', raw_value: pageUrl }]; + const variants: Variant[] = []; + if (isRecord(skuMap)) { + evidence.push({ ref: '1688:sku-map', kind: 'state', path: 'skuMapOriginal', raw_value: `组合记录 ${Object.keys(skuMap).length} 条` }); + for (const [combination, row] of Object.entries(skuMap)) { + if (!isRecord(row)) continue; + const sourceSku = stringAt(row, ['skuId', 'sku_id', 'skuCode']); + // A skuMapOriginal entry itself is a source-provided sellable-combination + // record. Never expand it with the Cartesian product of skuProps. + const sourceVariantKey = sourceSku ?? `1688:${combination}`; + if (!combination || combination === 'undefined') continue; + variants.push({ sourceVariantKey, ...(sourceSku ? { sourceSku } : {}), displayName: selectionsForCombination(combination, axes).map((item) => item.valueName).join(' / ') || `1688 SKU ${sourceSku ?? combination}`, selections: selectionsForCombination(combination, axes), ...(moneyAt(row, ['discountPrice', 'price', 'salePrice']) ? { sourcePrice: { amount: moneyAt(row, ['discountPrice', 'price', 'salePrice'])!, currencyCode: 'CNY' } } : {}), confidence: 'verified', evidenceRefs: ['1688:sku-map'] }); + } + } + return basePayload('1688', offerId(pageUrl), root, pageUrl, axes, uniqueVariants(variants), evidence, '1688-state-1.0.0'); +} + +function collectTaobao(root: Document, pageUrl: string, platform: 'taobao' | 'tmall'): CollectedProductPayload { + const state = pageState(root); const skuBase = findFirst(state, 'skuBase'); const skuCore = findFirst(state, 'skuCore'); + const axes = parseTaobaoAxes(isRecord(skuBase) ? skuBase.props : undefined); const evidence: CollectedProductPayload['evidence'] = [{ ref: 'page:url', kind: 'page', path: 'location.href', raw_value: pageUrl }]; + const variants: Variant[] = []; + const skus = isRecord(skuBase) && Array.isArray(skuBase.skus) ? skuBase.skus : []; + const sku2info = isRecord(skuCore) && isRecord(skuCore.sku2info) ? skuCore.sku2info : {}; + if (skus.length) evidence.push({ ref: 'taobao:sku-base', kind: 'state', path: 'skuBase.skus', raw_value: `组合记录 ${skus.length} 条` }); + for (const sku of skus) { + if (!isRecord(sku)) continue; + const skuId = stringAt(sku, ['skuId', 'sku_id']); const propPath = stringAt(sku, ['propPath', 'prop_path']); + // skuId + propPath is the required combination proof. Props alone only + // become axes and cause a single default SKU on the server. + if (!skuId || !propPath) continue; + const info = isRecord(sku2info[skuId]) ? sku2info[skuId] : {}; + const selections = selectionsForPropPath(propPath, axes); + variants.push({ sourceVariantKey: skuId, sourceSku: skuId, displayName: selections.map((item) => item.valueName).join(' / ') || `SKU ${skuId}`, selections, ...(moneyAt(info, ['price', 'promotionPrice', 'discountPrice']) ? { sourcePrice: { amount: moneyAt(info, ['price', 'promotionPrice', 'discountPrice'])!, currencyCode: 'CNY' } } : {}), confidence: 'verified', evidenceRefs: ['taobao:sku-base'] }); + } + return basePayload(platform, itemId(pageUrl), root, pageUrl, axes, uniqueVariants(variants), evidence, `${platform}-state-1.0.0`); +} + +function basePayload(platform: Exclude, itemKey: string, root: Document, pageUrl: string, axes: Axis[], variants: Variant[], evidence: CollectedProductPayload['evidence'], parserVersion: string): CollectedProductPayload { + const canonical = root.querySelector('link[rel="canonical"]')?.href ?? pageUrl.split('#')[0]; const title = root.querySelector('h1')?.textContent?.replace(/\s+/g, ' ').trim() || root.querySelector('meta[property="og:title"]')?.content?.trim(); + const assets = Array.from(root.images).map((image) => image.currentSrc || image.src).filter((url) => /^https:\/\//.test(url)).filter((url, index, all) => all.indexOf(url) === index).slice(0, 40).map((url, sortOrder) => ({ url, role: 'source_main' as const, relation: 'product_candidate' as const, sortOrder, evidenceRef: 'page:gallery' })); + return { source: { platform, itemKey, canonicalUrl: canonical, collectedAt: new Date().toISOString(), parserVersion }, common: { ...(title ? { title: { value: title, raw_value: title, confidence: 'verified' as const } } : {}), attributes: [], categories: [] }, variantAxes: axes, variants, assets, evidence }; +} + +function pageState(root: Document) { const all: unknown[] = []; for (const script of Array.from(root.scripts)) { const text = script.textContent?.trim() ?? ''; if (text.length < 2 || text.length > 5_000_000) continue; if (script.type === 'application/ld+json') { try { all.push(JSON.parse(text)); } catch { /* ignored evidence */ } } for (const marker of ['skuMapOriginal', 'skuBase', 'skuCore', 'skuProps']) { const object = objectAfterMarker(text, marker); if (object) all.push(object); } } return all; } +function objectAfterMarker(text: string, marker: string): unknown | null { const position = text.indexOf(marker); if (position < 0) return null; const start = text.slice(position).search(/[\[{]/); if (start < 0) return null; const first = position + start; const open = text[first]; const close = open === '{' ? '}' : ']'; let depth = 0; let quote = ''; let escaped = false; for (let index = first; index < text.length && index - first < 2_000_000; index += 1) { const char = text[index]; if (quote) { if (escaped) escaped = false; else if (char === '\\') escaped = true; else if (char === quote) quote = ''; continue; } if (char === '"' || char === "'") { quote = char; continue; } if (char === open) depth += 1; if (char === close && --depth === 0) { try { return JSON.parse(text.slice(first, index + 1)); } catch { return null; } } } return null; } +function findFirst(states: unknown[], key: string): unknown { for (const state of states) { const found = findKey(state, key); if (found !== undefined) return found; } return undefined; } +function findKey(value: unknown, key: string, seen = new Set()): unknown { if (!value || typeof value !== 'object' || seen.has(value)) return undefined; seen.add(value); if (isRecord(value) && key in value) return value[key]; for (const child of Array.isArray(value) ? value : Object.values(value)) { const found = findKey(child, key, seen); if (found !== undefined) return found; } return undefined; } +function parse1688Axes(value: unknown): Axis[] { const entries = Array.isArray(value) ? value : isRecord(value) && Array.isArray(value.skuProps) ? value.skuProps : []; return entries.flatMap((entry, axisIndex) => { if (!isRecord(entry)) return []; const axisKey = stringAt(entry, ['prop', 'propId', 'id']) ?? `axis:${axisIndex}`; const name = stringAt(entry, ['name', 'propName']) ?? axisKey; const values = (Array.isArray(entry.value) ? entry.value : Array.isArray(entry.values) ? entry.values : []).flatMap((item, valueIndex) => isRecord(item) ? [{ key: stringAt(item, ['valueId', 'vid', 'id']) ?? `${axisKey}:${valueIndex}`, name: stringAt(item, ['name', 'valueName']) ?? `${valueIndex + 1}`, ...(urlAt(item, ['imageUrl', 'image', 'imageUrlLarge']) ? { image_url: urlAt(item, ['imageUrl', 'image', 'imageUrlLarge'])! } : {}) }] : []); return values.length ? [{ key: axisKey, name, values }] : []; }); } +function parseTaobaoAxes(value: unknown): Axis[] { return (Array.isArray(value) ? value : []).flatMap((entry, index) => { if (!isRecord(entry)) return []; const key = stringAt(entry, ['pid', 'propId']) ?? `axis:${index}`; const name = stringAt(entry, ['name', 'propName']) ?? key; const values = (Array.isArray(entry.values) ? entry.values : []).flatMap((item, valueIndex) => isRecord(item) ? [{ key: stringAt(item, ['vid', 'valueId']) ?? `${key}:${valueIndex}`, name: stringAt(item, ['name', 'valueName']) ?? `${valueIndex + 1}`, ...(urlAt(item, ['image', 'imageUrl']) ? { image_url: urlAt(item, ['image', 'imageUrl'])! } : {}) }] : []); return values.length ? [{ key, name, values }] : []; }); } +function selectionsForPropPath(path: string, axes: Axis[]) { return path.split(/[;,]/).flatMap((pair) => { const [axisKey, valueKey] = pair.split(':'); const axis = axes.find((item) => item.key === axisKey); const value = axis?.values.find((item) => item.key === valueKey); return axis && value ? [{ axisKey, valueKey, valueName: value.name }] : []; }); } +function selectionsForCombination(combination: string, axes: Axis[]) { const ids = combination.split(/[;,&]/).map((value) => value.trim()).filter(Boolean); return axes.flatMap((axis) => { const value = axis.values.find((item) => ids.includes(item.key)); return value ? [{ axisKey: axis.key, valueKey: value.key, valueName: value.name }] : []; }); } +function stringAt(value: Json, keys: string[]) { for (const key of keys) if (typeof value[key] === 'string' || typeof value[key] === 'number') return String(value[key]); return undefined; } +function moneyAt(value: Json, keys: string[]) { const raw = stringAt(value, keys)?.replace(/,/g, '.'); return raw && /^\d+(\.\d{1,4})?$/.test(raw) ? raw : undefined; } +function urlAt(value: Json, keys: string[]) { const raw = stringAt(value, keys); return raw && /^https:\/\//.test(raw) ? raw : undefined; } +function uniqueVariants(variants: Variant[]) { const seen = new Set(); return variants.filter((variant) => !seen.has(variant.sourceVariantKey) && !!seen.add(variant.sourceVariantKey)); } +function offerId(url: string) { return url.match(/\/offer\/(\d+)\.html/)?.[1] ?? `1688:${new URL(url).pathname}`; } +function itemId(url: string) { return new URL(url).searchParams.get('id') ?? new URL(url).pathname; } +function isRecord(value: unknown): value is Json { return !!value && typeof value === 'object' && !Array.isArray(value); } diff --git a/apps/extension/wxt.config.ts b/apps/extension/wxt.config.ts index 2685756..2cd811c 100644 --- a/apps/extension/wxt.config.ts +++ b/apps/extension/wxt.config.ts @@ -14,6 +14,7 @@ export default defineConfig({ description: '将已确认的来源商品采集到 Atlas 工作空间。', permissions: ['storage', 'identity'], host_permissions: [ + 'http://localhost:8200/*', '*://*.ozon.ru/*', '*://*.ozon.kz/*', '*://*.ozon.by/*', diff --git a/apps/web/src/pages/auth/LoginPage.tsx b/apps/web/src/pages/auth/LoginPage.tsx index 8372371..6a53b6b 100644 --- a/apps/web/src/pages/auth/LoginPage.tsx +++ b/apps/web/src/pages/auth/LoginPage.tsx @@ -32,6 +32,14 @@ const LoginPage = () => { }); applyAuthSession(session); message.success('登录成功'); + const continuation = new URLSearchParams(location.search).get('continue'); + if (continuation) { + const target = new URL(continuation); + if (target.origin === (import.meta.env.VITE_API_PUBLIC_ORIGIN ?? 'http://localhost:8200') && target.pathname === '/api/extension/authorize') { + window.location.assign(target.toString()); + return; + } + } // 守卫记住的来源路径优先，默认进入在线商品首页 const from = (location.state as { from?: string } | null)?.from; navigate(from && from.startsWith('/') ? from : '/online-products', { diff --git a/apps/web/src/pages/collection/CollectionInspectionPage.tsx b/apps/web/src/pages/collection/CollectionInspectionPage.tsx new file mode 100644 index 0000000..b2144cf --- /dev/null +++ b/apps/web/src/pages/collection/CollectionInspectionPage.tsx @@ -0,0 +1,45 @@ +import { useEffect, useState } from 'react'; +import { useParams } from 'react-router-dom'; +import { Alert, Card, Collapse, Descriptions, Spin, Table, Tag, Typography } from 'antd'; +import type { CollectionItemInspection } from '@atlas/contracts'; +import { collectionApi } from '@/services/collection'; +import { apiErrorMessage } from '@/services/api'; + +/** Temporary parser inspection surface. This is deliberately read-only: it lets + * product and parser work be checked before these values become editor fields. */ +export default function CollectionInspectionPage() { + const { id } = useParams<{ id: string }>(); + const [record, setRecord] = useState(); + const [error, setError] = useState(); + useEffect(() => { if (!id) return; void collectionApi.inspection(id).then(setRecord).catch((reason) => setError(apiErrorMessage(reason))); }, [id]); + if (error) return ; + if (!record) return
; + return 只读测试}> + + + SKU 草稿 + value ?? '默认 SKU（待确认）' }, + { title: '来源货号', dataIndex: 'source_sku', render: (value) => value ?? '—' }, + { title: '名称', dataIndex: 'display_name', render: (value) => value ?? '—' }, + ]} /> + 素材转存任务 +
查看来源 }, + { title: '错误', dataIndex: 'error_payload', render: (value) => value ? {JSON.stringify(value)} : '—' }, + ]} /> + }, + { key: 'evidence', label: '原始证据（raw_evidence）', children: }, + ]} /> + ; +} +function Json({ value }: { value: unknown }) { return 
{JSON.stringify(value, null, 2)}
; } diff --git a/apps/web/src/router/index.tsx b/apps/web/src/router/index.tsx index a03c0ab..9df14e1 100644 --- a/apps/web/src/router/index.tsx +++ b/apps/web/src/router/index.tsx @@ -3,6 +3,7 @@ import { Navigate, useLocation, createBrowserRouter } from 'react-router-dom'; import MainLayout from '@/layouts/MainLayout'; import SessionSplash from '@/components/SessionSplash'; import CollectionPage from '@/pages/collection/CollectionPage'; +import CollectionInspectionPage from '@/pages/collection/CollectionInspectionPage'; import LoginPage from '@/pages/auth/LoginPage'; import OnlineProductsPage from '@/pages/online-products/OnlineProductsPage'; import PasswordSettingsPage from '@/pages/settings/PasswordSettingsPage'; @@ -56,6 +57,7 @@ export const router = createBrowserRouter([ { index: true, element: }, { path: 'online-products', element: }, { path: 'collection', element: }, + { path: 'collection-test/:id', element: }, { path: 'shops', element: }, { path: 'watermarks', element: }, { path: 'pricing', element: }, diff --git a/apps/web/src/services/collection.ts b/apps/web/src/services/collection.ts index d234ecf..a596f42 100644 --- a/apps/web/src/services/collection.ts +++ b/apps/web/src/services/collection.ts @@ -1,8 +1,9 @@ -import type { CollectionItem, CreateCollectionItemRequest, CursorPaginationMeta, ListCollectionItemsQuery, UpdateCollectionItemRequest } from '@atlas/contracts'; +import type { CollectionItem, CollectionItemInspection, CreateCollectionItemRequest, CursorPaginationMeta, ListCollectionItemsQuery, UpdateCollectionItemRequest } from '@atlas/contracts'; import { api } from './api'; export const collectionApi = { list: (platformId: string, query: Partial = {}) => api.getList('/collection-items', { headers: { 'X-Atlas-Platform-Id': platformId }, params: { limit: 20, ...query } }), create: (payload: CreateCollectionItemRequest) => api.post<{ collection_item: CollectionItem; product_id: string; editor_entry_path: string }>('/collection-items', payload, { headers: { 'Idempotency-Key': crypto.randomUUID() } }), + inspection: (id: string) => api.get(`/collection-items/${id}`), update: (id: string, payload: UpdateCollectionItemRequest, revision: number) => api.patch(`/collection-items/${id}`, payload, { headers: { 'Idempotency-Key': crypto.randomUUID(), 'If-Match': `"${revision}"` } }), bulkDelete: (ids: string[]) => api.post<{ deleted_count: number }>('/collection-items/bulk-delete', { ids }, { headers: { 'Idempotency-Key': crypto.randomUUID() } }), clear: (expected_active_count: number) => api.post<{ deleted_count: number }>('/collection-items/clear', { confirm: true, expected_active_count }, { headers: { 'Idempotency-Key': crypto.randomUUID() } }), diff --git a/apps/worker/src/collection-assets.ts b/apps/worker/src/collection-assets.ts new file mode 100644 index 0000000..6c37503 --- /dev/null +++ b/apps/worker/src/collection-assets.ts @@ -0,0 +1,81 @@ +import { createHash } from 'node:crypto'; +import { lookup } from 'node:dns/promises'; +import { isIP } from 'node:net'; +import { CreateBucketCommand, HeadBucketCommand, PutObjectCommand, type S3Client } from '@aws-sdk/client-s3'; +import type { Database } from '@atlas/database'; +import type { OutboxJob } from './outbox-dispatcher.js'; + +const MAX_BYTES = 20 * 1024 * 1024; +type Transfer = { id: string; source_url: string; source_variant_id: string | null }; + +/** Downloads source images only after a public-address check. A failed image is + * terminal for this transfer, not for the product collection transaction. */ +export async function materializeCollectionAssets(database: Database, storage: S3Client, bucket: string, job: OutboxJob) { + await ensureBucket(storage, bucket); + const transfers = await database.workspaceTransaction(job.workspaceId, async (tx) => { + const rows = await tx.query(`UPDATE atlas.collection_asset_transfers SET status = 'running', updated_at = now() + WHERE id IN (SELECT id FROM atlas.collection_asset_transfers WHERE collection_item_id = $1 AND status = 'queued' ORDER BY created_at FOR UPDATE SKIP LOCKED) + RETURNING id, source_url, source_variant_id`, [String(job.payload.collection_item_id)]); + return rows.rows; + }); + for (const transfer of transfers) await materializeOne(database, storage, bucket, job.workspaceId, transfer); + await database.workspaceTransaction(job.workspaceId, async (tx) => { + await tx.query(`UPDATE atlas.collection_items SET status = CASE WHEN EXISTS (SELECT 1 FROM atlas.collection_asset_transfers WHERE collection_item_id = $1 AND status = 'running') THEN 'materializing'::atlas.collection_item_status ELSE 'ready_for_edit'::atlas.collection_item_status END, updated_at = now(), revision = revision + 1 WHERE id = $1 AND deleted_at IS NULL`, [String(job.payload.collection_item_id)]); + }); +} + +async function materializeOne(database: Database, storage: S3Client, bucket: string, workspaceId: string, transfer: Transfer) { + try { + const downloaded = await downloadImage(transfer.source_url); + const checksum = createHash('sha256').update(downloaded.bytes).digest('hex'); + const objectKey = `collections/${workspaceId}/${checksum}`; + await storage.send(new PutObjectCommand({ Bucket: bucket, Key: objectKey, Body: downloaded.bytes, ContentType: downloaded.contentType })); + await database.workspaceTransaction(workspaceId, async (tx) => { + const existing = await tx.query<{ id: string }>('SELECT id FROM atlas.media_assets WHERE checksum_sha256 = $1', [checksum]); + const assetId = existing.rows[0]?.id; + if (assetId) { + await tx.query(`UPDATE atlas.collection_asset_transfers SET status = 'succeeded', asset_id = $2, completed_at = now(), updated_at = now(), error_payload = NULL WHERE id = $1 AND status = 'running'`, [transfer.id, assetId]); + return; + } + const asset = await tx.query<{ id: string }>(`INSERT INTO atlas.media_assets (workspace_id, storage_provider, bucket, object_key, content_type, byte_size, checksum_sha256, width_px, height_px) + VALUES (current_setting('app.workspace_id')::uuid, 'local_minio', $1, $2, $3, $4, $5, $6, $7) RETURNING id`, [bucket, objectKey, downloaded.contentType, downloaded.bytes.byteLength, checksum, downloaded.width, downloaded.height]); + await tx.query(`UPDATE atlas.collection_asset_transfers SET status = 'succeeded', asset_id = $2, completed_at = now(), updated_at = now(), error_payload = NULL WHERE id = $1 AND status = 'running'`, [transfer.id, asset.rows[0].id]); + }); + } catch (error) { + await database.workspaceTransaction(workspaceId, (tx) => tx.query(`UPDATE atlas.collection_asset_transfers SET status = 'failed', completed_at = now(), updated_at = now(), error_payload = jsonb_build_object('code', 'SOURCE_ASSET_DOWNLOAD_FAILED', 'message', $2) WHERE id = $1 AND status = 'running'`, [transfer.id, error instanceof Error ? error.message.slice(0, 300) : '素材下载失败'])); + } +} + +async function downloadImage(source: string) { + let url = new URL(source); + for (let redirect = 0; redirect < 4; redirect += 1) { + await assertPublicUrl(url); + const response = await fetch(url, { redirect: 'manual', signal: AbortSignal.timeout(20_000), headers: { Accept: 'image/avif,image/webp,image/png,image/jpeg,*/*;q=0.8', 'User-Agent': 'Atlas-Collector/0.1' } }); + if ([301, 302, 303, 307, 308].includes(response.status)) { + const location = response.headers.get('location'); if (!location) throw new Error('来源素材重定向缺少地址'); url = new URL(location, url); continue; + } + if (!response.ok || !response.body) throw new Error(`来源素材请求失败（HTTP ${response.status}）`); + const chunks: Buffer[] = []; let total = 0; + for await (const chunk of response.body as AsyncIterable) { total += chunk.byteLength; if (total > MAX_BYTES) throw new Error('来源素材超过 20MB'); chunks.push(Buffer.from(chunk)); } + const bytes = Buffer.concat(chunks, total); const image = detectImage(bytes); if (!image) throw new Error('来源素材不是支持的 PNG、JPEG 或 WebP 图片'); + return { bytes, ...image }; + } + throw new Error('来源素材重定向次数过多'); +} +async function assertPublicUrl(url: URL) { + if (url.protocol !== 'https:') throw new Error('来源素材必须使用 HTTPS'); + if (url.username || url.password || url.port || url.hostname === 'localhost' || isPrivateIp(url.hostname)) throw new Error('来源素材地址不可访问'); + const addresses = await lookup(url.hostname, { all: true, verbatim: true }); if (!addresses.length || addresses.some((entry) => isPrivateIp(entry.address))) throw new Error('来源素材地址不可访问'); +} +function isPrivateIp(value: string) { + if (isIP(value) === 4) return /^(10\.|127\.|0\.|169\.254\.|192\.168\.|172\.(1[6-9]|2\d|3[01])\.)/.test(value); + return value === '::1' || value.toLowerCase().startsWith('fc') || value.toLowerCase().startsWith('fd') || value.toLowerCase().startsWith('fe80:'); +} +async function ensureBucket(storage: S3Client, bucket: string) { try { await storage.send(new HeadBucketCommand({ Bucket: bucket })); } catch { await storage.send(new CreateBucketCommand({ Bucket: bucket })); } } +function detectImage(bytes: Buffer): { contentType: 'image/png' | 'image/jpeg' | 'image/webp'; width: number; height: number } | null { + if (bytes.length >= 24 && bytes.subarray(0, 8).equals(Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]))) return dimensions('image/png', bytes.readUInt32BE(16), bytes.readUInt32BE(20)); + if (bytes.length >= 30 && bytes.subarray(0, 4).toString() === 'RIFF' && bytes.subarray(8, 12).toString() === 'WEBP' && bytes.subarray(12, 16).toString() === 'VP8X') return dimensions('image/webp', 1 + bytes.readUIntLE(24, 3), 1 + bytes.readUIntLE(27, 3)); + if (bytes.length > 9 && bytes[0] === 0xff && bytes[1] === 0xd8) for (let offset = 2; offset + 9 < bytes.length;) { if (bytes[offset] !== 0xff) return null; const marker = bytes[offset + 1], length = bytes.readUInt16BE(offset + 2); if (length < 2 || offset + 2 + length > bytes.length) return null; if ([0xc0, 0xc1, 0xc2, 0xc3, 0xc5, 0xc6, 0xc7, 0xc9, 0xca, 0xcb, 0xcd, 0xce, 0xcf].includes(marker)) return dimensions('image/jpeg', bytes.readUInt16BE(offset + 7), bytes.readUInt16BE(offset + 5)); offset += length + 2; } + return null; +} +function dimensions(contentType: 'image/png' | 'image/jpeg' | 'image/webp', width: number, height: number) { return width >= 1 && height >= 1 && width <= 8192 && height <= 8192 ? { contentType, width, height } : null; } diff --git a/apps/worker/src/config.ts b/apps/worker/src/config.ts index 58c58bb..6c492bf 100644 --- a/apps/worker/src/config.ts +++ b/apps/worker/src/config.ts @@ -33,6 +33,7 @@ export const workerConfig = { storeSecretEncryptionKey: required('STORE_SECRET_ENCRYPTION_KEY'), objectStorageEndpoint: process.env.OBJECT_STORAGE_ENDPOINT ?? 'http://localhost:9000', + objectStorageBucket: process.env.OBJECT_STORAGE_BUCKET ?? 'atlas-assets', objectStorageForcePathStyle: (process.env.OBJECT_STORAGE_FORCE_PATH_STYLE ?? 'true') === 'true', objectStorageAccessKey: objectStorageCredential( diff --git a/apps/worker/src/main.ts b/apps/worker/src/main.ts index 806a555..c9f4d9c 100644 --- a/apps/worker/src/main.ts +++ b/apps/worker/src/main.ts @@ -12,6 +12,7 @@ import { } from './ozon-client.js'; import { decryptStoreSecret } from './store-secret.crypto.js'; import { MediaUploadCleanup } from './media-upload-cleanup.js'; +import { materializeCollectionAssets } from './collection-assets.js'; const connection = new Redis(workerConfig.redisUrl, { maxRetriesPerRequest: null, @@ -42,6 +43,8 @@ const worker = new Worker( await verifyStoreCredentials(database, job.data as OutboxJob & { payload: { store_id: string; operation_id: string; secret_id?: string } }); } else if (job.name === 'collection.cleanup.requested') { await cleanupCollection(database, job.data as OutboxJob & { payload: { operation_id: string; collection_item_ids: string[] } }); + } else if (job.name === 'collection.assets.materialize.requested' || job.name === 'collection.assets.retry.requested') { + await materializeCollectionAssets(database, objectStorage, workerConfig.objectStorageBucket, job.data); } }, { connection, concurrency: 5 }, diff --git a/docs/api/collection-and-extension.md b/docs/api/collection-and-extension.md index a3e8533..bec1245 100644 --- a/docs/api/collection-and-extension.md +++ b/docs/api/collection-and-extension.md @@ -57,7 +57,7 @@ Headers：扩展 Bearer token 与 `Idempotency-Key` 必填。请求体为已定 { "data": { "collection_item": { "id": "uuid", "status": "materializing", "source_title": "商品标题", "variant_count": 3 }, "product_id": "uuid", "reused": false, - "editor_entry_path": "/collection-items/uuid/edit", + "editor_entry_path": "/products/uuid", "materialization": { "total": 12, "queued": 12, "succeeded": 0, "failed": 0 } } } ``` diff --git a/docs/project/HANDOFF.md b/docs/project/HANDOFF.md index 6ea834a..6619dce 100644 --- a/docs/project/HANDOFF.md +++ b/docs/project/HANDOFF.md @@ -1,12 +1,12 @@ # Atlas Seller Kit 交接说明 -> 更新：2026-09-04。此文是新 AI 对话、架构讨论与开发工作的首读入口。开始开发前，必须先阅读并遵守 [`development-collaboration-contract.md`](./development-collaboration-contract.md)。 +> 更新：2026-09-06。此文是新 AI 对话、架构讨论与开发工作的首读入口。开始开发前，必须先阅读并遵守 [`development-collaboration-contract.md`](./development-collaboration-contract.md)。 ## 1. 项目定位 这是一个全新的云端产品，不是 `ozon-seller-kit` 的本地版重构。目标用户可拥有多个门店，每个门店只能绑定一个平台；平台自身可支持多个市场、国家、语言和币种。因此语言、币种、类目和发布规则不能写死为 Ozon 或中文/俄文。 -当前已完成 pnpm monorepo 初始化：包含 Web、NestJS API、BullMQ Worker、WXT 浏览器扩展、共享 API 契约、数据库包、本地基础设施 Compose 与依赖锁文件。API 已实现健康/readiness、集中配置与安全基础、认证、个人工作空间上下文、平台目录/上下文、门店、水印/素材与采集箱基础管理；Worker 已具备 Ozon 凭据验证及采集记录清理消费链路，仓库同步与来源素材转存尚未接入。Web 已具备完整前端框架（布局、路由、主题）、登录/注册页、页头平台选择器、店铺管理、水印管理与真实采集箱页面；浏览器扩展仍为可运行骨架。不要从旧项目复制 SQLite 模型、无鉴权 API、旧 `/trial/:id` 或商品编辑页组件。可选择性复用已经验证过的纯计价公式、采集扩展思路和 AI 任务经验，但必须先适配多租户和云端边界。 +当前已完成 pnpm monorepo 初始化：包含 Web、NestJS API、BullMQ Worker、WXT 浏览器扩展、共享 API 契约、数据库包、本地基础设施 Compose 与依赖锁文件。API 已实现健康/readiness、集中配置与安全基础、认证、个人工作空间上下文、平台目录/上下文、门店、水印/素材、采集箱与扩展采集入库；Worker 已具备 Ozon 凭据验证、采集记录清理和来源素材转存消费链路，仓库同步尚未接入。Web 已具备完整前端框架（布局、路由、主题）、登录/注册页、页头平台选择器、店铺管理、水印管理、真实采集箱页面和采集数据测试页；浏览器扩展已提供 PKCE 授权、悬浮窗及 Ozon/1688/淘宝/天猫首版页面状态解析。不要从旧项目复制 SQLite 模型、无鉴权 API、旧 `/trial/:id` 或商品编辑页组件。可选择性复用已经验证过的纯计价公式、采集扩展思路和 AI 任务经验，但必须先适配多租户和云端边界。 当前运行时基线：Node 24.20.0、pnpm 11.25.0、PostgreSQL 18.6、Redis 7.4.11、MinIO `RELEASE.2025-09-07T16-13-09Z`；Web 默认端口为 `8100`，API 默认端口为 `8200`，扩展 dev server 默认端口为 `8110`（仅本地开发用），Worker 无 HTTP 端口。完整目录、依赖边界与版本表见 [`architecture/project-structure.md`](../architecture/project-structure.md) 和 [`architecture/technology-and-deployment.md`](../architecture/technology-and-deployment.md)。 @@ -87,6 +87,21 @@ - Web 的采集箱已替换占位页，支持容量显示、标题/来源筛选、手动创建、批量删除、清空确认、空/加载/错误状态。商品编辑入口明确保留给下一任务，未伪造可发布能力。 - `apps/api/test/collection.e2e.test.ts` 覆盖手动创建、幂等重放、列表读取、乐观锁冲突和软删除；该测试、本次受影响的 API/Worker/Web TypeScript 检查及 Web 生产构建均已通过。任务清单见 [`m2.1-collection-box-foundation.md`](./m2.1-collection-box-foundation.md)。 +### 1.9 开发进度（2026-09-05）：M2.2 / M2.3 扩展采集首个闭环 + +- 已实现扩展 PKCE 授权码、扩展专用 access/refresh token 轮换、现有 Atlas Web 会话的静默授权分支，以及扩展会话检查/撤销。扩展不读取 Web token 或 refresh Cookie；回调 URI 必须按已安装扩展 ID 精确登记。 +- 已实现 `POST /api/extension/materials`：从扩展身份推导工作空间，创建或复用未绑定目标平台/门店的采集记录、SPU、SKU 草稿和来源素材任务。采集载荷严格限制为来源证据、规范化字段和素材 URL，不能携带店铺、Cookie、API Key 或目标平台。 +- Worker 已消费素材转存 outbox 事件，执行 HTTPS 公网地址校验、大小/图片格式限制、哈希去重与 MinIO/Kodo 转存；单个素材失败不回滚采集箱商品。 +- 浏览器扩展已实现 Ozon 公开商品页的小型 Shadow DOM 悬浮窗及未登录/已登录状态。Ozon 首版解析器以当前公开 SKU 和公开变体状态为证据，不生成笛卡尔积假 SKU。Ozon Seller 四接口本地观察、Ozon 变体详情逐页补全及 1688/淘宝/天猫解析仍未开始。 +- 详情、边界和本地配置见 [`m2.2-m2.3-extension-collection.md`](./m2.2-m2.3-extension-collection.md)。 + +### 1.10 开发进度（2026-09-06）：M2.4 来源解析器与采集测试页 + +- 扩展浮窗现覆盖 1688 商品详情、淘宝商品详情和天猫商品详情。1688 仅遍历 `skuMapOriginal` 中实际存在的组合；淘宝/天猫仅在同时取得 `skuBase.skus[].skuId + propPath` 时创建多 SKU，并以 `skuCore.sku2info` 补充参考价格。规格轴本身从不展开为笛卡尔积。 +- 缺少真实组合证据时，扩展只保存规格轴，服务端创建一个默认 SKU 并标记为待确认；颜色等轴值图片仍作为候选素材，绝不伪造来源 SKU 图片绑定。 +- “编辑上架”暂时不进入未完成的商品编辑模块，改为打开 `/collection-test/{collectionItemId}`。该只读测试页展示实际保存的规范化载荷、原始证据、SKU 草稿和素材转存任务，用于验证每个平台的采集结果。`GET /api/collection-items/{id}` 已提供该工作空间隔离的检查读模型。 +- 已通过 API/Worker TypeScript 检查、Web 与扩展生产构建；`apps/api/test/extension.e2e.test.ts` 已覆盖 PKCE 授权、扩展会话、采集幂等复用和测试页所用检查读模型。尚未在真实来源商品页执行样本回归，接手者应按第 6 节先完成该验证。 + ## 2. 已确认产品决策 | 主题 | 决策 | @@ -139,17 +154,17 @@ User → Workspace → Membership / Role - 所有接口遵循统一鉴权、工作空间隔离、页头平台上下文、金额字符串、错误格式、`Idempotency-Key`、`If-Match` 和异步 `operation` 约定。 - **响应信封是强制契约：** 成功单资源或同步动作返回 `{ "data": T }`，列表返回 `{ "data": T[], "meta": PaginationMeta }`，失败返回 `{ "error": ApiError }`，`204 No Content` 不返回响应体。项目不采用完整 JSON:API；任何调用方不得自行增加嵌套 `data` 或以 `data: null` 表示错误。完整规则见 [`api/conventions.md`](../api/conventions.md)。 - 关键异步动作必须在同一事务写入业务记录、`operations`（如需用户可见进度）和 `outbox_events`；Dispatcher 至少一次投递 BullMQ，Worker 必须幂等。 -- DDL 已通过本地隔离 PostgreSQL 执行验证；迁移执行器、认证/工作空间基础、幂等与 Outbox 写入闭环、Outbox Dispatcher、Ozon 凭据验证 Worker、上传意图清理 Worker 与采集记录清理 Worker 均已实现。仓库同步、来源素材转存、发布和在线商品同步 Worker 仍待实现。 +- DDL 已通过本地隔离 PostgreSQL 执行验证；迁移执行器、认证/工作空间基础、幂等与 Outbox 写入闭环、Outbox Dispatcher、Ozon 凭据验证 Worker、上传意图清理 Worker、采集记录清理 Worker 与来源素材转存 Worker 均已实现。仓库同步、发布和在线商品同步 Worker 仍待实现。 仍需在对应模块开始编码前完成的 P1/P2 决策和数据模型补丁，集中记录在 [`reviews/implementation-readiness-open-items.md`](../reviews/implementation-readiness-open-items.md)。 ## 6. 下一步建议 -下一步建议按采集入口顺序推进： +下一步建议按以下顺序推进： -1. M2.2：实现扩展 PKCE 授权、`POST /api/extension/materials` 与来源素材转存 Worker；素材只传来源 URL 和净化证据，不传 Cookie、店铺凭据或二进制。 -2. M2.3：实现插件悬浮窗与 Ozon 公开商品页解析器，验证真实 SKU 组合、素材关联和重复采集复用。 -3. M2.4：实现 1688、淘宝、天猫解析器及各自的稳定 SKU 证据规则。 +1. 用允许保存的单 SKU、单规格轴和颜色×尺码页面样本，在 Ozon、1688、淘宝、天猫逐项验证采集结果；页面状态未暴露组合数据时补 MAIN world bridge 或站点适配器，但绝不以规格轴猜测 SKU。 +2. 完成 Ozon 公开变体队列的逐 SKU hydrate；再独立实施 Seller 网页四接口本地观察实验，默认关闭、仅保存净化结果且不影响标准采集。 +3. 将当前 `/collection-test/{collectionItemId}` 只读检查面替换为正式商品编辑入口：按页头平台创建发布配置、预选同平台默认门店，并接入 SKU/媒体/类目编辑。 4. 进入商品编辑/在线同步前，关闭评审中的 P1：SKU 与平台发布配置同商品归属、未映射属性持久化、统一乐观锁 Repository 规则；平台能力 Schema 已在 M2.1 按固定键落地。在线商品归档策略和 Client ID 编辑边界仍待确认。 账户资料的已确认补充：`users.avatar_asset_id` 只保存素材引用，绝不保存公开 URL；`users.revision` 用于 `PATCH /api/auth/me` 的 `If-Match`。资料更新已实现幂等与版本冲突检测；修改密码已实现旧密码校验、按 IP/用户限流、撤销旧会话并轮换新会话。头像上传必须复用素材模块的预签名直传与校验流程，不引入 base64 直传；素材模块尚未实现时头像只能为 `null`。 @@ -165,4 +180,6 @@ User → Workspace → Membership / Role - [数据库角色与 RLS](../architecture/database-security.md) - [实现前待决与待落地事项](../reviews/implementation-readiness-open-items.md) - [浏览器采集插件需求与技术方案](../requirements/collector-browser-extension.md) +- [M2.2 / M2.3 扩展采集交接](./m2.2-m2.3-extension-collection.md) +- [M2.4 来源解析与测试页交接](./m2.4-source-parsers-and-inspection.md) - [技术栈、异步任务与部署方案](../architecture/technology-and-deployment.md) diff --git a/docs/project/m2.2-m2.3-extension-collection.md b/docs/project/m2.2-m2.3-extension-collection.md new file mode 100644 index 0000000..976a65f --- /dev/null +++ b/docs/project/m2.2-m2.3-extension-collection.md @@ -0,0 +1,34 @@ +# M2.2 / M2.3：扩展授权、素材转存与 Ozon 公开页采集 + +> 状态：已完成首个可运行闭环（2026-09-05）。Seller 网页四接口观察实验不属于本批次，仍待独立任务实施。 + +## 已交付 + +1. 扩展使用 PKCE 授权码登录，而不是读取 Atlas Web 的 localStorage 或 refresh Cookie。`GET /api/extension/authorize` 会优先复用用户浏览器已有的 Atlas Web 会话：会话有效则无感签发一次性 code；没有会话才跳转登录页，成功登录后回到原授权请求。 +2. 服务端保存 code 与扩展 refresh token 的哈希，code 60 秒且一次性使用，扩展 access token 15 分钟、refresh token 30 天并轮换。令牌含扩展专用 audience，不能被普通 Web/API 身份守卫接受。 +3. 扩展上报仅携带来源、规范化 SKU/规格轴、候选素材 URL 和证据；`POST /api/extension/materials` 从扩展令牌解析工作空间，不接受目标平台、门店、Cookie 或 API Key。相同工作空间、来源平台和来源 item key 复用同一采集记录。 +4. 上报事务创建来源采集记录、SPU、来源确认 SKU 和素材转存任务；素材异步事件经 Transactional Outbox 进入 Worker。Worker 仅下载 HTTPS 公网地址，限制重定向、私网地址、20MB 和 PNG/JPEG/WebP，按 SHA-256 去重后写入对象存储。单张失败不会使商品草稿丢失。 +5. Ozon (`ru/kz/by`) 商品详情页显示 Shadow DOM 小浮窗。未登录只显示“登录 Atlas ERP”；已登录显示“编辑上架 / 采集商品 / 进入 ERP”。编辑上架成功后打开现有 `/products/{productId}` ERP 编辑入口；采集商品只提示进入采集箱。 +6. Ozon 解析器保留当前公开 SKU，并只从带 `webProducts`、`webAspects` 或 `aspects` 标识的页面状态发现关联 SKU；不会扫描整页 SKU 或把规格轴做笛卡尔积。来源未给可确认组合时，仍保留当前 SKU。主图作为商品候选素材上报，不伪造 SKU 图片关系。 + +## 配置与安全前提 + +- `EXTENSION_COLLECTOR_REDIRECT_URI` 必须精确等于已安装扩展的 `chrome.identity.getRedirectURL('callback')`；不可用通配符代替。 +- 本地默认 API 为 `http://localhost:8200`。生产构建应为 Atlas API 域配置扩展 host permission 与 `ATLAS_API_ORIGIN`，并登记对应扩展 ID 的回调 URI。 +- 扩展 refresh token 仅存 Chrome extension storage；Web refresh Cookie 从不返回给扩展。Worker 不接触浏览器 Cookie。 + +## 尚未实现 + +- Ozon 变体队列逐页 hydrate、SKU 级公开详情补全；当前只消费已加载公开页面状态。 +- 1688、淘宝、天猫解析器，以及 Ozon Seller 网页四接口本地观察实验。 +- 采集箱详情页的素材关联可视化、重试 UI 和商品编辑/发布闭环。 + +## 验收命令 + +```sh +./packages/contracts/node_modules/.bin/tsc -p packages/contracts/tsconfig.json +./apps/api/node_modules/.bin/tsc -p apps/api/tsconfig.json --noEmit +./apps/worker/node_modules/.bin/tsc -p apps/worker/tsconfig.json --noEmit +(cd apps/extension && ./node_modules/.bin/wxt prepare && ./node_modules/.bin/tsc -p tsconfig.json --noEmit && ./node_modules/.bin/wxt build) +(cd apps/web && ./node_modules/.bin/vite build) +``` diff --git a/docs/project/m2.4-source-parsers-and-inspection.md b/docs/project/m2.4-source-parsers-and-inspection.md new file mode 100644 index 0000000..f61bb92 --- /dev/null +++ b/docs/project/m2.4-source-parsers-and-inspection.md @@ -0,0 +1,27 @@ +# M2.4：1688、淘宝、天猫解析器与采集数据测试页 + +> 状态：已完成第一版页面状态解析与结果检查面（2026-09-06）。 + +## 真实 SKU 证据规则 + +| 来源 | 创建多 SKU 的唯一证据 | 禁止行为 | +|---|---|---| +| 1688 | `skuMapOriginal` 的实际组合行；优先使用行内 `skuId`，否则以来源组合键为稳定 key。 | 用 `skuProps` 的规格值排列组合制造 SKU。 | +| 淘宝 / 天猫 | `skuBase.skus[]` 的 `skuId + propPath`；`propPath` 映射回 `skuBase.props`。 | 只有 `props` 时把颜色、尺码等规格轴展开成 SKU。 | + +`skuCore.sku2info[skuId]` 只补充已验证 SKU 的参考价格，不能单独证明一个 SKU 存在。来源未给出组合表时，服务端保留规格轴、创建默认 SKU，并让该记录进入 `needs_review`。 + +## 测试页 + +扩展的“编辑上架”在采集成功后打开 `/collection-test/{collectionItemId}`，而不是进入尚未完成的商品编辑模块。页面只读显示： + +- 采集来源、解析器版本、状态、SKU 数和素材任务数； +- 实际创建的 SKU 草稿及其来源 key； +- 每张候选素材的转存状态和失败原因； +- 持久化的 `normalized_payload` 与 `raw_evidence` JSON。 + +这让测试者能直接比对页面上的规格组合和服务端保存数据。它不是商品编辑页，也不会把来源价格、属性或媒体关系当作已确认发布数据。 + +## 已知边界 + +页面状态结构会随来源站灰度变动；当前解析器只读取已在页面脚本中可见的 JSON 对象。若页面数据仅存在于运行时内存或经签名接口延迟获取，本版会保留默认 SKU/规格轴而不猜测组合。后续需要使用允许的快照样本补充 MAIN world bridge 和针对站点版本的适配器测试。 diff --git a/docs/requirements/collector-browser-extension.md b/docs/requirements/collector-browser-extension.md index 6ef6019..f24e86c 100644 --- a/docs/requirements/collector-browser-extension.md +++ b/docs/requirements/collector-browser-extension.md @@ -49,7 +49,7 @@ | 操作 | 流程 | 成功结果 | |---|---|---| -| 编辑上架 | 扫描当前页 → 上报采集记录 → 创建/复用未绑定目标的商品基础草稿 → 打开 ERP 编辑入口 | 新标签打开 `/collection-items/{collectionItemId}/edit`；ERP 按页头当前平台取得发布配置并预选门店后进入 `/products/{productId}/editor`，素材继续后台转存。 | +| 编辑上架 | 扫描当前页 → 上报采集记录 → 创建/复用未绑定目标的商品基础草稿 → 打开 ERP 编辑入口 | 新标签打开当前已有的 `/products/{productId}` 入口；后续编辑模块在此入口按页头当前平台取得发布配置并预选门店，素材继续后台转存。 | | 采集商品 | 扫描当前页 → 上报采集记录 | 停留来源页，提示“已进入采集箱”；用户以后从采集箱编辑。 | | 进入 ERP | 不扫描、不上报 | 新标签打开 Atlas ERP 首页。 | diff --git a/packages/contracts/src/collection.ts b/packages/contracts/src/collection.ts index cf348a8..da8dada 100644 --- a/packages/contracts/src/collection.ts +++ b/packages/contracts/src/collection.ts @@ -31,6 +31,18 @@ export type CollectionItem = { created_at: string; updated_at: string; revision: number; }; +/** Development-only read model used by the collection inspection page. It keeps + * raw evidence separate from editable product fields so parser behaviour can be + * reviewed before the product editor consumes it. */ +export type CollectionItemInspection = CollectionItem & { + normalized_payload: unknown; + raw_evidence: unknown; + parser_version: string | null; + collected_at: string | null; + variants: Array<{ id: string; source_variant_key: string | null; source_sku: string | null; display_name: string | null; sort_order: number }>; + asset_transfers: Array<{ id: string; source_variant_id: string | null; role: string; source_url: string; status: string; asset_id: string | null; error_payload: unknown; created_at: string; completed_at: string | null }>; +}; + /** * 浏览器扩展与 API 共享的采集载荷最小契约。 * 领域字段会随采集模块实现逐步补全，不能在扩展侧绕过此契约直接构造 API 请求。 @@ -44,13 +56,25 @@ export const collectedProductPayloadSchema = z.object({ parserVersion: z.string().min(1), }), common: z.object({ - title: z.string().min(1).optional(), - }), - variantAxes: z.array(z.unknown()), - variants: z.array(z.unknown()), - assets: z.array(z.unknown()), - evidence: z.array(z.unknown()), -}); + title: z.object({ value: z.string().min(1).max(1_000), raw_value: z.string().max(4_000).optional(), confidence: z.enum(['verified', 'partial', 'ambiguous']).default('partial') }).optional(), + description: z.object({ value: z.string().max(100_000), raw_value: z.string().max(120_000).optional(), confidence: z.enum(['verified', 'partial', 'ambiguous']).default('partial') }).optional(), + sourcePrice: z.object({ amount: z.string().regex(/^\d+(\.\d{1,4})?$/), currencyCode: z.string().length(3).toUpperCase(), raw_value: z.string().max(200).optional() }).optional(), + attributes: z.array(z.object({ key: z.string().min(1).max(200), name: z.string().min(1).max(300), value: z.string().max(2_000), raw_value: z.string().max(4_000).optional() })).max(500).default([]), + categories: z.array(z.object({ key: z.string().min(1).max(300), name: z.string().min(1).max(500) })).max(50).default([]), + }).strict(), + variantAxes: z.array(z.object({ key: z.string().min(1).max(300), name: z.string().min(1).max(300), values: z.array(z.object({ key: z.string().min(1).max(300), name: z.string().min(1).max(300), image_url: z.url().optional() })).max(200) })).max(20), + variants: z.array(z.object({ + sourceVariantKey: z.string().min(1).max(500), sourceSku: z.string().max(500).optional(), displayName: z.string().min(1).max(1_000), + selections: z.array(z.object({ axisKey: z.string().min(1).max(300), valueKey: z.string().min(1).max(300), valueName: z.string().min(1).max(300) })).max(50), + sourcePrice: z.object({ amount: z.string().regex(/^\d+(\.\d{1,4})?$/), currencyCode: z.string().length(3).toUpperCase() }).optional(), + confidence: z.enum(['verified', 'partial']), evidenceRefs: z.array(z.string().min(1).max(300)).max(100), + }).strict()).max(100), + assets: z.array(z.object({ + url: z.url(), role: z.enum(['source_main', 'source_sku', 'source_detail']), variantSourceKey: z.string().min(1).max(500).optional(), + relation: z.enum(['variant_direct', 'axis_value', 'product_candidate', 'inherited_product_default']), sortOrder: z.number().int().min(0).max(10_000).default(0), refererOrigin: z.url().optional(), evidenceRef: z.string().max(300).optional(), + }).strict()).max(500), + evidence: z.array(z.object({ ref: z.string().min(1).max(300), kind: z.enum(['page', 'state', 'json_ld', 'dom']), path: z.string().max(1_000).optional(), raw_value: z.string().max(20_000).optional() }).strict()).max(500), +}).strict(); export type CollectedProductPayload = z.infer< typeof collectedProductPayloadSchema diff --git a/packages/database/migrations/010_extension_collection_auth.sql b/packages/database/migrations/010_extension_collection_auth.sql new file mode 100644 index 0000000..73f3fd4 --- /dev/null +++ b/packages/database/migrations/010_extension_collection_auth.sql @@ -0,0 +1,40 @@ +SET search_path TO atlas, public; + +CREATE TABLE extension_authorization_codes ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + workspace_id UUID NOT NULL REFERENCES workspaces(id) ON DELETE RESTRICT, + user_id UUID NOT NULL REFERENCES users(id) ON DELETE RESTRICT, + client_id TEXT NOT NULL, + redirect_uri TEXT NOT NULL, + code_hash TEXT NOT NULL UNIQUE, + code_challenge TEXT NOT NULL, + expires_at TIMESTAMPTZ NOT NULL, + consumed_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CHECK (expires_at > created_at), + UNIQUE (workspace_id, id) +); + +CREATE TABLE extension_sessions ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + workspace_id UUID NOT NULL REFERENCES workspaces(id) ON DELETE RESTRICT, + user_id UUID NOT NULL REFERENCES users(id) ON DELETE RESTRICT, + client_id TEXT NOT NULL, + refresh_token_hash TEXT NOT NULL UNIQUE, + expires_at TIMESTAMPTZ NOT NULL, + revoked_at TIMESTAMPTZ, + last_used_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + UNIQUE (workspace_id, id) +); +CREATE INDEX ix_extension_sessions_active ON extension_sessions (workspace_id, user_id, client_id, expires_at) + WHERE revoked_at IS NULL; + +ALTER TABLE extension_authorization_codes ENABLE ROW LEVEL SECURITY; +ALTER TABLE extension_authorization_codes FORCE ROW LEVEL SECURITY; +CREATE POLICY workspace_isolation ON extension_authorization_codes + USING (workspace_id = app_current_workspace_id()) WITH CHECK (workspace_id = app_current_workspace_id()); +ALTER TABLE extension_sessions ENABLE ROW LEVEL SECURITY; +ALTER TABLE extension_sessions FORCE ROW LEVEL SECURITY; +CREATE POLICY workspace_isolation ON extension_sessions + USING (workspace_id = app_current_workspace_id()) WITH CHECK (workspace_id = app_current_workspace_id());