import type { CollectedProductPayload } from '@atlas/contracts'; import { collectOzonProduct, isOzonProductPage, type OzonPageJsonFetcher, } from './ozon-parser.js'; import { collectPlatformDomAssets, normaliseImageUrl } from './product-media.js'; import { formatCollectedDescription } from './description-format.js'; type SourcePlatform = '1688' | 'taobao' | 'tmall' | 'ozon'; type Json = Record; type Axis = CollectedProductPayload['variantAxes'][number]; type Variant = CollectedProductPayload['variants'][number]; export function supportedProductPlatform( url = location.href, ): SourcePlatform | null { if (isOzonProductPage(url)) return 'ozon'; if (/^https:\/\/detail\.1688\.com\/offer\/\d+\.html/.test(url)) return '1688'; if (/^https:\/\/item\.taobao\.com\/item\.htm/.test(url)) return 'taobao'; if (/^https:\/\/detail\.tmall\.com\/item\.htm/.test(url)) return 'tmall'; return null; } /** Returns only the public, allowlisted product-state projection supplied by * the platform-specific MAIN-world bridge. */ export type MarketplacePageStateFetcher = ( platform: Exclude, ) => Promise; export async function collectCurrentProduct( root: Document = document, pageUrl = location.href, fetchOzonPageJson?: OzonPageJsonFetcher, fetchMarketplacePageState?: MarketplacePageStateFetcher, ): Promise { const platform = supportedProductPlatform(pageUrl); if (platform === 'ozon') return collectOzonProduct(root, pageUrl, fetchOzonPageJson); if (!platform) throw new Error('当前页面不是支持的商品详情页'); const pageState = fetchMarketplacePageState ? await fetchMarketplacePageState(platform) : undefined; return platform === '1688' ? collect1688(root, pageUrl, pageState) : collectTaobao(root, pageUrl, platform, pageState); } function collect1688(root: Document, pageUrl: string, bridgedState?: unknown): CollectedProductPayload { const state = [bridgedState, ...pageState(root)]; const skuInfoMap = findFirst(state, 'skuInfoMap'); const skuMap = isRecord(skuInfoMap) && Object.keys(skuInfoMap).length ? skuInfoMap : findFirst(state, 'skuMapOriginal'); const skuProps = findFirst(state, 'skuProps'); const axes = parse1688Axes(skuProps); const evidence: CollectedProductPayload['evidence'] = [ { ref: 'page:url', kind: 'page', path: 'location.href', raw_value: pageUrl, }, ]; const variants: Variant[] = []; const skuRows = Array.isArray(skuMap) ? skuMap.map((row) => [stringAt(isRecord(row) ? row : {}, ['specAttrs', 'propPath', 'skuId']) ?? '', row] as const) : isRecord(skuMap) ? Object.entries(skuMap) : []; if (skuRows.length) { evidence.push({ ref: '1688:sku-map', kind: 'state', path: isRecord(skuInfoMap) ? 'skuModel.skuInfoMap' : 'skuMapOriginal', raw_value: `组合记录 ${skuRows.length} 条`, }); for (const [combination, row] of skuRows) { if (!isRecord(row)) continue; const sourceSku = stringAt(row, ['skuId', 'sku_id', 'skuCode']); const specAttrs = stringAt(row, ['specAttrs', 'propPath']) ?? combination; const selections = selectionsFor1688Combination(specAttrs, axes); // 来源记录本身证明可售组合,不能用规格轴的笛卡尔积扩充 SKU。 const sourceVariantKey = sourceSku ?? `1688:${combination}`; if (!combination || combination === 'undefined') continue; variants.push({ sourceVariantKey, ...(sourceSku ? { sourceSku } : {}), displayName: selections .map((item) => item.valueName) .join(' / ') || specAttrs, selections, ...(moneyAt(row, ['price']) ? { sourcePrice: { amount: moneyAt(row, ['price'])!, currencyCode: 'CNY', }, } : {}), confidence: 'verified', evidenceRefs: ['1688:sku-map'], }); } } const payload = basePayload( '1688', offerId(pageUrl), root, pageUrl, axes, uniqueVariants(variants), evidence, '1688-structured-2.1.0', state, ); // SKU 图片由规格值和真实可售组合关联,独立于页面当时选中的 DOM 状态。 for (const variant of payload.variants) { for (const selection of variant.selections) { const rawUrl = axes.find((axis) => axis.key === selection.axisKey) ?.values.find((value) => value.key === selection.valueKey)?.image_url; const url = rawUrl && normaliseImageUrl(rawUrl, pageUrl, '1688'); if (!url || payload.assets.length >= 500) continue; if (payload.assets.some((asset) => asset.variantSourceKey === variant.sourceVariantKey && asset.url === url)) continue; payload.assets.push({ url, mediaType: 'image', role: 'source_sku', relation: 'axis_value', variantSourceKey: variant.sourceVariantKey, sortOrder: payload.assets.length, evidenceRef: '1688:sku-map', }); } } return payload; } function collectTaobao( root: Document, pageUrl: string, platform: 'taobao' | 'tmall', bridgedState?: unknown, ): CollectedProductPayload { const state = [bridgedState, ...pageState(root)]; const skuBase = findFirst(state, 'skuBase'); const skuCore = findFirst(state, 'skuCore'); const axes = parseTaobaoAxes(isRecord(skuBase) ? skuBase.props : undefined); const evidence: CollectedProductPayload['evidence'] = [ { ref: 'page:url', kind: 'page', path: 'location.href', raw_value: pageUrl, }, ]; const variants: Variant[] = []; const skus = isRecord(skuBase) && Array.isArray(skuBase.skus) ? skuBase.skus : []; const sku2info = isRecord(skuCore) && isRecord(skuCore.sku2info) ? skuCore.sku2info : {}; if (skus.length) evidence.push({ ref: 'taobao:sku-base', kind: 'state', path: 'skuBase.skus', raw_value: `组合记录 ${skus.length} 条`, }); for (const sku of skus) { if (!isRecord(sku)) continue; const skuId = stringAt(sku, ['skuId', 'sku_id']); const propPath = stringAt(sku, ['propPath', 'prop_path']); // skuId + propPath is the required combination proof. Props alone only // become axes and cause a single default SKU on the server. if (!skuId || !propPath) continue; const info = isRecord(sku2info[skuId]) ? sku2info[skuId] : {}; const selections = selectionsForPropPath(propPath, axes); variants.push({ sourceVariantKey: skuId, sourceSku: skuId, displayName: selections.map((item) => item.valueName).join(' / ') || `SKU ${skuId}`, selections, ...(moneyAt(info, ['price', 'promotionPrice', 'discountPrice']) ? { sourcePrice: { amount: moneyAt(info, [ 'price', 'promotionPrice', 'discountPrice', ])!, currencyCode: 'CNY', }, } : {}), confidence: 'verified', evidenceRefs: ['taobao:sku-base'], }); } return basePayload( platform, itemId(pageUrl), root, pageUrl, axes, uniqueVariants(variants), evidence, `${platform}-structured-2.0.1`, state, ); } function basePayload( platform: Exclude, itemKey: string, root: Document, pageUrl: string, axes: Axis[], variants: Variant[], evidence: CollectedProductPayload['evidence'], parserVersion: string, states: unknown[], ): CollectedProductPayload { const canonical = root.querySelector('link[rel="canonical"]')?.href ?? pageUrl.split('#')[0]; const facts = extractMarketplaceFacts(root, states, platform); const title = facts.title ?? ( root.querySelector('h1')?.textContent?.replace(/\s+/g, ' ').trim() || root .querySelector('meta[property="og:title"]') ?.content?.trim() ); const assets = collectPlatformDomAssets(platform, root, canonical); evidence.push({ ref: `${platform}:dom:image-profile`, kind: 'dom', path: 'platform-scoped gallery / sku / detail selectors', raw_value: `采集 ${assets.length} 张受限商品区域图片`, }); const videoCount = addMarketplaceVideos(platform, states, assets, canonical); if (videoCount) evidence.push({ ref: `${platform}:structured:videos`, kind: 'state', path: platform === '1688' ? 'gallery.fields.video|videos' : 'item.videos', raw_value: `采集 ${videoCount} 个结构化商品视频`, }); if (facts.attributes.length) evidence.push({ ref: `${platform}:structured:attributes`, kind: 'state', path: 'allowlisted product state parameter lists', raw_value: `采集 ${facts.attributes.length} 项结构化特征` }); if (facts.description) evidence.push({ ref: `${platform}:json-ld:description`, kind: 'json_ld', path: 'Product.description', raw_value: `采集 ${facts.description.length} 个字符描述` }); return { source: { platform, itemKey, canonicalUrl: canonical, collectedAt: new Date().toISOString(), parserVersion, }, common: { ...(title ? { title: { value: title, raw_value: title, confidence: 'verified' as const, }, } : {}), ...(facts.price ? { sourcePrice: { amount: facts.price, currencyCode: 'CNY', raw_value: facts.price } } : {}), ...(facts.description ? { description: { value: facts.description, raw_value: facts.description, confidence: 'verified' as const } } : {}), attributes: facts.attributes, categories: facts.categories, }, variantAxes: axes, variants, assets, evidence, }; } /** The old collector's proven video sources are structured product state. Do * not broaden this to page video elements: they also contain feeds and reviews. */ function addMarketplaceVideos( platform: Exclude, states: unknown[], assets: CollectedProductPayload['assets'], pageUrl: string, ) { const gallery = findFirst(states, 'gallery'); const item = findFirst(states, 'item'); const galleryFields = isRecord(gallery) && isRecord(gallery.fields) ? gallery.fields : gallery; const raw = platform === '1688' ? [ ...(isRecord(galleryFields) && Array.isArray(galleryFields.videos) ? galleryFields.videos : []), ...(isRecord(galleryFields) && galleryFields.video !== undefined ? [galleryFields.video] : []), ] : isRecord(item) && Array.isArray(item.videos) ? item.videos : []; let added = 0; const seen = new Set(assets.map((asset) => `video:${asset.url}`)); for (const entry of raw) { if (assets.length >= 500) break; const record = isRecord(entry) ? entry : undefined; const url = directVideoUrl( typeof entry === 'string' ? entry : stringAt(record ?? {}, ['url', 'videoUrl', 'src']), pageUrl, ); if (!url || seen.has(`video:${url}`)) continue; seen.add(`video:${url}`); const posterUrl = imageUrl( stringAt(record ?? {}, ['videoThumbnailURL', 'thumbnailUrl', 'coverUrl', 'coverImage', 'poster']), pageUrl, ); assets.push({ url, mediaType: 'video', ...(posterUrl ? { posterUrl } : {}), role: 'source_main', relation: 'product_candidate', sortOrder: assets.length, evidenceRef: `${platform}:structured:videos`, }); added += 1; } return added; } function directVideoUrl(value: string | undefined, pageUrl: string) { if (!value) return undefined; try { const url = new URL(value, pageUrl); if (url.protocol !== 'https:' || /\.m3u8(?:$|[?#])/i.test(url.pathname)) return undefined; return /\.(mp4|webm)(?:$|[?#])/i.test(url.pathname) ? url.href : undefined; } catch { return undefined; } } function imageUrl(value: string | undefined, pageUrl: string) { if (!value) return undefined; try { const url = new URL(value, pageUrl); return url.protocol === 'https:' ? url.href : undefined; } catch { return undefined; } } function extractMarketplaceFacts(root: Document, states: unknown[], platform: Exclude) { const attributes: CollectedProductPayload['common']['attributes'] = []; const seen = new Set(); for (const key of ['basicParamList', 'enhanceParamList', 'attributes', 'props']) { const list = findFirst(states, key); if (!Array.isArray(list)) continue; for (const [index, entry] of list.entries()) { if (!isRecord(entry)) continue; const name = stringAt(entry, ['propertyName', 'name', 'key', 'title']); const value = stringAt(entry, ['valueName', 'value', 'text']); if (!name || !value || seen.has(`${name}\u0000${value}`)) continue; seen.add(`${name}\u0000${value}`); attributes.push({ key: `${platform}:${stringAt(entry, ['propertyId', 'id', 'key']) ?? index}`.slice(0, 200), name: name.slice(0, 300), value: value.slice(0, 2_000), raw_value: value.slice(0, 4_000) }); } } const product = jsonLdProduct(root); const item = findFirst(states, 'item'); const temp = findFirst(states, 'tempModel'); const tradeModel = findFirst(states, 'tradeModel'); const priceVO = findFirst(states, 'priceVO'); const price = moneyAt(isRecord(tradeModel) ? tradeModel : {}, ['minPrice', 'price']) ?? moneyAt(isRecord(priceVO) ? priceVO : {}, ['priceText', 'price', 'extraPrice']) ?? decimal(product?.offers && isRecord(product.offers) ? product.offers.price : undefined); const categoryNames = Array.from(root.querySelectorAll('[class*="breadcrumb"] a, [class*="BreadCrumb"] a, nav a[href*="category"]')).map((link) => link.textContent?.replace(/\s+/g, ' ').trim() ?? '').filter(Boolean).slice(0, 50); return { title: (isRecord(item) ? stringAt(item, ['title', 'name']) : undefined) ?? (isRecord(temp) ? stringAt(temp, ['offerTitle']) : undefined) ?? (product ? stringAt(product, ['name']) : undefined), price, description: formatCollectedDescription( (product ? stringAt(product, ['description']) : undefined) ?? scopedDescription(root, platform) ?? '', ) || undefined, attributes, categories: [...new Set(categoryNames)].map((name) => ({ key: `${platform}:category:${name.toLowerCase()}`.slice(0, 300), name })), }; } function scopedDescription(root: Document, platform: Exclude) { const selectors = platform === '1688' ? ['.de-description-detail', '#detailContentContainer', '.html-description'] : ['[class*="tabDetailWrap--"]', '[class*="detailInfo--"]']; for (const selector of selectors) { const element = root.querySelector(selector); const value = formatCollectedDescription( element?.innerText ?? element?.textContent ?? '', ); if (value && value.length >= 10) return value.slice(0, 100_000); } return undefined; } function jsonLdProduct(root: Document): Json | undefined { for (const script of Array.from(root.scripts)) { if (script.type !== 'application/ld+json' || !script.textContent?.trim()) continue; try { const found = findJsonLdProduct(JSON.parse(script.textContent)); if (found) return found; } catch { /* ignored */ } } return undefined; } function findJsonLdProduct(value: unknown): Json | undefined { if (Array.isArray(value)) { for (const child of value) { const found = findJsonLdProduct(child); if (found) return found; } return undefined; } if (!isRecord(value)) return undefined; const type = value['@type']; if (type === 'Product' || (Array.isArray(type) && type.includes('Product'))) return value; return Array.isArray(value['@graph']) ? findJsonLdProduct(value['@graph']) : undefined; } function decimal(value: unknown) { const raw = typeof value === 'string' || typeof value === 'number' ? String(value).replace(',', '.') : ''; return /^\d+(\.\d{1,4})?$/.test(raw) ? raw : undefined; } function pageState(root: Document) { const all: unknown[] = []; for (const script of Array.from(root.scripts)) { const text = script.textContent?.trim() ?? ''; if (text.length < 2 || text.length > 5_000_000) continue; if (script.type === 'application/ld+json') { try { all.push(JSON.parse(text)); } catch { /* ignored evidence */ } } for (const marker of [ 'skuInfoMap', 'skuMapOriginal', 'skuBase', 'skuCore', 'skuProps', ]) { const object = objectAfterMarker(text, marker); if (object) all.push({ [marker]: object }); } } return all; } function objectAfterMarker(text: string, marker: string): unknown | null { const position = text.indexOf(marker); if (position < 0) return null; const start = text.slice(position).search(/[\[{]/); if (start < 0) return null; const first = position + start; const open = text[first]; const close = open === '{' ? '}' : ']'; let depth = 0; let quote = ''; let escaped = false; for ( let index = first; index < text.length && index - first < 2_000_000; index += 1 ) { const char = text[index]; if (quote) { if (escaped) escaped = false; else if (char === '\\') escaped = true; else if (char === quote) quote = ''; continue; } if (char === '"' || char === "'") { quote = char; continue; } if (char === open) depth += 1; if (char === close && --depth === 0) { try { return JSON.parse(text.slice(first, index + 1)); } catch { return null; } } } return null; } function findFirst(states: unknown[], key: string): unknown { for (const state of states) { const found = findKey(state, key); if (found !== undefined) return found; } return undefined; } function findKey( value: unknown, key: string, seen = new Set(), ): unknown { if (!value || typeof value !== 'object' || seen.has(value)) return undefined; seen.add(value); if (isRecord(value) && key in value) return value[key]; for (const child of Array.isArray(value) ? value : Object.values(value)) { const found = findKey(child, key, seen); if (found !== undefined) return found; } return undefined; } function parse1688Axes(value: unknown): Axis[] { const entries = Array.isArray(value) ? value : isRecord(value) && Array.isArray(value.skuProps) ? value.skuProps : []; return entries.flatMap((entry, axisIndex) => { if (!isRecord(entry)) return []; const axisKey = stringAt(entry, ['prop', 'propId', 'id']) ?? `axis:${axisIndex}`; const name = stringAt(entry, ['name', 'propName']) ?? axisKey; const values = ( Array.isArray(entry.value) ? entry.value : Array.isArray(entry.values) ? entry.values : [] ).flatMap((item, valueIndex) => isRecord(item) ? [ { key: stringAt(item, ['valueId', 'vid', 'id']) ?? `${axisKey}:${valueIndex}`, name: stringAt(item, ['name', 'valueName']) ?? `${valueIndex + 1}`, ...(urlAt(item, [ 'imageUrl', 'image', 'imageUrlLarge', ]) ? { image_url: urlAt(item, [ 'imageUrl', 'image', 'imageUrlLarge', ])!, } : {}), }, ] : [], ); return values.length ? [{ key: axisKey, name, values }] : []; }); } function parseTaobaoAxes(value: unknown): Axis[] { return (Array.isArray(value) ? value : []).flatMap((entry, index) => { if (!isRecord(entry)) return []; const key = stringAt(entry, ['pid', 'propId']) ?? `axis:${index}`; const name = stringAt(entry, ['name', 'propName']) ?? key; const values = ( Array.isArray(entry.values) ? entry.values : [] ).flatMap((item, valueIndex) => isRecord(item) ? [ { key: stringAt(item, ['vid', 'valueId']) ?? `${key}:${valueIndex}`, name: stringAt(item, ['name', 'valueName']) ?? `${valueIndex + 1}`, ...(urlAt(item, ['image', 'imageUrl']) ? { image_url: urlAt(item, [ 'image', 'imageUrl', ])!, } : {}), }, ] : [], ); return values.length ? [{ key, name, values }] : []; }); } function selectionsForPropPath(path: string, axes: Axis[]) { return path.split(/[;,]/).flatMap((pair) => { const [axisKey, valueKey] = pair.split(':'); const axis = axes.find((item) => item.key === axisKey); const value = axis?.values.find((item) => item.key === valueKey); return axis && value ? [{ axisKey, valueKey, valueName: value.name }] : []; }); } function selectionsForCombination(combination: string, axes: Axis[]) { const ids = combination .split(/[;,&]/) .map((value) => value.trim()) .filter(Boolean); return axes.flatMap((axis) => { const value = axis.values.find((item) => ids.includes(item.key)); return value ? [ { axisKey: axis.key, valueKey: value.key, valueName: value.name, }, ] : []; }); } function selectionsFor1688Combination(combination: string, axes: Axis[]) { const selected = selectionsForCombination(combination, axes); if (selected.length === axes.length) return selected; const parts = combination.split(/[;,&|]/).map((part) => part.trim()).filter(Boolean); return axes.flatMap((axis) => { const existing = selected.find((item) => item.axisKey === axis.key); if (existing) return [existing]; const value = axis.values.find((item) => parts.some((part) => part === item.name || part === `${axis.name}:${item.name}`) ); return value ? [{ axisKey: axis.key, valueKey: value.key, valueName: value.name }] : []; }); } function stringAt(value: Json, keys: string[]) { for (const key of keys) if (typeof value[key] === 'string' || typeof value[key] === 'number') return String(value[key]); return undefined; } function moneyAt(value: Json, keys: string[]) { const raw = stringAt(value, keys)?.replace(/,/g, '.'); return raw && /^\d+(\.\d{1,4})?$/.test(raw) ? raw : undefined; } function urlAt(value: Json, keys: string[]) { const raw = stringAt(value, keys); return raw && /^https:\/\//.test(raw) ? raw : undefined; } function uniqueVariants(variants: Variant[]) { const seen = new Set(); return variants.filter( (variant) => !seen.has(variant.sourceVariantKey) && !!seen.add(variant.sourceVariantKey), ); } function offerId(url: string) { return ( url.match(/\/offer\/(\d+)\.html/)?.[1] ?? `1688:${new URL(url).pathname}` ); } function itemId(url: string) { return new URL(url).searchParams.get('id') ?? new URL(url).pathname; } function isRecord(value: unknown): value is Json { return !!value && typeof value === 'object' && !Array.isArray(value); }