import type { CollectedProductPayload } from '@atlas/contracts'; import { dedupeImageUrl, normaliseImageUrl } from './product-media.js'; export type OzonPageJsonRequest = { kind: 'modal-aspects' | 'product' | 'product-detail'; sku: string; }; export type OzonPageJsonFetcher = ( request: OzonPageJsonRequest, ) => Promise; type Category = CollectedProductPayload['common']['categories'][number]; type Axis = CollectedProductPayload['variantAxes'][number]; type Variant = CollectedProductPayload['variants'][number]; type Asset = CollectedProductPayload['assets'][number]; type JsonRecord = Record; type Candidate = { sku: string; title?: string; price?: string; selections: Variant['selections']; evidenceRefs: Set; hydrated: boolean; }; export function isOzonProductPage(url = location.href) { return /^https:\/\/www\.ozon\.(ru|kz|by)\/product\//.test(url); } /** * Ozon does not expose a stable public SPU / variant-group identifier. The * public contract is the set of real variant SKUs in webAspects. We retain the * entered SKU as provenance and let the API merge re-collections by SKU-set * overlap instead of inventing a group ID. */ export async function collectOzonProduct( documentRoot: Document = document, pageUrl = location.href, fetchPageJson?: OzonPageJsonFetcher, ): Promise { const canonical = documentRoot.querySelector('link[rel="canonical"]') ?.href ?? pageUrl.split('?')[0]; const currentSku = skuFromUrl(canonical); if (!currentSku) throw new Error('未识别到 Ozon 商品 SKU'); const evidence: CollectedProductPayload['evidence'] = [ { ref: 'page:url', kind: 'page', path: 'location.href', raw_value: canonical, }, ]; const candidates = new Map(); const axes = new Map(); const assets: Asset[] = []; const assetKeys = new Set(); const localStates = readLocalWidgetStates(documentRoot); let attributes = extractOzonCharacteristics(localStates); let description = extractOzonDescription(localStates); const jsonLdDescription = extractOzonJsonLdDescription(documentRoot); const heading = firstState(localStates, 'webProductHeading'); const priceState = firstState(localStates, 'webPrice'); const title = stringAt(heading, ['title']) ?? text(documentRoot.querySelector('h1')) ?? documentRoot .querySelector('meta[property="og:title"]') ?.content?.trim(); const commonPrice = money(stringAt(priceState, ['price'])) ?? parsePrice(documentRoot); const currency = currencyFor(documentRoot, stringAt(priceState, ['price'])); addCandidate(candidates, currentSku, { title, price: commonPrice, evidenceRef: 'page:url', }); const pageVariants = discoverVariants( localStates, candidates, axes, 'ozon:aspects:page', canonical, ); if (pageVariants) { evidence.push({ ref: 'ozon:aspects:page', kind: 'state', path: 'state-webAspects-*.data-state.aspects[].variants[]', raw_value: `发现 ${pageVariants} 个不同的公开 SKU`, }); } addGalleryAssets( localStates, currentSku, assets, assetKeys, 'ozon:gallery:page', canonical, ); addGalleryVideos( localStates, currentSku, assets, assetKeys, 'ozon:gallery:page', canonical, ); if (attributes.length) { evidence.push({ ref: 'ozon:characteristics:page', kind: 'state', path: 'webShortCharacteristics|webDetailedCharacteristics|webCharacteristics.characteristics[]', raw_value: `发现 ${attributes.length} 项商品特征`, }); } if (jsonLdDescription) { evidence.push({ ref: 'ozon:json-ld:product.description', kind: 'json_ld', path: 'script[type="application/ld+json"] Product.description', raw_value: `JSON-LD 商品描述 ${jsonLdDescription.value.length} 个字符`, }); } const categories = collectCategories(documentRoot, canonical); if (categories.length) { evidence.push({ ref: 'ozon:breadcrumb', kind: 'json_ld', path: 'BreadcrumbList/itemListElement', raw_value: categories.map((category) => category.name).join(' > '), }); } if (fetchPageJson) { try { const modalStates = widgetStatesFromPageJson( await fetchPageJson({ kind: 'modal-aspects', sku: currentSku }), ); const modalVariants = discoverVariants( modalStates, candidates, axes, 'ozon:aspects:modal', canonical, ); evidence.push({ ref: 'ozon:aspects:modal', kind: 'state', path: 'webAspectsModal-*.aspects[].variants[]', raw_value: `发现 ${modalVariants} 个不同的公开 SKU`, }); } catch (error) { evidence.push( fetchFailureEvidence('ozon:aspects:modal', currentSku, error), ); } // Rich content is a separate, fixed Ozon page segment. Ask for it // once for the entered SKU; never scrape arbitrary images after // scrolling through recommendations or buyer reviews. try { const detailStates = widgetStatesFromPageJson( await fetchPageJson({ kind: 'product-detail', sku: currentSku, }), ); const added = addDetailAssets( detailStates, assets, assetKeys, 'ozon:detail:page', canonical, ); description = description ?? extractOzonDescription(detailStates); const detailAttributes = extractOzonCharacteristics(detailStates); attributes = mergeOzonCharacteristics(attributes, detailAttributes); evidence.push({ ref: 'ozon:detail:page', kind: 'state', path: 'webDescription|webRichContent|webFeatures image fields', raw_value: `发现 ${added} 张商品详情图`, }); if (detailAttributes.length) { evidence.push({ ref: 'ozon:characteristics:detail', kind: 'state', path: 'webShortCharacteristics|webDetailedCharacteristics|webCharacteristics.characteristics[]', raw_value: `详情段补全 ${detailAttributes.length} 项商品特征`, }); } } catch (error) { evidence.push( fetchFailureEvidence('ozon:detail:page', currentSku, error), ); } // A detail page can expose a SKU omitted by the initial modal. Append // new real SKUs to this bounded queue; never construct combinations. const pending = [...candidates.keys()]; const queued = new Set(pending); const hydrated = new Set(); while (pending.length && hydrated.size < 100) { const batch = pending.splice(0, 3); const results = await Promise.all( batch.map(async (sku) => { try { return { sku, states: widgetStatesFromPageJson( await fetchPageJson({ kind: 'product', sku }), ), }; } catch (error) { return { sku, error }; } }), ); for (const result of results) { hydrated.add(result.sku); if ('error' in result) { evidence.push( fetchFailureEvidence( `ozon:hydrate:${result.sku}`, result.sku, result.error, ), ); continue; } const detail = firstState(result.states, 'webProductHeading'); const detailPrice = firstState(result.states, 'webPrice'); addCandidate(candidates, result.sku, { title: stringAt(detail, ['title']), price: money(stringAt(detailPrice, ['price'])), evidenceRef: `ozon:hydrate:${result.sku}`, hydrated: true, }); discoverVariants( result.states, candidates, axes, `ozon:hydrate:${result.sku}`, canonical, ); addGalleryAssets( result.states, result.sku, assets, assetKeys, `ozon:gallery:${result.sku}`, canonical, ); addGalleryVideos( result.states, result.sku, assets, assetKeys, `ozon:gallery:${result.sku}`, canonical, ); if (result.sku === currentSku) { const detailedAttributes = extractOzonCharacteristics( result.states, ); attributes = mergeOzonCharacteristics( attributes, detailedAttributes, ); if (detailedAttributes.length) { evidence.push({ ref: 'ozon:characteristics:current', kind: 'state', path: 'webCharacteristics-*.characteristics[]', raw_value: `当前 SKU 补全 ${detailedAttributes.length} 项商品特征`, }); } } for (const sku of candidates.keys()) { if (!queued.has(sku) && queued.size < 100) { queued.add(sku); pending.push(sku); } } } } evidence.push({ ref: 'ozon:variant-hydration', kind: 'state', path: 'page/json/v2 product queue', raw_value: `发现 ${candidates.size} 个 SKU,已尝试补全 ${hydrated.size} 个`, }); } else { evidence.push({ ref: 'ozon:variant-hydration', kind: 'state', path: 'page/json/v2 product queue', raw_value: '未连接 Ozon 页面桥接;仅保存当前页面已渲染的变体状态', }); } // Ozon's rich widget is preferred because it is the dedicated description // segment. JSON-LD is the proven old-collector fallback for pages where // Ozon only places the seller description in the initial document. description = description ?? jsonLdDescription; addAspectCoverAssets(axes, assets, assetKeys, canonical); const variants = [...candidates.values()] .slice(0, 100) .map((candidate) => ({ sourceVariantKey: candidate.sku, sourceSku: candidate.sku, displayName: candidate.title ?? `Ozon SKU ${candidate.sku}`, selections: candidate.selections, ...(candidate.price ? { sourcePrice: { amount: candidate.price, currencyCode: currency, }, } : {}), confidence: candidate.hydrated || candidate.sku === currentSku ? 'verified' : 'partial', evidenceRefs: [...candidate.evidenceRefs], })); return { source: { platform: 'ozon', // Entered SKU remains audit provenance. The API reuses by sourceSku overlap. itemKey: currentSku, canonicalUrl: canonical, collectedAt: new Date().toISOString(), parserVersion: 'ozon-public-2.3.0', }, common: { ...(title ? { title: { value: title, raw_value: title, confidence: 'verified' as const, }, } : {}), ...(commonPrice ? { sourcePrice: { amount: commonPrice, currencyCode: currency, raw_value: commonPrice, }, } : {}), ...(description ? { description: { value: description.value, raw_value: description.rawValue, confidence: 'verified' as const, }, } : {}), attributes, categories, }, variantAxes: [...axes.values()].slice(0, 20).map((axis) => ({ ...axis, values: axis.values.slice(0, 200), })), variants, assets, evidence, }; } function addCandidate( candidates: Map, sku: string, input: { title?: string; price?: string; selection?: Variant['selections'][number]; evidenceRef: string; hydrated?: boolean; }, ) { const candidate = candidates.get(sku) ?? { sku, selections: [], evidenceRefs: new Set(), hydrated: false, }; candidate.title = input.title ?? candidate.title; candidate.price = input.price ?? candidate.price; candidate.hydrated = candidate.hydrated || input.hydrated === true; candidate.evidenceRefs.add(input.evidenceRef); if ( input.selection && !candidate.selections.some( (selection) => selection.axisKey === input.selection?.axisKey && selection.valueKey === input.selection?.valueKey, ) ) { candidate.selections.push(input.selection); } candidates.set(sku, candidate); } function discoverVariants( states: JsonRecord[], candidates: Map, axes: Map, evidenceRef: string, pageUrl: string, ) { const discovered = new Set(); for (const state of states) { if (!Array.isArray(state.aspects)) continue; state.aspects.forEach((rawAspect, index) => { if (!isRecord(rawAspect) || !Array.isArray(rawAspect.variants)) return; const axisKey = stringAt(rawAspect, ['aspectKey', 'key', 'id']) ?? `ozon:aspect:${index}`; const axisName = stringAt(rawAspect, ['aspectName', 'name']) ?? axisKey; const axis = axes.get(axisKey) ?? { key: axisKey, name: axisName, values: [], }; for (const rawVariant of rawAspect.variants) { if (!isRecord(rawVariant)) continue; const sku = stringAt(rawVariant, ['sku']); if (!sku) continue; discovered.add(sku); if (!candidates.has(sku) && candidates.size >= 100) continue; const data = isRecord(rawVariant.data) ? rawVariant.data : {}; const valueName = stringAt(data, ['searchableText', 'title']) ?? `SKU ${sku}`; const valueKey = `${axisKey}:${valueName.toLocaleLowerCase()}`; if (!axis.values.some((value) => value.key === valueKey)) { const cover = ozonImageUrl( stringAt(data, ['coverImage']), pageUrl, ); axis.values.push({ key: valueKey, name: valueName, ...(cover ? { image_url: cover } : {}), }); } addCandidate(candidates, sku, { title: stringAt(data, ['title', 'searchableText']), price: money(stringAt(data, ['price', 'cardPrice'])), selection: { axisKey, valueKey, valueName }, evidenceRef, }); } if (axis.values.length) axes.set(axisKey, axis); }); } return discovered.size; } /** Adds variant-axis covers without pretending that they identify one SKU. */ function addAspectCoverAssets( axes: Map, assets: Asset[], assetKeys: Set, pageUrl: string, ) { for (const axis of axes.values()) { for (const value of axis.values) { const url = ozonImageUrl(value.image_url, pageUrl); if (!url || assets.length >= 500) continue; value.image_url = url; const key = `sku:${axis.key}:${value.key}:${dedupeImageUrl(url, 'ozon')}`; if (assetKeys.has(key)) continue; assetKeys.add(key); assets.push({ url, mediaType: 'image', role: 'source_sku', relation: 'axis_value', sortOrder: assets.length, evidenceRef: `ozon:aspect-cover:${axis.key}:${value.key}`, }); } } } /** * Detail widgets are allowlisted by the bridge. Reading image-shaped fields * from only these widgets is safe; scanning page-wide image nodes is not. */ function addDetailAssets( states: JsonRecord[], assets: Asset[], assetKeys: Set, evidenceRef: string, pageUrl: string, ) { const urls = new Set(); for (const state of states) { const widget = String(state.__atlasWidgetName ?? '').replace( /^state-/, '', ); if (!/^web(?:Description|RichContent|Features)-/.test(widget)) continue; collectDetailImageUrls(state, urls); } let added = 0; for (const rawUrl of urls) { if (assets.length >= 500) break; const url = ozonImageUrl(rawUrl, pageUrl); if (!url) continue; const key = `detail:${dedupeImageUrl(url, 'ozon')}`; if (assetKeys.has(key)) continue; assetKeys.add(key); assets.push({ url, mediaType: 'image', role: 'source_detail', relation: 'product_candidate', sortOrder: added++, evidenceRef, }); } return added; } function collectDetailImageUrls( value: unknown, urls: Set, visited = new Set(), ) { if (!value || typeof value !== 'object' || visited.has(value)) return; visited.add(value); if (Array.isArray(value)) { value.forEach((entry) => collectDetailImageUrls(entry, urls, visited)); return; } for (const [key, child] of Object.entries(value as JsonRecord)) { if ( typeof child === 'string' && /^(src|url|image|imageurl|coverimage)$/i.test(key) && /^https?:\/\//.test(child) ) { urls.add(child); } else if (child && typeof child === 'object') { collectDetailImageUrls(child, urls, visited); } } } type OzonCharacteristic = CollectedProductPayload['common']['attributes'][number]; /** * Ozon renders a short parameter list in the initial document and exposes the * complete list in the normal product page state. Keep this reader limited to * those named widgets so arbitrary page copy never turns into product params. */ export function extractOzonCharacteristics( states: JsonRecord[], ): OzonCharacteristic[] { const attributes: OzonCharacteristic[] = []; for (const state of states) { const widget = String(state.__atlasWidgetName ?? '').replace( /^state-/, '', ); if ( !/^web(?:ShortCharacteristics|DetailedCharacteristics|Characteristics)-/.test( widget, ) || !Array.isArray(state.characteristics) ) continue; for (const [index, entry] of state.characteristics.entries()) { if (!isRecord(entry)) continue; const name = richText(entry.title); const value = Array.isArray(entry.values) ? entry.values.map(richText).filter(Boolean).join(', ') : richText(entry.value); if (!name || !value) continue; const rawId = stringAt(entry, ['id']); attributes.push({ key: `ozon:${rawId ?? `characteristic:${normaliseKey(name, index)}`}`.slice( 0, 200, ), name: name.slice(0, 300), value: value.slice(0, 2_000), raw_value: value.slice(0, 4_000), }); } } return mergeOzonCharacteristics([], attributes); } function mergeOzonCharacteristics( base: OzonCharacteristic[], additions: OzonCharacteristic[], ) { const merged = [...base]; const seen = new Set( base.map((attribute) => `${attribute.name}\u0000${attribute.value}`), ); for (const attribute of additions) { if (merged.length >= 500) break; const identity = `${attribute.name}\u0000${attribute.value}`; if (seen.has(identity)) continue; seen.add(identity); merged.push(attribute); } return merged; } type OzonDescription = { value: string; rawValue: string }; /** Extracts only Ozon's rich-description payload, converting it to reviewable text. */ export function extractOzonDescription( states: JsonRecord[], ): OzonDescription | undefined { for (const state of states) { const widget = String(state.__atlasWidgetName ?? '').replace( /^state-/, '', ); if (!/^web(?:Description|RichContent)-/.test(widget)) continue; const raw = findRichAnnotationJson(state); if (!raw) continue; let value = raw; try { value = richDescriptionText(JSON.parse(raw)); } catch { // Some older page responses already contain plain text here. } value = value .replace(/\s+\n/g, '\n') .replace(/\n{3,}/g, '\n\n') .trim(); if (!value) continue; return { value: value.slice(0, 100_000), rawValue: raw.slice(0, 120_000), }; } return undefined; } /** Reads only schema.org Product.description as a fallback, never arbitrary * JSON-LD page copy. This mirrors the mature collector's JSON-LD path. */ export function extractOzonJsonLdDescription( root: Document, ): OzonDescription | undefined { for (const script of Array.from(root.scripts)) { if (script.type !== 'application/ld+json' || !script.textContent?.trim()) continue; try { const product = findJsonLdProduct(JSON.parse(script.textContent)); const raw = product && typeof product.description === 'string' ? product.description.trim() : ''; if (!raw) continue; return { value: raw.replace(/\s+\n/g, '\n').replace(/\n{3,}/g, '\n\n').slice(0, 100_000), rawValue: raw.slice(0, 120_000), }; } catch { // A malformed analytics JSON-LD block is not product evidence. } } return undefined; } function findJsonLdProduct( value: unknown, visited = new Set(), ): JsonRecord | undefined { if (!value || typeof value !== 'object' || visited.has(value)) return undefined; visited.add(value); if (Array.isArray(value)) { for (const child of value) { const product = findJsonLdProduct(child, visited); if (product) return product; } return undefined; } const record = value as JsonRecord; const type = record['@type']; if (type === 'Product' || (Array.isArray(type) && type.includes('Product'))) return record; if (Array.isArray(record['@graph'])) { const product = findJsonLdProduct(record['@graph'], visited); if (product) return product; } return undefined; } function findRichAnnotationJson( value: unknown, visited = new Set(), ): string | undefined { if (!value || typeof value !== 'object' || visited.has(value)) return undefined; visited.add(value); if (Array.isArray(value)) { for (const child of value) { const found = findRichAnnotationJson(child, visited); if (found) return found; } return undefined; } const record = value as JsonRecord; if ( typeof record.richAnnotationJson === 'string' && record.richAnnotationJson.trim() ) return record.richAnnotationJson; for (const child of Object.values(record)) { const found = findRichAnnotationJson(child, visited); if (found) return found; } return undefined; } function richDescriptionText( value: unknown, visited = new Set(), ): string { if (typeof value === 'string') return value.trim(); if (!value || typeof value !== 'object' || visited.has(value)) return ''; visited.add(value); if (Array.isArray(value)) return value .map((child) => richDescriptionText(child, visited)) .filter(Boolean) .join('\n'); const record = value as JsonRecord; const direct = typeof record.text === 'string' ? record.text : typeof record.content === 'string' ? record.content : ''; const children = Object.entries(record) .filter( ([key]) => !['text', 'content', 'type', 'src', 'url', 'image'].includes( key, ), ) .map(([, child]) => richDescriptionText(child, visited)) .filter(Boolean); return [direct, ...children].filter(Boolean).join('\n'); } function richText(value: unknown): string { if (typeof value === 'string' || typeof value === 'number') return String(value).replace(/\s+/g, ' ').trim(); if (Array.isArray(value)) return value.map(richText).filter(Boolean).join(' ').trim(); if (!isRecord(value)) return ''; return richText(value.textRs ?? value.text ?? value.content); } function normaliseKey(value: string, index: number) { return ( value .toLocaleLowerCase() .replace(/[^\p{L}\p{N}]+/gu, '-') .replace(/^-+|-+$/g, '') .slice(0, 150) || String(index) ); } function addGalleryAssets( states: JsonRecord[], sku: string, assets: Asset[], assetKeys: Set, evidenceRef: string, pageUrl: string, ) { const gallery = firstState(states, 'webGallery'); if (!gallery) return; const urls = extractOzonGalleryImageUrls(gallery, pageUrl); for (const [index, url] of urls.entries()) { if (assets.length >= 500) return; const key = `main:${sku}:${url ? dedupeImageUrl(url, 'ozon') : ''}`; if (assetKeys.has(key)) continue; assetKeys.add(key); assets.push({ url, mediaType: 'image', // A gallery is product photography for this real SKU. It is not // a SKU-option thumbnail merely because it is the second image. role: 'source_main', variantSourceKey: sku, relation: 'variant_direct', sortOrder: index, evidenceRef, }); } } /** Adds only Ozon's named product-gallery videos. Page-wide