R524809
1201 lines
41 KiB
TypeScript
1201 lines
41 KiB
TypeScript
import type { CollectedProductPayload } from '@atlas/contracts';
|
|
import { dedupeImageUrl, normaliseImageUrl } from './product-media.js';
|
|
|
|
export type OzonPageJsonRequest = {
|
|
kind: 'modal-aspects' | 'product' | 'product-detail';
|
|
sku: string;
|
|
};
|
|
|
|
export type OzonPageJsonFetcher = (
|
|
request: OzonPageJsonRequest,
|
|
) => Promise<unknown>;
|
|
|
|
type Category = CollectedProductPayload['common']['categories'][number];
|
|
type Axis = CollectedProductPayload['variantAxes'][number];
|
|
type Variant = CollectedProductPayload['variants'][number];
|
|
type Asset = CollectedProductPayload['assets'][number];
|
|
type JsonRecord = Record<string, unknown>;
|
|
|
|
type Candidate = {
|
|
sku: string;
|
|
title?: string;
|
|
price?: string;
|
|
selections: Variant['selections'];
|
|
evidenceRefs: Set<string>;
|
|
hydrated: boolean;
|
|
};
|
|
|
|
export function isOzonProductPage(url = location.href) {
|
|
return /^https:\/\/www\.ozon\.(ru|kz|by)\/product\//.test(url);
|
|
}
|
|
|
|
/**
|
|
* Ozon does not expose a stable public SPU / variant-group identifier. The
|
|
* public contract is the set of real variant SKUs in webAspects. We retain the
|
|
* entered SKU as provenance and let the API merge re-collections by SKU-set
|
|
* overlap instead of inventing a group ID.
|
|
*/
|
|
export async function collectOzonProduct(
|
|
documentRoot: Document = document,
|
|
pageUrl = location.href,
|
|
fetchPageJson?: OzonPageJsonFetcher,
|
|
): Promise<CollectedProductPayload> {
|
|
const canonical =
|
|
documentRoot.querySelector<HTMLLinkElement>('link[rel="canonical"]')
|
|
?.href ?? pageUrl.split('?')[0];
|
|
const currentSku = skuFromUrl(canonical);
|
|
if (!currentSku) throw new Error('未识别到 Ozon 商品 SKU');
|
|
|
|
const evidence: CollectedProductPayload['evidence'] = [
|
|
{
|
|
ref: 'page:url',
|
|
kind: 'page',
|
|
path: 'location.href',
|
|
raw_value: canonical,
|
|
},
|
|
];
|
|
const candidates = new Map<string, Candidate>();
|
|
const axes = new Map<string, Axis>();
|
|
const assets: Asset[] = [];
|
|
const assetKeys = new Set<string>();
|
|
const localStates = readLocalWidgetStates(documentRoot);
|
|
let attributes = extractOzonCharacteristics(localStates);
|
|
let description = extractOzonDescription(localStates);
|
|
const jsonLdDescription = extractOzonJsonLdDescription(documentRoot);
|
|
const heading = firstState(localStates, 'webProductHeading');
|
|
const priceState = firstState(localStates, 'webPrice');
|
|
const title =
|
|
stringAt(heading, ['title']) ??
|
|
text(documentRoot.querySelector('h1')) ??
|
|
documentRoot
|
|
.querySelector<HTMLMetaElement>('meta[property="og:title"]')
|
|
?.content?.trim();
|
|
const commonPrice =
|
|
money(stringAt(priceState, ['price'])) ?? parsePrice(documentRoot);
|
|
const currency = currencyFor(documentRoot, stringAt(priceState, ['price']));
|
|
|
|
addCandidate(candidates, currentSku, {
|
|
title,
|
|
price: commonPrice,
|
|
evidenceRef: 'page:url',
|
|
});
|
|
const pageVariants = discoverVariants(
|
|
localStates,
|
|
candidates,
|
|
axes,
|
|
'ozon:aspects:page',
|
|
canonical,
|
|
);
|
|
if (pageVariants) {
|
|
evidence.push({
|
|
ref: 'ozon:aspects:page',
|
|
kind: 'state',
|
|
path: 'state-webAspects-*.data-state.aspects[].variants[]',
|
|
raw_value: `发现 ${pageVariants} 个不同的公开 SKU`,
|
|
});
|
|
}
|
|
addGalleryAssets(
|
|
localStates,
|
|
currentSku,
|
|
assets,
|
|
assetKeys,
|
|
'ozon:gallery:page',
|
|
canonical,
|
|
);
|
|
addGalleryVideos(
|
|
localStates,
|
|
currentSku,
|
|
assets,
|
|
assetKeys,
|
|
'ozon:gallery:page',
|
|
canonical,
|
|
);
|
|
if (attributes.length) {
|
|
evidence.push({
|
|
ref: 'ozon:characteristics:page',
|
|
kind: 'state',
|
|
path: 'webShortCharacteristics|webDetailedCharacteristics|webCharacteristics.characteristics[]',
|
|
raw_value: `发现 ${attributes.length} 项商品特征`,
|
|
});
|
|
}
|
|
if (jsonLdDescription) {
|
|
evidence.push({
|
|
ref: 'ozon:json-ld:product.description',
|
|
kind: 'json_ld',
|
|
path: 'script[type="application/ld+json"] Product.description',
|
|
raw_value: `JSON-LD 商品描述 ${jsonLdDescription.value.length} 个字符`,
|
|
});
|
|
}
|
|
|
|
const categories = collectCategories(documentRoot, canonical);
|
|
if (categories.length) {
|
|
evidence.push({
|
|
ref: 'ozon:breadcrumb',
|
|
kind: 'json_ld',
|
|
path: 'BreadcrumbList/itemListElement',
|
|
raw_value: categories.map((category) => category.name).join(' > '),
|
|
});
|
|
}
|
|
|
|
if (fetchPageJson) {
|
|
try {
|
|
const modalStates = widgetStatesFromPageJson(
|
|
await fetchPageJson({ kind: 'modal-aspects', sku: currentSku }),
|
|
);
|
|
const modalVariants = discoverVariants(
|
|
modalStates,
|
|
candidates,
|
|
axes,
|
|
'ozon:aspects:modal',
|
|
canonical,
|
|
);
|
|
evidence.push({
|
|
ref: 'ozon:aspects:modal',
|
|
kind: 'state',
|
|
path: 'webAspectsModal-*.aspects[].variants[]',
|
|
raw_value: `发现 ${modalVariants} 个不同的公开 SKU`,
|
|
});
|
|
} catch (error) {
|
|
evidence.push(
|
|
fetchFailureEvidence('ozon:aspects:modal', currentSku, error),
|
|
);
|
|
}
|
|
|
|
// Rich content is a separate, fixed Ozon page segment. Ask for it
|
|
// once for the entered SKU; never scrape arbitrary images after
|
|
// scrolling through recommendations or buyer reviews.
|
|
try {
|
|
const detailStates = widgetStatesFromPageJson(
|
|
await fetchPageJson({
|
|
kind: 'product-detail',
|
|
sku: currentSku,
|
|
}),
|
|
);
|
|
const added = addDetailAssets(
|
|
detailStates,
|
|
assets,
|
|
assetKeys,
|
|
'ozon:detail:page',
|
|
canonical,
|
|
);
|
|
description = description ?? extractOzonDescription(detailStates);
|
|
const detailAttributes = extractOzonCharacteristics(detailStates);
|
|
attributes = mergeOzonCharacteristics(attributes, detailAttributes);
|
|
evidence.push({
|
|
ref: 'ozon:detail:page',
|
|
kind: 'state',
|
|
path: 'webDescription|webRichContent|webFeatures image fields',
|
|
raw_value: `发现 ${added} 张商品详情图`,
|
|
});
|
|
if (detailAttributes.length) {
|
|
evidence.push({
|
|
ref: 'ozon:characteristics:detail',
|
|
kind: 'state',
|
|
path: 'webShortCharacteristics|webDetailedCharacteristics|webCharacteristics.characteristics[]',
|
|
raw_value: `详情段补全 ${detailAttributes.length} 项商品特征`,
|
|
});
|
|
}
|
|
} catch (error) {
|
|
evidence.push(
|
|
fetchFailureEvidence('ozon:detail:page', currentSku, error),
|
|
);
|
|
}
|
|
|
|
// A detail page can expose a SKU omitted by the initial modal. Append
|
|
// new real SKUs to this bounded queue; never construct combinations.
|
|
const pending = [...candidates.keys()];
|
|
const queued = new Set(pending);
|
|
const hydrated = new Set<string>();
|
|
while (pending.length && hydrated.size < 100) {
|
|
const batch = pending.splice(0, 3);
|
|
const results = await Promise.all(
|
|
batch.map(async (sku) => {
|
|
try {
|
|
return {
|
|
sku,
|
|
states: widgetStatesFromPageJson(
|
|
await fetchPageJson({ kind: 'product', sku }),
|
|
),
|
|
};
|
|
} catch (error) {
|
|
return { sku, error };
|
|
}
|
|
}),
|
|
);
|
|
for (const result of results) {
|
|
hydrated.add(result.sku);
|
|
if ('error' in result) {
|
|
evidence.push(
|
|
fetchFailureEvidence(
|
|
`ozon:hydrate:${result.sku}`,
|
|
result.sku,
|
|
result.error,
|
|
),
|
|
);
|
|
continue;
|
|
}
|
|
const detail = firstState(result.states, 'webProductHeading');
|
|
const detailPrice = firstState(result.states, 'webPrice');
|
|
addCandidate(candidates, result.sku, {
|
|
title: stringAt(detail, ['title']),
|
|
price: money(stringAt(detailPrice, ['price'])),
|
|
evidenceRef: `ozon:hydrate:${result.sku}`,
|
|
hydrated: true,
|
|
});
|
|
discoverVariants(
|
|
result.states,
|
|
candidates,
|
|
axes,
|
|
`ozon:hydrate:${result.sku}`,
|
|
canonical,
|
|
);
|
|
addGalleryAssets(
|
|
result.states,
|
|
result.sku,
|
|
assets,
|
|
assetKeys,
|
|
`ozon:gallery:${result.sku}`,
|
|
canonical,
|
|
);
|
|
addGalleryVideos(
|
|
result.states,
|
|
result.sku,
|
|
assets,
|
|
assetKeys,
|
|
`ozon:gallery:${result.sku}`,
|
|
canonical,
|
|
);
|
|
if (result.sku === currentSku) {
|
|
const detailedAttributes = extractOzonCharacteristics(
|
|
result.states,
|
|
);
|
|
attributes = mergeOzonCharacteristics(
|
|
attributes,
|
|
detailedAttributes,
|
|
);
|
|
if (detailedAttributes.length) {
|
|
evidence.push({
|
|
ref: 'ozon:characteristics:current',
|
|
kind: 'state',
|
|
path: 'webCharacteristics-*.characteristics[]',
|
|
raw_value: `当前 SKU 补全 ${detailedAttributes.length} 项商品特征`,
|
|
});
|
|
}
|
|
}
|
|
for (const sku of candidates.keys()) {
|
|
if (!queued.has(sku) && queued.size < 100) {
|
|
queued.add(sku);
|
|
pending.push(sku);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
evidence.push({
|
|
ref: 'ozon:variant-hydration',
|
|
kind: 'state',
|
|
path: 'page/json/v2 product queue',
|
|
raw_value: `发现 ${candidates.size} 个 SKU，已尝试补全 ${hydrated.size} 个`,
|
|
});
|
|
} else {
|
|
evidence.push({
|
|
ref: 'ozon:variant-hydration',
|
|
kind: 'state',
|
|
path: 'page/json/v2 product queue',
|
|
raw_value: '未连接 Ozon 页面桥接；仅保存当前页面已渲染的变体状态',
|
|
});
|
|
}
|
|
|
|
// Ozon's rich widget is preferred because it is the dedicated description
|
|
// segment. JSON-LD is the proven old-collector fallback for pages where
|
|
// Ozon only places the seller description in the initial document.
|
|
description = description ?? jsonLdDescription;
|
|
|
|
addAspectCoverAssets(axes, assets, assetKeys, canonical);
|
|
|
|
const variants = [...candidates.values()]
|
|
.slice(0, 100)
|
|
.map<Variant>((candidate) => ({
|
|
sourceVariantKey: candidate.sku,
|
|
sourceSku: candidate.sku,
|
|
displayName: candidate.title ?? `Ozon SKU ${candidate.sku}`,
|
|
selections: candidate.selections,
|
|
...(candidate.price
|
|
? {
|
|
sourcePrice: {
|
|
amount: candidate.price,
|
|
currencyCode: currency,
|
|
},
|
|
}
|
|
: {}),
|
|
confidence:
|
|
candidate.hydrated || candidate.sku === currentSku
|
|
? 'verified'
|
|
: 'partial',
|
|
evidenceRefs: [...candidate.evidenceRefs],
|
|
}));
|
|
|
|
return {
|
|
source: {
|
|
platform: 'ozon',
|
|
// Entered SKU remains audit provenance. The API reuses by sourceSku overlap.
|
|
itemKey: currentSku,
|
|
canonicalUrl: canonical,
|
|
collectedAt: new Date().toISOString(),
|
|
parserVersion: 'ozon-public-2.3.0',
|
|
},
|
|
common: {
|
|
...(title
|
|
? {
|
|
title: {
|
|
value: title,
|
|
raw_value: title,
|
|
confidence: 'verified' as const,
|
|
},
|
|
}
|
|
: {}),
|
|
...(commonPrice
|
|
? {
|
|
sourcePrice: {
|
|
amount: commonPrice,
|
|
currencyCode: currency,
|
|
raw_value: commonPrice,
|
|
},
|
|
}
|
|
: {}),
|
|
...(description
|
|
? {
|
|
description: {
|
|
value: description.value,
|
|
raw_value: description.rawValue,
|
|
confidence: 'verified' as const,
|
|
},
|
|
}
|
|
: {}),
|
|
attributes,
|
|
categories,
|
|
},
|
|
variantAxes: [...axes.values()].slice(0, 20).map((axis) => ({
|
|
...axis,
|
|
values: axis.values.slice(0, 200),
|
|
})),
|
|
variants,
|
|
assets,
|
|
evidence,
|
|
};
|
|
}
|
|
|
|
function addCandidate(
|
|
candidates: Map<string, Candidate>,
|
|
sku: string,
|
|
input: {
|
|
title?: string;
|
|
price?: string;
|
|
selection?: Variant['selections'][number];
|
|
evidenceRef: string;
|
|
hydrated?: boolean;
|
|
},
|
|
) {
|
|
const candidate = candidates.get(sku) ?? {
|
|
sku,
|
|
selections: [],
|
|
evidenceRefs: new Set<string>(),
|
|
hydrated: false,
|
|
};
|
|
candidate.title = input.title ?? candidate.title;
|
|
candidate.price = input.price ?? candidate.price;
|
|
candidate.hydrated = candidate.hydrated || input.hydrated === true;
|
|
candidate.evidenceRefs.add(input.evidenceRef);
|
|
if (
|
|
input.selection &&
|
|
!candidate.selections.some(
|
|
(selection) =>
|
|
selection.axisKey === input.selection?.axisKey &&
|
|
selection.valueKey === input.selection?.valueKey,
|
|
)
|
|
) {
|
|
candidate.selections.push(input.selection);
|
|
}
|
|
candidates.set(sku, candidate);
|
|
}
|
|
|
|
function discoverVariants(
|
|
states: JsonRecord[],
|
|
candidates: Map<string, Candidate>,
|
|
axes: Map<string, Axis>,
|
|
evidenceRef: string,
|
|
pageUrl: string,
|
|
) {
|
|
const discovered = new Set<string>();
|
|
for (const state of states) {
|
|
if (!Array.isArray(state.aspects)) continue;
|
|
state.aspects.forEach((rawAspect, index) => {
|
|
if (!isRecord(rawAspect) || !Array.isArray(rawAspect.variants))
|
|
return;
|
|
const axisKey =
|
|
stringAt(rawAspect, ['aspectKey', 'key', 'id']) ??
|
|
`ozon:aspect:${index}`;
|
|
const axisName =
|
|
stringAt(rawAspect, ['aspectName', 'name']) ?? axisKey;
|
|
const axis = axes.get(axisKey) ?? {
|
|
key: axisKey,
|
|
name: axisName,
|
|
values: [],
|
|
};
|
|
for (const rawVariant of rawAspect.variants) {
|
|
if (!isRecord(rawVariant)) continue;
|
|
const sku = stringAt(rawVariant, ['sku']);
|
|
if (!sku) continue;
|
|
discovered.add(sku);
|
|
if (!candidates.has(sku) && candidates.size >= 100) continue;
|
|
const data = isRecord(rawVariant.data) ? rawVariant.data : {};
|
|
const valueName =
|
|
stringAt(data, ['searchableText', 'title']) ?? `SKU ${sku}`;
|
|
const valueKey = `${axisKey}:${valueName.toLocaleLowerCase()}`;
|
|
if (!axis.values.some((value) => value.key === valueKey)) {
|
|
const cover = ozonImageUrl(
|
|
stringAt(data, ['coverImage']),
|
|
pageUrl,
|
|
);
|
|
axis.values.push({
|
|
key: valueKey,
|
|
name: valueName,
|
|
...(cover ? { image_url: cover } : {}),
|
|
});
|
|
}
|
|
addCandidate(candidates, sku, {
|
|
title: stringAt(data, ['title', 'searchableText']),
|
|
price: money(stringAt(data, ['price', 'cardPrice'])),
|
|
selection: { axisKey, valueKey, valueName },
|
|
evidenceRef,
|
|
});
|
|
}
|
|
if (axis.values.length) axes.set(axisKey, axis);
|
|
});
|
|
}
|
|
return discovered.size;
|
|
}
|
|
|
|
/** Adds variant-axis covers without pretending that they identify one SKU. */
|
|
function addAspectCoverAssets(
|
|
axes: Map<string, Axis>,
|
|
assets: Asset[],
|
|
assetKeys: Set<string>,
|
|
pageUrl: string,
|
|
) {
|
|
for (const axis of axes.values()) {
|
|
for (const value of axis.values) {
|
|
const url = ozonImageUrl(value.image_url, pageUrl);
|
|
if (!url || assets.length >= 500) continue;
|
|
value.image_url = url;
|
|
const key = `sku:${axis.key}:${value.key}:${dedupeImageUrl(url, 'ozon')}`;
|
|
if (assetKeys.has(key)) continue;
|
|
assetKeys.add(key);
|
|
assets.push({
|
|
url,
|
|
mediaType: 'image',
|
|
role: 'source_sku',
|
|
relation: 'axis_value',
|
|
sortOrder: assets.length,
|
|
evidenceRef: `ozon:aspect-cover:${axis.key}:${value.key}`,
|
|
});
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Detail widgets are allowlisted by the bridge. Reading image-shaped fields
|
|
* from only these widgets is safe; scanning page-wide image nodes is not.
|
|
*/
|
|
function addDetailAssets(
|
|
states: JsonRecord[],
|
|
assets: Asset[],
|
|
assetKeys: Set<string>,
|
|
evidenceRef: string,
|
|
pageUrl: string,
|
|
) {
|
|
const urls = new Set<string>();
|
|
for (const state of states) {
|
|
const widget = String(state.__atlasWidgetName ?? '').replace(
|
|
/^state-/,
|
|
'',
|
|
);
|
|
if (!/^web(?:Description|RichContent|Features)-/.test(widget)) continue;
|
|
collectDetailImageUrls(state, urls);
|
|
}
|
|
let added = 0;
|
|
for (const rawUrl of urls) {
|
|
if (assets.length >= 500) break;
|
|
const url = ozonImageUrl(rawUrl, pageUrl);
|
|
if (!url) continue;
|
|
const key = `detail:${dedupeImageUrl(url, 'ozon')}`;
|
|
if (assetKeys.has(key)) continue;
|
|
assetKeys.add(key);
|
|
assets.push({
|
|
url,
|
|
mediaType: 'image',
|
|
role: 'source_detail',
|
|
relation: 'product_candidate',
|
|
sortOrder: added++,
|
|
evidenceRef,
|
|
});
|
|
}
|
|
return added;
|
|
}
|
|
|
|
function collectDetailImageUrls(
|
|
value: unknown,
|
|
urls: Set<string>,
|
|
visited = new Set<unknown>(),
|
|
) {
|
|
if (!value || typeof value !== 'object' || visited.has(value)) return;
|
|
visited.add(value);
|
|
if (Array.isArray(value)) {
|
|
value.forEach((entry) => collectDetailImageUrls(entry, urls, visited));
|
|
return;
|
|
}
|
|
for (const [key, child] of Object.entries(value as JsonRecord)) {
|
|
if (
|
|
typeof child === 'string' &&
|
|
/^(src|url|image|imageurl|coverimage)$/i.test(key) &&
|
|
/^https?:\/\//.test(child)
|
|
) {
|
|
urls.add(child);
|
|
} else if (child && typeof child === 'object') {
|
|
collectDetailImageUrls(child, urls, visited);
|
|
}
|
|
}
|
|
}
|
|
|
|
type OzonCharacteristic =
|
|
CollectedProductPayload['common']['attributes'][number];
|
|
|
|
/**
|
|
* Ozon renders a short parameter list in the initial document and exposes the
|
|
* complete list in the normal product page state. Keep this reader limited to
|
|
* those named widgets so arbitrary page copy never turns into product params.
|
|
*/
|
|
export function extractOzonCharacteristics(
|
|
states: JsonRecord[],
|
|
): OzonCharacteristic[] {
|
|
const attributes: OzonCharacteristic[] = [];
|
|
for (const state of states) {
|
|
const widget = String(state.__atlasWidgetName ?? '').replace(
|
|
/^state-/,
|
|
'',
|
|
);
|
|
if (
|
|
!/^web(?:ShortCharacteristics|DetailedCharacteristics|Characteristics)-/.test(
|
|
widget,
|
|
) ||
|
|
!Array.isArray(state.characteristics)
|
|
)
|
|
continue;
|
|
for (const [index, entry] of state.characteristics.entries()) {
|
|
if (!isRecord(entry)) continue;
|
|
const name = richText(entry.title);
|
|
const value = Array.isArray(entry.values)
|
|
? entry.values.map(richText).filter(Boolean).join(', ')
|
|
: richText(entry.value);
|
|
if (!name || !value) continue;
|
|
const rawId = stringAt(entry, ['id']);
|
|
attributes.push({
|
|
key: `ozon:${rawId ?? `characteristic:${normaliseKey(name, index)}`}`.slice(
|
|
0,
|
|
200,
|
|
),
|
|
name: name.slice(0, 300),
|
|
value: value.slice(0, 2_000),
|
|
raw_value: value.slice(0, 4_000),
|
|
});
|
|
}
|
|
}
|
|
return mergeOzonCharacteristics([], attributes);
|
|
}
|
|
|
|
function mergeOzonCharacteristics(
|
|
base: OzonCharacteristic[],
|
|
additions: OzonCharacteristic[],
|
|
) {
|
|
const merged = [...base];
|
|
const seen = new Set(
|
|
base.map((attribute) => `${attribute.name}\u0000${attribute.value}`),
|
|
);
|
|
for (const attribute of additions) {
|
|
if (merged.length >= 500) break;
|
|
const identity = `${attribute.name}\u0000${attribute.value}`;
|
|
if (seen.has(identity)) continue;
|
|
seen.add(identity);
|
|
merged.push(attribute);
|
|
}
|
|
return merged;
|
|
}
|
|
|
|
type OzonDescription = { value: string; rawValue: string };
|
|
|
|
/** Extracts only Ozon's rich-description payload, converting it to reviewable text. */
|
|
export function extractOzonDescription(
|
|
states: JsonRecord[],
|
|
): OzonDescription | undefined {
|
|
for (const state of states) {
|
|
const widget = String(state.__atlasWidgetName ?? '').replace(
|
|
/^state-/,
|
|
'',
|
|
);
|
|
if (!/^web(?:Description|RichContent)-/.test(widget)) continue;
|
|
const raw = findRichAnnotationJson(state);
|
|
if (!raw) continue;
|
|
let value = raw;
|
|
try {
|
|
value = richDescriptionText(JSON.parse(raw));
|
|
} catch {
|
|
// Some older page responses already contain plain text here.
|
|
}
|
|
value = value
|
|
.replace(/\s+\n/g, '\n')
|
|
.replace(/\n{3,}/g, '\n\n')
|
|
.trim();
|
|
if (!value) continue;
|
|
return {
|
|
value: value.slice(0, 100_000),
|
|
rawValue: raw.slice(0, 120_000),
|
|
};
|
|
}
|
|
return undefined;
|
|
}
|
|
|
|
/** Reads only schema.org Product.description as a fallback, never arbitrary
|
|
* JSON-LD page copy. This mirrors the mature collector's JSON-LD path. */
|
|
export function extractOzonJsonLdDescription(
|
|
root: Document,
|
|
): OzonDescription | undefined {
|
|
for (const script of Array.from(root.scripts)) {
|
|
if (script.type !== 'application/ld+json' || !script.textContent?.trim())
|
|
continue;
|
|
try {
|
|
const product = findJsonLdProduct(JSON.parse(script.textContent));
|
|
const raw = product && typeof product.description === 'string'
|
|
? product.description.trim()
|
|
: '';
|
|
if (!raw) continue;
|
|
return {
|
|
value: raw.replace(/\s+\n/g, '\n').replace(/\n{3,}/g, '\n\n').slice(0, 100_000),
|
|
rawValue: raw.slice(0, 120_000),
|
|
};
|
|
} catch {
|
|
// A malformed analytics JSON-LD block is not product evidence.
|
|
}
|
|
}
|
|
return undefined;
|
|
}
|
|
|
|
function findJsonLdProduct(
|
|
value: unknown,
|
|
visited = new Set<unknown>(),
|
|
): JsonRecord | undefined {
|
|
if (!value || typeof value !== 'object' || visited.has(value)) return undefined;
|
|
visited.add(value);
|
|
if (Array.isArray(value)) {
|
|
for (const child of value) {
|
|
const product = findJsonLdProduct(child, visited);
|
|
if (product) return product;
|
|
}
|
|
return undefined;
|
|
}
|
|
const record = value as JsonRecord;
|
|
const type = record['@type'];
|
|
if (type === 'Product' || (Array.isArray(type) && type.includes('Product')))
|
|
return record;
|
|
if (Array.isArray(record['@graph'])) {
|
|
const product = findJsonLdProduct(record['@graph'], visited);
|
|
if (product) return product;
|
|
}
|
|
return undefined;
|
|
}
|
|
|
|
function findRichAnnotationJson(
|
|
value: unknown,
|
|
visited = new Set<unknown>(),
|
|
): string | undefined {
|
|
if (!value || typeof value !== 'object' || visited.has(value))
|
|
return undefined;
|
|
visited.add(value);
|
|
if (Array.isArray(value)) {
|
|
for (const child of value) {
|
|
const found = findRichAnnotationJson(child, visited);
|
|
if (found) return found;
|
|
}
|
|
return undefined;
|
|
}
|
|
const record = value as JsonRecord;
|
|
if (
|
|
typeof record.richAnnotationJson === 'string' &&
|
|
record.richAnnotationJson.trim()
|
|
)
|
|
return record.richAnnotationJson;
|
|
for (const child of Object.values(record)) {
|
|
const found = findRichAnnotationJson(child, visited);
|
|
if (found) return found;
|
|
}
|
|
return undefined;
|
|
}
|
|
|
|
function richDescriptionText(
|
|
value: unknown,
|
|
visited = new Set<unknown>(),
|
|
): string {
|
|
if (typeof value === 'string') return value.trim();
|
|
if (!value || typeof value !== 'object' || visited.has(value)) return '';
|
|
visited.add(value);
|
|
if (Array.isArray(value))
|
|
return value
|
|
.map((child) => richDescriptionText(child, visited))
|
|
.filter(Boolean)
|
|
.join('\n');
|
|
const record = value as JsonRecord;
|
|
const direct =
|
|
typeof record.text === 'string'
|
|
? record.text
|
|
: typeof record.content === 'string'
|
|
? record.content
|
|
: '';
|
|
const children = Object.entries(record)
|
|
.filter(
|
|
([key]) =>
|
|
!['text', 'content', 'type', 'src', 'url', 'image'].includes(
|
|
key,
|
|
),
|
|
)
|
|
.map(([, child]) => richDescriptionText(child, visited))
|
|
.filter(Boolean);
|
|
return [direct, ...children].filter(Boolean).join('\n');
|
|
}
|
|
|
|
function richText(value: unknown): string {
|
|
if (typeof value === 'string' || typeof value === 'number')
|
|
return String(value).replace(/\s+/g, ' ').trim();
|
|
if (Array.isArray(value))
|
|
return value.map(richText).filter(Boolean).join(' ').trim();
|
|
if (!isRecord(value)) return '';
|
|
return richText(value.textRs ?? value.text ?? value.content);
|
|
}
|
|
|
|
function normaliseKey(value: string, index: number) {
|
|
return (
|
|
value
|
|
.toLocaleLowerCase()
|
|
.replace(/[^\p{L}\p{N}]+/gu, '-')
|
|
.replace(/^-+|-+$/g, '')
|
|
.slice(0, 150) || String(index)
|
|
);
|
|
}
|
|
|
|
function addGalleryAssets(
|
|
states: JsonRecord[],
|
|
sku: string,
|
|
assets: Asset[],
|
|
assetKeys: Set<string>,
|
|
evidenceRef: string,
|
|
pageUrl: string,
|
|
) {
|
|
const gallery = firstState(states, 'webGallery');
|
|
if (!gallery) return;
|
|
const urls = extractOzonGalleryImageUrls(gallery, pageUrl);
|
|
for (const [index, url] of urls.entries()) {
|
|
if (assets.length >= 500) return;
|
|
const key = `main:${sku}:${url ? dedupeImageUrl(url, 'ozon') : ''}`;
|
|
if (assetKeys.has(key)) continue;
|
|
assetKeys.add(key);
|
|
assets.push({
|
|
url,
|
|
mediaType: 'image',
|
|
// A gallery is product photography for this real SKU. It is not
|
|
// a SKU-option thumbnail merely because it is the second image.
|
|
role: 'source_main',
|
|
variantSourceKey: sku,
|
|
relation: 'variant_direct',
|
|
sortOrder: index,
|
|
evidenceRef,
|
|
});
|
|
}
|
|
}
|
|
|
|
/** Adds only Ozon's named product-gallery videos. Page-wide <video> nodes can
|
|
* belong to reviews or recommendations and are intentionally never considered. */
|
|
function addGalleryVideos(
|
|
states: JsonRecord[],
|
|
sku: string,
|
|
assets: Asset[],
|
|
assetKeys: Set<string>,
|
|
evidenceRef: string,
|
|
pageUrl: string,
|
|
) {
|
|
const galleries = states.filter((state) => {
|
|
const widget = String(state.__atlasWidgetName ?? '').replace(/^state-/, '');
|
|
return /^web(?:Gallery|Video)-/.test(widget);
|
|
});
|
|
for (const gallery of galleries) {
|
|
for (const video of extractOzonGalleryVideos(gallery, pageUrl)) {
|
|
if (assets.length >= 500) return;
|
|
const key = `video:${sku}:${video.url}`;
|
|
if (assetKeys.has(key)) continue;
|
|
assetKeys.add(key);
|
|
assets.push({
|
|
url: video.url,
|
|
mediaType: 'video',
|
|
...(video.posterUrl ? { posterUrl: video.posterUrl } : {}),
|
|
role: 'source_main',
|
|
variantSourceKey: sku,
|
|
relation: 'variant_direct',
|
|
sortOrder: assets.length,
|
|
evidenceRef,
|
|
});
|
|
}
|
|
}
|
|
}
|
|
|
|
/** Pure helper for the strictly named Ozon gallery/video state. HLS, blobs,
|
|
* and non-HTTPS URLs stay out of the collection contract in this first phase. */
|
|
export function extractOzonGalleryVideos(gallery: JsonRecord, pageUrl: string) {
|
|
const videos: Array<{ url: string; posterUrl?: string }> = [];
|
|
const seen = new Set<string>();
|
|
// Ozon changes whether the direct file lives in videos[], video, or a
|
|
// nested gallery payload. The old collector walked this *one allowlisted
|
|
// widget* for direct video extensions; retaining that scope is critical.
|
|
for (const { value, parent } of nestedVideoValues(gallery)) {
|
|
const url = directVideoUrl(
|
|
typeof value === 'string'
|
|
? value
|
|
: stringAt(value, ['url', 'videoUrl', 'src']),
|
|
pageUrl,
|
|
);
|
|
if (!url) continue;
|
|
const posterUrl = ozonImageUrl(
|
|
(typeof value === 'string'
|
|
? undefined
|
|
: stringAt(value, ['coverImage', 'coverUrl', 'poster', 'previewImage']))
|
|
?? stringAt(parent, ['coverImage', 'coverUrl', 'poster', 'previewImage']),
|
|
pageUrl,
|
|
);
|
|
if (seen.has(url)) {
|
|
const existing = videos.find((video) => video.url === url);
|
|
if (existing && !existing.posterUrl && posterUrl)
|
|
existing.posterUrl = posterUrl;
|
|
continue;
|
|
}
|
|
seen.add(url);
|
|
videos.push({ url, ...(posterUrl ? { posterUrl } : {}) });
|
|
}
|
|
return videos;
|
|
}
|
|
|
|
function nestedVideoValues(root: JsonRecord) {
|
|
const values: Array<{ value: string | JsonRecord; parent?: JsonRecord }> = [];
|
|
const pending: Array<{ value: unknown; parent?: JsonRecord }> = [{ value: root }];
|
|
const seen = new Set<object>();
|
|
while (pending.length) {
|
|
const current = pending.pop()!;
|
|
if (typeof current.value === 'string') {
|
|
if (/\.(mp4|webm)(?:$|[?#])/i.test(current.value))
|
|
values.push({ value: current.value, parent: current.parent });
|
|
continue;
|
|
}
|
|
if (!current.value || typeof current.value !== 'object') continue;
|
|
if (seen.has(current.value)) continue;
|
|
seen.add(current.value);
|
|
if (Array.isArray(current.value)) {
|
|
for (const child of current.value)
|
|
pending.push({ value: child, parent: current.parent });
|
|
continue;
|
|
}
|
|
const record = current.value as JsonRecord;
|
|
if (stringAt(record, ['url', 'videoUrl', 'src']))
|
|
values.push({ value: record, parent: current.parent });
|
|
for (const child of Object.values(record))
|
|
pending.push({ value: child, parent: record });
|
|
}
|
|
return values;
|
|
}
|
|
|
|
/** Pure helper used by fixture tests and the structured Ozon gallery reader. */
|
|
export function extractOzonGalleryImageUrls(
|
|
gallery: JsonRecord,
|
|
pageUrl: string,
|
|
) {
|
|
const urls: string[] = [];
|
|
const seen = new Set<string>();
|
|
for (const image of [
|
|
gallery.coverImage,
|
|
...(Array.isArray(gallery.images) ? gallery.images : []),
|
|
]) {
|
|
const url = ozonImageUrl(
|
|
typeof image === 'string'
|
|
? image
|
|
: isRecord(image)
|
|
? stringAt(image, ['src', 'url'])
|
|
: undefined,
|
|
pageUrl,
|
|
);
|
|
if (!url) continue;
|
|
const key = dedupeImageUrl(url, 'ozon');
|
|
if (seen.has(key)) continue;
|
|
seen.add(key);
|
|
urls.push(url);
|
|
}
|
|
return urls;
|
|
}
|
|
|
|
function readLocalWidgetStates(root: Document): JsonRecord[] {
|
|
const states: JsonRecord[] = [];
|
|
for (const element of Array.from(
|
|
root.querySelectorAll<HTMLElement>('[id^="state-web"][data-state]'),
|
|
)) {
|
|
const raw = element.getAttribute('data-state');
|
|
if (!raw?.trim()) continue;
|
|
try {
|
|
const state = JSON.parse(raw);
|
|
if (isRecord(state)) states.push(withWidgetName(state, element.id));
|
|
} catch {
|
|
// Partially updated state must not fail the whole collection.
|
|
}
|
|
}
|
|
return states;
|
|
}
|
|
|
|
function widgetStatesFromPageJson(value: unknown): JsonRecord[] {
|
|
if (!isRecord(value) || !isRecord(value.widgetStates)) return [];
|
|
const states: JsonRecord[] = [];
|
|
for (const [widgetName, raw] of Object.entries(value.widgetStates)) {
|
|
if (
|
|
!/^web(Aspects|AspectsModal|ProductHeading|Price|Gallery|Video|Description|RichContent|Features|ShortCharacteristics|DetailedCharacteristics|Characteristics)-/.test(
|
|
widgetName,
|
|
)
|
|
)
|
|
continue;
|
|
try {
|
|
const state = typeof raw === 'string' ? JSON.parse(raw) : raw;
|
|
if (isRecord(state)) states.push(withWidgetName(state, widgetName));
|
|
} catch {
|
|
// One malformed Ozon widget only loses that evidence.
|
|
}
|
|
}
|
|
return states;
|
|
}
|
|
|
|
function withWidgetName(state: JsonRecord, widgetName: string): JsonRecord {
|
|
return { ...state, __atlasWidgetName: widgetName };
|
|
}
|
|
|
|
function firstState(
|
|
states: JsonRecord[],
|
|
prefix: string,
|
|
): JsonRecord | undefined {
|
|
return states.find((state) =>
|
|
String(state.__atlasWidgetName ?? '')
|
|
.replace(/^state-/, '')
|
|
.startsWith(prefix),
|
|
);
|
|
}
|
|
|
|
function fetchFailureEvidence(ref: string, sku: string, error: unknown) {
|
|
const message = error instanceof Error ? error.message : '请求失败';
|
|
return {
|
|
ref,
|
|
kind: 'state' as const,
|
|
path: `page/json/v2 product ${sku}`,
|
|
raw_value: `SKU ${sku} 补全失败：${message.slice(0, 500)}`,
|
|
};
|
|
}
|
|
|
|
/** JSON-LD 面包屑优先；Ozon 不输出时读取可见面包屑。 */
|
|
function collectCategories(root: Document, pageUrl: string): Category[] {
|
|
for (const script of Array.from(root.scripts)) {
|
|
if (
|
|
script.type !== 'application/ld+json' ||
|
|
!script.textContent?.trim()
|
|
)
|
|
continue;
|
|
try {
|
|
const categories = categoriesFromJsonLd(
|
|
JSON.parse(script.textContent),
|
|
pageUrl,
|
|
);
|
|
if (categories.length) return categories;
|
|
} catch {
|
|
// Invalid dynamic JSON-LD does not affect collection.
|
|
}
|
|
}
|
|
const links = Array.from(
|
|
root.querySelectorAll<HTMLAnchorElement>(
|
|
'[data-widget*="breadCrumbs"] a, [data-widget*="breadcrumb"] a, nav[aria-label*="хлеб"] a, nav[aria-label*="bread"] a',
|
|
),
|
|
);
|
|
return uniqueCategories(
|
|
links.map((link) =>
|
|
category(link.textContent, link.getAttribute('href'), pageUrl),
|
|
),
|
|
);
|
|
}
|
|
|
|
function categoriesFromJsonLd(value: unknown, pageUrl: string): Category[] {
|
|
const list = findBreadcrumbList(value);
|
|
if (!list || !Array.isArray(list.itemListElement)) return [];
|
|
return uniqueCategories(
|
|
list.itemListElement.map((entry) => {
|
|
if (!isRecord(entry)) return null;
|
|
const item = entry.item;
|
|
const itemUrl =
|
|
typeof item === 'string'
|
|
? item
|
|
: isRecord(item)
|
|
? stringAt(item, ['@id', 'url'])
|
|
: undefined;
|
|
return category(
|
|
stringAt(entry, ['name']) ??
|
|
(isRecord(item) ? stringAt(item, ['name']) : undefined),
|
|
itemUrl,
|
|
pageUrl,
|
|
);
|
|
}),
|
|
);
|
|
}
|
|
|
|
function findBreadcrumbList(
|
|
value: unknown,
|
|
visited = new Set<unknown>(),
|
|
): JsonRecord | null {
|
|
if (!isRecord(value) || visited.has(value)) return null;
|
|
visited.add(value);
|
|
const type = value['@type'];
|
|
if (
|
|
(typeof type === 'string' && type === 'BreadcrumbList') ||
|
|
(Array.isArray(type) && type.includes('BreadcrumbList'))
|
|
)
|
|
return value;
|
|
for (const child of Object.values(value)) {
|
|
if (Array.isArray(child)) {
|
|
for (const entry of child) {
|
|
const found = findBreadcrumbList(entry, visited);
|
|
if (found) return found;
|
|
}
|
|
} else {
|
|
const found = findBreadcrumbList(child, visited);
|
|
if (found) return found;
|
|
}
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function category(
|
|
rawName: unknown,
|
|
rawUrl: unknown,
|
|
pageUrl: string,
|
|
): Category | null {
|
|
const name =
|
|
typeof rawName === 'string' ? rawName.replace(/\s+/g, ' ').trim() : '';
|
|
if (
|
|
!name ||
|
|
/^(ozon|главная|home|首页|全部分类|все категории)$/i.test(name)
|
|
)
|
|
return null;
|
|
let key = name.toLowerCase();
|
|
if (typeof rawUrl === 'string' && rawUrl) {
|
|
try {
|
|
const url = new URL(rawUrl, pageUrl);
|
|
key = `${url.hostname}${url.pathname.replace(/\/$/, '')}${url.search}`;
|
|
} catch {
|
|
// Use the normalised name as a stable fallback key.
|
|
}
|
|
}
|
|
return { key: `ozon:${key}`, name };
|
|
}
|
|
|
|
function uniqueCategories(items: Array<Category | null>): Category[] {
|
|
const seen = new Set<string>();
|
|
return items.filter(
|
|
(item): item is Category =>
|
|
!!item && !seen.has(item.key) && !!seen.add(item.key),
|
|
);
|
|
}
|
|
|
|
function isRecord(value: unknown): value is JsonRecord {
|
|
return !!value && typeof value === 'object' && !Array.isArray(value);
|
|
}
|
|
|
|
function stringAt(value: JsonRecord | undefined, keys: string[]) {
|
|
if (!value) return undefined;
|
|
for (const key of keys) {
|
|
const item = value[key];
|
|
if (
|
|
(typeof item === 'string' || typeof item === 'number') &&
|
|
String(item).trim()
|
|
)
|
|
return String(item).trim();
|
|
}
|
|
return undefined;
|
|
}
|
|
|
|
function skuFromUrl(url: string) {
|
|
return url.match(/-(\d+)(?:\/|$|\?)/)?.[1] ?? null;
|
|
}
|
|
|
|
function text(element: Element | null) {
|
|
const value = element?.textContent?.replace(/\s+/g, ' ').trim();
|
|
return value || undefined;
|
|
}
|
|
|
|
function money(raw: string | undefined) {
|
|
if (!raw) return undefined;
|
|
const amount = raw.replace(/[^\d,\.]/g, '').replace(',', '.');
|
|
return /^\d+(\.\d{1,4})?$/.test(amount) ? amount : undefined;
|
|
}
|
|
|
|
function parsePrice(root: Document) {
|
|
const meta = root.querySelector<HTMLMetaElement>(
|
|
'meta[property="product:price:amount"]',
|
|
)?.content;
|
|
if (meta && /^\d+(\.\d+)?$/.test(meta)) return meta;
|
|
const candidate = Array.from(
|
|
root.querySelectorAll(
|
|
'[data-widget*="webPrice"], [data-widget*="price"]',
|
|
),
|
|
)
|
|
.map((node) => node.textContent ?? '')
|
|
.find((value) => /\d/.test(value));
|
|
return money(candidate);
|
|
}
|
|
|
|
function currencyFor(root: Document, price: string | undefined) {
|
|
const meta = root
|
|
.querySelector<HTMLMetaElement>(
|
|
'meta[property="product:price:currency"]',
|
|
)
|
|
?.content?.toUpperCase();
|
|
if (meta && /^[A-Z]{3}$/.test(meta)) return meta;
|
|
if (price?.includes('¥')) return 'CNY';
|
|
if (price?.includes('₸')) return 'KZT';
|
|
if (price?.includes('Br')) return 'BYN';
|
|
return 'RUB';
|
|
}
|
|
|
|
function httpsUrl(value: string | undefined) {
|
|
return value && /^https:\/\//.test(value) ? value : undefined;
|
|
}
|
|
|
|
function ozonImageUrl(value: string | undefined, pageUrl: string) {
|
|
if (!value) return undefined;
|
|
const url = normaliseImageUrl(value, pageUrl, 'ozon');
|
|
return url?.startsWith('https://') ? url : undefined;
|
|
}
|
|
|
|
function directVideoUrl(value: string | undefined, pageUrl: string) {
|
|
if (!value) return undefined;
|
|
try {
|
|
const url = new URL(value, pageUrl);
|
|
if (url.protocol !== 'https:' || /\.m3u8(?:$|[?#])/i.test(url.pathname))
|
|
return undefined;
|
|
return /\.(mp4|webm)(?:$|[?#])/i.test(url.pathname) ? url.href : undefined;
|
|
} catch {
|
|
return undefined;
|
|
}
|
|
}
|