Update seller store experience

This commit is contained in:
R524809
2026-09-08 16:19:38 +08:00
parent ceb644a3f9
commit 319887726e
25 changed files with 31417 additions and 127 deletions
+203 -2
View File
@@ -61,6 +61,7 @@ export async function collectOzonProduct(
const localStates = readLocalWidgetStates(documentRoot);
let attributes = extractOzonCharacteristics(localStates);
let description = extractOzonDescription(localStates);
const jsonLdDescription = extractOzonJsonLdDescription(documentRoot);
const heading = firstState(localStates, 'webProductHeading');
const priceState = firstState(localStates, 'webPrice');
const title =
@@ -101,6 +102,14 @@ export async function collectOzonProduct(
'ozon:gallery:page',
canonical,
);
addGalleryVideos(
localStates,
currentSku,
assets,
assetKeys,
'ozon:gallery:page',
canonical,
);
if (attributes.length) {
evidence.push({
ref: 'ozon:characteristics:page',
@@ -109,6 +118,14 @@ export async function collectOzonProduct(
raw_value: `发现 ${attributes.length} 项商品特征`,
});
}
if (jsonLdDescription) {
evidence.push({
ref: 'ozon:json-ld:product.description',
kind: 'json_ld',
path: 'script[type="application/ld+json"] Product.description',
raw_value: `JSON-LD 商品描述 ${jsonLdDescription.value.length} 个字符`,
});
}
const categories = collectCategories(documentRoot, canonical);
if (categories.length) {
@@ -162,12 +179,22 @@ export async function collectOzonProduct(
canonical,
);
description = description ?? extractOzonDescription(detailStates);
const detailAttributes = extractOzonCharacteristics(detailStates);
attributes = mergeOzonCharacteristics(attributes, detailAttributes);
evidence.push({
ref: 'ozon:detail:page',
kind: 'state',
path: 'webDescription|webRichContent|webFeatures image fields',
raw_value: `发现 ${added} 张商品详情图`,
});
if (detailAttributes.length) {
evidence.push({
ref: 'ozon:characteristics:detail',
kind: 'state',
path: 'webShortCharacteristics|webDetailedCharacteristics|webCharacteristics.characteristics[]',
raw_value: `详情段补全 ${detailAttributes.length} 项商品特征`,
});
}
} catch (error) {
evidence.push(
fetchFailureEvidence('ozon:detail:page', currentSku, error),
@@ -230,6 +257,14 @@ export async function collectOzonProduct(
`ozon:gallery:${result.sku}`,
canonical,
);
addGalleryVideos(
result.states,
result.sku,
assets,
assetKeys,
`ozon:gallery:${result.sku}`,
canonical,
);
if (result.sku === currentSku) {
const detailedAttributes = extractOzonCharacteristics(
result.states,
@@ -270,6 +305,11 @@ export async function collectOzonProduct(
});
}
// Ozon's rich widget is preferred because it is the dedicated description
// segment. JSON-LD is the proven old-collector fallback for pages where
// Ozon only places the seller description in the initial document.
description = description ?? jsonLdDescription;
addAspectCoverAssets(axes, assets, assetKeys, canonical);
const variants = [...candidates.values()]
@@ -301,7 +341,7 @@ export async function collectOzonProduct(
itemKey: currentSku,
canonicalUrl: canonical,
collectedAt: new Date().toISOString(),
parserVersion: 'ozon-public-2.2.0',
parserVersion: 'ozon-public-2.3.0',
},
common: {
...(title
@@ -452,6 +492,7 @@ function addAspectCoverAssets(
assetKeys.add(key);
assets.push({
url,
mediaType: 'image',
role: 'source_sku',
relation: 'axis_value',
sortOrder: assets.length,
@@ -491,6 +532,7 @@ function addDetailAssets(
assetKeys.add(key);
assets.push({
url,
mediaType: 'image',
role: 'source_detail',
relation: 'product_candidate',
sortOrder: added++,
@@ -621,6 +663,55 @@ export function extractOzonDescription(
return undefined;
}
/** Reads only schema.org Product.description as a fallback, never arbitrary
* JSON-LD page copy. This mirrors the mature collector's JSON-LD path. */
export function extractOzonJsonLdDescription(
root: Document,
): OzonDescription | undefined {
for (const script of Array.from(root.scripts)) {
if (script.type !== 'application/ld+json' || !script.textContent?.trim())
continue;
try {
const product = findJsonLdProduct(JSON.parse(script.textContent));
const raw = product && typeof product.description === 'string'
? product.description.trim()
: '';
if (!raw) continue;
return {
value: raw.replace(/\s+\n/g, '\n').replace(/\n{3,}/g, '\n\n').slice(0, 100_000),
rawValue: raw.slice(0, 120_000),
};
} catch {
// A malformed analytics JSON-LD block is not product evidence.
}
}
return undefined;
}
function findJsonLdProduct(
value: unknown,
visited = new Set<unknown>(),
): JsonRecord | undefined {
if (!value || typeof value !== 'object' || visited.has(value)) return undefined;
visited.add(value);
if (Array.isArray(value)) {
for (const child of value) {
const product = findJsonLdProduct(child, visited);
if (product) return product;
}
return undefined;
}
const record = value as JsonRecord;
const type = record['@type'];
if (type === 'Product' || (Array.isArray(type) && type.includes('Product')))
return record;
if (Array.isArray(record['@graph'])) {
const product = findJsonLdProduct(record['@graph'], visited);
if (product) return product;
}
return undefined;
}
function findRichAnnotationJson(
value: unknown,
visited = new Set<unknown>(),
@@ -716,6 +807,7 @@ function addGalleryAssets(
assetKeys.add(key);
assets.push({
url,
mediaType: 'image',
// A gallery is product photography for this real SKU. It is not
// a SKU-option thumbnail merely because it is the second image.
role: 'source_main',
@@ -727,6 +819,103 @@ function addGalleryAssets(
}
}
/** Adds only Ozon's named product-gallery videos. Page-wide <video> nodes can
* belong to reviews or recommendations and are intentionally never considered. */
function addGalleryVideos(
states: JsonRecord[],
sku: string,
assets: Asset[],
assetKeys: Set<string>,
evidenceRef: string,
pageUrl: string,
) {
const galleries = states.filter((state) => {
const widget = String(state.__atlasWidgetName ?? '').replace(/^state-/, '');
return /^web(?:Gallery|Video)-/.test(widget);
});
for (const gallery of galleries) {
for (const video of extractOzonGalleryVideos(gallery, pageUrl)) {
if (assets.length >= 500) return;
const key = `video:${sku}:${video.url}`;
if (assetKeys.has(key)) continue;
assetKeys.add(key);
assets.push({
url: video.url,
mediaType: 'video',
...(video.posterUrl ? { posterUrl: video.posterUrl } : {}),
role: 'source_main',
variantSourceKey: sku,
relation: 'variant_direct',
sortOrder: assets.length,
evidenceRef,
});
}
}
}
/** Pure helper for the strictly named Ozon gallery/video state. HLS, blobs,
* and non-HTTPS URLs stay out of the collection contract in this first phase. */
export function extractOzonGalleryVideos(gallery: JsonRecord, pageUrl: string) {
const videos: Array<{ url: string; posterUrl?: string }> = [];
const seen = new Set<string>();
// Ozon changes whether the direct file lives in videos[], video, or a
// nested gallery payload. The old collector walked this *one allowlisted
// widget* for direct video extensions; retaining that scope is critical.
for (const { value, parent } of nestedVideoValues(gallery)) {
const url = directVideoUrl(
typeof value === 'string'
? value
: stringAt(value, ['url', 'videoUrl', 'src']),
pageUrl,
);
if (!url) continue;
const posterUrl = ozonImageUrl(
(typeof value === 'string'
? undefined
: stringAt(value, ['coverImage', 'coverUrl', 'poster', 'previewImage']))
?? stringAt(parent, ['coverImage', 'coverUrl', 'poster', 'previewImage']),
pageUrl,
);
if (seen.has(url)) {
const existing = videos.find((video) => video.url === url);
if (existing && !existing.posterUrl && posterUrl)
existing.posterUrl = posterUrl;
continue;
}
seen.add(url);
videos.push({ url, ...(posterUrl ? { posterUrl } : {}) });
}
return videos;
}
function nestedVideoValues(root: JsonRecord) {
const values: Array<{ value: string | JsonRecord; parent?: JsonRecord }> = [];
const pending: Array<{ value: unknown; parent?: JsonRecord }> = [{ value: root }];
const seen = new Set<object>();
while (pending.length) {
const current = pending.pop()!;
if (typeof current.value === 'string') {
if (/\.(mp4|webm)(?:$|[?#])/i.test(current.value))
values.push({ value: current.value, parent: current.parent });
continue;
}
if (!current.value || typeof current.value !== 'object') continue;
if (seen.has(current.value)) continue;
seen.add(current.value);
if (Array.isArray(current.value)) {
for (const child of current.value)
pending.push({ value: child, parent: current.parent });
continue;
}
const record = current.value as JsonRecord;
if (stringAt(record, ['url', 'videoUrl', 'src']))
values.push({ value: record, parent: current.parent });
for (const child of Object.values(record))
pending.push({ value: child, parent: record });
}
return values;
}
/** Pure helper used by fixture tests and the structured Ozon gallery reader. */
export function extractOzonGalleryImageUrls(
gallery: JsonRecord,
@@ -777,7 +966,7 @@ function widgetStatesFromPageJson(value: unknown): JsonRecord[] {
const states: JsonRecord[] = [];
for (const [widgetName, raw] of Object.entries(value.widgetStates)) {
if (
!/^web(Aspects|AspectsModal|ProductHeading|Price|Gallery|Description|RichContent|Features|ShortCharacteristics|DetailedCharacteristics|Characteristics)-/.test(
!/^web(Aspects|AspectsModal|ProductHeading|Price|Gallery|Video|Description|RichContent|Features|ShortCharacteristics|DetailedCharacteristics|Characteristics)-/.test(
widgetName,
)
)
@@ -997,3 +1186,15 @@ function ozonImageUrl(value: string | undefined, pageUrl: string) {
const url = normaliseImageUrl(value, pageUrl, 'ozon');
return url?.startsWith('https://') ? url : undefined;
}
function directVideoUrl(value: string | undefined, pageUrl: string) {
if (!value) return undefined;
try {
const url = new URL(value, pageUrl);
if (url.protocol !== 'https:' || /\.m3u8(?:$|[?#])/i.test(url.pathname))
return undefined;
return /\.(mp4|webm)(?:$|[?#])/i.test(url.pathname) ? url.href : undefined;
} catch {
return undefined;
}
}
+1
View File
@@ -180,6 +180,7 @@ export function collectPlatformDomAssets(
seen.add(key);
assets.push({
url,
mediaType: 'image',
role: group.role,
relation: group.relation,
sortOrder: assets.length,
+177 -14
View File
@@ -22,22 +22,32 @@ export function supportedProductPlatform(
return null;
}
/** Returns only the public, allowlisted product-state projection supplied by
* the platform-specific MAIN-world bridge. */
export type MarketplacePageStateFetcher = (
platform: Exclude<SourcePlatform, 'ozon'>,
) => Promise<unknown>;
export async function collectCurrentProduct(
root: Document = document,
pageUrl = location.href,
fetchOzonPageJson?: OzonPageJsonFetcher,
fetchMarketplacePageState?: MarketplacePageStateFetcher,
): Promise<CollectedProductPayload> {
const platform = supportedProductPlatform(pageUrl);
if (platform === 'ozon')
return collectOzonProduct(root, pageUrl, fetchOzonPageJson);
if (!platform) throw new Error('当前页面不是支持的商品详情页');
const pageState = fetchMarketplacePageState
? await fetchMarketplacePageState(platform)
: undefined;
return platform === '1688'
? collect1688(root, pageUrl)
: collectTaobao(root, pageUrl, platform);
? collect1688(root, pageUrl, pageState)
: collectTaobao(root, pageUrl, platform, pageState);
}
function collect1688(root: Document, pageUrl: string): CollectedProductPayload {
const state = pageState(root);
function collect1688(root: Document, pageUrl: string, bridgedState?: unknown): CollectedProductPayload {
const state = [bridgedState, ...pageState(root)];
const skuMap = findFirst(state, 'skuMapOriginal');
const skuProps = findFirst(state, 'skuProps');
const axes = parse1688Axes(skuProps);
@@ -50,14 +60,19 @@ function collect1688(root: Document, pageUrl: string): CollectedProductPayload {
},
];
const variants: Variant[] = [];
if (isRecord(skuMap)) {
const skuRows = Array.isArray(skuMap)
? skuMap.map((row) => [stringAt(isRecord(row) ? row : {}, ['specAttrs', 'propPath', 'skuId']) ?? '', row] as const)
: isRecord(skuMap)
? Object.entries(skuMap)
: [];
if (skuRows.length) {
evidence.push({
ref: '1688:sku-map',
kind: 'state',
path: 'skuMapOriginal',
raw_value: `组合记录 ${Object.keys(skuMap).length} 条`,
raw_value: `组合记录 ${skuRows.length} 条`,
});
for (const [combination, row] of Object.entries(skuMap)) {
for (const [combination, row] of skuRows) {
if (!isRecord(row)) continue;
const sourceSku = stringAt(row, ['skuId', 'sku_id', 'skuCode']);
// A skuMapOriginal entry itself is a source-provided sellable-combination
@@ -97,7 +112,8 @@ function collect1688(root: Document, pageUrl: string): CollectedProductPayload {
axes,
uniqueVariants(variants),
evidence,
'1688-state-1.0.0',
'1688-structured-2.0.0',
state,
);
}
@@ -105,8 +121,9 @@ function collectTaobao(
root: Document,
pageUrl: string,
platform: 'taobao' | 'tmall',
bridgedState?: unknown,
): CollectedProductPayload {
const state = pageState(root);
const state = [bridgedState, ...pageState(root)];
const skuBase = findFirst(state, 'skuBase');
const skuCore = findFirst(state, 'skuCore');
const axes = parseTaobaoAxes(isRecord(skuBase) ? skuBase.props : undefined);
@@ -170,7 +187,8 @@ function collectTaobao(
axes,
uniqueVariants(variants),
evidence,
`${platform}-state-1.0.0`,
`${platform}-structured-2.0.0`,
state,
);
}
@@ -183,15 +201,18 @@ function basePayload(
variants: Variant[],
evidence: CollectedProductPayload['evidence'],
parserVersion: string,
states: unknown[],
): CollectedProductPayload {
const canonical =
root.querySelector<HTMLLinkElement>('link[rel="canonical"]')?.href ??
pageUrl.split('#')[0];
const title =
const facts = extractMarketplaceFacts(root, states, platform);
const title = facts.title ?? (
root.querySelector('h1')?.textContent?.replace(/\s+/g, ' ').trim() ||
root
.querySelector<HTMLMetaElement>('meta[property="og:title"]')
?.content?.trim();
?.content?.trim()
);
const assets = collectPlatformDomAssets(platform, root, canonical);
evidence.push({
ref: `${platform}:dom:image-profile`,
@@ -199,6 +220,15 @@ function basePayload(
path: 'platform-scoped gallery / sku / detail selectors',
raw_value: `采集 ${assets.length} 张受限商品区域图片`,
});
const videoCount = addMarketplaceVideos(platform, states, assets, canonical);
if (videoCount) evidence.push({
ref: `${platform}:structured:videos`,
kind: 'state',
path: platform === '1688' ? 'gallery.fields.video|videos' : 'item.videos',
raw_value: `采集 ${videoCount} 个结构化商品视频`,
});
if (facts.attributes.length) evidence.push({ ref: `${platform}:structured:attributes`, kind: 'state', path: 'allowlisted product state parameter lists', raw_value: `采集 ${facts.attributes.length} 项结构化特征` });
if (facts.description) evidence.push({ ref: `${platform}:json-ld:description`, kind: 'json_ld', path: 'Product.description', raw_value: `采集 ${facts.description.length} 个字符描述` });
return {
source: {
platform,
@@ -217,8 +247,10 @@ function basePayload(
},
}
: {}),
attributes: [],
categories: [],
...(facts.price ? { sourcePrice: { amount: facts.price, currencyCode: 'CNY', raw_value: facts.price } } : {}),
...(facts.description ? { description: { value: facts.description, raw_value: facts.description, confidence: 'verified' as const } } : {}),
attributes: facts.attributes,
categories: facts.categories,
},
variantAxes: axes,
variants,
@@ -227,6 +259,137 @@ function basePayload(
};
}
/** The old collector's proven video sources are structured product state. Do
* not broaden this to page video elements: they also contain feeds and reviews. */
function addMarketplaceVideos(
platform: Exclude<SourcePlatform, 'ozon'>,
states: unknown[],
assets: CollectedProductPayload['assets'],
pageUrl: string,
) {
const gallery = findFirst(states, 'gallery');
const item = findFirst(states, 'item');
const galleryFields = isRecord(gallery) && isRecord(gallery.fields)
? gallery.fields
: gallery;
const raw = platform === '1688'
? [
...(isRecord(galleryFields) && Array.isArray(galleryFields.videos) ? galleryFields.videos : []),
...(isRecord(galleryFields) && galleryFields.video !== undefined ? [galleryFields.video] : []),
]
: isRecord(item) && Array.isArray(item.videos)
? item.videos
: [];
let added = 0;
const seen = new Set(assets.map((asset) => `video:${asset.url}`));
for (const entry of raw) {
if (assets.length >= 500) break;
const record = isRecord(entry) ? entry : undefined;
const url = directVideoUrl(
typeof entry === 'string' ? entry : stringAt(record ?? {}, ['url', 'videoUrl', 'src']),
pageUrl,
);
if (!url || seen.has(`video:${url}`)) continue;
seen.add(`video:${url}`);
const posterUrl = imageUrl(
stringAt(record ?? {}, ['videoThumbnailURL', 'thumbnailUrl', 'coverUrl', 'coverImage', 'poster']),
pageUrl,
);
assets.push({
url,
mediaType: 'video',
...(posterUrl ? { posterUrl } : {}),
role: 'source_main',
relation: 'product_candidate',
sortOrder: assets.length,
evidenceRef: `${platform}:structured:videos`,
});
added += 1;
}
return added;
}
function directVideoUrl(value: string | undefined, pageUrl: string) {
if (!value) return undefined;
try {
const url = new URL(value, pageUrl);
if (url.protocol !== 'https:' || /\.m3u8(?:$|[?#])/i.test(url.pathname)) return undefined;
return /\.(mp4|webm)(?:$|[?#])/i.test(url.pathname) ? url.href : undefined;
} catch {
return undefined;
}
}
function imageUrl(value: string | undefined, pageUrl: string) {
if (!value) return undefined;
try {
const url = new URL(value, pageUrl);
return url.protocol === 'https:' ? url.href : undefined;
} catch {
return undefined;
}
}
function extractMarketplaceFacts(root: Document, states: unknown[], platform: Exclude<SourcePlatform, 'ozon'>) {
const attributes: CollectedProductPayload['common']['attributes'] = [];
const seen = new Set<string>();
for (const key of ['basicParamList', 'enhanceParamList', 'attributes', 'props']) {
const list = findFirst(states, key);
if (!Array.isArray(list)) continue;
for (const [index, entry] of list.entries()) {
if (!isRecord(entry)) continue;
const name = stringAt(entry, ['propertyName', 'name', 'key', 'title']);
const value = stringAt(entry, ['valueName', 'value', 'text']);
if (!name || !value || seen.has(`${name}\u0000${value}`)) continue;
seen.add(`${name}\u0000${value}`);
attributes.push({ key: `${platform}:${stringAt(entry, ['propertyId', 'id', 'key']) ?? index}`.slice(0, 200), name: name.slice(0, 300), value: value.slice(0, 2_000), raw_value: value.slice(0, 4_000) });
}
}
const product = jsonLdProduct(root);
const item = findFirst(states, 'item');
const temp = findFirst(states, 'tempModel');
const tradeModel = findFirst(states, 'tradeModel');
const priceVO = findFirst(states, 'priceVO');
const price = moneyAt(isRecord(tradeModel) ? tradeModel : {}, ['minPrice', 'price'])
?? moneyAt(isRecord(priceVO) ? priceVO : {}, ['priceText', 'price', 'extraPrice'])
?? decimal(product?.offers && isRecord(product.offers) ? product.offers.price : undefined);
const categoryNames = Array.from(root.querySelectorAll<HTMLAnchorElement>('[class*="breadcrumb"] a, [class*="BreadCrumb"] a, nav a[href*="category"]')).map((link) => link.textContent?.replace(/\s+/g, ' ').trim() ?? '').filter(Boolean).slice(0, 50);
return {
title: (isRecord(item) ? stringAt(item, ['title', 'name']) : undefined) ?? (isRecord(temp) ? stringAt(temp, ['offerTitle']) : undefined) ?? (product ? stringAt(product, ['name']) : undefined),
price,
description: (product ? stringAt(product, ['description']) : undefined) ?? scopedDescription(root, platform),
attributes,
categories: [...new Set(categoryNames)].map((name) => ({ key: `${platform}:category:${name.toLowerCase()}`.slice(0, 300), name })),
};
}
function scopedDescription(root: Document, platform: Exclude<SourcePlatform, 'ozon'>) {
const selectors = platform === '1688'
? ['.de-description-detail', '#detailContentContainer', '.html-description']
: ['[class*="tabDetailWrap--"]', '[class*="detailInfo--"]'];
for (const selector of selectors) {
const value = root.querySelector(selector)?.textContent?.replace(/\s+/g, ' ').trim();
if (value && value.length >= 10) return value.slice(0, 100_000);
}
return undefined;
}
function jsonLdProduct(root: Document): Json | undefined {
for (const script of Array.from(root.scripts)) {
if (script.type !== 'application/ld+json' || !script.textContent?.trim()) continue;
try { const found = findJsonLdProduct(JSON.parse(script.textContent)); if (found) return found; } catch { /* ignored */ }
}
return undefined;
}
function findJsonLdProduct(value: unknown): Json | undefined {
if (Array.isArray(value)) { for (const child of value) { const found = findJsonLdProduct(child); if (found) return found; } return undefined; }
if (!isRecord(value)) return undefined;
const type = value['@type'];
if (type === 'Product' || (Array.isArray(type) && type.includes('Product'))) return value;
return Array.isArray(value['@graph']) ? findJsonLdProduct(value['@graph']) : undefined;
}
function decimal(value: unknown) { const raw = typeof value === 'string' || typeof value === 'number' ? String(value).replace(',', '.') : ''; return /^\d+(\.\d{1,4})?$/.test(raw) ? raw : undefined; }
function pageState(root: Document) {
const all: unknown[] = [];
for (const script of Array.from(root.scripts)) {