Update seller store experience
This commit is contained in:
Vendored
+1
@@ -8,6 +8,7 @@ declare module "wxt/browser" {
|
||||
| "/background.js"
|
||||
| "/content-scripts/collector.js"
|
||||
| "/content-scripts/ozon-page-json.js"
|
||||
| "/content-scripts/source-page-state.js"
|
||||
type HtmlPublicPath = Extract<PublicPath, `${string}.html`>
|
||||
export interface WxtRuntime {
|
||||
getURL(path: PublicPath): string;
|
||||
|
||||
@@ -3,6 +3,7 @@ import {
|
||||
supportedProductPlatform,
|
||||
} from '../lib/source-parsers.js';
|
||||
import type { OzonPageJsonFetcher } from '../lib/ozon-parser.js';
|
||||
import type { MarketplacePageStateFetcher } from '../lib/source-parsers.js';
|
||||
|
||||
export default defineContentScript({
|
||||
matches: [
|
||||
@@ -72,6 +73,9 @@ export default defineContentScript({
|
||||
supportedProductPlatform() === 'ozon'
|
||||
? requestOzonPageJson
|
||||
: undefined,
|
||||
supportedProductPlatform() && supportedProductPlatform() !== 'ozon'
|
||||
? requestMarketplacePageState
|
||||
: undefined,
|
||||
);
|
||||
const response = await browser.runtime.sendMessage({
|
||||
type: 'atlas.collect',
|
||||
@@ -128,6 +132,23 @@ const requestOzonPageJson: OzonPageJsonFetcher = ({ kind, sku }) =>
|
||||
);
|
||||
});
|
||||
|
||||
const requestMarketplacePageState: MarketplacePageStateFetcher = (platform) =>
|
||||
new Promise((resolve) => {
|
||||
const requestId = crypto.randomUUID();
|
||||
const timeout = window.setTimeout(() => {
|
||||
window.removeEventListener('message', receive);
|
||||
resolve({});
|
||||
}, 3_000);
|
||||
const receive = (event: MessageEvent<unknown>) => {
|
||||
if (event.source !== window || !isMarketplaceStateResponse(event.data, requestId)) return;
|
||||
window.clearTimeout(timeout);
|
||||
window.removeEventListener('message', receive);
|
||||
resolve(event.data.data);
|
||||
};
|
||||
window.addEventListener('message', receive);
|
||||
window.postMessage({ source: 'atlas-collector', type: 'source-page-state.request', requestId, platform }, location.origin);
|
||||
});
|
||||
|
||||
function isBridgeResponse(
|
||||
value: unknown,
|
||||
requestId: string,
|
||||
@@ -149,3 +170,10 @@ function isBridgeResponse(
|
||||
(message.ok || typeof message.error === 'string')
|
||||
);
|
||||
}
|
||||
|
||||
function isMarketplaceStateResponse(value: unknown, requestId: string): value is { source: 'atlas-collector'; type: 'source-page-state.response'; requestId: string; data: unknown } {
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value)) return false;
|
||||
const message = value as Record<string, unknown>;
|
||||
return message.source === 'atlas-collector' && message.type === 'source-page-state.response'
|
||||
&& message.requestId === requestId && !!message.data && typeof message.data === 'object';
|
||||
}
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
/**
|
||||
* A deliberately narrow MAIN-world bridge for the mature 1688/Taobao/Tmall
|
||||
* structured collectors. It never exposes cookies, headers, arbitrary window
|
||||
* keys, or a generic fetch primitive; only public product-state projections.
|
||||
*/
|
||||
export default defineContentScript({
|
||||
matches: [
|
||||
'*://detail.1688.com/*',
|
||||
'*://item.taobao.com/*',
|
||||
'*://detail.tmall.com/*',
|
||||
],
|
||||
world: 'MAIN',
|
||||
runAt: 'document_idle',
|
||||
main() {
|
||||
window.addEventListener('message', (event) => {
|
||||
if (event.source !== window || !isRequest(event.data)) return;
|
||||
const data = event.data.platform === '1688'
|
||||
? project1688((window as unknown as Record<string, unknown>).context)
|
||||
: projectTaobao(window as unknown as Record<string, unknown>);
|
||||
window.postMessage({
|
||||
source: 'atlas-collector',
|
||||
type: 'source-page-state.response',
|
||||
requestId: event.data.requestId,
|
||||
data,
|
||||
}, location.origin);
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
type Request = { source: 'atlas-collector'; type: 'source-page-state.request'; requestId: string; platform: '1688' | 'taobao' | 'tmall' };
|
||||
function isRequest(value: unknown): value is Request {
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value)) return false;
|
||||
const data = value as Record<string, unknown>;
|
||||
return data.source === 'atlas-collector' && data.type === 'source-page-state.request'
|
||||
&& typeof data.requestId === 'string' && /^[a-zA-Z0-9-]{16,120}$/.test(data.requestId)
|
||||
&& (data.platform === '1688' || data.platform === 'taobao' || data.platform === 'tmall');
|
||||
}
|
||||
|
||||
const safeClone = (value: unknown) => { try { return JSON.parse(JSON.stringify(value)); } catch { return undefined; } };
|
||||
function find(value: unknown, key: string): unknown {
|
||||
const pending: unknown[] = [value];
|
||||
const seen = new Set<object>();
|
||||
let visited = 0;
|
||||
while (pending.length && visited < 300_000) {
|
||||
const current = pending.shift();
|
||||
if (!current || typeof current !== 'object' || seen.has(current)) continue;
|
||||
seen.add(current);
|
||||
visited += 1;
|
||||
const record = current as Record<string, unknown>;
|
||||
if (key in record) return record[key];
|
||||
pending.push(...Object.values(record));
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
function project1688(context: unknown) {
|
||||
const projected: Record<string, unknown> = {};
|
||||
for (const key of ['gallery', 'tempModel', 'skuModel', 'tradeModel', 'skuMapOriginal', 'pieceWeightScaleInfo', 'productAttributes']) {
|
||||
const value = find(context, key);
|
||||
if (value !== undefined) projected[key] = safeClone(value);
|
||||
}
|
||||
return projected;
|
||||
}
|
||||
function projectTaobao(windowState: Record<string, unknown>) {
|
||||
const ice = windowState.__ICE_APP_CONTEXT__ as Record<string, unknown> | undefined;
|
||||
const result = ice?.loaderData && typeof ice.loaderData === 'object'
|
||||
? (((ice.loaderData as Record<string, unknown>).home as Record<string, unknown> | undefined)?.data as Record<string, unknown> | undefined)?.res
|
||||
: undefined;
|
||||
return { product: safeClone(result) };
|
||||
}
|
||||
@@ -61,6 +61,7 @@ export async function collectOzonProduct(
|
||||
const localStates = readLocalWidgetStates(documentRoot);
|
||||
let attributes = extractOzonCharacteristics(localStates);
|
||||
let description = extractOzonDescription(localStates);
|
||||
const jsonLdDescription = extractOzonJsonLdDescription(documentRoot);
|
||||
const heading = firstState(localStates, 'webProductHeading');
|
||||
const priceState = firstState(localStates, 'webPrice');
|
||||
const title =
|
||||
@@ -101,6 +102,14 @@ export async function collectOzonProduct(
|
||||
'ozon:gallery:page',
|
||||
canonical,
|
||||
);
|
||||
addGalleryVideos(
|
||||
localStates,
|
||||
currentSku,
|
||||
assets,
|
||||
assetKeys,
|
||||
'ozon:gallery:page',
|
||||
canonical,
|
||||
);
|
||||
if (attributes.length) {
|
||||
evidence.push({
|
||||
ref: 'ozon:characteristics:page',
|
||||
@@ -109,6 +118,14 @@ export async function collectOzonProduct(
|
||||
raw_value: `发现 ${attributes.length} 项商品特征`,
|
||||
});
|
||||
}
|
||||
if (jsonLdDescription) {
|
||||
evidence.push({
|
||||
ref: 'ozon:json-ld:product.description',
|
||||
kind: 'json_ld',
|
||||
path: 'script[type="application/ld+json"] Product.description',
|
||||
raw_value: `JSON-LD 商品描述 ${jsonLdDescription.value.length} 个字符`,
|
||||
});
|
||||
}
|
||||
|
||||
const categories = collectCategories(documentRoot, canonical);
|
||||
if (categories.length) {
|
||||
@@ -162,12 +179,22 @@ export async function collectOzonProduct(
|
||||
canonical,
|
||||
);
|
||||
description = description ?? extractOzonDescription(detailStates);
|
||||
const detailAttributes = extractOzonCharacteristics(detailStates);
|
||||
attributes = mergeOzonCharacteristics(attributes, detailAttributes);
|
||||
evidence.push({
|
||||
ref: 'ozon:detail:page',
|
||||
kind: 'state',
|
||||
path: 'webDescription|webRichContent|webFeatures image fields',
|
||||
raw_value: `发现 ${added} 张商品详情图`,
|
||||
});
|
||||
if (detailAttributes.length) {
|
||||
evidence.push({
|
||||
ref: 'ozon:characteristics:detail',
|
||||
kind: 'state',
|
||||
path: 'webShortCharacteristics|webDetailedCharacteristics|webCharacteristics.characteristics[]',
|
||||
raw_value: `详情段补全 ${detailAttributes.length} 项商品特征`,
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
evidence.push(
|
||||
fetchFailureEvidence('ozon:detail:page', currentSku, error),
|
||||
@@ -230,6 +257,14 @@ export async function collectOzonProduct(
|
||||
`ozon:gallery:${result.sku}`,
|
||||
canonical,
|
||||
);
|
||||
addGalleryVideos(
|
||||
result.states,
|
||||
result.sku,
|
||||
assets,
|
||||
assetKeys,
|
||||
`ozon:gallery:${result.sku}`,
|
||||
canonical,
|
||||
);
|
||||
if (result.sku === currentSku) {
|
||||
const detailedAttributes = extractOzonCharacteristics(
|
||||
result.states,
|
||||
@@ -270,6 +305,11 @@ export async function collectOzonProduct(
|
||||
});
|
||||
}
|
||||
|
||||
// Ozon's rich widget is preferred because it is the dedicated description
|
||||
// segment. JSON-LD is the proven old-collector fallback for pages where
|
||||
// Ozon only places the seller description in the initial document.
|
||||
description = description ?? jsonLdDescription;
|
||||
|
||||
addAspectCoverAssets(axes, assets, assetKeys, canonical);
|
||||
|
||||
const variants = [...candidates.values()]
|
||||
@@ -301,7 +341,7 @@ export async function collectOzonProduct(
|
||||
itemKey: currentSku,
|
||||
canonicalUrl: canonical,
|
||||
collectedAt: new Date().toISOString(),
|
||||
parserVersion: 'ozon-public-2.2.0',
|
||||
parserVersion: 'ozon-public-2.3.0',
|
||||
},
|
||||
common: {
|
||||
...(title
|
||||
@@ -452,6 +492,7 @@ function addAspectCoverAssets(
|
||||
assetKeys.add(key);
|
||||
assets.push({
|
||||
url,
|
||||
mediaType: 'image',
|
||||
role: 'source_sku',
|
||||
relation: 'axis_value',
|
||||
sortOrder: assets.length,
|
||||
@@ -491,6 +532,7 @@ function addDetailAssets(
|
||||
assetKeys.add(key);
|
||||
assets.push({
|
||||
url,
|
||||
mediaType: 'image',
|
||||
role: 'source_detail',
|
||||
relation: 'product_candidate',
|
||||
sortOrder: added++,
|
||||
@@ -621,6 +663,55 @@ export function extractOzonDescription(
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/** Reads only schema.org Product.description as a fallback, never arbitrary
|
||||
* JSON-LD page copy. This mirrors the mature collector's JSON-LD path. */
|
||||
export function extractOzonJsonLdDescription(
|
||||
root: Document,
|
||||
): OzonDescription | undefined {
|
||||
for (const script of Array.from(root.scripts)) {
|
||||
if (script.type !== 'application/ld+json' || !script.textContent?.trim())
|
||||
continue;
|
||||
try {
|
||||
const product = findJsonLdProduct(JSON.parse(script.textContent));
|
||||
const raw = product && typeof product.description === 'string'
|
||||
? product.description.trim()
|
||||
: '';
|
||||
if (!raw) continue;
|
||||
return {
|
||||
value: raw.replace(/\s+\n/g, '\n').replace(/\n{3,}/g, '\n\n').slice(0, 100_000),
|
||||
rawValue: raw.slice(0, 120_000),
|
||||
};
|
||||
} catch {
|
||||
// A malformed analytics JSON-LD block is not product evidence.
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function findJsonLdProduct(
|
||||
value: unknown,
|
||||
visited = new Set<unknown>(),
|
||||
): JsonRecord | undefined {
|
||||
if (!value || typeof value !== 'object' || visited.has(value)) return undefined;
|
||||
visited.add(value);
|
||||
if (Array.isArray(value)) {
|
||||
for (const child of value) {
|
||||
const product = findJsonLdProduct(child, visited);
|
||||
if (product) return product;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
const record = value as JsonRecord;
|
||||
const type = record['@type'];
|
||||
if (type === 'Product' || (Array.isArray(type) && type.includes('Product')))
|
||||
return record;
|
||||
if (Array.isArray(record['@graph'])) {
|
||||
const product = findJsonLdProduct(record['@graph'], visited);
|
||||
if (product) return product;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function findRichAnnotationJson(
|
||||
value: unknown,
|
||||
visited = new Set<unknown>(),
|
||||
@@ -716,6 +807,7 @@ function addGalleryAssets(
|
||||
assetKeys.add(key);
|
||||
assets.push({
|
||||
url,
|
||||
mediaType: 'image',
|
||||
// A gallery is product photography for this real SKU. It is not
|
||||
// a SKU-option thumbnail merely because it is the second image.
|
||||
role: 'source_main',
|
||||
@@ -727,6 +819,103 @@ function addGalleryAssets(
|
||||
}
|
||||
}
|
||||
|
||||
/** Adds only Ozon's named product-gallery videos. Page-wide <video> nodes can
|
||||
* belong to reviews or recommendations and are intentionally never considered. */
|
||||
function addGalleryVideos(
|
||||
states: JsonRecord[],
|
||||
sku: string,
|
||||
assets: Asset[],
|
||||
assetKeys: Set<string>,
|
||||
evidenceRef: string,
|
||||
pageUrl: string,
|
||||
) {
|
||||
const galleries = states.filter((state) => {
|
||||
const widget = String(state.__atlasWidgetName ?? '').replace(/^state-/, '');
|
||||
return /^web(?:Gallery|Video)-/.test(widget);
|
||||
});
|
||||
for (const gallery of galleries) {
|
||||
for (const video of extractOzonGalleryVideos(gallery, pageUrl)) {
|
||||
if (assets.length >= 500) return;
|
||||
const key = `video:${sku}:${video.url}`;
|
||||
if (assetKeys.has(key)) continue;
|
||||
assetKeys.add(key);
|
||||
assets.push({
|
||||
url: video.url,
|
||||
mediaType: 'video',
|
||||
...(video.posterUrl ? { posterUrl: video.posterUrl } : {}),
|
||||
role: 'source_main',
|
||||
variantSourceKey: sku,
|
||||
relation: 'variant_direct',
|
||||
sortOrder: assets.length,
|
||||
evidenceRef,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Pure helper for the strictly named Ozon gallery/video state. HLS, blobs,
|
||||
* and non-HTTPS URLs stay out of the collection contract in this first phase. */
|
||||
export function extractOzonGalleryVideos(gallery: JsonRecord, pageUrl: string) {
|
||||
const videos: Array<{ url: string; posterUrl?: string }> = [];
|
||||
const seen = new Set<string>();
|
||||
// Ozon changes whether the direct file lives in videos[], video, or a
|
||||
// nested gallery payload. The old collector walked this *one allowlisted
|
||||
// widget* for direct video extensions; retaining that scope is critical.
|
||||
for (const { value, parent } of nestedVideoValues(gallery)) {
|
||||
const url = directVideoUrl(
|
||||
typeof value === 'string'
|
||||
? value
|
||||
: stringAt(value, ['url', 'videoUrl', 'src']),
|
||||
pageUrl,
|
||||
);
|
||||
if (!url) continue;
|
||||
const posterUrl = ozonImageUrl(
|
||||
(typeof value === 'string'
|
||||
? undefined
|
||||
: stringAt(value, ['coverImage', 'coverUrl', 'poster', 'previewImage']))
|
||||
?? stringAt(parent, ['coverImage', 'coverUrl', 'poster', 'previewImage']),
|
||||
pageUrl,
|
||||
);
|
||||
if (seen.has(url)) {
|
||||
const existing = videos.find((video) => video.url === url);
|
||||
if (existing && !existing.posterUrl && posterUrl)
|
||||
existing.posterUrl = posterUrl;
|
||||
continue;
|
||||
}
|
||||
seen.add(url);
|
||||
videos.push({ url, ...(posterUrl ? { posterUrl } : {}) });
|
||||
}
|
||||
return videos;
|
||||
}
|
||||
|
||||
function nestedVideoValues(root: JsonRecord) {
|
||||
const values: Array<{ value: string | JsonRecord; parent?: JsonRecord }> = [];
|
||||
const pending: Array<{ value: unknown; parent?: JsonRecord }> = [{ value: root }];
|
||||
const seen = new Set<object>();
|
||||
while (pending.length) {
|
||||
const current = pending.pop()!;
|
||||
if (typeof current.value === 'string') {
|
||||
if (/\.(mp4|webm)(?:$|[?#])/i.test(current.value))
|
||||
values.push({ value: current.value, parent: current.parent });
|
||||
continue;
|
||||
}
|
||||
if (!current.value || typeof current.value !== 'object') continue;
|
||||
if (seen.has(current.value)) continue;
|
||||
seen.add(current.value);
|
||||
if (Array.isArray(current.value)) {
|
||||
for (const child of current.value)
|
||||
pending.push({ value: child, parent: current.parent });
|
||||
continue;
|
||||
}
|
||||
const record = current.value as JsonRecord;
|
||||
if (stringAt(record, ['url', 'videoUrl', 'src']))
|
||||
values.push({ value: record, parent: current.parent });
|
||||
for (const child of Object.values(record))
|
||||
pending.push({ value: child, parent: record });
|
||||
}
|
||||
return values;
|
||||
}
|
||||
|
||||
/** Pure helper used by fixture tests and the structured Ozon gallery reader. */
|
||||
export function extractOzonGalleryImageUrls(
|
||||
gallery: JsonRecord,
|
||||
@@ -777,7 +966,7 @@ function widgetStatesFromPageJson(value: unknown): JsonRecord[] {
|
||||
const states: JsonRecord[] = [];
|
||||
for (const [widgetName, raw] of Object.entries(value.widgetStates)) {
|
||||
if (
|
||||
!/^web(Aspects|AspectsModal|ProductHeading|Price|Gallery|Description|RichContent|Features|ShortCharacteristics|DetailedCharacteristics|Characteristics)-/.test(
|
||||
!/^web(Aspects|AspectsModal|ProductHeading|Price|Gallery|Video|Description|RichContent|Features|ShortCharacteristics|DetailedCharacteristics|Characteristics)-/.test(
|
||||
widgetName,
|
||||
)
|
||||
)
|
||||
@@ -997,3 +1186,15 @@ function ozonImageUrl(value: string | undefined, pageUrl: string) {
|
||||
const url = normaliseImageUrl(value, pageUrl, 'ozon');
|
||||
return url?.startsWith('https://') ? url : undefined;
|
||||
}
|
||||
|
||||
function directVideoUrl(value: string | undefined, pageUrl: string) {
|
||||
if (!value) return undefined;
|
||||
try {
|
||||
const url = new URL(value, pageUrl);
|
||||
if (url.protocol !== 'https:' || /\.m3u8(?:$|[?#])/i.test(url.pathname))
|
||||
return undefined;
|
||||
return /\.(mp4|webm)(?:$|[?#])/i.test(url.pathname) ? url.href : undefined;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -180,6 +180,7 @@ export function collectPlatformDomAssets(
|
||||
seen.add(key);
|
||||
assets.push({
|
||||
url,
|
||||
mediaType: 'image',
|
||||
role: group.role,
|
||||
relation: group.relation,
|
||||
sortOrder: assets.length,
|
||||
|
||||
@@ -22,22 +22,32 @@ export function supportedProductPlatform(
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Returns only the public, allowlisted product-state projection supplied by
|
||||
* the platform-specific MAIN-world bridge. */
|
||||
export type MarketplacePageStateFetcher = (
|
||||
platform: Exclude<SourcePlatform, 'ozon'>,
|
||||
) => Promise<unknown>;
|
||||
|
||||
export async function collectCurrentProduct(
|
||||
root: Document = document,
|
||||
pageUrl = location.href,
|
||||
fetchOzonPageJson?: OzonPageJsonFetcher,
|
||||
fetchMarketplacePageState?: MarketplacePageStateFetcher,
|
||||
): Promise<CollectedProductPayload> {
|
||||
const platform = supportedProductPlatform(pageUrl);
|
||||
if (platform === 'ozon')
|
||||
return collectOzonProduct(root, pageUrl, fetchOzonPageJson);
|
||||
if (!platform) throw new Error('当前页面不是支持的商品详情页');
|
||||
const pageState = fetchMarketplacePageState
|
||||
? await fetchMarketplacePageState(platform)
|
||||
: undefined;
|
||||
return platform === '1688'
|
||||
? collect1688(root, pageUrl)
|
||||
: collectTaobao(root, pageUrl, platform);
|
||||
? collect1688(root, pageUrl, pageState)
|
||||
: collectTaobao(root, pageUrl, platform, pageState);
|
||||
}
|
||||
|
||||
function collect1688(root: Document, pageUrl: string): CollectedProductPayload {
|
||||
const state = pageState(root);
|
||||
function collect1688(root: Document, pageUrl: string, bridgedState?: unknown): CollectedProductPayload {
|
||||
const state = [bridgedState, ...pageState(root)];
|
||||
const skuMap = findFirst(state, 'skuMapOriginal');
|
||||
const skuProps = findFirst(state, 'skuProps');
|
||||
const axes = parse1688Axes(skuProps);
|
||||
@@ -50,14 +60,19 @@ function collect1688(root: Document, pageUrl: string): CollectedProductPayload {
|
||||
},
|
||||
];
|
||||
const variants: Variant[] = [];
|
||||
if (isRecord(skuMap)) {
|
||||
const skuRows = Array.isArray(skuMap)
|
||||
? skuMap.map((row) => [stringAt(isRecord(row) ? row : {}, ['specAttrs', 'propPath', 'skuId']) ?? '', row] as const)
|
||||
: isRecord(skuMap)
|
||||
? Object.entries(skuMap)
|
||||
: [];
|
||||
if (skuRows.length) {
|
||||
evidence.push({
|
||||
ref: '1688:sku-map',
|
||||
kind: 'state',
|
||||
path: 'skuMapOriginal',
|
||||
raw_value: `组合记录 ${Object.keys(skuMap).length} 条`,
|
||||
raw_value: `组合记录 ${skuRows.length} 条`,
|
||||
});
|
||||
for (const [combination, row] of Object.entries(skuMap)) {
|
||||
for (const [combination, row] of skuRows) {
|
||||
if (!isRecord(row)) continue;
|
||||
const sourceSku = stringAt(row, ['skuId', 'sku_id', 'skuCode']);
|
||||
// A skuMapOriginal entry itself is a source-provided sellable-combination
|
||||
@@ -97,7 +112,8 @@ function collect1688(root: Document, pageUrl: string): CollectedProductPayload {
|
||||
axes,
|
||||
uniqueVariants(variants),
|
||||
evidence,
|
||||
'1688-state-1.0.0',
|
||||
'1688-structured-2.0.0',
|
||||
state,
|
||||
);
|
||||
}
|
||||
|
||||
@@ -105,8 +121,9 @@ function collectTaobao(
|
||||
root: Document,
|
||||
pageUrl: string,
|
||||
platform: 'taobao' | 'tmall',
|
||||
bridgedState?: unknown,
|
||||
): CollectedProductPayload {
|
||||
const state = pageState(root);
|
||||
const state = [bridgedState, ...pageState(root)];
|
||||
const skuBase = findFirst(state, 'skuBase');
|
||||
const skuCore = findFirst(state, 'skuCore');
|
||||
const axes = parseTaobaoAxes(isRecord(skuBase) ? skuBase.props : undefined);
|
||||
@@ -170,7 +187,8 @@ function collectTaobao(
|
||||
axes,
|
||||
uniqueVariants(variants),
|
||||
evidence,
|
||||
`${platform}-state-1.0.0`,
|
||||
`${platform}-structured-2.0.0`,
|
||||
state,
|
||||
);
|
||||
}
|
||||
|
||||
@@ -183,15 +201,18 @@ function basePayload(
|
||||
variants: Variant[],
|
||||
evidence: CollectedProductPayload['evidence'],
|
||||
parserVersion: string,
|
||||
states: unknown[],
|
||||
): CollectedProductPayload {
|
||||
const canonical =
|
||||
root.querySelector<HTMLLinkElement>('link[rel="canonical"]')?.href ??
|
||||
pageUrl.split('#')[0];
|
||||
const title =
|
||||
const facts = extractMarketplaceFacts(root, states, platform);
|
||||
const title = facts.title ?? (
|
||||
root.querySelector('h1')?.textContent?.replace(/\s+/g, ' ').trim() ||
|
||||
root
|
||||
.querySelector<HTMLMetaElement>('meta[property="og:title"]')
|
||||
?.content?.trim();
|
||||
?.content?.trim()
|
||||
);
|
||||
const assets = collectPlatformDomAssets(platform, root, canonical);
|
||||
evidence.push({
|
||||
ref: `${platform}:dom:image-profile`,
|
||||
@@ -199,6 +220,15 @@ function basePayload(
|
||||
path: 'platform-scoped gallery / sku / detail selectors',
|
||||
raw_value: `采集 ${assets.length} 张受限商品区域图片`,
|
||||
});
|
||||
const videoCount = addMarketplaceVideos(platform, states, assets, canonical);
|
||||
if (videoCount) evidence.push({
|
||||
ref: `${platform}:structured:videos`,
|
||||
kind: 'state',
|
||||
path: platform === '1688' ? 'gallery.fields.video|videos' : 'item.videos',
|
||||
raw_value: `采集 ${videoCount} 个结构化商品视频`,
|
||||
});
|
||||
if (facts.attributes.length) evidence.push({ ref: `${platform}:structured:attributes`, kind: 'state', path: 'allowlisted product state parameter lists', raw_value: `采集 ${facts.attributes.length} 项结构化特征` });
|
||||
if (facts.description) evidence.push({ ref: `${platform}:json-ld:description`, kind: 'json_ld', path: 'Product.description', raw_value: `采集 ${facts.description.length} 个字符描述` });
|
||||
return {
|
||||
source: {
|
||||
platform,
|
||||
@@ -217,8 +247,10 @@ function basePayload(
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
attributes: [],
|
||||
categories: [],
|
||||
...(facts.price ? { sourcePrice: { amount: facts.price, currencyCode: 'CNY', raw_value: facts.price } } : {}),
|
||||
...(facts.description ? { description: { value: facts.description, raw_value: facts.description, confidence: 'verified' as const } } : {}),
|
||||
attributes: facts.attributes,
|
||||
categories: facts.categories,
|
||||
},
|
||||
variantAxes: axes,
|
||||
variants,
|
||||
@@ -227,6 +259,137 @@ function basePayload(
|
||||
};
|
||||
}
|
||||
|
||||
/** The old collector's proven video sources are structured product state. Do
|
||||
* not broaden this to page video elements: they also contain feeds and reviews. */
|
||||
function addMarketplaceVideos(
|
||||
platform: Exclude<SourcePlatform, 'ozon'>,
|
||||
states: unknown[],
|
||||
assets: CollectedProductPayload['assets'],
|
||||
pageUrl: string,
|
||||
) {
|
||||
const gallery = findFirst(states, 'gallery');
|
||||
const item = findFirst(states, 'item');
|
||||
const galleryFields = isRecord(gallery) && isRecord(gallery.fields)
|
||||
? gallery.fields
|
||||
: gallery;
|
||||
const raw = platform === '1688'
|
||||
? [
|
||||
...(isRecord(galleryFields) && Array.isArray(galleryFields.videos) ? galleryFields.videos : []),
|
||||
...(isRecord(galleryFields) && galleryFields.video !== undefined ? [galleryFields.video] : []),
|
||||
]
|
||||
: isRecord(item) && Array.isArray(item.videos)
|
||||
? item.videos
|
||||
: [];
|
||||
let added = 0;
|
||||
const seen = new Set(assets.map((asset) => `video:${asset.url}`));
|
||||
for (const entry of raw) {
|
||||
if (assets.length >= 500) break;
|
||||
const record = isRecord(entry) ? entry : undefined;
|
||||
const url = directVideoUrl(
|
||||
typeof entry === 'string' ? entry : stringAt(record ?? {}, ['url', 'videoUrl', 'src']),
|
||||
pageUrl,
|
||||
);
|
||||
if (!url || seen.has(`video:${url}`)) continue;
|
||||
seen.add(`video:${url}`);
|
||||
const posterUrl = imageUrl(
|
||||
stringAt(record ?? {}, ['videoThumbnailURL', 'thumbnailUrl', 'coverUrl', 'coverImage', 'poster']),
|
||||
pageUrl,
|
||||
);
|
||||
assets.push({
|
||||
url,
|
||||
mediaType: 'video',
|
||||
...(posterUrl ? { posterUrl } : {}),
|
||||
role: 'source_main',
|
||||
relation: 'product_candidate',
|
||||
sortOrder: assets.length,
|
||||
evidenceRef: `${platform}:structured:videos`,
|
||||
});
|
||||
added += 1;
|
||||
}
|
||||
return added;
|
||||
}
|
||||
|
||||
function directVideoUrl(value: string | undefined, pageUrl: string) {
|
||||
if (!value) return undefined;
|
||||
try {
|
||||
const url = new URL(value, pageUrl);
|
||||
if (url.protocol !== 'https:' || /\.m3u8(?:$|[?#])/i.test(url.pathname)) return undefined;
|
||||
return /\.(mp4|webm)(?:$|[?#])/i.test(url.pathname) ? url.href : undefined;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
function imageUrl(value: string | undefined, pageUrl: string) {
|
||||
if (!value) return undefined;
|
||||
try {
|
||||
const url = new URL(value, pageUrl);
|
||||
return url.protocol === 'https:' ? url.href : undefined;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
function extractMarketplaceFacts(root: Document, states: unknown[], platform: Exclude<SourcePlatform, 'ozon'>) {
|
||||
const attributes: CollectedProductPayload['common']['attributes'] = [];
|
||||
const seen = new Set<string>();
|
||||
for (const key of ['basicParamList', 'enhanceParamList', 'attributes', 'props']) {
|
||||
const list = findFirst(states, key);
|
||||
if (!Array.isArray(list)) continue;
|
||||
for (const [index, entry] of list.entries()) {
|
||||
if (!isRecord(entry)) continue;
|
||||
const name = stringAt(entry, ['propertyName', 'name', 'key', 'title']);
|
||||
const value = stringAt(entry, ['valueName', 'value', 'text']);
|
||||
if (!name || !value || seen.has(`${name}\u0000${value}`)) continue;
|
||||
seen.add(`${name}\u0000${value}`);
|
||||
attributes.push({ key: `${platform}:${stringAt(entry, ['propertyId', 'id', 'key']) ?? index}`.slice(0, 200), name: name.slice(0, 300), value: value.slice(0, 2_000), raw_value: value.slice(0, 4_000) });
|
||||
}
|
||||
}
|
||||
const product = jsonLdProduct(root);
|
||||
const item = findFirst(states, 'item');
|
||||
const temp = findFirst(states, 'tempModel');
|
||||
const tradeModel = findFirst(states, 'tradeModel');
|
||||
const priceVO = findFirst(states, 'priceVO');
|
||||
const price = moneyAt(isRecord(tradeModel) ? tradeModel : {}, ['minPrice', 'price'])
|
||||
?? moneyAt(isRecord(priceVO) ? priceVO : {}, ['priceText', 'price', 'extraPrice'])
|
||||
?? decimal(product?.offers && isRecord(product.offers) ? product.offers.price : undefined);
|
||||
const categoryNames = Array.from(root.querySelectorAll<HTMLAnchorElement>('[class*="breadcrumb"] a, [class*="BreadCrumb"] a, nav a[href*="category"]')).map((link) => link.textContent?.replace(/\s+/g, ' ').trim() ?? '').filter(Boolean).slice(0, 50);
|
||||
return {
|
||||
title: (isRecord(item) ? stringAt(item, ['title', 'name']) : undefined) ?? (isRecord(temp) ? stringAt(temp, ['offerTitle']) : undefined) ?? (product ? stringAt(product, ['name']) : undefined),
|
||||
price,
|
||||
description: (product ? stringAt(product, ['description']) : undefined) ?? scopedDescription(root, platform),
|
||||
attributes,
|
||||
categories: [...new Set(categoryNames)].map((name) => ({ key: `${platform}:category:${name.toLowerCase()}`.slice(0, 300), name })),
|
||||
};
|
||||
}
|
||||
|
||||
function scopedDescription(root: Document, platform: Exclude<SourcePlatform, 'ozon'>) {
|
||||
const selectors = platform === '1688'
|
||||
? ['.de-description-detail', '#detailContentContainer', '.html-description']
|
||||
: ['[class*="tabDetailWrap--"]', '[class*="detailInfo--"]'];
|
||||
for (const selector of selectors) {
|
||||
const value = root.querySelector(selector)?.textContent?.replace(/\s+/g, ' ').trim();
|
||||
if (value && value.length >= 10) return value.slice(0, 100_000);
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function jsonLdProduct(root: Document): Json | undefined {
|
||||
for (const script of Array.from(root.scripts)) {
|
||||
if (script.type !== 'application/ld+json' || !script.textContent?.trim()) continue;
|
||||
try { const found = findJsonLdProduct(JSON.parse(script.textContent)); if (found) return found; } catch { /* ignored */ }
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
function findJsonLdProduct(value: unknown): Json | undefined {
|
||||
if (Array.isArray(value)) { for (const child of value) { const found = findJsonLdProduct(child); if (found) return found; } return undefined; }
|
||||
if (!isRecord(value)) return undefined;
|
||||
const type = value['@type'];
|
||||
if (type === 'Product' || (Array.isArray(type) && type.includes('Product'))) return value;
|
||||
return Array.isArray(value['@graph']) ? findJsonLdProduct(value['@graph']) : undefined;
|
||||
}
|
||||
function decimal(value: unknown) { const raw = typeof value === 'string' || typeof value === 'number' ? String(value).replace(',', '.') : ''; return /^\d+(\.\d{1,4})?$/.test(raw) ? raw : undefined; }
|
||||
|
||||
function pageState(root: Document) {
|
||||
const all: unknown[] = [];
|
||||
for (const script of Array.from(root.scripts)) {
|
||||
|
||||
@@ -5,7 +5,9 @@ import test from 'node:test';
|
||||
import {
|
||||
extractOzonCharacteristics,
|
||||
extractOzonDescription,
|
||||
extractOzonJsonLdDescription,
|
||||
extractOzonGalleryImageUrls,
|
||||
extractOzonGalleryVideos,
|
||||
} from '../lib/ozon-parser.ts';
|
||||
import { dedupeImageUrl, normaliseImageUrl } from '../lib/product-media.ts';
|
||||
|
||||
@@ -73,6 +75,41 @@ test('Ozon rich annotation converts to description text without images', () => {
|
||||
assert.equal(description?.value, '耐用的背包\n适合旅行');
|
||||
});
|
||||
|
||||
test('Ozon JSON-LD Product description is the fallback when rich content is absent', () => {
|
||||
const root = {
|
||||
scripts: [{
|
||||
type: 'application/ld+json',
|
||||
textContent: JSON.stringify({
|
||||
'@context': 'http://schema.org',
|
||||
'@type': 'Product',
|
||||
name: '喷雾器',
|
||||
description: '来自 Ozon JSON-LD 的完整商品描述',
|
||||
}),
|
||||
}],
|
||||
} as unknown as Document;
|
||||
assert.equal(
|
||||
extractOzonJsonLdDescription(root)?.value,
|
||||
'来自 Ozon JSON-LD 的完整商品描述',
|
||||
);
|
||||
});
|
||||
|
||||
test('Ozon full characteristics merge with the five short header characteristics', () => {
|
||||
const attributes = extractOzonCharacteristics([
|
||||
{
|
||||
__atlasWidgetName: 'webShortCharacteristics-fixture',
|
||||
characteristics: [{ id: 'type', title: { textRs: [{ content: '类型' }] }, values: [{ text: '喷雾器' }] }],
|
||||
},
|
||||
{
|
||||
__atlasWidgetName: 'webCharacteristics-fixture',
|
||||
characteristics: [{ id: 'weight', title: { textRs: [{ content: '商品重量,克' }] }, values: [{ text: '385' }] }],
|
||||
},
|
||||
]);
|
||||
assert.deepEqual(attributes.map(({ name, value }) => ({ name, value })), [
|
||||
{ name: '类型', value: '喷雾器' },
|
||||
{ name: '商品重量,克', value: '385' },
|
||||
]);
|
||||
});
|
||||
|
||||
test('Ozon thumbnail variants normalise to one original URL', () => {
|
||||
const original = 'https://ir.ozone.ru/s3/multimedia-1-9/13597160877.jpg';
|
||||
const thumbnail =
|
||||
@@ -84,6 +121,29 @@ test('Ozon thumbnail variants normalise to one original URL', () => {
|
||||
);
|
||||
});
|
||||
|
||||
test('Ozon gallery only accepts direct product MP4/WebM videos and retains poster', () => {
|
||||
assert.deepEqual(
|
||||
extractOzonGalleryVideos(
|
||||
{
|
||||
videos: [
|
||||
{
|
||||
url: 'https://cdn.ozon.ru/product/demo.mp4',
|
||||
coverImage: 'https://ir.ozone.ru/s3/multimedia-1/demo.jpg',
|
||||
},
|
||||
{ videoUrl: 'https://cdn.ozon.ru/product/demo.mp4' },
|
||||
{ url: 'https://cdn.ozon.ru/product/stream.m3u8' },
|
||||
{ url: 'blob:https://www.ozon.ru/opaque' },
|
||||
],
|
||||
},
|
||||
'https://www.ozon.ru/product/demo-1/',
|
||||
),
|
||||
[{
|
||||
url: 'https://cdn.ozon.ru/product/demo.mp4',
|
||||
posterUrl: 'https://ir.ozone.ru/s3/multimedia-1/demo.jpg',
|
||||
}],
|
||||
);
|
||||
});
|
||||
|
||||
test('Alibaba transformed image URLs normalise before de-duplication', () => {
|
||||
const original = 'https://cbu01.alicdn.com/img/ibank/O1CN01.jpg';
|
||||
const transformed = `${original}_400x400.jpg?width=400`;
|
||||
|
||||
Reference in New Issue
Block a user