Files
atlas-seller-kit/apps/extension/lib/source-parsers.ts
T

626 lines
24 KiB
TypeScript

import type { CollectedProductPayload } from '@atlas/contracts';
import {
collectOzonProduct,
isOzonProductPage,
type OzonPageJsonFetcher,
} from './ozon-parser.js';
import { collectPlatformDomAssets } from './product-media.js';
import { formatCollectedDescription } from './description-format.js';
type SourcePlatform = '1688' | 'taobao' | 'tmall' | 'ozon';
type Json = Record<string, unknown>;
type Axis = CollectedProductPayload['variantAxes'][number];
type Variant = CollectedProductPayload['variants'][number];
export function supportedProductPlatform(
url = location.href,
): SourcePlatform | null {
if (isOzonProductPage(url)) return 'ozon';
if (/^https:\/\/detail\.1688\.com\/offer\/\d+\.html/.test(url))
return '1688';
if (/^https:\/\/item\.taobao\.com\/item\.htm/.test(url)) return 'taobao';
if (/^https:\/\/detail\.tmall\.com\/item\.htm/.test(url)) return 'tmall';
return null;
}
/** Returns only the public, allowlisted product-state projection supplied by
* the platform-specific MAIN-world bridge. */
export type MarketplacePageStateFetcher = (
platform: Exclude<SourcePlatform, 'ozon'>,
) => Promise<unknown>;
export async function collectCurrentProduct(
root: Document = document,
pageUrl = location.href,
fetchOzonPageJson?: OzonPageJsonFetcher,
fetchMarketplacePageState?: MarketplacePageStateFetcher,
): Promise<CollectedProductPayload> {
const platform = supportedProductPlatform(pageUrl);
if (platform === 'ozon')
return collectOzonProduct(root, pageUrl, fetchOzonPageJson);
if (!platform) throw new Error('当前页面不是支持的商品详情页');
const pageState = fetchMarketplacePageState
? await fetchMarketplacePageState(platform)
: undefined;
return platform === '1688'
? collect1688(root, pageUrl, pageState)
: collectTaobao(root, pageUrl, platform, pageState);
}
function collect1688(root: Document, pageUrl: string, bridgedState?: unknown): CollectedProductPayload {
const state = [bridgedState, ...pageState(root)];
const skuMap = findFirst(state, 'skuMapOriginal');
const skuProps = findFirst(state, 'skuProps');
const axes = parse1688Axes(skuProps);
const evidence: CollectedProductPayload['evidence'] = [
{
ref: 'page:url',
kind: 'page',
path: 'location.href',
raw_value: pageUrl,
},
];
const variants: Variant[] = [];
const skuRows = Array.isArray(skuMap)
? skuMap.map((row) => [stringAt(isRecord(row) ? row : {}, ['specAttrs', 'propPath', 'skuId']) ?? '', row] as const)
: isRecord(skuMap)
? Object.entries(skuMap)
: [];
if (skuRows.length) {
evidence.push({
ref: '1688:sku-map',
kind: 'state',
path: 'skuMapOriginal',
raw_value: `组合记录 ${skuRows.length} 条`,
});
for (const [combination, row] of skuRows) {
if (!isRecord(row)) continue;
const sourceSku = stringAt(row, ['skuId', 'sku_id', 'skuCode']);
// A skuMapOriginal entry itself is a source-provided sellable-combination
// record. Never expand it with the Cartesian product of skuProps.
const sourceVariantKey = sourceSku ?? `1688:${combination}`;
if (!combination || combination === 'undefined') continue;
variants.push({
sourceVariantKey,
...(sourceSku ? { sourceSku } : {}),
displayName:
selectionsForCombination(combination, axes)
.map((item) => item.valueName)
.join(' / ') || `1688 SKU ${sourceSku ?? combination}`,
selections: selectionsForCombination(combination, axes),
...(moneyAt(row, ['discountPrice', 'price', 'salePrice'])
? {
sourcePrice: {
amount: moneyAt(row, [
'discountPrice',
'price',
'salePrice',
])!,
currencyCode: 'CNY',
},
}
: {}),
confidence: 'verified',
evidenceRefs: ['1688:sku-map'],
});
}
}
return basePayload(
'1688',
offerId(pageUrl),
root,
pageUrl,
axes,
uniqueVariants(variants),
evidence,
'1688-structured-2.0.1',
state,
);
}
function collectTaobao(
root: Document,
pageUrl: string,
platform: 'taobao' | 'tmall',
bridgedState?: unknown,
): CollectedProductPayload {
const state = [bridgedState, ...pageState(root)];
const skuBase = findFirst(state, 'skuBase');
const skuCore = findFirst(state, 'skuCore');
const axes = parseTaobaoAxes(isRecord(skuBase) ? skuBase.props : undefined);
const evidence: CollectedProductPayload['evidence'] = [
{
ref: 'page:url',
kind: 'page',
path: 'location.href',
raw_value: pageUrl,
},
];
const variants: Variant[] = [];
const skus =
isRecord(skuBase) && Array.isArray(skuBase.skus) ? skuBase.skus : [];
const sku2info =
isRecord(skuCore) && isRecord(skuCore.sku2info) ? skuCore.sku2info : {};
if (skus.length)
evidence.push({
ref: 'taobao:sku-base',
kind: 'state',
path: 'skuBase.skus',
raw_value: `组合记录 ${skus.length} 条`,
});
for (const sku of skus) {
if (!isRecord(sku)) continue;
const skuId = stringAt(sku, ['skuId', 'sku_id']);
const propPath = stringAt(sku, ['propPath', 'prop_path']);
// skuId + propPath is the required combination proof. Props alone only
// become axes and cause a single default SKU on the server.
if (!skuId || !propPath) continue;
const info = isRecord(sku2info[skuId]) ? sku2info[skuId] : {};
const selections = selectionsForPropPath(propPath, axes);
variants.push({
sourceVariantKey: skuId,
sourceSku: skuId,
displayName:
selections.map((item) => item.valueName).join(' / ') ||
`SKU ${skuId}`,
selections,
...(moneyAt(info, ['price', 'promotionPrice', 'discountPrice'])
? {
sourcePrice: {
amount: moneyAt(info, [
'price',
'promotionPrice',
'discountPrice',
])!,
currencyCode: 'CNY',
},
}
: {}),
confidence: 'verified',
evidenceRefs: ['taobao:sku-base'],
});
}
return basePayload(
platform,
itemId(pageUrl),
root,
pageUrl,
axes,
uniqueVariants(variants),
evidence,
`${platform}-structured-2.0.1`,
state,
);
}
function basePayload(
platform: Exclude<SourcePlatform, 'ozon'>,
itemKey: string,
root: Document,
pageUrl: string,
axes: Axis[],
variants: Variant[],
evidence: CollectedProductPayload['evidence'],
parserVersion: string,
states: unknown[],
): CollectedProductPayload {
const canonical =
root.querySelector<HTMLLinkElement>('link[rel="canonical"]')?.href ??
pageUrl.split('#')[0];
const facts = extractMarketplaceFacts(root, states, platform);
const title = facts.title ?? (
root.querySelector('h1')?.textContent?.replace(/\s+/g, ' ').trim() ||
root
.querySelector<HTMLMetaElement>('meta[property="og:title"]')
?.content?.trim()
);
const assets = collectPlatformDomAssets(platform, root, canonical);
evidence.push({
ref: `${platform}:dom:image-profile`,
kind: 'dom',
path: 'platform-scoped gallery / sku / detail selectors',
raw_value: `采集 ${assets.length} 张受限商品区域图片`,
});
const videoCount = addMarketplaceVideos(platform, states, assets, canonical);
if (videoCount) evidence.push({
ref: `${platform}:structured:videos`,
kind: 'state',
path: platform === '1688' ? 'gallery.fields.video|videos' : 'item.videos',
raw_value: `采集 ${videoCount} 个结构化商品视频`,
});
if (facts.attributes.length) evidence.push({ ref: `${platform}:structured:attributes`, kind: 'state', path: 'allowlisted product state parameter lists', raw_value: `采集 ${facts.attributes.length} 项结构化特征` });
if (facts.description) evidence.push({ ref: `${platform}:json-ld:description`, kind: 'json_ld', path: 'Product.description', raw_value: `采集 ${facts.description.length} 个字符描述` });
return {
source: {
platform,
itemKey,
canonicalUrl: canonical,
collectedAt: new Date().toISOString(),
parserVersion,
},
common: {
...(title
? {
title: {
value: title,
raw_value: title,
confidence: 'verified' as const,
},
}
: {}),
...(facts.price ? { sourcePrice: { amount: facts.price, currencyCode: 'CNY', raw_value: facts.price } } : {}),
...(facts.description ? { description: { value: facts.description, raw_value: facts.description, confidence: 'verified' as const } } : {}),
attributes: facts.attributes,
categories: facts.categories,
},
variantAxes: axes,
variants,
assets,
evidence,
};
}
/** The old collector's proven video sources are structured product state. Do
* not broaden this to page video elements: they also contain feeds and reviews. */
function addMarketplaceVideos(
platform: Exclude<SourcePlatform, 'ozon'>,
states: unknown[],
assets: CollectedProductPayload['assets'],
pageUrl: string,
) {
const gallery = findFirst(states, 'gallery');
const item = findFirst(states, 'item');
const galleryFields = isRecord(gallery) && isRecord(gallery.fields)
? gallery.fields
: gallery;
const raw = platform === '1688'
? [
...(isRecord(galleryFields) && Array.isArray(galleryFields.videos) ? galleryFields.videos : []),
...(isRecord(galleryFields) && galleryFields.video !== undefined ? [galleryFields.video] : []),
]
: isRecord(item) && Array.isArray(item.videos)
? item.videos
: [];
let added = 0;
const seen = new Set(assets.map((asset) => `video:${asset.url}`));
for (const entry of raw) {
if (assets.length >= 500) break;
const record = isRecord(entry) ? entry : undefined;
const url = directVideoUrl(
typeof entry === 'string' ? entry : stringAt(record ?? {}, ['url', 'videoUrl', 'src']),
pageUrl,
);
if (!url || seen.has(`video:${url}`)) continue;
seen.add(`video:${url}`);
const posterUrl = imageUrl(
stringAt(record ?? {}, ['videoThumbnailURL', 'thumbnailUrl', 'coverUrl', 'coverImage', 'poster']),
pageUrl,
);
assets.push({
url,
mediaType: 'video',
...(posterUrl ? { posterUrl } : {}),
role: 'source_main',
relation: 'product_candidate',
sortOrder: assets.length,
evidenceRef: `${platform}:structured:videos`,
});
added += 1;
}
return added;
}
function directVideoUrl(value: string | undefined, pageUrl: string) {
if (!value) return undefined;
try {
const url = new URL(value, pageUrl);
if (url.protocol !== 'https:' || /\.m3u8(?:$|[?#])/i.test(url.pathname)) return undefined;
return /\.(mp4|webm)(?:$|[?#])/i.test(url.pathname) ? url.href : undefined;
} catch {
return undefined;
}
}
function imageUrl(value: string | undefined, pageUrl: string) {
if (!value) return undefined;
try {
const url = new URL(value, pageUrl);
return url.protocol === 'https:' ? url.href : undefined;
} catch {
return undefined;
}
}
function extractMarketplaceFacts(root: Document, states: unknown[], platform: Exclude<SourcePlatform, 'ozon'>) {
const attributes: CollectedProductPayload['common']['attributes'] = [];
const seen = new Set<string>();
for (const key of ['basicParamList', 'enhanceParamList', 'attributes', 'props']) {
const list = findFirst(states, key);
if (!Array.isArray(list)) continue;
for (const [index, entry] of list.entries()) {
if (!isRecord(entry)) continue;
const name = stringAt(entry, ['propertyName', 'name', 'key', 'title']);
const value = stringAt(entry, ['valueName', 'value', 'text']);
if (!name || !value || seen.has(`${name}\u0000${value}`)) continue;
seen.add(`${name}\u0000${value}`);
attributes.push({ key: `${platform}:${stringAt(entry, ['propertyId', 'id', 'key']) ?? index}`.slice(0, 200), name: name.slice(0, 300), value: value.slice(0, 2_000), raw_value: value.slice(0, 4_000) });
}
}
const product = jsonLdProduct(root);
const item = findFirst(states, 'item');
const temp = findFirst(states, 'tempModel');
const tradeModel = findFirst(states, 'tradeModel');
const priceVO = findFirst(states, 'priceVO');
const price = moneyAt(isRecord(tradeModel) ? tradeModel : {}, ['minPrice', 'price'])
?? moneyAt(isRecord(priceVO) ? priceVO : {}, ['priceText', 'price', 'extraPrice'])
?? decimal(product?.offers && isRecord(product.offers) ? product.offers.price : undefined);
const categoryNames = Array.from(root.querySelectorAll<HTMLAnchorElement>('[class*="breadcrumb"] a, [class*="BreadCrumb"] a, nav a[href*="category"]')).map((link) => link.textContent?.replace(/\s+/g, ' ').trim() ?? '').filter(Boolean).slice(0, 50);
return {
title: (isRecord(item) ? stringAt(item, ['title', 'name']) : undefined) ?? (isRecord(temp) ? stringAt(temp, ['offerTitle']) : undefined) ?? (product ? stringAt(product, ['name']) : undefined),
price,
description: formatCollectedDescription(
(product ? stringAt(product, ['description']) : undefined) ??
scopedDescription(root, platform) ??
'',
) || undefined,
attributes,
categories: [...new Set(categoryNames)].map((name) => ({ key: `${platform}:category:${name.toLowerCase()}`.slice(0, 300), name })),
};
}
function scopedDescription(root: Document, platform: Exclude<SourcePlatform, 'ozon'>) {
const selectors = platform === '1688'
? ['.de-description-detail', '#detailContentContainer', '.html-description']
: ['[class*="tabDetailWrap--"]', '[class*="detailInfo--"]'];
for (const selector of selectors) {
const element = root.querySelector<HTMLElement>(selector);
const value = formatCollectedDescription(
element?.innerText ?? element?.textContent ?? '',
);
if (value && value.length >= 10) return value.slice(0, 100_000);
}
return undefined;
}
function jsonLdProduct(root: Document): Json | undefined {
for (const script of Array.from(root.scripts)) {
if (script.type !== 'application/ld+json' || !script.textContent?.trim()) continue;
try { const found = findJsonLdProduct(JSON.parse(script.textContent)); if (found) return found; } catch { /* ignored */ }
}
return undefined;
}
function findJsonLdProduct(value: unknown): Json | undefined {
if (Array.isArray(value)) { for (const child of value) { const found = findJsonLdProduct(child); if (found) return found; } return undefined; }
if (!isRecord(value)) return undefined;
const type = value['@type'];
if (type === 'Product' || (Array.isArray(type) && type.includes('Product'))) return value;
return Array.isArray(value['@graph']) ? findJsonLdProduct(value['@graph']) : undefined;
}
function decimal(value: unknown) { const raw = typeof value === 'string' || typeof value === 'number' ? String(value).replace(',', '.') : ''; return /^\d+(\.\d{1,4})?$/.test(raw) ? raw : undefined; }
function pageState(root: Document) {
const all: unknown[] = [];
for (const script of Array.from(root.scripts)) {
const text = script.textContent?.trim() ?? '';
if (text.length < 2 || text.length > 5_000_000) continue;
if (script.type === 'application/ld+json') {
try {
all.push(JSON.parse(text));
} catch {
/* ignored evidence */
}
}
for (const marker of [
'skuMapOriginal',
'skuBase',
'skuCore',
'skuProps',
]) {
const object = objectAfterMarker(text, marker);
if (object) all.push(object);
}
}
return all;
}
function objectAfterMarker(text: string, marker: string): unknown | null {
const position = text.indexOf(marker);
if (position < 0) return null;
const start = text.slice(position).search(/[\[{]/);
if (start < 0) return null;
const first = position + start;
const open = text[first];
const close = open === '{' ? '}' : ']';
let depth = 0;
let quote = '';
let escaped = false;
for (
let index = first;
index < text.length && index - first < 2_000_000;
index += 1
) {
const char = text[index];
if (quote) {
if (escaped) escaped = false;
else if (char === '\\') escaped = true;
else if (char === quote) quote = '';
continue;
}
if (char === '"' || char === "'") {
quote = char;
continue;
}
if (char === open) depth += 1;
if (char === close && --depth === 0) {
try {
return JSON.parse(text.slice(first, index + 1));
} catch {
return null;
}
}
}
return null;
}
function findFirst(states: unknown[], key: string): unknown {
for (const state of states) {
const found = findKey(state, key);
if (found !== undefined) return found;
}
return undefined;
}
function findKey(
value: unknown,
key: string,
seen = new Set<unknown>(),
): unknown {
if (!value || typeof value !== 'object' || seen.has(value))
return undefined;
seen.add(value);
if (isRecord(value) && key in value) return value[key];
for (const child of Array.isArray(value) ? value : Object.values(value)) {
const found = findKey(child, key, seen);
if (found !== undefined) return found;
}
return undefined;
}
function parse1688Axes(value: unknown): Axis[] {
const entries = Array.isArray(value)
? value
: isRecord(value) && Array.isArray(value.skuProps)
? value.skuProps
: [];
return entries.flatMap((entry, axisIndex) => {
if (!isRecord(entry)) return [];
const axisKey =
stringAt(entry, ['prop', 'propId', 'id']) ?? `axis:${axisIndex}`;
const name = stringAt(entry, ['name', 'propName']) ?? axisKey;
const values = (
Array.isArray(entry.value)
? entry.value
: Array.isArray(entry.values)
? entry.values
: []
).flatMap((item, valueIndex) =>
isRecord(item)
? [
{
key:
stringAt(item, ['valueId', 'vid', 'id']) ??
`${axisKey}:${valueIndex}`,
name:
stringAt(item, ['name', 'valueName']) ??
`${valueIndex + 1}`,
...(urlAt(item, [
'imageUrl',
'image',
'imageUrlLarge',
])
? {
image_url: urlAt(item, [
'imageUrl',
'image',
'imageUrlLarge',
])!,
}
: {}),
},
]
: [],
);
return values.length ? [{ key: axisKey, name, values }] : [];
});
}
function parseTaobaoAxes(value: unknown): Axis[] {
return (Array.isArray(value) ? value : []).flatMap((entry, index) => {
if (!isRecord(entry)) return [];
const key = stringAt(entry, ['pid', 'propId']) ?? `axis:${index}`;
const name = stringAt(entry, ['name', 'propName']) ?? key;
const values = (
Array.isArray(entry.values) ? entry.values : []
).flatMap((item, valueIndex) =>
isRecord(item)
? [
{
key:
stringAt(item, ['vid', 'valueId']) ??
`${key}:${valueIndex}`,
name:
stringAt(item, ['name', 'valueName']) ??
`${valueIndex + 1}`,
...(urlAt(item, ['image', 'imageUrl'])
? {
image_url: urlAt(item, [
'image',
'imageUrl',
])!,
}
: {}),
},
]
: [],
);
return values.length ? [{ key, name, values }] : [];
});
}
function selectionsForPropPath(path: string, axes: Axis[]) {
return path.split(/[;,]/).flatMap((pair) => {
const [axisKey, valueKey] = pair.split(':');
const axis = axes.find((item) => item.key === axisKey);
const value = axis?.values.find((item) => item.key === valueKey);
return axis && value
? [{ axisKey, valueKey, valueName: value.name }]
: [];
});
}
function selectionsForCombination(combination: string, axes: Axis[]) {
const ids = combination
.split(/[;,&]/)
.map((value) => value.trim())
.filter(Boolean);
return axes.flatMap((axis) => {
const value = axis.values.find((item) => ids.includes(item.key));
return value
? [
{
axisKey: axis.key,
valueKey: value.key,
valueName: value.name,
},
]
: [];
});
}
function stringAt(value: Json, keys: string[]) {
for (const key of keys)
if (typeof value[key] === 'string' || typeof value[key] === 'number')
return String(value[key]);
return undefined;
}
function moneyAt(value: Json, keys: string[]) {
const raw = stringAt(value, keys)?.replace(/,/g, '.');
return raw && /^\d+(\.\d{1,4})?$/.test(raw) ? raw : undefined;
}
function urlAt(value: Json, keys: string[]) {
const raw = stringAt(value, keys);
return raw && /^https:\/\//.test(raw) ? raw : undefined;
}
function uniqueVariants(variants: Variant[]) {
const seen = new Set<string>();
return variants.filter(
(variant) =>
!seen.has(variant.sourceVariantKey) &&
!!seen.add(variant.sourceVariantKey),
);
}
function offerId(url: string) {
return (
url.match(/\/offer\/(\d+)\.html/)?.[1] ??
`1688:${new URL(url).pathname}`
);
}
function itemId(url: string) {
return new URL(url).searchParams.get('id') ?? new URL(url).pathname;
}
function isRecord(value: unknown): value is Json {
return !!value && typeof value === 'object' && !Array.isArray(value);
}