R524809
455 lines
16 KiB
TypeScript
455 lines
16 KiB
TypeScript
import type { CollectedProductPayload } from '@atlas/contracts';
|
|
import {
|
|
collectOzonProduct,
|
|
isOzonProductPage,
|
|
type OzonPageJsonFetcher,
|
|
} from './ozon-parser.js';
|
|
import { collectPlatformDomAssets } from './product-media.js';
|
|
|
|
type SourcePlatform = '1688' | 'taobao' | 'tmall' | 'ozon';
|
|
type Json = Record<string, unknown>;
|
|
type Axis = CollectedProductPayload['variantAxes'][number];
|
|
type Variant = CollectedProductPayload['variants'][number];
|
|
|
|
export function supportedProductPlatform(
|
|
url = location.href,
|
|
): SourcePlatform | null {
|
|
if (isOzonProductPage(url)) return 'ozon';
|
|
if (/^https:\/\/detail\.1688\.com\/offer\/\d+\.html/.test(url))
|
|
return '1688';
|
|
if (/^https:\/\/item\.taobao\.com\/item\.htm/.test(url)) return 'taobao';
|
|
if (/^https:\/\/detail\.tmall\.com\/item\.htm/.test(url)) return 'tmall';
|
|
return null;
|
|
}
|
|
|
|
export async function collectCurrentProduct(
|
|
root: Document = document,
|
|
pageUrl = location.href,
|
|
fetchOzonPageJson?: OzonPageJsonFetcher,
|
|
): Promise<CollectedProductPayload> {
|
|
const platform = supportedProductPlatform(pageUrl);
|
|
if (platform === 'ozon')
|
|
return collectOzonProduct(root, pageUrl, fetchOzonPageJson);
|
|
if (!platform) throw new Error('当前页面不是支持的商品详情页');
|
|
return platform === '1688'
|
|
? collect1688(root, pageUrl)
|
|
: collectTaobao(root, pageUrl, platform);
|
|
}
|
|
|
|
function collect1688(root: Document, pageUrl: string): CollectedProductPayload {
|
|
const state = pageState(root);
|
|
const skuMap = findFirst(state, 'skuMapOriginal');
|
|
const skuProps = findFirst(state, 'skuProps');
|
|
const axes = parse1688Axes(skuProps);
|
|
const evidence: CollectedProductPayload['evidence'] = [
|
|
{
|
|
ref: 'page:url',
|
|
kind: 'page',
|
|
path: 'location.href',
|
|
raw_value: pageUrl,
|
|
},
|
|
];
|
|
const variants: Variant[] = [];
|
|
if (isRecord(skuMap)) {
|
|
evidence.push({
|
|
ref: '1688:sku-map',
|
|
kind: 'state',
|
|
path: 'skuMapOriginal',
|
|
raw_value: `组合记录 ${Object.keys(skuMap).length} 条`,
|
|
});
|
|
for (const [combination, row] of Object.entries(skuMap)) {
|
|
if (!isRecord(row)) continue;
|
|
const sourceSku = stringAt(row, ['skuId', 'sku_id', 'skuCode']);
|
|
// A skuMapOriginal entry itself is a source-provided sellable-combination
|
|
// record. Never expand it with the Cartesian product of skuProps.
|
|
const sourceVariantKey = sourceSku ?? `1688:${combination}`;
|
|
if (!combination || combination === 'undefined') continue;
|
|
variants.push({
|
|
sourceVariantKey,
|
|
...(sourceSku ? { sourceSku } : {}),
|
|
displayName:
|
|
selectionsForCombination(combination, axes)
|
|
.map((item) => item.valueName)
|
|
.join(' / ') || `1688 SKU ${sourceSku ?? combination}`,
|
|
selections: selectionsForCombination(combination, axes),
|
|
...(moneyAt(row, ['discountPrice', 'price', 'salePrice'])
|
|
? {
|
|
sourcePrice: {
|
|
amount: moneyAt(row, [
|
|
'discountPrice',
|
|
'price',
|
|
'salePrice',
|
|
])!,
|
|
currencyCode: 'CNY',
|
|
},
|
|
}
|
|
: {}),
|
|
confidence: 'verified',
|
|
evidenceRefs: ['1688:sku-map'],
|
|
});
|
|
}
|
|
}
|
|
return basePayload(
|
|
'1688',
|
|
offerId(pageUrl),
|
|
root,
|
|
pageUrl,
|
|
axes,
|
|
uniqueVariants(variants),
|
|
evidence,
|
|
'1688-state-1.0.0',
|
|
);
|
|
}
|
|
|
|
function collectTaobao(
|
|
root: Document,
|
|
pageUrl: string,
|
|
platform: 'taobao' | 'tmall',
|
|
): CollectedProductPayload {
|
|
const state = pageState(root);
|
|
const skuBase = findFirst(state, 'skuBase');
|
|
const skuCore = findFirst(state, 'skuCore');
|
|
const axes = parseTaobaoAxes(isRecord(skuBase) ? skuBase.props : undefined);
|
|
const evidence: CollectedProductPayload['evidence'] = [
|
|
{
|
|
ref: 'page:url',
|
|
kind: 'page',
|
|
path: 'location.href',
|
|
raw_value: pageUrl,
|
|
},
|
|
];
|
|
const variants: Variant[] = [];
|
|
const skus =
|
|
isRecord(skuBase) && Array.isArray(skuBase.skus) ? skuBase.skus : [];
|
|
const sku2info =
|
|
isRecord(skuCore) && isRecord(skuCore.sku2info) ? skuCore.sku2info : {};
|
|
if (skus.length)
|
|
evidence.push({
|
|
ref: 'taobao:sku-base',
|
|
kind: 'state',
|
|
path: 'skuBase.skus',
|
|
raw_value: `组合记录 ${skus.length} 条`,
|
|
});
|
|
for (const sku of skus) {
|
|
if (!isRecord(sku)) continue;
|
|
const skuId = stringAt(sku, ['skuId', 'sku_id']);
|
|
const propPath = stringAt(sku, ['propPath', 'prop_path']);
|
|
// skuId + propPath is the required combination proof. Props alone only
|
|
// become axes and cause a single default SKU on the server.
|
|
if (!skuId || !propPath) continue;
|
|
const info = isRecord(sku2info[skuId]) ? sku2info[skuId] : {};
|
|
const selections = selectionsForPropPath(propPath, axes);
|
|
variants.push({
|
|
sourceVariantKey: skuId,
|
|
sourceSku: skuId,
|
|
displayName:
|
|
selections.map((item) => item.valueName).join(' / ') ||
|
|
`SKU ${skuId}`,
|
|
selections,
|
|
...(moneyAt(info, ['price', 'promotionPrice', 'discountPrice'])
|
|
? {
|
|
sourcePrice: {
|
|
amount: moneyAt(info, [
|
|
'price',
|
|
'promotionPrice',
|
|
'discountPrice',
|
|
])!,
|
|
currencyCode: 'CNY',
|
|
},
|
|
}
|
|
: {}),
|
|
confidence: 'verified',
|
|
evidenceRefs: ['taobao:sku-base'],
|
|
});
|
|
}
|
|
return basePayload(
|
|
platform,
|
|
itemId(pageUrl),
|
|
root,
|
|
pageUrl,
|
|
axes,
|
|
uniqueVariants(variants),
|
|
evidence,
|
|
`${platform}-state-1.0.0`,
|
|
);
|
|
}
|
|
|
|
function basePayload(
|
|
platform: Exclude<SourcePlatform, 'ozon'>,
|
|
itemKey: string,
|
|
root: Document,
|
|
pageUrl: string,
|
|
axes: Axis[],
|
|
variants: Variant[],
|
|
evidence: CollectedProductPayload['evidence'],
|
|
parserVersion: string,
|
|
): CollectedProductPayload {
|
|
const canonical =
|
|
root.querySelector<HTMLLinkElement>('link[rel="canonical"]')?.href ??
|
|
pageUrl.split('#')[0];
|
|
const title =
|
|
root.querySelector('h1')?.textContent?.replace(/\s+/g, ' ').trim() ||
|
|
root
|
|
.querySelector<HTMLMetaElement>('meta[property="og:title"]')
|
|
?.content?.trim();
|
|
const assets = collectPlatformDomAssets(platform, root, canonical);
|
|
evidence.push({
|
|
ref: `${platform}:dom:image-profile`,
|
|
kind: 'dom',
|
|
path: 'platform-scoped gallery / sku / detail selectors',
|
|
raw_value: `采集 ${assets.length} 张受限商品区域图片`,
|
|
});
|
|
return {
|
|
source: {
|
|
platform,
|
|
itemKey,
|
|
canonicalUrl: canonical,
|
|
collectedAt: new Date().toISOString(),
|
|
parserVersion,
|
|
},
|
|
common: {
|
|
...(title
|
|
? {
|
|
title: {
|
|
value: title,
|
|
raw_value: title,
|
|
confidence: 'verified' as const,
|
|
},
|
|
}
|
|
: {}),
|
|
attributes: [],
|
|
categories: [],
|
|
},
|
|
variantAxes: axes,
|
|
variants,
|
|
assets,
|
|
evidence,
|
|
};
|
|
}
|
|
|
|
function pageState(root: Document) {
|
|
const all: unknown[] = [];
|
|
for (const script of Array.from(root.scripts)) {
|
|
const text = script.textContent?.trim() ?? '';
|
|
if (text.length < 2 || text.length > 5_000_000) continue;
|
|
if (script.type === 'application/ld+json') {
|
|
try {
|
|
all.push(JSON.parse(text));
|
|
} catch {
|
|
/* ignored evidence */
|
|
}
|
|
}
|
|
for (const marker of [
|
|
'skuMapOriginal',
|
|
'skuBase',
|
|
'skuCore',
|
|
'skuProps',
|
|
]) {
|
|
const object = objectAfterMarker(text, marker);
|
|
if (object) all.push(object);
|
|
}
|
|
}
|
|
return all;
|
|
}
|
|
function objectAfterMarker(text: string, marker: string): unknown | null {
|
|
const position = text.indexOf(marker);
|
|
if (position < 0) return null;
|
|
const start = text.slice(position).search(/[\[{]/);
|
|
if (start < 0) return null;
|
|
const first = position + start;
|
|
const open = text[first];
|
|
const close = open === '{' ? '}' : ']';
|
|
let depth = 0;
|
|
let quote = '';
|
|
let escaped = false;
|
|
for (
|
|
let index = first;
|
|
index < text.length && index - first < 2_000_000;
|
|
index += 1
|
|
) {
|
|
const char = text[index];
|
|
if (quote) {
|
|
if (escaped) escaped = false;
|
|
else if (char === '\\') escaped = true;
|
|
else if (char === quote) quote = '';
|
|
continue;
|
|
}
|
|
if (char === '"' || char === "'") {
|
|
quote = char;
|
|
continue;
|
|
}
|
|
if (char === open) depth += 1;
|
|
if (char === close && --depth === 0) {
|
|
try {
|
|
return JSON.parse(text.slice(first, index + 1));
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
}
|
|
return null;
|
|
}
|
|
function findFirst(states: unknown[], key: string): unknown {
|
|
for (const state of states) {
|
|
const found = findKey(state, key);
|
|
if (found !== undefined) return found;
|
|
}
|
|
return undefined;
|
|
}
|
|
function findKey(
|
|
value: unknown,
|
|
key: string,
|
|
seen = new Set<unknown>(),
|
|
): unknown {
|
|
if (!value || typeof value !== 'object' || seen.has(value))
|
|
return undefined;
|
|
seen.add(value);
|
|
if (isRecord(value) && key in value) return value[key];
|
|
for (const child of Array.isArray(value) ? value : Object.values(value)) {
|
|
const found = findKey(child, key, seen);
|
|
if (found !== undefined) return found;
|
|
}
|
|
return undefined;
|
|
}
|
|
function parse1688Axes(value: unknown): Axis[] {
|
|
const entries = Array.isArray(value)
|
|
? value
|
|
: isRecord(value) && Array.isArray(value.skuProps)
|
|
? value.skuProps
|
|
: [];
|
|
return entries.flatMap((entry, axisIndex) => {
|
|
if (!isRecord(entry)) return [];
|
|
const axisKey =
|
|
stringAt(entry, ['prop', 'propId', 'id']) ?? `axis:${axisIndex}`;
|
|
const name = stringAt(entry, ['name', 'propName']) ?? axisKey;
|
|
const values = (
|
|
Array.isArray(entry.value)
|
|
? entry.value
|
|
: Array.isArray(entry.values)
|
|
? entry.values
|
|
: []
|
|
).flatMap((item, valueIndex) =>
|
|
isRecord(item)
|
|
? [
|
|
{
|
|
key:
|
|
stringAt(item, ['valueId', 'vid', 'id']) ??
|
|
`${axisKey}:${valueIndex}`,
|
|
name:
|
|
stringAt(item, ['name', 'valueName']) ??
|
|
`${valueIndex + 1}`,
|
|
...(urlAt(item, [
|
|
'imageUrl',
|
|
'image',
|
|
'imageUrlLarge',
|
|
])
|
|
? {
|
|
image_url: urlAt(item, [
|
|
'imageUrl',
|
|
'image',
|
|
'imageUrlLarge',
|
|
])!,
|
|
}
|
|
: {}),
|
|
},
|
|
]
|
|
: [],
|
|
);
|
|
return values.length ? [{ key: axisKey, name, values }] : [];
|
|
});
|
|
}
|
|
function parseTaobaoAxes(value: unknown): Axis[] {
|
|
return (Array.isArray(value) ? value : []).flatMap((entry, index) => {
|
|
if (!isRecord(entry)) return [];
|
|
const key = stringAt(entry, ['pid', 'propId']) ?? `axis:${index}`;
|
|
const name = stringAt(entry, ['name', 'propName']) ?? key;
|
|
const values = (
|
|
Array.isArray(entry.values) ? entry.values : []
|
|
).flatMap((item, valueIndex) =>
|
|
isRecord(item)
|
|
? [
|
|
{
|
|
key:
|
|
stringAt(item, ['vid', 'valueId']) ??
|
|
`${key}:${valueIndex}`,
|
|
name:
|
|
stringAt(item, ['name', 'valueName']) ??
|
|
`${valueIndex + 1}`,
|
|
...(urlAt(item, ['image', 'imageUrl'])
|
|
? {
|
|
image_url: urlAt(item, [
|
|
'image',
|
|
'imageUrl',
|
|
])!,
|
|
}
|
|
: {}),
|
|
},
|
|
]
|
|
: [],
|
|
);
|
|
return values.length ? [{ key, name, values }] : [];
|
|
});
|
|
}
|
|
function selectionsForPropPath(path: string, axes: Axis[]) {
|
|
return path.split(/[;,]/).flatMap((pair) => {
|
|
const [axisKey, valueKey] = pair.split(':');
|
|
const axis = axes.find((item) => item.key === axisKey);
|
|
const value = axis?.values.find((item) => item.key === valueKey);
|
|
return axis && value
|
|
? [{ axisKey, valueKey, valueName: value.name }]
|
|
: [];
|
|
});
|
|
}
|
|
function selectionsForCombination(combination: string, axes: Axis[]) {
|
|
const ids = combination
|
|
.split(/[;,&]/)
|
|
.map((value) => value.trim())
|
|
.filter(Boolean);
|
|
return axes.flatMap((axis) => {
|
|
const value = axis.values.find((item) => ids.includes(item.key));
|
|
return value
|
|
? [
|
|
{
|
|
axisKey: axis.key,
|
|
valueKey: value.key,
|
|
valueName: value.name,
|
|
},
|
|
]
|
|
: [];
|
|
});
|
|
}
|
|
function stringAt(value: Json, keys: string[]) {
|
|
for (const key of keys)
|
|
if (typeof value[key] === 'string' || typeof value[key] === 'number')
|
|
return String(value[key]);
|
|
return undefined;
|
|
}
|
|
function moneyAt(value: Json, keys: string[]) {
|
|
const raw = stringAt(value, keys)?.replace(/,/g, '.');
|
|
return raw && /^\d+(\.\d{1,4})?$/.test(raw) ? raw : undefined;
|
|
}
|
|
function urlAt(value: Json, keys: string[]) {
|
|
const raw = stringAt(value, keys);
|
|
return raw && /^https:\/\//.test(raw) ? raw : undefined;
|
|
}
|
|
function uniqueVariants(variants: Variant[]) {
|
|
const seen = new Set<string>();
|
|
return variants.filter(
|
|
(variant) =>
|
|
!seen.has(variant.sourceVariantKey) &&
|
|
!!seen.add(variant.sourceVariantKey),
|
|
);
|
|
}
|
|
function offerId(url: string) {
|
|
return (
|
|
url.match(/\/offer\/(\d+)\.html/)?.[1] ??
|
|
`1688:${new URL(url).pathname}`
|
|
);
|
|
}
|
|
function itemId(url: string) {
|
|
return new URL(url).searchParams.get('id') ?? new URL(url).pathname;
|
|
}
|
|
function isRecord(value: unknown): value is Json {
|
|
return !!value && typeof value === 'object' && !Array.isArray(value);
|
|
}
|