feat(extension): 采集插件功能开发
This commit is contained in:
@@ -0,0 +1,83 @@
|
||||
import type { CollectedProductPayload } from '@atlas/contracts';
|
||||
import { collectOzonProduct, isOzonProductPage } from './ozon-parser.js';
|
||||
|
||||
type SourcePlatform = '1688' | 'taobao' | 'tmall' | 'ozon';
|
||||
type Json = Record<string, unknown>;
|
||||
type Axis = CollectedProductPayload['variantAxes'][number];
|
||||
type Variant = CollectedProductPayload['variants'][number];
|
||||
|
||||
export function supportedProductPlatform(url = location.href): SourcePlatform | null {
|
||||
if (isOzonProductPage(url)) return 'ozon';
|
||||
if (/^https:\/\/detail\.1688\.com\/offer\/\d+\.html/.test(url)) return '1688';
|
||||
if (/^https:\/\/item\.taobao\.com\/item\.htm/.test(url)) return 'taobao';
|
||||
if (/^https:\/\/detail\.tmall\.com\/item\.htm/.test(url)) return 'tmall';
|
||||
return null;
|
||||
}
|
||||
|
||||
export function collectCurrentProduct(root: Document = document, pageUrl = location.href): CollectedProductPayload {
|
||||
const platform = supportedProductPlatform(pageUrl);
|
||||
if (platform === 'ozon') return collectOzonProduct(root, pageUrl);
|
||||
if (!platform) throw new Error('当前页面不是支持的商品详情页');
|
||||
return platform === '1688' ? collect1688(root, pageUrl) : collectTaobao(root, pageUrl, platform);
|
||||
}
|
||||
|
||||
function collect1688(root: Document, pageUrl: string): CollectedProductPayload {
|
||||
const state = pageState(root); const skuMap = findFirst(state, 'skuMapOriginal'); const skuProps = findFirst(state, 'skuProps');
|
||||
const axes = parse1688Axes(skuProps); const evidence: CollectedProductPayload['evidence'] = [{ ref: 'page:url', kind: 'page', path: 'location.href', raw_value: pageUrl }];
|
||||
const variants: Variant[] = [];
|
||||
if (isRecord(skuMap)) {
|
||||
evidence.push({ ref: '1688:sku-map', kind: 'state', path: 'skuMapOriginal', raw_value: `组合记录 ${Object.keys(skuMap).length} 条` });
|
||||
for (const [combination, row] of Object.entries(skuMap)) {
|
||||
if (!isRecord(row)) continue;
|
||||
const sourceSku = stringAt(row, ['skuId', 'sku_id', 'skuCode']);
|
||||
// A skuMapOriginal entry itself is a source-provided sellable-combination
|
||||
// record. Never expand it with the Cartesian product of skuProps.
|
||||
const sourceVariantKey = sourceSku ?? `1688:${combination}`;
|
||||
if (!combination || combination === 'undefined') continue;
|
||||
variants.push({ sourceVariantKey, ...(sourceSku ? { sourceSku } : {}), displayName: selectionsForCombination(combination, axes).map((item) => item.valueName).join(' / ') || `1688 SKU ${sourceSku ?? combination}`, selections: selectionsForCombination(combination, axes), ...(moneyAt(row, ['discountPrice', 'price', 'salePrice']) ? { sourcePrice: { amount: moneyAt(row, ['discountPrice', 'price', 'salePrice'])!, currencyCode: 'CNY' } } : {}), confidence: 'verified', evidenceRefs: ['1688:sku-map'] });
|
||||
}
|
||||
}
|
||||
return basePayload('1688', offerId(pageUrl), root, pageUrl, axes, uniqueVariants(variants), evidence, '1688-state-1.0.0');
|
||||
}
|
||||
|
||||
function collectTaobao(root: Document, pageUrl: string, platform: 'taobao' | 'tmall'): CollectedProductPayload {
|
||||
const state = pageState(root); const skuBase = findFirst(state, 'skuBase'); const skuCore = findFirst(state, 'skuCore');
|
||||
const axes = parseTaobaoAxes(isRecord(skuBase) ? skuBase.props : undefined); const evidence: CollectedProductPayload['evidence'] = [{ ref: 'page:url', kind: 'page', path: 'location.href', raw_value: pageUrl }];
|
||||
const variants: Variant[] = [];
|
||||
const skus = isRecord(skuBase) && Array.isArray(skuBase.skus) ? skuBase.skus : [];
|
||||
const sku2info = isRecord(skuCore) && isRecord(skuCore.sku2info) ? skuCore.sku2info : {};
|
||||
if (skus.length) evidence.push({ ref: 'taobao:sku-base', kind: 'state', path: 'skuBase.skus', raw_value: `组合记录 ${skus.length} 条` });
|
||||
for (const sku of skus) {
|
||||
if (!isRecord(sku)) continue;
|
||||
const skuId = stringAt(sku, ['skuId', 'sku_id']); const propPath = stringAt(sku, ['propPath', 'prop_path']);
|
||||
// skuId + propPath is the required combination proof. Props alone only
|
||||
// become axes and cause a single default SKU on the server.
|
||||
if (!skuId || !propPath) continue;
|
||||
const info = isRecord(sku2info[skuId]) ? sku2info[skuId] : {};
|
||||
const selections = selectionsForPropPath(propPath, axes);
|
||||
variants.push({ sourceVariantKey: skuId, sourceSku: skuId, displayName: selections.map((item) => item.valueName).join(' / ') || `SKU ${skuId}`, selections, ...(moneyAt(info, ['price', 'promotionPrice', 'discountPrice']) ? { sourcePrice: { amount: moneyAt(info, ['price', 'promotionPrice', 'discountPrice'])!, currencyCode: 'CNY' } } : {}), confidence: 'verified', evidenceRefs: ['taobao:sku-base'] });
|
||||
}
|
||||
return basePayload(platform, itemId(pageUrl), root, pageUrl, axes, uniqueVariants(variants), evidence, `${platform}-state-1.0.0`);
|
||||
}
|
||||
|
||||
function basePayload(platform: Exclude<SourcePlatform, 'ozon'>, itemKey: string, root: Document, pageUrl: string, axes: Axis[], variants: Variant[], evidence: CollectedProductPayload['evidence'], parserVersion: string): CollectedProductPayload {
|
||||
const canonical = root.querySelector<HTMLLinkElement>('link[rel="canonical"]')?.href ?? pageUrl.split('#')[0]; const title = root.querySelector('h1')?.textContent?.replace(/\s+/g, ' ').trim() || root.querySelector<HTMLMetaElement>('meta[property="og:title"]')?.content?.trim();
|
||||
const assets = Array.from(root.images).map((image) => image.currentSrc || image.src).filter((url) => /^https:\/\//.test(url)).filter((url, index, all) => all.indexOf(url) === index).slice(0, 40).map((url, sortOrder) => ({ url, role: 'source_main' as const, relation: 'product_candidate' as const, sortOrder, evidenceRef: 'page:gallery' }));
|
||||
return { source: { platform, itemKey, canonicalUrl: canonical, collectedAt: new Date().toISOString(), parserVersion }, common: { ...(title ? { title: { value: title, raw_value: title, confidence: 'verified' as const } } : {}), attributes: [], categories: [] }, variantAxes: axes, variants, assets, evidence };
|
||||
}
|
||||
|
||||
function pageState(root: Document) { const all: unknown[] = []; for (const script of Array.from(root.scripts)) { const text = script.textContent?.trim() ?? ''; if (text.length < 2 || text.length > 5_000_000) continue; if (script.type === 'application/ld+json') { try { all.push(JSON.parse(text)); } catch { /* ignored evidence */ } } for (const marker of ['skuMapOriginal', 'skuBase', 'skuCore', 'skuProps']) { const object = objectAfterMarker(text, marker); if (object) all.push(object); } } return all; }
|
||||
function objectAfterMarker(text: string, marker: string): unknown | null { const position = text.indexOf(marker); if (position < 0) return null; const start = text.slice(position).search(/[\[{]/); if (start < 0) return null; const first = position + start; const open = text[first]; const close = open === '{' ? '}' : ']'; let depth = 0; let quote = ''; let escaped = false; for (let index = first; index < text.length && index - first < 2_000_000; index += 1) { const char = text[index]; if (quote) { if (escaped) escaped = false; else if (char === '\\') escaped = true; else if (char === quote) quote = ''; continue; } if (char === '"' || char === "'") { quote = char; continue; } if (char === open) depth += 1; if (char === close && --depth === 0) { try { return JSON.parse(text.slice(first, index + 1)); } catch { return null; } } } return null; }
|
||||
function findFirst(states: unknown[], key: string): unknown { for (const state of states) { const found = findKey(state, key); if (found !== undefined) return found; } return undefined; }
|
||||
function findKey(value: unknown, key: string, seen = new Set<unknown>()): unknown { if (!value || typeof value !== 'object' || seen.has(value)) return undefined; seen.add(value); if (isRecord(value) && key in value) return value[key]; for (const child of Array.isArray(value) ? value : Object.values(value)) { const found = findKey(child, key, seen); if (found !== undefined) return found; } return undefined; }
|
||||
function parse1688Axes(value: unknown): Axis[] { const entries = Array.isArray(value) ? value : isRecord(value) && Array.isArray(value.skuProps) ? value.skuProps : []; return entries.flatMap((entry, axisIndex) => { if (!isRecord(entry)) return []; const axisKey = stringAt(entry, ['prop', 'propId', 'id']) ?? `axis:${axisIndex}`; const name = stringAt(entry, ['name', 'propName']) ?? axisKey; const values = (Array.isArray(entry.value) ? entry.value : Array.isArray(entry.values) ? entry.values : []).flatMap((item, valueIndex) => isRecord(item) ? [{ key: stringAt(item, ['valueId', 'vid', 'id']) ?? `${axisKey}:${valueIndex}`, name: stringAt(item, ['name', 'valueName']) ?? `${valueIndex + 1}`, ...(urlAt(item, ['imageUrl', 'image', 'imageUrlLarge']) ? { image_url: urlAt(item, ['imageUrl', 'image', 'imageUrlLarge'])! } : {}) }] : []); return values.length ? [{ key: axisKey, name, values }] : []; }); }
|
||||
function parseTaobaoAxes(value: unknown): Axis[] { return (Array.isArray(value) ? value : []).flatMap((entry, index) => { if (!isRecord(entry)) return []; const key = stringAt(entry, ['pid', 'propId']) ?? `axis:${index}`; const name = stringAt(entry, ['name', 'propName']) ?? key; const values = (Array.isArray(entry.values) ? entry.values : []).flatMap((item, valueIndex) => isRecord(item) ? [{ key: stringAt(item, ['vid', 'valueId']) ?? `${key}:${valueIndex}`, name: stringAt(item, ['name', 'valueName']) ?? `${valueIndex + 1}`, ...(urlAt(item, ['image', 'imageUrl']) ? { image_url: urlAt(item, ['image', 'imageUrl'])! } : {}) }] : []); return values.length ? [{ key, name, values }] : []; }); }
|
||||
function selectionsForPropPath(path: string, axes: Axis[]) { return path.split(/[;,]/).flatMap((pair) => { const [axisKey, valueKey] = pair.split(':'); const axis = axes.find((item) => item.key === axisKey); const value = axis?.values.find((item) => item.key === valueKey); return axis && value ? [{ axisKey, valueKey, valueName: value.name }] : []; }); }
|
||||
function selectionsForCombination(combination: string, axes: Axis[]) { const ids = combination.split(/[;,&]/).map((value) => value.trim()).filter(Boolean); return axes.flatMap((axis) => { const value = axis.values.find((item) => ids.includes(item.key)); return value ? [{ axisKey: axis.key, valueKey: value.key, valueName: value.name }] : []; }); }
|
||||
function stringAt(value: Json, keys: string[]) { for (const key of keys) if (typeof value[key] === 'string' || typeof value[key] === 'number') return String(value[key]); return undefined; }
|
||||
function moneyAt(value: Json, keys: string[]) { const raw = stringAt(value, keys)?.replace(/,/g, '.'); return raw && /^\d+(\.\d{1,4})?$/.test(raw) ? raw : undefined; }
|
||||
function urlAt(value: Json, keys: string[]) { const raw = stringAt(value, keys); return raw && /^https:\/\//.test(raw) ? raw : undefined; }
|
||||
function uniqueVariants(variants: Variant[]) { const seen = new Set<string>(); return variants.filter((variant) => !seen.has(variant.sourceVariantKey) && !!seen.add(variant.sourceVariantKey)); }
|
||||
function offerId(url: string) { return url.match(/\/offer\/(\d+)\.html/)?.[1] ?? `1688:${new URL(url).pathname}`; }
|
||||
function itemId(url: string) { return new URL(url).searchParams.get('id') ?? new URL(url).pathname; }
|
||||
function isRecord(value: unknown): value is Json { return !!value && typeof value === 'object' && !Array.isArray(value); }
|
||||
Reference in New Issue
Block a user