391 lines
14 KiB
TypeScript
391 lines
14 KiB
TypeScript
import type { CollectedProductPayload } from '@atlas/contracts';
|
|
|
|
type SupportedMediaPlatform = 'ozon' | '1688' | 'taobao' | 'tmall';
|
|
type DomMediaPlatform = Exclude<SupportedMediaPlatform, 'ozon'>;
|
|
type Asset = CollectedProductPayload['assets'][number];
|
|
|
|
type ImageGroup = {
|
|
role: Asset['role'];
|
|
relation: Asset['relation'];
|
|
selectors: string[];
|
|
sourceProps?: Array<
|
|
| 'data-lazyload-src'
|
|
| 'data-src'
|
|
| 'srcset'
|
|
| 'currentSrc'
|
|
| 'src'
|
|
| 'backgroundImage'
|
|
>;
|
|
excludeWithin?: string[];
|
|
minWidth?: number;
|
|
minHeight?: number;
|
|
};
|
|
|
|
/**
|
|
* These profiles are ported from the proven collector extension. The crucial
|
|
* invariant is that images always come from a product widget, never from a
|
|
* page-wide `document.images` scan (which also includes logos, ads and
|
|
* recommendation cards).
|
|
*/
|
|
const DOM_IMAGE_PROFILES: Record<DomMediaPlatform, ImageGroup[]> = {
|
|
'1688': [
|
|
{
|
|
role: 'source_main',
|
|
relation: 'inherited_product_default',
|
|
// 画廊缩略项虽小，但背景地址指向商品原图；位置与控件类型比尺寸更可靠。
|
|
selectors: ['.od-picture-gallery-list > .v-image-cover'],
|
|
sourceProps: ['backgroundImage'],
|
|
excludeWithin: ['.video-image-cover', '.v-image-cover:has(.cpv-icon, .cpv-tag-name)'],
|
|
},
|
|
{
|
|
role: 'source_main',
|
|
relation: 'inherited_product_default',
|
|
selectors: [
|
|
'[data-module="od_picture_gallery"] img',
|
|
'.module-od-picture-gallery img',
|
|
'#recyclerview .detail-gallery-turn-wrapper .detail-gallery-img',
|
|
'#screen .od-gallery-turn-item-wrapper .od-gallery-img',
|
|
'#content .od-scroller-item .v-image-cover',
|
|
'#content .od-picture-gallery-list .v-image-cover',
|
|
'#dt-tab img',
|
|
'.detail-gallery-turn img.detail-gallery-img',
|
|
'.img-list-wrapper img.od-gallery-img',
|
|
],
|
|
excludeWithin: ['.video-image-cover', '.v-image-cover:has(.cpv-icon, .cpv-tag-name)'],
|
|
minWidth: 200,
|
|
minHeight: 200,
|
|
},
|
|
{
|
|
role: 'source_sku',
|
|
relation: 'axis_value',
|
|
selectors: [
|
|
'[data-module="od_sku_selection"] img',
|
|
'.module-od-sku-selection img',
|
|
'.pc-sku-wrapper .prop-item-inner-wrapper',
|
|
'.sku-item-wrapper',
|
|
'.specification-cell',
|
|
'.sku-filter-button',
|
|
'.expand-view-item',
|
|
'.feature-item img',
|
|
],
|
|
sourceProps: ['backgroundImage'],
|
|
minWidth: 20,
|
|
minHeight: 20,
|
|
},
|
|
{
|
|
role: 'source_detail',
|
|
relation: 'product_candidate',
|
|
selectors: [
|
|
'#detail img',
|
|
'.de-description-detail img',
|
|
'#detailContentContainer img',
|
|
'.html-description img',
|
|
],
|
|
minWidth: 300,
|
|
minHeight: 100,
|
|
},
|
|
],
|
|
taobao: [
|
|
{
|
|
role: 'source_main',
|
|
relation: 'inherited_product_default',
|
|
selectors: [
|
|
'[class*="picGallery--"] img',
|
|
'#picGalleryEle img',
|
|
'[class*="thumbnailPic--"]',
|
|
],
|
|
},
|
|
{
|
|
role: 'source_sku',
|
|
relation: 'axis_value',
|
|
selectors: [
|
|
'[class*="valueItem--"]',
|
|
'[class*="valueItemImgWrap--"]',
|
|
],
|
|
minWidth: 20,
|
|
minHeight: 20,
|
|
},
|
|
taobaoDetailProfile(),
|
|
],
|
|
tmall: [
|
|
{
|
|
role: 'source_main',
|
|
relation: 'inherited_product_default',
|
|
selectors: [
|
|
'[class*="picGallery--"] img',
|
|
'#picGalleryEle img',
|
|
'[class*="thumbnailPic--"]',
|
|
],
|
|
},
|
|
{
|
|
role: 'source_sku',
|
|
relation: 'axis_value',
|
|
selectors: [
|
|
'[class*="valueItem--"]',
|
|
'[class*="valueItemImgWrap--"]',
|
|
],
|
|
minWidth: 20,
|
|
minHeight: 20,
|
|
},
|
|
taobaoDetailProfile(),
|
|
],
|
|
};
|
|
|
|
function taobaoDetailProfile(): ImageGroup {
|
|
return {
|
|
role: 'source_detail',
|
|
relation: 'product_candidate',
|
|
selectors: [
|
|
'[class*="tabDetailWrap--"] img',
|
|
'[class*="detailInfo--"] img',
|
|
],
|
|
// The old collector verified that review photos and recommendation
|
|
// cards can be large enough to pass a size-only filter.
|
|
excludeWithin: [
|
|
'[class*="Comment--"]',
|
|
'[class*="comments--"]',
|
|
'[class*="userInfo--"]',
|
|
'[class*="rate"]',
|
|
'[class*="RecommendInfo--"]',
|
|
'[data-spm="recommends"]',
|
|
'[class*="recommend--"]',
|
|
'[class*="cardPic--"]',
|
|
],
|
|
minWidth: 300,
|
|
minHeight: 100,
|
|
};
|
|
}
|
|
|
|
export function collectPlatformDomAssets(
|
|
platform: DomMediaPlatform,
|
|
root: Document,
|
|
pageUrl: string,
|
|
): Asset[] {
|
|
const assets: Asset[] = [];
|
|
const seen = new Set<string>();
|
|
for (const group of DOM_IMAGE_PROFILES[platform]) {
|
|
for (const selector of group.selectors) {
|
|
for (const element of Array.from(
|
|
root.querySelectorAll<HTMLElement>(selector),
|
|
)) {
|
|
if (
|
|
group.excludeWithin?.some((excluded) =>
|
|
element.closest(excluded),
|
|
)
|
|
)
|
|
continue;
|
|
if (!passesDimensions(element, group)) continue;
|
|
const url = imageUrlFromElement(
|
|
element,
|
|
group.sourceProps,
|
|
pageUrl,
|
|
platform,
|
|
);
|
|
if (!url) continue;
|
|
// Keep the old collector's group-level de-duplication: an
|
|
// aspect cover may legitimately also be a gallery image.
|
|
const key = `${group.role}:${dedupeImageUrl(url, platform)}`;
|
|
if (seen.has(key)) continue;
|
|
seen.add(key);
|
|
assets.push({
|
|
url,
|
|
mediaType: 'image',
|
|
role: group.role,
|
|
relation: group.relation,
|
|
sortOrder: assets.length,
|
|
evidenceRef: `${platform}:dom:${group.role}`,
|
|
});
|
|
}
|
|
}
|
|
}
|
|
if (platform === '1688') add1688GalleryVideos(root, pageUrl, assets);
|
|
return assets;
|
|
}
|
|
|
|
/** 只读取商品画廊内播放器的公开直链，不扫描全页视频或根据封面猜测地址。 */
|
|
function add1688GalleryVideos(root: Document, pageUrl: string, assets: Asset[]) {
|
|
const selectors = [
|
|
'[data-module="od_picture_gallery"] video',
|
|
'.module-od-picture-gallery video',
|
|
'.od-picture-gallery video',
|
|
'.od-picture-gallery-list video',
|
|
'.detail-gallery-turn video',
|
|
'.od-gallery-turn-item-wrapper video',
|
|
];
|
|
const seen = new Set<string>();
|
|
for (const selector of selectors) {
|
|
for (const video of Array.from(root.querySelectorAll<HTMLVideoElement>(selector))) {
|
|
const candidates = [video.currentSrc, video.getAttribute('src'),
|
|
...Array.from(video.querySelectorAll('source')).map((source) => source.getAttribute('src'))];
|
|
const url = candidates.map((candidate) => normaliseVideoUrl(candidate ?? '', pageUrl)).find(Boolean);
|
|
if (!url || seen.has(url) || assets.length >= 500) continue;
|
|
seen.add(url);
|
|
const cover = root.querySelector<HTMLElement>('.od-picture-gallery-list > .video-image-cover');
|
|
const poster = video.poster || video.getAttribute('poster');
|
|
const posterUrl = (poster ? normaliseImageUrl(poster, pageUrl, '1688') : undefined) ??
|
|
(cover ? imageUrlFromElement(cover, ['backgroundImage'], pageUrl, '1688') : undefined);
|
|
assets.push({
|
|
url, mediaType: 'video', role: 'source_main', relation: 'product_candidate',
|
|
...(posterUrl?.startsWith('https://') ? { posterUrl } : {}),
|
|
sortOrder: assets.length, evidenceRef: '1688:dom:source_video',
|
|
});
|
|
}
|
|
}
|
|
}
|
|
|
|
export function normaliseVideoUrl(raw: string, pageUrl: string): string | undefined {
|
|
if (!raw.trim()) return undefined;
|
|
try {
|
|
const url = new URL(raw, pageUrl);
|
|
return url.protocol === 'https:' && /\.(mp4|webm)$/i.test(url.pathname) ? url.href : undefined;
|
|
} catch {
|
|
return undefined;
|
|
}
|
|
}
|
|
|
|
/** Restores the original CDN URL before payload storage and de-duplication. */
|
|
export function normaliseImageUrl(
|
|
raw: string,
|
|
pageUrl: string,
|
|
platform: SupportedMediaPlatform,
|
|
): string | undefined {
|
|
if (!raw.trim() || /^(?:undefined|null|none)$/i.test(raw.trim())) return undefined;
|
|
let url: URL;
|
|
try {
|
|
url = new URL(raw, pageUrl);
|
|
} catch {
|
|
return undefined;
|
|
}
|
|
if (url.protocol !== 'https:' && url.protocol !== 'http:') return undefined;
|
|
|
|
if (platform === 'ozon') {
|
|
// Ozon thumbnails use /wc50/, /wc100/, /c600/, etc. The same media
|
|
// without this segment is its original CDN resource.
|
|
url.pathname = url.pathname
|
|
.replace(/\/(?:wc|c)\d+\//i, '/')
|
|
.replace(/\/{2,}/g, '/');
|
|
} else {
|
|
// Alibaba image URLs commonly append a transform after the file
|
|
// extension (e.g. image.jpg_400x400.jpg). Keep the original image.
|
|
const original = url.pathname.match(
|
|
/^(.+?\.(?:jpe?g|png|gif|bmp|heic|webp|avif))_/i,
|
|
);
|
|
if (original) url.pathname = original[1];
|
|
}
|
|
for (const parameter of ['width', 'height', 'size', 'quality', 'w', 'h']) {
|
|
url.searchParams.delete(parameter);
|
|
}
|
|
return url.toString();
|
|
}
|
|
|
|
export function dedupeImageUrl(url: string, platform: SupportedMediaPlatform) {
|
|
const normalised = normaliseImageUrl(url, url, platform) ?? url;
|
|
try {
|
|
const parsed = new URL(normalised);
|
|
parsed.search = '';
|
|
parsed.hash = '';
|
|
return parsed.toString();
|
|
} catch {
|
|
return normalised;
|
|
}
|
|
}
|
|
|
|
function imageUrlFromElement(
|
|
element: HTMLElement,
|
|
sourceProps: ImageGroup['sourceProps'] = [
|
|
'data-lazyload-src',
|
|
'data-src',
|
|
'srcset',
|
|
'currentSrc',
|
|
'src',
|
|
'backgroundImage',
|
|
],
|
|
pageUrl: string,
|
|
platform: SupportedMediaPlatform,
|
|
) {
|
|
const nestedTarget = element.matches('img, source')
|
|
? null
|
|
: element.querySelector<HTMLElement>('img, source');
|
|
const imageTarget = element.matches('img, source') ? element : nestedTarget;
|
|
for (const prop of sourceProps) {
|
|
// 1688 uses background images on the selected SKU option container.
|
|
// For every other property, prefer the actual nested image/source.
|
|
const target = prop === 'backgroundImage' ? element : imageTarget;
|
|
if (!target) continue;
|
|
let raw = '';
|
|
if (prop === 'srcset')
|
|
raw = bestSrcset(target.getAttribute('srcset') ?? '');
|
|
else if (prop === 'currentSrc')
|
|
raw = (target as HTMLImageElement).currentSrc ?? '';
|
|
else if (prop === 'backgroundImage') {
|
|
// 优先使用内联地址，避免占位值 undefined 被计算样式解析成一个页面相对 URL。
|
|
const style =
|
|
target.style.backgroundImage ||
|
|
target.ownerDocument.defaultView?.getComputedStyle(target).backgroundImage || '';
|
|
raw = style.match(/url\(["']?(.*?)["']?\)/i)?.[1] ?? '';
|
|
// Some 1688 SKU selectors directly match an <img> despite the
|
|
// group being background-image-first.
|
|
if (!raw && target.tagName === 'IMG') {
|
|
raw =
|
|
(target as HTMLImageElement).currentSrc ||
|
|
target.getAttribute('src') ||
|
|
'';
|
|
}
|
|
if (!raw && nestedTarget && nestedTarget !== target) {
|
|
const nestedStyle =
|
|
nestedTarget.ownerDocument.defaultView?.getComputedStyle(
|
|
nestedTarget,
|
|
).backgroundImage ?? nestedTarget.style.backgroundImage;
|
|
raw =
|
|
nestedStyle.match(/url\(["']?(.*?)["']?\)/i)?.[1] ??
|
|
((nestedTarget as HTMLImageElement).currentSrc ||
|
|
nestedTarget.getAttribute('src') ||
|
|
'');
|
|
}
|
|
} else raw = target.getAttribute(prop) ?? '';
|
|
const url = raw && normaliseImageUrl(raw, pageUrl, platform);
|
|
if (url) return url;
|
|
}
|
|
return undefined;
|
|
}
|
|
|
|
function passesDimensions(element: HTMLElement, group: ImageGroup) {
|
|
if (!group.minWidth && !group.minHeight) return true;
|
|
const image = element.matches('img')
|
|
? (element as HTMLImageElement)
|
|
: element.querySelector<HTMLImageElement>('img');
|
|
const measured = image ?? element;
|
|
const width = image?.naturalWidth || measured.offsetWidth || 0;
|
|
const height = image?.naturalHeight || measured.offsetHeight || 0;
|
|
// Browser snapshots and unrendered lazy images often report 0. As in the
|
|
// old collector, only reject when a real measured size proves it is tiny.
|
|
return !(
|
|
width > 0 &&
|
|
height > 0 &&
|
|
(width < (group.minWidth ?? 0) || height < (group.minHeight ?? 0))
|
|
);
|
|
}
|
|
|
|
function bestSrcset(srcset: string) {
|
|
let best = '';
|
|
let bestSize = -1;
|
|
for (const part of srcset
|
|
.split(',')
|
|
.map((value) => value.trim())
|
|
.filter(Boolean)) {
|
|
const [url, descriptor = ''] = part.split(/\s+/, 2);
|
|
const width = descriptor.match(/^(\d+)w$/)?.[1];
|
|
const density = descriptor.match(/^(\d+(?:\.\d+)?)x$/)?.[1];
|
|
const size = width
|
|
? Number(width)
|
|
: density
|
|
? Number(density) * 1_000
|
|
: 0;
|
|
if (size >= bestSize) {
|
|
best = url;
|
|
bestSize = size;
|
|
}
|
|
}
|
|
return best;
|
|
}
|