feat: 汉字数据修改

2026-08-26 13:39:02 +08:00
parent 2f8a837ed8
commit 2ae9207373
8 changed files with 64013 additions and 14004 deletions
@@ -0,0 +1,53 @@
+/**
+ * 检查 char_common_stroke.json 中的汉字 key 是否都在 char_introduce_v2.json 中存在
+ * 输出 introduce_v2 中缺失的汉字
+ */
+const fs = require('fs');
+const path = require('path');
+
+const SKIP_KEYS = new Set(['_schema', '_verified_before']);
+
+const charDir = path.join(__dirname, '../data/char');
+const strokePath = path.join(charDir, 'char_common_stroke.json');
+const introduceV2Path = path.join(charDir, 'char_introduce_v2.json');
+
+function main() {
+    if (!fs.existsSync(strokePath)) {
+        console.error('未找到:', strokePath);
+        process.exit(1);
+    }
+    if (!fs.existsSync(introduceV2Path)) {
+        console.error('未找到:', introduceV2Path);
+        process.exit(1);
+    }
+
+    const stroke = JSON.parse(fs.readFileSync(strokePath, 'utf-8'));
+    const introduceV2 = JSON.parse(fs.readFileSync(introduceV2Path, 'utf-8'));
+
+    const strokeKeys = Object.keys(stroke);
+    const introduceKeys = new Set(
+        Object.keys(introduceV2).filter((k) => !SKIP_KEYS.has(k))
+    );
+
+    const missing = strokeKeys.filter((char) => !introduceKeys.has(char));
+
+    console.log(`char_common_stroke.json 汉字数: ${strokeKeys.length}`);
+    console.log(`char_introduce_v2.json 汉字数: ${introduceKeys.size}`);
+    console.log(`introduce_v2 中缺失: ${missing.length} 个\n`);
+
+    if (missing.length === 0) {
+        console.log('全部覆盖，无缺失。');
+        return;
+    }
+
+    console.log('缺失汉字列表:');
+    console.log(missing.join(''));
+    console.log('\n缺失汉字（每行一字）:');
+    missing.forEach((c) => console.log(c));
+
+    const outPath = path.join(charDir, 'introduce_v2_missing.txt');
+    fs.writeFileSync(outPath, missing.join('\n') + '\n', 'utf-8');
+    console.log(`\n已写入: ${outPath}`);
+}
+
+main();
@@ -0,0 +1,102 @@
+/**
+ * 从 char_introduce.json 转换为 hanzi_readings.json
+ * 结构：单音字 [pinyin, gloss, introduce]；多音字 [[pinyin, gloss, introduce], ...]
+ * 去除 schema 中的 meaning，第二项统一为 gloss（简单释义/代表词语）
+ */
+const fs = require('fs');
+const path = require('path');
+
+const SKIP_KEYS = new Set(['_schema', '_verified_before']);
+
+const SCHEMA =
+    '单音字: [pinyin, gloss, introduce]; 多音字: [[pinyin, gloss, introduce], ...]; 判断: 首元素为 string 则单音，为 array 则多音';
+
+const charDir = path.join(__dirname, '../data/char');
+const inputPath = path.join(charDir, 'char_introduce.json');
+const outputPath = path.join(charDir, 'hanzi_readings.json');
+
+/**
+ * 一条读音规范为 [pinyin, gloss, introduce]，仅保留前三项
+ */
+function normalizeReading(arr) {
+    if (!Array.isArray(arr) || arr.length < 3) return null;
+    const pinyin = arr[0];
+    const gloss = arr[1] ?? null;
+    const introduce = arr[2];
+    if (typeof pinyin !== 'string' || typeof introduce !== 'string') return null;
+    return [pinyin, gloss, introduce];
+}
+
+/**
+ * 将 value 转为新结构（单音一维 / 多音二维）
+ */
+function transformValue(value) {
+    if (!Array.isArray(value) || value.length === 0) return null;
+    const first = value[0];
+    if (typeof first === 'string') {
+        return normalizeReading(value);
+    }
+    if (Array.isArray(first)) {
+        const readings = value.map((sub) => normalizeReading(sub)).filter(Boolean);
+        return readings.length > 0 ? readings : null;
+    }
+    return null;
+}
+
+/**
+ * 按 JSON 文件中 key 出现顺序提取汉字 key
+ */
+function getKeyOrderFromJsonText(text) {
+    const keys = [];
+    const re = /^\s*"([^"]+)":\s*(?:\[|")/gm;
+    let m;
+    while ((m = re.exec(text)) !== null) {
+        const key = m[1];
+        if (!SKIP_KEYS.has(key)) keys.push(key);
+    }
+    return keys;
+}
+
+function main() {
+    if (!fs.existsSync(inputPath)) {
+        console.error('未找到:', inputPath);
+        process.exit(1);
+    }
+
+    const text = fs.readFileSync(inputPath, 'utf-8');
+    const raw = JSON.parse(text);
+    const keyOrder = getKeyOrderFromJsonText(text);
+
+    const out = {
+        _schema: SCHEMA,
+    };
+    if (raw._verified_before !== undefined) {
+        out._verified_before = raw._verified_before;
+    }
+
+    let ok = 0;
+    let fail = 0;
+    const failed = [];
+
+    for (const char of keyOrder) {
+        const transformed = transformValue(raw[char]);
+        if (transformed) {
+            out[char] = transformed;
+            ok++;
+        } else {
+            fail++;
+            failed.push(char);
+        }
+    }
+
+    fs.writeFileSync(outputPath, JSON.stringify(out, null, 4) + '\n', 'utf-8');
+
+    console.log(`输入: ${inputPath}`);
+    console.log(`输出: ${outputPath}`);
+    console.log(`成功: ${ok} 个汉字`);
+    if (fail > 0) {
+        console.log(`失败: ${fail} 个`, failed.join(''));
+    }
+}
+
+main();