feat: 汉字数据修改

This commit is contained in:
R524809
2026-08-26 13:39:02 +08:00
parent 2f8a837ed8
commit 2ae9207373
8 changed files with 64013 additions and 14004 deletions
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
@@ -1,5 +1,5 @@
{ {
"_schema": "单音字: [pinyin, meaning|null, introduce]; 多音字: [[pinyin, meaning|null, introduce], ...]; 判断: 首元素为string则单音,为array则多音", "_schema": "单音字: [pinyin, gloss, introduce]; 多音字: [[pinyin, gloss, introduce], ...]; 判断: 首元素为 string 则单音,为 array 则多音",
"_verified_before": "佣", "_verified_before": "佣",
"一": ["yī", null, "一,一二三四的一"], "一": ["yī", null, "一,一二三四的一"],
"乙": ["yǐ", null, "乙,甲乙丙丁的乙"], "乙": ["yǐ", null, "乙,甲乙丙丁的乙"],
File diff suppressed because it is too large Load Diff
+2
View File
@@ -8,6 +8,8 @@
"generate-introduce": "cross-env node scripts/generate_char_introduce.js", "generate-introduce": "cross-env node scripts/generate_char_introduce.js",
"generate-voice": "node scripts/generate_char_voice.js", "generate-voice": "node scripts/generate_char_voice.js",
"generate-voice-v2": "node scripts/generate_char_voice_v2.js", "generate-voice-v2": "node scripts/generate_char_voice_v2.js",
"check-introduce-v2": "node scripts/check_introduce_v2_coverage.js",
"convert-hanzi-readings": "node scripts/convert_to_hanzi_readings.js",
"deploy-assets": "node scripts/deploy-assets.js", "deploy-assets": "node scripts/deploy-assets.js",
"deploy-assets:local": "node scripts/deploy-assets.js --local", "deploy-assets:local": "node scripts/deploy-assets.js --local",
"deploy-assets:server": "node scripts/deploy-assets.js --server" "deploy-assets:server": "node scripts/deploy-assets.js --server"
@@ -0,0 +1,53 @@
/**
* 检查 char_common_stroke.json 中的汉字 key 是否都在 char_introduce_v2.json 中存在
* 输出 introduce_v2 中缺失的汉字
*/
const fs = require('fs');
const path = require('path');
const SKIP_KEYS = new Set(['_schema', '_verified_before']);
const charDir = path.join(__dirname, '../data/char');
const strokePath = path.join(charDir, 'char_common_stroke.json');
const introduceV2Path = path.join(charDir, 'char_introduce_v2.json');
function main() {
if (!fs.existsSync(strokePath)) {
console.error('未找到:', strokePath);
process.exit(1);
}
if (!fs.existsSync(introduceV2Path)) {
console.error('未找到:', introduceV2Path);
process.exit(1);
}
const stroke = JSON.parse(fs.readFileSync(strokePath, 'utf-8'));
const introduceV2 = JSON.parse(fs.readFileSync(introduceV2Path, 'utf-8'));
const strokeKeys = Object.keys(stroke);
const introduceKeys = new Set(
Object.keys(introduceV2).filter((k) => !SKIP_KEYS.has(k))
);
const missing = strokeKeys.filter((char) => !introduceKeys.has(char));
console.log(`char_common_stroke.json 汉字数: ${strokeKeys.length}`);
console.log(`char_introduce_v2.json 汉字数: ${introduceKeys.size}`);
console.log(`introduce_v2 中缺失: ${missing.length}\n`);
if (missing.length === 0) {
console.log('全部覆盖,无缺失。');
return;
}
console.log('缺失汉字列表:');
console.log(missing.join(''));
console.log('\n缺失汉字(每行一字):');
missing.forEach((c) => console.log(c));
const outPath = path.join(charDir, 'introduce_v2_missing.txt');
fs.writeFileSync(outPath, missing.join('\n') + '\n', 'utf-8');
console.log(`\n已写入: ${outPath}`);
}
main();
@@ -0,0 +1,102 @@
/**
* 从 char_introduce.json 转换为 hanzi_readings.json
* 结构:单音字 [pinyin, gloss, introduce];多音字 [[pinyin, gloss, introduce], ...]
* 去除 schema 中的 meaning,第二项统一为 gloss(简单释义/代表词语)
*/
const fs = require('fs');
const path = require('path');
const SKIP_KEYS = new Set(['_schema', '_verified_before']);
const SCHEMA =
'单音字: [pinyin, gloss, introduce]; 多音字: [[pinyin, gloss, introduce], ...]; 判断: 首元素为 string 则单音,为 array 则多音';
const charDir = path.join(__dirname, '../data/char');
const inputPath = path.join(charDir, 'char_introduce.json');
const outputPath = path.join(charDir, 'hanzi_readings.json');
/**
* 一条读音规范为 [pinyin, gloss, introduce],仅保留前三项
*/
function normalizeReading(arr) {
if (!Array.isArray(arr) || arr.length < 3) return null;
const pinyin = arr[0];
const gloss = arr[1] ?? null;
const introduce = arr[2];
if (typeof pinyin !== 'string' || typeof introduce !== 'string') return null;
return [pinyin, gloss, introduce];
}
/**
* 将 value 转为新结构(单音一维 / 多音二维)
*/
function transformValue(value) {
if (!Array.isArray(value) || value.length === 0) return null;
const first = value[0];
if (typeof first === 'string') {
return normalizeReading(value);
}
if (Array.isArray(first)) {
const readings = value.map((sub) => normalizeReading(sub)).filter(Boolean);
return readings.length > 0 ? readings : null;
}
return null;
}
/**
* 按 JSON 文件中 key 出现顺序提取汉字 key
*/
function getKeyOrderFromJsonText(text) {
const keys = [];
const re = /^\s*"([^"]+)":\s*(?:\[|")/gm;
let m;
while ((m = re.exec(text)) !== null) {
const key = m[1];
if (!SKIP_KEYS.has(key)) keys.push(key);
}
return keys;
}
function main() {
if (!fs.existsSync(inputPath)) {
console.error('未找到:', inputPath);
process.exit(1);
}
const text = fs.readFileSync(inputPath, 'utf-8');
const raw = JSON.parse(text);
const keyOrder = getKeyOrderFromJsonText(text);
const out = {
_schema: SCHEMA,
};
if (raw._verified_before !== undefined) {
out._verified_before = raw._verified_before;
}
let ok = 0;
let fail = 0;
const failed = [];
for (const char of keyOrder) {
const transformed = transformValue(raw[char]);
if (transformed) {
out[char] = transformed;
ok++;
} else {
fail++;
failed.push(char);
}
}
fs.writeFileSync(outputPath, JSON.stringify(out, null, 4) + '\n', 'utf-8');
console.log(`输入: ${inputPath}`);
console.log(`输出: ${outputPath}`);
console.log(`成功: ${ok} 个汉字`);
if (fail > 0) {
console.log(`失败: ${fail}`, failed.join(''));
}
}
main();