feat(curator): Chinese from CC-CEDICT, with the HSK levels of 2025

Chinese comes from CC-CEDICT (CC BY-SA 4.0), made for learners of
Chinese: `curate cedict` writes two dictionaries of its entries,
`cc-cedict-hans-en` (zh-Hans) and `cc-cedict-hant-en` (zh-Hant), so a
studyset finds its words as it writes them, each finding a word by
either script, by pinyin with or without tones and by its meanings.
Readings are pinyin with tone marks, a syllable to a character; measure
words are a remark beside the meanings.

The HSK levels are those of the 2025 examination syllabus, which HSK
3.0's examination follows since July 2026 (decided 2026-10-14: level
tags, credited), as hsk30 extracted them (MIT): `hsk:1` … `hsk:6`, and
`hsk:7` for 7 to 9, each given to the entry read as the syllabus reads
the word (的 de, not 的 dì; 书 shū, not the Book of Documents, Shū),
ranking the words, HSK 1 first.

`curate update --languages de,el,en,zh` takes over from `--wiktionary`:
Chinese from MDBG's file of the day as often as EDRDG's files, the rest
from Wiktionary's dump.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Signed-off-by: Tizian 「ティツィアーン」 <raisondetredev@pm.me>

halkyon_learn_curator/test/cedict_test.dart

@@ -0,0 +1,133 @@
+import 'dart:convert';
+import 'dart:io';
+
+import 'package:halkyon_learn_curator/src/sources/cedict.dart';
+import 'package:test/test.dart';
+
+void main() {
+  late String sample;
+  late HskLevels hsk;
+
+  setUpAll(() async {
+    sample = await File('test/fixtures/cedict-sample.txt').readAsString();
+    hsk = await readHskLevels(File('data/hsk/syllabus2025_words.tsv'));
+  });
+
+  List<Map<String, Object?>> linesOf({required bool traditional}) => [
+    for (final line in cedictDictionary(
+      sample,
+      traditional: traditional,
+      hsk: hsk,
+    ))
+      jsonDecode(line) as Map<String, Object?>,
+  ];
+
+  Map<String, Object?> entryOf(String id, {bool traditional = false}) =>
+      linesOf(traditional: traditional)
+          .skip(1)
+          .singleWhere((line) => line['id'] == id);
+
+  test('a file in simplified characters and one in traditional ones, each '
+      'a dictionary of its own', () {
+    final simplified = linesOf(traditional: false).first;
+    expect(simplified['source'], 'cc-cedict-hans');
+    expect(simplified['language'], 'zh-Hans');
+    expect(simplified['glossLanguage'], 'en');
+    expect(simplified['sourceVersion'], '2026-10-13');
+    expect(
+      [
+        for (final credit in simplified['credits']! as List)
+          (credit as Map)['name'],
+      ],
+      ['CC-CEDICT', 'HSK 3.0 vocabulary (examination syllabus, 2025)'],
+    );
+    final traditional = linesOf(traditional: true).first;
+    expect(traditional['source'], 'cc-cedict-hant');
+    expect(traditional['language'], 'zh-Hant');
+    expect(cedictFileName(traditional: true), 'cc-cedict-hant-en.jsonl.gz');
+  });
+
+  test('a word: its characters in either script, its pinyin with tone '
+      'marks, found typed without tones, in text by its characters', () {
+    final xuexi = entryOf('學習 学习 [xue2 xi2]');
+    expect(xuexi['headword'], '学习');
+    expect(xuexi['reading'], 'xué xí');
+    expect(
+      (xuexi['keys']! as List).cast<String>(),
+      containsAll([
+        '学习',
+        '學習',
+        'xuexi',
+        'xuéxí',
+        'xue2xi2',
+        'to learn',
+        'learn',
+        'study',
+      ]),
+    );
+    expect(xuexi['textKeys'], ['学习', '學習']);
+    final entry = xuexi['entry']! as Map;
+    expect(entry['forms'], [
+      {'text': '學習'},
+    ]);
+    expect(entry['senses'], [
+      {
+        'glosses': ['to learn', 'to study'],
+      },
+    ]);
+    expect(entryOf('學習 学习 [xue2 xi2]', traditional: true)['headword'], '學習');
+  });
+
+  test('HSK levels go to the entry read as the syllabus reads the word: 的 '
+      'de, not dì; 书 shū, not the Book of Documents, Shū', () {
+    expect(entryOf('的 的 [de5]')['labels'], contains('hsk:1'));
+    expect(entryOf('的 的 [di4]')['labels'], isNot(contains('hsk:1')));
+    expect(entryOf('書 书 [shu1]')['labels'], contains('hsk:1'));
+    expect(entryOf('書 书 [Shu1]')['labels'], isEmpty);
+    expect(entryOf('中國 中国 [Zhong1 guo2]')['labels'], contains('hsk:1'));
+    expect(entryOf('銀行 银行 [yin2 hang2]')['labels'], contains('hsk:3'));
+    // HSK 1 before HSK 3, both before words of no level.
+    expect(
+      entryOf('的 的 [de5]')['rank']! as int,
+      lessThan(entryOf('銀行 银行 [yin2 hang2]')['rank']! as int),
+    );
+    expect(
+      entryOf('銀行 银行 [yin2 hang2]')['rank']! as int,
+      lessThan(entryOf('的 的 [di4]')['rank']! as int),
+    );
+  });
+
+  test('measure words are a remark beside the meanings, pinyin marked', () {
+    final book = entryOf('書 书 [shu1]')['entry']! as Map;
+    expect(book['classifiers'], ['本 běn', '冊|册 cè', '部 bù']);
+    expect(
+      [
+        for (final sense in book['senses']! as List)
+          ...(sense as Map)['glosses']! as List,
+      ],
+      ['book', 'letter', 'document', 'to write'],
+    );
+  });
+
+  group('pinyin', () {
+    test('takes its tone marks where they belong', () {
+      expect(pinyinWithToneMarks('xue2 xi2'), 'xué xí');
+      expect(pinyinWithToneMarks('Zhong1 guo2'), 'Zhōng guó');
+      expect(pinyinWithToneMarks('nu:3 er2'), 'nǚ ér');
+      expect(pinyinWithToneMarks('lu:e4'), 'lüè');
+      expect(
+        pinyinWithToneMarks('shuo1 gui4 liu2 xiong2'),
+        'shuō guì liú xióng',
+      );
+      expect(pinyinWithToneMarks('dou1 le5 r5'), 'dōu le r');
+      expect(pinyinWithToneMarks('A A zhi4'), 'A A zhì');
+    });
+
+    test('is typed without its tones', () {
+      expect(tonelessPinyin('xué xí'), 'xuexi');
+      expect(tonelessPinyin('nǚ’ér'), 'nuer');
+      expect(tonelessPinyin('yìsī-bùgǒu'), 'yisibugou');
+      expect(tonelessPinyin('nu:3 er2'), 'nuer');
+    });
+  });
+}