feat(curator): Chinese from CC-CEDICT, with the HSK levels of 2025

Chinese comes from CC-CEDICT (CC BY-SA 4.0), made for learners of
Chinese: `curate cedict` writes two dictionaries of its entries,
`cc-cedict-hans-en` (zh-Hans) and `cc-cedict-hant-en` (zh-Hant), so a
studyset finds its words as it writes them, each finding a word by
either script, by pinyin with or without tones and by its meanings.
Readings are pinyin with tone marks, a syllable to a character; measure
words are a remark beside the meanings.

The HSK levels are those of the 2025 examination syllabus, which HSK
3.0's examination follows since July 2026 (decided 2026-10-14: level
tags, credited), as hsk30 extracted them (MIT): `hsk:1` … `hsk:6`, and
`hsk:7` for 7 to 9, each given to the entry read as the syllabus reads
the word (的 de, not 的 dì; 书 shū, not the Book of Documents, Shū),
ranking the words, HSK 1 first.

`curate update --languages de,el,en,zh` takes over from `--wiktionary`:
Chinese from MDBG's file of the day as often as EDRDG's files, the rest
from Wiktionary's dump.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Signed-off-by: Tizian 「ティツィアーン」 <raisondetredev@pm.me>

halkyon_learn_curator/bin/curate.dart

@@ -2,6 +2,7 @@ import 'dart:convert';
 import 'dart:io';
 
 import 'package:args/args.dart';
+import 'package:halkyon_learn_curator/src/sources/cedict.dart';
 import 'package:halkyon_learn_curator/src/sources/jlpt.dart';
 import 'package:halkyon_learn_curator/src/sources/jmdict.dart';
 import 'package:halkyon_learn_curator/src/sources/jmdict_xml.dart';
@@ -33,13 +34,19 @@ import 'package:halkyon_learn_curator/src/wordfreq.dart';
 ///       --input raw-wiktextract-data.jsonl.gz --languages de,el,en \
 ///       --output ../halkyon_learn_server/dictionaries
 ///
+/// CC-CEDICT into its two dictionary files, simplified and traditional:
+///
+///     dart run halkyon_learn_curator:curate cedict \
+///       --input cedict_1_0_ts_utf-8_mdbg.txt.gz \
+///       --output ../halkyon_learn_server/dictionaries
+///
 /// Or brings a server's dictionaries up to date from EDRDG's files of the
-/// day and, for the languages named, Wiktionary's latest dump, once or
-/// every so many days:
+/// day and, for the languages named, Wiktionary's latest dump and MDBG's
+/// CC-CEDICT of the day (Chinese), once or every so many days:
 ///
 ///     dart run halkyon_learn_curator:curate update \
 ///       --into ../halkyon_learn_server/dictionaries [--every 7] \
-///       [--wiktionary de,el,en]
+///       [--languages de,el,en,zh]
 Future<void> main(List<String> args) async {
   final parser = ArgParser()
     ..addOption(
@@ -85,19 +92,15 @@ Future<void> main(List<String> args) async {
       'languages',
       help:
           'wiktionary: the languages to write dictionaries of, by their '
-          'BCP 47 tags (de,el,en).',
+          'BCP 47 tags (de,el,en); update: the languages besides Japanese '
+          'to keep current (de,el,en,zh; Chinese from CC-CEDICT, the rest '
+          "from Wiktionary's dump), none without.",
     )
     ..addOption(
       'version',
       help:
           "wiktionary: the dump's date (2026-10-03); the input file's date "
           'without.',
-    )
-    ..addOption(
-      'wiktionary',
-      help:
-          "update: the languages to keep current from Wiktionary's dump "
-          '(de,el,en); none without.',
     );
   final ArgResults options;
   try {
@@ -113,14 +116,57 @@ Future<void> main(List<String> args) async {
     if (into == null || (every != null && (days == null || days < 1))) {
       _fail(
         'Usage: curate update --into <folder> [--data <folder>] '
-        '[--every <days>] [--wiktionary <languages>]\n${parser.usage}',
+        '[--every <days>] [--languages <languages>]\n${parser.usage}',
       );
     }
     await _update(
       Directory(into),
       Directory(options.option('data')!),
       days,
-      wiktionary: _languages(options.option('wiktionary')),
+      languages: _languages(options.option('languages')),
+    );
+    return;
+  }
+  if (source == 'cedict') {
+    final input = options.option('input');
+    final output = options.option('output');
+    if (input == null || output == null) {
+      _fail(
+        'Usage: curate cedict --input <file> --output <folder> '
+        '[--data <folder>]\n${parser.usage}',
+      );
+    }
+    final read = File(input).openRead();
+    final text =
+        await (input.endsWith('.gz') ? read.transform(gzip.decoder) : read)
+            .transform(utf8.decoder)
+            .join();
+    final data = options.option('data')!;
+    final hsk = await readHskLevels(hskFile(Directory(data)));
+    final frequencies = File('$data/wordfreq/small_zh.msgpack.gz');
+    final words = frequencies.existsSync()
+        ? await WordFrequencies.read(frequencies)
+        : WordFrequencies.none;
+    final folder = await Directory(output).create(recursive: true);
+    for (final traditional in [false, true]) {
+      await Stream.fromIterable(
+            cedictDictionary(
+              text,
+              traditional: traditional,
+              hsk: hsk,
+              frequencies: words,
+            ),
+          )
+          .transform(utf8.encoder)
+          .transform(gzip.encoder)
+          .pipe(
+            File('${folder.path}/${cedictFileName(traditional: traditional)}')
+                .openWrite(),
+          );
+    }
+    stdout.writeln(
+      'Wrote ${cedictFileName(traditional: false)} and '
+      '${cedictFileName(traditional: true)} to $output.',
     );
     return;
   }
@@ -234,15 +280,16 @@ Future<void> main(List<String> args) async {
 /// Updates the dictionaries in [into]; with [every], keeps running and
 /// updates them whenever they are that many days old (as the files' dates
 /// say, so a restart doesn't download them again), trying again after six
-/// hours when an update fails. The languages of [wiktionary] are looked at
-/// each time it wakes, and converted again only when Wiktionary's dump is
-/// newer than theirs.
+/// hours when an update fails. Of [languages], Chinese comes from
+/// CC-CEDICT as often as EDRDG's files, the rest from Wiktionary, looked at
+/// each time it wakes and converted again only when the dump is newer.
 Future<void> _update(
   Directory into,
   Directory data,
   int? every, {
-  Set<String> wiktionary = const {},
+  Set<String> languages = const {},
 }) async {
+  final (:wiktionary, :chinese) = dictionarySources(languages);
   void say(String message) =>
       stdout.writeln('${DateTime.now().toUtc().toIso8601String()} $message');
   void written(List<File> files) => say(
@@ -275,8 +322,25 @@ Future<void> _update(
     }
   }
 
+  // CC-CEDICT once its files are as old as EDRDG's may be.
+  Future<bool> chineseOnce() async {
+    final updated = cedictUpdated(into);
+    if (!chinese ||
+        updated != null &&
+            DateTime.now().difference(updated) < Duration(days: every ?? 1)) {
+      return true;
+    }
+    try {
+      written(await updateCedict(into: into, data: data));
+      return true;
+    } on Object catch (error) {
+      say('Updating CC-CEDICT failed: $error');
+      return false;
+    }
+  }
+
   if (every == null) {
-    if (!await once() || !await wiktionaryOnce()) {
+    if (!await once() || !await chineseOnce() || !await wiktionaryOnce()) {
       exit(1);
     }
     return;
@@ -290,7 +354,7 @@ Future<void> _update(
       }
       continue;
     }
-    final retry = !await wiktionaryOnce();
+    final retry = !await chineseOnce() | !await wiktionaryOnce();
     final next = retry && due.difference(now) > const Duration(hours: 6)
         ? DateTime.now().add(const Duration(hours: 6))
         : due;