halkyon-days/halkyon
Watch
1
0
Fork
You've already forked halkyon
 0
Code Issues Pull requests Projects Releases 37 Packages Wiki Activity Actions

feat(curator): characters of Chinese and Korean, the Greek alphabet, Korean words from KRDict

Browse source 
For parity with Japanese's KANJIDIC2 and kana:

- Unihan (Unicode License v3) makes characters' dictionaries: the 8,105
  standard simplified characters (unihan-hans) and the traditional ones in
  use (unihan-hant), told apart and ranked by CC-CEDICT's words (Unihan
  calls 这 its own traditional variant; 後 ranks as the 后 of 以后), with
  pinyin, meanings, strokes, Kangxi radical, HSK 2025 character levels,
  the 3,500 common ones and Hong Kong's grades; and Korean Hanja
  (unihan-ko) with their sounds, the 1,800 of education, and their native
  readings from Wiktionary (물 수), with the 40 Hangul letters as kana are.
- The Greek alphabet, the curator's own (CC0): letters with names,
  transliteration and sound.
- KRDict, the Basic Korean Dictionary for learners (CC BY-SA 2.0 KR,
  downloaded without an account), replaces Wiktionary for Korean's words:
  learners' levels, pronunciation where it differs from the spelling
  (같이 [가치]), Hanja, English senses; Wiktionary's update keeps the
  words' forms (가다's 99) and the Hanja's readings beside it for both.
- The update converts KRDict when its page offers another export and the
  characters weekly, and removes the files of languages no longer kept
  or of sources replaced.

Measured on the real data: Unihan 3 s; KRDict 54,870 entries in 24 s
and under 600 MB.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Signed-off-by: Tizian 「ティツィアーン」 <raisondetredev@pm.me>
This commit is contained in:
Tizian 「ティツィアーン」 2026-10-14 12:08:10 +02:00
commit 8820cefde3
Signed by: raisondetredev
GPG key ID: 07F6FCF405488B64
1 changed files with 6997 additions and 85 deletions
Download patch file Download diff file Expand all files Collapse all files

216
halkyon_learn_curator/bin/curate.dart
View file

@ -9,7 +9,9 @@ import 'package:halkyon_learn_curator/src/sources/jmdict.dart';
import 'package:halkyon_learn_curator/src/sources/jmdict_xml.dart';
import 'package:halkyon_learn_curator/src/sources/kanjidic.dart';
import 'package:halkyon_learn_curator/src/sources/kanjivg.dart';
import 'package:halkyon_learn_curator/src/sources/krdict.dart';
import 'package:halkyon_learn_curator/src/sources/tatoeba.dart';
import 'package:halkyon_learn_curator/src/sources/unihan.dart';
import 'package:halkyon_learn_curator/src/sources/wiktionary.dart';
import 'package:halkyon_learn_curator/src/update.dart';
import 'package:halkyon_learn_curator/src/wordfreq.dart';
@ -42,6 +44,22 @@ import 'package:halkyon_learn_curator/src/wordfreq.dart';
/// --input cedict_1_0_ts_utf-8_mdbg.txt.gz \
/// --output ../halkyon_learn_server/dictionaries
///
/// Unihan into dictionary files of Chinese characters (simplified and
/// traditional) and of Korean Hanja, for the languages named, the Hanja's
/// native readings from a Wiktionary update's `.wiktionary-ko.json.gz`:
///
/// dart run halkyon_learn_curator:curate unihan \
/// --input Unihan.zip --languages ko,zh [--cedict <file>] \
/// [--korean <file>] \
/// --output ../halkyon_learn_server/dictionaries
///
/// KRDict's JSON export into the dictionary file of Korean words, with
/// the forms of each a Wiktionary update kept (`.wiktionary-ko.json.gz`):
///
/// dart run halkyon_learn_curator:curate krdict \
/// --input krdict_json_20260919.zip [--korean <file>] \
/// --output ../halkyon_learn_server/dictionaries
///
/// Tatoeba's sentences as examples into a dictionary file of words (its
/// exports of the dictionary's language, of its meaning language, and of
/// the links between them, bzip2 or not):
@ -119,6 +137,19 @@ Future<void> main(List<String> args) async {
'links',
help: "tatoeba: Tatoeba's links between the two languages.",
)
..addOption(
'cedict',
help:
'unihan: CC-CEDICT, whose words tell traditional characters in use '
'and rank the characters.',
)
..addOption(
'korean',
help:
'unihan, krdict: what a Wiktionary update kept for Korean '
"(.wiktionary-ko.json.gz: the Hanja's native readings, the "
"words' forms).",
)
..addOption(
'version',
help:
@ -243,6 +274,77 @@ Future<void> main(List<String> args) async {
);
return;
}
if (source == 'krdict') {
final input = options.option('input');
final output = options.option('output');
if (input == null || output == null) {
_fail(
'Usage: curate krdict --input <zip> --output <folder> '
'[--korean <file>] [--data <folder>]\n${parser.usage}',
);
}
final korean = options.option('korean');
final frequencies = File(
'${options.option('data')}/wordfreq/small_ko.msgpack.gz',
);
final folder = await Directory(output).create(recursive: true);
final (:version, :entries) = krdictExport(await File(input).readAsBytes());
await krdictDictionary(
entries,
sourceVersion: version,
frequencies: frequencies.existsSync()
? await WordFrequencies.read(frequencies)
: null,
korean: korean == null
? null
: WiktionaryKorean.fromJson(
utf8.decode(gzip.decode(await File(korean).readAsBytes())),
),
)
.transform(utf8.encoder)
.transform(gzip.encoder)
.pipe(File('${folder.path}/$krdictFileName').openWrite());
stdout.writeln('Wrote $krdictFileName to $output.');
return;
}
if (source == 'unihan') {
final input = options.option('input');
final output = options.option('output');
final languages = _languages(options.option('languages'));
if (input == null || output == null || languages.isEmpty) {
_fail(
'Usage: curate unihan --input <Unihan.zip> --languages <languages> '
'--output <folder> [--cedict <file>] [--korean <file>] '
'[--data <folder>]'
'\n${parser.usage}',
);
}
final korean = options.option('korean');
final cedict = options.option('cedict');
final written = await unihanDictionaries(
await File(input).readAsBytes(),
into: await Directory(output).create(recursive: true),
languages: languages,
data: Directory(options.option('data')!),
cedict: cedict == null
? null
: await (cedict.endsWith('.gz')
? File(cedict).openRead().transform(gzip.decoder)
: File(cedict).openRead())
.transform(utf8.decoder)
.join(),
eumhun: korean == null
? const {}
: WiktionaryKorean.fromJson(
utf8.decode(gzip.decode(await File(korean).readAsBytes())),
).hanja,
);
stdout.writeln(
'Wrote ${[for (final file in written) file.uri.pathSegments.last].join(', ')} '
'to $output.',
);
return;
}
if (source == 'wiktionary') {
final input = options.option('input');
final output = options.option('output');
@ -354,66 +456,96 @@ Future<void> main(List<String> args) async {
/// updates them whenever they are that many days old (as the files' dates
/// say, so a restart doesn't download them again), trying again after six
/// hours when an update fails. Of [languages], Chinese comes from
/// CC-CEDICT as often as EDRDG's files, the rest from Wiktionary, looked at
/// each time it wakes and converted again only when the dump is newer.
/// CC-CEDICT as often as EDRDG's files, and its characters from Unihan;
/// Korean from KRDict, looked at each time it wakes and converted again
/// only when it offers another export, and its Hanja from Unihan; the rest
/// from Wiktionary, looked at each time it wakes and converted again only
/// when the dump is newer (Korean's forms and the Hanja's readings come
/// from it too); Greek's alphabet with the characters. Files of languages
/// no longer kept go.
Future<void> _update(
Directory into,
Directory data,
int? every, {
Set<String> languages = const {},
}) async {
final (:wiktionary, :chinese) = dictionarySources(languages);
final (:wiktionary, :chinese, :korean, :greek) = dictionarySources(languages);
void say(String message) =>
stdout.writeln('${DateTime.now().toUtc().toIso8601String()} $message');
void written(List<File> files) => say(
'Updated ${[for (final file in files) file.uri.pathSegments.last].join(', ')}.',
);
Future<bool> once() async {
try {
written(await updateDictionaries(into: into, data: data));
return true;
} on Object catch (error) {
say('Updating the dictionaries failed: $error');
return false;
}
}
Future<bool> wiktionaryOnce() async {
try {
final files = await updateWiktionary(
into: into,
data: data,
languages: wiktionary,
void written(List<File> files) {
if (files.isNotEmpty) {
say(
'Updated ${[for (final file in files) file.uri.pathSegments.last].join(', ')}.',
);
if (files.isNotEmpty) {
written(files);
}
}
}
/// Runs [step], saying what it wrote; false when it failed.
Future<bool> attempt(String name, Future<List<File>> Function() step) async {
try {
written(await step());
return true;
} on Object catch (error) {
say('Updating Wiktionary failed: $error');
say('Updating $name failed: $error');
return false;
}
}
// CC-CEDICT once its files are as old as EDRDG's may be.
Future<bool> chineseOnce() async {
final updated = cedictUpdated(into);
if (!chinese ||
updated != null &&
DateTime.now().difference(updated) < Duration(days: every ?? 1)) {
return true;
}
try {
written(await updateCedict(into: into, data: data));
return true;
} on Object catch (error) {
say('Updating CC-CEDICT failed: $error');
return false;
Future<bool> once() => attempt(
'the dictionaries',
() => updateDictionaries(into: into, data: data),
);
// Files as old as EDRDG's may be: CC-CEDICT's, and the characters'.
bool due(DateTime? updated) =>
updated == null ||
DateTime.now().difference(updated) >= Duration(days: every ?? 1);
Future<bool> chineseOnce() async =>
!chinese ||
!due(cedictUpdated(into)) ||
await attempt('CC-CEDICT', () => updateCedict(into: into, data: data));
Future<bool> wiktionaryOnce() => attempt(
'Wiktionary',
() => updateWiktionary(
into: into,
data: data,
languages: wiktionary,
korean: korean,
),
);
Future<bool> koreanOnce() async =>
!korean ||
await attempt('KRDict', () => updateKrdict(into: into, data: data));
Future<bool> charactersOnce() async =>
!due(
charactersUpdated(into, chinese: chinese, korean: korean, greek: greek),
) ||
await attempt(
'the characters',
() => updateCharacters(
into: into,
data: data,
chinese: chinese,
korean: korean,
greek: greek,
),
);
// In this order: Korean's words and Hanja take what Wiktionary kept.
Future<bool> languagesOnce() async {
final removed = removeUnwanted(into, languages);
if (removed.isNotEmpty) {
say(
'Removed ${[for (final file in removed) file.uri.pathSegments.last].join(', ')}.',
);
}
return !(!await chineseOnce() |
!await wiktionaryOnce() |
!await koreanOnce() |
!await charactersOnce());
}
if (every == null) {
if (!await once() || !await chineseOnce() || !await wiktionaryOnce()) {
if (!await once() || !await languagesOnce()) {
exit(1);
}
return;
@ -427,7 +559,7 @@ Future<void> _update(
}
continue;
}
final retry = !await chineseOnce() | !await wiktionaryOnce();
final retry = !await languagesOnce();
final next = retry && due.difference(now) > const Duration(hours: 6)
? DateTime.now().add(const Duration(hours: 6))
: due;