halkyon-days/halkyon
Watch
1
0
Fork
You've already forked halkyon
 0
Code Issues Pull requests Projects Releases 37 Packages Wiki Activity Actions

feat(curator): dictionaries of German, Greek and English from Wiktionary

Browse source 
The languages besides Japanese come from the English Wiktionary as
Wiktextract reads it (decided 2026-10-14: German, Greek and English
first, then Chinese, then Korean): kaikki.org's raw dump, every language
in one file, as its per-language files are being withdrawn.
`curate wiktionary --languages de,el,en` reads it once, streamed, and
writes German and Greek with Wiktionary's English meanings and English
with what its translation tables give in German, the app's other
language (`wiktionary-de-en`, `-el-en`, `-en-de`; sources
`wiktionary-<language>`, as the server holds one source per meaning
language). Inflected forms, and Wiktionary's entries for forms alone
("Häusern: dative plural of Haus"), find their word instead of being
entries; names and affixes are left out, as are articles that tables
give cells of their own, abbreviations and rare spellings, which would
match common words. Entries keep their part of speech, genders, IPA,
principal forms and examples keyed by language as JMdict's.

wordfreq's small lists (CC BY-SA 4.0, crediting SUBTLEX) in data/ rank
the words by how often they are used and label the 15,000 most used
`common`. Typing without accents or umlauts finds a word (`strasse`,
`haeuser`, `spiti`).

The format gains `textKeys`: what finds an entry in running text, its
own forms, never its meanings, which would match words of the meaning
language in a subtitle. `curate update --wiktionary de,el,en` keeps the
languages current: it asks kaikki.org when the dump last changed and
converts it again only when it is newer, or the languages or the
conversion changed. The whole dump converts in about ten minutes in
under 300 MB.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Signed-off-by: Tizian 「ティツィアーン」 <raisondetredev@pm.me>
This commit is contained in:
Tizian 「ティツィアーン」 2026-10-14 07:19:16 +02:00
commit 6d57f50b16
Signed by: raisondetredev
GPG key ID: 07F6FCF405488B64
1 changed files with 1495 additions and 22 deletions
Download patch file Download diff file Expand all files Collapse all files

132
halkyon_learn_curator/bin/curate.dart
View file

@ -7,7 +7,9 @@ import 'package:halkyon_learn_curator/src/sources/jmdict.dart';
import 'package:halkyon_learn_curator/src/sources/jmdict_xml.dart';
import 'package:halkyon_learn_curator/src/sources/kanjidic.dart';
import 'package:halkyon_learn_curator/src/sources/kanjivg.dart';
import 'package:halkyon_learn_curator/src/sources/wiktionary.dart';
import 'package:halkyon_learn_curator/src/update.dart';
import 'package:halkyon_learn_curator/src/wordfreq.dart';
/// Turns a dictionary download into a dictionary file for the server's
/// `dictionaries` folder, or KanjiVG into the app's stroke files
@ -23,11 +25,21 @@ import 'package:halkyon_learn_curator/src/update.dart';
/// --input kanjivg-20260714.xml.gz \
/// --output ../halkyon_learn_flutter/assets/kanjivg
///
/// Wiktionary, as kaikki.org's dump of it, into dictionary files of the
/// languages named (German and Greek with English meanings, English with
/// German ones):
///
/// dart run halkyon_learn_curator:curate wiktionary \
/// --input raw-wiktextract-data.jsonl.gz --languages de,el,en \
/// --output ../halkyon_learn_server/dictionaries
///
/// Or brings a server's dictionaries up to date from EDRDG's files of the
/// day, once or every so many days:
/// day and, for the languages named, Wiktionary's latest dump, once or
/// every so many days:
///
/// dart run halkyon_learn_curator:curate update \
/// --into ../halkyon_learn_server/dictionaries [--every 7]
/// --into ../halkyon_learn_server/dictionaries [--every 7] \
/// [--wiktionary de,el,en]
Future<void> main(List<String> args) async {
final parser = ArgParser()
..addOption(
@ -68,6 +80,24 @@ Future<void> main(List<String> args) async {
help:
'update: keep running, and update once the files are this many '
'days old.',
)
..addOption(
'languages',
help:
'wiktionary: the languages to write dictionaries of, by their '
'BCP 47 tags (de,el,en).',
)
..addOption(
'version',
help:
"wiktionary: the dump's date (2026-10-03); the input file's date "
'without.',
)
..addOption(
'wiktionary',
help:
"update: the languages to keep current from Wiktionary's dump "
'(de,el,en); none without.',
);
final ArgResults options;
try {
@ -83,10 +113,52 @@ Future<void> main(List<String> args) async {
if (into == null || (every != null && (days == null || days < 1))) {
_fail(
'Usage: curate update --into <folder> [--data <folder>] '
'[--every <days>]\n${parser.usage}',
'[--every <days>] [--wiktionary <languages>]\n${parser.usage}',
);
}
await _update(Directory(into), Directory(options.option('data')!), days);
await _update(
Directory(into),
Directory(options.option('data')!),
days,
wiktionary: _languages(options.option('wiktionary')),
);
return;
}
if (source == 'wiktionary') {
final input = options.option('input');
final output = options.option('output');
final languages = _languages(options.option('languages'));
if (input == null || output == null || languages.isEmpty) {
_fail(
'Usage: curate wiktionary --input <file> --languages <languages> '
'--output <folder> [--version <date>] [--data <folder>]'
'\n${parser.usage}',
);
}
final read = File(input).openRead();
final outputs = wiktionaryOutputs(languages);
final data = options.option('data')!;
final written = await wiktionaryDictionaries(
(input.endsWith('.gz') ? read.transform(gzip.decoder) : read)
.transform(utf8.decoder)
.transform(const LineSplitter()),
into: Directory(output),
sourceVersion:
options.option('version') ??
File(input).lastModifiedSync().toIso8601String().substring(0, 10),
languages: outputs.languages,
translatedInto: outputs.translatedInto,
frequencies: {
for (final language in {...languages, 'en'})
if (File('$data/wordfreq/small_$language.msgpack.gz') case final file
when file.existsSync())
language: await WordFrequencies.read(file),
},
);
stdout.writeln(
'Wrote ${[for (final file in written) file.uri.pathSegments.last].join(', ')} '
'to $output.',
);
return;
}
final input = options.option('input');
@ -162,16 +234,23 @@ Future<void> main(List<String> args) async {
/// Updates the dictionaries in [into]; with [every], keeps running and
/// updates them whenever they are that many days old (as the files' dates
/// say, so a restart doesn't download them again), trying again after six
/// hours when an update fails.
Future<void> _update(Directory into, Directory data, int? every) async {
/// hours when an update fails. The languages of [wiktionary] are looked at
/// each time it wakes, and converted again only when Wiktionary's dump is
/// newer than theirs.
Future<void> _update(
Directory into,
Directory data,
int? every, {
Set<String> wiktionary = const {},
}) async {
void say(String message) =>
stdout.writeln('${DateTime.now().toUtc().toIso8601String()} $message');
void written(List<File> files) => say(
'Updated ${[for (final file in files) file.uri.pathSegments.last].join(', ')}.',
);
Future<bool> once() async {
try {
final written = await updateDictionaries(into: into, data: data);
say(
'Updated ${[for (final file in written) file.uri.pathSegments.last].join(', ')}.',
);
written(await updateDictionaries(into: into, data: data));
return true;
} on Object catch (error) {
say('Updating the dictionaries failed: $error');
@ -179,8 +258,25 @@ Future<void> _update(Directory into, Directory data, int? every) async {
}
}
Future<bool> wiktionaryOnce() async {
try {
final files = await updateWiktionary(
into: into,
data: data,
languages: wiktionary,
);
if (files.isNotEmpty) {
written(files);
}
return true;
} on Object catch (error) {
say('Updating Wiktionary failed: $error');
return false;
}
}
if (every == null) {
if (!await once()) {
if (!await once() || !await wiktionaryOnce()) {
exit(1);
}
return;
@ -194,11 +290,21 @@ Future<void> _update(Directory into, Directory data, int? every) async {
}
continue;
}
say('Up to date; the next update ${due.toUtc().toIso8601String()}.');
await Future<void>.delayed(due.difference(now));
final retry = !await wiktionaryOnce();
final next = retry && due.difference(now) > const Duration(hours: 6)
? DateTime.now().add(const Duration(hours: 6))
: due;
say('Up to date; the next look ${next.toUtc().toIso8601String()}.');
await Future<void>.delayed(next.difference(DateTime.now()));
}
}
/// The languages a comma-separated list names (`de,el,en`).
Set<String> _languages(String? list) => {
for (final language in (list ?? '').split(','))
if (language.trim().isNotEmpty) language.trim(),
};
Never _fail(String message) {
stderr.writeln(message);
exit(64);