feat(curator): curate update keeps a server's dictionaries current

EDRDG's licence asks of servers showing its dictionaries that they update
them at least monthly. `curate update --into <folder>` downloads EDRDG's
files of the day (JMdict with Tatoeba's sentences, KANJIDIC2), converts
them with the JLPT levels the curator now brings along (yomitan-jlpt-vocab's
lists, CC BY-SA 4.0, and the kanji levels of kanji-data's kanji.json: both
static, in data/ with their origin), and puts each file in place at once
(written beside it as .part, then renamed over it), so the server never
reads half a file; a failure leaves the files as they were. With --every 7
it keeps running and updates whenever the files are a week old, as their
dates say, so a restart doesn't download again; after a failure it tries
again in six hours. A real update takes some 50 seconds and 400 MB.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01MMm7usAo6q9ZZRoPkWjoxi
Signed-off-by: Tizian 「ティツィアーン」 <raisondetredev@pm.me>
This commit is contained in:
Tizian 「ティツィアーン」 2026-10-09 16:40:19 +02:00
commit 82b38fadd8
Signed by: raisondetredev
GPG key ID: 07F6FCF405488B64

View file

@ -7,6 +7,7 @@ import 'package:halkyon_learn_curator/src/sources/jmdict.dart';
import 'package:halkyon_learn_curator/src/sources/jmdict_xml.dart';
import 'package:halkyon_learn_curator/src/sources/kanjidic.dart';
import 'package:halkyon_learn_curator/src/sources/kanjivg.dart';
import 'package:halkyon_learn_curator/src/update.dart';
/// Turns a dictionary download into a dictionary file for the server's
/// `dictionaries` folder, or KanjiVG into the app's stroke files
@ -21,6 +22,12 @@ import 'package:halkyon_learn_curator/src/sources/kanjivg.dart';
/// dart run halkyon_learn_curator:curate kanjivg \
/// --input kanjivg-20260714.xml.gz \
/// --output ../halkyon_learn_flutter/assets/kanjivg
///
/// Or brings a server's dictionaries up to date from EDRDG's files of the
/// day, once or every so many days:
///
/// dart run halkyon_learn_curator:curate update \
/// --into ../halkyon_learn_server/dictionaries [--every 7]
Future<void> main(List<String> args) async {
final parser = ArgParser()
..addOption(
@ -49,6 +56,18 @@ Future<void> main(List<String> args) async {
help:
'The dictionary file to write, gzipped when it ends in .gz; for '
'kanjivg the folder to write the stroke files into.',
)
..addOption('into', help: 'update: the folder of the dictionary files.')
..addOption(
'data',
help: "update: the curator's data folder (the JLPT lists).",
defaultsTo: 'data',
)
..addOption(
'every',
help:
'update: keep running, and update once the files are this many '
'days old.',
);
final ArgResults options;
try {
@ -57,6 +76,19 @@ Future<void> main(List<String> args) async {
_fail('${error.message}\n${parser.usage}');
}
final [source, ...] = options.rest.isEmpty ? [''] : options.rest;
if (source == 'update') {
final into = options.option('into');
final every = options.option('every');
final days = every == null ? null : int.tryParse(every);
if (into == null || (every != null && (days == null || days < 1))) {
_fail(
'Usage: curate update --into <folder> [--data <folder>] '
'[--every <days>]\n${parser.usage}',
);
}
await _update(Directory(into), Directory(options.option('data')!), days);
return;
}
final input = options.option('input');
final output = options.option('output');
if (!['jmdict', 'kanjidic', 'kanjivg'].contains(source) ||
@ -127,6 +159,46 @@ Future<void> main(List<String> args) async {
);
}
/// Updates the dictionaries in [into]; with [every], keeps running and
/// updates them whenever they are that many days old (as the files' dates
/// say, so a restart doesn't download them again), trying again after six
/// hours when an update fails.
Future<void> _update(Directory into, Directory data, int? every) async {
void say(String message) =>
stdout.writeln('${DateTime.now().toUtc().toIso8601String()} $message');
Future<bool> once() async {
try {
final written = await updateDictionaries(into: into, data: data);
say(
'Updated ${[for (final file in written) file.uri.pathSegments.last].join(', ')}.',
);
return true;
} on Object catch (error) {
say('Updating the dictionaries failed: $error');
return false;
}
}
if (every == null) {
if (!await once()) {
exit(1);
}
return;
}
for (;;) {
final due = lastUpdated(into)?.add(Duration(days: every));
final now = DateTime.now();
if (due == null || !now.isBefore(due)) {
if (!await once()) {
await Future<void>.delayed(const Duration(hours: 6));
}
continue;
}
say('Up to date; the next update ${due.toUtc().toIso8601String()}.');
await Future<void>.delayed(due.difference(now));
}
}
Never _fail(String message) {
stderr.writeln(message);
exit(64);