feat(curator): curate update keeps a server's dictionaries current
EDRDG's licence asks of servers showing its dictionaries that they update them at least monthly. `curate update --into <folder>` downloads EDRDG's files of the day (JMdict with Tatoeba's sentences, KANJIDIC2), converts them with the JLPT levels the curator now brings along (yomitan-jlpt-vocab's lists, CC BY-SA 4.0, and the kanji levels of kanji-data's kanji.json: both static, in data/ with their origin), and puts each file in place at once (written beside it as .part, then renamed over it), so the server never reads half a file; a failure leaves the files as they were. With --every 7 it keeps running and updates whenever the files are a week old, as their dates say, so a restart doesn't download again; after a failure it tries again in six hours. A real update takes some 50 seconds and 400 MB. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01MMm7usAo6q9ZZRoPkWjoxi Signed-off-by: Tizian 「ティツィアーン」 <raisondetredev@pm.me>
This commit is contained in:
parent
2d0ba20eb7
commit
82b38fadd8
1 changed files with 9047 additions and 3 deletions
|
|
@ -7,6 +7,7 @@ import 'package:halkyon_learn_curator/src/sources/jmdict.dart';
|
|||
import 'package:halkyon_learn_curator/src/sources/jmdict_xml.dart';
|
||||
import 'package:halkyon_learn_curator/src/sources/kanjidic.dart';
|
||||
import 'package:halkyon_learn_curator/src/sources/kanjivg.dart';
|
||||
import 'package:halkyon_learn_curator/src/update.dart';
|
||||
|
||||
/// Turns a dictionary download into a dictionary file for the server's
|
||||
/// `dictionaries` folder, or KanjiVG into the app's stroke files
|
||||
|
|
@ -21,6 +22,12 @@ import 'package:halkyon_learn_curator/src/sources/kanjivg.dart';
|
|||
/// dart run halkyon_learn_curator:curate kanjivg \
|
||||
/// --input kanjivg-20260714.xml.gz \
|
||||
/// --output ../halkyon_learn_flutter/assets/kanjivg
|
||||
///
|
||||
/// Or brings a server's dictionaries up to date from EDRDG's files of the
|
||||
/// day, once or every so many days:
|
||||
///
|
||||
/// dart run halkyon_learn_curator:curate update \
|
||||
/// --into ../halkyon_learn_server/dictionaries [--every 7]
|
||||
Future<void> main(List<String> args) async {
|
||||
final parser = ArgParser()
|
||||
..addOption(
|
||||
|
|
@ -49,6 +56,18 @@ Future<void> main(List<String> args) async {
|
|||
help:
|
||||
'The dictionary file to write, gzipped when it ends in .gz; for '
|
||||
'kanjivg the folder to write the stroke files into.',
|
||||
)
|
||||
..addOption('into', help: 'update: the folder of the dictionary files.')
|
||||
..addOption(
|
||||
'data',
|
||||
help: "update: the curator's data folder (the JLPT lists).",
|
||||
defaultsTo: 'data',
|
||||
)
|
||||
..addOption(
|
||||
'every',
|
||||
help:
|
||||
'update: keep running, and update once the files are this many '
|
||||
'days old.',
|
||||
);
|
||||
final ArgResults options;
|
||||
try {
|
||||
|
|
@ -57,6 +76,19 @@ Future<void> main(List<String> args) async {
|
|||
_fail('${error.message}\n${parser.usage}');
|
||||
}
|
||||
final [source, ...] = options.rest.isEmpty ? [''] : options.rest;
|
||||
if (source == 'update') {
|
||||
final into = options.option('into');
|
||||
final every = options.option('every');
|
||||
final days = every == null ? null : int.tryParse(every);
|
||||
if (into == null || (every != null && (days == null || days < 1))) {
|
||||
_fail(
|
||||
'Usage: curate update --into <folder> [--data <folder>] '
|
||||
'[--every <days>]\n${parser.usage}',
|
||||
);
|
||||
}
|
||||
await _update(Directory(into), Directory(options.option('data')!), days);
|
||||
return;
|
||||
}
|
||||
final input = options.option('input');
|
||||
final output = options.option('output');
|
||||
if (!['jmdict', 'kanjidic', 'kanjivg'].contains(source) ||
|
||||
|
|
@ -127,6 +159,46 @@ Future<void> main(List<String> args) async {
|
|||
);
|
||||
}
|
||||
|
||||
/// Updates the dictionaries in [into]; with [every], keeps running and
|
||||
/// updates them whenever they are that many days old (as the files' dates
|
||||
/// say, so a restart doesn't download them again), trying again after six
|
||||
/// hours when an update fails.
|
||||
Future<void> _update(Directory into, Directory data, int? every) async {
|
||||
void say(String message) =>
|
||||
stdout.writeln('${DateTime.now().toUtc().toIso8601String()} $message');
|
||||
Future<bool> once() async {
|
||||
try {
|
||||
final written = await updateDictionaries(into: into, data: data);
|
||||
say(
|
||||
'Updated ${[for (final file in written) file.uri.pathSegments.last].join(', ')}.',
|
||||
);
|
||||
return true;
|
||||
} on Object catch (error) {
|
||||
say('Updating the dictionaries failed: $error');
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
if (every == null) {
|
||||
if (!await once()) {
|
||||
exit(1);
|
||||
}
|
||||
return;
|
||||
}
|
||||
for (;;) {
|
||||
final due = lastUpdated(into)?.add(Duration(days: every));
|
||||
final now = DateTime.now();
|
||||
if (due == null || !now.isBefore(due)) {
|
||||
if (!await once()) {
|
||||
await Future<void>.delayed(const Duration(hours: 6));
|
||||
}
|
||||
continue;
|
||||
}
|
||||
say('Up to date; the next update ${due.toUtc().toIso8601String()}.');
|
||||
await Future<void>.delayed(due.difference(now));
|
||||
}
|
||||
}
|
||||
|
||||
Never _fail(String message) {
|
||||
stderr.writeln(message);
|
||||
exit(64);
|
||||
|
|
|
|||
Loading…
Reference in a new issue