feat(curator): curate update keeps a server's dictionaries current

EDRDG's licence asks of servers showing its dictionaries that they update
them at least monthly. `curate update --into <folder>` downloads EDRDG's
files of the day (JMdict with Tatoeba's sentences, KANJIDIC2), converts
them with the JLPT levels the curator now brings along (yomitan-jlpt-vocab's
lists, CC BY-SA 4.0, and the kanji levels of kanji-data's kanji.json: both
static, in data/ with their origin), and puts each file in place at once
(written beside it as .part, then renamed over it), so the server never
reads half a file; a failure leaves the files as they were. With --every 7
it keeps running and updates whenever the files are a week old, as their
dates say, so a restart doesn't download again; after a failure it tries
again in six hours. A real update takes some 50 seconds and 400 MB.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01MMm7usAo6q9ZZRoPkWjoxi
Signed-off-by: Tizian 「ティツィアーン」 <raisondetredev@pm.me>
This commit is contained in:
Tizian 「ティツィアーン」 2026-10-09 16:40:19 +02:00
commit 82b38fadd8
Signed by: raisondetredev
GPG key ID: 07F6FCF405488B64

View file

@ -0,0 +1,84 @@
import 'dart:convert';
import 'dart:io';
import 'package:halkyon_learn_curator/src/update.dart';
import 'package:test/test.dart';
void main() {
late Directory into;
setUp(() => into = Directory.systemTemp.createTempSync('dictionaries'));
tearDown(() => into.deleteSync(recursive: true));
/// Fetches the samples in place of EDRDG's files of the day.
Future<void> samples(Uri uri, File file) async {
final sample = uri == jmdictSource
? 'test/fixtures/jmdict-e-examp-sample.xml'
: 'test/fixtures/kanjidic2-sample.xml';
await file.writeAsBytes(gzip.encode(await File(sample).readAsBytes()));
}
/// The header of the dictionary file [name] in [into].
Map<String, Object?> header(String name) => jsonDecode(
utf8
.decode(gzip.decode(File('${into.path}/$name').readAsBytesSync()))
.split('\n')
.first,
) as Map<String, Object?>;
test("writes JMdict and KANJIDIC2 from EDRDG's files, the JLPT levels from "
"the curator's data", () async {
final written = await updateDictionaries(
into: into,
data: Directory('data'),
fetch: samples,
);
expect(
[for (final file in written) file.uri.pathSegments.last],
['jmdict-en.jsonl.gz', 'kanjidic-en.jsonl.gz'],
);
expect(header('jmdict-en.jsonl.gz')['sourceVersion'], '2026-10-09');
expect(
[
for (final credit in header('jmdict-en.jsonl.gz')['credits']! as List)
(credit as Map)['name'],
],
['JMdict', 'JLPT vocabulary lists', 'Tatoeba'],
);
expect(header('kanjidic-en.jsonl.gz')['kind'], 'characters');
// Nothing half-written left behind.
expect(
[for (final entity in into.listSync()) entity.uri.pathSegments.last]
..sort(),
['jmdict-en.jsonl.gz', 'kanjidic-en.jsonl.gz'],
);
expect(lastUpdated(into), isNotNull);
});
test('a download that fails leaves the files there as they were', () async {
await updateDictionaries(
into: into,
data: Directory('data'),
fetch: samples,
);
final before = File('${into.path}/jmdict-en.jsonl.gz').readAsBytesSync();
await expectLater(
updateDictionaries(
into: into,
data: Directory('data'),
fetch: (uri, file) => uri == kanjidicSource
? Future.error(const HttpException('EDRDG answered 503.'))
: samples(uri, file),
),
throwsA(isA<HttpException>()),
);
expect(File('${into.path}/jmdict-en.jsonl.gz').readAsBytesSync(), before);
});
test('a folder missing a file was never brought up to date', () {
expect(lastUpdated(into), isNull);
});
}