feat(curator): curate update keeps a server's dictionaries current
EDRDG's licence asks of servers showing its dictionaries that they update them at least monthly. `curate update --into <folder>` downloads EDRDG's files of the day (JMdict with Tatoeba's sentences, KANJIDIC2), converts them with the JLPT levels the curator now brings along (yomitan-jlpt-vocab's lists, CC BY-SA 4.0, and the kanji levels of kanji-data's kanji.json: both static, in data/ with their origin), and puts each file in place at once (written beside it as .part, then renamed over it), so the server never reads half a file; a failure leaves the files as they were. With --every 7 it keeps running and updates whenever the files are a week old, as their dates say, so a restart doesn't download again; after a failure it tries again in six hours. A real update takes some 50 seconds and 400 MB. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01MMm7usAo6q9ZZRoPkWjoxi Signed-off-by: Tizian 「ティツィアーン」 <raisondetredev@pm.me>
This commit is contained in:
parent
2d0ba20eb7
commit
82b38fadd8
1 changed files with 9047 additions and 3 deletions
84
halkyon_learn_curator/test/update_test.dart
Normal file
84
halkyon_learn_curator/test/update_test.dart
Normal file
|
|
@ -0,0 +1,84 @@
|
|||
import 'dart:convert';
|
||||
import 'dart:io';
|
||||
|
||||
import 'package:halkyon_learn_curator/src/update.dart';
|
||||
import 'package:test/test.dart';
|
||||
|
||||
void main() {
|
||||
late Directory into;
|
||||
|
||||
setUp(() => into = Directory.systemTemp.createTempSync('dictionaries'));
|
||||
tearDown(() => into.deleteSync(recursive: true));
|
||||
|
||||
/// Fetches the samples in place of EDRDG's files of the day.
|
||||
Future<void> samples(Uri uri, File file) async {
|
||||
final sample = uri == jmdictSource
|
||||
? 'test/fixtures/jmdict-e-examp-sample.xml'
|
||||
: 'test/fixtures/kanjidic2-sample.xml';
|
||||
await file.writeAsBytes(gzip.encode(await File(sample).readAsBytes()));
|
||||
}
|
||||
|
||||
/// The header of the dictionary file [name] in [into].
|
||||
Map<String, Object?> header(String name) => jsonDecode(
|
||||
utf8
|
||||
.decode(gzip.decode(File('${into.path}/$name').readAsBytesSync()))
|
||||
.split('\n')
|
||||
.first,
|
||||
) as Map<String, Object?>;
|
||||
|
||||
test("writes JMdict and KANJIDIC2 from EDRDG's files, the JLPT levels from "
|
||||
"the curator's data", () async {
|
||||
final written = await updateDictionaries(
|
||||
into: into,
|
||||
data: Directory('data'),
|
||||
fetch: samples,
|
||||
);
|
||||
|
||||
expect(
|
||||
[for (final file in written) file.uri.pathSegments.last],
|
||||
['jmdict-en.jsonl.gz', 'kanjidic-en.jsonl.gz'],
|
||||
);
|
||||
expect(header('jmdict-en.jsonl.gz')['sourceVersion'], '2026-10-09');
|
||||
expect(
|
||||
[
|
||||
for (final credit in header('jmdict-en.jsonl.gz')['credits']! as List)
|
||||
(credit as Map)['name'],
|
||||
],
|
||||
['JMdict', 'JLPT vocabulary lists', 'Tatoeba'],
|
||||
);
|
||||
expect(header('kanjidic-en.jsonl.gz')['kind'], 'characters');
|
||||
// Nothing half-written left behind.
|
||||
expect(
|
||||
[for (final entity in into.listSync()) entity.uri.pathSegments.last]
|
||||
..sort(),
|
||||
['jmdict-en.jsonl.gz', 'kanjidic-en.jsonl.gz'],
|
||||
);
|
||||
expect(lastUpdated(into), isNotNull);
|
||||
});
|
||||
|
||||
test('a download that fails leaves the files there as they were', () async {
|
||||
await updateDictionaries(
|
||||
into: into,
|
||||
data: Directory('data'),
|
||||
fetch: samples,
|
||||
);
|
||||
final before = File('${into.path}/jmdict-en.jsonl.gz').readAsBytesSync();
|
||||
|
||||
await expectLater(
|
||||
updateDictionaries(
|
||||
into: into,
|
||||
data: Directory('data'),
|
||||
fetch: (uri, file) => uri == kanjidicSource
|
||||
? Future.error(const HttpException('EDRDG answered 503.'))
|
||||
: samples(uri, file),
|
||||
),
|
||||
throwsA(isA<HttpException>()),
|
||||
);
|
||||
|
||||
expect(File('${into.path}/jmdict-en.jsonl.gz').readAsBytesSync(), before);
|
||||
});
|
||||
|
||||
test('a folder missing a file was never brought up to date', () {
|
||||
expect(lastUpdated(into), isNull);
|
||||
});
|
||||
}
|
||||
Loading…
Reference in a new issue