feat(curator): curate update keeps a server's dictionaries current
EDRDG's licence asks of servers showing its dictionaries that they update
them at least monthly. `curate update --into <folder>` downloads EDRDG's
files of the day (JMdict with Tatoeba's sentences, KANJIDIC2), converts
them with the JLPT levels the curator now brings along (yomitan-jlpt-vocab's
lists, CC BY-SA 4.0, and the kanji levels of kanji-data's kanji.json: both
static, in data/ with their origin), and puts each file in place at once
(written beside it as .part, then renamed over it), so the server never
reads half a file; a failure leaves the files as they were. With --every 7
it keeps running and updates whenever the files are a week old, as their
dates say, so a restart doesn't download again; after a failure it tries
again in six hours. A real update takes some 50 seconds and 400 MB.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01MMm7usAo6q9ZZRoPkWjoxi
Signed-off-by: Tizian 「ティツィアーン」 <raisondetredev@pm.me>
2026-10-09 16:40:19 +02:00
|
|
|
import 'dart:convert';
|
|
|
|
|
import 'dart:io';
|
|
|
|
|
|
|
|
|
|
import 'package:halkyon_learn_curator/src/update.dart';
|
|
|
|
|
import 'package:test/test.dart';
|
|
|
|
|
|
|
|
|
|
void main() {
|
|
|
|
|
late Directory into;
|
|
|
|
|
|
|
|
|
|
setUp(() => into = Directory.systemTemp.createTempSync('dictionaries'));
|
|
|
|
|
tearDown(() => into.deleteSync(recursive: true));
|
|
|
|
|
|
|
|
|
|
/// Fetches the samples in place of EDRDG's files of the day.
|
|
|
|
|
Future<void> samples(Uri uri, File file) async {
|
|
|
|
|
final sample = uri == jmdictSource
|
|
|
|
|
? 'test/fixtures/jmdict-e-examp-sample.xml'
|
|
|
|
|
: 'test/fixtures/kanjidic2-sample.xml';
|
|
|
|
|
await file.writeAsBytes(gzip.encode(await File(sample).readAsBytes()));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/// The header of the dictionary file [name] in [into].
|
|
|
|
|
Map<String, Object?> header(String name) => jsonDecode(
|
|
|
|
|
utf8
|
|
|
|
|
.decode(gzip.decode(File('${into.path}/$name').readAsBytesSync()))
|
|
|
|
|
.split('\n')
|
|
|
|
|
.first,
|
|
|
|
|
) as Map<String, Object?>;
|
|
|
|
|
|
|
|
|
|
test("writes JMdict and KANJIDIC2 from EDRDG's files, the JLPT levels from "
|
|
|
|
|
"the curator's data", () async {
|
|
|
|
|
final written = await updateDictionaries(
|
|
|
|
|
into: into,
|
|
|
|
|
data: Directory('data'),
|
|
|
|
|
fetch: samples,
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
expect(
|
|
|
|
|
[for (final file in written) file.uri.pathSegments.last],
|
|
|
|
|
['jmdict-en.jsonl.gz', 'kanjidic-en.jsonl.gz'],
|
|
|
|
|
);
|
|
|
|
|
expect(header('jmdict-en.jsonl.gz')['sourceVersion'], '2026-10-09');
|
|
|
|
|
expect(
|
|
|
|
|
[
|
|
|
|
|
for (final credit in header('jmdict-en.jsonl.gz')['credits']! as List)
|
|
|
|
|
(credit as Map)['name'],
|
|
|
|
|
],
|
|
|
|
|
['JMdict', 'JLPT vocabulary lists', 'Tatoeba'],
|
|
|
|
|
);
|
|
|
|
|
expect(header('kanjidic-en.jsonl.gz')['kind'], 'characters');
|
feat(dictionary): the radicals, each listing the kanji written with it
Beside Words and Kanji & kana, a Radicals tab lists the 214 Kangxi
radicals by their strokes, with the forms Japanese writes them as inside
kanji (亻, 氵, 艹, …). One chosen lists the kanji KANJIDIC2 files under
it, in the dictionary's order, to add or to see how one is written; the
tab shares the characters' bloc, so a kanji added there shows as held in
a search too.
The curator labels each kanji with its classical radical (`radical:N`),
so the server's existing label queries serve the tab unchanged. Its
files now say which conversion made them (`.conversion`): an instance
running a curator that converts differently converts again at once,
rather than at its weekly update, so the radicals appear right after a
deploy.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01MMm7usAo6q9ZZRoPkWjoxi
Signed-off-by: Tizian 「ティツィアーン」 <raisondetredev@pm.me>
2026-10-10 02:13:06 +02:00
|
|
|
// Nothing half-written left behind; which conversion made them.
|
feat(curator): curate update keeps a server's dictionaries current
EDRDG's licence asks of servers showing its dictionaries that they update
them at least monthly. `curate update --into <folder>` downloads EDRDG's
files of the day (JMdict with Tatoeba's sentences, KANJIDIC2), converts
them with the JLPT levels the curator now brings along (yomitan-jlpt-vocab's
lists, CC BY-SA 4.0, and the kanji levels of kanji-data's kanji.json: both
static, in data/ with their origin), and puts each file in place at once
(written beside it as .part, then renamed over it), so the server never
reads half a file; a failure leaves the files as they were. With --every 7
it keeps running and updates whenever the files are a week old, as their
dates say, so a restart doesn't download again; after a failure it tries
again in six hours. A real update takes some 50 seconds and 400 MB.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01MMm7usAo6q9ZZRoPkWjoxi
Signed-off-by: Tizian 「ティツィアーン」 <raisondetredev@pm.me>
2026-10-09 16:40:19 +02:00
|
|
|
expect(
|
|
|
|
|
[for (final entity in into.listSync()) entity.uri.pathSegments.last]
|
|
|
|
|
..sort(),
|
feat(dictionary): the radicals, each listing the kanji written with it
Beside Words and Kanji & kana, a Radicals tab lists the 214 Kangxi
radicals by their strokes, with the forms Japanese writes them as inside
kanji (亻, 氵, 艹, …). One chosen lists the kanji KANJIDIC2 files under
it, in the dictionary's order, to add or to see how one is written; the
tab shares the characters' bloc, so a kanji added there shows as held in
a search too.
The curator labels each kanji with its classical radical (`radical:N`),
so the server's existing label queries serve the tab unchanged. Its
files now say which conversion made them (`.conversion`): an instance
running a curator that converts differently converts again at once,
rather than at its weekly update, so the radicals appear right after a
deploy.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01MMm7usAo6q9ZZRoPkWjoxi
Signed-off-by: Tizian 「ティツィアーン」 <raisondetredev@pm.me>
2026-10-10 02:13:06 +02:00
|
|
|
['.conversion', 'jmdict-en.jsonl.gz', 'kanjidic-en.jsonl.gz'],
|
feat(curator): curate update keeps a server's dictionaries current
EDRDG's licence asks of servers showing its dictionaries that they update
them at least monthly. `curate update --into <folder>` downloads EDRDG's
files of the day (JMdict with Tatoeba's sentences, KANJIDIC2), converts
them with the JLPT levels the curator now brings along (yomitan-jlpt-vocab's
lists, CC BY-SA 4.0, and the kanji levels of kanji-data's kanji.json: both
static, in data/ with their origin), and puts each file in place at once
(written beside it as .part, then renamed over it), so the server never
reads half a file; a failure leaves the files as they were. With --every 7
it keeps running and updates whenever the files are a week old, as their
dates say, so a restart doesn't download again; after a failure it tries
again in six hours. A real update takes some 50 seconds and 400 MB.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01MMm7usAo6q9ZZRoPkWjoxi
Signed-off-by: Tizian 「ティツィアーン」 <raisondetredev@pm.me>
2026-10-09 16:40:19 +02:00
|
|
|
);
|
feat(dictionary): the radicals, each listing the kanji written with it
Beside Words and Kanji & kana, a Radicals tab lists the 214 Kangxi
radicals by their strokes, with the forms Japanese writes them as inside
kanji (亻, 氵, 艹, …). One chosen lists the kanji KANJIDIC2 files under
it, in the dictionary's order, to add or to see how one is written; the
tab shares the characters' bloc, so a kanji added there shows as held in
a search too.
The curator labels each kanji with its classical radical (`radical:N`),
so the server's existing label queries serve the tab unchanged. Its
files now say which conversion made them (`.conversion`): an instance
running a curator that converts differently converts again at once,
rather than at its weekly update, so the radicals appear right after a
deploy.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01MMm7usAo6q9ZZRoPkWjoxi
Signed-off-by: Tizian 「ティツィアーン」 <raisondetredev@pm.me>
2026-10-10 02:13:06 +02:00
|
|
|
expect(File('${into.path}/.conversion').readAsStringSync(), '$conversion');
|
feat(curator): curate update keeps a server's dictionaries current
EDRDG's licence asks of servers showing its dictionaries that they update
them at least monthly. `curate update --into <folder>` downloads EDRDG's
files of the day (JMdict with Tatoeba's sentences, KANJIDIC2), converts
them with the JLPT levels the curator now brings along (yomitan-jlpt-vocab's
lists, CC BY-SA 4.0, and the kanji levels of kanji-data's kanji.json: both
static, in data/ with their origin), and puts each file in place at once
(written beside it as .part, then renamed over it), so the server never
reads half a file; a failure leaves the files as they were. With --every 7
it keeps running and updates whenever the files are a week old, as their
dates say, so a restart doesn't download again; after a failure it tries
again in six hours. A real update takes some 50 seconds and 400 MB.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01MMm7usAo6q9ZZRoPkWjoxi
Signed-off-by: Tizian 「ティツィアーン」 <raisondetredev@pm.me>
2026-10-09 16:40:19 +02:00
|
|
|
expect(lastUpdated(into), isNotNull);
|
|
|
|
|
});
|
|
|
|
|
|
feat(dictionary): the radicals, each listing the kanji written with it
Beside Words and Kanji & kana, a Radicals tab lists the 214 Kangxi
radicals by their strokes, with the forms Japanese writes them as inside
kanji (亻, 氵, 艹, …). One chosen lists the kanji KANJIDIC2 files under
it, in the dictionary's order, to add or to see how one is written; the
tab shares the characters' bloc, so a kanji added there shows as held in
a search too.
The curator labels each kanji with its classical radical (`radical:N`),
so the server's existing label queries serve the tab unchanged. Its
files now say which conversion made them (`.conversion`): an instance
running a curator that converts differently converts again at once,
rather than at its weekly update, so the radicals appear right after a
deploy.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01MMm7usAo6q9ZZRoPkWjoxi
Signed-off-by: Tizian 「ティツィアーン」 <raisondetredev@pm.me>
2026-10-10 02:13:06 +02:00
|
|
|
test('files another conversion made, or one from before the curator said, '
|
|
|
|
|
'are converted again at once', () async {
|
|
|
|
|
await updateDictionaries(
|
|
|
|
|
into: into,
|
|
|
|
|
data: Directory('data'),
|
|
|
|
|
fetch: samples,
|
|
|
|
|
);
|
|
|
|
|
final made = File('${into.path}/$conversionFile')
|
|
|
|
|
..writeAsStringSync('${conversion - 1}');
|
|
|
|
|
expect(lastUpdated(into), isNull);
|
|
|
|
|
|
|
|
|
|
made.deleteSync();
|
|
|
|
|
expect(lastUpdated(into), isNull);
|
|
|
|
|
});
|
|
|
|
|
|
feat(curator): curate update keeps a server's dictionaries current
EDRDG's licence asks of servers showing its dictionaries that they update
them at least monthly. `curate update --into <folder>` downloads EDRDG's
files of the day (JMdict with Tatoeba's sentences, KANJIDIC2), converts
them with the JLPT levels the curator now brings along (yomitan-jlpt-vocab's
lists, CC BY-SA 4.0, and the kanji levels of kanji-data's kanji.json: both
static, in data/ with their origin), and puts each file in place at once
(written beside it as .part, then renamed over it), so the server never
reads half a file; a failure leaves the files as they were. With --every 7
it keeps running and updates whenever the files are a week old, as their
dates say, so a restart doesn't download again; after a failure it tries
again in six hours. A real update takes some 50 seconds and 400 MB.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01MMm7usAo6q9ZZRoPkWjoxi
Signed-off-by: Tizian 「ティツィアーン」 <raisondetredev@pm.me>
2026-10-09 16:40:19 +02:00
|
|
|
test('a download that fails leaves the files there as they were', () async {
|
|
|
|
|
await updateDictionaries(
|
|
|
|
|
into: into,
|
|
|
|
|
data: Directory('data'),
|
|
|
|
|
fetch: samples,
|
|
|
|
|
);
|
|
|
|
|
final before = File('${into.path}/jmdict-en.jsonl.gz').readAsBytesSync();
|
|
|
|
|
|
|
|
|
|
await expectLater(
|
|
|
|
|
updateDictionaries(
|
|
|
|
|
into: into,
|
|
|
|
|
data: Directory('data'),
|
|
|
|
|
fetch: (uri, file) => uri == kanjidicSource
|
|
|
|
|
? Future.error(const HttpException('EDRDG answered 503.'))
|
|
|
|
|
: samples(uri, file),
|
|
|
|
|
),
|
|
|
|
|
throwsA(isA<HttpException>()),
|
|
|
|
|
);
|
|
|
|
|
|
|
|
|
|
expect(File('${into.path}/jmdict-en.jsonl.gz').readAsBytesSync(), before);
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
test('a folder missing a file was never brought up to date', () {
|
|
|
|
|
expect(lastUpdated(into), isNull);
|
|
|
|
|
});
|
|
|
|
|
}
|