Tizian 「ティツィアーン」
649499be23
Its server, client, app and curator, and the tools that write its assets (maps, Anki fixtures, icons), as they are; paths are fixed in the commits that follow. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Tizian 「ティツィアーン」 <raisondetredev@pm.me>
478 lines
15 KiB
Dart
478 lines
15 KiB
Dart
import 'dart:convert';
|
|
import 'dart:io';
|
|
|
|
import 'package:archive/archive.dart';
|
|
|
|
import 'package:halkyon_learn_curator/src/sources/krdict.dart';
|
|
import 'package:halkyon_learn_curator/src/sources/wiktionary.dart';
|
|
import 'package:halkyon_learn_curator/src/update.dart';
|
|
import 'package:test/test.dart';
|
|
|
|
void main() {
|
|
late Directory into;
|
|
|
|
setUp(() => into = Directory.systemTemp.createTempSync('dictionaries'));
|
|
tearDown(() => into.deleteSync(recursive: true));
|
|
|
|
/// Fetches the samples in place of EDRDG's files of the day.
|
|
Future<void> samples(Uri uri, File file) async {
|
|
final sample = uri == jmdictSource
|
|
? 'test/fixtures/jmdict-e-examp-sample.xml'
|
|
: 'test/fixtures/kanjidic2-sample.xml';
|
|
await file.writeAsBytes(gzip.encode(await File(sample).readAsBytes()));
|
|
}
|
|
|
|
/// Tatoeba's exports, as the fixtures hold a few of their lines, packed
|
|
/// as Tatoeba packs them; null for any other address.
|
|
List<int>? tatoeba(Uri uri) {
|
|
final name = uri.pathSegments.last.replaceAll('.bz2', '');
|
|
final fixture = File('test/fixtures/tatoeba-$name');
|
|
return fixture.existsSync()
|
|
? BZip2Encoder().encodeBytes(fixture.readAsBytesSync())
|
|
: null;
|
|
}
|
|
|
|
/// The header of the dictionary file [name] in [into].
|
|
Map<String, Object?> header(String name) => jsonDecode(
|
|
utf8
|
|
.decode(gzip.decode(File('${into.path}/$name').readAsBytesSync()))
|
|
.split('\n')
|
|
.first,
|
|
) as Map<String, Object?>;
|
|
|
|
test("writes JMdict and KANJIDIC2 from EDRDG's files, the JLPT levels from "
|
|
"the curator's data", () async {
|
|
final written = await updateDictionaries(
|
|
into: into,
|
|
data: Directory('data'),
|
|
fetch: samples,
|
|
);
|
|
|
|
expect(
|
|
[for (final file in written) file.uri.pathSegments.last],
|
|
['jmdict-en.jsonl.gz', 'kanjidic-en.jsonl.gz'],
|
|
);
|
|
expect(header('jmdict-en.jsonl.gz')['sourceVersion'], '2026-10-09');
|
|
expect(
|
|
[
|
|
for (final credit in header('jmdict-en.jsonl.gz')['credits']! as List)
|
|
(credit as Map)['name'],
|
|
],
|
|
['JMdict', 'JLPT vocabulary lists', 'Tatoeba'],
|
|
);
|
|
expect(header('kanjidic-en.jsonl.gz')['kind'], 'characters');
|
|
// Nothing half-written left behind; which conversion made them.
|
|
expect(
|
|
[for (final entity in into.listSync()) entity.uri.pathSegments.last]
|
|
..sort(),
|
|
['.conversion', 'jmdict-en.jsonl.gz', 'kanjidic-en.jsonl.gz'],
|
|
);
|
|
expect(File('${into.path}/.conversion').readAsStringSync(), '$conversion');
|
|
expect(lastUpdated(into), isNotNull);
|
|
});
|
|
|
|
test('files another conversion made, or one from before the curator said, '
|
|
'are converted again at once', () async {
|
|
await updateDictionaries(
|
|
into: into,
|
|
data: Directory('data'),
|
|
fetch: samples,
|
|
);
|
|
final made = File('${into.path}/$conversionFile')
|
|
..writeAsStringSync('${conversion - 1}');
|
|
expect(lastUpdated(into), isNull);
|
|
|
|
made.deleteSync();
|
|
expect(lastUpdated(into), isNull);
|
|
});
|
|
|
|
test('a download that fails leaves the files there as they were', () async {
|
|
await updateDictionaries(
|
|
into: into,
|
|
data: Directory('data'),
|
|
fetch: samples,
|
|
);
|
|
final before = File('${into.path}/jmdict-en.jsonl.gz').readAsBytesSync();
|
|
|
|
await expectLater(
|
|
updateDictionaries(
|
|
into: into,
|
|
data: Directory('data'),
|
|
fetch: (uri, file) => uri == kanjidicSource
|
|
? Future.error(const HttpException('EDRDG answered 503.'))
|
|
: samples(uri, file),
|
|
),
|
|
throwsA(isA<HttpException>()),
|
|
);
|
|
|
|
expect(File('${into.path}/jmdict-en.jsonl.gz').readAsBytesSync(), before);
|
|
});
|
|
|
|
test('a folder missing a file was never brought up to date', () {
|
|
expect(lastUpdated(into), isNull);
|
|
});
|
|
|
|
group("Wiktionary's dump", () {
|
|
var dumped = DateTime.utc(2026, 10, 3, 8, 24);
|
|
var opened = 0;
|
|
|
|
setUp(() {
|
|
dumped = DateTime.utc(2026, 10, 3, 8, 24);
|
|
opened = 0;
|
|
});
|
|
|
|
Future<DateTime> modified(Uri uri) async => dumped;
|
|
|
|
/// The sample in place of kaikki.org's dump.
|
|
Future<Download> open(Uri uri) async {
|
|
opened++;
|
|
return (
|
|
modified: dumped,
|
|
bytes: Stream.value(
|
|
gzip.encode(
|
|
File('test/fixtures/wiktionary-sample.jsonl').readAsBytesSync(),
|
|
),
|
|
),
|
|
);
|
|
}
|
|
|
|
Future<List<File>> update(Set<String> languages, {bool korean = false}) =>
|
|
updateWiktionary(
|
|
into: into,
|
|
data: Directory('data'),
|
|
languages: languages,
|
|
korean: korean,
|
|
modified: modified,
|
|
open: open,
|
|
fetch: (uri, file) => file.writeAsBytes(
|
|
tatoeba(uri) ?? (throw HttpException('$uri', uri: uri)),
|
|
),
|
|
);
|
|
|
|
test(
|
|
'is converted for the languages named, German and Greek with English '
|
|
'meanings, English with German ones, and ranked by frequency',
|
|
() async {
|
|
final written = await update({'de', 'el', 'en'});
|
|
|
|
expect(
|
|
[for (final file in written) file.uri.pathSegments.last]..sort(),
|
|
[
|
|
'wiktionary-de-en.jsonl.gz',
|
|
'wiktionary-el-en.jsonl.gz',
|
|
'wiktionary-en-de.jsonl.gz',
|
|
],
|
|
);
|
|
expect(
|
|
header('wiktionary-de-en.jsonl.gz')['sourceVersion'],
|
|
'2026-10-03',
|
|
);
|
|
expect([
|
|
for (final credit
|
|
in header('wiktionary-de-en.jsonl.gz')['credits']! as List)
|
|
(credit as Map)['name'],
|
|
], contains('wordfreq'));
|
|
expect(File('${into.path}/$wiktionaryStateFile').existsSync(), isTrue);
|
|
},
|
|
);
|
|
|
|
test(
|
|
'is downloaded again only once it changed, or the languages did',
|
|
() async {
|
|
await update({'de'});
|
|
expect(await update({'de'}), isEmpty);
|
|
expect(opened, 1);
|
|
|
|
dumped = DateTime.utc(2026, 11, 2);
|
|
expect(await update({'de'}), hasLength(1));
|
|
expect(opened, 2);
|
|
|
|
expect(await update({'de', 'el'}), hasLength(2));
|
|
expect(opened, 3);
|
|
},
|
|
);
|
|
|
|
test('nothing is asked for without languages', () async {
|
|
expect(await update({}), isEmpty);
|
|
expect(opened, 0);
|
|
});
|
|
|
|
test("keeps what Korean's dictionaries take from it, even without "
|
|
'languages of its words', () async {
|
|
expect(await update({}, korean: true), isEmpty);
|
|
expect(opened, 1);
|
|
|
|
final kept = WiktionaryKorean.fromJson(
|
|
utf8.decode(
|
|
gzip.decode(
|
|
File('${into.path}/$wiktionaryKoreanFile').readAsBytesSync(),
|
|
),
|
|
),
|
|
);
|
|
expect(kept.hanja['水'], [(hun: '물', eum: '수')]);
|
|
expect(await update({}, korean: true), isEmpty);
|
|
expect(opened, 1);
|
|
});
|
|
});
|
|
|
|
test('CC-CEDICT is converted in both scripts with the HSK levels, and '
|
|
'said to be up to date by its files', () async {
|
|
expect(cedictUpdated(into), isNull);
|
|
|
|
final written = await updateCedict(
|
|
into: into,
|
|
data: Directory('data'),
|
|
fetch: (uri, file) => file.writeAsBytes(
|
|
uri == cedictSource
|
|
? gzip.encode(
|
|
File('test/fixtures/cedict-sample.txt').readAsBytesSync(),
|
|
)
|
|
: tatoeba(uri) ?? (throw HttpException('$uri', uri: uri)),
|
|
),
|
|
);
|
|
|
|
expect(
|
|
[for (final file in written) file.uri.pathSegments.last],
|
|
['cc-cedict-hans-en.jsonl.gz', 'cc-cedict-hant-en.jsonl.gz'],
|
|
);
|
|
expect(header('cc-cedict-hant-en.jsonl.gz')['language'], 'zh-Hant');
|
|
expect([
|
|
for (final credit
|
|
in header('cc-cedict-hans-en.jsonl.gz')['credits']! as List)
|
|
(credit as Map)['name'],
|
|
], contains('Tatoeba'));
|
|
expect(cedictUpdated(into), isNotNull);
|
|
// Nothing of the downloads left beside the files.
|
|
expect(
|
|
[for (final entity in into.listSync()) entity.uri.pathSegments.last]
|
|
..sort(),
|
|
['cc-cedict-hans-en.jsonl.gz', 'cc-cedict-hant-en.jsonl.gz'],
|
|
);
|
|
});
|
|
|
|
test("a dictionary keeps its words without Tatoeba's sentences when they "
|
|
"can't be had", () async {
|
|
final written = await updateCedict(
|
|
into: into,
|
|
data: Directory('data'),
|
|
fetch: (uri, file) => uri == cedictSource
|
|
? file.writeAsBytes(
|
|
gzip.encode(
|
|
File('test/fixtures/cedict-sample.txt').readAsBytesSync(),
|
|
),
|
|
)
|
|
: throw HttpException('$uri answered 503.', uri: uri),
|
|
);
|
|
|
|
expect(written, hasLength(2));
|
|
expect([
|
|
for (final credit
|
|
in header('cc-cedict-hans-en.jsonl.gz')['credits']! as List)
|
|
(credit as Map)['name'],
|
|
], isNot(contains('Tatoeba')));
|
|
});
|
|
|
|
test('Chinese comes from CC-CEDICT, Korean from KRDict, the other '
|
|
"languages' words from Wiktionary", () {
|
|
final all = dictionarySources({'de', 'el', 'en', 'ko', 'zh'});
|
|
expect(all.wiktionary, {'de', 'el', 'en'});
|
|
expect(all.chinese, isTrue);
|
|
expect(all.korean, isTrue);
|
|
expect(all.greek, isTrue);
|
|
final traditional = dictionarySources({'zh-Hant'});
|
|
expect(traditional.wiktionary, isEmpty);
|
|
expect(traditional.chinese, isTrue);
|
|
expect(traditional.korean, isFalse);
|
|
expect(dictionarySources({}).chinese, isFalse);
|
|
});
|
|
|
|
test('files of languages no longer kept, or of a source replaced, go; an '
|
|
"operator's own stay", () {
|
|
for (final name in [
|
|
'jmdict-en.jsonl.gz',
|
|
'wiktionary-de-en.jsonl.gz',
|
|
'wiktionary-ko-en.jsonl.gz',
|
|
'wiktionary-el-en.jsonl.gz',
|
|
'krdict-ko-en.jsonl.gz',
|
|
'unihan-ko-en.jsonl.gz',
|
|
'greek-alphabet-en.jsonl.gz',
|
|
'my-own.jsonl.gz',
|
|
]) {
|
|
File('${into.path}/$name').writeAsStringSync('');
|
|
}
|
|
|
|
final removed = removeUnwanted(into, {'de', 'ko'});
|
|
|
|
expect([for (final file in removed) file.uri.pathSegments.last]..sort(), [
|
|
'greek-alphabet-en.jsonl.gz',
|
|
'wiktionary-el-en.jsonl.gz',
|
|
'wiktionary-ko-en.jsonl.gz',
|
|
]);
|
|
expect(
|
|
[for (final entity in into.listSync()) entity.uri.pathSegments.last]
|
|
..sort(),
|
|
[
|
|
'jmdict-en.jsonl.gz',
|
|
'krdict-ko-en.jsonl.gz',
|
|
'my-own.jsonl.gz',
|
|
'unihan-ko-en.jsonl.gz',
|
|
'wiktionary-de-en.jsonl.gz',
|
|
],
|
|
);
|
|
});
|
|
|
|
test("a replaced source stays until what replaces it is there: Korean's "
|
|
"words from Wiktionary until KRDict's are", () {
|
|
File('${into.path}/wiktionary-ko-en.jsonl.gz').writeAsStringSync('');
|
|
|
|
expect(removeUnwanted(into, {'ko'}), isEmpty);
|
|
|
|
File('${into.path}/krdict-ko-en.jsonl.gz').writeAsStringSync('');
|
|
expect(
|
|
[
|
|
for (final file in removeUnwanted(into, {'ko'}))
|
|
file.uri.pathSegments.last,
|
|
],
|
|
['wiktionary-ko-en.jsonl.gz'],
|
|
);
|
|
});
|
|
|
|
group("KRDict's export", () {
|
|
var number = '217';
|
|
var downloads = 0;
|
|
|
|
setUp(() {
|
|
number = '217';
|
|
downloads = 0;
|
|
});
|
|
|
|
/// KRDict's page and the fixture packed as its export; Tatoeba's.
|
|
Future<void> fetch(Uri uri, File file) async {
|
|
if (uri == krdictPage) {
|
|
await file.writeAsString(
|
|
"<button onclick=\"batchDownload('215')\"></button>"
|
|
"<button onclick=\"batchDownload('216')\"></button>"
|
|
"<button onclick=\"batchDownload('$number')\"></button>",
|
|
);
|
|
} else if (uri == krdictDownloadOf(number)) {
|
|
downloads++;
|
|
await file.writeAsBytes(
|
|
ZipEncoder().encodeBytes(
|
|
Archive()..add(
|
|
ArchiveFile.bytes(
|
|
'1_5000_20260919.json',
|
|
File('test/fixtures/krdict-sample.json').readAsBytesSync(),
|
|
),
|
|
),
|
|
),
|
|
);
|
|
} else {
|
|
await file.writeAsBytes(
|
|
tatoeba(uri) ?? (throw HttpException('$uri', uri: uri)),
|
|
);
|
|
}
|
|
}
|
|
|
|
test('is converted with the forms Wiktionary kept, and again only '
|
|
'once its page offers another', () async {
|
|
final kept = WiktionaryKorean()..forms['가다'] = {'갔어요'};
|
|
File('${into.path}/$wiktionaryKoreanFile')
|
|
.writeAsBytesSync(gzip.encode(utf8.encode(kept.toJson())));
|
|
|
|
final written = await updateKrdict(
|
|
into: into,
|
|
data: Directory('data'),
|
|
fetch: fetch,
|
|
);
|
|
|
|
expect(
|
|
[for (final file in written) file.uri.pathSegments.last],
|
|
[krdictFileName],
|
|
);
|
|
expect(header(krdictFileName)['sourceVersion'], '2026-09-19');
|
|
expect([
|
|
for (final credit in header(krdictFileName)['credits']! as List)
|
|
(credit as Map)['name'],
|
|
], contains('Wiktionary'));
|
|
expect(
|
|
await updateKrdict(into: into, data: Directory('data'), fetch: fetch),
|
|
isEmpty,
|
|
);
|
|
expect(downloads, 1);
|
|
|
|
number = '230';
|
|
expect(
|
|
await updateKrdict(into: into, data: Directory('data'), fetch: fetch),
|
|
hasLength(1),
|
|
);
|
|
expect(downloads, 2);
|
|
expect(
|
|
[for (final entity in into.listSync()) entity.uri.pathSegments.last]
|
|
..sort(),
|
|
['.krdict', wiktionaryKoreanFile, krdictFileName]..sort(),
|
|
);
|
|
});
|
|
});
|
|
|
|
test('the characters of Chinese in both scripts, of Korean with the Hangul '
|
|
'letters and the Greek alphabet, up to date until a Wiktionary update '
|
|
'keeps new Hanja readings', () async {
|
|
Future<void> fetch(Uri uri, File file) async => await file.writeAsBytes(
|
|
uri == unihanSource
|
|
? ZipEncoder().encodeBytes(
|
|
Archive()..add(
|
|
ArchiveFile.bytes(
|
|
'Unihan_Readings.txt',
|
|
File('test/fixtures/unihan-sample.txt').readAsBytesSync(),
|
|
),
|
|
),
|
|
)
|
|
: gzip.encode(
|
|
File('test/fixtures/cedict-characters-sample.txt')
|
|
.readAsBytesSync(),
|
|
),
|
|
);
|
|
expect(
|
|
charactersUpdated(into, chinese: true, korean: true, greek: true),
|
|
isNull,
|
|
);
|
|
|
|
final written = await updateCharacters(
|
|
into: into,
|
|
data: Directory('data'),
|
|
chinese: true,
|
|
korean: true,
|
|
greek: true,
|
|
fetch: fetch,
|
|
);
|
|
|
|
expect(
|
|
[for (final file in written) file.uri.pathSegments.last],
|
|
[
|
|
'unihan-hans-en.jsonl.gz',
|
|
'unihan-hant-en.jsonl.gz',
|
|
'unihan-ko-en.jsonl.gz',
|
|
'greek-alphabet-en.jsonl.gz',
|
|
],
|
|
);
|
|
expect(header('unihan-hans-en.jsonl.gz')['sourceVersion'], '18.0.0');
|
|
expect([
|
|
for (final credit
|
|
in header('unihan-hans-en.jsonl.gz')['credits']! as List)
|
|
(credit as Map)['name'],
|
|
], containsAll(['Unihan Database', 'CC-CEDICT', 'wordfreq']));
|
|
final updated = charactersUpdated(
|
|
into,
|
|
chinese: true,
|
|
korean: true,
|
|
greek: true,
|
|
);
|
|
expect(updated, isNotNull);
|
|
|
|
File('${into.path}/$wiktionaryKoreanFile')
|
|
..writeAsBytesSync(gzip.encode(utf8.encode(WiktionaryKorean().toJson())))
|
|
..setLastModifiedSync(updated!.add(const Duration(minutes: 1)));
|
|
expect(charactersUpdated(into, korean: true), isNull);
|
|
expect(charactersUpdated(into, chinese: true), isNotNull);
|
|
});
|
|
}
|