Tizian 「ティツィアーン」
3257905536
The curator puts Tatoeba's readings on a dictionary's examples by their Tatoeba ID (`withTatoebaReadings`, `curate readings`, and in `update`): furigana over JMdict's Japanese from its jpn transcriptions, pinyin with tone marks over Chinese from its cmn ones, each word over as many Han characters as it has syllables. An example gets `ruby`, `[start, end, reading]` each, and `rubyUnreviewed` where no one wrote or checked them (MeCab's furigana, the automatic pinyin). JMdict's conversion goes to 3, so instances convert again at once. A word's example hints carry them as ruby marks, the word still in bold. The hints sheet shows them from the start with a switch above (Furigana, Pinyin), kept on the device (`DeviceSettings.hintReadings`), and below a hint whose readings no one reviewed says so. Whats-New: Example sentences in a word's hints now show furigana or pinyin over them, with a switch to hide them. Whats-New-De: Beispielsätze in den Hinweisen eines Worts zeigen jetzt Furigana oder Pinyin darüber, mit einem Schalter zum Ausblenden. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Tizian 「ティツィアーン」 <raisondetredev@pm.me>
521 lines
16 KiB
Dart
521 lines
16 KiB
Dart
import 'dart:convert';
|
|
import 'dart:io';
|
|
|
|
import 'package:archive/archive.dart';
|
|
|
|
import 'package:halkyon_learn_curator/src/sources/krdict.dart';
|
|
import 'package:halkyon_learn_curator/src/sources/wiktionary.dart';
|
|
import 'package:halkyon_learn_curator/src/update.dart';
|
|
import 'package:test/test.dart';
|
|
|
|
void main() {
|
|
late Directory into;
|
|
|
|
setUp(() => into = Directory.systemTemp.createTempSync('dictionaries'));
|
|
tearDown(() => into.deleteSync(recursive: true));
|
|
|
|
/// Tatoeba's exports, as the fixtures hold a few of their lines, packed
|
|
/// as Tatoeba packs them; null for any other address.
|
|
List<int>? tatoeba(Uri uri) {
|
|
final name = uri.pathSegments.last.replaceAll('.bz2', '');
|
|
final fixture = File('test/fixtures/tatoeba-$name');
|
|
return fixture.existsSync()
|
|
? BZip2Encoder().encodeBytes(fixture.readAsBytesSync())
|
|
: null;
|
|
}
|
|
|
|
/// Fetches the samples in place of EDRDG's files of the day, and
|
|
/// Tatoeba's transcriptions.
|
|
Future<void> samples(Uri uri, File file) async {
|
|
if (tatoeba(uri) case final packed?) {
|
|
await file.writeAsBytes(packed);
|
|
return;
|
|
}
|
|
final sample = uri == jmdictSource
|
|
? 'test/fixtures/jmdict-e-examp-sample.xml'
|
|
: 'test/fixtures/kanjidic2-sample.xml';
|
|
await file.writeAsBytes(gzip.encode(await File(sample).readAsBytes()));
|
|
}
|
|
|
|
/// The examples of every sense in the dictionary file [name] in [into].
|
|
List<Map<String, Object?>> examplesIn(String name) => [
|
|
for (final line
|
|
in utf8
|
|
.decode(gzip.decode(File('${into.path}/$name').readAsBytesSync()))
|
|
.split('\n')
|
|
.skip(1))
|
|
if (line.isNotEmpty)
|
|
for (final sense
|
|
in ((jsonDecode(line) as Map)['entry'] as Map)['senses'] as List)
|
|
for (final example in (sense as Map)['examples'] as List? ?? const [])
|
|
example as Map<String, Object?>,
|
|
];
|
|
|
|
/// The header of the dictionary file [name] in [into].
|
|
Map<String, Object?> header(String name) => jsonDecode(
|
|
utf8
|
|
.decode(gzip.decode(File('${into.path}/$name').readAsBytesSync()))
|
|
.split('\n')
|
|
.first,
|
|
) as Map<String, Object?>;
|
|
|
|
test("writes JMdict and KANJIDIC2 from EDRDG's files, the JLPT levels from "
|
|
"the curator's data", () async {
|
|
final written = await updateDictionaries(
|
|
into: into,
|
|
data: Directory('data'),
|
|
fetch: samples,
|
|
);
|
|
|
|
expect(
|
|
[for (final file in written) file.uri.pathSegments.last],
|
|
['jmdict-en.jsonl.gz', 'kanjidic-en.jsonl.gz'],
|
|
);
|
|
expect(header('jmdict-en.jsonl.gz')['sourceVersion'], '2026-10-09');
|
|
expect(
|
|
[
|
|
for (final credit in header('jmdict-en.jsonl.gz')['credits']! as List)
|
|
(credit as Map)['name'],
|
|
],
|
|
['JMdict', 'JLPT vocabulary lists', 'Tatoeba'],
|
|
);
|
|
expect(header('kanjidic-en.jsonl.gz')['kind'], 'characters');
|
|
// Furigana over the sentences Tatoeba transcribed, those MeCab's alone
|
|
// marked.
|
|
final examples = {
|
|
for (final example in examplesIn('jmdict-en.jsonl.gz'))
|
|
'${example['tatoeba']}': example,
|
|
};
|
|
expect(examples['193344']?['ruby'], [
|
|
[3, 4, 'くだ'],
|
|
[4, 5, 'もの'],
|
|
[6, 7, 'た'],
|
|
]);
|
|
expect(examples['193344'], isNot(contains('rubyUnreviewed')));
|
|
expect(examples['81964']?['rubyUnreviewed'], isTrue);
|
|
// Nothing half-written left behind; which conversion made them.
|
|
expect(
|
|
[for (final entity in into.listSync()) entity.uri.pathSegments.last]
|
|
..sort(),
|
|
['.conversion', 'jmdict-en.jsonl.gz', 'kanjidic-en.jsonl.gz'],
|
|
);
|
|
expect(File('${into.path}/.conversion').readAsStringSync(), '$conversion');
|
|
expect(lastUpdated(into), isNotNull);
|
|
});
|
|
|
|
test('files another conversion made, or one from before the curator said, '
|
|
'are converted again at once', () async {
|
|
await updateDictionaries(
|
|
into: into,
|
|
data: Directory('data'),
|
|
fetch: samples,
|
|
);
|
|
final made = File('${into.path}/$conversionFile')
|
|
..writeAsStringSync('${conversion - 1}');
|
|
expect(lastUpdated(into), isNull);
|
|
|
|
made.deleteSync();
|
|
expect(lastUpdated(into), isNull);
|
|
});
|
|
|
|
test('a download that fails leaves the files there as they were', () async {
|
|
await updateDictionaries(
|
|
into: into,
|
|
data: Directory('data'),
|
|
fetch: samples,
|
|
);
|
|
final before = File('${into.path}/jmdict-en.jsonl.gz').readAsBytesSync();
|
|
|
|
await expectLater(
|
|
updateDictionaries(
|
|
into: into,
|
|
data: Directory('data'),
|
|
fetch: (uri, file) => uri == kanjidicSource
|
|
? Future.error(const HttpException('EDRDG answered 503.'))
|
|
: samples(uri, file),
|
|
),
|
|
throwsA(isA<HttpException>()),
|
|
);
|
|
|
|
expect(File('${into.path}/jmdict-en.jsonl.gz').readAsBytesSync(), before);
|
|
});
|
|
|
|
test('a folder missing a file was never brought up to date', () {
|
|
expect(lastUpdated(into), isNull);
|
|
});
|
|
|
|
group("Wiktionary's dump", () {
|
|
var dumped = DateTime.utc(2026, 10, 3, 8, 24);
|
|
var opened = 0;
|
|
|
|
setUp(() {
|
|
dumped = DateTime.utc(2026, 10, 3, 8, 24);
|
|
opened = 0;
|
|
});
|
|
|
|
Future<DateTime> modified(Uri uri) async => dumped;
|
|
|
|
/// The sample in place of kaikki.org's dump.
|
|
Future<Download> open(Uri uri) async {
|
|
opened++;
|
|
return (
|
|
modified: dumped,
|
|
bytes: Stream.value(
|
|
gzip.encode(
|
|
File('test/fixtures/wiktionary-sample.jsonl').readAsBytesSync(),
|
|
),
|
|
),
|
|
);
|
|
}
|
|
|
|
Future<List<File>> update(Set<String> languages, {bool korean = false}) =>
|
|
updateWiktionary(
|
|
into: into,
|
|
data: Directory('data'),
|
|
languages: languages,
|
|
korean: korean,
|
|
modified: modified,
|
|
open: open,
|
|
fetch: (uri, file) => file.writeAsBytes(
|
|
tatoeba(uri) ?? (throw HttpException('$uri', uri: uri)),
|
|
),
|
|
);
|
|
|
|
test(
|
|
'is converted for the languages named, German and Greek with English '
|
|
'meanings, English with German ones, and ranked by frequency',
|
|
() async {
|
|
final written = await update({'de', 'el', 'en'});
|
|
|
|
expect(
|
|
[for (final file in written) file.uri.pathSegments.last]..sort(),
|
|
[
|
|
'wiktionary-de-en.jsonl.gz',
|
|
'wiktionary-el-en.jsonl.gz',
|
|
'wiktionary-en-de.jsonl.gz',
|
|
],
|
|
);
|
|
expect(
|
|
header('wiktionary-de-en.jsonl.gz')['sourceVersion'],
|
|
'2026-10-03',
|
|
);
|
|
expect([
|
|
for (final credit
|
|
in header('wiktionary-de-en.jsonl.gz')['credits']! as List)
|
|
(credit as Map)['name'],
|
|
], contains('wordfreq'));
|
|
expect(File('${into.path}/$wiktionaryStateFile').existsSync(), isTrue);
|
|
},
|
|
);
|
|
|
|
test(
|
|
'is downloaded again only once it changed, or the languages did',
|
|
() async {
|
|
await update({'de'});
|
|
expect(await update({'de'}), isEmpty);
|
|
expect(opened, 1);
|
|
|
|
dumped = DateTime.utc(2026, 11, 2);
|
|
expect(await update({'de'}), hasLength(1));
|
|
expect(opened, 2);
|
|
|
|
expect(await update({'de', 'el'}), hasLength(2));
|
|
expect(opened, 3);
|
|
},
|
|
);
|
|
|
|
test('nothing is asked for without languages', () async {
|
|
expect(await update({}), isEmpty);
|
|
expect(opened, 0);
|
|
});
|
|
|
|
test("keeps what Korean's dictionaries take from it, even without "
|
|
'languages of its words', () async {
|
|
expect(await update({}, korean: true), isEmpty);
|
|
expect(opened, 1);
|
|
|
|
final kept = WiktionaryKorean.fromJson(
|
|
utf8.decode(
|
|
gzip.decode(
|
|
File('${into.path}/$wiktionaryKoreanFile').readAsBytesSync(),
|
|
),
|
|
),
|
|
);
|
|
expect(kept.hanja['水'], [(hun: '물', eum: '수')]);
|
|
expect(await update({}, korean: true), isEmpty);
|
|
expect(opened, 1);
|
|
});
|
|
});
|
|
|
|
test('CC-CEDICT is converted in both scripts with the HSK levels, and '
|
|
'said to be up to date by its files', () async {
|
|
expect(cedictUpdated(into), isNull);
|
|
|
|
final written = await updateCedict(
|
|
into: into,
|
|
data: Directory('data'),
|
|
fetch: (uri, file) => file.writeAsBytes(
|
|
uri == cedictSource
|
|
? gzip.encode(
|
|
File('test/fixtures/cedict-sample.txt').readAsBytesSync(),
|
|
)
|
|
: tatoeba(uri) ?? (throw HttpException('$uri', uri: uri)),
|
|
),
|
|
);
|
|
|
|
expect(
|
|
[for (final file in written) file.uri.pathSegments.last],
|
|
['cc-cedict-hans-en.jsonl.gz', 'cc-cedict-hant-en.jsonl.gz'],
|
|
);
|
|
expect(header('cc-cedict-hant-en.jsonl.gz')['language'], 'zh-Hant');
|
|
expect([
|
|
for (final credit
|
|
in header('cc-cedict-hans-en.jsonl.gz')['credits']! as List)
|
|
(credit as Map)['name'],
|
|
], contains('Tatoeba'));
|
|
expect(cedictUpdated(into), isNotNull);
|
|
// Pinyin over the sentences Tatoeba transcribed.
|
|
expect(
|
|
[
|
|
for (final example in examplesIn('cc-cedict-hans-en.jsonl.gz'))
|
|
if (example['tatoeba'] == 750344) example['ruby'],
|
|
].first,
|
|
[
|
|
[0, 1, 'wǒ'],
|
|
[1, 3, 'xuéxí'],
|
|
],
|
|
);
|
|
// Nothing of the downloads left beside the files.
|
|
expect(
|
|
[for (final entity in into.listSync()) entity.uri.pathSegments.last]
|
|
..sort(),
|
|
['cc-cedict-hans-en.jsonl.gz', 'cc-cedict-hant-en.jsonl.gz'],
|
|
);
|
|
});
|
|
|
|
test("a dictionary keeps its words without Tatoeba's sentences when they "
|
|
"can't be had", () async {
|
|
final written = await updateCedict(
|
|
into: into,
|
|
data: Directory('data'),
|
|
fetch: (uri, file) => uri == cedictSource
|
|
? file.writeAsBytes(
|
|
gzip.encode(
|
|
File('test/fixtures/cedict-sample.txt').readAsBytesSync(),
|
|
),
|
|
)
|
|
: throw HttpException('$uri answered 503.', uri: uri),
|
|
);
|
|
|
|
expect(written, hasLength(2));
|
|
expect([
|
|
for (final credit
|
|
in header('cc-cedict-hans-en.jsonl.gz')['credits']! as List)
|
|
(credit as Map)['name'],
|
|
], isNot(contains('Tatoeba')));
|
|
});
|
|
|
|
test('Chinese comes from CC-CEDICT, Korean from KRDict, the other '
|
|
"languages' words from Wiktionary", () {
|
|
final all = dictionarySources({'de', 'el', 'en', 'ko', 'zh'});
|
|
expect(all.wiktionary, {'de', 'el', 'en'});
|
|
expect(all.chinese, isTrue);
|
|
expect(all.korean, isTrue);
|
|
expect(all.greek, isTrue);
|
|
final traditional = dictionarySources({'zh-Hant'});
|
|
expect(traditional.wiktionary, isEmpty);
|
|
expect(traditional.chinese, isTrue);
|
|
expect(traditional.korean, isFalse);
|
|
expect(dictionarySources({}).chinese, isFalse);
|
|
});
|
|
|
|
test('files of languages no longer kept, or of a source replaced, go; an '
|
|
"operator's own stay", () {
|
|
for (final name in [
|
|
'jmdict-en.jsonl.gz',
|
|
'wiktionary-de-en.jsonl.gz',
|
|
'wiktionary-ko-en.jsonl.gz',
|
|
'wiktionary-el-en.jsonl.gz',
|
|
'krdict-ko-en.jsonl.gz',
|
|
'unihan-ko-en.jsonl.gz',
|
|
'greek-alphabet-en.jsonl.gz',
|
|
'my-own.jsonl.gz',
|
|
]) {
|
|
File('${into.path}/$name').writeAsStringSync('');
|
|
}
|
|
|
|
final removed = removeUnwanted(into, {'de', 'ko'});
|
|
|
|
expect([for (final file in removed) file.uri.pathSegments.last]..sort(), [
|
|
'greek-alphabet-en.jsonl.gz',
|
|
'wiktionary-el-en.jsonl.gz',
|
|
'wiktionary-ko-en.jsonl.gz',
|
|
]);
|
|
expect(
|
|
[for (final entity in into.listSync()) entity.uri.pathSegments.last]
|
|
..sort(),
|
|
[
|
|
'jmdict-en.jsonl.gz',
|
|
'krdict-ko-en.jsonl.gz',
|
|
'my-own.jsonl.gz',
|
|
'unihan-ko-en.jsonl.gz',
|
|
'wiktionary-de-en.jsonl.gz',
|
|
],
|
|
);
|
|
});
|
|
|
|
test("a replaced source stays until what replaces it is there: Korean's "
|
|
"words from Wiktionary until KRDict's are", () {
|
|
File('${into.path}/wiktionary-ko-en.jsonl.gz').writeAsStringSync('');
|
|
|
|
expect(removeUnwanted(into, {'ko'}), isEmpty);
|
|
|
|
File('${into.path}/krdict-ko-en.jsonl.gz').writeAsStringSync('');
|
|
expect(
|
|
[
|
|
for (final file in removeUnwanted(into, {'ko'}))
|
|
file.uri.pathSegments.last,
|
|
],
|
|
['wiktionary-ko-en.jsonl.gz'],
|
|
);
|
|
});
|
|
|
|
group("KRDict's export", () {
|
|
var number = '217';
|
|
var downloads = 0;
|
|
|
|
setUp(() {
|
|
number = '217';
|
|
downloads = 0;
|
|
});
|
|
|
|
/// KRDict's page and the fixture packed as its export; Tatoeba's.
|
|
Future<void> fetch(Uri uri, File file) async {
|
|
if (uri == krdictPage) {
|
|
await file.writeAsString(
|
|
"<button onclick=\"batchDownload('215')\"></button>"
|
|
"<button onclick=\"batchDownload('216')\"></button>"
|
|
"<button onclick=\"batchDownload('$number')\"></button>",
|
|
);
|
|
} else if (uri == krdictDownloadOf(number)) {
|
|
downloads++;
|
|
await file.writeAsBytes(
|
|
ZipEncoder().encodeBytes(
|
|
Archive()..add(
|
|
ArchiveFile.bytes(
|
|
'1_5000_20260919.json',
|
|
File('test/fixtures/krdict-sample.json').readAsBytesSync(),
|
|
),
|
|
),
|
|
),
|
|
);
|
|
} else {
|
|
await file.writeAsBytes(
|
|
tatoeba(uri) ?? (throw HttpException('$uri', uri: uri)),
|
|
);
|
|
}
|
|
}
|
|
|
|
test('is converted with the forms Wiktionary kept, and again only '
|
|
'once its page offers another', () async {
|
|
final kept = WiktionaryKorean()..forms['가다'] = {'갔어요'};
|
|
File('${into.path}/$wiktionaryKoreanFile')
|
|
.writeAsBytesSync(gzip.encode(utf8.encode(kept.toJson())));
|
|
|
|
final written = await updateKrdict(
|
|
into: into,
|
|
data: Directory('data'),
|
|
fetch: fetch,
|
|
);
|
|
|
|
expect(
|
|
[for (final file in written) file.uri.pathSegments.last],
|
|
[krdictFileName],
|
|
);
|
|
expect(header(krdictFileName)['sourceVersion'], '2026-09-19');
|
|
expect([
|
|
for (final credit in header(krdictFileName)['credits']! as List)
|
|
(credit as Map)['name'],
|
|
], contains('Wiktionary'));
|
|
expect(
|
|
await updateKrdict(into: into, data: Directory('data'), fetch: fetch),
|
|
isEmpty,
|
|
);
|
|
expect(downloads, 1);
|
|
|
|
number = '230';
|
|
expect(
|
|
await updateKrdict(into: into, data: Directory('data'), fetch: fetch),
|
|
hasLength(1),
|
|
);
|
|
expect(downloads, 2);
|
|
expect(
|
|
[for (final entity in into.listSync()) entity.uri.pathSegments.last]
|
|
..sort(),
|
|
['.krdict', wiktionaryKoreanFile, krdictFileName]..sort(),
|
|
);
|
|
});
|
|
});
|
|
|
|
test('the characters of Chinese in both scripts, of Korean with the Hangul '
|
|
'letters and the Greek alphabet, up to date until a Wiktionary update '
|
|
'keeps new Hanja readings', () async {
|
|
Future<void> fetch(Uri uri, File file) async => await file.writeAsBytes(
|
|
uri == unihanSource
|
|
? ZipEncoder().encodeBytes(
|
|
Archive()..add(
|
|
ArchiveFile.bytes(
|
|
'Unihan_Readings.txt',
|
|
File('test/fixtures/unihan-sample.txt').readAsBytesSync(),
|
|
),
|
|
),
|
|
)
|
|
: gzip.encode(
|
|
File('test/fixtures/cedict-characters-sample.txt')
|
|
.readAsBytesSync(),
|
|
),
|
|
);
|
|
expect(
|
|
charactersUpdated(into, chinese: true, korean: true, greek: true),
|
|
isNull,
|
|
);
|
|
|
|
final written = await updateCharacters(
|
|
into: into,
|
|
data: Directory('data'),
|
|
chinese: true,
|
|
korean: true,
|
|
greek: true,
|
|
fetch: fetch,
|
|
);
|
|
|
|
expect(
|
|
[for (final file in written) file.uri.pathSegments.last],
|
|
[
|
|
'unihan-hans-en.jsonl.gz',
|
|
'unihan-hant-en.jsonl.gz',
|
|
'unihan-ko-en.jsonl.gz',
|
|
'greek-alphabet-en.jsonl.gz',
|
|
],
|
|
);
|
|
expect(header('unihan-hans-en.jsonl.gz')['sourceVersion'], '18.0.0');
|
|
expect([
|
|
for (final credit
|
|
in header('unihan-hans-en.jsonl.gz')['credits']! as List)
|
|
(credit as Map)['name'],
|
|
], containsAll(['Unihan Database', 'CC-CEDICT', 'wordfreq']));
|
|
final updated = charactersUpdated(
|
|
into,
|
|
chinese: true,
|
|
korean: true,
|
|
greek: true,
|
|
);
|
|
expect(updated, isNotNull);
|
|
|
|
File('${into.path}/$wiktionaryKoreanFile')
|
|
..writeAsBytesSync(gzip.encode(utf8.encode(WiktionaryKorean().toJson())))
|
|
..setLastModifiedSync(updated!.add(const Duration(minutes: 1)));
|
|
expect(charactersUpdated(into, korean: true), isNull);
|
|
expect(charactersUpdated(into, chinese: true), isNotNull);
|
|
});
|
|
}
|