chore: Halkyon Learn moves into apps/learn
Its server, client, app and curator, and the tools that write its assets (maps, Anki fixtures, icons), as they are; paths are fixed in the commits that follow. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Tizian 「ティツィアーン」 <raisondetredev@pm.me>
This commit is contained in:
parent
11abd9afce
commit
649499be23
GPG key ID:
07F6FCF405488B64
1 changed files with 0 additions and 0 deletions
478
apps/learn/halkyon_learn_curator/test/update_test.dart
Normal file
478
apps/learn/halkyon_learn_curator/test/update_test.dart
Normal file
|
|
@ -0,0 +1,478 @@
|
|||
import 'dart:convert';
|
||||
import 'dart:io';
|
||||
|
||||
import 'package:archive/archive.dart';
|
||||
|
||||
import 'package:halkyon_learn_curator/src/sources/krdict.dart';
|
||||
import 'package:halkyon_learn_curator/src/sources/wiktionary.dart';
|
||||
import 'package:halkyon_learn_curator/src/update.dart';
|
||||
import 'package:test/test.dart';
|
||||
|
||||
void main() {
|
||||
late Directory into;
|
||||
|
||||
setUp(() => into = Directory.systemTemp.createTempSync('dictionaries'));
|
||||
tearDown(() => into.deleteSync(recursive: true));
|
||||
|
||||
/// Fetches the samples in place of EDRDG's files of the day.
|
||||
Future<void> samples(Uri uri, File file) async {
|
||||
final sample = uri == jmdictSource
|
||||
? 'test/fixtures/jmdict-e-examp-sample.xml'
|
||||
: 'test/fixtures/kanjidic2-sample.xml';
|
||||
await file.writeAsBytes(gzip.encode(await File(sample).readAsBytes()));
|
||||
}
|
||||
|
||||
/// Tatoeba's exports, as the fixtures hold a few of their lines, packed
|
||||
/// as Tatoeba packs them; null for any other address.
|
||||
List<int>? tatoeba(Uri uri) {
|
||||
final name = uri.pathSegments.last.replaceAll('.bz2', '');
|
||||
final fixture = File('test/fixtures/tatoeba-$name');
|
||||
return fixture.existsSync()
|
||||
? BZip2Encoder().encodeBytes(fixture.readAsBytesSync())
|
||||
: null;
|
||||
}
|
||||
|
||||
/// The header of the dictionary file [name] in [into].
|
||||
Map<String, Object?> header(String name) => jsonDecode(
|
||||
utf8
|
||||
.decode(gzip.decode(File('${into.path}/$name').readAsBytesSync()))
|
||||
.split('\n')
|
||||
.first,
|
||||
) as Map<String, Object?>;
|
||||
|
||||
test("writes JMdict and KANJIDIC2 from EDRDG's files, the JLPT levels from "
|
||||
"the curator's data", () async {
|
||||
final written = await updateDictionaries(
|
||||
into: into,
|
||||
data: Directory('data'),
|
||||
fetch: samples,
|
||||
);
|
||||
|
||||
expect(
|
||||
[for (final file in written) file.uri.pathSegments.last],
|
||||
['jmdict-en.jsonl.gz', 'kanjidic-en.jsonl.gz'],
|
||||
);
|
||||
expect(header('jmdict-en.jsonl.gz')['sourceVersion'], '2026-10-09');
|
||||
expect(
|
||||
[
|
||||
for (final credit in header('jmdict-en.jsonl.gz')['credits']! as List)
|
||||
(credit as Map)['name'],
|
||||
],
|
||||
['JMdict', 'JLPT vocabulary lists', 'Tatoeba'],
|
||||
);
|
||||
expect(header('kanjidic-en.jsonl.gz')['kind'], 'characters');
|
||||
// Nothing half-written left behind; which conversion made them.
|
||||
expect(
|
||||
[for (final entity in into.listSync()) entity.uri.pathSegments.last]
|
||||
..sort(),
|
||||
['.conversion', 'jmdict-en.jsonl.gz', 'kanjidic-en.jsonl.gz'],
|
||||
);
|
||||
expect(File('${into.path}/.conversion').readAsStringSync(), '$conversion');
|
||||
expect(lastUpdated(into), isNotNull);
|
||||
});
|
||||
|
||||
test('files another conversion made, or one from before the curator said, '
|
||||
'are converted again at once', () async {
|
||||
await updateDictionaries(
|
||||
into: into,
|
||||
data: Directory('data'),
|
||||
fetch: samples,
|
||||
);
|
||||
final made = File('${into.path}/$conversionFile')
|
||||
..writeAsStringSync('${conversion - 1}');
|
||||
expect(lastUpdated(into), isNull);
|
||||
|
||||
made.deleteSync();
|
||||
expect(lastUpdated(into), isNull);
|
||||
});
|
||||
|
||||
test('a download that fails leaves the files there as they were', () async {
|
||||
await updateDictionaries(
|
||||
into: into,
|
||||
data: Directory('data'),
|
||||
fetch: samples,
|
||||
);
|
||||
final before = File('${into.path}/jmdict-en.jsonl.gz').readAsBytesSync();
|
||||
|
||||
await expectLater(
|
||||
updateDictionaries(
|
||||
into: into,
|
||||
data: Directory('data'),
|
||||
fetch: (uri, file) => uri == kanjidicSource
|
||||
? Future.error(const HttpException('EDRDG answered 503.'))
|
||||
: samples(uri, file),
|
||||
),
|
||||
throwsA(isA<HttpException>()),
|
||||
);
|
||||
|
||||
expect(File('${into.path}/jmdict-en.jsonl.gz').readAsBytesSync(), before);
|
||||
});
|
||||
|
||||
test('a folder missing a file was never brought up to date', () {
|
||||
expect(lastUpdated(into), isNull);
|
||||
});
|
||||
|
||||
group("Wiktionary's dump", () {
|
||||
var dumped = DateTime.utc(2026, 10, 3, 8, 24);
|
||||
var opened = 0;
|
||||
|
||||
setUp(() {
|
||||
dumped = DateTime.utc(2026, 10, 3, 8, 24);
|
||||
opened = 0;
|
||||
});
|
||||
|
||||
Future<DateTime> modified(Uri uri) async => dumped;
|
||||
|
||||
/// The sample in place of kaikki.org's dump.
|
||||
Future<Download> open(Uri uri) async {
|
||||
opened++;
|
||||
return (
|
||||
modified: dumped,
|
||||
bytes: Stream.value(
|
||||
gzip.encode(
|
||||
File('test/fixtures/wiktionary-sample.jsonl').readAsBytesSync(),
|
||||
),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
Future<List<File>> update(Set<String> languages, {bool korean = false}) =>
|
||||
updateWiktionary(
|
||||
into: into,
|
||||
data: Directory('data'),
|
||||
languages: languages,
|
||||
korean: korean,
|
||||
modified: modified,
|
||||
open: open,
|
||||
fetch: (uri, file) => file.writeAsBytes(
|
||||
tatoeba(uri) ?? (throw HttpException('$uri', uri: uri)),
|
||||
),
|
||||
);
|
||||
|
||||
test(
|
||||
'is converted for the languages named, German and Greek with English '
|
||||
'meanings, English with German ones, and ranked by frequency',
|
||||
() async {
|
||||
final written = await update({'de', 'el', 'en'});
|
||||
|
||||
expect(
|
||||
[for (final file in written) file.uri.pathSegments.last]..sort(),
|
||||
[
|
||||
'wiktionary-de-en.jsonl.gz',
|
||||
'wiktionary-el-en.jsonl.gz',
|
||||
'wiktionary-en-de.jsonl.gz',
|
||||
],
|
||||
);
|
||||
expect(
|
||||
header('wiktionary-de-en.jsonl.gz')['sourceVersion'],
|
||||
'2026-10-03',
|
||||
);
|
||||
expect([
|
||||
for (final credit
|
||||
in header('wiktionary-de-en.jsonl.gz')['credits']! as List)
|
||||
(credit as Map)['name'],
|
||||
], contains('wordfreq'));
|
||||
expect(File('${into.path}/$wiktionaryStateFile').existsSync(), isTrue);
|
||||
},
|
||||
);
|
||||
|
||||
test(
|
||||
'is downloaded again only once it changed, or the languages did',
|
||||
() async {
|
||||
await update({'de'});
|
||||
expect(await update({'de'}), isEmpty);
|
||||
expect(opened, 1);
|
||||
|
||||
dumped = DateTime.utc(2026, 11, 2);
|
||||
expect(await update({'de'}), hasLength(1));
|
||||
expect(opened, 2);
|
||||
|
||||
expect(await update({'de', 'el'}), hasLength(2));
|
||||
expect(opened, 3);
|
||||
},
|
||||
);
|
||||
|
||||
test('nothing is asked for without languages', () async {
|
||||
expect(await update({}), isEmpty);
|
||||
expect(opened, 0);
|
||||
});
|
||||
|
||||
test("keeps what Korean's dictionaries take from it, even without "
|
||||
'languages of its words', () async {
|
||||
expect(await update({}, korean: true), isEmpty);
|
||||
expect(opened, 1);
|
||||
|
||||
final kept = WiktionaryKorean.fromJson(
|
||||
utf8.decode(
|
||||
gzip.decode(
|
||||
File('${into.path}/$wiktionaryKoreanFile').readAsBytesSync(),
|
||||
),
|
||||
),
|
||||
);
|
||||
expect(kept.hanja['水'], [(hun: '물', eum: '수')]);
|
||||
expect(await update({}, korean: true), isEmpty);
|
||||
expect(opened, 1);
|
||||
});
|
||||
});
|
||||
|
||||
test('CC-CEDICT is converted in both scripts with the HSK levels, and '
|
||||
'said to be up to date by its files', () async {
|
||||
expect(cedictUpdated(into), isNull);
|
||||
|
||||
final written = await updateCedict(
|
||||
into: into,
|
||||
data: Directory('data'),
|
||||
fetch: (uri, file) => file.writeAsBytes(
|
||||
uri == cedictSource
|
||||
? gzip.encode(
|
||||
File('test/fixtures/cedict-sample.txt').readAsBytesSync(),
|
||||
)
|
||||
: tatoeba(uri) ?? (throw HttpException('$uri', uri: uri)),
|
||||
),
|
||||
);
|
||||
|
||||
expect(
|
||||
[for (final file in written) file.uri.pathSegments.last],
|
||||
['cc-cedict-hans-en.jsonl.gz', 'cc-cedict-hant-en.jsonl.gz'],
|
||||
);
|
||||
expect(header('cc-cedict-hant-en.jsonl.gz')['language'], 'zh-Hant');
|
||||
expect([
|
||||
for (final credit
|
||||
in header('cc-cedict-hans-en.jsonl.gz')['credits']! as List)
|
||||
(credit as Map)['name'],
|
||||
], contains('Tatoeba'));
|
||||
expect(cedictUpdated(into), isNotNull);
|
||||
// Nothing of the downloads left beside the files.
|
||||
expect(
|
||||
[for (final entity in into.listSync()) entity.uri.pathSegments.last]
|
||||
..sort(),
|
||||
['cc-cedict-hans-en.jsonl.gz', 'cc-cedict-hant-en.jsonl.gz'],
|
||||
);
|
||||
});
|
||||
|
||||
test("a dictionary keeps its words without Tatoeba's sentences when they "
|
||||
"can't be had", () async {
|
||||
final written = await updateCedict(
|
||||
into: into,
|
||||
data: Directory('data'),
|
||||
fetch: (uri, file) => uri == cedictSource
|
||||
? file.writeAsBytes(
|
||||
gzip.encode(
|
||||
File('test/fixtures/cedict-sample.txt').readAsBytesSync(),
|
||||
),
|
||||
)
|
||||
: throw HttpException('$uri answered 503.', uri: uri),
|
||||
);
|
||||
|
||||
expect(written, hasLength(2));
|
||||
expect([
|
||||
for (final credit
|
||||
in header('cc-cedict-hans-en.jsonl.gz')['credits']! as List)
|
||||
(credit as Map)['name'],
|
||||
], isNot(contains('Tatoeba')));
|
||||
});
|
||||
|
||||
test('Chinese comes from CC-CEDICT, Korean from KRDict, the other '
|
||||
"languages' words from Wiktionary", () {
|
||||
final all = dictionarySources({'de', 'el', 'en', 'ko', 'zh'});
|
||||
expect(all.wiktionary, {'de', 'el', 'en'});
|
||||
expect(all.chinese, isTrue);
|
||||
expect(all.korean, isTrue);
|
||||
expect(all.greek, isTrue);
|
||||
final traditional = dictionarySources({'zh-Hant'});
|
||||
expect(traditional.wiktionary, isEmpty);
|
||||
expect(traditional.chinese, isTrue);
|
||||
expect(traditional.korean, isFalse);
|
||||
expect(dictionarySources({}).chinese, isFalse);
|
||||
});
|
||||
|
||||
test('files of languages no longer kept, or of a source replaced, go; an '
|
||||
"operator's own stay", () {
|
||||
for (final name in [
|
||||
'jmdict-en.jsonl.gz',
|
||||
'wiktionary-de-en.jsonl.gz',
|
||||
'wiktionary-ko-en.jsonl.gz',
|
||||
'wiktionary-el-en.jsonl.gz',
|
||||
'krdict-ko-en.jsonl.gz',
|
||||
'unihan-ko-en.jsonl.gz',
|
||||
'greek-alphabet-en.jsonl.gz',
|
||||
'my-own.jsonl.gz',
|
||||
]) {
|
||||
File('${into.path}/$name').writeAsStringSync('');
|
||||
}
|
||||
|
||||
final removed = removeUnwanted(into, {'de', 'ko'});
|
||||
|
||||
expect([for (final file in removed) file.uri.pathSegments.last]..sort(), [
|
||||
'greek-alphabet-en.jsonl.gz',
|
||||
'wiktionary-el-en.jsonl.gz',
|
||||
'wiktionary-ko-en.jsonl.gz',
|
||||
]);
|
||||
expect(
|
||||
[for (final entity in into.listSync()) entity.uri.pathSegments.last]
|
||||
..sort(),
|
||||
[
|
||||
'jmdict-en.jsonl.gz',
|
||||
'krdict-ko-en.jsonl.gz',
|
||||
'my-own.jsonl.gz',
|
||||
'unihan-ko-en.jsonl.gz',
|
||||
'wiktionary-de-en.jsonl.gz',
|
||||
],
|
||||
);
|
||||
});
|
||||
|
||||
test("a replaced source stays until what replaces it is there: Korean's "
|
||||
"words from Wiktionary until KRDict's are", () {
|
||||
File('${into.path}/wiktionary-ko-en.jsonl.gz').writeAsStringSync('');
|
||||
|
||||
expect(removeUnwanted(into, {'ko'}), isEmpty);
|
||||
|
||||
File('${into.path}/krdict-ko-en.jsonl.gz').writeAsStringSync('');
|
||||
expect(
|
||||
[
|
||||
for (final file in removeUnwanted(into, {'ko'}))
|
||||
file.uri.pathSegments.last,
|
||||
],
|
||||
['wiktionary-ko-en.jsonl.gz'],
|
||||
);
|
||||
});
|
||||
|
||||
group("KRDict's export", () {
|
||||
var number = '217';
|
||||
var downloads = 0;
|
||||
|
||||
setUp(() {
|
||||
number = '217';
|
||||
downloads = 0;
|
||||
});
|
||||
|
||||
/// KRDict's page and the fixture packed as its export; Tatoeba's.
|
||||
Future<void> fetch(Uri uri, File file) async {
|
||||
if (uri == krdictPage) {
|
||||
await file.writeAsString(
|
||||
"<button onclick=\"batchDownload('215')\"></button>"
|
||||
"<button onclick=\"batchDownload('216')\"></button>"
|
||||
"<button onclick=\"batchDownload('$number')\"></button>",
|
||||
);
|
||||
} else if (uri == krdictDownloadOf(number)) {
|
||||
downloads++;
|
||||
await file.writeAsBytes(
|
||||
ZipEncoder().encodeBytes(
|
||||
Archive()..add(
|
||||
ArchiveFile.bytes(
|
||||
'1_5000_20260919.json',
|
||||
File('test/fixtures/krdict-sample.json').readAsBytesSync(),
|
||||
),
|
||||
),
|
||||
),
|
||||
);
|
||||
} else {
|
||||
await file.writeAsBytes(
|
||||
tatoeba(uri) ?? (throw HttpException('$uri', uri: uri)),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
test('is converted with the forms Wiktionary kept, and again only '
|
||||
'once its page offers another', () async {
|
||||
final kept = WiktionaryKorean()..forms['가다'] = {'갔어요'};
|
||||
File('${into.path}/$wiktionaryKoreanFile')
|
||||
.writeAsBytesSync(gzip.encode(utf8.encode(kept.toJson())));
|
||||
|
||||
final written = await updateKrdict(
|
||||
into: into,
|
||||
data: Directory('data'),
|
||||
fetch: fetch,
|
||||
);
|
||||
|
||||
expect(
|
||||
[for (final file in written) file.uri.pathSegments.last],
|
||||
[krdictFileName],
|
||||
);
|
||||
expect(header(krdictFileName)['sourceVersion'], '2026-09-19');
|
||||
expect([
|
||||
for (final credit in header(krdictFileName)['credits']! as List)
|
||||
(credit as Map)['name'],
|
||||
], contains('Wiktionary'));
|
||||
expect(
|
||||
await updateKrdict(into: into, data: Directory('data'), fetch: fetch),
|
||||
isEmpty,
|
||||
);
|
||||
expect(downloads, 1);
|
||||
|
||||
number = '230';
|
||||
expect(
|
||||
await updateKrdict(into: into, data: Directory('data'), fetch: fetch),
|
||||
hasLength(1),
|
||||
);
|
||||
expect(downloads, 2);
|
||||
expect(
|
||||
[for (final entity in into.listSync()) entity.uri.pathSegments.last]
|
||||
..sort(),
|
||||
['.krdict', wiktionaryKoreanFile, krdictFileName]..sort(),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
test('the characters of Chinese in both scripts, of Korean with the Hangul '
|
||||
'letters and the Greek alphabet, up to date until a Wiktionary update '
|
||||
'keeps new Hanja readings', () async {
|
||||
Future<void> fetch(Uri uri, File file) async => await file.writeAsBytes(
|
||||
uri == unihanSource
|
||||
? ZipEncoder().encodeBytes(
|
||||
Archive()..add(
|
||||
ArchiveFile.bytes(
|
||||
'Unihan_Readings.txt',
|
||||
File('test/fixtures/unihan-sample.txt').readAsBytesSync(),
|
||||
),
|
||||
),
|
||||
)
|
||||
: gzip.encode(
|
||||
File('test/fixtures/cedict-characters-sample.txt')
|
||||
.readAsBytesSync(),
|
||||
),
|
||||
);
|
||||
expect(
|
||||
charactersUpdated(into, chinese: true, korean: true, greek: true),
|
||||
isNull,
|
||||
);
|
||||
|
||||
final written = await updateCharacters(
|
||||
into: into,
|
||||
data: Directory('data'),
|
||||
chinese: true,
|
||||
korean: true,
|
||||
greek: true,
|
||||
fetch: fetch,
|
||||
);
|
||||
|
||||
expect(
|
||||
[for (final file in written) file.uri.pathSegments.last],
|
||||
[
|
||||
'unihan-hans-en.jsonl.gz',
|
||||
'unihan-hant-en.jsonl.gz',
|
||||
'unihan-ko-en.jsonl.gz',
|
||||
'greek-alphabet-en.jsonl.gz',
|
||||
],
|
||||
);
|
||||
expect(header('unihan-hans-en.jsonl.gz')['sourceVersion'], '18.0.0');
|
||||
expect([
|
||||
for (final credit
|
||||
in header('unihan-hans-en.jsonl.gz')['credits']! as List)
|
||||
(credit as Map)['name'],
|
||||
], containsAll(['Unihan Database', 'CC-CEDICT', 'wordfreq']));
|
||||
final updated = charactersUpdated(
|
||||
into,
|
||||
chinese: true,
|
||||
korean: true,
|
||||
greek: true,
|
||||
);
|
||||
expect(updated, isNotNull);
|
||||
|
||||
File('${into.path}/$wiktionaryKoreanFile')
|
||||
..writeAsBytesSync(gzip.encode(utf8.encode(WiktionaryKorean().toJson())))
|
||||
..setLastModifiedSync(updated!.add(const Duration(minutes: 1)));
|
||||
expect(charactersUpdated(into, korean: true), isNull);
|
||||
expect(charactersUpdated(into, chinese: true), isNotNull);
|
||||
});
|
||||
}
|
||||
Loading…
Reference in a new issue