Tizian 「ティツィアーン」
649499be23
Its server, client, app and curator, and the tools that write its assets (maps, Anki fixtures, icons), as they are; paths are fixed in the commits that follow. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Tizian 「ティツィアーン」 <raisondetredev@pm.me>
340 lines
12 KiB
Dart
340 lines
12 KiB
Dart
import 'dart:convert';
|
|
import 'dart:io';
|
|
|
|
import 'package:halkyon_learn_curator/src/fold.dart';
|
|
import 'package:halkyon_learn_curator/src/sources/wiktionary.dart';
|
|
import 'package:halkyon_learn_curator/src/wordfreq.dart';
|
|
import 'package:test/test.dart';
|
|
|
|
void main() {
|
|
late Directory into;
|
|
|
|
setUp(() => into = Directory.systemTemp.createTempSync('wiktionary'));
|
|
tearDown(() => into.deleteSync(recursive: true));
|
|
|
|
/// The sample's lines, as the dump's.
|
|
Stream<String> sample() =>
|
|
File('test/fixtures/wiktionary-sample.jsonl')
|
|
.openRead()
|
|
.transform(utf8.decoder)
|
|
.transform(const LineSplitter());
|
|
|
|
/// The lines of the dictionary file [name] in [into], decoded.
|
|
List<Map<String, Object?>> linesOf(String name) => [
|
|
for (final line
|
|
in utf8
|
|
.decode(gzip.decode(File('${into.path}/$name').readAsBytesSync()))
|
|
.split('\n'))
|
|
if (line.isNotEmpty) jsonDecode(line) as Map<String, Object?>,
|
|
];
|
|
|
|
Map<String, Object?> entryOf(String name, String id) =>
|
|
linesOf(name).skip(1).singleWhere((line) => line['id'] == id);
|
|
|
|
Future<List<File>> convert({
|
|
Set<String> languages = const {'de', 'el'},
|
|
Set<String> translatedInto = const {'de'},
|
|
Map<String, WordFrequencies> frequencies = const {},
|
|
}) => wiktionaryDictionaries(
|
|
sample(),
|
|
into: into,
|
|
sourceVersion: '2026-10-03',
|
|
languages: languages,
|
|
translatedInto: translatedInto,
|
|
frequencies: frequencies,
|
|
);
|
|
|
|
test('writes a file for each language, and for English with what its '
|
|
'translation tables give', () async {
|
|
final written = await convert();
|
|
|
|
expect([for (final file in written) file.uri.pathSegments.last]..sort(), [
|
|
'wiktionary-de-en.jsonl.gz',
|
|
'wiktionary-el-en.jsonl.gz',
|
|
'wiktionary-en-de.jsonl.gz',
|
|
]);
|
|
final header = linesOf('wiktionary-de-en.jsonl.gz').first;
|
|
expect(header['source'], 'wiktionary-de');
|
|
expect(header['language'], 'de');
|
|
expect(header['glossLanguage'], 'en');
|
|
expect(header['sourceVersion'], '2026-10-03');
|
|
expect(header['license'], 'CC-BY-SA-4.0');
|
|
expect(
|
|
[
|
|
for (final credit in header['credits']! as List)
|
|
(credit as Map)['name'],
|
|
],
|
|
['Wiktionary', 'Wiktextract (kaikki.org)'],
|
|
);
|
|
expect(
|
|
linesOf('wiktionary-en-de.jsonl.gz').first['source'],
|
|
'wiktionary-en',
|
|
);
|
|
// Nothing half-written left behind.
|
|
expect(
|
|
[for (final entity in into.listSync()) entity.uri.pathSegments.last]
|
|
..sort(),
|
|
[
|
|
'wiktionary-de-en.jsonl.gz',
|
|
'wiktionary-el-en.jsonl.gz',
|
|
'wiktionary-en-de.jsonl.gz',
|
|
],
|
|
);
|
|
});
|
|
|
|
test('a German noun: its gender, its plural beside it, its meanings with '
|
|
'their examples, every form finding it in text', () async {
|
|
await convert();
|
|
|
|
final haus = entryOf('wiktionary-de-en.jsonl.gz', 'Haus/noun');
|
|
expect(haus['headword'], 'Haus');
|
|
expect(haus.containsKey('reading'), isFalse);
|
|
final entry = haus['entry']! as Map<String, Object?>;
|
|
expect(entry['pos'], 'noun');
|
|
expect(entry['genders'], ['n']);
|
|
expect(entry['ipa'], '[haʊ̯s]');
|
|
expect(
|
|
[for (final form in entry['inflections']! as List) (form as Map)['text']],
|
|
['Hauses', 'Häuser'],
|
|
);
|
|
final senses = (entry['senses']! as List).cast<Map<String, Object?>>();
|
|
expect(senses.first['glosses'], ['house, building']);
|
|
expect((senses.first['examples']! as List).first, {
|
|
'text': 'Haus',
|
|
'de': 'In dem Haus haben wir mal gewohnt.',
|
|
'en': 'We used to live in that house.',
|
|
});
|
|
// Its own forms, and those Wiktionary has entries for alone.
|
|
final textKeys = (haus['textKeys']! as List).cast<String>();
|
|
expect(textKeys, containsAll(['haus', 'hauses', 'häuser', 'häusern']));
|
|
final keys = (haus['keys']! as List).cast<String>();
|
|
expect(keys, containsAll(['haus', 'häusern', 'house, building', 'house']));
|
|
// Typed without umlauts.
|
|
expect(
|
|
(haus['looseKeys']! as List).cast<String>(),
|
|
containsAll(['hauser', 'haeuser']),
|
|
);
|
|
});
|
|
|
|
test(
|
|
'forms, names and other languages are no entries of their own',
|
|
() async {
|
|
await convert();
|
|
|
|
final ids = [
|
|
for (final line in linesOf('wiktionary-de-en.jsonl.gz').skip(1))
|
|
line['id'],
|
|
];
|
|
// Häuser, Häusern, aß and gegessen are forms; Haus and Häusern are also
|
|
// places, which are names.
|
|
expect(ids, containsAll(['Haus/noun', 'essen/verb', 'Straße/noun']));
|
|
expect(ids, isNot(contains('Häusern/noun')));
|
|
expect(ids, isNot(contains('aß/verb')));
|
|
expect(ids, isNot(contains('Haus/name')));
|
|
expect([
|
|
for (final line in linesOf('wiktionary-de-en.jsonl.gz').skip(1))
|
|
line['headword'],
|
|
], isNot(contains('maison')));
|
|
final essen = entryOf('wiktionary-de-en.jsonl.gz', 'essen/verb');
|
|
expect(
|
|
(essen['textKeys']! as List).cast<String>(),
|
|
containsAll(['isst', 'aß', 'gegessen']),
|
|
);
|
|
// A contraction is a word of its own: im, of in and dem.
|
|
expect(ids, contains('im/contraction'));
|
|
// Typed with `ss`.
|
|
expect(
|
|
(entryOf('wiktionary-de-en.jsonl.gz', 'Straße/noun')['looseKeys']!
|
|
as List)
|
|
.cast<String>(),
|
|
contains('strasse'),
|
|
);
|
|
},
|
|
);
|
|
|
|
test(
|
|
'a Greek word: found by its romanisation and without its accents',
|
|
() async {
|
|
await convert();
|
|
|
|
final spiti = entryOf('wiktionary-el-en.jsonl.gz', 'σπίτι/noun');
|
|
final entry = spiti['entry']! as Map<String, Object?>;
|
|
expect(entry['romanization'], 'spíti');
|
|
expect(entry['genders'], ['n']);
|
|
expect(
|
|
(spiti['textKeys']! as List).cast<String>(),
|
|
containsAll(['σπίτι', 'σπιτιού']),
|
|
);
|
|
expect((spiti['keys']! as List).cast<String>(), contains('spíti'));
|
|
// The articles its table gives cells of their own are no forms of it.
|
|
expect(
|
|
(spiti['textKeys']! as List).cast<String>(),
|
|
isNot(anyOf(contains('το'), contains('του'), contains('τα'))),
|
|
);
|
|
expect(
|
|
(spiti['looseKeys']! as List).cast<String>(),
|
|
containsAll(['σπιτι', 'σπιτιου']),
|
|
);
|
|
},
|
|
);
|
|
|
|
test('a Korean word: its Hanja and romanisation find it, its conjugations '
|
|
'in text; a counter word its table names is no form of it', () async {
|
|
await convert(languages: {'ko'}, translatedInto: {});
|
|
|
|
final student = entryOf('wiktionary-ko-en.jsonl.gz', '학생/noun');
|
|
final entry = student['entry']! as Map<String, Object?>;
|
|
expect(entry['hanja'], ['學生']);
|
|
expect(entry['romanization'], 'haksaeng');
|
|
expect(
|
|
(student['keys']! as List).cast<String>(),
|
|
containsAll(['학생', '學生', 'haksaeng', 'student']),
|
|
);
|
|
expect((student['textKeys']! as List).cast<String>(), ['학생']);
|
|
final eat = entryOf('wiktionary-ko-en.jsonl.gz', '먹다/verb');
|
|
expect(
|
|
(eat['textKeys']! as List).cast<String>(),
|
|
containsAll(['먹다', '먹어요', '먹었어요', '먹습니다']),
|
|
);
|
|
expect(
|
|
(eat['textKeys']! as List).cast<String>(),
|
|
isNot(contains('meokda')),
|
|
);
|
|
});
|
|
|
|
test("for Korean's dictionaries without its words: each Hanja's native "
|
|
"reading and sound, each word's forms", () async {
|
|
final korean = WiktionaryKorean();
|
|
final written = await wiktionaryDictionaries(
|
|
sample(),
|
|
into: into,
|
|
sourceVersion: '2026-10-03',
|
|
korean: korean,
|
|
);
|
|
|
|
expect(written, isEmpty);
|
|
expect(korean.hanja['水'], [(hun: '물', eum: '수')]);
|
|
expect(korean.forms['먹다'], containsAll(['먹어요', '먹었어요']));
|
|
expect(korean.forms['먹다'], isNot(contains('먹다')));
|
|
// Kept between updates as JSON.
|
|
final kept = WiktionaryKorean.fromJson(korean.toJson());
|
|
expect(kept.hanja['水'], [(hun: '물', eum: '수')]);
|
|
expect(kept.forms['먹다'], korean.forms['먹다']);
|
|
});
|
|
|
|
test('an English word with its German translations, a meaning each, found '
|
|
'by its forms and by the German words', () async {
|
|
await convert();
|
|
|
|
final house = entryOf('wiktionary-en-de.jsonl.gz', 'house/noun/1');
|
|
final entry = house['entry']! as Map<String, Object?>;
|
|
final senses = (entry['senses']! as List).cast<Map<String, Object?>>();
|
|
expect(senses.first, {
|
|
'glosses': ['Haus'],
|
|
'info': ['human abode'],
|
|
});
|
|
// Regional words (Alemannic German) left out.
|
|
expect([
|
|
for (final sense in senses) ...sense['glosses']! as List,
|
|
], isNot(contains('Huus')));
|
|
expect(
|
|
(house['textKeys']! as List).cast<String>(),
|
|
containsAll(['house', 'houses']),
|
|
);
|
|
expect((house['keys']! as List).cast<String>(), contains('haus'));
|
|
final go = entryOf('wiktionary-en-de.jsonl.gz', 'go/verb/1');
|
|
expect(
|
|
(go['textKeys']! as List).cast<String>(),
|
|
containsAll(['goes', 'went', 'gone']),
|
|
);
|
|
});
|
|
|
|
test('frequencies rank the words, the most used first, and say which are '
|
|
'common', () async {
|
|
final german = await WordFrequencies.read(
|
|
File('data/wordfreq/small_de.msgpack.gz'),
|
|
);
|
|
await convert(languages: {'de'}, frequencies: {'de': german});
|
|
|
|
final header = linesOf('wiktionary-de-en.jsonl.gz').first;
|
|
expect(
|
|
[
|
|
for (final credit in header['credits']! as List)
|
|
(credit as Map)['name'],
|
|
],
|
|
['Wiktionary', 'Wiktextract (kaikki.org)', 'wordfreq', 'SUBTLEX'],
|
|
);
|
|
final ranked = [
|
|
for (final line in linesOf('wiktionary-de-en.jsonl.gz').skip(1))
|
|
(line['headword'], line['rank']! as int, line['labels']),
|
|
]..sort((a, b) => a.$2.compareTo(b.$2));
|
|
expect(
|
|
[
|
|
for (final (word, _, _) in ranked)
|
|
if (['im', 'Haus', 'Straße'].contains(word)) word,
|
|
],
|
|
['im', 'Haus', 'Straße'],
|
|
);
|
|
expect(ranked.firstWhere((each) => each.$1 == 'Haus').$3, ['common']);
|
|
});
|
|
|
|
test("a word's frequency goes to its first entries alone, not to every "
|
|
'homograph of a syllable', () async {
|
|
final korean = await WordFrequencies.read(
|
|
File('data/wordfreq/small_ko.msgpack.gz'),
|
|
);
|
|
// Five entries of 이, as Wiktionary has thirteen.
|
|
await wiktionaryDictionaries(
|
|
Stream.fromIterable([
|
|
for (var number = 1; number <= 5; number++)
|
|
jsonEncode({
|
|
'word': '이',
|
|
'pos': 'noun',
|
|
'lang_code': 'ko',
|
|
'etymology_number': number,
|
|
'senses': [
|
|
{
|
|
'glosses': ['meaning $number'],
|
|
},
|
|
],
|
|
}),
|
|
]),
|
|
into: into,
|
|
sourceVersion: '2026-10-03',
|
|
languages: {'ko'},
|
|
frequencies: {'ko': korean},
|
|
);
|
|
|
|
final entries = linesOf('wiktionary-ko-en.jsonl.gz').skip(1).toList();
|
|
expect(
|
|
[for (final entry in entries) entry['labels']],
|
|
[
|
|
['common'],
|
|
['common'],
|
|
['common'],
|
|
<String>[],
|
|
<String>[],
|
|
],
|
|
);
|
|
expect(entries[3]['rank']! as int, greaterThan(entries[2]['rank']! as int));
|
|
});
|
|
|
|
group('folding', () {
|
|
test('case-folds as Unicode does', () {
|
|
expect(casefold('Straße'), 'strasse');
|
|
expect(casefold('ΣΠΊΤΙΣ'), 'σπίτισ');
|
|
});
|
|
|
|
test('drops accents of Latin and Greek letters', () {
|
|
expect(withoutAccents('Häuser'), 'hauser');
|
|
expect(withoutAccents('σπιτιού'), 'σπιτιου');
|
|
expect(withoutAccents('café'), 'cafe');
|
|
expect(withoutAccents('学習'), '学習');
|
|
});
|
|
|
|
test('writes umlauts out', () {
|
|
expect(umlautsWrittenOut('Häuser'), 'haeuser');
|
|
expect(umlautsWrittenOut('Haus'), isNull);
|
|
});
|
|
});
|
|
}
|