import 'dart:convert'; import 'dart:io'; import 'package:halkyon_learn_curator/src/sources/cedict.dart'; import 'package:halkyon_learn_curator/src/sources/tatoeba.dart'; import 'package:halkyon_learn_curator/src/sources/wiktionary.dart'; import 'package:test/test.dart'; void main() { late Directory into; setUp(() => into = Directory.systemTemp.createTempSync('tatoeba')); tearDown(() => into.deleteSync(recursive: true)); Stream lines(String fixture) => File('test/fixtures/$fixture') .openRead() .transform(utf8.decoder) .transform(const LineSplitter()); /// [file]'s lines with the fixtures' sentences of [code] as examples. Future>> enriched(File file, String code) async { final links = await readTatoebaLinks(lines('tatoeba-$code-eng_links.tsv')); final sentences = await readTatoebaSentences( lines('tatoeba-${code}_sentences.tsv'), wanted: links.containsKey, ); final linked = {for (final ids in links.values) ...ids}; final translations = await readTatoebaSentences( lines('tatoeba-eng_sentences.tsv'), wanted: linked.contains, ); return [ for (final line in await withTatoebaExamples( file, sentences: sentences, translations: translations, links: links, ).join().then((text) => text.split('\n'))) if (line.isNotEmpty) jsonDecode(line) as Map, ]; } List> examplesOf(Map line) => [ for (final example in ((((line['entry']! as Map)['senses']! as List).first as Map)['examples'] as List? ?? const [])) example as Map, ]; test("a German word's sentences come first in its first sense, those " 'without a translation left out, the header crediting Tatoeba', () async { final [german, ...] = await wiktionaryDictionaries( lines('wiktionary-sample.jsonl'), into: into, sourceVersion: '2026-10-03', languages: {'de'}, ); final result = await enriched(german, 'deu'); expect([ for (final credit in result.first['credits']! as List) (credit as Map)['name'], ], contains('Tatoeba')); final haus = result .skip(1) .singleWhere((line) => line['id'] == 'Haus/noun'); final examples = examplesOf(haus); // Three of Tatoeba's, before Wiktionary's own. // Those nearest seven words long first, then the shortest. expect(examples.take(3).map((example) => example['tatoeba']), [ 9984782, 1550095, 605418, ]); expect(examples.first['text'], 'Haus'); expect(examples.first['de'], 'Er ist im Haus.'); expect(examples.first['en'], isA()); expect(examples[3]['de'], 'In dem Haus haben wir mal gewohnt.'); // A form finds its word: aß is essen's. final essen = result .skip(1) .singleWhere((line) => line['id'] == 'essen/verb'); expect(examplesOf(essen).first['text'], 'aß'); }); test( 'Chinese sentences are cut by the longest words the dictionary has', () async { final file = File('${into.path}/cc-cedict-hans-en.jsonl'); await file.writeAsString( cedictDictionary( await File('test/fixtures/cedict-sample.txt').readAsString(), traditional: false, ).join(), ); final result = await enriched(file, 'cmn'); final xuexi = result .skip(1) .singleWhere((line) => line['id'] == '學習 学习 [xue2 xi2]'); expect(examplesOf(xuexi), isNotEmpty); expect(examplesOf(xuexi).first['text'], '学习'); expect(examplesOf(xuexi).first['zh-Hans'], contains('学习')); }, ); test('Korean words are found with their particles taken off', () async { final [korean, ...] = await wiktionaryDictionaries( lines('wiktionary-sample.jsonl'), into: into, sourceVersion: '2026-10-03', languages: {'ko'}, ); final result = await enriched(korean, 'kor'); final home = result.skip(1).singleWhere((line) => line['id'] == '집/noun/1'); expect(examplesOf(home), isNotEmpty); expect(examplesOf(home).first['text'], '집'); expect(examplesOf(home).first['ko'], contains('집에서')); }); test("Tatoeba's codes of the languages", () { expect(tatoebaCode('de'), 'deu'); expect(tatoebaCode('zh-Hant'), 'cmn'); expect(tatoebaCode('ja'), isNull); }); }