feat(curator): Tatoeba's sentences as examples in every new language

JMdict brings Tatoeba's sentences for Japanese; the other languages'
dictionaries had none (CC-CEDICT) or Wiktionary's own, often
quotations. Each file now takes Tatoeba's sentences of the week (its
per-language exports, CC BY 2.0 FR) as its entries' first examples,
which become a card's hints: each sentence with a translation into the
dictionary's meaning language goes to the entries its words find, cut
as the app cuts a line of subtitles (at spaces, Korean's particles off,
Chinese by its longest words), up to three for each, those using the
word as its headword writes it first (Haus rather than zu Hause; 学习
rather than 學習 in the simplified dictionary), then those nearest seven
words long. German's common words nearly all get some (8,849 of
9,292), in ten seconds and under 300 MB; a dictionary whose sentences
can't be had keeps its words. `curate tatoeba` does the same for a file
made by hand.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Signed-off-by: Tizian 「ティツィアーン」 <raisondetredev@pm.me>

halkyon_learn_curator/test/tatoeba_test.dart

@@ -0,0 +1,133 @@
+import 'dart:convert';
+import 'dart:io';
+
+import 'package:halkyon_learn_curator/src/sources/cedict.dart';
+import 'package:halkyon_learn_curator/src/sources/tatoeba.dart';
+import 'package:halkyon_learn_curator/src/sources/wiktionary.dart';
+import 'package:test/test.dart';
+
+void main() {
+  late Directory into;
+
+  setUp(() => into = Directory.systemTemp.createTempSync('tatoeba'));
+  tearDown(() => into.deleteSync(recursive: true));
+
+  Stream<String> lines(String fixture) =>
+      File('test/fixtures/$fixture')
+          .openRead()
+          .transform(utf8.decoder)
+          .transform(const LineSplitter());
+
+  /// [file]'s lines with the fixtures' sentences of [code] as examples.
+  Future<List<Map<String, Object?>>> enriched(File file, String code) async {
+    final links = await readTatoebaLinks(lines('tatoeba-$code-eng_links.tsv'));
+    final sentences = await readTatoebaSentences(
+      lines('tatoeba-${code}_sentences.tsv'),
+      wanted: links.containsKey,
+    );
+    final linked = {for (final ids in links.values) ...ids};
+    final translations = await readTatoebaSentences(
+      lines('tatoeba-eng_sentences.tsv'),
+      wanted: linked.contains,
+    );
+    return [
+      for (final line in await withTatoebaExamples(
+        file,
+        sentences: sentences,
+        translations: translations,
+        links: links,
+      ).join().then((text) => text.split('\n')))
+        if (line.isNotEmpty) jsonDecode(line) as Map<String, Object?>,
+    ];
+  }
+
+  List<Map<String, Object?>> examplesOf(Map<String, Object?> line) => [
+    for (final example
+        in ((((line['entry']! as Map)['senses']! as List).first
+                    as Map)['examples']
+                as List? ??
+            const []))
+      example as Map<String, Object?>,
+  ];
+
+  test("a German word's sentences come first in its first sense, those "
+      'without a translation left out, the header crediting Tatoeba', () async {
+    final [german, ...] = await wiktionaryDictionaries(
+      lines('wiktionary-sample.jsonl'),
+      into: into,
+      sourceVersion: '2026-10-03',
+      languages: {'de'},
+    );
+
+    final result = await enriched(german, 'deu');
+
+    expect([
+      for (final credit in result.first['credits']! as List)
+        (credit as Map)['name'],
+    ], contains('Tatoeba'));
+    final haus = result
+        .skip(1)
+        .singleWhere((line) => line['id'] == 'Haus/noun');
+    final examples = examplesOf(haus);
+    // Three of Tatoeba's, before Wiktionary's own.
+    // Those nearest seven words long first, then the shortest.
+    expect(examples.take(3).map((example) => example['tatoeba']), [
+      9984782,
+      1550095,
+      605418,
+    ]);
+    expect(examples.first['text'], 'Haus');
+    expect(examples.first['de'], 'Er ist im Haus.');
+    expect(examples.first['en'], isA<String>());
+    expect(examples[3]['de'], 'In dem Haus haben wir mal gewohnt.');
+    // A form finds its word: aß is essen's.
+    final essen = result
+        .skip(1)
+        .singleWhere((line) => line['id'] == 'essen/verb');
+    expect(examplesOf(essen).first['text'], 'aß');
+  });
+
+  test(
+    'Chinese sentences are cut by the longest words the dictionary has',
+    () async {
+      final file = File('${into.path}/cc-cedict-hans-en.jsonl');
+      await file.writeAsString(
+        cedictDictionary(
+          await File('test/fixtures/cedict-sample.txt').readAsString(),
+          traditional: false,
+        ).join(),
+      );
+
+      final result = await enriched(file, 'cmn');
+
+      final xuexi = result
+          .skip(1)
+          .singleWhere((line) => line['id'] == '學習 学习 [xue2 xi2]');
+      expect(examplesOf(xuexi), isNotEmpty);
+      expect(examplesOf(xuexi).first['text'], '学习');
+      expect(examplesOf(xuexi).first['zh-Hans'], contains('学习'));
+    },
+  );
+
+  test('Korean words are found with their particles taken off', () async {
+    final [korean, ...] = await wiktionaryDictionaries(
+      lines('wiktionary-sample.jsonl'),
+      into: into,
+      sourceVersion: '2026-10-03',
+      languages: {'ko'},
+    );
+
+    final result = await enriched(korean, 'kor');
+
+    final home = result.skip(1).singleWhere((line) => line['id'] == '집/noun/1');
+    expect(examplesOf(home), isNotEmpty);
+    expect(examplesOf(home).first['text'], '집');
+    expect(examplesOf(home).first['ko'], contains('집에서'));
+  });
+
+  test("Tatoeba's codes of the languages", () {
+    expect(tatoebaCode('de'), 'deu');
+    expect(tatoebaCode('zh-Hant'), 'cmn');
+    expect(tatoebaCode('ja'), isNull);
+  });
+}