feat(curator): Tatoeba's sentences as examples in every new language

JMdict brings Tatoeba's sentences for Japanese; the other languages'
dictionaries had none (CC-CEDICT) or Wiktionary's own, often
quotations. Each file now takes Tatoeba's sentences of the week (its
per-language exports, CC BY 2.0 FR) as its entries' first examples,
which become a card's hints: each sentence with a translation into the
dictionary's meaning language goes to the entries its words find, cut
as the app cuts a line of subtitles (at spaces, Korean's particles off,
Chinese by its longest words), up to three for each, those using the
word as its headword writes it first (Haus rather than zu Hause; 学习
rather than 學習 in the simplified dictionary), then those nearest seven
words long. German's common words nearly all get some (8,849 of
9,292), in ten seconds and under 300 MB; a dictionary whose sentences
can't be had keeps its words. `curate tatoeba` does the same for a file
made by hand.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Signed-off-by: Tizian 「ティツィアーン」 <raisondetredev@pm.me>

halkyon_learn_curator/test/update_test.dart

@@ -1,6 +1,8 @@
 import 'dart:convert';
 import 'dart:io';
 
+import 'package:archive/archive.dart';
+
 import 'package:halkyon_learn_curator/src/update.dart';
 import 'package:test/test.dart';
 
@@ -18,6 +20,16 @@ void main() {
     await file.writeAsBytes(gzip.encode(await File(sample).readAsBytes()));
   }
 
+  /// Tatoeba's exports, as the fixtures hold a few of their lines, packed
+  /// as Tatoeba packs them; null for any other address.
+  List<int>? tatoeba(Uri uri) {
+    final name = uri.pathSegments.last.replaceAll('.bz2', '');
+    final fixture = File('test/fixtures/tatoeba-$name');
+    return fixture.existsSync()
+        ? BZip2Encoder().encodeBytes(fixture.readAsBytesSync())
+        : null;
+  }
+
   /// The header of the dictionary file [name] in [into].
   Map<String, Object?> header(String name) => jsonDecode(
     utf8
@@ -128,6 +140,9 @@ void main() {
       languages: languages,
       modified: modified,
       open: open,
+      fetch: (uri, file) => file.writeAsBytes(
+        tatoeba(uri) ?? (throw HttpException('$uri', uri: uri)),
+      ),
     );
 
     test(
@@ -186,14 +201,13 @@ void main() {
     final written = await updateCedict(
       into: into,
       data: Directory('data'),
-      fetch: (uri, file) async {
-        expect(uri, cedictSource);
-        await file.writeAsBytes(
-          gzip.encode(
-            File('test/fixtures/cedict-sample.txt').readAsBytesSync(),
-          ),
-        );
-      },
+      fetch: (uri, file) => file.writeAsBytes(
+        uri == cedictSource
+            ? gzip.encode(
+                File('test/fixtures/cedict-sample.txt').readAsBytesSync(),
+              )
+            : tatoeba(uri) ?? (throw HttpException('$uri', uri: uri)),
+      ),
     );
 
     expect(
@@ -201,7 +215,40 @@ void main() {
       ['cc-cedict-hans-en.jsonl.gz', 'cc-cedict-hant-en.jsonl.gz'],
     );
     expect(header('cc-cedict-hant-en.jsonl.gz')['language'], 'zh-Hant');
+    expect([
+      for (final credit
+          in header('cc-cedict-hans-en.jsonl.gz')['credits']! as List)
+        (credit as Map)['name'],
+    ], contains('Tatoeba'));
     expect(cedictUpdated(into), isNotNull);
+    // Nothing of the downloads left beside the files.
+    expect(
+      [for (final entity in into.listSync()) entity.uri.pathSegments.last]
+        ..sort(),
+      ['cc-cedict-hans-en.jsonl.gz', 'cc-cedict-hant-en.jsonl.gz'],
+    );
+  });
+
+  test("a dictionary keeps its words without Tatoeba's sentences when they "
+      "can't be had", () async {
+    final written = await updateCedict(
+      into: into,
+      data: Directory('data'),
+      fetch: (uri, file) => uri == cedictSource
+          ? file.writeAsBytes(
+              gzip.encode(
+                File('test/fixtures/cedict-sample.txt').readAsBytesSync(),
+              ),
+            )
+          : throw HttpException('$uri answered 503.', uri: uri),
+    );
+
+    expect(written, hasLength(2));
+    expect([
+      for (final credit
+          in header('cc-cedict-hans-en.jsonl.gz')['credits']! as List)
+        (credit as Map)['name'],
+    ], isNot(contains('Tatoeba')));
   });
 
   test('Chinese comes from CC-CEDICT, the other languages from Wiktionary', () {