feat(learn): furigana and pinyin over hints, from Tatoeba's transcriptions

The curator puts Tatoeba's readings on a dictionary's examples by their
Tatoeba ID (`withTatoebaReadings`, `curate readings`, and in `update`):
furigana over JMdict's Japanese from its jpn transcriptions, pinyin with
tone marks over Chinese from its cmn ones, each word over as many Han
characters as it has syllables. An example gets `ruby`, `[start, end,
reading]` each, and `rubyUnreviewed` where no one wrote or checked them
(MeCab's furigana, the automatic pinyin). JMdict's conversion goes to 3,
so instances convert again at once.

A word's example hints carry them as ruby marks, the word still in bold.
The hints sheet shows them from the start with a switch above
(Furigana, Pinyin), kept on the device (`DeviceSettings.hintReadings`),
and below a hint whose readings no one reviewed says so.

Whats-New: Example sentences in a word's hints now show furigana or pinyin over them, with a switch to hide them.
Whats-New-De: Beispielsätze in den Hinweisen eines Worts zeigen jetzt Furigana oder Pinyin darüber, mit einem Schalter zum Ausblenden.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Signed-off-by: Tizian 「ティツィアーン」 <raisondetredev@pm.me>

apps/learn/halkyon_learn_curator/test/fixtures/tatoeba-cmn_transcriptions.tsv

@@ -0,0 +1,8 @@
+750344	cmn	Hant		我學習。
+750344	cmn	Latn		Wo3 xue2xi2.
+332662	cmn	Hant		您學習嗎？
+332662	cmn	Latn	Yorwba	Nin2 xue2xi2 ma5?
+4456365	cmn	Hant		她學習數學
+4456365	cmn	Latn		Ta1 xue2xi2 shu4xue2
+333376	cmn	Hant		我妹妹目前不學習。
+333376	cmn	Latn	Yorwba	Wo3 mei4mei5 mu4qian2 bu4 xue2xi2.

apps/learn/halkyon_learn_curator/test/fixtures/tatoeba-jpn_transcriptions.tsv

@@ -0,0 +1,3 @@
+193344	jpn	Hrkt	Waldelfe	もっと[果物|くだ|もの]を[食|た]べるべきです。
+81964	jpn	Hrkt		[僕|ぼく]は[脚本|きゃく|ほん][家|か]で[食|た]べていく[決心|けっ|しん]をした。
+232489	jpn	Hrkt		あなたはどこでテレビを[見|み]ますか。

apps/learn/halkyon_learn_curator/test/tatoeba_readings_test.dart

@@ -0,0 +1,167 @@
+import 'dart:convert';
+import 'dart:io';
+
+import 'package:halkyon_learn_curator/src/sources/tatoeba_readings.dart';
+import 'package:test/test.dart';
+
+void main() {
+  Stream<String> lines(String fixture) =>
+      File('test/fixtures/$fixture')
+          .openRead()
+          .transform(utf8.decoder)
+          .transform(const LineSplitter());
+
+  group('Japanese', () {
+    test('a reading over each kanji where Tatoeba gives one each, over the '
+        'whole word where it gives one for all', () {
+      expect(rubyOf('ja', '僕は脚本家で食べる。', '[僕|ぼく]は[脚本|きゃく|ほん][家|か]で[食|た]べる。'), [
+        (0, 1, 'ぼく'),
+        (2, 3, 'きゃく'),
+        (3, 4, 'ほん'),
+        (4, 5, 'か'),
+        (6, 7, 'た'),
+      ]);
+      expect(rubyOf('ja', '一人いくら？', '[一人|ひとり]いくら？'), [(0, 2, 'ひとり')]);
+    });
+
+    test('none for a transcription of another sentence than the example, '
+        'nor for one without any reading', () {
+      expect(rubyOf('ja', '僕は行く。', '[俺|おれ]は[行|い]く。'), isNull);
+      expect(rubyOf('ja', 'ごめん。', 'ごめん。'), isNull);
+      expect(rubyOf('ja', '僕。', '[僕|ぼく。'), isNull);
+    });
+  });
+
+  group('Chinese', () {
+    test('pinyin with tone marks over each word, its syllables counted '
+        "against the sentence's characters, whichever script", () {
+      expect(rubyOf('zh', '我学习。', 'Wo3 xue2xi2.'), [
+        (0, 1, 'wǒ'),
+        (1, 3, 'xuéxí'),
+      ]);
+      expect(rubyOf('zh', '您學習嗎？', 'Nin2 xue2xi2 ma5?'), [
+        (0, 1, 'nín'),
+        (1, 3, 'xuéxí'),
+        (3, 4, 'ma'),
+      ]);
+      expect(rubyOf('zh', '我妹妹去了', 'wo3 mei4mei5 qu4 le5'), [
+        (0, 1, 'wǒ'),
+        (1, 3, 'mèimei'),
+        (3, 4, 'qù'),
+        (4, 5, 'le'),
+      ]);
+    });
+
+    test('marks the vowel pinyin marks: a or e first, the o of ou, else the '
+        'last; ü written u: or v', () {
+      expect(rubyOf('zh', '好够对女绿', 'hao3 gou4 dui4 nu:3 lv4'), [
+        (0, 1, 'hǎo'),
+        (1, 2, 'gòu'),
+        (2, 3, 'duì'),
+        (3, 4, 'nǚ'),
+        (4, 5, 'lǜ'),
+      ]);
+    });
+
+    test("none where the syllables don't match the characters in number", () {
+      expect(rubyOf('zh', '我有3个', 'wo3 you3 san1 ge4'), isNull);
+      expect(rubyOf('zh', '我学习', 'wo3 xue2'), isNull);
+    });
+  });
+
+  test('another language has none', () {
+    expect(rubyOf('ko', '집', 'jip'), isNull);
+  });
+
+  test("reads Tatoeba's transcriptions of one script, those no one wrote or "
+      'checked (MeCab, or the automatic pinyin) unreviewed', () async {
+    final japanese = await readTatoebaTranscriptions(
+      lines('tatoeba-jpn_transcriptions.tsv'),
+    );
+    expect(japanese[193344], (
+      text: 'もっと[果物|くだ|もの]を[食|た]べるべきです。',
+      reviewed: true,
+    ));
+    expect(japanese[81964]?.reviewed, isFalse);
+
+    final chinese = await readTatoebaTranscriptions(
+      lines('tatoeba-cmn_transcriptions.tsv'),
+    );
+    expect(chinese.keys, unorderedEquals([750344, 332662, 4456365, 333376]));
+    expect(chinese[750344], (text: 'Wo3 xue2xi2.', reviewed: false));
+    expect(chinese[332662]?.reviewed, isTrue);
+  });
+
+  test("puts readings on a dictionary's examples by their Tatoeba ID, those "
+      'unreviewed marked, other lines as they were', () async {
+    final header = jsonEncode({
+      'format': 'halkyon-dictionary',
+      'language': 'zh-Hans',
+      'credits': <Object?>[],
+    });
+    String entry(List<Map<String, Object?>> examples) => jsonEncode({
+      'headword': '学习',
+      'entry': {
+        'senses': [
+          {
+            'gloss': ['to learn'],
+            'examples': examples,
+          },
+          {
+            'gloss': ['study'],
+            'examples': [
+              {'text': '学习', 'zh-Hans': '您学习吗？', 'tatoeba': '332662'},
+            ],
+          },
+        ],
+      },
+    });
+    final written = [
+      for (final line in await withTatoebaReadings(
+        Stream.fromIterable([
+          header,
+          entry([
+            {
+              'text': '学习',
+              'zh-Hans': '我学习。',
+              'en': 'I study.',
+              'tatoeba': 750344,
+            },
+            {'text': '学习', 'zh-Hans': '我学。', 'tatoeba': 4456365},
+            {'text': '学习', 'zh-Hans': '我学习。'},
+          ]),
+        ]),
+        {
+          750344: (text: 'Wo3 xue2xi2.', reviewed: false),
+          332662: (text: 'Nin2 xue2xi2 ma5?', reviewed: true),
+          4456365: (text: 'Ta1 xue2xi2 shu4xue2', reviewed: false),
+        },
+      ).toList())
+        jsonDecode(line) as Map<String, Object?>,
+    ];
+
+    expect(written.first, jsonDecode(header));
+    final senses = (written[1]['entry']! as Map)['senses']! as List;
+    final [first, second, third] = (senses[0] as Map)['examples']! as List;
+    expect(first, {
+      'text': '学习',
+      'zh-Hans': '我学习。',
+      'en': 'I study.',
+      'tatoeba': 750344,
+      'ruby': [
+        [0, 1, 'wǒ'],
+        [1, 3, 'xuéxí'],
+      ],
+      'rubyUnreviewed': true,
+    });
+    expect(second, isNot(contains('ruby')));
+    expect(third, isNot(contains('ruby')));
+    final [other] = (senses[1] as Map)['examples']! as List;
+    expect((other as Map)['ruby'], [
+      [0, 1, 'nín'],
+      [1, 3, 'xuéxí'],
+      [3, 4, 'ma'],
+    ]);
+    expect(other, isNot(contains('rubyUnreviewed')));
+  });
+}

apps/learn/halkyon_learn_curator/test/update_test.dart

@@ -14,14 +14,6 @@ void main() {
   setUp(() => into = Directory.systemTemp.createTempSync('dictionaries'));
   tearDown(() => into.deleteSync(recursive: true));
 
-  /// Fetches the samples in place of EDRDG's files of the day.
-  Future<void> samples(Uri uri, File file) async {
-    final sample = uri == jmdictSource
-        ? 'test/fixtures/jmdict-e-examp-sample.xml'
-        : 'test/fixtures/kanjidic2-sample.xml';
-    await file.writeAsBytes(gzip.encode(await File(sample).readAsBytes()));
-  }
-
   /// Tatoeba's exports, as the fixtures hold a few of their lines, packed
   /// as Tatoeba packs them; null for any other address.
   List<int>? tatoeba(Uri uri) {
@@ -32,6 +24,33 @@ void main() {
         : null;
   }
 
+  /// Fetches the samples in place of EDRDG's files of the day, and
+  /// Tatoeba's transcriptions.
+  Future<void> samples(Uri uri, File file) async {
+    if (tatoeba(uri) case final packed?) {
+      await file.writeAsBytes(packed);
+      return;
+    }
+    final sample = uri == jmdictSource
+        ? 'test/fixtures/jmdict-e-examp-sample.xml'
+        : 'test/fixtures/kanjidic2-sample.xml';
+    await file.writeAsBytes(gzip.encode(await File(sample).readAsBytes()));
+  }
+
+  /// The examples of every sense in the dictionary file [name] in [into].
+  List<Map<String, Object?>> examplesIn(String name) => [
+    for (final line
+        in utf8
+            .decode(gzip.decode(File('${into.path}/$name').readAsBytesSync()))
+            .split('\n')
+            .skip(1))
+      if (line.isNotEmpty)
+        for (final sense
+            in ((jsonDecode(line) as Map)['entry'] as Map)['senses'] as List)
+          for (final example in (sense as Map)['examples'] as List? ?? const [])
+            example as Map<String, Object?>,
+  ];
+
   /// The header of the dictionary file [name] in [into].
   Map<String, Object?> header(String name) => jsonDecode(
     utf8
@@ -61,6 +80,19 @@ void main() {
       ['JMdict', 'JLPT vocabulary lists', 'Tatoeba'],
     );
     expect(header('kanjidic-en.jsonl.gz')['kind'], 'characters');
+    // Furigana over the sentences Tatoeba transcribed, those MeCab's alone
+    // marked.
+    final examples = {
+      for (final example in examplesIn('jmdict-en.jsonl.gz'))
+        '${example['tatoeba']}': example,
+    };
+    expect(examples['193344']?['ruby'], [
+      [3, 4, 'くだ'],
+      [4, 5, 'もの'],
+      [6, 7, 'た'],
+    ]);
+    expect(examples['193344'], isNot(contains('rubyUnreviewed')));
+    expect(examples['81964']?['rubyUnreviewed'], isTrue);
     // Nothing half-written left behind; which conversion made them.
     expect(
       [for (final entity in into.listSync()) entity.uri.pathSegments.last]
@@ -242,6 +274,17 @@ void main() {
         (credit as Map)['name'],
     ], contains('Tatoeba'));
     expect(cedictUpdated(into), isNotNull);
+    // Pinyin over the sentences Tatoeba transcribed.
+    expect(
+      [
+        for (final example in examplesIn('cc-cedict-hans-en.jsonl.gz'))
+          if (example['tatoeba'] == 750344) example['ruby'],
+      ].first,
+      [
+        [0, 1, 'wǒ'],
+        [1, 3, 'xuéxí'],
+      ],
+    );
     // Nothing of the downloads left beside the files.
     expect(
       [for (final entity in into.listSync()) entity.uri.pathSegments.last]