halkyon/apps/learn/halkyon_learn_curator/test/tatoeba_test.dart

import 'dart:convert';
import 'dart:io';

import 'package:halkyon_learn_curator/src/sources/cedict.dart';
import 'package:halkyon_learn_curator/src/sources/tatoeba.dart';
import 'package:halkyon_learn_curator/src/sources/wiktionary.dart';
import 'package:test/test.dart';

void main() {
  late Directory into;

  setUp(() => into = Directory.systemTemp.createTempSync('tatoeba'));
  tearDown(() => into.deleteSync(recursive: true));

  Stream<String> lines(String fixture) =>
      File('test/fixtures/$fixture')
          .openRead()
          .transform(utf8.decoder)
          .transform(const LineSplitter());

  /// [file]'s lines with the fixtures' sentences of [code] as examples.
  Future<List<Map<String, Object?>>> enriched(File file, String code) async {
    final links = await readTatoebaLinks(lines('tatoeba-$code-eng_links.tsv'));
    final sentences = await readTatoebaSentences(
      lines('tatoeba-${code}_sentences.tsv'),
      wanted: links.containsKey,
    );
    final linked = {for (final ids in links.values) ...ids};
    final translations = await readTatoebaSentences(
      lines('tatoeba-eng_sentences.tsv'),
      wanted: linked.contains,
    );
    return [
      for (final line in await withTatoebaExamples(
        file,
        sentences: sentences,
        translations: translations,
        links: links,
      ).join().then((text) => text.split('\n')))
        if (line.isNotEmpty) jsonDecode(line) as Map<String, Object?>,
    ];
  }

  List<Map<String, Object?>> examplesOf(Map<String, Object?> line) => [
    for (final example
        in ((((line['entry']! as Map)['senses']! as List).first
                    as Map)['examples']
                as List? ??
            const []))
      example as Map<String, Object?>,
  ];

  test("a German word's sentences come first in its first sense, those "
      'without a translation left out, the header crediting Tatoeba', () async {
    final [german, ...] = await wiktionaryDictionaries(
      lines('wiktionary-sample.jsonl'),
      into: into,
      sourceVersion: '2026-10-03',
      languages: {'de'},
    );

    final result = await enriched(german, 'deu');

    expect([
      for (final credit in result.first['credits']! as List)
        (credit as Map)['name'],
    ], contains('Tatoeba'));
    final haus = result
        .skip(1)
        .singleWhere((line) => line['id'] == 'Haus/noun');
    final examples = examplesOf(haus);
    // Three of Tatoeba's, before Wiktionary's own.
    // Those nearest seven words long first, then the shortest.
    expect(examples.take(3).map((example) => example['tatoeba']), [
      9984782,
      1550095,
      605418,
    ]);
    expect(examples.first['text'], 'Haus');
    expect(examples.first['de'], 'Er ist im Haus.');
    expect(examples.first['en'], isA<String>());
    expect(examples[3]['de'], 'In dem Haus haben wir mal gewohnt.');
    // A form finds its word: aß is essen's.
    final essen = result
        .skip(1)
        .singleWhere((line) => line['id'] == 'essen/verb');
    expect(examplesOf(essen).first['text'], 'aß');
  });

  test(
    'Chinese sentences are cut by the longest words the dictionary has',
    () async {
      final file = File('${into.path}/cc-cedict-hans-en.jsonl');
      await file.writeAsString(
        cedictDictionary(
          await File('test/fixtures/cedict-sample.txt').readAsString(),
          traditional: false,
        ).join(),
      );

      final result = await enriched(file, 'cmn');

      final xuexi = result
          .skip(1)
          .singleWhere((line) => line['id'] == '學習 学习 [xue2 xi2]');
      expect(examplesOf(xuexi), isNotEmpty);
      expect(examplesOf(xuexi).first['text'], '学习');
      expect(examplesOf(xuexi).first['zh-Hans'], contains('学习'));
    },
  );

  test('Korean words are found with their particles taken off', () async {
    final [korean, ...] = await wiktionaryDictionaries(
      lines('wiktionary-sample.jsonl'),
      into: into,
      sourceVersion: '2026-10-03',
      languages: {'ko'},
    );

    final result = await enriched(korean, 'kor');

    final home = result.skip(1).singleWhere((line) => line['id'] == '집/noun/1');
    expect(examplesOf(home), isNotEmpty);
    expect(examplesOf(home).first['text'], '집');
    expect(examplesOf(home).first['ko'], contains('집에서'));
  });

  test("Tatoeba's codes of the languages", () {
    expect(tatoebaCode('de'), 'deu');
    expect(tatoebaCode('zh-Hant'), 'cmn');
    expect(tatoebaCode('ja'), isNull);
  });
}