halkyon/apps/learn/halkyon_learn_curator/test/wiktionary_test.dart

import 'dart:convert';
import 'dart:io';

import 'package:halkyon_learn_curator/src/fold.dart';
import 'package:halkyon_learn_curator/src/sources/wiktionary.dart';
import 'package:halkyon_learn_curator/src/wordfreq.dart';
import 'package:test/test.dart';

void main() {
  late Directory into;

  setUp(() => into = Directory.systemTemp.createTempSync('wiktionary'));
  tearDown(() => into.deleteSync(recursive: true));

  /// The sample's lines, as the dump's.
  Stream<String> sample() =>
      File('test/fixtures/wiktionary-sample.jsonl')
          .openRead()
          .transform(utf8.decoder)
          .transform(const LineSplitter());

  /// The lines of the dictionary file [name] in [into], decoded.
  List<Map<String, Object?>> linesOf(String name) => [
    for (final line
        in utf8
            .decode(gzip.decode(File('${into.path}/$name').readAsBytesSync()))
            .split('\n'))
      if (line.isNotEmpty) jsonDecode(line) as Map<String, Object?>,
  ];

  Map<String, Object?> entryOf(String name, String id) =>
      linesOf(name).skip(1).singleWhere((line) => line['id'] == id);

  Future<List<File>> convert({
    Set<String> languages = const {'de', 'el'},
    Set<String> translatedInto = const {'de'},
    Map<String, WordFrequencies> frequencies = const {},
  }) => wiktionaryDictionaries(
    sample(),
    into: into,
    sourceVersion: '2026-10-03',
    languages: languages,
    translatedInto: translatedInto,
    frequencies: frequencies,
  );

  test('writes a file for each language, and for English with what its '
      'translation tables give', () async {
    final written = await convert();

    expect([for (final file in written) file.uri.pathSegments.last]..sort(), [
      'wiktionary-de-en.jsonl.gz',
      'wiktionary-el-en.jsonl.gz',
      'wiktionary-en-de.jsonl.gz',
    ]);
    final header = linesOf('wiktionary-de-en.jsonl.gz').first;
    expect(header['source'], 'wiktionary-de');
    expect(header['language'], 'de');
    expect(header['glossLanguage'], 'en');
    expect(header['sourceVersion'], '2026-10-03');
    expect(header['license'], 'CC-BY-SA-4.0');
    expect(
      [
        for (final credit in header['credits']! as List)
          (credit as Map)['name'],
      ],
      ['Wiktionary', 'Wiktextract (kaikki.org)'],
    );
    expect(
      linesOf('wiktionary-en-de.jsonl.gz').first['source'],
      'wiktionary-en',
    );
    // Nothing half-written left behind.
    expect(
      [for (final entity in into.listSync()) entity.uri.pathSegments.last]
        ..sort(),
      [
        'wiktionary-de-en.jsonl.gz',
        'wiktionary-el-en.jsonl.gz',
        'wiktionary-en-de.jsonl.gz',
      ],
    );
  });

  test('a German noun: its gender, its plural beside it, its meanings with '
      'their examples, every form finding it in text', () async {
    await convert();

    final haus = entryOf('wiktionary-de-en.jsonl.gz', 'Haus/noun');
    expect(haus['headword'], 'Haus');
    expect(haus.containsKey('reading'), isFalse);
    final entry = haus['entry']! as Map<String, Object?>;
    expect(entry['pos'], 'noun');
    expect(entry['genders'], ['n']);
    expect(entry['ipa'], '[haʊ̯s]');
    expect(
      [for (final form in entry['inflections']! as List) (form as Map)['text']],
      ['Hauses', 'Häuser'],
    );
    final senses = (entry['senses']! as List).cast<Map<String, Object?>>();
    expect(senses.first['glosses'], ['house, building']);
    expect((senses.first['examples']! as List).first, {
      'text': 'Haus',
      'de': 'In dem Haus haben wir mal gewohnt.',
      'en': 'We used to live in that house.',
    });
    // Its own forms, and those Wiktionary has entries for alone.
    final textKeys = (haus['textKeys']! as List).cast<String>();
    expect(textKeys, containsAll(['haus', 'hauses', 'häuser', 'häusern']));
    final keys = (haus['keys']! as List).cast<String>();
    expect(keys, containsAll(['haus', 'häusern', 'house, building', 'house']));
    // Typed without umlauts.
    expect(
      (haus['looseKeys']! as List).cast<String>(),
      containsAll(['hauser', 'haeuser']),
    );
  });

  test(
    'forms, names and other languages are no entries of their own',
    () async {
      await convert();

      final ids = [
        for (final line in linesOf('wiktionary-de-en.jsonl.gz').skip(1))
          line['id'],
      ];
      // Häuser, Häusern, aß and gegessen are forms; Haus and Häusern are also
      // places, which are names.
      expect(ids, containsAll(['Haus/noun', 'essen/verb', 'Straße/noun']));
      expect(ids, isNot(contains('Häusern/noun')));
      expect(ids, isNot(contains('aß/verb')));
      expect(ids, isNot(contains('Haus/name')));
      expect([
        for (final line in linesOf('wiktionary-de-en.jsonl.gz').skip(1))
          line['headword'],
      ], isNot(contains('maison')));
      final essen = entryOf('wiktionary-de-en.jsonl.gz', 'essen/verb');
      expect(
        (essen['textKeys']! as List).cast<String>(),
        containsAll(['isst', 'aß', 'gegessen']),
      );
      // A contraction is a word of its own: im, of in and dem.
      expect(ids, contains('im/contraction'));
      // Typed with `ss`.
      expect(
        (entryOf('wiktionary-de-en.jsonl.gz', 'Straße/noun')['looseKeys']!
                as List)
            .cast<String>(),
        contains('strasse'),
      );
    },
  );

  test(
    'a Greek word: found by its romanisation and without its accents',
    () async {
      await convert();

      final spiti = entryOf('wiktionary-el-en.jsonl.gz', 'σπίτι/noun');
      final entry = spiti['entry']! as Map<String, Object?>;
      expect(entry['romanization'], 'spíti');
      expect(entry['genders'], ['n']);
      expect(
        (spiti['textKeys']! as List).cast<String>(),
        containsAll(['σπίτι', 'σπιτιού']),
      );
      expect((spiti['keys']! as List).cast<String>(), contains('spíti'));
      // The articles its table gives cells of their own are no forms of it.
      expect(
        (spiti['textKeys']! as List).cast<String>(),
        isNot(anyOf(contains('το'), contains('του'), contains('τα'))),
      );
      expect(
        (spiti['looseKeys']! as List).cast<String>(),
        containsAll(['σπιτι', 'σπιτιου']),
      );
    },
  );

  test('a Korean word: its Hanja and romanisation find it, its conjugations '
      'in text; a counter word its table names is no form of it', () async {
    await convert(languages: {'ko'}, translatedInto: {});

    final student = entryOf('wiktionary-ko-en.jsonl.gz', '학생/noun');
    final entry = student['entry']! as Map<String, Object?>;
    expect(entry['hanja'], ['學生']);
    expect(entry['romanization'], 'haksaeng');
    expect(
      (student['keys']! as List).cast<String>(),
      containsAll(['학생', '學生', 'haksaeng', 'student']),
    );
    expect((student['textKeys']! as List).cast<String>(), ['학생']);
    final eat = entryOf('wiktionary-ko-en.jsonl.gz', '먹다/verb');
    expect(
      (eat['textKeys']! as List).cast<String>(),
      containsAll(['먹다', '먹어요', '먹었어요', '먹습니다']),
    );
    expect(
      (eat['textKeys']! as List).cast<String>(),
      isNot(contains('meokda')),
    );
  });

  test("for Korean's dictionaries without its words: each Hanja's native "
      "reading and sound, each word's forms", () async {
    final korean = WiktionaryKorean();
    final written = await wiktionaryDictionaries(
      sample(),
      into: into,
      sourceVersion: '2026-10-03',
      korean: korean,
    );

    expect(written, isEmpty);
    expect(korean.hanja['水'], [(hun: '물', eum: '수')]);
    expect(korean.forms['먹다'], containsAll(['먹어요', '먹었어요']));
    expect(korean.forms['먹다'], isNot(contains('먹다')));
    // Kept between updates as JSON.
    final kept = WiktionaryKorean.fromJson(korean.toJson());
    expect(kept.hanja['水'], [(hun: '물', eum: '수')]);
    expect(kept.forms['먹다'], korean.forms['먹다']);
  });

  test('an English word with its German translations, a meaning each, found '
      'by its forms and by the German words', () async {
    await convert();

    final house = entryOf('wiktionary-en-de.jsonl.gz', 'house/noun/1');
    final entry = house['entry']! as Map<String, Object?>;
    final senses = (entry['senses']! as List).cast<Map<String, Object?>>();
    expect(senses.first, {
      'glosses': ['Haus'],
      'info': ['human abode'],
    });
    // Regional words (Alemannic German) left out.
    expect([
      for (final sense in senses) ...sense['glosses']! as List,
    ], isNot(contains('Huus')));
    expect(
      (house['textKeys']! as List).cast<String>(),
      containsAll(['house', 'houses']),
    );
    expect((house['keys']! as List).cast<String>(), contains('haus'));
    final go = entryOf('wiktionary-en-de.jsonl.gz', 'go/verb/1');
    expect(
      (go['textKeys']! as List).cast<String>(),
      containsAll(['goes', 'went', 'gone']),
    );
  });

  test('frequencies rank the words, the most used first, and say which are '
      'common', () async {
    final german = await WordFrequencies.read(
      File('data/wordfreq/small_de.msgpack.gz'),
    );
    await convert(languages: {'de'}, frequencies: {'de': german});

    final header = linesOf('wiktionary-de-en.jsonl.gz').first;
    expect(
      [
        for (final credit in header['credits']! as List)
          (credit as Map)['name'],
      ],
      ['Wiktionary', 'Wiktextract (kaikki.org)', 'wordfreq', 'SUBTLEX'],
    );
    final ranked = [
      for (final line in linesOf('wiktionary-de-en.jsonl.gz').skip(1))
        (line['headword'], line['rank']! as int, line['labels']),
    ]..sort((a, b) => a.$2.compareTo(b.$2));
    expect(
      [
        for (final (word, _, _) in ranked)
          if (['im', 'Haus', 'Straße'].contains(word)) word,
      ],
      ['im', 'Haus', 'Straße'],
    );
    expect(ranked.firstWhere((each) => each.$1 == 'Haus').$3, ['common']);
  });

  test("a word's frequency goes to its first entries alone, not to every "
      'homograph of a syllable', () async {
    final korean = await WordFrequencies.read(
      File('data/wordfreq/small_ko.msgpack.gz'),
    );
    // Five entries of 이, as Wiktionary has thirteen.
    await wiktionaryDictionaries(
      Stream.fromIterable([
        for (var number = 1; number <= 5; number++)
          jsonEncode({
            'word': '이',
            'pos': 'noun',
            'lang_code': 'ko',
            'etymology_number': number,
            'senses': [
              {
                'glosses': ['meaning $number'],
              },
            ],
          }),
      ]),
      into: into,
      sourceVersion: '2026-10-03',
      languages: {'ko'},
      frequencies: {'ko': korean},
    );

    final entries = linesOf('wiktionary-ko-en.jsonl.gz').skip(1).toList();
    expect(
      [for (final entry in entries) entry['labels']],
      [
        ['common'],
        ['common'],
        ['common'],
        <String>[],
        <String>[],
      ],
    );
    expect(entries[3]['rank']! as int, greaterThan(entries[2]['rank']! as int));
  });

  group('folding', () {
    test('case-folds as Unicode does', () {
      expect(casefold('Straße'), 'strasse');
      expect(casefold('ΣΠΊΤΙΣ'), 'σπίτισ');
    });

    test('drops accents of Latin and Greek letters', () {
      expect(withoutAccents('Häuser'), 'hauser');
      expect(withoutAccents('σπιτιού'), 'σπιτιου');
      expect(withoutAccents('café'), 'cafe');
      expect(withoutAccents('学習'), '学習');
    });

    test('writes umlauts out', () {
      expect(umlautsWrittenOut('Häuser'), 'haeuser');
      expect(umlautsWrittenOut('Haus'), isNull);
    });
  });
}