chore: Halkyon Learn moves into apps/learn

Its server, client, app and curator, and the tools that write its
assets (maps, Anki fixtures, icons), as they are; paths are fixed in
the commits that follow.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Signed-off-by: Tizian 「ティツィアーン」 <raisondetredev@pm.me>

apps/learn/halkyon_learn_curator/test/wiktionary_test.dart

@@ -0,0 +1,340 @@
+import 'dart:convert';
+import 'dart:io';
+
+import 'package:halkyon_learn_curator/src/fold.dart';
+import 'package:halkyon_learn_curator/src/sources/wiktionary.dart';
+import 'package:halkyon_learn_curator/src/wordfreq.dart';
+import 'package:test/test.dart';
+
+void main() {
+  late Directory into;
+
+  setUp(() => into = Directory.systemTemp.createTempSync('wiktionary'));
+  tearDown(() => into.deleteSync(recursive: true));
+
+  /// The sample's lines, as the dump's.
+  Stream<String> sample() =>
+      File('test/fixtures/wiktionary-sample.jsonl')
+          .openRead()
+          .transform(utf8.decoder)
+          .transform(const LineSplitter());
+
+  /// The lines of the dictionary file [name] in [into], decoded.
+  List<Map<String, Object?>> linesOf(String name) => [
+    for (final line
+        in utf8
+            .decode(gzip.decode(File('${into.path}/$name').readAsBytesSync()))
+            .split('\n'))
+      if (line.isNotEmpty) jsonDecode(line) as Map<String, Object?>,
+  ];
+
+  Map<String, Object?> entryOf(String name, String id) =>
+      linesOf(name).skip(1).singleWhere((line) => line['id'] == id);
+
+  Future<List<File>> convert({
+    Set<String> languages = const {'de', 'el'},
+    Set<String> translatedInto = const {'de'},
+    Map<String, WordFrequencies> frequencies = const {},
+  }) => wiktionaryDictionaries(
+    sample(),
+    into: into,
+    sourceVersion: '2026-10-03',
+    languages: languages,
+    translatedInto: translatedInto,
+    frequencies: frequencies,
+  );
+
+  test('writes a file for each language, and for English with what its '
+      'translation tables give', () async {
+    final written = await convert();
+
+    expect([for (final file in written) file.uri.pathSegments.last]..sort(), [
+      'wiktionary-de-en.jsonl.gz',
+      'wiktionary-el-en.jsonl.gz',
+      'wiktionary-en-de.jsonl.gz',
+    ]);
+    final header = linesOf('wiktionary-de-en.jsonl.gz').first;
+    expect(header['source'], 'wiktionary-de');
+    expect(header['language'], 'de');
+    expect(header['glossLanguage'], 'en');
+    expect(header['sourceVersion'], '2026-10-03');
+    expect(header['license'], 'CC-BY-SA-4.0');
+    expect(
+      [
+        for (final credit in header['credits']! as List)
+          (credit as Map)['name'],
+      ],
+      ['Wiktionary', 'Wiktextract (kaikki.org)'],
+    );
+    expect(
+      linesOf('wiktionary-en-de.jsonl.gz').first['source'],
+      'wiktionary-en',
+    );
+    // Nothing half-written left behind.
+    expect(
+      [for (final entity in into.listSync()) entity.uri.pathSegments.last]
+        ..sort(),
+      [
+        'wiktionary-de-en.jsonl.gz',
+        'wiktionary-el-en.jsonl.gz',
+        'wiktionary-en-de.jsonl.gz',
+      ],
+    );
+  });
+
+  test('a German noun: its gender, its plural beside it, its meanings with '
+      'their examples, every form finding it in text', () async {
+    await convert();
+
+    final haus = entryOf('wiktionary-de-en.jsonl.gz', 'Haus/noun');
+    expect(haus['headword'], 'Haus');
+    expect(haus.containsKey('reading'), isFalse);
+    final entry = haus['entry']! as Map<String, Object?>;
+    expect(entry['pos'], 'noun');
+    expect(entry['genders'], ['n']);
+    expect(entry['ipa'], '[haʊ̯s]');
+    expect(
+      [for (final form in entry['inflections']! as List) (form as Map)['text']],
+      ['Hauses', 'Häuser'],
+    );
+    final senses = (entry['senses']! as List).cast<Map<String, Object?>>();
+    expect(senses.first['glosses'], ['house, building']);
+    expect((senses.first['examples']! as List).first, {
+      'text': 'Haus',
+      'de': 'In dem Haus haben wir mal gewohnt.',
+      'en': 'We used to live in that house.',
+    });
+    // Its own forms, and those Wiktionary has entries for alone.
+    final textKeys = (haus['textKeys']! as List).cast<String>();
+    expect(textKeys, containsAll(['haus', 'hauses', 'häuser', 'häusern']));
+    final keys = (haus['keys']! as List).cast<String>();
+    expect(keys, containsAll(['haus', 'häusern', 'house, building', 'house']));
+    // Typed without umlauts.
+    expect(
+      (haus['looseKeys']! as List).cast<String>(),
+      containsAll(['hauser', 'haeuser']),
+    );
+  });
+
+  test(
+    'forms, names and other languages are no entries of their own',
+    () async {
+      await convert();
+
+      final ids = [
+        for (final line in linesOf('wiktionary-de-en.jsonl.gz').skip(1))
+          line['id'],
+      ];
+      // Häuser, Häusern, aß and gegessen are forms; Haus and Häusern are also
+      // places, which are names.
+      expect(ids, containsAll(['Haus/noun', 'essen/verb', 'Straße/noun']));
+      expect(ids, isNot(contains('Häusern/noun')));
+      expect(ids, isNot(contains('aß/verb')));
+      expect(ids, isNot(contains('Haus/name')));
+      expect([
+        for (final line in linesOf('wiktionary-de-en.jsonl.gz').skip(1))
+          line['headword'],
+      ], isNot(contains('maison')));
+      final essen = entryOf('wiktionary-de-en.jsonl.gz', 'essen/verb');
+      expect(
+        (essen['textKeys']! as List).cast<String>(),
+        containsAll(['isst', 'aß', 'gegessen']),
+      );
+      // A contraction is a word of its own: im, of in and dem.
+      expect(ids, contains('im/contraction'));
+      // Typed with `ss`.
+      expect(
+        (entryOf('wiktionary-de-en.jsonl.gz', 'Straße/noun')['looseKeys']!
+                as List)
+            .cast<String>(),
+        contains('strasse'),
+      );
+    },
+  );
+
+  test(
+    'a Greek word: found by its romanisation and without its accents',
+    () async {
+      await convert();
+
+      final spiti = entryOf('wiktionary-el-en.jsonl.gz', 'σπίτι/noun');
+      final entry = spiti['entry']! as Map<String, Object?>;
+      expect(entry['romanization'], 'spíti');
+      expect(entry['genders'], ['n']);
+      expect(
+        (spiti['textKeys']! as List).cast<String>(),
+        containsAll(['σπίτι', 'σπιτιού']),
+      );
+      expect((spiti['keys']! as List).cast<String>(), contains('spíti'));
+      // The articles its table gives cells of their own are no forms of it.
+      expect(
+        (spiti['textKeys']! as List).cast<String>(),
+        isNot(anyOf(contains('το'), contains('του'), contains('τα'))),
+      );
+      expect(
+        (spiti['looseKeys']! as List).cast<String>(),
+        containsAll(['σπιτι', 'σπιτιου']),
+      );
+    },
+  );
+
+  test('a Korean word: its Hanja and romanisation find it, its conjugations '
+      'in text; a counter word its table names is no form of it', () async {
+    await convert(languages: {'ko'}, translatedInto: {});
+
+    final student = entryOf('wiktionary-ko-en.jsonl.gz', '학생/noun');
+    final entry = student['entry']! as Map<String, Object?>;
+    expect(entry['hanja'], ['學生']);
+    expect(entry['romanization'], 'haksaeng');
+    expect(
+      (student['keys']! as List).cast<String>(),
+      containsAll(['학생', '學生', 'haksaeng', 'student']),
+    );
+    expect((student['textKeys']! as List).cast<String>(), ['학생']);
+    final eat = entryOf('wiktionary-ko-en.jsonl.gz', '먹다/verb');
+    expect(
+      (eat['textKeys']! as List).cast<String>(),
+      containsAll(['먹다', '먹어요', '먹었어요', '먹습니다']),
+    );
+    expect(
+      (eat['textKeys']! as List).cast<String>(),
+      isNot(contains('meokda')),
+    );
+  });
+
+  test("for Korean's dictionaries without its words: each Hanja's native "
+      "reading and sound, each word's forms", () async {
+    final korean = WiktionaryKorean();
+    final written = await wiktionaryDictionaries(
+      sample(),
+      into: into,
+      sourceVersion: '2026-10-03',
+      korean: korean,
+    );
+
+    expect(written, isEmpty);
+    expect(korean.hanja['水'], [(hun: '물', eum: '수')]);
+    expect(korean.forms['먹다'], containsAll(['먹어요', '먹었어요']));
+    expect(korean.forms['먹다'], isNot(contains('먹다')));
+    // Kept between updates as JSON.
+    final kept = WiktionaryKorean.fromJson(korean.toJson());
+    expect(kept.hanja['水'], [(hun: '물', eum: '수')]);
+    expect(kept.forms['먹다'], korean.forms['먹다']);
+  });
+
+  test('an English word with its German translations, a meaning each, found '
+      'by its forms and by the German words', () async {
+    await convert();
+
+    final house = entryOf('wiktionary-en-de.jsonl.gz', 'house/noun/1');
+    final entry = house['entry']! as Map<String, Object?>;
+    final senses = (entry['senses']! as List).cast<Map<String, Object?>>();
+    expect(senses.first, {
+      'glosses': ['Haus'],
+      'info': ['human abode'],
+    });
+    // Regional words (Alemannic German) left out.
+    expect([
+      for (final sense in senses) ...sense['glosses']! as List,
+    ], isNot(contains('Huus')));
+    expect(
+      (house['textKeys']! as List).cast<String>(),
+      containsAll(['house', 'houses']),
+    );
+    expect((house['keys']! as List).cast<String>(), contains('haus'));
+    final go = entryOf('wiktionary-en-de.jsonl.gz', 'go/verb/1');
+    expect(
+      (go['textKeys']! as List).cast<String>(),
+      containsAll(['goes', 'went', 'gone']),
+    );
+  });
+
+  test('frequencies rank the words, the most used first, and say which are '
+      'common', () async {
+    final german = await WordFrequencies.read(
+      File('data/wordfreq/small_de.msgpack.gz'),
+    );
+    await convert(languages: {'de'}, frequencies: {'de': german});
+
+    final header = linesOf('wiktionary-de-en.jsonl.gz').first;
+    expect(
+      [
+        for (final credit in header['credits']! as List)
+          (credit as Map)['name'],
+      ],
+      ['Wiktionary', 'Wiktextract (kaikki.org)', 'wordfreq', 'SUBTLEX'],
+    );
+    final ranked = [
+      for (final line in linesOf('wiktionary-de-en.jsonl.gz').skip(1))
+        (line['headword'], line['rank']! as int, line['labels']),
+    ]..sort((a, b) => a.$2.compareTo(b.$2));
+    expect(
+      [
+        for (final (word, _, _) in ranked)
+          if (['im', 'Haus', 'Straße'].contains(word)) word,
+      ],
+      ['im', 'Haus', 'Straße'],
+    );
+    expect(ranked.firstWhere((each) => each.$1 == 'Haus').$3, ['common']);
+  });
+
+  test("a word's frequency goes to its first entries alone, not to every "
+      'homograph of a syllable', () async {
+    final korean = await WordFrequencies.read(
+      File('data/wordfreq/small_ko.msgpack.gz'),
+    );
+    // Five entries of 이, as Wiktionary has thirteen.
+    await wiktionaryDictionaries(
+      Stream.fromIterable([
+        for (var number = 1; number <= 5; number++)
+          jsonEncode({
+            'word': '이',
+            'pos': 'noun',
+            'lang_code': 'ko',
+            'etymology_number': number,
+            'senses': [
+              {
+                'glosses': ['meaning $number'],
+              },
+            ],
+          }),
+      ]),
+      into: into,
+      sourceVersion: '2026-10-03',
+      languages: {'ko'},
+      frequencies: {'ko': korean},
+    );
+
+    final entries = linesOf('wiktionary-ko-en.jsonl.gz').skip(1).toList();
+    expect(
+      [for (final entry in entries) entry['labels']],
+      [
+        ['common'],
+        ['common'],
+        ['common'],
+        <String>[],
+        <String>[],
+      ],
+    );
+    expect(entries[3]['rank']! as int, greaterThan(entries[2]['rank']! as int));
+  });
+
+  group('folding', () {
+    test('case-folds as Unicode does', () {
+      expect(casefold('Straße'), 'strasse');
+      expect(casefold('ΣΠΊΤΙΣ'), 'σπίτισ');
+    });
+
+    test('drops accents of Latin and Greek letters', () {
+      expect(withoutAccents('Häuser'), 'hauser');
+      expect(withoutAccents('σπιτιού'), 'σπιτιου');
+      expect(withoutAccents('café'), 'cafe');
+      expect(withoutAccents('学習'), '学習');
+    });
+
+    test('writes umlauts out', () {
+      expect(umlautsWrittenOut('Häuser'), 'haeuser');
+      expect(umlautsWrittenOut('Haus'), isNull);
+    });
+  });
+}