feat(curator): dictionaries of German, Greek and English from Wiktionary

The languages besides Japanese come from the English Wiktionary as
Wiktextract reads it (decided 2026-10-14: German, Greek and English
first, then Chinese, then Korean): kaikki.org's raw dump, every language
in one file, as its per-language files are being withdrawn.
`curate wiktionary --languages de,el,en` reads it once, streamed, and
writes German and Greek with Wiktionary's English meanings and English
with what its translation tables give in German, the app's other
language (`wiktionary-de-en`, `-el-en`, `-en-de`; sources
`wiktionary-<language>`, as the server holds one source per meaning
language). Inflected forms, and Wiktionary's entries for forms alone
("Häusern: dative plural of Haus"), find their word instead of being
entries; names and affixes are left out, as are articles that tables
give cells of their own, abbreviations and rare spellings, which would
match common words. Entries keep their part of speech, genders, IPA,
principal forms and examples keyed by language as JMdict's.

wordfreq's small lists (CC BY-SA 4.0, crediting SUBTLEX) in data/ rank
the words by how often they are used and label the 15,000 most used
`common`. Typing without accents or umlauts finds a word (`strasse`,
`haeuser`, `spiti`).

The format gains `textKeys`: what finds an entry in running text, its
own forms, never its meanings, which would match words of the meaning
language in a subtitle. `curate update --wiktionary de,el,en` keeps the
languages current: it asks kaikki.org when the dump last changed and
converts it again only when it is newer, or the languages or the
conversion changed. The whole dump converts in about ten minutes in
under 300 MB.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Signed-off-by: Tizian 「ティツィアーン」 <raisondetredev@pm.me>

halkyon_learn_curator/test/wiktionary_test.dart

@@ -0,0 +1,247 @@
+import 'dart:convert';
+import 'dart:io';
+
+import 'package:halkyon_learn_curator/src/fold.dart';
+import 'package:halkyon_learn_curator/src/sources/wiktionary.dart';
+import 'package:halkyon_learn_curator/src/wordfreq.dart';
+import 'package:test/test.dart';
+
+void main() {
+  late Directory into;
+
+  setUp(() => into = Directory.systemTemp.createTempSync('wiktionary'));
+  tearDown(() => into.deleteSync(recursive: true));
+
+  /// The sample's lines, as the dump's.
+  Stream<String> sample() =>
+      File('test/fixtures/wiktionary-sample.jsonl')
+          .openRead()
+          .transform(utf8.decoder)
+          .transform(const LineSplitter());
+
+  /// The lines of the dictionary file [name] in [into], decoded.
+  List<Map<String, Object?>> linesOf(String name) => [
+    for (final line
+        in utf8
+            .decode(gzip.decode(File('${into.path}/$name').readAsBytesSync()))
+            .split('\n'))
+      if (line.isNotEmpty) jsonDecode(line) as Map<String, Object?>,
+  ];
+
+  Map<String, Object?> entryOf(String name, String id) =>
+      linesOf(name).skip(1).singleWhere((line) => line['id'] == id);
+
+  Future<List<File>> convert({
+    Set<String> languages = const {'de', 'el'},
+    Set<String> translatedInto = const {'de'},
+    Map<String, WordFrequencies> frequencies = const {},
+  }) => wiktionaryDictionaries(
+    sample(),
+    into: into,
+    sourceVersion: '2026-10-03',
+    languages: languages,
+    translatedInto: translatedInto,
+    frequencies: frequencies,
+  );
+
+  test('writes a file for each language, and for English with what its '
+      'translation tables give', () async {
+    final written = await convert();
+
+    expect([for (final file in written) file.uri.pathSegments.last]..sort(), [
+      'wiktionary-de-en.jsonl.gz',
+      'wiktionary-el-en.jsonl.gz',
+      'wiktionary-en-de.jsonl.gz',
+    ]);
+    final header = linesOf('wiktionary-de-en.jsonl.gz').first;
+    expect(header['source'], 'wiktionary-de');
+    expect(header['language'], 'de');
+    expect(header['glossLanguage'], 'en');
+    expect(header['sourceVersion'], '2026-10-03');
+    expect(header['license'], 'CC-BY-SA-4.0');
+    expect(
+      [
+        for (final credit in header['credits']! as List)
+          (credit as Map)['name'],
+      ],
+      ['Wiktionary', 'Wiktextract (kaikki.org)'],
+    );
+    expect(
+      linesOf('wiktionary-en-de.jsonl.gz').first['source'],
+      'wiktionary-en',
+    );
+    // Nothing half-written left behind.
+    expect(
+      [for (final entity in into.listSync()) entity.uri.pathSegments.last]
+        ..sort(),
+      [
+        'wiktionary-de-en.jsonl.gz',
+        'wiktionary-el-en.jsonl.gz',
+        'wiktionary-en-de.jsonl.gz',
+      ],
+    );
+  });
+
+  test('a German noun: its gender, its plural beside it, its meanings with '
+      'their examples, every form finding it in text', () async {
+    await convert();
+
+    final haus = entryOf('wiktionary-de-en.jsonl.gz', 'Haus/noun');
+    expect(haus['headword'], 'Haus');
+    expect(haus.containsKey('reading'), isFalse);
+    final entry = haus['entry']! as Map<String, Object?>;
+    expect(entry['pos'], 'noun');
+    expect(entry['genders'], ['n']);
+    expect(entry['ipa'], '[haʊ̯s]');
+    expect(
+      [for (final form in entry['inflections']! as List) (form as Map)['text']],
+      ['Hauses', 'Häuser'],
+    );
+    final senses = (entry['senses']! as List).cast<Map<String, Object?>>();
+    expect(senses.first['glosses'], ['house, building']);
+    expect((senses.first['examples']! as List).first, {
+      'text': 'Haus',
+      'de': 'In dem Haus haben wir mal gewohnt.',
+      'en': 'We used to live in that house.',
+    });
+    // Its own forms, and those Wiktionary has entries for alone.
+    final textKeys = (haus['textKeys']! as List).cast<String>();
+    expect(textKeys, containsAll(['haus', 'hauses', 'häuser', 'häusern']));
+    final keys = (haus['keys']! as List).cast<String>();
+    expect(keys, containsAll(['haus', 'häusern', 'house, building', 'house']));
+    // Typed without umlauts.
+    expect(
+      (haus['looseKeys']! as List).cast<String>(),
+      containsAll(['hauser', 'haeuser']),
+    );
+  });
+
+  test(
+    'forms, names and other languages are no entries of their own',
+    () async {
+      await convert();
+
+      final ids = [
+        for (final line in linesOf('wiktionary-de-en.jsonl.gz').skip(1))
+          line['id'],
+      ];
+      // Häuser, Häusern, aß and gegessen are forms; Haus and Häusern are also
+      // places, which are names.
+      expect(ids, containsAll(['Haus/noun', 'essen/verb', 'Straße/noun']));
+      expect(ids, isNot(contains('Häusern/noun')));
+      expect(ids, isNot(contains('aß/verb')));
+      expect(ids, isNot(contains('Haus/name')));
+      expect([
+        for (final line in linesOf('wiktionary-de-en.jsonl.gz').skip(1))
+          line['headword'],
+      ], isNot(contains('maison')));
+      final essen = entryOf('wiktionary-de-en.jsonl.gz', 'essen/verb');
+      expect(
+        (essen['textKeys']! as List).cast<String>(),
+        containsAll(['isst', 'aß', 'gegessen']),
+      );
+      // Typed with `ss`.
+      expect(
+        (entryOf('wiktionary-de-en.jsonl.gz', 'Straße/noun')['looseKeys']!
+                as List)
+            .cast<String>(),
+        contains('strasse'),
+      );
+    },
+  );
+
+  test(
+    'a Greek word: found by its romanisation and without its accents',
+    () async {
+      await convert();
+
+      final spiti = entryOf('wiktionary-el-en.jsonl.gz', 'σπίτι/noun');
+      final entry = spiti['entry']! as Map<String, Object?>;
+      expect(entry['romanization'], 'spíti');
+      expect(entry['genders'], ['n']);
+      expect(
+        (spiti['textKeys']! as List).cast<String>(),
+        containsAll(['σπίτι', 'σπιτιού']),
+      );
+      expect((spiti['keys']! as List).cast<String>(), contains('spíti'));
+    // The articles its table gives cells of their own are no forms of it.
+    expect(
+      (spiti['textKeys']! as List).cast<String>(),
+      isNot(anyOf(contains('το'), contains('του'), contains('τα'))),
+    );
+      expect(
+        (spiti['looseKeys']! as List).cast<String>(),
+        containsAll(['σπιτι', 'σπιτιου']),
+      );
+    },
+  );
+
+  test('an English word with its German translations, a meaning each, found '
+      'by its forms and by the German words', () async {
+    await convert();
+
+    final house = entryOf('wiktionary-en-de.jsonl.gz', 'house/noun/1');
+    final entry = house['entry']! as Map<String, Object?>;
+    final senses = (entry['senses']! as List).cast<Map<String, Object?>>();
+    expect(senses.first, {
+      'glosses': ['Haus'],
+      'info': ['human abode'],
+    });
+    // Regional words (Alemannic German) left out.
+    expect([
+      for (final sense in senses) ...sense['glosses']! as List,
+    ], isNot(contains('Huus')));
+    expect(
+      (house['textKeys']! as List).cast<String>(),
+      containsAll(['house', 'houses']),
+    );
+    expect((house['keys']! as List).cast<String>(), contains('haus'));
+    final go = entryOf('wiktionary-en-de.jsonl.gz', 'go/verb/1');
+    expect(
+      (go['textKeys']! as List).cast<String>(),
+      containsAll(['goes', 'went', 'gone']),
+    );
+  });
+
+  test('frequencies rank the words, the most used first, and say which are '
+      'common', () async {
+    final german = await WordFrequencies.read(
+      File('data/wordfreq/small_de.msgpack.gz'),
+    );
+    await convert(languages: {'de'}, frequencies: {'de': german});
+
+    final header = linesOf('wiktionary-de-en.jsonl.gz').first;
+    expect(
+      [
+        for (final credit in header['credits']! as List)
+          (credit as Map)['name'],
+      ],
+      ['Wiktionary', 'Wiktextract (kaikki.org)', 'wordfreq', 'SUBTLEX'],
+    );
+    final ranked = [
+      for (final line in linesOf('wiktionary-de-en.jsonl.gz').skip(1))
+        (line['headword'], line['rank']! as int, line['labels']),
+    ]..sort((a, b) => a.$2.compareTo(b.$2));
+    expect(ranked.first.$1, 'Haus');
+    expect(ranked.first.$3, ['common']);
+  });
+
+  group('folding', () {
+    test('case-folds as Unicode does', () {
+      expect(casefold('Straße'), 'strasse');
+      expect(casefold('ΣΠΊΤΙΣ'), 'σπίτισ');
+    });
+
+    test('drops accents of Latin and Greek letters', () {
+      expect(withoutAccents('Häuser'), 'hauser');
+      expect(withoutAccents('σπιτιού'), 'σπιτιου');
+      expect(withoutAccents('café'), 'cafe');
+      expect(withoutAccents('学習'), '学習');
+    });
+
+    test('writes umlauts out', () {
+      expect(umlautsWrittenOut('Häuser'), 'haeuser');
+      expect(umlautsWrittenOut('Haus'), isNull);
+    });
+  });
+}