halkyon/halkyon_learn_curator/test/jmdict_test.dart

import 'dart:convert';
import 'dart:io';

import 'package:halkyon_learn_curator/src/sources/jlpt.dart';
import 'package:halkyon_learn_curator/src/sources/jmdict.dart';
import 'package:test/test.dart';

void main() {
  late List<Map<String, Object?>> lines;

  setUpAll(() async {
    final jlpt = await readJlptLevels(Directory('test/fixtures/jlpt'));
    lines = [
      for (final line in await jmdictDictionary(
        File('test/fixtures/jmdict-eng-sample.json')
            .openRead()
            .transform(utf8.decoder),
        jlpt: jlpt,
      ).toList())
        jsonDecode(line) as Map<String, Object?>,
    ];
  });

  Map<String, Object?> entry(String headword) =>
      lines.skip(1).firstWhere((line) => line['headword'] == headword);

  test('the header says what the file holds, and whose it is', () {
    final header = lines.first;
    expect(header['format'], 'halkyon-dictionary');
    expect(header['formatVersion'], 1);
    expect(header['source'], 'jmdict');
    expect(header['sourceVersion'], '3.6.2 (2026-10-05)');
    expect((header['language'], header['glossLanguage']), ('ja', 'en'));
    expect(header['license'], 'CC-BY-SA-4.0');
    expect(
      [
        for (final credit in header['credits']! as List)
          (credit as Map)['name'],
      ],
      ['JMdict', 'jmdict-simplified', 'JLPT vocabulary lists'],
    );
    expect(lines, hasLength(7));
  });

  test('a word: its headword and reading, labels, and the keys a search '
      'matches, in kanji, kana, romaji and its meanings', () {
    final word = entry('食べる');

    expect(word['id'], '1358280');
    expect(word['reading'], 'たべる');
    expect(word['labels'], ['common', 'jlpt:n5']);
    expect(
      word['keys'],
      containsAll(['食べる', '喰べる', 'たべる', 'taberu', 'to eat', 'eat']),
    );
    // A form for searching only isn't shown.
    final forms = (word['entry']! as Map)['forms']! as List;
    expect([for (final form in forms) (form as Map)['text']], ['食べる']);
  });

  test('a word written in kana has no reading of its own; katakana is '
      'searched as hiragana too', () {
    final word = entry('テレビ');

    expect(word.containsKey('reading'), isFalse);
    expect(word['keys'], containsAll(['テレビ', 'てれび', 'terebi', 'tv']));
  });

  test('long vowels typed once are loose keys, found after the right '
      'spellings', () {
    final tokyo = entry('東京');
    expect(tokyo['keys'], containsAll(['toukyou', 'toukei']));
    // Its other reading, とうけい, typed with one o.
    expect(tokyo['looseKeys'], ['tokei']);
    // "tokyo" is a key already, its meaning in English.
    expect(tokyo['keys'], contains('tokyo'));
    expect(entry('食べる').containsKey('looseKeys'), isFalse);
  });

  test('the reading is one that belongs to the headword', () {
    expect(entry('四')['reading'], 'し');
  });

  test("the headword is the JLPT lists' spelling, at the easiest level a "
      'word is listed at', () {
    final word = entry('歯磨き');

    expect(word['labels'], ['common', 'jlpt:n2']);
  });

  group('The headword', () {
    /// A JMdict entry in jmdict-simplified's shape: written forms and
    /// readings as (text, tags), and its first sense's misc codes.
    Map<String, Object?> word(
      List<(String, List<String>)> kanji,
      List<(String, List<String>)> kana, {
      List<String> misc = const [],
    }) => {
      'id': '1',
      'kanji': [
        for (final (text, tags) in kanji)
          {'text': text, 'common': false, 'tags': tags},
      ],
      'kana': [
        for (final (text, tags) in kana)
          {
            'text': text,
            'common': true,
            'tags': tags,
            'appliesToKanji': ['*'],
          },
      ],
      'sense': [
        {
          'partOfSpeech': ['conj'],
          'misc': misc,
          'gloss': [
            {'text': 'and then'},
          ],
        },
      ],
    };

    test('is a usual reading rather than a rare or search-only kanji '
        'spelling', () {
      expect(
        jmdictEntry(
          word(
            [
              ('而して', ['rK']),
            ],
            [('そして', [])],
          ),
        ).headword,
        'そして',
      );
      expect(
        jmdictEntry(
          word(
            [
              ('居らっしゃる', ['sK']),
            ],
            [('いらっしゃる', [])],
          ),
        ).headword,
        'いらっしゃる',
      );
    });

    test('is in kana when JMdict says the word is usually written in kana, '
        'or the JLPT lists give no kanji', () {
      final arigatou = word([('有難う', [])], [('ありがとう', [])]);

      expect(
        jmdictEntry(
          arigatou,
          jlpt: {'1': (level: 5, place: 0, spelling: null)},
        ).headword,
        'ありがとう',
      );
      expect(jmdictEntry(arigatou).headword, '有難う');
      // Usually in kana, even where the lists give kanji (其の for その).
      final uk = word([('有難う', [])], [('ありがとう', [])], misc: ['uk']);
      expect(jmdictEntry(uk).headword, 'ありがとう');
      expect(
        jmdictEntry(
          uk,
          jlpt: {'1': (level: 5, place: 0, spelling: '有難う')},
        ).headword,
        'ありがとう',
      );
      expect(
        jmdictEntry(
          arigatou,
          jlpt: {'1': (level: 5, place: 0, spelling: '有難う')},
        ).headword,
        '有難う',
      );
    });

    test('is a rare spelling only when nothing else is usual', () {
      expect(
        jmdictEntry(
          word(
            [
              ('彼処', ['rK']),
            ],
            [
              ('あすこ', ['ik']),
            ],
          ),
        ).headword,
        '彼処',
      );
    });
  });

  test("a meaning's example sentences come along: the form the word takes, "
      'the sentence, its English and where it is from', () {
    final entry = jmdictEntry({
      'id': '1358280',
      'kanji': [
        {'text': '食べる', 'common': true, 'tags': <String>[]},
      ],
      'kana': [
        {
          'text': 'たべる',
          'common': true,
          'tags': <String>[],
          'appliesToKanji': ['*'],
        },
      ],
      'sense': [
        {
          'partOfSpeech': ['v1'],
          'gloss': [
            {'text': 'to eat'},
          ],
          'examples': [
            {
              'source': {'type': 'tatoeba', 'value': '193344'},
              'text': '食べる',
              'sentences': [
                {'lang': 'jpn', 'text': 'もっと果物を食べるべきです。'},
                {'lang': 'eng', 'text': 'You should eat more fruit.'},
              ],
            },
          ],
        },
      ],
    });

    final senses = entry.entry['senses']! as List;
    expect((senses.single as Map)['examples'], [
      {
        'text': '食べる',
        'ja': 'もっと果物を食べるべきです。',
        'en': 'You should eat more fruit.',
        'tatoeba': '193344',
      },
    ]);
  });

  test('the examples edition credits Tatoeba for its sentences', () async {
    final header = jsonDecode(
      await jmdictDictionary(
        File('test/fixtures/jmdict-eng-sample.json')
            .openRead()
            .transform(utf8.decoder),
        examples: true,
      ).first,
    ) as Map<String, Object?>;

    expect([
      for (final credit in header['credits']! as List) (credit as Map)['name'],
    ], contains('Tatoeba'));
  });

  test('words rank N5 first, then down the levels, then the other common '
      'words, then the rest', () {
    int rank(String headword) => entry(headword)['rank']! as int;

    expect(rank('食べる'), lessThan(rank('歯磨き')));
    expect(rank('歯磨き'), lessThan(rank('東京')));
    expect(rank('東京'), lessThan(rank('〃')));
  });
}