halkyon/apps/learn/halkyon_learn_curator/test/cedict_test.dart

import 'dart:convert';
import 'dart:io';

import 'package:halkyon_learn_curator/src/sources/cedict.dart';
import 'package:test/test.dart';

void main() {
  late String sample;
  late HskLevels hsk;

  setUpAll(() async {
    sample = await File('test/fixtures/cedict-sample.txt').readAsString();
    hsk = await readHskLevels(File('data/hsk/syllabus2025_words.tsv'));
  });

  List<Map<String, Object?>> linesOf({required bool traditional}) => [
    for (final line in cedictDictionary(
      sample,
      traditional: traditional,
      hsk: hsk,
    ))
      jsonDecode(line) as Map<String, Object?>,
  ];

  Map<String, Object?> entryOf(String id, {bool traditional = false}) =>
      linesOf(traditional: traditional)
          .skip(1)
          .singleWhere((line) => line['id'] == id);

  test('a file in simplified characters and one in traditional ones, each '
      'a dictionary of its own', () {
    final simplified = linesOf(traditional: false).first;
    expect(simplified['source'], 'cc-cedict-hans');
    expect(simplified['language'], 'zh-Hans');
    expect(simplified['glossLanguage'], 'en');
    expect(simplified['sourceVersion'], '2026-10-13');
    expect(
      [
        for (final credit in simplified['credits']! as List)
          (credit as Map)['name'],
      ],
      ['CC-CEDICT', 'HSK 3.0 vocabulary (examination syllabus, 2025)'],
    );
    final traditional = linesOf(traditional: true).first;
    expect(traditional['source'], 'cc-cedict-hant');
    expect(traditional['language'], 'zh-Hant');
    expect(cedictFileName(traditional: true), 'cc-cedict-hant-en.jsonl.gz');
  });

  test('a word: its characters in either script, its pinyin with tone '
      'marks, found typed without tones, in text by its characters', () {
    final xuexi = entryOf('學習 学习 [xue2 xi2]');
    expect(xuexi['headword'], '学习');
    expect(xuexi['reading'], 'xué xí');
    expect(
      (xuexi['keys']! as List).cast<String>(),
      containsAll([
        '学习',
        '學習',
        'xuexi',
        'xuéxí',
        'xue2xi2',
        'to learn',
        'learn',
        'study',
      ]),
    );
    expect(xuexi['textKeys'], ['学习', '學習']);
    final entry = xuexi['entry']! as Map;
    expect(entry['forms'], [
      {'text': '學習'},
    ]);
    expect(entry['senses'], [
      {
        'glosses': ['to learn', 'to study'],
      },
    ]);
    expect(entryOf('學習 学习 [xue2 xi2]', traditional: true)['headword'], '學習');
  });

  test('HSK levels go to the entry read as the syllabus reads the word: 的 '
      'de, not dì; 书 shū, not the Book of Documents, Shū', () {
    expect(entryOf('的 的 [de5]')['labels'], contains('hsk:1'));
    expect(entryOf('的 的 [di4]')['labels'], isNot(contains('hsk:1')));
    // Read with its tones: 说 shuō, not shuì.
    expect(entryOf('說 说 [shuo1]')['labels'], contains('hsk:1'));
    expect(entryOf('說 说 [shui4]')['labels'], isEmpty);
    expect(entryOf('書 书 [shu1]')['labels'], contains('hsk:1'));
    expect(entryOf('書 书 [Shu1]')['labels'], isEmpty);
    expect(entryOf('中國 中国 [Zhong1 guo2]')['labels'], contains('hsk:1'));
    expect(entryOf('銀行 银行 [yin2 hang2]')['labels'], contains('hsk:3'));
    // HSK 1 before HSK 3, both before words of no level.
    expect(
      entryOf('的 的 [de5]')['rank']! as int,
      lessThan(entryOf('銀行 银行 [yin2 hang2]')['rank']! as int),
    );
    expect(
      entryOf('銀行 银行 [yin2 hang2]')['rank']! as int,
      lessThan(entryOf('的 的 [di4]')['rank']! as int),
    );
  });

  test('an entry that is only a variant of another is no entry, but finds '
      'that one', () {
    final ids = [
      for (final line in linesOf(traditional: false).skip(1)) line['id'],
    ];
    expect(ids, isNot(contains('咊 和 [he2]')));
    expect(
      (entryOf('和 和 [he2]')['textKeys']! as List).cast<String>(),
      contains('咊'),
    );
  });

  test('measure words are a remark beside the meanings, pinyin marked', () {
    final book = entryOf('書 书 [shu1]')['entry']! as Map;
    expect(book['classifiers'], ['本 běn', '冊|册 cè', '部 bù']);
    expect(
      [
        for (final sense in book['senses']! as List)
          ...(sense as Map)['glosses']! as List,
      ],
      ['book', 'letter', 'document', 'to write'],
    );
  });

  group('pinyin', () {
    test('takes its tone marks where they belong', () {
      expect(pinyinWithToneMarks('xue2 xi2'), 'xué xí');
      expect(pinyinWithToneMarks('Zhong1 guo2'), 'Zhōng guó');
      expect(pinyinWithToneMarks('nu:3 er2'), 'nǚ ér');
      expect(pinyinWithToneMarks('lu:e4'), 'lüè');
      expect(
        pinyinWithToneMarks('shuo1 gui4 liu2 xiong2'),
        'shuō guì liú xióng',
      );
      expect(pinyinWithToneMarks('dou1 le5 r5'), 'dōu le r');
      expect(pinyinWithToneMarks('A A zhi4'), 'A A zhì');
    });

    test('is typed without its tones', () {
      expect(tonelessPinyin('xué xí'), 'xuexi');
      expect(tonelessPinyin('nǚ’ér'), 'nuer');
      expect(tonelessPinyin('yìsī-bùgǒu'), 'yisibugou');
      expect(tonelessPinyin('nu:3 er2'), 'nuer');
    });
  });
}