halkyon/apps/learn/halkyon_learn_curator/test/unihan_test.dart

import 'dart:convert';
import 'dart:io';

import 'package:archive/archive.dart';
import 'package:halkyon_learn_curator/src/sources/cedict.dart';
import 'package:halkyon_learn_curator/src/sources/unihan.dart';
import 'package:halkyon_learn_curator/src/wordfreq.dart';
import 'package:test/test.dart';

void main() {
  final unihan = Unihan.parse(
    File('test/fixtures/unihan-sample.txt').readAsLinesSync(),
  );
  final cedict = File('test/fixtures/cedict-characters-sample.txt')
      .readAsStringSync();
  // The words' places as wordfreq's list has them (的 0, 了 3, 后 41, …).
  final frequencies = WordFrequencies(const {
    '了': 3,
    '我们': 14,
    '说': 24,
    '为了': 170,
    '以后': 160,
    '后': 41,
    '学习': 380,
    '了解': 520,
    '这': 15,
    '水': 600,
    '银行': 1900,
    '皇后': 9000,
  });
  final places = cedictCharacterPlaces(cedict, frequencies);

  /// The entries of a dictionary file's [lines], by character.
  Map<String, Map<String, Object?>> entriesOf(Iterable<String> lines) => {
    for (final line in lines.skip(1))
      if (jsonDecode(line) case final Map<String, Object?> entry)
        entry['headword']! as String: entry,
  };

  test("the simplified characters are the standard's, with pinyin, the "
      'usual reading first, meanings, strokes, radical and HSK level', () {
    final lines = chineseCharacters(
      unihan,
      traditional: false,
      sourceVersion: '18.0.0',
      hsk: const {'说': 1, '行': 1, '后': 1},
      places: places,
    ).toList();

    final header = jsonDecode(lines.first) as Map<String, Object?>;
    expect(header['source'], 'unihan-hans');
    expect(header['kind'], 'characters');
    expect(header['language'], 'zh-Hans');
    expect(header['sourceVersion'], '18.0.0');
    final entries = entriesOf(lines);
    // No traditional character among them.
    expect(entries.keys, isNot(contains('說')));
    expect(entries.keys, containsAll(['说', '这', '后', '了', '行', '水']));
    final xing = entries['行']!;
    expect(((xing['entry']! as Map)['pinyin'] as List).take(2), [
      'xíng',
      'háng',
    ]);
    expect((xing['entry']! as Map)['strokes'], 6);
    expect(xing['labels'], ['hsk:1', 'common', 'radical:144']);
    final shuo = entries['说']!;
    expect((shuo['entry']! as Map)['meanings'], [
      'speak',
      'say',
      'talk',
      'scold',
      'upbraid',
    ]);
    expect((shuo['entry']! as Map)['traditional'], ['說', '説']);
    // Found by its pinyin with marks, without, and numbered; written in
    // the other script, loosely.
    expect(shuo['keys'], containsAll(['shuō', 'shuo', 'shuo1', 'speak']));
    expect(shuo['looseKeys'], ['說', '説']);
  });

  test('the traditional characters are those words in use write: 這 and 了, '
      'not 这; 後 the usual form of 后, ranked and levelled as it', () {
    final entries = entriesOf(
      chineseCharacters(
        unihan,
        traditional: true,
        hsk: const {'后': 1, '了': 1},
        places: places,
      ),
    );

    expect(entries.keys, containsAll(['這', '了', '瞭', '後', '后', '說']));
    expect(entries.keys, isNot(contains('这')));
    // 後 takes 后's place and level; 后, the queen, ranks by its own words.
    expect(entries['後']!['labels'], contains('hsk:1'));
    expect(entries['后']!['labels'], isNot(contains('hsk:1')));
    expect(
      entries['後']!['rank']! as int,
      lessThan(entries['后']!['rank']! as int),
    );
    // 了 stands for itself; 瞭 is a form of it of its own.
    expect(entries['了']!['labels'], contains('hsk:1'));
    expect(entries['瞭']!['labels'], isNot(contains('hsk:1')));
    expect((entries['說']!['entry']! as Map)['simplified'], ['说']);
    // Hong Kong's primary schools teach 說 in the first grade.
    expect(entries['說']!['labels'], contains('grade:1'));
  });

  test("CC-CEDICT's words tell each script's characters their places: a "
      "lone simplified character's to the traditional form longer words use "
      'most', () {
    expect(places.simplified['后'], 41);
    expect(places.traditional['後'], 41);
    expect(places.traditional['后'], 9000);
    expect(places.traditional['了'], 3);
    // 瞭解 is only a variant of 了解 there, and 了 the form 為了 uses.
    expect(places.traditional, isNot(contains('瞭')));
    expect(places.traditional, isNot(contains('这')));
  });

  test('Korean Hanja with their sounds, the taught one first, their native '
      'readings from Wiktionary, and the Hangul letters before them', () {
    final lines = koreanHanja(
      unihan,
      sourceVersion: '18.0.0',
      eumhun: const {
        '水': [(hun: '물', eum: '수')],
      },
    ).toList();

    final header = jsonDecode(lines.first) as Map<String, Object?>;
    expect(header['language'], 'ko');
    expect(header['license'], 'CC-BY-SA-4.0');
    expect([
      for (final credit in header['credits']! as List) (credit as Map)['name'],
    ], containsAll(['Unihan Database', 'Wiktionary']));
    final entries = entriesOf(lines);
    final water = entries['水']!;
    expect(water['entry'], {
      'meanings': ['water', 'liquid', 'lotion', 'juice'],
      'eum': ['수'],
      'hun': ['물'],
      'eumhun': ['물 수'],
      'strokes': 4,
    });
    expect(water['labels'], ['education', 'radical:85']);
    expect(water['keys'], containsAll(['水', '수', '물', '물 수', 'water']));
    // 行 is taught as 행, also read 항.
    expect((entries['行']!['entry']! as Map)['eum'], ['행', '항']);
    final giyeok = entries['ㄱ']!;
    expect(giyeok['labels'], ['hangul']);
    expect(giyeok['entry'], {
      'romaji': 'g, k',
      'script': 'hangul',
      'name': '기역',
    });
    expect(giyeok['rank']! as int, lessThan(water['rank']! as int));
  });

  test("Unihan's zip is read whole, with the version of Unicode it is of", () {
    final archive = Archive()
      ..add(
        ArchiveFile.bytes(
          'Unihan_Readings.txt',
          File('test/fixtures/unihan-sample.txt').readAsBytesSync(),
        ),
      );
    final (:unihan, :version) = Unihan.ofZip(ZipEncoder().encodeBytes(archive));

    expect(version, '18.0.0');
    expect(unihan.characters, contains('水'));
  });
}