halkyon/halkyon_learn_curator/test/update_test.dart

import 'dart:convert';
import 'dart:io';

import 'package:archive/archive.dart';

import 'package:halkyon_learn_curator/src/sources/krdict.dart';
import 'package:halkyon_learn_curator/src/sources/wiktionary.dart';
import 'package:halkyon_learn_curator/src/update.dart';
import 'package:test/test.dart';

void main() {
  late Directory into;

  setUp(() => into = Directory.systemTemp.createTempSync('dictionaries'));
  tearDown(() => into.deleteSync(recursive: true));

  /// Fetches the samples in place of EDRDG's files of the day.
  Future<void> samples(Uri uri, File file) async {
    final sample = uri == jmdictSource
        ? 'test/fixtures/jmdict-e-examp-sample.xml'
        : 'test/fixtures/kanjidic2-sample.xml';
    await file.writeAsBytes(gzip.encode(await File(sample).readAsBytes()));
  }

  /// Tatoeba's exports, as the fixtures hold a few of their lines, packed
  /// as Tatoeba packs them; null for any other address.
  List<int>? tatoeba(Uri uri) {
    final name = uri.pathSegments.last.replaceAll('.bz2', '');
    final fixture = File('test/fixtures/tatoeba-$name');
    return fixture.existsSync()
        ? BZip2Encoder().encodeBytes(fixture.readAsBytesSync())
        : null;
  }

  /// The header of the dictionary file [name] in [into].
  Map<String, Object?> header(String name) => jsonDecode(
    utf8
        .decode(gzip.decode(File('${into.path}/$name').readAsBytesSync()))
        .split('\n')
        .first,
  ) as Map<String, Object?>;

  test("writes JMdict and KANJIDIC2 from EDRDG's files, the JLPT levels from "
      "the curator's data", () async {
    final written = await updateDictionaries(
      into: into,
      data: Directory('data'),
      fetch: samples,
    );

    expect(
      [for (final file in written) file.uri.pathSegments.last],
      ['jmdict-en.jsonl.gz', 'kanjidic-en.jsonl.gz'],
    );
    expect(header('jmdict-en.jsonl.gz')['sourceVersion'], '2026-10-09');
    expect(
      [
        for (final credit in header('jmdict-en.jsonl.gz')['credits']! as List)
          (credit as Map)['name'],
      ],
      ['JMdict', 'JLPT vocabulary lists', 'Tatoeba'],
    );
    expect(header('kanjidic-en.jsonl.gz')['kind'], 'characters');
    // Nothing half-written left behind; which conversion made them.
    expect(
      [for (final entity in into.listSync()) entity.uri.pathSegments.last]
        ..sort(),
      ['.conversion', 'jmdict-en.jsonl.gz', 'kanjidic-en.jsonl.gz'],
    );
    expect(File('${into.path}/.conversion').readAsStringSync(), '$conversion');
    expect(lastUpdated(into), isNotNull);
  });

  test('files another conversion made, or one from before the curator said, '
      'are converted again at once', () async {
    await updateDictionaries(
      into: into,
      data: Directory('data'),
      fetch: samples,
    );
    final made = File('${into.path}/$conversionFile')
      ..writeAsStringSync('${conversion - 1}');
    expect(lastUpdated(into), isNull);

    made.deleteSync();
    expect(lastUpdated(into), isNull);
  });

  test('a download that fails leaves the files there as they were', () async {
    await updateDictionaries(
      into: into,
      data: Directory('data'),
      fetch: samples,
    );
    final before = File('${into.path}/jmdict-en.jsonl.gz').readAsBytesSync();

    await expectLater(
      updateDictionaries(
        into: into,
        data: Directory('data'),
        fetch: (uri, file) => uri == kanjidicSource
            ? Future.error(const HttpException('EDRDG answered 503.'))
            : samples(uri, file),
      ),
      throwsA(isA<HttpException>()),
    );

    expect(File('${into.path}/jmdict-en.jsonl.gz').readAsBytesSync(), before);
  });

  test('a folder missing a file was never brought up to date', () {
    expect(lastUpdated(into), isNull);
  });

  group("Wiktionary's dump", () {
    var dumped = DateTime.utc(2026, 10, 3, 8, 24);
    var opened = 0;

    setUp(() {
      dumped = DateTime.utc(2026, 10, 3, 8, 24);
      opened = 0;
    });

    Future<DateTime> modified(Uri uri) async => dumped;

    /// The sample in place of kaikki.org's dump.
    Future<Download> open(Uri uri) async {
      opened++;
      return (
        modified: dumped,
        bytes: Stream.value(
          gzip.encode(
            File('test/fixtures/wiktionary-sample.jsonl').readAsBytesSync(),
          ),
        ),
      );
    }

    Future<List<File>> update(Set<String> languages, {bool korean = false}) =>
        updateWiktionary(
          into: into,
          data: Directory('data'),
          languages: languages,
          korean: korean,
          modified: modified,
          open: open,
          fetch: (uri, file) => file.writeAsBytes(
            tatoeba(uri) ?? (throw HttpException('$uri', uri: uri)),
          ),
        );

    test(
      'is converted for the languages named, German and Greek with English '
      'meanings, English with German ones, and ranked by frequency',
      () async {
        final written = await update({'de', 'el', 'en'});

        expect(
          [for (final file in written) file.uri.pathSegments.last]..sort(),
          [
            'wiktionary-de-en.jsonl.gz',
            'wiktionary-el-en.jsonl.gz',
            'wiktionary-en-de.jsonl.gz',
          ],
        );
        expect(
          header('wiktionary-de-en.jsonl.gz')['sourceVersion'],
          '2026-10-03',
        );
        expect([
          for (final credit
              in header('wiktionary-de-en.jsonl.gz')['credits']! as List)
            (credit as Map)['name'],
        ], contains('wordfreq'));
        expect(File('${into.path}/$wiktionaryStateFile').existsSync(), isTrue);
      },
    );

    test(
      'is downloaded again only once it changed, or the languages did',
      () async {
        await update({'de'});
        expect(await update({'de'}), isEmpty);
        expect(opened, 1);

        dumped = DateTime.utc(2026, 11, 2);
        expect(await update({'de'}), hasLength(1));
        expect(opened, 2);

        expect(await update({'de', 'el'}), hasLength(2));
        expect(opened, 3);
      },
    );

    test('nothing is asked for without languages', () async {
      expect(await update({}), isEmpty);
      expect(opened, 0);
    });

    test("keeps what Korean's dictionaries take from it, even without "
        'languages of its words', () async {
      expect(await update({}, korean: true), isEmpty);
      expect(opened, 1);

      final kept = WiktionaryKorean.fromJson(
        utf8.decode(
          gzip.decode(
            File('${into.path}/$wiktionaryKoreanFile').readAsBytesSync(),
          ),
        ),
      );
      expect(kept.hanja['水'], [(hun: '물', eum: '수')]);
      expect(await update({}, korean: true), isEmpty);
      expect(opened, 1);
    });
  });

  test('CC-CEDICT is converted in both scripts with the HSK levels, and '
      'said to be up to date by its files', () async {
    expect(cedictUpdated(into), isNull);

    final written = await updateCedict(
      into: into,
      data: Directory('data'),
      fetch: (uri, file) => file.writeAsBytes(
        uri == cedictSource
            ? gzip.encode(
                File('test/fixtures/cedict-sample.txt').readAsBytesSync(),
              )
            : tatoeba(uri) ?? (throw HttpException('$uri', uri: uri)),
      ),
    );

    expect(
      [for (final file in written) file.uri.pathSegments.last],
      ['cc-cedict-hans-en.jsonl.gz', 'cc-cedict-hant-en.jsonl.gz'],
    );
    expect(header('cc-cedict-hant-en.jsonl.gz')['language'], 'zh-Hant');
    expect([
      for (final credit
          in header('cc-cedict-hans-en.jsonl.gz')['credits']! as List)
        (credit as Map)['name'],
    ], contains('Tatoeba'));
    expect(cedictUpdated(into), isNotNull);
    // Nothing of the downloads left beside the files.
    expect(
      [for (final entity in into.listSync()) entity.uri.pathSegments.last]
        ..sort(),
      ['cc-cedict-hans-en.jsonl.gz', 'cc-cedict-hant-en.jsonl.gz'],
    );
  });

  test("a dictionary keeps its words without Tatoeba's sentences when they "
      "can't be had", () async {
    final written = await updateCedict(
      into: into,
      data: Directory('data'),
      fetch: (uri, file) => uri == cedictSource
          ? file.writeAsBytes(
              gzip.encode(
                File('test/fixtures/cedict-sample.txt').readAsBytesSync(),
              ),
            )
          : throw HttpException('$uri answered 503.', uri: uri),
    );

    expect(written, hasLength(2));
    expect([
      for (final credit
          in header('cc-cedict-hans-en.jsonl.gz')['credits']! as List)
        (credit as Map)['name'],
    ], isNot(contains('Tatoeba')));
  });

  test('Chinese comes from CC-CEDICT, Korean from KRDict, the other '
      "languages' words from Wiktionary", () {
    final all = dictionarySources({'de', 'el', 'en', 'ko', 'zh'});
    expect(all.wiktionary, {'de', 'el', 'en'});
    expect(all.chinese, isTrue);
    expect(all.korean, isTrue);
    expect(all.greek, isTrue);
    final traditional = dictionarySources({'zh-Hant'});
    expect(traditional.wiktionary, isEmpty);
    expect(traditional.chinese, isTrue);
    expect(traditional.korean, isFalse);
    expect(dictionarySources({}).chinese, isFalse);
  });

  test('files of languages no longer kept, or of a source replaced, go; an '
      "operator's own stay", () {
    for (final name in [
      'jmdict-en.jsonl.gz',
      'wiktionary-de-en.jsonl.gz',
      'wiktionary-ko-en.jsonl.gz',
      'wiktionary-el-en.jsonl.gz',
      'krdict-ko-en.jsonl.gz',
      'unihan-ko-en.jsonl.gz',
      'greek-alphabet-en.jsonl.gz',
      'my-own.jsonl.gz',
    ]) {
      File('${into.path}/$name').writeAsStringSync('');
    }

    final removed = removeUnwanted(into, {'de', 'ko'});

    expect([for (final file in removed) file.uri.pathSegments.last]..sort(), [
      'greek-alphabet-en.jsonl.gz',
      'wiktionary-el-en.jsonl.gz',
      'wiktionary-ko-en.jsonl.gz',
    ]);
    expect(
      [for (final entity in into.listSync()) entity.uri.pathSegments.last]
        ..sort(),
      [
        'jmdict-en.jsonl.gz',
        'krdict-ko-en.jsonl.gz',
        'my-own.jsonl.gz',
        'unihan-ko-en.jsonl.gz',
        'wiktionary-de-en.jsonl.gz',
      ],
    );
  });

  test("a replaced source stays until what replaces it is there: Korean's "
      'words from Wiktionary until KRDict\'s are', () {
    File('${into.path}/wiktionary-ko-en.jsonl.gz').writeAsStringSync('');

    expect(removeUnwanted(into, {'ko'}), isEmpty);

    File('${into.path}/krdict-ko-en.jsonl.gz').writeAsStringSync('');
    expect(
      [
        for (final file in removeUnwanted(into, {'ko'}))
          file.uri.pathSegments.last,
      ],
      ['wiktionary-ko-en.jsonl.gz'],
    );
  });

  group("KRDict's export", () {
    var number = '217';
    var downloads = 0;

    setUp(() {
      number = '217';
      downloads = 0;
    });

    /// KRDict's page and the fixture packed as its export; Tatoeba's.
    Future<void> fetch(Uri uri, File file) async {
      if (uri == krdictPage) {
        await file.writeAsString(
          "<button onclick=\"batchDownload('215')\"></button>"
          "<button onclick=\"batchDownload('216')\"></button>"
          "<button onclick=\"batchDownload('$number')\"></button>",
        );
      } else if (uri == krdictDownloadOf(number)) {
        downloads++;
        await file.writeAsBytes(
          ZipEncoder().encodeBytes(
            Archive()..add(
              ArchiveFile.bytes(
                '1_5000_20260919.json',
                File('test/fixtures/krdict-sample.json').readAsBytesSync(),
              ),
            ),
          ),
        );
      } else {
        await file.writeAsBytes(
          tatoeba(uri) ?? (throw HttpException('$uri', uri: uri)),
        );
      }
    }

    test('is converted with the forms Wiktionary kept, and again only '
        'once its page offers another', () async {
      final kept = WiktionaryKorean()..forms['가다'] = {'갔어요'};
      File('${into.path}/$wiktionaryKoreanFile')
          .writeAsBytesSync(gzip.encode(utf8.encode(kept.toJson())));

      final written = await updateKrdict(
        into: into,
        data: Directory('data'),
        fetch: fetch,
      );

      expect(
        [for (final file in written) file.uri.pathSegments.last],
        [krdictFileName],
      );
      expect(header(krdictFileName)['sourceVersion'], '2026-09-19');
      expect([
        for (final credit in header(krdictFileName)['credits']! as List)
          (credit as Map)['name'],
      ], contains('Wiktionary'));
      expect(
        await updateKrdict(into: into, data: Directory('data'), fetch: fetch),
        isEmpty,
      );
      expect(downloads, 1);

      number = '230';
      expect(
        await updateKrdict(into: into, data: Directory('data'), fetch: fetch),
        hasLength(1),
      );
      expect(downloads, 2);
      expect(
        [for (final entity in into.listSync()) entity.uri.pathSegments.last]
          ..sort(),
        ['.krdict', wiktionaryKoreanFile, krdictFileName]..sort(),
      );
    });
  });

  test('the characters of Chinese in both scripts, of Korean with the Hangul '
      'letters and the Greek alphabet, up to date until a Wiktionary update '
      'keeps new Hanja readings', () async {
    Future<void> fetch(Uri uri, File file) async => await file.writeAsBytes(
      uri == unihanSource
          ? ZipEncoder().encodeBytes(
              Archive()..add(
                ArchiveFile.bytes(
                  'Unihan_Readings.txt',
                  File('test/fixtures/unihan-sample.txt').readAsBytesSync(),
                ),
              ),
            )
          : gzip.encode(
              File('test/fixtures/cedict-characters-sample.txt')
                  .readAsBytesSync(),
            ),
    );
    expect(
      charactersUpdated(into, chinese: true, korean: true, greek: true),
      isNull,
    );

    final written = await updateCharacters(
      into: into,
      data: Directory('data'),
      chinese: true,
      korean: true,
      greek: true,
      fetch: fetch,
    );

    expect(
      [for (final file in written) file.uri.pathSegments.last],
      [
        'unihan-hans-en.jsonl.gz',
        'unihan-hant-en.jsonl.gz',
        'unihan-ko-en.jsonl.gz',
        'greek-alphabet-en.jsonl.gz',
      ],
    );
    expect(header('unihan-hans-en.jsonl.gz')['sourceVersion'], '18.0.0');
    expect([
      for (final credit
          in header('unihan-hans-en.jsonl.gz')['credits']! as List)
        (credit as Map)['name'],
    ], containsAll(['Unihan Database', 'CC-CEDICT', 'wordfreq']));
    final updated = charactersUpdated(
      into,
      chinese: true,
      korean: true,
      greek: true,
    );
    expect(updated, isNotNull);

    File('${into.path}/$wiktionaryKoreanFile')
      ..writeAsBytesSync(gzip.encode(utf8.encode(WiktionaryKorean().toJson())))
      ..setLastModifiedSync(updated!.add(const Duration(minutes: 1)));
    expect(charactersUpdated(into, korean: true), isNull);
    expect(charactersUpdated(into, chinese: true), isNotNull);
  });
}