halkyon/halkyon_learn_curator/test/update_test.dart

import 'dart:convert';
import 'dart:io';

import 'package:halkyon_learn_curator/src/update.dart';
import 'package:test/test.dart';

void main() {
  late Directory into;

  setUp(() => into = Directory.systemTemp.createTempSync('dictionaries'));
  tearDown(() => into.deleteSync(recursive: true));

  /// Fetches the samples in place of EDRDG's files of the day.
  Future<void> samples(Uri uri, File file) async {
    final sample = uri == jmdictSource
        ? 'test/fixtures/jmdict-e-examp-sample.xml'
        : 'test/fixtures/kanjidic2-sample.xml';
    await file.writeAsBytes(gzip.encode(await File(sample).readAsBytes()));
  }

  /// The header of the dictionary file [name] in [into].
  Map<String, Object?> header(String name) => jsonDecode(
    utf8
        .decode(gzip.decode(File('${into.path}/$name').readAsBytesSync()))
        .split('\n')
        .first,
  ) as Map<String, Object?>;

  test("writes JMdict and KANJIDIC2 from EDRDG's files, the JLPT levels from "
      "the curator's data", () async {
    final written = await updateDictionaries(
      into: into,
      data: Directory('data'),
      fetch: samples,
    );

    expect(
      [for (final file in written) file.uri.pathSegments.last],
      ['jmdict-en.jsonl.gz', 'kanjidic-en.jsonl.gz'],
    );
    expect(header('jmdict-en.jsonl.gz')['sourceVersion'], '2026-10-09');
    expect(
      [
        for (final credit in header('jmdict-en.jsonl.gz')['credits']! as List)
          (credit as Map)['name'],
      ],
      ['JMdict', 'JLPT vocabulary lists', 'Tatoeba'],
    );
    expect(header('kanjidic-en.jsonl.gz')['kind'], 'characters');
    // Nothing half-written left behind; which conversion made them.
    expect(
      [for (final entity in into.listSync()) entity.uri.pathSegments.last]
        ..sort(),
      ['.conversion', 'jmdict-en.jsonl.gz', 'kanjidic-en.jsonl.gz'],
    );
    expect(File('${into.path}/.conversion').readAsStringSync(), '$conversion');
    expect(lastUpdated(into), isNotNull);
  });

  test('files another conversion made, or one from before the curator said, '
      'are converted again at once', () async {
    await updateDictionaries(
      into: into,
      data: Directory('data'),
      fetch: samples,
    );
    final made = File('${into.path}/$conversionFile')
      ..writeAsStringSync('${conversion - 1}');
    expect(lastUpdated(into), isNull);

    made.deleteSync();
    expect(lastUpdated(into), isNull);
  });

  test('a download that fails leaves the files there as they were', () async {
    await updateDictionaries(
      into: into,
      data: Directory('data'),
      fetch: samples,
    );
    final before = File('${into.path}/jmdict-en.jsonl.gz').readAsBytesSync();

    await expectLater(
      updateDictionaries(
        into: into,
        data: Directory('data'),
        fetch: (uri, file) => uri == kanjidicSource
            ? Future.error(const HttpException('EDRDG answered 503.'))
            : samples(uri, file),
      ),
      throwsA(isA<HttpException>()),
    );

    expect(File('${into.path}/jmdict-en.jsonl.gz').readAsBytesSync(), before);
  });

  test('a folder missing a file was never brought up to date', () {
    expect(lastUpdated(into), isNull);
  });

  group("Wiktionary's dump", () {
    var dumped = DateTime.utc(2026, 10, 3, 8, 24);
    var opened = 0;

    setUp(() {
      dumped = DateTime.utc(2026, 10, 3, 8, 24);
      opened = 0;
    });

    Future<DateTime> modified(Uri uri) async => dumped;

    /// The sample in place of kaikki.org's dump.
    Future<Download> open(Uri uri) async {
      opened++;
      return (
        modified: dumped,
        bytes: Stream.value(
          gzip.encode(
            File('test/fixtures/wiktionary-sample.jsonl').readAsBytesSync(),
          ),
        ),
      );
    }

    Future<List<File>> update(Set<String> languages) => updateWiktionary(
      into: into,
      data: Directory('data'),
      languages: languages,
      modified: modified,
      open: open,
    );

    test(
      'is converted for the languages named, German and Greek with English '
      'meanings, English with German ones, and ranked by frequency',
      () async {
        final written = await update({'de', 'el', 'en'});

        expect(
          [for (final file in written) file.uri.pathSegments.last]..sort(),
          [
            'wiktionary-de-en.jsonl.gz',
            'wiktionary-el-en.jsonl.gz',
            'wiktionary-en-de.jsonl.gz',
          ],
        );
        expect(
          header('wiktionary-de-en.jsonl.gz')['sourceVersion'],
          '2026-10-03',
        );
        expect([
          for (final credit
              in header('wiktionary-de-en.jsonl.gz')['credits']! as List)
            (credit as Map)['name'],
        ], contains('wordfreq'));
        expect(File('${into.path}/$wiktionaryStateFile').existsSync(), isTrue);
      },
    );

    test(
      'is downloaded again only once it changed, or the languages did',
      () async {
        await update({'de'});
        expect(await update({'de'}), isEmpty);
        expect(opened, 1);

        dumped = DateTime.utc(2026, 11, 2);
        expect(await update({'de'}), hasLength(1));
        expect(opened, 2);

        expect(await update({'de', 'el'}), hasLength(2));
        expect(opened, 3);
      },
    );

    test('nothing is asked for without languages', () async {
      expect(await update({}), isEmpty);
      expect(opened, 0);
    });
  });
}