halkyon/halkyon_learn_curator/bin/curate.dart

import 'dart:convert';
import 'dart:io';

import 'package:archive/archive.dart';
import 'package:args/args.dart';
import 'package:halkyon_learn_curator/src/sources/cedict.dart';
import 'package:halkyon_learn_curator/src/sources/jlpt.dart';
import 'package:halkyon_learn_curator/src/sources/jmdict.dart';
import 'package:halkyon_learn_curator/src/sources/jmdict_xml.dart';
import 'package:halkyon_learn_curator/src/sources/kanjidic.dart';
import 'package:halkyon_learn_curator/src/sources/kanjivg.dart';
import 'package:halkyon_learn_curator/src/sources/krdict.dart';
import 'package:halkyon_learn_curator/src/sources/tatoeba.dart';
import 'package:halkyon_learn_curator/src/sources/unihan.dart';
import 'package:halkyon_learn_curator/src/sources/wiktionary.dart';
import 'package:halkyon_learn_curator/src/update.dart';
import 'package:halkyon_learn_curator/src/wordfreq.dart';

/// Turns a dictionary download into a dictionary file for the server's
/// `dictionaries` folder, or KanjiVG into the app's stroke files
/// (README.md):
///
///     dart run halkyon_learn_curator:curate jmdict \
///       --input JMdict_e_examp.gz --jlpt original_data \
///       --output jmdict-en.jsonl.gz
///     dart run halkyon_learn_curator:curate kanjidic \
///       --input kanjidic2.xml.gz --jlpt kanji.json \
///       --output kanjidic-en.jsonl.gz
///     dart run halkyon_learn_curator:curate kanjivg \
///       --input kanjivg-20260714.xml.gz \
///       --output ../halkyon_learn_flutter/assets/kanjivg
///
/// Wiktionary, as kaikki.org's dump of it, into dictionary files of the
/// languages named (German and Greek with English meanings, English with
/// German ones):
///
///     dart run halkyon_learn_curator:curate wiktionary \
///       --input raw-wiktextract-data.jsonl.gz --languages de,el,en \
///       --output ../halkyon_learn_server/dictionaries
///
/// CC-CEDICT into its two dictionary files, simplified and traditional:
///
///     dart run halkyon_learn_curator:curate cedict \
///       --input cedict_1_0_ts_utf-8_mdbg.txt.gz \
///       --output ../halkyon_learn_server/dictionaries
///
/// Unihan into dictionary files of Chinese characters (simplified and
/// traditional) and of Korean Hanja, for the languages named, the Hanja's
/// native readings from a Wiktionary update's `.wiktionary-ko.json.gz`:
///
///     dart run halkyon_learn_curator:curate unihan \
///       --input Unihan.zip --languages ko,zh [--cedict <file>] \
///       [--korean <file>] \
///       --output ../halkyon_learn_server/dictionaries
///
/// KRDict's JSON export into the dictionary file of Korean words, with
/// the forms of each a Wiktionary update kept (`.wiktionary-ko.json.gz`):
///
///     dart run halkyon_learn_curator:curate krdict \
///       --input krdict_json_20260919.zip [--korean <file>] \
///       --output ../halkyon_learn_server/dictionaries
///
/// Tatoeba's sentences as examples into a dictionary file of words (its
/// exports of the dictionary's language, of its meaning language, and of
/// the links between them, bzip2 or not):
///
///     dart run halkyon_learn_curator:curate tatoeba \
///       --input wiktionary-de-en.jsonl.gz --sentences deu_sentences.tsv.bz2 \
///       --translations eng_sentences.tsv.bz2 --links deu-eng_links.tsv.bz2 \
///       --output wiktionary-de-en.jsonl.gz
///
/// Or brings a server's dictionaries up to date from EDRDG's files of the
/// day and, for the languages named, Wiktionary's latest dump and MDBG's
/// CC-CEDICT of the day (Chinese), once or every so many days:
///
///     dart run halkyon_learn_curator:curate update \
///       --into ../halkyon_learn_server/dictionaries [--every 7] \
///       [--languages de,el,en,zh]
Future<void> main(List<String> args) async {
  final parser = ArgParser()
    ..addOption(
      'input',
      abbr: 'i',
      help: 'The source file, unpacked or gzipped (.gz).',
    )
    ..addOption(
      'jlpt',
      help:
          "jmdict: yomitan-jlpt-vocab's original_data folder (n1.csv … "
          "n5.csv); kanjidic: kanji-data's kanji.json.",
    )
    ..addFlag(
      'examples',
      help:
          "The input is JMdict's examples edition (EDRDG's "
          "JMdict_e_examp.gz, jmdict-simplified's jmdict-examples-eng-*.json): "
          'credit Tatoeba for its sentences. On by default for a file named '
          'so.',
      defaultsTo: null,
    )
    ..addOption(
      'output',
      abbr: 'o',
      help:
          'The dictionary file to write, gzipped when it ends in .gz; for '
          'kanjivg the folder to write the stroke files into.',
    )
    ..addOption('into', help: 'update: the folder of the dictionary files.')
    ..addOption(
      'data',
      help: "update: the curator's data folder (the JLPT lists).",
      defaultsTo: 'data',
    )
    ..addOption(
      'every',
      help:
          'update: keep running, and update once the files are this many '
          'days old.',
    )
    ..addOption(
      'languages',
      help:
          'wiktionary: the languages to write dictionaries of, by their '
          'BCP 47 tags (de,el,en); update: the languages besides Japanese '
          'to keep current (de,el,en,zh; Chinese from CC-CEDICT, the rest '
          "from Wiktionary's dump), none without.",
    )
    ..addOption(
      'sentences',
      help: "tatoeba: Tatoeba's sentences in the dictionary's language.",
    )
    ..addOption(
      'translations',
      help: "tatoeba: Tatoeba's sentences in the meaning language.",
    )
    ..addOption(
      'links',
      help: "tatoeba: Tatoeba's links between the two languages.",
    )
    ..addOption(
      'cedict',
      help:
          'unihan: CC-CEDICT, whose words tell traditional characters in use '
          'and rank the characters.',
    )
    ..addOption(
      'korean',
      help:
          'unihan, krdict: what a Wiktionary update kept for Korean '
          "(.wiktionary-ko.json.gz: the Hanja's native readings, the "
          "words' forms).",
    )
    ..addOption(
      'version',
      help:
          "wiktionary: the dump's date (2026-10-03); the input file's date "
          'without.',
    );
  final ArgResults options;
  try {
    options = parser.parse(args);
  } on FormatException catch (error) {
    _fail('${error.message}\n${parser.usage}');
  }
  final [source, ...] = options.rest.isEmpty ? [''] : options.rest;
  if (source == 'update') {
    final into = options.option('into');
    final every = options.option('every');
    final days = every == null ? null : int.tryParse(every);
    if (into == null || (every != null && (days == null || days < 1))) {
      _fail(
        'Usage: curate update --into <folder> [--data <folder>] '
        '[--every <days>] [--languages <languages>]\n${parser.usage}',
      );
    }
    await _update(
      Directory(into),
      Directory(options.option('data')!),
      days,
      languages: _languages(options.option('languages')),
    );
    return;
  }
  if (source == 'tatoeba') {
    final input = options.option('input');
    final output = options.option('output');
    final files = [
      for (final name in ['sentences', 'translations', 'links'])
        options.option(name),
    ];
    if (input == null || output == null || files.contains(null)) {
      _fail(
        'Usage: curate tatoeba --input <dictionary> --sentences <file> '
        '--translations <file> --links <file> --output <dictionary>'
        '\n${parser.usage}',
      );
    }
    Stream<String> lines(String path) {
      final bytes = path.endsWith('.bz2')
          ? Stream.value(
              BZip2Decoder().decodeBytes(File(path).readAsBytesSync()),
            )
          : File(path).openRead();
      return bytes.transform(utf8.decoder).transform(const LineSplitter());
    }

    final links = await readTatoebaLinks(lines(files[2]!));
    final sentences = await readTatoebaSentences(
      lines(files[0]!),
      wanted: links.containsKey,
    );
    final linked = {for (final ids in links.values) ...ids};
    final translations = await readTatoebaSentences(
      lines(files[1]!),
      wanted: linked.contains,
    );
    final written = File('$output.part');
    final bytes = withTatoebaExamples(
      File(input),
      sentences: sentences,
      translations: translations,
      links: links,
    ).transform(utf8.encoder);
    await (output.endsWith('.gz') ? bytes.transform(gzip.encoder) : bytes).pipe(
      written.openWrite(),
    );
    await written.rename(output);
    stdout.writeln(
      'Wrote $output with ${sentences.length} sentences of Tatoeba to '
      'choose from.',
    );
    return;
  }
  if (source == 'cedict') {
    final input = options.option('input');
    final output = options.option('output');
    if (input == null || output == null) {
      _fail(
        'Usage: curate cedict --input <file> --output <folder> '
        '[--data <folder>]\n${parser.usage}',
      );
    }
    final read = File(input).openRead();
    final text =
        await (input.endsWith('.gz') ? read.transform(gzip.decoder) : read)
            .transform(utf8.decoder)
            .join();
    final data = options.option('data')!;
    final hsk = await readHskLevels(hskFile(Directory(data)));
    final frequencies = File('$data/wordfreq/small_zh.msgpack.gz');
    final words = frequencies.existsSync()
        ? await WordFrequencies.read(frequencies)
        : WordFrequencies.none;
    final folder = await Directory(output).create(recursive: true);
    for (final traditional in [false, true]) {
      await Stream.fromIterable(
            cedictDictionary(
              text,
              traditional: traditional,
              hsk: hsk,
              frequencies: words,
            ),
          )
          .transform(utf8.encoder)
          .transform(gzip.encoder)
          .pipe(
            File('${folder.path}/${cedictFileName(traditional: traditional)}')
                .openWrite(),
          );
    }
    stdout.writeln(
      'Wrote ${cedictFileName(traditional: false)} and '
      '${cedictFileName(traditional: true)} to $output.',
    );
    return;
  }
  if (source == 'krdict') {
    final input = options.option('input');
    final output = options.option('output');
    if (input == null || output == null) {
      _fail(
        'Usage: curate krdict --input <zip> --output <folder> '
        '[--korean <file>] [--data <folder>]\n${parser.usage}',
      );
    }
    final korean = options.option('korean');
    final frequencies = File(
      '${options.option('data')}/wordfreq/small_ko.msgpack.gz',
    );
    final folder = await Directory(output).create(recursive: true);
    final (:version, :entries) = krdictExport(await File(input).readAsBytes());
    await krdictDictionary(
          entries,
          sourceVersion: version,
          frequencies: frequencies.existsSync()
              ? await WordFrequencies.read(frequencies)
              : null,
          korean: korean == null
              ? null
              : WiktionaryKorean.fromJson(
                  utf8.decode(gzip.decode(await File(korean).readAsBytes())),
                ),
        )
        .transform(utf8.encoder)
        .transform(gzip.encoder)
        .pipe(File('${folder.path}/$krdictFileName').openWrite());
    stdout.writeln('Wrote $krdictFileName to $output.');
    return;
  }
  if (source == 'unihan') {
    final input = options.option('input');
    final output = options.option('output');
    final languages = _languages(options.option('languages'));
    if (input == null || output == null || languages.isEmpty) {
      _fail(
        'Usage: curate unihan --input <Unihan.zip> --languages <languages> '
        '--output <folder> [--cedict <file>] [--korean <file>] '
        '[--data <folder>]'
        '\n${parser.usage}',
      );
    }
    final korean = options.option('korean');
    final cedict = options.option('cedict');
    final written = await unihanDictionaries(
      await File(input).readAsBytes(),
      into: await Directory(output).create(recursive: true),
      languages: languages,
      data: Directory(options.option('data')!),
      cedict: cedict == null
          ? null
          : await (cedict.endsWith('.gz')
                    ? File(cedict).openRead().transform(gzip.decoder)
                    : File(cedict).openRead())
                .transform(utf8.decoder)
                .join(),
      eumhun: korean == null
          ? const {}
          : WiktionaryKorean.fromJson(
              utf8.decode(gzip.decode(await File(korean).readAsBytes())),
            ).hanja,
    );
    stdout.writeln(
      'Wrote ${[for (final file in written) file.uri.pathSegments.last].join(', ')} '
      'to $output.',
    );
    return;
  }
  if (source == 'wiktionary') {
    final input = options.option('input');
    final output = options.option('output');
    final languages = _languages(options.option('languages'));
    if (input == null || output == null || languages.isEmpty) {
      _fail(
        'Usage: curate wiktionary --input <file> --languages <languages> '
        '--output <folder> [--version <date>] [--data <folder>]'
        '\n${parser.usage}',
      );
    }
    final read = File(input).openRead();
    final outputs = wiktionaryOutputs(languages);
    final data = options.option('data')!;
    final written = await wiktionaryDictionaries(
      (input.endsWith('.gz') ? read.transform(gzip.decoder) : read)
          .transform(utf8.decoder)
          .transform(const LineSplitter()),
      into: Directory(output),
      sourceVersion:
          options.option('version') ??
          File(input).lastModifiedSync().toIso8601String().substring(0, 10),
      languages: outputs.languages,
      translatedInto: outputs.translatedInto,
      frequencies: {
        for (final language in {...languages, 'en'})
          if (File('$data/wordfreq/small_$language.msgpack.gz') case final file
              when file.existsSync())
            language: await WordFrequencies.read(file),
      },
    );
    stdout.writeln(
      'Wrote ${[for (final file in written) file.uri.pathSegments.last].join(', ')} '
      'to $output.',
    );
    return;
  }
  final input = options.option('input');
  final output = options.option('output');
  if (!['jmdict', 'kanjidic', 'kanjivg'].contains(source) ||
      input == null ||
      output == null) {
    _fail(
      'Usage: curate jmdict|kanjidic|kanjivg --input <file> [--jlpt <path>] '
      '--output <path>\n${parser.usage}',
    );
  }
  final read = File(input).openRead();
  final text = (input.endsWith('.gz') ? read.transform(gzip.decoder) : read)
      .transform(utf8.decoder);
  final jlptPath = options.option('jlpt');

  if (source == 'kanjivg') {
    final strokes = kanjivgStrokes(await text.join());
    final folder = await Directory(output).create(recursive: true);
    final files = strokeFiles(strokes);
    for (final MapEntry(key: name, value: characters) in files.entries) {
      await File('${folder.path}/$name.json.gz')
          .writeAsBytes(gzip.encode(utf8.encode(jsonEncode(characters))));
    }
    stdout.writeln(
      'Wrote the strokes of ${strokes.length} characters to '
      '${files.length} files in $output.',
    );
    return;
  }

  final Stream<String> lines;
  final int withLevel;
  if (source == 'kanjidic') {
    final jlpt = jlptPath == null
        ? const <String, int>{}
        : await readKanjiJlpt(File(jlptPath));
    lines = Stream.fromIterable(
      kanjidicDictionary(await text.join(), jlpt: jlpt),
    );
    withLevel = jlpt.length;
  } else {
    final jlpt = jlptPath == null
        ? const <String, JlptLevel>{}
        : await readJlptLevels(Directory(jlptPath));
    final name = File(input).uri.pathSegments.last;
    final examples = options.wasParsed('examples')
        ? options.flag('examples')
        : name.contains('examp');
    // jmdict-simplified's JSON (`jmdict-eng-….json`), else EDRDG's XML.
    lines = name.contains('.json')
        ? jmdictDictionary(text, jlpt: jlpt, examples: examples)
        : jmdictXmlDictionary(text, jlpt: jlpt, examples: examples);
    withLevel = jlpt.length;
  }
  var entries = -1;
  final bytes = lines
      .map((line) {
        entries++;
        return line;
      })
      .transform(utf8.encoder);
  await (output.endsWith('.gz') ? bytes.transform(gzip.encoder) : bytes).pipe(
    File(output).openWrite(),
  );
  stdout.writeln(
    'Wrote $entries entries to $output'
    '${withLevel == 0 ? '' : ' ($withLevel with a JLPT level)'}.',
  );
}

/// Updates the dictionaries in [into]; with [every], keeps running and
/// updates them whenever they are that many days old (as the files' dates
/// say, so a restart doesn't download them again), trying again after six
/// hours when an update fails. Of [languages], Chinese comes from
/// CC-CEDICT as often as EDRDG's files, and its characters from Unihan;
/// Korean from KRDict, looked at each time it wakes and converted again
/// only when it offers another export, and its Hanja from Unihan; the rest
/// from Wiktionary, looked at each time it wakes and converted again only
/// when the dump is newer (Korean's forms and the Hanja's readings come
/// from it too); Greek's alphabet with the characters. Files of languages
/// no longer kept go.
Future<void> _update(
  Directory into,
  Directory data,
  int? every, {
  Set<String> languages = const {},
}) async {
  final (:wiktionary, :chinese, :korean, :greek) = dictionarySources(languages);
  void say(String message) =>
      stdout.writeln('${DateTime.now().toUtc().toIso8601String()} $message');
  void written(List<File> files) {
    if (files.isNotEmpty) {
      say(
        'Updated ${[for (final file in files) file.uri.pathSegments.last].join(', ')}.',
      );
    }
  }

  /// Runs [step], saying what it wrote; false when it failed.
  Future<bool> attempt(String name, Future<List<File>> Function() step) async {
    try {
      written(await step());
      return true;
    } on Object catch (error) {
      say('Updating $name failed: $error');
      return false;
    }
  }

  Future<bool> once() => attempt(
    'the dictionaries',
    () => updateDictionaries(into: into, data: data),
  );

  // Files as old as EDRDG's may be: CC-CEDICT's, and the characters'.
  bool due(DateTime? updated) =>
      updated == null ||
      DateTime.now().difference(updated) >= Duration(days: every ?? 1);
  Future<bool> chineseOnce() async =>
      !chinese ||
      !due(cedictUpdated(into)) ||
      await attempt('CC-CEDICT', () => updateCedict(into: into, data: data));
  Future<bool> wiktionaryOnce() => attempt(
    'Wiktionary',
    () => updateWiktionary(
      into: into,
      data: data,
      languages: wiktionary,
      korean: korean,
    ),
  );
  Future<bool> koreanOnce() async =>
      !korean ||
      await attempt('KRDict', () => updateKrdict(into: into, data: data));
  Future<bool> charactersOnce() async =>
      !due(
        charactersUpdated(into, chinese: chinese, korean: korean, greek: greek),
      ) ||
      await attempt(
        'the characters',
        () => updateCharacters(
          into: into,
          data: data,
          chinese: chinese,
          korean: korean,
          greek: greek,
        ),
      );
  // In this order: Korean's words and Hanja take what Wiktionary kept.
  Future<bool> languagesOnce() async {
    final removed = removeUnwanted(into, languages);
    if (removed.isNotEmpty) {
      say(
        'Removed ${[for (final file in removed) file.uri.pathSegments.last].join(', ')}.',
      );
    }
    return !(!await chineseOnce() |
        !await wiktionaryOnce() |
        !await koreanOnce() |
        !await charactersOnce());
  }

  if (every == null) {
    if (!await once() || !await languagesOnce()) {
      exit(1);
    }
    return;
  }
  for (;;) {
    final due = lastUpdated(into)?.add(Duration(days: every));
    final now = DateTime.now();
    if (due == null || !now.isBefore(due)) {
      if (!await once()) {
        await Future<void>.delayed(const Duration(hours: 6));
      }
      continue;
    }
    final retry = !await languagesOnce();
    final next = retry && due.difference(now) > const Duration(hours: 6)
        ? DateTime.now().add(const Duration(hours: 6))
        : due;
    say('Up to date; the next look ${next.toUtc().toIso8601String()}.');
    await Future<void>.delayed(next.difference(DateTime.now()));
  }
}

/// The languages a comma-separated list names (`de,el,en`).
Set<String> _languages(String? list) => {
  for (final language in (list ?? '').split(','))
    if (language.trim().isNotEmpty) language.trim(),
};

Never _fail(String message) {
  stderr.writeln(message);
  exit(64);
}