halkyon/halkyon_learn_curator/bin/curate.dart

import 'dart:convert';
import 'dart:io';

import 'package:args/args.dart';
import 'package:halkyon_learn_curator/src/sources/jlpt.dart';
import 'package:halkyon_learn_curator/src/sources/jmdict.dart';
import 'package:halkyon_learn_curator/src/sources/kanjidic.dart';
import 'package:halkyon_learn_curator/src/sources/kanjivg.dart';

/// Turns a dictionary download into a dictionary file for the server's
/// `dictionaries` folder, or KanjiVG into the app's stroke files
/// (README.md):
///
///     dart run halkyon_learn_curator:curate jmdict \
///       --input jmdict-examples-eng-3.6.2.json --jlpt original_data \
///       --output jmdict-en.jsonl.gz
///     dart run halkyon_learn_curator:curate kanjidic \
///       --input kanjidic2.xml.gz --jlpt kanji.json \
///       --output kanjidic-en.jsonl.gz
///     dart run halkyon_learn_curator:curate kanjivg \
///       --input kanjivg-20260714.xml.gz \
///       --output ../halkyon_learn_flutter/assets/kanjivg
Future<void> main(List<String> args) async {
  final parser = ArgParser()
    ..addOption(
      'input',
      abbr: 'i',
      help: 'The source file, unpacked or gzipped (.gz).',
    )
    ..addOption(
      'jlpt',
      help:
          "jmdict: yomitan-jlpt-vocab's original_data folder (n1.csv … "
          "n5.csv); kanjidic: kanji-data's kanji.json.",
    )
    ..addFlag(
      'examples',
      help:
          "The input is jmdict-simplified's examples edition "
          '(jmdict-examples-eng-*.json): credit Tatoeba for its sentences. '
          'On by default for a file named so.',
      defaultsTo: null,
    )
    ..addOption(
      'output',
      abbr: 'o',
      help:
          'The dictionary file to write, gzipped when it ends in .gz; for '
          'kanjivg the folder to write the stroke files into.',
    );
  final ArgResults options;
  try {
    options = parser.parse(args);
  } on FormatException catch (error) {
    _fail('${error.message}\n${parser.usage}');
  }
  final [source, ...] = options.rest.isEmpty ? [''] : options.rest;
  final input = options.option('input');
  final output = options.option('output');
  if (!['jmdict', 'kanjidic', 'kanjivg'].contains(source) ||
      input == null ||
      output == null) {
    _fail(
      'Usage: curate jmdict|kanjidic|kanjivg --input <file> [--jlpt <path>] '
      '--output <path>\n${parser.usage}',
    );
  }
  final read = File(input).openRead();
  final text = (input.endsWith('.gz') ? read.transform(gzip.decoder) : read)
      .transform(utf8.decoder);
  final jlptPath = options.option('jlpt');

  if (source == 'kanjivg') {
    final strokes = kanjivgStrokes(await text.join());
    final folder = await Directory(output).create(recursive: true);
    final files = strokeFiles(strokes);
    for (final MapEntry(key: name, value: characters) in files.entries) {
      await File('${folder.path}/$name.json.gz')
          .writeAsBytes(gzip.encode(utf8.encode(jsonEncode(characters))));
    }
    stdout.writeln(
      'Wrote the strokes of ${strokes.length} characters to '
      '${files.length} files in $output.',
    );
    return;
  }

  final Stream<String> lines;
  final int withLevel;
  if (source == 'kanjidic') {
    final jlpt = jlptPath == null
        ? const <String, int>{}
        : await readKanjiJlpt(File(jlptPath));
    lines = Stream.fromIterable(
      kanjidicDictionary(await text.join(), jlpt: jlpt),
    );
    withLevel = jlpt.length;
  } else {
    final jlpt = jlptPath == null
        ? const <String, JlptLevel>{}
        : await readJlptLevels(Directory(jlptPath));
    lines = jmdictDictionary(
      text,
      jlpt: jlpt,
      examples: options.wasParsed('examples')
          ? options.flag('examples')
          : File(input).uri.pathSegments.last.contains('examples'),
    );
    withLevel = jlpt.length;
  }
  var entries = -1;
  final bytes = lines
      .map((line) {
        entries++;
        return line;
      })
      .transform(utf8.encoder);
  await (output.endsWith('.gz') ? bytes.transform(gzip.encoder) : bytes).pipe(
    File(output).openWrite(),
  );
  stdout.writeln(
    'Wrote $entries entries to $output'
    '${withLevel == 0 ? '' : ' ($withLevel with a JLPT level)'}.',
  );
}

Never _fail(String message) {
  stderr.writeln(message);
  exit(64);
}