import 'dart:convert'; import 'dart:io'; import 'package:args/args.dart'; import 'package:halkyon_learn_curator/src/sources/jlpt.dart'; import 'package:halkyon_learn_curator/src/sources/jmdict.dart'; import 'package:halkyon_learn_curator/src/sources/jmdict_xml.dart'; import 'package:halkyon_learn_curator/src/sources/kanjidic.dart'; import 'package:halkyon_learn_curator/src/sources/kanjivg.dart'; import 'package:halkyon_learn_curator/src/update.dart'; /// Turns a dictionary download into a dictionary file for the server's /// `dictionaries` folder, or KanjiVG into the app's stroke files /// (README.md): /// /// dart run halkyon_learn_curator:curate jmdict \ /// --input JMdict_e_examp.gz --jlpt original_data \ /// --output jmdict-en.jsonl.gz /// dart run halkyon_learn_curator:curate kanjidic \ /// --input kanjidic2.xml.gz --jlpt kanji.json \ /// --output kanjidic-en.jsonl.gz /// dart run halkyon_learn_curator:curate kanjivg \ /// --input kanjivg-20260714.xml.gz \ /// --output ../halkyon_learn_flutter/assets/kanjivg /// /// Or brings a server's dictionaries up to date from EDRDG's files of the /// day, once or every so many days: /// /// dart run halkyon_learn_curator:curate update \ /// --into ../halkyon_learn_server/dictionaries [--every 7] Future main(List args) async { final parser = ArgParser() ..addOption( 'input', abbr: 'i', help: 'The source file, unpacked or gzipped (.gz).', ) ..addOption( 'jlpt', help: "jmdict: yomitan-jlpt-vocab's original_data folder (n1.csv … " "n5.csv); kanjidic: kanji-data's kanji.json.", ) ..addFlag( 'examples', help: "The input is JMdict's examples edition (EDRDG's " "JMdict_e_examp.gz, jmdict-simplified's jmdict-examples-eng-*.json): " 'credit Tatoeba for its sentences. On by default for a file named ' 'so.', defaultsTo: null, ) ..addOption( 'output', abbr: 'o', help: 'The dictionary file to write, gzipped when it ends in .gz; for ' 'kanjivg the folder to write the stroke files into.', ) ..addOption('into', help: 'update: the folder of the dictionary files.') ..addOption( 'data', help: "update: the curator's data folder (the JLPT lists).", defaultsTo: 'data', ) ..addOption( 'every', help: 'update: keep running, and update once the files are this many ' 'days old.', ); final ArgResults options; try { options = parser.parse(args); } on FormatException catch (error) { _fail('${error.message}\n${parser.usage}'); } final [source, ...] = options.rest.isEmpty ? [''] : options.rest; if (source == 'update') { final into = options.option('into'); final every = options.option('every'); final days = every == null ? null : int.tryParse(every); if (into == null || (every != null && (days == null || days < 1))) { _fail( 'Usage: curate update --into [--data ] ' '[--every ]\n${parser.usage}', ); } await _update(Directory(into), Directory(options.option('data')!), days); return; } final input = options.option('input'); final output = options.option('output'); if (!['jmdict', 'kanjidic', 'kanjivg'].contains(source) || input == null || output == null) { _fail( 'Usage: curate jmdict|kanjidic|kanjivg --input [--jlpt ] ' '--output \n${parser.usage}', ); } final read = File(input).openRead(); final text = (input.endsWith('.gz') ? read.transform(gzip.decoder) : read) .transform(utf8.decoder); final jlptPath = options.option('jlpt'); if (source == 'kanjivg') { final strokes = kanjivgStrokes(await text.join()); final folder = await Directory(output).create(recursive: true); final files = strokeFiles(strokes); for (final MapEntry(key: name, value: characters) in files.entries) { await File('${folder.path}/$name.json.gz') .writeAsBytes(gzip.encode(utf8.encode(jsonEncode(characters)))); } stdout.writeln( 'Wrote the strokes of ${strokes.length} characters to ' '${files.length} files in $output.', ); return; } final Stream lines; final int withLevel; if (source == 'kanjidic') { final jlpt = jlptPath == null ? const {} : await readKanjiJlpt(File(jlptPath)); lines = Stream.fromIterable( kanjidicDictionary(await text.join(), jlpt: jlpt), ); withLevel = jlpt.length; } else { final jlpt = jlptPath == null ? const {} : await readJlptLevels(Directory(jlptPath)); final name = File(input).uri.pathSegments.last; final examples = options.wasParsed('examples') ? options.flag('examples') : name.contains('examp'); // jmdict-simplified's JSON (`jmdict-eng-….json`), else EDRDG's XML. lines = name.contains('.json') ? jmdictDictionary(text, jlpt: jlpt, examples: examples) : jmdictXmlDictionary(text, jlpt: jlpt, examples: examples); withLevel = jlpt.length; } var entries = -1; final bytes = lines .map((line) { entries++; return line; }) .transform(utf8.encoder); await (output.endsWith('.gz') ? bytes.transform(gzip.encoder) : bytes).pipe( File(output).openWrite(), ); stdout.writeln( 'Wrote $entries entries to $output' '${withLevel == 0 ? '' : ' ($withLevel with a JLPT level)'}.', ); } /// Updates the dictionaries in [into]; with [every], keeps running and /// updates them whenever they are that many days old (as the files' dates /// say, so a restart doesn't download them again), trying again after six /// hours when an update fails. Future _update(Directory into, Directory data, int? every) async { void say(String message) => stdout.writeln('${DateTime.now().toUtc().toIso8601String()} $message'); Future once() async { try { final written = await updateDictionaries(into: into, data: data); say( 'Updated ${[for (final file in written) file.uri.pathSegments.last].join(', ')}.', ); return true; } on Object catch (error) { say('Updating the dictionaries failed: $error'); return false; } } if (every == null) { if (!await once()) { exit(1); } return; } for (;;) { final due = lastUpdated(into)?.add(Duration(days: every)); final now = DateTime.now(); if (due == null || !now.isBefore(due)) { if (!await once()) { await Future.delayed(const Duration(hours: 6)); } continue; } say('Up to date; the next update ${due.toUtc().toIso8601String()}.'); await Future.delayed(due.difference(now)); } } Never _fail(String message) { stderr.writeln(message); exit(64); }