68 lines
2.1 KiB
Dart
68 lines
2.1 KiB
Dart
|
import 'dart:convert';
|
||
|
|
import 'dart:io';
|
||
|
|
|
||
|
|
import 'package:args/args.dart';
|
||
|
|
import 'package:halkyon_learn_curator/src/sources/jlpt.dart';
|
||
|
|
import 'package:halkyon_learn_curator/src/sources/jmdict.dart';
|
||
|
|
|
||
|
|
/// Turns a dictionary download into a dictionary file for the server's
|
||
|
|
/// `dictionaries` folder (README.md):
|
||
|
|
///
|
||
|
|
/// dart run halkyon_learn_curator:curate jmdict \
|
||
|
|
/// --input jmdict-eng-3.6.2.json --jlpt original_data \
|
||
|
|
/// --output jmdict-en.jsonl.gz
|
||
|
|
Future<void> main(List<String> args) async {
|
||
|
|
final parser = ArgParser()
|
||
|
|
..addOption('input', abbr: 'i', help: 'The source file, unpacked.')
|
||
|
|
..addOption(
|
||
|
|
'jlpt',
|
||
|
|
help: "yomitan-jlpt-vocab's original_data folder (n1.csv … n5.csv).",
|
||
|
|
)
|
||
|
|
..addOption(
|
||
|
|
'output',
|
||
|
|
abbr: 'o',
|
||
|
|
help: 'The dictionary file to write; gzipped when it ends in .gz.',
|
||
|
|
);
|
||
|
|
final ArgResults options;
|
||
|
|
try {
|
||
|
|
options = parser.parse(args);
|
||
|
|
} on FormatException catch (error) {
|
||
|
|
_fail('${error.message}\n${parser.usage}');
|
||
|
|
}
|
||
|
|
final [source, ...] = options.rest.isEmpty ? [''] : options.rest;
|
||
|
|
final input = options.option('input');
|
||
|
|
final output = options.option('output');
|
||
|
|
if (source != 'jmdict' || input == null || output == null) {
|
||
|
|
_fail(
|
||
|
|
'Usage: curate jmdict --input <file> [--jlpt <folder>] --output '
|
||
|
|
'<file>\n${parser.usage}',
|
||
|
|
);
|
||
|
|
}
|
||
|
|
|
||
|
|
final jlpt = switch (options.option('jlpt')) {
|
||
|
|
final folder? => await readJlptLevels(Directory(folder)),
|
||
|
|
null => const <String, JlptLevel>{},
|
||
|
|
};
|
||
|
|
var entries = -1;
|
||
|
|
final lines =
|
||
|
|
jmdictDictionary(
|
||
|
|
File(input).openRead().transform(utf8.decoder),
|
||
|
|
jlpt: jlpt,
|
||
|
|
).map((line) {
|
||
|
|
entries++;
|
||
|
|
return line;
|
||
|
|
});
|
||
|
|
final bytes = lines.transform(utf8.encoder);
|
||
|
|
await (output.endsWith('.gz') ? bytes.transform(gzip.encoder) : bytes).pipe(
|
||
|
|
File(output).openWrite(),
|
||
|
|
);
|
||
|
|
stdout.writeln(
|
||
|
|
'Wrote $entries entries to $output'
|
||
|
|
'${jlpt.isEmpty ? '' : ' (${jlpt.length} with a JLPT level)'}.',
|
||
|
|
);
|
||
|
|
}
|
||
|
|
|
||
|
|
Never _fail(String message) {
|
||
|
|
stderr.writeln(message);
|
||
|
|
exit(64);
|
||
|
|
}
|