Tizian 「ティツィアーン」
3257905536
The curator puts Tatoeba's readings on a dictionary's examples by their Tatoeba ID (`withTatoebaReadings`, `curate readings`, and in `update`): furigana over JMdict's Japanese from its jpn transcriptions, pinyin with tone marks over Chinese from its cmn ones, each word over as many Han characters as it has syllables. An example gets `ruby`, `[start, end, reading]` each, and `rubyUnreviewed` where no one wrote or checked them (MeCab's furigana, the automatic pinyin). JMdict's conversion goes to 3, so instances convert again at once. A word's example hints carry them as ruby marks, the word still in bold. The hints sheet shows them from the start with a switch above (Furigana, Pinyin), kept on the device (`DeviceSettings.hintReadings`), and below a hint whose readings no one reviewed says so. Whats-New: Example sentences in a word's hints now show furigana or pinyin over them, with a switch to hide them. Whats-New-De: Beispielsätze in den Hinweisen eines Worts zeigen jetzt Furigana oder Pinyin darüber, mit einem Schalter zum Ausblenden. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Tizian 「ティツィアーン」 <raisondetredev@pm.me>
752 lines
26 KiB
Dart
752 lines
26 KiB
Dart
import 'dart:convert';
|
|
import 'dart:io';
|
|
|
|
import 'package:archive/archive.dart';
|
|
import 'package:args/args.dart';
|
|
import 'package:halkyon_learn_curator/src/mecab.dart';
|
|
import 'package:halkyon_learn_curator/src/sources/cedict.dart';
|
|
import 'package:halkyon_learn_curator/src/sources/hangul_strokes.dart';
|
|
import 'package:halkyon_learn_curator/src/sources/jlpt.dart';
|
|
import 'package:halkyon_learn_curator/src/sources/jmdict.dart';
|
|
import 'package:halkyon_learn_curator/src/sources/jmdict_xml.dart';
|
|
import 'package:halkyon_learn_curator/src/sources/kanjidic.dart';
|
|
import 'package:halkyon_learn_curator/src/sources/kanjivg.dart';
|
|
import 'package:halkyon_learn_curator/src/sources/krdict.dart';
|
|
import 'package:halkyon_learn_curator/src/sources/makemeahanzi.dart';
|
|
import 'package:halkyon_learn_curator/src/sources/tatoeba.dart';
|
|
import 'package:halkyon_learn_curator/src/sources/tatoeba_readings.dart';
|
|
import 'package:halkyon_learn_curator/src/sources/unihan.dart';
|
|
import 'package:halkyon_learn_curator/src/sources/wiktionary.dart';
|
|
import 'package:halkyon_learn_curator/src/update.dart';
|
|
import 'package:halkyon_learn_curator/src/wordfreq.dart';
|
|
|
|
/// Turns a dictionary download into a dictionary file for the server's
|
|
/// `dictionaries` folder, or KanjiVG into the app's stroke files
|
|
/// (README.md):
|
|
///
|
|
/// dart run halkyon_learn_curator:curate jmdict \
|
|
/// --input JMdict_e_examp.gz --jlpt original_data \
|
|
/// --output jmdict-en.jsonl.gz
|
|
/// dart run halkyon_learn_curator:curate kanjidic \
|
|
/// --input kanjidic2.xml.gz --jlpt kanji.json \
|
|
/// --output kanjidic-en.jsonl.gz
|
|
/// dart run halkyon_learn_curator:curate kanjivg \
|
|
/// --input kanjivg-20260714.xml.gz \
|
|
/// --output ../halkyon_learn_flutter/assets/kanjivg
|
|
///
|
|
/// The Hangul letters' strokes, the curator's own, into the app's stroke
|
|
/// files:
|
|
///
|
|
/// dart run halkyon_learn_curator:curate hangul \
|
|
/// --output ../halkyon_learn_flutter/assets/hangul
|
|
///
|
|
/// Make Me a Hanzi into the app's stroke files of Chinese characters:
|
|
///
|
|
/// dart run halkyon_learn_curator:curate makemeahanzi \
|
|
/// --input graphics.txt --dictionary dictionary.txt \
|
|
/// --output ../halkyon_learn_flutter/assets/makemeahanzi
|
|
///
|
|
/// Wiktionary, as kaikki.org's dump of it, into dictionary files of the
|
|
/// languages named (German and Greek with English meanings, English with
|
|
/// German ones):
|
|
///
|
|
/// dart run halkyon_learn_curator:curate wiktionary \
|
|
/// --input raw-wiktextract-data.jsonl.gz --languages de,el,en \
|
|
/// --output ../halkyon_learn_server/dictionaries
|
|
///
|
|
/// MeCab's compiled IPADIC into the pack the app's devices download (ADR
|
|
/// 0012), from Ubuntu's `mecab-ipadic-utf8` unless another folder is given
|
|
/// (`update` packs it too, where that package is installed):
|
|
///
|
|
/// dart run halkyon_learn_curator:curate ipadic \
|
|
/// [--input /var/lib/mecab/dic/ipadic-utf8] \
|
|
/// --output ../halkyon_learn_server/dictionaries
|
|
///
|
|
/// CC-CEDICT into its two dictionary files, simplified and traditional:
|
|
///
|
|
/// dart run halkyon_learn_curator:curate cedict \
|
|
/// --input cedict_1_0_ts_utf-8_mdbg.txt.gz \
|
|
/// --output ../halkyon_learn_server/dictionaries
|
|
///
|
|
/// Unihan into dictionary files of Chinese characters (simplified and
|
|
/// traditional) and of Korean Hanja, for the languages named, the Hanja's
|
|
/// native readings from a Wiktionary update's `.wiktionary-ko.json.gz`:
|
|
///
|
|
/// dart run halkyon_learn_curator:curate unihan \
|
|
/// --input Unihan.zip --languages ko,zh [--cedict <file>] \
|
|
/// [--korean <file>] \
|
|
/// --output ../halkyon_learn_server/dictionaries
|
|
///
|
|
/// KRDict's JSON export into the dictionary file of Korean words, with
|
|
/// the forms of each a Wiktionary update kept (`.wiktionary-ko.json.gz`):
|
|
///
|
|
/// dart run halkyon_learn_curator:curate krdict \
|
|
/// --input krdict_json_20260919.zip [--korean <file>] \
|
|
/// --output ../halkyon_learn_server/dictionaries
|
|
///
|
|
/// Tatoeba's sentences as examples into a dictionary file of words (its
|
|
/// exports of the dictionary's language, of its meaning language, and of
|
|
/// the links between them, bzip2 or not):
|
|
///
|
|
/// dart run halkyon_learn_curator:curate tatoeba \
|
|
/// --input wiktionary-de-en.jsonl.gz --sentences deu_sentences.tsv.bz2 \
|
|
/// --translations eng_sentences.tsv.bz2 --links deu-eng_links.tsv.bz2 \
|
|
/// --output wiktionary-de-en.jsonl.gz
|
|
///
|
|
/// Readings over a dictionary file's examples, from Tatoeba's
|
|
/// transcriptions of their sentences (furigana over Japanese, pinyin over
|
|
/// Chinese; its `jpn_transcriptions.tsv` or `cmn_transcriptions.tsv`, bzip2
|
|
/// or not):
|
|
///
|
|
/// dart run halkyon_learn_curator:curate readings \
|
|
/// --input jmdict-en.jsonl.gz \
|
|
/// --transcriptions jpn_transcriptions.tsv.bz2 \
|
|
/// --output jmdict-en.jsonl.gz
|
|
///
|
|
/// Or brings a server's dictionaries up to date from EDRDG's files of the
|
|
/// day and, for the languages named, Wiktionary's latest dump and MDBG's
|
|
/// CC-CEDICT of the day (Chinese), once or every so many days:
|
|
///
|
|
/// dart run halkyon_learn_curator:curate update \
|
|
/// --into ../halkyon_learn_server/dictionaries [--every 7] \
|
|
/// [--languages de,el,en,zh]
|
|
Future<void> main(List<String> args) async {
|
|
final parser = ArgParser()
|
|
..addOption(
|
|
'input',
|
|
abbr: 'i',
|
|
help: 'The source file, unpacked or gzipped (.gz).',
|
|
)
|
|
..addOption(
|
|
'jlpt',
|
|
help:
|
|
"jmdict: yomitan-jlpt-vocab's original_data folder (n1.csv … "
|
|
"n5.csv); kanjidic: kanji-data's kanji.json.",
|
|
)
|
|
..addFlag(
|
|
'examples',
|
|
help:
|
|
"The input is JMdict's examples edition (EDRDG's "
|
|
"JMdict_e_examp.gz, jmdict-simplified's jmdict-examples-eng-*.json): "
|
|
'credit Tatoeba for its sentences. On by default for a file named '
|
|
'so.',
|
|
defaultsTo: null,
|
|
)
|
|
..addOption(
|
|
'output',
|
|
abbr: 'o',
|
|
help:
|
|
'The dictionary file to write, gzipped when it ends in .gz; for '
|
|
'kanjivg the folder to write the stroke files into.',
|
|
)
|
|
..addOption('into', help: 'update: the folder of the dictionary files.')
|
|
..addOption(
|
|
'data',
|
|
help: "update: the curator's data folder (the JLPT lists).",
|
|
defaultsTo: 'data',
|
|
)
|
|
..addOption(
|
|
'every',
|
|
help:
|
|
'update: keep running, and update once the files are this many '
|
|
'days old.',
|
|
)
|
|
..addOption(
|
|
'languages',
|
|
help:
|
|
'wiktionary: the languages to write dictionaries of, by their '
|
|
'BCP 47 tags (de,el,en); update: the languages besides Japanese '
|
|
'to keep current (de,el,en,zh; Chinese from CC-CEDICT, the rest '
|
|
"from Wiktionary's dump), none without.",
|
|
)
|
|
..addOption(
|
|
'sentences',
|
|
help: "tatoeba: Tatoeba's sentences in the dictionary's language.",
|
|
)
|
|
..addOption(
|
|
'translations',
|
|
help: "tatoeba: Tatoeba's sentences in the meaning language.",
|
|
)
|
|
..addOption(
|
|
'links',
|
|
help: "tatoeba: Tatoeba's links between the two languages.",
|
|
)
|
|
..addOption(
|
|
'transcriptions',
|
|
help: "readings: Tatoeba's transcriptions of the dictionary's language.",
|
|
)
|
|
..addOption(
|
|
'cedict',
|
|
help:
|
|
'unihan: CC-CEDICT, whose words tell traditional characters in use '
|
|
'and rank the characters.',
|
|
)
|
|
..addOption(
|
|
'dictionary',
|
|
help: "makemeahanzi: its dictionary.txt, for the characters' components.",
|
|
)
|
|
..addOption(
|
|
'korean',
|
|
help:
|
|
'unihan, krdict: what a Wiktionary update kept for Korean '
|
|
"(.wiktionary-ko.json.gz: the Hanja's native readings, the "
|
|
"words' forms).",
|
|
)
|
|
..addOption(
|
|
'version',
|
|
help:
|
|
"wiktionary: the dump's date (2026-10-03); the input file's date "
|
|
'without.',
|
|
);
|
|
final ArgResults options;
|
|
try {
|
|
options = parser.parse(args);
|
|
} on FormatException catch (error) {
|
|
_fail('${error.message}\n${parser.usage}');
|
|
}
|
|
final [source, ...] = options.rest.isEmpty ? [''] : options.rest;
|
|
if (source == 'update') {
|
|
final into = options.option('into');
|
|
final every = options.option('every');
|
|
final days = every == null ? null : int.tryParse(every);
|
|
if (into == null || (every != null && (days == null || days < 1))) {
|
|
_fail(
|
|
'Usage: curate update --into <folder> [--data <folder>] '
|
|
'[--every <days>] [--languages <languages>]\n${parser.usage}',
|
|
);
|
|
}
|
|
await _update(
|
|
Directory(into),
|
|
Directory(options.option('data')!),
|
|
days,
|
|
languages: _languages(options.option('languages')),
|
|
// Without --languages, the files of other languages are left as
|
|
// they are: an operator may keep them up to date otherwise.
|
|
removesOthers: options.option('languages') != null,
|
|
);
|
|
return;
|
|
}
|
|
if (source == 'ipadic') {
|
|
final output = options.option('output');
|
|
if (output == null) {
|
|
_fail(
|
|
'Usage: curate ipadic [--input <compiled IPADIC>] --output <folder>'
|
|
'\n${parser.usage}',
|
|
);
|
|
}
|
|
final packed = await packIpadic(
|
|
Directory(options.option('input') ?? systemIpadic),
|
|
Directory(output),
|
|
licence: File(systemIpadicLicence),
|
|
);
|
|
stdout.writeln(
|
|
packed == null
|
|
? '$ipadicFile in $output is up to date.'
|
|
: 'Wrote ${packed.path}.',
|
|
);
|
|
return;
|
|
}
|
|
if (source == 'tatoeba') {
|
|
final input = options.option('input');
|
|
final output = options.option('output');
|
|
final files = [
|
|
for (final name in ['sentences', 'translations', 'links'])
|
|
options.option(name),
|
|
];
|
|
if (input == null || output == null || files.contains(null)) {
|
|
_fail(
|
|
'Usage: curate tatoeba --input <dictionary> --sentences <file> '
|
|
'--translations <file> --links <file> --output <dictionary>'
|
|
'\n${parser.usage}',
|
|
);
|
|
}
|
|
Stream<String> lines(String path) {
|
|
final bytes = path.endsWith('.bz2')
|
|
? Stream.value(
|
|
BZip2Decoder().decodeBytes(File(path).readAsBytesSync()),
|
|
)
|
|
: File(path).openRead();
|
|
return bytes.transform(utf8.decoder).transform(const LineSplitter());
|
|
}
|
|
|
|
final links = await readTatoebaLinks(lines(files[2]!));
|
|
final sentences = await readTatoebaSentences(
|
|
lines(files[0]!),
|
|
wanted: links.containsKey,
|
|
);
|
|
final linked = {for (final ids in links.values) ...ids};
|
|
final translations = await readTatoebaSentences(
|
|
lines(files[1]!),
|
|
wanted: linked.contains,
|
|
);
|
|
final written = File('$output.part');
|
|
final bytes = withTatoebaExamples(
|
|
File(input),
|
|
sentences: sentences,
|
|
translations: translations,
|
|
links: links,
|
|
).transform(utf8.encoder);
|
|
await (output.endsWith('.gz') ? bytes.transform(gzip.encoder) : bytes).pipe(
|
|
written.openWrite(),
|
|
);
|
|
await written.rename(output);
|
|
stdout.writeln(
|
|
'Wrote $output with ${sentences.length} sentences of Tatoeba to '
|
|
'choose from.',
|
|
);
|
|
return;
|
|
}
|
|
if (source == 'readings') {
|
|
final input = options.option('input');
|
|
final output = options.option('output');
|
|
final transcriptions = options.option('transcriptions');
|
|
if (input == null || output == null || transcriptions == null) {
|
|
_fail(
|
|
'Usage: curate readings --input <dictionary> --transcriptions <file> '
|
|
'--output <dictionary>\n${parser.usage}',
|
|
);
|
|
}
|
|
final read = await readTatoebaTranscriptions(
|
|
(transcriptions.endsWith('.bz2')
|
|
? Stream.value(
|
|
BZip2Decoder().decodeBytes(
|
|
File(transcriptions).readAsBytesSync(),
|
|
),
|
|
)
|
|
: File(transcriptions).openRead())
|
|
.transform(utf8.decoder)
|
|
.transform(const LineSplitter()),
|
|
);
|
|
final bytes = File(input).openRead();
|
|
final written = File('$output.part');
|
|
final lines = withTatoebaReadings(
|
|
(input.endsWith('.gz') ? bytes.transform(gzip.decoder) : bytes)
|
|
.transform(utf8.decoder)
|
|
.transform(const LineSplitter()),
|
|
read,
|
|
).transform(utf8.encoder);
|
|
await (output.endsWith('.gz') ? lines.transform(gzip.encoder) : lines).pipe(
|
|
written.openWrite(),
|
|
);
|
|
await written.rename(output);
|
|
stdout.writeln(
|
|
'Wrote $output with ${read.length} transcriptions to read from.',
|
|
);
|
|
return;
|
|
}
|
|
if (source == 'cedict') {
|
|
final input = options.option('input');
|
|
final output = options.option('output');
|
|
if (input == null || output == null) {
|
|
_fail(
|
|
'Usage: curate cedict --input <file> --output <folder> '
|
|
'[--data <folder>]\n${parser.usage}',
|
|
);
|
|
}
|
|
final read = File(input).openRead();
|
|
final text =
|
|
await (input.endsWith('.gz') ? read.transform(gzip.decoder) : read)
|
|
.transform(utf8.decoder)
|
|
.join();
|
|
final data = options.option('data')!;
|
|
final hsk = await readHskLevels(hskFile(Directory(data)));
|
|
final frequencies = File('$data/wordfreq/small_zh.msgpack.gz');
|
|
final words = frequencies.existsSync()
|
|
? await WordFrequencies.read(frequencies)
|
|
: WordFrequencies.none;
|
|
final folder = await Directory(output).create(recursive: true);
|
|
for (final traditional in [false, true]) {
|
|
await Stream.fromIterable(
|
|
cedictDictionary(
|
|
text,
|
|
traditional: traditional,
|
|
hsk: hsk,
|
|
frequencies: words,
|
|
),
|
|
)
|
|
.transform(utf8.encoder)
|
|
.transform(gzip.encoder)
|
|
.pipe(
|
|
File('${folder.path}/${cedictFileName(traditional: traditional)}')
|
|
.openWrite(),
|
|
);
|
|
}
|
|
stdout.writeln(
|
|
'Wrote ${cedictFileName(traditional: false)} and '
|
|
'${cedictFileName(traditional: true)} to $output.',
|
|
);
|
|
return;
|
|
}
|
|
if (source == 'krdict') {
|
|
final input = options.option('input');
|
|
final output = options.option('output');
|
|
if (input == null || output == null) {
|
|
_fail(
|
|
'Usage: curate krdict --input <zip> --output <folder> '
|
|
'[--korean <file>] [--data <folder>]\n${parser.usage}',
|
|
);
|
|
}
|
|
final korean = options.option('korean');
|
|
final frequencies = File(
|
|
'${options.option('data')}/wordfreq/small_ko.msgpack.gz',
|
|
);
|
|
final folder = await Directory(output).create(recursive: true);
|
|
final (:version, :entries) = krdictExport(await File(input).readAsBytes());
|
|
await krdictDictionary(
|
|
entries,
|
|
sourceVersion: version,
|
|
frequencies: frequencies.existsSync()
|
|
? await WordFrequencies.read(frequencies)
|
|
: null,
|
|
korean: korean == null
|
|
? null
|
|
: WiktionaryKorean.fromJson(
|
|
utf8.decode(gzip.decode(await File(korean).readAsBytes())),
|
|
),
|
|
)
|
|
.transform(utf8.encoder)
|
|
.transform(gzip.encoder)
|
|
.pipe(File('${folder.path}/$krdictFileName').openWrite());
|
|
stdout.writeln('Wrote $krdictFileName to $output.');
|
|
return;
|
|
}
|
|
if (source == 'unihan') {
|
|
final input = options.option('input');
|
|
final output = options.option('output');
|
|
final languages = _languages(options.option('languages'));
|
|
if (input == null || output == null || languages.isEmpty) {
|
|
_fail(
|
|
'Usage: curate unihan --input <Unihan.zip> --languages <languages> '
|
|
'--output <folder> [--cedict <file>] [--korean <file>] '
|
|
'[--data <folder>]'
|
|
'\n${parser.usage}',
|
|
);
|
|
}
|
|
final korean = options.option('korean');
|
|
final cedict = options.option('cedict');
|
|
final written = await unihanDictionaries(
|
|
await File(input).readAsBytes(),
|
|
into: await Directory(output).create(recursive: true),
|
|
languages: languages,
|
|
data: Directory(options.option('data')!),
|
|
cedict: cedict == null
|
|
? null
|
|
: await (cedict.endsWith('.gz')
|
|
? File(cedict).openRead().transform(gzip.decoder)
|
|
: File(cedict).openRead())
|
|
.transform(utf8.decoder)
|
|
.join(),
|
|
eumhun: korean == null
|
|
? const {}
|
|
: WiktionaryKorean.fromJson(
|
|
utf8.decode(gzip.decode(await File(korean).readAsBytes())),
|
|
).hanja,
|
|
);
|
|
stdout.writeln(
|
|
'Wrote ${[for (final file in written) file.uri.pathSegments.last].join(', ')} '
|
|
'to $output.',
|
|
);
|
|
return;
|
|
}
|
|
if (source == 'wiktionary') {
|
|
final input = options.option('input');
|
|
final output = options.option('output');
|
|
final languages = _languages(options.option('languages'));
|
|
if (input == null || output == null || languages.isEmpty) {
|
|
_fail(
|
|
'Usage: curate wiktionary --input <file> --languages <languages> '
|
|
'--output <folder> [--version <date>] [--data <folder>]'
|
|
'\n${parser.usage}',
|
|
);
|
|
}
|
|
final read = File(input).openRead();
|
|
final outputs = wiktionaryOutputs(languages);
|
|
final data = options.option('data')!;
|
|
final written = await wiktionaryDictionaries(
|
|
(input.endsWith('.gz') ? read.transform(gzip.decoder) : read)
|
|
.transform(utf8.decoder)
|
|
.transform(const LineSplitter()),
|
|
into: Directory(output),
|
|
sourceVersion:
|
|
options.option('version') ??
|
|
File(input).lastModifiedSync().toIso8601String().substring(0, 10),
|
|
languages: outputs.languages,
|
|
translatedInto: outputs.translatedInto,
|
|
frequencies: {
|
|
for (final language in {...languages, 'en'})
|
|
if (File('$data/wordfreq/small_$language.msgpack.gz') case final file
|
|
when file.existsSync())
|
|
language: await WordFrequencies.read(file),
|
|
},
|
|
);
|
|
stdout.writeln(
|
|
'Wrote ${[for (final file in written) file.uri.pathSegments.last].join(', ')} '
|
|
'to $output.',
|
|
);
|
|
return;
|
|
}
|
|
if (source == 'hangul') {
|
|
final output = options.option('output');
|
|
if (output == null) {
|
|
_fail('Usage: curate hangul --output <folder>\n${parser.usage}');
|
|
}
|
|
final strokes = hangulStrokes();
|
|
final folder = await Directory(output).create(recursive: true);
|
|
final files = strokeFiles(strokes);
|
|
for (final MapEntry(key: name, value: characters) in files.entries) {
|
|
await File('${folder.path}/$name.json.gz')
|
|
.writeAsBytes(gzip.encode(utf8.encode(jsonEncode(characters))));
|
|
}
|
|
stdout.writeln(
|
|
'Wrote the strokes of ${strokes.length} Hangul letters to $output.',
|
|
);
|
|
return;
|
|
}
|
|
if (source == 'makemeahanzi') {
|
|
final input = options.option('input');
|
|
final dictionary = options.option('dictionary');
|
|
final output = options.option('output');
|
|
if (input == null || output == null) {
|
|
_fail(
|
|
'Usage: curate makemeahanzi --input graphics.txt '
|
|
'[--dictionary dictionary.txt] --output <folder>\n${parser.usage}',
|
|
);
|
|
}
|
|
final strokes = makemeahanziStrokes(
|
|
await File(input).readAsLines(),
|
|
dictionary: dictionary == null
|
|
? const []
|
|
: await File(dictionary).readAsLines(),
|
|
);
|
|
final folder = await Directory(output).create(recursive: true);
|
|
final files = strokeFiles(
|
|
strokes,
|
|
notice: makemeahanziNotice(
|
|
DateTime.now().toUtc().toIso8601String().substring(0, 10),
|
|
),
|
|
);
|
|
for (final MapEntry(key: name, value: characters) in files.entries) {
|
|
await File('${folder.path}/$name.json.gz')
|
|
.writeAsBytes(gzip.encode(utf8.encode(jsonEncode(characters))));
|
|
}
|
|
stdout.writeln(
|
|
'Wrote the strokes of ${strokes.length} characters to '
|
|
'${files.length} files in $output.',
|
|
);
|
|
return;
|
|
}
|
|
final input = options.option('input');
|
|
final output = options.option('output');
|
|
if (!['jmdict', 'kanjidic', 'kanjivg'].contains(source) ||
|
|
input == null ||
|
|
output == null) {
|
|
_fail(
|
|
'Usage: curate jmdict|kanjidic|kanjivg --input <file> [--jlpt <path>] '
|
|
'--output <path>\n${parser.usage}',
|
|
);
|
|
}
|
|
final read = File(input).openRead();
|
|
final text = (input.endsWith('.gz') ? read.transform(gzip.decoder) : read)
|
|
.transform(utf8.decoder);
|
|
final jlptPath = options.option('jlpt');
|
|
|
|
if (source == 'kanjivg') {
|
|
final strokes = kanjivgStrokes(await text.join());
|
|
final folder = await Directory(output).create(recursive: true);
|
|
final files = strokeFiles(strokes);
|
|
for (final MapEntry(key: name, value: characters) in files.entries) {
|
|
await File('${folder.path}/$name.json.gz')
|
|
.writeAsBytes(gzip.encode(utf8.encode(jsonEncode(characters))));
|
|
}
|
|
stdout.writeln(
|
|
'Wrote the strokes of ${strokes.length} characters to '
|
|
'${files.length} files in $output.',
|
|
);
|
|
return;
|
|
}
|
|
|
|
final Stream<String> lines;
|
|
final int withLevel;
|
|
if (source == 'kanjidic') {
|
|
final jlpt = jlptPath == null
|
|
? const <String, int>{}
|
|
: await readKanjiJlpt(File(jlptPath));
|
|
lines = Stream.fromIterable(
|
|
kanjidicDictionary(await text.join(), jlpt: jlpt),
|
|
);
|
|
withLevel = jlpt.length;
|
|
} else {
|
|
final jlpt = jlptPath == null
|
|
? const <String, JlptLevel>{}
|
|
: await readJlptLevels(Directory(jlptPath));
|
|
final name = File(input).uri.pathSegments.last;
|
|
final examples = options.wasParsed('examples')
|
|
? options.flag('examples')
|
|
: name.contains('examp');
|
|
// jmdict-simplified's JSON (`jmdict-eng-….json`), else EDRDG's XML.
|
|
lines = name.contains('.json')
|
|
? jmdictDictionary(text, jlpt: jlpt, examples: examples)
|
|
: jmdictXmlDictionary(text, jlpt: jlpt, examples: examples);
|
|
withLevel = jlpt.length;
|
|
}
|
|
var entries = -1;
|
|
final bytes = lines
|
|
.map((line) {
|
|
entries++;
|
|
return line;
|
|
})
|
|
.transform(utf8.encoder);
|
|
await (output.endsWith('.gz') ? bytes.transform(gzip.encoder) : bytes).pipe(
|
|
File(output).openWrite(),
|
|
);
|
|
stdout.writeln(
|
|
'Wrote $entries entries to $output'
|
|
'${withLevel == 0 ? '' : ' ($withLevel with a JLPT level)'}.',
|
|
);
|
|
}
|
|
|
|
/// Updates the dictionaries in [into]; with [every], keeps running and
|
|
/// updates them whenever they are that many days old (as the files' dates
|
|
/// say, so a restart doesn't download them again), trying again after six
|
|
/// hours when an update fails. Of [languages], Chinese comes from
|
|
/// CC-CEDICT as often as EDRDG's files, and its characters from Unihan;
|
|
/// Korean from KRDict, looked at each time it wakes and converted again
|
|
/// only when it offers another export, and its Hanja from Unihan; the rest
|
|
/// from Wiktionary, looked at each time it wakes and converted again only
|
|
/// when the dump is newer (Korean's forms and the Hanja's readings come
|
|
/// from it too); Greek's alphabet with the characters. Files of languages
|
|
/// no longer kept go.
|
|
Future<void> _update(
|
|
Directory into,
|
|
Directory data,
|
|
int? every, {
|
|
Set<String> languages = const {},
|
|
bool removesOthers = true,
|
|
}) async {
|
|
final (:wiktionary, :chinese, :korean, :greek) = dictionarySources(languages);
|
|
void say(String message) =>
|
|
stdout.writeln('${DateTime.now().toUtc().toIso8601String()} $message');
|
|
void written(List<File> files) {
|
|
if (files.isNotEmpty) {
|
|
say(
|
|
'Updated ${[for (final file in files) file.uri.pathSegments.last].join(', ')}.',
|
|
);
|
|
}
|
|
}
|
|
|
|
/// Runs [step], saying what it wrote; false when it failed.
|
|
Future<bool> attempt(String name, Future<List<File>> Function() step) async {
|
|
try {
|
|
written(await step());
|
|
return true;
|
|
} on Object catch (error) {
|
|
say('Updating $name failed: $error');
|
|
return false;
|
|
}
|
|
}
|
|
|
|
Future<bool> once() => attempt(
|
|
'the dictionaries',
|
|
() => updateDictionaries(into: into, data: data),
|
|
);
|
|
|
|
// Files as old as EDRDG's may be: CC-CEDICT's, and the characters'.
|
|
bool due(DateTime? updated) =>
|
|
updated == null ||
|
|
DateTime.now().difference(updated) >= Duration(days: every ?? 1);
|
|
Future<bool> chineseOnce() async =>
|
|
!chinese ||
|
|
!due(cedictUpdated(into)) ||
|
|
await attempt('CC-CEDICT', () => updateCedict(into: into, data: data));
|
|
Future<bool> wiktionaryOnce() => attempt(
|
|
'Wiktionary',
|
|
() => updateWiktionary(
|
|
into: into,
|
|
data: data,
|
|
languages: wiktionary,
|
|
korean: korean,
|
|
),
|
|
);
|
|
Future<bool> koreanOnce() async =>
|
|
!korean ||
|
|
await attempt('KRDict', () => updateKrdict(into: into, data: data));
|
|
Future<bool> charactersOnce() async =>
|
|
!due(
|
|
charactersUpdated(into, chinese: chinese, korean: korean, greek: greek),
|
|
) ||
|
|
await attempt(
|
|
'the characters',
|
|
() => updateCharacters(
|
|
into: into,
|
|
data: data,
|
|
chinese: chinese,
|
|
korean: korean,
|
|
greek: greek,
|
|
),
|
|
);
|
|
// MeCab's IPADIC for the devices, where the image installed it; packed
|
|
// again only when it changed.
|
|
Future<bool> ipadicOnce() async =>
|
|
!Directory(systemIpadic).existsSync() ||
|
|
await attempt('IPADIC', () async {
|
|
final packed = await packIpadic(
|
|
Directory(systemIpadic),
|
|
into,
|
|
licence: File(systemIpadicLicence),
|
|
);
|
|
return [?packed];
|
|
});
|
|
// In this order: Korean's words and Hanja take what Wiktionary kept.
|
|
// What is no longer wanted goes last, once what replaces it is there.
|
|
Future<bool> languagesOnce() async {
|
|
final done =
|
|
!(!await chineseOnce() |
|
|
!await wiktionaryOnce() |
|
|
!await koreanOnce() |
|
|
!await charactersOnce() |
|
|
!await ipadicOnce());
|
|
final removed = removesOthers
|
|
? removeUnwanted(into, languages)
|
|
: const <File>[];
|
|
if (removed.isNotEmpty) {
|
|
say(
|
|
'Removed ${[for (final file in removed) file.uri.pathSegments.last].join(', ')}.',
|
|
);
|
|
}
|
|
return done;
|
|
}
|
|
|
|
if (every == null) {
|
|
if (!await once() || !await languagesOnce()) {
|
|
exit(1);
|
|
}
|
|
return;
|
|
}
|
|
for (;;) {
|
|
final due = lastUpdated(into)?.add(Duration(days: every));
|
|
final now = DateTime.now();
|
|
if (due == null || !now.isBefore(due)) {
|
|
if (!await once()) {
|
|
await Future<void>.delayed(const Duration(hours: 6));
|
|
}
|
|
continue;
|
|
}
|
|
final retry = !await languagesOnce();
|
|
final next = retry && due.difference(now) > const Duration(hours: 6)
|
|
? DateTime.now().add(const Duration(hours: 6))
|
|
: due;
|
|
say('Up to date; the next look ${next.toUtc().toIso8601String()}.');
|
|
await Future<void>.delayed(next.difference(DateTime.now()));
|
|
}
|
|
}
|
|
|
|
/// The languages a comma-separated list names (`de,el,en`).
|
|
Set<String> _languages(String? list) => {
|
|
for (final language in (list ?? '').split(','))
|
|
if (language.trim().isNotEmpty) language.trim(),
|
|
};
|
|
|
|
Never _fail(String message) {
|
|
stderr.writeln(message);
|
|
exit(64);
|
|
}
|