chore: Halkyon Learn moves into apps/learn
Its server, client, app and curator, and the tools that write its assets (maps, Anki fixtures, icons), as they are; paths are fixed in the commits that follow. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Tizian 「ティツィアーン」 <raisondetredev@pm.me>
This commit is contained in:
parent
11abd9afce
commit
649499be23
GPG key ID:
07F6FCF405488B64
1 changed files with 0 additions and 0 deletions
699
apps/learn/halkyon_learn_curator/bin/curate.dart
Normal file
699
apps/learn/halkyon_learn_curator/bin/curate.dart
Normal file
|
|
@ -0,0 +1,699 @@
|
|||
import 'dart:convert';
|
||||
import 'dart:io';
|
||||
|
||||
import 'package:archive/archive.dart';
|
||||
import 'package:args/args.dart';
|
||||
import 'package:halkyon_learn_curator/src/mecab.dart';
|
||||
import 'package:halkyon_learn_curator/src/sources/cedict.dart';
|
||||
import 'package:halkyon_learn_curator/src/sources/hangul_strokes.dart';
|
||||
import 'package:halkyon_learn_curator/src/sources/jlpt.dart';
|
||||
import 'package:halkyon_learn_curator/src/sources/jmdict.dart';
|
||||
import 'package:halkyon_learn_curator/src/sources/jmdict_xml.dart';
|
||||
import 'package:halkyon_learn_curator/src/sources/kanjidic.dart';
|
||||
import 'package:halkyon_learn_curator/src/sources/kanjivg.dart';
|
||||
import 'package:halkyon_learn_curator/src/sources/krdict.dart';
|
||||
import 'package:halkyon_learn_curator/src/sources/makemeahanzi.dart';
|
||||
import 'package:halkyon_learn_curator/src/sources/tatoeba.dart';
|
||||
import 'package:halkyon_learn_curator/src/sources/unihan.dart';
|
||||
import 'package:halkyon_learn_curator/src/sources/wiktionary.dart';
|
||||
import 'package:halkyon_learn_curator/src/update.dart';
|
||||
import 'package:halkyon_learn_curator/src/wordfreq.dart';
|
||||
|
||||
/// Turns a dictionary download into a dictionary file for the server's
|
||||
/// `dictionaries` folder, or KanjiVG into the app's stroke files
|
||||
/// (README.md):
|
||||
///
|
||||
/// dart run halkyon_learn_curator:curate jmdict \
|
||||
/// --input JMdict_e_examp.gz --jlpt original_data \
|
||||
/// --output jmdict-en.jsonl.gz
|
||||
/// dart run halkyon_learn_curator:curate kanjidic \
|
||||
/// --input kanjidic2.xml.gz --jlpt kanji.json \
|
||||
/// --output kanjidic-en.jsonl.gz
|
||||
/// dart run halkyon_learn_curator:curate kanjivg \
|
||||
/// --input kanjivg-20260714.xml.gz \
|
||||
/// --output ../halkyon_learn_flutter/assets/kanjivg
|
||||
///
|
||||
/// The Hangul letters' strokes, the curator's own, into the app's stroke
|
||||
/// files:
|
||||
///
|
||||
/// dart run halkyon_learn_curator:curate hangul \
|
||||
/// --output ../halkyon_learn_flutter/assets/hangul
|
||||
///
|
||||
/// Make Me a Hanzi into the app's stroke files of Chinese characters:
|
||||
///
|
||||
/// dart run halkyon_learn_curator:curate makemeahanzi \
|
||||
/// --input graphics.txt --dictionary dictionary.txt \
|
||||
/// --output ../halkyon_learn_flutter/assets/makemeahanzi
|
||||
///
|
||||
/// Wiktionary, as kaikki.org's dump of it, into dictionary files of the
|
||||
/// languages named (German and Greek with English meanings, English with
|
||||
/// German ones):
|
||||
///
|
||||
/// dart run halkyon_learn_curator:curate wiktionary \
|
||||
/// --input raw-wiktextract-data.jsonl.gz --languages de,el,en \
|
||||
/// --output ../halkyon_learn_server/dictionaries
|
||||
///
|
||||
/// MeCab's compiled IPADIC into the pack the app's devices download (ADR
|
||||
/// 0012), from Ubuntu's `mecab-ipadic-utf8` unless another folder is given
|
||||
/// (`update` packs it too, where that package is installed):
|
||||
///
|
||||
/// dart run halkyon_learn_curator:curate ipadic \
|
||||
/// [--input /var/lib/mecab/dic/ipadic-utf8] \
|
||||
/// --output ../halkyon_learn_server/dictionaries
|
||||
///
|
||||
/// CC-CEDICT into its two dictionary files, simplified and traditional:
|
||||
///
|
||||
/// dart run halkyon_learn_curator:curate cedict \
|
||||
/// --input cedict_1_0_ts_utf-8_mdbg.txt.gz \
|
||||
/// --output ../halkyon_learn_server/dictionaries
|
||||
///
|
||||
/// Unihan into dictionary files of Chinese characters (simplified and
|
||||
/// traditional) and of Korean Hanja, for the languages named, the Hanja's
|
||||
/// native readings from a Wiktionary update's `.wiktionary-ko.json.gz`:
|
||||
///
|
||||
/// dart run halkyon_learn_curator:curate unihan \
|
||||
/// --input Unihan.zip --languages ko,zh [--cedict <file>] \
|
||||
/// [--korean <file>] \
|
||||
/// --output ../halkyon_learn_server/dictionaries
|
||||
///
|
||||
/// KRDict's JSON export into the dictionary file of Korean words, with
|
||||
/// the forms of each a Wiktionary update kept (`.wiktionary-ko.json.gz`):
|
||||
///
|
||||
/// dart run halkyon_learn_curator:curate krdict \
|
||||
/// --input krdict_json_20260919.zip [--korean <file>] \
|
||||
/// --output ../halkyon_learn_server/dictionaries
|
||||
///
|
||||
/// Tatoeba's sentences as examples into a dictionary file of words (its
|
||||
/// exports of the dictionary's language, of its meaning language, and of
|
||||
/// the links between them, bzip2 or not):
|
||||
///
|
||||
/// dart run halkyon_learn_curator:curate tatoeba \
|
||||
/// --input wiktionary-de-en.jsonl.gz --sentences deu_sentences.tsv.bz2 \
|
||||
/// --translations eng_sentences.tsv.bz2 --links deu-eng_links.tsv.bz2 \
|
||||
/// --output wiktionary-de-en.jsonl.gz
|
||||
///
|
||||
/// Or brings a server's dictionaries up to date from EDRDG's files of the
|
||||
/// day and, for the languages named, Wiktionary's latest dump and MDBG's
|
||||
/// CC-CEDICT of the day (Chinese), once or every so many days:
|
||||
///
|
||||
/// dart run halkyon_learn_curator:curate update \
|
||||
/// --into ../halkyon_learn_server/dictionaries [--every 7] \
|
||||
/// [--languages de,el,en,zh]
|
||||
Future<void> main(List<String> args) async {
|
||||
final parser = ArgParser()
|
||||
..addOption(
|
||||
'input',
|
||||
abbr: 'i',
|
||||
help: 'The source file, unpacked or gzipped (.gz).',
|
||||
)
|
||||
..addOption(
|
||||
'jlpt',
|
||||
help:
|
||||
"jmdict: yomitan-jlpt-vocab's original_data folder (n1.csv … "
|
||||
"n5.csv); kanjidic: kanji-data's kanji.json.",
|
||||
)
|
||||
..addFlag(
|
||||
'examples',
|
||||
help:
|
||||
"The input is JMdict's examples edition (EDRDG's "
|
||||
"JMdict_e_examp.gz, jmdict-simplified's jmdict-examples-eng-*.json): "
|
||||
'credit Tatoeba for its sentences. On by default for a file named '
|
||||
'so.',
|
||||
defaultsTo: null,
|
||||
)
|
||||
..addOption(
|
||||
'output',
|
||||
abbr: 'o',
|
||||
help:
|
||||
'The dictionary file to write, gzipped when it ends in .gz; for '
|
||||
'kanjivg the folder to write the stroke files into.',
|
||||
)
|
||||
..addOption('into', help: 'update: the folder of the dictionary files.')
|
||||
..addOption(
|
||||
'data',
|
||||
help: "update: the curator's data folder (the JLPT lists).",
|
||||
defaultsTo: 'data',
|
||||
)
|
||||
..addOption(
|
||||
'every',
|
||||
help:
|
||||
'update: keep running, and update once the files are this many '
|
||||
'days old.',
|
||||
)
|
||||
..addOption(
|
||||
'languages',
|
||||
help:
|
||||
'wiktionary: the languages to write dictionaries of, by their '
|
||||
'BCP 47 tags (de,el,en); update: the languages besides Japanese '
|
||||
'to keep current (de,el,en,zh; Chinese from CC-CEDICT, the rest '
|
||||
"from Wiktionary's dump), none without.",
|
||||
)
|
||||
..addOption(
|
||||
'sentences',
|
||||
help: "tatoeba: Tatoeba's sentences in the dictionary's language.",
|
||||
)
|
||||
..addOption(
|
||||
'translations',
|
||||
help: "tatoeba: Tatoeba's sentences in the meaning language.",
|
||||
)
|
||||
..addOption(
|
||||
'links',
|
||||
help: "tatoeba: Tatoeba's links between the two languages.",
|
||||
)
|
||||
..addOption(
|
||||
'cedict',
|
||||
help:
|
||||
'unihan: CC-CEDICT, whose words tell traditional characters in use '
|
||||
'and rank the characters.',
|
||||
)
|
||||
..addOption(
|
||||
'dictionary',
|
||||
help: "makemeahanzi: its dictionary.txt, for the characters' components.",
|
||||
)
|
||||
..addOption(
|
||||
'korean',
|
||||
help:
|
||||
'unihan, krdict: what a Wiktionary update kept for Korean '
|
||||
"(.wiktionary-ko.json.gz: the Hanja's native readings, the "
|
||||
"words' forms).",
|
||||
)
|
||||
..addOption(
|
||||
'version',
|
||||
help:
|
||||
"wiktionary: the dump's date (2026-10-03); the input file's date "
|
||||
'without.',
|
||||
);
|
||||
final ArgResults options;
|
||||
try {
|
||||
options = parser.parse(args);
|
||||
} on FormatException catch (error) {
|
||||
_fail('${error.message}\n${parser.usage}');
|
||||
}
|
||||
final [source, ...] = options.rest.isEmpty ? [''] : options.rest;
|
||||
if (source == 'update') {
|
||||
final into = options.option('into');
|
||||
final every = options.option('every');
|
||||
final days = every == null ? null : int.tryParse(every);
|
||||
if (into == null || (every != null && (days == null || days < 1))) {
|
||||
_fail(
|
||||
'Usage: curate update --into <folder> [--data <folder>] '
|
||||
'[--every <days>] [--languages <languages>]\n${parser.usage}',
|
||||
);
|
||||
}
|
||||
await _update(
|
||||
Directory(into),
|
||||
Directory(options.option('data')!),
|
||||
days,
|
||||
languages: _languages(options.option('languages')),
|
||||
// Without --languages, the files of other languages are left as
|
||||
// they are: an operator may keep them up to date otherwise.
|
||||
removesOthers: options.option('languages') != null,
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (source == 'ipadic') {
|
||||
final output = options.option('output');
|
||||
if (output == null) {
|
||||
_fail(
|
||||
'Usage: curate ipadic [--input <compiled IPADIC>] --output <folder>'
|
||||
'\n${parser.usage}',
|
||||
);
|
||||
}
|
||||
final packed = await packIpadic(
|
||||
Directory(options.option('input') ?? systemIpadic),
|
||||
Directory(output),
|
||||
licence: File(systemIpadicLicence),
|
||||
);
|
||||
stdout.writeln(
|
||||
packed == null
|
||||
? '$ipadicFile in $output is up to date.'
|
||||
: 'Wrote ${packed.path}.',
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (source == 'tatoeba') {
|
||||
final input = options.option('input');
|
||||
final output = options.option('output');
|
||||
final files = [
|
||||
for (final name in ['sentences', 'translations', 'links'])
|
||||
options.option(name),
|
||||
];
|
||||
if (input == null || output == null || files.contains(null)) {
|
||||
_fail(
|
||||
'Usage: curate tatoeba --input <dictionary> --sentences <file> '
|
||||
'--translations <file> --links <file> --output <dictionary>'
|
||||
'\n${parser.usage}',
|
||||
);
|
||||
}
|
||||
Stream<String> lines(String path) {
|
||||
final bytes = path.endsWith('.bz2')
|
||||
? Stream.value(
|
||||
BZip2Decoder().decodeBytes(File(path).readAsBytesSync()),
|
||||
)
|
||||
: File(path).openRead();
|
||||
return bytes.transform(utf8.decoder).transform(const LineSplitter());
|
||||
}
|
||||
|
||||
final links = await readTatoebaLinks(lines(files[2]!));
|
||||
final sentences = await readTatoebaSentences(
|
||||
lines(files[0]!),
|
||||
wanted: links.containsKey,
|
||||
);
|
||||
final linked = {for (final ids in links.values) ...ids};
|
||||
final translations = await readTatoebaSentences(
|
||||
lines(files[1]!),
|
||||
wanted: linked.contains,
|
||||
);
|
||||
final written = File('$output.part');
|
||||
final bytes = withTatoebaExamples(
|
||||
File(input),
|
||||
sentences: sentences,
|
||||
translations: translations,
|
||||
links: links,
|
||||
).transform(utf8.encoder);
|
||||
await (output.endsWith('.gz') ? bytes.transform(gzip.encoder) : bytes).pipe(
|
||||
written.openWrite(),
|
||||
);
|
||||
await written.rename(output);
|
||||
stdout.writeln(
|
||||
'Wrote $output with ${sentences.length} sentences of Tatoeba to '
|
||||
'choose from.',
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (source == 'cedict') {
|
||||
final input = options.option('input');
|
||||
final output = options.option('output');
|
||||
if (input == null || output == null) {
|
||||
_fail(
|
||||
'Usage: curate cedict --input <file> --output <folder> '
|
||||
'[--data <folder>]\n${parser.usage}',
|
||||
);
|
||||
}
|
||||
final read = File(input).openRead();
|
||||
final text =
|
||||
await (input.endsWith('.gz') ? read.transform(gzip.decoder) : read)
|
||||
.transform(utf8.decoder)
|
||||
.join();
|
||||
final data = options.option('data')!;
|
||||
final hsk = await readHskLevels(hskFile(Directory(data)));
|
||||
final frequencies = File('$data/wordfreq/small_zh.msgpack.gz');
|
||||
final words = frequencies.existsSync()
|
||||
? await WordFrequencies.read(frequencies)
|
||||
: WordFrequencies.none;
|
||||
final folder = await Directory(output).create(recursive: true);
|
||||
for (final traditional in [false, true]) {
|
||||
await Stream.fromIterable(
|
||||
cedictDictionary(
|
||||
text,
|
||||
traditional: traditional,
|
||||
hsk: hsk,
|
||||
frequencies: words,
|
||||
),
|
||||
)
|
||||
.transform(utf8.encoder)
|
||||
.transform(gzip.encoder)
|
||||
.pipe(
|
||||
File('${folder.path}/${cedictFileName(traditional: traditional)}')
|
||||
.openWrite(),
|
||||
);
|
||||
}
|
||||
stdout.writeln(
|
||||
'Wrote ${cedictFileName(traditional: false)} and '
|
||||
'${cedictFileName(traditional: true)} to $output.',
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (source == 'krdict') {
|
||||
final input = options.option('input');
|
||||
final output = options.option('output');
|
||||
if (input == null || output == null) {
|
||||
_fail(
|
||||
'Usage: curate krdict --input <zip> --output <folder> '
|
||||
'[--korean <file>] [--data <folder>]\n${parser.usage}',
|
||||
);
|
||||
}
|
||||
final korean = options.option('korean');
|
||||
final frequencies = File(
|
||||
'${options.option('data')}/wordfreq/small_ko.msgpack.gz',
|
||||
);
|
||||
final folder = await Directory(output).create(recursive: true);
|
||||
final (:version, :entries) = krdictExport(await File(input).readAsBytes());
|
||||
await krdictDictionary(
|
||||
entries,
|
||||
sourceVersion: version,
|
||||
frequencies: frequencies.existsSync()
|
||||
? await WordFrequencies.read(frequencies)
|
||||
: null,
|
||||
korean: korean == null
|
||||
? null
|
||||
: WiktionaryKorean.fromJson(
|
||||
utf8.decode(gzip.decode(await File(korean).readAsBytes())),
|
||||
),
|
||||
)
|
||||
.transform(utf8.encoder)
|
||||
.transform(gzip.encoder)
|
||||
.pipe(File('${folder.path}/$krdictFileName').openWrite());
|
||||
stdout.writeln('Wrote $krdictFileName to $output.');
|
||||
return;
|
||||
}
|
||||
if (source == 'unihan') {
|
||||
final input = options.option('input');
|
||||
final output = options.option('output');
|
||||
final languages = _languages(options.option('languages'));
|
||||
if (input == null || output == null || languages.isEmpty) {
|
||||
_fail(
|
||||
'Usage: curate unihan --input <Unihan.zip> --languages <languages> '
|
||||
'--output <folder> [--cedict <file>] [--korean <file>] '
|
||||
'[--data <folder>]'
|
||||
'\n${parser.usage}',
|
||||
);
|
||||
}
|
||||
final korean = options.option('korean');
|
||||
final cedict = options.option('cedict');
|
||||
final written = await unihanDictionaries(
|
||||
await File(input).readAsBytes(),
|
||||
into: await Directory(output).create(recursive: true),
|
||||
languages: languages,
|
||||
data: Directory(options.option('data')!),
|
||||
cedict: cedict == null
|
||||
? null
|
||||
: await (cedict.endsWith('.gz')
|
||||
? File(cedict).openRead().transform(gzip.decoder)
|
||||
: File(cedict).openRead())
|
||||
.transform(utf8.decoder)
|
||||
.join(),
|
||||
eumhun: korean == null
|
||||
? const {}
|
||||
: WiktionaryKorean.fromJson(
|
||||
utf8.decode(gzip.decode(await File(korean).readAsBytes())),
|
||||
).hanja,
|
||||
);
|
||||
stdout.writeln(
|
||||
'Wrote ${[for (final file in written) file.uri.pathSegments.last].join(', ')} '
|
||||
'to $output.',
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (source == 'wiktionary') {
|
||||
final input = options.option('input');
|
||||
final output = options.option('output');
|
||||
final languages = _languages(options.option('languages'));
|
||||
if (input == null || output == null || languages.isEmpty) {
|
||||
_fail(
|
||||
'Usage: curate wiktionary --input <file> --languages <languages> '
|
||||
'--output <folder> [--version <date>] [--data <folder>]'
|
||||
'\n${parser.usage}',
|
||||
);
|
||||
}
|
||||
final read = File(input).openRead();
|
||||
final outputs = wiktionaryOutputs(languages);
|
||||
final data = options.option('data')!;
|
||||
final written = await wiktionaryDictionaries(
|
||||
(input.endsWith('.gz') ? read.transform(gzip.decoder) : read)
|
||||
.transform(utf8.decoder)
|
||||
.transform(const LineSplitter()),
|
||||
into: Directory(output),
|
||||
sourceVersion:
|
||||
options.option('version') ??
|
||||
File(input).lastModifiedSync().toIso8601String().substring(0, 10),
|
||||
languages: outputs.languages,
|
||||
translatedInto: outputs.translatedInto,
|
||||
frequencies: {
|
||||
for (final language in {...languages, 'en'})
|
||||
if (File('$data/wordfreq/small_$language.msgpack.gz') case final file
|
||||
when file.existsSync())
|
||||
language: await WordFrequencies.read(file),
|
||||
},
|
||||
);
|
||||
stdout.writeln(
|
||||
'Wrote ${[for (final file in written) file.uri.pathSegments.last].join(', ')} '
|
||||
'to $output.',
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (source == 'hangul') {
|
||||
final output = options.option('output');
|
||||
if (output == null) {
|
||||
_fail('Usage: curate hangul --output <folder>\n${parser.usage}');
|
||||
}
|
||||
final strokes = hangulStrokes();
|
||||
final folder = await Directory(output).create(recursive: true);
|
||||
final files = strokeFiles(strokes);
|
||||
for (final MapEntry(key: name, value: characters) in files.entries) {
|
||||
await File('${folder.path}/$name.json.gz')
|
||||
.writeAsBytes(gzip.encode(utf8.encode(jsonEncode(characters))));
|
||||
}
|
||||
stdout.writeln(
|
||||
'Wrote the strokes of ${strokes.length} Hangul letters to $output.',
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (source == 'makemeahanzi') {
|
||||
final input = options.option('input');
|
||||
final dictionary = options.option('dictionary');
|
||||
final output = options.option('output');
|
||||
if (input == null || output == null) {
|
||||
_fail(
|
||||
'Usage: curate makemeahanzi --input graphics.txt '
|
||||
'[--dictionary dictionary.txt] --output <folder>\n${parser.usage}',
|
||||
);
|
||||
}
|
||||
final strokes = makemeahanziStrokes(
|
||||
await File(input).readAsLines(),
|
||||
dictionary: dictionary == null
|
||||
? const []
|
||||
: await File(dictionary).readAsLines(),
|
||||
);
|
||||
final folder = await Directory(output).create(recursive: true);
|
||||
final files = strokeFiles(
|
||||
strokes,
|
||||
notice: makemeahanziNotice(
|
||||
DateTime.now().toUtc().toIso8601String().substring(0, 10),
|
||||
),
|
||||
);
|
||||
for (final MapEntry(key: name, value: characters) in files.entries) {
|
||||
await File('${folder.path}/$name.json.gz')
|
||||
.writeAsBytes(gzip.encode(utf8.encode(jsonEncode(characters))));
|
||||
}
|
||||
stdout.writeln(
|
||||
'Wrote the strokes of ${strokes.length} characters to '
|
||||
'${files.length} files in $output.',
|
||||
);
|
||||
return;
|
||||
}
|
||||
final input = options.option('input');
|
||||
final output = options.option('output');
|
||||
if (!['jmdict', 'kanjidic', 'kanjivg'].contains(source) ||
|
||||
input == null ||
|
||||
output == null) {
|
||||
_fail(
|
||||
'Usage: curate jmdict|kanjidic|kanjivg --input <file> [--jlpt <path>] '
|
||||
'--output <path>\n${parser.usage}',
|
||||
);
|
||||
}
|
||||
final read = File(input).openRead();
|
||||
final text = (input.endsWith('.gz') ? read.transform(gzip.decoder) : read)
|
||||
.transform(utf8.decoder);
|
||||
final jlptPath = options.option('jlpt');
|
||||
|
||||
if (source == 'kanjivg') {
|
||||
final strokes = kanjivgStrokes(await text.join());
|
||||
final folder = await Directory(output).create(recursive: true);
|
||||
final files = strokeFiles(strokes);
|
||||
for (final MapEntry(key: name, value: characters) in files.entries) {
|
||||
await File('${folder.path}/$name.json.gz')
|
||||
.writeAsBytes(gzip.encode(utf8.encode(jsonEncode(characters))));
|
||||
}
|
||||
stdout.writeln(
|
||||
'Wrote the strokes of ${strokes.length} characters to '
|
||||
'${files.length} files in $output.',
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
final Stream<String> lines;
|
||||
final int withLevel;
|
||||
if (source == 'kanjidic') {
|
||||
final jlpt = jlptPath == null
|
||||
? const <String, int>{}
|
||||
: await readKanjiJlpt(File(jlptPath));
|
||||
lines = Stream.fromIterable(
|
||||
kanjidicDictionary(await text.join(), jlpt: jlpt),
|
||||
);
|
||||
withLevel = jlpt.length;
|
||||
} else {
|
||||
final jlpt = jlptPath == null
|
||||
? const <String, JlptLevel>{}
|
||||
: await readJlptLevels(Directory(jlptPath));
|
||||
final name = File(input).uri.pathSegments.last;
|
||||
final examples = options.wasParsed('examples')
|
||||
? options.flag('examples')
|
||||
: name.contains('examp');
|
||||
// jmdict-simplified's JSON (`jmdict-eng-….json`), else EDRDG's XML.
|
||||
lines = name.contains('.json')
|
||||
? jmdictDictionary(text, jlpt: jlpt, examples: examples)
|
||||
: jmdictXmlDictionary(text, jlpt: jlpt, examples: examples);
|
||||
withLevel = jlpt.length;
|
||||
}
|
||||
var entries = -1;
|
||||
final bytes = lines
|
||||
.map((line) {
|
||||
entries++;
|
||||
return line;
|
||||
})
|
||||
.transform(utf8.encoder);
|
||||
await (output.endsWith('.gz') ? bytes.transform(gzip.encoder) : bytes).pipe(
|
||||
File(output).openWrite(),
|
||||
);
|
||||
stdout.writeln(
|
||||
'Wrote $entries entries to $output'
|
||||
'${withLevel == 0 ? '' : ' ($withLevel with a JLPT level)'}.',
|
||||
);
|
||||
}
|
||||
|
||||
/// Updates the dictionaries in [into]; with [every], keeps running and
|
||||
/// updates them whenever they are that many days old (as the files' dates
|
||||
/// say, so a restart doesn't download them again), trying again after six
|
||||
/// hours when an update fails. Of [languages], Chinese comes from
|
||||
/// CC-CEDICT as often as EDRDG's files, and its characters from Unihan;
|
||||
/// Korean from KRDict, looked at each time it wakes and converted again
|
||||
/// only when it offers another export, and its Hanja from Unihan; the rest
|
||||
/// from Wiktionary, looked at each time it wakes and converted again only
|
||||
/// when the dump is newer (Korean's forms and the Hanja's readings come
|
||||
/// from it too); Greek's alphabet with the characters. Files of languages
|
||||
/// no longer kept go.
|
||||
Future<void> _update(
|
||||
Directory into,
|
||||
Directory data,
|
||||
int? every, {
|
||||
Set<String> languages = const {},
|
||||
bool removesOthers = true,
|
||||
}) async {
|
||||
final (:wiktionary, :chinese, :korean, :greek) = dictionarySources(languages);
|
||||
void say(String message) =>
|
||||
stdout.writeln('${DateTime.now().toUtc().toIso8601String()} $message');
|
||||
void written(List<File> files) {
|
||||
if (files.isNotEmpty) {
|
||||
say(
|
||||
'Updated ${[for (final file in files) file.uri.pathSegments.last].join(', ')}.',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Runs [step], saying what it wrote; false when it failed.
|
||||
Future<bool> attempt(String name, Future<List<File>> Function() step) async {
|
||||
try {
|
||||
written(await step());
|
||||
return true;
|
||||
} on Object catch (error) {
|
||||
say('Updating $name failed: $error');
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
Future<bool> once() => attempt(
|
||||
'the dictionaries',
|
||||
() => updateDictionaries(into: into, data: data),
|
||||
);
|
||||
|
||||
// Files as old as EDRDG's may be: CC-CEDICT's, and the characters'.
|
||||
bool due(DateTime? updated) =>
|
||||
updated == null ||
|
||||
DateTime.now().difference(updated) >= Duration(days: every ?? 1);
|
||||
Future<bool> chineseOnce() async =>
|
||||
!chinese ||
|
||||
!due(cedictUpdated(into)) ||
|
||||
await attempt('CC-CEDICT', () => updateCedict(into: into, data: data));
|
||||
Future<bool> wiktionaryOnce() => attempt(
|
||||
'Wiktionary',
|
||||
() => updateWiktionary(
|
||||
into: into,
|
||||
data: data,
|
||||
languages: wiktionary,
|
||||
korean: korean,
|
||||
),
|
||||
);
|
||||
Future<bool> koreanOnce() async =>
|
||||
!korean ||
|
||||
await attempt('KRDict', () => updateKrdict(into: into, data: data));
|
||||
Future<bool> charactersOnce() async =>
|
||||
!due(
|
||||
charactersUpdated(into, chinese: chinese, korean: korean, greek: greek),
|
||||
) ||
|
||||
await attempt(
|
||||
'the characters',
|
||||
() => updateCharacters(
|
||||
into: into,
|
||||
data: data,
|
||||
chinese: chinese,
|
||||
korean: korean,
|
||||
greek: greek,
|
||||
),
|
||||
);
|
||||
// MeCab's IPADIC for the devices, where the image installed it; packed
|
||||
// again only when it changed.
|
||||
Future<bool> ipadicOnce() async =>
|
||||
!Directory(systemIpadic).existsSync() ||
|
||||
await attempt('IPADIC', () async {
|
||||
final packed = await packIpadic(
|
||||
Directory(systemIpadic),
|
||||
into,
|
||||
licence: File(systemIpadicLicence),
|
||||
);
|
||||
return [?packed];
|
||||
});
|
||||
// In this order: Korean's words and Hanja take what Wiktionary kept.
|
||||
// What is no longer wanted goes last, once what replaces it is there.
|
||||
Future<bool> languagesOnce() async {
|
||||
final done =
|
||||
!(!await chineseOnce() |
|
||||
!await wiktionaryOnce() |
|
||||
!await koreanOnce() |
|
||||
!await charactersOnce() |
|
||||
!await ipadicOnce());
|
||||
final removed = removesOthers
|
||||
? removeUnwanted(into, languages)
|
||||
: const <File>[];
|
||||
if (removed.isNotEmpty) {
|
||||
say(
|
||||
'Removed ${[for (final file in removed) file.uri.pathSegments.last].join(', ')}.',
|
||||
);
|
||||
}
|
||||
return done;
|
||||
}
|
||||
|
||||
if (every == null) {
|
||||
if (!await once() || !await languagesOnce()) {
|
||||
exit(1);
|
||||
}
|
||||
return;
|
||||
}
|
||||
for (;;) {
|
||||
final due = lastUpdated(into)?.add(Duration(days: every));
|
||||
final now = DateTime.now();
|
||||
if (due == null || !now.isBefore(due)) {
|
||||
if (!await once()) {
|
||||
await Future<void>.delayed(const Duration(hours: 6));
|
||||
}
|
||||
continue;
|
||||
}
|
||||
final retry = !await languagesOnce();
|
||||
final next = retry && due.difference(now) > const Duration(hours: 6)
|
||||
? DateTime.now().add(const Duration(hours: 6))
|
||||
: due;
|
||||
say('Up to date; the next look ${next.toUtc().toIso8601String()}.');
|
||||
await Future<void>.delayed(next.difference(DateTime.now()));
|
||||
}
|
||||
}
|
||||
|
||||
/// The languages a comma-separated list names (`de,el,en`).
|
||||
Set<String> _languages(String? list) => {
|
||||
for (final language in (list ?? '').split(','))
|
||||
if (language.trim().isNotEmpty) language.trim(),
|
||||
};
|
||||
|
||||
Never _fail(String message) {
|
||||
stderr.writeln(message);
|
||||
exit(64);
|
||||
}
|
||||
Loading…
Reference in a new issue