halkyon-days/halkyon
Watch
1
0
Fork
You've already forked halkyon
 0
Code Issues Pull requests Projects Releases 37 Packages Wiki Activity Actions

feat(curator): KANJIDIC2 and the kana as a dictionary of characters; KanjiVG's strokes for the app

Browse source 
A dictionary file now says whether it holds words or characters (`kind`).
The kanjidic adapter turns KANJIDIC2 into one entry per kanji that has a
Japanese reading or an English meaning (12,633), with its readings,
meanings, school grade, strokes, frequency and JLPT level (Jonathan
Waller's, through kanji-data's kanji.json), labelled `jlpt:nX`,
`grade:N`/`grade:secondary`, followed by the 71 hiragana and katakana of
the kana table with their romaji. A kun reading is found whole; its kanji
part only as a loose key, so typing "ta" brings た before every kanji read
so.

`curate kanjivg` writes KanjiVG's strokes and components as the app's
stroke files, one gzipped JSON file per 256 code points, keyed by the code
point in each entry's id (KanjiVG's 橅 names itself 機).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01MMm7usAo6q9ZZRoPkWjoxi
Signed-off-by: Tizian 「ティツィアーン」 <raisondetredev@pm.me>
This commit is contained in:
Tizian 「ティツィアーン」 2026-10-09 14:37:22 +02:00
commit e42cdfbfcb
Signed by: raisondetredev
GPG key ID: 07F6FCF405488B64
1 changed files with 1036 additions and 29 deletions
Download patch file Download diff file Expand all files Collapse all files

95
halkyon_learn_curator/bin/curate.dart
View file

@ -4,19 +4,34 @@ import 'dart:io';
import 'package:args/args.dart';
import 'package:halkyon_learn_curator/src/sources/jlpt.dart';
import 'package:halkyon_learn_curator/src/sources/jmdict.dart';
import 'package:halkyon_learn_curator/src/sources/kanjidic.dart';
import 'package:halkyon_learn_curator/src/sources/kanjivg.dart';
/// Turns a dictionary download into a dictionary file for the server's
/// `dictionaries` folder (README.md):
/// `dictionaries` folder, or KanjiVG into the app's stroke files
/// (README.md):
///
/// dart run halkyon_learn_curator:curate jmdict \
/// --input jmdict-examples-eng-3.6.2.json --jlpt original_data \
/// --output jmdict-en.jsonl.gz
/// dart run halkyon_learn_curator:curate kanjidic \
/// --input kanjidic2.xml.gz --jlpt kanji.json \
/// --output kanjidic-en.jsonl.gz
/// dart run halkyon_learn_curator:curate kanjivg \
/// --input kanjivg-20260714.xml.gz \
/// --output ../halkyon_learn_flutter/assets/kanjivg
Future<void> main(List<String> args) async {
final parser = ArgParser()
..addOption('input', abbr: 'i', help: 'The source file, unpacked.')
..addOption(
'input',
abbr: 'i',
help: 'The source file, unpacked or gzipped (.gz).',
)
..addOption(
'jlpt',
help: "yomitan-jlpt-vocab's original_data folder (n1.csv … n5.csv).",
help:
"jmdict: yomitan-jlpt-vocab's original_data folder (n1.csv … "
"n5.csv); kanjidic: kanji-data's kanji.json.",
)
..addFlag(
'examples',
@ -29,7 +44,9 @@ Future<void> main(List<String> args) async {
..addOption(
'output',
abbr: 'o',
help: 'The dictionary file to write; gzipped when it ends in .gz.',
help:
'The dictionary file to write, gzipped when it ends in .gz; for '
'kanjivg the folder to write the stroke files into.',
);
final ArgResults options;
try {
@ -40,36 +57,70 @@ Future<void> main(List<String> args) async {
final [source, ...] = options.rest.isEmpty ? [''] : options.rest;
final input = options.option('input');
final output = options.option('output');
if (source != 'jmdict' || input == null || output == null) {
if (!['jmdict', 'kanjidic', 'kanjivg'].contains(source) ||
input == null ||
output == null) {
_fail(
'Usage: curate jmdict --input <file> [--jlpt <folder>] --output '
'<file>\n${parser.usage}',
'Usage: curate jmdict|kanjidic|kanjivg --input <file> [--jlpt <path>] '
'--output <path>\n${parser.usage}',
);
}
final read = File(input).openRead();
final text = (input.endsWith('.gz') ? read.transform(gzip.decoder) : read)
.transform(utf8.decoder);
final jlptPath = options.option('jlpt');
final jlpt = switch (options.option('jlpt')) {
final folder? => await readJlptLevels(Directory(folder)),
null => const <String, JlptLevel>{},
};
if (source == 'kanjivg') {
final strokes = kanjivgStrokes(await text.join());
final folder = await Directory(output).create(recursive: true);
final files = strokeFiles(strokes);
for (final MapEntry(key: name, value: characters) in files.entries) {
await File('${folder.path}/$name.json.gz')
.writeAsBytes(gzip.encode(utf8.encode(jsonEncode(characters))));
}
stdout.writeln(
'Wrote the strokes of ${strokes.length} characters to '
'${files.length} files in $output.',
);
return;
}
final Stream<String> lines;
final int withLevel;
if (source == 'kanjidic') {
final jlpt = jlptPath == null
? const <String, int>{}
: await readKanjiJlpt(File(jlptPath));
lines = Stream.fromIterable(
kanjidicDictionary(await text.join(), jlpt: jlpt),
);
withLevel = jlpt.length;
} else {
final jlpt = jlptPath == null
? const <String, JlptLevel>{}
: await readJlptLevels(Directory(jlptPath));
lines = jmdictDictionary(
text,
jlpt: jlpt,
examples: options.wasParsed('examples')
? options.flag('examples')
: File(input).uri.pathSegments.last.contains('examples'),
);
withLevel = jlpt.length;
}
var entries = -1;
final lines =
jmdictDictionary(
File(input).openRead().transform(utf8.decoder),
jlpt: jlpt,
examples: options.wasParsed('examples')
? options.flag('examples')
: File(input).uri.pathSegments.last.contains('examples'),
).map((line) {
final bytes = lines
.map((line) {
entries++;
return line;
});
final bytes = lines.transform(utf8.encoder);
})
.transform(utf8.encoder);
await (output.endsWith('.gz') ? bytes.transform(gzip.encoder) : bytes).pipe(
File(output).openWrite(),
);
stdout.writeln(
'Wrote $entries entries to $output'
'${jlpt.isEmpty ? '' : ' (${jlpt.length} with a JLPT level)'}.',
'${withLevel == 0 ? '' : ' ($withLevel with a JLPT level)'}.',
);
}