feat(curator): characters of Chinese and Korean, the Greek alphabet, Korean words from KRDict
For parity with Japanese's KANJIDIC2 and kana: - Unihan (Unicode License v3) makes characters' dictionaries: the 8,105 standard simplified characters (unihan-hans) and the traditional ones in use (unihan-hant), told apart and ranked by CC-CEDICT's words (Unihan calls 这 its own traditional variant; 後 ranks as the 后 of 以后), with pinyin, meanings, strokes, Kangxi radical, HSK 2025 character levels, the 3,500 common ones and Hong Kong's grades; and Korean Hanja (unihan-ko) with their sounds, the 1,800 of education, and their native readings from Wiktionary (물 수), with the 40 Hangul letters as kana are. - The Greek alphabet, the curator's own (CC0): letters with names, transliteration and sound. - KRDict, the Basic Korean Dictionary for learners (CC BY-SA 2.0 KR, downloaded without an account), replaces Wiktionary for Korean's words: learners' levels, pronunciation where it differs from the spelling (같이 [가치]), Hanja, English senses; Wiktionary's update keeps the words' forms (가다's 99) and the Hanja's readings beside it for both. - The update converts KRDict when its page offers another export and the characters weekly, and removes the files of languages no longer kept or of sources replaced. Measured on the real data: Unihan 3 s; KRDict 54,870 entries in 24 s and under 600 MB. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Tizian 「ティツィアーン」 <raisondetredev@pm.me>
This commit is contained in:
parent
4e6581b9a5
commit
8820cefde3
GPG key ID:
07F6FCF405488B64
1 changed files with 6997 additions and 85 deletions
|
|
@ -3,6 +3,8 @@ import 'dart:io';
|
|||
|
||||
import 'package:archive/archive.dart';
|
||||
|
||||
import 'package:halkyon_learn_curator/src/sources/krdict.dart';
|
||||
import 'package:halkyon_learn_curator/src/sources/wiktionary.dart';
|
||||
import 'package:halkyon_learn_curator/src/update.dart';
|
||||
import 'package:test/test.dart';
|
||||
|
||||
|
|
@ -134,16 +136,18 @@ void main() {
|
|||
);
|
||||
}
|
||||
|
||||
Future<List<File>> update(Set<String> languages) => updateWiktionary(
|
||||
into: into,
|
||||
data: Directory('data'),
|
||||
languages: languages,
|
||||
modified: modified,
|
||||
open: open,
|
||||
fetch: (uri, file) => file.writeAsBytes(
|
||||
tatoeba(uri) ?? (throw HttpException('$uri', uri: uri)),
|
||||
),
|
||||
);
|
||||
Future<List<File>> update(Set<String> languages, {bool korean = false}) =>
|
||||
updateWiktionary(
|
||||
into: into,
|
||||
data: Directory('data'),
|
||||
languages: languages,
|
||||
korean: korean,
|
||||
modified: modified,
|
||||
open: open,
|
||||
fetch: (uri, file) => file.writeAsBytes(
|
||||
tatoeba(uri) ?? (throw HttpException('$uri', uri: uri)),
|
||||
),
|
||||
);
|
||||
|
||||
test(
|
||||
'is converted for the languages named, German and Greek with English '
|
||||
|
|
@ -192,6 +196,23 @@ void main() {
|
|||
expect(await update({}), isEmpty);
|
||||
expect(opened, 0);
|
||||
});
|
||||
|
||||
test("keeps what Korean's dictionaries take from it, even without "
|
||||
'languages of its words', () async {
|
||||
expect(await update({}, korean: true), isEmpty);
|
||||
expect(opened, 1);
|
||||
|
||||
final kept = WiktionaryKorean.fromJson(
|
||||
utf8.decode(
|
||||
gzip.decode(
|
||||
File('${into.path}/$wiktionaryKoreanFile').readAsBytesSync(),
|
||||
),
|
||||
),
|
||||
);
|
||||
expect(kept.hanja['水'], [(hun: '물', eum: '수')]);
|
||||
expect(await update({}, korean: true), isEmpty);
|
||||
expect(opened, 1);
|
||||
});
|
||||
});
|
||||
|
||||
test('CC-CEDICT is converted in both scripts with the HSK levels, and '
|
||||
|
|
@ -251,13 +272,191 @@ void main() {
|
|||
], isNot(contains('Tatoeba')));
|
||||
});
|
||||
|
||||
test('Chinese comes from CC-CEDICT, the other languages from Wiktionary', () {
|
||||
final all = dictionarySources({'de', 'el', 'en', 'zh'});
|
||||
test('Chinese comes from CC-CEDICT, Korean from KRDict, the other '
|
||||
"languages' words from Wiktionary", () {
|
||||
final all = dictionarySources({'de', 'el', 'en', 'ko', 'zh'});
|
||||
expect(all.wiktionary, {'de', 'el', 'en'});
|
||||
expect(all.chinese, isTrue);
|
||||
expect(all.korean, isTrue);
|
||||
expect(all.greek, isTrue);
|
||||
final traditional = dictionarySources({'zh-Hant'});
|
||||
expect(traditional.wiktionary, isEmpty);
|
||||
expect(traditional.chinese, isTrue);
|
||||
expect(traditional.korean, isFalse);
|
||||
expect(dictionarySources({}).chinese, isFalse);
|
||||
});
|
||||
|
||||
test('files of languages no longer kept, or of a source replaced, go; an '
|
||||
"operator's own stay", () {
|
||||
for (final name in [
|
||||
'jmdict-en.jsonl.gz',
|
||||
'wiktionary-de-en.jsonl.gz',
|
||||
'wiktionary-ko-en.jsonl.gz',
|
||||
'wiktionary-el-en.jsonl.gz',
|
||||
'krdict-ko-en.jsonl.gz',
|
||||
'unihan-ko-en.jsonl.gz',
|
||||
'greek-alphabet-en.jsonl.gz',
|
||||
'my-own.jsonl.gz',
|
||||
]) {
|
||||
File('${into.path}/$name').writeAsStringSync('');
|
||||
}
|
||||
|
||||
final removed = removeUnwanted(into, {'de', 'ko'});
|
||||
|
||||
expect([for (final file in removed) file.uri.pathSegments.last]..sort(), [
|
||||
'greek-alphabet-en.jsonl.gz',
|
||||
'wiktionary-el-en.jsonl.gz',
|
||||
'wiktionary-ko-en.jsonl.gz',
|
||||
]);
|
||||
expect(
|
||||
[for (final entity in into.listSync()) entity.uri.pathSegments.last]
|
||||
..sort(),
|
||||
[
|
||||
'jmdict-en.jsonl.gz',
|
||||
'krdict-ko-en.jsonl.gz',
|
||||
'my-own.jsonl.gz',
|
||||
'unihan-ko-en.jsonl.gz',
|
||||
'wiktionary-de-en.jsonl.gz',
|
||||
],
|
||||
);
|
||||
});
|
||||
|
||||
group("KRDict's export", () {
|
||||
var number = '217';
|
||||
var downloads = 0;
|
||||
|
||||
setUp(() {
|
||||
number = '217';
|
||||
downloads = 0;
|
||||
});
|
||||
|
||||
/// KRDict's page and the fixture packed as its export; Tatoeba's.
|
||||
Future<void> fetch(Uri uri, File file) async {
|
||||
if (uri == krdictPage) {
|
||||
await file.writeAsString(
|
||||
"<button onclick=\"batchDownload('215')\"></button>"
|
||||
"<button onclick=\"batchDownload('216')\"></button>"
|
||||
"<button onclick=\"batchDownload('$number')\"></button>",
|
||||
);
|
||||
} else if (uri == krdictDownloadOf(number)) {
|
||||
downloads++;
|
||||
await file.writeAsBytes(
|
||||
ZipEncoder().encodeBytes(
|
||||
Archive()..add(
|
||||
ArchiveFile.bytes(
|
||||
'1_5000_20260919.json',
|
||||
File('test/fixtures/krdict-sample.json').readAsBytesSync(),
|
||||
),
|
||||
),
|
||||
),
|
||||
);
|
||||
} else {
|
||||
await file.writeAsBytes(
|
||||
tatoeba(uri) ?? (throw HttpException('$uri', uri: uri)),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
test('is converted with the forms Wiktionary kept, and again only '
|
||||
'once its page offers another', () async {
|
||||
final kept = WiktionaryKorean()..forms['가다'] = {'갔어요'};
|
||||
File('${into.path}/$wiktionaryKoreanFile')
|
||||
.writeAsBytesSync(gzip.encode(utf8.encode(kept.toJson())));
|
||||
|
||||
final written = await updateKrdict(
|
||||
into: into,
|
||||
data: Directory('data'),
|
||||
fetch: fetch,
|
||||
);
|
||||
|
||||
expect(
|
||||
[for (final file in written) file.uri.pathSegments.last],
|
||||
[krdictFileName],
|
||||
);
|
||||
expect(header(krdictFileName)['sourceVersion'], '2026-09-19');
|
||||
expect([
|
||||
for (final credit in header(krdictFileName)['credits']! as List)
|
||||
(credit as Map)['name'],
|
||||
], contains('Wiktionary'));
|
||||
expect(
|
||||
await updateKrdict(into: into, data: Directory('data'), fetch: fetch),
|
||||
isEmpty,
|
||||
);
|
||||
expect(downloads, 1);
|
||||
|
||||
number = '230';
|
||||
expect(
|
||||
await updateKrdict(into: into, data: Directory('data'), fetch: fetch),
|
||||
hasLength(1),
|
||||
);
|
||||
expect(downloads, 2);
|
||||
expect(
|
||||
[for (final entity in into.listSync()) entity.uri.pathSegments.last]
|
||||
..sort(),
|
||||
['.krdict', wiktionaryKoreanFile, krdictFileName]..sort(),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
test('the characters of Chinese in both scripts, of Korean with the Hangul '
|
||||
'letters and the Greek alphabet, up to date until a Wiktionary update '
|
||||
'keeps new Hanja readings', () async {
|
||||
Future<void> fetch(Uri uri, File file) async => await file.writeAsBytes(
|
||||
uri == unihanSource
|
||||
? ZipEncoder().encodeBytes(
|
||||
Archive()..add(
|
||||
ArchiveFile.bytes(
|
||||
'Unihan_Readings.txt',
|
||||
File('test/fixtures/unihan-sample.txt').readAsBytesSync(),
|
||||
),
|
||||
),
|
||||
)
|
||||
: gzip.encode(
|
||||
File('test/fixtures/cedict-characters-sample.txt')
|
||||
.readAsBytesSync(),
|
||||
),
|
||||
);
|
||||
expect(
|
||||
charactersUpdated(into, chinese: true, korean: true, greek: true),
|
||||
isNull,
|
||||
);
|
||||
|
||||
final written = await updateCharacters(
|
||||
into: into,
|
||||
data: Directory('data'),
|
||||
chinese: true,
|
||||
korean: true,
|
||||
greek: true,
|
||||
fetch: fetch,
|
||||
);
|
||||
|
||||
expect(
|
||||
[for (final file in written) file.uri.pathSegments.last],
|
||||
[
|
||||
'unihan-hans-en.jsonl.gz',
|
||||
'unihan-hant-en.jsonl.gz',
|
||||
'unihan-ko-en.jsonl.gz',
|
||||
'greek-alphabet-en.jsonl.gz',
|
||||
],
|
||||
);
|
||||
expect(header('unihan-hans-en.jsonl.gz')['sourceVersion'], '18.0.0');
|
||||
expect([
|
||||
for (final credit
|
||||
in header('unihan-hans-en.jsonl.gz')['credits']! as List)
|
||||
(credit as Map)['name'],
|
||||
], containsAll(['Unihan Database', 'CC-CEDICT', 'wordfreq']));
|
||||
final updated = charactersUpdated(
|
||||
into,
|
||||
chinese: true,
|
||||
korean: true,
|
||||
greek: true,
|
||||
);
|
||||
expect(updated, isNotNull);
|
||||
|
||||
File('${into.path}/$wiktionaryKoreanFile')
|
||||
..writeAsBytesSync(gzip.encode(utf8.encode(WiktionaryKorean().toJson())))
|
||||
..setLastModifiedSync(updated!.add(const Duration(minutes: 1)));
|
||||
expect(charactersUpdated(into, korean: true), isNull);
|
||||
expect(charactersUpdated(into, chinese: true), isNotNull);
|
||||
});
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue