Tizian 「ティツィアーン」
a875a91f40
A package of its own in the workspace, without Serverpod (AGPL, like the tooling): an adapter per source writes one format for all, documented in its README, which the server will load. The format is JSON Lines: a header (source, version, languages, licence, credits), then entries with a headword and reading, labels (`common`, `jlpt:n5`), a rank, the keys a search matches, and the entry as cards take it. Everything specific to a language happens in its adapter (Japanese: forms, readings in kana and Hepburn romaji, long vowels typed once, meanings without "to" and articles), so the server's search knows no language. JMdict comes as jmdict-simplified's JSON, read one entry at a time: all 218,863 entries in some 20 seconds and 300 MB, 16.8 MB gzipped. The JLPT lists (yomitan-jlpt-vocab, CC BY-SA 4.0, from Waller's CC BY lists) add levels by JMdict entry, the easiest when a word is on several, and give the headword its common spelling. Words rank N5 to N1, then the other common words, then the rest. The server's image leaves the package out, as it does the client's. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01MMm7usAo6q9ZZRoPkWjoxi Signed-off-by: Tizian 「ティツィアーン」 <raisondetredev@pm.me>
92 lines
3 KiB
Dart
92 lines
3 KiB
Dart
import 'dart:convert';
|
|
import 'dart:io';
|
|
|
|
import 'package:halkyon_learn_curator/src/sources/jlpt.dart';
|
|
import 'package:halkyon_learn_curator/src/sources/jmdict.dart';
|
|
import 'package:test/test.dart';
|
|
|
|
void main() {
|
|
late List<Map<String, Object?>> lines;
|
|
|
|
setUpAll(() async {
|
|
final jlpt = await readJlptLevels(Directory('test/fixtures/jlpt'));
|
|
lines = [
|
|
for (final line in await jmdictDictionary(
|
|
File('test/fixtures/jmdict-eng-sample.json')
|
|
.openRead()
|
|
.transform(utf8.decoder),
|
|
jlpt: jlpt,
|
|
).toList())
|
|
jsonDecode(line) as Map<String, Object?>,
|
|
];
|
|
});
|
|
|
|
Map<String, Object?> entry(String headword) =>
|
|
lines.skip(1).firstWhere((line) => line['headword'] == headword);
|
|
|
|
test('the header says what the file holds, and whose it is', () {
|
|
final header = lines.first;
|
|
expect(header['format'], 'halkyon-dictionary');
|
|
expect(header['formatVersion'], 1);
|
|
expect(header['source'], 'jmdict');
|
|
expect(header['sourceVersion'], '3.6.2 (2026-10-05)');
|
|
expect((header['language'], header['glossLanguage']), ('ja', 'en'));
|
|
expect(header['license'], 'CC-BY-SA-4.0');
|
|
expect(
|
|
[
|
|
for (final credit in header['credits']! as List)
|
|
(credit as Map)['name'],
|
|
],
|
|
['JMdict', 'jmdict-simplified', 'JLPT vocabulary lists'],
|
|
);
|
|
expect(lines, hasLength(7));
|
|
});
|
|
|
|
test('a word: its headword and reading, labels, and the keys a search '
|
|
'matches, in kanji, kana, romaji and its meanings', () {
|
|
final word = entry('食べる');
|
|
|
|
expect(word['id'], '1358280');
|
|
expect(word['reading'], 'たべる');
|
|
expect(word['labels'], ['common', 'jlpt:n5']);
|
|
expect(
|
|
word['keys'],
|
|
containsAll(['食べる', '喰べる', 'たべる', 'taberu', 'to eat', 'eat']),
|
|
);
|
|
// A form for searching only isn't shown.
|
|
final forms = (word['entry']! as Map)['forms']! as List;
|
|
expect([for (final form in forms) (form as Map)['text']], ['食べる']);
|
|
});
|
|
|
|
test('a word written in kana has no reading of its own; katakana is '
|
|
'searched as hiragana too', () {
|
|
final word = entry('テレビ');
|
|
|
|
expect(word.containsKey('reading'), isFalse);
|
|
expect(word['keys'], containsAll(['テレビ', 'てれび', 'terebi', 'tv']));
|
|
});
|
|
|
|
test('long vowels are found typed once', () {
|
|
expect(entry('東京')['keys'], containsAll(['toukyou', 'tokyo']));
|
|
});
|
|
|
|
test('the reading is one that belongs to the headword', () {
|
|
expect(entry('四')['reading'], 'し');
|
|
});
|
|
|
|
test("the headword is the JLPT lists' spelling, at the easiest level a "
|
|
'word is listed at', () {
|
|
final word = entry('歯磨き');
|
|
|
|
expect(word['labels'], ['common', 'jlpt:n2']);
|
|
});
|
|
|
|
test('words rank N5 first, then down the levels, then the other common '
|
|
'words, then the rest', () {
|
|
int rank(String headword) => entry(headword)['rank']! as int;
|
|
|
|
expect(rank('食べる'), lessThan(rank('歯磨き')));
|
|
expect(rank('歯磨き'), lessThan(rank('東京')));
|
|
expect(rank('東京'), lessThan(rank('〃')));
|
|
});
|
|
}
|