halkyon-days/halkyon
Watch
1
0
Fork
You've already forked halkyon
 0
Code Issues Pull requests Projects Releases 37 Packages Wiki Activity Actions

fix: dictionary headwords, search order and a loader that can't crash

Browse source 
The curator:
- A word JMdict says is usually written in kana gets a kana headword,
  even where the JLPT lists give kanji (その, not 其の), and a rare or
  search-only spelling is never the headword while a usual reading
  exists (そして, not 而して); 9,611 headwords change, 96 of them N5.
- Keys hold full-width letters as typed (Ｔシャツ as tシャツ), デュ/テュ/
  フュ/イェ romanised, long a, i and e typed once as loose keys (kado for
  カード), and a blank JLPT line is skipped.

The server:
- A search is one query ordered by tier, then rank: a loose or prefix
  match no longer pushes an exact one out of the list (75-150 ms over
  all of JMdict, before 150-450).
- Queries find long vowels with macrons spelt おお or ー and typed once
  (ōkii, kōhī), ん typed n' or nn, and full-width letters.
- Pages of a label are ordered by rank and id, so entries that share a
  rank are neither repeated nor skipped.
- The loader never throws (an unreadable folder ended the server from
  the background), refuses two files of one source rather than loading
  both at every start, reads no rows back when replacing a source, and
  takes its time from clock.
- Dictionary files stay out of the Docker build context.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01MMm7usAo6q9ZZRoPkWjoxi
Signed-off-by: Tizian 「ティツィアーン」 <raisondetredev@pm.me>
This commit is contained in:
Tizian 「ティツィアーン」 2026-10-09 12:15:16 +02:00
commit 50ca46f6b1
Signed by: raisondetredev
GPG key ID: 07F6FCF405488B64
5 changed files with 467 additions and 105 deletions
Download patch file Download diff file Expand all files Collapse all files

18
halkyon_learn_server/lib/server.dart
View file

@ -44,14 +44,24 @@ Future<void> run(List<String> args) async {
// The dictionaries the operator put into the server's folder, those that
// changed since; in the background, as loading takes a minute or two.
unawaited(
pod.withSession(
unawaited(_loadDictionaries(pod));
}
/// Loads the dictionaries the operator put into the server's folder (or
/// `DICTIONARIES_DIR`). [loadDictionaries] throws nothing; a session that
/// can't be had is reported here, as an error in the background would end
/// the server.
Future<void> _loadDictionaries(Serverpod pod) async {
try {
await pod.withSession(
(session) => loadDictionaries(
session,
Directory(Platform.environment['DICTIONARIES_DIR'] ?? 'dictionaries'),
),
),
);
);
} on Object catch (error) {
stderr.writeln('Loading the dictionaries failed: $error');
}
}
/// The identifier of the daily pruning.

89
halkyon_learn_server/lib/src/dictionary/dictionary_endpoint.dart
View file

@ -39,52 +39,47 @@ class DictionaryEndpoint extends Endpoint {
return const [];
}
final count = (limit ?? 30).clamp(1, maxSearchResults);
Future<List<DictionaryEntry>> matching(String Function(String) pattern) =>
DictionaryEntry.db.find(
session,
where: (t) {
Expression<Object?> keys = Constant.bool(false);
for (final variant in variants) {
final like = pattern(likeEscaped(variant));
keys = keys | t.searchKeys.like(like) | t.looseKeys.like(like);
}
return t.language.equals(language) &
t.glossLanguage.equals(glossLanguage) &
keys;
},
orderBy: (t) => t.rank,
limit: count,
);
// SELECT … WHERE "searchKeys" LIKE '%|eat|%' OR "looseKeys" LIKE … ORDER
// BY "rank" LIMIT …, then LIKE '%|eat%' for keys starting with it.
final candidates = {
for (final entry in [
...await matching((key) => '%|$key|%'),
...await matching((key) => '%|$key%'),
])
entry.id: entry,
}.values.toList();
int tier(DictionaryEntry entry) {
bool any(String keys, String Function(String) pattern) => variants.any(
(variant) => keys.contains(pattern(variant.replaceAll('|', ' '))),
);
return any(entry.searchKeys, (key) => '|$key|')
? 0
: any(entry.looseKeys, (key) => '|$key|')
? 1
: any(entry.searchKeys, (key) => '|$key')
? 2
: 3;
}
final tiers = {for (final entry in candidates) entry.id: tier(entry)};
candidates.sort(
(a, b) => switch (tiers[a.id]!.compareTo(tiers[b.id]!)) {
0 => a.rank.compareTo(b.rank),
final order => order,
},
// One scan, in the order the tiers give: a key that is the query, a
// loose key that is, a key starting with it, a loose key starting with
// it; within each the dictionary's order. A key that is the query also
// starts with it, so the prefix patterns pick every candidate.
String any(String column, String pattern) => [
for (var i = 0; i < variants.length; i++) '"$column" LIKE @$pattern$i',
].join(' OR ');
final rows = await session.db.unsafeQuery(
'SELECT "id" FROM "dictionary_entry" '
'WHERE "language" = @language AND "glossLanguage" = @glossLanguage '
'AND (${any('searchKeys', 'starts')} OR ${any('looseKeys', 'starts')}) '
'ORDER BY CASE '
'WHEN ${any('searchKeys', 'whole')} THEN 0 '
'WHEN ${any('looseKeys', 'whole')} THEN 1 '
'WHEN ${any('searchKeys', 'starts')} THEN 2 '
'ELSE 3 END, "rank", "id" '
'LIMIT @limit',
parameters: QueryParameters.named({
'language': language,
'glossLanguage': glossLanguage,
'limit': count,
for (final (i, variant) in variants.indexed) ...{
// LIKE '%|eat|%' for a key that is "eat", '%|eat%' for one
// starting with it.
'whole$i': '%|${likeEscaped(variant)}|%',
'starts$i': '%|${likeEscaped(variant)}%',
},
}),
);
return [for (final entry in candidates.take(count)) _hit(entry)];
final ids = [for (final row in rows) row.first! as int];
final byId = {
for (final entry in await DictionaryEntry.db.find(
session,
where: (t) => t.id.inSet(ids.toSet()),
))
entry.id: entry,
};
return [
for (final id in ids)
if (byId[id] case final entry?) _hit(entry),
];
}
/// How many entries of the dictionary of [language] with meanings in
@ -118,7 +113,9 @@ class DictionaryEndpoint extends Endpoint {
t.language.equals(language) &
t.glossLanguage.equals(glossLanguage) &
t.labels.like('%|${likeEscaped(label)}|%'),
orderBy: (t) => t.rank,
// Many share a rank (every other common word): the id breaks the tie,
// so pages neither repeat nor skip one.
orderByList: (t) => [t.rank, t.id],
offset: (offset ?? 0).clamp(0, 1 << 30),
limit: (limit ?? 100).clamp(1, maxPage),
))

81
halkyon_learn_server/lib/src/dictionary/dictionary_loader.dart
View file

@ -1,6 +1,7 @@
import 'dart:convert';
import 'dart:io';
import 'package:clock/clock.dart';
import 'package:crypto/crypto.dart';
import 'package:halkyon_learn_server/src/dictionary/dictionary_search.dart';
import 'package:halkyon_learn_server/src/generated/protocol.dart';
@ -19,27 +20,63 @@ const _batch = 1000;
/// file the curator wrote into the server's `dictionaries` folder, and the
/// server loads it on its next start. A file that can't be read is logged
/// and left out, the others still load. No folder, nothing to load.
///
/// Two files of one source in one meaning language (an old version left
/// beside the new one) are refused: neither loads, as which is meant is
/// unclear, and loading both would replace the source twice at every start.
/// Nothing here throws: the server starts it in the background, where an
/// error would end the process.
Future<void> loadDictionaries(Session session, Directory directory) async {
if (!directory.existsSync()) {
return;
}
final files = [
for (final entity in directory.listSync())
if (entity is File &&
(entity.path.endsWith('.jsonl') || entity.path.endsWith('.jsonl.gz')))
entity,
]..sort((a, b) => a.path.compareTo(b.path));
for (final file in files) {
try {
await loadDictionaryFile(session, file);
} on Object catch (error, st) {
session.log(
'Loading the dictionary ${file.path} failed.',
level: LogLevel.error,
exception: error,
stackTrace: st,
);
void failed(String message, Object error, StackTrace st) => session.log(
message,
level: LogLevel.error,
exception: error,
stackTrace: st,
);
try {
if (!directory.existsSync()) {
return;
}
final files = [
for (final entity in directory.listSync())
if (entity is File &&
(entity.path.endsWith('.jsonl') ||
entity.path.endsWith('.jsonl.gz')))
entity,
]..sort((a, b) => a.path.compareTo(b.path));
final bySource = <String, List<File>>{};
for (final file in files) {
try {
final header = await _header(await _lines(file).first);
(bySource['${header['source']} (${header['glossLanguage']})'] ??= [])
.add(file);
} on Object catch (error, st) {
failed('Reading the dictionary ${file.path} failed.', error, st);
}
}
for (final MapEntry(key: source, value: files) in bySource.entries) {
if (files.length > 1) {
session.log(
'Several dictionary files of $source: '
'${[for (final file in files) file.uri.pathSegments.last].join(', ')}. '
'None of them was loaded; keep one.',
level: LogLevel.error,
);
continue;
}
try {
await loadDictionaryFile(session, files.single);
} on Object catch (error, st) {
failed(
'Loading the dictionary ${files.single.path} failed.',
error,
st,
);
}
}
} on Object catch (error, st) {
failed('Loading the dictionaries in ${directory.path} failed.', error, st);
}
}
@ -66,11 +103,14 @@ Future<bool> loadDictionaryFile(Session session, File file) async {
final name = file.uri.pathSegments.last;
var count = 0;
await session.db.transaction((transaction) async {
// Neither the old rows nor the new ones are read back: a whole source
// is hundreds of thousands of them.
await DictionaryEntry.db.deleteWhere(
session,
where: (t) =>
t.source.equals(source) & t.glossLanguage.equals(glossLanguage),
transaction: transaction,
noReturn: true,
);
final batch = <DictionaryEntry>[];
Future<void> flush() async {
@ -78,6 +118,7 @@ Future<bool> loadDictionaryFile(Session session, File file) async {
session,
batch,
transaction: transaction,
noReturn: true,
);
count += batch.length;
batch.clear();
@ -126,7 +167,7 @@ Future<bool> loadDictionaryFile(Session session, File file) async {
entryCount: count,
fileName: name,
fileHash: hash,
loadedAt: DateTime.now().toUtc(),
loadedAt: clock.now().toUtc(),
);
if (loaded == null) {
await DictionarySource.db.insertRow(

52
halkyon_learn_server/lib/src/dictionary/dictionary_search.dart
View file

@ -1,19 +1,30 @@
/// What a search for [query] in a dictionary of [language] looks for: the
/// query as typed, in lower case, trimmed. In Japanese also with katakana as
/// hiragana, and long vowels with macrons written out (`tōkyō` as
/// `toukyou`), as the keys hold them. Empty for nothing to look for.
/// query as typed, in lower case, trimmed, with full-width letters
/// (`Ｔシャツ`) as people type them. In Japanese also with katakana as
/// hiragana; long vowels with macrons written out the ways the keys hold
/// them (`tōkyō` as `toukyou`, `ōkii` as `ookii`) and once (`tokyo`, a
/// loose key); ん typed as `n'` or `nn` (`kin'en`, `honnya`) as the keys'
/// `n`. Empty for nothing to look for.
///
/// Everything else specific to a language is in the keys, which the curator
/// made. Romaji in the Kunrei style (`si`, `tu`) isn't turned into Hepburn:
/// meanings would turn with it (`time` into `chime`).
List<String> queryVariants(String language, String query) {
final typed = query.trim().toLowerCase().replaceAll(RegExp(r'\s+'), ' ');
final typed = _halfWidth(
query,
).trim().toLowerCase().replaceAll(RegExp(r'\s+'), ' ');
if (typed.isEmpty) {
return const [];
}
return {
typed,
if (language == 'ja') ...[_toHiragana(typed), _withoutMacrons(typed)],
if (language == 'ja') ...[
_toHiragana(typed),
for (final long in _longVowels) _withoutMacrons(typed, long),
typed
.replaceAll(RegExp("['’]"), '')
.replaceAllMapped(RegExp('nn(?=[aiueoy])'), (_) => 'n'),
],
}.toList();
}
@ -35,9 +46,28 @@ String _toHiragana(String text) => String.fromCharCodes([
if (code >= 0x30A1 && code <= 0x30F6) code - 0x60 else code,
]);
String _withoutMacrons(String text) => text
.replaceAll('ā', 'aa')
.replaceAll('ī', 'ii')
.replaceAll('ū', 'uu')
.replaceAll('ē', 'ee')
.replaceAll('ō', 'ou');
/// How a long vowel with a macron is spelt in the keys: ō as `ou`
/// (とうきょう), as `oo` (おおきい, コーヒー's ー), or once (loose keys);
/// ē as `ei` (せんせい) or `ee`.
const _longVowels = [
{'ā': 'aa', 'ī': 'ii', 'ū': 'uu', 'ē': 'ei', 'ō': 'ou'},
{'ā': 'aa', 'ī': 'ii', 'ū': 'uu', 'ē': 'ee', 'ō': 'oo'},
{'ā': 'a', 'ī': 'i', 'ū': 'u', 'ē': 'e', 'ō': 'o'},
];
String _withoutMacrons(String text, Map<String, String> long) => [
for (final char in text.split('')) long[char] ?? char,
].join();
/// [text] with full-width Latin letters, digits and signs as the ordinary
/// ones, as the curator writes the keys.
String _halfWidth(String text) => String.fromCharCodes([
for (final code in text.runes)
// ！ (U+FF01) to ～ (U+FF5E) lie 0xFEE0 above ! to ~.
if (code >= 0xFF01 && code <= 0xFF5E)
code - 0xFEE0
else if (code == 0x3000)
0x20
else
code,
]);

129
halkyon_learn_server/test/integration/dictionary/dictionary_test.dart
View file

@ -156,6 +156,17 @@ void main() {
hit.headword,
];
Future<List<String>> searchFirst(String query, int limit) async => [
for (final hit in await endpoints.dictionary.search(
signedIn,
'ja',
'en',
query,
limit: limit,
))
hit.headword,
];
test('a dictionary file loads with its credits, and again only once '
'it changed, replacing what it held', () async {
final file = write([taberu, terebi, toukyou]);
@ -215,6 +226,66 @@ void main() {
expect(await search('numeral'), ['数詞']);
});
test('the best matches come first even when the rest would fill the '
'list: an exact key before a loose one ranked higher', () async {
await loadDictionaryFile(session, write([suushi, sushi]));
expect(await searchFirst('sushi', 1), ['寿司']);
});
test('long vowels with macrons find words written with おお or ー, and '
'with the vowel once', () async {
final ookii = entry(
'1',
'大きい',
reading: 'おおきい',
rank: 10,
keys: ['大きい', 'おおきい', 'ookii', 'big'],
looseKeys: ['okii'],
);
final koohii = entry(
'2',
'コーヒー',
rank: 20,
keys: ['コーヒー', 'こーひー', 'koohii', 'coffee'],
looseKeys: ['kohi'],
);
await loadDictionaryFile(session, write([ookii, koohii]));
expect(await search('ōkii'), ['大きい']);
expect(await search('kōhī'), ['コーヒー']);
});
test("ん typed as n' or nn, and full-width letters, find their "
'words', () async {
final kinen = entry(
'1',
'禁煙',
reading: 'きんえん',
rank: 10,
keys: ['禁煙', 'きんえん', 'kinen', 'no smoking'],
);
final honya = entry(
'2',
'本屋',
reading: 'ほんや',
rank: 20,
keys: ['本屋', 'ほんや', 'honya', 'bookshop'],
);
final tshirt = entry(
'3',
'Ｔシャツ',
rank: 30,
keys: ['tシャツ', 'てぃーしゃつ', 't-shirt'],
);
await loadDictionaryFile(session, write([kinen, honya, tshirt]));
expect(await search("kin'en"), ['禁煙']);
expect(await search('honnya'), ['本屋']);
expect(await search('Ｔシャツ'), ['Ｔシャツ']);
expect(await search('Tシャツ'), ['Ｔシャツ']);
});
test("the query's wildcards and bars are taken as they are", () async {
await loadDictionaryFile(session, write([taberu, terebi]));
@ -223,6 +294,59 @@ void main() {
expect(await search('|'), isEmpty);
});
test('two files of one source are refused, and neither loads, as '
'which one is meant is unclear', () async {
write([taberu], name: 'jmdict-en-old.jsonl');
write([terebi]);
await loadDictionaries(session, folder);
expect(await DictionaryEntry.db.count(session), 0);
});
test("a folder that can't be read is logged, never thrown: the "
'server goes on', () async {
final locked = Directory('${folder.path}/locked')..createSync();
Process.runSync('chmod', ['000', locked.path]);
try {
await expectLater(loadDictionaries(session, locked), completes);
} finally {
Process.runSync('chmod', ['700', locked.path]);
}
});
test('pages of a label never repeat or skip entries that rank the '
'same', () async {
await loadDictionaryFile(
session,
write([
for (var i = 0; i < 40; i++)
entry(
'$i',
'w$i',
labels: ['common'],
rank: 600000,
keys: ['w$i'],
),
]),
);
final seen = [
for (var offset = 0; offset < 40; offset += 7)
for (final hit in await endpoints.dictionary.labelled(
signedIn,
'ja',
'en',
'common',
offset: offset,
limit: 7,
))
hit.sourceId,
];
expect(seen, hasLength(40));
expect(seen.toSet(), hasLength(40));
});
test("a label picks its entries, in the dictionary's order, a page at "
'a time', () async {
await loadDictionaryFile(session, write([toukyou, terebi, taberu]));
@ -285,7 +409,10 @@ void main() {
test('a query looks for itself, and in Japanese for its hiragana and '
'its long vowels written out', () {
expect(queryVariants('ja', ' Tōkyō '), ['tōkyō', 'toukyou']);
expect(
queryVariants('ja', ' Tōkyō '),
containsAll(['tōkyō', 'toukyou', 'tookyoo', 'tokyo']),
);
expect(queryVariants('ja', 'タベル'), ['タベル', 'たべる']);
expect(queryVariants('en', 'Tōkyō'), ['tōkyō']);
expect(queryVariants('ja', ' '), isEmpty);