feat: dictionaries hold words or characters, and are asked by their source

A dictionary source now says what it holds (`kind`: `words`, the default
for files from before, or `characters`), a string so that an app older
than a new kind leaves it out. Search, countLabelled and labelled take the
dictionary by its source and meaning language, as the unique index names
it, rather than by the language: with KANJIDIC2 beside JMdict, a search
for words would otherwise bring kanji, and JLPT N5 would hold both. The
search reads the language for its query variants from the source's row;
a dictionary the instance doesn't hold finds nothing. The rank index goes
by source too.

Migration: drops and recreates dictionary_entry_rank_idx on (source,
glossLanguage, rank), and adds dictionary_source.kind, text NOT NULL
DEFAULT 'words', so a JMdict loaded before is a dictionary of words. Both
tables are reference data, never synced; the old server keeps working
until the switch (its queries by language just lose the index).

In the app, a studyset's dictionary is one of words unless a kind is
asked for.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01MMm7usAo6q9ZZRoPkWjoxi
Signed-off-by: Tizian 「ティツィアーン」 <raisondetredev@pm.me>
This commit is contained in:
Tizian 「ティツィアーン」 2026-10-09 14:47:46 +02:00
commit a6699b30cb
Signed by: raisondetredev
GPG key ID: 07F6FCF405488B64
15 changed files with 11756 additions and 84 deletions

View file

@ -3,8 +3,9 @@ import 'package:halkyon_learn_server/src/generated/protocol.dart';
import 'package:serverpod/serverpod.dart';
/// The dictionaries the instance holds, to look words up in and to take
/// words from by a label (all JLPT N5 words). Reference data: any signed-in
/// person may read all of it.
/// words from by a label (all JLPT N5 words). A dictionary is one source
/// in one meaning language (`jmdict` in `en`). Reference data: any
/// signed-in person may read all of it.
class DictionaryEndpoint extends Endpoint {
@override
bool get requireLogin => true;
@ -22,19 +23,28 @@ class DictionaryEndpoint extends Endpoint {
orderBy: (t) => t.language,
);
/// The entries of the dictionary of [language] with meanings in
/// The entries of the dictionary of [source] with meanings in
/// [glossLanguage] that [query] finds, best first: those with a key that
/// is the query, then a loose key that is (`sushi` for すうし), then a key
/// starting with it, then a loose key; each in the dictionary's order. At
/// most [limit] (30 unless given).
/// most [limit] (30 unless given). None for a dictionary the instance
/// doesn't hold.
Future<List<DictionaryHit>> search(
Session session,
String language,
String source,
String glossLanguage,
String query, {
int? limit,
}) async {
final variants = queryVariants(language, query);
final dictionary = await DictionarySource.db.findFirstRow(
session,
where: (t) =>
t.source.equals(source) & t.glossLanguage.equals(glossLanguage),
);
if (dictionary == null) {
return const [];
}
final variants = queryVariants(dictionary.language, query);
if (variants.isEmpty) {
return const [];
}
@ -48,7 +58,7 @@ class DictionaryEndpoint extends Endpoint {
].join(' OR ');
final rows = await session.db.unsafeQuery(
'SELECT "id" FROM "dictionary_entry" '
'WHERE "language" = @language AND "glossLanguage" = @glossLanguage '
'WHERE "source" = @source AND "glossLanguage" = @glossLanguage '
'AND (${any('searchKeys', 'starts')} OR ${any('looseKeys', 'starts')}) '
'ORDER BY CASE '
'WHEN ${any('searchKeys', 'whole')} THEN 0 '
@ -57,7 +67,7 @@ class DictionaryEndpoint extends Endpoint {
'ELSE 3 END, "rank", "id" '
'LIMIT @limit',
parameters: QueryParameters.named({
'language': language,
'source': source,
'glossLanguage': glossLanguage,
'limit': count,
for (final (i, variant) in variants.indexed) ...{
@ -82,17 +92,17 @@ class DictionaryEndpoint extends Endpoint {
];
}
/// How many entries of the dictionary of [language] with meanings in
/// How many entries of the dictionary of [source] with meanings in
/// [glossLanguage] carry [label].
Future<int> countLabelled(
Session session,
String language,
String source,
String glossLanguage,
String label,
) => DictionaryEntry.db.count(
session,
where: (t) =>
t.language.equals(language) &
t.source.equals(source) &
t.glossLanguage.equals(glossLanguage) &
t.labels.like('%|${likeEscaped(label)}|%'),
);
@ -101,7 +111,7 @@ class DictionaryEndpoint extends Endpoint {
/// them (100 unless given, at most [maxPage]) after the first [offset].
Future<List<DictionaryHit>> labelled(
Session session,
String language,
String source,
String glossLanguage,
String label, {
int? offset,
@ -110,7 +120,7 @@ class DictionaryEndpoint extends Endpoint {
for (final entry in await DictionaryEntry.db.find(
session,
where: (t) =>
t.language.equals(language) &
t.source.equals(source) &
t.glossLanguage.equals(glossLanguage) &
t.labels.like('%|${likeEscaped(label)}|%'),
// Many share a rank (every other common word): the id breaks the tie,

View file

@ -35,6 +35,6 @@ indexes:
dictionary_entry_source_idx:
fields: source, glossLanguage, sourceId
unique: true
# A language's entries in their order, for what is picked by a label.
# A dictionary's entries in their order, for what is picked by a label.
dictionary_entry_rank_idx:
fields: language, glossLanguage, rank
fields: source, glossLanguage, rank

View file

@ -156,6 +156,8 @@ Future<bool> loadDictionaryFile(Session session, File file) async {
id: loaded?.id,
source: source,
sourceVersion: header['sourceVersion']! as String,
// Files written before kinds were all of words.
kind: header['kind'] as String? ?? 'words',
language: language,
glossLanguage: glossLanguage,
license: header['license']! as String,

View file

@ -8,6 +8,10 @@ fields:
source: String
### Its version (`3.6.2 (2026-10-05)`).
sourceVersion: String
### What it holds: `words` (JMdict) or `characters` (KANJIDIC2). A string
### rather than an enum, so an app older than a new kind can leave out
### what it doesn't know.
kind: String, default='words'
### The words' language (`ja`).
language: String
### The meanings' language (`en`).

View file

@ -23,6 +23,7 @@ abstract class DictionarySource
this.id,
required this.source,
required this.sourceVersion,
String? kind,
required this.language,
required this.glossLanguage,
required this.license,
@ -31,12 +32,13 @@ abstract class DictionarySource
required this.fileName,
required this.fileHash,
required this.loadedAt,
});
}) : kind = kind ?? 'words';
factory DictionarySource({
int? id,
required String source,
required String sourceVersion,
String? kind,
required String language,
required String glossLanguage,
required String license,
@ -52,6 +54,7 @@ abstract class DictionarySource
id: jsonSerialization['id'] as int?,
source: jsonSerialization['source'] as String,
sourceVersion: jsonSerialization['sourceVersion'] as String,
kind: jsonSerialization['kind'] as String?,
language: jsonSerialization['language'] as String,
glossLanguage: jsonSerialization['glossLanguage'] as String,
license: jsonSerialization['license'] as String,
@ -81,6 +84,11 @@ abstract class DictionarySource
/// Its version (`3.6.2 (2026-10-05)`).
String sourceVersion;
/// What it holds: `words` (JMdict) or `characters` (KANJIDIC2). A string
/// rather than an enum, so an app older than a new kind can leave out
/// what it doesn't know.
String kind;
/// The words' language (`ja`).
String language;
@ -114,6 +122,7 @@ abstract class DictionarySource
int? id,
String? source,
String? sourceVersion,
String? kind,
String? language,
String? glossLanguage,
String? license,
@ -130,6 +139,7 @@ abstract class DictionarySource
if (id != null) 'id': id,
'source': source,
'sourceVersion': sourceVersion,
'kind': kind,
'language': language,
'glossLanguage': glossLanguage,
'license': license,
@ -148,6 +158,7 @@ abstract class DictionarySource
if (id != null) 'id': id,
'source': source,
'sourceVersion': sourceVersion,
'kind': kind,
'language': language,
'glossLanguage': glossLanguage,
'license': license,
@ -194,6 +205,7 @@ class _DictionarySourceImpl extends DictionarySource {
int? id,
required String source,
required String sourceVersion,
String? kind,
required String language,
required String glossLanguage,
required String license,
@ -206,6 +218,7 @@ class _DictionarySourceImpl extends DictionarySource {
id: id,
source: source,
sourceVersion: sourceVersion,
kind: kind,
language: language,
glossLanguage: glossLanguage,
license: license,
@ -224,6 +237,7 @@ class _DictionarySourceImpl extends DictionarySource {
Object? id = _Undefined,
String? source,
String? sourceVersion,
String? kind,
String? language,
String? glossLanguage,
String? license,
@ -237,6 +251,7 @@ class _DictionarySourceImpl extends DictionarySource {
id: id is int? ? id : this.id,
source: source ?? this.source,
sourceVersion: sourceVersion ?? this.sourceVersion,
kind: kind ?? this.kind,
language: language ?? this.language,
glossLanguage: glossLanguage ?? this.glossLanguage,
license: license ?? this.license,
@ -264,6 +279,11 @@ class DictionarySourceUpdateTable
value,
);
_is.ColumnValue<String, String> kind(String value) => _is.ColumnValue(
table.kind,
value,
);
_is.ColumnValue<String, String> language(String value) => _is.ColumnValue(
table.language,
value,
@ -323,6 +343,11 @@ class DictionarySourceTable extends _is.Table<int?> {
'sourceVersion',
this,
);
kind = _is.ColumnString(
'kind',
this,
hasDefault: true,
);
language = _is.ColumnString(
'language',
this,
@ -365,6 +390,11 @@ class DictionarySourceTable extends _is.Table<int?> {
/// Its version (`3.6.2 (2026-10-05)`).
late final _is.ColumnString sourceVersion;
/// What it holds: `words` (JMdict) or `characters` (KANJIDIC2). A string
/// rather than an enum, so an app older than a new kind can leave out
/// what it doesn't know.
late final _is.ColumnString kind;
/// The words' language (`ja`).
late final _is.ColumnString language;
@ -393,6 +423,7 @@ class DictionarySourceTable extends _is.Table<int?> {
id,
source,
sourceVersion,
kind,
language,
glossLanguage,
license,

View file

@ -334,8 +334,8 @@ class Endpoints extends _is.EndpointDispatch {
'search': _is.MethodConnector(
name: 'search',
params: {
'language': _is.ParameterDescription(
name: 'language',
'source': _is.ParameterDescription(
name: 'source',
type: _is.getType<String>(),
nullable: false,
),
@ -363,7 +363,7 @@ class Endpoints extends _is.EndpointDispatch {
(endpoints['dictionary'] as _ilgtwlto.DictionaryEndpoint)
.search(
session,
params['language'],
params['source'],
params['glossLanguage'],
params['query'],
limit: params['limit'],
@ -372,8 +372,8 @@ class Endpoints extends _is.EndpointDispatch {
'countLabelled': _is.MethodConnector(
name: 'countLabelled',
params: {
'language': _is.ParameterDescription(
name: 'language',
'source': _is.ParameterDescription(
name: 'source',
type: _is.getType<String>(),
nullable: false,
),
@ -396,7 +396,7 @@ class Endpoints extends _is.EndpointDispatch {
(endpoints['dictionary'] as _ilgtwlto.DictionaryEndpoint)
.countLabelled(
session,
params['language'],
params['source'],
params['glossLanguage'],
params['label'],
),
@ -404,8 +404,8 @@ class Endpoints extends _is.EndpointDispatch {
'labelled': _is.MethodConnector(
name: 'labelled',
params: {
'language': _is.ParameterDescription(
name: 'language',
'source': _is.ParameterDescription(
name: 'source',
type: _is.getType<String>(),
nullable: false,
),
@ -438,7 +438,7 @@ class Endpoints extends _is.EndpointDispatch {
(endpoints['dictionary'] as _ilgtwlto.DictionaryEndpoint)
.labelled(
session,
params['language'],
params['source'],
params['glossLanguage'],
params['label'],
offset: params['offset'],

View file

@ -625,7 +625,7 @@ class Protocol extends _is.DatabaseSerializationManager {
elements: [
_isp.IndexElementDefinition(
type: _isp.IndexElementDefinitionType.column,
definition: 'language',
definition: 'source',
),
_isp.IndexElementDefinition(
type: _isp.IndexElementDefinitionType.column,
@ -668,6 +668,13 @@ class Protocol extends _is.DatabaseSerializationManager {
isNullable: false,
dartType: 'String',
),
_isp.ColumnDefinition(
name: 'kind',
columnType: _isp.ColumnType.text,
isNullable: false,
dartType: 'String',
columnDefault: '\'words\'',
),
_isp.ColumnDefinition(
name: 'language',
columnType: _isp.ColumnType.text,

File diff suppressed because it is too large Load diff

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,78 @@
{
"__className__": "serverpod.DatabaseMigration",
"actions": [
{
"__className__": "serverpod.DatabaseMigrationAction",
"type": "alterTable",
"alterTable": {
"__className__": "serverpod.TableMigration",
"name": "dictionary_entry",
"schema": "public",
"addColumns": [],
"deleteColumns": [],
"modifyColumns": [],
"addIndexes": [
{
"__className__": "serverpod.IndexDefinition",
"indexName": "dictionary_entry_rank_idx",
"elements": [
{
"__className__": "serverpod.IndexElementDefinition",
"type": 0,
"definition": "source"
},
{
"__className__": "serverpod.IndexElementDefinition",
"type": 0,
"definition": "glossLanguage"
},
{
"__className__": "serverpod.IndexElementDefinition",
"type": 0,
"definition": "rank"
}
],
"type": "btree",
"isUnique": false,
"isPrimary": false
}
],
"deleteIndexes": [
"dictionary_entry_rank_idx"
],
"addForeignKeys": [],
"deleteForeignKeys": [],
"warnings": []
}
},
{
"__className__": "serverpod.DatabaseMigrationAction",
"type": "alterTable",
"alterTable": {
"__className__": "serverpod.TableMigration",
"name": "dictionary_source",
"schema": "public",
"addColumns": [
{
"__className__": "serverpod.ColumnDefinition",
"name": "kind",
"fieldName": "kind",
"columnType": 0,
"isNullable": false,
"columnDefault": "'words'",
"dartType": "String"
}
],
"deleteColumns": [],
"modifyColumns": [],
"addIndexes": [],
"deleteIndexes": [],
"addForeignKeys": [],
"deleteForeignKeys": [],
"warnings": []
}
}
],
"warnings": [],
"migrationApiVersion": 1
}

View file

@ -0,0 +1,54 @@
BEGIN;
--
-- ACTION ALTER TABLE
--
DROP INDEX "dictionary_entry_rank_idx";
CREATE INDEX "dictionary_entry_rank_idx" ON "dictionary_entry" USING btree ("source", "glossLanguage", "rank");
--
-- ACTION ALTER TABLE
--
ALTER TABLE "dictionary_source" ADD COLUMN "kind" text NOT NULL DEFAULT 'words'::text;
--
-- MIGRATION VERSION FOR halkyon_learn
--
INSERT INTO "serverpod_migrations" ("module", "version", "timestamp")
VALUES ('halkyon_learn', '20261009123850448-dictionary-kind', now())
ON CONFLICT ("module")
DO UPDATE SET "version" = '20261009123850448-dictionary-kind', "timestamp" = now();
--
-- MIGRATION VERSION FOR serverpod
--
INSERT INTO "serverpod_migrations" ("module", "version", "timestamp")
VALUES ('serverpod', '20260824182259319', now())
ON CONFLICT ("module")
DO UPDATE SET "version" = '20260824182259319', "timestamp" = now();
--
-- MIGRATION VERSION FOR serverpod_auth_idp
--
INSERT INTO "serverpod_migrations" ("module", "version", "timestamp")
VALUES ('serverpod_auth_idp', '20260924105404509', now())
ON CONFLICT ("module")
DO UPDATE SET "version" = '20260924105404509', "timestamp" = now();
--
-- MIGRATION VERSION FOR serverpod_offline_sync
--
INSERT INTO "serverpod_migrations" ("module", "version", "timestamp")
VALUES ('serverpod_offline_sync', '20260914143806119', now())
ON CONFLICT ("module")
DO UPDATE SET "version" = '20260914143806119', "timestamp" = now();
--
-- MIGRATION VERSION FOR serverpod_auth_core
--
INSERT INTO "serverpod_migrations" ("module", "version", "timestamp")
VALUES ('serverpod_auth_core', '20260924105232991', now())
ON CONFLICT ("module")
DO UPDATE SET "version" = '20260924105232991', "timestamp" = now();
COMMIT;

View file

@ -16,3 +16,4 @@
20261007205440868-card-buried-at
20261008165051407-pdf-position
20261008223835206-dictionaries
20261009123850448-dictionary-kind

View file

@ -9,11 +9,17 @@ import 'package:test/test.dart';
import '../test_tools/serverpod_test_tools.dart';
/// The header of a dictionary file of JMdict at [version].
Map<String, Object?> header({String version = '3.6.2 (2026-10-05)'}) => {
/// The header of a dictionary file of JMdict at [version], or of another
/// [source] of another [kind].
Map<String, Object?> header({
String version = '3.6.2 (2026-10-05)',
String source = 'jmdict',
String? kind = 'words',
}) => {
'format': 'halkyon-dictionary',
'formatVersion': 1,
'source': 'jmdict',
'source': source,
'kind': ?kind,
'sourceVersion': version,
'language': 'ja',
'glossLanguage': 'en',
@ -91,6 +97,14 @@ final sushi = entry(
rank: 600000,
keys: ['寿司', 'すし', 'sushi'],
);
// A kanji, as KANJIDIC2's file gives it.
final shoku = entry(
'食',
'食',
labels: ['jlpt:n5', 'grade:2'],
rank: 10328,
keys: ['食', 'ショク', 'しょく', 'shoku', 'たべる', 'taberu', 'eat'],
);
// Ranks first, but "eat" only starts one of its keys.
final kuidaore = entry(
'9000001',
@ -149,7 +163,7 @@ void main() {
Future<List<String>> search(String query) async => [
for (final hit in await endpoints.dictionary.search(
signedIn,
'ja',
'jmdict',
'en',
query,
))
@ -159,7 +173,7 @@ void main() {
Future<List<String>> searchFirst(String query, int limit) async => [
for (final hit in await endpoints.dictionary.search(
signedIn,
'ja',
'jmdict',
'en',
query,
limit: limit,
@ -187,6 +201,93 @@ void main() {
expect(await search('taberu'), ['食べる']);
});
test('a dictionary says what it holds; a file from before kinds holds '
'words', () async {
await loadDictionaryFile(
session,
write([taberu], head: header(kind: null)),
);
await loadDictionaryFile(
session,
write(
[shoku],
name: 'kanjidic-en.jsonl',
head: header(source: 'kanjidic', kind: 'characters'),
),
);
expect(
{
for (final source in await endpoints.dictionary.dictionaries(
signedIn,
))
source.source: source.kind,
},
{'jmdict': 'words', 'kanjidic': 'characters'},
);
});
test("a dictionary's search and labels never reach another's entries "
'of the same language', () async {
await loadDictionaryFile(session, write([taberu]));
await loadDictionaryFile(
session,
write(
[shoku],
name: 'kanjidic-en.jsonl',
head: header(source: 'kanjidic', kind: 'characters'),
),
);
expect(await search('taberu'), ['食べる']);
expect(
[
for (final hit in await endpoints.dictionary.search(
signedIn,
'kanjidic',
'en',
'taberu',
))
hit.headword,
],
['食'],
);
expect(
await endpoints.dictionary.countLabelled(
signedIn,
'kanjidic',
'en',
'jlpt:n5',
),
1,
);
expect(
[
for (final hit in await endpoints.dictionary.labelled(
signedIn,
'jmdict',
'en',
'jlpt:n5',
))
hit.headword,
],
['食べる'],
);
});
test("a dictionary the instance doesn't hold finds nothing", () async {
await loadDictionaryFile(session, write([taberu]));
expect(
await endpoints.dictionary.search(signedIn, 'jmdict', 'de', 'taberu'),
isEmpty,
);
expect(
await endpoints.dictionary.search(signedIn, 'other', 'en', 'taberu'),
isEmpty,
);
});
test('the folder loads every file it can read, gzipped too, and '
"leaves out one it can't", () async {
write([taberu], name: 'jmdict-en.jsonl.gz');
@ -335,7 +436,7 @@ void main() {
for (var offset = 0; offset < 40; offset += 7)
for (final hit in await endpoints.dictionary.labelled(
signedIn,
'ja',
'jmdict',
'en',
'common',
offset: offset,
@ -354,7 +455,7 @@ void main() {
expect(
await endpoints.dictionary.countLabelled(
signedIn,
'ja',
'jmdict',
'en',
'jlpt:n5',
),
@ -365,7 +466,7 @@ void main() {
[
for (final hit in await endpoints.dictionary.labelled(
signedIn,
'ja',
'jmdict',
'en',
'jlpt:n5',
offset: offset,
@ -387,7 +488,7 @@ void main() {
final hit = (await endpoints.dictionary.search(
signedIn,
'ja',
'jmdict',
'en',
'taberu',
)).single;
@ -400,7 +501,12 @@ void main() {
test('the dictionaries are for those signed in', () {
expect(
() => endpoints.dictionary.search(sessionBuilder, 'ja', 'en', 'x'),
() => endpoints.dictionary.search(
sessionBuilder,
'jmdict',
'en',
'x',
),
throwsA(isA<ServerpodUnauthenticatedException>()),
);
});

View file

@ -659,7 +659,7 @@ class _DictionaryEndpoint {
_ida.Future<List<_ism30i9n.DictionaryHit>> search(
_ist.TestSessionBuilder sessionBuilder,
String language,
String source,
String glossLanguage,
String query, {
int? limit,
@ -676,7 +676,7 @@ class _DictionaryEndpoint {
endpointPath: 'dictionary',
methodName: 'search',
parameters: _ist.testObjectToJson({
'language': language,
'source': source,
'glossLanguage': glossLanguage,
'query': query,
'limit': limit,
@ -698,7 +698,7 @@ class _DictionaryEndpoint {
_ida.Future<int> countLabelled(
_ist.TestSessionBuilder sessionBuilder,
String language,
String source,
String glossLanguage,
String label,
) async {
@ -714,7 +714,7 @@ class _DictionaryEndpoint {
endpointPath: 'dictionary',
methodName: 'countLabelled',
parameters: _ist.testObjectToJson({
'language': language,
'source': source,
'glossLanguage': glossLanguage,
'label': label,
}),
@ -735,7 +735,7 @@ class _DictionaryEndpoint {
_ida.Future<List<_ism30i9n.DictionaryHit>> labelled(
_ist.TestSessionBuilder sessionBuilder,
String language,
String source,
String glossLanguage,
String label, {
int? offset,
@ -753,7 +753,7 @@ class _DictionaryEndpoint {
endpointPath: 'dictionary',
methodName: 'labelled',
parameters: _ist.testObjectToJson({
'language': language,
'source': source,
'glossLanguage': glossLanguage,
'label': label,
'offset': offset,