halkyon-days/halkyon
Watch
1
0
Fork
You've already forked halkyon
 0
Code Issues Pull requests Projects Releases 37 Packages Wiki Activity Actions

feat(server): dictionaries, loaded from files and searched

Browse source 
The server now holds dictionaries for everyone: `dictionary_entry` and
`dictionary_source`, reference data owned by no one, never synced. It
loads the dictionary files in its `dictionaries` folder at every start,
those whose SHA-256 changed: in the background, each source replaced in
one transaction, so searches meanwhile find the old entries; a file it
can't read is logged and left out. All of JMdict loads in under half a
minute. On an instance the folder is deploy/dictionaries, mounted
read-only (compose.yaml); the server's README says how.

DictionaryEndpoint (signed in): the dictionaries with their credits; a
search (exact key, exact loose key, key starting with the query, loose
key starting with it, each in the source's order; katakana as hiragana,
macrons written out), some 150-450 ms over all of JMdict; and the
entries with a label, counted and a page at a time (all JLPT N5 words).

The curator now keeps spellings that aren't quite right apart, as
`looseKeys` (romaji with long vowels typed once): "sushi" finds 寿司
before 数詞 (すうし).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01MMm7usAo6q9ZZRoPkWjoxi
Signed-off-by: Tizian 「ティツィアーン」 <raisondetredev@pm.me>
This commit is contained in:
Tizian 「ティツィアーン」 2026-10-09 00:51:27 +02:00
commit 3013bc04b3
Signed by: raisondetredev
GPG key ID: 07F6FCF405488B64
6 changed files with 16048 additions and 73 deletions
Download patch file Download diff file Expand all files Collapse all files

6909
halkyon_learn_server/migrations/20261008223835206-dictionaries/definition.json Normal file
View file

File diff suppressed because it is too large Load diff

1740
halkyon_learn_server/migrations/20261008223835206-dictionaries/definition.sql Normal file
View file

File diff suppressed because it is too large Load diff

2644
halkyon_learn_server/migrations/20261008223835206-dictionaries/definition_project.json Normal file
View file

File diff suppressed because it is too large Load diff

294
halkyon_learn_server/migrations/20261008223835206-dictionaries/migration.json Normal file
View file

@ -0,0 +1,294 @@
{
"__className__": "serverpod.DatabaseMigration",
"actions": [
{
"__className__": "serverpod.DatabaseMigrationAction",
"type": "createTable",
"createTable": {
"__className__": "serverpod.TableDefinition",
"name": "dictionary_entry",
"dartName": "DictionaryEntry",
"module": "halkyon_learn",
"schema": "public",
"columns": [
{
"__className__": "serverpod.ColumnDefinition",
"name": "id",
"fieldName": "id",
"columnType": 6,
"isNullable": false,
"columnDefault": "serial",
"dartType": "int?"
},
{
"__className__": "serverpod.ColumnDefinition",
"name": "source",
"fieldName": "source",
"columnType": 0,
"isNullable": false,
"dartType": "String"
},
{
"__className__": "serverpod.ColumnDefinition",
"name": "sourceId",
"fieldName": "sourceId",
"columnType": 0,
"isNullable": false,
"dartType": "String"
},
{
"__className__": "serverpod.ColumnDefinition",
"name": "language",
"fieldName": "language",
"columnType": 0,
"isNullable": false,
"dartType": "String"
},
{
"__className__": "serverpod.ColumnDefinition",
"name": "glossLanguage",
"fieldName": "glossLanguage",
"columnType": 0,
"isNullable": false,
"dartType": "String"
},
{
"__className__": "serverpod.ColumnDefinition",
"name": "headword",
"fieldName": "headword",
"columnType": 0,
"isNullable": false,
"dartType": "String"
},
{
"__className__": "serverpod.ColumnDefinition",
"name": "reading",
"fieldName": "reading",
"columnType": 0,
"isNullable": true,
"dartType": "String?"
},
{
"__className__": "serverpod.ColumnDefinition",
"name": "rank",
"fieldName": "rank",
"columnType": 6,
"isNullable": false,
"dartType": "int"
},
{
"__className__": "serverpod.ColumnDefinition",
"name": "labels",
"fieldName": "labels",
"columnType": 0,
"isNullable": false,
"dartType": "String"
},
{
"__className__": "serverpod.ColumnDefinition",
"name": "searchKeys",
"fieldName": "searchKeys",
"columnType": 0,
"isNullable": false,
"dartType": "String"
},
{
"__className__": "serverpod.ColumnDefinition",
"name": "looseKeys",
"fieldName": "looseKeys",
"columnType": 0,
"isNullable": false,
"dartType": "String"
},
{
"__className__": "serverpod.ColumnDefinition",
"name": "entry",
"fieldName": "entry",
"columnType": 0,
"isNullable": false,
"dartType": "String"
}
],
"foreignKeys": [],
"indexes": [
{
"__className__": "serverpod.IndexDefinition",
"indexName": "dictionary_entry_source_idx",
"elements": [
{
"__className__": "serverpod.IndexElementDefinition",
"type": 0,
"definition": "source"
},
{
"__className__": "serverpod.IndexElementDefinition",
"type": 0,
"definition": "glossLanguage"
},
{
"__className__": "serverpod.IndexElementDefinition",
"type": 0,
"definition": "sourceId"
}
],
"type": "btree",
"isUnique": true,
"isPrimary": false
},
{
"__className__": "serverpod.IndexDefinition",
"indexName": "dictionary_entry_rank_idx",
"elements": [
{
"__className__": "serverpod.IndexElementDefinition",
"type": 0,
"definition": "language"
},
{
"__className__": "serverpod.IndexElementDefinition",
"type": 0,
"definition": "glossLanguage"
},
{
"__className__": "serverpod.IndexElementDefinition",
"type": 0,
"definition": "rank"
}
],
"type": "btree",
"isUnique": false,
"isPrimary": false
}
],
"managed": true
}
},
{
"__className__": "serverpod.DatabaseMigrationAction",
"type": "createTable",
"createTable": {
"__className__": "serverpod.TableDefinition",
"name": "dictionary_source",
"dartName": "DictionarySource",
"module": "halkyon_learn",
"schema": "public",
"columns": [
{
"__className__": "serverpod.ColumnDefinition",
"name": "id",
"fieldName": "id",
"columnType": 6,
"isNullable": false,
"columnDefault": "serial",
"dartType": "int?"
},
{
"__className__": "serverpod.ColumnDefinition",
"name": "source",
"fieldName": "source",
"columnType": 0,
"isNullable": false,
"dartType": "String"
},
{
"__className__": "serverpod.ColumnDefinition",
"name": "sourceVersion",
"fieldName": "sourceVersion",
"columnType": 0,
"isNullable": false,
"dartType": "String"
},
{
"__className__": "serverpod.ColumnDefinition",
"name": "language",
"fieldName": "language",
"columnType": 0,
"isNullable": false,
"dartType": "String"
},
{
"__className__": "serverpod.ColumnDefinition",
"name": "glossLanguage",
"fieldName": "glossLanguage",
"columnType": 0,
"isNullable": false,
"dartType": "String"
},
{
"__className__": "serverpod.ColumnDefinition",
"name": "license",
"fieldName": "license",
"columnType": 0,
"isNullable": false,
"dartType": "String"
},
{
"__className__": "serverpod.ColumnDefinition",
"name": "credits",
"fieldName": "credits",
"columnType": 8,
"isNullable": false,
"dartType": "List<protocol:DictionaryCredit>"
},
{
"__className__": "serverpod.ColumnDefinition",
"name": "entryCount",
"fieldName": "entryCount",
"columnType": 6,
"isNullable": false,
"dartType": "int"
},
{
"__className__": "serverpod.ColumnDefinition",
"name": "fileName",
"fieldName": "fileName",
"columnType": 0,
"isNullable": false,
"dartType": "String"
},
{
"__className__": "serverpod.ColumnDefinition",
"name": "fileHash",
"fieldName": "fileHash",
"columnType": 0,
"isNullable": false,
"dartType": "String"
},
{
"__className__": "serverpod.ColumnDefinition",
"name": "loadedAt",
"fieldName": "loadedAt",
"columnType": 4,
"isNullable": false,
"dartType": "DateTime"
}
],
"foreignKeys": [],
"indexes": [
{
"__className__": "serverpod.IndexDefinition",
"indexName": "dictionary_source_idx",
"elements": [
{
"__className__": "serverpod.IndexElementDefinition",
"type": 0,
"definition": "source"
},
{
"__className__": "serverpod.IndexElementDefinition",
"type": 0,
"definition": "glossLanguage"
}
],
"type": "btree",
"isUnique": true,
"isPrimary": false
}
],
"managed": true
}
}
],
"warnings": [],
"migrationApiVersion": 1
}

87
halkyon_learn_server/migrations/20261008223835206-dictionaries/migration.sql Normal file
View file

@ -0,0 +1,87 @@
BEGIN;
--
-- ACTION CREATE TABLE
--
CREATE TABLE "dictionary_entry" (
"id" bigserial PRIMARY KEY,
"source" text NOT NULL,
"sourceId" text NOT NULL,
"language" text NOT NULL,
"glossLanguage" text NOT NULL,
"headword" text NOT NULL,
"reading" text,
"rank" bigint NOT NULL,
"labels" text NOT NULL,
"searchKeys" text NOT NULL,
"looseKeys" text NOT NULL,
"entry" text NOT NULL
);
-- Indexes
CREATE UNIQUE INDEX "dictionary_entry_source_idx" ON "dictionary_entry" USING btree ("source", "glossLanguage", "sourceId");
CREATE INDEX "dictionary_entry_rank_idx" ON "dictionary_entry" USING btree ("language", "glossLanguage", "rank");
--
-- ACTION CREATE TABLE
--
CREATE TABLE "dictionary_source" (
"id" bigserial PRIMARY KEY,
"source" text NOT NULL,
"sourceVersion" text NOT NULL,
"language" text NOT NULL,
"glossLanguage" text NOT NULL,
"license" text NOT NULL,
"credits" json NOT NULL,
"entryCount" bigint NOT NULL,
"fileName" text NOT NULL,
"fileHash" text NOT NULL,
"loadedAt" timestamp without time zone NOT NULL
);
-- Indexes
CREATE UNIQUE INDEX "dictionary_source_idx" ON "dictionary_source" USING btree ("source", "glossLanguage");
--
-- MIGRATION VERSION FOR halkyon_learn
--
INSERT INTO "serverpod_migrations" ("module", "version", "timestamp")
VALUES ('halkyon_learn', '20261008223835206-dictionaries', now())
ON CONFLICT ("module")
DO UPDATE SET "version" = '20261008223835206-dictionaries', "timestamp" = now();
--
-- MIGRATION VERSION FOR serverpod
--
INSERT INTO "serverpod_migrations" ("module", "version", "timestamp")
VALUES ('serverpod', '20260824182259319', now())
ON CONFLICT ("module")
DO UPDATE SET "version" = '20260824182259319', "timestamp" = now();
--
-- MIGRATION VERSION FOR serverpod_auth_idp
--
INSERT INTO "serverpod_migrations" ("module", "version", "timestamp")
VALUES ('serverpod_auth_idp', '20260924105404509', now())
ON CONFLICT ("module")
DO UPDATE SET "version" = '20260924105404509', "timestamp" = now();
--
-- MIGRATION VERSION FOR serverpod_offline_sync
--
INSERT INTO "serverpod_migrations" ("module", "version", "timestamp")
VALUES ('serverpod_offline_sync', '20260914143806119', now())
ON CONFLICT ("module")
DO UPDATE SET "version" = '20260914143806119', "timestamp" = now();
--
-- MIGRATION VERSION FOR serverpod_auth_core
--
INSERT INTO "serverpod_migrations" ("module", "version", "timestamp")
VALUES ('serverpod_auth_core', '20260924105232991', now())
ON CONFLICT ("module")
DO UPDATE SET "version" = '20260924105232991', "timestamp" = now();
COMMIT;

1
halkyon_learn_server/migrations/migration_registry.txt
View file

@ -15,3 +15,4 @@
20261003110855698-pdf-sources
20261007205440868-card-buried-at
20261008165051407-pdf-position
20261008223835206-dictionaries