feat(server): dictionaries, loaded from files and searched
The server now holds dictionaries for everyone: `dictionary_entry` and `dictionary_source`, reference data owned by no one, never synced. It loads the dictionary files in its `dictionaries` folder at every start, those whose SHA-256 changed: in the background, each source replaced in one transaction, so searches meanwhile find the old entries; a file it can't read is logged and left out. All of JMdict loads in under half a minute. On an instance the folder is deploy/dictionaries, mounted read-only (compose.yaml); the server's README says how. DictionaryEndpoint (signed in): the dictionaries with their credits; a search (exact key, exact loose key, key starting with the query, loose key starting with it, each in the source's order; katakana as hiragana, macrons written out), some 150-450 ms over all of JMdict; and the entries with a label, counted and a page at a time (all JLPT N5 words). The curator now keeps spellings that aren't quite right apart, as `looseKeys` (romaji with long vowels typed once): "sushi" finds 寿司 before 数詞 (すうし). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01MMm7usAo6q9ZZRoPkWjoxi Signed-off-by: Tizian 「ティツィアーン」 <raisondetredev@pm.me>
This commit is contained in:
parent
a875a91f40
commit
3013bc04b3
GPG key ID:
07F6FCF405488B64
5 changed files with 16048 additions and 73 deletions
File diff suppressed because it is too large
Load diff
File diff suppressed because it is too large
Load diff
File diff suppressed because it is too large
Load diff
|
|
@ -0,0 +1,294 @@
|
|||
{
|
||||
"__className__": "serverpod.DatabaseMigration",
|
||||
"actions": [
|
||||
{
|
||||
"__className__": "serverpod.DatabaseMigrationAction",
|
||||
"type": "createTable",
|
||||
"createTable": {
|
||||
"__className__": "serverpod.TableDefinition",
|
||||
"name": "dictionary_entry",
|
||||
"dartName": "DictionaryEntry",
|
||||
"module": "halkyon_learn",
|
||||
"schema": "public",
|
||||
"columns": [
|
||||
{
|
||||
"__className__": "serverpod.ColumnDefinition",
|
||||
"name": "id",
|
||||
"fieldName": "id",
|
||||
"columnType": 6,
|
||||
"isNullable": false,
|
||||
"columnDefault": "serial",
|
||||
"dartType": "int?"
|
||||
},
|
||||
{
|
||||
"__className__": "serverpod.ColumnDefinition",
|
||||
"name": "source",
|
||||
"fieldName": "source",
|
||||
"columnType": 0,
|
||||
"isNullable": false,
|
||||
"dartType": "String"
|
||||
},
|
||||
{
|
||||
"__className__": "serverpod.ColumnDefinition",
|
||||
"name": "sourceId",
|
||||
"fieldName": "sourceId",
|
||||
"columnType": 0,
|
||||
"isNullable": false,
|
||||
"dartType": "String"
|
||||
},
|
||||
{
|
||||
"__className__": "serverpod.ColumnDefinition",
|
||||
"name": "language",
|
||||
"fieldName": "language",
|
||||
"columnType": 0,
|
||||
"isNullable": false,
|
||||
"dartType": "String"
|
||||
},
|
||||
{
|
||||
"__className__": "serverpod.ColumnDefinition",
|
||||
"name": "glossLanguage",
|
||||
"fieldName": "glossLanguage",
|
||||
"columnType": 0,
|
||||
"isNullable": false,
|
||||
"dartType": "String"
|
||||
},
|
||||
{
|
||||
"__className__": "serverpod.ColumnDefinition",
|
||||
"name": "headword",
|
||||
"fieldName": "headword",
|
||||
"columnType": 0,
|
||||
"isNullable": false,
|
||||
"dartType": "String"
|
||||
},
|
||||
{
|
||||
"__className__": "serverpod.ColumnDefinition",
|
||||
"name": "reading",
|
||||
"fieldName": "reading",
|
||||
"columnType": 0,
|
||||
"isNullable": true,
|
||||
"dartType": "String?"
|
||||
},
|
||||
{
|
||||
"__className__": "serverpod.ColumnDefinition",
|
||||
"name": "rank",
|
||||
"fieldName": "rank",
|
||||
"columnType": 6,
|
||||
"isNullable": false,
|
||||
"dartType": "int"
|
||||
},
|
||||
{
|
||||
"__className__": "serverpod.ColumnDefinition",
|
||||
"name": "labels",
|
||||
"fieldName": "labels",
|
||||
"columnType": 0,
|
||||
"isNullable": false,
|
||||
"dartType": "String"
|
||||
},
|
||||
{
|
||||
"__className__": "serverpod.ColumnDefinition",
|
||||
"name": "searchKeys",
|
||||
"fieldName": "searchKeys",
|
||||
"columnType": 0,
|
||||
"isNullable": false,
|
||||
"dartType": "String"
|
||||
},
|
||||
{
|
||||
"__className__": "serverpod.ColumnDefinition",
|
||||
"name": "looseKeys",
|
||||
"fieldName": "looseKeys",
|
||||
"columnType": 0,
|
||||
"isNullable": false,
|
||||
"dartType": "String"
|
||||
},
|
||||
{
|
||||
"__className__": "serverpod.ColumnDefinition",
|
||||
"name": "entry",
|
||||
"fieldName": "entry",
|
||||
"columnType": 0,
|
||||
"isNullable": false,
|
||||
"dartType": "String"
|
||||
}
|
||||
],
|
||||
"foreignKeys": [],
|
||||
"indexes": [
|
||||
{
|
||||
"__className__": "serverpod.IndexDefinition",
|
||||
"indexName": "dictionary_entry_source_idx",
|
||||
"elements": [
|
||||
{
|
||||
"__className__": "serverpod.IndexElementDefinition",
|
||||
"type": 0,
|
||||
"definition": "source"
|
||||
},
|
||||
{
|
||||
"__className__": "serverpod.IndexElementDefinition",
|
||||
"type": 0,
|
||||
"definition": "glossLanguage"
|
||||
},
|
||||
{
|
||||
"__className__": "serverpod.IndexElementDefinition",
|
||||
"type": 0,
|
||||
"definition": "sourceId"
|
||||
}
|
||||
],
|
||||
"type": "btree",
|
||||
"isUnique": true,
|
||||
"isPrimary": false
|
||||
},
|
||||
{
|
||||
"__className__": "serverpod.IndexDefinition",
|
||||
"indexName": "dictionary_entry_rank_idx",
|
||||
"elements": [
|
||||
{
|
||||
"__className__": "serverpod.IndexElementDefinition",
|
||||
"type": 0,
|
||||
"definition": "language"
|
||||
},
|
||||
{
|
||||
"__className__": "serverpod.IndexElementDefinition",
|
||||
"type": 0,
|
||||
"definition": "glossLanguage"
|
||||
},
|
||||
{
|
||||
"__className__": "serverpod.IndexElementDefinition",
|
||||
"type": 0,
|
||||
"definition": "rank"
|
||||
}
|
||||
],
|
||||
"type": "btree",
|
||||
"isUnique": false,
|
||||
"isPrimary": false
|
||||
}
|
||||
],
|
||||
"managed": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"__className__": "serverpod.DatabaseMigrationAction",
|
||||
"type": "createTable",
|
||||
"createTable": {
|
||||
"__className__": "serverpod.TableDefinition",
|
||||
"name": "dictionary_source",
|
||||
"dartName": "DictionarySource",
|
||||
"module": "halkyon_learn",
|
||||
"schema": "public",
|
||||
"columns": [
|
||||
{
|
||||
"__className__": "serverpod.ColumnDefinition",
|
||||
"name": "id",
|
||||
"fieldName": "id",
|
||||
"columnType": 6,
|
||||
"isNullable": false,
|
||||
"columnDefault": "serial",
|
||||
"dartType": "int?"
|
||||
},
|
||||
{
|
||||
"__className__": "serverpod.ColumnDefinition",
|
||||
"name": "source",
|
||||
"fieldName": "source",
|
||||
"columnType": 0,
|
||||
"isNullable": false,
|
||||
"dartType": "String"
|
||||
},
|
||||
{
|
||||
"__className__": "serverpod.ColumnDefinition",
|
||||
"name": "sourceVersion",
|
||||
"fieldName": "sourceVersion",
|
||||
"columnType": 0,
|
||||
"isNullable": false,
|
||||
"dartType": "String"
|
||||
},
|
||||
{
|
||||
"__className__": "serverpod.ColumnDefinition",
|
||||
"name": "language",
|
||||
"fieldName": "language",
|
||||
"columnType": 0,
|
||||
"isNullable": false,
|
||||
"dartType": "String"
|
||||
},
|
||||
{
|
||||
"__className__": "serverpod.ColumnDefinition",
|
||||
"name": "glossLanguage",
|
||||
"fieldName": "glossLanguage",
|
||||
"columnType": 0,
|
||||
"isNullable": false,
|
||||
"dartType": "String"
|
||||
},
|
||||
{
|
||||
"__className__": "serverpod.ColumnDefinition",
|
||||
"name": "license",
|
||||
"fieldName": "license",
|
||||
"columnType": 0,
|
||||
"isNullable": false,
|
||||
"dartType": "String"
|
||||
},
|
||||
{
|
||||
"__className__": "serverpod.ColumnDefinition",
|
||||
"name": "credits",
|
||||
"fieldName": "credits",
|
||||
"columnType": 8,
|
||||
"isNullable": false,
|
||||
"dartType": "List<protocol:DictionaryCredit>"
|
||||
},
|
||||
{
|
||||
"__className__": "serverpod.ColumnDefinition",
|
||||
"name": "entryCount",
|
||||
"fieldName": "entryCount",
|
||||
"columnType": 6,
|
||||
"isNullable": false,
|
||||
"dartType": "int"
|
||||
},
|
||||
{
|
||||
"__className__": "serverpod.ColumnDefinition",
|
||||
"name": "fileName",
|
||||
"fieldName": "fileName",
|
||||
"columnType": 0,
|
||||
"isNullable": false,
|
||||
"dartType": "String"
|
||||
},
|
||||
{
|
||||
"__className__": "serverpod.ColumnDefinition",
|
||||
"name": "fileHash",
|
||||
"fieldName": "fileHash",
|
||||
"columnType": 0,
|
||||
"isNullable": false,
|
||||
"dartType": "String"
|
||||
},
|
||||
{
|
||||
"__className__": "serverpod.ColumnDefinition",
|
||||
"name": "loadedAt",
|
||||
"fieldName": "loadedAt",
|
||||
"columnType": 4,
|
||||
"isNullable": false,
|
||||
"dartType": "DateTime"
|
||||
}
|
||||
],
|
||||
"foreignKeys": [],
|
||||
"indexes": [
|
||||
{
|
||||
"__className__": "serverpod.IndexDefinition",
|
||||
"indexName": "dictionary_source_idx",
|
||||
"elements": [
|
||||
{
|
||||
"__className__": "serverpod.IndexElementDefinition",
|
||||
"type": 0,
|
||||
"definition": "source"
|
||||
},
|
||||
{
|
||||
"__className__": "serverpod.IndexElementDefinition",
|
||||
"type": 0,
|
||||
"definition": "glossLanguage"
|
||||
}
|
||||
],
|
||||
"type": "btree",
|
||||
"isUnique": true,
|
||||
"isPrimary": false
|
||||
}
|
||||
],
|
||||
"managed": true
|
||||
}
|
||||
}
|
||||
],
|
||||
"warnings": [],
|
||||
"migrationApiVersion": 1
|
||||
}
|
||||
|
|
@ -0,0 +1,87 @@
|
|||
BEGIN;
|
||||
|
||||
--
|
||||
-- ACTION CREATE TABLE
|
||||
--
|
||||
CREATE TABLE "dictionary_entry" (
|
||||
"id" bigserial PRIMARY KEY,
|
||||
"source" text NOT NULL,
|
||||
"sourceId" text NOT NULL,
|
||||
"language" text NOT NULL,
|
||||
"glossLanguage" text NOT NULL,
|
||||
"headword" text NOT NULL,
|
||||
"reading" text,
|
||||
"rank" bigint NOT NULL,
|
||||
"labels" text NOT NULL,
|
||||
"searchKeys" text NOT NULL,
|
||||
"looseKeys" text NOT NULL,
|
||||
"entry" text NOT NULL
|
||||
);
|
||||
|
||||
-- Indexes
|
||||
CREATE UNIQUE INDEX "dictionary_entry_source_idx" ON "dictionary_entry" USING btree ("source", "glossLanguage", "sourceId");
|
||||
CREATE INDEX "dictionary_entry_rank_idx" ON "dictionary_entry" USING btree ("language", "glossLanguage", "rank");
|
||||
|
||||
--
|
||||
-- ACTION CREATE TABLE
|
||||
--
|
||||
CREATE TABLE "dictionary_source" (
|
||||
"id" bigserial PRIMARY KEY,
|
||||
"source" text NOT NULL,
|
||||
"sourceVersion" text NOT NULL,
|
||||
"language" text NOT NULL,
|
||||
"glossLanguage" text NOT NULL,
|
||||
"license" text NOT NULL,
|
||||
"credits" json NOT NULL,
|
||||
"entryCount" bigint NOT NULL,
|
||||
"fileName" text NOT NULL,
|
||||
"fileHash" text NOT NULL,
|
||||
"loadedAt" timestamp without time zone NOT NULL
|
||||
);
|
||||
|
||||
-- Indexes
|
||||
CREATE UNIQUE INDEX "dictionary_source_idx" ON "dictionary_source" USING btree ("source", "glossLanguage");
|
||||
|
||||
|
||||
--
|
||||
-- MIGRATION VERSION FOR halkyon_learn
|
||||
--
|
||||
INSERT INTO "serverpod_migrations" ("module", "version", "timestamp")
|
||||
VALUES ('halkyon_learn', '20261008223835206-dictionaries', now())
|
||||
ON CONFLICT ("module")
|
||||
DO UPDATE SET "version" = '20261008223835206-dictionaries', "timestamp" = now();
|
||||
|
||||
--
|
||||
-- MIGRATION VERSION FOR serverpod
|
||||
--
|
||||
INSERT INTO "serverpod_migrations" ("module", "version", "timestamp")
|
||||
VALUES ('serverpod', '20260824182259319', now())
|
||||
ON CONFLICT ("module")
|
||||
DO UPDATE SET "version" = '20260824182259319', "timestamp" = now();
|
||||
|
||||
--
|
||||
-- MIGRATION VERSION FOR serverpod_auth_idp
|
||||
--
|
||||
INSERT INTO "serverpod_migrations" ("module", "version", "timestamp")
|
||||
VALUES ('serverpod_auth_idp', '20260924105404509', now())
|
||||
ON CONFLICT ("module")
|
||||
DO UPDATE SET "version" = '20260924105404509', "timestamp" = now();
|
||||
|
||||
--
|
||||
-- MIGRATION VERSION FOR serverpod_offline_sync
|
||||
--
|
||||
INSERT INTO "serverpod_migrations" ("module", "version", "timestamp")
|
||||
VALUES ('serverpod_offline_sync', '20260914143806119', now())
|
||||
ON CONFLICT ("module")
|
||||
DO UPDATE SET "version" = '20260914143806119', "timestamp" = now();
|
||||
|
||||
--
|
||||
-- MIGRATION VERSION FOR serverpod_auth_core
|
||||
--
|
||||
INSERT INTO "serverpod_migrations" ("module", "version", "timestamp")
|
||||
VALUES ('serverpod_auth_core', '20260924105232991', now())
|
||||
ON CONFLICT ("module")
|
||||
DO UPDATE SET "version" = '20260924105232991', "timestamp" = now();
|
||||
|
||||
|
||||
COMMIT;
|
||||
Loading…
Reference in a new issue