halkyon-days/halkyon
Watch
1
0
Fork
You've already forked halkyon
 0
Code Issues Pull requests Projects Releases 37 Packages Wiki Activity Actions

perf(dictionary): a search finds keys by their first characters, indexed

Browse source 
A search matched LIKE '%|eat%' over every entry's barred keys, which no
index can answer, so each one read the whole dictionary: 43-57 ms in
JMdict's 218,863 entries. dictionary_search_key holds a row per key and
loose key with its first one, two and three characters in columns of
their own and one index on them, so the keys a query starts are an
equality the index answers whatever the database's collation (Serverpod's
models can't declare COLLATE "C" or text_pattern_ops). A search now takes
5-26 ms, at 1.4 million rows and 190 MB for JMdict, and its load takes
72 s instead of 33 s.

The loader fills the table beside the entries and loads a dictionary
again when it has none. Until then, in the minutes after the deploying
server starts, the search reads the barred keys as before; those columns
stay until a later release, since the server before a deploy's switch
still searches them. The migration only creates the table and its index.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Signed-off-by: Tizian 「ティツィアーン」 <raisondetredev@pm.me>
This commit is contained in:
Tizian 「ティツィアーン」 2026-10-17 13:22:24 +02:00
commit 8e3a4e56dc
Signed by: raisondetredev
GPG key ID: 07F6FCF405488B64
5 changed files with 17709 additions and 19 deletions
Download patch file Download diff file Expand all files Collapse all files

9082
halkyon_learn_server/migrations/20261017110130653/definition.json Normal file
View file

File diff suppressed because it is too large Load diff

2191
halkyon_learn_server/migrations/20261017110130653/definition.sql Normal file
View file

File diff suppressed because it is too large Load diff

4817
halkyon_learn_server/migrations/20261017110130653/definition_project.json Normal file
View file

File diff suppressed because it is too large Load diff

139
halkyon_learn_server/migrations/20261017110130653/migration.json Normal file
View file

@ -0,0 +1,139 @@
{
"__className__": "serverpod.DatabaseMigration",
"actions": [
{
"__className__": "serverpod.DatabaseMigrationAction",
"type": "createTable",
"createTable": {
"__className__": "serverpod.TableDefinition",
"name": "dictionary_search_key",
"dartName": "DictionarySearchKey",
"module": "halkyon_learn",
"schema": "public",
"columns": [
{
"__className__": "serverpod.ColumnDefinition",
"name": "id",
"fieldName": "id",
"columnType": 6,
"isNullable": false,
"columnDefault": "serial",
"dartType": "int?"
},
{
"__className__": "serverpod.ColumnDefinition",
"name": "source",
"fieldName": "source",
"columnType": 0,
"isNullable": false,
"dartType": "String"
},
{
"__className__": "serverpod.ColumnDefinition",
"name": "glossLanguage",
"fieldName": "glossLanguage",
"columnType": 0,
"isNullable": false,
"dartType": "String"
},
{
"__className__": "serverpod.ColumnDefinition",
"name": "key",
"fieldName": "key",
"columnType": 0,
"isNullable": false,
"dartType": "String"
},
{
"__className__": "serverpod.ColumnDefinition",
"name": "loose",
"fieldName": "loose",
"columnType": 1,
"isNullable": false,
"dartType": "bool"
},
{
"__className__": "serverpod.ColumnDefinition",
"name": "start1",
"fieldName": "start1",
"columnType": 0,
"isNullable": false,
"dartType": "String"
},
{
"__className__": "serverpod.ColumnDefinition",
"name": "start2",
"fieldName": "start2",
"columnType": 0,
"isNullable": false,
"dartType": "String"
},
{
"__className__": "serverpod.ColumnDefinition",
"name": "start3",
"fieldName": "start3",
"columnType": 0,
"isNullable": false,
"dartType": "String"
},
{
"__className__": "serverpod.ColumnDefinition",
"name": "sourceId",
"fieldName": "sourceId",
"columnType": 0,
"isNullable": false,
"dartType": "String"
},
{
"__className__": "serverpod.ColumnDefinition",
"name": "rank",
"fieldName": "rank",
"columnType": 6,
"isNullable": false,
"dartType": "int"
}
],
"foreignKeys": [],
"indexes": [
{
"__className__": "serverpod.IndexDefinition",
"indexName": "dictionary_search_key_start_idx",
"elements": [
{
"__className__": "serverpod.IndexElementDefinition",
"type": 0,
"definition": "source"
},
{
"__className__": "serverpod.IndexElementDefinition",
"type": 0,
"definition": "glossLanguage"
},
{
"__className__": "serverpod.IndexElementDefinition",
"type": 0,
"definition": "start1"
},
{
"__className__": "serverpod.IndexElementDefinition",
"type": 0,
"definition": "start2"
},
{
"__className__": "serverpod.IndexElementDefinition",
"type": 0,
"definition": "start3"
}
],
"type": "btree",
"isUnique": false,
"isPrimary": false
}
],
"managed": true
}
}
],
"warnings": [],
"migrationApiVersion": 1
}

64
halkyon_learn_server/migrations/20261017110130653/migration.sql Normal file
View file

@ -0,0 +1,64 @@
BEGIN;
--
-- ACTION CREATE TABLE
--
CREATE TABLE "dictionary_search_key" (
"id" bigserial PRIMARY KEY,
"source" text NOT NULL,
"glossLanguage" text NOT NULL,
"key" text NOT NULL,
"loose" boolean NOT NULL,
"start1" text NOT NULL,
"start2" text NOT NULL,
"start3" text NOT NULL,
"sourceId" text NOT NULL,
"rank" bigint NOT NULL
);
-- Indexes
CREATE INDEX "dictionary_search_key_start_idx" ON "dictionary_search_key" USING btree ("source", "glossLanguage", "start1", "start2", "start3");
--
-- MIGRATION VERSION FOR halkyon_learn
--
INSERT INTO "serverpod_migrations" ("module", "version", "timestamp")
VALUES ('halkyon_learn', '20261017110130653', now())
ON CONFLICT ("module")
DO UPDATE SET "version" = '20261017110130653', "timestamp" = now();
--
-- MIGRATION VERSION FOR serverpod
--
INSERT INTO "serverpod_migrations" ("module", "version", "timestamp")
VALUES ('serverpod', '20260824182259319', now())
ON CONFLICT ("module")
DO UPDATE SET "version" = '20260824182259319', "timestamp" = now();
--
-- MIGRATION VERSION FOR serverpod_auth_idp
--
INSERT INTO "serverpod_migrations" ("module", "version", "timestamp")
VALUES ('serverpod_auth_idp', '20260924105404509', now())
ON CONFLICT ("module")
DO UPDATE SET "version" = '20260924105404509', "timestamp" = now();
--
-- MIGRATION VERSION FOR serverpod_offline_sync
--
INSERT INTO "serverpod_migrations" ("module", "version", "timestamp")
VALUES ('serverpod_offline_sync', '20260914143806119', now())
ON CONFLICT ("module")
DO UPDATE SET "version" = '20260914143806119', "timestamp" = now();
--
-- MIGRATION VERSION FOR serverpod_auth_core
--
INSERT INTO "serverpod_migrations" ("module", "version", "timestamp")
VALUES ('serverpod_auth_core', '20260924105232991', now())
ON CONFLICT ("module")
DO UPDATE SET "version" = '20260924105232991', "timestamp" = now();
COMMIT;