halkyon-days/halkyon
Watch
1
0
Fork
You've already forked halkyon
 0
Code Issues Pull requests Projects Releases 40 Packages Wiki Activity Actions

feat(server): dictionaries, loaded from files and searched

Browse source 
The server now holds dictionaries for everyone: `dictionary_entry` and
`dictionary_source`, reference data owned by no one, never synced. It
loads the dictionary files in its `dictionaries` folder at every start,
those whose SHA-256 changed: in the background, each source replaced in
one transaction, so searches meanwhile find the old entries; a file it
can't read is logged and left out. All of JMdict loads in under half a
minute. On an instance the folder is deploy/dictionaries, mounted
read-only (compose.yaml); the server's README says how.

DictionaryEndpoint (signed in): the dictionaries with their credits; a
search (exact key, exact loose key, key starting with the query, loose
key starting with it, each in the source's order; katakana as hiragana,
macrons written out), some 150-450 ms over all of JMdict; and the
entries with a label, counted and a page at a time (all JLPT N5 words).

The curator now keeps spellings that aren't quite right apart, as
`looseKeys` (romaji with long vowels typed once): "sushi" finds 寿司
before 数詞 (すうし).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01MMm7usAo6q9ZZRoPkWjoxi
Signed-off-by: Tizian 「ティツィアーン」 <raisondetredev@pm.me>
This commit is contained in:
Tizian 「ティツィアーン」 2026-10-09 00:51:27 +02:00
commit 3013bc04b3
Signed by: raisondetredev
GPG key ID: 07F6FCF405488B64
1 changed files with 16048 additions and 73 deletions
Download patch file Download diff file Expand all files Collapse all files

294
halkyon_learn_server/migrations/20261008223835206-dictionaries/migration.json Normal file
View file

@ -0,0 +1,294 @@
{
"__className__": "serverpod.DatabaseMigration",
"actions": [
{
"__className__": "serverpod.DatabaseMigrationAction",
"type": "createTable",
"createTable": {
"__className__": "serverpod.TableDefinition",
"name": "dictionary_entry",
"dartName": "DictionaryEntry",
"module": "halkyon_learn",
"schema": "public",
"columns": [
{
"__className__": "serverpod.ColumnDefinition",
"name": "id",
"fieldName": "id",
"columnType": 6,
"isNullable": false,
"columnDefault": "serial",
"dartType": "int?"
},
{
"__className__": "serverpod.ColumnDefinition",
"name": "source",
"fieldName": "source",
"columnType": 0,
"isNullable": false,
"dartType": "String"
},
{
"__className__": "serverpod.ColumnDefinition",
"name": "sourceId",
"fieldName": "sourceId",
"columnType": 0,
"isNullable": false,
"dartType": "String"
},
{
"__className__": "serverpod.ColumnDefinition",
"name": "language",
"fieldName": "language",
"columnType": 0,
"isNullable": false,
"dartType": "String"
},
{
"__className__": "serverpod.ColumnDefinition",
"name": "glossLanguage",
"fieldName": "glossLanguage",
"columnType": 0,
"isNullable": false,
"dartType": "String"
},
{
"__className__": "serverpod.ColumnDefinition",
"name": "headword",
"fieldName": "headword",
"columnType": 0,
"isNullable": false,
"dartType": "String"
},
{
"__className__": "serverpod.ColumnDefinition",
"name": "reading",
"fieldName": "reading",
"columnType": 0,
"isNullable": true,
"dartType": "String?"
},
{
"__className__": "serverpod.ColumnDefinition",
"name": "rank",
"fieldName": "rank",
"columnType": 6,
"isNullable": false,
"dartType": "int"
},
{
"__className__": "serverpod.ColumnDefinition",
"name": "labels",
"fieldName": "labels",
"columnType": 0,
"isNullable": false,
"dartType": "String"
},
{
"__className__": "serverpod.ColumnDefinition",
"name": "searchKeys",
"fieldName": "searchKeys",
"columnType": 0,
"isNullable": false,
"dartType": "String"
},
{
"__className__": "serverpod.ColumnDefinition",
"name": "looseKeys",
"fieldName": "looseKeys",
"columnType": 0,
"isNullable": false,
"dartType": "String"
},
{
"__className__": "serverpod.ColumnDefinition",
"name": "entry",
"fieldName": "entry",
"columnType": 0,
"isNullable": false,
"dartType": "String"
}
],
"foreignKeys": [],
"indexes": [
{
"__className__": "serverpod.IndexDefinition",
"indexName": "dictionary_entry_source_idx",
"elements": [
{
"__className__": "serverpod.IndexElementDefinition",
"type": 0,
"definition": "source"
},
{
"__className__": "serverpod.IndexElementDefinition",
"type": 0,
"definition": "glossLanguage"
},
{
"__className__": "serverpod.IndexElementDefinition",
"type": 0,
"definition": "sourceId"
}
],
"type": "btree",
"isUnique": true,
"isPrimary": false
},
{
"__className__": "serverpod.IndexDefinition",
"indexName": "dictionary_entry_rank_idx",
"elements": [
{
"__className__": "serverpod.IndexElementDefinition",
"type": 0,
"definition": "language"
},
{
"__className__": "serverpod.IndexElementDefinition",
"type": 0,
"definition": "glossLanguage"
},
{
"__className__": "serverpod.IndexElementDefinition",
"type": 0,
"definition": "rank"
}
],
"type": "btree",
"isUnique": false,
"isPrimary": false
}
],
"managed": true
}
},
{
"__className__": "serverpod.DatabaseMigrationAction",
"type": "createTable",
"createTable": {
"__className__": "serverpod.TableDefinition",
"name": "dictionary_source",
"dartName": "DictionarySource",
"module": "halkyon_learn",
"schema": "public",
"columns": [
{
"__className__": "serverpod.ColumnDefinition",
"name": "id",
"fieldName": "id",
"columnType": 6,
"isNullable": false,
"columnDefault": "serial",
"dartType": "int?"
},
{
"__className__": "serverpod.ColumnDefinition",
"name": "source",
"fieldName": "source",
"columnType": 0,
"isNullable": false,
"dartType": "String"
},
{
"__className__": "serverpod.ColumnDefinition",
"name": "sourceVersion",
"fieldName": "sourceVersion",
"columnType": 0,
"isNullable": false,
"dartType": "String"
},
{
"__className__": "serverpod.ColumnDefinition",
"name": "language",
"fieldName": "language",
"columnType": 0,
"isNullable": false,
"dartType": "String"
},
{
"__className__": "serverpod.ColumnDefinition",
"name": "glossLanguage",
"fieldName": "glossLanguage",
"columnType": 0,
"isNullable": false,
"dartType": "String"
},
{
"__className__": "serverpod.ColumnDefinition",
"name": "license",
"fieldName": "license",
"columnType": 0,
"isNullable": false,
"dartType": "String"
},
{
"__className__": "serverpod.ColumnDefinition",
"name": "credits",
"fieldName": "credits",
"columnType": 8,
"isNullable": false,
"dartType": "List<protocol:DictionaryCredit>"
},
{
"__className__": "serverpod.ColumnDefinition",
"name": "entryCount",
"fieldName": "entryCount",
"columnType": 6,
"isNullable": false,
"dartType": "int"
},
{
"__className__": "serverpod.ColumnDefinition",
"name": "fileName",
"fieldName": "fileName",
"columnType": 0,
"isNullable": false,
"dartType": "String"
},
{
"__className__": "serverpod.ColumnDefinition",
"name": "fileHash",
"fieldName": "fileHash",
"columnType": 0,
"isNullable": false,
"dartType": "String"
},
{
"__className__": "serverpod.ColumnDefinition",
"name": "loadedAt",
"fieldName": "loadedAt",
"columnType": 4,
"isNullable": false,
"dartType": "DateTime"
}
],
"foreignKeys": [],
"indexes": [
{
"__className__": "serverpod.IndexDefinition",
"indexName": "dictionary_source_idx",
"elements": [
{
"__className__": "serverpod.IndexElementDefinition",
"type": 0,
"definition": "source"
},
{
"__className__": "serverpod.IndexElementDefinition",
"type": 0,
"definition": "glossLanguage"
}
],
"type": "btree",
"isUnique": true,
"isPrimary": false
}
],
"managed": true
}
}
],
"warnings": [],
"migrationApiVersion": 1
}