chore: Halkyon Learn moves into apps/learn
Its server, client, app and curator, and the tools that write its assets (maps, Anki fixtures, icons), as they are; paths are fixed in the commits that follow. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Tizian 「ティツィアーン」 <raisondetredev@pm.me>
This commit is contained in:
parent
11abd9afce
commit
649499be23
GPG key ID:
07F6FCF405488B64
2 changed files with 0 additions and 0 deletions
|
|
@ -0,0 +1,245 @@
|
|||
import 'dart:convert';
|
||||
import 'dart:io';
|
||||
|
||||
import 'package:crypto/crypto.dart';
|
||||
import 'package:halkyon_learn_server/src/dictionary/device_dictionaries.dart';
|
||||
import 'package:halkyon_learn_server/src/dictionary/dictionary_loader.dart';
|
||||
import 'package:halkyon_learn_server/src/generated/protocol.dart';
|
||||
import 'package:serverpod/serverpod.dart';
|
||||
import 'package:test/test.dart';
|
||||
|
||||
import '../test_tools/serverpod_test_tools.dart';
|
||||
|
||||
/// A dictionary file's header, as the curator writes it.
|
||||
Map<String, Object?> _header() => {
|
||||
'format': 'halkyon-dictionary',
|
||||
'formatVersion': 1,
|
||||
'source': 'jmdict',
|
||||
'kind': 'words',
|
||||
'sourceVersion': '3.6.2 (2026-10-05)',
|
||||
'language': 'ja',
|
||||
'glossLanguage': 'en',
|
||||
'license': 'CC-BY-SA-4.0',
|
||||
'credits': [
|
||||
{
|
||||
'name': 'JMdict',
|
||||
'by': 'EDRDG',
|
||||
'license': 'CC-BY-SA-4.0',
|
||||
'url': 'https://www.edrdg.org/edrdg/licence.html',
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
/// An entry, as the curator writes it.
|
||||
Map<String, Object?> _entry(
|
||||
String id,
|
||||
String headword, {
|
||||
String? reading,
|
||||
List<String> labels = const [],
|
||||
required int rank,
|
||||
required List<String> keys,
|
||||
}) => {
|
||||
'id': id,
|
||||
'headword': headword,
|
||||
'reading': ?reading,
|
||||
'labels': labels,
|
||||
'rank': rank,
|
||||
'keys': keys,
|
||||
'entry': {
|
||||
'senses': [
|
||||
{'glosses': keys.last.split(', ')},
|
||||
],
|
||||
},
|
||||
};
|
||||
|
||||
void main() {
|
||||
withServerpod(
|
||||
'Dictionaries for devices',
|
||||
rollbackDatabase: RollbackDatabase.disabled,
|
||||
(sessionBuilder, endpoints) {
|
||||
late Session session;
|
||||
late Directory folder;
|
||||
late Directory exports;
|
||||
late DeviceDictionaries devices;
|
||||
|
||||
setUp(() async {
|
||||
session = sessionBuilder.build();
|
||||
folder = Directory.systemTemp.createTempSync('dictionaries');
|
||||
exports = Directory.systemTemp.createTempSync('exports');
|
||||
devices = DeviceDictionaries.of(session.serverpod)
|
||||
..dictionaries = folder
|
||||
..exports = exports
|
||||
..inSession = ((work) => work(session));
|
||||
final file = File('${folder.path}/jmdict-en.jsonl')
|
||||
..writeAsStringSync(
|
||||
[
|
||||
jsonEncode(_header()),
|
||||
jsonEncode(
|
||||
_entry(
|
||||
'1358280',
|
||||
'食べる',
|
||||
reading: 'たべる',
|
||||
labels: ['common', 'jlpt:n5'],
|
||||
rank: 100368,
|
||||
keys: ['食べる', 'たべる', 'taberu', 'to eat', 'eat'],
|
||||
),
|
||||
),
|
||||
jsonEncode(
|
||||
_entry(
|
||||
'1080510',
|
||||
'テレビ',
|
||||
labels: ['common', 'jlpt:n5'],
|
||||
rank: 100410,
|
||||
keys: ['テレビ', 'てれび', 'terebi', 'television', 'tv'],
|
||||
),
|
||||
),
|
||||
].join('\n'),
|
||||
);
|
||||
await loadDictionaryFile(session, file);
|
||||
});
|
||||
|
||||
tearDown(() async {
|
||||
await DictionaryEntry.db.deleteWhere(session, where: (t) => t.id > 0);
|
||||
await DictionaryKey.db.deleteWhere(session, where: (t) => t.id > 0);
|
||||
await DictionarySource.db.deleteWhere(session, where: (t) => t.id > 0);
|
||||
folder.deleteSync(recursive: true);
|
||||
exports.deleteSync(recursive: true);
|
||||
});
|
||||
|
||||
final signedIn = sessionBuilder.copyWith(
|
||||
authentication: AuthenticationOverride.authenticationInfo(
|
||||
const Uuid().v7(),
|
||||
{},
|
||||
),
|
||||
);
|
||||
|
||||
/// The curator's IPADIC pack, in the dictionaries' folder.
|
||||
File ipadic() =>
|
||||
File('${folder.path}/mecab-ipadic.tar.gz')
|
||||
..writeAsBytesSync(gzip.encode(utf8.encode('IPADIC, packed')));
|
||||
|
||||
test('lists, for Japanese, IPADIC and JMdict as this instance holds it, '
|
||||
'each with where to get it, its hash and size, once both are '
|
||||
'ready', () async {
|
||||
final pack = ipadic();
|
||||
expect(
|
||||
await endpoints.dictionary.deviceFiles(signedIn, 'ja'),
|
||||
isEmpty,
|
||||
reason: 'the export is still being made',
|
||||
);
|
||||
|
||||
await devices.prepare(session);
|
||||
final files = await endpoints.dictionary.deviceFiles(signedIn, 'ja');
|
||||
|
||||
expect(
|
||||
[for (final file in files) file.name],
|
||||
[
|
||||
'mecab-ipadic',
|
||||
'jmdict-en',
|
||||
],
|
||||
);
|
||||
final [ipadicFile, jmdictFile] = files;
|
||||
expect(
|
||||
ipadicFile.sha256,
|
||||
sha256.convert(pack.readAsBytesSync()).toString(),
|
||||
);
|
||||
expect(ipadicFile.size, pack.lengthSync());
|
||||
expect(
|
||||
ipadicFile.url,
|
||||
endsWith(
|
||||
'/device-dictionaries/mecab-ipadic-'
|
||||
'${ipadicFile.sha256.substring(0, 16)}.tar.gz',
|
||||
),
|
||||
);
|
||||
final served = devices.servedFile(jmdictFile.url.split('/').last)!;
|
||||
expect(
|
||||
jmdictFile.sha256,
|
||||
sha256.convert(served.readAsBytesSync()).toString(),
|
||||
);
|
||||
});
|
||||
|
||||
test('the export holds every entry as the instance finds and versions '
|
||||
'it: its keys, labels, version and the entry as cards take it, '
|
||||
'after a header with the credits', () async {
|
||||
ipadic();
|
||||
await devices.prepare(session);
|
||||
final file = (await endpoints.dictionary.deviceFiles(
|
||||
signedIn,
|
||||
'ja',
|
||||
)).last;
|
||||
|
||||
final lines = utf8
|
||||
.decode(
|
||||
gzip.decode(
|
||||
devices.servedFile(file.url.split('/').last)!.readAsBytesSync(),
|
||||
),
|
||||
)
|
||||
.trim()
|
||||
.split('\n')
|
||||
.map((line) => jsonDecode(line) as Map<String, Object?>)
|
||||
.toList();
|
||||
final [header, ...entries] = lines;
|
||||
expect(header['format'], deviceExportFormat);
|
||||
expect(header['source'], 'jmdict');
|
||||
expect(header['language'], 'ja');
|
||||
expect(
|
||||
(header['credits']! as List).single,
|
||||
containsPair('name', 'JMdict'),
|
||||
);
|
||||
expect(entries, hasLength(2));
|
||||
final taberu = entries.singleWhere((entry) => entry['id'] == '1358280');
|
||||
expect(taberu['headword'], '食べる');
|
||||
expect(taberu['reading'], 'たべる');
|
||||
expect(taberu['labels'], ['common', 'jlpt:n5']);
|
||||
expect(taberu['keys'], containsAll(['食べる', 'たべる']));
|
||||
expect(taberu['version'], matches(RegExp(r'^[0-9a-f]{16}$')));
|
||||
final stored = await DictionaryEntry.db.findFirstRow(
|
||||
session,
|
||||
where: (t) => t.sourceId.equals('1358280'),
|
||||
);
|
||||
expect(taberu['version'], stored!.version);
|
||||
expect(taberu['entry'], stored.entry);
|
||||
});
|
||||
|
||||
test('lists nothing for other languages, nor without IPADIC, as one '
|
||||
'file is no use without the other', () async {
|
||||
await devices.prepare(session);
|
||||
|
||||
expect(await endpoints.dictionary.deviceFiles(signedIn, 'ja'), isEmpty);
|
||||
ipadic();
|
||||
expect(await endpoints.dictionary.deviceFiles(signedIn, 'de'), isEmpty);
|
||||
expect(
|
||||
await endpoints.dictionary.deviceFiles(signedIn, 'ja-JP'),
|
||||
hasLength(2),
|
||||
);
|
||||
});
|
||||
|
||||
test('serves no file but those listed', () async {
|
||||
ipadic();
|
||||
await devices.prepare(session);
|
||||
await endpoints.dictionary.deviceFiles(signedIn, 'ja');
|
||||
|
||||
for (final name in [
|
||||
'mecab-ipadic.tar.gz',
|
||||
'jmdict-en.jsonl',
|
||||
'..%2Fpasswords.yaml',
|
||||
'',
|
||||
]) {
|
||||
expect(devices.servedFile(name), isNull, reason: name);
|
||||
}
|
||||
});
|
||||
|
||||
test('refuses to list them to nobody signed in', () async {
|
||||
await expectLater(
|
||||
endpoints.dictionary.deviceFiles(
|
||||
sessionBuilder.copyWith(
|
||||
authentication: AuthenticationOverride.unauthenticated(),
|
||||
),
|
||||
'ja',
|
||||
),
|
||||
throwsA(isA<ServerpodUnauthenticatedException>()),
|
||||
);
|
||||
});
|
||||
},
|
||||
);
|
||||
}
|
||||
|
|
@ -0,0 +1,909 @@
|
|||
import 'dart:convert';
|
||||
import 'dart:io';
|
||||
|
||||
import 'package:halkyon_learn_server/src/dictionary/dictionary_endpoint.dart';
|
||||
import 'package:halkyon_learn_server/src/dictionary/dictionary_loader.dart';
|
||||
import 'package:halkyon_learn_server/src/dictionary/dictionary_search.dart';
|
||||
import 'package:halkyon_learn_server/src/generated/protocol.dart';
|
||||
import 'package:serverpod/serverpod.dart';
|
||||
import 'package:test/test.dart';
|
||||
|
||||
import '../test_tools/serverpod_test_tools.dart';
|
||||
|
||||
/// The header of a dictionary file of JMdict at [version], or of another
|
||||
/// [source] of another [kind].
|
||||
Map<String, Object?> header({
|
||||
String version = '3.6.2 (2026-10-05)',
|
||||
String source = 'jmdict',
|
||||
String? kind = 'words',
|
||||
}) => {
|
||||
'format': 'halkyon-dictionary',
|
||||
'formatVersion': 1,
|
||||
'source': source,
|
||||
'kind': ?kind,
|
||||
'sourceVersion': version,
|
||||
'language': 'ja',
|
||||
'glossLanguage': 'en',
|
||||
'license': 'CC-BY-SA-4.0',
|
||||
'credits': [
|
||||
{
|
||||
'name': 'JMdict',
|
||||
'by': 'EDRDG',
|
||||
'license': 'CC-BY-SA-4.0',
|
||||
'url': 'https://www.edrdg.org/edrdg/licence.html',
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
/// An entry as the curator writes it.
|
||||
Map<String, Object?> entry(
|
||||
String id,
|
||||
String headword, {
|
||||
String? reading,
|
||||
List<String> labels = const [],
|
||||
required int rank,
|
||||
required List<String> keys,
|
||||
List<String> looseKeys = const [],
|
||||
}) => {
|
||||
'id': id,
|
||||
'headword': headword,
|
||||
'reading': ?reading,
|
||||
'labels': labels,
|
||||
'rank': rank,
|
||||
'keys': keys,
|
||||
'looseKeys': ?(looseKeys.isEmpty ? null : looseKeys),
|
||||
'entry': {
|
||||
'senses': [
|
||||
{'glosses': keys.last.split(', ')},
|
||||
],
|
||||
},
|
||||
};
|
||||
|
||||
final taberu = entry(
|
||||
'1358280',
|
||||
'食べる',
|
||||
reading: 'たべる',
|
||||
labels: ['common', 'jlpt:n5'],
|
||||
rank: 100368,
|
||||
keys: ['食べる', 'たべる', 'taberu', 'to eat', 'eat'],
|
||||
);
|
||||
final terebi = entry(
|
||||
'1080510',
|
||||
'テレビ',
|
||||
labels: ['common', 'jlpt:n5'],
|
||||
rank: 100410,
|
||||
keys: ['テレビ', 'てれび', 'terebi', 'television', 'tv'],
|
||||
);
|
||||
final toukyou = entry(
|
||||
'1447690',
|
||||
'東京',
|
||||
reading: 'とうきょう',
|
||||
labels: ['common'],
|
||||
rank: 600000,
|
||||
keys: ['東京', 'とうきょう', 'toukyou', 'tokyo'],
|
||||
);
|
||||
// Ranks before 寿司, but "sushi" is only its long vowel typed once.
|
||||
final suushi = entry(
|
||||
'1356640',
|
||||
'数詞',
|
||||
reading: 'すうし',
|
||||
rank: 500000,
|
||||
keys: ['数詞', 'すうし', 'suushi', 'numeral'],
|
||||
looseKeys: ['sushi'],
|
||||
);
|
||||
final sushi = entry(
|
||||
'1595650',
|
||||
'寿司',
|
||||
reading: 'すし',
|
||||
rank: 600000,
|
||||
keys: ['寿司', 'すし', 'sushi'],
|
||||
);
|
||||
// A kanji, as KANJIDIC2's file gives it.
|
||||
final shoku = entry(
|
||||
'食',
|
||||
'食',
|
||||
labels: ['jlpt:n5', 'grade:2'],
|
||||
rank: 10328,
|
||||
keys: ['食', 'ショク', 'しょく', 'shoku', 'たべる', 'taberu', 'eat'],
|
||||
);
|
||||
// Ranks first, but "eat" only starts one of its keys.
|
||||
final kuidaore = entry(
|
||||
'9000001',
|
||||
'食い倒れ',
|
||||
reading: 'くいだおれ',
|
||||
rank: 50,
|
||||
keys: ['食い倒れ', 'くいだおれ', 'kuidaore', 'eating oneself into ruin'],
|
||||
);
|
||||
|
||||
void main() {
|
||||
withServerpod(
|
||||
'Dictionaries',
|
||||
rollbackDatabase: RollbackDatabase.disabled,
|
||||
(sessionBuilder, endpoints) {
|
||||
late Session session;
|
||||
late Directory folder;
|
||||
|
||||
setUp(() {
|
||||
session = sessionBuilder.build();
|
||||
folder = Directory.systemTemp.createTempSync('dictionaries');
|
||||
});
|
||||
|
||||
tearDown(() async {
|
||||
await DictionaryEntry.db.deleteWhere(session, where: (t) => t.id > 0);
|
||||
await DictionaryKey.db.deleteWhere(session, where: (t) => t.id > 0);
|
||||
await DictionarySearchKey.db.deleteWhere(
|
||||
session,
|
||||
where: (t) => t.id > 0,
|
||||
);
|
||||
await DictionarySource.db.deleteWhere(
|
||||
session,
|
||||
where: (t) => t.id > 0,
|
||||
);
|
||||
folder.deleteSync(recursive: true);
|
||||
});
|
||||
|
||||
/// A dictionary file named [name] in the folder, of [entries].
|
||||
File write(
|
||||
List<Map<String, Object?>> entries, {
|
||||
String name = 'jmdict-en.jsonl',
|
||||
Map<String, Object?>? head,
|
||||
}) {
|
||||
final text = [
|
||||
jsonEncode(head ?? header()),
|
||||
for (final entry in entries) jsonEncode(entry),
|
||||
].join('\n');
|
||||
final file = File('${folder.path}/$name');
|
||||
name.endsWith('.gz')
|
||||
? file.writeAsBytesSync(gzip.encode(utf8.encode(text)))
|
||||
: file.writeAsStringSync(text);
|
||||
return file;
|
||||
}
|
||||
|
||||
final signedIn = sessionBuilder.copyWith(
|
||||
authentication: AuthenticationOverride.authenticationInfo(
|
||||
const Uuid().v7(),
|
||||
{},
|
||||
),
|
||||
);
|
||||
|
||||
Future<List<String>> search(String query) async => [
|
||||
for (final hit in await endpoints.dictionary.search(
|
||||
signedIn,
|
||||
'jmdict',
|
||||
'en',
|
||||
query,
|
||||
))
|
||||
hit.headword,
|
||||
];
|
||||
|
||||
Future<List<String>> searchFirst(String query, int limit) async => [
|
||||
for (final hit in await endpoints.dictionary.search(
|
||||
signedIn,
|
||||
'jmdict',
|
||||
'en',
|
||||
query,
|
||||
limit: limit,
|
||||
))
|
||||
hit.headword,
|
||||
];
|
||||
|
||||
test('a dictionary file loads with its credits, and again only once '
|
||||
'it changed, replacing what it held', () async {
|
||||
final file = write([taberu, terebi, toukyou]);
|
||||
|
||||
expect(await loadDictionaryFile(session, file), isTrue);
|
||||
expect(await DictionaryEntry.db.count(session), 3);
|
||||
var source = (await DictionarySource.db.find(session)).single;
|
||||
expect(source.entryCount, 3);
|
||||
expect(source.credits.single.name, 'JMdict');
|
||||
|
||||
expect(await loadDictionaryFile(session, file), isFalse);
|
||||
|
||||
write([taberu, toukyou], head: header(version: '3.6.3 (2026-11-01)'));
|
||||
expect(await loadDictionaryFile(session, file), isTrue);
|
||||
source = (await DictionarySource.db.find(session)).single;
|
||||
expect(source.sourceVersion, '3.6.3 (2026-11-01)');
|
||||
expect(await search('terebi'), isEmpty);
|
||||
expect(await search('taberu'), ['食べる']);
|
||||
});
|
||||
|
||||
test('a dictionary says what it holds; a file from before kinds holds '
|
||||
'words', () async {
|
||||
await loadDictionaryFile(
|
||||
session,
|
||||
write([taberu], head: header(kind: null)),
|
||||
);
|
||||
await loadDictionaryFile(
|
||||
session,
|
||||
write(
|
||||
[shoku],
|
||||
name: 'kanjidic-en.jsonl',
|
||||
head: header(source: 'kanjidic', kind: 'characters'),
|
||||
),
|
||||
);
|
||||
|
||||
expect(
|
||||
{
|
||||
for (final source in await endpoints.dictionary.dictionaries(
|
||||
signedIn,
|
||||
))
|
||||
source.source: source.kind,
|
||||
},
|
||||
{'jmdict': 'words', 'kanjidic': 'characters'},
|
||||
);
|
||||
});
|
||||
|
||||
test("a dictionary's search and labels never reach another's entries "
|
||||
'of the same language', () async {
|
||||
await loadDictionaryFile(session, write([taberu]));
|
||||
await loadDictionaryFile(
|
||||
session,
|
||||
write(
|
||||
[shoku],
|
||||
name: 'kanjidic-en.jsonl',
|
||||
head: header(source: 'kanjidic', kind: 'characters'),
|
||||
),
|
||||
);
|
||||
|
||||
expect(await search('taberu'), ['食べる']);
|
||||
expect(
|
||||
[
|
||||
for (final hit in await endpoints.dictionary.search(
|
||||
signedIn,
|
||||
'kanjidic',
|
||||
'en',
|
||||
'taberu',
|
||||
))
|
||||
hit.headword,
|
||||
],
|
||||
['食'],
|
||||
);
|
||||
expect(
|
||||
await endpoints.dictionary.countLabelled(
|
||||
signedIn,
|
||||
'kanjidic',
|
||||
'en',
|
||||
'jlpt:n5',
|
||||
),
|
||||
1,
|
||||
);
|
||||
expect(
|
||||
[
|
||||
for (final hit in await endpoints.dictionary.labelled(
|
||||
signedIn,
|
||||
'jmdict',
|
||||
'en',
|
||||
'jlpt:n5',
|
||||
))
|
||||
hit.headword,
|
||||
],
|
||||
['食べる'],
|
||||
);
|
||||
});
|
||||
|
||||
test("a dictionary the instance doesn't hold finds nothing", () async {
|
||||
await loadDictionaryFile(session, write([taberu]));
|
||||
|
||||
expect(
|
||||
await endpoints.dictionary.search(signedIn, 'jmdict', 'de', 'taberu'),
|
||||
isEmpty,
|
||||
);
|
||||
expect(
|
||||
await endpoints.dictionary.search(signedIn, 'other', 'en', 'taberu'),
|
||||
isEmpty,
|
||||
);
|
||||
});
|
||||
|
||||
test('the folder loads every file it can read, gzipped too, and '
|
||||
"leaves out one it can't", () async {
|
||||
write([taberu], name: 'jmdict-en.jsonl.gz');
|
||||
write([], name: 'broken.jsonl', head: {'format': 'something else'});
|
||||
|
||||
await loadDictionaries(session, folder);
|
||||
|
||||
expect(await search('taberu'), ['食べる']);
|
||||
});
|
||||
|
||||
test('a search finds a word by its kanji, kana (katakana too), romaji '
|
||||
'with long vowels as typed, and its meanings', () async {
|
||||
await loadDictionaryFile(session, write([taberu, terebi, toukyou]));
|
||||
|
||||
expect(await search('食べる'), ['食べる']);
|
||||
expect(await search('たべ'), ['食べる']);
|
||||
expect(await search('タベル'), ['食べる']);
|
||||
expect(await search('TABERU'), ['食べる']);
|
||||
expect(await search('tokyo'), ['東京']);
|
||||
expect(await search('tōkyō'), ['東京']);
|
||||
expect(await search('television'), ['テレビ']);
|
||||
expect(await search(''), isEmpty);
|
||||
});
|
||||
|
||||
test('a key that is the query comes before one that starts with it, '
|
||||
'whatever their order otherwise', () async {
|
||||
await loadDictionaryFile(session, write([taberu, kuidaore]));
|
||||
|
||||
expect(await search('eat'), ['食べる', '食い倒れ']);
|
||||
});
|
||||
|
||||
test('a loose key, a spelling not quite right, comes after the '
|
||||
'right ones', () async {
|
||||
await loadDictionaryFile(session, write([suushi, sushi]));
|
||||
|
||||
expect(await search('sushi'), ['寿司', '数詞']);
|
||||
expect(await search('numeral'), ['数詞']);
|
||||
});
|
||||
|
||||
test('the best matches come first even when the rest would fill the '
|
||||
'list: an exact key before a loose one ranked higher', () async {
|
||||
await loadDictionaryFile(session, write([suushi, sushi]));
|
||||
|
||||
expect(await searchFirst('sushi', 1), ['寿司']);
|
||||
});
|
||||
|
||||
test('long vowels with macrons find words written with おお or ー, and '
|
||||
'with the vowel once', () async {
|
||||
final ookii = entry(
|
||||
'1',
|
||||
'大きい',
|
||||
reading: 'おおきい',
|
||||
rank: 10,
|
||||
keys: ['大きい', 'おおきい', 'ookii', 'big'],
|
||||
looseKeys: ['okii'],
|
||||
);
|
||||
final koohii = entry(
|
||||
'2',
|
||||
'コーヒー',
|
||||
rank: 20,
|
||||
keys: ['コーヒー', 'こーひー', 'koohii', 'coffee'],
|
||||
looseKeys: ['kohi'],
|
||||
);
|
||||
await loadDictionaryFile(session, write([ookii, koohii]));
|
||||
|
||||
expect(await search('ōkii'), ['大きい']);
|
||||
expect(await search('kōhī'), ['コーヒー']);
|
||||
});
|
||||
|
||||
test("ん typed as n' or nn, and full-width letters, find their "
|
||||
'words', () async {
|
||||
final kinen = entry(
|
||||
'1',
|
||||
'禁煙',
|
||||
reading: 'きんえん',
|
||||
rank: 10,
|
||||
keys: ['禁煙', 'きんえん', 'kinen', 'no smoking'],
|
||||
);
|
||||
final honya = entry(
|
||||
'2',
|
||||
'本屋',
|
||||
reading: 'ほんや',
|
||||
rank: 20,
|
||||
keys: ['本屋', 'ほんや', 'honya', 'bookshop'],
|
||||
);
|
||||
final tshirt = entry(
|
||||
'3',
|
||||
'Ｔシャツ',
|
||||
rank: 30,
|
||||
keys: ['tシャツ', 'てぃーしゃつ', 't-shirt'],
|
||||
);
|
||||
await loadDictionaryFile(session, write([kinen, honya, tshirt]));
|
||||
|
||||
expect(await search("kin'en"), ['禁煙']);
|
||||
expect(await search('honnya'), ['本屋']);
|
||||
expect(await search('Ｔシャツ'), ['Ｔシャツ']);
|
||||
expect(await search('Tシャツ'), ['Ｔシャツ']);
|
||||
});
|
||||
|
||||
test("the query's wildcards and bars are taken as they are", () async {
|
||||
await loadDictionaryFile(session, write([taberu, terebi]));
|
||||
|
||||
expect(await search('%'), isEmpty);
|
||||
expect(await search('_'), isEmpty);
|
||||
expect(await search('|'), isEmpty);
|
||||
});
|
||||
|
||||
test('a search finds a key by its first character, its first two, '
|
||||
'three or more, a character beyond 16 bits counting as one, and '
|
||||
'a key shorter than the query never', () async {
|
||||
final shikaru = entry(
|
||||
'1590120',
|
||||
'𠮟る',
|
||||
reading: 'しかる',
|
||||
rank: 120000,
|
||||
keys: ['𠮟る', 'しかる', 'shikaru', 'to scold', 'scold'],
|
||||
);
|
||||
await loadDictionaryFile(session, write([taberu, terebi, shikaru]));
|
||||
|
||||
expect(await search('𠮟'), ['𠮟る']);
|
||||
expect(await search('𠮟る'), ['𠮟る']);
|
||||
expect(await search('t'), ['食べる', 'テレビ', '𠮟る']);
|
||||
expect(await search('te'), ['テレビ']);
|
||||
expect(await search('tel'), ['テレビ']);
|
||||
expect(await search('tele'), ['テレビ']);
|
||||
expect(await search('televisions'), isEmpty);
|
||||
expect(await search('scold'), ['𠮟る']);
|
||||
});
|
||||
|
||||
test('search keys go with their dictionary: replaced with it, and a '
|
||||
'dictionary loaded before they had rows is searched as before '
|
||||
'until it loads again', () async {
|
||||
final file = write([taberu, terebi]);
|
||||
await loadDictionaryFile(session, file);
|
||||
// Every key and loose key, "to eat" with its space too.
|
||||
expect(await DictionarySearchKey.db.count(session), 10);
|
||||
|
||||
write([terebi], head: header(version: '3.6.3 (2026-11-01)'));
|
||||
await loadDictionaryFile(session, file);
|
||||
expect(await search('taberu'), isEmpty);
|
||||
expect(await DictionarySearchKey.db.count(session), 5);
|
||||
|
||||
await DictionarySearchKey.db.deleteWhere(
|
||||
session,
|
||||
where: (t) => t.id > 0,
|
||||
);
|
||||
expect(await search('tele'), ['テレビ']);
|
||||
expect(await loadDictionaryFile(session, file), isTrue);
|
||||
expect(await DictionarySearchKey.db.count(session), 5);
|
||||
expect(await search('tele'), ['テレビ']);
|
||||
expect(await loadDictionaryFile(session, file), isFalse);
|
||||
});
|
||||
|
||||
test('loads asked for at once leave the dictionary loaded once, '
|
||||
'whole', () async {
|
||||
write([taberu, terebi]);
|
||||
|
||||
await Future.wait([
|
||||
loadDictionaries(session, folder),
|
||||
loadDictionaries(session, folder),
|
||||
]);
|
||||
|
||||
expect(await DictionaryEntry.db.count(session), 2);
|
||||
expect(await DictionarySource.db.count(session), 1);
|
||||
});
|
||||
|
||||
test('a dictionary whose file is gone is unloaded, never while a file '
|
||||
"can't be read", () async {
|
||||
write([taberu]);
|
||||
write(
|
||||
[terebi],
|
||||
name: 'other-en.jsonl',
|
||||
head: header(source: 'other'),
|
||||
);
|
||||
await loadDictionaries(session, folder);
|
||||
expect(await DictionarySource.db.count(session), 2);
|
||||
|
||||
File('${folder.path}/other-en.jsonl').deleteSync();
|
||||
File('${folder.path}/broken.jsonl').writeAsStringSync('not json');
|
||||
await loadDictionaries(session, folder);
|
||||
expect(await DictionarySource.db.count(session), 2);
|
||||
|
||||
File('${folder.path}/broken.jsonl').deleteSync();
|
||||
await loadDictionaries(session, folder);
|
||||
expect(
|
||||
[
|
||||
for (final source in await DictionarySource.db.find(session))
|
||||
source.source,
|
||||
],
|
||||
['jmdict'],
|
||||
);
|
||||
expect(
|
||||
await DictionaryEntry.db.count(
|
||||
session,
|
||||
where: (t) => t.source.equals('other'),
|
||||
),
|
||||
0,
|
||||
);
|
||||
expect(
|
||||
await DictionaryKey.db.count(
|
||||
session,
|
||||
where: (t) => t.source.equals('other'),
|
||||
),
|
||||
0,
|
||||
);
|
||||
expect(
|
||||
await DictionarySearchKey.db.count(
|
||||
session,
|
||||
where: (t) => t.source.equals('other'),
|
||||
),
|
||||
0,
|
||||
);
|
||||
});
|
||||
|
||||
test('two files of one source are refused, and neither loads, as '
|
||||
'which one is meant is unclear', () async {
|
||||
write([taberu], name: 'jmdict-en-old.jsonl');
|
||||
write([terebi]);
|
||||
|
||||
await loadDictionaries(session, folder);
|
||||
|
||||
expect(await DictionaryEntry.db.count(session), 0);
|
||||
});
|
||||
|
||||
test("a folder that can't be read is logged, never thrown: the "
|
||||
'server goes on', () async {
|
||||
final locked = Directory('${folder.path}/locked')..createSync();
|
||||
Process.runSync('chmod', ['000', locked.path]);
|
||||
try {
|
||||
await expectLater(loadDictionaries(session, locked), completes);
|
||||
} finally {
|
||||
Process.runSync('chmod', ['700', locked.path]);
|
||||
}
|
||||
});
|
||||
|
||||
test('pages of a label never repeat or skip entries that rank the '
|
||||
'same', () async {
|
||||
await loadDictionaryFile(
|
||||
session,
|
||||
write([
|
||||
for (var i = 0; i < 40; i++)
|
||||
entry(
|
||||
'$i',
|
||||
'w$i',
|
||||
labels: ['common'],
|
||||
rank: 600000,
|
||||
keys: ['w$i'],
|
||||
),
|
||||
]),
|
||||
);
|
||||
|
||||
final seen = [
|
||||
for (var offset = 0; offset < 40; offset += 7)
|
||||
for (final hit in await endpoints.dictionary.labelled(
|
||||
signedIn,
|
||||
'jmdict',
|
||||
'en',
|
||||
'common',
|
||||
offset: offset,
|
||||
limit: 7,
|
||||
))
|
||||
hit.sourceId,
|
||||
];
|
||||
expect(seen, hasLength(40));
|
||||
expect(seen.toSet(), hasLength(40));
|
||||
});
|
||||
|
||||
test("a label picks its entries, in the dictionary's order, a page at "
|
||||
'a time', () async {
|
||||
await loadDictionaryFile(session, write([toukyou, terebi, taberu]));
|
||||
|
||||
expect(
|
||||
await endpoints.dictionary.countLabelled(
|
||||
signedIn,
|
||||
'jmdict',
|
||||
'en',
|
||||
'jlpt:n5',
|
||||
),
|
||||
2,
|
||||
);
|
||||
final pages = [
|
||||
for (final offset in [0, 1, 2])
|
||||
[
|
||||
for (final hit in await endpoints.dictionary.labelled(
|
||||
signedIn,
|
||||
'jmdict',
|
||||
'en',
|
||||
'jlpt:n5',
|
||||
offset: offset,
|
||||
limit: 1,
|
||||
))
|
||||
hit.headword,
|
||||
],
|
||||
];
|
||||
expect(pages, [
|
||||
['食べる'],
|
||||
['テレビ'],
|
||||
<String>[],
|
||||
]);
|
||||
});
|
||||
|
||||
test('a hit carries what a card needs: its source and id, labels and '
|
||||
'the entry', () async {
|
||||
await loadDictionaryFile(session, write([taberu]));
|
||||
|
||||
final hit = (await endpoints.dictionary.search(
|
||||
signedIn,
|
||||
'jmdict',
|
||||
'en',
|
||||
'taberu',
|
||||
)).single;
|
||||
|
||||
expect((hit.source, hit.sourceId), ('jmdict', '1358280'));
|
||||
expect(hit.reading, 'たべる');
|
||||
expect(hit.labels, ['common', 'jlpt:n5']);
|
||||
expect(jsonDecode(hit.entry), isA<Map<String, Object?>>());
|
||||
});
|
||||
|
||||
test("a hit carries its entry's version, which changes with what a "
|
||||
'card takes of it', () async {
|
||||
Future<String?> version() async => (await endpoints.dictionary.search(
|
||||
signedIn,
|
||||
'jmdict',
|
||||
'en',
|
||||
'taberu',
|
||||
)).single.version;
|
||||
await loadDictionaryFile(session, write([taberu]));
|
||||
final before = await version();
|
||||
|
||||
final corrected = write([
|
||||
{
|
||||
...taberu,
|
||||
'entry': {
|
||||
'senses': [
|
||||
{
|
||||
'glosses': ['to eat', 'to consume'],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
], head: header(version: '3.6.3 (2026-11-01)'));
|
||||
await loadDictionaryFile(session, corrected);
|
||||
final after = await version();
|
||||
|
||||
expect(before, hasLength(16));
|
||||
expect(after, isNot(before));
|
||||
expect(entryVersion('食べる', 'たべる', '{}'), hasLength(16));
|
||||
});
|
||||
|
||||
test('changed: of the entries named, those whose version is another '
|
||||
'now; none gone from the source', () async {
|
||||
await loadDictionaryFile(session, write([taberu, terebi]));
|
||||
final versions = {
|
||||
for (final hit in await endpoints.dictionary.labelled(
|
||||
signedIn,
|
||||
'jmdict',
|
||||
'en',
|
||||
'jlpt:n5',
|
||||
))
|
||||
hit.sourceId: hit.version!,
|
||||
};
|
||||
|
||||
Future<List<String>> changed(Map<String, String> asked) async => [
|
||||
for (final hit in await endpoints.dictionary.changed(
|
||||
signedIn,
|
||||
'jmdict',
|
||||
'en',
|
||||
asked,
|
||||
))
|
||||
hit.sourceId,
|
||||
];
|
||||
|
||||
expect(await changed(versions), isEmpty);
|
||||
expect(
|
||||
await changed({...versions, '1358280': 'an older one'}),
|
||||
['1358280'],
|
||||
);
|
||||
expect(await changed({'404': 'gone'}), isEmpty);
|
||||
expect(await changed({}), isEmpty);
|
||||
expect(
|
||||
() => changed({
|
||||
for (var i = 0; i <= DictionaryEndpoint.maxCompared; i++) '$i': 'v',
|
||||
}),
|
||||
throwsA(anything),
|
||||
);
|
||||
});
|
||||
|
||||
test('entries: those named, in the order named; none gone from the '
|
||||
'source, nor from another dictionary', () async {
|
||||
await loadDictionaryFile(session, write([taberu, terebi]));
|
||||
final ids = [
|
||||
for (final hit in await endpoints.dictionary.labelled(
|
||||
signedIn,
|
||||
'jmdict',
|
||||
'en',
|
||||
'jlpt:n5',
|
||||
))
|
||||
hit.sourceId,
|
||||
];
|
||||
|
||||
Future<List<String>> entries(List<String> asked) async => [
|
||||
for (final hit in await endpoints.dictionary.entries(
|
||||
signedIn,
|
||||
'jmdict',
|
||||
'en',
|
||||
asked,
|
||||
))
|
||||
hit.sourceId,
|
||||
];
|
||||
|
||||
expect(await entries(ids.reversed.toList()), ids.reversed);
|
||||
expect(await entries([ids.first, '404']), [ids.first]);
|
||||
expect(
|
||||
await endpoints.dictionary.entries(signedIn, 'jmdict', 'de', ids),
|
||||
isEmpty,
|
||||
);
|
||||
expect(await entries([]), isEmpty);
|
||||
expect(
|
||||
() => entries([
|
||||
for (var i = 0; i <= DictionaryEndpoint.maxCompared; i++) '$i',
|
||||
]),
|
||||
throwsA(anything),
|
||||
);
|
||||
});
|
||||
|
||||
test('withKeys: the entries with a key that is one asked for, each '
|
||||
'with that key; a key only starting one finds nothing, nor does '
|
||||
"another dictionary's entry", () async {
|
||||
await loadDictionaryFile(
|
||||
session,
|
||||
write([taberu, terebi, toukyou, kuidaore]),
|
||||
);
|
||||
|
||||
Future<List<(String, String)>> withKeys(
|
||||
List<String> keys, {
|
||||
String glossLanguage = 'en',
|
||||
}) async => [
|
||||
for (final found in await endpoints.dictionary.withKeys(
|
||||
signedIn,
|
||||
'jmdict',
|
||||
glossLanguage,
|
||||
keys,
|
||||
))
|
||||
(found.key, found.hit.headword),
|
||||
];
|
||||
|
||||
expect(
|
||||
(await withKeys([
|
||||
'食べる',
|
||||
'たべ',
|
||||
'テレビ',
|
||||
'てれび',
|
||||
'東京',
|
||||
'EAT',
|
||||
'nothing',
|
||||
'',
|
||||
'a|b',
|
||||
])).toSet(),
|
||||
{
|
||||
('食べる', '食べる'),
|
||||
('テレビ', 'テレビ'),
|
||||
('てれび', 'テレビ'),
|
||||
('東京', '東京'),
|
||||
('eat', '食べる'),
|
||||
},
|
||||
);
|
||||
expect(await withKeys(['食べる'], glossLanguage: 'de'), isEmpty);
|
||||
expect(await withKeys([]), isEmpty);
|
||||
expect(
|
||||
() => withKeys([
|
||||
for (var i = 0; i <= DictionaryEndpoint.maxKeys; i++) '$i',
|
||||
]),
|
||||
throwsA(anything),
|
||||
);
|
||||
});
|
||||
|
||||
test('withKeys finds an entry that names its text keys by them alone: '
|
||||
'a German word by its forms, never by its English meanings, which '
|
||||
'would match English words in German text', () async {
|
||||
await loadDictionaryFile(
|
||||
session,
|
||||
write(
|
||||
[
|
||||
{
|
||||
...entry(
|
||||
'Haus/noun',
|
||||
'Haus',
|
||||
rank: 5584,
|
||||
keys: ['haus', 'häuser', 'house', 'house, building'],
|
||||
),
|
||||
'textKeys': ['haus', 'häuser', 'häusern'],
|
||||
},
|
||||
],
|
||||
name: 'wiktionary-de-en.jsonl',
|
||||
head: {
|
||||
...header(source: 'wiktionary-de'),
|
||||
'language': 'de',
|
||||
},
|
||||
),
|
||||
);
|
||||
|
||||
final found = await endpoints.dictionary.withKeys(
|
||||
signedIn,
|
||||
'wiktionary-de',
|
||||
'en',
|
||||
['Häusern', 'house', 'haus'],
|
||||
);
|
||||
|
||||
expect({for (final hit in found) hit.key}, {'häusern', 'haus'});
|
||||
// A search still finds it by its meaning.
|
||||
expect(
|
||||
[
|
||||
for (final hit in await endpoints.dictionary.search(
|
||||
signedIn,
|
||||
'wiktionary-de',
|
||||
'en',
|
||||
'house',
|
||||
))
|
||||
hit.headword,
|
||||
],
|
||||
['Haus'],
|
||||
);
|
||||
});
|
||||
|
||||
test('withKeys finds the keys loaded with their entries, and no more '
|
||||
'once a load replaced them; a dictionary loaded before keys had '
|
||||
'rows loads again', () async {
|
||||
final file = write([taberu, terebi]);
|
||||
await loadDictionaryFile(session, file);
|
||||
expect(
|
||||
await DictionaryKey.db.count(session),
|
||||
// 食べる, たべる, taberu, eat; テレビ, てれび, terebi, television,
|
||||
// tv: "to eat" has a space.
|
||||
9,
|
||||
);
|
||||
|
||||
write([terebi], head: header(version: '3.6.3 (2026-11-01)'));
|
||||
await loadDictionaryFile(session, file);
|
||||
expect(
|
||||
await endpoints.dictionary.withKeys(signedIn, 'jmdict', 'en', [
|
||||
'食べる',
|
||||
]),
|
||||
isEmpty,
|
||||
);
|
||||
|
||||
await DictionaryKey.db.deleteWhere(session, where: (t) => t.id > 0);
|
||||
expect(await loadDictionaryFile(session, file), isTrue);
|
||||
expect(await DictionaryKey.db.count(session), 5);
|
||||
expect(await loadDictionaryFile(session, file), isFalse);
|
||||
});
|
||||
|
||||
test('withKeys: for a key many entries have, the first few in the '
|
||||
"dictionary's order", () async {
|
||||
await loadDictionaryFile(
|
||||
session,
|
||||
write([
|
||||
for (var i = 0; i < DictionaryEndpoint.maxPerKey + 2; i++)
|
||||
entry(
|
||||
'${2000 + i}',
|
||||
'はし$i',
|
||||
rank: 100 - i,
|
||||
keys: ['はし$i', 'はし', 'hashi'],
|
||||
),
|
||||
]),
|
||||
);
|
||||
|
||||
final found = await endpoints.dictionary.withKeys(
|
||||
signedIn,
|
||||
'jmdict',
|
||||
'en',
|
||||
['はし'],
|
||||
);
|
||||
|
||||
expect(found, hasLength(DictionaryEndpoint.maxPerKey));
|
||||
expect(
|
||||
found.first.hit.headword,
|
||||
'はし${DictionaryEndpoint.maxPerKey + 1}',
|
||||
);
|
||||
expect(found.map((each) => each.key).toSet(), {'はし'});
|
||||
});
|
||||
|
||||
test('the dictionaries are for those signed in', () {
|
||||
expect(
|
||||
() => endpoints.dictionary.search(
|
||||
sessionBuilder,
|
||||
'jmdict',
|
||||
'en',
|
||||
'x',
|
||||
),
|
||||
throwsA(isA<ServerpodUnauthenticatedException>()),
|
||||
);
|
||||
expect(
|
||||
() => endpoints.dictionary.withKeys(sessionBuilder, 'jmdict', 'en', [
|
||||
'食べる',
|
||||
]),
|
||||
throwsA(isA<ServerpodUnauthenticatedException>()),
|
||||
);
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
test('a query looks for itself, and in Japanese for its hiragana and '
|
||||
'its long vowels written out', () {
|
||||
expect(
|
||||
queryVariants('ja', ' Tōkyō '),
|
||||
containsAll(['tōkyō', 'toukyou', 'tookyoo', 'tokyo']),
|
||||
);
|
||||
expect(queryVariants('ja', 'タベル'), ['タベル', 'たべる']);
|
||||
expect(queryVariants('en', 'Tōkyō'), ['tōkyō']);
|
||||
expect(queryVariants('ja', ' '), isEmpty);
|
||||
expect(likeEscaped(r'50%_a|b\c'), r'50\%\_a b\\c');
|
||||
});
|
||||
}
|
||||
Loading…
Reference in a new issue