Once the person agrees, the device holds MeCab's IPADIC and the
instance's JMdict and cuts lines as Jisho.org does, so their text never
leaves it (ADR 0012). MeCab comes through mecab_for_dart, whose build
hook compiles it; its tokens are grouped by Ve's rules, written anew in
Dart, with に kept a word of its own. Each word is found as it stands or
as a form of a word, else by its dictionary form, else as the longest
words it holds from token to token; longer words over token boundaries
are offered as before.
The curator packs Ubuntu's compiled IPADIC beside the dictionary files
(curate ipadic, and every update run). The server exports its JMdict
from its tables, so the device finds the same keys and keeps the same
versions, and serves both at /device-dictionaries/, named in
dictionary.deviceFiles('ja') with their hashes; Caddy forwards the path.
The app checks each file against its hash, unpacks IPADIC, builds a
SQLite database from the export on an isolate, and asks the instance
for newer files every 30 days, downloading only what changed. The Watch
screen offers the download once per device with its size (34 MB, some
150 MB once set up); Settings downloads and removes them. Without them,
or should cutting on the device fail, lines go to the instance as
before. MeCab's, IPADIC's and Ve's licences join the licence list.
Every build now needs a C++ compiler; mecab_for_dart holds code_assets
at 1.x.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Signed-off-by: Tizian 「ティツィアーン」 <raisondetredev@pm.me>
137 lines
4.4 KiB
Dart
137 lines
4.4 KiB
Dart
import 'dart:io';
|
|
|
|
import 'package:clock/clock.dart';
|
|
import 'package:flutter_test/flutter_test.dart';
|
|
import 'package:halkyon_learn_flutter/src/feature/dictionary/data/device_dictionaries.dart';
|
|
import 'package:halkyon_learn_flutter/src/feature/dictionary/data/device_dictionary_gateway.dart';
|
|
import 'package:halkyon_learn_flutter/src/feature/dictionary/data/japanese_analyser.dart';
|
|
import 'package:halkyon_learn_flutter/src/feature/dictionary/model/device_dictionary.dart';
|
|
import 'package:halkyon_learn_flutter/src/feature/dictionary/model/sentence_words.dart';
|
|
|
|
/// An instance offering [offered] for devices, each file's bytes by its
|
|
/// address in [bytes]; offering none by default.
|
|
class FakeDeviceDictionaryGateway({
|
|
var List<DeviceFile> offered = const [],
|
|
final Map<Uri, List<int>> bytes = const {},
|
|
}) implements DeviceDictionaryGateway {
|
|
/// The addresses downloaded, in order.
|
|
final downloaded = <Uri>[];
|
|
|
|
/// How many times the files were asked for.
|
|
int asked = 0;
|
|
|
|
/// Thrown by [files] while set: the instance can't be reached.
|
|
Exception? fail;
|
|
|
|
@override
|
|
Future<List<DeviceFile>> files(String language) async {
|
|
asked++;
|
|
if (fail case final error?) {
|
|
throw error;
|
|
}
|
|
return language == 'ja' ? offered : const [];
|
|
}
|
|
|
|
@override
|
|
Stream<List<int>> download(Uri url) async* {
|
|
downloaded.add(url);
|
|
final all = bytes[url] ?? (throw HttpException('Not found', uri: url));
|
|
// In chunks, as a download comes.
|
|
for (var at = 0; at < all.length; at += 4096) {
|
|
yield all.sublist(at, at + 4096 > all.length ? all.length : at + 4096);
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Cuts each sentence into the parts [cut] gives, and looks a stretch up
|
|
/// as [stretch] gives it; records what it was asked.
|
|
class FakeJapaneseAnalyser({
|
|
final List<SentencePart> Function(String sentence)? cut,
|
|
final SentencePart Function(String text, int start)? stretch,
|
|
}) implements JapaneseAnalyser {
|
|
/// The sentences asked for, each call's.
|
|
final asked = <List<String>>[];
|
|
|
|
/// The folders and databases asked to read.
|
|
final read = <String>[];
|
|
|
|
@override
|
|
Future<List<List<SentencePart>>> wordsOf({
|
|
required String ipadic,
|
|
required String database,
|
|
required List<String> sentences,
|
|
}) async {
|
|
asked.add(sentences);
|
|
read
|
|
..add(ipadic)
|
|
..add(database);
|
|
return [
|
|
for (final sentence in sentences)
|
|
cut?.call(sentence) ?? [SentencePart(text: sentence, start: 0)],
|
|
];
|
|
}
|
|
|
|
@override
|
|
Future<SentencePart> wordOfStretch({
|
|
required String database,
|
|
required String text,
|
|
required int start,
|
|
}) async {
|
|
read.add(database);
|
|
return stretch?.call(text, start) ?? SentencePart(text: text, start: start);
|
|
}
|
|
}
|
|
|
|
/// Dictionaries on a device that holds none, nor is offered any: lines are
|
|
/// cut through the instance.
|
|
DeviceDictionaries noDeviceDictionaries() => DeviceDictionaries(
|
|
FakeDeviceDictionaryGateway(),
|
|
FakeJapaneseAnalyser(),
|
|
folder: Directory('${Directory.systemTemp.path}/halkyon-no-dictionaries'),
|
|
);
|
|
|
|
/// The files an instance offers devices for Japanese, IPADIC and JMdict, as
|
|
/// large as learn.halkyon.eu's would be: 34.0 MB together.
|
|
List<DeviceFile> offeredJapanese() => [
|
|
DeviceFile(
|
|
name: 'mecab-ipadic',
|
|
url: Uri.parse('https://learn.example.org/device-dictionaries/ipadic'),
|
|
sha256: '11' * 32,
|
|
size: 13953908,
|
|
),
|
|
DeviceFile(
|
|
name: 'jmdict-en',
|
|
url: Uri.parse('https://learn.example.org/device-dictionaries/jmdict'),
|
|
sha256: '22' * 32,
|
|
size: 21749430,
|
|
),
|
|
];
|
|
|
|
/// What a device holds once it set them up: 150.0 MB, checked just now.
|
|
DeviceInstallation installedJapanese() => DeviceInstallation(
|
|
ipadic: 'ipadic-1111111111111111',
|
|
database: 'jmdict-en-2222222222222222.sqlite',
|
|
source: 'jmdict',
|
|
glossLanguage: 'en',
|
|
files: {'mecab-ipadic': '11' * 32, 'jmdict-en': '22' * 32},
|
|
bytes: 150 * 1024 * 1024,
|
|
installedAt: clock.now().toUtc(),
|
|
checkedAt: clock.now().toUtc(),
|
|
);
|
|
|
|
/// Dictionaries on a device, kept in a folder of their own that goes after
|
|
/// the test, offered by [gateway] and cut with [analyser].
|
|
DeviceDictionaries deviceDictionariesIn(
|
|
FakeDeviceDictionaryGateway gateway, {
|
|
FakeJapaneseAnalyser? analyser,
|
|
DeviceDictionaryStatus initial = const DeviceDictionaryStatus(),
|
|
}) {
|
|
final folder = Directory.systemTemp.createTempSync('dictionaries');
|
|
addTearDown(() => folder.deleteSync(recursive: true));
|
|
return DeviceDictionaries(
|
|
gateway,
|
|
analyser ?? FakeJapaneseAnalyser(),
|
|
folder: folder,
|
|
initial: initial,
|
|
);
|
|
}
|