feat(learn): LanguageTool as a sidecar, reached through the server

compose.languagetool.yaml runs LanguageTool's open-source server
(meyay/languagetool 6.8-13, pinned by digest) on `edge` with no site and
no published port, as user 61081 on a read-only filesystem, without a
log. The server alone reaches it (LANGUAGETOOL_URL): grammar.check is
for those signed in, takes 20,000 characters, 60 checks a minute each
(counted in memory), maps a studyset's language to LanguageTool's code
and keeps nothing. instance.info() names the languages it checks; the
privacy page says what goes there. sidecars/README.md has the steps.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Signed-off-by: Tizian 「ティツィアーン」 <raisondetredev@pm.me>
This commit is contained in:
Tizian 「ティツィアーン」 2026-10-25 23:13:46 +01:00
commit f766a880ac
Signed by: raisondetredev
GPG key ID: 07F6FCF405488B64
22 changed files with 2235 additions and 5 deletions

View file

@ -6,6 +6,7 @@ import 'package:halkyon_learn_server/src/auth/learn_base.dart';
import 'package:halkyon_learn_server/src/dictionary/device_dictionaries.dart';
import 'package:halkyon_learn_server/src/dictionary/dictionary_loader.dart';
import 'package:halkyon_learn_server/src/generated/serverpod.dart';
import 'package:halkyon_learn_server/src/grammar/grammar_checking.dart';
import 'package:halkyon_learn_server/src/instance/jellyfin_suggestion.dart';
import 'package:serverpod_auth_idp_server/core.dart';
@ -53,6 +54,8 @@ Future<void> run(List<String> args) async {
// Whether a Jellyfin server answers where the instance would suggest one
// (decided 2026-10-13), so instance.info() can name it from the first ask.
unawaited(JellyfinSuggestion.of(pod).look());
// Which languages the instance's LanguageTool checks (ADR 0024), likewise.
unawaited(GrammarChecking.of(pod).look());
// The accounts nobody signs into any more, daily (decided 2026-09-30;
// every anonymous account left since ADR 0020, D25). Scheduling always

View file

@ -55,7 +55,8 @@ HoldsRegister learnAccounts([Map<String, String>? environment]) =>
/// What the instance's privacy page says of Learn's data (ADR 0019, point 4,
/// and ADR 0021, D39): what a private studyset shows the instance, what is
/// published, what goes to it to be answered, conversations with
/// assistants (D13, D52), and the learner count.
/// assistants (D13, D52), spelling and grammar checked (ADR 0024), and the
/// learner count.
const List<PrivacyPagePoint> _privacyPagePoints = [
(
english:
@ -119,6 +120,20 @@ const List<PrivacyPagePoint> _privacyPagePoints = [
'aus Jellyfin gehen an die Instanz, um beantwortet zu werden, außer '
'dein Gerät hat die Wörterbücher; sie bewahrt nichts davon auf.',
),
(
english:
'Where the instance checks spelling and grammar, what you check and, '
'while “Check grammar while writing” is on, what you write goes to '
'its own LanguageTool, a paragraph at a time; in a private studyset '
'only once you agreed to it there. Neither the instance nor '
'LanguageTool keeps any of it.',
german:
'Wo die Instanz Rechtschreibung und Grammatik prüft, geht, was du '
'prüfst, und, solange „Grammatik beim Schreiben prüfen“ an ist, was du '
'schreibst, an ihr eigenes LanguageTool, ein Absatz nach dem anderen; '
'in einem privaten Lernset erst, nachdem du dort zugestimmt hast. '
'Weder die Instanz noch LanguageTool bewahrt etwas davon auf.',
),
(
english:
'While “Count my learning in Explore” is on, the instance keeps that '

View file

@ -34,6 +34,7 @@ import '../discussion/discussion_endpoint.dart' as _ienplr78;
import '../explore/explore_endpoint.dart' as _i897jtwc;
import '../explore/learning_count_endpoint.dart' as _is6pj3q5;
import '../feedback/feedback_endpoint.dart' as _i3n4p88h;
import '../grammar/grammar_endpoint.dart' as _icdgmtmg;
import '../instance/instance_endpoint.dart' as _i3g8bfc2;
import '../publish/publish_endpoint.dart' as _iyiav298;
export 'future_calls.dart' show ServerpodFutureCallsGetter;
@ -96,6 +97,12 @@ class Endpoints extends _is.EndpointDispatch {
'inbox',
null,
),
'grammar': _icdgmtmg.GrammarEndpoint()
..initialize(
server,
'grammar',
null,
),
'instance': _i3g8bfc2.InstanceEndpoint()
..initialize(
server,
@ -1154,6 +1161,37 @@ class Endpoints extends _is.EndpointDispatch {
),
},
);
connectors['grammar'] = _is.EndpointConnector(
name: 'grammar',
endpoint: endpoints['grammar']!,
methodConnectors: {
'check': _is.MethodConnector(
name: 'check',
params: {
'text': _is.ParameterDescription(
name: 'text',
type: _is.getType<String>(),
nullable: false,
),
'language': _is.ParameterDescription(
name: 'language',
type: _is.getType<String?>(),
nullable: true,
),
},
call:
(
_is.Session session,
Map<String, dynamic> params,
) async =>
(endpoints['grammar'] as _icdgmtmg.GrammarEndpoint).check(
session,
text: params['text'],
language: params['language'],
),
),
},
);
connectors['instance'] = _is.EndpointConnector(
name: 'instance',
endpoint: endpoints['instance']!,

View file

@ -0,0 +1,113 @@
/* AUTOMATICALLY GENERATED CODE DO NOT MODIFY */
/* To generate run: "serverpod generate" */
// ignore_for_file: implementation_imports
// ignore_for_file: library_private_types_in_public_api
// ignore_for_file: non_constant_identifier_names
// ignore_for_file: public_member_api_docs
// ignore_for_file: type_literal_in_constant_pattern
// ignore_for_file: use_super_parameters
// ignore_for_file: invalid_use_of_internal_member
// ignore_for_file: no_leading_underscores_for_library_prefixes
import 'package:halkyon_learn_server/src/generated/protocol.dart' as _i5lsenq5;
import 'package:serverpod/serverpod.dart' as _is;
import '../grammar/grammar_match.dart' as _idb2htis;
/// What LanguageTool found in a text (ADR 0024). Neither the text nor this is
/// kept.
abstract class GrammarCheck
implements _is.SerializableModel, _is.ProtocolSerialization {
GrammarCheck._({
required this.matches,
required this.language,
required this.truncated,
});
factory GrammarCheck({
required List<_idb2htis.GrammarMatch> matches,
required String language,
required bool truncated,
}) = _GrammarCheckImpl;
factory GrammarCheck.fromJson(Map<String, dynamic> jsonSerialization) {
return GrammarCheck(
matches: _i5lsenq5.Protocol().deserialize<List<_idb2htis.GrammarMatch>>(
jsonSerialization['matches'],
),
language: jsonSerialization['language'] as String,
truncated: _is.BoolJsonExtension.fromJson(jsonSerialization['truncated']),
);
}
/// What it found, in the text's order; at most 200.
List<_idb2htis.GrammarMatch> matches;
/// The language it checked in (`de-DE`): the one asked for, or the one it
/// recognised where none was.
String language;
/// Whether it found more than [matches] hold.
bool truncated;
/// Returns a shallow copy of this [GrammarCheck]
/// with some or all fields replaced by the given arguments.
@_is.useResult
GrammarCheck copyWith({
List<_idb2htis.GrammarMatch>? matches,
String? language,
bool? truncated,
});
@override
Map<String, dynamic> toJson() {
return {
'__className__': 'GrammarCheck',
'matches': matches.toJson(valueToJson: (v) => v.toJson()),
'language': language,
'truncated': truncated,
};
}
@override
Map<String, dynamic> toJsonForProtocol() {
return {
'__className__': 'GrammarCheck',
'matches': matches.toJson(valueToJson: (v) => v.toJsonForProtocol()),
'language': language,
'truncated': truncated,
};
}
@override
String toString() {
return _is.SerializationManager.encode(this);
}
}
class _GrammarCheckImpl extends GrammarCheck {
_GrammarCheckImpl({
required List<_idb2htis.GrammarMatch> matches,
required String language,
required bool truncated,
}) : super._(
matches: matches,
language: language,
truncated: truncated,
);
/// Returns a shallow copy of this [GrammarCheck]
/// with some or all fields replaced by the given arguments.
@_is.useResult
@override
GrammarCheck copyWith({
List<_idb2htis.GrammarMatch>? matches,
String? language,
bool? truncated,
}) {
return GrammarCheck(
matches: matches ?? this.matches.map((e0) => e0.copyWith()).toList(),
language: language ?? this.language,
truncated: truncated ?? this.truncated,
);
}
}

View file

@ -0,0 +1,166 @@
/* AUTOMATICALLY GENERATED CODE DO NOT MODIFY */
/* To generate run: "serverpod generate" */
// ignore_for_file: implementation_imports
// ignore_for_file: library_private_types_in_public_api
// ignore_for_file: non_constant_identifier_names
// ignore_for_file: public_member_api_docs
// ignore_for_file: type_literal_in_constant_pattern
// ignore_for_file: use_super_parameters
// ignore_for_file: invalid_use_of_internal_member
// ignore_for_file: no_leading_underscores_for_library_prefixes
import 'package:halkyon_learn_server/src/generated/protocol.dart' as _i5lsenq5;
import 'package:serverpod/serverpod.dart' as _is;
/// Something LanguageTool found in a text it checked (ADR 0024): where, what
/// is wrong, and what could stand there instead.
abstract class GrammarMatch
implements _is.SerializableModel, _is.ProtocolSerialization {
GrammarMatch._({
required this.offset,
required this.length,
required this.message,
required this.shortMessage,
required this.replacements,
required this.rule,
required this.category,
});
factory GrammarMatch({
required int offset,
required int length,
required String message,
required String shortMessage,
required List<String> replacements,
required String rule,
required String category,
}) = _GrammarMatchImpl;
factory GrammarMatch.fromJson(Map<String, dynamic> jsonSerialization) {
return GrammarMatch(
offset: jsonSerialization['offset'] as int,
length: jsonSerialization['length'] as int,
message: jsonSerialization['message'] as String,
shortMessage: jsonSerialization['shortMessage'] as String,
replacements: _i5lsenq5.Protocol().deserialize<List<String>>(
jsonSerialization['replacements'],
),
rule: jsonSerialization['rule'] as String,
category: jsonSerialization['category'] as String,
);
}
/// Where it starts in the text checked, in UTF-16 code units, as Dart's
/// strings and LanguageTool's (Java's) count them.
int offset;
/// How long it is, counted so.
int length;
/// What is wrong, in a sentence, as the rule of the language checked says
/// it.
String message;
/// The same in a few words; may be empty.
String shortMessage;
/// What could stand there instead, the likeliest first; at most 8, none
/// where the rule only points something out.
List<String> replacements;
/// LanguageTool's rule (`MORFOLOGIK_RULE_EN_US`).
String rule;
/// Its category (`TYPOS`, `GRAMMAR`, `STYLE`).
String category;
/// Returns a shallow copy of this [GrammarMatch]
/// with some or all fields replaced by the given arguments.
@_is.useResult
GrammarMatch copyWith({
int? offset,
int? length,
String? message,
String? shortMessage,
List<String>? replacements,
String? rule,
String? category,
});
@override
Map<String, dynamic> toJson() {
return {
'__className__': 'GrammarMatch',
'offset': offset,
'length': length,
'message': message,
'shortMessage': shortMessage,
'replacements': replacements.toJson(),
'rule': rule,
'category': category,
};
}
@override
Map<String, dynamic> toJsonForProtocol() {
return {
'__className__': 'GrammarMatch',
'offset': offset,
'length': length,
'message': message,
'shortMessage': shortMessage,
'replacements': replacements.toJson(),
'rule': rule,
'category': category,
};
}
@override
String toString() {
return _is.SerializationManager.encode(this);
}
}
class _GrammarMatchImpl extends GrammarMatch {
_GrammarMatchImpl({
required int offset,
required int length,
required String message,
required String shortMessage,
required List<String> replacements,
required String rule,
required String category,
}) : super._(
offset: offset,
length: length,
message: message,
shortMessage: shortMessage,
replacements: replacements,
rule: rule,
category: category,
);
/// Returns a shallow copy of this [GrammarMatch]
/// with some or all fields replaced by the given arguments.
@_is.useResult
@override
GrammarMatch copyWith({
int? offset,
int? length,
String? message,
String? shortMessage,
List<String>? replacements,
String? rule,
String? category,
}) {
return GrammarMatch(
offset: offset ?? this.offset,
length: length ?? this.length,
message: message ?? this.message,
shortMessage: shortMessage ?? this.shortMessage,
replacements: replacements ?? this.replacements.map((e0) => e0).toList(),
rule: rule ?? this.rule,
category: category ?? this.category,
);
}
}

View file

@ -0,0 +1,52 @@
/* AUTOMATICALLY GENERATED CODE DO NOT MODIFY */
/* To generate run: "serverpod generate" */
// ignore_for_file: implementation_imports
// ignore_for_file: library_private_types_in_public_api
// ignore_for_file: non_constant_identifier_names
// ignore_for_file: public_member_api_docs
// ignore_for_file: type_literal_in_constant_pattern
// ignore_for_file: use_super_parameters
// ignore_for_file: invalid_use_of_internal_member
// ignore_for_file: no_leading_underscores_for_library_prefixes
import 'package:serverpod/serverpod.dart' as _is;
/// Why a text wasn't checked (ADR 0024).
enum GrammarRefusal implements _is.SerializableModel {
/// This instance checks nothing (no `LANGUAGETOOL_URL`), or its
/// LanguageTool didn't answer.
unavailable,
/// Its LanguageTool doesn't check the language asked for.
unknownLanguage,
/// Longer than one check takes (`GrammarEndpoint.maxText`).
tooLong,
/// More checks in the last minute than a person may ask for.
tooOften;
static GrammarRefusal fromJson(String name) {
switch (name) {
case 'unavailable':
return GrammarRefusal.unavailable;
case 'unknownLanguage':
return GrammarRefusal.unknownLanguage;
case 'tooLong':
return GrammarRefusal.tooLong;
case 'tooOften':
return GrammarRefusal.tooOften;
default:
throw ArgumentError(
'Value "$name" cannot be converted to "GrammarRefusal"',
);
}
}
@override
String toJson() => name;
@override
String toString() => name;
}

View file

@ -0,0 +1,77 @@
/* AUTOMATICALLY GENERATED CODE DO NOT MODIFY */
/* To generate run: "serverpod generate" */
// ignore_for_file: implementation_imports
// ignore_for_file: library_private_types_in_public_api
// ignore_for_file: non_constant_identifier_names
// ignore_for_file: public_member_api_docs
// ignore_for_file: type_literal_in_constant_pattern
// ignore_for_file: use_super_parameters
// ignore_for_file: invalid_use_of_internal_member
// ignore_for_file: no_leading_underscores_for_library_prefixes
import 'package:serverpod/serverpod.dart' as _is;
import '../grammar/grammar_refusal.dart' as _i6zxv7br;
/// A text wasn't checked (ADR 0024).
abstract class GrammarRefusedException
implements
_is.SerializableException,
_is.SerializableModel,
_is.ProtocolSerialization {
GrammarRefusedException._({required this.reason});
factory GrammarRefusedException({required _i6zxv7br.GrammarRefusal reason}) =
_GrammarRefusedExceptionImpl;
factory GrammarRefusedException.fromJson(
Map<String, dynamic> jsonSerialization,
) {
return GrammarRefusedException(
reason: _i6zxv7br.GrammarRefusal.fromJson(
(jsonSerialization['reason'] as String),
),
);
}
/// Why.
_i6zxv7br.GrammarRefusal reason;
/// Returns a shallow copy of this [GrammarRefusedException]
/// with some or all fields replaced by the given arguments.
@_is.useResult
GrammarRefusedException copyWith({_i6zxv7br.GrammarRefusal? reason});
@override
Map<String, dynamic> toJson() {
return {
'__className__': 'GrammarRefusedException',
'reason': reason.toJson(),
};
}
@override
Map<String, dynamic> toJsonForProtocol() {
return {
'__className__': 'GrammarRefusedException',
'reason': reason.toJson(),
};
}
@override
String toString() {
return 'GrammarRefusedException(reason: $reason)';
}
}
class _GrammarRefusedExceptionImpl extends GrammarRefusedException {
_GrammarRefusedExceptionImpl({required _i6zxv7br.GrammarRefusal reason})
: super._(reason: reason);
/// Returns a shallow copy of this [GrammarRefusedException]
/// with some or all fields replaced by the given arguments.
@_is.useResult
@override
GrammarRefusedException copyWith({_i6zxv7br.GrammarRefusal? reason}) {
return GrammarRefusedException(reason: reason ?? this.reason);
}
}

View file

@ -10,6 +10,7 @@
// ignore_for_file: invalid_use_of_internal_member
// ignore_for_file: no_leading_underscores_for_library_prefixes
import 'package:halkyon_learn_server/src/generated/protocol.dart' as _i5lsenq5;
import 'package:serverpod/serverpod.dart' as _is;
/// What this instance offers, so the app can adapt: every instance is run by
@ -21,6 +22,7 @@ abstract class InstanceInfo
this.minimumAppBuild,
this.accountDomain,
this.jellyfinUrl,
this.grammarLanguages,
this.emailSignIn,
this.dataLocation,
this.backupLocation,
@ -31,6 +33,7 @@ abstract class InstanceInfo
int? minimumAppBuild,
String? accountDomain,
String? jellyfinUrl,
List<String>? grammarLanguages,
bool? emailSignIn,
String? dataLocation,
String? backupLocation,
@ -42,6 +45,11 @@ abstract class InstanceInfo
minimumAppBuild: jsonSerialization['minimumAppBuild'] as int?,
accountDomain: jsonSerialization['accountDomain'] as String?,
jellyfinUrl: jsonSerialization['jellyfinUrl'] as String?,
grammarLanguages: jsonSerialization['grammarLanguages'] == null
? null
: _i5lsenq5.Protocol().deserialize<List<String>>(
jsonSerialization['grammarLanguages'],
),
emailSignIn: jsonSerialization['emailSignIn'] == null
? null
: _is.BoolJsonExtension.fromJson(jsonSerialization['emailSignIn']),
@ -71,6 +79,11 @@ abstract class InstanceInfo
/// none, and from servers that predate it.
String? jellyfinUrl;
/// The languages its LanguageTool checks spelling and grammar in (ADR
/// 0024), each without its variants (`de`, `ja`). Null where it has none,
/// while it hasn't answered, and from servers that predate it.
List<String>? grammarLanguages;
/// Whether people can sign in with an email address here (ADR 0010): a
/// code mailed to it. Null from servers that predate it: no.
bool? emailSignIn;
@ -93,6 +106,7 @@ abstract class InstanceInfo
int? minimumAppBuild,
String? accountDomain,
String? jellyfinUrl,
List<String>? grammarLanguages,
bool? emailSignIn,
String? dataLocation,
String? backupLocation,
@ -105,6 +119,8 @@ abstract class InstanceInfo
if (minimumAppBuild != null) 'minimumAppBuild': minimumAppBuild,
if (accountDomain != null) 'accountDomain': accountDomain,
if (jellyfinUrl != null) 'jellyfinUrl': jellyfinUrl,
if (grammarLanguages != null)
'grammarLanguages': grammarLanguages?.toJson(),
if (emailSignIn != null) 'emailSignIn': emailSignIn,
if (dataLocation != null) 'dataLocation': dataLocation,
if (backupLocation != null) 'backupLocation': backupLocation,
@ -119,6 +135,8 @@ abstract class InstanceInfo
if (minimumAppBuild != null) 'minimumAppBuild': minimumAppBuild,
if (accountDomain != null) 'accountDomain': accountDomain,
if (jellyfinUrl != null) 'jellyfinUrl': jellyfinUrl,
if (grammarLanguages != null)
'grammarLanguages': grammarLanguages?.toJson(),
if (emailSignIn != null) 'emailSignIn': emailSignIn,
if (dataLocation != null) 'dataLocation': dataLocation,
if (backupLocation != null) 'backupLocation': backupLocation,
@ -139,6 +157,7 @@ class _InstanceInfoImpl extends InstanceInfo {
int? minimumAppBuild,
String? accountDomain,
String? jellyfinUrl,
List<String>? grammarLanguages,
bool? emailSignIn,
String? dataLocation,
String? backupLocation,
@ -147,6 +166,7 @@ class _InstanceInfoImpl extends InstanceInfo {
minimumAppBuild: minimumAppBuild,
accountDomain: accountDomain,
jellyfinUrl: jellyfinUrl,
grammarLanguages: grammarLanguages,
emailSignIn: emailSignIn,
dataLocation: dataLocation,
backupLocation: backupLocation,
@ -161,6 +181,7 @@ class _InstanceInfoImpl extends InstanceInfo {
Object? minimumAppBuild = _Undefined,
Object? accountDomain = _Undefined,
Object? jellyfinUrl = _Undefined,
Object? grammarLanguages = _Undefined,
Object? emailSignIn = _Undefined,
Object? dataLocation = _Undefined,
Object? backupLocation = _Undefined,
@ -174,6 +195,9 @@ class _InstanceInfoImpl extends InstanceInfo {
? accountDomain
: this.accountDomain,
jellyfinUrl: jellyfinUrl is String? ? jellyfinUrl : this.jellyfinUrl,
grammarLanguages: grammarLanguages is List<String>?
? grammarLanguages
: this.grammarLanguages?.map((e0) => e0).toList(),
emailSignIn: emailSignIn is bool? ? emailSignIn : this.emailSignIn,
dataLocation: dataLocation is String? ? dataLocation : this.dataLocation,
backupLocation: backupLocation is String?

View file

@ -81,6 +81,10 @@ import 'feedback/feedback_person.dart' as _i8yyqgue;
import 'feedback/feedback_seen.dart' as _izm0nigq;
import 'feedback/feedback_thread.dart' as _ihfcaz5v;
import 'feedback/feedback_view.dart' as _in90wtl8;
import 'grammar/grammar_check.dart' as _i9fqesmy;
import 'grammar/grammar_match.dart' as _ic4rezvx;
import 'grammar/grammar_refusal.dart' as _i0nzmh1m;
import 'grammar/grammar_refused_exception.dart' as _ixpk31s8;
import 'instance/instance_info.dart' as _iskrg2g3;
import 'learning_path/learning_path.dart' as _ivt5m868;
import 'learning_path/learning_path_answer.dart' as _iy9sv3bs;
@ -146,6 +150,10 @@ export 'feedback/feedback_person.dart';
export 'feedback/feedback_seen.dart';
export 'feedback/feedback_thread.dart';
export 'feedback/feedback_view.dart';
export 'grammar/grammar_check.dart';
export 'grammar/grammar_match.dart';
export 'grammar/grammar_refusal.dart';
export 'grammar/grammar_refused_exception.dart';
export 'instance/instance_info.dart';
export 'learning_path/learning_path.dart';
export 'learning_path/learning_path_answer.dart';
@ -3901,6 +3909,18 @@ class Protocol extends _is.DatabaseSerializationManager {
if (t == _in90wtl8.FeedbackView) {
return _in90wtl8.FeedbackView.fromJson(data) as T;
}
if (t == _i9fqesmy.GrammarCheck) {
return _i9fqesmy.GrammarCheck.fromJson(data) as T;
}
if (t == _ic4rezvx.GrammarMatch) {
return _ic4rezvx.GrammarMatch.fromJson(data) as T;
}
if (t == _i0nzmh1m.GrammarRefusal) {
return _i0nzmh1m.GrammarRefusal.fromJson(data) as T;
}
if (t == _ixpk31s8.GrammarRefusedException) {
return _ixpk31s8.GrammarRefusedException.fromJson(data) as T;
}
if (t == _iskrg2g3.InstanceInfo) {
return _iskrg2g3.InstanceInfo.fromJson(data) as T;
}
@ -4137,6 +4157,22 @@ class Protocol extends _is.DatabaseSerializationManager {
if (t == _is.getType<_in90wtl8.FeedbackView?>()) {
return (data != null ? _in90wtl8.FeedbackView.fromJson(data) : null) as T;
}
if (t == _is.getType<_i9fqesmy.GrammarCheck?>()) {
return (data != null ? _i9fqesmy.GrammarCheck.fromJson(data) : null) as T;
}
if (t == _is.getType<_ic4rezvx.GrammarMatch?>()) {
return (data != null ? _ic4rezvx.GrammarMatch.fromJson(data) : null) as T;
}
if (t == _is.getType<_i0nzmh1m.GrammarRefusal?>()) {
return (data != null ? _i0nzmh1m.GrammarRefusal.fromJson(data) : null)
as T;
}
if (t == _is.getType<_ixpk31s8.GrammarRefusedException?>()) {
return (data != null
? _ixpk31s8.GrammarRefusedException.fromJson(data)
: null)
as T;
}
if (t == _is.getType<_iskrg2g3.InstanceInfo?>()) {
return (data != null ? _iskrg2g3.InstanceInfo.fromJson(data) : null) as T;
}
@ -4250,6 +4286,18 @@ class Protocol extends _is.DatabaseSerializationManager {
.toList()
as T;
}
if (t == List<_ic4rezvx.GrammarMatch>) {
return (data as List)
.map((e) => deserialize<_ic4rezvx.GrammarMatch>(e))
.toList()
as T;
}
if (t == _is.getType<List<String>?>()) {
return (data != null
? (data as List).map((e) => deserialize<String>(e)).toList()
: null)
as T;
}
if (t == List<_i7kd0wkp.ReleaseInfo>) {
return (data as List)
.map((e) => deserialize<_i7kd0wkp.ReleaseInfo>(e))
@ -4397,6 +4445,10 @@ class Protocol extends _is.DatabaseSerializationManager {
_izm0nigq.FeedbackSeen => 'FeedbackSeen',
_ihfcaz5v.FeedbackThread => 'FeedbackThread',
_in90wtl8.FeedbackView => 'FeedbackView',
_i9fqesmy.GrammarCheck => 'GrammarCheck',
_ic4rezvx.GrammarMatch => 'GrammarMatch',
_i0nzmh1m.GrammarRefusal => 'GrammarRefusal',
_ixpk31s8.GrammarRefusedException => 'GrammarRefusedException',
_iskrg2g3.InstanceInfo => 'InstanceInfo',
_ivt5m868.LearningPath => 'LearningPath',
_iy9sv3bs.LearningPathAnswer => 'LearningPathAnswer',
@ -4521,6 +4573,14 @@ class Protocol extends _is.DatabaseSerializationManager {
return 'FeedbackThread';
case _in90wtl8.FeedbackView():
return 'FeedbackView';
case _i9fqesmy.GrammarCheck():
return 'GrammarCheck';
case _ic4rezvx.GrammarMatch():
return 'GrammarMatch';
case _i0nzmh1m.GrammarRefusal():
return 'GrammarRefusal';
case _ixpk31s8.GrammarRefusedException():
return 'GrammarRefusedException';
case _iskrg2g3.InstanceInfo():
return 'InstanceInfo';
case _ivt5m868.LearningPath():
@ -4733,6 +4793,18 @@ class Protocol extends _is.DatabaseSerializationManager {
if (dataClassName == 'FeedbackView') {
return deserialize<_in90wtl8.FeedbackView>(data['data']);
}
if (dataClassName == 'GrammarCheck') {
return deserialize<_i9fqesmy.GrammarCheck>(data['data']);
}
if (dataClassName == 'GrammarMatch') {
return deserialize<_ic4rezvx.GrammarMatch>(data['data']);
}
if (dataClassName == 'GrammarRefusal') {
return deserialize<_i0nzmh1m.GrammarRefusal>(data['data']);
}
if (dataClassName == 'GrammarRefusedException') {
return deserialize<_ixpk31s8.GrammarRefusedException>(data['data']);
}
if (dataClassName == 'InstanceInfo') {
return deserialize<_iskrg2g3.InstanceInfo>(data['data']);
}

View file

@ -50,6 +50,8 @@ inbox:
- unblock:
- blockedPeople:
- file:
grammar:
- check:
instance:
- info:
publish:

View file

@ -0,0 +1,11 @@
### What LanguageTool found in a text (ADR 0024). Neither the text nor this is
### kept.
class: GrammarCheck
fields:
### What it found, in the text's order; at most 200.
matches: List<GrammarMatch>
### The language it checked in (`de-DE`): the one asked for, or the one it
### recognised where none was.
language: String
### Whether it found more than [matches] hold.
truncated: bool

View file

@ -0,0 +1,140 @@
import 'dart:async';
import 'dart:collection';
import 'dart:io';
import 'package:clock/clock.dart';
import 'package:halkyon_learn_server/src/generated/protocol.dart';
import 'package:halkyon_learn_server/src/grammar/language_tool.dart';
import 'package:serverpod/serverpod.dart';
/// Checking spelling and grammar on this instance (ADR 0024), through its
/// LanguageTool where it has one: which languages it checks, and checks for
/// a person, as often as [perMinute] allows. Neither the text nor what was
/// found is kept or logged; the counts live in memory alone.
class GrammarChecking {
GrammarChecking({this.languageTool});
/// The checking of [pod], made the first time it is asked for, from the
/// operator's `LANGUAGETOOL_URL` (empty: none). Tests set their own.
factory GrammarChecking.of(Serverpod pod) => _ofPod[pod] ??= GrammarChecking(
languageTool: switch (Platform.environment['LANGUAGETOOL_URL']?.trim()) {
final url? when url.isNotEmpty => LanguageTool(Uri.parse(url)),
_ => null,
},
);
static final _ofPod = Expando<GrammarChecking>('Grammar checking');
/// Makes [checking] the one of [pod].
static void use(Serverpod pod, GrammarChecking checking) =>
_ofPod[pod] = checking;
/// The instance's LanguageTool; none: it checks nothing.
final LanguageTool? languageTool;
/// The longest text one check takes, in UTF-16 code units: a paragraph
/// of a note at most, as the app checks one at a time.
static const maxText = 20000;
/// The most checks a person may ask for in a minute. The app checks a
/// paragraph once its writer pauses, and only those that changed.
static const perMinute = 60;
List<String>? _codes;
DateTime? _lookedAt;
Future<void>? _looking;
final _asked = HashMap<UuidValue, Queue<DateTime>>();
/// How long a look at the languages counts.
static const _fresh = Duration(hours: 1);
/// The languages checked here (`de`, `ja`); null without a LanguageTool,
/// or while it hasn't answered. Looks again in the background once the
/// last look is an hour old, and never waits.
List<String>? get languages {
if (_lookedAt case final lookedAt?
when clock.now().difference(lookedAt) > _fresh) {
unawaited(look());
}
return switch (_codes) {
final codes? => checkedLanguages(codes),
null => null,
};
}
/// Asks LanguageTool which languages it checks; one look at a time. A
/// failed look forgets none it knew.
Future<void> look() => _looking ??= () async {
try {
_codes = await languageTool?.languages() ?? _codes;
} on LanguageToolFailed {
// Unknown until the next look.
} finally {
_lookedAt = clock.now();
_looking = null;
}
}();
/// What LanguageTool finds in [text] of [person], checked in [language]
/// (a language tag, `de` or `en-GB`; none: the one it recognises).
/// Refused without a LanguageTool or when it fails (`unavailable`), for a
/// language it lacks, a text longer than [maxText] and more than
/// [perMinute] checks in the last minute; an empty text needs no check.
Future<GrammarCheck> check(
UuidValue person,
String text, {
String? language,
}) async {
final languageTool = this.languageTool;
if (languageTool == null) {
throw GrammarRefusedException(reason: GrammarRefusal.unavailable);
}
if (text.length > maxText) {
throw GrammarRefusedException(reason: GrammarRefusal.tooLong);
}
if (text.trim().isEmpty) {
return GrammarCheck(
matches: const [],
language: language ?? '',
truncated: false,
);
}
if (_codes == null) {
await look();
}
final codes = _codes;
if (codes == null) {
throw GrammarRefusedException(reason: GrammarRefusal.unavailable);
}
final code = languageToolCode(language, codes);
if (code == null) {
throw GrammarRefusedException(reason: GrammarRefusal.unknownLanguage);
}
_count(person);
try {
return await languageTool.check(text, code);
} on LanguageToolFailed {
throw GrammarRefusedException(reason: GrammarRefusal.unavailable);
}
}
/// Counts a check of [person]'s, refusing it beyond [perMinute] in the
/// last minute.
void _count(UuidValue person) {
final now = clock.now();
final asked = _asked.putIfAbsent(person, Queue.new);
while (asked.isNotEmpty &&
now.difference(asked.first) >= const Duration(minutes: 1)) {
asked.removeFirst();
}
if (asked.length >= perMinute) {
throw GrammarRefusedException(reason: GrammarRefusal.tooOften);
}
asked.addLast(now);
// Those who checked nothing for a minute are forgotten.
_asked.removeWhere(
(_, times) =>
times.isEmpty ||
now.difference(times.last) >= const Duration(minutes: 1),
);
}
}

View file

@ -0,0 +1,26 @@
import 'package:halkyon_learn_server/src/generated/protocol.dart';
import 'package:halkyon_learn_server/src/grammar/grammar_checking.dart';
import 'package:serverpod/serverpod.dart';
import 'package:serverpod_auth_idp_server/core.dart';
/// Spelling and grammar checked by the instance's LanguageTool (ADR 0024),
/// for those signed in: LanguageTool has no sign-in of its own, so the
/// server alone reaches it. Neither the text nor what was found is kept or
/// logged. Refusals are `GrammarRefusedException`s.
class GrammarEndpoint extends Endpoint {
@override
bool get requireLogin => true;
/// What LanguageTool finds in [text], checked in [language] (a language
/// tag, `de` or `en-GB`; none: the one it recognises). Refused where the
/// instance checks nothing or its LanguageTool fails, for a language it
/// lacks, a text longer than `GrammarChecking.maxText`, and beyond
/// `GrammarChecking.perMinute` checks a minute.
Future<GrammarCheck> check(
Session session, {
required String text,
String? language,
}) => GrammarChecking.of(
session.serverpod,
).check(session.authenticated!.authUserId, text, language: language);
}

View file

@ -0,0 +1,21 @@
### Something LanguageTool found in a text it checked (ADR 0024): where, what
### is wrong, and what could stand there instead.
class: GrammarMatch
fields:
### Where it starts in the text checked, in UTF-16 code units, as Dart's
### strings and LanguageTool's (Java's) count them.
offset: int
### How long it is, counted so.
length: int
### What is wrong, in a sentence, as the rule of the language checked says
### it.
message: String
### The same in a few words; may be empty.
shortMessage: String
### What could stand there instead, the likeliest first; at most 8, none
### where the rule only points something out.
replacements: List<String>
### LanguageTool's rule (`MORFOLOGIK_RULE_EN_US`).
rule: String
### Its category (`TYPOS`, `GRAMMAR`, `STYLE`).
category: String

View file

@ -0,0 +1,13 @@
### Why a text wasn't checked (ADR 0024).
enum: GrammarRefusal
serialized: byName
values:
### This instance checks nothing (no `LANGUAGETOOL_URL`), or its
### LanguageTool didn't answer.
- unavailable
### Its LanguageTool doesn't check the language asked for.
- unknownLanguage
### Longer than one check takes (`GrammarEndpoint.maxText`).
- tooLong
### More checks in the last minute than a person may ask for.
- tooOften

View file

@ -0,0 +1,5 @@
### A text wasn't checked (ADR 0024).
exception: GrammarRefusedException
fields:
### Why.
reason: GrammarRefusal

View file

@ -0,0 +1,208 @@
import 'dart:async';
import 'dart:convert';
import 'dart:io';
import 'package:halkyon_learn_server/src/generated/protocol.dart';
/// The instance's LanguageTool (ADR 0024): the open-source server of
/// `deploy/compose.languagetool.yaml`, or another the operator names
/// (`LANGUAGETOOL_URL`). Only the server talks to it, as it has no sign-in
/// of its own; what it is sent is never kept or logged.
class LanguageTool {
LanguageTool(this.address);
/// Where it answers (`http://languagetool:8081/`).
final Uri address;
/// How long a request may take.
static const _timeout = Duration(seconds: 10);
/// The variants spelling is checked in where LanguageTool is to recognise
/// the language: without one it checks English, German, Portuguese and
/// Catalan for grammar alone.
static const preferredVariants = 'en-US,de-DE,pt-PT,ca-ES';
/// The codes of the languages it checks (`de-DE`, `ja-JP`).
Future<List<String>> languages() async {
final answer = await _request('GET', 'v2/languages');
if (answer is! List<Object?>) {
throw const LanguageToolFailed('no list of languages');
}
return [
for (final language in answer)
if (language case {'longCode': final String code}) code,
];
}
/// What it finds in [text], checked in [language] (one of [languages], or
/// `auto` to recognise it).
Future<GrammarCheck> check(String text, String language) async {
final answer = await _request('POST', 'v2/check', {
'text': text,
'language': language,
if (language == 'auto') 'preferredVariants': preferredVariants,
});
if (answer is! Map<String, Object?>) {
throw const LanguageToolFailed('no check');
}
return grammarCheckOf(answer);
}
/// [method] on [path], with [form] form-encoded as LanguageTool takes
/// it; its JSON answer.
Future<Object?> _request(
String method,
String path, [
Map<String, String>? form,
]) async {
final client = HttpClient()..connectionTimeout = _timeout;
try {
final request = await client
.openUrl(method, address.resolve(path))
.timeout(_timeout);
if (form != null) {
request.headers.contentType = ContentType(
'application',
'x-www-form-urlencoded',
charset: 'utf-8',
);
request.write(Uri(queryParameters: form).query);
}
final response = await request.close().timeout(_timeout);
final body = await utf8.decodeStream(response).timeout(_timeout);
if (response.statusCode != HttpStatus.ok) {
// Never the body: LanguageTool's errors may quote what it was sent.
throw LanguageToolFailed('answered ${response.statusCode}');
}
return jsonDecode(body);
} on LanguageToolFailed {
rethrow;
} on Object catch (error) {
throw LanguageToolFailed(error.runtimeType.toString());
} finally {
client.close(force: true);
}
}
}
/// LanguageTool didn't answer as it does: [reason] says how, never with
/// what it was sent.
class LanguageToolFailed implements Exception {
const LanguageToolFailed(this.reason);
final String reason;
@override
String toString() => 'LanguageTool failed: $reason';
}
/// The most matches one check returns, so a text full of them can't send
/// a device an answer without end.
const maxMatches = 200;
/// The most suggestions one match returns.
const maxReplacements = 8;
/// LanguageTool's answer to `/v2/check`, as the app gets it: its matches,
/// those it can place, at most [maxMatches] with at most [maxReplacements]
/// suggestions each, and the language it checked in.
GrammarCheck grammarCheckOf(Map<String, Object?> answer) {
final found = switch (answer['matches']) {
final List<Object?> matches => matches,
_ => const <Object?>[],
};
final matches = [
for (final match in found)
if (match case {
'offset': final int offset,
'length': final int length,
'message': final String message,
})
GrammarMatch(
offset: offset,
length: length,
message: message,
shortMessage: switch (match['shortMessage']) {
final String short => short,
_ => '',
},
replacements: [
if (match['replacements'] case final List<Object?> replacements)
for (final replacement in replacements)
if (replacement case {'value': final String value}
when value.isNotEmpty)
value,
].take(maxReplacements).toList(),
rule: switch (match['rule']) {
{'id': final String id} => id,
_ => '',
},
category: switch (match['rule']) {
{'category': {'id': final String id}} => id,
_ => '',
},
),
];
return GrammarCheck(
matches: matches.take(maxMatches).toList(),
language: switch (answer['language']) {
{'code': 'auto', 'detectedLanguage': {'code': final String code}} => code,
{'code': final String code} => code,
_ => '',
},
truncated: matches.length > maxMatches,
);
}
/// Where a language alone has several variants, the one LanguageTool checks
/// spelling in: it has none for `en` or `de` as such.
const _spellingVariants = {
'en': 'en-US',
'de': 'de-DE',
'pt': 'pt-PT',
'ca': 'ca-ES',
};
/// The code [tag] (a studyset's language, `de` or `en-GB`) is checked in,
/// of LanguageTool's [codes]: the tag as it stands, else the variant its
/// language spells in, its only variant, or the language alone; `auto` for
/// none, to recognise it; null where LanguageTool lacks it. Traditional
/// Chinese has none: LanguageTool's one Chinese is simplified (`zh-CN`).
String? languageToolCode(String? tag, Iterable<String> codes) {
final wanted = tag?.trim() ?? '';
if (wanted.isEmpty) {
return 'auto';
}
final byLower = {for (final code in codes) code.toLowerCase(): code};
final subtags = wanted.toLowerCase().split(RegExp('[-_]'));
final language = subtags.first;
// A language alone only after its variants: LanguageTool lists `en` too,
// which checks no spelling.
if (subtags.length > 1) {
if (byLower[subtags.join('-')] case final exact?) {
return exact;
}
}
if (language == 'zh' &&
subtags.skip(1).any({'hant', 'tw', 'hk', 'mo'}.contains)) {
return null;
}
if (_spellingVariants[language] case final variant?
when byLower.containsKey(variant.toLowerCase())) {
return byLower[variant.toLowerCase()];
}
final variants = [
for (final code in codes)
if (code.toLowerCase().startsWith('$language-')) code,
];
if (variants.length == 1) {
return variants.single;
}
return byLower[language];
}
/// The languages LanguageTool's [codes] check, each once and without its
/// variants (`de`, `ja`), in order: what the app asks about.
List<String> checkedLanguages(Iterable<String> codes) => {
for (final code in codes) code.split('-').first.toLowerCase(),
}.toList()..sort();

View file

@ -1,5 +1,6 @@
import 'package:halkyon_base_server/base.dart';
import 'package:halkyon_learn_server/src/generated/protocol.dart';
import 'package:halkyon_learn_server/src/grammar/grammar_checking.dart';
import 'package:halkyon_learn_server/src/instance/jellyfin_suggestion.dart';
import 'package:serverpod/serverpod.dart';
@ -11,8 +12,8 @@ import 'package:serverpod/serverpod.dart';
class InstanceEndpoint extends Endpoint {
/// This instance's offer: email sign-in only where it can mail codes,
/// where its source is, the oldest app build it works with, the domain its
/// people's handles are on, the Jellyfin server it suggests, and where it
/// keeps people's data and its backups.
/// people's handles are on, the Jellyfin server it suggests, the languages
/// it checks writing in, and where it keeps people's data and its backups.
Future<InstanceInfo> info(Session session) async {
final pod = session.serverpod;
return InstanceInfo(
@ -20,6 +21,7 @@ class InstanceEndpoint extends Endpoint {
minimumAppBuild: HalkyonBase.of(pod).minimumAppBuild,
accountDomain: accountDomain(pod),
jellyfinUrl: JellyfinSuggestion.of(pod).url?.toString(),
grammarLanguages: GrammarChecking.of(pod).languages,
emailSignIn: emailSignInAvailable(pod),
dataLocation: dataLocation(),
backupLocation: backupLocation(),

View file

@ -19,6 +19,10 @@ fields:
### instance, once a Jellyfin server answers there. Null where there is
### none, and from servers that predate it.
jellyfinUrl: String?
### The languages its LanguageTool checks spelling and grammar in (ADR
### 0024), each without its variants (`de`, `ja`). Null where it has none,
### while it hasn't answered, and from servers that predate it.
grammarLanguages: List<String>?
### Whether people can sign in with an email address here (ADR 0010): a
### code mailed to it. Null from servers that predate it: no.
emailSignIn: bool?

View file

@ -0,0 +1,140 @@
import 'dart:convert';
import 'dart:io';
import 'package:halkyon_learn_server/src/generated/protocol.dart';
import 'package:halkyon_learn_server/src/grammar/grammar_checking.dart';
import 'package:halkyon_learn_server/src/grammar/language_tool.dart';
import 'package:serverpod/serverpod.dart';
import 'package:test/test.dart';
import '../support/auth_fixtures.dart';
import '../test_tools/serverpod_test_tools.dart';
void main() {
withServerpod('Checking spelling and grammar', (sessionBuilder, endpoints) {
late HttpServer languageTool;
final sent = <String>[];
TestSessionBuilder signedIn() => sessionBuilder.copyWith(
authentication: AuthenticationOverride.authenticationInfo(
const Uuid().v7(),
{},
),
);
setUp(() async {
useLearnBase(sessionBuilder);
sent.clear();
// LanguageTool as it answers, for German and English.
languageTool = (await HttpServer.bind(InternetAddress.loopbackIPv4, 0))
..listen((request) async {
final body = await utf8.decodeStream(request);
sent.add(body);
request.response.headers.contentType = ContentType.json;
request.response.write(
jsonEncode(
request.uri.path == '/v2/languages'
? [
{'name': 'German (Germany)', 'longCode': 'de-DE'},
{'name': 'English (US)', 'longCode': 'en-US'},
]
: {
'language': {'code': 'en-US'},
'matches': [
{
'message': 'Possible spelling mistake found.',
'shortMessage': 'Spelling mistake',
'replacements': [
{'value': 'sentence'},
],
'offset': 5,
'length': 8,
'rule': {
'id': 'MORFOLOGIK_RULE_EN_US',
'category': {'id': 'TYPOS'},
},
},
],
},
),
);
await request.response.close();
});
GrammarChecking.use(
sessionBuilder.build().serverpod,
GrammarChecking(
languageTool: LanguageTool(
Uri.parse('http://127.0.0.1:${languageTool.port}/'),
),
),
);
});
tearDown(() => languageTool.close(force: true));
test('checks what a person sends, in the language asked for', () async {
final check = await endpoints.grammar.check(
signedIn(),
text: 'This sentance.',
language: 'en',
);
expect(check.language, 'en-US');
expect(check.matches.single.replacements, ['sentence']);
expect(Uri.splitQueryString(sent.last), {
'text': 'This sentance.',
'language': 'en-US',
});
});
test(
'is for those signed in alone, and sends nothing for others',
() async {
await expectLater(
endpoints.grammar.check(sessionBuilder, text: 'A text.'),
throwsA(isA<ServerpodUnauthenticatedException>()),
);
expect(sent, isEmpty);
},
);
test('refuses a language its LanguageTool lacks', () async {
await expectLater(
endpoints.grammar.check(signedIn(), text: '안녕.', language: 'ko'),
throwsA(
isA<GrammarRefusedException>().having(
(refused) => refused.reason,
'reason',
GrammarRefusal.unknownLanguage,
),
),
);
});
test('names the languages it checks in what the instance offers', () async {
await GrammarChecking.of(sessionBuilder.build().serverpod).look();
final info = await endpoints.instance.info(sessionBuilder);
expect(info.grammarLanguages, ['de', 'en']);
});
test('an instance without LanguageTool names none and checks '
'nothing', () async {
GrammarChecking.use(sessionBuilder.build().serverpod, GrammarChecking());
final info = await endpoints.instance.info(sessionBuilder);
expect(info.grammarLanguages, isNull);
await expectLater(
endpoints.grammar.check(signedIn(), text: 'A text.'),
throwsA(
isA<GrammarRefusedException>().having(
(refused) => refused.reason,
'reason',
GrammarRefusal.unavailable,
),
),
);
});
});
}

Some files were not shown because too many files have changed in this diff Show more