chore: Halkyon Learn moves into apps/learn
Its server, client, app and curator, and the tools that write its assets (maps, Anki fixtures, icons), as they are; paths are fixed in the commits that follow. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Tizian 「ティツィアーン」 <raisondetredev@pm.me>
This commit is contained in:
parent
11abd9afce
commit
649499be23
191 changed files with 0 additions and 0 deletions
134
apps/learn/halkyon_learn_server/lib/server.dart
Normal file
134
apps/learn/halkyon_learn_server/lib/server.dart
Normal file
|
|
@ -0,0 +1,134 @@
|
|||
import 'dart:async';
|
||||
import 'dart:io';
|
||||
|
||||
import 'package:halkyon_base_server/base.dart';
|
||||
import 'package:halkyon_learn_server/src/assistant/connector_routes.dart';
|
||||
import 'package:halkyon_learn_server/src/auth/learn_base.dart';
|
||||
import 'package:halkyon_learn_server/src/dictionary/device_dictionaries.dart';
|
||||
import 'package:halkyon_learn_server/src/dictionary/dictionary_loader.dart';
|
||||
import 'package:halkyon_learn_server/src/generated/serverpod.dart';
|
||||
import 'package:halkyon_learn_server/src/instance/jellyfin_suggestion.dart';
|
||||
import 'package:halkyon_learn_server/src/profile/profile_page.dart';
|
||||
import 'package:serverpod_auth_idp_server/core.dart';
|
||||
|
||||
/// The starting point of the Serverpod server.
|
||||
Future<void> run(List<String> args) async {
|
||||
// Initialize Serverpod. The generated Serverpod class is already connected
|
||||
// with your project's generated code.
|
||||
final pod = Serverpod(args);
|
||||
// What every Halkyon server has (ADR 0014): sign-in, the account and its
|
||||
// deletion, the pruning, the instance's basics, set up for Learn.
|
||||
HalkyonBase.configure(pod, learnBase());
|
||||
pod
|
||||
// Token managers validate and issue authentication keys; identity
|
||||
// providers are the ways people can sign in (halkyon_base's
|
||||
// auth_config.dart).
|
||||
..initializeAuthServices(
|
||||
tokenManagerBuilders: tokenManagerBuilders,
|
||||
identityProviderBuilders: identityProviderBuilders(pod),
|
||||
)
|
||||
// Files live in Postgres until the self-hosted S3-compatible storage arrives later.
|
||||
..addCloudStorage(DatabaseCloudStorage('private'))
|
||||
..addCloudStorage(DatabaseCloudStorage('public'));
|
||||
|
||||
// The assistant connector, on the web server (ADR 0006).
|
||||
addConnectorRoutes(pod);
|
||||
// Profile pages, `/@handle`, which profile links link back to (ADR 0011).
|
||||
addProfilePageRoute(pod);
|
||||
// What devices download to look Japanese up by themselves (ADR 0012).
|
||||
addDeviceDictionaryRoute(pod);
|
||||
|
||||
// Start the server.
|
||||
await pod.start();
|
||||
|
||||
// Whether a Jellyfin server answers where the instance would suggest one
|
||||
// (decided 2026-10-13), so instance.info() can name it from the first ask.
|
||||
unawaited(JellyfinSuggestion.of(pod).look());
|
||||
|
||||
// Abandoned anonymous people are pruned daily (decided 2026-09-30).
|
||||
// Scheduling always adds an entry, so the previous one goes first.
|
||||
await pod.futureCalls.cancel(_pruning);
|
||||
await pod.futureCalls
|
||||
.callRecurring(identifier: _pruning)
|
||||
.cron('17 3 * * *')
|
||||
.pruning
|
||||
.pruneAnonymousPeople();
|
||||
|
||||
// The IP addresses of sign-up attempts, hourly, once the rate limit is
|
||||
// done with them (decided 2026-10-10).
|
||||
await pod.futureCalls.cancel(_attemptPruning);
|
||||
await pod.futureCalls
|
||||
.callRecurring(identifier: _attemptPruning)
|
||||
.cron('27 * * * *')
|
||||
.pruning
|
||||
.forgetSignUpAttempts();
|
||||
|
||||
// The assistant connector's expired codes and tokens (ADR 0006, rule 5).
|
||||
await pod.futureCalls.cancel(_connectorPruning);
|
||||
await pod.futureCalls
|
||||
.callRecurring(identifier: _connectorPruning)
|
||||
.cron('57 3 * * *')
|
||||
.pruning
|
||||
.forgetConnectorSecrets();
|
||||
|
||||
// So are the files nobody can reach any more (ADR 0004, rule 5).
|
||||
await pod.futureCalls.cancel(_filePruning);
|
||||
await pod.futureCalls
|
||||
.callRecurring(identifier: _filePruning)
|
||||
.cron('47 3 * * *')
|
||||
.filePruning
|
||||
.pruneFiles();
|
||||
|
||||
// How many people learn each published studyset here (ADR 0005, rule 4).
|
||||
await pod.futureCalls.cancel(_learnerCount);
|
||||
await pod.futureCalls
|
||||
.callRecurring(identifier: _learnerCount)
|
||||
.cron('37 4 * * *')
|
||||
.learnerCount
|
||||
.countLearners();
|
||||
|
||||
// The dictionaries in the server's folder, those that changed since; in
|
||||
// the background, as loading takes a minute or two. Then again hourly, as
|
||||
// the dictionaries service replaces them weekly (and fills its volume
|
||||
// first after an instance's first start).
|
||||
unawaited(_loadDictionaries(pod));
|
||||
await pod.futureCalls.cancel(_dictionaryReload);
|
||||
await pod.futureCalls
|
||||
.callRecurring(identifier: _dictionaryReload)
|
||||
.cron('7 * * * *')
|
||||
.dictionaryReload
|
||||
.reloadDictionaries();
|
||||
}
|
||||
|
||||
/// Loads the dictionaries in the server's folder ([dictionaryFolder]).
|
||||
/// [loadDictionaries] throws nothing; a session that can't be had is
|
||||
/// reported here, as an error in the background would end the server.
|
||||
Future<void> _loadDictionaries(Serverpod pod) async {
|
||||
try {
|
||||
await pod.withSession((session) async {
|
||||
await loadDictionaries(session, dictionaryFolder());
|
||||
// What devices download, ready before the first asks (ADR 0012).
|
||||
await DeviceDictionaries.of(pod).prepare(session);
|
||||
});
|
||||
} on Object catch (error) {
|
||||
stderr.writeln('Loading the dictionaries failed: $error');
|
||||
}
|
||||
}
|
||||
|
||||
/// The identifier of the daily pruning.
|
||||
const _pruning = 'prune-abandoned-anonymous-people';
|
||||
|
||||
/// The identifier of the hourly forgetting of sign-up attempts.
|
||||
const _attemptPruning = 'forget-sign-up-attempts';
|
||||
|
||||
/// The identifier of the daily forgetting of the connector's expired secrets.
|
||||
const _connectorPruning = 'forget-connector-secrets';
|
||||
|
||||
/// The identifier of the daily pruning of files.
|
||||
const _filePruning = 'prune-unreferenced-files';
|
||||
|
||||
/// The identifier of the hourly rereading of the dictionaries.
|
||||
const _dictionaryReload = 'reload-dictionaries';
|
||||
|
||||
/// The identifier of the daily count of each published studyset's learners.
|
||||
const _learnerCount = 'count-learners';
|
||||
33
apps/learn/halkyon_learn_server/lib/src/asset/asset.spy.yaml
Normal file
33
apps/learn/halkyon_learn_server/lib/src/asset/asset.spy.yaml
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
### A file in a person's content, an image or a sound (ADR 0004). This row is synced;
|
||||
### the file itself is in the server's private cloud storage, reachable only
|
||||
### through an endpoint that checks the owner, and each device caches it by its
|
||||
### hash. Content refers to it by the row's id.
|
||||
###
|
||||
### A file added twice (the same image in two imported cards) is one file in
|
||||
### storage, since its path is its hash; two rows for it may exist when two
|
||||
### devices added it while offline, and count as one.
|
||||
class: Asset
|
||||
table: asset
|
||||
database: sync
|
||||
fields:
|
||||
### The file's SHA-256, in lowercase hex: its name in storage and in every
|
||||
### device's cache, and the check that a download is the file that was added.
|
||||
sha256: String
|
||||
### Its media type, e.g. 'image/png' or 'audio/mpeg'.
|
||||
mimeType: String
|
||||
### Its size in bytes.
|
||||
byteSize: int
|
||||
### The encryption key's id; always null (ADR 0002), as on studysets.
|
||||
keyId: String?
|
||||
### The content as UTF-8 JSON: {"schemaVersion", "fileName"}; the name it came
|
||||
### with is content, like a studyset's title.
|
||||
payload: ByteData
|
||||
### The encryption nonce; always null, like keyId.
|
||||
nonce: ByteData?
|
||||
indexes:
|
||||
# Deleting a space finds its rows through this index (see studyset.spy.yaml).
|
||||
asset_space_idx:
|
||||
fields: spaceId
|
||||
# A space's file with a given hash, to add a file only once.
|
||||
asset_hash_idx:
|
||||
fields: spaceId, sha256
|
||||
|
|
@ -0,0 +1,170 @@
|
|||
import 'dart:typed_data';
|
||||
|
||||
import 'package:clock/clock.dart';
|
||||
import 'package:crypto/crypto.dart';
|
||||
import 'package:halkyon_learn_server/src/asset/file_limits.dart';
|
||||
import 'package:halkyon_learn_server/src/asset/file_pruning.dart';
|
||||
import 'package:halkyon_learn_server/src/generated/protocol.dart';
|
||||
import 'package:serverpod/protocol.dart' show CloudStorageEntry;
|
||||
import 'package:serverpod/serverpod.dart';
|
||||
import 'package:serverpod_offline_sync_server/serverpod_offline_sync_server.dart';
|
||||
|
||||
/// The files of people's content (ADR 0004, rule 5): images, sounds and
|
||||
/// PDFs, stored in the server's private storage at a path derived from the space
|
||||
/// and the file's SHA-256, so a file added twice is stored once.
|
||||
///
|
||||
/// Uploading goes into the caller's own space, by the file's hash, before or
|
||||
/// after its asset row syncs: the server asks for the file only when it
|
||||
/// lacks it, and keeps it only when its content has that hash. Downloading
|
||||
/// goes by the asset row, from a space the caller may read; any other asks
|
||||
/// as if the file weren't there. Files no asset row refers to any more are
|
||||
/// pruned ([pruneUnreferencedFiles]).
|
||||
class AssetEndpoint extends Endpoint {
|
||||
@override
|
||||
bool get requireLogin => true;
|
||||
|
||||
/// The largest file accepted: a PDF (decided 2026-10-03).
|
||||
static const maxBytes = 50 * 1024 * 1024;
|
||||
|
||||
/// The largest file of any other kind, an image or a sound. A larger one
|
||||
/// is checked to be a PDF once stored, and deleted otherwise.
|
||||
static const maxOtherBytes = 20 * 1024 * 1024;
|
||||
|
||||
/// How to upload the caller's file with [sha256] (lowercase hex) of
|
||||
/// [byteSize] bytes; null when the server has it already. Throws a
|
||||
/// [FileLimitExceededException] when it doesn't fit (see [FileLimits]).
|
||||
Future<String?> startUpload(
|
||||
Session session,
|
||||
String sha256,
|
||||
int byteSize,
|
||||
) async {
|
||||
_checkHash(sha256);
|
||||
if (byteSize <= 0 || byteSize > maxBytes) {
|
||||
throw ArgumentError.value(byteSize, 'byteSize', 'is out of range');
|
||||
}
|
||||
final space = _caller(session);
|
||||
final path = assetPath(space, sha256);
|
||||
if (await session.storage.fileExists(storageId: assetStorage, path: path)) {
|
||||
// The file is claimed anew, so pruning leaves it alone for a while
|
||||
// even if the row that claims it hasn't synced yet.
|
||||
// UPDATE "serverpod_cloud_storage" SET "addedTime" = $1
|
||||
// WHERE "storageId" = 'private' AND "path" = $2
|
||||
await CloudStorageEntry.db.updateWhere(
|
||||
session,
|
||||
columnValues: (t) => [t.addedTime(clock.now().toUtc())],
|
||||
where: (t) => t.storageId.equals(assetStorage) & t.path.equals(path),
|
||||
);
|
||||
return null;
|
||||
}
|
||||
checkRoom(await roomOf(session, space), byteSize);
|
||||
return await session.storage.createUploadDescription(
|
||||
storageId: assetStorage,
|
||||
path: path,
|
||||
options: UploadOptions(contentLength: byteSize, maxFileSize: maxBytes),
|
||||
);
|
||||
}
|
||||
|
||||
/// Completes the upload of the caller's file with [sha256]; whether it is
|
||||
/// stored now. A file whose content has another hash is deleted, and so is
|
||||
/// one over [maxOtherBytes] that isn't a PDF, and one that no longer fits,
|
||||
/// with a [FileLimitExceededException].
|
||||
Future<bool> finishUpload(Session session, String sha256) async {
|
||||
_checkHash(sha256);
|
||||
final path = assetPath(_caller(session), sha256);
|
||||
if (!await session.storage.verifyUpload(
|
||||
storageId: assetStorage,
|
||||
path: path,
|
||||
)) {
|
||||
return false;
|
||||
}
|
||||
final bytes = await session.storage.retrieveFile(
|
||||
storageId: assetStorage,
|
||||
path: path,
|
||||
);
|
||||
if (_hashOf(bytes) != sha256 ||
|
||||
(bytes.lengthInBytes > maxOtherBytes && !_isPdf(bytes))) {
|
||||
await session.storage.deleteFile(storageId: assetStorage, path: path);
|
||||
return false;
|
||||
}
|
||||
// Checked again now that it is stored: uploads started side by side each
|
||||
// fitted on their own.
|
||||
try {
|
||||
checkRoom(
|
||||
await roomOf(session, _caller(session), finishedOnly: true),
|
||||
0,
|
||||
);
|
||||
} on FileLimitExceededException {
|
||||
await session.storage.deleteFile(storageId: assetStorage, path: path);
|
||||
rethrow;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/// How much room the caller's files take, and may take.
|
||||
Future<FileUsage> usage(Session session) async {
|
||||
final room = await roomOf(session, _caller(session));
|
||||
return FileUsage(usedBytes: room.used, limitBytes: effectiveLimit(room));
|
||||
}
|
||||
|
||||
/// The file of the asset [assetId]. Throws a [NotFoundException] when the
|
||||
/// asset isn't in a space the caller may read, or its file isn't uploaded
|
||||
/// yet.
|
||||
Future<ByteData> download(Session session, UuidValue assetId) async {
|
||||
final asset = await Asset.db.findById(session, assetId);
|
||||
final spaceId = asset?.spaceId;
|
||||
final space = spaceId == null
|
||||
? null
|
||||
: await OfflineSyncSpace.db.findById(session, spaceId);
|
||||
final caller = _caller(session);
|
||||
if (asset == null ||
|
||||
space == null ||
|
||||
!(space.uuidSpaceId == caller ||
|
||||
await OfflineSyncSpaces(
|
||||
session,
|
||||
).roleOf(user: caller, space: space.uuidSpaceId) !=
|
||||
null)) {
|
||||
throw NotFoundException(what: 'file');
|
||||
}
|
||||
try {
|
||||
return await session.storage.retrieveFile(
|
||||
storageId: assetStorage,
|
||||
path: assetPath(space.uuidSpaceId, asset.sha256),
|
||||
);
|
||||
} on CloudStorageFileNotFoundException {
|
||||
throw NotFoundException(what: 'file');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Where the file with [sha256] of the space [space] is stored.
|
||||
String assetPath(UuidValue space, String sha256) =>
|
||||
'$assetPathPrefix${space.uuid}/$sha256';
|
||||
|
||||
/// The start of every stored file's path; a space's files are below
|
||||
/// `<prefix><space>/`.
|
||||
const assetPathPrefix = 'spaces/';
|
||||
|
||||
/// The storage of every person's files.
|
||||
const assetStorage = 'private';
|
||||
|
||||
UuidValue _caller(Session session) =>
|
||||
UuidValue.withValidation(session.authenticated!.userIdentifier);
|
||||
|
||||
void _checkHash(String sha256) {
|
||||
if (!RegExp(r'^[0-9a-f]{64}$').hasMatch(sha256)) {
|
||||
throw ArgumentError.value(sha256, 'sha256', 'is no SHA-256 in hex');
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether [bytes] are a PDF's: they start as every PDF does.
|
||||
bool _isPdf(ByteData bytes) {
|
||||
const signature = '%PDF-';
|
||||
return bytes.lengthInBytes >= signature.length &&
|
||||
String.fromCharCodes(
|
||||
Uint8List.sublistView(bytes, 0, signature.length),
|
||||
) ==
|
||||
signature;
|
||||
}
|
||||
|
||||
String _hashOf(ByteData bytes) =>
|
||||
sha256.convert(Uint8List.sublistView(bytes)).toString();
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
### More room for one person's files than the instance's default (decided
|
||||
### 2026-10-02): the operator grants it on request, by adding a row. Only the
|
||||
### server reads it; deleting the person deletes it.
|
||||
class: FileAllowance
|
||||
table: file_allowance
|
||||
fields:
|
||||
### The person: the AuthUser's id.
|
||||
authUserId: UuidValue
|
||||
### How many bytes of files they may keep.
|
||||
maxBytes: int
|
||||
### Why, for the operator (e.g. "asked by mail, 2026-10-02").
|
||||
note: String?
|
||||
indexes:
|
||||
file_allowance_person_idx:
|
||||
fields: authUserId
|
||||
unique: true
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
### A file doesn't fit: it would take the caller past what they may keep, or the
|
||||
### instance past what it keeps in all. Nothing was stored.
|
||||
exception: FileLimitExceededException
|
||||
fields:
|
||||
### Whether the instance is full, rather than the caller's own room.
|
||||
instanceFull: bool
|
||||
### What the caller may keep, in bytes.
|
||||
limitBytes: int
|
||||
### What the caller keeps now, in bytes.
|
||||
usedBytes: int
|
||||
145
apps/learn/halkyon_learn_server/lib/src/asset/file_limits.dart
Normal file
145
apps/learn/halkyon_learn_server/lib/src/asset/file_limits.dart
Normal file
|
|
@ -0,0 +1,145 @@
|
|||
import 'dart:io';
|
||||
|
||||
import 'package:halkyon_base_server/base.dart';
|
||||
import 'package:halkyon_learn_server/src/asset/asset_endpoint.dart';
|
||||
import 'package:halkyon_learn_server/src/generated/protocol.dart';
|
||||
import 'package:serverpod/serverpod.dart';
|
||||
|
||||
/// How much room people's files may take (decided 2026-10-02): an anonymous
|
||||
/// person little, since anonymous accounts are meant to become identified
|
||||
/// ones; an identified person more, and more again by a `FileAllowance` the
|
||||
/// operator grants; and all of them together no more than the instance has
|
||||
/// room for. Each comes from the environment, in MiB, with a default:
|
||||
/// `FILE_LIMIT_ANONYMOUS_MB` (100), `FILE_LIMIT_IDENTIFIED_MB` (500) and
|
||||
/// `FILE_LIMIT_INSTANCE_MB` (15000; 0 for none). Every file is in the
|
||||
/// database and in each dump the server keeps: with three dumps, and a
|
||||
/// fourth while a deploy writes its own, files take five times their size on
|
||||
/// the disk.
|
||||
class FileLimits {
|
||||
/// Limits of [anonymousBytes], [identifiedBytes] and [instanceBytes] (null
|
||||
/// for none).
|
||||
const FileLimits({
|
||||
required this.anonymousBytes,
|
||||
required this.identifiedBytes,
|
||||
required this.instanceBytes,
|
||||
});
|
||||
|
||||
/// The limits the environment sets, else the defaults.
|
||||
factory FileLimits.fromEnvironment([Map<String, String>? environment]) {
|
||||
final env = environment ?? Platform.environment;
|
||||
int mib(String name, int fallback) =>
|
||||
(int.tryParse(env[name] ?? '') ?? fallback) * _mib;
|
||||
final instance = mib('FILE_LIMIT_INSTANCE_MB', 15000);
|
||||
return FileLimits(
|
||||
anonymousBytes: mib('FILE_LIMIT_ANONYMOUS_MB', 100),
|
||||
identifiedBytes: mib('FILE_LIMIT_IDENTIFIED_MB', 500),
|
||||
instanceBytes: instance > 0 ? instance : null,
|
||||
);
|
||||
}
|
||||
|
||||
/// What an anonymous person may keep, in bytes.
|
||||
final int anonymousBytes;
|
||||
|
||||
/// What an identified person may keep without an allowance, in bytes.
|
||||
final int identifiedBytes;
|
||||
|
||||
/// What everyone together may keep, in bytes; null for no limit.
|
||||
final int? instanceBytes;
|
||||
}
|
||||
|
||||
/// The limits the server works with.
|
||||
FileLimits get fileLimits => _fileLimits ??= FileLimits.fromEnvironment();
|
||||
|
||||
/// Makes [limits] the ones the server works with, for tests; null goes back
|
||||
/// to the environment's.
|
||||
set fileLimits(FileLimits? limits) => _fileLimits = limits;
|
||||
|
||||
FileLimits? _fileLimits;
|
||||
|
||||
const _mib = 1024 * 1024;
|
||||
|
||||
/// How much room the caller has: what their files take, what they may keep,
|
||||
/// and what the instance's files take in all.
|
||||
typedef Room = ({int used, int limit, int instanceUsed});
|
||||
|
||||
/// The room of the caller of [session], whose space is [space]. Uploads not
|
||||
/// finished yet take room too, unless [finishedOnly]: an upload starting
|
||||
/// counts them, or unfinished uploads could fill the disk unnoticed; an
|
||||
/// upload finishing counts only what is kept, itself included.
|
||||
Future<Room> roomOf(
|
||||
Session session,
|
||||
UuidValue space, {
|
||||
bool finishedOnly = false,
|
||||
}) async {
|
||||
final limits = fileLimits;
|
||||
final person = UuidValue.withValidation(
|
||||
session.authenticated!.userIdentifier,
|
||||
);
|
||||
final allowance = await FileAllowance.db.findFirstRow(
|
||||
session,
|
||||
where: (t) => t.authUserId.equals(person),
|
||||
);
|
||||
return (
|
||||
used: await _storedBytes(
|
||||
session,
|
||||
'$assetPathPrefix${space.uuid}/',
|
||||
finishedOnly: finishedOnly,
|
||||
),
|
||||
limit:
|
||||
allowance?.maxBytes ??
|
||||
(session.authenticated!.scopes.contains(identified)
|
||||
? limits.identifiedBytes
|
||||
: limits.anonymousBytes),
|
||||
instanceUsed: await _storedBytes(
|
||||
session,
|
||||
assetPathPrefix,
|
||||
finishedOnly: finishedOnly,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
/// Throws a [FileLimitExceededException] unless [room] takes [bytes] more.
|
||||
void checkRoom(Room room, int bytes) {
|
||||
final instance = fileLimits.instanceBytes;
|
||||
final instanceFull = instance != null && room.instanceUsed + bytes > instance;
|
||||
if (instanceFull || room.used + bytes > room.limit) {
|
||||
throw FileLimitExceededException(
|
||||
instanceFull: instanceFull,
|
||||
limitBytes: room.limit,
|
||||
usedBytes: room.used,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// What the caller may keep in all: their limit, or less when the instance
|
||||
/// has less room left.
|
||||
int effectiveLimit(Room room) {
|
||||
final instance = fileLimits.instanceBytes;
|
||||
if (instance == null) {
|
||||
return room.limit;
|
||||
}
|
||||
final left = instance - room.instanceUsed;
|
||||
return room.used + left < room.limit ? room.used + left : room.limit;
|
||||
}
|
||||
|
||||
/// What the stored files below [prefix] take: unfinished uploads included,
|
||||
/// unless [finishedOnly].
|
||||
Future<int> _storedBytes(
|
||||
Session session,
|
||||
String prefix, {
|
||||
required bool finishedOnly,
|
||||
}) async {
|
||||
// SELECT SUM(octet_length("byteData")) FROM "serverpod_cloud_storage"
|
||||
// WHERE "storageId" = 'private' AND "path" LIKE $1 [AND "verified"]
|
||||
final rows = await session.db.unsafeQuery(
|
||||
'SELECT COALESCE(SUM(octet_length("byteData")), 0)::bigint '
|
||||
'FROM "serverpod_cloud_storage" '
|
||||
'WHERE "storageId" = @storage AND "path" LIKE @prefix'
|
||||
'${finishedOnly ? ' AND "verified"' : ''}',
|
||||
parameters: QueryParameters.named({
|
||||
'storage': assetStorage,
|
||||
'prefix': '$prefix%',
|
||||
}),
|
||||
);
|
||||
return rows.first.first! as int;
|
||||
}
|
||||
|
|
@ -0,0 +1,99 @@
|
|||
import 'package:halkyon_learn_server/src/asset/asset_endpoint.dart';
|
||||
import 'package:halkyon_learn_server/src/generated/protocol.dart';
|
||||
import 'package:serverpod/serverpod.dart';
|
||||
import 'package:serverpod_offline_sync_server/serverpod_offline_sync_server.dart';
|
||||
|
||||
/// How long a stored file stays after it was last claimed (uploaded, or
|
||||
/// asked for again), whether or not an asset row refers to it: a device
|
||||
/// uploads a file around the time its row syncs, not necessarily after.
|
||||
const filePruningGrace = Duration(days: 7);
|
||||
|
||||
/// Deletes the stored files nobody can reach any more (ADR 0004, rule 5), at
|
||||
/// most [limit] per call, and returns how many: last claimed before
|
||||
/// [filePruningGrace] ago, and either never verified (an upload that wasn't
|
||||
/// finished) or not referred to by any asset row of their space that still
|
||||
/// exists (a deleted row is a tombstone, which the sync engine hides).
|
||||
Future<int> pruneUnreferencedFiles(
|
||||
Session session, {
|
||||
required DateTime now,
|
||||
int limit = 500,
|
||||
}) async {
|
||||
// The columns hold UTC as timestamp without time zone.
|
||||
final timestamp = now
|
||||
.toUtc()
|
||||
.subtract(filePruningGrace)
|
||||
.toIso8601String()
|
||||
.replaceFirst('Z', '');
|
||||
final candidates = await session.db.unsafeQuery(
|
||||
'SELECT "path", "verified" FROM "serverpod_cloud_storage" '
|
||||
'WHERE "storageId" = @storage AND "path" LIKE @prefix '
|
||||
'AND "addedTime" < @cutoff::timestamp '
|
||||
'ORDER BY "addedTime" LIMIT @limit',
|
||||
parameters: QueryParameters.named({
|
||||
'storage': assetStorage,
|
||||
'prefix': '$assetPathPrefix%',
|
||||
'cutoff': timestamp,
|
||||
'limit': limit,
|
||||
}),
|
||||
);
|
||||
|
||||
// The candidates' hashes, by space; a path of another form isn't ours to
|
||||
// judge.
|
||||
final hashesBySpace = <UuidValue, Set<String>>{};
|
||||
final unreferenced = <String>{};
|
||||
for (final [path as String, verified as bool] in candidates) {
|
||||
final match = _assetPathPattern.firstMatch(path);
|
||||
if (match == null) {
|
||||
continue;
|
||||
}
|
||||
if (!verified) {
|
||||
unreferenced.add(path);
|
||||
} else {
|
||||
(hashesBySpace[UuidValue.withValidation(match[1]!)] ??= {}).add(
|
||||
match[2]!,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
for (final MapEntry(key: space, value: hashes) in hashesBySpace.entries) {
|
||||
final spaceId = (await OfflineSyncSpace.db.findFirstRow(
|
||||
session,
|
||||
where: (t) => t.uuidSpaceId.equals(space),
|
||||
))?.id;
|
||||
final referenced = spaceId == null
|
||||
? const <String>{}
|
||||
: {
|
||||
for (final asset in await Asset.db.find(
|
||||
session,
|
||||
where: (t) => t.spaceId.equals(spaceId) & t.sha256.inSet(hashes),
|
||||
))
|
||||
asset.sha256,
|
||||
};
|
||||
unreferenced.addAll([
|
||||
for (final hash in hashes.difference(referenced)) assetPath(space, hash),
|
||||
]);
|
||||
}
|
||||
if (unreferenced.isEmpty) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
// A file claimed again meanwhile stays. Plain SQL, since the ORM's delete
|
||||
// would read every deleted file to return it.
|
||||
return await session.db.unsafeExecute(
|
||||
'DELETE FROM "serverpod_cloud_storage" '
|
||||
'WHERE "storageId" = @storage AND "path" = ANY(@paths::text[]) '
|
||||
'AND "addedTime" < @cutoff::timestamp',
|
||||
parameters: QueryParameters.named({
|
||||
'storage': assetStorage,
|
||||
'paths': unreferenced.toList(),
|
||||
'cutoff': timestamp,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
/// A stored file's path: `spaces/<space>/<sha256>`.
|
||||
final _assetPathPattern = RegExp(
|
||||
'^${RegExp.escape(assetPathPrefix)}'
|
||||
'([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})/'
|
||||
r'([0-9a-f]{64})$',
|
||||
);
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
import 'package:clock/clock.dart';
|
||||
import 'package:halkyon_learn_server/src/asset/file_pruning.dart';
|
||||
import 'package:serverpod/serverpod.dart';
|
||||
|
||||
/// Deletes the stored files nobody can reach any more; the server schedules
|
||||
/// it daily when it starts (`server.dart`).
|
||||
class FilePruningFutureCall extends FutureCall {
|
||||
/// Prunes one batch and logs how many files went.
|
||||
Future<void> pruneFiles(Session session) async {
|
||||
final pruned = await pruneUnreferencedFiles(session, now: clock.now());
|
||||
session.log('Pruned $pruned unreferenced files.');
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
### How much room the caller's files take, and how much they may take.
|
||||
class: FileUsage
|
||||
fields:
|
||||
### What their files take now, in bytes.
|
||||
usedBytes: int
|
||||
### What they may keep, in bytes: their own limit, or less when the instance
|
||||
### has less room left.
|
||||
limitBytes: int
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
### An app that registered to connect people's studysets (ADR 0006): Claude, or
|
||||
### another MCP client, through OAuth dynamic client registration (RFC 7591).
|
||||
### Anyone may register one; it can do nothing until a person connects it with
|
||||
### a code from their app. One nobody connected is pruned after a week.
|
||||
class: AssistantClient
|
||||
table: assistant_client
|
||||
serverOnly: true
|
||||
fields:
|
||||
### The client_id given out: random and unguessable, never reused.
|
||||
clientId: String
|
||||
### What it calls itself, shown when a person connects it.
|
||||
name: String
|
||||
### Where authorization may send the person back to, compared exactly.
|
||||
redirectUris: List<String>
|
||||
### SHA-256 of its secret, for a client that authenticates with one at the
|
||||
### token endpoint; null for a public client (PKCE alone).
|
||||
secretHash: String?
|
||||
### When it registered.
|
||||
createdAt: DateTime
|
||||
indexes:
|
||||
assistant_client_id_idx:
|
||||
fields: clientId
|
||||
unique: true
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
### A code for connecting an assistant, as the app shows it (ADR 0006, rule 3).
|
||||
class: AssistantCode
|
||||
fields:
|
||||
### The code, as `K7QX-M2PD`.
|
||||
code: String
|
||||
### When it stops working, in UTC.
|
||||
expiresAt: DateTime
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
### A code a person's app showed for connecting an assistant (ADR 0006, rule 3):
|
||||
### eight characters, valid for ten minutes, used once. Kept only as a hash, so
|
||||
### the table never holds a code that works.
|
||||
class: AssistantConnectCode
|
||||
table: assistant_connect_code
|
||||
serverOnly: true
|
||||
fields:
|
||||
### Whose account the code connects.
|
||||
authUserId: UuidValue
|
||||
### SHA-256 of the code as typed, normalized (`normalizeConnectCode`).
|
||||
codeHash: String
|
||||
### When it stops working, in UTC.
|
||||
expiresAt: DateTime
|
||||
indexes:
|
||||
assistant_connect_code_hash_idx:
|
||||
fields: codeHash
|
||||
unique: true
|
||||
assistant_connect_code_person_idx:
|
||||
fields: authUserId
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
### A person's grant to one assistant (ADR 0006, rule 5): made when they enter a
|
||||
### code on the client's authorization page. The app lists them, and
|
||||
### disconnecting one deletes it with its codes and tokens.
|
||||
class: AssistantConnection
|
||||
table: assistant_connection
|
||||
serverOnly: true
|
||||
fields:
|
||||
### Whose content the assistant reads.
|
||||
authUserId: UuidValue
|
||||
### The assistant; the connection goes with it.
|
||||
client: AssistantClient?, relation(onDelete=Cascade)
|
||||
### When the person connected it.
|
||||
createdAt: DateTime
|
||||
### When the assistant last read something, to the minute; null before.
|
||||
lastUsedAt: DateTime?
|
||||
indexes:
|
||||
assistant_connection_person_idx:
|
||||
fields: authUserId
|
||||
# Deleting a client cascades through this index; Postgres doesn't index
|
||||
# foreign keys by itself.
|
||||
assistant_connection_client_idx:
|
||||
fields: clientId
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
### A connected assistant, as the app lists it.
|
||||
class: AssistantConnectionInfo
|
||||
fields:
|
||||
### The connection's id, to disconnect it.
|
||||
id: int
|
||||
### What the assistant calls itself.
|
||||
name: String
|
||||
### The host the person was sent back to on connecting, e.g. `claude.ai`.
|
||||
host: String
|
||||
### When the person connected it.
|
||||
connectedAt: DateTime
|
||||
### When it last read something; null before.
|
||||
lastUsedAt: DateTime?
|
||||
|
|
@ -0,0 +1,113 @@
|
|||
import 'package:clock/clock.dart';
|
||||
import 'package:halkyon_base_server/base.dart';
|
||||
import 'package:halkyon_learn_server/src/assistant/connector_oauth.dart';
|
||||
import 'package:halkyon_learn_server/src/assistant/connector_secrets.dart';
|
||||
import 'package:halkyon_learn_server/src/generated/protocol.dart';
|
||||
import 'package:serverpod/serverpod.dart';
|
||||
import 'package:serverpod_auth_idp_server/core.dart';
|
||||
|
||||
/// Connecting assistants to the signed-in person's account (ADR 0006): a
|
||||
/// code to type on an assistant's authorization page, and the assistants
|
||||
/// connected, to disconnect. Anonymous people too: the code is how they
|
||||
/// connect without signing in anywhere.
|
||||
class AssistantEndpoint extends Endpoint {
|
||||
@override
|
||||
bool get requireLogin => true;
|
||||
|
||||
/// A new code for connecting an assistant, valid for ten minutes and
|
||||
/// once; the person's earlier codes stop working. Only its hash is kept.
|
||||
Future<AssistantCode> connectCode(Session session) async {
|
||||
final person = session.authenticated!.authUserId;
|
||||
final code = newShortCode();
|
||||
final expiresAt = clock.now().toUtc().add(connectCodeLifetime);
|
||||
await session.db.transaction((transaction) async {
|
||||
// DELETE FROM "assistant_connect_code" WHERE "authUserId" = $1
|
||||
await AssistantConnectCode.db.deleteWhere(
|
||||
session,
|
||||
where: (t) => t.authUserId.equals(person),
|
||||
transaction: transaction,
|
||||
);
|
||||
await AssistantConnectCode.db.insertRow(
|
||||
session,
|
||||
AssistantConnectCode(
|
||||
authUserId: person,
|
||||
codeHash: hashSecret(normalizeShortCode(code)),
|
||||
expiresAt: expiresAt,
|
||||
),
|
||||
transaction: transaction,
|
||||
);
|
||||
});
|
||||
return AssistantCode(code: code, expiresAt: expiresAt);
|
||||
}
|
||||
|
||||
/// The assistants connected to the person's account, the newest first.
|
||||
Future<List<AssistantConnectionInfo>> connections(Session session) async {
|
||||
final person = session.authenticated!.authUserId;
|
||||
final connections = await AssistantConnection.db.find(
|
||||
session,
|
||||
where: (t) => t.authUserId.equals(person),
|
||||
orderBy: (t) => t.createdAt.desc(),
|
||||
include: AssistantConnection.include(client: AssistantClient.include()),
|
||||
);
|
||||
return [
|
||||
for (final connection in connections)
|
||||
AssistantConnectionInfo(
|
||||
id: connection.id!,
|
||||
name: connection.client?.name ?? '',
|
||||
host: switch (connection.client?.redirectUris.firstOrNull) {
|
||||
final uri? => Uri.tryParse(uri)?.host ?? '',
|
||||
null => '',
|
||||
},
|
||||
connectedAt: connection.createdAt,
|
||||
lastUsedAt: connection.lastUsedAt,
|
||||
),
|
||||
];
|
||||
}
|
||||
|
||||
/// Disconnects the person's assistant [id]: its tokens stop working at
|
||||
/// once. Someone else's connection is one that doesn't exist.
|
||||
Future<void> disconnect(Session session, int id) async {
|
||||
final person = session.authenticated!.authUserId;
|
||||
// DELETE FROM "assistant_connection" WHERE "id" = $1 AND "authUserId" = $2
|
||||
// (its grants and tokens cascade)
|
||||
final deleted = await AssistantConnection.db.deleteWhere(
|
||||
session,
|
||||
where: (t) => t.id.equals(id) & t.authUserId.equals(person),
|
||||
);
|
||||
if (deleted.isEmpty) {
|
||||
throw NotFoundException(what: 'connection');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Deletes what the connector no longer needs (ADR 0006, rule 5), as of
|
||||
/// [now], and returns how many rows went: codes, authorization codes and
|
||||
/// tokens past their time, and clients nobody connected within a week.
|
||||
Future<int> forgetExpiredConnectorSecrets(
|
||||
Session session, {
|
||||
required DateTime now,
|
||||
}) async {
|
||||
final codes = await AssistantConnectCode.db.deleteWhere(
|
||||
session,
|
||||
where: (t) => t.expiresAt < now,
|
||||
);
|
||||
final grants = await AssistantGrant.db.deleteWhere(
|
||||
session,
|
||||
where: (t) => t.expiresAt < now,
|
||||
);
|
||||
final tokens = await AssistantToken.db.deleteWhere(
|
||||
session,
|
||||
where: (t) => t.refreshExpiresAt < now,
|
||||
);
|
||||
final connected = {
|
||||
for (final connection in await AssistantConnection.db.find(session))
|
||||
connection.clientId,
|
||||
};
|
||||
final clients = await AssistantClient.db.deleteWhere(
|
||||
session,
|
||||
where: (t) =>
|
||||
(t.createdAt < now.subtract(unconnectedClientLifetime)) &
|
||||
t.id.notInSet(connected),
|
||||
);
|
||||
return codes.length + grants.length + tokens.length + clients.length;
|
||||
}
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
### An authorization code on its way to the assistant (OAuth 2.1, section 4.1):
|
||||
### valid for five minutes and used once, bound to the redirect URI, the PKCE
|
||||
### challenge and the resource it was asked for. Kept only as a hash.
|
||||
class: AssistantGrant
|
||||
table: assistant_grant
|
||||
serverOnly: true
|
||||
fields:
|
||||
### The connection the code was made for; it goes with it.
|
||||
connection: AssistantConnection?, relation(onDelete=Cascade)
|
||||
### SHA-256 of the code.
|
||||
codeHash: String
|
||||
### The redirect URI the authorization request named; the token request
|
||||
### has to name it again.
|
||||
redirectUri: String
|
||||
### The PKCE challenge (S256) the token request's verifier has to match.
|
||||
codeChallenge: String
|
||||
### The MCP server the tokens are for (RFC 8707).
|
||||
resource: String
|
||||
### The scopes granted, space-separated.
|
||||
scope: String
|
||||
### When the code stops working, in UTC.
|
||||
expiresAt: DateTime
|
||||
indexes:
|
||||
assistant_grant_hash_idx:
|
||||
fields: codeHash
|
||||
unique: true
|
||||
# Disconnecting cascades through this index.
|
||||
assistant_grant_connection_idx:
|
||||
fields: connectionId
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
### An assistant's access and refresh token for one connection (ADR 0006, rule
|
||||
### 5): the access token works for an hour, the refresh token for 30 days and
|
||||
### only once, each use giving a new pair. Kept only as hashes.
|
||||
class: AssistantToken
|
||||
table: assistant_token
|
||||
serverOnly: true
|
||||
fields:
|
||||
### The connection; the tokens go with it.
|
||||
connection: AssistantConnection?, relation(onDelete=Cascade)
|
||||
### SHA-256 of the access token.
|
||||
accessHash: String
|
||||
### When the access token stops working, in UTC.
|
||||
accessExpiresAt: DateTime
|
||||
### SHA-256 of the refresh token.
|
||||
refreshHash: String
|
||||
### When the refresh token stops working, in UTC.
|
||||
refreshExpiresAt: DateTime
|
||||
### The MCP server the tokens are for (RFC 8707); the server refuses others.
|
||||
resource: String
|
||||
### The scopes granted, space-separated.
|
||||
scope: String
|
||||
indexes:
|
||||
assistant_token_access_idx:
|
||||
fields: accessHash
|
||||
unique: true
|
||||
assistant_token_refresh_idx:
|
||||
fields: refreshHash
|
||||
unique: true
|
||||
# Disconnecting cascades through this index.
|
||||
assistant_token_connection_idx:
|
||||
fields: connectionId
|
||||
|
|
@ -0,0 +1,90 @@
|
|||
import 'dart:convert';
|
||||
|
||||
/// An HTTP request to the assistant connector as its handlers read it:
|
||||
/// plain values, so tests can make one. The web server's routes
|
||||
/// (`connector_routes.dart`) turn Relic's requests into these.
|
||||
typedef ConnectorRequest = ({
|
||||
/// The method, upper case: `GET`, `POST`.
|
||||
String method,
|
||||
|
||||
/// The query's parameters, each its first value.
|
||||
Map<String, String> query,
|
||||
|
||||
/// The headers, by lower-case name, several values joined by commas.
|
||||
Map<String, String> headers,
|
||||
|
||||
/// The body as text; empty without.
|
||||
String body,
|
||||
});
|
||||
|
||||
/// A request with [method], [query], [headers] and [body], for tests and
|
||||
/// the routes.
|
||||
ConnectorRequest connectorRequest(
|
||||
String method, {
|
||||
Map<String, String> query = const {},
|
||||
Map<String, String> headers = const {},
|
||||
String body = '',
|
||||
}) => (
|
||||
method: method,
|
||||
query: query,
|
||||
headers: {
|
||||
for (final MapEntry(:key, :value) in headers.entries)
|
||||
key.toLowerCase(): value,
|
||||
},
|
||||
body: body,
|
||||
);
|
||||
|
||||
/// What a connector handler answers.
|
||||
class const ConnectorResponse(
|
||||
/// The HTTP status.
|
||||
final int status, {
|
||||
|
||||
/// The body.
|
||||
final String body = '',
|
||||
|
||||
/// Its media type.
|
||||
final String contentType = 'application/json',
|
||||
|
||||
/// Further headers, by name.
|
||||
final Map<String, String> headers = const {},
|
||||
}) {
|
||||
/// [json] as the body, kept by nobody along the way: OAuth answers carry
|
||||
/// tokens (RFC 6749, 5.1).
|
||||
factory json(
|
||||
int status,
|
||||
Object? json, {
|
||||
Map<String, String> headers = const {},
|
||||
}) => ConnectorResponse(
|
||||
status,
|
||||
body: jsonEncode(json),
|
||||
headers: {'Cache-Control': 'no-store', ...headers},
|
||||
);
|
||||
|
||||
/// An OAuth error (RFC 6749, 5.2): [error] and what went wrong.
|
||||
factory oauthError(int status, String error, String description) =>
|
||||
ConnectorResponse.json(status, {
|
||||
'error': error,
|
||||
'error_description': description,
|
||||
});
|
||||
|
||||
/// A redirect to [location].
|
||||
factory redirect(Uri location) => ConnectorResponse(
|
||||
302,
|
||||
contentType: 'text/plain',
|
||||
headers: {'Location': '$location', 'Cache-Control': 'no-store'},
|
||||
);
|
||||
}
|
||||
|
||||
/// The form fields of [request]'s body where it is
|
||||
/// `application/x-www-form-urlencoded`; empty for anything else.
|
||||
Map<String, String> formOf(ConnectorRequest request) {
|
||||
final type = request.headers['content-type'] ?? '';
|
||||
if (!type.toLowerCase().startsWith('application/x-www-form-urlencoded')) {
|
||||
return const {};
|
||||
}
|
||||
try {
|
||||
return Uri.splitQueryString(request.body);
|
||||
} on FormatException {
|
||||
return const {};
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,57 @@
|
|||
import 'package:halkyon_learn_server/src/assistant/connector_http.dart';
|
||||
import 'package:halkyon_learn_server/src/assistant/connector_oauth.dart';
|
||||
import 'package:halkyon_learn_server/src/assistant/connector_tools.dart';
|
||||
import 'package:halkyon_learn_server/src/assistant/mcp_protocol.dart';
|
||||
import 'package:serverpod/serverpod.dart';
|
||||
|
||||
/// The MCP endpoint (ADR 0006, rule 1): Streamable HTTP, each POST one
|
||||
/// JSON-RPC message, answered with JSON; no sessions and no stream, so GET
|
||||
/// and DELETE are refused (405), as the transport allows. Every request
|
||||
/// carries an access token of [connector], and the tools read that
|
||||
/// connection's person only.
|
||||
Future<ConnectorResponse> answerMcpRequest(
|
||||
Session session,
|
||||
ConnectorRequest request,
|
||||
Connector connector,
|
||||
) async {
|
||||
if (request.method != 'POST') {
|
||||
return const ConnectorResponse(
|
||||
405,
|
||||
contentType: 'text/plain',
|
||||
headers: {'Allow': 'POST'},
|
||||
);
|
||||
}
|
||||
// A browser's page on a plain-HTTP origin is how DNS rebinding would
|
||||
// reach a server; clients call from their own servers, or from https.
|
||||
if (request.headers['origin'] case final origin?
|
||||
when !origin.startsWith('https://')) {
|
||||
return ConnectorResponse.json(403, {'error': 'origin not allowed'});
|
||||
}
|
||||
if (request.headers['mcp-protocol-version'] case final version?
|
||||
when !mcpProtocolVersions.contains(version)) {
|
||||
return ConnectorResponse.json(400, {
|
||||
'error': 'unsupported protocol version',
|
||||
'supported': mcpProtocolVersions,
|
||||
});
|
||||
}
|
||||
final connection = await connector.connectionOf(
|
||||
session,
|
||||
request.headers['authorization'],
|
||||
);
|
||||
if (connection == null) {
|
||||
return connector.unauthorized();
|
||||
}
|
||||
final (:message, :parseError) = decodeMcp(request.body);
|
||||
if (parseError != null) {
|
||||
return ConnectorResponse.json(400, parseError);
|
||||
}
|
||||
final answer = await answerMcp(
|
||||
message,
|
||||
tools: connectorTools(session, connection.authUserId),
|
||||
server: (name: 'halkyon-learn', title: 'Halkyon Learn', version: '1'),
|
||||
instructions: connectorInstructions,
|
||||
);
|
||||
return answer == null
|
||||
? const ConnectorResponse(202, contentType: 'text/plain')
|
||||
: ConnectorResponse.json(200, answer);
|
||||
}
|
||||
|
|
@ -0,0 +1,585 @@
|
|||
import 'dart:convert';
|
||||
|
||||
import 'package:clock/clock.dart';
|
||||
import 'package:halkyon_base_server/base.dart';
|
||||
import 'package:halkyon_learn_server/src/assistant/connector_http.dart';
|
||||
import 'package:halkyon_learn_server/src/assistant/connector_page.dart';
|
||||
import 'package:halkyon_learn_server/src/assistant/connector_secrets.dart';
|
||||
import 'package:halkyon_learn_server/src/generated/protocol.dart';
|
||||
import 'package:serverpod/serverpod.dart';
|
||||
|
||||
/// How long a connect code works (ADR 0006, rule 3).
|
||||
const connectCodeLifetime = Duration(minutes: 10);
|
||||
|
||||
/// How long an authorization code works.
|
||||
const grantLifetime = Duration(minutes: 5);
|
||||
|
||||
/// How long an access token works.
|
||||
const accessTokenLifetime = Duration(hours: 1);
|
||||
|
||||
/// How long a refresh token works, from the last refresh.
|
||||
const refreshTokenLifetime = Duration(days: 30);
|
||||
|
||||
/// How long a client nobody connected stays registered.
|
||||
const unconnectedClientLifetime = Duration(days: 7);
|
||||
|
||||
/// The one scope: reading the person's content.
|
||||
const readScope = 'studysets:read';
|
||||
|
||||
/// How many clients may register in an hour: registration is open to
|
||||
/// anyone, so this keeps a flood from filling the table before pruning
|
||||
/// (clients nobody connects go after a week). Claude registers once per
|
||||
/// person adding the connector.
|
||||
const maxRegistrationsPerHour = 200;
|
||||
|
||||
/// The OAuth side of the assistant connector (ADR 0006): its metadata, client
|
||||
/// registration, the authorization page, the token endpoint, and checking
|
||||
/// a bearer token. [base] is where the web server is reached from outside,
|
||||
/// as `https://learn.halkyon.eu`, without a slash at the end; the MCP
|
||||
/// endpoint is [resource]. [instance] names the instance on the page.
|
||||
class Connector({required final String base, required final String instance}) {
|
||||
/// The MCP endpoint, as the tokens are bound to it (RFC 8707).
|
||||
String get resource => '$base/mcp';
|
||||
|
||||
/// Where the protected resource metadata is (RFC 9728), as 401s name it.
|
||||
String get resourceMetadataUrl =>
|
||||
'$base/.well-known/oauth-protected-resource';
|
||||
|
||||
/// The protected resource metadata (RFC 9728): the MCP endpoint, and this
|
||||
/// instance as its authorization server.
|
||||
ConnectorResponse resourceMetadata() => ConnectorResponse.json(200, {
|
||||
'resource': resource,
|
||||
'authorization_servers': [base],
|
||||
'scopes_supported': [readScope],
|
||||
'bearer_methods_supported': ['header'],
|
||||
'resource_name': 'Halkyon Learn',
|
||||
});
|
||||
|
||||
/// The authorization server metadata (RFC 8414).
|
||||
ConnectorResponse serverMetadata() => ConnectorResponse.json(200, {
|
||||
'issuer': base,
|
||||
'authorization_endpoint': '$base/oauth/authorize',
|
||||
'token_endpoint': '$base/oauth/token',
|
||||
'registration_endpoint': '$base/oauth/register',
|
||||
'response_types_supported': ['code'],
|
||||
'grant_types_supported': ['authorization_code', 'refresh_token'],
|
||||
'code_challenge_methods_supported': ['S256'],
|
||||
'token_endpoint_auth_methods_supported': [
|
||||
'none',
|
||||
'client_secret_post',
|
||||
'client_secret_basic',
|
||||
],
|
||||
'scopes_supported': [readScope],
|
||||
'authorization_response_iss_parameter_supported': true,
|
||||
// Fetching a client's metadata from its URL would make the instance
|
||||
// request addresses anyone names; clients register instead.
|
||||
'client_id_metadata_document_supported': false,
|
||||
});
|
||||
|
||||
/// Registers a client (RFC 7591): its name and redirect URIs, and a
|
||||
/// secret if it asks to authenticate with one.
|
||||
Future<ConnectorResponse> register(
|
||||
Session session,
|
||||
ConnectorRequest request,
|
||||
) async {
|
||||
final Map<String, Object?> metadata;
|
||||
try {
|
||||
metadata = switch (jsonDecode(request.body)) {
|
||||
final Map<String, Object?> json => json,
|
||||
_ => throw const FormatException(),
|
||||
};
|
||||
} on FormatException {
|
||||
return ConnectorResponse.oauthError(
|
||||
400,
|
||||
'invalid_client_metadata',
|
||||
'The body is not a JSON object.',
|
||||
);
|
||||
}
|
||||
final redirects = switch (metadata['redirect_uris']) {
|
||||
final List<Object?> uris when uris.isNotEmpty && uris.length <= 5 => uris,
|
||||
_ => null,
|
||||
};
|
||||
if (redirects == null || !redirects.every(_acceptableRedirect)) {
|
||||
return ConnectorResponse.oauthError(
|
||||
400,
|
||||
'invalid_redirect_uri',
|
||||
'Give one to five redirect URIs, each https, or http on localhost.',
|
||||
);
|
||||
}
|
||||
final method = switch (metadata['token_endpoint_auth_method']) {
|
||||
null => 'none',
|
||||
final String method when _authMethods.contains(method) => method,
|
||||
_ => null,
|
||||
};
|
||||
if (method == null) {
|
||||
return ConnectorResponse.oauthError(
|
||||
400,
|
||||
'invalid_client_metadata',
|
||||
'token_endpoint_auth_method must be one of ${_authMethods.join(', ')}.',
|
||||
);
|
||||
}
|
||||
if (metadata['grant_types'] case final List<Object?> types
|
||||
when !types.contains('authorization_code')) {
|
||||
return ConnectorResponse.oauthError(
|
||||
400,
|
||||
'invalid_client_metadata',
|
||||
'Only the authorization_code grant is offered.',
|
||||
);
|
||||
}
|
||||
final name = switch (metadata['client_name']) {
|
||||
final String name when name.trim().isNotEmpty => _plain(name, 100),
|
||||
_ => 'MCP client',
|
||||
};
|
||||
final hourAgo = clock.now().toUtc().subtract(const Duration(hours: 1));
|
||||
if (await AssistantClient.db.count(
|
||||
session,
|
||||
where: (t) => t.createdAt > hourAgo,
|
||||
) >=
|
||||
maxRegistrationsPerHour) {
|
||||
return ConnectorResponse.oauthError(
|
||||
429,
|
||||
'temporarily_unavailable',
|
||||
'Too many registrations in the last hour; try again later.',
|
||||
);
|
||||
}
|
||||
final secret = method == 'none' ? null : randomSecret();
|
||||
final client = await AssistantClient.db.insertRow(
|
||||
session,
|
||||
AssistantClient(
|
||||
clientId: randomSecret(16),
|
||||
name: name,
|
||||
redirectUris: [for (final uri in redirects) '$uri'],
|
||||
secretHash: secret == null ? null : hashSecret(secret),
|
||||
createdAt: clock.now().toUtc(),
|
||||
),
|
||||
);
|
||||
return ConnectorResponse.json(201, {
|
||||
'client_id': client.clientId,
|
||||
'client_id_issued_at': client.createdAt.millisecondsSinceEpoch ~/ 1000,
|
||||
'client_secret': ?secret,
|
||||
if (secret != null) 'client_secret_expires_at': 0,
|
||||
'client_name': client.name,
|
||||
'redirect_uris': client.redirectUris,
|
||||
'grant_types': ['authorization_code', 'refresh_token'],
|
||||
'response_types': ['code'],
|
||||
'token_endpoint_auth_method': method,
|
||||
'scope': readScope,
|
||||
});
|
||||
}
|
||||
|
||||
/// The authorization page: asks for a connect code (GET), and grants
|
||||
/// access on a right one (POST), sending the person back to the client.
|
||||
///
|
||||
/// A request naming an unknown client or a redirect URI it didn't
|
||||
/// register gets an error page and never a redirect (RFC 6749, 4.1.2.1);
|
||||
/// other faults go back to the client as errors.
|
||||
Future<ConnectorResponse> authorize(
|
||||
Session session,
|
||||
ConnectorRequest request,
|
||||
) async {
|
||||
final post = request.method == 'POST';
|
||||
final fields = post ? formOf(request) : request.query;
|
||||
final language = pageLanguage(request.headers['accept-language']);
|
||||
final clientId = fields['client_id'];
|
||||
final client = clientId == null
|
||||
? null
|
||||
: await AssistantClient.db.findFirstRow(
|
||||
session,
|
||||
where: (t) => t.clientId.equals(clientId),
|
||||
);
|
||||
final redirect = fields['redirect_uri'];
|
||||
if (client == null ||
|
||||
redirect == null ||
|
||||
!client.redirectUris.contains(redirect)) {
|
||||
return ConnectorResponse(
|
||||
400,
|
||||
contentType: 'text/html; charset=utf-8',
|
||||
body: authorizeProblemPage(language),
|
||||
headers: pageHeaders(null),
|
||||
);
|
||||
}
|
||||
final redirectUri = Uri.parse(redirect);
|
||||
final state = fields['state'];
|
||||
ConnectorResponse refuse(String error, String description) =>
|
||||
ConnectorResponse.redirect(
|
||||
redirectUri.replace(
|
||||
queryParameters: {
|
||||
...redirectUri.queryParameters,
|
||||
'error': error,
|
||||
'error_description': description,
|
||||
'state': ?state,
|
||||
'iss': base,
|
||||
},
|
||||
),
|
||||
);
|
||||
if (fields['response_type'] != 'code') {
|
||||
return refuse('unsupported_response_type', 'Only code is offered.');
|
||||
}
|
||||
final challenge = fields['code_challenge'];
|
||||
if (challenge == null ||
|
||||
challenge.isEmpty ||
|
||||
fields['code_challenge_method'] != 'S256') {
|
||||
return refuse('invalid_request', 'PKCE with S256 is required.');
|
||||
}
|
||||
final asked = fields['resource'];
|
||||
if (asked != null && _withoutSlash(asked) != resource) {
|
||||
return refuse('invalid_target', 'The resource is $resource.');
|
||||
}
|
||||
final scope = fields['scope'];
|
||||
if (scope != null &&
|
||||
scope.trim().isNotEmpty &&
|
||||
!scope.split(' ').every((each) => each.isEmpty || each == readScope)) {
|
||||
return refuse('invalid_scope', 'The scope offered is $readScope.');
|
||||
}
|
||||
final page = (
|
||||
language: language,
|
||||
client: client.name,
|
||||
returnsTo: redirectUri.host.isEmpty ? redirect : redirectUri.host,
|
||||
instance: instance,
|
||||
fields: {
|
||||
for (final name in _carried) name: ?fields[name],
|
||||
},
|
||||
);
|
||||
if (!post) {
|
||||
return ConnectorResponse(
|
||||
200,
|
||||
contentType: 'text/html; charset=utf-8',
|
||||
body: authorizePage(page),
|
||||
headers: pageHeaders(redirectUri),
|
||||
);
|
||||
}
|
||||
final now = clock.now().toUtc();
|
||||
final typed = normalizeShortCode(fields['connect_code'] ?? '');
|
||||
final code = typed.isEmpty
|
||||
? null
|
||||
: await AssistantConnectCode.db.findFirstRow(
|
||||
session,
|
||||
where: (t) =>
|
||||
t.codeHash.equals(hashSecret(typed)) & (t.expiresAt > now),
|
||||
);
|
||||
if (code == null) {
|
||||
return ConnectorResponse(
|
||||
200,
|
||||
contentType: 'text/html; charset=utf-8',
|
||||
body: authorizePage(page, wrongCode: true),
|
||||
headers: pageHeaders(redirectUri),
|
||||
);
|
||||
}
|
||||
final grant = randomSecret();
|
||||
final granted = await session.db.transaction((transaction) async {
|
||||
// Used once: of two requests racing with it, only the one that deletes
|
||||
// it goes on.
|
||||
final spent = await AssistantConnectCode.db.deleteWhere(
|
||||
session,
|
||||
where: (t) => t.id.equals(code.id),
|
||||
transaction: transaction,
|
||||
);
|
||||
if (spent.isEmpty) {
|
||||
return false;
|
||||
}
|
||||
// One connection per person and client: connecting again renews it.
|
||||
final connection =
|
||||
await AssistantConnection.db.findFirstRow(
|
||||
session,
|
||||
where: (t) =>
|
||||
t.authUserId.equals(code.authUserId) &
|
||||
t.clientId.equals(client.id),
|
||||
transaction: transaction,
|
||||
) ??
|
||||
await AssistantConnection.db.insertRow(
|
||||
session,
|
||||
AssistantConnection(
|
||||
authUserId: code.authUserId,
|
||||
clientId: client.id!,
|
||||
createdAt: now,
|
||||
),
|
||||
transaction: transaction,
|
||||
);
|
||||
await AssistantGrant.db.insertRow(
|
||||
session,
|
||||
AssistantGrant(
|
||||
connectionId: connection.id!,
|
||||
codeHash: hashSecret(grant),
|
||||
redirectUri: redirect,
|
||||
codeChallenge: challenge,
|
||||
resource: resource,
|
||||
scope: readScope,
|
||||
expiresAt: now.add(grantLifetime),
|
||||
),
|
||||
transaction: transaction,
|
||||
);
|
||||
return true;
|
||||
});
|
||||
if (!granted) {
|
||||
return ConnectorResponse(
|
||||
200,
|
||||
contentType: 'text/html; charset=utf-8',
|
||||
body: authorizePage(page, wrongCode: true),
|
||||
headers: pageHeaders(redirectUri),
|
||||
);
|
||||
}
|
||||
return ConnectorResponse.redirect(
|
||||
redirectUri.replace(
|
||||
queryParameters: {
|
||||
...redirectUri.queryParameters,
|
||||
'code': grant,
|
||||
'state': ?state,
|
||||
'iss': base,
|
||||
},
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
/// The token endpoint (RFC 6749, 3.2): an authorization code for tokens,
|
||||
/// or a refresh token for new ones, the old one spent.
|
||||
Future<ConnectorResponse> token(
|
||||
Session session,
|
||||
ConnectorRequest request,
|
||||
) async {
|
||||
final form = formOf(request);
|
||||
final now = clock.now().toUtc();
|
||||
final client = await _authenticate(session, request, form);
|
||||
if (client == null) {
|
||||
return ConnectorResponse.oauthError(
|
||||
401,
|
||||
'invalid_client',
|
||||
'The client is unknown, or its secret is wrong.',
|
||||
);
|
||||
}
|
||||
if (form['resource'] case final asked?
|
||||
when _withoutSlash(asked) != resource) {
|
||||
return ConnectorResponse.oauthError(
|
||||
400,
|
||||
'invalid_target',
|
||||
'The resource is $resource.',
|
||||
);
|
||||
}
|
||||
switch (form['grant_type']) {
|
||||
case 'authorization_code':
|
||||
final code = form['code'];
|
||||
final grant = code == null
|
||||
? null
|
||||
: await AssistantGrant.db.findFirstRow(
|
||||
session,
|
||||
where: (t) => t.codeHash.equals(hashSecret(code)),
|
||||
include: AssistantGrant.include(
|
||||
connection: AssistantConnection.include(),
|
||||
),
|
||||
);
|
||||
// Used once, whatever comes of it: of two requests racing with it,
|
||||
// only the one that deletes it goes on.
|
||||
if (grant == null ||
|
||||
(await AssistantGrant.db.deleteWhere(
|
||||
session,
|
||||
where: (t) => t.id.equals(grant.id),
|
||||
)).isEmpty) {
|
||||
return _invalidGrant('The code is unknown or used.');
|
||||
}
|
||||
if (!grant.expiresAt.isAfter(now) ||
|
||||
grant.connection?.clientId != client.id ||
|
||||
form['redirect_uri'] != grant.redirectUri ||
|
||||
!pkceMatches(form['code_verifier'] ?? '', grant.codeChallenge)) {
|
||||
return _invalidGrant(
|
||||
'The code expired, or the client, the redirect URI or the '
|
||||
'code verifier differ.',
|
||||
);
|
||||
}
|
||||
return await _issue(session, grant.connectionId, grant.scope, now);
|
||||
case 'refresh_token':
|
||||
final refresh = form['refresh_token'];
|
||||
final tokens = refresh == null
|
||||
? null
|
||||
: await AssistantToken.db.findFirstRow(
|
||||
session,
|
||||
where: (t) => t.refreshHash.equals(hashSecret(refresh)),
|
||||
include: AssistantToken.include(
|
||||
connection: AssistantConnection.include(),
|
||||
),
|
||||
);
|
||||
if (tokens == null ||
|
||||
!tokens.refreshExpiresAt.isAfter(now) ||
|
||||
tokens.connection?.clientId != client.id) {
|
||||
return _invalidGrant('The refresh token is unknown or expired.');
|
||||
}
|
||||
// Spent: a new pair replaces it, for the one request that deletes it.
|
||||
if ((await AssistantToken.db.deleteWhere(
|
||||
session,
|
||||
where: (t) => t.id.equals(tokens.id),
|
||||
)).isEmpty) {
|
||||
return _invalidGrant('The refresh token is used.');
|
||||
}
|
||||
return await _issue(session, tokens.connectionId, tokens.scope, now);
|
||||
default:
|
||||
return ConnectorResponse.oauthError(
|
||||
400,
|
||||
'unsupported_grant_type',
|
||||
'The grants offered are authorization_code and refresh_token.',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// The connection [authorization] (`Bearer …`) is a valid access token of,
|
||||
/// for this connector; null for none. Notes when it was last used, to the
|
||||
/// minute.
|
||||
Future<AssistantConnection?> connectionOf(
|
||||
Session session,
|
||||
String? authorization,
|
||||
) async {
|
||||
final match = RegExp(
|
||||
r'^Bearer\s+(\S+)$',
|
||||
caseSensitive: false,
|
||||
).firstMatch(authorization?.trim() ?? '');
|
||||
if (match == null) {
|
||||
return null;
|
||||
}
|
||||
final now = clock.now().toUtc();
|
||||
final tokens = await AssistantToken.db.findFirstRow(
|
||||
session,
|
||||
where: (t) =>
|
||||
t.accessHash.equals(hashSecret(match.group(1)!)) &
|
||||
(t.accessExpiresAt > now),
|
||||
include: AssistantToken.include(
|
||||
connection: AssistantConnection.include(),
|
||||
),
|
||||
);
|
||||
final connection = tokens?.connection;
|
||||
if (tokens == null || connection == null || tokens.resource != resource) {
|
||||
return null;
|
||||
}
|
||||
final last = connection.lastUsedAt;
|
||||
if (last == null || now.difference(last) >= const Duration(minutes: 1)) {
|
||||
await AssistantConnection.db.updateRow(
|
||||
session,
|
||||
connection.copyWith(lastUsedAt: now),
|
||||
);
|
||||
}
|
||||
return connection;
|
||||
}
|
||||
|
||||
/// What a request without a valid token gets: 401, naming where to find
|
||||
/// out how to get one (RFC 9728, 5.1).
|
||||
ConnectorResponse unauthorized() => ConnectorResponse.json(
|
||||
401,
|
||||
{'error': 'invalid_token'},
|
||||
headers: {
|
||||
'WWW-Authenticate':
|
||||
'Bearer resource_metadata="$resourceMetadataUrl", '
|
||||
'scope="$readScope"',
|
||||
},
|
||||
);
|
||||
|
||||
Future<ConnectorResponse> _issue(
|
||||
Session session,
|
||||
int connectionId,
|
||||
String scope,
|
||||
DateTime now,
|
||||
) async {
|
||||
final access = randomSecret();
|
||||
final refresh = randomSecret();
|
||||
await AssistantToken.db.insertRow(
|
||||
session,
|
||||
AssistantToken(
|
||||
connectionId: connectionId,
|
||||
accessHash: hashSecret(access),
|
||||
accessExpiresAt: now.add(accessTokenLifetime),
|
||||
refreshHash: hashSecret(refresh),
|
||||
refreshExpiresAt: now.add(refreshTokenLifetime),
|
||||
resource: resource,
|
||||
scope: scope,
|
||||
),
|
||||
);
|
||||
return ConnectorResponse.json(200, {
|
||||
'access_token': access,
|
||||
'token_type': 'Bearer',
|
||||
'expires_in': accessTokenLifetime.inSeconds,
|
||||
'refresh_token': refresh,
|
||||
'scope': scope,
|
||||
});
|
||||
}
|
||||
|
||||
/// The client of a token request, authenticated as it registered: by its
|
||||
/// id alone, or with its secret in the form or in Basic authorization.
|
||||
Future<AssistantClient?> _authenticate(
|
||||
Session session,
|
||||
ConnectorRequest request,
|
||||
Map<String, String> form,
|
||||
) async {
|
||||
var id = form['client_id'];
|
||||
var secret = form['client_secret'];
|
||||
final basic = RegExp(
|
||||
r'^Basic\s+(\S+)$',
|
||||
caseSensitive: false,
|
||||
).firstMatch(request.headers['authorization']?.trim() ?? '');
|
||||
if (basic != null) {
|
||||
try {
|
||||
final pair = utf8.decode(base64.decode(basic.group(1)!));
|
||||
final colon = pair.indexOf(':');
|
||||
if (colon > 0) {
|
||||
id = Uri.decodeQueryComponent(pair.substring(0, colon));
|
||||
secret = Uri.decodeQueryComponent(pair.substring(colon + 1));
|
||||
}
|
||||
} on FormatException {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
if (id == null) {
|
||||
return null;
|
||||
}
|
||||
final client = await AssistantClient.db.findFirstRow(
|
||||
session,
|
||||
where: (t) => t.clientId.equals(id),
|
||||
);
|
||||
if (client == null) {
|
||||
return null;
|
||||
}
|
||||
return switch (client.secretHash) {
|
||||
null => client,
|
||||
final hash when secret != null && hashSecret(secret) == hash => client,
|
||||
_ => null,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
ConnectorResponse _invalidGrant(String description) =>
|
||||
ConnectorResponse.oauthError(400, 'invalid_grant', description);
|
||||
|
||||
/// How a client may authenticate at the token endpoint.
|
||||
const _authMethods = ['none', 'client_secret_post', 'client_secret_basic'];
|
||||
|
||||
/// The fields the authorization page carries from its GET to its POST.
|
||||
const _carried = [
|
||||
'response_type',
|
||||
'client_id',
|
||||
'redirect_uri',
|
||||
'state',
|
||||
'scope',
|
||||
'code_challenge',
|
||||
'code_challenge_method',
|
||||
'resource',
|
||||
];
|
||||
|
||||
/// Whether [uri] may be a redirect URI: absolute, without a fragment, https,
|
||||
/// or http on the device itself (a desktop client's loopback).
|
||||
bool _acceptableRedirect(Object? uri) {
|
||||
if (uri is! String || uri.length > 512) {
|
||||
return false;
|
||||
}
|
||||
final parsed = Uri.tryParse(uri);
|
||||
if (parsed == null || !parsed.isAbsolute || parsed.hasFragment) {
|
||||
return false;
|
||||
}
|
||||
return switch (parsed.scheme) {
|
||||
'https' => parsed.host.isNotEmpty,
|
||||
'http' => const ['localhost', '127.0.0.1', '::1'].contains(parsed.host),
|
||||
_ => false,
|
||||
};
|
||||
}
|
||||
|
||||
/// [text] without control characters, at most [length] long.
|
||||
String _plain(String text, int length) {
|
||||
final plain = text.replaceAll(RegExp(r'[\x00-\x1F\x7F]'), ' ').trim();
|
||||
return plain.length <= length ? plain : plain.substring(0, length);
|
||||
}
|
||||
|
||||
String _withoutSlash(String uri) =>
|
||||
uri.endsWith('/') ? uri.substring(0, uri.length - 1) : uri;
|
||||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue