refactor(base,learn): people's files kept by halkyon_base
ADR 0043, D2, D4, D6 and D7, the first step towards Todo's attachments: Learn's file store moves into the module, for every server that keeps files, and Learn becomes its first host. halkyon_base gets the store as Learn had it: uploads into the caller's own space by SHA-256, downloads by a row the caller may read, each person's room (FILE_LIMIT_IDENTIFIED_MB, FILE_LIMIT_INSTANCE_MB, the operator's file_allowance, which moves into the module unchanged) and the pruning of files no live row names. It stays free of the sync engine: a host hands it a FileStore saying which of its rows name a file and what it takes (HalkyonBase.files); without one the files endpoint refuses. Its tests, with refusals, run on a stand-in host. Learn hands it learnFiles: its asset rows, read by their space's members, a readable file over 20 MB only as a PDF, up to 100 MB. Its app calls the module; its own asset endpoint stays for released apps alone, forwarding to the module in Learn's own models, whose names are on the wire. Its file tests still pass through it, end to end. Migrations: the module's creates file_allowance. Learn's server migration holds version rows alone, as tables are matched by name (ADR 0014); its device migration creates the module's empty table, as every module's. Todo's, Train's and Meditate's create the empty table. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Tizian 「ティツィアーン」 <raisondetredev@pm.me>
This commit is contained in:
parent
59fb9c4d6d
commit
b7f0aaaec7
GPG key ID:
07F6FCF405488B64
1 changed files with 112084 additions and 348 deletions
|
|
@ -0,0 +1,344 @@
|
|||
import 'dart:convert';
|
||||
import 'dart:typed_data';
|
||||
|
||||
import 'package:crypto/crypto.dart';
|
||||
import 'package:halkyon_base_server/base.dart';
|
||||
import 'package:serverpod/protocol.dart' show CloudStorageEntry;
|
||||
import 'package:serverpod/serverpod.dart';
|
||||
import 'package:test/test.dart';
|
||||
|
||||
import '../support/fixtures.dart';
|
||||
import '../test_tools/serverpod_test_tools.dart';
|
||||
|
||||
/// A host's row naming a file: the space it is in, the file's hash, and who
|
||||
/// may read it (its space's members).
|
||||
typedef _Row = ({UuidValue space, String sha256, Set<UuidValue> readers});
|
||||
|
||||
void main() {
|
||||
withServerpod('Files (ADR 0043)', (sessionBuilder, endpoints) {
|
||||
late Session session;
|
||||
late UuidValue alice;
|
||||
late UuidValue bob;
|
||||
|
||||
/// The host's live rows, by their ID.
|
||||
late Map<UuidValue, _Row> rows;
|
||||
final image = utf8.encode('an image');
|
||||
final imageHash = sha256.convert(image).toString();
|
||||
|
||||
/// The file store a host gives, over [rows], taking what [accepts] does.
|
||||
FileStore store({AcceptsFile accepts = FileStore.acceptsAny}) => FileStore(
|
||||
fileOfRow: (session, rowId) async {
|
||||
final row = rows[rowId];
|
||||
final caller = UuidValue.withValidation(
|
||||
session.authenticated!.userIdentifier,
|
||||
);
|
||||
return row == null || !row.readers.contains(caller)
|
||||
? null
|
||||
: (space: row.space, sha256: row.sha256);
|
||||
},
|
||||
liveHashes: (session, space, hashes) async => {
|
||||
for (final row in rows.values)
|
||||
if (row.space == space && (hashes?.contains(row.sha256) ?? true))
|
||||
row.sha256,
|
||||
},
|
||||
maxBytes: 64,
|
||||
accepts: accepts,
|
||||
);
|
||||
|
||||
setUp(() {
|
||||
useBase(sessionBuilder, files: store());
|
||||
session = sessionBuilder.build();
|
||||
alice = const Uuid().v7obj();
|
||||
bob = const Uuid().v7obj();
|
||||
rows = {};
|
||||
});
|
||||
|
||||
tearDown(() async {
|
||||
fileLimits = null;
|
||||
await FileAllowance.db.deleteWhere(
|
||||
session,
|
||||
where: (t) => t.authUserId.inSet({alice, bob}),
|
||||
);
|
||||
await session.db.unsafeExecute(
|
||||
'DELETE FROM "serverpod_cloud_storage" WHERE "path" LIKE \'spaces/%\'',
|
||||
);
|
||||
});
|
||||
|
||||
/// [person]'s session, `identified`.
|
||||
TestSessionBuilder signedIn(UuidValue person) => sessionBuilder.copyWith(
|
||||
authentication: AuthenticationOverride.authenticationInfo(person.uuid, {
|
||||
identified,
|
||||
}),
|
||||
);
|
||||
|
||||
/// Adds [bytes] as [person], as an app does.
|
||||
Future<bool> add(
|
||||
UuidValue person,
|
||||
List<int> bytes, {
|
||||
String? as,
|
||||
bool? encrypted,
|
||||
}) async {
|
||||
final hash = as ?? sha256.convert(bytes).toString();
|
||||
final description = await endpoints.files.startUpload(
|
||||
signedIn(person),
|
||||
hash,
|
||||
bytes.length,
|
||||
encrypted: encrypted,
|
||||
);
|
||||
if (description != null) {
|
||||
await _upload(session, description, bytes);
|
||||
}
|
||||
return await endpoints.files.finishUpload(
|
||||
signedIn(person),
|
||||
hash,
|
||||
encrypted: encrypted,
|
||||
);
|
||||
}
|
||||
|
||||
/// A row of [space] naming [hash], which [readers] may read.
|
||||
UuidValue row(UuidValue space, String hash, Set<UuidValue> readers) {
|
||||
final id = const Uuid().v7obj();
|
||||
rows[id] = (space: space, sha256: hash, readers: readers);
|
||||
return id;
|
||||
}
|
||||
|
||||
test('refuses callers who are not signed in', () async {
|
||||
final nobody = sessionBuilder.copyWith(
|
||||
authentication: AuthenticationOverride.unauthenticated(),
|
||||
);
|
||||
await expectLater(
|
||||
endpoints.files.startUpload(nobody, imageHash, image.length),
|
||||
throwsA(isA<ServerpodUnauthenticatedException>()),
|
||||
);
|
||||
await expectLater(
|
||||
endpoints.files.download(nobody, const Uuid().v7obj()),
|
||||
throwsA(isA<ServerpodUnauthenticatedException>()),
|
||||
);
|
||||
});
|
||||
|
||||
test('a host without a file store keeps no files', () async {
|
||||
useBase(sessionBuilder);
|
||||
await expectLater(
|
||||
endpoints.files.startUpload(signedIn(alice), imageHash, image.length),
|
||||
throwsA(isA<StateError>()),
|
||||
);
|
||||
await expectLater(
|
||||
endpoints.files.usage(signedIn(alice)),
|
||||
throwsA(isA<StateError>()),
|
||||
);
|
||||
});
|
||||
|
||||
test("stores a file in the caller's space once, and serves it by a row "
|
||||
'its reader may read', () async {
|
||||
expect(await add(alice, image), isTrue);
|
||||
expect(
|
||||
await endpoints.files.startUpload(
|
||||
signedIn(alice),
|
||||
imageHash,
|
||||
image.length,
|
||||
),
|
||||
isNull,
|
||||
reason: 'it is there',
|
||||
);
|
||||
final id = row(alice, imageHash, {alice, bob});
|
||||
|
||||
expect(
|
||||
Uint8List.sublistView(
|
||||
await endpoints.files.download(signedIn(bob), id),
|
||||
),
|
||||
image,
|
||||
);
|
||||
});
|
||||
|
||||
test("serves nothing of a row the caller can't read, an unknown row, or "
|
||||
'a file not uploaded, alike', () async {
|
||||
await add(alice, image);
|
||||
final hers = row(alice, imageHash, {alice});
|
||||
final missing = row(alice, sha256.convert([1]).toString(), {alice});
|
||||
|
||||
for (final (person, id) in [
|
||||
(bob, hers),
|
||||
(alice, const Uuid().v7obj()),
|
||||
(alice, missing),
|
||||
]) {
|
||||
await expectLater(
|
||||
endpoints.files.download(signedIn(person), id),
|
||||
throwsA(isA<StoredFileNotFoundException>()),
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test('keeps no file whose content has another hash', () async {
|
||||
expect(
|
||||
await add(alice, image, as: sha256.convert([7]).toString()),
|
||||
false,
|
||||
);
|
||||
expect(await _stored(session), isEmpty);
|
||||
});
|
||||
|
||||
test('refuses what is no SHA-256, and sizes out of range', () async {
|
||||
for (final (hash, size) in [
|
||||
('not a hash', 1),
|
||||
(imageHash.toUpperCase(), 1),
|
||||
(imageHash, 0),
|
||||
(imageHash, 65),
|
||||
]) {
|
||||
await expectLater(
|
||||
endpoints.files.startUpload(signedIn(alice), hash, size),
|
||||
throwsA(isA<ArgumentError>()),
|
||||
reason: '$hash $size',
|
||||
);
|
||||
}
|
||||
// Sealed, it may be larger by what sealing adds.
|
||||
expect(
|
||||
await endpoints.files.startUpload(
|
||||
signedIn(alice),
|
||||
imageHash,
|
||||
65,
|
||||
encrypted: true,
|
||||
),
|
||||
isNotNull,
|
||||
);
|
||||
});
|
||||
|
||||
test('keeps a file said to be sealed only when it starts as one; the '
|
||||
"host's own rule decides the rest", () async {
|
||||
expect(await add(alice, image, encrypted: true), isFalse);
|
||||
final sealed = [...ascii.encode('HLKF'), 1, 9, 9];
|
||||
expect(await add(alice, sealed, encrypted: true), isTrue);
|
||||
|
||||
useBase(
|
||||
sessionBuilder,
|
||||
files: store(accepts: FileStore.acceptsSealedOnly),
|
||||
);
|
||||
expect(await add(bob, image), isFalse);
|
||||
expect(await add(bob, sealed, encrypted: true), isTrue);
|
||||
});
|
||||
|
||||
group('room', () {
|
||||
test('a person keeps no more than their room; an allowance gives one '
|
||||
'person more', () async {
|
||||
fileLimits = const FileLimits(identifiedBytes: 10, instanceBytes: null);
|
||||
await expectLater(
|
||||
endpoints.files.startUpload(signedIn(alice), imageHash, 11),
|
||||
throwsA(isA<FileLimitExceededException>()),
|
||||
);
|
||||
|
||||
await FileAllowance.db.insertRow(
|
||||
session,
|
||||
FileAllowance(authUserId: alice, maxBytes: 64),
|
||||
);
|
||||
expect(
|
||||
await endpoints.files.startUpload(signedIn(alice), imageHash, 11),
|
||||
isNotNull,
|
||||
);
|
||||
await expectLater(
|
||||
endpoints.files.startUpload(signedIn(bob), imageHash, 11),
|
||||
throwsA(isA<FileLimitExceededException>()),
|
||||
);
|
||||
});
|
||||
|
||||
test("counts the files the space's live rows name, and those claimed "
|
||||
'lately', () async {
|
||||
await add(alice, image);
|
||||
row(alice, imageHash, {alice});
|
||||
final usage = await endpoints.files.usage(signedIn(alice));
|
||||
expect(usage.usedBytes, image.length);
|
||||
|
||||
rows.clear();
|
||||
await _claimed(
|
||||
session,
|
||||
alice,
|
||||
imageHash,
|
||||
DateTime.now().toUtc().subtract(const Duration(days: 8)),
|
||||
);
|
||||
expect(
|
||||
(await endpoints.files.usage(signedIn(alice))).usedBytes,
|
||||
0,
|
||||
reason: 'no live row names it, and it was claimed long ago',
|
||||
);
|
||||
});
|
||||
|
||||
test("the operator's allowance goes with the person", () async {
|
||||
await FileAllowance.db.insertRow(
|
||||
session,
|
||||
FileAllowance(authUserId: alice, maxBytes: 64),
|
||||
);
|
||||
await deletePerson(session, alice);
|
||||
expect(
|
||||
await FileAllowance.db.find(
|
||||
session,
|
||||
where: (t) => t.authUserId.equals(alice),
|
||||
),
|
||||
isEmpty,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
group('pruning', () {
|
||||
final longAgo = DateTime.now().toUtc().subtract(const Duration(days: 8));
|
||||
|
||||
test('deletes files no live row names, claimed long ago; keeps the '
|
||||
'rest', () async {
|
||||
final other = utf8.encode('another');
|
||||
final otherHash = sha256.convert(other).toString();
|
||||
await add(alice, image);
|
||||
await add(alice, other);
|
||||
row(alice, imageHash, {alice});
|
||||
await _claimed(session, alice, imageHash, longAgo);
|
||||
await _claimed(session, alice, otherHash, longAgo);
|
||||
|
||||
expect(await pruneUnreferencedFiles(session, now: DateTime.now()), 1);
|
||||
expect(await _stored(session), {storedFilePath(alice, imageHash)});
|
||||
});
|
||||
|
||||
test('keeps a file claimed lately, named or not', () async {
|
||||
await add(alice, image);
|
||||
expect(await pruneUnreferencedFiles(session, now: DateTime.now()), 0);
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/// Uploads [bytes] as the upload [description] says, standing in for the
|
||||
/// app's HTTP upload, as in Halkyon Learn's tests.
|
||||
Future<void> _upload(
|
||||
Session session,
|
||||
String description,
|
||||
List<int> bytes,
|
||||
) async {
|
||||
final url = Uri.parse(
|
||||
(jsonDecode(description) as Map<String, Object?>)['url']! as String,
|
||||
);
|
||||
final storage =
|
||||
Serverpod.instance.storage[url.queryParameters['storage']!]!
|
||||
as DatabaseCloudStorage;
|
||||
await storage.storeUnverifiedFile(
|
||||
session: session,
|
||||
path: url.queryParameters['path']!,
|
||||
byteData: ByteData.sublistView(Uint8List.fromList(bytes)),
|
||||
);
|
||||
}
|
||||
|
||||
/// The stored files' paths.
|
||||
Future<Set<String>> _stored(Session session) async => {
|
||||
for (final [path as String] in await session.db.unsafeQuery(
|
||||
'SELECT "path" FROM "serverpod_cloud_storage" WHERE "path" LIKE '
|
||||
"'spaces/%'",
|
||||
))
|
||||
path,
|
||||
};
|
||||
|
||||
/// Makes the stored file with [hash] of [space] last claimed [at].
|
||||
Future<void> _claimed(
|
||||
Session session,
|
||||
UuidValue space,
|
||||
String hash,
|
||||
DateTime at,
|
||||
) => CloudStorageEntry.db.updateWhere(
|
||||
session,
|
||||
columnValues: (t) => [t.addedTime(at)],
|
||||
where: (t) =>
|
||||
t.storageId.equals(fileStorage) &
|
||||
t.path.equals(storedFilePath(space, hash)),
|
||||
);
|
||||
Loading…
Reference in a new issue