Limit how much room people's files take
An anonymous person may keep 100 MB of images and sounds, an identified one 500 MB, and one person more by a file_allowance row the operator adds; all of them together 15 GB, since every file is in the database and in each dump the server keeps (decided 2026-10-02). Each comes from the environment (FILE_LIMIT_*_MB), with these defaults, documented in deploy/.env.example. The server checks when an upload starts, counting unfinished uploads so they can't fill the disk unnoticed, and again when it finishes, counting what is kept, so uploads started side by side can't overshoot; a refusal is a FileLimitExceededException, and an upload refused as it finishes is deleted. deletePerson deletes the allowance with the person. Migration: the new table file_allowance, only added. The app keeps a refused file waiting on the device and stops that round of uploads; the Anki import warns when a deck's files may not fit in the room left (asset.usage()). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Tizian 「ティツィアーン」 <raisondetredev@pm.me>
This commit is contained in:
parent
53b5d56e79
commit
ca69f74fd9
1 changed files with 9884 additions and 20 deletions
|
|
@ -3,7 +3,9 @@ import 'dart:typed_data';
|
|||
|
||||
import 'package:crypto/crypto.dart';
|
||||
import 'package:halkyon_learn_server/src/asset/asset_endpoint.dart';
|
||||
import 'package:halkyon_learn_server/src/asset/file_limits.dart';
|
||||
import 'package:halkyon_learn_server/src/asset/file_pruning.dart';
|
||||
import 'package:halkyon_learn_server/src/auth/scopes.dart';
|
||||
import 'package:halkyon_learn_server/src/generated/protocol.dart';
|
||||
import 'package:serverpod/protocol.dart' show CloudStorageEntry;
|
||||
import 'package:serverpod/serverpod.dart';
|
||||
|
|
@ -36,6 +38,11 @@ void main() {
|
|||
});
|
||||
|
||||
tearDown(() async {
|
||||
fileLimits = null;
|
||||
await FileAllowance.db.deleteWhere(
|
||||
session,
|
||||
where: (t) => t.authUserId.inSet(spaces),
|
||||
);
|
||||
for (final space in spaces) {
|
||||
await session.db.unsafeExecute(
|
||||
'DELETE FROM "serverpod_cloud_storage" WHERE "path" LIKE @prefix',
|
||||
|
|
@ -51,26 +58,37 @@ void main() {
|
|||
spaces.clear();
|
||||
});
|
||||
|
||||
TestSessionBuilder signedIn(UuidValue person) => sessionBuilder.copyWith(
|
||||
TestSessionBuilder signedIn(
|
||||
UuidValue person, {
|
||||
Set<Scope> scopes = const {},
|
||||
}) => sessionBuilder.copyWith(
|
||||
authentication: AuthenticationOverride.authenticationInfo(
|
||||
person.uuid,
|
||||
{},
|
||||
scopes,
|
||||
),
|
||||
);
|
||||
|
||||
/// Adds [bytes] as [person], as the app does: asks how to upload them,
|
||||
/// uploads them unless the server has them, and finishes the upload.
|
||||
Future<bool> add(UuidValue person, List<int> bytes, {String? as}) async {
|
||||
Future<bool> add(
|
||||
UuidValue person,
|
||||
List<int> bytes, {
|
||||
String? as,
|
||||
Set<Scope> scopes = const {},
|
||||
}) async {
|
||||
final hash = as ?? sha256.convert(bytes).toString();
|
||||
final description = await endpoints.asset.startUpload(
|
||||
signedIn(person),
|
||||
signedIn(person, scopes: scopes),
|
||||
hash,
|
||||
bytes.length,
|
||||
);
|
||||
if (description != null) {
|
||||
await _upload(session, description, bytes);
|
||||
}
|
||||
return await endpoints.asset.finishUpload(signedIn(person), hash);
|
||||
return await endpoints.asset.finishUpload(
|
||||
signedIn(person, scopes: scopes),
|
||||
hash,
|
||||
);
|
||||
}
|
||||
|
||||
test('refuses callers who are not signed in', () async {
|
||||
|
|
@ -273,6 +291,158 @@ void main() {
|
|||
);
|
||||
});
|
||||
|
||||
group('room', () {
|
||||
setUp(() {
|
||||
fileLimits = const FileLimits(
|
||||
anonymousBytes: 10,
|
||||
identifiedBytes: 20,
|
||||
instanceBytes: 1000,
|
||||
);
|
||||
});
|
||||
|
||||
Matcher refused({required bool instanceFull, required int used}) =>
|
||||
throwsA(
|
||||
isA<FileLimitExceededException>()
|
||||
.having((e) => e.instanceFull, 'instanceFull', instanceFull)
|
||||
.having((e) => e.usedBytes, 'usedBytes', used),
|
||||
);
|
||||
|
||||
test('an anonymous person keeps no more than their room', () async {
|
||||
expect(await add(alice, List.filled(8, 1)), isTrue);
|
||||
|
||||
await expectLater(
|
||||
endpoints.asset.startUpload(
|
||||
signedIn(alice),
|
||||
sha256.convert([2, 2, 2]).toString(),
|
||||
3,
|
||||
),
|
||||
refused(instanceFull: false, used: 8),
|
||||
);
|
||||
});
|
||||
|
||||
test('an identified person keeps more', () async {
|
||||
expect(
|
||||
await add(alice, List.filled(15, 1), scopes: {identified}),
|
||||
isTrue,
|
||||
);
|
||||
await expectLater(
|
||||
add(bob, List.filled(15, 1)),
|
||||
refused(instanceFull: false, used: 0),
|
||||
);
|
||||
});
|
||||
|
||||
test('an allowance gives one person more, and nobody else', () async {
|
||||
await FileAllowance.db.insertRow(
|
||||
session,
|
||||
FileAllowance(authUserId: alice, maxBytes: 40),
|
||||
);
|
||||
|
||||
expect(await add(alice, List.filled(30, 1)), isTrue);
|
||||
await expectLater(
|
||||
add(bob, List.filled(30, 2)),
|
||||
refused(instanceFull: false, used: 0),
|
||||
);
|
||||
});
|
||||
|
||||
test('the instance keeps no more than its room', () async {
|
||||
// Whatever other tests left behind counts too: the room is set
|
||||
// from what the instance holds now.
|
||||
fileLimits = FileLimits(
|
||||
anonymousBytes: 10,
|
||||
identifiedBytes: 20,
|
||||
instanceBytes: await _instanceBytes(session) + 12,
|
||||
);
|
||||
expect(await add(alice, List.filled(8, 1)), isTrue);
|
||||
|
||||
await expectLater(
|
||||
add(bob, List.filled(5, 2)),
|
||||
refused(instanceFull: true, used: 0),
|
||||
);
|
||||
});
|
||||
|
||||
test('uploads started side by side are checked again as they '
|
||||
'finish', () async {
|
||||
final first = List.filled(6, 1);
|
||||
final second = List.filled(6, 2);
|
||||
String hash(List<int> bytes) => sha256.convert(bytes).toString();
|
||||
// Both started before either arrives: each fits on its own.
|
||||
final descriptions = [
|
||||
for (final bytes in [first, second])
|
||||
(await endpoints.asset.startUpload(
|
||||
signedIn(alice),
|
||||
hash(bytes),
|
||||
bytes.length,
|
||||
))!,
|
||||
];
|
||||
await _upload(session, descriptions[0], first);
|
||||
await _upload(session, descriptions[1], second);
|
||||
|
||||
expect(
|
||||
await endpoints.asset.finishUpload(signedIn(alice), hash(first)),
|
||||
isTrue,
|
||||
);
|
||||
await expectLater(
|
||||
endpoints.asset.finishUpload(signedIn(alice), hash(second)),
|
||||
refused(instanceFull: false, used: 12),
|
||||
);
|
||||
expect(
|
||||
await session.storage.fileExists(
|
||||
storageId: assetStorage,
|
||||
path: assetPath(alice, hash(second)),
|
||||
),
|
||||
isFalse,
|
||||
);
|
||||
});
|
||||
|
||||
test('uploads never finished take room from the next', () async {
|
||||
final unfinished = List.filled(6, 1);
|
||||
await _upload(
|
||||
session,
|
||||
(await endpoints.asset.startUpload(
|
||||
signedIn(alice),
|
||||
sha256.convert(unfinished).toString(),
|
||||
unfinished.length,
|
||||
))!,
|
||||
unfinished,
|
||||
);
|
||||
|
||||
await expectLater(
|
||||
endpoints.asset.startUpload(
|
||||
signedIn(alice),
|
||||
sha256.convert(List.filled(6, 2)).toString(),
|
||||
6,
|
||||
),
|
||||
refused(instanceFull: false, used: 6),
|
||||
);
|
||||
});
|
||||
|
||||
test('tells how much room the files take, and may take', () async {
|
||||
await add(alice, List.filled(4, 1));
|
||||
|
||||
final usage = await endpoints.asset.usage(signedIn(alice));
|
||||
expect((usage.usedBytes, usage.limitBytes), (4, 10));
|
||||
|
||||
fileLimits = FileLimits(
|
||||
anonymousBytes: 10,
|
||||
identifiedBytes: 20,
|
||||
instanceBytes: await _instanceBytes(session) + 2,
|
||||
);
|
||||
final tight = await endpoints.asset.usage(signedIn(alice));
|
||||
expect((tight.usedBytes, tight.limitBytes), (4, 6));
|
||||
});
|
||||
|
||||
test('refuses callers who are not signed in', () async {
|
||||
await expectLater(
|
||||
endpoints.asset.usage(
|
||||
sessionBuilder.copyWith(
|
||||
authentication: AuthenticationOverride.unauthenticated(),
|
||||
),
|
||||
),
|
||||
throwsA(isA<ServerpodUnauthenticatedException>()),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
group('pruning', () {
|
||||
final now = DateTime.now().toUtc();
|
||||
DateTime daysAgo(int days) => now.subtract(Duration(days: days));
|
||||
|
|
@ -445,6 +615,14 @@ Future<void> _upload(
|
|||
);
|
||||
}
|
||||
|
||||
/// What the stored files of everyone take, in bytes.
|
||||
Future<int> _instanceBytes(Session session) async =>
|
||||
(await session.db.unsafeQuery(
|
||||
'SELECT COALESCE(SUM(octet_length("byteData")), 0)::bigint '
|
||||
'FROM "serverpod_cloud_storage" WHERE "path" LIKE \'spaces/%\'',
|
||||
)).first.first!
|
||||
as int;
|
||||
|
||||
/// Makes the stored file with [hash] of [space] last claimed [at].
|
||||
Future<void> _claimed(
|
||||
Session session,
|
||||
|
|
|
|||
Loading…
Reference in a new issue