Limit how much room people's files take
An anonymous person may keep 100 MB of images and sounds, an identified one 500 MB, and one person more by a file_allowance row the operator adds; all of them together 15 GB, since every file is in the database and in each dump the server keeps (decided 2026-10-02). Each comes from the environment (FILE_LIMIT_*_MB), with these defaults, documented in deploy/.env.example. The server checks when an upload starts, counting unfinished uploads so they can't fill the disk unnoticed, and again when it finishes, counting what is kept, so uploads started side by side can't overshoot; a refusal is a FileLimitExceededException, and an upload refused as it finishes is deleted. deletePerson deletes the allowance with the person. Migration: the new table file_allowance, only added. The app keeps a refused file waiting on the device and stops that round of uploads; the Anki import warns when a deck's files may not fit in the room left (asset.usage()). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Tizian 「ティツィアーン」 <raisondetredev@pm.me>
This commit is contained in:
parent
53b5d56e79
commit
ca69f74fd9
12 changed files with 9884 additions and 20 deletions
|
|
@ -413,6 +413,13 @@
|
|||
}
|
||||
},
|
||||
"ankiImportMediaSubtitle": "Keep the deck's images and sounds on the cards.",
|
||||
"ankiImportMediaMayNotFit": "Its files take {size} MB, more than the {left} MB you have left: those that don't fit stay on this device, and your other devices won't show them.",
|
||||
"@ankiImportMediaMayNotFit": {
|
||||
"placeholders": {
|
||||
"size": {"type": "String"},
|
||||
"left": {"type": "String"}
|
||||
}
|
||||
},
|
||||
"ankiImportRunning": "Importing…",
|
||||
"ankiImportConfirm": "Import",
|
||||
"ankiImportNotAPackage": "This isn't an Anki deck. In Anki, export a deck as an .apkg file and import that.",
|
||||
|
|
|
|||
|
|
@ -2158,6 +2158,12 @@ abstract class AppLocalizations {
|
|||
/// **'Keep the deck\'s images and sounds on the cards.'**
|
||||
String get ankiImportMediaSubtitle;
|
||||
|
||||
/// No description provided for @ankiImportMediaMayNotFit.
|
||||
///
|
||||
/// In en, this message translates to:
|
||||
/// **'Its files take {size} MB, more than the {left} MB you have left: those that don\'t fit stay on this device, and your other devices won\'t show them.'**
|
||||
String ankiImportMediaMayNotFit(String size, String left);
|
||||
|
||||
/// No description provided for @ankiImportRunning.
|
||||
///
|
||||
/// In en, this message translates to:
|
||||
|
|
|
|||
|
|
@ -1272,6 +1272,11 @@ class AppLocalizationsEn extends AppLocalizations {
|
|||
String get ankiImportMediaSubtitle =>
|
||||
'Keep the deck\'s images and sounds on the cards.';
|
||||
|
||||
@override
|
||||
String ankiImportMediaMayNotFit(String size, String left) {
|
||||
return 'Its files take $size MB, more than the $left MB you have left: those that don\'t fit stay on this device, and your other devices won\'t show them.';
|
||||
}
|
||||
|
||||
@override
|
||||
String get ankiImportRunning => 'Importing…';
|
||||
|
||||
|
|
|
|||
|
|
@ -143,6 +143,11 @@ class AssetRepository(
|
|||
continue;
|
||||
}
|
||||
await _files.clearPending(hash);
|
||||
} on FileLimitReached catch (error) {
|
||||
// The others wouldn't fit either: they stay on this device, and
|
||||
// the next round asks again, once room may have been made.
|
||||
log('No room for more files', name: 'Assets', error: error);
|
||||
return;
|
||||
} on Object catch (error) {
|
||||
// Without a connection the others fail too: next time.
|
||||
log('Uploading a file failed', name: 'Assets', error: error);
|
||||
|
|
@ -167,6 +172,17 @@ class AssetRepository(
|
|||
await _server.finishUpload(hash);
|
||||
}
|
||||
|
||||
/// How much room the person's files take on the server, and may take; null
|
||||
/// when that can't be asked now (no connection).
|
||||
Future<FileRoom?> room() async {
|
||||
try {
|
||||
return await _server.room();
|
||||
} on Exception catch (error) {
|
||||
log('Asking for the room of files failed', name: 'Assets', error: error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/// Stops uploading; an upload under way finishes first.
|
||||
Future<void> dispose() async {
|
||||
await _triggers.cancel();
|
||||
|
|
|
|||
|
|
@ -1,18 +1,40 @@
|
|||
import 'dart:typed_data';
|
||||
|
||||
/// How much room the person's files take on the server, and may take, in
|
||||
/// bytes.
|
||||
typedef FileRoom = ({int usedBytes, int limitBytes});
|
||||
|
||||
/// The server keeps no more of the person's files: they would take more
|
||||
/// room than the person may use, or the instance has left ([instanceFull]).
|
||||
class const FileLimitReached({
|
||||
/// Whether the instance is full, rather than the person's own room.
|
||||
required final bool instanceFull,
|
||||
|
||||
/// What the person may keep, in bytes.
|
||||
required final int limitBytes,
|
||||
}) implements Exception {
|
||||
@override
|
||||
String toString() =>
|
||||
'FileLimitReached(instanceFull: $instanceFull, limit: $limitBytes)';
|
||||
}
|
||||
|
||||
/// The server's copy of the person's files.
|
||||
abstract interface class AssetServerGateway() {
|
||||
/// How to upload the file with [sha256] of [byteSize] bytes; null when the
|
||||
/// server has it already.
|
||||
/// server has it already. Throws [FileLimitReached] when it doesn't fit.
|
||||
Future<String?> startUpload(String sha256, int byteSize);
|
||||
|
||||
/// Uploads [bytes] as [description] says; whether that worked.
|
||||
Future<bool> upload(String description, Uint8List bytes);
|
||||
|
||||
/// Completes the upload of the file with [sha256]; whether the server keeps
|
||||
/// it, which it does only if its content has that hash.
|
||||
/// it, which it does only if its content has that hash. Throws
|
||||
/// [FileLimitReached] when it no longer fits.
|
||||
Future<bool> finishUpload(String sha256);
|
||||
|
||||
/// How much room the person's files take, and may take.
|
||||
Future<FileRoom> room();
|
||||
|
||||
/// The file of the asset [id]; null when the server has none for this
|
||||
/// person (not uploaded yet, or not theirs).
|
||||
Future<Uint8List?> download(String id);
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ class ServerpodAssetServerGateway(final protocol.Client _client)
|
|||
implements AssetServerGateway {
|
||||
@override
|
||||
Future<String?> startUpload(String sha256, int byteSize) =>
|
||||
_client.asset.startUpload(sha256, byteSize);
|
||||
_refusing(() => _client.asset.startUpload(sha256, byteSize));
|
||||
|
||||
@override
|
||||
Future<bool> upload(String description, Uint8List bytes) =>
|
||||
|
|
@ -18,7 +18,25 @@ class ServerpodAssetServerGateway(final protocol.Client _client)
|
|||
|
||||
@override
|
||||
Future<bool> finishUpload(String sha256) =>
|
||||
_client.asset.finishUpload(sha256);
|
||||
_refusing(() => _client.asset.finishUpload(sha256));
|
||||
|
||||
@override
|
||||
Future<FileRoom> room() async {
|
||||
final usage = await _client.asset.usage();
|
||||
return (usedBytes: usage.usedBytes, limitBytes: usage.limitBytes);
|
||||
}
|
||||
|
||||
/// [call], its refusal for want of room as a [FileLimitReached].
|
||||
Future<T> _refusing<T>(Future<T> Function() call) async {
|
||||
try {
|
||||
return await call();
|
||||
} on protocol.FileLimitExceededException catch (error) {
|
||||
throw FileLimitReached(
|
||||
instanceFull: error.instanceFull,
|
||||
limitBytes: error.limitBytes,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@override
|
||||
Future<Uint8List?> download(String id) async {
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ import 'dart:typed_data';
|
|||
|
||||
import 'package:equatable/equatable.dart';
|
||||
import 'package:flutter_bloc/flutter_bloc.dart';
|
||||
import 'package:halkyon_learn_flutter/src/core/asset/data/asset_repository.dart';
|
||||
import 'package:halkyon_learn_flutter/src/feature/anki/data/anki_importer.dart';
|
||||
import 'package:halkyon_learn_flutter/src/feature/anki/data/anki_package.dart';
|
||||
import 'package:halkyon_learn_flutter/src/feature/anki/data/anki_package_reader.dart';
|
||||
|
|
@ -12,6 +13,7 @@ import 'package:halkyon_learn_flutter/src/feature/anki/data/anki_package_reader.
|
|||
class AnkiImportBloc(
|
||||
final AnkiPackageReader _reader,
|
||||
final AnkiImporter _importer,
|
||||
final AssetRepository _assets,
|
||||
final String _studysetId,
|
||||
) extends Bloc<AnkiImportEvent, AnkiImportState> {
|
||||
this : super(const AnkiImportStateReading()) {
|
||||
|
|
@ -40,11 +42,20 @@ class AnkiImportBloc(
|
|||
}
|
||||
_package = package;
|
||||
final collection = package.collection;
|
||||
final media = referencedMedia(collection);
|
||||
final room = media.isEmpty ? null : await _assets.room();
|
||||
final summary = AnkiImportSummary(
|
||||
notes: collection.notes.length,
|
||||
decks: collection.deckPaths.length,
|
||||
hasReviews: collection.hasReviews,
|
||||
files: referencedMedia(collection).length,
|
||||
files: media.length,
|
||||
fileBytes: media.fold(
|
||||
0,
|
||||
(sum, name) => sum + (package.mediaSize(name) ?? 0),
|
||||
),
|
||||
roomLeft: room == null
|
||||
? null
|
||||
: (room.limitBytes - room.usedBytes).clamp(0, room.limitBytes),
|
||||
);
|
||||
emit(
|
||||
AnkiImportStateReady(
|
||||
|
|
@ -101,12 +112,31 @@ class const AnkiImportSummary({
|
|||
|
||||
/// How many images and sounds its notes show.
|
||||
required final int files,
|
||||
|
||||
/// How much room those take, in bytes.
|
||||
final int fileBytes = 0,
|
||||
|
||||
/// How much room the person has left on the server, in bytes; null when
|
||||
/// unknown (no connection, or no files to import).
|
||||
final int? roomLeft,
|
||||
}) with Equatable {
|
||||
/// Whether its notes are in more than one deck.
|
||||
bool get hasSubdecks => decks > 1;
|
||||
|
||||
/// Whether its files would take more room than the person has left: those
|
||||
/// that don't fit stay on this device. (Files they have already don't take
|
||||
/// more, so it may fit nonetheless.)
|
||||
bool get filesMayNotFit => roomLeft != null && fileBytes > roomLeft!;
|
||||
|
||||
@override
|
||||
List<Object?> get props => [notes, decks, hasReviews, files];
|
||||
List<Object?> get props => [
|
||||
notes,
|
||||
decks,
|
||||
hasReviews,
|
||||
files,
|
||||
fileBytes,
|
||||
roomLeft,
|
||||
];
|
||||
}
|
||||
|
||||
/// How to import a package.
|
||||
|
|
|
|||
|
|
@ -57,6 +57,15 @@ class AnkiPackage._(
|
|||
final Map<String, _MediaEntry> _media,
|
||||
final bool _compressed,
|
||||
) {
|
||||
/// The size of the media file [name] once decoded, in bytes; null if the
|
||||
/// package doesn't ship it.
|
||||
int? mediaSize(String name) {
|
||||
final entry = _media[name];
|
||||
return entry == null
|
||||
? null
|
||||
: entry.size ?? _archive.findFile(entry.zipName)?.size;
|
||||
}
|
||||
|
||||
/// The media file [name] as the notes refer to it; null if the package
|
||||
/// doesn't ship it or it is damaged.
|
||||
Uint8List? mediaBytes(String name) {
|
||||
|
|
|
|||
|
|
@ -41,6 +41,7 @@ Future<void> importFromAnki(
|
|||
person.reviews,
|
||||
person.assets,
|
||||
),
|
||||
person.assets,
|
||||
studysetId,
|
||||
)..add(AnkiImportEventStarted(file.bytes)),
|
||||
child: const _AnkiImport(),
|
||||
|
|
@ -221,7 +222,14 @@ class const _Options({
|
|||
onChanged: (value) =>
|
||||
change(options.copyWith(includeMedia: value ?? false)),
|
||||
title: Text(l10n.ankiImportMedia(summary.files)),
|
||||
subtitle: Text(l10n.ankiImportMediaSubtitle),
|
||||
subtitle: Text(
|
||||
options.includeMedia && summary.filesMayNotFit
|
||||
? l10n.ankiImportMediaMayNotFit(
|
||||
_megabytes(summary.fileBytes),
|
||||
_megabytes(summary.roomLeft!),
|
||||
)
|
||||
: l10n.ankiImportMediaSubtitle,
|
||||
),
|
||||
contentPadding: EdgeInsets.zero,
|
||||
controlAffinity: ListTileControlAffinity.leading,
|
||||
),
|
||||
|
|
@ -240,3 +248,6 @@ class const _Options({
|
|||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// [bytes] in megabytes, one decimal.
|
||||
String _megabytes(int bytes) => (bytes / (1024 * 1024)).toStringAsFixed(1);
|
||||
|
|
|
|||
|
|
@ -172,6 +172,40 @@ void main() {
|
|||
},
|
||||
);
|
||||
|
||||
test('keeps waiting while the server has no room, and asks no '
|
||||
'further that round', () async {
|
||||
await addImage();
|
||||
await repository.add(
|
||||
Uint8List.fromList(utf8.encode('another image')),
|
||||
fileName: 'dog.png',
|
||||
mimeType: 'image/png',
|
||||
);
|
||||
server.limitBytes = 4;
|
||||
|
||||
await repository.uploadPending();
|
||||
|
||||
expect(server.files, isEmpty);
|
||||
expect(cache.pending, hasLength(2));
|
||||
expect(server.started, hasLength(1));
|
||||
|
||||
server.limitBytes = 1 << 20;
|
||||
await repository.uploadPending();
|
||||
expect(cache.pending, isEmpty);
|
||||
});
|
||||
|
||||
test('tells how much room the files take, and may take', () async {
|
||||
server.files[imageHash] = image;
|
||||
server.limitBytes = 100;
|
||||
|
||||
expect(await repository.room(), (
|
||||
usedBytes: image.length,
|
||||
limitBytes: 100,
|
||||
));
|
||||
|
||||
server.failCalls = Exception('offline');
|
||||
expect(await repository.room(), isNull);
|
||||
});
|
||||
|
||||
test('stops waiting for a file gone from this device', () async {
|
||||
await addImage();
|
||||
cache.files.clear();
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ import 'package:halkyon_learn_flutter/src/feature/studyset/data/studyset_payload
|
|||
import 'package:halkyon_learn_flutter/src/feature/studyset/model/studyset_content.dart';
|
||||
import 'package:material_ui/material_ui.dart';
|
||||
|
||||
import '../../../support/fake_asset_server_gateway.dart';
|
||||
import '../../../support/fake_card_gateway.dart';
|
||||
import '../../../support/fake_file_picker.dart';
|
||||
import '../../../support/fake_studyset_gateway.dart';
|
||||
|
|
@ -17,10 +18,12 @@ import '../data/anki_package_test.dart' show fixture;
|
|||
void main() {
|
||||
late FakeCardGateway cardGateway;
|
||||
late FakeFilePicker files;
|
||||
late FakeAssetServerGateway server;
|
||||
|
||||
setUp(() {
|
||||
cardGateway = FakeCardGateway();
|
||||
files = FakeFilePicker();
|
||||
server = FakeAssetServerGateway();
|
||||
});
|
||||
|
||||
/// Opens the studyset Japanese, in a window wide enough for Add material
|
||||
|
|
@ -39,6 +42,7 @@ void main() {
|
|||
),
|
||||
cardGateway: cardGateway,
|
||||
files: files,
|
||||
assetServer: server,
|
||||
),
|
||||
location: RoutePaths.studyset('japanese'),
|
||||
);
|
||||
|
|
@ -86,6 +90,21 @@ void main() {
|
|||
expect(await cards.cardsOf('japanese'), hasLength(4));
|
||||
});
|
||||
|
||||
testWidgets('warns when the files may not fit in the room left', (
|
||||
tester,
|
||||
) async {
|
||||
server.limitBytes = 100;
|
||||
|
||||
await startImport(tester, fixture('anki_latest.apkg'));
|
||||
|
||||
expect(find.textContaining('more than the'), findsOneWidget);
|
||||
|
||||
await tester.tap(find.text('Include media (2 files)'));
|
||||
await tester.pump();
|
||||
|
||||
expect(find.textContaining('more than the'), findsNothing);
|
||||
});
|
||||
|
||||
testWidgets('says why a file is no deck it can import', (tester) async {
|
||||
await startImport(tester, Uint8List.fromList([1, 2, 3]));
|
||||
|
||||
|
|
|
|||
|
|
@ -31,11 +31,23 @@ class FakeAssetServerGateway() implements AssetServerGateway {
|
|||
/// When set, uploads arrive with this content instead of what was sent.
|
||||
Uint8List? corruptUploads;
|
||||
|
||||
/// What the person may keep, in bytes; past it, uploads are refused.
|
||||
int limitBytes = 1 << 30;
|
||||
|
||||
int get _usedBytes =>
|
||||
files.values.fold(0, (sum, bytes) => sum + bytes.length);
|
||||
|
||||
@override
|
||||
Future<String?> startUpload(String sha256, int byteSize) async {
|
||||
_check();
|
||||
started.add(sha256);
|
||||
return files.containsKey(sha256) ? null : 'upload:$sha256';
|
||||
if (files.containsKey(sha256)) {
|
||||
return null;
|
||||
}
|
||||
if (_usedBytes + byteSize > limitBytes) {
|
||||
throw FileLimitReached(instanceFull: false, limitBytes: limitBytes);
|
||||
}
|
||||
return 'upload:$sha256';
|
||||
}
|
||||
|
||||
@override
|
||||
|
|
@ -60,6 +72,12 @@ class FakeAssetServerGateway() implements AssetServerGateway {
|
|||
return true;
|
||||
}
|
||||
|
||||
@override
|
||||
Future<FileRoom> room() async {
|
||||
_check();
|
||||
return (usedBytes: _usedBytes, limitBytes: limitBytes);
|
||||
}
|
||||
|
||||
@override
|
||||
Future<Uint8List?> download(String id) async {
|
||||
_check();
|
||||
|
|
|
|||
Loading…
Reference in a new issue