refactor(app,learn): people's files on the device shared in halkyon_app

ADR 0043's recommendation, for Todo's attachments: what Learn had for its
files on the device moves into halkyon_app, unchanged but for its names,
and Learn uses it from there.

- FileCache, DirectoryFileCache (Learn's AssetFileCache): the files by
  their SHA-256 in a folder of the person's, and which wait to be
  uploaded.
- FileServerGateway, ServerpodFileServerGateway (Learn's
  AssetServerGateway): halkyon_base's files endpoint, taking the module's
  caller rather than Learn's client; FileLimitReached, FileRoom.
- Their fakes in halkyon_app's testing: MemoryFileCache,
  FakeFileServerGateway, sha256Of.
- SealedFile: the format every app seals a file in (HLKF, version 1),
  sealing with halkyon_app's ContentCipher; Learn's Privacy seals and
  opens its files with it, for the same context as before, so every file
  sealed so far opens as it did.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Signed-off-by: Tizian 「ティツィアーン」 <raisondetredev@pm.me>

packages/halkyon_app/lib/halkyon_app.dart

@@ -99,8 +99,11 @@ export 'src/auth/widget/session_gate.dart';
 export 'src/auth/widget/start_screen.dart';
 export 'src/auth/widget/wait_screen.dart';
 export 'src/base/base_client.dart';
+export 'src/files/file_cache.dart';
 export 'src/files/file_picker.dart';
 export 'src/files/file_saver.dart';
+export 'src/files/file_server_gateway.dart';
+export 'src/files/serverpod_file_server_gateway.dart';
 export 'src/handle/bloc/handle_check_cubit.dart';
 export 'src/handle/model/handle_problem.dart';
 export 'src/handle/widget/handle_field.dart';
@@ -163,6 +166,7 @@ export 'src/password/widget/secret_screen.dart';
 export 'src/person/person_folders.dart';
 export 'src/privacy/content_cipher.dart';
 export 'src/privacy/payload_envelope.dart';
+export 'src/privacy/sealed_file.dart';
 export 'src/profile/bloc/public_profile_bloc.dart';
 export 'src/profile/data/avatar_image.dart';
 export 'src/profile/data/base_public_profile_gateway.dart';

packages/halkyon_app/lib/src/files/file_cache.dart

@@ -0,0 +1,100 @@
+import 'dart:io';
+import 'dart:typed_data';
+
+import 'package:path/path.dart' as p;
+
+/// The files on this device, by their SHA-256, and which of them wait to be
+/// uploaded.
+abstract interface class FileCache() {
+  /// The file with [sha256]; null if this device hasn't got it.
+  Future<Uint8List?> read(String sha256);
+
+  /// Keeps [bytes], whose SHA-256 is [sha256].
+  Future<void> write(String sha256, Uint8List bytes);
+
+  /// Where the file with [sha256] is, or would be.
+  String pathOf(String sha256);
+
+  /// The files that wait to be uploaded.
+  Future<Set<String>> pendingUploads();
+
+  /// Notes that the file with [sha256] waits to be uploaded.
+  Future<void> markPending(String sha256);
+
+  /// Notes that the file with [sha256] doesn't wait any more.
+  Future<void> clearPending(String sha256);
+}
+
+/// [FileCache] in a folder of the person's: each file named by its hash,
+/// and an empty file of the same name in `uploads/` for each that waits.
+///
+/// A hash is part of a path here, and rows synced from elsewhere carry them,
+/// so anything but 64 lowercase hex digits is refused.
+class DirectoryFileCache(final Directory _folder) implements FileCache {
+  Directory get _uploads => Directory(p.join(_folder.path, 'uploads'));
+
+  @override
+  Future<Uint8List?> read(String sha256) async {
+    try {
+      return await File(pathOf(sha256)).readAsBytes();
+    } on PathNotFoundException {
+      return null;
+    }
+  }
+
+  @override
+  Future<void> write(String sha256, Uint8List bytes) async {
+    final file = File(pathOf(sha256));
+    if (file.existsSync()) {
+      return;
+    }
+    await _folder.create(recursive: true);
+    // Written aside and renamed, so no half-written file ever has the name.
+    final partial = File('${file.path}.partial');
+    await partial.writeAsBytes(bytes, flush: true);
+    await partial.rename(file.path);
+  }
+
+  @override
+  String pathOf(String sha256) => p.join(_folder.path, checkedSha256(sha256));
+
+  @override
+  Future<Set<String>> pendingUploads() async {
+    if (!_uploads.existsSync()) {
+      return {};
+    }
+    return {
+      await for (final entry in _uploads.list())
+        if (entry is File && isSha256(p.basename(entry.path)))
+          p.basename(entry.path),
+    };
+  }
+
+  @override
+  Future<void> markPending(String sha256) async {
+    await _uploads.create(recursive: true);
+    await File(
+      p.join(_uploads.path, checkedSha256(sha256)),
+    ).writeAsBytes(const []);
+  }
+
+  @override
+  Future<void> clearPending(String sha256) async {
+    try {
+      await File(p.join(_uploads.path, checkedSha256(sha256))).delete();
+    } on PathNotFoundException {
+      // Not waiting.
+    }
+  }
+}
+
+/// Whether [value] is a SHA-256 as files are named by: 64 lowercase hex
+/// digits.
+bool isSha256(String value) => _sha256.hasMatch(value);
+
+/// [value], if it [isSha256]; throws an [ArgumentError] otherwise.
+String checkedSha256(String value) => isSha256(value)
+    ? value
+    : throw ArgumentError.value(value, 'sha256', 'is no SHA-256 in hex');
+
+final _sha256 = RegExp(r'^[0-9a-f]{64}$');

packages/halkyon_app/lib/src/files/file_server_gateway.dart

@@ -0,0 +1,47 @@
+import 'dart:typed_data';
+
+/// How much room the person's files take on the server, and may take, in
+/// bytes.
+typedef FileRoom = ({int usedBytes, int limitBytes});
+
+/// The server keeps no more of the person's files: they would take more
+/// room than the person may use, or the instance has left ([instanceFull]).
+class const FileLimitReached({
+  /// Whether the instance is full, rather than the person's own room.
+  required final bool instanceFull,
+
+  /// What the person may keep, in bytes.
+  required final int limitBytes,
+}) implements Exception {
+  @override
+  String toString() =>
+      'FileLimitReached(instanceFull: $instanceFull, limit: $limitBytes)';
+}
+
+/// The server's copy of the person's files (Halkyon's ADR 0043), which
+/// halkyon_base keeps for the app.
+abstract interface class FileServerGateway() {
+  /// How to upload the file with [sha256] of [byteSize] bytes; null when the
+  /// server has it already. Throws [FileLimitReached] when it doesn't fit.
+  Future<String?> startUpload(
+    String sha256,
+    int byteSize, {
+    bool encrypted = false,
+  });
+
+  /// Uploads [bytes] as [description] says; whether that worked.
+  Future<bool> upload(String description, Uint8List bytes);
+
+  /// Completes the upload of the file with [sha256]; whether the server keeps
+  /// it, which it does only if its content has that hash. Throws
+  /// [FileLimitReached] when it no longer fits.
+  Future<bool> finishUpload(String sha256, {bool encrypted = false});
+
+  /// How much room the person's files take, and may take.
+  Future<FileRoom> room();
+
+  /// The file the app's row [id] names (Learn's asset, Todo's attachment);
+  /// null when the server has none for this person (not uploaded yet, or
+  /// not theirs).
+  Future<Uint8List?> download(String id);
+}

packages/halkyon_app/lib/src/files/serverpod_file_server_gateway.dart

@@ -0,0 +1,69 @@
+import 'dart:typed_data';
+
+import 'package:halkyon_app/src/files/file_server_gateway.dart';
+import 'package:halkyon_base_client/halkyon_base_client.dart' as protocol;
+
+/// [FileServerGateway] on halkyon_base's `files` endpoint (Halkyon's ADR
+/// 0043), uploading through Serverpod's own upload route: every app's files
+/// (Learn's assets, Todo's attachments). Learn's own `asset` endpoint stays
+/// for its released apps alone (D7).
+class ServerpodFileServerGateway(final protocol.Caller _base)
+    implements FileServerGateway {
+  @override
+  Future<String?> startUpload(
+    String sha256,
+    int byteSize, {
+    bool encrypted = false,
+  }) => _refusing(
+    () => _base.files.startUpload(
+      sha256,
+      byteSize,
+      encrypted: encrypted,
+    ),
+  );
+
+  @override
+  Future<bool> upload(String description, Uint8List bytes) =>
+      protocol.FileUploader(description)
+          .uploadByteData(ByteData.sublistView(bytes));
+
+  @override
+  Future<bool> finishUpload(String sha256, {bool encrypted = false}) =>
+      _refusing(
+        () => _base.files.finishUpload(
+          sha256,
+          encrypted: encrypted,
+        ),
+      );
+
+  @override
+  Future<FileRoom> room() async {
+    final usage = await _base.files.usage();
+    return (usedBytes: usage.usedBytes, limitBytes: usage.limitBytes);
+  }
+
+  /// [call], its refusal for want of room as a [FileLimitReached].
+  Future<T> _refusing<T>(Future<T> Function() call) async {
+    try {
+      return await call();
+    } on protocol.FileLimitExceededException catch (error) {
+      throw FileLimitReached(
+        instanceFull: error.instanceFull,
+        limitBytes: error.limitBytes,
+      );
+    }
+  }
+
+  @override
+  Future<Uint8List?> download(String id) async {
+    try {
+      return Uint8List.sublistView(
+        await _base.files.download(
+          protocol.UuidValue.withValidation(id),
+        ),
+      );
+    } on protocol.StoredFileNotFoundException {
+      return null;
+    }
+  }
+}

packages/halkyon_app/lib/src/privacy/sealed_file.dart

@@ -0,0 +1,86 @@
+import 'dart:convert';
+import 'dart:typed_data';
+
+import 'package:halkyon_app/src/privacy/content_cipher.dart';
+
+/// A file sealed as every app seals one (Halkyon Learn's ADR 0019, D5;
+/// Halkyon's ADR 0043): `HLKF`, the format's version (1), the length of the
+/// key's ID and the ID itself, the nonce (24 bytes), then the sealed
+/// payload (the signature and the ciphertext). The server checks the first
+/// five bytes alone (halkyon_base's `finishFileUpload`).
+abstract final class SealedFile() {
+  /// What a sealed file starts with.
+  static final magic = utf8.encode('HLKF');
+
+  /// The format's version.
+  static const version = 1;
+
+  static const _nonceLength = 24;
+
+  /// [plain] sealed with [cipher]'s key [keyId] for the [context] it belongs
+  /// to (its row, so it opens for no other). Throws a [StateError] for a key
+  /// not open there.
+  static Uint8List seal(
+    ContentCipher cipher,
+    Uint8List plain, {
+    required String keyId,
+    required List<int> context,
+  }) {
+    final sealed = cipher.seal(plain, keyId: keyId, context: context);
+    final id = utf8.encode(keyId);
+    return Uint8List.fromList([
+      ...magic,
+      version,
+      id.length,
+      ...id,
+      ...sealed.nonce,
+      ...sealed.payload,
+    ]);
+  }
+
+  /// What [sealed] holds, for the [context] it belongs to; null where it
+  /// isn't a sealed file or doesn't open with [cipher] (its key isn't open,
+  /// it was changed, or it belongs elsewhere).
+  static Uint8List? open(
+    ContentCipher cipher,
+    Uint8List sealed, {
+    required List<int> context,
+  }) {
+    final header = magic.length + 2;
+    if (sealed.length < header ||
+        !_startsWith(sealed, magic) ||
+        sealed[magic.length] != version) {
+      return null;
+    }
+    final idLength = sealed[magic.length + 1];
+    if (sealed.length < header + idLength + _nonceLength) {
+      return null;
+    }
+    final String keyId;
+    try {
+      keyId = utf8.decode(
+        Uint8List.sublistView(sealed, header, header + idLength),
+      );
+    } on FormatException {
+      return null;
+    }
+    final nonceAt = header + idLength;
+    return cipher.unseal(
+      SealedPayload(
+        keyId: keyId,
+        nonce: Uint8List.sublistView(sealed, nonceAt, nonceAt + _nonceLength),
+        payload: Uint8List.sublistView(sealed, nonceAt + _nonceLength),
+      ),
+      context: context,
+    );
+  }
+
+  static bool _startsWith(Uint8List bytes, List<int> prefix) {
+    for (var i = 0; i < prefix.length; i++) {
+      if (bytes[i] != prefix[i]) {
+        return false;
+      }
+    }
+    return true;
+  }
+}

packages/halkyon_app/lib/src/testing/fake_file_server_gateway.dart

@@ -0,0 +1,106 @@
+import 'dart:typed_data';
+
+import 'package:crypto/crypto.dart';
+import 'package:halkyon_app/src/files/file_server_gateway.dart';
+
+/// A [FileServerGateway] in memory, holding files by hash as the server
+/// does, and the rows' files by id for downloads.
+class FakeFileServerGateway() implements FileServerGateway {
+  /// The files the server keeps, by hash.
+  final files = <String, Uint8List>{};
+
+  /// The hash of each row naming a file the server knows, by id: what a download
+  /// serves.
+  final rows = <String, String>{};
+
+  /// What was uploaded and not finished yet, by hash.
+  final _uploaded = <String, Uint8List>{};
+
+  /// The hashes uploads were started for, in order.
+  final started = <String>[];
+
+  /// The ids downloads were asked for, in order.
+  final downloads = <String>[];
+
+  /// The hashes of the uploads said to be encrypted (ADR 0019).
+  final encrypted = <String>{};
+
+  /// When set, every call fails with it, as without a connection.
+  Exception? failCalls;
+
+  /// When true, uploads don't arrive.
+  bool failUploads = false;
+
+  /// When set, uploads arrive with this content instead of what was sent.
+  Uint8List? corruptUploads;
+
+  /// What the person may keep, in bytes; past it, uploads are refused.
+  int limitBytes = 1 << 30;
+
+  int get _usedBytes =>
+      files.values.fold(0, (sum, bytes) => sum + bytes.length);
+
+  @override
+  Future<String?> startUpload(
+    String sha256,
+    int byteSize, {
+    bool encrypted = false,
+  }) async {
+    _check();
+    started.add(sha256);
+    if (encrypted) {
+      this.encrypted.add(sha256);
+    }
+    if (files.containsKey(sha256)) {
+      return null;
+    }
+    if (_usedBytes + byteSize > limitBytes) {
+      throw FileLimitReached(instanceFull: false, limitBytes: limitBytes);
+    }
+    return 'upload:$sha256';
+  }
+
+  @override
+  Future<bool> upload(String description, Uint8List bytes) async {
+    _check();
+    if (failUploads) {
+      return false;
+    }
+    _uploaded[description.substring('upload:'.length)] =
+        corruptUploads ?? bytes;
+    return true;
+  }
+
+  @override
+  Future<bool> finishUpload(String sha256, {bool encrypted = false}) async {
+    _check();
+    final bytes = _uploaded.remove(sha256);
+    if (bytes == null || sha256Of(bytes) != sha256) {
+      return false;
+    }
+    files[sha256] = bytes;
+    return true;
+  }
+
+  @override
+  Future<FileRoom> room() async {
+    _check();
+    return (usedBytes: _usedBytes, limitBytes: limitBytes);
+  }
+
+  @override
+  Future<Uint8List?> download(String id) async {
+    _check();
+    downloads.add(id);
+    return files[rows[id]];
+  }
+
+  void _check() {
+    if (failCalls case final error?) {
+      throw error;
+    }
+  }
+}
+
+/// The SHA-256 of [bytes] in lowercase hex.
+String sha256Of(List<int> bytes) => sha256.convert(bytes).toString();

packages/halkyon_app/lib/src/testing/memory_file_cache.dart

@@ -0,0 +1,31 @@
+import 'dart:typed_data';
+
+import 'package:halkyon_app/src/files/file_cache.dart';
+
+/// A [FileCache] in memory.
+class MemoryFileCache() implements FileCache {
+  /// The files on the "device", by hash.
+  final files = <String, Uint8List>{};
+
+  /// The hashes of the files waiting to be uploaded.
+  final pending = <String>{};
+
+  @override
+  Future<Uint8List?> read(String sha256) async => files[sha256];
+
+  @override
+  Future<void> write(String sha256, Uint8List bytes) async =>
+      files[checkedSha256(sha256)] = bytes;
+
+  @override
+  String pathOf(String sha256) => '/cache/${checkedSha256(sha256)}';
+
+  @override
+  Future<Set<String>> pendingUploads() async => {...pending};
+
+  @override
+  Future<void> markPending(String sha256) async => pending.add(sha256);
+
+  @override
+  Future<void> clearPending(String sha256) async => pending.remove(sha256);
+}

packages/halkyon_app/lib/testing.dart

@@ -13,6 +13,7 @@ export 'src/testing/fake_assistant_gateway.dart';
 export 'src/testing/fake_auth_gateway.dart';
 export 'src/testing/fake_content_cipher.dart';
 export 'src/testing/fake_file_picker.dart';
+export 'src/testing/fake_file_server_gateway.dart';
 export 'src/testing/fake_instance_checker.dart';
 export 'src/testing/fake_instance_operator_gateway.dart';
 export 'src/testing/fake_network_watch.dart';
@@ -24,6 +25,7 @@ export 'src/testing/fake_simplelogin_gateway.dart';
 export 'src/testing/fake_speech_input.dart';
 export 'src/testing/memory_account_key_store.dart';
 export 'src/testing/memory_account_wait_store.dart';
+export 'src/testing/memory_file_cache.dart';
 export 'src/testing/memory_file_saver.dart';
 export 'src/testing/memory_instance_store.dart';
 export 'src/testing/memory_look_store.dart';

packages/halkyon_app/pubspec.yaml

@@ -10,6 +10,8 @@ environment:
 
 dependencies:
   clock: ^1.1.3
+  # The files' hashes (FakeFileServerGateway; the apps hash with it too).
+  crypto: ^3.0.7
   equatable: ^3.0.0
   # chmod(2) for files saved on the desktop: the own user's alone (FileSaver).
   ffi: ^2.2.0

packages/halkyon_app/test/files/directory_file_cache_test.dart

@@ -0,0 +1,62 @@
+import 'dart:io';
+import 'dart:typed_data';
+
+import 'package:flutter_test/flutter_test.dart';
+import 'package:halkyon_app/halkyon_app.dart';
+import 'package:path/path.dart' as p;
+
+void main() {
+  late Directory folder;
+  late DirectoryFileCache cache;
+  final hash = 'a' * 64;
+  final other = 'b' * 64;
+
+  setUp(() async {
+    folder = await Directory.systemTemp.createTemp('files');
+    cache = DirectoryFileCache(Directory(p.join(folder.path, 'files')));
+  });
+  tearDown(() => folder.delete(recursive: true));
+
+  test('keeps a file by its hash, and has none it was not given', () async {
+    expect(await cache.read(hash), isNull);
+
+    await cache.write(hash, Uint8List.fromList([1, 2, 3]));
+
+    expect(await cache.read(hash), [1, 2, 3]);
+    expect(File(cache.pathOf(hash)).readAsBytesSync(), [1, 2, 3]);
+    expect(
+      Directory(p.join(folder.path, 'files')).listSync().map((e) => e.path),
+      [cache.pathOf(hash)],
+      reason: 'nothing half-written is left beside it',
+    );
+  });
+
+  test('keeps the file it has when given it again', () async {
+    await cache.write(hash, Uint8List.fromList([1]));
+    await cache.write(hash, Uint8List.fromList([2]));
+
+    expect(await cache.read(hash), [1]);
+  });
+
+  test('notes which files wait to be uploaded', () async {
+    expect(await cache.pendingUploads(), isEmpty);
+
+    await cache.markPending(hash);
+    await cache.markPending(other);
+    await cache.clearPending(hash);
+    await cache.clearPending('c' * 64);
+
+    expect(await cache.pendingUploads(), {other});
+  });
+
+  test('refuses anything but a SHA-256 in a path', () async {
+    for (final forged in ['../sync.sqlite', 'A' * 64, '${'a' * 63}/', '']) {
+      expect(() => cache.pathOf(forged), throwsArgumentError);
+      await expectLater(cache.markPending(forged), throwsArgumentError);
+      await expectLater(
+        cache.write(forged, Uint8List.fromList([1])),
+        throwsArgumentError,
+      );
+    }
+  });
+}