fix(privacy): the instance's side of the reading of private studysets (ADR 0019)

An upload said to be encrypted must start as a sealed file does (HLKF,
version 1), so a readable file can't skip the PDF check by saying so. A
person's room counts the files a live asset row names and those claimed
within the week of grace, so a private studyset's rewrite near the limit
no longer fails for a week while pruning keeps the replaced files. Copy
reports are limited to 60 an hour per person. Tests: what goes with the
person, `counted` refused to anonymous people.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Signed-off-by: Tizian 「ティツィアーン」 <raisondetredev@pm.me>

apps/learn/halkyon_learn_server/lib/src/asset/asset_endpoint.dart

@@ -76,8 +76,9 @@ class AssetEndpoint extends Endpoint {
   /// stored now. A file whose content has another hash is deleted, and so is
   /// a readable one over [maxOtherBytes] that isn't a PDF, and one that no
   /// longer fits, with a [FileLimitExceededException]. An [encrypted] file
-  /// (ADR 0019) can't be looked into: its size alone counts, as the quotas'
+  /// (ADR 0019) can't be looked into: it must start as a sealed file does
-  /// sizes do.
+  /// (`HLKF`, version 1), or it is deleted, and its size alone counts, as
+  /// the quotas' sizes do.
   Future<bool> finishUpload(
     Session session,
     String sha256, {
@@ -96,9 +97,9 @@ class AssetEndpoint extends Endpoint {
       path: path,
     );
     if (_hashOf(bytes) != sha256 ||
-        (!(encrypted ?? false) &&
+        (encrypted ?? false
-            bytes.lengthInBytes > maxOtherBytes &&
+            ? !_isSealed(bytes)
-            !_isPdf(bytes)) ||
+            : bytes.lengthInBytes > maxOtherBytes && !_isPdf(bytes)) ||
         bytes.lengthInBytes > maxBytes + encryptionOverhead) {
       await session.storage.deleteFile(storageId: assetStorage, path: path);
       return false;
@@ -183,5 +184,17 @@ bool _isPdf(ByteData bytes) {
           signature;
 }
 
+/// Whether [bytes] are a sealed file's (ADR 0019): they start as the app
+/// seals every file, `HLKF` and its format's version, 1. What follows can't
+/// be looked into, but a readable file said to be sealed can't skip the
+/// PDF check unnoticed.
+bool _isSealed(ByteData bytes) {
+  const magic = 'HLKF';
+  return bytes.lengthInBytes > magic.length + 1 &&
+      String.fromCharCodes(Uint8List.sublistView(bytes, 0, magic.length)) ==
+          magic &&
+      bytes.getUint8(magic.length) == 1;
+}
+
 String _hashOf(ByteData bytes) =>
     sha256.convert(Uint8List.sublistView(bytes)).toString();

apps/learn/halkyon_learn_server/lib/src/asset/file_limits.dart

@@ -1,9 +1,12 @@
 import 'dart:io';
 
 import 'package:halkyon_base_server/base.dart';
+import 'package:clock/clock.dart';
 import 'package:halkyon_learn_server/src/asset/asset_endpoint.dart';
+import 'package:halkyon_learn_server/src/asset/file_pruning.dart';
 import 'package:halkyon_learn_server/src/generated/protocol.dart';
 import 'package:serverpod/serverpod.dart';
+import 'package:serverpod_offline_sync_server/serverpod_offline_sync_server.dart';
 
 /// How much room people's files may take (decided 2026-10-02): an anonymous
 /// person little, since anonymous accounts are meant to become identified
@@ -80,11 +83,7 @@ Future<Room> roomOf(
     where: (t) => t.authUserId.equals(person),
   );
   return (
-    used: await _storedBytes(
+    used: await _spaceBytes(session, space, finishedOnly: finishedOnly),
-      session,
-      '$assetPathPrefix${space.uuid}/',
-      finishedOnly: finishedOnly,
-    ),
     limit:
         allowance?.maxBytes ??
         (session.authenticated!.scopes.contains(identified)
@@ -122,6 +121,65 @@ int effectiveLimit(Room room) {
   return room.used + left < room.limit ? room.used + left : room.limit;
 }
 
+/// What the stored files of [space] take: those a live asset row of it
+/// names, and those claimed within [filePruningGrace] (an upload's row may
+/// sync after it), unfinished uploads included unless [finishedOnly]. A
+/// file no live row names any more, as one a private studyset's rewrite
+/// replaced (ADR 0019), counts no longer, though pruning keeps it a while.
+Future<int> _spaceBytes(
+  Session session,
+  UuidValue space, {
+  required bool finishedOnly,
+}) async {
+  final prefix = '$assetPathPrefix${space.uuid}/';
+  final cutoff = clock
+      .now()
+      .toUtc()
+      .subtract(filePruningGrace)
+      .toIso8601String()
+      .replaceFirst('Z', '');
+  // SELECT "path", octet_length("byteData"), "addedTime" >= $3
+  //   FROM "serverpod_cloud_storage"
+  //   WHERE "storageId" = 'private' AND "path" LIKE $1 [AND "verified"]
+  final rows = await session.db.unsafeQuery(
+    'SELECT "path", octet_length("byteData")::bigint, '
+    '"addedTime" >= @cutoff::timestamp '
+    'FROM "serverpod_cloud_storage" '
+    'WHERE "storageId" = @storage AND "path" LIKE @prefix'
+    '${finishedOnly ? ' AND "verified"' : ''}',
+    parameters: QueryParameters.named({
+      'storage': assetStorage,
+      'prefix': '$prefix%',
+      'cutoff': cutoff,
+    }),
+  );
+  if (rows.isEmpty) {
+    return 0;
+  }
+  // SELECT "id" FROM "offline_sync_spaces" WHERE "uuidSpaceId" = $1
+  final spaceId = (await OfflineSyncSpace.db.findFirstRow(
+    session,
+    where: (t) => t.uuidSpaceId.equals(space),
+  ))?.id;
+  // SELECT "sha256" FROM "asset" WHERE "spaceId" = $1 (tombstones hidden)
+  final live = spaceId == null
+      ? const <String>{}
+      : {
+          for (final asset in await Asset.db.find(
+            session,
+            where: (t) => t.spaceId.equals(spaceId),
+          ))
+            asset.sha256,
+        };
+  var used = 0;
+  for (final [path as String, bytes as int, recent as bool] in rows) {
+    if (recent || live.contains(path.substring(prefix.length))) {
+      used += bytes;
+    }
+  }
+  return used;
+}
+
 /// What the stored files below [prefix] take: unfinished uploads included,
 /// unless [finishedOnly].
 Future<int> _storedBytes(

apps/learn/halkyon_learn_server/lib/src/explore/learning_count_endpoint.dart

@@ -4,6 +4,19 @@ import 'package:halkyon_learn_server/src/generated/protocol.dart';
 import 'package:serverpod/serverpod.dart';
 import 'package:serverpod_auth_idp_server/core.dart';
 
+/// The most reports a person's devices may send an hour: each device
+/// reports after its syncs, when what it reports changed.
+const copyReportsPerHour = 60;
+
+final _copyReports = DatabaseRateLimiter(
+  RateLimiterConfig(
+    domain: 'learning_count',
+    source: 'copy_reports_per_person',
+    maxAttempts: copyReportsPerHour,
+    timeframe: const Duration(hours: 1),
+  ),
+);
+
 /// Whether the caller is counted as learning what their copies come from
 /// ("Learn this here", ADR 0005, rule 8), and which publications those are
 /// (ADR 0019, D3): their settings and private copies are encrypted, so their
@@ -67,8 +80,9 @@ class LearningCountEndpoint extends Endpoint {
 
   /// Replaces the copies the caller's device reported before with
   /// [copies]; whether they were kept, which they are only while the
-  /// caller chose to be counted. Which of them are really the caller's,
+  /// caller chose to be counted, and for at most [copyReportsPerHour]
-  /// and still there, the count works out from their own rows.
+  /// reports an hour. Which of them are really the caller's, and still
+  /// there, the count works out from their own rows.
   Future<bool> reportCopies(Session session, List<CopyReport> copies) async {
     if (copies.length > maxPublications ||
         copies.fold(0, (sum, copy) => sum + copy.studysetIds.length) >
@@ -76,6 +90,9 @@ class LearningCountEndpoint extends Endpoint {
       throw ArgumentError.value(copies.length, 'copies', 'too many');
     }
     final person = session.authenticated!.authUserId;
+    if (!await _copyReports.tryRecordAttempt(session, key: person.uuid)) {
+      return false;
+    }
     return await session.db.transaction((transaction) async {
       // SELECT count(*) FROM "counted_learner" WHERE "authUserId" = $1
       if (await CountedLearner.db.count(

apps/learn/halkyon_learn_server/test/integration/asset/asset_test.dart

@@ -191,7 +191,9 @@ void main() {
         'look into it, a little larger than 50 MB too, and no larger',
         () async {
           const size = AssetEndpoint.maxOtherBytes + 1;
-          final sealed = Uint8List(size)..fillRange(0, size, 7);
+          final sealed = Uint8List(size)
+            ..fillRange(0, size, 7)
+            ..setAll(0, [...utf8.encode('HLKF'), 1]);
           final readable = Uint8List(size)..fillRange(0, size, 8);
           const most =
               AssetEndpoint.maxBytes + AssetEndpoint.encryptionOverhead;
@@ -228,6 +230,25 @@ void main() {
         },
       );
 
+      test(
+        'keeps no file said to be encrypted that is no sealed file, as a '
+        'readable one would skip the PDF check',
+        () async {
+          const size = AssetEndpoint.maxOtherBytes + 1;
+          final readable = Uint8List(size)..fillRange(0, size, 9);
+          final wrongVersion = Uint8List(size)
+            ..fillRange(0, size, 9)
+            ..setAll(0, [...utf8.encode('HLKF'), 2]);
+
+          expect(await add(alice, readable, encrypted: true), isFalse);
+          expect(await add(alice, wrongVersion, encrypted: true), isFalse);
+          expect(
+            await add(alice, utf8.encode('HLKF'), encrypted: true),
+            isFalse,
+          );
+        },
+      );
+
       test('finishes no upload that never came', () async {
         expect(
           await endpoints.asset.finishUpload(signedIn(alice), imageHash),
@@ -488,6 +509,40 @@ void main() {
           );
         });
 
+        test(
+          'counts no file a live row no longer names once its week of grace '
+          'is over, as one a rewrite replaced (ADR 0019)',
+          () async {
+            final replaced = List.filled(6, 3);
+            final named = List.filled(3, 4);
+            String hash(List<int> bytes) => sha256.convert(bytes).toString();
+            final gone = await _assetIn(session, alice, hash(replaced));
+            await add(alice, replaced);
+            await _deleteAsset(session, alice, gone);
+            await _assetIn(session, alice, hash(named));
+            await add(alice, named);
+            final old = DateTime.now().toUtc().subtract(
+              const Duration(days: 8),
+            );
+            await _claimed(session, alice, hash(replaced), old);
+            await _claimed(session, alice, hash(named), old);
+
+            final usage = await endpoints.asset.usage(signedIn(alice));
+            expect(usage.usedBytes, 3);
+          },
+        );
+
+        test(
+          'counts a file no row names yet while it is recent, so uploads '
+          "without rows can't fill the room unnoticed",
+          () async {
+            await add(alice, List.filled(4, 5));
+
+            final usage = await endpoints.asset.usage(signedIn(alice));
+            expect(usage.usedBytes, 4);
+          },
+        );
+
         test('tells how much room the files take, and may take', () async {
           await add(alice, List.filled(4, 1));
 

apps/learn/halkyon_learn_server/test/integration/explore/learner_count_test.dart

@@ -2,6 +2,7 @@ import 'dart:typed_data';
 
 import 'package:halkyon_base_server/base.dart';
 import 'package:halkyon_learn_server/src/explore/learner_count.dart';
+import 'package:halkyon_learn_server/src/explore/learning_count_endpoint.dart';
 import 'package:halkyon_learn_server/src/generated/protocol.dart';
 import 'package:serverpod/serverpod.dart';
 import 'package:serverpod_auth_idp_server/core.dart';
@@ -222,8 +223,51 @@ void main() {
 
         await learner();
         expect(await counted(), 1);
+        final gone = people.last;
         await deletePerson(session, people.removeLast());
         expect(await counted(), 0);
+        // Nothing of theirs stays behind, whatever the count says.
+        expect(
+          await CountedLearner.db.count(
+            session,
+            where: (t) => t.authUserId.equals(gone),
+          ),
+          0,
+        );
+        expect(
+          await LearningCopy.db.count(
+            session,
+            where: (t) => t.authUserId.equals(gone),
+          ),
+          0,
+        );
+      });
+
+      test('a person reports at most an hour\'s worth of times', () async {
+        final copy = await learner();
+        final person = people.last;
+        final report = [
+          CopyReport(
+            publicationId: japanese.publicId.uuid,
+            studysetIds: [copy],
+          ),
+        ];
+        // The learner reported once already.
+        for (var i = 1; i < copyReportsPerHour; i++) {
+          expect(
+            await endpoints.learningCount.reportCopies(
+              signedIn(person),
+              report,
+            ),
+            isTrue,
+          );
+        }
+
+        expect(
+          await endpoints.learningCount.reportCopies(signedIn(person), []),
+          isFalse,
+        );
+        expect(await counted(), 1);
       });
 
       test('reports are kept only while the person chose to be counted, '
@@ -274,6 +318,10 @@ void main() {
           ),
           throwsA(isA<ServerpodInsufficientAccessException>()),
         );
+        await expectLater(
+          endpoints.learningCount.counted(signedIn(person, anonymous: true)),
+          throwsA(isA<ServerpodInsufficientAccessException>()),
+        );
         expect(await CountedLearner.db.count(session), 0);
       });
     },