feat: Replace the isDeleted flag by an int CLFlag (#55)

Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>

packages/serverpod_offline_sync_client/lib/src/crdt/extensions.dart

@@ -24,10 +24,13 @@ extension CrdtDataRowExtension on CrdtDataRow {
 
 /// Extensions for the [CrdtDataDeleted] class.
 extension CrdtDataDeletedExtension on CrdtDataDeleted {
-  /// The HLC represented by this tombstone.
+  /// Whether this visibility generation hides the row.
+  bool get isDeleted => clFlag.isEven;
+
+  /// The HLC represented by this visibility generation.
   Hlc get hlc => node == null
       ? (throw StateError(
-          'Fetch the CRDT tombstone with the node included to get the HLC.',
+          'Fetch the CRDT visibility generation with the node included to get the HLC.',
         ))
       : toHlcForNode(node!.uuidNodeId);
 }

packages/serverpod_offline_sync_client/lib/src/crdt/sync.dart

@@ -466,7 +466,8 @@ class CrdtSync {
         tableName: tableName,
         uuidRowId: tombstone.row!.uuidRowId,
         uuidNodeId: tombstone.node!.uuidNodeId,
-        isDeleted: tombstone.isDeleted,
+        clFlag: tombstone.clFlag,
+        reason: tombstone.reason,
       );
     }
   }

packages/serverpod_offline_sync_client/lib/src/database/merge.dart

@@ -332,17 +332,18 @@ extension CrdtMergeRecorderExtension on CrdtMutationRecorder {
     final row = rows[rowKey];
     if (row == null) return;
 
-    final incomingHlc = delete.hlc;
     final currentTombstone = tombstones[rowKey];
-    if (incomingHlc <= row.hlc.maxBetween(currentTombstone?.hlc)) {
-      return;
-    }
+    final currentClFlag = currentTombstone?.clFlag ?? 1;
+    final currentHlc = currentTombstone?.hlc ?? row.hlc;
+    if (delete.clFlag < currentClFlag) return;
+    if (delete.clFlag == currentClFlag && delete.hlc <= currentHlc) return;
 
     tombstones[rowKey] = await _upsertMergeTombstone(
       row,
       _requireRemoteNode(remoteNodes, delete.uuidNodeId),
-      incomingHlc,
-      delete.isDeleted,
+      delete.hlc,
+      delete.clFlag,
+      delete.reason,
       currentTombstone,
       transaction,
     );
@@ -441,7 +442,8 @@ extension CrdtMergeRecorderExtension on CrdtMutationRecorder {
     CrdtDataRow row,
     CrdtNode remoteNode,
     Hlc incomingHlc,
-    bool isDeleted,
+    int clFlag,
+    CrdtDataDeletedReason reason,
     CrdtDataDeleted? currentTombstone,
     Transaction transaction,
   ) async {
@@ -453,7 +455,8 @@ extension CrdtMergeRecorderExtension on CrdtMutationRecorder {
           nodeId: remoteNode.id!,
           hlcDatetime: incomingHlc.datetime,
           hlcCounter: incomingHlc.counter,
-          isDeleted: isDeleted,
+          clFlag: clFlag,
+          reason: reason,
         ),
         transaction: transaction,
       );
@@ -470,13 +473,14 @@ extension CrdtMergeRecorderExtension on CrdtMutationRecorder {
       node: remoteNode,
       hlcDatetime: incomingHlc.datetime,
       hlcCounter: incomingHlc.counter,
-      isDeleted: isDeleted,
+      clFlag: clFlag,
+      reason: reason,
     );
 
     await CrdtDataDeleted.db.update(
       _session,
       [updatedTombstone],
-      columns: (t) => [t.nodeId, t.hlcDatetime, t.hlcCounter, t.isDeleted],
+      columns: (t) => [t.nodeId, t.hlcDatetime, t.hlcCounter, t.clFlag, t.reason],
       transaction: transaction,
     );
 
@@ -558,12 +562,14 @@ extension CrdtMergeRecorderExtension on CrdtMutationRecorder {
 
     final rowKey = (insert.tableName, insert.uuidRowId);
     final currentTombstone = tombstones[rowKey];
-    if (currentTombstone == null || incomingHlc > currentTombstone.hlc) {
+    if ((currentTombstone?.clFlag ?? 1) == 1 &&
+        incomingHlc > (currentTombstone?.hlc ?? currentRow.hlc)) {
       tombstones[rowKey] = await _upsertMergeTombstone(
         currentRow,
         remoteNode,
         incomingHlc,
-        false,
+        1,
+        CrdtDataDeletedReason.userInsert,
         currentTombstone,
         transaction,
       );

packages/serverpod_offline_sync_client/lib/src/database/recorder.dart

@@ -240,7 +240,12 @@ class CrdtMutationRecorder {
       );
 
       await _touchCrdtRows(crdtDataRows, hlcManager, transaction);
-      await _markCrdtRowsDeleted(crdtDataRows, false, transaction);
+      await _markCrdtRowsDeleted(
+        crdtDataRows,
+        false,
+        CrdtDataDeletedReason.userReinsert,
+        transaction,
+      );
       await _recordUpdatedFields(reinsertedRows, crdtDataRows, null, transaction);
     });
   }
@@ -347,7 +352,7 @@ LEFT JOIN "crdt_data_tombstone" tomb
   ON tomb."rowId" = r."id"
 WHERE c."id" IN ($whereRowIds)
   AND c."$childColumn" IS NOT NULL
-  AND (p."id" IS NULL OR tomb."isDeleted" = TRUE)
+  AND (p."id" IS NULL OR tomb."clFlag" % 2 = 0)
 ''';
     });
 
@@ -543,6 +548,7 @@ WHERE c."id" IN ($whereRowIds)
       deletedRows.uuidRowIds,
       transaction,
       null,
+      CrdtDataDeletedReason.userDelete,
     );
   }
 
@@ -551,6 +557,7 @@ WHERE c."id" IN ($whereRowIds)
     Set<UuidValue> rowIds,
     Transaction transaction,
     Set<String>? processedRowIds,
+    CrdtDataDeletedReason reason,
   ) async {
     if (rowIds.isEmpty) return;
     if (!_isCrdtTrackedTableName(tableName)) return;
@@ -585,7 +592,7 @@ WHERE c."id" IN ($whereRowIds)
         processing,
       );
       await _releaseUniqueConflicts(tableName, visibleRowIds, transaction);
-      await _markCrdtRowsDeleted(visibleCrdtRows, true, transaction);
+      await _markCrdtRowsDeleted(visibleCrdtRows, true, reason, transaction);
     } finally {
       for (final rowId in unprocessedRowIds) {
         processing.remove('$tableName:$rowId');
@@ -665,6 +672,7 @@ WHERE c."id" IN ($whereRowIds)
   Future<void> _markCrdtRowsDeleted(
     List<CrdtDataRow> crdtDataRows,
     bool isDeleted,
+    CrdtDataDeletedReason reason,
     Transaction transaction,
   ) async {
     final existingTombs = await CrdtDataDeleted.db.find(
@@ -682,6 +690,7 @@ WHERE c."id" IN ($whereRowIds)
       final rowPk = row.id!;
       final hlc = hlcManager.increment();
       final existing = tombByCrdtRowPk[rowPk];
+      final clFlag = _nextClFlag(existing, isDeleted);
 
       if (existing == null) {
         toInsert.add(
@@ -690,7 +699,8 @@ WHERE c."id" IN ($whereRowIds)
             nodeId: hlcManager.normalizedNodeId,
             hlcDatetime: hlc.datetime,
             hlcCounter: hlc.counter,
-            isDeleted: isDeleted,
+            clFlag: clFlag,
+            reason: reason,
           ),
         );
       } else {
@@ -699,7 +709,8 @@ WHERE c."id" IN ($whereRowIds)
             nodeId: hlcManager.normalizedNodeId,
             hlcDatetime: hlc.datetime,
             hlcCounter: hlc.counter,
-            isDeleted: isDeleted,
+            clFlag: clFlag,
+            reason: reason,
           ),
         );
       }
@@ -843,6 +854,7 @@ WHERE c."id" IN ($whereRowIds)
               childIds,
               transaction,
               processing,
+              CrdtDataDeletedReason.cascadeDelete,
             );
         }
       }
@@ -925,7 +937,7 @@ LEFT JOIN "crdt_data_rows" r
 LEFT JOIN "crdt_data_tombstone" tomb
   ON tomb."rowId" = r."id"
 WHERE d."${_escapeIdentifier(columnName)}" = ${_sqlLiteral(value)}
-  AND (tomb."id" IS NULL OR tomb."isDeleted" = FALSE)
+  AND (tomb."id" IS NULL OR tomb."clFlag" % 2 = 1)
 ''',
       transaction: transaction,
     );
@@ -1059,6 +1071,12 @@ WHERE "id" IN (${_sqlLiteralList(rowIds)})
 
 String _escapeIdentifier(String identifier) => identifier.replaceAll('"', '""');
 
+int _nextClFlag(CrdtDataDeleted? current, bool isDeleted) {
+  var next = (current?.clFlag ?? 1) + 1;
+  if (next.isEven != isDeleted) next++;
+  return next;
+}
+
 List<_UniqueIndexConflictRelease> _syncableUniqueIndexesForTable(
   TableDefinition table,
 ) {

packages/serverpod_offline_sync_client/lib/src/database/tombstone.dart

@@ -5,7 +5,7 @@ import '../protocol/protocol.dart';
 
 /// Merges [where] with CRDT tombstone predicates and mutates [include] in place.
 ///
-/// This method ensures that no rows marked as deleted on the tombstone are
+/// This method ensures that no rows marked as deleted by the CLFlag are
 /// returned for queries with [where] clauses or [include] graphs. If [include]
 /// is null, only the root table and [where] are merged.
 ///
@@ -85,7 +85,7 @@ extension on Table {
   ///   - [CrdtDataRow.id] = [CrdtDataDeleted.rowId].
   ///
   /// Returns null when [Table] does not use a UUID primary key. Keeps rows
-  /// when there is no tombstone row or [CrdtDataDeleted.isDeleted] is false
+  /// when there is no tombstone row or [CrdtDataDeleted.clFlag] is odd
   /// (LEFT JOIN null-safe).
   Expression? get whereNotDeletedOnTombstone {
     if (id is! ColumnUuid) return null;
@@ -98,6 +98,6 @@ extension on Table {
           CrdtDataRowTable(tableRelation: foreignTableRelation),
     );
     final tomb = crdtRowTable.deleted;
-    return (tomb.id.equals(null)) | (tomb.isDeleted.equals(false));
+    return (tomb.id.equals(null)) | Expression('${tomb.clFlag} % 2 = 1');
   }
 }

packages/serverpod_offline_sync_server/lib/src/models/data/deleted.spy.yaml

@@ -1,20 +1,24 @@
-### CRDT tombstone table.
+### CRDT visibility generation table.
 ###
-### This table stores all tombstones for deleted data.
+### This table stores materialized monotone CLFlag metadata after a visibility
+### change event has been recorded.
 class: CrdtDataDeleted
 extends: BaseHlc
 table: crdt_data_tombstone
 database: all
 fields:
-  ### Row that was deleted.
+  ### Row whose visibility is tracked.
   row: CrdtDataRow?, relation(name=row_deleted, field=rowId, onDelete=Cascade)
   rowId: int
 
   ### The node that updated the field.
   node: CrdtNode?, relation
 
-  ### When the row was deleted.
-  isDeleted: bool
+  ### Monotone causal-length flag. Odd values are visible and even values are deleted.
+  clFlag: int
+
+  ### Why the row entered its current visibility generation.
+  reason: CrdtDataDeletedReason
 
 indexes:
   crdt_data_tombstone_row_idx:

packages/serverpod_offline_sync_server/lib/src/models/data/deleted_reason.spy.yaml

@@ -0,0 +1,10 @@
+### Why a CRDT row entered its current visibility generation.
+enum: CrdtDataDeletedReason
+serialized: byIndex
+values:
+  - userInsert
+  - userDelete
+  - userReinsert
+  - restrictRestore
+  - cascadeDelete
+  - uniqueLoser

packages/serverpod_offline_sync_server/lib/src/models/data/row.spy.yaml

@@ -22,7 +22,10 @@ fields:
   ### The node that inserted the row with the HLC timestamp.
   node: CrdtNode?, relation
 
-  ### The tombstone for this row.
+  ### The visibility CLFlag for this row.
+  ###
+  ### Only present after the first visibility change event has been recorded.
+  ### If null, the row is visible and has never been deleted/hidden.
   deleted: CrdtDataDeleted?, relation(name=row_deleted, optional)
 
   ### The fields for this row.

packages/serverpod_offline_sync_server/lib/src/models/merge/delete.spy.yaml

@@ -1,6 +1,9 @@
-### A remote tombstone update to merge.
+### A remote visibility CLFlag update to merge.
 class: CrdtMergeDelete
 extends: CrdtMergeChange
 fields:
-  ### Whether the row is deleted or restored.
-  isDeleted: bool
+  ### Monotone causal-length flag. Odd values are visible and even values are deleted.
+  clFlag: int
+
+  ### Why the row entered its current visibility generation.
+  reason: CrdtDataDeletedReason