test: Add a DST and tests reproducing found failures (#110)

Co-authored-by: Marcelo Soares <marcelo.me.soares@gmail.com>

.github/workflows/ci.yaml

@@ -98,6 +98,29 @@ jobs:
       #     file-path: coverage-results.txt
       #     comment-tag: coverage-results
 
+  simulation:
+    name: deterministic simulation
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    # Deliberately not on pull requests. The suite seeds from the clock, so it
+    # searches fresh schedules on every run and can fail for a defect unrelated
+    # to the change under review - which is what makes it worth running, and
+    # what makes it a poor merge gate. It runs on pushes to the release
+    # branches and on demand, where a failure is a finding rather than a
+    # blocked review.
+    if: github.event_name != 'pull_request'
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/setup
+
+      - name: Run the simulation suite
+        run: melos test-dst
+        env:
+          # Wider than a local run, and still inside the job timeout. A failure
+          # prints DST_SEED_BASE, which replays that simulation exactly.
+          DST_SEEDS: 50
+          DST_ROUNDS: 20
+
   benchmarks:
     name: run benchmarks
     runs-on: ubuntu-latest

packages/serverpod_offline_sync/lib/src/crdt/integrity_violation.dart

@@ -1,3 +1,4 @@
+import 'package:clock/clock.dart';
 import 'package:serverpod_database/serverpod_database.dart';
 
 import '../generated/protocol.dart';
@@ -20,7 +21,7 @@ Future<CrdtSyncIntegrityViolation> recordCrdtSyncIntegrityViolation(
     transaction: transaction,
   );
 
-  final now = DateTime.now().toUtc();
+  final now = clock.now().toUtc();
   final newViolation = existing != null
       ? existing.copyWith(
           crdtDataRowId: violation.crdtDataRowId ?? existing.crdtDataRowId,

packages/serverpod_offline_sync/lib/src/crdt/sync.dart

@@ -1,5 +1,6 @@
 import 'dart:async';
 
+import 'package:clock/clock.dart';
 import 'package:serverpod_database/serverpod_database.dart';
 import 'package:uuid/uuid.dart';
 
@@ -463,7 +464,7 @@ class CrdtSync {
     if (role.canWrite) return;
 
     final firstChange = changes.first;
-    final now = DateTime.now().toUtc();
+    final now = clock.now().toUtc();
     final violation = CrdtSyncIntegrityViolation(
       type: CrdtSyncViolationType.unauthorizedWrite,
       domainTableName: firstChange.tableName,
@@ -963,7 +964,7 @@ class CrdtSync {
       session,
       pending.ownerScopeId,
     );
-    final now = DateTime.now().toUtc();
+    final now = clock.now().toUtc();
     final violation = CrdtSyncIntegrityViolation(
       type: pending.type,
       domainTableName: pending.tableName,

packages/serverpod_offline_sync/lib/src/database/merge.dart

@@ -316,6 +316,7 @@ extension CrdtMergeRecorderExtension on CrdtMutationRecorder {
               {insert.uuidRowId},
               savepoint,
               safeInsert.attempts,
+              mergeCache: (fields: context.fields, node: remoteNode),
             );
             return insertedRow;
           },
@@ -897,7 +898,7 @@ extension CrdtMergeRecorderExtension on CrdtMutationRecorder {
     CrdtNode? node,
     Hlc? hlc,
   }) async {
-    final now = DateTime.now().toUtc();
+    final now = clock.now().toUtc();
     throw CrdtSyncIntegrityViolationException(
       CrdtSyncIntegrityViolation(
         type: CrdtSyncViolationType.ownershipCollision,

packages/serverpod_offline_sync/lib/src/database/merge_utils/foreign_key_projector.dart

@@ -276,12 +276,26 @@ class CrdtForeignKeyProjector {
     await _upsertProjections(projectionWrites, transaction);
   }
 
+  /// Records the foreign key attempts carried by an incoming insert.
+  ///
+  /// Foreign key columns keep their attempted value in [CrdtDataForeignKey],
+  /// which hangs off a [CrdtDataField], so an insert has to materialize field
+  /// metadata for them even though no update has happened yet.
+  ///
+  /// [mergeCache] is the merge context's field cache and the node authoring the
+  /// batch. The cache is loaded once before a batch is applied, so without this
+  /// it would not know about the fields created here, and a later change to the
+  /// same column in the same batch would take the "no field yet" path and
+  /// insert a second row for the same `(rowId, columnId)`.
+  ///
+  /// The local write path passes nothing: it has no batch to cache for.
   Future<void> recordInsertAttempts(
     String tableName,
     Set<UuidValue> rowIds,
     Transaction transaction,
-    ForeignKeyAttemptsByField attemptedValues,
+    ForeignKeyAttemptsByField attemptedValues, {
-  ) async {
+    MergeFieldCache? mergeCache,
+  }) async {
     if (rowIds.isEmpty) return;
 
     final foreignKeyColumns = _foreignKeys.foreignKeyColumnsFor(tableName);
@@ -324,7 +338,10 @@ class CrdtForeignKeyProjector {
       columnNames: columnIds.keys.toSet(),
       transaction: transaction,
     );
-    final fieldsToInsert = <CrdtDataField>[];
+    // Field and key travel together so the write-back cannot pair them up
+    // wrongly; zipping two lists by index would quietly depend on `insert`
+    // returning rows in the order it was given them.
+    final pending = <({CrdtDataField field, MergeFieldKey key})>[];
     for (final crdtRow in crdtRows) {
       final values = valuesByRowId[crdtRow.uuidRowId];
       if (values == null) continue;
@@ -335,24 +352,35 @@ class CrdtForeignKeyProjector {
           continue;
         }
 
-        fieldsToInsert.add(
+        pending.add((
-          CrdtDataField(
+          field: CrdtDataField(
             rowId: crdtRow.id!,
             columnId: columnId,
             nodeId: crdtRow.nodeId,
             hlcDatetime: crdtRow.hlcDatetime,
             hlcCounter: crdtRow.hlcCounter,
           ),
-        );
+          key: (tableName, crdtRow.uuidRowId, columnName),
+        ));
       }
     }
 
-    if (fieldsToInsert.isNotEmpty) {
+    if (pending.isNotEmpty) {
-      await CrdtDataField.db.insert(
+      final insertedFields = await CrdtDataField.db.insert(
         _context.databaseSession,
-        fieldsToInsert,
+        [for (final entry in pending) entry.field],
         transaction: transaction,
       );
+      if (mergeCache != null) {
+        for (final (index, field) in insertedFields.indexed) {
+          // The node is attached because a cached field is read back through
+          // `CrdtDataField.hlc`, which resolves the node's uuid and throws
+          // without it.
+          mergeCache.fields[pending[index].key] = field.copyWith(
+            node: mergeCache.node,
+          );
+        }
+      }
     }
 
     await recordAttemptsForRows(
@@ -971,7 +999,7 @@ class CrdtForeignKeyProjector {
       for (final child
           in state.rowsByTable[edge.childTableName] ??
               const <_ProjectedForeignKeyRow>[]) {
-        if (finalHidden.contains(child.key)) continue;
+        final isHidden = finalHidden.contains(child.key);
 
         final attemptedValue = _attemptedValue(child, edge, state);
         final currentVisibleValue = child.values[edge.childColumn].toUuidValue();
@@ -1015,6 +1043,31 @@ class CrdtForeignKeyProjector {
           }
         }
 
+        // A hidden row is repaired like any other, so replicas that hid it by
+        // different routes still agree about its columns. Two values it must
+        // not materialize, though:
+        //
+        // - one whose parent row is absent entirely, which the physical
+        //   foreign key constraint rejects - that is why a row whose parent is
+        //   missing is hidden with its safe value left in place;
+        // - one on a unique-indexed column that it would be adopting *back*
+        //   after a repair. A hidden row is invisible to unique conflict
+        //   resolution but still occupies the physical unique index, so
+        //   re-adopting the value when the parent returns wedges every later
+        //   legitimate claim on it.
+        if (isHidden && desiredVisibleValue != null) {
+          if (_parentRowForValue(edge, desiredVisibleValue, state) == null) {
+            continue;
+          }
+          if (overrideReason == null &&
+              _uniqueResolver.isUniqueIndexedColumn(
+                edge.childTableName,
+                edge.childColumn,
+              )) {
+            continue;
+          }
+        }
+
         if (!currentVisibleValue.sameUuidValue(desiredVisibleValue)) {
           child.values[edge.childColumn] = desiredVisibleValue;
           changedUpdatesByRow.putIfAbsent(child.key, () => {})[edge.childColumn] =

packages/serverpod_offline_sync/lib/src/database/merge_utils/types.dart

@@ -23,6 +23,18 @@ typedef MergeContext = ({
   Map<MergeRowKey, DomainRowOwner> domainOwners,
 });
 
+/// The merge context's field cache, with the node authoring the batch.
+///
+/// The two are one parameter because they are only meaningful together: a
+/// cached field is read back through `CrdtDataField.hlc`, which resolves the
+/// node's uuid and throws without it. Separate optional parameters would allow
+/// a cache with no node, whose entries throw when read.
+@internal
+typedef MergeFieldCache = ({
+  Map<MergeFieldKey, CrdtDataField> fields,
+  CrdtNode node,
+});
+
 /// Remote nodes and scope-node rows loaded for a merge.
 @internal
 typedef MergeNodes = ({

packages/serverpod_offline_sync/lib/src/database/merge_utils/unique_resolver.dart

@@ -378,6 +378,13 @@ class CrdtUniqueConflictResolver {
     );
   }
 
+  /// Whether [columnName] participates in any unique index of [tableName].
+  ///
+  /// Foreign key projection needs this to know when materializing a value onto
+  /// a hidden row would occupy the physical unique index.
+  bool isUniqueIndexedColumn(String tableName, String columnName) =>
+      _isUniqueIndexedColumn(tableName, columnName);
+
   bool _isUniqueIndexedColumn(String tableName, String columnName) {
     final tableDefinition = _context.tableDefinitionsByName[tableName];
     if (tableDefinition == null) return false;

packages/serverpod_offline_sync/lib/src/database/recorder.dart

@@ -1,3 +1,4 @@
+import 'package:clock/clock.dart';
 import 'package:meta/meta.dart';
 import 'package:serverpod_database/serverpod_database.dart';
 import 'package:serverpod_serialization/serverpod_serialization.dart';

packages/serverpod_offline_sync/lib/src/hlc/hlc.dart

@@ -75,7 +75,7 @@ class Hlc implements Comparable<Hlc> {
   factory Hlc.zero(UuidValue nodeId) => Hlc(DateTime.utc(1970), 0, nodeId);
 
   /// Instantiates an Hlc using the wall clock.
-  factory Hlc.now(UuidValue nodeId) => Hlc(DateTime.now(), 0, nodeId);
+  factory Hlc.now(UuidValue nodeId) => Hlc(clock.now(), 0, nodeId);
 
   /// Parses the node segment of an HLC string: a UUID, or legacy `uuid:workerId`.
   static UuidValue parseNodeSegment(String segment) {

packages/serverpod_offline_sync/test/hlc/hlc_comparison_test.dart

@@ -1,14 +1,12 @@
 import 'package:serverpod_offline_sync/serverpod_offline_sync.dart';
 import 'package:test/test.dart';
-import 'package:uuid/uuid.dart';
 
-final _dateTime = DateTime.parse('2001-09-09T01:46:40.000Z');
+import 'hlc_fixtures.dart';
-final _nodeId = const Uuid().v4obj();
 
 void main() {
   group('Given two Hlcs with same dateTime, counter and nodeId when comparing ', () {
-    final hlc1 = Hlc(_dateTime, 17, _nodeId);
+    final hlc1 = Hlc(hlcTime, 17, hlcNodeId);
-    final hlc2 = Hlc(_dateTime, 17, _nodeId);
+    final hlc2 = Hlc(hlcTime, 17, hlcNodeId);
 
     test('then they are equal.', () {
       expect(hlc1, hlc2);
@@ -34,10 +32,8 @@ void main() {
   group(
     'Given two Hlcs with same dateTime and counter and different nodeId when comparing ',
     () {
-      final node2 = const Uuid().v4obj();
+      final hlc1 = Hlc(hlcTime, 17, hlcNodeId);
-
+      final hlc2 = Hlc(hlcTime, 17, hlcSecondNodeId);
-      final hlc1 = Hlc(_dateTime, 17, _nodeId);
-      final hlc2 = Hlc(_dateTime, 17, node2);
 
       test('then they are not equal.', () {
         expect(hlc1, isNot(hlc2));
@@ -50,19 +46,16 @@ void main() {
         );
       });
 
-      test('then one is greater or less by uuid order.', () {
+      test('then the lower node uuid is the lesser Hlc.', () {
-        if (_nodeId.uuid.compareTo(node2.uuid) > 0) {
+        expect(hlc1 < hlc2, isTrue);
-          expect(hlc1 > hlc2, isTrue);
+        expect(hlc2 > hlc1, isTrue);
-        } else {
-          expect(hlc1 < hlc2, isTrue);
-        }
       });
     },
   );
 
   group('Given two Hlcs with same dateTime and different counter when comparing ', () {
-    final hlc1 = Hlc(_dateTime, 17, _nodeId);
+    final hlc1 = Hlc(hlcTime, 17, hlcNodeId);
-    final hlc2 = Hlc(_dateTime, 18, _nodeId);
+    final hlc2 = Hlc(hlcTime, 18, hlcNodeId);
 
     test('then they are not equal.', () {
       expect(hlc1, isNot(hlc2));
@@ -95,8 +88,8 @@ void main() {
   group(
     'Given two Hlcs with different dateTime when comparing ',
     () {
-      final hlc1 = Hlc(_dateTime, 0, _nodeId);
+      final hlc1 = Hlc(hlcTime, 0, hlcNodeId);
-      final hlc2 = Hlc(_dateTime.add(const Duration(milliseconds: 1)), 17, _nodeId);
+      final hlc2 = Hlc(hlcTime.advance(), 17, hlcNodeId);
 
       test('then they are not equal.', () {
         expect(hlc1, isNot(hlc2));

packages/serverpod_offline_sync/test/hlc/hlc_copy_with_test.dart

@@ -1,16 +1,14 @@
 import 'package:serverpod_offline_sync/serverpod_offline_sync.dart';
 import 'package:test/test.dart';
-import 'package:uuid/uuid.dart';
 
-final _dateTime = DateTime.parse('2001-09-09T01:46:40.000Z');
+import 'hlc_fixtures.dart';
-final _nodeId = const Uuid().v4obj();
 
 void main() {
   group('Given an HLC', () {
-    final hlc = Hlc(_dateTime, 17, _nodeId);
+    final hlc = Hlc(hlcTime, 17, hlcNodeId);
 
     test('when calling copyWith with new dateTime then result has new dateTime.', () {
-      final newDateTime = _dateTime.add(const Duration(minutes: 1));
+      final newDateTime = hlcTime.add(const Duration(minutes: 1));
       final newHlc = hlc.copyWith(datetime: newDateTime);
 
       expect(newHlc.datetime, newDateTime);
@@ -23,10 +21,9 @@ void main() {
     });
 
     test('when calling copyWith with new nodeId then result has new nodeId.', () {
-      final newNodeId = const Uuid().v4obj();
+      final newHlc = hlc.copyWith(nodeId: hlcSecondNodeId);
-      final newHlc = hlc.copyWith(nodeId: newNodeId);
 
-      expect(newHlc.nodeId, newNodeId);
+      expect(newHlc.nodeId, hlcSecondNodeId);
     });
   });
 }

packages/serverpod_offline_sync/test/hlc/hlc_fixtures.dart

@@ -0,0 +1,46 @@
+/// Constant fixtures for the `Hlc` tests.
+///
+/// Nothing here reads the wall clock or generates a random value, because both
+/// make an HLC test depend on when it runs.
+///
+/// `Hlc.increment` and `Hlc.merge` compare against `clock.now()` and reject a
+/// timestamp that drifts more than a minute from it, so a fixture built from
+/// `DateTime.now()` only passes while the run is close to the moment the
+/// fixture was created. On a runner that re-executes tests in the same process
+/// that stops being true, since a top-level `final` is initialized once per
+/// isolate and can outlive the values derived from it.
+///
+/// Node ids are fixed for the same reason: `Hlc.compareTo` breaks ties on the
+/// node uuid, so a random id makes the expected ordering vary between runs.
+library;
+
+import 'package:clock/clock.dart';
+import 'package:uuid/uuid.dart';
+
+/// The instant the HLC tests treat as the current wall time.
+final hlcTime = DateTime.parse('2001-09-09T01:46:40.000Z');
+
+/// A node id that sorts before [hlcSecondNodeId].
+final hlcNodeId = UuidValue.withValidation(
+  '11111111-1111-4111-8111-111111111111',
+);
+
+/// A node id that sorts after [hlcNodeId].
+final hlcSecondNodeId = UuidValue.withValidation(
+  '22222222-2222-4222-8222-222222222222',
+);
+
+/// Runs [body] with the wall clock pinned to [wallTime].
+T atWallTime<T>(DateTime wallTime, T Function() body) =>
+    withClock(Clock.fixed(wallTime), body);
+
+/// Offsets for building fixtures relative to [hlcTime].
+extension HlcFixtureTime on DateTime {
+  /// This instant one millisecond later.
+  DateTime advance([Duration duration = const Duration(milliseconds: 1)]) =>
+      add(duration);
+
+  /// This instant one millisecond earlier.
+  DateTime retreat([Duration duration = const Duration(milliseconds: 1)]) =>
+      subtract(duration);
+}

packages/serverpod_offline_sync/test/hlc/hlc_increment_test.dart

@@ -1,18 +1,17 @@
-import 'package:clock/clock.dart';
 import 'package:serverpod_offline_sync/serverpod_offline_sync.dart';
 import 'package:test/test.dart';
-import 'package:uuid/uuid.dart';
 
-final _dateTime = DateTime.parse('2001-09-09T01:46:40.000Z');
+import 'hlc_fixtures.dart';
-final _nodeId = const Uuid().v4obj();
 
+/// [Hlc.increment] derives its result from `clock.now()`, so every increment
+/// here runs with the wall clock pinned to an explicit instant.
 void main() {
   group('Given an HLC with lower canonical time than wall time', () {
-    final hlc = Hlc(_dateTime, 17, _nodeId);
+    final hlc = Hlc(hlcTime, 17, hlcNodeId);
-    final wallTime = _dateTime.advance();
+    final wallTime = hlcTime.advance();
 
     test('when incrementing then dateTime becomes wall time and counter resets.', () {
-      withClock(Clock.fixed(wallTime), () {
+      atWallTime(wallTime, () {
         final sendHlc = hlc.increment();
 
         expect(sendHlc, isNot(hlc));
@@ -24,11 +23,11 @@ void main() {
   });
 
   group('Given an HLC with equal canonical time and wall time', () {
-    final hlc = Hlc(_dateTime, 17, _nodeId);
+    final hlc = Hlc(hlcTime, 17, hlcNodeId);
-    final wallTime = _dateTime;
+    final wallTime = hlcTime;
 
     test('when incrementing then counter increments and dateTime is unchanged.', () {
-      withClock(Clock.fixed(wallTime), () {
+      atWallTime(wallTime, () {
         final sendHlc = hlc.increment();
 
         expect(sendHlc, isNot(hlc));
@@ -40,11 +39,11 @@ void main() {
   });
 
   group('Given an HLC with higher canonical time than wall time', () {
-    final hlc = Hlc(_dateTime, 17, _nodeId);
+    final hlc = Hlc(hlcTime, 17, hlcNodeId);
-    final wallTime = _dateTime.retreat();
+    final wallTime = hlcTime.retreat();
 
     test('when incrementing then counter increments and dateTime is unchanged.', () {
-      withClock(Clock.fixed(wallTime), () {
+      atWallTime(wallTime, () {
         final sendHlc = hlc.increment();
 
         expect(sendHlc, isNot(hlc));
@@ -56,29 +55,24 @@ void main() {
   });
 
   group('Given an HLC with canonical time more than one minute ahead of wall time', () {
-    final hlc = Hlc(_dateTime.add(const Duration(minutes: 1, seconds: 5)), 0, _nodeId);
+    final hlc = Hlc(hlcTime.add(const Duration(minutes: 1, seconds: 5)), 0, hlcNodeId);
-    final wallTime = _dateTime;
+    final wallTime = hlcTime;
 
     test('when incrementing then ClockDriftException is thrown.', () {
-      withClock(Clock.fixed(wallTime), () {
+      atWallTime(wallTime, () {
         expect(hlc.increment, throwsA(isA<ClockDriftException>()));
       });
     });
   });
 
   group('Given an HLC with counter at maximum and canonical time at wall time', () {
-    final hlc = Hlc(_dateTime, 0xFFFF, _nodeId);
+    final hlc = Hlc(hlcTime, 0xFFFF, hlcNodeId);
-    final wallTime = _dateTime;
+    final wallTime = hlcTime;
 
     test('when incrementing at same time then OverflowException is thrown.', () {
-      withClock(Clock.fixed(wallTime), () {
+      atWallTime(wallTime, () {
         expect(hlc.increment, throwsA(isA<OverflowException>()));
       });
     });
   });
 }
-
-extension on DateTime {
-  DateTime advance() => add(const Duration(milliseconds: 1));
-  DateTime retreat() => subtract(const Duration(milliseconds: 1));
-}

packages/serverpod_offline_sync/test/hlc/hlc_merge_test.dart

@@ -1,82 +1,85 @@
-import 'package:clock/clock.dart';
 import 'package:serverpod_offline_sync/serverpod_offline_sync.dart';
 import 'package:test/test.dart';
-import 'package:uuid/uuid.dart';
 
-final _dateTime = DateTime.now().toUtc();
+import 'hlc_fixtures.dart';
-final _nodeId = const Uuid().v4obj();
-final _nodeIdAfter = const Uuid().v4obj();
 
+/// [Hlc.merge] compares the remote timestamp against `clock.now()`, so every
+/// merge here runs with the wall clock pinned to [hlcTime]. Without that the
+/// outcome depends on how far real time has drifted from the fixtures.
 void main() {
   group('Given local canonical Hlc and remote with higher time', () {
-    final canonical = Hlc(_dateTime, 17, _nodeId);
+    final canonical = Hlc(hlcTime, 17, hlcNodeId);
-    final remote = Hlc(_dateTime.advance(), 2, _nodeIdAfter);
+    final remote = Hlc(hlcTime.advance(), 2, hlcSecondNodeId);
 
     test('when merging then result has remote time and counter with local nodeId.', () {
-      final hlc = canonical.merge(remote);
+      final hlc = atWallTime(hlcTime, () => canonical.merge(remote));
 
       expect(hlc, Hlc(remote.datetime, remote.counter, canonical.nodeId));
     });
   });
 
   group('Given local canonical Hlc and remote with same time but higher counter', () {
-    final canonical = Hlc(_dateTime, 17, _nodeId);
+    final canonical = Hlc(hlcTime, 17, hlcNodeId);
-    final remote = Hlc(_dateTime, 18, _nodeIdAfter);
+    final remote = Hlc(hlcTime, 18, hlcSecondNodeId);
 
     test('when merging then result has remote counter with local nodeId.', () {
-      final hlc = canonical.merge(remote);
+      final hlc = atWallTime(hlcTime, () => canonical.merge(remote));
 
       expect(hlc, Hlc(canonical.datetime, remote.counter, canonical.nodeId));
     });
   });
 
   group('Given local canonical Hlc and remote with same time and counter', () {
-    final canonical = Hlc(_dateTime, 17, _nodeId);
+    final canonical = Hlc(hlcTime, 17, hlcNodeId);
-    final remote = Hlc(_dateTime, 17, _nodeIdAfter);
+    final remote = Hlc(hlcTime, 17, hlcSecondNodeId);
 
     test('when merging then local is returned.', () {
-      final hlc = canonical.merge(remote);
+      final hlc = atWallTime(hlcTime, () => canonical.merge(remote));
 
       expect(hlc, equals(canonical));
     });
   });
 
   group('Given local canonical Hlc and remote with lower time', () {
-    final canonical = Hlc(_dateTime, 17, _nodeId);
+    final canonical = Hlc(hlcTime, 17, hlcNodeId);
-    final remote = Hlc(_dateTime.retreat(), 17, _nodeIdAfter);
+    final remote = Hlc(hlcTime.retreat(), 17, hlcSecondNodeId);
 
     test('when merging then local is returned.', () {
-      final hlc = canonical.merge(remote);
+      final hlc = atWallTime(hlcTime, () => canonical.merge(remote));
 
       expect(hlc, equals(canonical));
     });
   });
 
   group('Given local canonical Hlc and remote with same nodeId and same time', () {
-    final canonical = Hlc(_dateTime, 17, _nodeId);
+    final canonical = Hlc(hlcTime, 17, hlcNodeId);
-    final remote = Hlc(_dateTime, 17, _nodeId);
+    final remote = Hlc(hlcTime, 17, hlcNodeId);
 
     test('when merging then node id is not checked and local is returned.', () {
-      expect(canonical.merge(remote), canonical);
+      final hlc = atWallTime(hlcTime, () => canonical.merge(remote));
+
+      expect(hlc, canonical);
     });
   });
 
   group('Given local canonical Hlc and remote with same nodeId and lower time', () {
-    final canonical = Hlc(_dateTime, 17, _nodeId);
+    final canonical = Hlc(hlcTime, 17, hlcNodeId);
-    final remote = Hlc(_dateTime.retreat(), 17, _nodeId);
+    final remote = Hlc(hlcTime.retreat(), 17, hlcNodeId);
 
     test('when merging then node id is not checked and local is returned.', () {
-      expect(canonical.merge(remote), canonical);
+      final hlc = atWallTime(hlcTime, () => canonical.merge(remote));
+
+      expect(hlc, canonical);
     });
   });
 
   group('Given local canonical Hlc and remote with same nodeId and higher time', () {
-    final canonical = Hlc(_dateTime, 17, _nodeId);
+    final canonical = Hlc(hlcTime, 17, hlcNodeId);
-    final remote = Hlc(_dateTime.advance(), 0, _nodeId);
+    final remote = Hlc(hlcTime.advance(), 0, hlcNodeId);
 
     test('when merging then DuplicateNodeException is thrown.', () {
       expect(
-        () => canonical.merge(remote),
+        () => atWallTime(hlcTime, () => canonical.merge(remote)),
         throwsA(isA<DuplicateNodeException>()),
       );
     });
@@ -85,29 +88,19 @@ void main() {
   group(
     'Given local canonical Hlc and remote with time more than one minute ahead of wall',
     () {
-      final canonical = Hlc(_dateTime, 17, _nodeId);
+      final canonical = Hlc(hlcTime, 17, hlcNodeId);
       final remote = Hlc(
-        _dateTime.advance(const Duration(minutes: 2)),
+        hlcTime.advance(const Duration(minutes: 2)),
         17,
-        _nodeIdAfter,
+        hlcSecondNodeId,
       );
 
       test('when merging then ClockDriftException is thrown.', () {
-        withClock(Clock.fixed(_dateTime.advance()), () {
+        expect(
-          expect(
+          () => atWallTime(hlcTime, () => canonical.merge(remote)),
-            () => canonical.merge(remote),
+          throwsA(isA<ClockDriftException>()),
-            throwsA(isA<ClockDriftException>()),
+        );
-          );
-        });
       });
     },
   );
 }
-
-extension on DateTime {
-  DateTime advance([Duration duration = const Duration(milliseconds: 1)]) =>
-      add(duration);
-
-  DateTime retreat([Duration duration = const Duration(milliseconds: 1)]) =>
-      subtract(duration);
-}

packages/serverpod_offline_sync/test/hlc/hlc_serialization_test.dart

@@ -1,74 +1,73 @@
 import 'package:serverpod_offline_sync/serverpod_offline_sync.dart';
 import 'package:test/test.dart';
-import 'package:uuid/uuid.dart';
 
-const _isoTime = '2001-09-09T01:46:40.000Z';
+import 'hlc_fixtures.dart';
-final _dateTime = DateTime.parse(_isoTime);
+
-final _nodeId = const Uuid().v4obj();
+final _isoTime = hlcTime.toIso8601String();
 
 void main() {
   test('Given an HLC built with dateTime, counter and nodeId '
       'when reading properties '
       'then dateTime, counter and nodeId match.', () {
-    final hlc = Hlc(_dateTime, 0x17, _nodeId);
+    final hlc = Hlc(hlcTime, 0x17, hlcNodeId);
 
-    expect(hlc.datetime, _dateTime);
+    expect(hlc.datetime, hlcTime);
     expect(hlc.counter, 0x17);
-    expect(hlc.nodeId, _nodeId);
+    expect(hlc.nodeId, hlcNodeId);
   });
 
   test('Given Hlc.zero '
       'when building for a nodeId '
       'then it equals epoch with zero counter.', () {
-    final zero = Hlc.zero(_nodeId);
+    final zero = Hlc.zero(hlcNodeId);
 
     expect(zero.unixTimestamp, 0);
     expect(zero.counter, 0);
-    expect(zero.nodeId, _nodeId);
+    expect(zero.nodeId, hlcNodeId);
   });
 
   test('Given an ISO HLC string '
       'when parsing '
       'then it returns the equivalent Hlc.', () {
-    final hlc = Hlc(_dateTime, 0x17, _nodeId);
+    final hlc = Hlc(hlcTime, 0x17, hlcNodeId);
 
-    expect(Hlc.parse('${_dateTime.toIso8601String()}-0017-$_nodeId'), hlc);
+    expect(Hlc.parse('${hlcTime.toIso8601String()}-0017-$hlcNodeId'), hlc);
   });
 
   test('Given a Unix timestamp HLC string '
       'when parsing '
       'then it returns the equivalent Hlc.', () {
-    final hlc = Hlc(_dateTime, 0x17, _nodeId);
+    final hlc = Hlc(hlcTime, 0x17, hlcNodeId);
 
-    expect(Hlc.parse('${_dateTime.millisecondsSinceEpoch}-0017-$_nodeId'), hlc);
+    expect(Hlc.parse('${hlcTime.millisecondsSinceEpoch}-0017-$hlcNodeId'), hlc);
   });
 
   test('Given legacy string with :worker suffix '
       'when parsing '
       'then node id is the uuid only.', () {
-    final hlc = Hlc.parse('$_isoTime-0017-$_nodeId:7');
+    final hlc = Hlc.parse('$_isoTime-0017-$hlcNodeId:7');
-    expect(hlc.nodeId, _nodeId);
+    expect(hlc.nodeId, hlcNodeId);
     expect(hlc.counter, 0x17);
-    expect(hlc.datetime, _dateTime);
+    expect(hlc.datetime, hlcTime);
   });
 
   test('Given a parsed Hlc '
       'when calling toString '
       'then it returns the ISO HLC string.', () {
-    final hlc = Hlc.parse('$_isoTime-0017-$_nodeId');
+    final hlc = Hlc.parse('$_isoTime-0017-$hlcNodeId');
 
-    expect(hlc.toString(), '$_isoTime-0017-$_nodeId');
+    expect(hlc.toString(), '$_isoTime-0017-$hlcNodeId');
   });
 
   test('Given an HLC '
       'when converting to JSON '
       'then it returns the equivalent string.', () {
-    final hlc = Hlc(_dateTime, 0x17, _nodeId);
+    final hlc = Hlc(hlcTime, 0x17, hlcNodeId);
 
     expect(hlc.toJson(), {
-      'datetime': _dateTime.toIso8601String(),
+      'datetime': hlcTime.toIso8601String(),
       'counter': '0017',
-      'node': _nodeId.toString(),
+      'node': hlcNodeId.toString(),
     });
   });
 }

pubspec.yaml

@@ -115,6 +115,17 @@ melos:
         melos exec --scope="serverpod_offline_sync" -- dart test test/ --reporter=failures-only &&
         melos exec --scope="serverpod_offline_sync_server" -- dart test test/ --reporter=failures-only --concurrency=1 &&
         melos exec --scope="serverpod_offline_sync_test_server" -- dart test test/ --reporter=failures-only --concurrency=1
+    test-dst:
+      # The deterministic simulation suite, which `test` skips: it is slow and
+      # seeds from the clock, so it searches fresh schedules on every run and
+      # can fail for a defect unrelated to the change that triggered it.
+      #
+      # Widen the search with DST_SEEDS and DST_ROUNDS; a failure prints the
+      # command that replays it. Concurrency is pinned to one so the reported
+      # failures cannot interleave, which matters when reading a wide sweep.
+      run: >-
+        melos exec --scope="serverpod_offline_sync_test_server" --
+        dart test -P dst --reporter=failures-only --concurrency=1
     reset:
       exec: rm -f test.db*
     publish-check:

test/serverpod_offline_sync_test_client/lib/migrations/20260901174859443/migration.json

@@ -0,0 +1,106 @@
+{
+  "__className__": "serverpod.DatabaseMigration",
+  "actions": [
+    {
+      "__className__": "serverpod.DatabaseMigrationAction",
+      "type": "createTable",
+      "createTable": {
+        "__className__": "serverpod.TableDefinition",
+        "name": "unique_cascade_child",
+        "dartName": "UniqueCascadeChild",
+        "module": "serverpod_offline_sync_test",
+        "schema": "public",
+        "columns": [
+          {
+            "__className__": "serverpod.ColumnDefinition",
+            "name": "id",
+            "fieldName": "id",
+            "columnType": 7,
+            "isNullable": false,
+            "columnDefault": "random_v7",
+            "dartType": "UuidValue?"
+          },
+          {
+            "__className__": "serverpod.ColumnDefinition",
+            "name": "scopeId",
+            "fieldName": "scopeId",
+            "columnType": 6,
+            "isNullable": true,
+            "dartType": "int?"
+          },
+          {
+            "__className__": "serverpod.ColumnDefinition",
+            "name": "name",
+            "fieldName": "name",
+            "columnType": 0,
+            "isNullable": false,
+            "dartType": "String"
+          },
+          {
+            "__className__": "serverpod.ColumnDefinition",
+            "name": "parentId",
+            "fieldName": "parentId",
+            "columnType": 7,
+            "isNullable": true,
+            "dartType": "UuidValue?"
+          }
+        ],
+        "foreignKeys": [
+          {
+            "__className__": "serverpod.ForeignKeyDefinition",
+            "constraintName": "unique_cascade_child_fk_0",
+            "columns": [
+              "scopeId"
+            ],
+            "referenceTable": "crdt_scopes",
+            "referenceTableSchema": "public",
+            "referenceColumns": [
+              "id"
+            ],
+            "onUpdate": 3,
+            "onDelete": 4
+          },
+          {
+            "__className__": "serverpod.ForeignKeyDefinition",
+            "constraintName": "unique_cascade_child_fk_1",
+            "columns": [
+              "parentId"
+            ],
+            "referenceTable": "person",
+            "referenceTableSchema": "public",
+            "referenceColumns": [
+              "id"
+            ],
+            "onUpdate": 3,
+            "onDelete": 4,
+            "deferrable": "initiallyDeferred"
+          }
+        ],
+        "indexes": [
+          {
+            "__className__": "serverpod.IndexDefinition",
+            "indexName": "unique_cascade_child__scopeId__name__unique_idx",
+            "elements": [
+              {
+                "__className__": "serverpod.IndexElementDefinition",
+                "type": 0,
+                "definition": "scopeId"
+              },
+              {
+                "__className__": "serverpod.IndexElementDefinition",
+                "type": 0,
+                "definition": "name"
+              }
+            ],
+            "type": "btree",
+            "isUnique": true,
+            "isPrimary": false
+          }
+        ],
+        "managed": true
+      }
+    }
+  ],
+  "warnings": [],
+  "migrationApiVersion": 1
+}