test: Harden DST authored-state checks and workloads (#123)
This commit is contained in:
parent
16d15eebe5
commit
ad3e1a49e4
61 changed files with 10312 additions and 458 deletions
31
.github/workflows/ci.yaml
vendored
31
.github/workflows/ci.yaml
vendored
|
|
@ -99,9 +99,9 @@ jobs:
|
||||||
# comment-tag: coverage-results
|
# comment-tag: coverage-results
|
||||||
|
|
||||||
simulation:
|
simulation:
|
||||||
name: deterministic simulation
|
name: deterministic simulation (${{ matrix.topology }})
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
timeout-minutes: 30
|
timeout-minutes: 60
|
||||||
# Deliberately not on pull requests. The suite seeds from the clock, so it
|
# Deliberately not on pull requests. The suite seeds from the clock, so it
|
||||||
# searches fresh schedules on every run and can fail for a defect unrelated
|
# searches fresh schedules on every run and can fail for a defect unrelated
|
||||||
# to the change under review - which is what makes it worth running, and
|
# to the change under review - which is what makes it worth running, and
|
||||||
|
|
@ -109,17 +109,32 @@ jobs:
|
||||||
# branches and on demand, where a failure is a finding rather than a
|
# branches and on demand, where a failure is a finding rather than a
|
||||||
# blocked review.
|
# blocked review.
|
||||||
if: github.event_name != 'pull_request'
|
if: github.event_name != 'pull_request'
|
||||||
|
strategy:
|
||||||
|
fail-fast: false
|
||||||
|
matrix:
|
||||||
|
# Keep the topology files below and shared ownership controls exhaustive
|
||||||
|
# when adding a new test file tagged dst.
|
||||||
|
topology: [convergence, cross_space]
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@v4
|
||||||
- uses: ./.github/actions/setup
|
- uses: ./.github/actions/setup
|
||||||
|
|
||||||
- name: Run the simulation suite
|
- name: Run the simulation topology and ownership controls
|
||||||
run: melos test-dst
|
# Four ten-minute simulation ceilings fit within each 60-minute job.
|
||||||
|
working-directory: test/serverpod_offline_sync_test_server
|
||||||
|
run: >-
|
||||||
|
dart test -P dst
|
||||||
|
test/dst/dst_${{ matrix.topology }}_test.dart
|
||||||
|
test/dst/dst_ownership_collision_test.dart
|
||||||
|
--concurrency=1
|
||||||
env:
|
env:
|
||||||
# Wider than a local run, and still inside the job timeout. A failure
|
# Depth matters: 200 rounds reproduces deferred engine failures that
|
||||||
# prints DST_SEED_BASE, which replays that simulation exactly.
|
# 20-round empty worlds missed. Alternate populated conflict graphs
|
||||||
DST_SEEDS: 50
|
# and sparse exploration; publish DST_METRICS even for failing seeds.
|
||||||
DST_ROUNDS: 20
|
DST_SEEDS: 4
|
||||||
|
DST_ROUNDS: 200
|
||||||
|
DST_PROFILE: mixed
|
||||||
|
DST_GRAPH_WIDTH: 2
|
||||||
|
|
||||||
benchmarks:
|
benchmarks:
|
||||||
name: run benchmarks
|
name: run benchmarks
|
||||||
|
|
|
||||||
16
README.md
16
README.md
|
|
@ -22,6 +22,7 @@ out-of-the-box with Serverpod's existing APIs.
|
||||||
- [Usage](#usage)
|
- [Usage](#usage)
|
||||||
- [Spaces and sharing](#spaces-and-sharing)
|
- [Spaces and sharing](#spaces-and-sharing)
|
||||||
- [Data modeling limitations](#data-modeling-limitations)
|
- [Data modeling limitations](#data-modeling-limitations)
|
||||||
|
- [Reserved values](#reserved-values)
|
||||||
- [How it works](#how-it-works)
|
- [How it works](#how-it-works)
|
||||||
- [Performance](#performance)
|
- [Performance](#performance)
|
||||||
- [Example](#example)
|
- [Example](#example)
|
||||||
|
|
@ -264,6 +265,21 @@ Respecting these limitations, all other database invariants — including
|
||||||
foreign-key actions — are preserved, with the exception of check constraints,
|
foreign-key actions — are preserved, with the exception of check constraints,
|
||||||
which Serverpod does not support either.
|
which Serverpod does not support either.
|
||||||
|
|
||||||
|
#### Reserved values
|
||||||
|
|
||||||
|
Besides the schema restrictions, some values are reserved by the sync layer and
|
||||||
|
cannot be authored by the user. Such values are used by the engine to resolve
|
||||||
|
unique conflicts and are not expected to represent a restriction to real
|
||||||
|
production usage. Trying to author a reserved value will immediately throw an
|
||||||
|
`OfflineSyncReservedValueException` exception to prevent later conflicts.
|
||||||
|
|
||||||
|
- Unique text values cannot be authored with a suffix matching
|
||||||
|
`__conflict__<UUID>`, `__hidden__<UUID>`, or `__park__<UUID>`. Non-unique
|
||||||
|
text fields are unrestricted.
|
||||||
|
- Non-nullable unique UUID columns that are not foreign keys reserve version 8
|
||||||
|
for generated alternatives. Primary keys, foreign keys, nullable UUID columns
|
||||||
|
and non-unique UUID columns are unrestricted.
|
||||||
|
|
||||||
## How it works
|
## How it works
|
||||||
|
|
||||||
- **Architecture.** The engine is a transparent layer between the application
|
- **Architecture.** The engine is a transparent layer between the application
|
||||||
|
|
|
||||||
|
|
@ -68,8 +68,28 @@ independently of the database dialect.
|
||||||
Create or update the row whenever the final domain value differs from the
|
Create or update the row whenever the final domain value differs from the
|
||||||
authored value. Remove it only when the domain value again equals the authored
|
authored value. Remove it only when the domain value again equals the authored
|
||||||
value and no projector still needs an override. At that point the domain column
|
value and no projector still needs an override. At that point the domain column
|
||||||
itself safely carries the authored value. No changes to the `CrdtDataField`
|
itself safely carries the authored value. A `CrdtDataField` row is needed when
|
||||||
existence policy, which is whenever its HLC is newer than the row HLC.
|
its HLC is newer than the row HLC or it holds a preserved attempted value.
|
||||||
|
Otherwise the field inherits the row's insertion HLC.
|
||||||
|
|
||||||
|
### Reinserting a deleted row
|
||||||
|
|
||||||
|
Inserting or upserting a deleted row keeps its identity and ownership and
|
||||||
|
advances its delete/restore generation. The complete row is authored at a new
|
||||||
|
insertion HLC, including preserved conflict claims, so unique claims receive
|
||||||
|
a new age. This is a full-row write even when the upsert supplies an update
|
||||||
|
column filter or predicate: those restrictions apply to visible existing rows.
|
||||||
|
|
||||||
|
Old field timestamps are removed. Only metadata needed to preserve a projected
|
||||||
|
value remains, at that same insertion HLC; deleting that metadata would cascade
|
||||||
|
to the only stored copy of its authored value. Projection uses the renewed
|
||||||
|
claim ages before redundant field records are removed.
|
||||||
|
|
||||||
|
Outbound sync sends the full authored row without redundant field updates.
|
||||||
|
Existing receivers still compare each field's HLC, so newer concurrent edits
|
||||||
|
survive. The delete/restore generation is never reset, preventing an older
|
||||||
|
deletion from hiding the restored row. These changes are atomic with the domain
|
||||||
|
write and roll back together on failure.
|
||||||
|
|
||||||
### Diagnostic projection reason
|
### Diagnostic projection reason
|
||||||
|
|
||||||
|
|
@ -151,8 +171,9 @@ must be FK-safe; under the current policy the FK is nullable and releases to
|
||||||
|
|
||||||
Local insert, update, upsert, delete, and tombstone restoration need this plan
|
Local insert, update, upsert, delete, and tombstone restoration need this plan
|
||||||
before the underlying write can violate an immediate unique constraint. A
|
before the underlying write can violate an immediate unique constraint. A
|
||||||
full-row passthrough of an existing projected value remains non-authored;
|
full-row passthrough translates an unchanged projected value back to its
|
||||||
explicit/narrowed field writes author a new attempted value and field HLC.
|
preserved authored value; explicit/narrowed field writes author the supplied
|
||||||
|
value. Reinsertion follows the full-row reauthoring rule above.
|
||||||
|
|
||||||
## Outbound and rebuild behavior
|
## Outbound and rebuild behavior
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -20,6 +20,20 @@ This is the `flag` policy. It is intentionally simple:
|
||||||
This policy is the only unique-conflict policy in space for the current
|
This policy is the only unique-conflict policy in space for the current
|
||||||
implementation.
|
implementation.
|
||||||
|
|
||||||
|
## Local writes
|
||||||
|
|
||||||
|
Local inserts, updates, upserts, and restores obey the database's unique
|
||||||
|
constraints. Two records cannot claim the same occupied value or non-null
|
||||||
|
composite tuple in one local write. The database enforces this against stored
|
||||||
|
rows and other rows in the batch; no additional domain reads are needed to
|
||||||
|
pre-check uniqueness. A rejected write rolls back its domain changes and sync
|
||||||
|
metadata together. Ordinary `ignoreConflicts` and upsert predicates still apply.
|
||||||
|
|
||||||
|
Claims accepted independently on disconnected replicas can still conflict when
|
||||||
|
merged. The policy below preserves those accepted claims and chooses their
|
||||||
|
visible values. Passing an unchanged displayed alternative through a full-row
|
||||||
|
save preserves the underlying claim and follows the normal field-touch rules.
|
||||||
|
|
||||||
## Current Policy: Flag
|
## Current Policy: Flag
|
||||||
|
|
||||||
Candidate construction, hidden-row release, canonical attempted-value lookup,
|
Candidate construction, hidden-row release, canonical attempted-value lookup,
|
||||||
|
|
@ -64,6 +78,34 @@ non-null constraint, the FK constraint, or the unique constraint itself.
|
||||||
The rewritten value is a materialized data value, not a synced conflict class.
|
The rewritten value is a materialized data value, not a synced conflict class.
|
||||||
The core CRDT protocol does not need to expose unique conflict objects to users.
|
The core CRDT protocol does not need to expose unique conflict objects to users.
|
||||||
|
|
||||||
|
### Reserved generated values
|
||||||
|
|
||||||
|
Authored values in unique text columns must not end in
|
||||||
|
`__conflict__<UUID>`, `__hidden__<UUID>`, or `__park__<UUID>`, where the UUID uses
|
||||||
|
the hexadecimal `8-4-4-4-12` shape. The marker is lowercase; hexadecimal letters
|
||||||
|
can use either case. These names belong exclusively to projection, including
|
||||||
|
hidden-row release and temporary writes. Non-unique text is unrestricted.
|
||||||
|
|
||||||
|
Non-nullable unique UUID columns that are not foreign keys reserve UUID version
|
||||||
|
8 for generated alternatives (`xxxxxxxx-xxxx-8xxx-xxxx-xxxxxxxxxxxx`). Conflict,
|
||||||
|
hidden-row, and temporary parking values use the same deterministic hash inputs
|
||||||
|
as before, with the version marker set to 8. Authored version-8 values in these
|
||||||
|
columns are rejected; ordinary UUID versions, including 4, 5, and 7, remain
|
||||||
|
supported. Primary keys, foreign keys, nullable UUID columns, and non-unique UUID
|
||||||
|
columns do not use synthetic UUID release and are not subject to this rule.
|
||||||
|
|
||||||
|
Local writes and incoming sync facts enforce this rule with
|
||||||
|
`OfflineSyncReservedValueException`. Rejected transactions do not advance
|
||||||
|
authored data or sync progress. A full-row update/upsert can echo its own
|
||||||
|
unchanged materialized alternative: it keeps the original authored claim and
|
||||||
|
uses the normal field-touch semantics. Explicitly authoring an alternative on
|
||||||
|
another record is rejected, including exchanges of displayed alternatives.
|
||||||
|
|
||||||
|
This keeps the existing deterministic text suffix construction. The resolver does
|
||||||
|
not search currently occupied values for a free alternative. The original claim
|
||||||
|
is what gets exported, so ordinary conflict projection and bootstrap remain
|
||||||
|
valid under the input rule.
|
||||||
|
|
||||||
## Why Flag Is The Current Default
|
## Why Flag Is The Current Default
|
||||||
|
|
||||||
The `flag` policy avoids the hard second-order effects caused by hiding or
|
The `flag` policy avoids the hard second-order effects caused by hiding or
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,7 @@
|
||||||
export 'src/crdt/extensions.dart';
|
export 'src/crdt/extensions.dart';
|
||||||
export 'src/crdt/merge.dart';
|
export 'src/crdt/merge.dart';
|
||||||
export 'src/database/database.dart';
|
export 'src/database/database.dart';
|
||||||
|
export 'src/database/exceptions.dart';
|
||||||
export 'src/database/recorder.dart';
|
export 'src/database/recorder.dart';
|
||||||
export 'src/database/schema.dart';
|
export 'src/database/schema.dart';
|
||||||
export 'src/database/session.dart'
|
export 'src/database/session.dart'
|
||||||
|
|
|
||||||
|
|
@ -432,7 +432,7 @@ class OfflineSyncDatabase implements Database {
|
||||||
(tx) async {
|
(tx) async {
|
||||||
final prepared = _prepareRowsForInsert(rows, tx);
|
final prepared = _prepareRowsForInsert(rows, tx);
|
||||||
final values = _recorder.withForeignKeyInsertDefaults(prepared.rows);
|
final values = _recorder.withForeignKeyInsertDefaults(prepared.rows);
|
||||||
final projectionUnchanged = await _recorder.prepareLocalUpsert(
|
final projection = await _recorder.prepareLocalUpsert(
|
||||||
values,
|
values,
|
||||||
conflictColumns,
|
conflictColumns,
|
||||||
updateColumns,
|
updateColumns,
|
||||||
|
|
@ -475,7 +475,8 @@ class OfflineSyncDatabase implements Database {
|
||||||
updatedRows,
|
updatedRows,
|
||||||
updateColumns,
|
updateColumns,
|
||||||
tx,
|
tx,
|
||||||
projectionUnchanged: projectionUnchanged,
|
projectionUnchanged: projection.projectionUnchanged,
|
||||||
|
domainBeforeUpsert: projection.domain,
|
||||||
);
|
);
|
||||||
if (noReturn) return <T>[];
|
if (noReturn) return <T>[];
|
||||||
_stripStampedRows(result, prepared);
|
_stripStampedRows(result, prepared);
|
||||||
|
|
@ -519,11 +520,24 @@ class OfflineSyncDatabase implements Database {
|
||||||
if (deletedRowIds.contains(row.id)) row,
|
if (deletedRowIds.contains(row.id)) row,
|
||||||
];
|
];
|
||||||
if (rowsToReinsert.isEmpty) return [];
|
if (rowsToReinsert.isEmpty) return [];
|
||||||
|
// Hidden targets were filtered out of the delegate's upsert results, so
|
||||||
|
// its duplicate-target check cannot see these restorations. Check the
|
||||||
|
// submitted identities before turning them into ordinary updates.
|
||||||
|
if (rowsToReinsert.map((row) => row.id).toSet().length != rowsToReinsert.length) {
|
||||||
|
throw DatabaseQueryException(
|
||||||
|
'ON CONFLICT DO UPDATE command cannot affect row a second time',
|
||||||
|
code: switch (dialect) {
|
||||||
|
DatabaseDialect.postgres => PgErrorCode.cardinalityViolation,
|
||||||
|
DatabaseDialect.sqlite => SqliteErrorCode.integrityConstraintViolation,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
final plannedReinserts = await _recorder.planLocalUpdates(
|
final plannedReinserts = await _recorder.planLocalUpdates(
|
||||||
rowsToReinsert,
|
rowsToReinsert,
|
||||||
null,
|
null,
|
||||||
transaction,
|
transaction,
|
||||||
|
restoring: true,
|
||||||
);
|
);
|
||||||
final reinsertedRows = await _delegate.update<T>(
|
final reinsertedRows = await _delegate.update<T>(
|
||||||
plannedReinserts.rows,
|
plannedReinserts.rows,
|
||||||
|
|
@ -789,6 +803,7 @@ class OfflineSyncDatabase implements Database {
|
||||||
);
|
);
|
||||||
|
|
||||||
final columns = columnValues.map((e) => e.column).toList();
|
final columns = columnValues.map((e) => e.column).toList();
|
||||||
|
_recorder.validateAuthoredRows(result, columns);
|
||||||
await _recorder.afterUpdate(result, columns, tx);
|
await _recorder.afterUpdate(result, columns, tx);
|
||||||
if (noReturn) return <T>[];
|
if (noReturn) return <T>[];
|
||||||
result.forEach(_stripSpaceId);
|
result.forEach(_stripSpaceId);
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,22 @@
|
||||||
|
/// A caller tried to author a unique value reserved for sync projection.
|
||||||
|
class OfflineSyncReservedValueException implements Exception {
|
||||||
|
/// Creates an exception identifying the rejected field and value.
|
||||||
|
const OfflineSyncReservedValueException({
|
||||||
|
required this.tableName,
|
||||||
|
required this.columnName,
|
||||||
|
required this.value,
|
||||||
|
});
|
||||||
|
|
||||||
|
/// The synchronized table containing the unique column.
|
||||||
|
final String tableName;
|
||||||
|
|
||||||
|
/// The unique column being authored.
|
||||||
|
final String columnName;
|
||||||
|
|
||||||
|
/// The rejected text or UUID value, rendered as a string.
|
||||||
|
final String value;
|
||||||
|
|
||||||
|
@override
|
||||||
|
String toString() =>
|
||||||
|
'Reserved generated unique value for $tableName.$columnName: $value';
|
||||||
|
}
|
||||||
|
|
@ -87,6 +87,29 @@ extension CrdtMergeRecorderExtension on CrdtMutationRecorder {
|
||||||
if (mergeSet.isEmpty) return;
|
if (mergeSet.isEmpty) return;
|
||||||
|
|
||||||
final operations = mergeSet.causallyOrderedChanges;
|
final operations = mergeSet.causallyOrderedChanges;
|
||||||
|
// Validate remote facts before touching rows or progress, including stale
|
||||||
|
// and repeated deliveries. Generated alternatives are never authored facts.
|
||||||
|
for (final operation in operations) {
|
||||||
|
if (!_context.isCrdtTrackedTableName(operation.tableName) ||
|
||||||
|
!_uniqueResolver.hasReservedUniqueColumns(operation.tableName)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
switch (operation) {
|
||||||
|
case CrdtMergeInsert():
|
||||||
|
for (final MapEntry(key: column, value: value)
|
||||||
|
in operation.databaseColumns.entries) {
|
||||||
|
_uniqueResolver.validateAuthoredValue(operation.tableName, column, value);
|
||||||
|
}
|
||||||
|
case CrdtMergeUpdate():
|
||||||
|
_uniqueResolver.validateAuthoredValue(
|
||||||
|
operation.tableName,
|
||||||
|
operation.columnName,
|
||||||
|
operation.value,
|
||||||
|
);
|
||||||
|
case CrdtMergeDelete():
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
// Nothing in this batch touches a foreign key or a unique index, so no
|
// Nothing in this batch touches a foreign key or a unique index, so no
|
||||||
// projection decision can change: skip both passes instead of loading
|
// projection decision can change: skip both passes instead of loading
|
||||||
// state for tables that cannot produce a candidate or a conflict.
|
// state for tables that cannot produce a candidate or a conflict.
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,3 @@
|
||||||
import 'dart:typed_data';
|
|
||||||
|
|
||||||
import 'package:meta/meta.dart';
|
import 'package:meta/meta.dart';
|
||||||
import 'package:serverpod_database/serverpod_database.dart';
|
import 'package:serverpod_database/serverpod_database.dart';
|
||||||
import 'package:serverpod_serialization/serverpod_serialization.dart';
|
import 'package:serverpod_serialization/serverpod_serialization.dart';
|
||||||
|
|
@ -62,41 +60,14 @@ int compareMergeRowKeys(MergeRowKey left, MergeRowKey right) {
|
||||||
return left.$2.uuid.compareTo(right.$2.uuid);
|
return left.$2.uuid.compareTo(right.$2.uuid);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Canonicalizes a domain/attempted value so UUID blobs round-trip as
|
/// Decodes UUID columns using their schema, preserving text and binary values
|
||||||
/// [UuidValue] rather than as SQLite byte lists.
|
/// even when their contents could also represent a UUID.
|
||||||
///
|
|
||||||
/// Only values already shaped like a UUID are converted. Anything else is
|
|
||||||
/// returned unchanged, so an ordinary string or number never pays for a thrown
|
|
||||||
/// [FormatException] on the projection and outbound paths.
|
|
||||||
@internal
|
@internal
|
||||||
Object? canonicalDomainValue(Object? value) {
|
Object? canonicalDomainValue(Object? value, ColumnDefinition? column) {
|
||||||
if (value is UuidValue) return value;
|
if (column?.columnType == ColumnType.uuid) return value.toUuidValue();
|
||||||
if (value is Uint8List) {
|
|
||||||
return value.length == 16 ? UuidValue.fromByteList(value) : value;
|
|
||||||
}
|
|
||||||
if (value is String) {
|
|
||||||
return _looksLikeUuid(value) ? UuidValue.withValidation(value) : value;
|
|
||||||
}
|
|
||||||
return value;
|
return value;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Whether [value] has the 8-4-4-4-12 shape of a canonical UUID string.
|
|
||||||
bool _looksLikeUuid(String value) {
|
|
||||||
if (value.length != 36) return false;
|
|
||||||
for (var index = 0; index < 36; index++) {
|
|
||||||
final code = value.codeUnitAt(index);
|
|
||||||
if (index == 8 || index == 13 || index == 18 || index == 23) {
|
|
||||||
if (code != 0x2d) return false;
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
final isDigit = code >= 0x30 && code <= 0x39;
|
|
||||||
final isLower = code >= 0x61 && code <= 0x66;
|
|
||||||
final isUpper = code >= 0x41 && code <= 0x46;
|
|
||||||
if (!isDigit && !isLower && !isUpper) return false;
|
|
||||||
}
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Extensions on [UuidValue?] used for UUID comparison.
|
/// Extensions on [UuidValue?] used for UUID comparison.
|
||||||
@internal
|
@internal
|
||||||
extension UuidValueComparisonExtension on UuidValue? {
|
extension UuidValueComparisonExtension on UuidValue? {
|
||||||
|
|
|
||||||
|
|
@ -306,7 +306,14 @@ class CrdtForeignKeyProjector {
|
||||||
if (projectionValuesEqual(attempt.value, values[columnName])) continue;
|
if (projectionValuesEqual(attempt.value, values[columnName])) continue;
|
||||||
|
|
||||||
projectionWrites.add(
|
projectionWrites.add(
|
||||||
(fieldId: fieldId, value: attempt.value, reason: attempt.reason),
|
(
|
||||||
|
fieldId: fieldId,
|
||||||
|
value: canonicalDomainValue(
|
||||||
|
attempt.value,
|
||||||
|
_context.columnsByTableAndName[tableName]?[columnName],
|
||||||
|
),
|
||||||
|
reason: attempt.reason,
|
||||||
|
),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -606,36 +613,6 @@ class CrdtForeignKeyProjector {
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Fields that currently carry a sparse attempted-value override.
|
|
||||||
///
|
|
||||||
/// Reinsert must not bump these field HLCs: the authored fact is unchanged
|
|
||||||
/// and the domain difference is projection.
|
|
||||||
Future<Set<MergeFieldKey>> findActiveAttemptedFields({
|
|
||||||
required String tableName,
|
|
||||||
required Set<UuidValue> rowIds,
|
|
||||||
required Transaction transaction,
|
|
||||||
}) async {
|
|
||||||
if (rowIds.isEmpty) return const {};
|
|
||||||
|
|
||||||
final columnNames = {
|
|
||||||
..._foreignKeys.foreignKeyColumnsFor(tableName),
|
|
||||||
..._uniqueResolver.uniqueColumnNamesFor(tableName),
|
|
||||||
};
|
|
||||||
if (columnNames.isEmpty) return const {};
|
|
||||||
|
|
||||||
final fields = await _loadFields(
|
|
||||||
tableName: tableName,
|
|
||||||
rowIds: rowIds,
|
|
||||||
columnNames: columnNames,
|
|
||||||
transaction: transaction,
|
|
||||||
);
|
|
||||||
return {
|
|
||||||
for (final field in fields)
|
|
||||||
if (field.attemptedValue != null)
|
|
||||||
(tableName, field.row!.uuidRowId, field.column!.name),
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Proves that an ordinary local write leaves the existing projection fixed.
|
/// Proves that an ordinary local write leaves the existing projection fixed.
|
||||||
///
|
///
|
||||||
/// Only new, unreferenced rows and additions to null FKs qualify. All parents
|
/// Only new, unreferenced rows and additions to null FKs qualify. All parents
|
||||||
|
|
@ -689,6 +666,7 @@ class CrdtForeignKeyProjector {
|
||||||
inserting || writtenColumns == null || writtenColumns.contains(column)
|
inserting || writtenColumns == null || writtenColumns.contains(column)
|
||||||
? supplied[column]
|
? supplied[column]
|
||||||
: previous[column],
|
: previous[column],
|
||||||
|
_context.columnsByTableAndName[tableName]?[column],
|
||||||
),
|
),
|
||||||
};
|
};
|
||||||
if (!inserting) {
|
if (!inserting) {
|
||||||
|
|
@ -779,6 +757,8 @@ class CrdtForeignKeyProjector {
|
||||||
/// authored value of existing rows before planning. When [materialize] is
|
/// authored value of existing rows before planning. When [materialize] is
|
||||||
/// false, only the in-memory plan is computed so a later physical insert is
|
/// false, only the in-memory plan is computed so a later physical insert is
|
||||||
/// not blocked by this rebuild.
|
/// not blocked by this rebuild.
|
||||||
|
/// Local writes preserve proposed visible unique values for database
|
||||||
|
/// constraint enforcement; only merge/rebuild passes arbitrate new conflicts.
|
||||||
///
|
///
|
||||||
/// [seedTables] names the tables this pass changed. Only their foreign key
|
/// [seedTables] names the tables this pass changed. Only their foreign key
|
||||||
/// components are loaded, which is equivalent to a full pass because no
|
/// components are loaded, which is equivalent to a full pass because no
|
||||||
|
|
@ -791,6 +771,8 @@ class CrdtForeignKeyProjector {
|
||||||
Set<String>? seedTables,
|
Set<String>? seedTables,
|
||||||
Set<MergeRowKey>? seedRows,
|
Set<MergeRowKey>? seedRows,
|
||||||
bool materialize = true,
|
bool materialize = true,
|
||||||
|
bool localWrite = false,
|
||||||
|
Set<MergeRowKey> restoringRows = const {},
|
||||||
}) async {
|
}) async {
|
||||||
final state = await _loadProjectionState(
|
final state = await _loadProjectionState(
|
||||||
transaction,
|
transaction,
|
||||||
|
|
@ -834,6 +816,8 @@ class CrdtForeignKeyProjector {
|
||||||
finalHidden: finalHidden,
|
finalHidden: finalHidden,
|
||||||
authoredOverlays: authoredOverlays,
|
authoredOverlays: authoredOverlays,
|
||||||
materialize: materialize,
|
materialize: materialize,
|
||||||
|
localWrite: localWrite,
|
||||||
|
restoringRows: restoringRows,
|
||||||
transaction: transaction,
|
transaction: transaction,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
@ -1557,9 +1541,15 @@ class CrdtForeignKeyProjector {
|
||||||
// parent a row points at now and the one it is about to point at.
|
// parent a row points at now and the one it is about to point at.
|
||||||
Set<Object?> valuesFor(MergeRowKey rowKey, String columnName) {
|
Set<Object?> valuesFor(MergeRowKey rowKey, String columnName) {
|
||||||
return <Object?>{
|
return <Object?>{
|
||||||
rows[rowKey]?.values[columnName],
|
canonicalDomainValue(
|
||||||
|
rows[rowKey]?.values[columnName],
|
||||||
|
_context.columnsByTableAndName[rowKey.$1]?[columnName],
|
||||||
|
),
|
||||||
if (unwrittenValues[rowKey]?.containsKey(columnName) ?? false)
|
if (unwrittenValues[rowKey]?.containsKey(columnName) ?? false)
|
||||||
unwrittenValues[rowKey]![columnName],
|
canonicalDomainValue(
|
||||||
|
unwrittenValues[rowKey]![columnName],
|
||||||
|
_context.columnsByTableAndName[rowKey.$1]?[columnName],
|
||||||
|
),
|
||||||
}..remove(null);
|
}..remove(null);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -2101,6 +2091,8 @@ class CrdtForeignKeyProjector {
|
||||||
required Set<MergeRowKey> finalHidden,
|
required Set<MergeRowKey> finalHidden,
|
||||||
required Map<MergeFieldKey, Object?> authoredOverlays,
|
required Map<MergeFieldKey, Object?> authoredOverlays,
|
||||||
required bool materialize,
|
required bool materialize,
|
||||||
|
required bool localWrite,
|
||||||
|
required Set<MergeRowKey> restoringRows,
|
||||||
required Transaction transaction,
|
required Transaction transaction,
|
||||||
}) async {
|
}) async {
|
||||||
final authoredByField = _authoredValuesByField(state, authoredOverlays);
|
final authoredByField = _authoredValuesByField(state, authoredOverlays);
|
||||||
|
|
@ -2119,6 +2111,25 @@ class CrdtForeignKeyProjector {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (localWrite) {
|
||||||
|
// Keep locally observable unique values until the physical write has
|
||||||
|
// passed its constraints. An unchanged projected field carries its
|
||||||
|
// current domain value; a selected field carries the caller's value.
|
||||||
|
// Resolving a new conflict here would hide a violation from the database.
|
||||||
|
for (final row in state.rows.values) {
|
||||||
|
if (state.pendingInsertKeys.contains(row.key)) continue;
|
||||||
|
for (final column in _uniqueResolver.uniqueColumnNamesFor(row.key.$1)) {
|
||||||
|
final key = (row.key.$1, row.key.$2, column);
|
||||||
|
row.values[column] = canonicalDomainValue(
|
||||||
|
authoredOverlays.containsKey(key)
|
||||||
|
? authoredOverlays[key]
|
||||||
|
: state.originalDomain[row.key]?[column],
|
||||||
|
_context.columnsByTableAndName[row.key.$1]?[column],
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
final uniqueReasons = _uniqueResolver.planUniqueProjection(
|
final uniqueReasons = _uniqueResolver.planUniqueProjection(
|
||||||
valuesByRow: {
|
valuesByRow: {
|
||||||
for (final row in state.rows.values) row.key: row.values,
|
for (final row in state.rows.values) row.key: row.values,
|
||||||
|
|
@ -2126,10 +2137,13 @@ class CrdtForeignKeyProjector {
|
||||||
crdtRows: {
|
crdtRows: {
|
||||||
for (final row in state.rows.values) row.key: row.crdtRow,
|
for (final row in state.rows.values) row.key: row.crdtRow,
|
||||||
},
|
},
|
||||||
hidden: finalHidden,
|
// Restores must pass the physical unique constraint before their
|
||||||
|
// tombstones are lifted. Releasing these inputs would hide collisions.
|
||||||
|
hidden: finalHidden.difference(restoringRows),
|
||||||
authoredByField: authoredByField,
|
authoredByField: authoredByField,
|
||||||
claimByField: claimByField,
|
claimByField: claimByField,
|
||||||
fieldHlcs: state.fieldHlcs,
|
fieldHlcs: state.fieldHlcs,
|
||||||
|
resolveVisibleConflicts: !localWrite,
|
||||||
);
|
);
|
||||||
|
|
||||||
final terminalReasons = <MergeFieldKey, CrdtProjectionReason>{
|
final terminalReasons = <MergeFieldKey, CrdtProjectionReason>{
|
||||||
|
|
@ -2185,6 +2199,7 @@ class CrdtForeignKeyProjector {
|
||||||
authoredOverlays.containsKey(fieldKey)
|
authoredOverlays.containsKey(fieldKey)
|
||||||
? authoredOverlays[fieldKey]
|
? authoredOverlays[fieldKey]
|
||||||
: state.attemptedValues[fieldKey]?.value ?? domainValue,
|
: state.attemptedValues[fieldKey]?.value ?? domainValue,
|
||||||
|
_context.columnsByTableAndName[row.key.$1]?[columnName],
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -2517,7 +2532,11 @@ class CrdtForeignKeyProjector {
|
||||||
updates.entries.toList()..sort((a, b) => a.key.compareTo(b.key)),
|
updates.entries.toList()..sort((a, b) => a.key.compareTo(b.key)),
|
||||||
);
|
);
|
||||||
final signature = sortedUpdates.entries
|
final signature = sortedUpdates.entries
|
||||||
.map((entry) => '${entry.key}:${entry.value.sqlLiteral()}')
|
.map(
|
||||||
|
(entry) =>
|
||||||
|
'${entry.key}:'
|
||||||
|
'${_context.encodeDomainColumnValue(rowKey.$1, entry.key, entry.value)}',
|
||||||
|
)
|
||||||
.join('\x1f');
|
.join('\x1f');
|
||||||
final groupKey = (rowKey.$1, signature);
|
final groupKey = (rowKey.$1, signature);
|
||||||
|
|
||||||
|
|
@ -2687,7 +2706,7 @@ class CrdtForeignKeyProjector {
|
||||||
final attempted = CrdtDataAttemptedValue(
|
final attempted = CrdtDataAttemptedValue(
|
||||||
id: current?.id,
|
id: current?.id,
|
||||||
fieldId: write.fieldId,
|
fieldId: write.fieldId,
|
||||||
value: canonicalDomainValue(write.value),
|
value: write.value,
|
||||||
projectionReason: write.reason,
|
projectionReason: write.reason,
|
||||||
);
|
);
|
||||||
if (current == null) {
|
if (current == null) {
|
||||||
|
|
|
||||||
|
|
@ -278,6 +278,32 @@ WHERE r."spaceId" = $spaceId AND r."tblId" = $childTableId
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Reauthors existing fields at their row's newly persisted insertion HLC.
|
||||||
|
///
|
||||||
|
/// This runs before reprojection so unique claims use their new age. Updating
|
||||||
|
/// in place preserves attempted values whose foreign key cascades on field
|
||||||
|
/// deletion. [rows] carries the newly stamped metadata returned by the row
|
||||||
|
/// update, so no field or domain reads are needed. Ordinary field records can
|
||||||
|
/// be removed once projection is complete.
|
||||||
|
Future<void> resetReinsertedFieldClocks(
|
||||||
|
List<CrdtDataRow> rows,
|
||||||
|
Transaction transaction,
|
||||||
|
) async {
|
||||||
|
for (final row in rows) {
|
||||||
|
await CrdtDataField.db.updateWhere(
|
||||||
|
databaseSession,
|
||||||
|
columnValues: (t) => [
|
||||||
|
t.nodeId(row.nodeId),
|
||||||
|
t.hlcDatetime(row.hlcDatetime),
|
||||||
|
t.hlcCounter(row.hlcCounter),
|
||||||
|
],
|
||||||
|
where: (t) => t.rowId.equals(row.id),
|
||||||
|
transaction: transaction,
|
||||||
|
noReturn: true,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
Future<void> recordFieldsUpdatedByTable(
|
Future<void> recordFieldsUpdatedByTable(
|
||||||
String tableName,
|
String tableName,
|
||||||
Set<UuidValue> rowIds,
|
Set<UuidValue> rowIds,
|
||||||
|
|
@ -651,6 +677,16 @@ WHERE (${domainColumnPredicate('spaceId', spaceId)})
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Encodes model or merge values using the same column-aware encoder as ORM
|
||||||
|
/// writes. The generated column retains semantics such as DateTime, Duration,
|
||||||
|
/// binary data, and structured JSON that a SQL storage type alone cannot.
|
||||||
|
String encodeDomainColumnValue(String tableName, String columnName, Object? value) {
|
||||||
|
final column = syncTableByName[tableName]!.columns.singleWhere(
|
||||||
|
(column) => column.columnName == columnName,
|
||||||
|
);
|
||||||
|
return ValueEncoder.instance.encodeColumnValue(column, value);
|
||||||
|
}
|
||||||
|
|
||||||
Future<void> updateDomainRows(
|
Future<void> updateDomainRows(
|
||||||
String tableName,
|
String tableName,
|
||||||
Set<UuidValue> rowIds,
|
Set<UuidValue> rowIds,
|
||||||
|
|
@ -659,18 +695,12 @@ WHERE (${domainColumnPredicate('spaceId', spaceId)})
|
||||||
) async {
|
) async {
|
||||||
if (rowIds.isEmpty || updates.isEmpty) return;
|
if (rowIds.isEmpty || updates.isEmpty) return;
|
||||||
|
|
||||||
final columns = {
|
|
||||||
for (final column in syncTableByName[tableName]!.columns)
|
|
||||||
column.columnName: column,
|
|
||||||
};
|
|
||||||
final assignments = updates.entries
|
final assignments = updates.entries
|
||||||
.map((entry) {
|
.map(
|
||||||
final value = ValueEncoder.instance.encodeColumnValue(
|
(e) =>
|
||||||
columns[entry.key]!,
|
'"${e.key.escapeIdentifier()}" = '
|
||||||
entry.value,
|
'${encodeDomainColumnValue(tableName, e.key, e.value)}',
|
||||||
);
|
)
|
||||||
return '"${entry.key.escapeIdentifier()}" = $value';
|
|
||||||
})
|
|
||||||
.join(', ');
|
.join(', ');
|
||||||
|
|
||||||
await database.unsafeExecute(
|
await database.unsafeExecute(
|
||||||
|
|
@ -735,10 +765,13 @@ WHERE "id" IN (${rowIds.sqlLiteralList()})
|
||||||
}
|
}
|
||||||
case CrdtUniqueConflictReleaseKind.syntheticUuid:
|
case CrdtUniqueConflictReleaseKind.syntheticUuid:
|
||||||
if (value != null) {
|
if (value != null) {
|
||||||
return const Uuid().v5obj(
|
final hash = const Uuid().v5(
|
||||||
Namespace.oid.value,
|
Namespace.oid.value,
|
||||||
'$tableName.${column.columnName}:${value}__${releaseSuffix}__$conflictingId',
|
'$tableName.${column.columnName}:${value}__${releaseSuffix}__$conflictingId',
|
||||||
);
|
);
|
||||||
|
// Keep the deterministic payload, but mark these custom UUIDs with
|
||||||
|
// version 8 so authored values can be rejected without domain reads.
|
||||||
|
return UuidValue.fromString(hash.replaceRange(14, 15, '8'));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -879,9 +912,16 @@ WHERE c."id" IN ($whereRowIds)
|
||||||
);
|
);
|
||||||
if (result.isEmpty) return null;
|
if (result.isEmpty) return null;
|
||||||
|
|
||||||
|
final foreignKeyIndex = result.first[0] as int;
|
||||||
return (
|
return (
|
||||||
foreignKeyIndex: result.first[0] as int,
|
foreignKeyIndex: foreignKeyIndex,
|
||||||
value: result.first[1] as Object,
|
// Raw SQLite UUID columns are blobs; report the authored identifier.
|
||||||
|
value: canonicalDomainValue(
|
||||||
|
result.first[1],
|
||||||
|
columnsByTableAndName[childTableName]?[foreignKeys[foreignKeyIndex]
|
||||||
|
.columns
|
||||||
|
.single],
|
||||||
|
)!,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,9 +1,11 @@
|
||||||
import 'package:meta/meta.dart';
|
import 'package:meta/meta.dart';
|
||||||
|
import 'package:serverpod_database/serverpod_database.dart' show ColumnType;
|
||||||
import 'package:serverpod_serialization/serverpod_serialization.dart';
|
import 'package:serverpod_serialization/serverpod_serialization.dart';
|
||||||
|
|
||||||
import '../../crdt/extensions.dart';
|
import '../../crdt/extensions.dart';
|
||||||
import '../../generated/protocol.dart';
|
import '../../generated/protocol.dart';
|
||||||
import '../../hlc/hlc.dart';
|
import '../../hlc/hlc.dart';
|
||||||
|
import '../exceptions.dart';
|
||||||
import '../unique_index_utils.dart';
|
import '../unique_index_utils.dart';
|
||||||
import 'database_helpers.dart';
|
import 'database_helpers.dart';
|
||||||
import 'recorder_context.dart';
|
import 'recorder_context.dart';
|
||||||
|
|
@ -21,6 +23,73 @@ class CrdtUniqueConflictResolver {
|
||||||
|
|
||||||
final CrdtRecorderContext _context;
|
final CrdtRecorderContext _context;
|
||||||
|
|
||||||
|
static final _reservedTextSuffix = RegExp(
|
||||||
|
'__(?:conflict|hidden|park)__[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-'
|
||||||
|
r'[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$',
|
||||||
|
);
|
||||||
|
final _uniqueTextColumns = <String, Set<String>>{};
|
||||||
|
final _syntheticUuidColumns = <String, Set<String>>{};
|
||||||
|
|
||||||
|
Set<String> _textColumnsFor(String tableName) => _uniqueTextColumns.putIfAbsent(
|
||||||
|
tableName,
|
||||||
|
() => {
|
||||||
|
for (final column in uniqueColumnNamesFor(tableName))
|
||||||
|
if (_context.columnsByTableAndName[tableName]?[column]?.columnType ==
|
||||||
|
ColumnType.text)
|
||||||
|
column,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
Set<String> _uuidColumnsFor(String tableName) => _syntheticUuidColumns.putIfAbsent(
|
||||||
|
tableName,
|
||||||
|
() => {
|
||||||
|
for (final index in uniqueIndexesFor(tableName))
|
||||||
|
for (final column in index.releaseColumns)
|
||||||
|
if (column.kind == CrdtUniqueConflictReleaseKind.syntheticUuid)
|
||||||
|
column.columnName,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
/// Whether authored values need a generated-namespace check.
|
||||||
|
bool hasReservedUniqueColumns(String tableName) =>
|
||||||
|
_textColumnsFor(tableName).isNotEmpty || _uuidColumnsFor(tableName).isNotEmpty;
|
||||||
|
|
||||||
|
/// Rejects authored values in the namespace owned by deterministic projection.
|
||||||
|
/// This checks the schema and input only, never the currently occupied names.
|
||||||
|
void validateAuthoredValue(String tableName, String columnName, Object? value) {
|
||||||
|
if (!isReservedValue(tableName, columnName, value)) return;
|
||||||
|
throw OfflineSyncReservedValueException(
|
||||||
|
tableName: tableName,
|
||||||
|
columnName: columnName,
|
||||||
|
value: value.toString(),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Generated text suffixes and version-8 UUIDs belong only to projection.
|
||||||
|
/// UUID checks apply only where the schema uses synthetic UUID release,
|
||||||
|
/// leaving primary keys, foreign keys, and nullable UUID columns unrestricted.
|
||||||
|
bool isReservedValue(String tableName, String columnName, Object? value) {
|
||||||
|
if (value is String &&
|
||||||
|
value.contains('__') &&
|
||||||
|
_textColumnsFor(tableName).contains(columnName) &&
|
||||||
|
_reservedTextSuffix.hasMatch(value)) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
if ((value is UuidValue || value is String) &&
|
||||||
|
_uuidColumnsFor(tableName).contains(columnName)) {
|
||||||
|
final uuid = value.toString();
|
||||||
|
return uuid.length == 36 && uuid[14] == '8';
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Validates a group of newly authored field facts.
|
||||||
|
void validateAuthoredFields(Map<MergeFieldKey, Object?> values) {
|
||||||
|
for (final MapEntry(key: key, value: value) in values.entries) {
|
||||||
|
validateAuthoredValue(key.$1, key.$3, value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Whether [tableName] has any synchronized unique index.
|
/// Whether [tableName] has any synchronized unique index.
|
||||||
bool tableHasUniqueIndexes(String tableName) =>
|
bool tableHasUniqueIndexes(String tableName) =>
|
||||||
uniqueIndexesFor(tableName).isNotEmpty;
|
uniqueIndexesFor(tableName).isNotEmpty;
|
||||||
|
|
@ -77,6 +146,7 @@ class CrdtUniqueConflictResolver {
|
||||||
required Map<MergeFieldKey, Object?> authoredByField,
|
required Map<MergeFieldKey, Object?> authoredByField,
|
||||||
required Map<MergeFieldKey, Object?> claimByField,
|
required Map<MergeFieldKey, Object?> claimByField,
|
||||||
required Map<MergeFieldKey, Hlc> fieldHlcs,
|
required Map<MergeFieldKey, Hlc> fieldHlcs,
|
||||||
|
bool resolveVisibleConflicts = true,
|
||||||
}) {
|
}) {
|
||||||
final reasons = <MergeFieldKey, CrdtProjectionReason>{};
|
final reasons = <MergeFieldKey, CrdtProjectionReason>{};
|
||||||
final tableNames = {for (final rowKey in valuesByRow.keys) rowKey.$1};
|
final tableNames = {for (final rowKey in valuesByRow.keys) rowKey.$1};
|
||||||
|
|
@ -107,6 +177,9 @@ class CrdtUniqueConflictResolver {
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Local writes retain their proposed unique values so the database
|
||||||
|
// enforces uniqueness. Hidden rows still release their old values.
|
||||||
|
if (!resolveVisibleConflicts) continue;
|
||||||
final groups = <String, List<MergeRowKey>>{};
|
final groups = <String, List<MergeRowKey>>{};
|
||||||
for (final rowKey in tableRowKeys) {
|
for (final rowKey in tableRowKeys) {
|
||||||
if (hidden.contains(rowKey)) continue;
|
if (hidden.contains(rowKey)) continue;
|
||||||
|
|
@ -250,6 +323,7 @@ class CrdtUniqueConflictResolver {
|
||||||
@internal
|
@internal
|
||||||
String canonicalProjectionValue(Object? value) {
|
String canonicalProjectionValue(Object? value) {
|
||||||
if (value == null) return '';
|
if (value == null) return '';
|
||||||
|
if (value is String) return value;
|
||||||
final uuid = tryUuidValue(value);
|
final uuid = tryUuidValue(value);
|
||||||
if (uuid != null) return uuid.uuid;
|
if (uuid != null) return uuid.uuid;
|
||||||
return value.toString();
|
return value.toString();
|
||||||
|
|
@ -259,6 +333,7 @@ String canonicalProjectionValue(Object? value) {
|
||||||
@internal
|
@internal
|
||||||
bool projectionValuesEqual(Object? left, Object? right) {
|
bool projectionValuesEqual(Object? left, Object? right) {
|
||||||
if (left == null || right == null) return left == right;
|
if (left == null || right == null) return left == right;
|
||||||
|
if (left is String && right is String) return left == right;
|
||||||
if (left is Map && right is Map) {
|
if (left is Map && right is Map) {
|
||||||
return left.length == right.length &&
|
return left.length == right.length &&
|
||||||
left.entries.every(
|
left.entries.every(
|
||||||
|
|
|
||||||
|
|
@ -372,8 +372,10 @@ class CrdtMutationRecorder {
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Returns whether primary-key upserts preserve the existing projection.
|
/// Captures the materialized values before an upsert so its accepted writes
|
||||||
Future<bool> prepareLocalUpsert<T extends TableRow>(
|
/// can replace old attempted values without authoring unchanged projections.
|
||||||
|
Future<({bool projectionUnchanged, Map<MergeRowKey, Map<String, Object?>> domain})>
|
||||||
|
prepareLocalUpsert<T extends TableRow>(
|
||||||
List<T> rows,
|
List<T> rows,
|
||||||
List<Column> conflictColumns,
|
List<Column> conflictColumns,
|
||||||
List<Column>? updateColumns,
|
List<Column>? updateColumns,
|
||||||
|
|
@ -384,15 +386,17 @@ class CrdtMutationRecorder {
|
||||||
for (final row in rows)
|
for (final row in rows)
|
||||||
if (row.id case final UuidValue rowId) rowId,
|
if (row.id case final UuidValue rowId) rowId,
|
||||||
};
|
};
|
||||||
if (conflictColumns.length == 1 &&
|
final targetsPrimaryKey =
|
||||||
conflictColumns.single.columnName == 'id' &&
|
conflictColumns.length == 1 && conflictColumns.single.columnName == 'id';
|
||||||
rowIds.length == rows.length) {
|
const emptyDomain = <MergeRowKey, Map<String, Object?>>{};
|
||||||
|
if (targetsPrimaryKey && rowIds.length == rows.length) {
|
||||||
final stored = await _context.findCrdtRows(tableName, rowIds, transaction);
|
final stored = await _context.findCrdtRows(tableName, rowIds, transaction);
|
||||||
if (stored.isEmpty &&
|
if (stored.isEmpty &&
|
||||||
!_foreignKeys.tablesWithDefaultDependencies.contains(tableName)) {
|
!_foreignKeys.tablesWithDefaultDependencies.contains(tableName)) {
|
||||||
// The full pass has no persisted seed or default to load. Classification
|
// The full pass has no persisted seed or default to load. Classification
|
||||||
// after the physical upsert still reads metadata again.
|
// after the physical upsert still reads metadata again.
|
||||||
return false;
|
validateAuthoredRows(rows, null);
|
||||||
|
return (projectionUnchanged: false, domain: emptyDomain);
|
||||||
}
|
}
|
||||||
if (stored.length == rowIds.length &&
|
if (stored.length == rowIds.length &&
|
||||||
await _foreignKeyProjector.canLeaveLocalProjectionUnchanged(
|
await _foreignKeyProjector.canLeaveLocalProjectionUnchanged(
|
||||||
|
|
@ -401,20 +405,82 @@ class CrdtMutationRecorder {
|
||||||
inserting: false,
|
inserting: false,
|
||||||
columns: updateColumns,
|
columns: updateColumns,
|
||||||
)) {
|
)) {
|
||||||
return true;
|
validateAuthoredRows(rows, updateColumns);
|
||||||
|
return (projectionUnchanged: true, domain: emptyDomain);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
await projectCurrent(tableName, rowIds, transaction);
|
if (!targetsPrimaryKey && _foreignKeyProjector.needsProjection(tableName, null)) {
|
||||||
return false;
|
final suppliedValues = [for (final row in rows) row.toJsonForDatabase() as Map];
|
||||||
|
rowIds.addAll(
|
||||||
|
await _context.findDomainRowIdsWhereColumnsIn(
|
||||||
|
tableName: tableName,
|
||||||
|
valuesByColumn: {
|
||||||
|
for (final column in conflictColumns)
|
||||||
|
if (column.columnName != 'spaceId')
|
||||||
|
column.columnName: {
|
||||||
|
for (final values in suppliedValues)
|
||||||
|
canonicalDomainValue(
|
||||||
|
values[column.columnName],
|
||||||
|
_context.columnsByTableAndName[tableName]?[column.columnName],
|
||||||
|
),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
transaction: transaction,
|
||||||
|
),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
final projected = await _foreignKeyProjector.project(
|
||||||
|
transaction,
|
||||||
|
seedTables: {tableName},
|
||||||
|
// Other conflict targets can update an existing row with a different id.
|
||||||
|
seedRows: {for (final rowId in rowIds) (tableName, rowId)},
|
||||||
|
);
|
||||||
|
if (_uniqueResolver.hasReservedUniqueColumns(tableName)) {
|
||||||
|
final written = updateColumns?.map((column) => column.columnName).toSet();
|
||||||
|
for (final row in rows) {
|
||||||
|
final supplied = _authoredValuesFromRow(row, null);
|
||||||
|
for (final MapEntry(key: column, value: value) in supplied.entries) {
|
||||||
|
if (!_uniqueResolver.isReservedValue(tableName, column, value)) continue;
|
||||||
|
final before =
|
||||||
|
projected.domain[(tableName, row.id)] ??
|
||||||
|
projected.domain.entries
|
||||||
|
.where(
|
||||||
|
(entry) =>
|
||||||
|
entry.key.$1 == tableName &&
|
||||||
|
conflictColumns.every(
|
||||||
|
(key) =>
|
||||||
|
key.columnName == 'spaceId' ||
|
||||||
|
(key.columnName == 'id'
|
||||||
|
? entry.key.$2 == row.id
|
||||||
|
: projectionValuesEqual(
|
||||||
|
entry.value[key.columnName],
|
||||||
|
supplied[key.columnName],
|
||||||
|
)),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.firstOrNull
|
||||||
|
?.value;
|
||||||
|
if (before != null &&
|
||||||
|
((written != null && !written.contains(column)) ||
|
||||||
|
projectionValuesEqual(before[column], value))) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
// A new claim must fail before the physical upsert can collide with
|
||||||
|
// another row's generated value. Unchanged echoes are checked after
|
||||||
|
// the write, when we know whether it updated or restored the row.
|
||||||
|
_uniqueResolver.validateAuthoredValue(tableName, column, value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return (projectionUnchanged: false, domain: projected.domain);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Plans FK/unique projection for rows that are about to be inserted.
|
/// Plans FK/unique projection for rows that are about to be inserted.
|
||||||
///
|
///
|
||||||
/// Releases hidden unique claims first and returns copies whose unique/FK
|
/// Releases hidden unique claims first and plans foreign key repairs while
|
||||||
/// columns already hold the planned domain values, so the physical write
|
/// preserving submitted unique values for database constraint enforcement.
|
||||||
/// cannot violate an immediate unique index. Authored values that differ
|
/// Authored values that differ from the planned domain are returned so they
|
||||||
/// from the planned domain are returned so they can be stored as attempted
|
/// can be stored as attempted values after the insert.
|
||||||
/// values after the insert.
|
|
||||||
Future<({List<T> rows, ProjectionAttemptsByField attempts})>
|
Future<({List<T> rows, ProjectionAttemptsByField attempts})>
|
||||||
planLocalInserts<T extends TableRow>(
|
planLocalInserts<T extends TableRow>(
|
||||||
List<T> rows,
|
List<T> rows,
|
||||||
|
|
@ -428,6 +494,8 @@ class CrdtMutationRecorder {
|
||||||
return (rows: rows, attempts: unplanned);
|
return (rows: rows, attempts: unplanned);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
validateAuthoredRows(rows, null);
|
||||||
|
|
||||||
if (await _foreignKeyProjector.canLeaveLocalProjectionUnchanged(
|
if (await _foreignKeyProjector.canLeaveLocalProjectionUnchanged(
|
||||||
rows,
|
rows,
|
||||||
transaction,
|
transaction,
|
||||||
|
|
@ -459,6 +527,7 @@ class CrdtMutationRecorder {
|
||||||
final planned = await _foreignKeyProjector.project(
|
final planned = await _foreignKeyProjector.project(
|
||||||
transaction,
|
transaction,
|
||||||
pendingInserts: pending,
|
pendingInserts: pending,
|
||||||
|
localWrite: true,
|
||||||
seedTables: {tableName},
|
seedTables: {tableName},
|
||||||
seedRows: {for (final row in pending) (tableName, row.rowId)},
|
seedRows: {for (final row in pending) (tableName, row.rowId)},
|
||||||
);
|
);
|
||||||
|
|
@ -496,8 +565,9 @@ class CrdtMutationRecorder {
|
||||||
planLocalUpdates<T extends TableRow>(
|
planLocalUpdates<T extends TableRow>(
|
||||||
List<T> rows,
|
List<T> rows,
|
||||||
List<Column>? columns,
|
List<Column>? columns,
|
||||||
Transaction transaction,
|
Transaction transaction, {
|
||||||
) async {
|
bool restoring = false,
|
||||||
|
}) async {
|
||||||
if (rows.isEmpty) return (rows: rows, projectionUnchanged: true);
|
if (rows.isEmpty) return (rows: rows, projectionUnchanged: true);
|
||||||
final tableName = rows.first.table.tableName;
|
final tableName = rows.first.table.tableName;
|
||||||
if (!_context.isCrdtTrackedTableName(tableName)) {
|
if (!_context.isCrdtTrackedTableName(tableName)) {
|
||||||
|
|
@ -524,9 +594,7 @@ class CrdtMutationRecorder {
|
||||||
row,
|
row,
|
||||||
columns,
|
columns,
|
||||||
).entries)
|
).entries)
|
||||||
(tableName, row.id as UuidValue, columnName): canonicalDomainValue(
|
(tableName, row.id as UuidValue, columnName): value,
|
||||||
value,
|
|
||||||
),
|
|
||||||
};
|
};
|
||||||
var overlays = authored;
|
var overlays = authored;
|
||||||
if (columns == null) {
|
if (columns == null) {
|
||||||
|
|
@ -548,9 +616,15 @@ class CrdtMutationRecorder {
|
||||||
fieldKey: value,
|
fieldKey: value,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
_uniqueResolver.validateAuthoredFields(overlays);
|
||||||
final planned = await _foreignKeyProjector.project(
|
final planned = await _foreignKeyProjector.project(
|
||||||
transaction,
|
transaction,
|
||||||
authoredOverlays: overlays,
|
authoredOverlays: overlays,
|
||||||
|
localWrite: true,
|
||||||
|
restoringRows: {
|
||||||
|
if (restoring)
|
||||||
|
for (final row in rows) (tableName, row.id as UuidValue),
|
||||||
|
},
|
||||||
seedTables: {tableName},
|
seedTables: {tableName},
|
||||||
seedRows: {
|
seedRows: {
|
||||||
for (final row in rows)
|
for (final row in rows)
|
||||||
|
|
@ -569,6 +643,30 @@ class CrdtMutationRecorder {
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Validates unprojected ORM writes, or projected inserts with their attempts.
|
||||||
|
void validateAuthoredRows<T extends TableRow>(
|
||||||
|
List<T> rows,
|
||||||
|
List<Column>? columns, {
|
||||||
|
ProjectionAttemptsByField attempts = const {},
|
||||||
|
}) {
|
||||||
|
if (rows.isEmpty) return;
|
||||||
|
final tableName = rows.first.table.tableName;
|
||||||
|
if (!_uniqueResolver.hasReservedUniqueColumns(tableName)) return;
|
||||||
|
for (final row in rows) {
|
||||||
|
for (final MapEntry(key: columnName, value: value) in _authoredValuesFromRow(
|
||||||
|
row,
|
||||||
|
columns,
|
||||||
|
).entries) {
|
||||||
|
final key = (tableName, row.id, columnName);
|
||||||
|
_uniqueResolver.validateAuthoredValue(
|
||||||
|
tableName,
|
||||||
|
columnName,
|
||||||
|
attempts.containsKey(key) ? attempts[key]!.value : value,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
Map<String, Object?> _authoredValuesFromRow<T extends TableRow>(
|
Map<String, Object?> _authoredValuesFromRow<T extends TableRow>(
|
||||||
T row,
|
T row,
|
||||||
List<Column>? columns,
|
List<Column>? columns,
|
||||||
|
|
@ -580,7 +678,11 @@ class CrdtMutationRecorder {
|
||||||
column.columnName,
|
column.columnName,
|
||||||
];
|
];
|
||||||
return {
|
return {
|
||||||
for (final columnName in columnNames) columnName: json[columnName],
|
for (final columnName in columnNames)
|
||||||
|
columnName: canonicalDomainValue(
|
||||||
|
json[columnName],
|
||||||
|
_context.columnsByTableAndName[row.table.tableName]?[columnName],
|
||||||
|
),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -659,6 +761,7 @@ class CrdtMutationRecorder {
|
||||||
Transaction transaction, {
|
Transaction transaction, {
|
||||||
ProjectionAttemptsByField attempts = const {},
|
ProjectionAttemptsByField attempts = const {},
|
||||||
}) async {
|
}) async {
|
||||||
|
validateAuthoredRows(insertedRows, null, attempts: attempts);
|
||||||
await _forTrackedRows(insertedRows, transaction, (
|
await _forTrackedRows(insertedRows, transaction, (
|
||||||
tableName,
|
tableName,
|
||||||
rowIds,
|
rowIds,
|
||||||
|
|
@ -700,39 +803,36 @@ class CrdtMutationRecorder {
|
||||||
rowIds,
|
rowIds,
|
||||||
hlcManager,
|
hlcManager,
|
||||||
) async {
|
) async {
|
||||||
final crdtDataRows = await _context.findRequiredCrdtRows(
|
final crdtDataRows = await _touchCrdtRows(
|
||||||
tableName,
|
await _context.findRequiredCrdtRows(
|
||||||
rowIds,
|
tableName,
|
||||||
'reinserted',
|
rowIds,
|
||||||
|
'reinserted',
|
||||||
|
transaction,
|
||||||
|
),
|
||||||
|
hlcManager,
|
||||||
transaction,
|
transaction,
|
||||||
);
|
);
|
||||||
|
// A reinsert authors the complete row at its new insertion timestamp.
|
||||||
await _touchCrdtRows(crdtDataRows, hlcManager, transaction);
|
// Retain attempted values while projecting, but discard their old ages.
|
||||||
|
await _context.resetReinsertedFieldClocks(crdtDataRows, transaction);
|
||||||
await _context.markCrdtRowsDeleted(
|
await _context.markCrdtRowsDeleted(
|
||||||
crdtDataRows,
|
crdtDataRows,
|
||||||
false,
|
false,
|
||||||
CrdtDataDeletedReason.userReinsert,
|
CrdtDataDeletedReason.userReinsert,
|
||||||
transaction,
|
transaction,
|
||||||
);
|
);
|
||||||
// Every field to skip carries an attempted value: a repaired foreign key
|
|
||||||
// and a restored authored value are both subsets of that set, over a
|
|
||||||
// subset of its columns.
|
|
||||||
final skippedFields = await _foreignKeyProjector.findActiveAttemptedFields(
|
|
||||||
tableName: tableName,
|
|
||||||
rowIds: rowIds,
|
|
||||||
transaction: transaction,
|
|
||||||
);
|
|
||||||
// Reinsert is a full-row passthrough. Projected unique/FK columns must
|
|
||||||
// keep their original claim HLC so restoration can reclaim or lose
|
|
||||||
// without authoring a newer unique claim.
|
|
||||||
await _recordUpdatedFields(
|
|
||||||
reinsertedRows,
|
|
||||||
crdtDataRows,
|
|
||||||
null,
|
|
||||||
transaction,
|
|
||||||
skippedFields: skippedFields,
|
|
||||||
);
|
|
||||||
await _maybeProject(tableName, rowIds, null, transaction);
|
await _maybeProject(tableName, rowIds, null, transaction);
|
||||||
|
// Only projected fields need a metadata row to hold their authored value.
|
||||||
|
// All other fields now inherit the insertion timestamp implicitly.
|
||||||
|
await CrdtDataField.db.deleteWhere(
|
||||||
|
_session,
|
||||||
|
where: (t) =>
|
||||||
|
t.rowId.inSet(crdtDataRows.map((row) => row.id!).toSet()) &
|
||||||
|
t.attemptedValue.id.equals(null),
|
||||||
|
transaction: transaction,
|
||||||
|
noReturn: true,
|
||||||
|
);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -742,6 +842,7 @@ class CrdtMutationRecorder {
|
||||||
List<Column>? columns,
|
List<Column>? columns,
|
||||||
Transaction transaction, {
|
Transaction transaction, {
|
||||||
bool projectionUnchanged = false,
|
bool projectionUnchanged = false,
|
||||||
|
Map<MergeRowKey, Map<String, Object?>> domainBeforeUpsert = const {},
|
||||||
}) async {
|
}) async {
|
||||||
await _foreignKeyProjector.assertVisibleTargets(updatedRows, columns, transaction);
|
await _foreignKeyProjector.assertVisibleTargets(updatedRows, columns, transaction);
|
||||||
|
|
||||||
|
|
@ -756,6 +857,24 @@ class CrdtMutationRecorder {
|
||||||
'updated',
|
'updated',
|
||||||
transaction,
|
transaction,
|
||||||
);
|
);
|
||||||
|
// Only rows returned by the physical upsert were accepted by updateWhere.
|
||||||
|
// Explicit columns author even an unchanged null; full-row passthrough
|
||||||
|
// keeps the claim behind an unchanged displayed alternative.
|
||||||
|
final authoredOverlays = <MergeFieldKey, Object?>{
|
||||||
|
for (final row in updatedRows)
|
||||||
|
if (domainBeforeUpsert.containsKey((tableName, row.id)))
|
||||||
|
for (final MapEntry(key: columnName, value: value)
|
||||||
|
in _authoredValuesFromRow(row, columns).entries)
|
||||||
|
if ((domainBeforeUpsert[(tableName, row.id)]?.containsKey(columnName) ??
|
||||||
|
false) &&
|
||||||
|
(columns != null ||
|
||||||
|
!projectionValuesEqual(
|
||||||
|
domainBeforeUpsert[(tableName, row.id)]![columnName],
|
||||||
|
value,
|
||||||
|
)))
|
||||||
|
(tableName, row.id as UuidValue, columnName): value,
|
||||||
|
};
|
||||||
|
_uniqueResolver.validateAuthoredFields(authoredOverlays);
|
||||||
final implicitForeignKeyRepairFields = columns == null
|
final implicitForeignKeyRepairFields = columns == null
|
||||||
? await _foreignKeyProjector.findImplicitRepairFields(
|
? await _foreignKeyProjector.findImplicitRepairFields(
|
||||||
tableName: tableName,
|
tableName: tableName,
|
||||||
|
|
@ -768,7 +887,9 @@ class CrdtMutationRecorder {
|
||||||
crdtDataRows,
|
crdtDataRows,
|
||||||
columns,
|
columns,
|
||||||
transaction,
|
transaction,
|
||||||
skippedFields: implicitForeignKeyRepairFields,
|
skippedFields: implicitForeignKeyRepairFields.difference(
|
||||||
|
authoredOverlays.keys.toSet(),
|
||||||
|
),
|
||||||
);
|
);
|
||||||
if (projectionUnchanged &&
|
if (projectionUnchanged &&
|
||||||
await _foreignKeyProjector.canLeaveLocalProjectionUnchanged(
|
await _foreignKeyProjector.canLeaveLocalProjectionUnchanged(
|
||||||
|
|
@ -785,6 +906,7 @@ class CrdtMutationRecorder {
|
||||||
rowIds,
|
rowIds,
|
||||||
updatedColumnNames,
|
updatedColumnNames,
|
||||||
transaction,
|
transaction,
|
||||||
|
authoredOverlays: authoredOverlays,
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
@ -793,13 +915,15 @@ class CrdtMutationRecorder {
|
||||||
String tableName,
|
String tableName,
|
||||||
Set<UuidValue> rowIds,
|
Set<UuidValue> rowIds,
|
||||||
Set<String>? columnNames,
|
Set<String>? columnNames,
|
||||||
Transaction transaction,
|
Transaction transaction, {
|
||||||
) async {
|
Map<MergeFieldKey, Object?> authoredOverlays = const {},
|
||||||
|
}) async {
|
||||||
if (_foreignKeyProjector.needsProjection(tableName, columnNames)) {
|
if (_foreignKeyProjector.needsProjection(tableName, columnNames)) {
|
||||||
await _foreignKeyProjector.project(
|
await _foreignKeyProjector.project(
|
||||||
transaction,
|
transaction,
|
||||||
seedTables: {tableName},
|
seedTables: {tableName},
|
||||||
seedRows: {for (final rowId in rowIds) (tableName, rowId)},
|
seedRows: {for (final rowId in rowIds) (tableName, rowId)},
|
||||||
|
authoredOverlays: authoredOverlays,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -843,14 +967,14 @@ class CrdtMutationRecorder {
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
Future<void> _touchCrdtRows(
|
Future<List<CrdtDataRow>> _touchCrdtRows(
|
||||||
List<CrdtDataRow> rows,
|
List<CrdtDataRow> rows,
|
||||||
HlcManager hlcManager,
|
HlcManager hlcManager,
|
||||||
Transaction transaction,
|
Transaction transaction,
|
||||||
) async {
|
) async {
|
||||||
if (rows.isEmpty) return;
|
if (rows.isEmpty) return [];
|
||||||
|
|
||||||
await CrdtDataRow.db.update(
|
return CrdtDataRow.db.update(
|
||||||
_session,
|
_session,
|
||||||
[for (final row in rows) _context.withNextHlc(row, hlcManager)],
|
[for (final row in rows) _context.withNextHlc(row, hlcManager)],
|
||||||
columns: (t) => [t.nodeId, t.hlcDatetime, t.hlcCounter],
|
columns: (t) => [t.nodeId, t.hlcDatetime, t.hlcCounter],
|
||||||
|
|
|
||||||
|
|
@ -900,7 +900,7 @@ class OfflineSyncEngine {
|
||||||
return (
|
return (
|
||||||
exists: true,
|
exists: true,
|
||||||
ownerSpaceId: spaceId,
|
ownerSpaceId: spaceId,
|
||||||
value: canonicalDomainValue(attempted.value),
|
value: attempted.value,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -1208,7 +1208,7 @@ extension on Map<String, dynamic> {
|
||||||
for (final field in attemptedValueFields) {
|
for (final field in attemptedValueFields) {
|
||||||
final attempted = field.attemptedValue;
|
final attempted = field.attemptedValue;
|
||||||
if (attempted == null) continue;
|
if (attempted == null) continue;
|
||||||
this[field.column!.name] = canonicalDomainValue(attempted.value);
|
this[field.column!.name] = attempted.value;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -2,7 +2,7 @@ tags:
|
||||||
integration: {}
|
integration: {}
|
||||||
dst:
|
dst:
|
||||||
# A simulation drives many real SQLite transactions across replicas.
|
# A simulation drives many real SQLite transactions across replicas.
|
||||||
timeout: 5x
|
timeout: 10m
|
||||||
skip: 'Simulation suite: run with `dart test -P dst`'
|
skip: 'Simulation suite: run with `dart test -P dst`'
|
||||||
|
|
||||||
# The simulation suite is excluded from an ordinary run. It is slow, and its
|
# The simulation suite is excluded from an ordinary run. It is slow, and its
|
||||||
|
|
|
||||||
|
|
@ -2,7 +2,8 @@
|
||||||
|
|
||||||
These suites drive several replicas through randomized operations and
|
These suites drive several replicas through randomized operations and
|
||||||
adversarial delivery, then check properties the engine must hold. A run is
|
adversarial delivery, then check properties the engine must hold. A run is
|
||||||
fully described by its seed, so a failure replays exactly.
|
described by its seed, rounds, workload profile, and graph width. Those inputs
|
||||||
|
replay exactly within the same source/SDK revision.
|
||||||
|
|
||||||
They complement the `integration/` suites rather than replacing them.
|
They complement the `integration/` suites rather than replacing them.
|
||||||
Integration tests pin specific scenarios and assert exact outcomes; these
|
Integration tests pin specific scenarios and assert exact outcomes; these
|
||||||
|
|
@ -27,8 +28,10 @@ directory are untagged and run with ordinary `dart test`.
|
||||||
| Variable | Default | Meaning |
|
| Variable | Default | Meaning |
|
||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
| `DST_SEEDS` | 8 | How many seeds the sweep runs |
|
| `DST_SEEDS` | 8 | How many seeds the sweep runs |
|
||||||
| `DST_ROUNDS` | 12 | Operation rounds per simulation |
|
| `DST_ROUNDS` | 40 | Operation rounds per simulation |
|
||||||
| `DST_SEED_BASE` | unix seconds | First seed; successive seeds increment |
|
| `DST_SEED_BASE` | unix seconds | First seed; successive seeds increment |
|
||||||
|
| `DST_PROFILE` | sparse | `sparse`, `populated`, or `mixed` (even seeds populated, odd sparse) |
|
||||||
|
| `DST_GRAPH_WIDTH` | 2 | Rows per table in each populated graph, minimum 2 |
|
||||||
| `DST_DEBUG_TABLE` | unset | Print every merge touching this table, in delivery order |
|
| `DST_DEBUG_TABLE` | unset | Print every merge touching this table, in delivery order |
|
||||||
|
|
||||||
A divergence is usually explained by which facts a replica had merged when it
|
A divergence is usually explained by which facts a replica had merged when it
|
||||||
|
|
@ -44,7 +47,7 @@ which defects those fixed seeds reach.
|
||||||
Nothing is lost to reproducibility: a seed determines its simulation entirely
|
Nothing is lost to reproducibility: a seed determines its simulation entirely
|
||||||
and a failure prints its replay command. Tests are named by position rather
|
and a failure prints its replay command. Tests are named by position rather
|
||||||
than by seed, so the suite stays stable while the schedules underneath it vary.
|
than by seed, so the suite stays stable while the schedules underneath it vary.
|
||||||
Pin `DST_SEED_BASE` to re-run an exact sweep.
|
Pin the seed, rounds, profile, and graph width to re-run an exact sweep.
|
||||||
|
|
||||||
The consequence to expect is that a run can fail for a defect unrelated to the
|
The consequence to expect is that a run can fail for a defect unrelated to the
|
||||||
change that triggered it. That is the suite doing its job; take the seed from
|
change that triggered it. That is the suite doing its job; take the seed from
|
||||||
|
|
@ -83,6 +86,10 @@ Determinism requires every source of variation to come from the seed.
|
||||||
node UUID, so an unpinned node id makes concurrent merge winners
|
node UUID, so an unpinned node id makes concurrent merge winners
|
||||||
nondeterministic.
|
nondeterministic.
|
||||||
|
|
||||||
|
Node UUIDs are assigned to replicas through a seeded permutation, independently
|
||||||
|
of the ordered clock offsets. Seeds can therefore explore both directions of
|
||||||
|
the HLC node tie-break while replaying the same assignment exactly.
|
||||||
|
|
||||||
Node identity is currently pinned by pre-creating the `CrdtNode` row and
|
Node identity is currently pinned by pre-creating the `CrdtNode` row and
|
||||||
attaching it to every space, which relies on how `OfflineSyncSpaceManager` resolves
|
attaching it to every space, which relies on how `OfflineSyncSpaceManager` resolves
|
||||||
the current node. `DstReplica._assertSeededNodeIdentity` fails loudly if that
|
the current node. `DstReplica._assertSeededNodeIdentity` fails loudly if that
|
||||||
|
|
@ -91,8 +98,9 @@ injectable node id on the engine would remove the need for the trick.
|
||||||
|
|
||||||
## What the adversary does
|
## What the adversary does
|
||||||
|
|
||||||
Reorders, delays, redelivers already-merged batches, and partitions replicas for
|
Reorders, delays, redelivers already-merged batches, and isolates incoming
|
||||||
a few rounds.
|
delivery to selected replicas for a few rounds. Sources can still send during
|
||||||
|
this receive isolation; this is not a bidirectional network partition.
|
||||||
|
|
||||||
Two moves are deliberately unavailable, both because the merge contract states
|
Two moves are deliberately unavailable, both because the merge contract states
|
||||||
that input arrives as a causally complete snapshot of the sender
|
that input arrives as a causally complete snapshot of the sender
|
||||||
|
|
@ -112,6 +120,10 @@ that input arrives as a causally complete snapshot of the sender
|
||||||
Delay, reorder, and redelivery are the honest moves - and redelivery is how
|
Delay, reorder, and redelivery are the honest moves - and redelivery is how
|
||||||
idempotence gets probed.
|
idempotence gets probed.
|
||||||
|
|
||||||
|
When the schedule ends, quiescence drains every pending batch and collects only
|
||||||
|
facts a receiver has not yet seen. It stops introducing deliberate duplicates,
|
||||||
|
so stable replicas cannot exhaust the convergence limit merely through resends.
|
||||||
|
|
||||||
## Layout
|
## Layout
|
||||||
|
|
||||||
| File | Role |
|
| File | Role |
|
||||||
|
|
@ -120,9 +132,18 @@ idempotence gets probed.
|
||||||
| `framework/dst_world.dart` | Replicas and operation generation |
|
| `framework/dst_world.dart` | Replicas and operation generation |
|
||||||
| `framework/dst_schema.dart` | Generated model adapters and declared unique indexes |
|
| `framework/dst_schema.dart` | Generated model adapters and declared unique indexes |
|
||||||
| `framework/dst_adversary.dart` | Delivery scheduling and quiescence |
|
| `framework/dst_adversary.dart` | Delivery scheduling and quiescence |
|
||||||
| `framework/dst_snapshot.dart` | Canonical snapshots and the property oracle |
|
| `framework/dst_snapshot.dart` | Portable domain/authored/projection snapshots and structural oracle |
|
||||||
|
| `framework/dst_authored.dart` | Pre-write evidence, primary visibility intents, and accepted-fact retention |
|
||||||
|
| `framework/dst_rejection.dart` | Concrete refusal prediction and accepted-error matching |
|
||||||
|
| `framework/dst_workload.dart` | Populated per-space graphs and deterministic semantic transitions |
|
||||||
|
| `framework/dst_coverage.dart` | Deduplicated authored transitions and observed graph shapes |
|
||||||
| `framework/dst_runner.dart` | One seeded run, and failure reporting |
|
| `framework/dst_runner.dart` | One seeded run, and failure reporting |
|
||||||
|
|
||||||
|
Every generated synchronized table participates, including `Types` with boolean,
|
||||||
|
UTC timestamp, text, integer, BigInt, finite real, binary, enum, and nullable
|
||||||
|
values. Untagged schema guards fail if a new table, FK target column, or delete
|
||||||
|
action needs generator or refusal-predictor support.
|
||||||
|
|
||||||
The simulated graph includes nullable and required cascade, no-action,
|
The simulated graph includes nullable and required cascade, no-action,
|
||||||
set-null, and set-default references, the mixed cascade/no-action chains, and
|
set-null, and set-default references, the mixed cascade/no-action chains, and
|
||||||
all outbound references of person and organization. Person, company, and town
|
all outbound references of person and organization. Person, company, and town
|
||||||
|
|
@ -154,6 +175,14 @@ composite, fixed-discriminator, overlapping, FK-only composite, and space-scoped
|
||||||
FK/text claims. The unique oracle reads all declared tuple components and space.
|
FK/text claims. The unique oracle reads all declared tuple components and space.
|
||||||
A null component releases the tuple, as in SQL.
|
A null component releases the tuple, as in SQL.
|
||||||
|
|
||||||
|
Locally observable unique collisions are expected refusals. The predictor uses
|
||||||
|
the pre-operation visible rows and submitted values, including composite tuples
|
||||||
|
and space partitions. It matches the violated index and checks that domain data,
|
||||||
|
authored metadata, and sync progress all roll back. Unexpected constraint errors
|
||||||
|
still fail the run. Conflict-resolution scenarios author competing claims on
|
||||||
|
disconnected replicas and merge them, so only independently accepted claims
|
||||||
|
enter arbitration.
|
||||||
|
|
||||||
The operation generator can insert, update, delete, restore a retained identity,
|
The operation generator can insert, update, delete, restore a retained identity,
|
||||||
upsert, pass a full model back through update, insert/update/delete a batch,
|
upsert, pass a full model back through update, insert/update/delete a batch,
|
||||||
swap unique tuples atomically, and perform predicate updates/deletes. Scripted regressions
|
swap unique tuples atomically, and perform predicate updates/deletes. Scripted regressions
|
||||||
|
|
@ -161,3 +190,49 @@ use `DstOperations.apply` to select the same paths directly. Reports retain
|
||||||
attempted and committed counts by table/action so a passing run does not hide
|
attempted and committed counts by table/action so a passing run does not hide
|
||||||
which paths it visited. Rejected local transactions remain separate from
|
which paths it visited. Rejected local transactions remain separate from
|
||||||
committed operations.
|
committed operations.
|
||||||
|
|
||||||
|
## Workload strength and metrics
|
||||||
|
|
||||||
|
The populated profile creates every declared table and FK edge, closes the
|
||||||
|
person/company/town cycles, and drives unique conflicts, a real tuple exchange,
|
||||||
|
restore/redelete, FK retarget/detach, and a constrained refusal. These are actual
|
||||||
|
ORM transactions, each checked immediately for structure, authored preservation,
|
||||||
|
and causal monotonicity. Complete exports distribute the populated spaces before
|
||||||
|
random scheduling starts. The sparse profile retains empty-world exploration.
|
||||||
|
|
||||||
|
Every run emits one `DST_METRICS` JSON record, including failed runs. It separates
|
||||||
|
setup attempts from scheduled commits and records attempted, committed, rejected,
|
||||||
|
skipped, unexpected failures, and committed transactions whose oracle failed.
|
||||||
|
A run failing during setup reports zero scheduled activity. Paths retain table
|
||||||
|
and action; network observations include merge counts, duplicate batches, maximum
|
||||||
|
batch size, total delivered changes, and receive-isolation events.
|
||||||
|
|
||||||
|
Coverage counts distinct field/tombstone events rather than repeated snapshot
|
||||||
|
appearances. FK edges and cycles are **authored graph** observations. Unique
|
||||||
|
projection coverage reports columns, not a claim that every overlapping index
|
||||||
|
conflicted. A swap counts only a validated exchange of two different tuples.
|
||||||
|
The counters do not claim every FK transition or every unique shape was stressed.
|
||||||
|
|
||||||
|
Passing simulations require scheduled commits and merges. At 100+ rounds the
|
||||||
|
minimum is 30 scheduled commits; shorter runs are smoke checks. Populated runs
|
||||||
|
also require all declared authored FK edges and the mandatory semantic paths.
|
||||||
|
Setup commits alone cannot satisfy the scheduled-activity gate.
|
||||||
|
|
||||||
|
CI alternates sparse and populated profiles across four consecutive seeds at
|
||||||
|
200 rounds in both topologies: 4,800 scheduled operation attempts, versus the
|
||||||
|
previous 6,000 attempts spread across fifty shallow 20-round worlds. This keeps
|
||||||
|
comparable attempt volume while exploring ten times the history depth and
|
||||||
|
connected graphs. Each topology runs in its own sixty-minute CI job with four
|
||||||
|
ten-minute per-seed ceilings; a failure in one job does not cancel the other.
|
||||||
|
Both jobs also run the ownership-collision controls. The local `melos test-dst`
|
||||||
|
command runs both topologies together, so it covers the combined CI matrix.
|
||||||
|
Known engine failures remain failures; CI is intentionally not a PR merge gate.
|
||||||
|
|
||||||
|
```sh
|
||||||
|
DST_SEED_BASE=114 DST_SEEDS=1 DST_ROUNDS=200 DST_PROFILE=populated DST_GRAPH_WIDTH=2 dart test -P dst
|
||||||
|
DST_SEED_BASE=62 DST_SEEDS=1 DST_ROUNDS=200 DST_PROFILE=populated DST_GRAPH_WIDTH=3 dart test -P dst
|
||||||
|
```
|
||||||
|
|
||||||
|
The oracle remains bounded: it does not independently arbitrate all unique
|
||||||
|
winners or FK fixed points. Collector concurrency/checkpoint lifecycle, crash
|
||||||
|
recovery, space grant/revoke, and transport framing remain outside this schedule.
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,75 @@
|
||||||
|
import 'package:serverpod_offline_sync_test_client/serverpod_offline_sync_test_client.dart';
|
||||||
|
import 'package:test/test.dart';
|
||||||
|
|
||||||
|
import '../integration/test_tools/client_session.dart';
|
||||||
|
import 'framework/dst_adversary.dart';
|
||||||
|
import 'framework/dst_random.dart';
|
||||||
|
import 'framework/dst_snapshot.dart';
|
||||||
|
import 'framework/dst_world.dart';
|
||||||
|
|
||||||
|
void main() {
|
||||||
|
initTestClientSession(createSessionPerTest: false);
|
||||||
|
|
||||||
|
test(
|
||||||
|
'Given two replicas and a schedule repeatedly redelivering one city insertion, '
|
||||||
|
'when the network drains after those duplicate deliveries, '
|
||||||
|
'then stable authored facts converge without a false non-idempotence failure.',
|
||||||
|
() async {
|
||||||
|
final ids = DstIds(DstRandom(916));
|
||||||
|
final space = ids.next();
|
||||||
|
final clock = DstClock();
|
||||||
|
final replicas = <DstReplica>[];
|
||||||
|
for (var index = 0; index < 2; index++) {
|
||||||
|
replicas.add(
|
||||||
|
await DstReplica.create(
|
||||||
|
name: 'r$index',
|
||||||
|
spaceUuids: [space],
|
||||||
|
nodeUuid: ids.next(),
|
||||||
|
clock: clock.clock,
|
||||||
|
),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
final author = replicas.first;
|
||||||
|
await author.withReplicaClock(
|
||||||
|
() => author.session.db.transactionForUser(
|
||||||
|
space,
|
||||||
|
(tx) => City.db.insertRow(
|
||||||
|
author.session,
|
||||||
|
City(id: ids.next(), name: 'city'),
|
||||||
|
transaction: tx,
|
||||||
|
),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
final expected = (await DstSnapshot.capture(author)).renderSpace(space);
|
||||||
|
final adversary = DstAdversary(_RepeatedDeliverySchedule(), replicas);
|
||||||
|
Future<void> check(DstReplica replica) async {
|
||||||
|
expect(DstOracle.invariants(await DstSnapshot.capture(replica)), isEmpty);
|
||||||
|
}
|
||||||
|
|
||||||
|
await adversary.step(check);
|
||||||
|
await adversary.step(check);
|
||||||
|
expect(adversary.duplicateBatches, greaterThan(0));
|
||||||
|
await adversary.quiesce(check);
|
||||||
|
|
||||||
|
for (final replica in replicas) {
|
||||||
|
expect((await DstSnapshot.capture(replica)).renderSpace(space), expected);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A concrete network schedule: always collect the first replica, always
|
||||||
|
/// choose resend, and never isolate delivery. Database/merge paths stay real.
|
||||||
|
class _RepeatedDeliverySchedule extends DstRandom {
|
||||||
|
_RepeatedDeliverySchedule() : super(916);
|
||||||
|
|
||||||
|
@override
|
||||||
|
bool chance(double probability) => switch (probability) {
|
||||||
|
DstAdversary.receiveIsolationProbability => false,
|
||||||
|
DstAdversary.collectionProbability || DstAdversary.resendProbability => true,
|
||||||
|
_ => throw StateError('Unspecified network decision with probability $probability'),
|
||||||
|
};
|
||||||
|
|
||||||
|
@override
|
||||||
|
T pick<T>(List<T> items) => items.first;
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,610 @@
|
||||||
|
import 'package:serverpod_offline_sync_server/serverpod_offline_sync_server.dart';
|
||||||
|
import 'package:serverpod_offline_sync_test_client/serverpod_offline_sync_test_client.dart';
|
||||||
|
import 'package:test/test.dart';
|
||||||
|
|
||||||
|
import '../integration/test_tools/client_session.dart';
|
||||||
|
import 'framework/dst_authored.dart';
|
||||||
|
import 'framework/dst_random.dart';
|
||||||
|
import 'framework/dst_snapshot.dart';
|
||||||
|
import 'framework/dst_world.dart';
|
||||||
|
|
||||||
|
void main() {
|
||||||
|
initTestClientSession(createSessionPerTest: false);
|
||||||
|
|
||||||
|
test(
|
||||||
|
'Given two accepted unique rows claiming the same name, '
|
||||||
|
'when the preserved losing claim is removed from storage, '
|
||||||
|
'then the independent authoring evidence and portable comparison detect the loss.',
|
||||||
|
() async {
|
||||||
|
final ids = DstIds(DstRandom(22));
|
||||||
|
final space = ids.next();
|
||||||
|
final replica = await _replica(ids, space);
|
||||||
|
final first = Unique(id: ids.next(), name: 'shared');
|
||||||
|
final second = Unique(id: ids.next(), name: 'shared');
|
||||||
|
final evidence = DstWriteEvidence(await DstSnapshot.capture(replica))
|
||||||
|
..write('unique', first.toJson())
|
||||||
|
..write('unique', second.toJson());
|
||||||
|
await replica.withReplicaClock(
|
||||||
|
() => replica.session.db.transactionForUser(
|
||||||
|
space,
|
||||||
|
(tx) => Unique.db.insertRow(replica.session, first, transaction: tx),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
final peer = await _replica(ids, space);
|
||||||
|
await peer.withReplicaClock(
|
||||||
|
() => peer.session.db.transactionForUser(
|
||||||
|
space,
|
||||||
|
(tx) => Unique.db.insertRow(peer.session, second, transaction: tx),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
await replica.merge(await peer.collect(space), space);
|
||||||
|
final accepted = await DstSnapshot.capture(replica);
|
||||||
|
expect(evidence.validate(accepted), isEmpty);
|
||||||
|
final oracle = DstAuthoredOracle()..accept(accepted);
|
||||||
|
expect(oracle.validate(accepted, space), isEmpty);
|
||||||
|
final attempts = await CrdtDataAttemptedValue.db.find(replica.rawSession);
|
||||||
|
expect(attempts, hasLength(1));
|
||||||
|
|
||||||
|
// Deliberate oracle mutation after a real supported write: this is a
|
||||||
|
// detector regression, not an allegation that production deletes it.
|
||||||
|
await CrdtDataAttemptedValue.db.delete(replica.rawSession, attempts);
|
||||||
|
final damaged = await DstSnapshot.capture(replica);
|
||||||
|
|
||||||
|
expect(
|
||||||
|
evidence.validate(damaged).map((v) => v.property),
|
||||||
|
contains('acceptedWrite'),
|
||||||
|
);
|
||||||
|
expect(
|
||||||
|
oracle.validate(damaged, space).map((v) => v.property),
|
||||||
|
contains('authoredRetention'),
|
||||||
|
);
|
||||||
|
expect(damaged.renderSpace(space), isNot(accepted.renderSpace(space)));
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
test(
|
||||||
|
'Given a live city with no authored deletion or outbound references, '
|
||||||
|
'when an oracle input falsely hides that city, '
|
||||||
|
'then the independent visibility rule rejects the snapshot.',
|
||||||
|
() async {
|
||||||
|
final ids = DstIds(DstRandom(23));
|
||||||
|
final space = ids.next();
|
||||||
|
final replica = await _replica(ids, space);
|
||||||
|
final city = City(id: ids.next(), name: 'live');
|
||||||
|
await replica.withReplicaClock(
|
||||||
|
() => replica.session.db.transactionForUser(
|
||||||
|
space,
|
||||||
|
(tx) => City.db.insertRow(replica.session, city, transaction: tx),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
final before = await DstSnapshot.capture(replica);
|
||||||
|
final row = before.rows['city']![city.id]!;
|
||||||
|
|
||||||
|
final damaged = _copy(
|
||||||
|
before,
|
||||||
|
rows: {
|
||||||
|
...before.rows,
|
||||||
|
'city': {city.id!: (spaceUuid: space, columns: row.columns, visible: false)},
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(DstOracle.invariants(before), isEmpty);
|
||||||
|
expect(DstOracle.invariants(damaged).map((v) => v.property), [
|
||||||
|
'legitimateVisibility',
|
||||||
|
]);
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
test(
|
||||||
|
'Given an acknowledged city and a later corrupted field clock, '
|
||||||
|
'when an unrelated unique insert is acknowledged, '
|
||||||
|
'then the oracle still rejects the preexisting corruption.',
|
||||||
|
() async {
|
||||||
|
final ids = DstIds(DstRandom(26));
|
||||||
|
final space = ids.next();
|
||||||
|
final replica = await _replica(ids, space);
|
||||||
|
final city = City(id: ids.next(), name: 'accepted');
|
||||||
|
await replica.withReplicaClock(
|
||||||
|
() => replica.session.db.transactionForUser(
|
||||||
|
space,
|
||||||
|
(tx) => City.db.insertRow(replica.session, city, transaction: tx),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
final accepted = await DstSnapshot.capture(replica);
|
||||||
|
final oracle = DstAuthoredOracle()..accept(accepted);
|
||||||
|
final key = ('city', city.id!, 'name');
|
||||||
|
final fabricatedClock = accepted.fieldHlc(key)!.increment();
|
||||||
|
final beforeUnrelated = _copy(accepted, fieldHlcs: {key: fabricatedClock});
|
||||||
|
final unique = Unique(id: ids.next(), name: 'unrelated');
|
||||||
|
|
||||||
|
await replica.withReplicaClock(
|
||||||
|
() => replica.session.db.transactionForUser(
|
||||||
|
space,
|
||||||
|
(tx) => Unique.db.insertRow(replica.session, unique, transaction: tx),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
final afterUnrelated = _copy(
|
||||||
|
await DstSnapshot.capture(replica),
|
||||||
|
fieldHlcs: {key: fabricatedClock},
|
||||||
|
);
|
||||||
|
oracle.accept(afterUnrelated, before: beforeUnrelated);
|
||||||
|
|
||||||
|
expect(oracle.validate(afterUnrelated, space).map((v) => v.property), [
|
||||||
|
'authoredRetention',
|
||||||
|
]);
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
test(
|
||||||
|
'Given a nullable FK with a persisted database default, '
|
||||||
|
'when the raw ORM inserts an omitted value and explicitly updates it to null, '
|
||||||
|
'then insertion uses the declared default and the narrowed update retains null.',
|
||||||
|
() async {
|
||||||
|
final ids = DstIds(DstRandom(27));
|
||||||
|
final space = ids.next();
|
||||||
|
final replica = await _replica(ids, space);
|
||||||
|
await replica.seedDefaultTown(space);
|
||||||
|
final child = UniqueSetDefaultChild(id: ids.next(), name: 'defaulted');
|
||||||
|
final inserted = await UniqueSetDefaultChild.db.insertRow(
|
||||||
|
replica.rawSession,
|
||||||
|
child,
|
||||||
|
);
|
||||||
|
|
||||||
|
await UniqueSetDefaultChild.db.updateRow(
|
||||||
|
replica.rawSession,
|
||||||
|
inserted.copyWith(parentId: null),
|
||||||
|
columns: (t) => [t.parentId],
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(inserted.parentId, dstDefaultTownId);
|
||||||
|
expect(
|
||||||
|
(await UniqueSetDefaultChild.db.findById(
|
||||||
|
replica.rawSession,
|
||||||
|
child.id!,
|
||||||
|
))!.parentId,
|
||||||
|
isNull,
|
||||||
|
);
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
test(
|
||||||
|
'Given a CRDT child inserted using its persisted FK default, '
|
||||||
|
'when an explicit narrowed update clears its reference, '
|
||||||
|
'then independent evidence records the default followed by authored null.',
|
||||||
|
() async {
|
||||||
|
final ids = DstIds(DstRandom(28));
|
||||||
|
final space = ids.next();
|
||||||
|
final replica = await _replica(ids, space);
|
||||||
|
await replica.seedDefaultTown(space);
|
||||||
|
final child = UniqueSetDefaultChild(id: ids.next(), name: 'defaulted');
|
||||||
|
final insertion = DstWriteEvidence(await DstSnapshot.capture(replica))
|
||||||
|
..write('unique_set_default_child', child.toJson(), insertDefaults: true);
|
||||||
|
await replica.withReplicaClock(
|
||||||
|
() => replica.session.db.transactionForUser(
|
||||||
|
space,
|
||||||
|
(tx) => UniqueSetDefaultChild.db.insertRow(
|
||||||
|
replica.session,
|
||||||
|
child,
|
||||||
|
transaction: tx,
|
||||||
|
),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
final beforeUpdate = await DstSnapshot.capture(replica);
|
||||||
|
final update = DstWriteEvidence(beforeUpdate)
|
||||||
|
..write(
|
||||||
|
'unique_set_default_child',
|
||||||
|
child.copyWith(parentId: null).toJson(),
|
||||||
|
columns: {'parentId'},
|
||||||
|
);
|
||||||
|
|
||||||
|
await replica.withReplicaClock(
|
||||||
|
() => replica.session.db.transactionForUser(
|
||||||
|
space,
|
||||||
|
(tx) => UniqueSetDefaultChild.db.updateRow(
|
||||||
|
replica.session,
|
||||||
|
child.copyWith(parentId: null),
|
||||||
|
columns: (t) => [t.parentId],
|
||||||
|
transaction: tx,
|
||||||
|
),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
final after = await DstSnapshot.capture(replica);
|
||||||
|
|
||||||
|
expect(insertion.validate(beforeUpdate), isEmpty);
|
||||||
|
expect(update.validate(after), isEmpty);
|
||||||
|
expect(
|
||||||
|
after.authoredValue(('unique_set_default_child', child.id!, 'parentId')),
|
||||||
|
isNull,
|
||||||
|
);
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
for (final action in ['restore', 'hidden upsert', 'visible upsert']) {
|
||||||
|
test(
|
||||||
|
'Given a nullable defaulted FK explicitly cleared before $action, '
|
||||||
|
'when that existing identity is written with a null model value, '
|
||||||
|
'then authoring evidence follows the existing-row default contract.',
|
||||||
|
() async {
|
||||||
|
final ids = DstIds(DstRandom(29));
|
||||||
|
final space = ids.next();
|
||||||
|
final replica = await _replica(ids, space);
|
||||||
|
await replica.seedDefaultTown(space);
|
||||||
|
final child = UniqueSetDefaultChild(id: ids.next(), name: 'defaulted');
|
||||||
|
await replica.withReplicaClock(
|
||||||
|
() => replica.session.db.transactionForUser(space, (tx) async {
|
||||||
|
await UniqueSetDefaultChild.db.insertRow(
|
||||||
|
replica.session,
|
||||||
|
child,
|
||||||
|
transaction: tx,
|
||||||
|
);
|
||||||
|
await UniqueSetDefaultChild.db.updateRow(
|
||||||
|
replica.session,
|
||||||
|
child.copyWith(parentId: null),
|
||||||
|
columns: (t) => [t.parentId],
|
||||||
|
transaction: tx,
|
||||||
|
);
|
||||||
|
if (action != 'visible upsert') {
|
||||||
|
await UniqueSetDefaultChild.db.deleteRow(
|
||||||
|
replica.session,
|
||||||
|
child,
|
||||||
|
transaction: tx,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
final before = await DstSnapshot.capture(replica);
|
||||||
|
final evidence = DstWriteEvidence(before)
|
||||||
|
..write(
|
||||||
|
'unique_set_default_child',
|
||||||
|
child.toJson(),
|
||||||
|
insertDefaults: action == 'visible upsert',
|
||||||
|
);
|
||||||
|
|
||||||
|
await replica.withReplicaClock(
|
||||||
|
() => replica.session.db.transactionForUser(space, (tx) async {
|
||||||
|
if (action == 'restore') {
|
||||||
|
await UniqueSetDefaultChild.db.insertRow(
|
||||||
|
replica.session,
|
||||||
|
child,
|
||||||
|
transaction: tx,
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
await replica.session.db.upsertRow(
|
||||||
|
child,
|
||||||
|
conflictColumns: [UniqueSetDefaultChild.t.id],
|
||||||
|
transaction: tx,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
final after = await DstSnapshot.capture(replica);
|
||||||
|
|
||||||
|
expect(evidence.validate(after), isEmpty);
|
||||||
|
expect(
|
||||||
|
after.authoredValue(('unique_set_default_child', child.id!, 'parentId')),
|
||||||
|
action == 'visible upsert' ? dstDefaultTownId.toString() : null,
|
||||||
|
);
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
test(
|
||||||
|
'Given a city restored after its original insertion reached another replica, '
|
||||||
|
'when the restored facts reach that replica and an empty bootstrap, '
|
||||||
|
'then different raw row anchors preserve identical canonical fields and tombstones.',
|
||||||
|
() async {
|
||||||
|
final ids = DstIds(DstRandom(30));
|
||||||
|
final space = ids.next();
|
||||||
|
final source = await _replica(ids, space);
|
||||||
|
final receiver = await _replica(ids, space);
|
||||||
|
final mirror = await _replica(ids, space);
|
||||||
|
final city = City(id: ids.next(), name: 'original');
|
||||||
|
await source.withReplicaClock(
|
||||||
|
() => source.session.db.transactionForUser(
|
||||||
|
space,
|
||||||
|
(tx) => City.db.insertRow(source.session, city, transaction: tx),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
await receiver.merge(await source.collect(space), space);
|
||||||
|
await source.withReplicaClock(
|
||||||
|
() => source.session.db.transactionForUser(
|
||||||
|
space,
|
||||||
|
(tx) => City.db.deleteRow(source.session, city, transaction: tx),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
await source.withReplicaClock(
|
||||||
|
() => source.session.db.transactionForUser(
|
||||||
|
space,
|
||||||
|
(tx) => City.db.insertRow(
|
||||||
|
source.session,
|
||||||
|
city.copyWith(name: 'restored'),
|
||||||
|
transaction: tx,
|
||||||
|
),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
final accepted = await DstSnapshot.capture(source);
|
||||||
|
final oracle = DstAuthoredOracle()..accept(accepted);
|
||||||
|
|
||||||
|
await receiver.merge(await source.collect(space), space);
|
||||||
|
await mirror.merge(await receiver.collect(space), space);
|
||||||
|
final received = await DstSnapshot.capture(receiver);
|
||||||
|
final bootstrapped = await DstSnapshot.capture(mirror);
|
||||||
|
final key = 'city/${city.id}';
|
||||||
|
|
||||||
|
expect(received.rowHlcs[key], isNot(accepted.rowHlcs[key]));
|
||||||
|
expect(received.renderRawMetadata(), isNot(accepted.renderRawMetadata()));
|
||||||
|
expect(received.tombstones[key], accepted.tombstones[key]);
|
||||||
|
expect(received.tombstones[key]!.clFlag, 3);
|
||||||
|
expect(received.renderSpace(space), accepted.renderSpace(space));
|
||||||
|
expect(bootstrapped.renderSpace(space), accepted.renderSpace(space));
|
||||||
|
expect(oracle.validate(received, space), isEmpty);
|
||||||
|
expect(oracle.validate(bootstrapped, space), isEmpty);
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
test(
|
||||||
|
'Given independently upserted visible cities sharing one identity, '
|
||||||
|
'when the newer insertion reaches an existing replica and an empty bootstrap, '
|
||||||
|
'then its redundant generation-one insertion marker has the same canonical facts.',
|
||||||
|
() async {
|
||||||
|
final ids = DstIds(DstRandom(31));
|
||||||
|
final space = ids.next();
|
||||||
|
final clock = DstClock();
|
||||||
|
final receiver = await DstReplica.create(
|
||||||
|
name: 'receiver',
|
||||||
|
spaceUuids: [space],
|
||||||
|
nodeUuid: ids.next(),
|
||||||
|
clock: clock.clock,
|
||||||
|
);
|
||||||
|
final source = await DstReplica.create(
|
||||||
|
name: 'source',
|
||||||
|
spaceUuids: [space],
|
||||||
|
nodeUuid: ids.next(),
|
||||||
|
clock: clock.clock,
|
||||||
|
);
|
||||||
|
final mirror = await DstReplica.create(
|
||||||
|
name: 'mirror',
|
||||||
|
spaceUuids: [space],
|
||||||
|
nodeUuid: ids.next(),
|
||||||
|
clock: clock.clock,
|
||||||
|
);
|
||||||
|
final city = City(id: ids.next(), name: 'original');
|
||||||
|
await receiver.withReplicaClock(
|
||||||
|
() => receiver.session.db.transactionForUser(
|
||||||
|
space,
|
||||||
|
(tx) => receiver.session.db.upsertRow(
|
||||||
|
city,
|
||||||
|
conflictColumns: [City.t.id],
|
||||||
|
transaction: tx,
|
||||||
|
),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
clock.advance(const Duration(milliseconds: 1));
|
||||||
|
await source.withReplicaClock(
|
||||||
|
() => source.session.db.transactionForUser(
|
||||||
|
space,
|
||||||
|
(tx) => source.session.db.upsertRow(
|
||||||
|
city.copyWith(name: 'newer'),
|
||||||
|
conflictColumns: [City.t.id],
|
||||||
|
transaction: tx,
|
||||||
|
),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
final accepted = await DstSnapshot.capture(source);
|
||||||
|
final oracle = DstAuthoredOracle()..accept(accepted);
|
||||||
|
|
||||||
|
await receiver.merge(await source.collect(space), space);
|
||||||
|
await mirror.merge(await receiver.collect(space), space);
|
||||||
|
final received = await DstSnapshot.capture(receiver);
|
||||||
|
final bootstrapped = await DstSnapshot.capture(mirror);
|
||||||
|
final key = 'city/${city.id}';
|
||||||
|
|
||||||
|
expect(accepted.tombstones[key], isNull);
|
||||||
|
expect(received.tombstones[key]!.clFlag, 1);
|
||||||
|
expect(received.tombstones[key]!.reason, CrdtDataDeletedReason.userInsert);
|
||||||
|
expect(received.rowHlcs[key], isNot(accepted.rowHlcs[key]));
|
||||||
|
expect(received.renderRawMetadata(), isNot(accepted.renderRawMetadata()));
|
||||||
|
expect(received.renderSpace(space), accepted.renderSpace(space));
|
||||||
|
expect(bootstrapped.renderSpace(space), accepted.renderSpace(space));
|
||||||
|
expect(oracle.validate(received, space), isEmpty);
|
||||||
|
expect(oracle.validate(bootstrapped, space), isEmpty);
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
test(
|
||||||
|
'Given an accepted live city without a delete event, '
|
||||||
|
'when a fabricated delete tombstone appears in the oracle input, '
|
||||||
|
'then the oracle rejects the unacknowledged visibility change.',
|
||||||
|
() async {
|
||||||
|
final ids = DstIds(DstRandom(32));
|
||||||
|
final space = ids.next();
|
||||||
|
final replica = await _replica(ids, space);
|
||||||
|
final city = City(id: ids.next(), name: 'live');
|
||||||
|
await replica.withReplicaClock(
|
||||||
|
() => replica.session.db.transactionForUser(
|
||||||
|
space,
|
||||||
|
(tx) => City.db.insertRow(replica.session, city, transaction: tx),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
final accepted = await DstSnapshot.capture(replica);
|
||||||
|
final oracle = DstAuthoredOracle()..accept(accepted);
|
||||||
|
final key = 'city/${city.id}';
|
||||||
|
|
||||||
|
final damaged = _copy(
|
||||||
|
accepted,
|
||||||
|
tombstones: {
|
||||||
|
key: (
|
||||||
|
hlc: accepted.rowHlcs[key]!.increment(),
|
||||||
|
clFlag: 2,
|
||||||
|
reason: CrdtDataDeletedReason.userDelete,
|
||||||
|
),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(oracle.validate(damaged, space).map((v) => v.property), [
|
||||||
|
'authoredRetention',
|
||||||
|
]);
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
test(
|
||||||
|
'Given an accepted live city without a newer insertion event, '
|
||||||
|
'when an insertion marker newer than every field clock appears in the oracle input, '
|
||||||
|
'then the oracle rejects the unacknowledged insertion marker.',
|
||||||
|
() async {
|
||||||
|
final ids = DstIds(DstRandom(32));
|
||||||
|
final space = ids.next();
|
||||||
|
final replica = await _replica(ids, space);
|
||||||
|
final city = City(id: ids.next(), name: 'live');
|
||||||
|
await replica.withReplicaClock(
|
||||||
|
() => replica.session.db.transactionForUser(
|
||||||
|
space,
|
||||||
|
(tx) => City.db.insertRow(replica.session, city, transaction: tx),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
final accepted = await DstSnapshot.capture(replica);
|
||||||
|
final oracle = DstAuthoredOracle()..accept(accepted);
|
||||||
|
final key = 'city/${city.id}';
|
||||||
|
|
||||||
|
final damaged = _copy(
|
||||||
|
accepted,
|
||||||
|
tombstones: {
|
||||||
|
key: (
|
||||||
|
hlc: accepted.rowHlcs[key]!.increment(),
|
||||||
|
clFlag: 1,
|
||||||
|
reason: CrdtDataDeletedReason.userInsert,
|
||||||
|
),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(oracle.validate(damaged, space).map((v) => v.property), [
|
||||||
|
'authoredRetention',
|
||||||
|
]);
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
for (final metadata in [
|
||||||
|
'field clock',
|
||||||
|
'inherited field clock',
|
||||||
|
'tombstone clock',
|
||||||
|
'tombstone reason',
|
||||||
|
'projection reason',
|
||||||
|
]) {
|
||||||
|
test(
|
||||||
|
'Given equal-looking snapshots of a deleted unique row, '
|
||||||
|
'when only its $metadata diverges, '
|
||||||
|
'then the portable comparison detects the different metadata.',
|
||||||
|
() async {
|
||||||
|
final ids = DstIds(DstRandom(24));
|
||||||
|
final space = ids.next();
|
||||||
|
final replica = await _replica(ids, space);
|
||||||
|
final row = Unique(id: ids.next(), name: 'retained');
|
||||||
|
await replica.withReplicaClock(
|
||||||
|
() => replica.session.db.transactionForUser(
|
||||||
|
space,
|
||||||
|
(tx) => Unique.db.insertRow(replica.session, row, transaction: tx),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
await replica.withReplicaClock(
|
||||||
|
() => replica.session.db.transactionForUser(
|
||||||
|
space,
|
||||||
|
(tx) => Unique.db.deleteRow(replica.session, row, transaction: tx),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
final before = await DstSnapshot.capture(replica);
|
||||||
|
final key = ('unique', row.id!, 'name');
|
||||||
|
final rowKey = 'unique/${row.id}';
|
||||||
|
final tombstone = before.tombstones[rowKey]!;
|
||||||
|
final projection = before.projections[key]!;
|
||||||
|
|
||||||
|
final damaged = _copy(
|
||||||
|
before,
|
||||||
|
fieldHlcs: metadata == 'field clock'
|
||||||
|
? {...before.fieldHlcs, key: before.fieldHlc(key)!.increment()}
|
||||||
|
: metadata == 'inherited field clock'
|
||||||
|
? (Map.of(before.fieldHlcs)..remove(key))
|
||||||
|
: null,
|
||||||
|
rowHlcs: metadata == 'inherited field clock'
|
||||||
|
? {...before.rowHlcs, rowKey: before.rowHlcs[rowKey]!.increment()}
|
||||||
|
: null,
|
||||||
|
tombstones: metadata.startsWith('tombstone')
|
||||||
|
? {
|
||||||
|
...before.tombstones,
|
||||||
|
rowKey: (
|
||||||
|
hlc: metadata == 'tombstone clock'
|
||||||
|
? tombstone.hlc.increment()
|
||||||
|
: tombstone.hlc,
|
||||||
|
clFlag: tombstone.clFlag,
|
||||||
|
reason: metadata == 'tombstone reason'
|
||||||
|
? CrdtDataDeletedReason.userCascadeDelete
|
||||||
|
: tombstone.reason,
|
||||||
|
),
|
||||||
|
}
|
||||||
|
: null,
|
||||||
|
projections: metadata == 'projection reason'
|
||||||
|
? {
|
||||||
|
...before.projections,
|
||||||
|
key: (
|
||||||
|
attemptedValue: projection.attemptedValue,
|
||||||
|
projectionReason: CrdtProjectionReason.uniqueConflict,
|
||||||
|
),
|
||||||
|
}
|
||||||
|
: null,
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(damaged.rows, before.rows);
|
||||||
|
expect(damaged.renderSpace(space), isNot(before.renderSpace(space)));
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
test(
|
||||||
|
'Given a field inheriting its insertion clock, '
|
||||||
|
'when an equal explicit field clock is added to the snapshot, '
|
||||||
|
'then portable equality accepts the equivalent sparse representation.',
|
||||||
|
() async {
|
||||||
|
final ids = DstIds(DstRandom(25));
|
||||||
|
final space = ids.next();
|
||||||
|
final replica = await _replica(ids, space);
|
||||||
|
final city = City(id: ids.next(), name: 'live');
|
||||||
|
await replica.withReplicaClock(
|
||||||
|
() => replica.session.db.transactionForUser(
|
||||||
|
space,
|
||||||
|
(tx) => City.db.insertRow(replica.session, city, transaction: tx),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
final before = await DstSnapshot.capture(replica);
|
||||||
|
final key = ('city', city.id!, 'name');
|
||||||
|
|
||||||
|
final equivalent = _copy(
|
||||||
|
before,
|
||||||
|
fieldHlcs: {...before.fieldHlcs, key: before.fieldHlc(key)!},
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(equivalent.renderSpace(space), before.renderSpace(space));
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
Future<DstReplica> _replica(DstIds ids, UuidValue space) => DstReplica.create(
|
||||||
|
name: 'replica',
|
||||||
|
spaceUuids: [space],
|
||||||
|
nodeUuid: ids.next(),
|
||||||
|
clock: DstClock().clock,
|
||||||
|
);
|
||||||
|
|
||||||
|
DstSnapshot _copy(
|
||||||
|
DstSnapshot source, {
|
||||||
|
Map<String, Map<UuidValue, DstRow>>? rows,
|
||||||
|
Map<String, Hlc>? rowHlcs,
|
||||||
|
Map<DstFieldKey, Hlc>? fieldHlcs,
|
||||||
|
Map<String, DstTombstone>? tombstones,
|
||||||
|
Map<DstFieldKey, DstProjection>? projections,
|
||||||
|
}) => DstSnapshot(
|
||||||
|
rows: rows ?? source.rows,
|
||||||
|
projections: projections ?? source.projections,
|
||||||
|
causalLengths: source.causalLengths,
|
||||||
|
rowHlcs: rowHlcs ?? source.rowHlcs,
|
||||||
|
fieldHlcs: fieldHlcs ?? source.fieldHlcs,
|
||||||
|
tombstones: tombstones ?? source.tombstones,
|
||||||
|
);
|
||||||
|
|
@ -30,9 +30,13 @@ void main() {
|
||||||
index: index,
|
index: index,
|
||||||
seed: seed,
|
seed: seed,
|
||||||
rounds: config.rounds,
|
rounds: config.rounds,
|
||||||
|
profile: config.profile,
|
||||||
|
graphWidth: config.graphWidth,
|
||||||
run: () => runDstSimulation(
|
run: () => runDstSimulation(
|
||||||
seed: seed,
|
seed: seed,
|
||||||
rounds: config.rounds,
|
rounds: config.rounds,
|
||||||
|
profile: config.profile,
|
||||||
|
graphWidth: config.graphWidth,
|
||||||
topology: DstTopology.singleSpace,
|
topology: DstTopology.singleSpace,
|
||||||
),
|
),
|
||||||
);
|
);
|
||||||
|
|
|
||||||
|
|
@ -36,9 +36,13 @@ void main() {
|
||||||
index: index,
|
index: index,
|
||||||
seed: seed,
|
seed: seed,
|
||||||
rounds: config.rounds,
|
rounds: config.rounds,
|
||||||
|
profile: config.profile,
|
||||||
|
graphWidth: config.graphWidth,
|
||||||
run: () => runDstSimulation(
|
run: () => runDstSimulation(
|
||||||
seed: seed,
|
seed: seed,
|
||||||
rounds: config.rounds,
|
rounds: config.rounds,
|
||||||
|
profile: config.profile,
|
||||||
|
graphWidth: config.graphWidth,
|
||||||
topology: DstTopology.overlappingSpaces,
|
topology: DstTopology.overlappingSpaces,
|
||||||
),
|
),
|
||||||
);
|
);
|
||||||
|
|
|
||||||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue