feat: Require scopeId on domain tables to ensure user isolation (#73)
This commit is contained in:
Marcelo Mendonça Soares
GitHub
parent
4f2e6b3210
commit
84cfbadc45
76 changed files with 4258 additions and 770 deletions
|
|
@ -11,7 +11,7 @@ dependencies:
|
|||
benchmark_harness: ^2.4.0
|
||||
intl: ^0.20.2
|
||||
path: ^1.9.1
|
||||
serverpod_database: 3.5.0-beta.9
|
||||
serverpod_database: 3.5.0-beta.10
|
||||
serverpod_offline_sync_server:
|
||||
path: ../packages/serverpod_offline_sync_server
|
||||
serverpod_offline_sync_shared:
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ import 'utils/conversion.dart';
|
|||
import 'utils/merge.dart';
|
||||
import 'utils/results.dart';
|
||||
import 'utils/runner.dart';
|
||||
import 'utils/scope.dart';
|
||||
|
||||
Future<void> main(List<String> args) async {
|
||||
final runningInCI = args.contains('--ci');
|
||||
|
|
@ -90,6 +91,38 @@ Future<void> main(List<String> args) async {
|
|||
runningInCI: runningInCI,
|
||||
);
|
||||
}
|
||||
|
||||
// Production-shaped CRDT metadata: many users and a crdt_data_rows table
|
||||
// much larger than the queried rows, exercising the scoped and unscoped
|
||||
// tombstone predicates.
|
||||
const scopeNoiseUsers = 100;
|
||||
final scopeNoiseCrdtRows = rowCount * 100;
|
||||
final scopeMeasurements = <ScopeMode, ScopeMeasurement>{};
|
||||
for (final scopeMode in ScopeMode.values) {
|
||||
scopeMeasurements[scopeMode] = await runWithProgress(
|
||||
'Running select scope benchmark (${scopeMode.name})',
|
||||
() => TombstoneScopeBenchmark(
|
||||
'select scope (${scopeMode.name})',
|
||||
mode: scopeMode,
|
||||
rowCount: rowCount,
|
||||
noiseUsers: scopeNoiseUsers,
|
||||
noiseCrdtRows: scopeNoiseCrdtRows,
|
||||
).measure(),
|
||||
skipProgress: runningInCI,
|
||||
);
|
||||
}
|
||||
printScopeImpact(
|
||||
ScopeBenchmarkResults(
|
||||
baseline: scopeMeasurements[ScopeMode.baseline]!,
|
||||
scoped: scopeMeasurements[ScopeMode.scoped]!,
|
||||
unscoped: scopeMeasurements[ScopeMode.unscoped]!,
|
||||
noiseUsers: scopeNoiseUsers,
|
||||
noiseCrdtRows: scopeNoiseCrdtRows,
|
||||
),
|
||||
rowCount: rowCount,
|
||||
runningInCI: runningInCI,
|
||||
);
|
||||
|
||||
final baselineStorageResult = await runWithProgress(
|
||||
'Running storage benchmark (baseline)',
|
||||
() => TypesTableStorageBenchmark(
|
||||
|
|
|
|||
|
|
@ -138,7 +138,7 @@ abstract class MergeScenarioBenchmark extends AsyncBenchmarkBase {
|
|||
Future<void> mergeSeedRows(List<TableRow<UuidValue?>> rows) {
|
||||
return _crdtSession.db.mergeChanges(
|
||||
[for (final row in rows) insertChangeFor(row)],
|
||||
userId: _userId,
|
||||
scopeId: _userId,
|
||||
);
|
||||
}
|
||||
|
||||
|
|
@ -199,7 +199,7 @@ abstract class MergeScenarioBenchmark extends AsyncBenchmarkBase {
|
|||
@override
|
||||
Future<void> run() async {
|
||||
final queriesBefore = _countingDb.queryCount;
|
||||
await _crdtSession.db.mergeChanges(_mergeSet, userId: _userId);
|
||||
await _crdtSession.db.mergeChanges(_mergeSet, scopeId: _userId);
|
||||
_timedQueries += _countingDb.queryCount - queriesBefore;
|
||||
_timedRuns++;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
|
||||
import 'benchmark.dart';
|
||||
import 'conversion.dart';
|
||||
import 'scope.dart';
|
||||
|
||||
void printPerformanceImpact(
|
||||
BenchmarkResults results, {
|
||||
|
|
@ -131,6 +132,40 @@ void printMergeImpact(
|
|||
if (runningInCI) print('```');
|
||||
}
|
||||
|
||||
void printScopeImpact(
|
||||
ScopeBenchmarkResults results, {
|
||||
required int rowCount,
|
||||
bool runningInCI = false,
|
||||
}) {
|
||||
String compare(
|
||||
double baseline,
|
||||
double scoped,
|
||||
double unscoped,
|
||||
) {
|
||||
String overhead(double value) =>
|
||||
'${formatter2.format((value - baseline) / baseline * 100)}%';
|
||||
return '${baseline.toFormattedDuration()} --> '
|
||||
'${scoped.toFormattedDuration()} scoped (+${overhead(scoped)}) / '
|
||||
'${unscoped.toFormattedDuration()} unscoped (+${overhead(unscoped)})';
|
||||
}
|
||||
|
||||
print(
|
||||
'${runningInCI ? '```' : ''}'
|
||||
'\n🔭 SELECT scope impact '
|
||||
'(${formatter0.format(results.noiseUsers)} extra users, '
|
||||
'${formatter0.format(results.noiseCrdtRows)} CRDT noise rows):',
|
||||
);
|
||||
print(
|
||||
' find all (${formatter0.format(rowCount)} rows): '
|
||||
'${compare(results.baseline.findAllMicros, results.scoped.findAllMicros, results.unscoped.findAllMicros)}',
|
||||
);
|
||||
print(
|
||||
' findById (per lookup): '
|
||||
'${compare(results.baseline.findByIdMicros, results.scoped.findByIdMicros, results.unscoped.findByIdMicros)}',
|
||||
);
|
||||
if (runningInCI) print('```');
|
||||
}
|
||||
|
||||
class BenchmarkResults {
|
||||
const BenchmarkResults({
|
||||
required this.operation,
|
||||
|
|
@ -143,6 +178,22 @@ class BenchmarkResults {
|
|||
final double crdt;
|
||||
}
|
||||
|
||||
class ScopeBenchmarkResults {
|
||||
const ScopeBenchmarkResults({
|
||||
required this.baseline,
|
||||
required this.scoped,
|
||||
required this.unscoped,
|
||||
required this.noiseUsers,
|
||||
required this.noiseCrdtRows,
|
||||
});
|
||||
|
||||
final ScopeMeasurement baseline;
|
||||
final ScopeMeasurement scoped;
|
||||
final ScopeMeasurement unscoped;
|
||||
final int noiseUsers;
|
||||
final int noiseCrdtRows;
|
||||
}
|
||||
|
||||
class MergeBenchmarkResults {
|
||||
const MergeBenchmarkResults({
|
||||
required this.title,
|
||||
|
|
|
|||
224
benchmark/utils/scope.dart
Normal file
224
benchmark/utils/scope.dart
Normal file
|
|
@ -0,0 +1,224 @@
|
|||
// Production-shaped tombstone scope benchmark: many users and a large
|
||||
// crdt_data_rows table, measuring scoped vs unscoped SELECT cost.
|
||||
|
||||
import 'dart:io';
|
||||
import 'dart:typed_data';
|
||||
|
||||
import 'package:path/path.dart' as p;
|
||||
import 'package:serverpod_database/serverpod_database.dart';
|
||||
import 'package:serverpod_offline_sync_server/serverpod_offline_sync_server.dart';
|
||||
import 'package:serverpod_offline_sync_test_client/serverpod_offline_sync_test_client.dart';
|
||||
|
||||
import 'benchmark.dart';
|
||||
import 'tables.dart';
|
||||
|
||||
/// Which session executes the measured selects.
|
||||
enum ScopeMode {
|
||||
/// Plain SQLite session without CRDT metadata.
|
||||
baseline,
|
||||
|
||||
/// CRDT session with a persistent user (single-user client pattern).
|
||||
scoped,
|
||||
|
||||
/// CRDT session without a user scope (server admin reads).
|
||||
unscoped,
|
||||
}
|
||||
|
||||
/// Timed results for one [ScopeMode], in average microseconds.
|
||||
typedef ScopeMeasurement = ({double findAllMicros, double findByIdMicros});
|
||||
|
||||
/// Measures SELECT cost against a database whose `crdt_data_rows` table holds
|
||||
/// production-shaped metadata instead of a single user's rows.
|
||||
///
|
||||
/// On top of the [rowCount] benchmark rows, the CRDT modes seed [noiseUsers]
|
||||
/// extra scopes tracking [noiseCrdtRows] unrelated rows spread across every
|
||||
/// synced table. Random UUIDs keep the noise ownership-conformant while still
|
||||
/// exercising large CRDT metadata tables.
|
||||
class TombstoneScopeBenchmark {
|
||||
TombstoneScopeBenchmark(
|
||||
this.name, {
|
||||
required this.mode,
|
||||
required this.rowCount,
|
||||
required this.noiseUsers,
|
||||
required this.noiseCrdtRows,
|
||||
});
|
||||
|
||||
final String name;
|
||||
final ScopeMode mode;
|
||||
final int rowCount;
|
||||
final int noiseUsers;
|
||||
final int noiseCrdtRows;
|
||||
|
||||
static const _clientUrl = 'http://localhost:8081/';
|
||||
static const _warmupMillis = 100;
|
||||
static const _measurementMillis = 2000;
|
||||
static const _findByIdLookupsPerCycle = 100;
|
||||
|
||||
late final File _dbFile;
|
||||
late final ClientDatabaseSession _plainSession;
|
||||
CrdtDatabaseSession? _crdtSession;
|
||||
var _hasOpenSession = false;
|
||||
|
||||
final UuidValue _userId = const Uuid().v7obj();
|
||||
|
||||
List<Types> _seededRows = [];
|
||||
List<UuidValue> _sampleIds = [];
|
||||
|
||||
DatabaseSession get _activeSession =>
|
||||
mode == ScopeMode.baseline ? _plainSession : _crdtSession!;
|
||||
|
||||
Future<void> _setup() async {
|
||||
final dbPath = p.join(
|
||||
Directory.systemTemp.path,
|
||||
'offline_sync_benchmark_$name.db',
|
||||
);
|
||||
_dbFile = File(dbPath);
|
||||
deleteDatabaseFiles(_dbFile);
|
||||
|
||||
_plainSession = await Client(_clientUrl).createSession(
|
||||
_dbFile.path,
|
||||
isDebugMode: true,
|
||||
);
|
||||
_hasOpenSession = true;
|
||||
|
||||
if (mode != ScopeMode.baseline) {
|
||||
_crdtSession = CrdtDatabaseSession.wraps(
|
||||
_plainSession,
|
||||
syncTables: benchmarkSyncTables,
|
||||
persistentUserId: mode == ScopeMode.scoped ? _userId : null,
|
||||
);
|
||||
await _crdtSession!.db.initialize();
|
||||
}
|
||||
|
||||
await _seedBenchmarkRows();
|
||||
if (mode != ScopeMode.baseline) {
|
||||
await _seedCrdtNoise();
|
||||
}
|
||||
|
||||
final sampleStep = (rowCount / _findByIdLookupsPerCycle).ceil();
|
||||
_sampleIds = [
|
||||
for (var i = 0; i < rowCount; i += sampleStep) _seededRows[i].id!,
|
||||
];
|
||||
}
|
||||
|
||||
Future<void> _seedBenchmarkRows() async {
|
||||
final baseTimestamp = DateTime.now();
|
||||
final rows = List.generate(
|
||||
rowCount,
|
||||
(i) => Types(
|
||||
aBool: i.isEven,
|
||||
aDateTime: baseTimestamp.add(Duration(seconds: i)),
|
||||
aText: 'Text $i',
|
||||
anInt: i,
|
||||
anInt64: BigInt.from(i),
|
||||
aReal: i.toDouble(),
|
||||
aBlob: ByteData(0),
|
||||
anEnum: null,
|
||||
optionalText: null,
|
||||
optionalUuid: const Uuid().v7obj(),
|
||||
),
|
||||
);
|
||||
|
||||
if (mode == ScopeMode.baseline) {
|
||||
_seededRows = await Types.db.insert(_plainSession, rows);
|
||||
return;
|
||||
}
|
||||
_seededRows = await _crdtSession!.db.transactionForUser(_userId, (tx) {
|
||||
return Types.db.insert(_crdtSession!, rows, transaction: tx);
|
||||
});
|
||||
}
|
||||
|
||||
/// Injects CRDT metadata noise with raw SQL, bypassing the recorder: the
|
||||
/// noise only needs to be visible to the tombstone predicates, which read
|
||||
/// `crdt_data_rows` alone.
|
||||
Future<void> _seedCrdtNoise() async {
|
||||
final db = _plainSession.db;
|
||||
final hiddenVisibility = CrdtDataRowVisibility.userDelete.index;
|
||||
|
||||
final benchUserId =
|
||||
(await db.unsafeQuery('SELECT MIN("id") FROM "crdt_scopes"')).first[0] as int;
|
||||
await db.unsafeExecute('''
|
||||
WITH RECURSIVE n(i) AS (SELECT 0 UNION ALL SELECT i + 1 FROM n WHERE i < $noiseUsers - 1)
|
||||
INSERT INTO "crdt_scopes" ("currentNodeId") SELECT NULL FROM n
|
||||
''');
|
||||
await db.unsafeExecute('''
|
||||
INSERT INTO "crdt_nodes" ("scopeId")
|
||||
SELECT u."id" FROM "crdt_scopes" u
|
||||
WHERE NOT EXISTS (SELECT 1 FROM "crdt_nodes" n WHERE n."scopeId" = u."id")
|
||||
''');
|
||||
|
||||
// Noise rows for every noise user, spread across all synced tables, with
|
||||
// 10% tombstoned. Random UUIDs keep them unrelated to the domain rows.
|
||||
await db.unsafeExecute('''
|
||||
WITH RECURSIVE n(i) AS (SELECT 0 UNION ALL SELECT i + 1 FROM n WHERE i < $noiseCrdtRows - 1),
|
||||
users(rowidx, uid) AS (
|
||||
SELECT ROW_NUMBER() OVER (ORDER BY "id") - 1, "id"
|
||||
FROM "crdt_scopes" WHERE "id" <> $benchUserId
|
||||
),
|
||||
tbls(tblidx, tid) AS (
|
||||
SELECT ROW_NUMBER() OVER (ORDER BY "id") - 1, "id" FROM "crdt_schema_tables"
|
||||
)
|
||||
INSERT INTO "crdt_data_rows"
|
||||
("hlcDatetime", "hlcCounter", "scopeId", "tblId", "uuidRowId", "nodeId", "visibility")
|
||||
SELECT 0, 0, u.uid, t.tid, randomblob(16),
|
||||
(SELECT nd."id" FROM "crdt_nodes" nd WHERE nd."scopeId" = u.uid LIMIT 1),
|
||||
CASE WHEN n.i % 10 = 0 THEN $hiddenVisibility ELSE 0 END
|
||||
FROM n
|
||||
JOIN users u ON u.rowidx = n.i % $noiseUsers
|
||||
JOIN tbls t ON t.tblidx = n.i % (SELECT COUNT(*) FROM "crdt_schema_tables")
|
||||
''');
|
||||
|
||||
await db.unsafeExecute('ANALYZE');
|
||||
}
|
||||
|
||||
Future<void> _runFindAll() async {
|
||||
final rows = await Types.db.find(_activeSession);
|
||||
if (rows.length != rowCount) {
|
||||
throw Exception(
|
||||
'Scope benchmark ($name) returned ${rows.length} rows '
|
||||
'but expected $rowCount.',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
Future<void> _runFindByIdBatch() async {
|
||||
for (final id in _sampleIds) {
|
||||
final row = await Types.db.findById(_activeSession, id);
|
||||
if (row == null) {
|
||||
throw Exception('Scope benchmark ($name) could not find row $id.');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Future<double> _measureQuery(Future<void> Function() run) async {
|
||||
Future<void> noPrepare() async {}
|
||||
await measurePreparedCycles(_warmupMillis, prepare: noPrepare, run: run);
|
||||
return measurePreparedCycles(
|
||||
_measurementMillis,
|
||||
prepare: noPrepare,
|
||||
run: run,
|
||||
);
|
||||
}
|
||||
|
||||
/// Seeds the database once and measures both query shapes.
|
||||
Future<ScopeMeasurement> measure() async {
|
||||
await _setup();
|
||||
try {
|
||||
final findAllMicros = await _measureQuery(_runFindAll);
|
||||
final findByIdBatchMicros = await _measureQuery(_runFindByIdBatch);
|
||||
return (
|
||||
findAllMicros: findAllMicros,
|
||||
findByIdMicros: findByIdBatchMicros / _sampleIds.length,
|
||||
);
|
||||
} finally {
|
||||
await _teardown();
|
||||
}
|
||||
}
|
||||
|
||||
Future<void> _teardown() async {
|
||||
if (!_hasOpenSession) return;
|
||||
await _plainSession.close();
|
||||
_hasOpenSession = false;
|
||||
deleteDatabaseFiles(_dbFile);
|
||||
}
|
||||
}
|
||||
|
|
@ -1,237 +0,0 @@
|
|||
# Foreign key invariant implementation review
|
||||
|
||||
Reviewed against `docs/foreign-key-invariants.md` on branch `foreign-key-handling` at `6834d7e`.
|
||||
Updated on the current branch after applying the local/merge semantic
|
||||
clarification and the FK projection fixes.
|
||||
|
||||
## Summary for distribution
|
||||
|
||||
The branch now aligns with the clarified spec on the major FK invariant points:
|
||||
|
||||
1. **Local and merge FK semantics are intentionally different and implemented.** Local `RESTRICT` / `NO ACTION`, `SET NULL`, `SET DEFAULT`, and `CASCADE` are material user actions. Merge-time FK repairs remain projection-only and do not advance ordinary `CrdtDataField` HLCs.
|
||||
2. **Missing-parent merge cases are handled before physical FK writes.** Incoming FK attempts are recorded in `CrdtDataForeignKey.attemptedValue`, while the constrained domain column receives a safe visible value before the write.
|
||||
3. **FK attempted-value storage is explicit.** `attemptedValue` is durable FK field-value storage; `visibleValue`, `hasOverride`, and `overrideReason` are projection metadata.
|
||||
4. **Projection recomputation is clarified.** FK-affecting work may use full fixed-point recomputation as a valid strategy/oracle, while unrelated non-FK updates skip FK projection.
|
||||
5. **The targeted spec coverage has been expanded.** The suite now covers local cascade user tombstones, multiple restrict children, invalid non-null `SET NULL`, missing-parent repairs, unique FK repair, replay metadata churn, batching convergence, and a permitted FK cycle.
|
||||
6. **The main easy performance cleanups are implemented.** CRDT field upsert logic is shared, projection rows are upserted in batches, parent/child projection lookups are indexed, loaded CRDT rows are reused for visibility writes, FK materialization writes are batched, FK edges are precomputed by parent/child table, list updates project once, and sync avoids FK projection N+1 lookups.
|
||||
|
||||
The detailed items below retain the original review structure but now record the updated status.
|
||||
|
||||
## Correctness work item status
|
||||
|
||||
### FK-1: Local cascade should be material; merge projection should stay non-authoring
|
||||
|
||||
**Spec reference:** Core Policy and Local Action Semantics: local `ON DELETE` actions are authored CRDT facts. `RESTRICT` / `NO ACTION` reject the local delete, `SET NULL` / `SET DEFAULT` update child FK fields and ordinary field HLCs, and `CASCADE` records user-delete tombstones for visible cascade descendants. Merge-time FK repairs remain deterministic projection and must not create ordinary user field updates.
|
||||
|
||||
**Current code after fix:**
|
||||
|
||||
- `CrdtMutationRecorder.insteadOfDelete` calls `_softDeleteRowsByTable`, then `_projectForeignKeys` (`packages/serverpod_offline_sync_client/lib/src/database/recorder.dart:620-634`).
|
||||
- `_softDeleteRowsByTable` calls `_applyForeignKeyDeleteActions` before marking the parent deleted, releases unique conflicts, records the parent tombstone, and recursively soft-deletes cascade descendants with the same user-delete reason.
|
||||
- `_applyForeignKeyDeleteActions` throws for `RESTRICT` / `NO ACTION`, mutates child FK columns for `SET NULL` / `SET DEFAULT`, records ordinary CRDT field updates through `recordFieldsUpdatedByTable`, and returns `CASCADE` descendants for recursive user tombstones. Under the revised spec, all of that local behavior is intended.
|
||||
- CRUD tests assert local cascade user-delete tombstones while preserving local restrict/set-null/set-default material-action expectations (`test/serverpod_offline_sync_test_server/test/integration/crud/delete_test.dart`).
|
||||
|
||||
**Status after fix:**
|
||||
|
||||
Resolved. The reviewed branch already treated local `SET NULL`, `SET DEFAULT`, `RESTRICT`, and `NO ACTION` as material database-style actions. Local `CASCADE` now also authors synced `CrdtDataDeletedReason.userDelete` tombstones for visible cascade descendants instead of relying on projection-only `foreignKeyCascade` visibility.
|
||||
|
||||
The inverse risk also matters: when projection runs after a remote merge, its `SET NULL` / `SET DEFAULT` materialization must not call `recordFieldsUpdatedByTable` or otherwise mint new user field HLCs. Those repairs are merge-time projections, not local authored actions.
|
||||
|
||||
**Scenarios verified:**
|
||||
|
||||
- Local delete of `City` with visible `Organization` and `Person` descendants under `ON DELETE CASCADE` should leave descendant domain rows present but attach `CrdtDataDeletedReason.userDelete` tombstones to every cascade descendant and set their visibility to `userDelete`.
|
||||
- Local delete of `Person` referenced by `Town.mayorId ON DELETE SET NULL` should continue to leave `Town` visible with `mayorId == null` and a newer ordinary `Town.mayorId` `CrdtDataField` HLC.
|
||||
- Local delete of `Town` referenced by `Company.townId ON DELETE SET DEFAULT` should continue to materialize the default as an ordinary child FK field update.
|
||||
- Local delete of a `RESTRICT` / `NO ACTION` parent should continue to fail while visible children exist and should not record a parent tombstone for the failed operation.
|
||||
- Remote/merge-time `SET NULL` and `SET DEFAULT` repairs should still update only the materialized domain FK value and `CrdtDataForeignKey` projection metadata, without bumping ordinary child `CrdtDataField` HLCs.
|
||||
|
||||
**Implemented direction:**
|
||||
|
||||
Keep local `RESTRICT`, `SET NULL`, and `SET DEFAULT` as material actions. Add recursive local cascade handling that records user-delete tombstones for visible cascade descendants. Keep merge-time projection pure: do not record ordinary field updates from `_materializeForeignKeyValues`, and keep derived cascade visibility as `foreignKeyCascade` only for rows hidden by merged facts rather than the current local transaction.
|
||||
|
||||
**Stash `e77b65745ee6277d7e695417547ac47ec60acd5f` evaluation:** not sufficient as-is. Directionally, the stash moves local cascade toward authored tombstones by adding `_cascadeUserDeleteFacts` and updating the cascade CRUD expectations, but it regresses two clarified requirements:
|
||||
|
||||
- It removes the existing local `SET NULL` / `SET DEFAULT` material actions from the soft-delete path instead of preserving them.
|
||||
- It adds `recordFieldsUpdatedByTable` calls inside `_materializeForeignKeyValues`, which turns remote/merge-time `SET NULL` and `SET DEFAULT` projection repairs into ordinary user field updates and makes repair HLCs depend on merge timing.
|
||||
|
||||
It also removes `_foreignKeysByReferencedTable`, which removes the existing composite-FK initialization guard and leaves composite FKs silently ignored by `_foreignKeyEdges`.
|
||||
|
||||
### FK-2: Missing-parent merge operations can fail before projection can repair them
|
||||
|
||||
**Spec reference:** Merge-Time Action Semantics and Invariants: for attempted FK values that point to a hidden or missing parent, projection must produce no visible orphan; merge-time `SET NULL` / `SET DEFAULT` repairs are pure functions of merged facts.
|
||||
|
||||
**Current code after fix:**
|
||||
|
||||
- Merge insert/update paths call `_safeIncomingForeignKeyData` before writing FK columns to constrained domain tables.
|
||||
- The original incoming FK value is recorded as an attempted value for `CrdtDataForeignKey.attemptedValue`.
|
||||
- Repairable missing-parent `SET NULL` / `SET DEFAULT` writes use a safe visible value before the physical write. Nullable unrepairable `RESTRICT` / `NO ACTION` / `CASCADE` missing-parent writes use `null` as the physical value and projection hides the row so no visible orphan remains.
|
||||
- Local user updates are still blocked by `_assertVisibleForeignKeyTargets`, which throws when the updated FK points at a missing or hidden parent.
|
||||
|
||||
**Status after fix:**
|
||||
|
||||
Resolved. The projection model now accepts the merged attempted FK fact without forcing that unsafe value into the constrained domain column first. Repairable actions materialize a safe visible value; unrepairable missing-parent attempts are hidden deterministically.
|
||||
|
||||
**Scenarios verified:**
|
||||
|
||||
- Remote `Town` insert/update with `mayorId` pointing at a parent row that is absent locally should converge to a visible town with a materialized `null` mayor and the attempted id preserved.
|
||||
- Remote `Company` insert/update with `townId` pointing at an absent attempted town and a visible default town should converge to the visible default, not throw during the write.
|
||||
- Remote child insert/update under `RESTRICT` / `NO ACTION` / `CASCADE` with a missing parent must follow an explicit deterministic policy that satisfies “no visible row references a hidden or missing parent.” The current spec implies no visible orphan, but the implementation needs a concrete policy for the “parent fact is absent” case.
|
||||
|
||||
**Implemented direction:**
|
||||
|
||||
Do not write an unsafe attempted FK into the physical FK column before projection. Options include computing the visible FK value before the domain write for FK columns, staging attempted values outside the constrained domain column, deferring constraints where supported, or making the attempted value durable in CRDT metadata and writing only the projected visible value to the domain table.
|
||||
|
||||
### FK-3: FK attempted-value storage must be explicit
|
||||
|
||||
**Spec reference:** Projection Metadata: `CrdtDataForeignKey.attemptedValue` is the durable FK field-value payload for FK columns because `CrdtDataField` stores only HLC metadata. `visibleValue`, `hasOverride`, and `overrideReason` are projection metadata and must not be used as business conflict authority.
|
||||
|
||||
**Current code:**
|
||||
|
||||
- `_attemptedForeignKeyValue` returns `projection.attemptedValue` whenever an override is active, otherwise it reads the current domain column (`packages/serverpod_offline_sync_client/lib/src/database/merge_utils/foreign_key_projector.dart:730-746`).
|
||||
- `_childrenReferencingParent` and closure/blocking decisions use `_attemptedForeignKeyValue` (`foreign_key_projector.dart:798-811`).
|
||||
- Sync also relies on projection metadata to reconstruct attempted values: `_applyProjectedForeignKeyAttempts` replaces row insert payload columns with `projection.attemptedValue`, and streamed update fields include `foreignKey` projection rows to return attempted values when an override is active.
|
||||
|
||||
**Status after spec clarification:**
|
||||
|
||||
Once `SET NULL` or `SET DEFAULT` is materialized, the physical domain column no longer contains the attempted FK. `CrdtDataField` only stores HLC metadata, not the attempted value. Therefore `CrdtDataForeignKey.attemptedValue` is explicitly the durable attempted FK value. This removes the earlier spec/code contradiction.
|
||||
|
||||
The remaining correctness requirement is that merge projection only updates the materialized domain value plus projection fields. It must not advance the ordinary `CrdtDataField` HLC for a repaired FK. The test suite now includes a set-null merge repair regression that snapshots the child FK field HLC and verifies projection does not advance it.
|
||||
|
||||
**Documented direction:**
|
||||
|
||||
Keep this storage model documented in the spec and model comment. Treat `CrdtDataForeignKey.attemptedValue` as FK field-value storage and keep `visibleValue`, `hasOverride`, and `overrideReason` as recomputable projection metadata.
|
||||
|
||||
### FK-4: `visibleValue` is populated when `hasOverride == false`
|
||||
|
||||
**Spec reference:** Projection Metadata: `visibleValue` is meaningful only with `hasOverride`; when `hasOverride` is false, it means “same as attempted.”
|
||||
|
||||
**Current code after fix:**
|
||||
|
||||
- `_recordForeignKeyAttemptsForRows` and `_materializeForeignKeyValues` write `visibleValue == null` when `hasOverride == false`.
|
||||
- Tests now assert inactive projections have `visibleValue == null`.
|
||||
|
||||
**Status after fix:**
|
||||
|
||||
Resolved. Inactive projections no longer duplicate the attempted value in `visibleValue`; `visibleValue == null` is interpreted with `hasOverride`.
|
||||
|
||||
**Implemented direction:**
|
||||
|
||||
Store `visibleValue == null` whenever `hasOverride == false`; tests assert that inactive projections leave `visibleValue` empty.
|
||||
|
||||
### FK-5: FK-affecting projection uses full fixed-point recomputation
|
||||
|
||||
**Spec reference:** Merge Pipeline steps 2-4 and invariant/test 15: FK-affecting work must compute the same fixed point as the affected FK closure. A full graph recomputation is valid for FK-affecting work; unrelated non-FK updates must skip FK projection.
|
||||
|
||||
**Current code after fix:**
|
||||
|
||||
- `_projectForeignKeys` still computes a full fixed point for FK-affecting operations, which is now an allowed implementation strategy and oracle.
|
||||
- `_tableColumnsMayAffectForeignKeys` and `_mergeOperationsMayAffectForeignKeys` skip projection for updates that touch no child FK column or parent reference column.
|
||||
- `Database.update(List<T>)` performs row writes in one transaction and calls `afterUpdate` once, so list updates trigger at most one projection pass.
|
||||
- Projection state now indexes parent and child lookups and batches projection/domain writes, reducing the cost of full fixed-point recomputation.
|
||||
|
||||
**Status after spec clarification and fix:**
|
||||
|
||||
Resolved against the clarified spec. The implementation uses full fixed-point recomputation only when an operation can affect FK state, and skips FK projection for unrelated non-FK updates. A narrower affected-closure loader remains a possible optimization, not a correctness requirement.
|
||||
|
||||
**Implemented direction:**
|
||||
|
||||
Gate projection by FK-affecting tables/columns; keep full recomputation as the fixed-point oracle for FK-affecting work; reduce full recomputation cost with indexed state and batched writes.
|
||||
|
||||
### FK-6: Composite foreign keys must be explicitly unsupported
|
||||
|
||||
**Spec reference:** Core Policy: supported foreign keys are single-column child references to single-column parent references. Schemas with composite foreign keys must fail recorder initialization instead of silently bypassing projection.
|
||||
|
||||
**Current code:**
|
||||
|
||||
- `_foreignKeysByReferencedTable` throws during recorder initialization if any target table definition contains a composite FK (`packages/serverpod_offline_sync_client/lib/src/database/recorder.dart:95-106`).
|
||||
- `_foreignKeyEdges` silently includes only FKs whose `columns.length == 1` and `referenceColumns.length == 1` (`recorder.dart:119-135`).
|
||||
|
||||
**Status after spec clarification:**
|
||||
|
||||
A schema with a composite FK fails during recorder initialization through `_foreignKeysByReferencedTable`. This is now documented as an explicit limitation rather than a spec divergence. Later refactors must keep that guard so composite FKs are not silently ignored by `_foreignKeyEdges`.
|
||||
|
||||
**Documented direction:**
|
||||
|
||||
Keep the initialization validation and add/keep a schema test that proves composite FKs fail fast if such a model is introduced.
|
||||
|
||||
### FK-7: Unique/FK stage order is only partially implemented and not proven by tests
|
||||
|
||||
**Spec reference:** Merge Pipeline step 6 and test 13: if FK repairs can change unique columns, recompute unique projection after FK projection or define/prove the opposite order.
|
||||
|
||||
**Current code after fix:**
|
||||
|
||||
- Unique conflict resolution runs before FK projection for incoming inserts/updates (`merge.dart:273-345`).
|
||||
- `_materializeForeignKeyValues` runs `_resolveUniqueConflictsAfterForeignKeyProjection` only for rows whose visible FK value changed (`foreign_key_projector.dart:588-679`).
|
||||
- The test schema now includes a unique nullable `SET NULL` FK via `UniqueSetNullChild.parentId`.
|
||||
|
||||
**Status after fix:**
|
||||
|
||||
Resolved for the covered nullable unique `SET NULL` case. The code keeps the pre-FK unique pass and the post-FK unique pass for rows whose FK projection changed; the integration suite now proves the repaired unique FK becomes visible as `null` with the attempted value preserved.
|
||||
|
||||
**Implemented direction:**
|
||||
|
||||
Added a test model with a unique nullable `SET NULL` FK and covered the post-FK unique pass for that repair path. Batching convergence is covered separately for the repaired `SET NULL` graph.
|
||||
|
||||
## Spec coverage status in tests
|
||||
|
||||
The integration suite now covers the high-risk cases from the review:
|
||||
|
||||
- **Test 2:** multiple visible restrict children where any one child keeps the parent visible.
|
||||
- **Local cascade:** local cascade deletes assert synced user-delete tombstones on cascade descendants.
|
||||
- **Test 3:** restrict child detachment after a blocked parent delete restores the preserved delete projection.
|
||||
- **Test 4/6/7 merge projection discipline:** remote `SET NULL` repairs assert that ordinary child `CrdtDataField` HLCs are not advanced by projection.
|
||||
- **Invalid `SET NULL`:** a non-nullable `ON DELETE SET NULL` relation proves deterministic blocking.
|
||||
- **Missing-parent repair:** remote insert/update references to absent parents cover repairable `SET NULL` / `SET DEFAULT` and unrepairable `RESTRICT` behavior before domain writes.
|
||||
- **Replay metadata churn:** cascade replay snapshots both `CrdtDataRow.visibility` and `CrdtDataForeignKey` projection rows.
|
||||
- **Test 12:** an existing permitted data cycle across `Person`, `Company`, and `Town` proves fixed-point evaluation terminates.
|
||||
- **Test 13:** a unique-indexed nullable `SET NULL` FK repair is covered.
|
||||
- **Batching convergence:** one-batch and split-batch merges converge for a repaired `SET NULL` graph.
|
||||
|
||||
Remaining optional coverage candidates:
|
||||
|
||||
- A dedicated fourth-degree chain comparing a hand-minimized affected closure to full recomputation. The current spec allows full recomputation as the implementation strategy.
|
||||
- A synthetic composite-FK schema test. Composite FKs are explicitly unsupported and the initialization guard is retained, but the test server does not include a composite FK model.
|
||||
|
||||
## Duplication and easy performance cleanup items
|
||||
|
||||
### PERF-1: Extract duplicated CRDT field upsert logic
|
||||
|
||||
Resolved. `_recordUpdatedFields` and `recordFieldsUpdatedByTable` now share `_upsertCrdtFieldsForRows(tableName, crdtRows, schemaColumns, skippedFields?)`. This also keeps merge-time FK projection away from ordinary field-HLC authoring.
|
||||
|
||||
### PERF-2: Batch projection upserts instead of querying one row at a time
|
||||
|
||||
Resolved. `_ForeignKeyProjectionRow` now carries the projection row id and override reason, and `_upsertForeignKeyProjections` batches inserts/updates using either loaded projection state or one bulk lookup for missing field ids.
|
||||
|
||||
### PERF-3: Index parent and child lookups inside projection state
|
||||
|
||||
Resolved. `_ForeignKeyProjectionState` now builds `(parentTable, parentColumn, value) -> row` and `(childTable, childColumn, attemptedValue) -> children` maps once, and hot closure/blocking/materialization paths use those indexes.
|
||||
|
||||
### PERF-4: Avoid re-querying CRDT rows already present in projection state
|
||||
|
||||
Resolved. `_ProjectedForeignKeyRow` now carries the loaded `CrdtDataRow` and current visibility, so `_setProjectedRowVisibility` can update without re-querying rows.
|
||||
|
||||
### PERF-5: Batch domain FK materialization writes
|
||||
|
||||
Resolved for equal update sets. `_materializeForeignKeyValues` collects domain FK changes, groups rows by table and identical update map, writes each group once, then runs the post-FK unique pass.
|
||||
|
||||
### PERF-6: Precompute foreign-key edges by parent and child table
|
||||
|
||||
Resolved. The recorder now precomputes `_foreignKeyEdgesByParentTable` and `_foreignKeyEdgesByChildTable`, and projector paths use them for child-table and parent-table loops.
|
||||
|
||||
### PERF-7: Avoid full projection per row in `Database.update(List<T>)`
|
||||
|
||||
Resolved at the recorder/projection layer. `CrdtDatabase.update(List<T>)` still applies the tombstone-aware row update predicate per row, but it records CRDT updates once for the full list and therefore runs FK projection at most once.
|
||||
|
||||
### PERF-8: Avoid N+1 sync projection lookups
|
||||
|
||||
Resolved for FK projection lookups. `_streamUpdates` includes `foreignKey` with each `CrdtDataField`, and `_streamInserts` preloads active projected FK attempt fields for all streamed CRDT rows.
|
||||
|
||||
## Validation notes
|
||||
|
||||
- Ran `dart analyze packages/serverpod_offline_sync_client test/serverpod_offline_sync_test_server --fatal-infos --fatal-warnings`; no issues found.
|
||||
- Ran `dart test test/serverpod_offline_sync_test_server/test/integration/merge/foreign_key_invariant_test.dart --reporter=expanded --concurrency=1`; it passed all 36 tests.
|
||||
- Ran `dart test test/serverpod_offline_sync_test_server/test/integration/crud/delete_test.dart --reporter=expanded --concurrency=1`; it passed all 20 tests.
|
||||
- Ran `dart test test/serverpod_offline_sync_test_server/test/integration/crud --reporter=expanded --concurrency=1`; it passed all 58 tests.
|
||||
- Ran `dart test test/serverpod_offline_sync_test_server/test/integration/merge --reporter=expanded --concurrency=1`; it passed all 71 tests.
|
||||
- Ran `dart test test/serverpod_offline_sync_test_server/test/integration/database --reporter=expanded --concurrency=1`; it passed all 3 tests.
|
||||
- Ran `dart test test/serverpod_offline_sync_test_server/test/integration/sync/sync_roundtrip_test.dart --reporter=expanded --concurrency=1`; it passed all 6 tests, including projected FK attempted-value insert/update payload roundtrips.
|
||||
- Attempted `dart test test/serverpod_offline_sync_test_server/test/integration/sync --reporter=expanded --concurrency=1`; `sync_roundtrip_test.dart`, `sync_merge_helpers_test.dart`, and `sync_hash_test.dart` passed, but `sync_flow_test.dart` and `sync_unique_conflict_test.dart` failed during load with `Database is not available in this session` from the `withServerpod` test harness before test bodies executed.
|
||||
740
docs/row-ownership.md
Normal file
740
docs/row-ownership.md
Normal file
|
|
@ -0,0 +1,740 @@
|
|||
# Row ownership and scope isolation
|
||||
|
||||
Status: implemented.
|
||||
The *Future: shared scopes* section is direction, not commitment.
|
||||
|
||||
## Summary
|
||||
|
||||
Synced rows are physically shared on the server while their CRDT metadata is
|
||||
tracked per user. Ownership is therefore implicit: nothing in the data model
|
||||
prevents two users from tracking the same `(table, uuidRowId)` pair, but the
|
||||
storage model cannot actually represent that state. Deletes survive it because
|
||||
visibility is per-user metadata; field values cannot, so concurrent updates by
|
||||
different users silently overwrite each other across tenant boundaries.
|
||||
|
||||
This document makes ownership explicit. The core decisions:
|
||||
|
||||
> **Every synced domain row is globally identified by `(table, uuidRowId)` and
|
||||
> owned by exactly one scope.** `scopeId` is not part of row identity; it is
|
||||
> ownership and isolation metadata on a globally identified row. Two scopes
|
||||
> never materialize or sync the same `(table, uuidRowId)`. Cross-scope UUID
|
||||
> reuse is a terminal ownership collision, not valid multi-tenant state. Sync
|
||||
> fails on that collision, records a durable violation, and does not advance
|
||||
> the sender past the rejected change. Sharing, when it arrives, will be
|
||||
> modeled as shared scopes that multiple identities subscribe to — never as
|
||||
> two scopes aliasing one row.
|
||||
|
||||
> **Every synced table declares `scopeId`, identically on server and
|
||||
> client.** The column is a hard requirement, never `scope=serverOnly`, so
|
||||
> both databases run one schema shape and the package has exactly one code
|
||||
> path. "Single user" is a usage pattern (a database that happens to contain
|
||||
> one scope), not a mode: there are no mode flags, no schema branching, and
|
||||
> no per-mode test matrix.
|
||||
|
||||
> **The sync layer owns the column: stamp if null, assert if set — and the
|
||||
> value leaves the way it came in.** On every insert, a null `scopeId` is
|
||||
> stamped from the effective scope; a non-null one must equal it or the
|
||||
> operation throws. Stamped values are stripped back to null on returned
|
||||
> rows and on scoped reads, so the database-local value never leaks into
|
||||
> endpoint payloads; explicitly provided values are kept on the way out.
|
||||
> One rule serves both personalities: users who never want to think about
|
||||
> tenancy never do, and users who pass it explicitly get verified
|
||||
> explicitness and keep seeing what they manage. After insert the column is
|
||||
> immutable.
|
||||
|
||||
The package contract for a synced table is two structural requirements, both
|
||||
validated at `initialize()` with paste-ready error messages:
|
||||
|
||||
```yaml
|
||||
fields:
|
||||
id: UuidValue?, defaultPersist=random_v7
|
||||
### Owner scope of this row. Maintained by the CRDT sync layer.
|
||||
scopeId: int?
|
||||
```
|
||||
|
||||
No model inheritance is needed: like the UUID primary key requirement, the
|
||||
contract is structural (a reserved column name with a required type, checked
|
||||
by `CrdtSchemaRegistry`), not nominal (a base class), which is the only
|
||||
option anyway since Serverpod models cannot extend models from an imported
|
||||
module. The same constraint shapes the implementation: with no common
|
||||
interface to call, the package stamps the field through dynamic dispatch —
|
||||
`(row as dynamic).copyWith(scopeId: …)` (or the plain field setter, since
|
||||
generated fields are mutable). The field name is a compile-time constant, so
|
||||
the serialization manager's `patchTableRow` would be the wrong tool here: its
|
||||
serialize–patch–deserialize round-trip costs 50–200× a dynamic call and is
|
||||
justified only where field names are data-driven, as in the merge path.
|
||||
|
||||
## Goals and constraints
|
||||
|
||||
1. **One ownership model, one schema shape, one code path.** Earlier drafts
|
||||
explored an optional column and then schema-presence modes; both were
|
||||
rejected because every mechanism (CRUD, merge, sync, unique and FK
|
||||
invariants) would need building and testing per branch. Requiring the
|
||||
column everywhere deletes the mode system, the schema-projection
|
||||
generator feature it needed, and the per-mode test matrix. The cost — an
|
||||
int column (~1–2 bytes per row in SQLite) and a visible read-only field
|
||||
on domain models — is accepted.
|
||||
2. **No external model inheritance.** Serverpod does not let application
|
||||
models extend models from an imported module, so a base class cannot
|
||||
carry shared columns into user tables. Requirements are enforced by
|
||||
runtime validation, and generic field access by dynamic dispatch or
|
||||
`patchTableRow`.
|
||||
3. **Roles are not hard-coded.** Nothing in the package assumes the server
|
||||
is multi-user or the client single-user. Multi-account devices and
|
||||
single-tenant servers are the same code path with different scope counts.
|
||||
4. **Application code cannot get tenancy wrong.** The package must verify
|
||||
any user-supplied scope against the effective scope regardless (a tenant
|
||||
id from application code can never be trusted), so correctness never
|
||||
rests on call-site discipline.
|
||||
5. **Multi-device per user keeps working unchanged.** Devices of one user
|
||||
share the user's scope and merge into one chain.
|
||||
6. **Sharing must not be foreclosed.** Collaborative data is a core
|
||||
local-first use case; the design must leave a clean path to it.
|
||||
|
||||
## Problem: ownership is implicit
|
||||
|
||||
The server holds one physical domain row per `(table, uuidRowId)`, while CRDT
|
||||
metadata (`crdt_data_rows` and its children) is keyed per
|
||||
`(scopeId, tblId, uuidRowId)`. Before this design was implemented, that
|
||||
metadata shape let multiple scopes track the same row even though the domain
|
||||
table could only materialize one copy. Symptoms, in increasing severity:
|
||||
|
||||
1. **Tombstone masking.** A tombstone recorded by one user (or for another
|
||||
table) used to hide unrelated rows sharing a UUID. Fixed by scoping the
|
||||
visibility predicates — but the fix required defining what an *unscoped*
|
||||
query means ("hidden only when hidden for every tracking user"), an
|
||||
aggregate query shape, and a covering index we could not afford on
|
||||
clients. Complexity that exists only because multiple trackers can.
|
||||
2. **Update leakage is unrepresentable.** Field values materialize into the
|
||||
single shared domain row. If two users' chains track it, each chain has
|
||||
its own LWW history, but there is only one cell to materialize into.
|
||||
Whoever merges last wins, across tenant boundaries, with no record of the
|
||||
conflict. This is not a policy gap — divergent per-user values cannot be
|
||||
stored at all.
|
||||
3. **Merge-insert overwrites other users' rows.** `_applyMergeInsertForMissingRow`
|
||||
treats "missing" as "no CRDT row *for the merging user*" and writes the
|
||||
domain row update-then-insert. A client that syncs an insert reusing
|
||||
another user's row UUID overwrites that user's data through the normal
|
||||
sync path. (Updates and deletes are already safe: they no-op when the
|
||||
merging user has no tracker.)
|
||||
4. **Unique-conflict resolution crosses tenants.** Domain unique indexes are
|
||||
global on the server and the flag policy (see
|
||||
`unique-constraint-invariants.md`) picks the winner by HLC with no notion
|
||||
of ownership. One user's insert can cause another user's newer row to have
|
||||
its unique value rewritten.
|
||||
|
||||
All four share one root cause: the metadata says who owns a row, but nothing
|
||||
enforces that the answer is unique, and the domain storage assumes it is.
|
||||
|
||||
"Leave it up to users" is rejected explicitly: symptom 2 means the storage
|
||||
model cannot represent what multi-scope tracking promises, so allowing it is
|
||||
not granting flexibility — it is permitting silent cross-tenant data
|
||||
corruption. Applications that need shared data need shared *scopes* (see
|
||||
*Future: shared scopes*), which preserve the one-chain-per-row property that
|
||||
makes LWW merging coherent.
|
||||
|
||||
## The `scopeId` column
|
||||
|
||||
### Contract
|
||||
|
||||
`scopeId` is a reserved column name, required on every synced table:
|
||||
|
||||
- **It is not part of row identity.** The domain primary key remains the
|
||||
required `id: UuidValue`, and `(table, uuidRowId)` remains globally unique
|
||||
in the physical domain table. A row with the same UUID in a different scope
|
||||
cannot be represented as a second domain row; it is treated as a corrupt or
|
||||
hostile ownership collision, the merge fails, and the violation is recorded.
|
||||
- The value references `crdt_scopes.id` — the scope that owns the row. Today
|
||||
a scope is a user; the name is chosen deliberately so shared scopes can be
|
||||
added later without renaming storage again.
|
||||
- **The value is database-local.** It is a normalization of the scope UUID
|
||||
into the local `crdt_scopes` table; the client's int for a scope is not
|
||||
the server's int for the same scope. The sync layer re-maps it on merge
|
||||
and never ships it as data, and application code cannot ship it by
|
||||
accident either, because scoped reads return it stripped (see *Keeping
|
||||
the value off the wire*). This is also why the field is nullable: a model
|
||||
outside the database honestly carries "not mine to say" instead of a
|
||||
meaningless local id.
|
||||
- `scope=serverOnly` is **rejected**: it would reintroduce asymmetric
|
||||
schemas and with them the mode branching this design exists to delete.
|
||||
The client-side registry sees the column missing and says exactly that.
|
||||
- The column is **owned by the sync layer** and immutable after insert.
|
||||
The value round-trips the way it came in: callers that never provide it
|
||||
never receive it (stamped values are stripped from returned rows, and
|
||||
scoped reads return null), explicit callers keep seeing what they passed,
|
||||
and unscoped admin reads always return it populated (see *Keeping the
|
||||
value off the wire*).
|
||||
|
||||
### Write semantics: stamp if null, assert if set
|
||||
|
||||
On every local insert (and batch insert), the database proxy resolves the
|
||||
effective scope — transaction scope, else persistent scope, else the
|
||||
operation throws, exactly as recording already requires — and applies one
|
||||
rule per row:
|
||||
|
||||
- `scopeId == null` → the row is stamped with the effective scope before the
|
||||
delegate insert (one write, not an UPDATE after the fact). Stamping uses a
|
||||
dynamic call, since no base interface can declare the field; this is the
|
||||
per-row hot path of every insert, which is why the heavier `patchTableRow`
|
||||
stays out of it. In-place mutation is preferred (generated fields are
|
||||
mutable, so no per-row allocation); dynamic `copyWith` is the fallback
|
||||
only if rows were ever immutable.
|
||||
- `scopeId != null` → it must equal the effective scope, or the operation
|
||||
throws. Explicit callers get verification for free.
|
||||
|
||||
Consequences of the rule:
|
||||
|
||||
- **Cross-scope writes cannot succeed.** A model carrying another scope's id
|
||||
trips the assertion; a model carrying null is stamped with the acting
|
||||
scope and then the scoped `WHERE` filter (which includes
|
||||
`scopeId = <scope>`) matches no foreign row, so the update or delete fails
|
||||
with "no rows" instead of touching another tenant's data.
|
||||
- **The column never appears in update SET lists**, exactly like `id`
|
||||
(`_updateRowWithoutRecording` already filters `columnName != 'id'`).
|
||||
A tampered model cannot attempt to move a row across scopes.
|
||||
- **Merges stamp from the merging scope** (each merge acts in exactly one
|
||||
scope), and incoming payloads have the field stripped by
|
||||
`_sanitizeMergeRowData` — it is not portable data.
|
||||
|
||||
The same effective-scope requirement applies to every CRUD entrypoint, not
|
||||
only inserts:
|
||||
|
||||
- `insert`/`insertRow`/batch insert: stamp-or-assert before the delegate write;
|
||||
- reinsert of a tombstoned row: only the same owning scope can revive it;
|
||||
a foreign-scope row id is a UUID ownership collision, not a reinsert target;
|
||||
- `update`/`updateRow`: the scoped `WHERE` includes `scopeId = <scope>`, and
|
||||
`scopeId` is removed from the SET list even if it appears in `columns`;
|
||||
- `updateWhere`/`updateById`: any `ColumnValue` for `scopeId` is rejected (or
|
||||
dropped before SQL generation, but rejection gives better caller feedback),
|
||||
and the scoped `WHERE` still includes `scopeId = <scope>`;
|
||||
- `delete`/`deleteRow`/`deleteWhere`: deletes only see rows owned by the
|
||||
effective scope, so a foreign row id returns "no rows" instead of recording a
|
||||
tombstone for another owner;
|
||||
- future `upsert`: must first resolve ownership of the conflicting row and may
|
||||
update only when the existing row's `scopeId` equals the effective scope.
|
||||
|
||||
### Keeping the value off the wire
|
||||
|
||||
The wire discipline is enforced by construction and respects the caller's
|
||||
choice: **the field leaves the package the way it came in.**
|
||||
|
||||
- **Stamped values stay internal.** When a model comes in with `scopeId`
|
||||
null, the proxy stamps the effective scope for the database write, and
|
||||
the rows handed back by that operation carry null again. The same applies
|
||||
to scoped reads, which have no input model to respect. Stripping applies to
|
||||
every synced model instance returned by the proxy, including nested
|
||||
`IncludeObject` and `IncludeList` result graphs. Callers who never provide
|
||||
the value never observe it, and cannot serialize the database-local int into
|
||||
endpoint payloads even by accident.
|
||||
- **Explicit values are respected.** When a model comes in with `scopeId`
|
||||
set (and verified by the assert arm), the rows returned by that operation
|
||||
keep it — the package does not hide a value the caller knowingly manages.
|
||||
In batch operations the rule applies per row.
|
||||
- **Unscoped (admin) reads keep it populated.** Within a scoped read the
|
||||
value is redundant — it always equals the acting scope — but for admin
|
||||
reads it is the only per-row ownership information, and those results are
|
||||
consumed server-side.
|
||||
|
||||
Mechanically this stays on the cheap path: rows returned by the delegate are
|
||||
fresh, package-owned instances, so stripping is a per-row in-place setter
|
||||
with no allocation, applied at the same proxy layer that injects the
|
||||
visibility predicates. The caller's own input instance is never left
|
||||
observably mutated by a stamp: the implementation either stamps a copy bound
|
||||
for the database or restores the field after the write.
|
||||
|
||||
This mirrors mature row-level-security practice, where the tenant column is
|
||||
populated from session context (`DEFAULT current_setting(...)`) precisely so
|
||||
application code cannot get it wrong, rather than passed through call sites.
|
||||
|
||||
A convenience accessor exposing the effective scope's normalized id (e.g. on
|
||||
the transaction or the database) may be added for explicit callers and
|
||||
custom filtering; it is not required by any flow, and its documentation
|
||||
carries the database-local warning above.
|
||||
|
||||
### Sync-layer exclusions
|
||||
|
||||
`scopeId` is infrastructure that happens to live in the domain row. Keyed by
|
||||
the reserved name, it is excluded from:
|
||||
|
||||
- `crdt_schema_columns` registration and field-level CRDT metadata (it has no
|
||||
HLC history; it is never merged as a field) — which also keeps it out of
|
||||
the synced-schema hash symmetrically on both sides;
|
||||
- update recording (`afterUpdate` ignores it) and merge payload sanitization;
|
||||
- outbound sync insert payloads and outbound sync update payloads. The sender
|
||||
must not serialize `scopeId` from the domain row; the receiver stamps the
|
||||
merge-local normalized scope instead.
|
||||
|
||||
### Validation
|
||||
|
||||
`CrdtSchemaRegistry` extends the existing UUID primary key check:
|
||||
|
||||
- every synced table must have a nullable `int` column named `scopeId`
|
||||
(missing or mistyped → error with the paste-ready yaml, including the
|
||||
"remove `scope=serverOnly`" hint when the server schema has the column and
|
||||
the client schema does not);
|
||||
- every non-primary unique index on a synced table must include `scopeId`
|
||||
(missing → error), except a foreign-key-only unique index whose referenced
|
||||
tables are also synced. Ordinary global unique indexes are forbidden for now
|
||||
because the package does not yet define deterministic cross-scope conflict
|
||||
resolution.
|
||||
|
||||
## Unique indexes
|
||||
|
||||
Per-scope uniqueness is declared by the application as an ordinary composite
|
||||
index, which now exists identically on both databases:
|
||||
|
||||
```yaml
|
||||
indexes:
|
||||
person_scope_name_unique_idx:
|
||||
fields: scopeId, name
|
||||
unique: true
|
||||
```
|
||||
|
||||
Ordinary global unique indexes are rejected until the package has a
|
||||
deterministic cross-scope arbitration policy and a way to record the
|
||||
losing-side release in that scope's CRDT chain. "Whoever syncs first" is not
|
||||
an acceptable CRDT rule. The only permitted global unique indexes are
|
||||
foreign-key-only indexes: every index element must be a column, and the
|
||||
indexed columns must be composed only of complete foreign-key column sets whose
|
||||
referenced tables are also synced. This covers Serverpod's relation indexes,
|
||||
including one-to-one and composite FK relation indexes, without forcing
|
||||
duplicate per-scope indexes for include-query performance. Those columns point
|
||||
at globally unique synced row ids, and FK enforcement already repairs or
|
||||
rejects cross-scope references, so they are not treated as global
|
||||
application-level arbitration. The model must still declare scoped composite
|
||||
unique indexes for application uniqueness. On a deployment that happens to
|
||||
contain one scope, a composite unique with a constant `scopeId` accepts exactly
|
||||
the same rows as a plain unique; nothing degrades.
|
||||
|
||||
The unique-conflict resolver discovers indexes from the table definitions,
|
||||
uses `scopeId` as part of the lookup predicate, and releases only the
|
||||
application columns in scoped indexes. Allowed FK-only global unique indexes
|
||||
are ignored by the CRDT resolver; same-scope application conflicts are handled
|
||||
by scoped composite indexes, and cross-scope references are handled by FK
|
||||
repair. Rows in different scopes never form a conflict group under the
|
||||
required composite index. The flag policy (see
|
||||
`unique-constraint-invariants.md`) is otherwise unchanged.
|
||||
|
||||
## Enforcement mechanics
|
||||
|
||||
1. **Local writes: stamp-or-assert** (see *Write semantics*), with the
|
||||
effective scope required exactly as it already is for CRDT recording.
|
||||
2. **Merge-insert ownership check.** `_applyMergeInsertForMissingRow` treats
|
||||
an existing domain row with another `scopeId` as a UUID ownership collision.
|
||||
This is a terminal sync error, not a recoverable conflict policy. The check
|
||||
is atomic with the physical write:
|
||||
1. Look up the domain row by `(table, uuidRowId)` before writing.
|
||||
2. If it exists with the merging scope, follow the same-scope
|
||||
update/reinsert path.
|
||||
3. If it exists with another scope, roll back the merge work for the
|
||||
incoming change, record a durable integrity violation with denormalized
|
||||
operation, table, row, scope, node UUID, and HLC fields, then fail the
|
||||
sync session.
|
||||
4. If it does not exist, insert with `scopeId` stamped from the merging
|
||||
scope.
|
||||
5. If the insert races another scope and hits the domain primary key,
|
||||
re-read the row and apply the same ownership decision before any CRDT
|
||||
metadata survives the transaction.
|
||||
|
||||
Step 5 carries an implementation constraint: today the merge records the
|
||||
scope's CRDT row (`_upsertMergeRow`) — and unique-conflict resolution and
|
||||
FK-safe handling, which may mutate neighboring rows — *before* the domain
|
||||
write where the collision surfaces. The insert application must therefore
|
||||
run inside a savepoint rolled back on collision (or be reordered
|
||||
domain-write-first), so that a rejected insert leaves behind no CRDT row,
|
||||
no released unique values, and no FK attempts. The durable violation is
|
||||
written after that rollback, outside the rolled-back merge work, so the
|
||||
diagnostic record survives even though the bad change does not.
|
||||
|
||||
The durable record is sparse exception state, not a second ownership index.
|
||||
`crdt_sync_integrity_violations` stores one deduplicated row per observed
|
||||
violation key: `(type, operation, domainTableName, uuidRowId,
|
||||
ownerScopeUuid, incomingScopeUuid)`. All other stored fields are
|
||||
denormalized scalars so the record remains useful after merge rollbacks and
|
||||
on databases that do not share the same local CRDT metadata rows. There are
|
||||
no foreign keys into CRDT metadata tables (`crdt_schema_tables`,
|
||||
`crdt_data_rows`, `crdt_nodes`): relations would often target rows that
|
||||
never committed or no longer exist when the violation is persisted outside
|
||||
the rolled-back merge transaction.
|
||||
|
||||
`type` and `operation` are serialized enum names. The current types are
|
||||
`ownershipCollision` and `missingDomainRow`; operations distinguish merge
|
||||
and outbound insert/update/delete paths. Stored alongside the key:
|
||||
`uuidNodeId` and `hlcDatetime` / `hlcCounter` for the rejected change's
|
||||
causal context; optional `crdtDataRowId` as a same-database hint to the
|
||||
local metadata row id when one was known (for example outbound sync with a
|
||||
stale tracker); `firstSeenAt`, `lastSeenAt`, and `occurrences`.
|
||||
`ownerScopeUuid` is nullable because a corrupt physical row can exist with
|
||||
a null owner;
|
||||
`incomingScopeUuid` is required and globally locatable. `crdtDataRowId` is
|
||||
often null on merge-insert collisions and is not portable across devices.
|
||||
The record intentionally stores neither the rejected payload nor a payload
|
||||
hash. The sync response returns the violation identifier and checkpoints do
|
||||
not advance past the rejected change, so retrying the same corrupt client
|
||||
state fails deterministically until the application resolves it.
|
||||
|
||||
Re-keying the incoming row to a synthetic UUID was considered (non-lossy,
|
||||
consistent with the unique flag policy) but rejected: row identity re-keying
|
||||
breaks the incoming chain's own foreign key references to that UUID, which
|
||||
would require alias tracking across the whole merge. UUIDv7 collisions
|
||||
between honest clients are negligible; a collision is a bug or an attack,
|
||||
and failing the sync is the only response that preserves both the existing
|
||||
owner's data and the sender's still-unaccepted local change.
|
||||
3. **Foreign keys stay within a scope.** A synced row may only reference
|
||||
synced rows of its own scope. Locally,
|
||||
`_assertVisibleForeignKeyTargets` gains a scope-equality check next to
|
||||
its visibility check; in merges, a foreign-scope target is treated
|
||||
exactly like an invisible target, so the existing attempt/override
|
||||
machinery (see `foreign-key-invariants.md`) repairs the reference instead
|
||||
of linking across scopes. Cascade deletes and SET NULL projections
|
||||
therefore never cross a scope boundary. References to non-synced tables
|
||||
are unaffected.
|
||||
4. **No duplicated metadata lookup index.** An earlier draft added a unique
|
||||
index on `crdt_data_rows (tblId, uuidRowId)` as defense-in-depth. It is
|
||||
dropped because the hot CRDT paths need the existing scoped index
|
||||
`(scopeId, tblId, uuidRowId)`, and adding the unscoped index duplicates
|
||||
most of the key space on every row. Instead, every consumer of CRDT row
|
||||
metadata is required to prove ownership against the physical row: reads key
|
||||
visibility through `domain.scopeId`; outbound sync serializes a domain row
|
||||
only when `domain.scopeId` equals the collecting scope; merge, FK, and
|
||||
unique helpers reject metadata that attempts to materialize or mutate a
|
||||
foreign-owned row. A stray foreign tracker is therefore inert by consumer
|
||||
discipline, while an observed attempt to use it is a durable sync violation.
|
||||
Deployments that need a stronger offline audit can run an explicit scan for
|
||||
duplicate `(tblId, uuidRowId)` trackers; the package does not pay that
|
||||
index cost on every database by default.
|
||||
|
||||
## Read path
|
||||
|
||||
Ownership on the row collapses every visibility predicate into an
|
||||
index-covered probe; the aggregate (`GROUP BY`/`MIN`) form, its full-scan
|
||||
cost (~95 ms per point lookup at 1M CRDT rows in the SQLite benchmark), and
|
||||
the covering index it wanted never ship.
|
||||
|
||||
- **Scoped query** (a `transactionForUser` read, or the persistent default
|
||||
scope): rows are filtered by `scopeId = <scope>` — the isolation promise,
|
||||
and the central point of the design — plus the existing correlated
|
||||
`NOT EXISTS` visibility probe on `(scopeId, tblId, uuidRowId)`.
|
||||
- **Unscoped query** (no transaction scope, no persistent scope — admin
|
||||
reads): no isolation filter; the visibility probe is keyed by the row's
|
||||
own column — `NOT EXISTS (… WHERE c.scopeId = t."scopeId" AND tblId = ?
|
||||
AND uuidRowId = t.id AND visibility > …)` — fully covered by the existing
|
||||
unique index. A row is visible while its owner sees it.
|
||||
|
||||
On a database containing one scope, the two views return the same rows; no
|
||||
code anywhere branches on scope count. Rows with a null `scopeId` — created
|
||||
behind the sync layer's back (raw SQL, seed fixtures) — are visible only to
|
||||
unscoped reads; seeding flows must stamp scopes, which ties into the
|
||||
reference-table open question.
|
||||
|
||||
Multi-scope deployments additionally gain Postgres row-level security as an
|
||||
option, since policies can reference `scopeId` directly — the package's
|
||||
isolation filter and RLS become two enforcements of one predicate.
|
||||
|
||||
## Sync path
|
||||
|
||||
Sync also consumes CRDT metadata, so it follows the same row-ownership rule
|
||||
as reads and writes:
|
||||
|
||||
- **Outbound inserts** collect `crdt_data_rows` for one scope, then fetch the
|
||||
physical domain row only if `domain.scopeId` equals that same scope. A
|
||||
stale or hostile tracker for a foreign-owned row fails the sync and records
|
||||
a durable violation before any payload is produced.
|
||||
- **Outbound updates** fetch the column value only after the field's CRDT row
|
||||
is proven to match the domain row owner. `scopeId` itself is never emitted
|
||||
as an update because it has no field metadata.
|
||||
- **Outbound deletes** are scoped by CRDT metadata; a foreign tracker can at
|
||||
worst try to produce a tombstone for a row the collecting scope never owned,
|
||||
so delete collection also verifies the domain owner when the domain row still
|
||||
exists and fails with a durable violation on mismatch.
|
||||
- **Inbound inserts and updates** sanitize any incoming `scopeId` before merge
|
||||
and stamp from the merge scope. Incoming row UUIDs that already belong to
|
||||
another scope fail the sync session as ownership collisions, record durable
|
||||
violations, and do not advance checkpoints past the rejected change.
|
||||
|
||||
This is the audit rule for all future sync helpers: no path may serialize,
|
||||
materialize, resolve conflicts for, or project a synced domain row through
|
||||
CRDT metadata unless the domain row's `scopeId` equals the effective CRDT
|
||||
scope, except explicit unscoped admin reads.
|
||||
|
||||
## API surface
|
||||
|
||||
The package is unreleased, so renames land directly, with no aliases and no
|
||||
data migration. Public API stays user-first (that is the semantics
|
||||
developers expect to write); internals and storage are named after what they
|
||||
actually key — scopes:
|
||||
|
||||
- **`transactionForUser(userId, fn)`** keeps its name and shape: it resolves
|
||||
the user's UUID to the local scope and pins the transaction to it — writes
|
||||
stamp and assert against it, reads are isolated to it. The optional
|
||||
`scopeId` parameter is reserved for shared scopes (see *Future*).
|
||||
- **`persistentUserId`** remains the way to pin a default scope outside
|
||||
explicit transactions — the standard single-user client configuration.
|
||||
- **`mergeChanges(scopeId: …)`** keys the chain being merged — the parameter
|
||||
is renamed from `userId` to what it always was.
|
||||
- **Internals and storage** adopt scope naming in one sweep: `crdt_users` →
|
||||
`crdt_scopes` (`CrdtUser` → `CrdtScope`, `uuidUserId` → `uuidScopeId`),
|
||||
`crdt_data_rows.userId` → `scopeId` (matching the domain column),
|
||||
`userForTransaction`, the effective-user helpers, and the HLC manager
|
||||
keys. Doing this now is what keeps the *Future: shared scopes* section a
|
||||
pure addition instead of a rename project.
|
||||
- **Unscoped reads** remain supported as admin reads.
|
||||
|
||||
Consequences for application code, stated as contract:
|
||||
|
||||
1. **Every synced table declares `id: UuidValue` and `scopeId: int?`.**
|
||||
Startup fails with a paste-ready error otherwise.
|
||||
2. **`id` remains globally unique per synced table.** The same
|
||||
`(table, uuidRowId)` cannot exist in two scopes. If merge observes that
|
||||
UUID already owned by another scope, it records a durable ownership
|
||||
violation and fails the sync.
|
||||
3. **`scopeId` round-trips the way it came in** — left null, the sync layer
|
||||
stamps it internally and returns it null (scoped reads return it null
|
||||
too); provided explicitly, it must match the effective scope or the
|
||||
operation throws, and it stays visible on the way out. The column is
|
||||
immutable after insert.
|
||||
4. **Scoped reads are isolated to their scope.** Reading across scopes is an
|
||||
explicit unscoped (admin) read.
|
||||
5. **Merge-inserts colliding with another scope's row fail the sync**, never
|
||||
apply over the owner's data, and never advance the sender past the rejected
|
||||
change.
|
||||
6. **Foreign keys on synced tables target same-scope rows.** Cross-scope
|
||||
references are repaired by the existing FK override machinery instead of
|
||||
being linked.
|
||||
7. **Application unique indexes on synced tables include `scopeId`.** Global
|
||||
unique indexes are rejected at startup until a deterministic cross-scope
|
||||
policy exists, except FK-only indexes whose referenced tables are synced.
|
||||
|
||||
## Implementation plan
|
||||
|
||||
Each phase lands independently green; later phases depend on earlier ones
|
||||
only where stated. No generator or Serverpod fork changes are required.
|
||||
|
||||
- **Phase 1 — naming, contract, validation.** The user→scope renames in
|
||||
internals and storage (above), done first so later phases build on the
|
||||
final names. `CrdtSchemaRegistry`: the `scopeId` presence/type check with
|
||||
the `serverOnly` hint, and the global unique-index rejection. Reserved-name
|
||||
exclusions: `crdt_schema_columns` registration, field-level merge, update
|
||||
recording, inbound merge sanitization, and outbound sync serialization.
|
||||
Test models gain `scopeId`.
|
||||
- **Phase 2 — write-path enforcement.** Stamp-or-assert in the proxy insert
|
||||
paths (dynamic dispatch on the known field); `scopeId` rejected from
|
||||
`updateWhere`/`updateById` column values and excluded from model-update SET
|
||||
lists; ownership assertion on update, delete, reinsert, and upsert; atomic
|
||||
merge-insert ownership check with durable fail-fast collision handling,
|
||||
including primary-key race handling; merge stamping from the merging scope;
|
||||
FK same-scope assertion and merge repair routing; unique resolver limited
|
||||
to scoped indexes, with ordinary global indexes rejected by validation.
|
||||
- **Phase 3 — read paths.** Isolation filter for scoped reads; row-keyed
|
||||
visibility probe for unscoped reads; stripping of `scopeId` from scoped
|
||||
root and include-graph read results; outbound sync owner checks before
|
||||
serializing inserts, updates, or deletes; delete the `GROUP BY`/`MIN` form
|
||||
and the server-only-index TODO from `tombstone.dart`.
|
||||
- **Phase 4 — tests and benchmarks.** The plan below, plus updating
|
||||
`TombstoneScopeBenchmark` to ownership-conformant seeding (one tracker per
|
||||
row); its current second-tracker fixtures survive only as the enforcement
|
||||
scenario.
|
||||
|
||||
### Test plan
|
||||
|
||||
Because no code branches on scope count or deployment role, mechanisms are
|
||||
tested **once**, not per combination:
|
||||
|
||||
- **Mechanism suites (run once).** All existing CRUD, tombstone visibility,
|
||||
unique-conflict, FK-invariant, merge, and sync suites continue on the
|
||||
uniform model; test models declare `scopeId` and are otherwise unchanged
|
||||
(the stamp rule means call sites do not change).
|
||||
- **Enforcement tests.** Stamping on insert; assert-on-mismatch throwing for
|
||||
insert and update; immutability of the column; round-trip symmetry (null
|
||||
in → null out, explicit in → kept, per row in batches); stripping on
|
||||
scoped root and included read results and retention on unscoped reads; the
|
||||
caller's input instance left unmutated; merge-insert collision fails sync,
|
||||
records a durable violation, and covers the primary-key race regression;
|
||||
stale foreign tracker fails before producing outbound insert/update/delete
|
||||
payloads and records a durable violation (the multi-user fixtures in
|
||||
`tombstone_scope_test.dart` become these tests); FK cross-scope repair;
|
||||
schema validation rejects ordinary global unique indexes and accepts
|
||||
FK-only unique indexes whose referenced tables are synced.
|
||||
- **Combination smoke tests (one each, not per mechanism).** A two-scope
|
||||
database proving read isolation and per-scope uniqueness on SQLite; one
|
||||
`withServerpod` sync roundtrip proving the wiring end to end. These exist
|
||||
to prove deployment shapes compose, not to re-verify mechanisms.
|
||||
|
||||
## Future: shared scopes
|
||||
|
||||
`crdt_scopes` is the entity everything is keyed by: sync, locking, HLC
|
||||
chains, and visibility. Sharing becomes an indirection in front of it, not a
|
||||
change to row ownership:
|
||||
|
||||
- an authenticated identity maps to one personal scope plus any number of
|
||||
shared scopes;
|
||||
- a shared list/document/workspace is a scope that several identities
|
||||
subscribe to and sync;
|
||||
- every row still has exactly one owning scope (its `scopeId`), so merging
|
||||
stays one chain per row and nothing in this document's enforcement
|
||||
changes.
|
||||
|
||||
This is the standard shape of sharing in local-first systems, and it is the
|
||||
reason the column and concept are named `scope` rather than `user`. The rest
|
||||
of this section sketches the direction in enough detail to keep today's
|
||||
decisions compatible with it; it is not committed design.
|
||||
|
||||
### Users authenticate, scopes act
|
||||
|
||||
The two concepts separate cleanly:
|
||||
|
||||
- a **user** is an authentication identity — what endpoints and the sync
|
||||
handshake verify;
|
||||
- a **scope** is the unit of replication and ownership — what transactions
|
||||
act in, what chains and tombstones are keyed by, what `scopeId`
|
||||
references.
|
||||
|
||||
Two conventions bridge them with no storage migration:
|
||||
|
||||
1. **A user's personal scope has the user's own UUID.** This is already true
|
||||
(the scope UUID keys the chain today), so every deployment without
|
||||
sharing is the degenerate case "each user has exactly one scope: their
|
||||
personal one". Shared scopes are new `crdt_scopes` rows whose UUID is no
|
||||
user's id.
|
||||
2. **Membership** is a server-side relation, `crdt_scope_members(scopeId,
|
||||
userUuid, role?)`, managed by application code (invitations and roles are
|
||||
app domain). Personal-scope membership is implicit; no row is stored for
|
||||
it.
|
||||
|
||||
### The transaction API
|
||||
|
||||
`transactionForUser` keeps its name and user-first semantics — that is what
|
||||
developers expect to write — and gains an optional scope:
|
||||
|
||||
```dart
|
||||
db.transactionForUser(userId, fn); // acts in the personal scope
|
||||
db.transactionForUser(userId, fn, scopeId: listId); // acts in a shared scope
|
||||
```
|
||||
|
||||
- Without `scopeId`, the scope resolves to the user's personal scope — by
|
||||
convention 1, simply `userId` itself, which is exactly today's behavior.
|
||||
- With `scopeId`, the `userId` argument is **not dead weight**: the package
|
||||
asserts `membership(userId, scopeId)` before acting. The parameter pair
|
||||
reads as "authenticated as `userId`, acting in `scopeId`, verified
|
||||
member" — the same predicate row-level security policies mirror. The
|
||||
caller remains responsible for `userId` being authenticated, exactly as
|
||||
today.
|
||||
- A transaction acts in **exactly one scope**. Cross-scope writes need
|
||||
separate transactions; this is deliberate honesty about CRDT semantics,
|
||||
since two scopes' chains replicate independently and remote replicas can
|
||||
never observe a cross-scope write atomically anyway.
|
||||
|
||||
### Reads: membership-wide; writes: scope-pinned
|
||||
|
||||
A user-scoped read in a sharing world naturally means "rows in any scope I am
|
||||
a member of": the isolation filter becomes `scopeId IN (<member scopes>)`
|
||||
instead of `scopeId = <scope>`. The visibility probe is the row-keyed form —
|
||||
the same probe as unscoped admin reads — so sharing simplifies the predicate
|
||||
matrix rather than growing it: reads differ only in their membership filter
|
||||
(one scope, a user's scopes, or none for admin).
|
||||
|
||||
This is also exactly the shape of a Postgres row-level security policy
|
||||
(`scopeId IN (SELECT scopeId FROM crdt_scope_members WHERE userUuid =
|
||||
current_setting(...))`), which is the standard practice for multi-tenant
|
||||
isolation and on the Serverpod backlog; the package's application-level
|
||||
filter and RLS become two enforcements of one predicate.
|
||||
|
||||
### Sync and clients
|
||||
|
||||
- A device syncs one chain per scope it holds: the sync session (or one
|
||||
session per scope) enumerates the authenticated user's member scopes,
|
||||
verified server-side against `crdt_scope_members` at the handshake; the
|
||||
membership table itself is not synced.
|
||||
- Every database is already sharing-capable schema-wise — the column is
|
||||
universal — so a sharing-enabled client simply holds several scopes' rows.
|
||||
One database file per scope remains a valid alternative layout.
|
||||
- Checkpoints stay per `(scope, node)`; a device is a node in each scope's
|
||||
chain it participates in.
|
||||
|
||||
### What sharing does not change
|
||||
|
||||
Every enforcement rule in this document survives unchanged: `(table,
|
||||
uuidRowId)` stays globally identified, rows have exactly one owning scope,
|
||||
merge-insert ownership checks compare scopes, unique conflict groups form
|
||||
within a scope, ordinary global unique indexes stay rejected, and
|
||||
stamp-or-assert applies with the transaction's resolved scope. Sharing adds a
|
||||
membership layer in front of scopes; it does not touch row ownership.
|
||||
|
||||
## Resolved during review
|
||||
|
||||
Decisions that were open questions in earlier drafts, kept here so they are
|
||||
not relitigated by accident:
|
||||
|
||||
- **Optional vs. required `scopeId`:** required, on every synced table and
|
||||
in both schemas. The optional-column draft and the schema-presence-mode
|
||||
draft were both rejected: each created a second ownership architecture (or
|
||||
a second schema shape) that every mechanism would have to support and
|
||||
test. Requiring the column everywhere deleted the mode system, the
|
||||
index-projection generator feature, and the per-mode test matrix, at the
|
||||
cost of ~1–2 bytes per row and a visible read-only field on domain models.
|
||||
- **Global UUID row identity:** retained. Serverpod's generated domain model
|
||||
and ORM expect one primary key column, so the package does not introduce
|
||||
composite `(scopeId, id)` identity or a surrogate physical row id. A
|
||||
duplicate `(table, uuidRowId)` in another scope is an ownership collision,
|
||||
not a valid second row.
|
||||
- **`scope=serverOnly` on the column:** rejected (would reintroduce
|
||||
asymmetric schemas); validation forbids it.
|
||||
- **Stamp-or-assert as the single write path:** chosen over required
|
||||
explicit passing (ceremony without added security — the package must
|
||||
verify any supplied value anyway, and the persistent-user client pattern
|
||||
has no transaction object in hand) and over dual documented paths (two
|
||||
mental models). Explicit callers still get verified explicitness through
|
||||
the assert arm.
|
||||
- **Nullability:** nullable, permanently. The int is database-local, so a
|
||||
non-nullable field would force clients to serialize a meaningless local
|
||||
id into endpoint payloads; null honestly means "not mine to say". The
|
||||
sync layer guarantees the value on every synced row.
|
||||
- **Wire transport prevented by construction, respecting the caller:** the
|
||||
value leaves the way it came in — stamped values are stripped from
|
||||
returned rows and scoped reads, including include graphs; explicitly
|
||||
provided (and verified) values are kept on the way out; unscoped admin
|
||||
reads keep the value as the only per-row ownership information server-side;
|
||||
outbound sync never serializes it. Mechanism: in-place dynamic mutation on
|
||||
package-owned instances (preferred over `copyWith`, the fallback only
|
||||
under immutability; `patchTableRow` is reserved for the merge path, where
|
||||
field names are data-driven and its 50–200× cost is justified), never
|
||||
leaving the caller's input observably mutated.
|
||||
- **Unscoped-read semantics:** an unscoped read is an admin view; a row is
|
||||
visible while its owner sees it. Scoped reads are isolated to their scope
|
||||
— confirmed as the central goal of the design.
|
||||
- **Fail vs. skip vs. re-key on colliding inserts:** fail and record a
|
||||
durable violation. Skipping can silently consume a client's rejected change
|
||||
and advance sync state without any recovery policy; re-keying row identity
|
||||
would break the incoming chain's own FK references.
|
||||
- **Metadata backstop index:** dropped only with the consumer audit rule:
|
||||
every read, write, sync, merge, FK, and unique path that consumes CRDT row
|
||||
metadata must prove `domain.scopeId` equals the effective metadata scope
|
||||
before materializing domain data.
|
||||
- **Audit surface:** a durable `crdt_sync_integrity_violations` table is part
|
||||
of the terminal sync-integrity contract. Session logs may mirror it, but
|
||||
they are not the source of truth.
|
||||
- **No compatibility aliases:** the package is unreleased, so renames
|
||||
(storage and API parameters) land without aliases. The storage migration
|
||||
preserves existing durable violation rows by mapping prior operation strings
|
||||
to the new enum names.
|
||||
|
||||
## Open questions
|
||||
|
||||
1. **Globally shared reference tables.** Read-only data synced to every user
|
||||
would violate "every row has one owning scope" and is out of scope; if
|
||||
the use case materializes, it likely becomes a third table kind rather
|
||||
than a relaxation of the contract. Related: rows seeded behind the sync
|
||||
layer's back carry a null `scopeId` and are visible only to admin reads.
|
||||
2. **`scopeId` naming collisions.** The name is reserved by convention; an
|
||||
application with a pre-existing, semantically different `scopeId` column
|
||||
on a synced table has no escape hatch other than renaming its column.
|
||||
Acceptable for now; revisit if it bites real users.
|
||||
3. **Membership revocation semantics (sharing).** When a user loses
|
||||
membership of a scope, the server stops streaming it at the next
|
||||
handshake, but the device still holds the scope's data and may hold
|
||||
unsynced local changes for it. Purge-vs-keep policy, and whether revoked
|
||||
unsynced changes are surfaced or dropped, need their own design.
|
||||
4. **Roles within a scope (sharing).** `crdt_scope_members.role` is sketched
|
||||
but undefined: read-only members would need write rejection at merge time
|
||||
(another fail-and-record case), and the interaction with offline-written
|
||||
changes by a demoted member overlaps with question 3.
|
||||
284
docs/sync-non-sync-relations.md
Normal file
284
docs/sync-non-sync-relations.md
Normal file
|
|
@ -0,0 +1,284 @@
|
|||
# Sync and non-sync foreign key boundaries
|
||||
|
||||
Status: proposal for a separate implementation issue.
|
||||
|
||||
## Summary
|
||||
|
||||
Foreign keys that cross the sync boundary are unsafe by default. A synced row is
|
||||
owned, collected, merged, hidden, and repaired through CRDT metadata. A
|
||||
non-synced row has none of that state. Letting arbitrary database relations
|
||||
cross that boundary lets the database mutate or constrain synced rows without
|
||||
the CRDT layer recording the corresponding facts.
|
||||
|
||||
There are only two allowed sync/non-sync relation shapes:
|
||||
|
||||
1. the package-owned ownership link from each synced domain row to
|
||||
`crdt_scopes`;
|
||||
2. a synced row referencing a non-synced, server-authoritative reference table,
|
||||
but only after the sync protocol can provision that reference data before
|
||||
CRDT merge.
|
||||
|
||||
The ownership link is:
|
||||
|
||||
```yaml
|
||||
fields:
|
||||
id: UuidValue?, defaultPersist=random_v7
|
||||
### Owner scope of this row. Maintained by the CRDT sync layer.
|
||||
scopeId: int?, relation(optional, parent=crdt_scopes, onDelete=Cascade)
|
||||
```
|
||||
|
||||
`scopeId == null` remains valid persisted state, but it means the row is
|
||||
orphaned from CRDT ownership: it is never synced, never collected for outbound
|
||||
sync, and not removed by deleting a scope. `scopeId != null` means the row is
|
||||
owned by that scope, participates in CRDT, and is physically purged when the
|
||||
scope is deleted.
|
||||
|
||||
All other foreign keys between synced and non-synced tables must fail
|
||||
`initialize()`. Until the reference-data sync phase exists, `initialize()` must
|
||||
also reject server-authoritative reference-table relations even when they use
|
||||
`onDelete=Restrict`.
|
||||
|
||||
## Table classes
|
||||
|
||||
- **Synced table:** any application table passed to the CRDT `syncTables`
|
||||
configuration.
|
||||
- **Non-synced table:** any other application table known to the Serverpod
|
||||
serialization manager.
|
||||
- **Protocol-provisioned reference table:** a non-synced table whose rows are
|
||||
authored by the server, sent through a reference-data phase in the sync
|
||||
protocol, applied before CRDT merge, and not removed while synced rows
|
||||
reference them.
|
||||
- **Package metadata table:** CRDT infrastructure tables owned by this package,
|
||||
especially `crdt_scopes`, `crdt_nodes`, `crdt_data_rows`,
|
||||
`crdt_data_fields`, `crdt_data_tombstone`, and FK projection tables.
|
||||
|
||||
`crdt_scopes` is not a synced domain table. The `scopeId -> crdt_scopes.id`
|
||||
relation is a special package-owned purge boundary, not an application-level
|
||||
foreign key.
|
||||
|
||||
## Allowed relations
|
||||
|
||||
### Synced -> synced
|
||||
|
||||
Allowed under the normal FK invariant rules:
|
||||
|
||||
- the child and parent are both in `syncTables`;
|
||||
- supported FK shapes are single-column child references to single-column
|
||||
parent references;
|
||||
- local writes assert same-scope visibility;
|
||||
- merge-time FK projection repairs hidden, missing, or foreign-scope targets.
|
||||
|
||||
### Non-synced -> non-synced
|
||||
|
||||
Ignored by CRDT. These relations are normal application/database relations.
|
||||
|
||||
### Synced -> `crdt_scopes` through `scopeId`
|
||||
|
||||
Allowed, and required once this proposal is implemented. The relation must be
|
||||
exactly:
|
||||
|
||||
- child table is a synced domain table;
|
||||
- child column is the reserved `scopeId` column;
|
||||
- child column type is nullable `int`;
|
||||
- parent table is `crdt_scopes`;
|
||||
- parent column is `id`;
|
||||
- `onDelete` is `Cascade`;
|
||||
- `onUpdate` is `NoAction` or unspecified database default.
|
||||
|
||||
This relation exists only to make terminal scope purge database-enforced. It
|
||||
does not change normal CRDT deletes, which remain soft deletes recorded in
|
||||
CRDT metadata.
|
||||
|
||||
### Future: synced -> protocol-provisioned reference table
|
||||
|
||||
Allowed when every referenced row is server-authoritative reference data, such
|
||||
as a global category table used to group synced rows, and when the sync
|
||||
protocol provisions that table before applying CRDT changes. The relation must
|
||||
be strictly constraining: it may prevent reference-table deletion, but it must
|
||||
never cause the database to mutate synced rows behind the CRDT recorder.
|
||||
|
||||
This exception must not be implemented as a schema-only relaxation. The same
|
||||
implementation issue that permits this relation must also add the reference-data
|
||||
sync phase described below.
|
||||
|
||||
The allowed shape is:
|
||||
|
||||
- child table is a synced domain table;
|
||||
- parent table is a non-synced application table;
|
||||
- parent table is declared in sync configuration as protocol-provisioned
|
||||
reference data;
|
||||
- the FK is single-column child reference to single-column parent reference;
|
||||
- `onDelete` is exactly `Restrict`;
|
||||
- `onUpdate` is `Restrict`, `NoAction`, or unspecified database default;
|
||||
- the referenced parent key is treated as immutable by application policy;
|
||||
- referenced parent rows are applied before any local insert, local update, or
|
||||
incoming merge can use them.
|
||||
|
||||
`NoAction` is intentionally not accepted for `onDelete`: the delete must fail
|
||||
immediately rather than being deferrable to transaction end. `Cascade`,
|
||||
`SetNull`, and `SetDefault` are forbidden because they can physically delete or
|
||||
mutate synced rows without CRDT tombstones or field HLCs.
|
||||
|
||||
This exception does not make the reference table offline-first. Creating,
|
||||
renaming, deleting, or otherwise syncing reference rows is outside CRDT. The
|
||||
server authors reference data, and clients cache the latest server snapshot or
|
||||
delta before CRDT merge.
|
||||
|
||||
This exception also does not expand the global unique-index exception. A unique
|
||||
index involving a FK to a protocol-provisioned reference table is
|
||||
application-level uniqueness and must include `scopeId` unless another explicit
|
||||
policy is added.
|
||||
|
||||
## Reference-data sync phase
|
||||
|
||||
The sync protocol must treat server-authoritative reference data as an ordered
|
||||
precondition for CRDT merge:
|
||||
|
||||
1. The sync handshake includes the client's reference-data versions or hashes.
|
||||
2. The server sends missing reference-data snapshots or deltas before CRDT
|
||||
changes that may reference them.
|
||||
3. The client applies those reference rows to local non-synced tables.
|
||||
4. Only after the reference-data phase succeeds does the client apply incoming
|
||||
CRDT operations.
|
||||
5. If an incoming CRDT row references missing reference data after that phase,
|
||||
the sync fails as a protocol violation. FK projection must not repair it.
|
||||
|
||||
Reference data has server-authoritative replacement/delta semantics, not CRDT
|
||||
semantics. Clients do not author reference rows, and conflicts are not resolved
|
||||
through HLCs, unique conflict projection, or FK projection.
|
||||
|
||||
## Forbidden relations
|
||||
|
||||
### Non-synced -> synced
|
||||
|
||||
Always forbidden.
|
||||
|
||||
A non-synced child has no CRDT row, no scope ownership, and no merge history.
|
||||
If it references a synced parent, then a CRDT soft delete can hide the parent
|
||||
while the database FK still sees the physical parent row. Delete actions and
|
||||
visibility checks become incoherent, and the current recorder can also reach
|
||||
paths that assume the child table has CRDT schema metadata.
|
||||
|
||||
### Synced -> other non-synced application table
|
||||
|
||||
Forbidden, except for:
|
||||
|
||||
- the special `scopeId -> crdt_scopes.id` relation;
|
||||
- the protocol-provisioned reference-table relation above.
|
||||
|
||||
The apparent safe use case is an app user/auth table, because that table is
|
||||
server-authoritative and must exist before offline data can be synced. That can
|
||||
fit the protocol-provisioned reference-table rule only when user rows are
|
||||
provisioned by the sync reference-data phase and never deleted while synced rows
|
||||
reference them. If account deletion must erase synced data, it should not rely
|
||||
on a direct app-user FK cascade. A direct unrestricted FK from synced rows to a
|
||||
non-synced table is unsafe:
|
||||
|
||||
- merge inserts and updates cannot project or repair missing non-synced
|
||||
parents;
|
||||
- database FK failures become raw merge failures, not deterministic CRDT
|
||||
projection;
|
||||
- deleting the non-synced parent with `CASCADE`, `SET NULL`, or `SET DEFAULT`
|
||||
can change synced child rows without CRDT tombstones or field HLCs;
|
||||
- `NoAction` is not strict enough for this boundary because it can be
|
||||
deferrable.
|
||||
|
||||
The app-user relation should be modeled outside synced domain rows. Account
|
||||
deletion should revoke the user and purge the user's `CrdtScope`, not rely on
|
||||
arbitrary application FKs into synced data.
|
||||
|
||||
### Synced -> package metadata table other than `crdt_scopes`
|
||||
|
||||
Forbidden. Domain rows must not reference CRDT metadata internals such as nodes,
|
||||
schema rows, field rows, tombstones, or ownership violations.
|
||||
|
||||
### Non-synced -> package metadata table
|
||||
|
||||
Forbidden unless the relation is part of a future explicit package API. An
|
||||
application table must not depend on metadata rows whose lifecycle is controlled
|
||||
by sync, merge, or scope purge.
|
||||
|
||||
## Scope purge semantics
|
||||
|
||||
Deleting a scope is a terminal GDPR/account-erasure operation, not a CRDT
|
||||
operation.
|
||||
|
||||
A safe purge workflow is:
|
||||
|
||||
1. Revoke the account or scope so no new sync session can start.
|
||||
2. Mark the scope UUID as deleted, or otherwise make auth reject future
|
||||
`getOrCreate(uuidScopeId)` attempts for that UUID.
|
||||
3. Delete `crdt_scopes.id`.
|
||||
4. Let database cascades remove:
|
||||
- synced domain rows with `scopeId = deletedScope.id`;
|
||||
- `crdt_nodes` for that scope;
|
||||
- `crdt_data_rows` for that scope;
|
||||
- row fields, tombstones, and FK projection rows through their existing
|
||||
metadata cascades.
|
||||
5. Delete or anonymize diagnostics that intentionally outlive metadata
|
||||
cascades, such as durable ownership violation rows. Those rows store
|
||||
denormalized table names and scope UUIDs only; they do not reference
|
||||
purged metadata through foreign keys.
|
||||
|
||||
If an old offline client later reconnects with data for a purged scope, sync
|
||||
must fail with a terminal "scope deleted/reset required" response. The server
|
||||
must not recreate the scope and must not accept the old CRDT chain, or erased
|
||||
data can be resurrected.
|
||||
|
||||
Rows with `scopeId == null` are not affected by scope purge. They are
|
||||
application-owned orphan/admin rows and remain outside CRDT.
|
||||
|
||||
## Initialize validation
|
||||
|
||||
`initialize()` must inspect all generated table definitions and classify every
|
||||
foreign key by child and parent table class.
|
||||
|
||||
Validation rules:
|
||||
|
||||
- `synced -> synced`: allowed only when the existing FK invariant support can
|
||||
handle the FK shape.
|
||||
- `non-synced -> non-synced`: ignored.
|
||||
- `synced -> crdt_scopes` through reserved `scopeId`: allowed only when it
|
||||
matches the exact allowed shape above.
|
||||
- `synced -> protocol-provisioned reference table`: allowed only when the
|
||||
reference-data sync phase is implemented, the parent table is declared as
|
||||
protocol-provisioned reference data, and the relation matches the exact
|
||||
allowed shape above. Without that protocol support, reject it even when the
|
||||
FK uses `onDelete=Restrict`.
|
||||
- every other `synced <-> non-synced` or `domain <-> package metadata` FK:
|
||||
fail initialization with an error naming the offending constraint.
|
||||
|
||||
The error message should explain the allowed alternatives:
|
||||
|
||||
- make both tables synced;
|
||||
- make both tables non-synced;
|
||||
- remove the relation;
|
||||
- use the future reference-data sync phase with `onDelete=Restrict` for
|
||||
immutable server-authoritative reference data;
|
||||
- for account deletion, use the package-owned scope purge relation instead of
|
||||
an application FK.
|
||||
|
||||
## Tests
|
||||
|
||||
The implementation issue should add schema/initialize tests for:
|
||||
|
||||
- accepting `scopeId: int?, relation(optional, parent=crdt_scopes,
|
||||
onDelete=Cascade)` on every synced table;
|
||||
- rejecting `sync -> non-sync` restricted reference-table FKs with
|
||||
`onDelete=Restrict` while the reference-data sync phase is absent;
|
||||
- accepting `sync -> non-sync` protocol-provisioned reference-table FKs with
|
||||
`onDelete=Restrict` after the reference-data sync phase is implemented;
|
||||
- applying reference-data deltas before CRDT merge when incoming rows reference
|
||||
those rows;
|
||||
- failing sync as a protocol violation when incoming CRDT rows reference
|
||||
missing reference data after the reference-data phase;
|
||||
- rejecting `non-sync -> sync`;
|
||||
- rejecting `sync -> non-sync` application FKs with `NoAction`, `SetNull`,
|
||||
`SetDefault`, and `Cascade`;
|
||||
- rejecting global unique indexes over protocol-provisioned reference FK
|
||||
columns unless the index includes `scopeId`;
|
||||
- rejecting synced domain relations to package metadata tables other than
|
||||
`crdt_scopes`;
|
||||
- preserving orphan rows with `scopeId == null` across scope purge;
|
||||
- deleting a scope removes owned domain rows and CRDT metadata for that scope;
|
||||
- stale clients for a purged scope cannot recreate the scope or sync old data.
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
export 'src/crdt/client_sync.dart';
|
||||
export 'src/crdt/exceptions.dart';
|
||||
export 'src/crdt/exceptions.dart' hide PendingOutboundIntegrityViolation;
|
||||
export 'src/crdt/merge.dart';
|
||||
export 'src/crdt/sync.dart';
|
||||
export 'src/database/database.dart';
|
||||
|
|
@ -7,4 +7,4 @@ export 'src/database/recorder.dart';
|
|||
export 'src/database/schema.dart';
|
||||
export 'src/database/session.dart' show CrdtDatabaseAccess, CrdtDatabaseSession;
|
||||
export 'src/managers/hlc.dart';
|
||||
export 'src/managers/user.dart';
|
||||
export 'src/managers/scope.dart';
|
||||
|
|
|
|||
6
packages/serverpod_offline_sync_client/lib/migrations/analysis_options.yaml
generated
Normal file
6
packages/serverpod_offline_sync_client/lib/migrations/analysis_options.yaml
generated
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
analyzer:
|
||||
errors:
|
||||
camel_case_types: ignore
|
||||
|
||||
formatter:
|
||||
trailing_commas: preserve
|
||||
|
|
@ -1,3 +1,5 @@
|
|||
import 'package:meta/meta.dart';
|
||||
import 'package:serverpod_offline_sync_shared/serverpod_offline_sync_shared.dart';
|
||||
import 'package:serverpod_serialization/serverpod_serialization.dart';
|
||||
|
||||
import '../protocol/protocol.dart';
|
||||
|
|
@ -82,6 +84,60 @@ final class SyncTablesHashMismatchException extends CrdtSyncException {
|
|||
'schema version before syncing.';
|
||||
}
|
||||
|
||||
/// Thrown when sync observes terminal CRDT metadata/domain-row inconsistency.
|
||||
final class CrdtSyncIntegrityViolationException extends CrdtSyncException {
|
||||
/// Creates a [CrdtSyncIntegrityViolationException].
|
||||
const CrdtSyncIntegrityViolationException(this.violation);
|
||||
|
||||
/// The integrity violation that caused sync to stop.
|
||||
final CrdtSyncIntegrityViolation violation;
|
||||
|
||||
@override
|
||||
String toString() {
|
||||
final persisted = violation.id == null
|
||||
? ''
|
||||
: ' Persisted violation id: ${violation.id}.';
|
||||
final row = '${violation.domainTableName}.${violation.uuidRowId}';
|
||||
return switch (violation.type) {
|
||||
CrdtSyncViolationType.ownershipCollision =>
|
||||
'CrdtSyncIntegrityViolationException: '
|
||||
'${violation.type.name}/${violation.operation.name} for $row '
|
||||
'belongs to scope ${violation.ownerScopeUuid}, but sync attempted '
|
||||
'scope ${violation.incomingScopeUuid}.$persisted',
|
||||
CrdtSyncViolationType.missingDomainRow =>
|
||||
'CrdtSyncIntegrityViolationException: '
|
||||
'${violation.type.name}/${violation.operation.name} references '
|
||||
'missing domain row $row for scope '
|
||||
'${violation.incomingScopeUuid}.$persisted',
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
@internal
|
||||
final class PendingOutboundIntegrityViolation implements Exception {
|
||||
const PendingOutboundIntegrityViolation({
|
||||
required this.crdtDataRowId,
|
||||
required this.type,
|
||||
required this.operation,
|
||||
required this.tableName,
|
||||
required this.rowId,
|
||||
required this.ownerScopeId,
|
||||
required this.incomingScopeUuid,
|
||||
required this.uuidNodeId,
|
||||
this.hlc,
|
||||
});
|
||||
|
||||
final int? crdtDataRowId;
|
||||
final CrdtSyncViolationType type;
|
||||
final CrdtSyncViolationOperation operation;
|
||||
final String tableName;
|
||||
final UuidValue rowId;
|
||||
final int? ownerScopeId;
|
||||
final UuidValue incomingScopeUuid;
|
||||
final UuidValue uuidNodeId;
|
||||
final Hlc? hlc;
|
||||
}
|
||||
|
||||
extension on Type {
|
||||
/// The name of the class without the leading underscore and without the 'Impl' suffix.
|
||||
String get className {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,54 @@
|
|||
import 'package:serverpod_database/serverpod_database.dart';
|
||||
|
||||
import '../protocol/protocol.dart';
|
||||
|
||||
/// Persists or updates the durable row for an integrity violation.
|
||||
Future<CrdtSyncIntegrityViolation> recordCrdtSyncIntegrityViolation(
|
||||
DatabaseSession session, {
|
||||
required CrdtSyncIntegrityViolation violation,
|
||||
Transaction? transaction,
|
||||
}) async {
|
||||
final existing = await CrdtSyncIntegrityViolation.db.findFirstRow(
|
||||
session,
|
||||
where: (t) =>
|
||||
t.type.equals(violation.type) &
|
||||
t.operation.equals(violation.operation) &
|
||||
t.domainTableName.equals(violation.domainTableName) &
|
||||
t.uuidRowId.equals(violation.uuidRowId) &
|
||||
t.ownerScopeUuid.equals(violation.ownerScopeUuid) &
|
||||
t.incomingScopeUuid.equals(violation.incomingScopeUuid),
|
||||
transaction: transaction,
|
||||
);
|
||||
|
||||
final now = DateTime.now().toUtc();
|
||||
final newViolation = existing != null
|
||||
? existing.copyWith(
|
||||
crdtDataRowId: violation.crdtDataRowId ?? existing.crdtDataRowId,
|
||||
uuidNodeId: violation.uuidNodeId ?? existing.uuidNodeId,
|
||||
hlcDatetime: violation.hlcDatetime,
|
||||
hlcCounter: violation.hlcCounter,
|
||||
lastSeenAt: now,
|
||||
occurrences: existing.occurrences + 1,
|
||||
)
|
||||
: violation.copyWith(
|
||||
firstSeenAt: now,
|
||||
lastSeenAt: now,
|
||||
occurrences: 1,
|
||||
);
|
||||
|
||||
final persisted = await CrdtSyncIntegrityViolation.db.upsertRow(
|
||||
session,
|
||||
newViolation,
|
||||
conflictColumns: (t) => [
|
||||
t.type,
|
||||
t.operation,
|
||||
t.domainTableName,
|
||||
t.uuidRowId,
|
||||
t.ownerScopeUuid,
|
||||
t.incomingScopeUuid,
|
||||
],
|
||||
transaction: transaction,
|
||||
);
|
||||
|
||||
return persisted!;
|
||||
}
|
||||
|
|
@ -5,16 +5,23 @@ import 'package:serverpod_offline_sync_shared/serverpod_offline_sync_shared.dart
|
|||
import 'package:uuid/uuid.dart';
|
||||
|
||||
import '../database/database.dart';
|
||||
import '../managers/user.dart';
|
||||
import '../managers/scope.dart';
|
||||
import '../protocol/protocol.dart';
|
||||
import '../utils/case_when.dart' show Case;
|
||||
import 'exceptions.dart';
|
||||
import 'extensions.dart';
|
||||
import 'integrity_violation.dart';
|
||||
import 'merge.dart';
|
||||
|
||||
/// Callback function for when a merge is successful.
|
||||
typedef CrdtSyncOnMergeSuccess = FutureOr<void> Function(Hlc syncedHlc);
|
||||
|
||||
/// A tuple representing the ownership of a domain row.
|
||||
typedef DomainRowOwner = ({bool exists, int? scopeId});
|
||||
|
||||
/// A cache of domain row owners by table name and row id.
|
||||
typedef DomainRowOwnerCache = Map<(String, UuidValue), DomainRowOwner>;
|
||||
|
||||
/// The shared CRDT synchronization logic used by both client and server nodes.
|
||||
class CrdtSync {
|
||||
/// Creates a new [CrdtSync] instance.
|
||||
|
|
@ -142,10 +149,18 @@ class CrdtSync {
|
|||
required UuidValue peerNodeId,
|
||||
required List<Hlc> nodeCheckpoints,
|
||||
}) async* {
|
||||
final crdtUser = await CrdtUserManager(session).getOrCreate(userId);
|
||||
yield* _streamInserts(session, crdtUser, nodeCheckpoints);
|
||||
yield* _streamUpdates(session, crdtUser, nodeCheckpoints);
|
||||
yield* _streamDeletes(session, crdtUser, nodeCheckpoints);
|
||||
final crdtUser = await CrdtScopeManager(session).getOrCreate(userId);
|
||||
try {
|
||||
await for (final change in _streamPendingChanges(
|
||||
session,
|
||||
crdtUser,
|
||||
nodeCheckpoints,
|
||||
)) {
|
||||
yield change;
|
||||
}
|
||||
} on PendingOutboundIntegrityViolation catch (violation) {
|
||||
await _recordAndThrowIntegrityViolation(session, violation);
|
||||
}
|
||||
}
|
||||
|
||||
/// Creates the [CrdtSyncSinceHlc] checkpoint for the post-connect handshake.
|
||||
|
|
@ -157,12 +172,12 @@ class CrdtSync {
|
|||
required UuidValue userId,
|
||||
required UuidValue peerNodeId,
|
||||
}) async {
|
||||
final crdtUser = await CrdtUserManager(session).getOrCreate(userId);
|
||||
final crdtUser = await CrdtScopeManager(session).getOrCreate(userId);
|
||||
|
||||
final nodes = await CrdtNode.db.find(
|
||||
session,
|
||||
where: (t) =>
|
||||
t.userId.equals(crdtUser.id) &
|
||||
t.scopeId.equals(crdtUser.id) &
|
||||
t.uuidNodeId.notEquals(crdtUser.currentNode!.uuidNodeId),
|
||||
);
|
||||
|
||||
|
|
@ -194,7 +209,7 @@ class CrdtSync {
|
|||
if (mergeSet.isEmpty) return null;
|
||||
final maxSyncedHlc = mergeSet.maxHlc;
|
||||
final crdtDb = await _openCrdtDatabase(session);
|
||||
await crdtDb.mergeChanges(mergeSet, userId: userId);
|
||||
await crdtDb.mergeChanges(mergeSet, scopeId: userId);
|
||||
if (maxSyncedHlc != null) {
|
||||
await crdtDb.recordSyncCheckpoint(otherNodeId, maxSyncedHlc, userId: userId);
|
||||
}
|
||||
|
|
@ -255,7 +270,7 @@ class CrdtSync {
|
|||
|
||||
var sessionCompleted = false;
|
||||
try {
|
||||
final crdtUser = await CrdtUserManager(session).getOrCreate(userId);
|
||||
final crdtUser = await CrdtScopeManager(session).getOrCreate(userId);
|
||||
final localNodeId = crdtUser.currentNode!.uuidNodeId;
|
||||
|
||||
yield CrdtSyncConnect(
|
||||
|
|
@ -363,10 +378,24 @@ class CrdtSync {
|
|||
return crdtDb;
|
||||
}
|
||||
|
||||
Stream<CrdtMergeChange> _streamPendingChanges(
|
||||
DatabaseSession session,
|
||||
CrdtScope crdtUser,
|
||||
List<Hlc> nodeCheckpoints,
|
||||
) async* {
|
||||
// Domain ownership is immutable while a collection runs, so read each
|
||||
// row's owner at most once across all three streams.
|
||||
final ownerCache = DomainRowOwnerCache();
|
||||
yield* _streamInserts(session, crdtUser, nodeCheckpoints, ownerCache);
|
||||
yield* _streamUpdates(session, crdtUser, nodeCheckpoints, ownerCache);
|
||||
yield* _streamDeletes(session, crdtUser, nodeCheckpoints, ownerCache);
|
||||
}
|
||||
|
||||
Stream<CrdtMergeInsert> _streamInserts(
|
||||
DatabaseSession session,
|
||||
CrdtUser crdtUser,
|
||||
CrdtScope crdtUser,
|
||||
List<Hlc> nodeCheckpoints,
|
||||
DomainRowOwnerCache ownerCache,
|
||||
) async* {
|
||||
final rows = await CrdtDataRow.db.find(
|
||||
session,
|
||||
|
|
@ -397,8 +426,35 @@ class CrdtSync {
|
|||
table,
|
||||
dartName,
|
||||
foreignKeyAttemptFieldsByRowId[row.id!],
|
||||
crdtUser.id!,
|
||||
ownerCache,
|
||||
);
|
||||
if (domainRow == null) continue;
|
||||
if (!domainRow.exists) {
|
||||
_throwPendingIntegrityViolation(
|
||||
crdtDataRowId: row.id,
|
||||
type: CrdtSyncViolationType.missingDomainRow,
|
||||
operation: CrdtSyncViolationOperation.outboundInsert,
|
||||
tableName: tableName,
|
||||
rowId: row.uuidRowId,
|
||||
ownerScopeId: null,
|
||||
incomingScopeUuid: crdtUser.uuidScopeId,
|
||||
uuidNodeId: row.node!.uuidNodeId,
|
||||
hlc: row.hlc,
|
||||
);
|
||||
}
|
||||
if (domainRow.ownerScopeId != crdtUser.id) {
|
||||
_throwPendingIntegrityViolation(
|
||||
crdtDataRowId: row.id,
|
||||
type: CrdtSyncViolationType.ownershipCollision,
|
||||
operation: CrdtSyncViolationOperation.outboundInsert,
|
||||
tableName: tableName,
|
||||
rowId: row.uuidRowId,
|
||||
ownerScopeId: domainRow.ownerScopeId,
|
||||
incomingScopeUuid: crdtUser.uuidScopeId,
|
||||
uuidNodeId: row.node!.uuidNodeId,
|
||||
hlc: row.hlc,
|
||||
);
|
||||
}
|
||||
|
||||
yield CrdtMergeInsert(
|
||||
hlcDatetime: row.hlcDatetime,
|
||||
|
|
@ -406,15 +462,16 @@ class CrdtSync {
|
|||
tableName: tableName,
|
||||
uuidRowId: row.uuidRowId,
|
||||
uuidNodeId: row.node!.uuidNodeId,
|
||||
data: domainRow,
|
||||
data: domainRow.row,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
Stream<CrdtMergeUpdate> _streamUpdates(
|
||||
DatabaseSession session,
|
||||
CrdtUser crdtUser,
|
||||
CrdtScope crdtUser,
|
||||
List<Hlc> nodeCheckpoints,
|
||||
DomainRowOwnerCache ownerCache,
|
||||
) async* {
|
||||
final fields = await CrdtDataField.db.find(
|
||||
session,
|
||||
|
|
@ -437,13 +494,41 @@ class CrdtSync {
|
|||
}
|
||||
|
||||
final columnName = field.column!.name;
|
||||
final decodedValue = await _fetchColumnValue(
|
||||
final columnValue = await _fetchOwnedColumnValue(
|
||||
session,
|
||||
tableName,
|
||||
field.row!.uuidRowId,
|
||||
columnName,
|
||||
field.foreignKey,
|
||||
crdtUser.id!,
|
||||
ownerCache,
|
||||
);
|
||||
if (!columnValue.exists) {
|
||||
_throwPendingIntegrityViolation(
|
||||
crdtDataRowId: field.row!.id,
|
||||
type: CrdtSyncViolationType.missingDomainRow,
|
||||
operation: CrdtSyncViolationOperation.outboundUpdate,
|
||||
tableName: tableName,
|
||||
rowId: field.row!.uuidRowId,
|
||||
ownerScopeId: null,
|
||||
incomingScopeUuid: crdtUser.uuidScopeId,
|
||||
uuidNodeId: field.node!.uuidNodeId,
|
||||
hlc: field.hlc,
|
||||
);
|
||||
}
|
||||
if (columnValue.ownerScopeId != crdtUser.id) {
|
||||
_throwPendingIntegrityViolation(
|
||||
crdtDataRowId: field.row!.id,
|
||||
type: CrdtSyncViolationType.ownershipCollision,
|
||||
operation: CrdtSyncViolationOperation.outboundUpdate,
|
||||
tableName: tableName,
|
||||
rowId: field.row!.uuidRowId,
|
||||
ownerScopeId: columnValue.ownerScopeId,
|
||||
incomingScopeUuid: crdtUser.uuidScopeId,
|
||||
uuidNodeId: field.node!.uuidNodeId,
|
||||
hlc: field.hlc,
|
||||
);
|
||||
}
|
||||
|
||||
yield CrdtMergeUpdate(
|
||||
hlcDatetime: field.hlcDatetime,
|
||||
|
|
@ -452,15 +537,16 @@ class CrdtSync {
|
|||
uuidRowId: field.row!.uuidRowId,
|
||||
uuidNodeId: field.node!.uuidNodeId,
|
||||
columnName: columnName,
|
||||
value: decodedValue,
|
||||
value: columnValue.value,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
Stream<CrdtMergeDelete> _streamDeletes(
|
||||
DatabaseSession session,
|
||||
CrdtUser crdtUser,
|
||||
CrdtScope crdtUser,
|
||||
List<Hlc> nodeCheckpoints,
|
||||
DomainRowOwnerCache ownerCache,
|
||||
) async* {
|
||||
final tombstones = await CrdtDataDeleted.db.find(
|
||||
session,
|
||||
|
|
@ -476,6 +562,25 @@ class CrdtSync {
|
|||
|
||||
final tableName = tombstone.row!.tbl!.name;
|
||||
if (!_syncTablesByName.containsKey(tableName)) continue;
|
||||
final owner = await _readDomainRowOwner(
|
||||
session,
|
||||
tableName,
|
||||
tombstone.row!.uuidRowId,
|
||||
ownerCache,
|
||||
);
|
||||
if (owner.exists && owner.scopeId != crdtUser.id) {
|
||||
_throwPendingIntegrityViolation(
|
||||
crdtDataRowId: tombstone.row!.id,
|
||||
type: CrdtSyncViolationType.ownershipCollision,
|
||||
operation: CrdtSyncViolationOperation.outboundDelete,
|
||||
tableName: tableName,
|
||||
rowId: tombstone.row!.uuidRowId,
|
||||
ownerScopeId: owner.scopeId,
|
||||
incomingScopeUuid: crdtUser.uuidScopeId,
|
||||
uuidNodeId: tombstone.node!.uuidNodeId,
|
||||
hlc: tombstone.hlc,
|
||||
);
|
||||
}
|
||||
|
||||
yield CrdtMergeDelete(
|
||||
hlcDatetime: tombstone.hlcDatetime,
|
||||
|
|
@ -491,10 +596,10 @@ class CrdtSync {
|
|||
|
||||
Expression _rowHlcAfterFilter(
|
||||
CrdtDataRowTable t,
|
||||
CrdtUser crdtUser,
|
||||
CrdtScope crdtUser,
|
||||
List<Hlc> nodeCheckpoints,
|
||||
) =>
|
||||
t.userId.equals(crdtUser.id) &
|
||||
t.scopeId.equals(crdtUser.id) &
|
||||
t.node.afterAnyCheckpointFilter(
|
||||
t.node.uuidNodeId,
|
||||
t.hlcDatetime,
|
||||
|
|
@ -504,10 +609,10 @@ class CrdtSync {
|
|||
|
||||
Expression _fieldHlcAfterFilter(
|
||||
CrdtDataFieldTable t,
|
||||
CrdtUser crdtUser,
|
||||
CrdtScope crdtUser,
|
||||
List<Hlc> nodeCheckpoints,
|
||||
) =>
|
||||
t.row.userId.equals(crdtUser.id) &
|
||||
t.row.scopeId.equals(crdtUser.id) &
|
||||
t.node.afterAnyCheckpointFilter(
|
||||
t.node.uuidNodeId,
|
||||
t.hlcDatetime,
|
||||
|
|
@ -517,10 +622,10 @@ class CrdtSync {
|
|||
|
||||
Expression _tombstoneHlcAfterFilter(
|
||||
CrdtDataDeletedTable t,
|
||||
CrdtUser crdtUser,
|
||||
CrdtScope crdtUser,
|
||||
List<Hlc> nodeCheckpoints,
|
||||
) =>
|
||||
t.row.userId.equals(crdtUser.id) &
|
||||
t.row.scopeId.equals(crdtUser.id) &
|
||||
t.node.afterAnyCheckpointFilter(
|
||||
t.node.uuidNodeId,
|
||||
t.hlcDatetime,
|
||||
|
|
@ -534,33 +639,45 @@ class CrdtSync {
|
|||
/// with an active override back to [CrdtDataForeignKey.attemptedValue] before
|
||||
/// deserializing. This is the inverse of inbound FK materialization: the wire
|
||||
/// payload carries attempted facts, not locally projected visible values.
|
||||
Future<dynamic> _fetchDomainRow(
|
||||
Future<({bool exists, int? ownerScopeId, dynamic row})> _fetchDomainRow(
|
||||
DatabaseSession session,
|
||||
String tableName,
|
||||
UuidValue rowId,
|
||||
Table table,
|
||||
String dartName,
|
||||
List<CrdtDataField>? foreignKeyAttemptFields,
|
||||
int scopeId,
|
||||
DomainRowOwnerCache ownerCache,
|
||||
) async {
|
||||
final cols = table.columns
|
||||
.map((column) => '"${_escapeIdentifier(column.columnName)}"')
|
||||
.join(', ');
|
||||
final encodedRowId = ValueEncoder.instance.convert(rowId);
|
||||
final encodedScopeId = ValueEncoder.instance.convert(scopeId);
|
||||
final escapedTableName = _escapeIdentifier(tableName);
|
||||
final result = await session.db.unsafeQuery(
|
||||
'SELECT $cols FROM "${_escapeIdentifier(tableName)}" '
|
||||
'WHERE "id" = $encodedRowId',
|
||||
'SELECT $cols FROM "$escapedTableName" '
|
||||
'WHERE "id" = $encodedRowId AND "scopeId" = $encodedScopeId '
|
||||
'LIMIT 1',
|
||||
);
|
||||
if (result.isEmpty) return null;
|
||||
if (result.isEmpty) {
|
||||
final owner = await _readDomainRowOwner(session, tableName, rowId, ownerCache);
|
||||
return (exists: owner.exists, ownerScopeId: owner.scopeId, row: null);
|
||||
}
|
||||
ownerCache[(tableName, rowId)] = (exists: true, scopeId: scopeId);
|
||||
|
||||
final columnMap = result.first.toColumnMap()
|
||||
// Domain columns hold visible/materialized FK values; restore attempted
|
||||
// values for override columns before building the outbound merge payload.
|
||||
..applyProjectedForeignKeyAttempts(foreignKeyAttemptFields);
|
||||
..applyProjectedForeignKeyAttempts(foreignKeyAttemptFields)
|
||||
// scopeId is local ownership metadata; it is never emitted on the wire.
|
||||
..remove('scopeId');
|
||||
|
||||
return session.db.serializationManager.deserializeByClassName({
|
||||
final row = session.db.serializationManager.deserializeByClassName({
|
||||
'className': dartName,
|
||||
'data': columnMap,
|
||||
});
|
||||
return (exists: true, ownerScopeId: scopeId, row: row);
|
||||
}
|
||||
|
||||
/// Resolves a column value for outbound update sync.
|
||||
|
|
@ -569,25 +686,135 @@ class CrdtSync {
|
|||
/// from [CrdtDataForeignKey.attemptedValue] instead of the materialized
|
||||
/// domain column value. Non-FK columns and FK columns without an override
|
||||
/// are read directly from the domain table.
|
||||
Future<dynamic> _fetchColumnValue(
|
||||
Future<({bool exists, int? ownerScopeId, dynamic value})> _fetchOwnedColumnValue(
|
||||
DatabaseSession session,
|
||||
String tableName,
|
||||
UuidValue rowId,
|
||||
String columnName,
|
||||
CrdtDataForeignKey? projection,
|
||||
int scopeId,
|
||||
DomainRowOwnerCache ownerCache,
|
||||
) async {
|
||||
if (projection != null && projection.hasOverride) {
|
||||
return _decodeColumnValue(tableName, columnName, projection.attemptedValue);
|
||||
final owner = await _readDomainRowOwner(session, tableName, rowId, ownerCache);
|
||||
if (!owner.exists || owner.scopeId != scopeId) {
|
||||
return (exists: owner.exists, ownerScopeId: owner.scopeId, value: null);
|
||||
}
|
||||
return (
|
||||
exists: true,
|
||||
ownerScopeId: scopeId,
|
||||
value: _decodeColumnValue(tableName, columnName, projection.attemptedValue),
|
||||
);
|
||||
}
|
||||
|
||||
final encodedValue = ValueEncoder.instance.convert(rowId);
|
||||
final encodedScopeId = ValueEncoder.instance.convert(scopeId);
|
||||
final escapedTableName = _escapeIdentifier(tableName);
|
||||
final result = await session.db.unsafeQuery(
|
||||
'SELECT "${_escapeIdentifier(columnName)}" '
|
||||
'FROM "${_escapeIdentifier(tableName)}" '
|
||||
'WHERE "id" = $encodedValue',
|
||||
'FROM "$escapedTableName" '
|
||||
'WHERE "id" = $encodedValue AND "scopeId" = $encodedScopeId '
|
||||
'LIMIT 1',
|
||||
);
|
||||
if (result.isEmpty) return null;
|
||||
return _decodeColumnValue(tableName, columnName, result.first[0]);
|
||||
if (result.isNotEmpty) {
|
||||
ownerCache[(tableName, rowId)] = (exists: true, scopeId: scopeId);
|
||||
return (
|
||||
exists: true,
|
||||
ownerScopeId: scopeId,
|
||||
value: _decodeColumnValue(tableName, columnName, result.first[0]),
|
||||
);
|
||||
}
|
||||
|
||||
final owner = await _readDomainRowOwner(session, tableName, rowId, ownerCache);
|
||||
return (exists: owner.exists, ownerScopeId: owner.scopeId, value: null);
|
||||
}
|
||||
|
||||
Future<DomainRowOwner> _readDomainRowOwner(
|
||||
DatabaseSession session,
|
||||
String tableName,
|
||||
UuidValue rowId,
|
||||
DomainRowOwnerCache ownerCache,
|
||||
) async {
|
||||
final cached = ownerCache[(tableName, rowId)];
|
||||
if (cached != null) return cached;
|
||||
|
||||
final encodedValue = ValueEncoder.instance.convert(rowId);
|
||||
final escapedTableName = _escapeIdentifier(tableName);
|
||||
final result = await session.db.unsafeQuery(
|
||||
'SELECT "scopeId" FROM "$escapedTableName" '
|
||||
'WHERE "id" = $encodedValue '
|
||||
'LIMIT 1',
|
||||
);
|
||||
final owner = result.isEmpty
|
||||
? (exists: false, scopeId: null)
|
||||
: (exists: true, scopeId: result.first[0] as int?);
|
||||
ownerCache[(tableName, rowId)] = owner;
|
||||
return owner;
|
||||
}
|
||||
|
||||
Never _throwPendingIntegrityViolation({
|
||||
required int? crdtDataRowId,
|
||||
required CrdtSyncViolationType type,
|
||||
required CrdtSyncViolationOperation operation,
|
||||
required String tableName,
|
||||
required UuidValue rowId,
|
||||
required int? ownerScopeId,
|
||||
required UuidValue incomingScopeUuid,
|
||||
required UuidValue uuidNodeId,
|
||||
Hlc? hlc,
|
||||
}) {
|
||||
throw PendingOutboundIntegrityViolation(
|
||||
crdtDataRowId: crdtDataRowId,
|
||||
type: type,
|
||||
operation: operation,
|
||||
tableName: tableName,
|
||||
rowId: rowId,
|
||||
ownerScopeId: ownerScopeId,
|
||||
incomingScopeUuid: incomingScopeUuid,
|
||||
uuidNodeId: uuidNodeId,
|
||||
hlc: hlc,
|
||||
);
|
||||
}
|
||||
|
||||
Future<Never> _recordAndThrowIntegrityViolation(
|
||||
DatabaseSession session,
|
||||
PendingOutboundIntegrityViolation pending,
|
||||
) async {
|
||||
final ownerScopeUuid = await _scopeUuidForNormalizedId(
|
||||
session,
|
||||
pending.ownerScopeId,
|
||||
);
|
||||
final now = DateTime.now().toUtc();
|
||||
final violation = CrdtSyncIntegrityViolation(
|
||||
type: pending.type,
|
||||
domainTableName: pending.tableName,
|
||||
uuidRowId: pending.rowId,
|
||||
ownerScopeUuid: ownerScopeUuid,
|
||||
incomingScopeUuid: pending.incomingScopeUuid,
|
||||
operation: pending.operation,
|
||||
uuidNodeId: pending.uuidNodeId,
|
||||
crdtDataRowId: pending.crdtDataRowId,
|
||||
hlcDatetime: pending.hlc?.datetime,
|
||||
hlcCounter: pending.hlc?.counter,
|
||||
firstSeenAt: now,
|
||||
lastSeenAt: now,
|
||||
occurrences: 1,
|
||||
);
|
||||
final persisted = await recordCrdtSyncIntegrityViolation(
|
||||
session,
|
||||
violation: violation,
|
||||
);
|
||||
throw CrdtSyncIntegrityViolationException(persisted);
|
||||
}
|
||||
|
||||
Future<UuidValue?> _scopeUuidForNormalizedId(
|
||||
DatabaseSession session,
|
||||
int? scopeId,
|
||||
) async {
|
||||
if (scopeId == null) return null;
|
||||
|
||||
final scope = await CrdtScope.db.findById(session, scopeId);
|
||||
return scope?.uuidScopeId;
|
||||
}
|
||||
|
||||
/// Loads FK projection metadata for outbound insert sync.
|
||||
|
|
@ -661,8 +888,10 @@ class CrdtSync {
|
|||
final definition = tableDefinitionsByName[table.tableName];
|
||||
final columns = [
|
||||
for (final column in definition?.columns ?? const <ColumnDefinition>[])
|
||||
column.name,
|
||||
if (definition == null) ...table.columns.map((column) => column.columnName),
|
||||
if (column.name != 'scopeId') column.name,
|
||||
if (definition == null)
|
||||
for (final column in table.columns)
|
||||
if (column.columnName != 'scopeId') column.columnName,
|
||||
]..sort();
|
||||
final foreignKeys = _canonicalForeignKeys(definition);
|
||||
final uniqueIndexes = _canonicalUniqueIndexes(definition);
|
||||
|
|
|
|||
|
|
@ -6,6 +6,8 @@ import 'package:serverpod_database/serverpod_database.dart';
|
|||
import 'package:serverpod_offline_sync_shared/serverpod_offline_sync_shared.dart';
|
||||
import 'package:uuid/uuid.dart';
|
||||
|
||||
import '../crdt/exceptions.dart';
|
||||
import '../crdt/integrity_violation.dart';
|
||||
import '../crdt/merge.dart';
|
||||
import '../crdt/sync.dart';
|
||||
import '../protocol/protocol.dart';
|
||||
|
|
@ -13,8 +15,10 @@ import 'recorder.dart';
|
|||
import 'session.dart';
|
||||
import 'tombstone.dart';
|
||||
|
||||
/// Map of transaction hashes to the user ID they are associated with.
|
||||
final userForTransaction = <Transaction, CrdtUser>{};
|
||||
part 'scope.dart';
|
||||
|
||||
/// Map of transaction hashes to the scope they are associated with.
|
||||
final scopeForTransaction = <Transaction, CrdtScope>{};
|
||||
|
||||
/// Database proxy that runs insert/update/delete ORM operations inside a
|
||||
/// transaction to record each change in the CRDT tables.
|
||||
|
|
@ -70,7 +74,7 @@ class CrdtDatabase implements Database {
|
|||
/// Returns the current node identifier for the effective user.
|
||||
Future<UuidValue> currentNodeId({UuidValue? userId}) async {
|
||||
final effectiveUserId = await _requireUserId(userId);
|
||||
final user = await _recorder.getOrCreateUser(effectiveUserId);
|
||||
final user = await _recorder.getOrCreateScope(effectiveUserId);
|
||||
return user.currentNode!.uuidNodeId;
|
||||
}
|
||||
|
||||
|
|
@ -105,27 +109,35 @@ class CrdtDatabase implements Database {
|
|||
);
|
||||
}
|
||||
|
||||
/// Merges remote CRDT changes into the local database for the given user.
|
||||
/// Merges remote CRDT changes into the local database for the given scope.
|
||||
///
|
||||
/// When [userId] is omitted, this uses the recorder's persistent user id.
|
||||
/// The merge locks the current user's CRDT tables and executes atomically
|
||||
/// When [scopeId] is omitted, this uses the recorder's persistent user id.
|
||||
/// The merge locks the current scope's CRDT tables and executes atomically
|
||||
/// inside [transactionForUser].
|
||||
Future<void> mergeChanges(
|
||||
CrdtMergeSet mergeSet, {
|
||||
UuidValue? userId,
|
||||
UuidValue? scopeId,
|
||||
}) async {
|
||||
if (mergeSet.isEmpty) return;
|
||||
|
||||
final effectiveUserId =
|
||||
userId ??
|
||||
final effectiveScopeId =
|
||||
scopeId ??
|
||||
_recorder.persistentUserId ??
|
||||
(throw StateError(
|
||||
'A user ID is required when merging changes without a persistent user.',
|
||||
'A scope ID is required when merging changes without a persistent user.',
|
||||
));
|
||||
await transactionForUser<void>(effectiveUserId, (tx) async {
|
||||
try {
|
||||
await transactionForUser<void>(effectiveScopeId, (tx) async {
|
||||
await _recorder.lockCurrentUser(tx);
|
||||
await _recorder.mergeChanges(mergeSet, tx);
|
||||
});
|
||||
} on CrdtSyncIntegrityViolationException catch (exception) {
|
||||
final persistedViolation = await recordCrdtSyncIntegrityViolation(
|
||||
_delegate.session,
|
||||
violation: exception.violation,
|
||||
);
|
||||
throw CrdtSyncIntegrityViolationException(persistedViolation);
|
||||
}
|
||||
}
|
||||
|
||||
@override
|
||||
|
|
@ -138,6 +150,24 @@ class CrdtDatabase implements Database {
|
|||
DatabaseSerializationManager get serializationManager =>
|
||||
_delegate.serializationManager;
|
||||
|
||||
/// Merges [where] with the CRDT visibility predicates scoped to the queried
|
||||
/// tables and the user associated with [transaction] (or the persistent
|
||||
/// user). Without a user scope (e.g. server-side admin reads), a row is
|
||||
/// only hidden when every user tracking it has it hidden.
|
||||
Expression? _whereVisibleWithTombstone<T extends TableRow>(
|
||||
Expression? where,
|
||||
Include? include,
|
||||
Transaction? transaction,
|
||||
) {
|
||||
return mergeWhereWithTombstone<T>(
|
||||
serializationManager,
|
||||
where,
|
||||
include,
|
||||
tableIdForName: _recorder.tableIdForName,
|
||||
scopeId: () => _recorder.scopeForQueries(transaction)?.id,
|
||||
);
|
||||
}
|
||||
|
||||
@override
|
||||
Future<List<T>> find<T extends TableRow>({
|
||||
Expression? where,
|
||||
|
|
@ -151,8 +181,8 @@ class CrdtDatabase implements Database {
|
|||
LockMode? lockMode,
|
||||
LockBehavior? lockBehavior,
|
||||
}) async {
|
||||
return _delegate.find<T>(
|
||||
where: mergeWhereWithTombstone<T>(serializationManager, where, include),
|
||||
final result = await _delegate.find<T>(
|
||||
where: _whereVisibleWithTombstone<T>(where, include, transaction),
|
||||
limit: limit,
|
||||
offset: offset,
|
||||
orderBy: orderBy,
|
||||
|
|
@ -165,6 +195,7 @@ class CrdtDatabase implements Database {
|
|||
lockMode: lockMode,
|
||||
lockBehavior: lockBehavior,
|
||||
);
|
||||
return _stripScopeIdFromScopedRead(result, include, transaction);
|
||||
}
|
||||
|
||||
@override
|
||||
|
|
@ -177,13 +208,15 @@ class CrdtDatabase implements Database {
|
|||
}) async {
|
||||
final table = serializationManager.getTableForType(T);
|
||||
final where = table?.id.equals(id);
|
||||
return _delegate.findFirstRow<T>(
|
||||
where: mergeWhereWithTombstone<T>(serializationManager, where, include),
|
||||
final result = await _delegate.findFirstRow<T>(
|
||||
where: _whereVisibleWithTombstone<T>(where, include, transaction),
|
||||
transaction: transaction,
|
||||
include: include,
|
||||
lockMode: lockMode,
|
||||
lockBehavior: lockBehavior,
|
||||
);
|
||||
if (result == null) return null;
|
||||
return _stripScopeIdFromScopedRead([result], include, transaction).single;
|
||||
}
|
||||
|
||||
@override
|
||||
|
|
@ -198,8 +231,8 @@ class CrdtDatabase implements Database {
|
|||
LockMode? lockMode,
|
||||
LockBehavior? lockBehavior,
|
||||
}) async {
|
||||
return _delegate.findFirstRow<T>(
|
||||
where: mergeWhereWithTombstone<T>(serializationManager, where, include),
|
||||
final result = await _delegate.findFirstRow<T>(
|
||||
where: _whereVisibleWithTombstone<T>(where, include, transaction),
|
||||
offset: offset,
|
||||
orderBy: orderBy,
|
||||
orderByList: orderByList,
|
||||
|
|
@ -211,6 +244,8 @@ class CrdtDatabase implements Database {
|
|||
lockMode: lockMode,
|
||||
lockBehavior: lockBehavior,
|
||||
);
|
||||
if (result == null) return null;
|
||||
return _stripScopeIdFromScopedRead([result], include, transaction).single;
|
||||
}
|
||||
|
||||
@override
|
||||
|
|
@ -219,17 +254,20 @@ class CrdtDatabase implements Database {
|
|||
Transaction? transaction,
|
||||
bool ignoreConflicts = false,
|
||||
}) async {
|
||||
if (rows.isEmpty) return [];
|
||||
return DatabaseUtil.runInTransactionOrSavepoint(
|
||||
_delegate,
|
||||
transaction,
|
||||
(tx) async {
|
||||
final prepared = _prepareRowsForInsert(rows, tx);
|
||||
final result = await _delegate.insert<T>(
|
||||
rows,
|
||||
prepared.rows,
|
||||
transaction: tx,
|
||||
ignoreConflicts: ignoreConflicts,
|
||||
);
|
||||
|
||||
await _recorder.afterInsert<T>(result, tx);
|
||||
_stripStampedRows(result, prepared);
|
||||
return result;
|
||||
},
|
||||
);
|
||||
|
|
@ -244,18 +282,22 @@ class CrdtDatabase implements Database {
|
|||
_delegate,
|
||||
transaction,
|
||||
(tx) async {
|
||||
final prepared = _prepareRowsForInsert([row], tx);
|
||||
final databaseRow = prepared.rows.single;
|
||||
late final T result;
|
||||
try {
|
||||
result = await _delegate.insertRow<T>(row, transaction: tx);
|
||||
result = await _delegate.insertRow<T>(databaseRow, transaction: tx);
|
||||
} on DatabaseQueryException {
|
||||
if (!await _recorder.isDeleted(row, tx)) rethrow;
|
||||
|
||||
result = await _delegate.updateRow<T>(row, transaction: tx);
|
||||
result = await _delegate.updateRow<T>(databaseRow, transaction: tx);
|
||||
await _recorder.afterReinsert([result], tx);
|
||||
_stripStampedRows([result], prepared);
|
||||
return result;
|
||||
}
|
||||
|
||||
await _recorder.afterInsert([result], tx);
|
||||
_stripStampedRows([result], prepared);
|
||||
return result;
|
||||
},
|
||||
);
|
||||
|
|
@ -281,6 +323,15 @@ class CrdtDatabase implements Database {
|
|||
Expression? updateWhere,
|
||||
Transaction? transaction,
|
||||
}) async {
|
||||
if (T == CrdtSyncIntegrityViolation) {
|
||||
return _delegate.upsertRow<T>(
|
||||
row,
|
||||
conflictColumns: conflictColumns,
|
||||
updateColumns: updateColumns,
|
||||
updateWhere: updateWhere,
|
||||
transaction: transaction,
|
||||
);
|
||||
}
|
||||
// TODO: Implement CRDT upsert.
|
||||
throw UnimplementedError('CRDT upsert is not implemented.');
|
||||
}
|
||||
|
|
@ -298,7 +349,12 @@ class CrdtDatabase implements Database {
|
|||
(tx) async {
|
||||
final updatedRows = [
|
||||
for (final row in rows)
|
||||
await _updateRowWithoutRecording(row, columns: columns, transaction: tx),
|
||||
await _updateRowWithoutRecording(
|
||||
row,
|
||||
stripScopeId: _shouldStripReturnedScopeId(row, tx),
|
||||
transaction: tx,
|
||||
columns: columns,
|
||||
),
|
||||
];
|
||||
|
||||
await _recorder.afterUpdate(updatedRows, columns, tx);
|
||||
|
|
@ -317,10 +373,12 @@ class CrdtDatabase implements Database {
|
|||
_delegate,
|
||||
transaction,
|
||||
(tx) async {
|
||||
final stripScopeId = _shouldStripReturnedScopeId(row, tx);
|
||||
final updatedRow = await _updateRowWithoutRecording(
|
||||
row,
|
||||
columns: columns,
|
||||
stripScopeId: stripScopeId,
|
||||
transaction: tx,
|
||||
columns: columns,
|
||||
);
|
||||
|
||||
await _recorder.afterUpdate([updatedRow], columns, tx);
|
||||
|
|
@ -332,18 +390,19 @@ class CrdtDatabase implements Database {
|
|||
Future<T> _updateRowWithoutRecording<T extends TableRow>(
|
||||
T row, {
|
||||
required Transaction transaction,
|
||||
required bool stripScopeId,
|
||||
List<Column>? columns,
|
||||
}) async {
|
||||
final values = row.toJsonForDatabase() as Map<String, dynamic>;
|
||||
final columnValues = (columns ?? row.table.managedColumns)
|
||||
.where((c) => c.columnName != 'id')
|
||||
.where((c) => c.columnName != 'id' && c.columnName != 'scopeId')
|
||||
.map((c) => ColumnValue(c, values[c.columnName]))
|
||||
.toList();
|
||||
|
||||
final where = row.table.id.equals(row.id);
|
||||
final updatedRows = await _delegate.updateWhere<T>(
|
||||
columnValues: columnValues,
|
||||
where: mergeWhereWithTombstone<T>(serializationManager, where, null)!,
|
||||
where: _whereVisibleWithTombstone<T>(where, null, transaction)!,
|
||||
transaction: transaction,
|
||||
);
|
||||
|
||||
|
|
@ -353,7 +412,9 @@ class CrdtDatabase implements Database {
|
|||
throw Exception('Failed to update row, no rows updated.');
|
||||
}
|
||||
|
||||
return updatedRows.single;
|
||||
final updatedRow = updatedRows.single;
|
||||
if (stripScopeId) _stripScopeId(updatedRow);
|
||||
return updatedRow;
|
||||
}
|
||||
|
||||
@override
|
||||
|
|
@ -386,13 +447,17 @@ class CrdtDatabase implements Database {
|
|||
bool orderDescending = false,
|
||||
Transaction? transaction,
|
||||
}) async {
|
||||
if (_recorder.isCrdtTracked<T>()) {
|
||||
_assertNoScopeIdColumnValues<T>(columnValues);
|
||||
}
|
||||
|
||||
return DatabaseUtil.runInTransactionOrSavepoint(
|
||||
_delegate,
|
||||
transaction,
|
||||
(tx) async {
|
||||
final result = await _delegate.updateWhere<T>(
|
||||
columnValues: columnValues,
|
||||
where: mergeWhereWithTombstone<T>(serializationManager, where, null)!,
|
||||
where: _whereVisibleWithTombstone<T>(where, null, tx)!,
|
||||
limit: limit,
|
||||
offset: offset,
|
||||
orderBy: orderBy,
|
||||
|
|
@ -405,6 +470,9 @@ class CrdtDatabase implements Database {
|
|||
|
||||
final columns = columnValues.map((e) => e.column).toList();
|
||||
await _recorder.afterUpdate(result, columns, tx);
|
||||
if (_recorder.isCrdtTracked<T>()) {
|
||||
result.forEach(_stripScopeId);
|
||||
}
|
||||
return result;
|
||||
},
|
||||
);
|
||||
|
|
@ -476,7 +544,7 @@ class CrdtDatabase implements Database {
|
|||
transaction,
|
||||
(tx) async {
|
||||
final rows = await _delegate.find<T>(
|
||||
where: mergeWhereWithTombstone<T>(serializationManager, where, null),
|
||||
where: _whereVisibleWithTombstone<T>(where, null, tx),
|
||||
orderBy: orderBy,
|
||||
orderByList: orderByList,
|
||||
// Remove this once the deprecated member is removed.
|
||||
|
|
@ -486,7 +554,7 @@ class CrdtDatabase implements Database {
|
|||
);
|
||||
|
||||
await _recorder.insteadOfDelete<T>(rows, tx);
|
||||
return rows;
|
||||
return _stripScopeIdFromScopedRead(rows, null, tx);
|
||||
},
|
||||
);
|
||||
}
|
||||
|
|
@ -499,7 +567,7 @@ class CrdtDatabase implements Database {
|
|||
Transaction? transaction,
|
||||
}) async {
|
||||
return _delegate.count<T>(
|
||||
where: mergeWhereWithTombstone<T>(serializationManager, where, null),
|
||||
where: _whereVisibleWithTombstone<T>(where, null, transaction),
|
||||
limit: limit,
|
||||
useCache: useCache,
|
||||
transaction: transaction,
|
||||
|
|
@ -514,7 +582,7 @@ class CrdtDatabase implements Database {
|
|||
LockBehavior lockBehavior = LockBehavior.wait,
|
||||
}) async {
|
||||
return _delegate.lockRows<T>(
|
||||
where: mergeWhereWithTombstone<T>(serializationManager, where, null)!,
|
||||
where: _whereVisibleWithTombstone<T>(where, null, transaction)!,
|
||||
lockMode: lockMode,
|
||||
transaction: transaction,
|
||||
lockBehavior: lockBehavior,
|
||||
|
|
@ -543,15 +611,15 @@ class CrdtDatabase implements Database {
|
|||
TransactionSettings? settings,
|
||||
}) async {
|
||||
// Ensure that the user exists with a node before starting the transaction.
|
||||
final user = await _recorder.getOrCreateUser(userId);
|
||||
final user = await _recorder.getOrCreateScope(userId);
|
||||
|
||||
return transaction<R>(
|
||||
(tx) async {
|
||||
try {
|
||||
userForTransaction[tx] = user;
|
||||
scopeForTransaction[tx] = user;
|
||||
return await transactionFunction(tx);
|
||||
} finally {
|
||||
userForTransaction.remove(tx);
|
||||
scopeForTransaction.remove(tx);
|
||||
}
|
||||
},
|
||||
settings: settings,
|
||||
|
|
|
|||
|
|
@ -2,11 +2,13 @@ part of 'recorder.dart';
|
|||
|
||||
typedef _MergeRowKey = (String, UuidValue);
|
||||
typedef _MergeFieldKey = (String, UuidValue, String);
|
||||
typedef _DomainOwner = ({bool exists, int? scopeId});
|
||||
typedef _MergeContext = ({
|
||||
Map<_MergeRowKey, CrdtDataRow> rows,
|
||||
Map<_MergeFieldKey, CrdtDataField> fields,
|
||||
Map<_MergeFieldKey, Hlc> incomingFieldHlcs,
|
||||
Map<_MergeRowKey, CrdtDataDeleted> tombstones,
|
||||
Map<_MergeRowKey, _DomainOwner> domainOwners,
|
||||
});
|
||||
|
||||
/// Adds merge-specific behavior to [CrdtMutationRecorder].
|
||||
|
|
@ -16,10 +18,10 @@ extension CrdtMergeRecorderExtension on CrdtMutationRecorder {
|
|||
|
||||
/// Locks the current user row so merges can serialize with other work.
|
||||
Future<void> lockCurrentUser(Transaction transaction) async {
|
||||
final user = _getEffectiveUser(transaction);
|
||||
final user = _getEffectiveScope(transaction);
|
||||
// Use a row lock without fetching the record since the merge path only
|
||||
// needs serialization against concurrent work for the same user.
|
||||
await CrdtUser.db.lockRows(
|
||||
await CrdtScope.db.lockRows(
|
||||
_session,
|
||||
where: (t) => t.id.equals(user.id),
|
||||
transaction: transaction,
|
||||
|
|
@ -39,7 +41,7 @@ extension CrdtMergeRecorderExtension on CrdtMutationRecorder {
|
|||
final shouldProjectForeignKeys = _mergeOperationsMayAffectForeignKeys(
|
||||
operations,
|
||||
);
|
||||
final currentUser = _getEffectiveUser(transaction);
|
||||
final currentUser = _getEffectiveScope(transaction);
|
||||
final remoteNodes = await _findOrCreateNodesForMerge(
|
||||
currentUser.id!,
|
||||
{for (final change in operations) change.uuidNodeId},
|
||||
|
|
@ -150,7 +152,7 @@ extension CrdtMergeRecorderExtension on CrdtMutationRecorder {
|
|||
|
||||
var nodes = await CrdtNode.db.find(
|
||||
_session,
|
||||
where: (t) => t.userId.equals(userId) & t.uuidNodeId.inSet(nodeIds),
|
||||
where: (t) => t.scopeId.equals(userId) & t.uuidNodeId.inSet(nodeIds),
|
||||
transaction: transaction,
|
||||
);
|
||||
final existingNodeIds = nodes.map((node) => node.uuidNodeId).toSet();
|
||||
|
|
@ -162,7 +164,7 @@ extension CrdtMergeRecorderExtension on CrdtMutationRecorder {
|
|||
[
|
||||
for (final uuidNodeId in missingNodeIds)
|
||||
CrdtNode(
|
||||
userId: userId,
|
||||
scopeId: userId,
|
||||
uuidNodeId: uuidNodeId,
|
||||
),
|
||||
],
|
||||
|
|
@ -172,7 +174,7 @@ extension CrdtMergeRecorderExtension on CrdtMutationRecorder {
|
|||
|
||||
nodes = await CrdtNode.db.find(
|
||||
_session,
|
||||
where: (t) => t.userId.equals(userId) & t.uuidNodeId.inSet(nodeIds),
|
||||
where: (t) => t.scopeId.equals(userId) & t.uuidNodeId.inSet(nodeIds),
|
||||
transaction: transaction,
|
||||
);
|
||||
}
|
||||
|
|
@ -197,6 +199,13 @@ extension CrdtMergeRecorderExtension on CrdtMutationRecorder {
|
|||
(update.tableName, update.uuidRowId, update.columnName): update.hlc,
|
||||
};
|
||||
final tombstones = <_MergeRowKey, CrdtDataDeleted>{};
|
||||
// Owning scope per row, snapshotted once so the per-operation ownership
|
||||
// checks never re-read the same immutable `scopeId`. Only rows that
|
||||
// already have a tracker are pre-read here — those are exactly the rows
|
||||
// whose ownership the update/delete/existing-insert paths verify. New
|
||||
// rows go through the insert path, which proves ownership at the physical
|
||||
// write and refreshes this entry afterwards.
|
||||
final domainOwners = <_MergeRowKey, _DomainOwner>{};
|
||||
|
||||
for (final MapEntry(key: tableName, value: rowIds) in rowIdsByTable.entries) {
|
||||
final loadedRows = await _findCrdtRows(
|
||||
|
|
@ -216,6 +225,22 @@ extension CrdtMergeRecorderExtension on CrdtMutationRecorder {
|
|||
}
|
||||
}
|
||||
|
||||
final trackedRowIds = loadedRows.map((row) => row.uuidRowId).toSet();
|
||||
if (trackedRowIds.isNotEmpty) {
|
||||
final ownerValues = await _readDomainColumnValues(
|
||||
tableName,
|
||||
trackedRowIds,
|
||||
['scopeId'],
|
||||
transaction,
|
||||
);
|
||||
for (final rowId in trackedRowIds) {
|
||||
final values = ownerValues[rowId];
|
||||
domainOwners[(tableName, rowId)] = values == null
|
||||
? (exists: false, scopeId: null)
|
||||
: (exists: true, scopeId: values['scopeId'] as int?);
|
||||
}
|
||||
}
|
||||
|
||||
final columnNames = columnNamesByTable[tableName];
|
||||
if (columnNames == null || columnNames.isEmpty || loadedRows.isEmpty) continue;
|
||||
|
||||
|
|
@ -252,6 +277,7 @@ extension CrdtMergeRecorderExtension on CrdtMutationRecorder {
|
|||
fields: fields,
|
||||
incomingFieldHlcs: incomingFieldHlcs,
|
||||
tombstones: tombstones,
|
||||
domainOwners: domainOwners,
|
||||
);
|
||||
}
|
||||
|
||||
|
|
@ -273,33 +299,61 @@ extension CrdtMergeRecorderExtension on CrdtMutationRecorder {
|
|||
final data = _sanitizeMergeRowData(insert.tableName, insert.databaseColumns);
|
||||
|
||||
if (currentRow == null) {
|
||||
final resolvedInsert = await _uniqueConflictResolver._resolveForIncomingInsert(
|
||||
insert,
|
||||
data,
|
||||
context,
|
||||
final mergingScope = _getEffectiveScope(transaction);
|
||||
final mergingScopeId = mergingScope.id!;
|
||||
try {
|
||||
// Insert-first inside a savepoint: the domain primary key resolves
|
||||
// creation races atomically, and `_applyMergeInsertForMissingRow`
|
||||
// raises `_ForeignScopeRowCollision` when the row turns out to be owned
|
||||
// by another scope. Rolling the savepoint back then leaves no CRDT
|
||||
// metadata, released unique values, or foreign key attempts behind.
|
||||
context.rows[rowKey] = await DatabaseUtil.runInTransactionOrSavepoint(
|
||||
_db,
|
||||
transaction,
|
||||
);
|
||||
(savepoint) async {
|
||||
final resolvedInsert = await _uniqueConflictResolver
|
||||
._resolveForIncomingInsert(insert, data, context, savepoint);
|
||||
|
||||
final safeInsert = await _safeIncomingForeignKeyData(
|
||||
insert.tableName,
|
||||
insert.uuidRowId,
|
||||
resolvedInsert.data,
|
||||
transaction,
|
||||
savepoint,
|
||||
);
|
||||
|
||||
context.rows[rowKey] = await _applyMergeInsertForMissingRow(
|
||||
final insertedRow = await _applyMergeInsertForMissingRow(
|
||||
insert,
|
||||
remoteNode,
|
||||
incomingHlc,
|
||||
safeInsert.data,
|
||||
transaction,
|
||||
savepoint,
|
||||
);
|
||||
await _recordForeignKeyInsertAttempts(
|
||||
insert.tableName,
|
||||
{insert.uuidRowId},
|
||||
transaction,
|
||||
savepoint,
|
||||
safeInsert.attempts,
|
||||
);
|
||||
return insertedRow;
|
||||
},
|
||||
);
|
||||
// The row is now owned by the merging scope (freshly inserted, or a
|
||||
// same-scope recovery), so same-merge field updates resolve ownership
|
||||
// from the cache instead of re-reading the domain row.
|
||||
context.domainOwners[rowKey] = (exists: true, scopeId: mergingScopeId);
|
||||
} on _ForeignScopeRowCollision catch (collision) {
|
||||
await _throwOwnershipCollision(
|
||||
operation: CrdtSyncViolationOperation.mergeInsert,
|
||||
tableName: insert.tableName,
|
||||
rowId: insert.uuidRowId,
|
||||
owningScopeId: collision.owningScopeId,
|
||||
incomingScope: mergingScope,
|
||||
metadataRowId: null,
|
||||
node: remoteNode,
|
||||
hlc: insert.hlc,
|
||||
transaction: transaction,
|
||||
);
|
||||
}
|
||||
} else {
|
||||
await _applyMergeInsertForExistingRow(
|
||||
insert,
|
||||
|
|
@ -322,13 +376,26 @@ extension CrdtMergeRecorderExtension on CrdtMutationRecorder {
|
|||
final rowKey = (update.tableName, update.uuidRowId);
|
||||
final row = context.rows[rowKey];
|
||||
if (row == null) return;
|
||||
final remoteNode = _requireRemoteNode(remoteNodes, update.uuidNodeId);
|
||||
if (!await _domainRowOwnedByEffectiveScope(
|
||||
operation: CrdtSyncViolationOperation.mergeUpdate,
|
||||
tableName: update.tableName,
|
||||
rowId: update.uuidRowId,
|
||||
context: context,
|
||||
transaction: transaction,
|
||||
metadataRowId: row.id,
|
||||
node: remoteNode,
|
||||
hlc: update.hlc,
|
||||
)) {
|
||||
return;
|
||||
}
|
||||
|
||||
final shouldApply = await _shouldMergeFieldMetadataIfNewer(
|
||||
tableName: update.tableName,
|
||||
rowId: update.uuidRowId,
|
||||
columnName: update.columnName,
|
||||
row: row,
|
||||
remoteNode: _requireRemoteNode(remoteNodes, update.uuidNodeId),
|
||||
remoteNode: remoteNode,
|
||||
incomingHlc: update.hlc,
|
||||
fields: context.fields,
|
||||
transaction: transaction,
|
||||
|
|
@ -373,6 +440,19 @@ extension CrdtMergeRecorderExtension on CrdtMutationRecorder {
|
|||
final row = context.rows[rowKey];
|
||||
if (row == null) return;
|
||||
if (!delete.reason.isSynced) return;
|
||||
final remoteNode = _requireRemoteNode(remoteNodes, delete.uuidNodeId);
|
||||
if (!await _domainRowOwnedByEffectiveScope(
|
||||
operation: CrdtSyncViolationOperation.mergeDelete,
|
||||
tableName: delete.tableName,
|
||||
rowId: delete.uuidRowId,
|
||||
context: context,
|
||||
transaction: transaction,
|
||||
metadataRowId: row.id,
|
||||
node: remoteNode,
|
||||
hlc: delete.hlc,
|
||||
)) {
|
||||
return;
|
||||
}
|
||||
|
||||
final currentTombstone = context.tombstones[rowKey];
|
||||
final currentClFlag = currentTombstone?.clFlag ?? 1;
|
||||
|
|
@ -382,7 +462,7 @@ extension CrdtMergeRecorderExtension on CrdtMutationRecorder {
|
|||
|
||||
context.tombstones[rowKey] = await _upsertMergeTombstone(
|
||||
row,
|
||||
_requireRemoteNode(remoteNodes, delete.uuidNodeId),
|
||||
remoteNode,
|
||||
delete.hlc,
|
||||
delete.clFlag,
|
||||
delete.reason,
|
||||
|
|
@ -412,7 +492,7 @@ extension CrdtMergeRecorderExtension on CrdtMutationRecorder {
|
|||
final insertedRow = await CrdtDataRow.db.insertRow(
|
||||
_session,
|
||||
CrdtDataRow(
|
||||
userId: _getHlcManager(transaction).normalizedUserId,
|
||||
scopeId: _getHlcManager(transaction).normalizedScopeId,
|
||||
tblId: tableId,
|
||||
uuidRowId: rowId,
|
||||
nodeId: remoteNode.id!,
|
||||
|
|
@ -555,16 +635,41 @@ extension CrdtMergeRecorderExtension on CrdtMutationRecorder {
|
|||
|
||||
final tableRow = _requireMergeInsertTableRow(insert);
|
||||
final patchedRow = _db.serializationManager.patchTableRow(tableRow, data);
|
||||
final mergingScopeId = _getHlcManager(transaction).normalizedScopeId;
|
||||
final scopedRow = _withScopeId(patchedRow, mergingScopeId);
|
||||
try {
|
||||
// Insert first: the domain primary key resolves creation races
|
||||
// atomically. The inner savepoint keeps a violation from poisoning the
|
||||
// surrounding transaction so the recovery update below can still run.
|
||||
await DatabaseUtil.runInTransactionOrSavepoint(
|
||||
_db,
|
||||
transaction,
|
||||
(savepoint) => _db.insertRow(scopedRow, transaction: savepoint),
|
||||
);
|
||||
} on DatabaseQueryException {
|
||||
final owner = await _readDomainRowOwner(
|
||||
insert.tableName,
|
||||
insert.uuidRowId,
|
||||
transaction,
|
||||
);
|
||||
// No row means the violation came from something other than the
|
||||
// primary key (e.g. a racing unique value); preserve the failure.
|
||||
if (!owner.exists) rethrow;
|
||||
if (owner.scopeId != mergingScopeId) {
|
||||
throw _ForeignScopeRowCollision(owner.scopeId);
|
||||
}
|
||||
|
||||
// Same-scope recovery (a tracked row whose metadata was lost, or a
|
||||
// race between this scope's own merges). scopeId is immutable and rows
|
||||
// are only soft-deleted, so ownership observed here cannot change for
|
||||
// the rest of the transaction and the update is safe.
|
||||
await _db.updateRow(
|
||||
patchedRow,
|
||||
columns: patchedRow.table.managedColumns
|
||||
scopedRow,
|
||||
columns: scopedRow.table.managedColumns
|
||||
.where((c) => data.containsKey(c.columnName))
|
||||
.toList(),
|
||||
transaction: transaction,
|
||||
);
|
||||
} on DatabaseUpdateRowException {
|
||||
await _db.insertRow(patchedRow, transaction: transaction);
|
||||
}
|
||||
|
||||
return insertedCrdtRow;
|
||||
|
|
@ -579,6 +684,19 @@ extension CrdtMergeRecorderExtension on CrdtMutationRecorder {
|
|||
_MergeContext context,
|
||||
Transaction transaction,
|
||||
) async {
|
||||
if (!await _domainRowOwnedByEffectiveScope(
|
||||
operation: CrdtSyncViolationOperation.mergeInsert,
|
||||
tableName: insert.tableName,
|
||||
rowId: insert.uuidRowId,
|
||||
context: context,
|
||||
transaction: transaction,
|
||||
metadataRowId: currentRow.id,
|
||||
node: remoteNode,
|
||||
hlc: insert.hlc,
|
||||
)) {
|
||||
return;
|
||||
}
|
||||
|
||||
final (_, columnsByName) = _schema[insert.tableName]!;
|
||||
final updatedValues = <String, Object?>{};
|
||||
for (final MapEntry(key: columnName, value: schemaColumn)
|
||||
|
|
@ -715,12 +833,133 @@ extension CrdtMergeRecorderExtension on CrdtMutationRecorder {
|
|||
|
||||
return {
|
||||
for (final MapEntry(key: columnName, value: value) in data.entries)
|
||||
if (columnName != 'id' && columns.containsKey(columnName))
|
||||
if (columnName != 'id' &&
|
||||
columnName != 'scopeId' &&
|
||||
columns.containsKey(columnName))
|
||||
columnName: columns[columnName]!.columnType == ColumnType.uuid
|
||||
? _uuidValueFromDatabase(value)
|
||||
: value,
|
||||
};
|
||||
}
|
||||
|
||||
Future<({bool exists, int? scopeId})> _readDomainRowOwner(
|
||||
String tableName,
|
||||
UuidValue rowId,
|
||||
Transaction transaction,
|
||||
) async {
|
||||
final values = await _readDomainColumnValues(
|
||||
tableName,
|
||||
{rowId},
|
||||
['scopeId'],
|
||||
transaction,
|
||||
);
|
||||
final row = values[rowId];
|
||||
if (row == null) return (exists: false, scopeId: null);
|
||||
return (exists: true, scopeId: row['scopeId'] as int?);
|
||||
}
|
||||
|
||||
/// Returns the owning scope of [rowId] from the merge snapshot taken in
|
||||
/// [_loadMergeContext], reading the domain row only if it was not captured
|
||||
/// there (e.g. a row a concurrent operation added since the snapshot).
|
||||
Future<_DomainOwner> _resolveDomainOwner(
|
||||
_MergeContext context,
|
||||
String tableName,
|
||||
UuidValue rowId,
|
||||
Transaction transaction,
|
||||
) async {
|
||||
final key = (tableName, rowId);
|
||||
final cached = context.domainOwners[key];
|
||||
if (cached != null) return cached;
|
||||
final owner = await _readDomainRowOwner(tableName, rowId, transaction);
|
||||
context.domainOwners[key] = owner;
|
||||
return owner;
|
||||
}
|
||||
|
||||
Future<bool> _domainRowOwnedByEffectiveScope({
|
||||
required CrdtSyncViolationOperation operation,
|
||||
required String tableName,
|
||||
required UuidValue rowId,
|
||||
required _MergeContext context,
|
||||
required Transaction transaction,
|
||||
int? metadataRowId,
|
||||
CrdtNode? node,
|
||||
Hlc? hlc,
|
||||
}) async {
|
||||
final owner = await _resolveDomainOwner(context, tableName, rowId, transaction);
|
||||
final mergingScope = _getEffectiveScope(transaction);
|
||||
if (owner.exists && owner.scopeId == mergingScope.id) return true;
|
||||
if (!owner.exists) return false;
|
||||
|
||||
await _throwOwnershipCollision(
|
||||
operation: operation,
|
||||
tableName: tableName,
|
||||
rowId: rowId,
|
||||
owningScopeId: owner.scopeId,
|
||||
incomingScope: mergingScope,
|
||||
metadataRowId: metadataRowId,
|
||||
node: node,
|
||||
hlc: hlc,
|
||||
transaction: transaction,
|
||||
);
|
||||
}
|
||||
|
||||
Future<Never> _throwOwnershipCollision({
|
||||
required CrdtSyncViolationOperation operation,
|
||||
required String tableName,
|
||||
required UuidValue rowId,
|
||||
required int? owningScopeId,
|
||||
required CrdtScope incomingScope,
|
||||
required int? metadataRowId,
|
||||
required Transaction transaction,
|
||||
CrdtNode? node,
|
||||
Hlc? hlc,
|
||||
}) async {
|
||||
final now = DateTime.now().toUtc();
|
||||
throw CrdtSyncIntegrityViolationException(
|
||||
CrdtSyncIntegrityViolation(
|
||||
type: CrdtSyncViolationType.ownershipCollision,
|
||||
domainTableName: tableName,
|
||||
uuidRowId: rowId,
|
||||
ownerScopeUuid: await _scopeUuidForNormalizedId(owningScopeId, transaction),
|
||||
incomingScopeUuid: incomingScope.uuidScopeId,
|
||||
operation: operation,
|
||||
uuidNodeId: node?.uuidNodeId,
|
||||
crdtDataRowId: metadataRowId,
|
||||
hlcDatetime: hlc?.datetime,
|
||||
hlcCounter: hlc?.counter,
|
||||
firstSeenAt: now,
|
||||
lastSeenAt: now,
|
||||
occurrences: 1,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
Future<UuidValue?> _scopeUuidForNormalizedId(
|
||||
int? scopeId,
|
||||
Transaction transaction,
|
||||
) async {
|
||||
if (scopeId == null) return null;
|
||||
|
||||
final scope = await CrdtScope.db.findById(
|
||||
_session,
|
||||
scopeId,
|
||||
transaction: transaction,
|
||||
);
|
||||
return scope?.uuidScopeId;
|
||||
}
|
||||
|
||||
T _withScopeId<T extends TableRow>(T row, int scopeId) {
|
||||
return (row as dynamic).copyWith(scopeId: scopeId) as T;
|
||||
}
|
||||
}
|
||||
|
||||
/// Signals that a merge insert lost a primary-key race to a row owned by
|
||||
/// another scope, so the savepoint around the insert application must be
|
||||
/// rolled back before the sync violation is recorded and thrown.
|
||||
class _ForeignScopeRowCollision implements Exception {
|
||||
_ForeignScopeRowCollision(this.owningScopeId);
|
||||
|
||||
final int? owningScopeId;
|
||||
}
|
||||
|
||||
extension on DatabaseSerializationManager {
|
||||
|
|
@ -728,7 +967,9 @@ extension on DatabaseSerializationManager {
|
|||
T patchTableRow<T extends TableRow>(T row, Map<String, Object?> data) {
|
||||
final rowJson = Map<String, dynamic>.from(row.toJson() as Map);
|
||||
for (final column in row.table.managedColumns) {
|
||||
if (column.columnName != 'id' && data.containsKey(column.columnName)) {
|
||||
if (column.columnName != 'id' &&
|
||||
column.columnName != 'scopeId' &&
|
||||
data.containsKey(column.columnName)) {
|
||||
rowJson[column.fieldName] = data[column.columnName];
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -409,10 +409,10 @@ extension _CrdtForeignKeyProjector on CrdtMutationRecorder {
|
|||
|
||||
for (final tableName in _syncTableByName.keys) {
|
||||
final (tableId, _) = _schema[tableName]!;
|
||||
final userId = _getHlcManager(transaction).normalizedUserId;
|
||||
final userId = _getHlcManager(transaction).normalizedScopeId;
|
||||
final crdtRows = await CrdtDataRow.db.find(
|
||||
_session,
|
||||
where: (t) => t.userId.equals(userId) & t.tblId.equals(tableId),
|
||||
where: (t) => t.scopeId.equals(userId) & t.tblId.equals(tableId),
|
||||
include: CrdtDataRow.include(deleted: CrdtDataDeleted.include()),
|
||||
orderBy: (t) => t.uuidRowId,
|
||||
transaction: transaction,
|
||||
|
|
@ -519,11 +519,11 @@ extension _CrdtForeignKeyProjector on CrdtMutationRecorder {
|
|||
if (rowIds.isEmpty || columnNames.isEmpty) return {};
|
||||
|
||||
final (tableId, _) = _schema[tableName]!;
|
||||
final userId = _getHlcManager(transaction).normalizedUserId;
|
||||
final userId = _getHlcManager(transaction).normalizedScopeId;
|
||||
final fields = await CrdtDataField.db.find(
|
||||
_session,
|
||||
where: (t) =>
|
||||
t.row.userId.equals(userId) &
|
||||
t.row.scopeId.equals(userId) &
|
||||
t.row.tblId.equals(tableId) &
|
||||
t.row.uuidRowId.inSet(rowIds) &
|
||||
t.column.name.inSet(columnNames),
|
||||
|
|
@ -970,6 +970,7 @@ extension _CrdtForeignKeyProjector on CrdtMutationRecorder {
|
|||
fields: <_MergeFieldKey, CrdtDataField>{},
|
||||
incomingFieldHlcs: <_MergeFieldKey, Hlc>{},
|
||||
tombstones: <_MergeRowKey, CrdtDataDeleted>{},
|
||||
domainOwners: <_MergeRowKey, _DomainOwner>{},
|
||||
);
|
||||
|
||||
for (final MapEntry(key: rowKey, value: updates) in changedUpdatesByRow.entries) {
|
||||
|
|
@ -1059,7 +1060,13 @@ extension _CrdtForeignKeyProjector on CrdtMutationRecorder {
|
|||
) async {
|
||||
final rowIds = await _findVisibleDomainRowIdsWhere(
|
||||
tableName: edge.parentTableName,
|
||||
predicates: [_domainColumnPredicate(edge.parentColumn, value)],
|
||||
predicates: [
|
||||
_domainColumnPredicate(
|
||||
'scopeId',
|
||||
_getHlcManager(transaction).normalizedScopeId,
|
||||
),
|
||||
_domainColumnPredicate(edge.parentColumn, value),
|
||||
],
|
||||
transaction: transaction,
|
||||
);
|
||||
return rowIds.isNotEmpty;
|
||||
|
|
|
|||
|
|
@ -259,7 +259,14 @@ class CrdtUniqueConflictResolver {
|
|||
if (values.containsKey(column.columnName))
|
||||
column.columnName: values[column.columnName],
|
||||
};
|
||||
if (columnValues.length != uniqueIndex.columns.length) continue;
|
||||
if (uniqueIndex.scoped) {
|
||||
columnValues['scopeId'] = _recorder
|
||||
._getHlcManager(transaction)
|
||||
.normalizedScopeId;
|
||||
}
|
||||
final expectedValueCount =
|
||||
uniqueIndex.columns.length + (uniqueIndex.scoped ? 1 : 0);
|
||||
if (columnValues.length != expectedValueCount) continue;
|
||||
if (columnValues.values.any((value) => value == null)) continue;
|
||||
|
||||
final uniqueConflictIds = await _findVisibleRowsByUniqueValues(
|
||||
|
|
|
|||
|
|
@ -2,14 +2,16 @@ import 'package:serverpod_database/serverpod_database.dart';
|
|||
import 'package:serverpod_offline_sync_shared/serverpod_offline_sync_shared.dart';
|
||||
import 'package:serverpod_serialization/serverpod_serialization.dart';
|
||||
|
||||
import '../crdt/exceptions.dart';
|
||||
import '../crdt/extensions.dart';
|
||||
import '../crdt/merge.dart';
|
||||
import '../managers/hlc.dart';
|
||||
import '../managers/user.dart';
|
||||
import '../managers/scope.dart';
|
||||
import '../protocol/protocol.dart';
|
||||
import 'database.dart';
|
||||
import 'schema.dart';
|
||||
import 'session.dart';
|
||||
import 'unique_index_utils.dart';
|
||||
|
||||
part 'merge.dart';
|
||||
part 'merge_utils/foreign_key_projector.dart';
|
||||
|
|
@ -36,6 +38,7 @@ typedef _ForeignKeyValueKey = (String tableName, String columnName, String value
|
|||
|
||||
typedef _UniqueIndexConflictRelease = ({
|
||||
List<_UniqueColumnConflictRelease> columns,
|
||||
bool scoped,
|
||||
});
|
||||
|
||||
typedef _UniqueColumnConflictRelease = ({
|
||||
|
|
@ -76,7 +79,7 @@ class CrdtMutationRecorder {
|
|||
persistentUserId: persistentUserId,
|
||||
);
|
||||
|
||||
late final CrdtUserManager _userManager = CrdtUserManager(_dbSession);
|
||||
late final CrdtScopeManager _scopeManager = CrdtScopeManager(_dbSession);
|
||||
|
||||
final Map<UuidValue, HlcManager> _hlcManagers = {};
|
||||
|
||||
|
|
@ -162,11 +165,11 @@ class CrdtMutationRecorder {
|
|||
/// Safe to call again after the database was wiped (e.g. test `tearDown`)
|
||||
/// for in-memory schema ids to match new rows.
|
||||
Future<void> initialize() async {
|
||||
_userManager.clearCache();
|
||||
_scopeManager.clearCache();
|
||||
_hlcManagers.clear();
|
||||
|
||||
if (persistentUserId != null) {
|
||||
await _userManager.getOrCreate(persistentUserId!);
|
||||
await _scopeManager.getOrCreate(persistentUserId!);
|
||||
}
|
||||
|
||||
final schemaRegistry = CrdtSchemaRegistry(_dbSession, syncTables: syncTables);
|
||||
|
|
@ -200,7 +203,8 @@ class CrdtMutationRecorder {
|
|||
|
||||
late final Map<String, Set<String>> _syncedTableColumnNamesForMerge = {
|
||||
for (final MapEntry(key: k, value: cols) in _columnsByTableAndName.entries)
|
||||
if (_syncTableByName.containsKey(k)) k: cols.keys.toSet(),
|
||||
if (_syncTableByName.containsKey(k))
|
||||
k: cols.keys.where((columnName) => columnName != 'scopeId').toSet(),
|
||||
};
|
||||
|
||||
/// Whether the given table is tracked by CRDT.
|
||||
|
|
@ -215,9 +219,25 @@ class CrdtMutationRecorder {
|
|||
return _syncTableByName.containsKey(tableName);
|
||||
}
|
||||
|
||||
/// Returns the [CrdtUser] for the given user ID, creating it when needed.
|
||||
Future<CrdtUser> getOrCreateUser(UuidValue userId) {
|
||||
return _userManager.getOrCreate(userId);
|
||||
/// Returns the local [CrdtSchemaTable] id for [tableName], or null when the
|
||||
/// table is not registered for CRDT synchronization.
|
||||
int? tableIdForName(String tableName) => _schema[tableName]?.$1;
|
||||
|
||||
/// Returns the user scoping CRDT visibility for queries, or null when no
|
||||
/// user is associated with [transaction] and no persistent user exists.
|
||||
CrdtScope? scopeForQueries(Transaction? transaction) {
|
||||
if (transaction != null) {
|
||||
final user = scopeForTransaction[transaction];
|
||||
if (user != null) return user;
|
||||
}
|
||||
final userId = persistentUserId;
|
||||
if (userId == null) return null;
|
||||
return _scopeManager.getCached(userId);
|
||||
}
|
||||
|
||||
/// Returns the [CrdtScope] for the given user ID, creating it when needed.
|
||||
Future<CrdtScope> getOrCreateScope(UuidValue userId) {
|
||||
return _scopeManager.getOrCreate(userId);
|
||||
}
|
||||
|
||||
/// Records the latest acknowledged sync checkpoint for [otherNodeId].
|
||||
|
|
@ -226,17 +246,17 @@ class CrdtMutationRecorder {
|
|||
UuidValue otherNodeId,
|
||||
Hlc syncedHlc,
|
||||
) async {
|
||||
final user = await _userManager.getOrCreate(userId);
|
||||
final user = await _scopeManager.getOrCreate(userId);
|
||||
await _db.transaction((transaction) async {
|
||||
var node = await CrdtNode.db.findFirstRow(
|
||||
_session,
|
||||
where: (t) => t.userId.equals(user.id) & t.uuidNodeId.equals(otherNodeId),
|
||||
where: (t) => t.scopeId.equals(user.id) & t.uuidNodeId.equals(otherNodeId),
|
||||
transaction: transaction,
|
||||
);
|
||||
|
||||
node ??= await CrdtNode.db.insertRow(
|
||||
_session,
|
||||
CrdtNode(userId: user.id!, uuidNodeId: otherNodeId),
|
||||
CrdtNode(scopeId: user.id!, uuidNodeId: otherNodeId),
|
||||
transaction: transaction,
|
||||
);
|
||||
|
||||
|
|
@ -421,7 +441,7 @@ class CrdtMutationRecorder {
|
|||
}) async {
|
||||
if (foreignKeys.isEmpty) return null;
|
||||
|
||||
final userId = _getHlcManager(transaction).normalizedUserId;
|
||||
final scopeId = _getHlcManager(transaction).normalizedScopeId;
|
||||
final escapedChildTable = _escapeIdentifier(childTableName);
|
||||
final whereRowIds = _sqlLiteralList(childRowIds);
|
||||
|
||||
|
|
@ -440,10 +460,14 @@ FROM "$escapedChildTable" c
|
|||
LEFT JOIN "$parentTable" p
|
||||
ON p."$parentColumn" = c."$childColumn"
|
||||
LEFT JOIN "crdt_data_rows" r
|
||||
ON r."userId" = $userId AND r."tblId" = $parentTableId AND r."uuidRowId" = p."id"
|
||||
ON r."scopeId" = $scopeId AND r."tblId" = $parentTableId AND r."uuidRowId" = p."id"
|
||||
WHERE c."id" IN ($whereRowIds)
|
||||
AND c."$childColumn" IS NOT NULL
|
||||
AND (p."id" IS NULL OR r."visibility" > $crdtRowLastVisibleVisibilityIndex)
|
||||
AND (
|
||||
p."id" IS NULL OR
|
||||
p."scopeId" <> $scopeId OR
|
||||
r."visibility" > $crdtRowLastVisibleVisibilityIndex
|
||||
)
|
||||
''';
|
||||
});
|
||||
|
||||
|
|
@ -487,7 +511,7 @@ WHERE c."id" IN ($whereRowIds)
|
|||
final hlc = hlcManager.increment();
|
||||
|
||||
return CrdtDataRow(
|
||||
userId: hlcManager.normalizedUserId,
|
||||
scopeId: hlcManager.normalizedScopeId,
|
||||
tblId: tableId,
|
||||
uuidRowId: rowId,
|
||||
nodeId: hlcManager.normalizedNodeId,
|
||||
|
|
@ -552,7 +576,7 @@ WHERE c."id" IN ($whereRowIds)
|
|||
|
||||
final table = updatedRows.first.table;
|
||||
final updatedColumnList = (columns ?? table.managedColumns)
|
||||
.where((c) => c.columnName != 'id')
|
||||
.where((c) => c.columnName != 'id' && c.columnName != 'scopeId')
|
||||
.toList();
|
||||
if (updatedColumnList.isEmpty) return;
|
||||
|
||||
|
|
@ -871,11 +895,13 @@ WHERE c."id" IN ($whereRowIds)
|
|||
CrdtDataRowInclude? include,
|
||||
}) {
|
||||
final (tableId, _) = _schema[tableName]!;
|
||||
final userId = _getHlcManager(transaction).normalizedUserId;
|
||||
final scopeId = _getHlcManager(transaction).normalizedScopeId;
|
||||
return CrdtDataRow.db.find(
|
||||
_session,
|
||||
where: (t) =>
|
||||
t.userId.equals(userId) & t.tblId.equals(tableId) & t.uuidRowId.inSet(rowIds),
|
||||
t.scopeId.equals(scopeId) &
|
||||
t.tblId.equals(tableId) &
|
||||
t.uuidRowId.inSet(rowIds),
|
||||
include: include,
|
||||
transaction: transaction,
|
||||
);
|
||||
|
|
@ -979,9 +1005,7 @@ WHERE c."id" IN ($whereRowIds)
|
|||
|
||||
final uniqueIndexes = _uniqueIndexesForTable(tableDefinition);
|
||||
for (final uniqueIndex in uniqueIndexes) {
|
||||
final columnNames = uniqueIndex.columns
|
||||
.map((column) => column.columnName)
|
||||
.toList();
|
||||
final columnNames = uniqueIndex.columnNames.toList();
|
||||
|
||||
final valuesByRowId = await _readDomainColumnValues(
|
||||
tableName,
|
||||
|
|
@ -1025,7 +1049,10 @@ WHERE c."id" IN ($whereRowIds)
|
|||
) {
|
||||
return _uniqueIndexesByTableName.putIfAbsent(
|
||||
tableDefinition.name,
|
||||
() => _syncableUniqueIndexesForTable(tableDefinition),
|
||||
() => _syncableUniqueIndexesForTable(
|
||||
tableDefinition,
|
||||
_syncTableByName.keys.toSet(),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
|
|
@ -1048,13 +1075,13 @@ WHERE c."id" IN ($whereRowIds)
|
|||
required Transaction transaction,
|
||||
}) async {
|
||||
final (tableId, _) = _schema[tableName]!;
|
||||
final userId = _getHlcManager(transaction).normalizedUserId;
|
||||
final scopeId = _getHlcManager(transaction).normalizedScopeId;
|
||||
final result = await _db.unsafeQuery(
|
||||
'''
|
||||
SELECT d."id"
|
||||
FROM "${_escapeIdentifier(tableName)}" d
|
||||
LEFT JOIN "crdt_data_rows" r
|
||||
ON r."userId" = $userId AND r."tblId" = $tableId AND r."uuidRowId" = d."id"
|
||||
ON r."scopeId" = $scopeId AND r."tblId" = $tableId AND r."uuidRowId" = d."id"
|
||||
WHERE (${predicates.join(') AND (')})
|
||||
AND (r."id" IS NULL OR r."visibility" <= $crdtRowLastVisibleVisibilityIndex)
|
||||
''',
|
||||
|
|
@ -1170,20 +1197,18 @@ WHERE "id" IN (${_sqlLiteralList(rowIds)})
|
|||
}
|
||||
|
||||
HlcManager _getHlcManager(Transaction transaction) {
|
||||
final user = _getEffectiveUser(transaction);
|
||||
final user = _getEffectiveScope(transaction);
|
||||
return _hlcManagers.putIfAbsent(
|
||||
user.uuidUserId,
|
||||
() => HlcManager.forUser(user),
|
||||
user.uuidScopeId,
|
||||
() => HlcManager.forScope(user),
|
||||
);
|
||||
}
|
||||
|
||||
CrdtUser _getEffectiveUser(Transaction transaction) {
|
||||
final user = userForTransaction[transaction];
|
||||
if (user != null) return user;
|
||||
if (persistentUserId == null) {
|
||||
throw StateError('No user ID found for transaction or persistent user ID.');
|
||||
}
|
||||
return _userManager.getCached(persistentUserId!);
|
||||
CrdtScope _getEffectiveScope(Transaction transaction) {
|
||||
return scopeForQueries(transaction) ??
|
||||
(throw StateError(
|
||||
'No user ID found for transaction or persistent user ID.',
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -1197,6 +1222,7 @@ int _nextClFlag(CrdtDataDeleted? current, bool isDeleted) {
|
|||
|
||||
List<_UniqueIndexConflictRelease> _syncableUniqueIndexesForTable(
|
||||
TableDefinition table,
|
||||
Set<String> syncTableNames,
|
||||
) {
|
||||
final uniqueIndexes = table.indexes
|
||||
.where(
|
||||
|
|
@ -1205,24 +1231,49 @@ List<_UniqueIndexConflictRelease> _syncableUniqueIndexesForTable(
|
|||
!index.isPrimary &&
|
||||
index.elements.every(
|
||||
(element) => element.type == IndexElementDefinitionType.column,
|
||||
),
|
||||
) &&
|
||||
(_isScopedUniqueIndex(index) ||
|
||||
isCrdtAllowedForeignKeyOnlyUniqueIndex(
|
||||
table,
|
||||
index,
|
||||
syncTableNames,
|
||||
)),
|
||||
)
|
||||
.toList();
|
||||
|
||||
final columnsByName = {for (final column in table.columns) column.name: column};
|
||||
return [
|
||||
for (final index in uniqueIndexes)
|
||||
(
|
||||
_uniqueIndexConflictReleaseForIndex(table, columnsByName, index),
|
||||
];
|
||||
}
|
||||
|
||||
bool _isScopedUniqueIndex(IndexDefinition index) {
|
||||
return index.elements.any(
|
||||
(element) =>
|
||||
element.type == IndexElementDefinitionType.column &&
|
||||
element.definition == 'scopeId',
|
||||
);
|
||||
}
|
||||
|
||||
_UniqueIndexConflictRelease _uniqueIndexConflictReleaseForIndex(
|
||||
TableDefinition table,
|
||||
Map<String, ColumnDefinition> columnsByName,
|
||||
IndexDefinition index,
|
||||
) {
|
||||
final columnNames = index.elements.map((element) => element.definition).toList();
|
||||
return (
|
||||
scoped: columnNames.contains('scopeId'),
|
||||
columns: [
|
||||
for (final columnName in index.elements.map((element) => element.definition))
|
||||
for (final columnName in columnNames)
|
||||
if (columnName != 'scopeId')
|
||||
_uniqueConflictReleaseForColumn(
|
||||
table,
|
||||
columnsByName[columnName],
|
||||
columnName,
|
||||
),
|
||||
],
|
||||
),
|
||||
];
|
||||
);
|
||||
}
|
||||
|
||||
_UniqueColumnConflictRelease _uniqueConflictReleaseForColumn(
|
||||
|
|
|
|||
|
|
@ -2,11 +2,16 @@ import 'package:serverpod_database/serverpod_database.dart';
|
|||
import 'package:uuid/uuid.dart';
|
||||
|
||||
import '../protocol/protocol.dart';
|
||||
import 'unique_index_utils.dart';
|
||||
|
||||
/// Manages the CRDT schema for a database.
|
||||
class CrdtSchemaRegistry {
|
||||
/// Creates a new instance of [CrdtSchemaRegistry].
|
||||
CrdtSchemaRegistry(this._session, {required this.syncTables}) {
|
||||
CrdtSchemaRegistry(
|
||||
this._session, {
|
||||
required this.syncTables,
|
||||
Iterable<TableDefinition>? tableDefinitions,
|
||||
}) {
|
||||
final tablesWithoutUuidPk = syncTables.where(
|
||||
(table) => table.id is! Column<UuidValue>,
|
||||
);
|
||||
|
|
@ -17,6 +22,47 @@ class CrdtSchemaRegistry {
|
|||
'${tablesWithoutUuidPk.map((t) => '"${t.tableName}"').join(', ')}',
|
||||
);
|
||||
}
|
||||
|
||||
final tablesWithoutScopeId = syncTables.where((table) {
|
||||
final scopeColumn = _scopeIdColumn(table);
|
||||
return scopeColumn == null || scopeColumn is! Column<int>;
|
||||
}).toList();
|
||||
if (tablesWithoutScopeId.isNotEmpty) {
|
||||
throw StateError(
|
||||
'CRDT can only synchronize tables with a nullable int scopeId column, '
|
||||
'but ${tablesWithoutScopeId.length} table(s) are missing it or declare '
|
||||
'it with the wrong type: '
|
||||
'${tablesWithoutScopeId.map((t) => '"${t.tableName}"').join(', ')}\n\n'
|
||||
'Add this field to every synced model:\n'
|
||||
'fields:\n'
|
||||
' id: UuidValue?, defaultPersist=random_v7\n'
|
||||
' ### Owner scope of this row. Maintained by the CRDT sync layer.\n'
|
||||
' scopeId: int?\n\n'
|
||||
'If the column is declared with scope=serverOnly, remove the scope; '
|
||||
'the column must exist on every database.',
|
||||
);
|
||||
}
|
||||
|
||||
final tableDefinitionsByName = {
|
||||
for (final tableDefinition
|
||||
in tableDefinitions ??
|
||||
_session.db.serializationManager.getTargetTableDefinitions())
|
||||
tableDefinition.name: tableDefinition,
|
||||
};
|
||||
final globalUniqueIndexes = _globalUniqueIndexViolations(
|
||||
syncTables,
|
||||
tableDefinitionsByName,
|
||||
);
|
||||
if (globalUniqueIndexes.isNotEmpty) {
|
||||
throw StateError(
|
||||
'CRDT can only synchronize tables with per-scope unique indexes, but '
|
||||
'${globalUniqueIndexes.length} unique index(es) do not include scopeId: '
|
||||
'${globalUniqueIndexes.join(', ')}. '
|
||||
'Add scopeId to the unique index fields. '
|
||||
'The only allowed global unique indexes are foreign-key-only indexes '
|
||||
'whose target tables are also synchronized.',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
final DatabaseSession _session;
|
||||
|
|
@ -27,7 +73,8 @@ class CrdtSchemaRegistry {
|
|||
late final _columnsPerTableName = {
|
||||
for (final table in syncTables)
|
||||
table.tableName: [
|
||||
for (final column in table.columns) column.columnName,
|
||||
for (final column in table.columns)
|
||||
if (column.columnName != 'scopeId') column.columnName,
|
||||
],
|
||||
};
|
||||
|
||||
|
|
@ -118,3 +165,48 @@ class CrdtSchemaRegistry {
|
|||
return foundColumnRows;
|
||||
}
|
||||
}
|
||||
|
||||
Column? _scopeIdColumn(Table table) {
|
||||
for (final column in table.columns) {
|
||||
if (column.columnName == 'scopeId') return column;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
List<String> _globalUniqueIndexViolations(
|
||||
List<Table> syncTables,
|
||||
Map<String, TableDefinition> tableDefinitionsByName,
|
||||
) {
|
||||
final syncTableNames = {for (final table in syncTables) table.tableName};
|
||||
return [
|
||||
for (final tableName in syncTableNames)
|
||||
for (final index
|
||||
in tableDefinitionsByName[tableName]?.indexes ?? const <IndexDefinition>[])
|
||||
if (_isForbiddenGlobalUniqueIndex(
|
||||
tableDefinitionsByName[tableName]!,
|
||||
index,
|
||||
syncTableNames,
|
||||
))
|
||||
'$tableName.${index.indexName}',
|
||||
];
|
||||
}
|
||||
|
||||
bool _isForbiddenGlobalUniqueIndex(
|
||||
TableDefinition tableDefinition,
|
||||
IndexDefinition index,
|
||||
Set<String> syncTableNames,
|
||||
) {
|
||||
if (!index.isUnique || index.isPrimary) return false;
|
||||
final includesScopeId = index.elements.any(
|
||||
(element) =>
|
||||
element.type == IndexElementDefinitionType.column &&
|
||||
element.definition == 'scopeId',
|
||||
);
|
||||
if (includesScopeId) return false;
|
||||
|
||||
return !isCrdtAllowedForeignKeyOnlyUniqueIndex(
|
||||
tableDefinition,
|
||||
index,
|
||||
syncTableNames,
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,172 @@
|
|||
part of 'database.dart';
|
||||
|
||||
extension _CrdtDatabaseScope on CrdtDatabase {
|
||||
({
|
||||
List<T> rows,
|
||||
Set<int> stampedIndexes,
|
||||
Set<Object> explicitRowIds,
|
||||
})
|
||||
_prepareRowsForInsert<T extends TableRow>(
|
||||
List<T> rows,
|
||||
Transaction transaction,
|
||||
) {
|
||||
if (!_recorder.isCrdtTracked<T>(rows.first.table)) {
|
||||
return (rows: rows, stampedIndexes: const {}, explicitRowIds: const {});
|
||||
}
|
||||
|
||||
final scope = _requireEffectiveScope(transaction);
|
||||
final effectiveScopeId = scope.id!;
|
||||
final databaseRows = <T>[];
|
||||
final stampedIndexes = <int>{};
|
||||
final explicitRowIds = <Object>{};
|
||||
|
||||
for (final (index, row) in rows.indexed) {
|
||||
final rowScopeId = _readScopeId(row);
|
||||
if (rowScopeId == null) {
|
||||
databaseRows.add(_copyWithScopeId(row, effectiveScopeId));
|
||||
stampedIndexes.add(index);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (rowScopeId != effectiveScopeId) {
|
||||
throw StateError(
|
||||
'Cannot write ${row.table.tableName} row ${row.id} with scopeId '
|
||||
'$rowScopeId while acting in scope $effectiveScopeId.',
|
||||
);
|
||||
}
|
||||
|
||||
databaseRows.add(row);
|
||||
final rowId = row.id;
|
||||
if (rowId != null) explicitRowIds.add(rowId as Object);
|
||||
}
|
||||
|
||||
return (
|
||||
rows: databaseRows,
|
||||
stampedIndexes: stampedIndexes,
|
||||
explicitRowIds: explicitRowIds,
|
||||
);
|
||||
}
|
||||
|
||||
bool _shouldStripReturnedScopeId<T extends TableRow>(
|
||||
T row,
|
||||
Transaction transaction,
|
||||
) {
|
||||
if (!_recorder.isCrdtTracked<T>(row.table)) return false;
|
||||
|
||||
final rowScopeId = _readScopeId(row);
|
||||
if (rowScopeId == null) return true;
|
||||
|
||||
final effectiveScopeId = _requireEffectiveScope(transaction).id!;
|
||||
if (rowScopeId != effectiveScopeId) {
|
||||
throw StateError(
|
||||
'Cannot write ${row.table.tableName} row ${row.id} with scopeId '
|
||||
'$rowScopeId while acting in scope $effectiveScopeId.',
|
||||
);
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
void _assertNoScopeIdColumnValues<T extends TableRow>(
|
||||
List<ColumnValue> columnValues,
|
||||
) {
|
||||
for (final columnValue in columnValues) {
|
||||
if (columnValue.column.columnName == 'scopeId') {
|
||||
final table = serializationManager.getTableForType(T)?.tableName ?? 'unknown';
|
||||
throw StateError(
|
||||
'scopeId is immutable and owned by the CRDT sync layer for $table.',
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
CrdtScope _requireEffectiveScope(Transaction transaction) {
|
||||
return _recorder.scopeForQueries(transaction) ??
|
||||
(throw StateError(
|
||||
'A scope ID is required for CRDT writes without a persistent user.',
|
||||
));
|
||||
}
|
||||
|
||||
int? _readScopeId(TableRow row) => (row as dynamic).scopeId as int?;
|
||||
|
||||
T _copyWithScopeId<T extends TableRow>(T row, int scopeId) {
|
||||
return (row as dynamic).copyWith(scopeId: scopeId) as T;
|
||||
}
|
||||
|
||||
void _stripStampedRows<T extends TableRow>(
|
||||
List<T> rows,
|
||||
({
|
||||
List<T> rows,
|
||||
Set<int> stampedIndexes,
|
||||
Set<Object> explicitRowIds,
|
||||
})
|
||||
prepared,
|
||||
) {
|
||||
if (rows.length == prepared.rows.length) {
|
||||
for (final (index, row) in rows.indexed) {
|
||||
if (prepared.stampedIndexes.contains(index)) _stripScopeId(row);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// `ignoreConflicts` dropped rows, so positions no longer align with the
|
||||
// input. Keep the value only on rows whose id was caller-provided on an
|
||||
// explicit (asserted) input row; everything else is treated as stamped.
|
||||
for (final row in rows) {
|
||||
final rowId = row.id;
|
||||
if (rowId == null || !prepared.explicitRowIds.contains(rowId)) {
|
||||
_stripScopeId(row);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void _stripScopeId(TableRow row) {
|
||||
(row as dynamic).scopeId = null;
|
||||
}
|
||||
|
||||
List<T> _stripScopeIdFromScopedRead<T extends TableRow>(
|
||||
List<T> rows,
|
||||
Include? include,
|
||||
Transaction? transaction,
|
||||
) {
|
||||
if (rows.isEmpty) return rows;
|
||||
if (!_recorder.isCrdtTracked<T>(rows.first.table)) return rows;
|
||||
if (_recorder.scopeForQueries(transaction) == null) return rows;
|
||||
|
||||
final hasNestedRows =
|
||||
include != null && include.includes.values.any((inc) => inc != null);
|
||||
if (!hasNestedRows) {
|
||||
// Hot path: delegate-returned rows are package-owned, so the root
|
||||
// field is stripped in place with no allocation.
|
||||
rows.forEach(_stripScopeId);
|
||||
return rows;
|
||||
}
|
||||
|
||||
// Nested models are only reachable through generated fields, which
|
||||
// cannot be accessed generically by name, so include graphs pay a
|
||||
// serialization round-trip per row to strip recursively.
|
||||
return [
|
||||
for (final row in rows) _copyWithScopeIdsStripped(row),
|
||||
];
|
||||
}
|
||||
|
||||
T _copyWithScopeIdsStripped<T extends TableRow>(T row) {
|
||||
final json = Map<String, dynamic>.from(row.toJson() as Map);
|
||||
_stripScopeIdsFromJson(json);
|
||||
return serializationManager.deserialize<T>(json);
|
||||
}
|
||||
|
||||
void _stripScopeIdsFromJson(Object? value) {
|
||||
if (value is Map) {
|
||||
if (value.containsKey('scopeId')) {
|
||||
value['scopeId'] = null;
|
||||
}
|
||||
value.values.forEach(_stripScopeIdsFromJson);
|
||||
return;
|
||||
}
|
||||
|
||||
if (value is Iterable) {
|
||||
value.forEach(_stripScopeIdsFromJson);
|
||||
}
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue