halkyon-days/serverpod-offline-sync
Watch
1
0
Fork
You've already forked serverpod-offline-sync
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

refactor: Move the core implementation to the shared package with no manual queries (#107)

Browse source
This commit is contained in:
Marcelo Mendonça Soares 2026-08-04 11:04:04 -03:00 • committed by GitHub
commit 9f2ce4d9e6
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
55 changed files with 1992 additions and 14018 deletions
Download patch file Download diff file Expand all files Collapse all files

13
packages/serverpod_offline_sync_client/lib/crdt.dart
View file

@ -1,12 +1,3 @@
export 'package:serverpod_offline_sync/serverpod_offline_sync.dart' hide Protocol;
export 'src/crdt/client_sync.dart';
export 'src/crdt/exceptions.dart' hide PendingOutboundIntegrityViolation;
export 'src/crdt/merge.dart';
export 'src/crdt/scope_membership.dart';
export 'src/crdt/sync.dart';
export 'src/database/database.dart';
export 'src/database/recorder.dart';
export 'src/database/schema.dart';
export 'src/database/session.dart' show CrdtDatabaseAccess, CrdtDatabaseSession;
export 'src/database/tombstone.dart' show IncludeTombstonedRows;
export 'src/managers/hlc.dart';
export 'src/managers/scope.dart';

4
packages/serverpod_offline_sync_client/lib/serverpod_offline_sync_client.dart
View file

@ -1,5 +1,5 @@
export 'package:serverpod_client/serverpod_client.dart';
export 'package:serverpod_offline_sync/serverpod_offline_sync.dart' hide Protocol;
export 'crdt.dart';
export 'src/crdt/extensions.dart';
export 'src/crdt/client_sync.dart';
export 'src/protocol/protocol.dart';

148
packages/serverpod_offline_sync_client/lib/src/crdt/client_sync.dart
View file

@ -1,147 +1,15 @@
import 'dart:async';
import 'package:serverpod_client/serverpod_client.dart';
import 'package:serverpod_database/serverpod_database.dart';
import 'package:serverpod_offline_sync/serverpod_offline_sync.dart';
import '../database/session.dart';
import '../protocol/protocol.dart';
import 'sync.dart';
/// A running continuous CRDT sync session.
class CrdtSyncSession {
CrdtSyncSession._({
required this.done,
required Future<void> Function() cancel,
}) : _cancel = cancel;
/// Completes when the sync session ends normally or after [cancel].
final Future<void> done;
final Future<void> Function() _cancel;
/// Stops the sync session and closes the transport. This method is idempotent
/// and can be called multiple times.
Future<void> cancel() => _cancel();
}
/// High-level client sync helpers built on top of the generated module caller.
class CrdtSyncClient {
/// Creates a [CrdtSyncClient] for the provided module caller.
CrdtSyncClient(this._caller);
final Caller _caller;
/// Pushes local pending changes for [session] to the remote peer once.
///
/// The [session] must be wrapped in a [CrdtDatabaseSession].
Future<void> syncOnce(
DatabaseSession session, {
CrdtSyncOnMergeSuccess? onMergeSuccess,
}) async {
await _startSyncSession(session, once: true, onMergeSuccess: onMergeSuccess).done;
}
/// Keeps synchronizing [session] with the remote peer until cancelled through
/// [CrdtSyncSession.cancel] or the remote stream closes.
///
/// The [session] must be wrapped in a [CrdtDatabaseSession].
CrdtSyncSession syncContinuously(
DatabaseSession session, {
CrdtSyncOnMergeSuccess? onMergeSuccess,
}) => _startSyncSession(session, once: false, onMergeSuccess: onMergeSuccess);
CrdtSyncSession _startSyncSession(
DatabaseSession session, {
required bool once,
CrdtSyncOnMergeSuccess? onMergeSuccess,
}) {
// The stream will be closed using [outboundChanges.closeOrSkip].
// ignore: close_sinks
final outboundChanges = StreamController<CrdtSyncStreamEvent>();
final doneCompleter = Completer<void>();
var cancelled = false;
try {
final remoteStream = _caller.crdtSync.sync(
changes: outboundChanges.stream,
once: once,
);
final subscription = session.crdtDb
.sync(
inbound: remoteStream,
once: once,
onMergeSuccess: onMergeSuccess,
mode: CrdtSyncPeerMode.follower,
)
.listen(
(event) {
outboundChanges.addIfNotClosed(event);
if (once && event is CrdtSyncClose) {
unawaited(outboundChanges.closeOrSkip());
}
},
onDone: doneCompleter.completeOrSkip,
onError: doneCompleter.completeErrorOrSkip,
cancelOnError: true,
);
return CrdtSyncSession._(
done: doneCompleter.future.whenComplete(outboundChanges.closeOrSkip),
cancel: () async {
if (cancelled) return;
cancelled = true;
unawaited(outboundChanges.closeOrSkip());
if (doneCompleter.isCompleted) return;
const waitTimeout = Duration(milliseconds: 200);
// Prefer graceful shutdown: closing outbound lets the server finish
// and complete [done] via [subscription.onDone].
await doneCompleter.future.timeout(waitTimeout, onTimeout: () {});
await subscription.cancel().timeout(waitTimeout, onTimeout: () {});
doneCompleter.completeOrSkip();
},
);
} catch (_) {
unawaited(outboundChanges.closeOrSkip());
rethrow;
}
}
}
import '../protocol/client.dart';
/// Exposes CRDT sync helpers from a generated client.
extension CrdtSyncClientExtension on ServerpodClientShared {
/// Returns CRDT sync helpers bound to this client.
CrdtSyncClient get crdt => CrdtSyncClient(Caller(this));
}
extension on StreamController<CrdtSyncStreamEvent> {
void addIfNotClosed(CrdtSyncStreamEvent event) {
if (isClosed) return;
add(event);
}
Future<void> closeOrSkip() async {
if (!isClosed) {
unawaited(close());
}
// The done event is only delivered once the stream has a listener. If the
// method stream never listened (e.g. opening it failed), awaiting [done]
// would hang forever.
if (hasListener) {
await done;
}
}
}
extension on Completer<void> {
void completeOrSkip() {
if (isCompleted) return;
complete();
}
void completeErrorOrSkip(Object error, StackTrace stackTrace) {
if (isCompleted) return;
completeError(error, stackTrace);
}
CrdtSyncClient get crdt => CrdtSyncClient(
({required changes, required once}) => Caller(this).crdtSync.sync(
changes: changes,
once: once,
),
);
}

177
packages/serverpod_offline_sync_client/lib/src/crdt/exceptions.dart
View file

@ -1,177 +0,0 @@
import 'package:meta/meta.dart';
import 'package:serverpod_serialization/serverpod_serialization.dart';
import '../protocol/protocol.dart';
/// Base type for CRDT sync session failures.
sealed class CrdtSyncException implements Exception {
const CrdtSyncException();
}
/// Thrown when the inbound sync stream closes before an expected event arrives.
final class CrdtSyncStreamClosedException extends CrdtSyncException {
/// Creates a [CrdtSyncStreamClosedException].
const CrdtSyncStreamClosedException({required this.phase});
/// The protocol phase that was interrupted.
final String phase;
@override
String toString() =>
'CrdtSyncStreamClosedException: sync stream closed before $phase event.';
}
/// Thrown when the inbound sync stream delivers an event of the wrong type.
final class CrdtSyncUnexpectedEventException<T extends CrdtSyncStreamEvent>
extends CrdtSyncException {
/// Creates a [CrdtSyncUnexpectedEventException].
const CrdtSyncUnexpectedEventException({
required this.expected,
required this.received,
});
/// A description of the expected event type or types.
final String expected;
/// The event that was received instead.
final CrdtSyncStreamEvent received;
@override
String toString() =>
'CrdtSyncUnexpectedEventException: expected $expected, but '
'received "${received.runtimeType.className}" instead.';
}
/// Thrown when the sync tables hash sent by a peer does not match locally.
final class SyncTablesHashMismatchException extends CrdtSyncException {
/// Creates a [SyncTablesHashMismatchException].
const SyncTablesHashMismatchException({
required this.received,
required this.expected,
});
/// The hash received from the remote peer.
final String received;
/// The hash computed locally from the configured sync tables.
final String expected;
@override
String toString() =>
'SyncTablesHashMismatchException: schema hash mismatch. Received '
'"$received", expected "$expected". Ensure both sides are on the same '
'schema version before syncing.';
}
/// Thrown when sync observes terminal CRDT metadata/domain-row inconsistency.
final class CrdtSyncIntegrityViolationException extends CrdtSyncException {
/// Creates a [CrdtSyncIntegrityViolationException].
const CrdtSyncIntegrityViolationException(this.violation);
/// The integrity violation that caused sync to stop.
final CrdtSyncIntegrityViolation violation;
@override
String toString() {
final persisted = violation.id == null
? ''
: ' Persisted violation id: ${violation.id}.';
final row = '${violation.domainTableName}.${violation.uuidRowId}';
return switch (violation.type) {
CrdtSyncViolationType.ownershipCollision =>
'CrdtSyncIntegrityViolationException: '
'${violation.type.name}/${violation.operation.name} for $row '
'belongs to scope ${violation.ownerScopeUuid}, but sync attempted '
'scope ${violation.incomingScopeUuid}.$persisted',
CrdtSyncViolationType.missingDomainRow =>
'CrdtSyncIntegrityViolationException: '
'${violation.type.name}/${violation.operation.name} references '
'missing domain row $row for scope '
'${violation.incomingScopeUuid}.$persisted',
CrdtSyncViolationType.unauthorizedWrite =>
'CrdtSyncIntegrityViolationException: '
'${violation.type.name}/${violation.operation.name} rejected '
'write to $row in scope ${violation.incomingScopeUuid}.$persisted',
};
}
}
/// Thrown when a user attempts to act in a scope they are not a member of.
final class CrdtScopeMembershipException implements Exception {
/// Creates a [CrdtScopeMembershipException].
const CrdtScopeMembershipException({
required this.userId,
required this.scopeId,
});
/// The authenticated user id supplied to `transactionForUser`.
final UuidValue userId;
/// The scope id the transaction attempted to act in.
final UuidValue scopeId;
@override
String toString() =>
'CrdtScopeMembershipException: user "$userId" is not a member of '
'scope "$scopeId".';
}
/// Thrown when a user attempts to write in a scope with a non-writable role.
final class CrdtScopeRoleException implements Exception {
/// Creates a [CrdtScopeRoleException].
const CrdtScopeRoleException({
required this.userId,
required this.scopeId,
this.role,
});
/// The authenticated user id supplied to `transactionForUser`.
final UuidValue userId;
/// The scope id the transaction attempted to write in.
final UuidValue scopeId;
/// The role that does not allow writes.
final CrdtScopeRole? role;
@override
String toString() =>
'CrdtScopeRoleException: user "$userId" with role "$role" cannot write '
'in scope "$scopeId".';
}
@internal
final class PendingOutboundIntegrityViolation implements Exception {
const PendingOutboundIntegrityViolation({
required this.crdtDataRowId,
required this.type,
required this.operation,
required this.tableName,
required this.rowId,
required this.ownerScopeId,
required this.incomingScopeUuid,
required this.uuidNodeId,
this.hlc,
});
final int? crdtDataRowId;
final CrdtSyncViolationType type;
final CrdtSyncViolationOperation operation;
final String tableName;
final UuidValue rowId;
final int? ownerScopeId;
final UuidValue incomingScopeUuid;
final UuidValue uuidNodeId;
final Hlc? hlc;
}
extension on Type {
/// The name of the class without the leading underscore and without the 'Impl' suffix.
String get className {
final name = toString();
if (name.startsWith('_') && name.endsWith('Impl')) {
return name.substring(1, name.length - 4);
}
return name;
}
}

78
packages/serverpod_offline_sync_client/lib/src/crdt/extensions.dart
View file

@ -1,78 +0,0 @@
import '../protocol/protocol.dart';
/// Last serialized index that still represents a visible CRDT row.
///
/// The [CrdtDataRowVisibility] is ordered by visibility, so the index is used
/// to compare the visibility states. The last visible state in the enum is
/// [CrdtDataRowVisibility.foreignKeyRestrictRestore].
final crdtRowLastVisibleVisibilityIndex =
CrdtDataRowVisibility.foreignKeyRestrictRestore.index;
/// Extensions for the [CrdtDataRowVisibility] enum.
extension CrdtDataRowVisibilityExtension on CrdtDataRowVisibility {
/// Whether this visibility state hides the row from domain queries.
///
bool get isHidden => index > crdtRowLastVisibleVisibilityIndex;
}
/// Extensions for the [CrdtDataField] class.
extension CrdtDataFieldExtension on CrdtDataField {
/// The HLC represented by this field.
Hlc get hlc => node == null
? (throw StateError(
'Fetch the CRDT field with the node included to get the HLC.',
))
: toHlcForNode(node!.uuidNodeId);
}
/// Extensions for the [CrdtDataRow] class.
extension CrdtDataRowExtension on CrdtDataRow {
/// Whether this row is currently hidden from domain queries.
bool get isHidden => visibility.isHidden;
/// The HLC represented by this row.
Hlc get hlc => node == null
? (throw StateError(
'Fetch the CRDT row with the node included to get the HLC.',
))
: toHlcForNode(node!.uuidNodeId);
}
/// Extensions for the [CrdtDataDeleted] class.
extension CrdtDataDeletedExtension on CrdtDataDeleted {
/// Whether this visibility generation hides the row.
bool get isDeleted => clFlag.isEven;
/// The HLC represented by this visibility generation.
Hlc get hlc => node == null
? (throw StateError(
'Fetch the CRDT visibility generation with the node included to get the HLC.',
))
: toHlcForNode(node!.uuidNodeId);
}
/// Extensions for the [CrdtDataForeignKey] class.
extension CrdtDataForeignKeyExtension on CrdtDataForeignKey {
/// Whether an active projection override exists for this FK field.
///
/// Derived from [CrdtDataForeignKey.overrideReason]: an override is active if
/// and only if [overrideReason] is non-null. This is the authoritative test
/// for whether the domain row's FK value differs from the attempted value.
bool get hasOverride => overrideReason != null;
}
/// Extensions for the [CrdtDataDeletedReason] enum.
extension CrdtDataDeletedReasonExtension on CrdtDataDeletedReason {
/// Whether this tombstone reason is synced across replicas. All reasons that
/// start with 'user' are synced.
bool get isSynced => name.startsWith('user');
}
/// CRDT write capability for a scope role.
///
/// The implicit personal scope resolves to [CrdtScopeRole.readWrite]. A missing
/// role means no shared membership row was found and does not grant writes.
extension CrdtScopeRoleWriteAccess on CrdtScopeRole? {
/// Whether this role may write CRDT rows in the scope.
bool get canWrite => this == CrdtScopeRole.readWrite;
}

54
packages/serverpod_offline_sync_client/lib/src/crdt/integrity_violation.dart
View file

@ -1,54 +0,0 @@
import 'package:serverpod_database/serverpod_database.dart';
import '../protocol/protocol.dart';
/// Persists or updates the durable row for an integrity violation.
Future<CrdtSyncIntegrityViolation> recordCrdtSyncIntegrityViolation(
DatabaseSession session, {
required CrdtSyncIntegrityViolation violation,
Transaction? transaction,
}) async {
final existing = await CrdtSyncIntegrityViolation.db.findFirstRow(
session,
where: (t) =>
t.type.equals(violation.type) &
t.operation.equals(violation.operation) &
t.domainTableName.equals(violation.domainTableName) &
t.uuidRowId.equals(violation.uuidRowId) &
t.ownerScopeUuid.equals(violation.ownerScopeUuid) &
t.incomingScopeUuid.equals(violation.incomingScopeUuid),
transaction: transaction,
);
final now = DateTime.now().toUtc();
final newViolation = existing != null
? existing.copyWith(
crdtDataRowId: violation.crdtDataRowId ?? existing.crdtDataRowId,
uuidNodeId: violation.uuidNodeId ?? existing.uuidNodeId,
hlcDatetime: violation.hlcDatetime,
hlcCounter: violation.hlcCounter,
lastSeenAt: now,
occurrences: existing.occurrences + 1,
)
: violation.copyWith(
firstSeenAt: now,
lastSeenAt: now,
occurrences: 1,
);
final persisted = await CrdtSyncIntegrityViolation.db.upsertRow(
session,
newViolation,
conflictColumns: (t) => [
t.type,
t.operation,
t.domainTableName,
t.uuidRowId,
t.ownerScopeUuid,
t.incomingScopeUuid,
],
transaction: transaction,
);
return persisted!;
}

190
packages/serverpod_offline_sync_client/lib/src/crdt/merge.dart
View file

@ -1,190 +0,0 @@
import 'dart:async';
import 'package:serverpod_client/serverpod_client.dart';
import 'package:serverpod_database/serverpod_database.dart';
import '../protocol/protocol.dart';
import 'exceptions.dart';
/// A group of CRDT merge changes collected for one sync batch.
typedef CrdtMergeSet = List<CrdtMergeChange>;
/// Merge metadata extracted from a [CrdtMergeSet].
typedef CrdtMergeMetadataLookup = ({
Map<String, Set<UuidValue>> rowIdsByTable,
Map<String, Set<String>> columnNamesByTable,
});
/// Extensions for stream iterators of [CrdtSyncStreamEvent].
extension CrdtSyncStreamEventStreamExtension on StreamIterator<CrdtSyncStreamEvent> {
/// Collects the next framed sync batch from this iterator.
///
/// Each batch is zero or more scope, handshake, and/or merge frames followed
/// by [CrdtSyncEndOfBatch]. If the stream is idle before a batch starts, an
/// empty batch is returned.
///
/// When [allowCloseBeforeBatch] is true, returns `null` if the transport
/// stream closes before the next batch starts. [CrdtSyncClose] is an
/// intentional peer request to end the streaming session and returns `null`
/// even after frames for a partial batch have already arrived. The caller
/// must not persist checkpoint progress for such a discarded partial batch.
/// Otherwise, closing before a batch starts is treated as a
/// [CrdtSyncStreamClosedException].
///
/// If the transport stream closes after a frame was already received without
/// a [CrdtSyncClose] control frame, the partial batch is still treated as an
/// error.
///
/// An idle timeout only ends an *empty* batch: once any frame has been
/// collected the timeout is ignored and collection waits for the explicit
/// terminator.
Future<CrdtSyncCycleBatch?> collectNextBatch({
bool allowCloseBeforeBatch = false,
}) async {
final batch = CrdtSyncCycleBatch();
while (await moveNext()) {
switch (current) {
case final CrdtSyncScopeSet event:
batch.scopeSet = event;
case final CrdtSyncSinceHlc event:
batch.sinceHlcs[event.uuidScopeId] = event;
case CrdtSyncMergeChunk(:final changes):
batch.changes.addAll(changes);
case CrdtSyncIdleTimeout():
if (batch.isEmpty) return batch;
case CrdtSyncEndOfBatch():
return batch;
case CrdtSyncClose():
return null;
default:
throw CrdtSyncUnexpectedEventException(
expected: 'a sync cycle frame',
received: current,
);
}
}
if (batch.isEmpty && allowCloseBeforeBatch) return null;
throw const CrdtSyncStreamClosedException(phase: 'end-of-batch');
}
/// Moves the iterator to the next event and throws if the stream is closed or
/// the next event is not of type [T].
Future<T> moveAndThrowIfNot<T extends CrdtSyncStreamEvent>() async {
while (await moveNext()) {
if (current is CrdtSyncIdleTimeout) continue;
if (current is T) return current as T;
throw CrdtSyncUnexpectedEventException(
expected: '"$T"',
received: current,
);
}
throw CrdtSyncStreamClosedException(phase: '"$T"');
}
}
/// One sync cycle's inbound frames.
class CrdtSyncCycleBatch {
/// The peer's scope announcement for this cycle, if it sent one.
CrdtSyncScopeSet? scopeSet;
/// The peer's resume vectors, keyed by scope.
final Map<UuidValue, CrdtSyncSinceHlc> sinceHlcs = {};
/// The peer's merge changes for this cycle.
final List<CrdtMergeChange> changes = [];
/// Whether the peer sent nothing this cycle (it was idle).
bool get isEmpty => scopeSet == null && sinceHlcs.isEmpty && changes.isEmpty;
}
/// Helpers for grouping merge changes into stream payload batches.
extension CrdtMergeChangeStreamExtension on Stream<CrdtMergeChange> {
/// Emits lists with at most [batchSize] changes from this stream.
Stream<CrdtMergeSet> chunked(int batchSize) async* {
var batch = <CrdtMergeChange>[];
await for (final change in this) {
batch.add(change);
if (batch.length < batchSize) continue;
yield batch;
batch = <CrdtMergeChange>[];
}
if (batch.isNotEmpty) yield batch;
}
}
/// CRDT merge helpers for [CrdtMergeSet].
extension CrdtMergeSetExtension on CrdtMergeSet {
/// The greatest HLC represented by the changes in this merge set.
Hlc? get maxHlc => fold<Hlc?>(
null,
(current, change) => change.hlc.maxBetween(current),
);
/// Insert changes in this set.
Iterable<CrdtMergeInsert> get inserts => whereType<CrdtMergeInsert>();
/// Update changes in this set.
Iterable<CrdtMergeUpdate> get updates => whereType<CrdtMergeUpdate>();
/// Delete changes in this set.
Iterable<CrdtMergeDelete> get deletes => whereType<CrdtMergeDelete>();
/// All merge changes in this set sorted by causal order.
List<CrdtMergeChange> get causallyOrderedChanges {
final operations = List<CrdtMergeChange>.from(this)
..sort((left, right) => left.hlc.compareTo(right.hlc));
return operations;
}
/// Collects the row and column metadata needed to load merge state.
CrdtMergeMetadataLookup collectMetadataLookup({
required Map<String, Set<String>> columnNamesByTableName,
}) {
final rowIdsToLoadByTable = <String, Set<UuidValue>>{};
final columnNamesToLoadByTable = <String, Set<String>>{};
for (final change in this) {
final availableColumnNames = columnNamesByTableName[change.tableName];
if (availableColumnNames == null) continue;
rowIdsToLoadByTable.putIfAbsent(change.tableName, () => {}).add(change.uuidRowId);
columnNamesToLoadByTable.putIfAbsent(change.tableName, () => {}).addAll(
switch (change) {
CrdtMergeInsert() => availableColumnNames.difference({'id'}),
CrdtMergeUpdate() => [change.columnName],
_ => [],
},
);
}
return (
rowIdsByTable: rowIdsToLoadByTable,
columnNamesByTable: columnNamesToLoadByTable,
);
}
}
/// Convenience helpers shared by every merge change.
extension CrdtMergeChangeExtension on CrdtMergeChange {
/// The HLC represented by this change.
Hlc get hlc => Hlc(hlcDatetime, hlcCounter, uuidNodeId);
}
/// Convenience helpers for insert changes.
extension CrdtMergeInsertExtension on CrdtMergeInsert {
/// The database column payload represented by this change.
Map<String, Object?> get databaseColumns {
final payload = switch (data) {
final TableRow row => row.toJsonForDatabase(),
final Map<String, dynamic> map => map,
_ => throw StateError(
'Unsupported merge insert payload type for $tableName: '
'${data.runtimeType}. Expected TableRow or Map<String, dynamic>.',
),
};
return Map<String, Object?>.from(payload as Map<String, dynamic>);
}
}

204
packages/serverpod_offline_sync_client/lib/src/crdt/scope_membership.dart
View file

@ -1,204 +0,0 @@
import 'package:meta/meta.dart';
import 'package:serverpod_database/serverpod_database.dart' hide Protocol;
import 'package:uuid/uuid.dart';
import '../protocol/protocol.dart';
/// Resolves shared-scope membership from the `crdt_scope_members` table.
///
/// The table is `database: all`, so the same code runs on every node:
/// authoritatively on the server (the source of truth) and against the local
/// read-only cache on a follower. A user's personal scope is implicit — a user
/// always belongs to the scope whose UUID equals their own user UUID — and
/// needs no membership row. Explicit rows grant access to additional shared
/// scopes.
///
/// Queries are raw SQL rather than the generated ORM on purpose: `find<T>`
/// resolves the table through the session's serialization manager keyed by the
/// model [Type], and a server session only registers the server-package model
/// classes — never this client-package [CrdtScopeMember]. Raw SQL is
/// serialization-manager-agnostic and runs identically on both ends, matching
/// the rest of the CRDT engine's cross-node queries.
class CrdtScopeMembership {
const CrdtScopeMembership._();
/// Returns the scope UUIDs [userUuid] may sync.
///
/// Includes the implicit personal scope and every explicit shared membership,
/// de-duplicated and sorted by UUID string so both peers iterate
/// deterministically.
static Future<List<UuidValue>> memberScopes(
DatabaseSession session,
UuidValue userUuid, {
Transaction? transaction,
}) async {
final grants = await memberGrants(
session,
userUuid,
transaction: transaction,
);
return [for (final grant in grants) grant.uuidScopeId];
}
/// Returns the authoritative scope grants for [userUuid].
///
/// Like [memberScopes] but carries each scope's `role`. The implicit
/// personal scope is included with [CrdtScopeRole.readWrite]. Used to build the
/// authoritative [CrdtSyncScopeSet] announcement so a follower can project
/// roles into its local cache.
static Future<List<CrdtScopeGrant>> memberGrants(
DatabaseSession session,
UuidValue userUuid, {
Transaction? transaction,
}) async {
final encodedUserUuid = ValueEncoder.instance.convert(userUuid);
final result = await session.db.unsafeQuery(
'SELECT s."${CrdtScope.t.uuidScopeId.columnName}", '
'm."${CrdtScopeMember.t.role.columnName}" '
'FROM "${CrdtScopeMember.t.tableName}" m '
'JOIN "${CrdtScope.t.tableName}" s '
'ON s."${CrdtScope.t.id.columnName}" = m."${CrdtScopeMember.t.scopeId.columnName}" '
'WHERE m."${CrdtScopeMember.t.userUuid.columnName}" = $encodedUserUuid',
transaction: transaction,
);
final grantsByUuid = <String, CrdtScopeGrant>{};
for (final row in result) {
final scopeUuid = Protocol().deserialize<UuidValue>(row[0]);
grantsByUuid[scopeUuid.uuid] = CrdtScopeGrant(
uuidScopeId: scopeUuid,
role: _roleFromDatabase(row[1]),
);
}
// Assigned after the shared rows so the personal grant stays readWrite
// even if a stray membership row exists for it, matching [roleOf].
grantsByUuid[userUuid.uuid] = CrdtScopeGrant(
uuidScopeId: userUuid,
role: CrdtScopeRole.readWrite,
);
return grantsByUuid.values.toList()
..sort((a, b) => a.uuidScopeId.uuid.compareTo(b.uuidScopeId.uuid));
}
/// The role [userUuid] holds in [scopeUuid], or null if none is recorded.
///
/// The implicit personal scope resolves to [CrdtScopeRole.readWrite]. On the
/// server this reads authoritative membership; on a client it reads the
/// projected membership cache.
static Future<CrdtScopeRole?> roleOf(
DatabaseSession session, {
required UuidValue userUuid,
required UuidValue scopeUuid,
Transaction? transaction,
}) async {
if (userUuid == scopeUuid) return CrdtScopeRole.readWrite;
final scopeId = await _scopeIdForUuid(
session,
scopeUuid,
transaction: transaction,
);
if (scopeId == null) return null;
final encodedUserUuid = ValueEncoder.instance.convert(userUuid);
final encodedScopeId = ValueEncoder.instance.convert(scopeId);
final result = await session.db.unsafeQuery(
'SELECT "${CrdtScopeMember.t.role.columnName}" '
'FROM "${CrdtScopeMember.t.tableName}" '
'WHERE "${CrdtScopeMember.t.userUuid.columnName}" = $encodedUserUuid '
'AND "${CrdtScopeMember.t.scopeId.columnName}" = $encodedScopeId '
'LIMIT 1',
transaction: transaction,
);
return result.isEmpty ? null : _roleFromDatabase(result.first[0]);
}
/// Reconciles the local `crdt_scope_members` cache from an authoritative
/// [grants] announcement — a follower's read-only projection of its own
/// memberships.
///
/// Upserts a row per shared grant (the implicit personal scope is skipped)
/// and deletes rows for [userUuid] whose scope is no longer granted, so a
/// revoked or demoted membership does not linger offline. Scopes must already
/// be materialized locally; a grant whose scope is unknown is skipped.
@internal
static Future<void> projectFollowerMembership(
DatabaseSession session, {
required UuidValue userUuid,
required List<CrdtScopeGrant> grants,
}) async {
final encodedUserUuid = ValueEncoder.instance.convert(userUuid);
final keptScopeIds = <int>[];
for (final grant in grants) {
if (grant.uuidScopeId == userUuid) continue; // personal scope is implicit
final scopeId = await _scopeIdForUuid(session, grant.uuidScopeId);
if (scopeId == null) continue; // not materialized locally yet
keptScopeIds.add(scopeId);
final encodedScopeId = ValueEncoder.instance.convert(scopeId);
final encodedRole = ValueEncoder.instance.convert(grant.role.toJson());
await session.db.unsafeExecute(
'INSERT INTO "${CrdtScopeMember.t.tableName}" '
'("${CrdtScopeMember.t.scopeId.columnName}", '
'"${CrdtScopeMember.t.userUuid.columnName}", '
'"${CrdtScopeMember.t.role.columnName}") '
'VALUES ($encodedScopeId, $encodedUserUuid, $encodedRole) '
'ON CONFLICT ("${CrdtScopeMember.t.userUuid.columnName}", '
'"${CrdtScopeMember.t.scopeId.columnName}") '
'DO UPDATE SET "${CrdtScopeMember.t.role.columnName}" = $encodedRole',
);
}
final notInClause = keptScopeIds.isEmpty
? ''
: 'AND "${CrdtScopeMember.t.scopeId.columnName}" NOT IN '
'(${keptScopeIds.map(ValueEncoder.instance.convert).join(', ')})';
await session.db.unsafeExecute(
'DELETE FROM "${CrdtScopeMember.t.tableName}" '
'WHERE "${CrdtScopeMember.t.userUuid.columnName}" = $encodedUserUuid '
'$notInClause',
);
}
/// Returns whether [userUuid] may act in [scopeUuid].
///
/// The implicit personal scope is accepted without a membership row.
static Future<bool> isMember(
DatabaseSession session, {
required UuidValue userUuid,
required UuidValue scopeUuid,
Transaction? transaction,
}) async {
final role = await roleOf(
session,
userUuid: userUuid,
scopeUuid: scopeUuid,
transaction: transaction,
);
return role != null;
}
}
CrdtScopeRole _roleFromDatabase(Object? value) {
if (value == null) {
throw StateError('crdt_scope_members.role must not be null.');
}
return CrdtScopeRole.fromJson(value as String);
}
Future<int?> _scopeIdForUuid(
DatabaseSession session,
UuidValue scopeUuid, {
Transaction? transaction,
}) async {
final encodedScopeUuid = ValueEncoder.instance.convert(scopeUuid);
final result = await session.db.unsafeQuery(
'SELECT "${CrdtScope.t.id.columnName}" FROM "${CrdtScope.t.tableName}" '
'WHERE "${CrdtScope.t.uuidScopeId.columnName}" = $encodedScopeUuid '
'LIMIT 1',
transaction: transaction,
);
return result.isEmpty ? null : result.first[0] as int;
}

272
packages/serverpod_offline_sync_client/lib/src/crdt/scope_sync.dart
View file

@ -1,272 +0,0 @@
import 'package:serverpod_database/serverpod_database.dart';
import 'package:uuid/uuid.dart';
import '../managers/scope.dart';
import '../protocol/protocol.dart';
import 'extensions.dart';
import 'merge.dart';
import 'scope_membership.dart';
/// How a peer decides which scopes it syncs.
enum CrdtSyncPeerMode {
/// Dictates the scope set from authoritative membership.
authoritative,
/// Adopts the scope set announced by the authoritative peer.
follower,
}
/// De-duplicates [scopeIds] and sorts them by UUID string so both peers iterate
/// scopes in the same deterministic order.
List<UuidValue> _sortedUniqueScopeIds(Iterable<UuidValue> scopeIds) {
final byUuid = {for (final scopeId in scopeIds) scopeId.uuid: scopeId};
return byUuid.values.toList()..sort((a, b) => a.uuid.compareTo(b.uuid));
}
/// De-duplicates [grants] by scope and sorts them by scope UUID string.
List<CrdtScopeGrant> _sortedUniqueGrants(Iterable<CrdtScopeGrant> grants) {
final byUuid = {for (final grant in grants) grant.uuidScopeId.uuid: grant};
return byUuid.values.toList()
..sort((a, b) => a.uuidScopeId.uuid.compareTo(b.uuidScopeId.uuid));
}
/// Whether the grant list [a] last announced equals the current list [b].
bool _grantsEqual(List<CrdtScopeGrant>? a, List<CrdtScopeGrant> b) {
if (a == null || a.length != b.length) return false;
for (var i = 0; i < b.length; i++) {
if (a[i].uuidScopeId != b[i].uuidScopeId || a[i].role != b[i].role) {
return false;
}
}
return true;
}
/// Owns the scope state of one sync session and every scope decision its driver
/// (`CrdtSync.sync`) makes: which scopes are active, their handshake and
/// checkpoint state, membership reconciliation, and per-frame authorization.
///
/// The driver owns the wire (yielding and reading frames, the once/continuous
/// control flow); this owns the scopes. It needs only a [DatabaseSession] and
/// the membership helpers, so it carries no protocol or transport concerns.
class CrdtScopeSyncSession {
/// Creates a scope session for [userId] acting in [mode] against [_session].
///
/// [peerNodeId] is the remote peer's CRDT node, learned from its connect
/// handshake before this session is constructed.
CrdtScopeSyncSession(
this._session, {
required UuidValue userId,
required CrdtSyncPeerMode mode,
required UuidValue peerNodeId,
}) : _userId = userId,
_mode = mode,
_peerNodeId = peerNodeId;
final DatabaseSession _session;
final UuidValue _userId;
final CrdtSyncPeerMode _mode;
final UuidValue _peerNodeId;
/// Session-lived scope manager. Reused across cycles so its in-memory cache
/// of already-materialized scopes survives, sparing a follower a per-cycle
/// `getOrCreate` round-trip for scopes it has already created this session.
late final CrdtScopeManager _scopeManager = CrdtScopeManager(_session);
/// The grant set this peer last announced (null until the first announcement).
List<CrdtScopeGrant>? _announcedGrants;
/// This peer's current local grants, refreshed by [reconcile].
List<CrdtScopeGrant> _localGrants = const [];
/// The grant set the peer last announced, adopted by [adoptPeerGrants].
List<CrdtScopeGrant> _peerGrants = const [];
/// The scopes cycled this round, sorted, materialized for a follower.
List<UuidValue> _activeScopeIds = const [];
/// UUID strings of [_activeScopeIds] and of [_peerGrants], for O(1) lookups.
Set<String> _activeUuids = const {};
Set<String> _peerUuids = const {};
/// Active scopes this peer may send local writes for.
Set<String> _writableUuids = const {};
/// Scopes for which this peer has already sent its [CrdtSyncSinceHlc].
final Set<String> _sinceHlcSent = {};
/// The peer's node id from its connect handshake.
UuidValue get peerNodeId => _peerNodeId;
/// What the peer has seen per scope (advances as we send), seeded from its
/// [CrdtSyncSinceHlc] — presence means the scope's handshake completed both
/// ways, so it can stream changes.
final Map<UuidValue, Map<UuidValue, Hlc>> _checkpointsByScope = {};
/// The grants this peer should announce this cycle.
List<CrdtScopeGrant> get localGrants => _localGrants;
/// The scopes cycled this round (sorted, deterministic on both peers).
List<UuidValue> get activeScopeIds => _activeScopeIds;
/// The authenticated user this sync session represents.
UuidValue get userId => _userId;
/// Whether this peer is authoritative for scope membership.
bool get isAuthoritative => _mode == CrdtSyncPeerMode.authoritative;
/// Whether the local grants changed since the last announcement.
bool get shouldAnnounce => !_grantsEqual(_announcedGrants, _localGrants);
/// Records that [localGrants] was just announced, so it is not re-sent until
/// it changes again.
void markAnnounced() => _announcedGrants = _localGrants;
/// Re-resolves local membership, then recomputes the active scope set.
///
/// Used at the top of every data-loop cycle so an authoritative peer picks up
/// its own membership changes. Establishment instead recomputes from the
/// *announced* grants (via [adoptPeerGrants]) without re-reading, so a grant
/// that lands mid-handshake cannot make this peer handshake a different scope
/// count than it announced.
Future<void> reconcile() async {
_localGrants = _sortedUniqueGrants(await _resolveLocalGrants());
await _recomputeActiveScopes();
_refreshWritableScopes();
}
/// Adopts the peer's announced [peerGrants] and recomputes the active scope
/// set from the grants already resolved (no membership re-read). A follower
/// materializes the announced scopes locally and projects their roles into the
/// members cache (projection needs the scope rows to exist); an authoritative
/// peer keeps cycling its own membership.
Future<void> adoptPeerGrants(List<CrdtScopeGrant> peerGrants) async {
_peerGrants = _sortedUniqueGrants(peerGrants);
_peerUuids = {for (final g in _peerGrants) g.uuidScopeId.uuid};
await _recomputeActiveScopes();
if (_mode == CrdtSyncPeerMode.follower) {
await CrdtScopeMembership.projectFollowerMembership(
_session,
userUuid: _userId,
grants: _peerGrants,
);
}
_refreshWritableScopes();
}
/// Recomputes the active scope set from the current local and peer grants
/// (materializing a follower's adopted scopes), then prunes in-session state
/// for scopes that left the set — e.g. a membership the peer revoked.
Future<void> _recomputeActiveScopes() async {
_activeScopeIds = _sortedUniqueScopeIds(
await _resolveOrderedScopeIds(
localScopeIds: [for (final g in _localGrants) g.uuidScopeId],
peerScopeIds: [for (final g in _peerGrants) g.uuidScopeId],
),
);
_activeUuids = {for (final s in _activeScopeIds) s.uuid};
_sinceHlcSent.removeWhere((uuid) => !_activeUuids.contains(uuid));
_checkpointsByScope.removeWhere((s, _) => !_activeUuids.contains(s.uuid));
}
void _refreshWritableScopes() {
if (_mode == CrdtSyncPeerMode.authoritative) {
_writableUuids = _activeUuids;
return;
}
// The peer's announced grants already carry the authoritative roles, so a
// follower derives writability in memory instead of re-reading the cache
// it projected from these same grants. The personal scope is always
// writable; consumers only consult scopes in the active set.
_writableUuids = {
_userId.uuid,
for (final grant in _peerGrants)
if (grant.role.canWrite) grant.uuidScopeId.uuid,
};
}
/// Marks that this peer is sending its [CrdtSyncSinceHlc] for [scopeId],
/// returning whether the scope was newly handshaked (so the driver sends one).
bool markHandshakeSent(UuidValue scopeId) => _sinceHlcSent.add(scopeId.uuid);
/// Records the peer's resume vector for [scopeId] from its [sinceHlc],
/// completing the scope's handshake from this peer's side.
void recordPeerHandshake(UuidValue scopeId, CrdtSyncSinceHlc sinceHlc) {
_checkpointsByScope[scopeId] = {
for (final checkpoint in sinceHlc.nodeCheckpoints) checkpoint.nodeId: checkpoint,
};
}
/// Checkpoints for writable scopes whose handshake completed both ways, keyed
/// by scope — the input to a pending-change collection pass.
Map<UuidValue, List<Hlc>> get sendableCheckpoints => {
for (final scopeId in _activeScopeIds)
if (_checkpointsByScope[scopeId] != null && _writableUuids.contains(scopeId.uuid))
scopeId: _checkpointsByScope[scopeId]!.values.toList(),
};
/// Whether any active scope still lacks the peer handshake state.
bool get hasIncompleteActiveHandshake => _activeScopeIds.any(
(scopeId) => _checkpointsByScope[scopeId] == null,
);
/// Advances the in-session checkpoint for [scopeId] past a just-sent [change],
/// so the next collection does not resend it.
void advanceCheckpoint(UuidValue scopeId, CrdtMergeChange change) {
final checkpoints = _checkpointsByScope[scopeId];
if (checkpoints == null) return;
checkpoints[change.uuidNodeId] = change.hlc.maxBetween(
checkpoints[change.uuidNodeId],
);
}
/// Whether this peer authorizes acting in [scopeId]: an authoritative peer
/// trusts its own membership, a follower trusts the peer's announced set. This
/// is the wire-side counterpart of the server's membership gate.
bool accepts(UuidValue scopeId) =>
(_mode == CrdtSyncPeerMode.follower ? _peerUuids : _activeUuids).contains(
scopeId.uuid,
);
/// The greatest checkpoint HLC tracked for [scopeId], or null if none.
Hlc? checkpointMaxOf(UuidValue scopeId) {
final checkpoints = _checkpointsByScope[scopeId];
if (checkpoints == null || checkpoints.isEmpty) return null;
return checkpoints.values.max;
}
/// Resolves the local grants this peer announces in its [CrdtSyncScopeSet].
///
/// Authoritative peers enumerate [CrdtScopeMembership.memberGrants].
/// Followers send an empty set because the authoritative peer never widens
/// access from follower-reported scope state.
Future<List<CrdtScopeGrant>> _resolveLocalGrants() async {
switch (_mode) {
case CrdtSyncPeerMode.authoritative:
return CrdtScopeMembership.memberGrants(_session, _userId);
case CrdtSyncPeerMode.follower:
await _scopeManager.getOrCreate(_userId);
return const [];
}
}
/// Resolves the ordered lockstep scopes cycled this round.
///
/// Authoritative peers dictate their own set; followers adopt the peer's
/// announced set, materializing each scope locally.
Future<List<UuidValue>> _resolveOrderedScopeIds({
required List<UuidValue> localScopeIds,
required List<UuidValue> peerScopeIds,
}) async {
switch (_mode) {
case CrdtSyncPeerMode.authoritative:
return localScopeIds;
case CrdtSyncPeerMode.follower:
final scopeIds = _sortedUniqueScopeIds(peerScopeIds);
for (final scopeId in scopeIds) {
await _scopeManager.getOrCreate(scopeId);
}
return scopeIds;
}
}
}

1159
packages/serverpod_offline_sync_client/lib/src/crdt/sync.dart
View file

File diff suppressed because it is too large Load diff

1045
packages/serverpod_offline_sync_client/lib/src/database/database.dart
View file

File diff suppressed because it is too large Load diff

976
packages/serverpod_offline_sync_client/lib/src/database/merge.dart
View file

@ -1,976 +0,0 @@
part of 'recorder.dart';
typedef _MergeRowKey = (String, UuidValue);
typedef _MergeFieldKey = (String, UuidValue, String);
typedef _DomainOwner = ({bool exists, int? scopeId});
typedef _MergeContext = ({
Map<_MergeRowKey, CrdtDataRow> rows,
Map<_MergeFieldKey, CrdtDataField> fields,
Map<_MergeFieldKey, Hlc> incomingFieldHlcs,
Map<_MergeRowKey, CrdtDataDeleted> tombstones,
Map<_MergeRowKey, _DomainOwner> domainOwners,
});
typedef _MergeNodes = ({
Map<UuidValue, CrdtNode> nodesByUuid,
Map<UuidValue, CrdtScopeNode> scopeNodesByUuid,
});
/// Adds merge-specific behavior to [CrdtMutationRecorder].
extension CrdtMergeRecorderExtension on CrdtMutationRecorder {
CrdtUniqueConflictResolver get _uniqueConflictResolver =>
CrdtUniqueConflictResolver(this);
/// Locks the current user row so merges can serialize with other work.
Future<void> lockCurrentUser(Transaction transaction) async {
final user = _getEffectiveScope(transaction);
// Use a row lock without fetching the record since the merge path only
// needs serialization against concurrent work for the same user.
await CrdtScope.db.lockRows(
_session,
where: (t) => t.id.equals(user.id),
transaction: transaction,
lockMode: LockMode.forUpdate,
lockBehavior: LockBehavior.wait,
);
}
/// Merges remote CRDT changes into the current database.
Future<void> mergeChanges(
CrdtMergeSet mergeSet,
Transaction transaction,
) async {
if (mergeSet.isEmpty) return;
final operations = mergeSet.causallyOrderedChanges;
final shouldProjectForeignKeys = _mergeOperationsMayAffectForeignKeys(
operations,
);
final currentUser = _getEffectiveScope(transaction);
final remoteNodes = await _findOrCreateNodesForMerge(
currentUser.id!,
{for (final change in operations) change.uuidNodeId},
transaction,
);
final nodesByUuid = remoteNodes.nodesByUuid;
final context = await _loadMergeContext(mergeSet, transaction);
for (final operation in operations) {
if (!_isCrdtTrackedTableName(operation.tableName)) {
continue;
}
switch (operation) {
case final CrdtMergeInsert insert:
await _applyMergeInsert(
insert,
nodesByUuid,
context,
transaction,
);
case final CrdtMergeUpdate update:
await _applyMergeUpdate(
update,
nodesByUuid,
context,
transaction,
);
case final CrdtMergeDelete delete:
await _applyMergeDelete(
delete,
nodesByUuid,
context,
transaction,
);
}
}
if (shouldProjectForeignKeys) {
await _projectForeignKeys(transaction);
}
await _updateHlcFromIncomingOperations(
operations,
remoteNodes,
transaction,
);
}
Future<void> _updateHlcFromIncomingOperations(
List<CrdtMergeChange> operations,
_MergeNodes remoteNodes,
Transaction transaction,
) async {
final maxIncomingHlc = operations.fold<Hlc?>(
null,
(current, change) => change.hlc.maxBetween(current),
);
final scopeNodesToUpdate = <CrdtScopeNode>[];
if (maxIncomingHlc != null) {
final hlcManager = _getHlcManager(transaction);
if (maxIncomingHlc.nodeId == hlcManager.uuidNodeId) {
if (maxIncomingHlc > hlcManager.lastHlc) {
hlcManager.lastHlc = maxIncomingHlc;
}
} else {
hlcManager.merge(maxIncomingHlc);
}
await _persistCurrentNodeHlc(hlcManager, transaction);
}
final maxIncomingHlcByNode = <UuidValue, Hlc>{};
for (final operation in operations) {
maxIncomingHlcByNode[operation.uuidNodeId] = operation.hlc.maxBetween(
maxIncomingHlcByNode[operation.uuidNodeId],
);
}
for (final MapEntry(key: nodeId, value: incomingHlc)
in maxIncomingHlcByNode.entries) {
final remoteNode = remoteNodes.scopeNodesByUuid[nodeId];
if (remoteNode == null) continue;
final updatedScopeNode = remoteNode.copyWith(
lastReceivedHlc: incomingHlc.maxBetween(remoteNode.lastReceivedHlc),
);
if (updatedScopeNode.lastReceivedHlc == remoteNode.lastReceivedHlc) {
continue;
}
scopeNodesToUpdate.add(updatedScopeNode);
remoteNodes.scopeNodesByUuid[nodeId] = updatedScopeNode;
}
if (scopeNodesToUpdate.isNotEmpty) {
await CrdtScopeNode.db.update(
_session,
scopeNodesToUpdate,
columns: (t) => [t.lastReceivedHlc],
transaction: transaction,
);
}
}
Future<_MergeNodes> _findOrCreateNodesForMerge(
int userId,
Set<UuidValue> nodeIds,
Transaction transaction,
) async {
if (nodeIds.isEmpty) {
return (
nodesByUuid: const <UuidValue, CrdtNode>{},
scopeNodesByUuid: const <UuidValue, CrdtScopeNode>{},
);
}
final nodesByUuid = {
for (final uuidNodeId in nodeIds)
uuidNodeId: await _findOrCreateNode(uuidNodeId, transaction),
};
final scopeNodesByUuid = {
for (final entry in nodesByUuid.entries)
entry.key: await _findOrCreateScopeNode(
userId,
entry.value.id!,
transaction,
),
};
return (
nodesByUuid: nodesByUuid,
scopeNodesByUuid: scopeNodesByUuid,
);
}
Future<_MergeContext> _loadMergeContext(
CrdtMergeSet mergeSet,
Transaction transaction,
) async {
final (:rowIdsByTable, :columnNamesByTable) = mergeSet.collectMetadataLookup(
columnNamesByTableName: _syncedTableColumnNamesForMerge,
);
final rows = <_MergeRowKey, CrdtDataRow>{};
final fields = <_MergeFieldKey, CrdtDataField>{};
final incomingFieldHlcs = {
for (final update in mergeSet.whereType<CrdtMergeUpdate>())
(update.tableName, update.uuidRowId, update.columnName): update.hlc,
};
final tombstones = <_MergeRowKey, CrdtDataDeleted>{};
// Owning scope per row, snapshotted once so the per-operation ownership
// checks never re-read the same immutable `scopeId`. Only rows that
// already have a tracker are pre-read here — those are exactly the rows
// whose ownership the update/delete/existing-insert paths verify. New
// rows go through the insert path, which proves ownership at the physical
// write and refreshes this entry afterwards.
final domainOwners = <_MergeRowKey, _DomainOwner>{};
for (final MapEntry(key: tableName, value: rowIds) in rowIdsByTable.entries) {
final loadedRows = await _findCrdtRows(
tableName,
rowIds,
transaction,
include: CrdtDataRow.include(
node: CrdtNode.include(),
deleted: CrdtDataDeleted.include(node: CrdtNode.include()),
),
);
for (final row in loadedRows) {
rows[(tableName, row.uuidRowId)] = row;
if (row.deleted != null) {
tombstones[(tableName, row.uuidRowId)] = row.deleted!;
}
}
final trackedRowIds = loadedRows.map((row) => row.uuidRowId).toSet();
if (trackedRowIds.isNotEmpty) {
final ownerValues = await _readDomainColumnValues(
tableName,
trackedRowIds,
['scopeId'],
transaction,
);
for (final rowId in trackedRowIds) {
final values = ownerValues[rowId];
domainOwners[(tableName, rowId)] = values == null
? (exists: false, scopeId: null)
: (exists: true, scopeId: values['scopeId'] as int?);
}
}
final columnNames = columnNamesByTable[tableName];
if (columnNames == null || columnNames.isEmpty || loadedRows.isEmpty) continue;
final rowPks = loadedRows.map((row) => row.id).whereType<int>().toSet();
if (rowPks.length != loadedRows.length) {
throw StateError(
'Some merge metadata rows for table $tableName are missing '
'persisted identifiers.',
);
}
final columnIdsByName = _schemaColumnIds(_schema, tableName, columnNames);
if (columnIdsByName.isEmpty) continue;
final loadedFields = await CrdtDataField.db.find(
_session,
where: (t) =>
t.rowId.inSet(rowPks) & t.columnId.inSet(columnIdsByName.values.toSet()),
include: CrdtDataField.include(
column: CrdtSchemaColumn.include(),
node: CrdtNode.include(),
),
transaction: transaction,
);
for (final field in loadedFields) {
final row = loadedRows.firstWhere((loadedRow) => loadedRow.id == field.rowId);
fields[(tableName, row.uuidRowId, field.column!.name)] = field;
}
}
return (
rows: rows,
fields: fields,
incomingFieldHlcs: incomingFieldHlcs,
tombstones: tombstones,
domainOwners: domainOwners,
);
}
Future<void> _applyMergeInsert(
CrdtMergeInsert insert,
Map<UuidValue, CrdtNode> remoteNodes,
_MergeContext context,
Transaction transaction,
) async {
final rowKey = (insert.tableName, insert.uuidRowId);
final incomingHlc = insert.hlc;
final remoteNode = _requireRemoteNode(remoteNodes, insert.uuidNodeId);
final currentRow = context.rows[rowKey];
if (currentRow != null && incomingHlc <= currentRow.hlc) {
return;
}
final data = _sanitizeMergeRowData(insert.tableName, insert.databaseColumns);
if (currentRow == null) {
final mergingScope = _getEffectiveScope(transaction);
final mergingScopeId = mergingScope.id!;
try {
// Insert-first inside a savepoint: the domain primary key resolves
// creation races atomically, and `_applyMergeInsertForMissingRow`
// raises `_ForeignScopeRowCollision` when the row turns out to be owned
// by another scope. Rolling the savepoint back then leaves no CRDT
// metadata, released unique values, or foreign key attempts behind.
context.rows[rowKey] = await DatabaseUtil.runInTransactionOrSavepoint(
_db,
transaction,
(savepoint) async {
final resolvedInsert = await _uniqueConflictResolver
._resolveForIncomingInsert(insert, data, context, savepoint);
final safeInsert = await _safeIncomingForeignKeyData(
insert.tableName,
insert.uuidRowId,
resolvedInsert.data,
savepoint,
);
final insertedRow = await _applyMergeInsertForMissingRow(
insert,
remoteNode,
incomingHlc,
safeInsert.data,
savepoint,
);
await _recordForeignKeyInsertAttempts(
insert.tableName,
{insert.uuidRowId},
savepoint,
safeInsert.attempts,
);
return insertedRow;
},
);
// The row is now owned by the merging scope (freshly inserted, or a
// same-scope recovery), so same-merge field updates resolve ownership
// from the cache instead of re-reading the domain row.
context.domainOwners[rowKey] = (exists: true, scopeId: mergingScopeId);
} on _ForeignScopeRowCollision catch (collision) {
await _throwOwnershipCollision(
operation: CrdtSyncViolationOperation.mergeInsert,
tableName: insert.tableName,
rowId: insert.uuidRowId,
owningScopeId: collision.owningScopeId,
incomingScope: mergingScope,
metadataRowId: null,
node: remoteNode,
hlc: insert.hlc,
transaction: transaction,
);
}
} else {
await _applyMergeInsertForExistingRow(
insert,
currentRow,
remoteNode,
incomingHlc,
data,
context,
transaction,
);
}
}
Future<void> _applyMergeUpdate(
CrdtMergeUpdate update,
Map<UuidValue, CrdtNode> remoteNodes,
_MergeContext context,
Transaction transaction,
) async {
final rowKey = (update.tableName, update.uuidRowId);
final row = context.rows[rowKey];
if (row == null) return;
final remoteNode = _requireRemoteNode(remoteNodes, update.uuidNodeId);
if (!await _domainRowOwnedByEffectiveScope(
operation: CrdtSyncViolationOperation.mergeUpdate,
tableName: update.tableName,
rowId: update.uuidRowId,
context: context,
transaction: transaction,
metadataRowId: row.id,
node: remoteNode,
hlc: update.hlc,
)) {
return;
}
final shouldApply = await _shouldMergeFieldMetadataIfNewer(
tableName: update.tableName,
rowId: update.uuidRowId,
columnName: update.columnName,
row: row,
remoteNode: remoteNode,
incomingHlc: update.hlc,
fields: context.fields,
transaction: transaction,
);
if (!shouldApply) return;
final resolvedUpdates = await _uniqueConflictResolver._resolveForIncomingUpdate(
update,
row,
context,
transaction,
);
final safeUpdate = await _safeIncomingForeignKeyData(
update.tableName,
update.uuidRowId,
resolvedUpdates,
transaction,
);
await _updateDomainRows(
update.tableName,
{update.uuidRowId},
safeUpdate.data,
transaction,
);
await _recordForeignKeyAttemptsForRows(
update.tableName,
{update.uuidRowId},
resolvedUpdates.keys.toSet(),
transaction,
attemptedValues: safeUpdate.attempts,
);
}
Future<void> _applyMergeDelete(
CrdtMergeDelete delete,
Map<UuidValue, CrdtNode> remoteNodes,
_MergeContext context,
Transaction transaction,
) async {
final rowKey = (delete.tableName, delete.uuidRowId);
final row = context.rows[rowKey];
if (row == null) return;
if (!delete.reason.isSynced) return;
final remoteNode = _requireRemoteNode(remoteNodes, delete.uuidNodeId);
if (!await _domainRowOwnedByEffectiveScope(
operation: CrdtSyncViolationOperation.mergeDelete,
tableName: delete.tableName,
rowId: delete.uuidRowId,
context: context,
transaction: transaction,
metadataRowId: row.id,
node: remoteNode,
hlc: delete.hlc,
)) {
return;
}
final currentTombstone = context.tombstones[rowKey];
final currentClFlag = currentTombstone?.clFlag ?? 1;
final currentHlc = currentTombstone?.hlc ?? row.hlc;
if (delete.clFlag < currentClFlag) return;
if (delete.clFlag == currentClFlag && delete.hlc <= currentHlc) return;
context.tombstones[rowKey] = await _upsertMergeTombstone(
row,
remoteNode,
delete.hlc,
delete.clFlag,
delete.reason,
currentTombstone,
transaction,
);
if (delete.reason.isSynced) {
await _applyRowVisibilityFromUserTombstone(
row,
context.tombstones[rowKey]!,
transaction,
);
}
}
Future<CrdtDataRow> _upsertMergeRow(
String tableName,
UuidValue rowId,
CrdtNode remoteNode,
Hlc incomingHlc,
CrdtDataRow? currentRow,
Transaction transaction,
) async {
if (currentRow == null) {
final (tableId, _) = _schema[tableName]!;
final insertedRow = await CrdtDataRow.db.insertRow(
_session,
CrdtDataRow(
scopeId: _getHlcManager(transaction).normalizedScopeId,
tblId: tableId,
uuidRowId: rowId,
nodeId: remoteNode.id!,
hlcDatetime: incomingHlc.datetime,
hlcCounter: incomingHlc.counter,
),
transaction: transaction,
);
return insertedRow.copyWith(node: remoteNode);
}
final updatedRow = currentRow.copyWith(
nodeId: remoteNode.id,
node: remoteNode,
hlcDatetime: incomingHlc.datetime,
hlcCounter: incomingHlc.counter,
);
await CrdtDataRow.db.update(
_session,
[updatedRow],
columns: (t) => [t.nodeId, t.hlcDatetime, t.hlcCounter],
transaction: transaction,
);
return updatedRow;
}
Future<CrdtDataField> _upsertMergeField(
CrdtDataRow row,
CrdtSchemaColumn column,
CrdtNode remoteNode,
Hlc incomingHlc,
CrdtDataField? currentField,
Transaction transaction,
) async {
if (currentField == null) {
final insertedField = await CrdtDataField.db.insertRow(
_session,
CrdtDataField(
rowId: row.id!,
columnId: column.id!,
nodeId: remoteNode.id!,
hlcDatetime: incomingHlc.datetime,
hlcCounter: incomingHlc.counter,
),
transaction: transaction,
);
return insertedField.copyWith(
row: row,
column: column,
node: remoteNode,
);
}
final updatedField = currentField.copyWith(
row: row,
column: column,
nodeId: remoteNode.id,
node: remoteNode,
hlcDatetime: incomingHlc.datetime,
hlcCounter: incomingHlc.counter,
);
await CrdtDataField.db.update(
_session,
[updatedField],
columns: (t) => [t.nodeId, t.hlcDatetime, t.hlcCounter],
transaction: transaction,
);
return updatedField;
}
Future<CrdtDataDeleted> _upsertMergeTombstone(
CrdtDataRow row,
CrdtNode remoteNode,
Hlc incomingHlc,
int clFlag,
CrdtDataDeletedReason reason,
CrdtDataDeleted? currentTombstone,
Transaction transaction,
) async {
if (currentTombstone == null) {
final insertedTombstone = await CrdtDataDeleted.db.insertRow(
_session,
CrdtDataDeleted(
rowId: row.id!,
nodeId: remoteNode.id!,
hlcDatetime: incomingHlc.datetime,
hlcCounter: incomingHlc.counter,
clFlag: clFlag,
reason: reason,
),
transaction: transaction,
);
return insertedTombstone.copyWith(
row: row,
node: remoteNode,
);
}
final updatedTombstone = currentTombstone.copyWith(
row: row,
nodeId: remoteNode.id,
node: remoteNode,
hlcDatetime: incomingHlc.datetime,
hlcCounter: incomingHlc.counter,
clFlag: clFlag,
reason: reason,
);
await CrdtDataDeleted.db.update(
_session,
[updatedTombstone],
columns: (t) => [t.nodeId, t.hlcDatetime, t.hlcCounter, t.clFlag, t.reason],
transaction: transaction,
);
return updatedTombstone;
}
Future<CrdtDataRow> _applyMergeInsertForMissingRow(
CrdtMergeInsert insert,
CrdtNode remoteNode,
Hlc incomingHlc,
Map<String, Object?> data,
Transaction transaction,
) async {
final insertedCrdtRow = await _upsertMergeRow(
insert.tableName,
insert.uuidRowId,
remoteNode,
incomingHlc,
null,
transaction,
);
final tableRow = _requireMergeInsertTableRow(insert);
final patchedRow = _db.serializationManager.patchTableRow(tableRow, data);
final mergingScopeId = _getHlcManager(transaction).normalizedScopeId;
final scopedRow = _withScopeId(patchedRow, mergingScopeId);
try {
// Insert first: the domain primary key resolves creation races
// atomically. The inner savepoint keeps a violation from poisoning the
// surrounding transaction so the recovery update below can still run.
await DatabaseUtil.runInTransactionOrSavepoint(
_db,
transaction,
(savepoint) => _db.insertRow(scopedRow, transaction: savepoint),
);
} on DatabaseQueryException {
final owner = await _readDomainRowOwner(
insert.tableName,
insert.uuidRowId,
transaction,
);
// No row means the violation came from something other than the
// primary key (e.g. a racing unique value); preserve the failure.
if (!owner.exists) rethrow;
if (owner.scopeId != mergingScopeId) {
throw _ForeignScopeRowCollision(owner.scopeId);
}
// Same-scope recovery (a tracked row whose metadata was lost, or a
// race between this scope's own merges). scopeId is immutable and rows
// are only soft-deleted, so ownership observed here cannot change for
// the rest of the transaction and the update is safe.
await _db.updateRow(
scopedRow,
columns: scopedRow.table.managedColumns
.where((c) => data.containsKey(c.columnName))
.toList(),
transaction: transaction,
);
}
return insertedCrdtRow;
}
Future<void> _applyMergeInsertForExistingRow(
CrdtMergeInsert insert,
CrdtDataRow currentRow,
CrdtNode remoteNode,
Hlc incomingHlc,
Map<String, Object?> data,
_MergeContext context,
Transaction transaction,
) async {
if (!await _domainRowOwnedByEffectiveScope(
operation: CrdtSyncViolationOperation.mergeInsert,
tableName: insert.tableName,
rowId: insert.uuidRowId,
context: context,
transaction: transaction,
metadataRowId: currentRow.id,
node: remoteNode,
hlc: insert.hlc,
)) {
return;
}
final (_, columnsByName) = _schema[insert.tableName]!;
final updatedValues = <String, Object?>{};
for (final MapEntry(key: columnName, value: schemaColumn)
in columnsByName.entries) {
if (columnName == 'id') continue;
final shouldApply = await _shouldMergeFieldMetadataIfNewer(
tableName: insert.tableName,
rowId: insert.uuidRowId,
columnName: columnName,
row: currentRow,
remoteNode: remoteNode,
incomingHlc: incomingHlc,
fields: context.fields,
transaction: transaction,
schemaColumn: schemaColumn,
);
if (!shouldApply) continue;
updatedValues[columnName] = data[columnName];
}
var updatesToApply = updatedValues;
if (updatesToApply.isNotEmpty) {
updatesToApply = await _uniqueConflictResolver._resolveForIncomingUpdates(
tableName: insert.tableName,
row: currentRow,
updates: updatesToApply,
context: context,
transaction: transaction,
);
}
if (updatesToApply.isNotEmpty) {
final safeUpdate = await _safeIncomingForeignKeyData(
insert.tableName,
insert.uuidRowId,
updatesToApply,
transaction,
);
await _updateDomainRows(
insert.tableName,
{insert.uuidRowId},
safeUpdate.data,
transaction,
);
await _recordForeignKeyAttemptsForRows(
insert.tableName,
{insert.uuidRowId},
updatesToApply.keys.toSet(),
transaction,
attemptedValues: safeUpdate.attempts,
);
}
final rowKey = (insert.tableName, insert.uuidRowId);
final currentTombstone = context.tombstones[rowKey];
if ((currentTombstone?.clFlag ?? 1) == 1 &&
incomingHlc > (currentTombstone?.hlc ?? currentRow.hlc)) {
context.tombstones[rowKey] = await _upsertMergeTombstone(
currentRow,
remoteNode,
incomingHlc,
1,
CrdtDataDeletedReason.userInsert,
currentTombstone,
transaction,
);
await _applyRowVisibilityFromUserTombstone(
currentRow,
context.tombstones[rowKey]!,
transaction,
);
}
}
Future<bool> _shouldMergeFieldMetadataIfNewer({
required String tableName,
required UuidValue rowId,
required String columnName,
required CrdtDataRow row,
required CrdtNode remoteNode,
required Hlc incomingHlc,
required Map<_MergeFieldKey, CrdtDataField> fields,
required Transaction transaction,
CrdtSchemaColumn? schemaColumn,
}) async {
final resolvedSchemaColumn =
schemaColumn ?? _schemaColumn(_schema, tableName, columnName);
if (resolvedSchemaColumn == null) return false;
final fieldKey = (tableName, rowId, columnName);
final currentField = fields[fieldKey];
final currentHlc = currentField?.hlc ?? row.hlc;
if (incomingHlc <= currentHlc) return false;
fields[fieldKey] = await _upsertMergeField(
row,
resolvedSchemaColumn,
remoteNode,
incomingHlc,
currentField,
transaction,
);
return true;
}
TableRow _requireMergeInsertTableRow(CrdtMergeInsert insert) {
final row = insert.data;
if (row is TableRow) return row;
throw StateError(
'Unsupported merge insert payload type for "${insert.tableName}": '
'${row.runtimeType}. Expected subclass of TableRow.',
);
}
CrdtNode _requireRemoteNode(
Map<UuidValue, CrdtNode> remoteNodes,
UuidValue uuidNodeId,
) {
return remoteNodes[uuidNodeId] ??
(throw StateError(
'Remote node $uuidNodeId not found for merge.',
));
}
Map<String, Object?> _sanitizeMergeRowData(
String tableName,
Map<String, Object?> data,
) {
final columns = _context._columnsByTableAndName[tableName];
if (columns == null) return {};
return {
for (final MapEntry(key: columnName, value: value) in data.entries)
if (columnName != 'id' &&
columnName != 'scopeId' &&
columns.containsKey(columnName))
columnName: columns[columnName]!.columnType == ColumnType.uuid
? _uuidValueFromDatabase(value)
: value,
};
}
Future<({bool exists, int? scopeId})> _readDomainRowOwner(
String tableName,
UuidValue rowId,
Transaction transaction,
) async {
final values = await _readDomainColumnValues(
tableName,
{rowId},
['scopeId'],
transaction,
);
final row = values[rowId];
if (row == null) return (exists: false, scopeId: null);
return (exists: true, scopeId: row['scopeId'] as int?);
}
/// Returns the owning scope of [rowId] from the merge snapshot taken in
/// [_loadMergeContext], reading the domain row only if it was not captured
/// there (e.g. a row a concurrent operation added since the snapshot).
Future<_DomainOwner> _resolveDomainOwner(
_MergeContext context,
String tableName,
UuidValue rowId,
Transaction transaction,
) async {
final key = (tableName, rowId);
final cached = context.domainOwners[key];
if (cached != null) return cached;
final owner = await _readDomainRowOwner(tableName, rowId, transaction);
context.domainOwners[key] = owner;
return owner;
}
Future<bool> _domainRowOwnedByEffectiveScope({
required CrdtSyncViolationOperation operation,
required String tableName,
required UuidValue rowId,
required _MergeContext context,
required Transaction transaction,
int? metadataRowId,
CrdtNode? node,
Hlc? hlc,
}) async {
final owner = await _resolveDomainOwner(context, tableName, rowId, transaction);
final mergingScope = _getEffectiveScope(transaction);
if (owner.exists && owner.scopeId == mergingScope.id) return true;
if (!owner.exists) return false;
await _throwOwnershipCollision(
operation: operation,
tableName: tableName,
rowId: rowId,
owningScopeId: owner.scopeId,
incomingScope: mergingScope,
metadataRowId: metadataRowId,
node: node,
hlc: hlc,
transaction: transaction,
);
}
Future<Never> _throwOwnershipCollision({
required CrdtSyncViolationOperation operation,
required String tableName,
required UuidValue rowId,
required int? owningScopeId,
required CrdtScope incomingScope,
required int? metadataRowId,
required Transaction transaction,
CrdtNode? node,
Hlc? hlc,
}) async {
final now = DateTime.now().toUtc();
throw CrdtSyncIntegrityViolationException(
CrdtSyncIntegrityViolation(
type: CrdtSyncViolationType.ownershipCollision,
domainTableName: tableName,
uuidRowId: rowId,
ownerScopeUuid: await _scopeUuidForNormalizedId(owningScopeId, transaction),
incomingScopeUuid: incomingScope.uuidScopeId,
operation: operation,
uuidNodeId: node?.uuidNodeId,
crdtDataRowId: metadataRowId,
hlcDatetime: hlc?.datetime,
hlcCounter: hlc?.counter,
firstSeenAt: now,
lastSeenAt: now,
occurrences: 1,
),
);
}
Future<UuidValue?> _scopeUuidForNormalizedId(
int? scopeId,
Transaction transaction,
) async {
if (scopeId == null) return null;
final scope = await CrdtScope.db.findById(
_session,
scopeId,
transaction: transaction,
);
return scope?.uuidScopeId;
}
T _withScopeId<T extends TableRow>(T row, int scopeId) {
return (row as dynamic).copyWith(scopeId: scopeId) as T;
}
}
/// Signals that a merge insert lost a primary-key race to a row owned by
/// another scope, so the savepoint around the insert application must be
/// rolled back before the sync violation is recorded and thrown.
class _ForeignScopeRowCollision implements Exception {
_ForeignScopeRowCollision(this.owningScopeId);
final int? owningScopeId;
}
extension on DatabaseSerializationManager {
/// Patches a [TableRow] with the given data.
T patchTableRow<T extends TableRow>(T row, Map<String, Object?> data) {
final rowJson = Map<String, dynamic>.from(row.toJson() as Map);
for (final column in row.table.managedColumns) {
if (column.columnName != 'id' &&
column.columnName != 'scopeId' &&
data.containsKey(column.columnName)) {
rowJson[column.fieldName] = data[column.columnName];
}
}
return deserialize<T>(rowJson);
}
}

1203
packages/serverpod_offline_sync_client/lib/src/database/merge_utils/foreign_key_projector.dart
View file

File diff suppressed because it is too large Load diff

383
packages/serverpod_offline_sync_client/lib/src/database/merge_utils/unique_resolver.dart
View file

@ -1,383 +0,0 @@
part of '../recorder.dart';
typedef _UniqueConflict = ({
CrdtDataRow row,
_UniqueIndexConflictRelease uniqueIndex,
});
/// Resolves unique constraint conflicts that appear while merging CRDT changes.
class CrdtUniqueConflictResolver {
/// Creates a resolver for the merge recorder.
CrdtUniqueConflictResolver(this._recorder);
final CrdtMutationRecorder _recorder;
/// Resolves unique conflicts for an incoming insert.
Future<({Map<String, Object?> data, bool changed})> _resolveForIncomingInsert(
CrdtMergeInsert insert,
Map<String, Object?> data,
_MergeContext context,
Transaction transaction,
) async {
final resolvedData = Map<String, Object?>.from(data);
var changed = false;
final conflicts = await _findVisibleUniqueConflicts(
tableName: insert.tableName,
rowId: insert.uuidRowId,
values: data,
transaction: transaction,
);
for (final conflict in conflicts) {
final incomingClaimHlc = _incomingInsertUniqueClaim(
insert: insert,
uniqueIndex: conflict.uniqueIndex,
context: context,
);
final conflictClaimHlc = await _uniqueIndexHlc(
tableName: insert.tableName,
row: conflict.row,
uniqueIndex: conflict.uniqueIndex,
fields: context.fields,
transaction: transaction,
);
if (incomingClaimHlc > conflictClaimHlc) {
resolvedData.addAll(
_uniqueConflictFreeValuesForData(
insert.tableName,
insert.uuidRowId,
resolvedData,
conflict.uniqueIndex,
),
);
changed = true;
} else {
await _releaseUniqueConflictForRow(
tableName: insert.tableName,
rowId: conflict.row.uuidRowId,
uniqueIndex: conflict.uniqueIndex,
transaction: transaction,
);
}
}
return (data: resolvedData, changed: changed);
}
Hlc _incomingInsertUniqueClaim({
required CrdtMergeInsert insert,
required _UniqueIndexConflictRelease uniqueIndex,
required _MergeContext context,
}) {
var uniqueHlc = insert.hlc;
for (final columnName in uniqueIndex.indexedColumns) {
uniqueHlc = uniqueHlc.maxBetween(
context.incomingFieldHlcs[(
insert.tableName,
insert.uuidRowId,
columnName,
)],
);
}
return uniqueHlc;
}
/// Resolves unique conflicts for a single incoming field update.
Future<Map<String, Object?>> _resolveForIncomingUpdate(
CrdtMergeUpdate update,
CrdtDataRow row,
_MergeContext context,
Transaction transaction,
) {
return _resolveForIncomingUpdates(
tableName: update.tableName,
row: row,
updates: {update.columnName: update.value},
context: context,
transaction: transaction,
);
}
/// Resolves unique conflicts for one or more incoming field updates.
Future<Map<String, Object?>> _resolveForIncomingUpdates({
required String tableName,
required CrdtDataRow row,
required Map<String, Object?> updates,
required _MergeContext context,
required Transaction transaction,
}) async {
if (updates.keys.every(
(columnName) => !_isUniqueIndexedColumn(tableName, columnName),
)) {
return updates;
}
final tableDefinition = _recorder._tableDefinitionsByName[tableName];
if (tableDefinition == null) {
return updates;
}
final uniqueColumnNamesToRead = {
for (final uniqueIndex in _recorder._uniqueIndexesForTable(tableDefinition))
...uniqueIndex.indexedColumns,
};
if (uniqueColumnNamesToRead.isEmpty) {
return updates;
}
final currentValues = await _recorder._readDomainColumnValues(
tableName,
{row.uuidRowId},
uniqueColumnNamesToRead.toList(),
transaction,
);
final values = {
...?currentValues[row.uuidRowId],
...updates,
};
final conflicts = await _findVisibleUniqueConflicts(
tableName: tableName,
rowId: row.uuidRowId,
values: values,
transaction: transaction,
);
final resolvedUpdates = Map<String, Object?>.from(updates);
for (final conflict in conflicts) {
final rowClaimHlc = await _uniqueIndexHlc(
tableName: tableName,
row: row,
uniqueIndex: conflict.uniqueIndex,
fields: context.fields,
transaction: transaction,
);
final conflictClaimHlc = await _uniqueIndexHlc(
tableName: tableName,
row: conflict.row,
uniqueIndex: conflict.uniqueIndex,
fields: context.fields,
transaction: transaction,
);
if (rowClaimHlc > conflictClaimHlc) {
final releasedValues = _uniqueConflictFreeValuesForData(
tableName,
row.uuidRowId,
values,
conflict.uniqueIndex,
);
for (final columnName in conflict.uniqueIndex.releaseColumnNames) {
if (releasedValues.containsKey(columnName)) {
resolvedUpdates[columnName] = releasedValues[columnName];
values[columnName] = releasedValues[columnName];
}
}
} else {
await _releaseUniqueConflictForRow(
tableName: tableName,
rowId: conflict.row.uuidRowId,
uniqueIndex: conflict.uniqueIndex,
transaction: transaction,
);
}
}
return resolvedUpdates;
}
Map<String, Object?> _uniqueConflictFreeValuesForData(
String tableName,
UuidValue rowId,
Map<String, Object?> data,
_UniqueIndexConflictRelease uniqueIndex,
) {
final tableDefinition = _recorder._tableDefinitionsByName[tableName];
if (tableDefinition == null) return data;
final released = Map<String, Object?>.from(data);
final values = {
for (final columnName in uniqueIndex.indexedColumns)
columnName: released[columnName],
};
if (values.values.any((value) => value == null)) return released;
for (final column in uniqueIndex.releaseColumns) {
released[column.columnName] = _recorder._conflictFreeValue(
column,
values[column.columnName],
tableName,
rowId,
'conflict',
);
}
return released;
}
Future<void> _releaseUniqueConflictForRow({
required String tableName,
required UuidValue rowId,
required _UniqueIndexConflictRelease uniqueIndex,
required Transaction transaction,
}) async {
final valuesByRowId = await _recorder._readDomainColumnValues(
tableName,
{rowId},
uniqueIndex.indexedColumns,
transaction,
);
final values = valuesByRowId[rowId];
if (values == null || values.values.any((value) => value == null)) return;
final releasedValues = _uniqueConflictFreeValuesForData(
tableName,
rowId,
values,
uniqueIndex,
);
await _recorder._updateDomainRows(tableName, {rowId}, releasedValues, transaction);
}
Future<List<_UniqueConflict>> _findVisibleUniqueConflicts({
required String tableName,
required UuidValue rowId,
required Map<String, Object?> values,
required Transaction transaction,
}) async {
final tableDefinition = _recorder._tableDefinitionsByName[tableName];
if (tableDefinition == null) return const [];
final uniqueIndexes = _recorder._uniqueIndexesForTable(tableDefinition);
if (uniqueIndexes.isEmpty) return const [];
final uniqueIndexesByConflictId = <UuidValue, Set<_UniqueIndexConflictRelease>>{};
for (final uniqueIndex in uniqueIndexes) {
final columnValues = {
for (final columnName in uniqueIndex.indexedColumns)
if (values.containsKey(columnName)) columnName: values[columnName],
};
if (uniqueIndex.scoped) {
columnValues['scopeId'] = _recorder
._getHlcManager(transaction)
.normalizedScopeId;
}
final expectedValueCount =
uniqueIndex.indexedColumns.length + (uniqueIndex.scoped ? 1 : 0);
if (columnValues.length != expectedValueCount) continue;
if (columnValues.values.any((value) => value == null)) continue;
final uniqueConflictIds = await _findVisibleRowsByUniqueValues(
tableName: tableName,
rowId: rowId,
values: columnValues,
transaction: transaction,
);
for (final conflictId in uniqueConflictIds) {
uniqueIndexesByConflictId.putIfAbsent(conflictId, () => {}).add(uniqueIndex);
}
}
if (uniqueIndexesByConflictId.isEmpty) return const [];
final conflictRows = await _recorder._findCrdtRows(
tableName,
uniqueIndexesByConflictId.keys.toSet(),
transaction,
include: CrdtDataRow.include(
node: CrdtNode.include(),
deleted: CrdtDataDeleted.include(node: CrdtNode.include()),
),
);
return [
for (final conflictRow in conflictRows)
for (final uniqueIndex in uniqueIndexesByConflictId[conflictRow.uuidRowId]!)
(
row: conflictRow,
uniqueIndex: uniqueIndex,
),
];
}
/// Returns the HLC for this row's current claim on [uniqueIndex].
///
/// Composite indexes use the newest field HLC among their columns; rows with
/// no field metadata fall back to the row insert HLC. Missing field metadata
/// is loaded lazily because conflicting rows may not be part of the preloaded
/// merge context.
Future<Hlc> _uniqueIndexHlc({
required String tableName,
required CrdtDataRow row,
required _UniqueIndexConflictRelease uniqueIndex,
required Map<_MergeFieldKey, CrdtDataField> fields,
required Transaction transaction,
}) async {
final uniqueColumnNames = {
...uniqueIndex.indexedColumns,
};
final missingColumnIds = <int>{};
final maxFieldHlc = <Hlc>[];
for (final columnName in uniqueColumnNames) {
final field = fields[(tableName, row.uuidRowId, columnName)];
if (field == null) {
final columnId = _schemaColumn(_recorder._schema, tableName, columnName)?.id;
if (columnId != null) missingColumnIds.add(columnId);
continue;
}
maxFieldHlc.add(field.hlc);
}
if (missingColumnIds.isNotEmpty && row.id != null) {
final loadedFields = await CrdtDataField.db.find(
_recorder._session,
where: (t) => t.rowId.equals(row.id) & t.columnId.inSet(missingColumnIds),
include: CrdtDataField.include(
column: CrdtSchemaColumn.include(),
node: CrdtNode.include(),
),
transaction: transaction,
);
for (final field in loadedFields) {
final columnName = field.column?.name;
if (columnName == null) continue;
fields[(tableName, row.uuidRowId, columnName)] = field;
maxFieldHlc.add(field.hlc);
}
}
if (maxFieldHlc.isEmpty) return row.hlc;
return maxFieldHlc.reduce((left, right) => left.maxBetween(right));
}
Future<Set<UuidValue>> _findVisibleRowsByUniqueValues({
required String tableName,
required UuidValue rowId,
required Map<String, Object?> values,
required Transaction transaction,
}) async {
return _recorder._findVisibleDomainRowIdsWhere(
tableName: tableName,
predicates: [
for (final MapEntry(key: columnName, value: value) in values.entries)
_domainColumnPredicate(columnName, value),
_domainColumnNotPredicate('id', rowId),
],
transaction: transaction,
);
}
bool _isUniqueIndexedColumn(String tableName, String columnName) {
final tableDefinition = _recorder._tableDefinitionsByName[tableName];
if (tableDefinition == null) return false;
return _recorder
._uniqueIndexesForTable(tableDefinition)
.any((i) => i.indexedColumns.contains(columnName));
}
}
extension on _UniqueIndexConflictRelease {
Set<String> get releaseColumnNames => {
for (final column in releaseColumns) column.columnName,
};
}

1525
packages/serverpod_offline_sync_client/lib/src/database/recorder.dart
View file

File diff suppressed because it is too large Load diff

277
packages/serverpod_offline_sync_client/lib/src/database/schema.dart
View file

@ -1,277 +0,0 @@
import 'package:serverpod_database/serverpod_database.dart';
import 'package:uuid/uuid.dart';
import '../protocol/protocol.dart';
import 'unique_index_utils.dart';
/// Manages the CRDT schema for a database.
class CrdtSchemaRegistry {
/// Creates a new instance of [CrdtSchemaRegistry].
CrdtSchemaRegistry(
this._session, {
required this.syncTables,
Iterable<TableDefinition>? tableDefinitions,
}) {
final tablesWithoutUuidPk = syncTables.where(
(table) => table.id is! Column<UuidValue>,
);
if (tablesWithoutUuidPk.isNotEmpty) {
throw StateError(
'CRDT can only synchronize tables with a UUID primary key, but '
'${tablesWithoutUuidPk.length} table(s) do not have a UUID primary key: '
'${tablesWithoutUuidPk.map((t) => '"${t.tableName}"').join(', ')}',
);
}
final tablesWithoutScopeId = syncTables.where((table) {
final scopeColumn = _scopeIdColumn(table);
return scopeColumn == null || scopeColumn is! Column<int>;
}).toList();
if (tablesWithoutScopeId.isNotEmpty) {
throw StateError(
'CRDT can only synchronize tables with a nullable int scopeId column, '
'but ${tablesWithoutScopeId.length} table(s) are missing it or declare '
'it with the wrong type: '
'${tablesWithoutScopeId.map((t) => '"${t.tableName}"').join(', ')}'
'$_scopeIdFieldHelp',
);
}
final tableDefinitionsByName = {
for (final tableDefinition
in tableDefinitions ??
_session.db.serializationManager.getTargetTableDefinitions())
tableDefinition.name: tableDefinition,
};
final globalUniqueIndexes = _globalUniqueIndexViolations(
syncTables,
tableDefinitionsByName,
);
if (globalUniqueIndexes.isNotEmpty) {
throw StateError(
'CRDT can only synchronize tables with per-scope unique indexes, but '
'${globalUniqueIndexes.length} unique index(es) do not include scopeId: '
'${globalUniqueIndexes.join(', ')}. '
'Add scopeId to the unique index fields. '
'The only allowed global unique indexes are foreign-key-only indexes '
'whose target tables are also synchronized.',
);
}
final requiredFKeyUniqueColumns = crdtRequiredForeignKeyOnlyUniqueColumnViolations(
syncTables,
tableDefinitionsByName,
);
if (requiredFKeyUniqueColumns.isNotEmpty) {
throw StateError(
'CRDT can only synchronize 1:1 relations when the foreign key column is '
'nullable, but the following foreign keys are non-nullable: '
'"${requiredFKeyUniqueColumns.join("', '")}". '
'Make the relation optional/nullable.',
);
}
final nonReleasableUniqueIndexes = crdtNonReleasableUniqueIndexViolations(
syncTables,
tableDefinitionsByName,
);
if (nonReleasableUniqueIndexes.isNotEmpty) {
throw StateError(
'CRDT unique conflict resolution requires at least one releasable '
'non-scope column for ${nonReleasableUniqueIndexes.length} unique '
'index(es): ${nonReleasableUniqueIndexes.join(', ')}. '
'Only nullable, text, and non-FK UUID unique columns are supported.',
);
}
final tablesMissingScopeIdRelation = _missingCrdtScopeRelations(
syncTables,
tableDefinitionsByName,
);
if (tablesMissingScopeIdRelation.isNotEmpty) {
throw StateError(
'CRDT synced tables must declare scopeId as a cascade relation to '
'crdt_scopes, but ${tablesMissingScopeIdRelation.length} table(s) are '
'missing this relation: '
'${tablesMissingScopeIdRelation.map((t) => '"$t"').join(', ')}'
'$_scopeIdFieldHelp',
);
}
}
/// Help text appended to schema validation errors that describes the required
/// `scopeId` ownership field on every synced model.
static const _scopeIdFieldHelp =
'\n\n'
'Add this field to every synced model:\n'
'fields:\n'
' id: UuidValue?, defaultPersist=random_v7\n'
' ### Owner scope of this row. Maintained by the CRDT sync layer.\n'
' scopeId: int?, relation(parent=crdt_scopes, onDelete=Cascade)\n\n'
'If the column is declared with scope=serverOnly, remove the scope; '
'the column must exist on every database.';
final DatabaseSession _session;
/// The list of tables to sync with CRDT.
final List<Table> syncTables;
late final _columnsPerTableName = {
for (final table in syncTables)
table.tableName: [
for (final column in table.columns)
if (column.columnName != 'scopeId') column.columnName,
],
};
/// Ensures that the CRDT schema is created for the database.
Future<(List<CrdtSchemaTable>, List<CrdtSchemaColumn>)> syncAndGetSchema() async {
return _session.db.transaction((transaction) async {
final tableRows = await _syncTableSchemas(transaction);
final columnRows = await _syncColumnSchemas(tableRows, transaction);
return (tableRows, columnRows);
});
}
/// Syncs the table schemas to the database.
///
/// New tables are inserted and no longer present tables are deleted. Table
/// renames are not taken into account. The returned list contains the
/// [CrdtSchemaTable]s with their IDs.
Future<List<CrdtSchemaTable>> _syncTableSchemas(Transaction transaction) async {
await CrdtSchemaTable.db.insert(
_session,
_columnsPerTableName.keys.map((name) => CrdtSchemaTable(name: name)).toList(),
transaction: transaction,
ignoreConflicts: true,
);
final foundTableRows = await CrdtSchemaTable.db.find(
_session,
transaction: transaction,
);
final deletedTableRows = [
for (final (ix, table) in foundTableRows.indexed.toList().reversed)
if (!_columnsPerTableName.containsKey(table.name)) foundTableRows.removeAt(ix),
];
await CrdtSchemaTable.db.delete(
_session,
deletedTableRows,
transaction: transaction,
);
return foundTableRows;
}
/// Syncs the column schemas to the database.
///
/// New columns are inserted and no longer present columns are deleted. Column
/// renames are not taken into account yet. The returned list contains the
/// [CrdtSchemaColumn]s with their IDs.
Future<List<CrdtSchemaColumn>> _syncColumnSchemas(
List<CrdtSchemaTable> tableRows,
Transaction transaction,
) async {
await CrdtSchemaColumn.db.insert(
_session,
[
for (final table in tableRows)
for (final column in _columnsPerTableName[table.name]!)
CrdtSchemaColumn(tblId: table.id!, name: column),
],
transaction: transaction,
ignoreConflicts: true,
);
final columnsPerTableId = {
for (final tableRow in tableRows)
tableRow.id!: _columnsPerTableName[tableRow.name]!,
};
final foundColumnRows = await CrdtSchemaColumn.db.find(
_session,
transaction: transaction,
);
final deletedColumnRows = [
for (final (ix, column) in foundColumnRows.indexed.toList().reversed)
if (!columnsPerTableId.containsKey(column.tblId) ||
!columnsPerTableId[column.tblId]!.contains(column.name))
foundColumnRows.removeAt(ix),
];
await CrdtSchemaColumn.db.delete(
_session,
deletedColumnRows,
transaction: transaction,
);
return foundColumnRows;
}
}
Column? _scopeIdColumn(Table table) {
for (final column in table.columns) {
if (column.columnName == 'scopeId') return column;
}
return null;
}
List<String> _globalUniqueIndexViolations(
List<Table> syncTables,
Map<String, TableDefinition> tableDefinitionsByName,
) {
final syncTableNames = {for (final table in syncTables) table.tableName};
return [
for (final tableName in syncTableNames)
for (final index
in tableDefinitionsByName[tableName]?.indexes ?? const <IndexDefinition>[])
if (_isForbiddenGlobalUniqueIndex(
tableDefinitionsByName[tableName]!,
index,
syncTableNames,
))
'$tableName.${index.indexName}',
];
}
bool _isForbiddenGlobalUniqueIndex(
TableDefinition tableDefinition,
IndexDefinition index,
Set<String> syncTableNames,
) {
if (!index.isUnique || index.isPrimary) return false;
if (isCrdtScopedUniqueIndex(index)) return false;
return !isCrdtAllowedForeignKeyOnlyUniqueIndex(
tableDefinition,
index,
syncTableNames,
);
}
List<String> _missingCrdtScopeRelations(
List<Table> syncTables,
Map<String, TableDefinition> tableDefinitionsByName,
) {
return [
for (final table in syncTables)
if (tableDefinitionsByName.containsKey(table.tableName) &&
!_hasCrdtScopeRelation(tableDefinitionsByName[table.tableName]!))
table.tableName,
];
}
bool _hasCrdtScopeRelation(TableDefinition tableDefinition) {
return tableDefinition.foreignKeys.any(
(fk) =>
fk.columns.length == 1 &&
fk.columns.single == 'scopeId' &&
fk.referenceTable == 'crdt_scopes' &&
fk.referenceColumns.length == 1 &&
fk.referenceColumns.single == 'id' &&
fk.onDelete == ForeignKeyAction.cascade,
);
}

217
packages/serverpod_offline_sync_client/lib/src/database/scope.dart
View file

@ -1,217 +0,0 @@
part of 'database.dart';
extension _CrdtDatabaseScope on CrdtDatabase {
({
List<T> rows,
Set<int> stampedIndexes,
Set<Object> explicitRowIds,
})
_prepareRowsForInsert<T extends TableRow>(
List<T> rows,
Transaction transaction,
) {
if (!_recorder.isCrdtTracked<T>(rows.first.table)) {
return (rows: rows, stampedIndexes: const {}, explicitRowIds: const {});
}
final scope = _requireEffectiveScope(transaction);
final effectiveScopeId = scope.id!;
final databaseRows = <T>[];
final stampedIndexes = <int>{};
final explicitRowIds = <Object>{};
for (final (index, row) in rows.indexed) {
final rowScopeId = _readScopeId(row);
if (rowScopeId == null) {
databaseRows.add(_copyWithScopeId(row, effectiveScopeId));
stampedIndexes.add(index);
continue;
}
if (rowScopeId != effectiveScopeId) {
throw StateError(
'Cannot write ${row.table.tableName} row ${row.id} with scopeId '
'$rowScopeId while acting in scope $effectiveScopeId.',
);
}
databaseRows.add(row);
final rowId = row.id;
if (rowId != null) explicitRowIds.add(rowId as Object);
}
return (
rows: databaseRows,
stampedIndexes: stampedIndexes,
explicitRowIds: explicitRowIds,
);
}
bool _shouldStripReturnedScopeId<T extends TableRow>(
T row,
Transaction transaction,
) {
if (!_recorder.isCrdtTracked<T>(row.table)) return false;
final rowScopeId = _readScopeId(row);
if (rowScopeId == null) return true;
final effectiveScopeId = _requireEffectiveScope(transaction).id!;
if (rowScopeId != effectiveScopeId) {
throw StateError(
'Cannot write ${row.table.tableName} row ${row.id} with scopeId '
'$rowScopeId while acting in scope $effectiveScopeId.',
);
}
return false;
}
void _assertNoScopeIdColumnValues<T extends TableRow>(
List<ColumnValue> columnValues,
) {
for (final columnValue in columnValues) {
if (columnValue.column.columnName == 'scopeId') {
final table = serializationManager.getTableForType(T)?.tableName ?? 'unknown';
throw StateError(
'scopeId is immutable and owned by the CRDT sync layer for $table.',
);
}
}
}
CrdtScope _requireEffectiveScope(Transaction transaction) {
return _recorder.scopeForQueries(transaction) ??
(throw StateError(
'A scope ID is required for CRDT writes without a persistent user.',
));
}
int? _readScopeId(TableRow row) => (row as dynamic).scopeId as int?;
T _copyWithScopeId<T extends TableRow>(T row, int scopeId) {
return (row as dynamic).copyWith(scopeId: scopeId) as T;
}
void _stripStampedRows<T extends TableRow>(
List<T> rows,
({
List<T> rows,
Set<int> stampedIndexes,
Set<Object> explicitRowIds,
})
prepared,
) {
if (rows.length == prepared.rows.length) {
for (final (index, row) in rows.indexed) {
if (prepared.stampedIndexes.contains(index)) _stripScopeId(row);
}
return;
}
// `ignoreConflicts` dropped rows, so positions no longer align with the
// input. Keep the value only on rows whose id was caller-provided on an
// explicit (asserted) input row; everything else is treated as stamped.
for (final row in rows) {
final rowId = row.id;
if (rowId == null || !prepared.explicitRowIds.contains(rowId)) {
_stripScopeId(row);
}
}
}
void _stripScopeId(TableRow row) {
(row as dynamic).scopeId = null;
}
List<T> _stripScopeIdFromScopedRead<T extends TableRow>(
List<T> rows,
Include? include,
Transaction? transaction,
) {
if (rows.isEmpty) return rows;
if (!_recorder.isCrdtTracked<T>(rows.first.table)) return rows;
if (_recorder.scopeForQueries(transaction) == null) return rows;
final hasNestedRows =
include != null && include.includes.values.any((inc) => inc != null);
if (!hasNestedRows) {
// Hot path: delegate-returned rows are package-owned, so the root
// field is stripped in place with no allocation.
rows.forEach(_stripScopeId);
return rows;
}
// Nested models are only reachable through generated fields, which
// cannot be accessed generically by name, so include graphs pay a
// serialization round-trip per row to strip recursively.
return [
for (final row in rows) _copyWithScopeIdsStripped(row),
];
}
T _copyWithScopeIdsStripped<T extends TableRow>(T row) {
final json = Map<String, dynamic>.from(row.toJson() as Map);
_stripScopeIdsFromJson(json);
return serializationManager.deserialize<T>(json);
}
void _stripScopeIdsFromJson(Object? value) {
if (value is Map) {
if (value.containsKey('scopeId')) {
value['scopeId'] = null;
}
value.values.forEach(_stripScopeIdsFromJson);
return;
}
if (value is Iterable) {
value.forEach(_stripScopeIdsFromJson);
}
}
}
/// Extension on [Table] that exposes [scopeEquals] for use in `where`
/// clauses to restrict membership-wide reads to a single scope.
extension ScopeUuidFilter on Table {
/// Expression that narrows a membership-wide read to [scopeUuid].
///
/// Reads are membership-wide and strip the database-local `scopeId`, so a
/// caller holding several scopes cannot otherwise restrict a query to one scope
/// or tell which scope a row came from. The placeholder is resolved to a
/// filter on the reserved `scopeId` column before the query runs. When the
/// scope UUID is not known locally the filter matches no rows.
///
/// Example:
/// ```dart
/// // Returns only persons in the shared scope.
/// final sharedPeople = await Person.db.find(
/// session,
/// where: (t) => t.scopeEquals(sharedScopeUuid),
/// );
///
/// // Combines with other filters.
/// final aliceInShared = await Person.db.find(
/// session,
/// where: (t) => t.name.equals('Alice') & t.scopeEquals(sharedScopeUuid),
/// );
/// ```
Expression scopeEquals(UuidValue scopeUuid) =>
_crdtScope.uuidScopeId.equals(scopeUuid);
CrdtScopeTable get _crdtScope {
return createRelationTable<CrdtScopeTable>(
relationFieldName: '${tableName}_crdt_scope',
field: _scopeIdColumn,
foreignField: CrdtScope.t.id,
tableRelation: tableRelation,
createTable: (foreignTableRelation) =>
CrdtScopeTable(tableRelation: foreignTableRelation),
);
}
Column get _scopeIdColumn => columns.singleWhere(
(column) => column.columnName == 'scopeId' && column is ColumnInt,
orElse: () => throw StateError('Table "$tableName" has no scopeId column.'),
);
}

123
packages/serverpod_offline_sync_client/lib/src/database/session.dart
View file

@ -1,123 +0,0 @@
import 'package:meta/meta.dart';
import 'package:serverpod_database/serverpod_database.dart';
import 'package:uuid/uuid.dart';
import '../crdt/sync.dart';
import 'database.dart';
import 'recorder.dart';
/// Wraps a [DatabaseSession] to provide a [CrdtDatabase] as [DatabaseSession.db].
class CrdtDatabaseSession implements DatabaseSession {
/// Creates a [CrdtDatabaseSession] instance.
CrdtDatabaseSession(
Database db, {
/// The list of tables to sync with CRDT.
required List<Table> syncTables,
/// Shared CRDT database metadata.
CrdtDatabaseContext? context,
/// Maximum number of merge changes sent in one sync stream message.
int syncBatchSize = CrdtSync.defaultSyncBatchSize,
/// Delay between continuous sync rounds.
Duration continuousSyncInterval = CrdtSync.defaultContinuousSyncInterval,
/// The user ID to use for all CRDT operations. This should only be used for
/// databases operating on the client side, where all data is for the same user.
/// Otherwise, the user ID must be passed through the transaction.
UuidValue? persistentUserId,
}) : _db = db is CrdtDatabase
? db
: CrdtDatabase(
db,
syncTables: syncTables,
context: context,
syncBatchSize: syncBatchSize,
continuousSyncInterval: continuousSyncInterval,
persistentUserId: persistentUserId,
);
/// Creates a [CrdtDatabaseSession] instance that wraps a [DatabaseSession].
factory CrdtDatabaseSession.wraps(
DatabaseSession session, {
/// The list of tables to sync with CRDT.
required List<Table> syncTables,
/// Shared CRDT database metadata.
CrdtDatabaseContext? context,
/// Maximum number of merge changes sent in one sync stream message.
int syncBatchSize = CrdtSync.defaultSyncBatchSize,
/// Delay between continuous sync rounds.
Duration continuousSyncInterval = CrdtSync.defaultContinuousSyncInterval,
/// The user ID to use for all CRDT operations. This should only be used for
/// databases operating on the client side, where all data is for the same user.
/// Otherwise, the user ID must be passed through the transaction.
UuidValue? persistentUserId,
}) => CrdtDatabaseSession(
session.db,
syncTables: syncTables,
context: context,
syncBatchSize: syncBatchSize,
continuousSyncInterval: continuousSyncInterval,
persistentUserId: persistentUserId,
);
final CrdtDatabase _db;
@override
CrdtDatabase get db => _db;
@override
LogQueryFunction? get logQuery => null;
@override
LogWarningFunction? get logWarning => null;
@override
Transaction? transaction;
}
/// Wraps a [Database] to provide a [DatabaseSession] as [DatabaseSession.db].
@internal
class BasicDatabaseSession implements DatabaseSession {
/// Creates a [BasicDatabaseSession] instance.
BasicDatabaseSession(this._db);
final Database _db;
@override
Database get db => _db;
@override
LogQueryFunction? get logQuery => null;
@override
LogWarningFunction? get logWarning => null;
@override
Transaction? transaction;
}
@internal
extension DatabaseSessionExtension on Database {
DatabaseSession get session => BasicDatabaseSession(this);
}
/// Convenience access to a CRDT-aware database from a wrapped session.
extension CrdtDatabaseAccess on DatabaseSession {
/// Returns the wrapped [CrdtDatabase] for this session.
CrdtDatabase get crdtDb {
final database = db;
if (database is CrdtDatabase) return database;
throw StateError(
'This database session is not wrapped with CrdtDatabaseSession. '
'Use CrdtDatabaseSession.wraps(...) before accessing crdtDb.',
);
}
}

249
packages/serverpod_offline_sync_client/lib/src/database/tombstone.dart
View file

@ -1,249 +0,0 @@
import 'package:meta/meta.dart';
import 'package:serverpod_database/serverpod_database.dart';
import '../crdt/extensions.dart';
import '../protocol/protocol.dart';
/// Resolves the local [CrdtSchemaTable] id for a domain table name. Returns
/// null when the table is not registered for CRDT synchronization.
typedef CrdtTableIdResolver = int? Function(String tableName);
/// Resolves the [CrdtScope] ids scoping the current query. Returns null when no
/// scope is available (e.g. server-side reads outside `transactionForUser`
/// without a persistent user), in which case the query is an admin read: rows
/// are not isolated and a row is hidden only when its owning scope hides it.
typedef CrdtScopeIdsResolver = List<int>? Function();
/// Private sentinel class used to signal that the CRDT visibility filter
/// should be bypassed. Detected via `is` type check instead of string
/// comparison.
///
/// It renders as the inert SQL literal `TRUE`, so it is always valid wherever
/// it appears in a `where` clause and never needs to be stripped out before the
/// query reaches the database.
final class _IncludeHiddenSentinel extends Expression<String> {
const _IncludeHiddenSentinel() : super('TRUE');
}
/// Extension on [Table] that exposes [includeHiddenRows] for use in `where`
/// clauses to bypass the CRDT tombstone/visibility filter.
extension IncludeTombstonedRows on Table {
/// Expression that bypasses the CRDT tombstone/visibility filter.
///
/// When included in a `where` clause, all rows are returned — including those
/// that are tombstoned (soft-deleted) — instead of being hidden.
///
/// Example:
/// ```dart
/// // Returns all persons, including tombstoned ones.
/// final allPersons = await Person.db.find(
/// session,
/// where: (t) => t.includeHiddenRows,
/// );
///
/// // Returns all persons named 'Alice', including tombstoned ones.
/// final alicePersons = await Person.db.find(
/// session,
/// where: (t) => t.name.equals('Alice') & t.includeHiddenRows,
/// );
/// ```
Expression get includeHiddenRows => const _IncludeHiddenSentinel();
}
/// Merges [where] with CRDT visibility predicates and mutates [include] in place.
///
/// This method ensures that no rows with a hidden [CrdtDataRow.visibility] are
/// returned for queries with [where] clauses or [include] graphs. If [include]
/// is null, only the root table and [where] are merged.
///
/// Each predicate only considers [CrdtDataRow]s recorded for the queried table
/// ([tableIdForName]) and, when scopes are available ([scopeIds]), isolates the
/// query to those scopes' rows. This prevents a tombstone recorded for another
/// table or scope from masking an unrelated row that reuses the same UUID.
/// Without a scope (admin reads), rows are not isolated and a row stays
/// visible as long as its owning scope still sees it.
///
/// Both resolvers are only invoked when a predicate is actually built (i.e.
/// for synced tables with UUID primary keys), so queries on CRDT-internal
/// tables never touch the recorder state.
///
/// We do **not** add a root predicate for keys whose nested include is an
/// [IncludeList] - visibility filtering for that path is already handled via the
/// list subquery [where] (and many-relations are not the same join shape as a
/// single [IncludeObject]).
///
/// If [where] contains the [IncludeTombstonedRows.includeHiddenRows] sentinel
/// anywhere in the expression tree, the visibility filter is skipped for the
/// root table. The sentinel itself renders as the inert SQL literal `TRUE`, so
/// it is left in place without producing invalid SQL.
@internal
Expression? mergeWhereWithTombstone<T extends TableRow>(
DatabaseSerializationManager serializationManager,
Expression? where,
Include? include, {
required CrdtTableIdResolver tableIdForName,
required CrdtScopeIdsResolver scopeIds,
}) {
final includeObjectPredicates = _walkIncludeGraphForTombstone(
include,
null,
tableIdForName: tableIdForName,
scopeIds: scopeIds,
);
final rootTable = serializationManager.getTableForType(T);
var merged = _includesHiddenSentinel(where)
? where
: _mergeWhereOptional(
where,
rootTable?.whereVisibleOnCrdtRow(tableIdForName, scopeIds),
);
merged = _mergeWhereOptional(merged, includeObjectPredicates);
return merged;
}
/// Walks [inc], mutating each [IncludeList.where] and returning predicates to
/// `AND` into the main query for [IncludeObject] joins.
Expression? _walkIncludeGraphForTombstone(
Include? inc,
Expression? includeObjectPredicates, {
required CrdtTableIdResolver tableIdForName,
required CrdtScopeIdsResolver scopeIds,
}) {
if (inc == null) return includeObjectPredicates;
if (inc is IncludeList) {
// List relation: filter inside the list subquery only.
if (!_includesHiddenSentinel(inc.where)) {
inc.where = _mergeWhereOptional(
inc.where,
inc.table.whereVisibleOnCrdtRow(tableIdForName, scopeIds),
);
}
return _walkIncludeGraphForTombstone(
inc.include,
includeObjectPredicates,
tableIdForName: tableIdForName,
scopeIds: scopeIds,
);
}
var acc = includeObjectPredicates;
final obj = inc;
final selfTable = obj.table;
for (final entry in obj.includes.entries) {
final nested = entry.value;
if (nested is IncludeList) {
// e.g. one-to-many: visibility applies via IncludeList.where, not the main WHERE.
acc = _walkIncludeGraphForTombstone(
nested,
acc,
tableIdForName: tableIdForName,
scopeIds: scopeIds,
);
continue;
}
if (nested is IncludeObject) {
// one-to-one / optional object: joined on the main SELECT, constrain via root WHERE.
final childTable = selfTable.getRelationTable(entry.key);
if (childTable != null) {
acc = _mergeWhereOptional(
acc,
childTable.whereVisibleOnCrdtRow(tableIdForName, scopeIds),
);
}
acc = _walkIncludeGraphForTombstone(
nested,
acc,
tableIdForName: tableIdForName,
scopeIds: scopeIds,
);
}
}
return acc;
}
Expression? _mergeWhereOptional(Expression? where, Expression? addition) {
if (addition == null) return where;
if (where == null) return addition;
return where & addition;
}
/// Returns `true` when [where] contains the [_IncludeHiddenSentinel] anywhere
/// in its expression tree.
///
/// Walks the tree via [Expression.depthFirst], which every expression type
/// implements, so the sentinel is detected regardless of how deeply it is
/// nested or which operators (`AND`, `OR`, `NOT`) combine it. The sentinel
/// renders as inert SQL (`TRUE`), so no rewriting of [where] is needed.
bool _includesHiddenSentinel(Expression? where) {
if (where == null) return false;
return where.depthFirst.any((e) => e is _IncludeHiddenSentinel);
}
extension on Table {
/// Creates a predicate that filters out hidden rows for the given [Table]
/// using materialized [CrdtDataRow.visibility] metadata.
///
/// Returns null when [Table] does not use a UUID primary key or is not
/// registered for CRDT synchronization. Keeps rows when the id is null
/// (unmatched [IncludeObject] joins) or no hidden CRDT row matches.
///
/// With scopes, rows are isolated to them (`scopeId IN (<scopes>)`) and
/// checked against those scopes' tombstones with a correlated `NOT EXISTS`
/// probe covered by the (scopeId, tblId, uuidRowId) unique index. Without a
/// scope (admin reads), there is no isolation filter and the probe is
/// keyed by the row's own `scopeId` column instead — covered by the same
/// index — so a row stays visible while its owning scope sees it.
Expression? whereVisibleOnCrdtRow(
CrdtTableIdResolver tableIdForName,
CrdtScopeIdsResolver scopeIds,
) {
if (id is! ColumnUuid) return null;
final tableId = tableIdForName(tableName);
if (tableId == null) return null;
// Synced tables are validated to carry the column at initialize(); a
// missing column here must fail closed, never skip the predicate.
final scopeColumn =
scopeIdColumn ??
(throw StateError(
'Synced table "$tableName" has no int scopeId column; '
'CrdtSchemaRegistry validation should have rejected it.',
));
final crdtRow = CrdtDataRow.t;
final effectiveScopeIds = scopeIds();
if (effectiveScopeIds != null && effectiveScopeIds.isEmpty) {
return id.equals(null) | const Expression('FALSE');
}
final scopeFilter = effectiveScopeIds == null
? '${crdtRow.scopeId} = $scopeColumn'
: '${crdtRow.scopeId} IN (${_sqlIntList(effectiveScopeIds)})';
final notExistsExpr = Expression(
'NOT EXISTS '
'(SELECT 1 FROM "${crdtRow.tableName}" '
'WHERE $scopeFilter '
'AND ${crdtRow.tblId} = $tableId '
'AND ${crdtRow.uuidRowId} = $id '
'AND ${crdtRow.visibility} > $crdtRowLastVisibleVisibilityIndex)',
);
return id.equals(null) |
((effectiveScopeIds != null)
? (scopeColumn.inSet(effectiveScopeIds.toSet()) & notExistsExpr)
: notExistsExpr);
}
ColumnInt? get scopeIdColumn {
for (final column in columns) {
if (column.columnName == 'scopeId' && column is ColumnInt) {
return column;
}
}
return null;
}
}
String _sqlIntList(Iterable<int> values) {
return values.map(ValueEncoder.instance.convert).join(', ');
}

190
packages/serverpod_offline_sync_client/lib/src/database/unique_index_utils.dart
View file

@ -1,190 +0,0 @@
import 'package:meta/meta.dart';
import 'package:serverpod_database/serverpod_database.dart';
@internal
bool isCrdtAllowedForeignKeyOnlyUniqueIndex(
TableDefinition tableDefinition,
IndexDefinition index,
Set<String> syncTableNames,
) {
return _foreignKeyOnlyUniqueIndexColumns(
tableDefinition,
index,
syncTableNames,
) !=
null;
}
List<String>? _foreignKeyOnlyUniqueIndexColumns(
TableDefinition tableDefinition,
IndexDefinition index,
Set<String> syncTableNames,
) {
final indexColumns = <String>[];
for (final element in index.elements) {
if (element.type != IndexElementDefinitionType.column) return null;
indexColumns.add(element.definition);
}
if (indexColumns.isEmpty || indexColumns.length != indexColumns.toSet().length) {
return null;
}
final indexColumnSet = indexColumns.toSet();
final coveredForeignKeyColumns = <String>{};
for (final foreignKey in tableDefinition.foreignKeys) {
if (!syncTableNames.contains(foreignKey.referenceTable)) continue;
final foreignKeyColumns = foreignKey.columns.toSet();
if (foreignKeyColumns.isEmpty) continue;
if (indexColumnSet.containsAll(foreignKeyColumns)) {
coveredForeignKeyColumns.addAll(foreignKeyColumns);
}
}
return indexColumns.every(coveredForeignKeyColumns.contains) ? indexColumns : null;
}
@internal
List<String> crdtRequiredForeignKeyOnlyUniqueColumnViolations(
List<Table> syncTables,
Map<String, TableDefinition> tableDefinitionsByName,
) {
final syncTableNames = {for (final table in syncTables) table.tableName};
final violations = <String>{};
for (final tableName in syncTableNames) {
final table = tableDefinitionsByName[tableName];
if (table == null) continue;
final columnsByName = {for (final column in table.columns) column.name: column};
for (final index in table.indexes) {
if (!index.isUnique || index.isPrimary || isCrdtScopedUniqueIndex(index)) {
continue;
}
final foreignKeyOnlyColumns = _foreignKeyOnlyUniqueIndexColumns(
table,
index,
syncTableNames,
);
if (foreignKeyOnlyColumns == null) continue;
for (final columnName in foreignKeyOnlyColumns) {
final column = columnsByName[columnName];
if (column == null) {
throw StateError(
'No column definition found for ${table.name}.$columnName.',
);
}
if (!column.isNullable) violations.add('${table.name}.$columnName');
}
}
}
return violations.toList();
}
@internal
bool isCrdtScopedUniqueIndex(IndexDefinition index) {
return index.elements.any(
(element) =>
element.type == IndexElementDefinitionType.column &&
element.definition == 'scopeId',
);
}
@internal
Iterable<IndexDefinition> crdtSyncableUniqueIndexesForTable(
TableDefinition table,
Set<String> syncTableNames,
) {
return table.indexes.where(
(index) =>
index.isUnique &&
!index.isPrimary &&
index.elements.every(
(element) => element.type == IndexElementDefinitionType.column,
) &&
(isCrdtScopedUniqueIndex(index) ||
isCrdtAllowedForeignKeyOnlyUniqueIndex(
table,
index,
syncTableNames,
)),
);
}
@internal
bool isCrdtForeignKeyColumn(TableDefinition table, String columnName) {
return table.foreignKeys.any(
(fk) => fk.columns.length == 1 && fk.columns.single == columnName,
);
}
/// How a unique-indexed column can be released to a conflict-free value, or
/// null when the column cannot be released deterministically.
@internal
enum CrdtUniqueConflictReleaseKind {
setNull,
textSuffix,
syntheticUuid,
}
@internal
CrdtUniqueConflictReleaseKind? crdtUniqueConflictReleaseKindForColumn(
TableDefinition table,
ColumnDefinition column,
) {
if (column.isNullable) return CrdtUniqueConflictReleaseKind.setNull;
if (column.columnType == ColumnType.text) {
return CrdtUniqueConflictReleaseKind.textSuffix;
}
if (column.columnType == ColumnType.uuid &&
!isCrdtForeignKeyColumn(table, column.name)) {
return CrdtUniqueConflictReleaseKind.syntheticUuid;
}
return null;
}
@internal
List<String> crdtNonReleasableUniqueIndexViolations(
List<Table> syncTables,
Map<String, TableDefinition> tableDefinitionsByName,
) {
final syncTableNames = {for (final table in syncTables) table.tableName};
return [
for (final tableName in syncTableNames)
if (tableDefinitionsByName.containsKey(tableName))
for (final index in crdtSyncableUniqueIndexesForTable(
tableDefinitionsByName[tableName]!,
syncTableNames,
))
if (!_hasReleasableNonScopeUniqueColumn(
tableDefinitionsByName[tableName]!,
index,
))
'$tableName.${index.indexName}',
];
}
bool _hasReleasableNonScopeUniqueColumn(
TableDefinition table,
IndexDefinition index,
) {
final columnsByName = {
for (final column in table.columns) column.name: column,
};
for (final element in index.elements) {
if (element.type != IndexElementDefinitionType.column) continue;
if (element.definition == 'scopeId') continue;
final column = columnsByName[element.definition];
if (column == null) {
throw StateError(
'No column definition found for ${table.name}.${element.definition}.',
);
}
if (crdtUniqueConflictReleaseKindForColumn(table, column) != null) {
return true;
}
}
return false;
}

Some files were not shown because too many files have changed in this diff Show more