feat: Implement the sync protocol (#54)
Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
This commit is contained in:
Marcelo Mendonça Soares
GitHub
parent
83fbef8d2c
commit
f85239c3a4
12 changed files with 3031 additions and 306 deletions
|
|
@ -1,7 +1,10 @@
|
|||
export 'src/crdt/client_sync.dart';
|
||||
export 'src/crdt/exceptions.dart';
|
||||
export 'src/crdt/merge.dart';
|
||||
export 'src/crdt/sync.dart';
|
||||
export 'src/database/database.dart';
|
||||
export 'src/database/recorder.dart';
|
||||
export 'src/database/schema.dart';
|
||||
export 'src/database/session.dart' show CrdtDatabaseSession;
|
||||
export 'src/database/session.dart' show CrdtDatabaseAccess, CrdtDatabaseSession;
|
||||
export 'src/managers/hlc.dart';
|
||||
export 'src/managers/user.dart';
|
||||
|
|
|
|||
|
|
@ -0,0 +1,127 @@
|
|||
import 'dart:async';
|
||||
|
||||
import 'package:serverpod_client/serverpod_client.dart';
|
||||
import 'package:serverpod_database/serverpod_database.dart';
|
||||
|
||||
import '../database/session.dart';
|
||||
import '../protocol/protocol.dart';
|
||||
import 'sync.dart';
|
||||
|
||||
/// A running continuous CRDT sync session.
|
||||
class CrdtSyncSession {
|
||||
CrdtSyncSession._({
|
||||
required this.done,
|
||||
required Future<void> Function() cancel,
|
||||
}) : _cancel = cancel;
|
||||
|
||||
/// Completes when the sync session ends normally or after [cancel].
|
||||
final Future<void> done;
|
||||
|
||||
final Future<void> Function() _cancel;
|
||||
|
||||
/// Stops the sync session and closes the transport. This method is idempotent
|
||||
/// and can be called multiple times.
|
||||
Future<void> cancel() => _cancel();
|
||||
}
|
||||
|
||||
/// High-level client sync helpers built on top of the generated module caller.
|
||||
class CrdtSyncClient {
|
||||
/// Creates a [CrdtSyncClient] for the provided module caller.
|
||||
CrdtSyncClient(this._caller);
|
||||
|
||||
final Caller _caller;
|
||||
|
||||
/// Pushes local pending changes for [session] to the remote peer once.
|
||||
///
|
||||
/// The [session] must be wrapped in a [CrdtDatabaseSession].
|
||||
Future<void> syncOnce(DatabaseSession session) async {
|
||||
await _startSyncSession(session, once: true).done;
|
||||
}
|
||||
|
||||
/// Keeps synchronizing [session] with the remote peer until cancelled through
|
||||
/// [CrdtSyncSession.cancel] or the remote stream closes.
|
||||
///
|
||||
/// The [session] must be wrapped in a [CrdtDatabaseSession].
|
||||
CrdtSyncSession syncContinuously(
|
||||
DatabaseSession session, {
|
||||
CrdtSyncOnMergeSuccess? onMergeSuccess,
|
||||
}) => _startSyncSession(session, once: false, onMergeSuccess: onMergeSuccess);
|
||||
|
||||
CrdtSyncSession _startSyncSession(
|
||||
DatabaseSession session, {
|
||||
required bool once,
|
||||
CrdtSyncOnMergeSuccess? onMergeSuccess,
|
||||
}) {
|
||||
// The stream will be closed using [outboundChanges.closeOrSkip].
|
||||
// ignore: close_sinks
|
||||
final outboundChanges = StreamController<CrdtSyncStreamEvent>();
|
||||
final doneCompleter = Completer<void>();
|
||||
var cancelled = false;
|
||||
|
||||
try {
|
||||
final remoteStream = _caller.crdtSync.sync(
|
||||
changes: outboundChanges.stream,
|
||||
once: once,
|
||||
);
|
||||
|
||||
final subscription = session.crdtDb
|
||||
.sync(inbound: remoteStream, once: once, onMergeSuccess: onMergeSuccess)
|
||||
.listen(
|
||||
outboundChanges.addIfNotClosed,
|
||||
onDone: doneCompleter.completeOrSkip,
|
||||
onError: doneCompleter.completeErrorOrSkip,
|
||||
cancelOnError: true,
|
||||
);
|
||||
|
||||
return CrdtSyncSession._(
|
||||
done: doneCompleter.future.whenComplete(outboundChanges.closeOrSkip),
|
||||
cancel: () async {
|
||||
if (cancelled) return;
|
||||
cancelled = true;
|
||||
|
||||
unawaited(outboundChanges.closeOrSkip());
|
||||
if (doneCompleter.isCompleted) return;
|
||||
const waitTimeout = Duration(milliseconds: 200);
|
||||
// Prefer graceful shutdown: closing outbound lets the server finish
|
||||
// and complete [done] via [subscription.onDone].
|
||||
await doneCompleter.future.timeout(waitTimeout, onTimeout: () {});
|
||||
await subscription.cancel().timeout(waitTimeout, onTimeout: () {});
|
||||
doneCompleter.completeOrSkip();
|
||||
},
|
||||
);
|
||||
} catch (_) {
|
||||
unawaited(outboundChanges.closeOrSkip());
|
||||
rethrow;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Exposes CRDT sync helpers from a generated client.
|
||||
extension CrdtSyncClientExtension on ServerpodClientShared {
|
||||
/// Returns CRDT sync helpers bound to this client.
|
||||
CrdtSyncClient get crdt => CrdtSyncClient(Caller(this));
|
||||
}
|
||||
|
||||
extension on StreamController<CrdtSyncStreamEvent> {
|
||||
void addIfNotClosed(CrdtSyncStreamEvent event) {
|
||||
if (isClosed) return;
|
||||
add(event);
|
||||
}
|
||||
|
||||
Future<void> closeOrSkip() async {
|
||||
if (isClosed) return;
|
||||
await close();
|
||||
}
|
||||
}
|
||||
|
||||
extension on Completer<void> {
|
||||
void completeOrSkip() {
|
||||
if (isCompleted) return;
|
||||
complete();
|
||||
}
|
||||
|
||||
void completeErrorOrSkip(Object error, StackTrace stackTrace) {
|
||||
if (isCompleted) return;
|
||||
completeError(error, stackTrace);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,94 @@
|
|||
import 'package:serverpod_serialization/serverpod_serialization.dart';
|
||||
|
||||
import '../protocol/protocol.dart';
|
||||
|
||||
/// Base type for CRDT sync session failures.
|
||||
sealed class CrdtSyncException implements Exception {
|
||||
const CrdtSyncException();
|
||||
}
|
||||
|
||||
/// Thrown when the inbound sync stream closes before an expected event arrives.
|
||||
final class CrdtSyncStreamClosedException extends CrdtSyncException {
|
||||
/// Creates a [CrdtSyncStreamClosedException].
|
||||
const CrdtSyncStreamClosedException({required this.phase});
|
||||
|
||||
/// The protocol phase that was interrupted.
|
||||
final String phase;
|
||||
|
||||
@override
|
||||
String toString() =>
|
||||
'CrdtSyncStreamClosedException: sync stream closed before $phase event.';
|
||||
}
|
||||
|
||||
/// Thrown when the inbound sync stream delivers an event of the wrong type.
|
||||
final class CrdtSyncUnexpectedEventException<T extends CrdtSyncStreamEvent>
|
||||
extends CrdtSyncException {
|
||||
/// Creates a [CrdtSyncUnexpectedEventException].
|
||||
const CrdtSyncUnexpectedEventException({
|
||||
required this.expected,
|
||||
required this.received,
|
||||
});
|
||||
|
||||
/// A description of the expected event type or types.
|
||||
final String expected;
|
||||
|
||||
/// The event that was received instead.
|
||||
final CrdtSyncStreamEvent received;
|
||||
|
||||
@override
|
||||
String toString() =>
|
||||
'CrdtSyncUnexpectedEventException: expected $expected, but '
|
||||
'received "${received.runtimeType.className}" instead.';
|
||||
}
|
||||
|
||||
/// Thrown when a peer's since-HLC frame does not reference the local node id.
|
||||
final class CrdtSyncInvalidSinceHlcException extends CrdtSyncException {
|
||||
/// Creates a [CrdtSyncInvalidSinceHlcException].
|
||||
const CrdtSyncInvalidSinceHlcException({
|
||||
required this.receivedNodeId,
|
||||
required this.expectedNodeId,
|
||||
});
|
||||
|
||||
/// The node id carried by the peer's since-HLC frame.
|
||||
final UuidValue receivedNodeId;
|
||||
|
||||
/// The local node id that was expected.
|
||||
final UuidValue expectedNodeId;
|
||||
|
||||
@override
|
||||
String toString() =>
|
||||
'CrdtSyncInvalidSinceHlcException: peer since HLC node id '
|
||||
'"$receivedNodeId" does not match local node id "$expectedNodeId".';
|
||||
}
|
||||
|
||||
/// Thrown when the sync tables hash sent by a peer does not match locally.
|
||||
final class SyncTablesHashMismatchException extends CrdtSyncException {
|
||||
/// Creates a [SyncTablesHashMismatchException].
|
||||
const SyncTablesHashMismatchException({
|
||||
required this.received,
|
||||
required this.expected,
|
||||
});
|
||||
|
||||
/// The hash received from the remote peer.
|
||||
final String received;
|
||||
|
||||
/// The hash computed locally from the configured sync tables.
|
||||
final String expected;
|
||||
|
||||
@override
|
||||
String toString() =>
|
||||
'SyncTablesHashMismatchException: schema hash mismatch. Received '
|
||||
'"$received", expected "$expected". Ensure both sides are on the same '
|
||||
'schema version before syncing.';
|
||||
}
|
||||
|
||||
extension on Type {
|
||||
/// The name of the class without the leading underscore and without the 'Impl' suffix.
|
||||
String get className {
|
||||
final name = toString();
|
||||
if (name.startsWith('_') && name.endsWith('Impl')) {
|
||||
return name.substring(1, name.length - 4);
|
||||
}
|
||||
return name;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,8 +1,14 @@
|
|||
import 'dart:async';
|
||||
|
||||
import 'package:serverpod_client/serverpod_client.dart';
|
||||
import 'package:serverpod_database/serverpod_database.dart';
|
||||
import 'package:serverpod_offline_sync_shared/serverpod_offline_sync_shared.dart';
|
||||
|
||||
import '../protocol/protocol.dart';
|
||||
import 'exceptions.dart';
|
||||
|
||||
/// A group of CRDT merge changes collected for one sync batch.
|
||||
typedef CrdtMergeSet = List<CrdtMergeChange>;
|
||||
|
||||
/// Merge metadata extracted from a [CrdtMergeSet].
|
||||
typedef CrdtMergeMetadataLookup = ({
|
||||
|
|
@ -10,21 +16,95 @@ typedef CrdtMergeMetadataLookup = ({
|
|||
Map<String, Set<String>> columnNamesByTable,
|
||||
});
|
||||
|
||||
/// CRDT merge helpers built on top of the generated Serverpod models.
|
||||
extension CrdtMergeSetExtension on CrdtMergeSet {
|
||||
/// Whether this merge set has no changes.
|
||||
bool get isEmpty => inserts.isEmpty && updates.isEmpty && deletes.isEmpty;
|
||||
/// Extensions for stream iterators of [CrdtSyncStreamEvent].
|
||||
extension CrdtSyncStreamEventStreamExtension on StreamIterator<CrdtSyncStreamEvent> {
|
||||
/// Collects the next framed sync batch from this iterator.
|
||||
///
|
||||
/// Each batch is zero or more [CrdtSyncMergeChunk] events followed by
|
||||
/// [CrdtSyncEndOfBatch]. If the stream is idle before a batch starts, an
|
||||
/// empty batch is returned.
|
||||
///
|
||||
/// When [allowCloseBeforeBatch] is true, returns `null` if the stream closes
|
||||
/// before the next batch starts. Otherwise, closing before a batch starts is
|
||||
/// treated as a [CrdtSyncStreamClosedException].
|
||||
///
|
||||
/// If the stream closes after a merge change was already received, the
|
||||
/// partial batch is still treated as an error.
|
||||
Future<CrdtMergeSet?> collectNextBatch({
|
||||
bool allowCloseBeforeBatch = false,
|
||||
}) async {
|
||||
final mergeSet = <CrdtMergeChange>[];
|
||||
|
||||
/// All merge changes in this set.
|
||||
Iterable<CrdtMergeChange> get changes sync* {
|
||||
yield* inserts;
|
||||
yield* updates;
|
||||
yield* deletes;
|
||||
while (await moveNext()) {
|
||||
switch (current) {
|
||||
case CrdtSyncMergeChunk(changes: final changes):
|
||||
mergeSet.addAll(changes);
|
||||
case CrdtSyncIdleTimeout():
|
||||
if (mergeSet.isEmpty) return mergeSet;
|
||||
case CrdtSyncEndOfBatch():
|
||||
return mergeSet;
|
||||
default:
|
||||
throw CrdtSyncUnexpectedEventException(
|
||||
expected: '"CrdtSyncMergeChunk" or "CrdtSyncEndOfBatch"',
|
||||
received: current,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (mergeSet.isEmpty && allowCloseBeforeBatch) return null;
|
||||
throw const CrdtSyncStreamClosedException(phase: 'end-of-batch');
|
||||
}
|
||||
|
||||
/// Moves the iterator to the next event and throws if the stream is closed or
|
||||
/// the next event is not of type [T].
|
||||
Future<T> moveAndThrowIfNot<T extends CrdtSyncStreamEvent>() async {
|
||||
while (await moveNext()) {
|
||||
if (current is CrdtSyncIdleTimeout) continue;
|
||||
if (current is T) return current as T;
|
||||
throw CrdtSyncUnexpectedEventException(
|
||||
expected: '"$T"',
|
||||
received: current,
|
||||
);
|
||||
}
|
||||
throw CrdtSyncStreamClosedException(phase: '"$T"');
|
||||
}
|
||||
}
|
||||
|
||||
/// Helpers for grouping merge changes into stream payload batches.
|
||||
extension CrdtMergeChangeStreamExtension on Stream<CrdtMergeChange> {
|
||||
/// Emits lists with at most [batchSize] changes from this stream.
|
||||
Stream<CrdtMergeSet> chunked(int batchSize) async* {
|
||||
var batch = <CrdtMergeChange>[];
|
||||
await for (final change in this) {
|
||||
batch.add(change);
|
||||
if (batch.length < batchSize) continue;
|
||||
yield batch;
|
||||
batch = <CrdtMergeChange>[];
|
||||
}
|
||||
if (batch.isNotEmpty) yield batch;
|
||||
}
|
||||
}
|
||||
|
||||
/// CRDT merge helpers for [CrdtMergeSet].
|
||||
extension CrdtMergeSetExtension on CrdtMergeSet {
|
||||
/// The greatest HLC represented by the changes in this merge set.
|
||||
Hlc? get maxHlc => fold<Hlc?>(
|
||||
null,
|
||||
(current, change) => change.hlc.maxBetween(current),
|
||||
);
|
||||
|
||||
/// Insert changes in this set.
|
||||
Iterable<CrdtMergeInsert> get inserts => whereType<CrdtMergeInsert>();
|
||||
|
||||
/// Update changes in this set.
|
||||
Iterable<CrdtMergeUpdate> get updates => whereType<CrdtMergeUpdate>();
|
||||
|
||||
/// Delete changes in this set.
|
||||
Iterable<CrdtMergeDelete> get deletes => whereType<CrdtMergeDelete>();
|
||||
|
||||
/// All merge changes in this set sorted by causal order.
|
||||
List<CrdtMergeChange> get causallyOrderedChanges {
|
||||
final operations = changes.toList()
|
||||
final operations = List<CrdtMergeChange>.from(this)
|
||||
..sort((left, right) => left.hlc.compareTo(right.hlc));
|
||||
return operations;
|
||||
}
|
||||
|
|
@ -36,7 +116,7 @@ extension CrdtMergeSetExtension on CrdtMergeSet {
|
|||
final rowIdsToLoadByTable = <String, Set<UuidValue>>{};
|
||||
final columnNamesToLoadByTable = <String, Set<String>>{};
|
||||
|
||||
for (final change in changes) {
|
||||
for (final change in this) {
|
||||
final availableColumnNames = columnNamesByTableName[change.tableName];
|
||||
if (availableColumnNames == null) continue;
|
||||
|
||||
|
|
|
|||
626
packages/serverpod_offline_sync_client/lib/src/crdt/sync.dart
Normal file
626
packages/serverpod_offline_sync_client/lib/src/crdt/sync.dart
Normal file
|
|
@ -0,0 +1,626 @@
|
|||
import 'dart:async';
|
||||
|
||||
import 'package:serverpod_database/serverpod_database.dart';
|
||||
import 'package:serverpod_offline_sync_shared/serverpod_offline_sync_shared.dart';
|
||||
import 'package:uuid/uuid.dart';
|
||||
|
||||
import '../database/database.dart';
|
||||
import '../managers/user.dart';
|
||||
import '../protocol/protocol.dart';
|
||||
import 'exceptions.dart';
|
||||
import 'merge.dart';
|
||||
|
||||
/// Callback function for when a merge is successful.
|
||||
typedef CrdtSyncOnMergeSuccess = FutureOr<void> Function(Hlc syncedHlc);
|
||||
|
||||
/// The shared CRDT synchronization logic used by both client and server nodes.
|
||||
class CrdtSync {
|
||||
/// Creates a new [CrdtSync] instance.
|
||||
CrdtSync({
|
||||
/// The list of tables to sync with CRDT.
|
||||
required List<Table> syncTables,
|
||||
|
||||
/// The serialization manager to use for deserializing merge changes.
|
||||
required DatabaseSerializationManager serializationManager,
|
||||
|
||||
/// Maximum number of merge changes sent in one sync stream message.
|
||||
int syncBatchSize = defaultSyncBatchSize,
|
||||
|
||||
/// Delay between continuous sync rounds.
|
||||
Duration continuousSyncInterval = defaultContinuousSyncInterval,
|
||||
}) : _syncTables = syncTables,
|
||||
_serializationManager = serializationManager,
|
||||
_syncBatchSize = syncBatchSize,
|
||||
_continuousSyncInterval = continuousSyncInterval {
|
||||
if (syncBatchSize < 1) {
|
||||
throw ArgumentError.value(syncBatchSize, 'syncBatchSize', 'Must be >= 1');
|
||||
}
|
||||
}
|
||||
|
||||
/// Default maximum number of merge changes sent in one stream message.
|
||||
static const defaultSyncBatchSize = 100;
|
||||
|
||||
/// Default delay between continuous sync rounds.
|
||||
static const defaultContinuousSyncInterval = Duration(milliseconds: 200);
|
||||
|
||||
final List<Table> _syncTables;
|
||||
final DatabaseSerializationManager _serializationManager;
|
||||
final int _syncBatchSize;
|
||||
final Duration _continuousSyncInterval;
|
||||
|
||||
static CrdtSync? _instance;
|
||||
|
||||
/// The singleton instance of [CrdtSync]. Throws a [StateError] if it is not
|
||||
/// initialized.
|
||||
static CrdtSync get instance =>
|
||||
_instance ??
|
||||
(throw StateError(
|
||||
'The CrdtSync has not been initialized. Call pod.initializeCrdtSync(...) '
|
||||
'during server startup to configure the CRDT sync.',
|
||||
));
|
||||
|
||||
/// Configures the shared singleton used by server endpoints.
|
||||
///
|
||||
/// Use [syncBatchSize] to control the maximum number of merge changes carried
|
||||
/// by each [CrdtSyncMergeChunk] stream event.
|
||||
///
|
||||
/// The [continuousSyncInterval] controls how long a continuous sync session
|
||||
/// waits after completing one sync round before checking for local changes.
|
||||
static void initialize({
|
||||
required List<Table> syncTables,
|
||||
required DatabaseSerializationManager serializationManager,
|
||||
int syncBatchSize = defaultSyncBatchSize,
|
||||
Duration continuousSyncInterval = defaultContinuousSyncInterval,
|
||||
}) {
|
||||
_instance = CrdtSync(
|
||||
syncTables: syncTables,
|
||||
serializationManager: serializationManager,
|
||||
syncBatchSize: syncBatchSize,
|
||||
continuousSyncInterval: continuousSyncInterval,
|
||||
);
|
||||
}
|
||||
|
||||
late final Map<String, Table> _syncTablesByName = {
|
||||
for (final table in _syncTables) table.tableName: table,
|
||||
};
|
||||
|
||||
late final Map<String, String> _classNamesByTableName = {
|
||||
for (final definition in _serializationManager.getTargetTableDefinitions())
|
||||
if (definition.dartName != null) definition.name: definition.dartName!,
|
||||
};
|
||||
|
||||
late final Map<String, Map<String, ColumnDefinition>> _columnDefinitionsByTableName =
|
||||
{
|
||||
for (final definition in _serializationManager.getTargetTableDefinitions())
|
||||
definition.name: {
|
||||
for (final column in definition.columns) column.name: column,
|
||||
},
|
||||
};
|
||||
|
||||
/// The deterministic hash representing the current synchronized schema.
|
||||
late final String currentSyncTablesHash = computeSyncTablesHash(
|
||||
_syncTables,
|
||||
tableDefinitions: _serializationManager.getTargetTableDefinitions(),
|
||||
);
|
||||
|
||||
/// Computes a deterministic fixed-size hash of the synchronized schema.
|
||||
///
|
||||
/// The [tableDefinitions] is the list of all table definitions in the
|
||||
/// database, which must include the definitions for all [syncTables].
|
||||
static String computeSyncTablesHash(
|
||||
List<Table> syncTables, {
|
||||
required List<TableDefinition> tableDefinitions,
|
||||
}) {
|
||||
final canonicalSignature = _computeCanonicalSyncTablesSignature(
|
||||
syncTables,
|
||||
tableDefinitions: tableDefinitions,
|
||||
);
|
||||
// Use two deterministic namespace-based UUIDv5 hashes to keep the payload
|
||||
// fixed-size while substantially reducing the practical collision risk.
|
||||
const uuid = Uuid();
|
||||
return '${uuid.v5(Namespace.url.value, canonicalSignature)}:'
|
||||
'${uuid.v5(Namespace.oid.value, canonicalSignature)}';
|
||||
}
|
||||
|
||||
/// Streams local CRDT changes since the last sync checkpoint with [otherNodeId].
|
||||
///
|
||||
/// Changes are emitted in insert, update, then delete order. Domain row and
|
||||
/// column payloads are resolved incrementally as each change is yielded.
|
||||
///
|
||||
/// When [sinceHlc] is provided, it is used instead of the stored checkpoint.
|
||||
Stream<CrdtMergeChange> collectPendingChanges(
|
||||
DatabaseSession session, {
|
||||
required UuidValue userId,
|
||||
required UuidValue otherNodeId,
|
||||
Hlc? sinceHlc,
|
||||
}) async* {
|
||||
final crdtUser = await CrdtUserManager(session).getOrCreate(userId);
|
||||
final syncCheckpoint =
|
||||
sinceHlc ??
|
||||
await _syncCheckpointForNode(
|
||||
session,
|
||||
crdtUser,
|
||||
otherNodeId,
|
||||
);
|
||||
|
||||
yield* _streamInserts(session, crdtUser, syncCheckpoint);
|
||||
yield* _streamUpdates(session, crdtUser, syncCheckpoint);
|
||||
yield* _streamDeletes(session, crdtUser, syncCheckpoint);
|
||||
}
|
||||
|
||||
/// Creates the [CrdtSyncSinceHlc] checkpoint for the post-connect handshake.
|
||||
///
|
||||
/// [CrdtSyncSinceHlc.lastSyncFromPeer] reflects the latest change this node
|
||||
/// has received from [peerNodeId], tagged with the peer's node id.
|
||||
Future<CrdtSyncSinceHlc> createSyncSinceHlc(
|
||||
DatabaseSession session, {
|
||||
required UuidValue userId,
|
||||
required UuidValue peerNodeId,
|
||||
}) async {
|
||||
final crdtUser = await CrdtUserManager(session).getOrCreate(userId);
|
||||
final lastSeenHlc = await _syncCheckpointForNode(session, crdtUser, peerNodeId);
|
||||
return CrdtSyncSinceHlc(lastSyncFromPeer: lastSeenHlc);
|
||||
}
|
||||
|
||||
/// Merges a remote [mergeSet] and records the sync checkpoint for [otherNodeId].
|
||||
///
|
||||
/// Throws if the merge fails. The sync stream should be closed so the next
|
||||
/// attempt resumes from the last persisted checkpoint.
|
||||
///
|
||||
/// Returns the greatest HLC synced in the batch, or `null` if the batch is
|
||||
/// empty.
|
||||
Future<Hlc?> mergeInboundBatch(
|
||||
DatabaseSession session, {
|
||||
required UuidValue userId,
|
||||
required UuidValue otherNodeId,
|
||||
required CrdtMergeSet mergeSet,
|
||||
}) async {
|
||||
if (mergeSet.isEmpty) return null;
|
||||
final maxSyncedHlc = mergeSet.maxHlc;
|
||||
final crdtDb = await _openCrdtDatabase(session);
|
||||
await crdtDb.mergeChanges(mergeSet, userId: userId);
|
||||
if (maxSyncedHlc != null) {
|
||||
await crdtDb.recordSyncCheckpoint(otherNodeId, maxSyncedHlc, userId: userId);
|
||||
}
|
||||
return maxSyncedHlc;
|
||||
}
|
||||
|
||||
/// Streams a framed outbound sync batch of merge changes.
|
||||
Stream<CrdtSyncStreamEvent> streamOutboundBatch(
|
||||
DatabaseSession session, {
|
||||
required UuidValue userId,
|
||||
required UuidValue peerNodeId,
|
||||
required Hlc sinceHlc,
|
||||
}) async* {
|
||||
yield* collectPendingChanges(
|
||||
session,
|
||||
userId: userId,
|
||||
otherNodeId: peerNodeId,
|
||||
sinceHlc: sinceHlc,
|
||||
)
|
||||
.chunked(_syncBatchSize)
|
||||
.map(
|
||||
(changes) => CrdtSyncMergeChunk(changes: changes),
|
||||
);
|
||||
yield CrdtSyncEndOfBatch();
|
||||
}
|
||||
|
||||
/// Runs a symmetric CRDT sync session over a bidirectional event stream.
|
||||
///
|
||||
/// Both peers send [CrdtSyncConnect], read the peer connect frame, then send
|
||||
/// and read [CrdtSyncSinceHlc] once. After that, both sides exchange framed
|
||||
/// data batches using (1) the peer's authoritative since HLC for the first
|
||||
/// send, and (2) [CrdtSyncSinceHlc.lastSyncFromPeer] for subsequent sends
|
||||
/// within the session.
|
||||
///
|
||||
/// If a batch fails to merge, the exception propagates and the stream closes.
|
||||
/// The next sync attempt resumes from the last persisted checkpoint on the
|
||||
/// side that failed to merge.
|
||||
///
|
||||
/// When [once] is true, each peer sends one batch, reads one batch, sends a
|
||||
/// [CrdtSyncClose] frame and awaits for the other to send its closing event
|
||||
/// to complete the session.
|
||||
Stream<CrdtSyncStreamEvent> sync(
|
||||
DatabaseSession session, {
|
||||
required UuidValue userId,
|
||||
required Stream<CrdtSyncStreamEvent> inbound,
|
||||
bool once = false,
|
||||
CrdtSyncOnMergeSuccess? onMergeSuccess,
|
||||
}) async* {
|
||||
final inboundIterator = StreamIterator(
|
||||
// Inject a timeout event if the inbound stream is idle for too long to
|
||||
// allow returning an empty batch and waiting for a change without closing
|
||||
// the stream.
|
||||
inbound.timeout(
|
||||
const Duration(seconds: 1),
|
||||
onTimeout: (sink) => sink.add(CrdtSyncIdleTimeout()),
|
||||
),
|
||||
);
|
||||
|
||||
var sessionCompleted = false;
|
||||
try {
|
||||
final crdtUser = await CrdtUserManager(session).getOrCreate(userId);
|
||||
final localNodeId = crdtUser.currentNode!.uuidNodeId;
|
||||
|
||||
yield CrdtSyncConnect(
|
||||
localNodeId: localNodeId,
|
||||
syncTablesHash: currentSyncTablesHash,
|
||||
);
|
||||
|
||||
final peerConnect = await inboundIterator.moveAndThrowIfNot<CrdtSyncConnect>();
|
||||
final peerNodeId = peerConnect.localNodeId;
|
||||
_validateSyncTablesHash(peerConnect.syncTablesHash);
|
||||
|
||||
yield await createSyncSinceHlc(
|
||||
session,
|
||||
userId: userId,
|
||||
peerNodeId: peerNodeId,
|
||||
);
|
||||
|
||||
final peerSinceHlc = await inboundIterator.moveAndThrowIfNot<CrdtSyncSinceHlc>();
|
||||
if (peerSinceHlc.lastSyncFromPeer.nodeId != localNodeId) {
|
||||
throw CrdtSyncInvalidSinceHlcException(
|
||||
receivedNodeId: peerSinceHlc.lastSyncFromPeer.nodeId,
|
||||
expectedNodeId: localNodeId,
|
||||
);
|
||||
}
|
||||
|
||||
var lastSentToPeer = peerSinceHlc.lastSyncFromPeer;
|
||||
|
||||
while (true) {
|
||||
final pendingLocalChanges = collectPendingChanges(
|
||||
session,
|
||||
userId: userId,
|
||||
otherNodeId: peerNodeId,
|
||||
sinceHlc: lastSentToPeer,
|
||||
);
|
||||
|
||||
var hasChanges = false;
|
||||
await for (final changes in pendingLocalChanges.chunked(_syncBatchSize)) {
|
||||
hasChanges = true;
|
||||
lastSentToPeer = lastSentToPeer.maxBetween(changes.maxHlc);
|
||||
yield CrdtSyncMergeChunk(changes: changes);
|
||||
}
|
||||
if (hasChanges || once) {
|
||||
yield CrdtSyncEndOfBatch();
|
||||
}
|
||||
|
||||
final inboundMergeSet = await inboundIterator.collectNextBatch(
|
||||
allowCloseBeforeBatch: !once,
|
||||
);
|
||||
if (inboundMergeSet == null) {
|
||||
sessionCompleted = true;
|
||||
return;
|
||||
}
|
||||
|
||||
final lastReceivedFromPeer = await mergeInboundBatch(
|
||||
session,
|
||||
userId: userId,
|
||||
otherNodeId: peerNodeId,
|
||||
mergeSet: inboundMergeSet,
|
||||
);
|
||||
|
||||
if (hasChanges || inboundMergeSet.isNotEmpty) {
|
||||
await onMergeSuccess?.call(
|
||||
lastSentToPeer.maxBetween(lastReceivedFromPeer),
|
||||
);
|
||||
}
|
||||
|
||||
if (once) {
|
||||
yield CrdtSyncClose();
|
||||
await inboundIterator.moveAndThrowIfNot<CrdtSyncClose>();
|
||||
sessionCompleted = true;
|
||||
return;
|
||||
}
|
||||
|
||||
// Wait for the configured interval before checking for local changes again.
|
||||
await Future<void>.delayed(_continuousSyncInterval);
|
||||
}
|
||||
} finally {
|
||||
// Cancelling inbound on normal completion races with WebSocket stream
|
||||
// teardown and produces "connection closed" errors on the peer. Keep
|
||||
// cleanup for abnormal exits so listener cancellation can unblock.
|
||||
if (!sessionCompleted) {
|
||||
// Best-effort cleanup, since the transport will close the socket anyway.
|
||||
const waitTimeout = Duration(milliseconds: 200);
|
||||
await inboundIterator.cancel().timeout(waitTimeout, onTimeout: () {});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void _validateSyncTablesHash(String syncTablesHash) {
|
||||
if (syncTablesHash != currentSyncTablesHash) {
|
||||
throw SyncTablesHashMismatchException(
|
||||
received: syncTablesHash,
|
||||
expected: currentSyncTablesHash,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
Future<CrdtDatabase> _openCrdtDatabase(DatabaseSession session) async {
|
||||
final db = session.db;
|
||||
if (db is CrdtDatabase) {
|
||||
return db;
|
||||
}
|
||||
|
||||
final crdtDb = CrdtDatabase(db, syncTables: _syncTables);
|
||||
await crdtDb.initialize();
|
||||
return crdtDb;
|
||||
}
|
||||
|
||||
Future<Hlc> _syncCheckpointForNode(
|
||||
DatabaseSession session,
|
||||
CrdtUser crdtUser,
|
||||
UuidValue otherNodeId,
|
||||
) async {
|
||||
final otherNode = await CrdtNode.db.findFirstRow(
|
||||
session,
|
||||
where: (t) => t.userId.equals(crdtUser.id) & t.uuidNodeId.equals(otherNodeId),
|
||||
);
|
||||
|
||||
return otherNode?.lastReceivedHlc ?? Hlc.zero(otherNodeId);
|
||||
}
|
||||
|
||||
Stream<CrdtMergeInsert> _streamInserts(
|
||||
DatabaseSession session,
|
||||
CrdtUser crdtUser,
|
||||
Hlc lastSyncHlc,
|
||||
) async* {
|
||||
final rows = await CrdtDataRow.db.find(
|
||||
session,
|
||||
where: (t) => _rowHlcAfterFilter(t, crdtUser, lastSyncHlc),
|
||||
include: CrdtDataRow.include(
|
||||
tbl: CrdtSchemaTable.include(),
|
||||
node: CrdtNode.include(),
|
||||
),
|
||||
);
|
||||
|
||||
for (final row in rows) {
|
||||
final tableName = row.tbl!.name;
|
||||
if (!_syncTablesByName.containsKey(tableName)) continue;
|
||||
|
||||
final table = _syncTablesByName[tableName]!;
|
||||
final dartName = _classNamesByTableName[tableName];
|
||||
if (dartName == null) continue;
|
||||
|
||||
final domainRow = await _fetchDomainRow(
|
||||
session,
|
||||
tableName,
|
||||
row.uuidRowId,
|
||||
table,
|
||||
dartName,
|
||||
);
|
||||
if (domainRow == null) continue;
|
||||
|
||||
yield CrdtMergeInsert(
|
||||
hlcDatetime: row.hlcDatetime,
|
||||
hlcCounter: row.hlcCounter,
|
||||
tableName: tableName,
|
||||
uuidRowId: row.uuidRowId,
|
||||
uuidNodeId: row.node!.uuidNodeId,
|
||||
data: domainRow,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
Stream<CrdtMergeUpdate> _streamUpdates(
|
||||
DatabaseSession session,
|
||||
CrdtUser crdtUser,
|
||||
Hlc lastSyncHlc,
|
||||
) async* {
|
||||
final fields = await CrdtDataField.db.find(
|
||||
session,
|
||||
where: (t) => _fieldHlcAfterFilter(t, crdtUser, lastSyncHlc),
|
||||
include: CrdtDataField.include(
|
||||
row: CrdtDataRow.include(tbl: CrdtSchemaTable.include()),
|
||||
column: CrdtSchemaColumn.include(),
|
||||
node: CrdtNode.include(),
|
||||
),
|
||||
);
|
||||
|
||||
for (final field in fields) {
|
||||
final tableName = field.row!.tbl!.name;
|
||||
if (!_syncTablesByName.containsKey(tableName)) continue;
|
||||
|
||||
final columnName = field.column!.name;
|
||||
final decodedValue = await _fetchColumnValue(
|
||||
session,
|
||||
tableName,
|
||||
field.row!.uuidRowId,
|
||||
columnName,
|
||||
);
|
||||
|
||||
yield CrdtMergeUpdate(
|
||||
hlcDatetime: field.hlcDatetime,
|
||||
hlcCounter: field.hlcCounter,
|
||||
tableName: tableName,
|
||||
uuidRowId: field.row!.uuidRowId,
|
||||
uuidNodeId: field.node!.uuidNodeId,
|
||||
columnName: columnName,
|
||||
value: decodedValue,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
Stream<CrdtMergeDelete> _streamDeletes(
|
||||
DatabaseSession session,
|
||||
CrdtUser crdtUser,
|
||||
Hlc lastSyncHlc,
|
||||
) async* {
|
||||
final tombstones = await CrdtDataDeleted.db.find(
|
||||
session,
|
||||
where: (t) => _tombstoneHlcAfterFilter(t, crdtUser, lastSyncHlc),
|
||||
include: CrdtDataDeleted.include(
|
||||
row: CrdtDataRow.include(tbl: CrdtSchemaTable.include()),
|
||||
node: CrdtNode.include(),
|
||||
),
|
||||
);
|
||||
|
||||
for (final tombstone in tombstones) {
|
||||
final tableName = tombstone.row!.tbl!.name;
|
||||
if (!_syncTablesByName.containsKey(tableName)) continue;
|
||||
|
||||
yield CrdtMergeDelete(
|
||||
hlcDatetime: tombstone.hlcDatetime,
|
||||
hlcCounter: tombstone.hlcCounter,
|
||||
tableName: tableName,
|
||||
uuidRowId: tombstone.row!.uuidRowId,
|
||||
uuidNodeId: tombstone.node!.uuidNodeId,
|
||||
isDeleted: tombstone.isDeleted,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
Expression _rowHlcAfterFilter(
|
||||
CrdtDataRowTable t,
|
||||
CrdtUser crdtUser,
|
||||
Hlc lastSyncHlc,
|
||||
) =>
|
||||
t.userId.equals(crdtUser.id) &
|
||||
((t.hlcDatetime > lastSyncHlc.datetime) |
|
||||
(t.hlcDatetime.equals(lastSyncHlc.datetime) &
|
||||
(t.hlcCounter > lastSyncHlc.counter)));
|
||||
|
||||
Expression _fieldHlcAfterFilter(
|
||||
CrdtDataFieldTable t,
|
||||
CrdtUser crdtUser,
|
||||
Hlc lastSyncHlc,
|
||||
) =>
|
||||
t.row.userId.equals(crdtUser.id) &
|
||||
((t.hlcDatetime > lastSyncHlc.datetime) |
|
||||
(t.hlcDatetime.equals(lastSyncHlc.datetime) &
|
||||
(t.hlcCounter > lastSyncHlc.counter)));
|
||||
|
||||
Expression _tombstoneHlcAfterFilter(
|
||||
CrdtDataDeletedTable t,
|
||||
CrdtUser crdtUser,
|
||||
Hlc lastSyncHlc,
|
||||
) =>
|
||||
t.row.userId.equals(crdtUser.id) &
|
||||
((t.hlcDatetime > lastSyncHlc.datetime) |
|
||||
(t.hlcDatetime.equals(lastSyncHlc.datetime) &
|
||||
(t.hlcCounter > lastSyncHlc.counter)));
|
||||
|
||||
Future<dynamic> _fetchDomainRow(
|
||||
DatabaseSession session,
|
||||
String tableName,
|
||||
UuidValue rowId,
|
||||
Table table,
|
||||
String dartName,
|
||||
) async {
|
||||
final cols = table.columns.map((column) => '"${column.columnName}"').join(', ');
|
||||
final encodedRowId = ValueEncoder.instance.convert(rowId);
|
||||
final result = await session.db.unsafeQuery(
|
||||
'SELECT $cols FROM "$tableName" WHERE "id" = $encodedRowId',
|
||||
);
|
||||
if (result.isEmpty) return null;
|
||||
|
||||
final columnMap = result.first.toColumnMap();
|
||||
return session.db.serializationManager.deserializeByClassName({
|
||||
'className': dartName,
|
||||
'data': columnMap,
|
||||
});
|
||||
}
|
||||
|
||||
Future<dynamic> _fetchColumnValue(
|
||||
DatabaseSession session,
|
||||
String tableName,
|
||||
UuidValue rowId,
|
||||
String columnName,
|
||||
) async {
|
||||
final encodedValue = ValueEncoder.instance.convert(rowId);
|
||||
final result = await session.db.unsafeQuery(
|
||||
'SELECT "$columnName" FROM "$tableName" WHERE "id" = $encodedValue',
|
||||
);
|
||||
if (result.isEmpty) return null;
|
||||
return _decodeColumnValue(tableName, columnName, result.first[0]);
|
||||
}
|
||||
|
||||
dynamic _decodeColumnValue(String tableName, String columnName, Object? value) {
|
||||
if (value == null) return null;
|
||||
|
||||
final definition = _columnDefinitionsByTableName[tableName]?[columnName];
|
||||
final dartType = definition?.dartType;
|
||||
if (dartType == null) return value;
|
||||
|
||||
final className = _classNameForDartType(dartType);
|
||||
return switch (className) {
|
||||
'bool' || 'double' || 'int' || 'String' => value,
|
||||
_ => _serializationManager.deserializeByClassName({
|
||||
'className': className,
|
||||
'data': value,
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
String _classNameForDartType(String dartType) {
|
||||
final withoutNullable = dartType.endsWith('?')
|
||||
? dartType.substring(0, dartType.length - 1)
|
||||
: dartType;
|
||||
return withoutNullable.split(':').last;
|
||||
}
|
||||
|
||||
static String _computeCanonicalSyncTablesSignature(
|
||||
List<Table> syncTables, {
|
||||
required List<TableDefinition> tableDefinitions,
|
||||
}) {
|
||||
final tableDefinitionsByName = {
|
||||
for (final definition in tableDefinitions) definition.name: definition,
|
||||
};
|
||||
|
||||
final sortedTables = syncTables.toList()
|
||||
..sort((left, right) => left.tableName.compareTo(right.tableName));
|
||||
|
||||
return sortedTables
|
||||
.map((table) {
|
||||
final definition = tableDefinitionsByName[table.tableName];
|
||||
final columns = [
|
||||
for (final column in definition?.columns ?? const <ColumnDefinition>[])
|
||||
column.name,
|
||||
if (definition == null) ...table.columns.map((column) => column.columnName),
|
||||
]..sort();
|
||||
final foreignKeys = _canonicalForeignKeys(definition);
|
||||
final uniqueIndexes = _canonicalUniqueIndexes(definition);
|
||||
return '${table.tableName}:'
|
||||
'${columns.join(',')}|'
|
||||
'fk[${foreignKeys.join(';')}]|'
|
||||
'uq[${uniqueIndexes.join(';')}]';
|
||||
})
|
||||
.join(';');
|
||||
}
|
||||
|
||||
static List<String> _canonicalForeignKeys(TableDefinition? definition) {
|
||||
if (definition == null) return const [];
|
||||
final entries = <String>[
|
||||
for (final fk in definition.foreignKeys)
|
||||
// Each foreign key must map all parameters.
|
||||
// ignore: no_adjacent_strings_in_list
|
||||
'${(fk.columns.toList()..sort()).join(',')}->'
|
||||
'${fk.referenceTableSchema}.${fk.referenceTable}'
|
||||
'(${(fk.referenceColumns.toList()..sort()).join(',')})'
|
||||
'|u:${fk.onUpdate?.toString() ?? '-'}'
|
||||
'|d:${fk.onDelete?.toString() ?? '-'}'
|
||||
'|m:${fk.matchType?.toString() ?? '-'}',
|
||||
]..sort();
|
||||
|
||||
return entries;
|
||||
}
|
||||
|
||||
static List<String> _canonicalUniqueIndexes(TableDefinition? definition) {
|
||||
if (definition == null) return const [];
|
||||
|
||||
final entries = <String>[
|
||||
for (final index in definition.indexes)
|
||||
if (index.isUnique && !index.isPrimary)
|
||||
() {
|
||||
final sortedElements = [
|
||||
for (final element in index.elements)
|
||||
'${element.type}:${element.definition}',
|
||||
]..sort();
|
||||
return sortedElements.join(',');
|
||||
}(),
|
||||
]..sort();
|
||||
|
||||
return entries;
|
||||
}
|
||||
}
|
||||
|
|
@ -3,10 +3,11 @@
|
|||
// ignore_for_file: invalid_use_of_internal_member
|
||||
|
||||
import 'package:serverpod_database/serverpod_database.dart';
|
||||
import 'package:serverpod_offline_sync_shared/serverpod_offline_sync_shared.dart';
|
||||
import 'package:uuid/uuid.dart';
|
||||
|
||||
import '../crdt/merge.dart';
|
||||
import '../managers/user.dart';
|
||||
import '../crdt/sync.dart';
|
||||
import '../protocol/protocol.dart';
|
||||
import 'recorder.dart';
|
||||
import 'session.dart';
|
||||
|
|
@ -25,25 +26,85 @@ class CrdtDatabase implements Database {
|
|||
/// The list of tables to sync with CRDT.
|
||||
required List<Table> syncTables,
|
||||
|
||||
/// Maximum number of merge changes sent in one sync stream message.
|
||||
int syncBatchSize = CrdtSync.defaultSyncBatchSize,
|
||||
|
||||
/// Delay between continuous sync rounds.
|
||||
Duration continuousSyncInterval = CrdtSync.defaultContinuousSyncInterval,
|
||||
|
||||
/// The user ID to use for all CRDT operations. This should only be used for
|
||||
/// databases operating on the client side, where all data is for the same user.
|
||||
/// Otherwise, the user ID must be passed through the transaction.
|
||||
UuidValue? persistentUserId,
|
||||
}) : _recorder = CrdtMutationRecorder(
|
||||
}) : _syncTables = syncTables,
|
||||
_syncBatchSize = syncBatchSize,
|
||||
_continuousSyncInterval = continuousSyncInterval,
|
||||
_recorder = CrdtMutationRecorder(
|
||||
_delegate,
|
||||
persistentUserId: persistentUserId,
|
||||
syncTables: syncTables,
|
||||
);
|
||||
|
||||
final Database _delegate;
|
||||
final List<Table> _syncTables;
|
||||
final int _syncBatchSize;
|
||||
final Duration _continuousSyncInterval;
|
||||
|
||||
final CrdtMutationRecorder _recorder;
|
||||
|
||||
late final CrdtSync _sync = CrdtSync(
|
||||
syncTables: _syncTables,
|
||||
serializationManager: serializationManager,
|
||||
syncBatchSize: _syncBatchSize,
|
||||
continuousSyncInterval: _continuousSyncInterval,
|
||||
);
|
||||
|
||||
/// Initializes the CRDT database.
|
||||
Future<void> initialize() async {
|
||||
await _recorder.initialize();
|
||||
}
|
||||
|
||||
/// The hash describing the synchronized schema configured for this database.
|
||||
String get syncTablesHash => _sync.currentSyncTablesHash;
|
||||
|
||||
/// Returns the current node identifier for the effective user.
|
||||
Future<UuidValue> currentNodeId({UuidValue? userId}) async {
|
||||
final effectiveUserId = await _requireUserId(userId);
|
||||
final user = await _recorder.getOrCreateUser(effectiveUserId);
|
||||
return user.currentNode!.uuidNodeId;
|
||||
}
|
||||
|
||||
/// Runs a symmetric CRDT sync session over a bidirectional event stream.
|
||||
Stream<CrdtSyncStreamEvent> sync({
|
||||
required Stream<CrdtSyncStreamEvent> inbound,
|
||||
UuidValue? userId,
|
||||
bool once = false,
|
||||
CrdtSyncOnMergeSuccess? onMergeSuccess,
|
||||
}) async* {
|
||||
final effectiveUserId = await _requireUserId(userId);
|
||||
yield* _sync.sync(
|
||||
_delegate.session,
|
||||
userId: effectiveUserId,
|
||||
inbound: inbound,
|
||||
once: once,
|
||||
onMergeSuccess: onMergeSuccess,
|
||||
);
|
||||
}
|
||||
|
||||
/// Records the latest acknowledged sync checkpoint for [otherNodeId].
|
||||
Future<void> recordSyncCheckpoint(
|
||||
UuidValue otherNodeId,
|
||||
Hlc syncedHlc, {
|
||||
UuidValue? userId,
|
||||
}) async {
|
||||
final effectiveUserId = await _requireUserId(userId);
|
||||
await _recorder.recordSyncCheckpoint(
|
||||
effectiveUserId,
|
||||
otherNodeId,
|
||||
syncedHlc,
|
||||
);
|
||||
}
|
||||
|
||||
/// Merges remote CRDT changes into the local database for the given user.
|
||||
///
|
||||
/// When [userId] is omitted, this uses the recorder's persistent user id.
|
||||
|
|
@ -200,6 +261,30 @@ class CrdtDatabase implements Database {
|
|||
);
|
||||
}
|
||||
|
||||
@override
|
||||
Future<List<T>> upsert<T extends TableRow>(
|
||||
List<T> rows, {
|
||||
required List<Column> conflictColumns,
|
||||
List<Column>? updateColumns,
|
||||
Expression? updateWhere,
|
||||
Transaction? transaction,
|
||||
}) async {
|
||||
// TODO: Implement CRDT upsert.
|
||||
throw UnimplementedError('CRDT upsert is not implemented.');
|
||||
}
|
||||
|
||||
@override
|
||||
Future<T?> upsertRow<T extends TableRow>(
|
||||
T row, {
|
||||
required List<Column> conflictColumns,
|
||||
List<Column>? updateColumns,
|
||||
Expression? updateWhere,
|
||||
Transaction? transaction,
|
||||
}) async {
|
||||
// TODO: Implement CRDT upsert.
|
||||
throw UnimplementedError('CRDT upsert is not implemented.');
|
||||
}
|
||||
|
||||
@override
|
||||
Future<List<T>> update<T extends TableRow>(
|
||||
List<T> rows, {
|
||||
|
|
@ -433,7 +518,7 @@ class CrdtDatabase implements Database {
|
|||
TransactionSettings? settings,
|
||||
}) async {
|
||||
// Ensure that the user exists with a node before starting the transaction.
|
||||
final user = await CrdtUserManager.getOrCreate(_delegate.session, userId);
|
||||
final user = await _recorder.getOrCreateUser(userId);
|
||||
|
||||
return transaction<R>(
|
||||
(tx) async {
|
||||
|
|
@ -448,6 +533,14 @@ class CrdtDatabase implements Database {
|
|||
);
|
||||
}
|
||||
|
||||
Future<UuidValue> _requireUserId(UuidValue? userId) async {
|
||||
return userId ??
|
||||
_recorder.persistentUserId ??
|
||||
(throw StateError(
|
||||
'A user ID is required when syncing without a persistent user.',
|
||||
));
|
||||
}
|
||||
|
||||
@override
|
||||
Future<int> unsafeExecute(
|
||||
String query, {
|
||||
|
|
|
|||
|
|
@ -69,18 +69,24 @@ extension CrdtMergeRecorderExtension on CrdtMutationRecorder {
|
|||
}
|
||||
}
|
||||
|
||||
_updateHlcFromIncomingOperations(operations, transaction);
|
||||
await _updateHlcFromIncomingOperations(
|
||||
operations,
|
||||
remoteNodes,
|
||||
transaction,
|
||||
);
|
||||
}
|
||||
|
||||
void _updateHlcFromIncomingOperations(
|
||||
Future<void> _updateHlcFromIncomingOperations(
|
||||
List<CrdtMergeChange> operations,
|
||||
Map<UuidValue, CrdtNode> remoteNodes,
|
||||
Transaction transaction,
|
||||
) {
|
||||
) async {
|
||||
final maxIncomingHlc = operations.fold<Hlc?>(
|
||||
null,
|
||||
(current, change) =>
|
||||
current == null || change.hlc > current ? change.hlc : current,
|
||||
(current, change) => change.hlc.maxBetween(current),
|
||||
);
|
||||
final nodesToUpdate = <CrdtNode>[];
|
||||
|
||||
if (maxIncomingHlc != null) {
|
||||
final hlcManager = _getHlcManager(transaction);
|
||||
if (maxIncomingHlc.nodeId == hlcManager.uuidNodeId) {
|
||||
|
|
@ -90,6 +96,37 @@ extension CrdtMergeRecorderExtension on CrdtMutationRecorder {
|
|||
} else {
|
||||
hlcManager.merge(maxIncomingHlc);
|
||||
}
|
||||
|
||||
nodesToUpdate.add(hlcManager.getNode());
|
||||
}
|
||||
|
||||
final maxIncomingHlcByNode = <UuidValue, Hlc>{};
|
||||
for (final operation in operations) {
|
||||
maxIncomingHlcByNode[operation.uuidNodeId] = operation.hlc.maxBetween(
|
||||
maxIncomingHlcByNode[operation.uuidNodeId],
|
||||
);
|
||||
}
|
||||
|
||||
for (final MapEntry(key: nodeId, value: incomingHlc)
|
||||
in maxIncomingHlcByNode.entries) {
|
||||
final remoteNode = remoteNodes[nodeId];
|
||||
if (remoteNode == null) continue;
|
||||
final updatedNode = remoteNode.copyWith(
|
||||
lastReceivedHlc: incomingHlc.maxBetween(remoteNode.lastReceivedHlc),
|
||||
);
|
||||
if (updatedNode.lastReceivedHlc == remoteNode.lastReceivedHlc) continue;
|
||||
|
||||
nodesToUpdate.add(updatedNode);
|
||||
remoteNodes[nodeId] = updatedNode;
|
||||
}
|
||||
|
||||
if (nodesToUpdate.isNotEmpty) {
|
||||
await CrdtNode.db.update(
|
||||
_session,
|
||||
nodesToUpdate,
|
||||
columns: (t) => [t.lastReceivedHlc],
|
||||
transaction: transaction,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -466,7 +503,7 @@ extension CrdtMergeRecorderExtension on CrdtMutationRecorder {
|
|||
try {
|
||||
await _db.updateRow(
|
||||
row,
|
||||
columns: _syncTableByName[insert.tableName]!.managedColumns
|
||||
columns: row.table.managedColumns
|
||||
.where((c) => data.containsKey(c.columnName))
|
||||
.toList(),
|
||||
transaction: transaction,
|
||||
|
|
|
|||
|
|
@ -54,12 +54,18 @@ class CrdtMutationRecorder {
|
|||
|
||||
final Database _db;
|
||||
|
||||
late final DatabaseSession _dbSession = _db.session;
|
||||
|
||||
late final _session = CrdtDatabaseSession(
|
||||
_db,
|
||||
syncTables: syncTables,
|
||||
persistentUserId: persistentUserId,
|
||||
);
|
||||
|
||||
late final CrdtUserManager _userManager = CrdtUserManager(_dbSession);
|
||||
|
||||
final Map<UuidValue, HlcManager> _hlcManagers = {};
|
||||
|
||||
late Map<String, (int, Map<String, CrdtSchemaColumn>)> _schema;
|
||||
|
||||
late final _tableDefinitionsByName = {
|
||||
|
|
@ -103,13 +109,14 @@ class CrdtMutationRecorder {
|
|||
/// Safe to call again after the database was wiped (e.g. test `tearDown`)
|
||||
/// for in-memory schema ids to match new rows.
|
||||
Future<void> initialize() async {
|
||||
final session = _db.session;
|
||||
_userManager.clearCache();
|
||||
_hlcManagers.clear();
|
||||
|
||||
if (persistentUserId != null) {
|
||||
await CrdtUserManager.getOrCreate(session, persistentUserId!);
|
||||
await _userManager.getOrCreate(persistentUserId!);
|
||||
}
|
||||
|
||||
final schemaRegistry = CrdtSchemaRegistry(session, syncTables: syncTables);
|
||||
final schemaRegistry = CrdtSchemaRegistry(_dbSession, syncTables: syncTables);
|
||||
final (tableRows, columnRows) = await schemaRegistry.syncAndGetSchema();
|
||||
|
||||
final columnsByTableId = <int, Map<String, CrdtSchemaColumn>>{};
|
||||
|
|
@ -155,6 +162,45 @@ class CrdtMutationRecorder {
|
|||
return _syncTableByName.containsKey(tableName);
|
||||
}
|
||||
|
||||
/// Returns the [CrdtUser] for the given user ID, creating it when needed.
|
||||
Future<CrdtUser> getOrCreateUser(UuidValue userId) {
|
||||
return _userManager.getOrCreate(userId);
|
||||
}
|
||||
|
||||
/// Records the latest acknowledged sync checkpoint for [otherNodeId].
|
||||
Future<void> recordSyncCheckpoint(
|
||||
UuidValue userId,
|
||||
UuidValue otherNodeId,
|
||||
Hlc syncedHlc,
|
||||
) async {
|
||||
final user = await _userManager.getOrCreate(userId);
|
||||
await _db.transaction((transaction) async {
|
||||
var node = await CrdtNode.db.findFirstRow(
|
||||
_session,
|
||||
where: (t) => t.userId.equals(user.id) & t.uuidNodeId.equals(otherNodeId),
|
||||
transaction: transaction,
|
||||
);
|
||||
|
||||
node ??= await CrdtNode.db.insertRow(
|
||||
_session,
|
||||
CrdtNode(userId: user.id!, uuidNodeId: otherNodeId),
|
||||
transaction: transaction,
|
||||
);
|
||||
|
||||
final currentSyncHlc = node.lastReceivedHlc;
|
||||
if (currentSyncHlc != null && currentSyncHlc >= syncedHlc) {
|
||||
return;
|
||||
}
|
||||
|
||||
await CrdtNode.db.updateRow(
|
||||
_session,
|
||||
node.copyWith(lastReceivedHlc: syncedHlc),
|
||||
columns: (t) => [t.lastReceivedHlc],
|
||||
transaction: transaction,
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
/// Insert the CRDT metadata for the inserted rows.
|
||||
Future<void> afterInsert<T extends TableRow>(
|
||||
List<T> insertedRows,
|
||||
|
|
@ -995,7 +1041,10 @@ WHERE "id" IN (${_sqlLiteralList(rowIds)})
|
|||
|
||||
HlcManager _getHlcManager(Transaction transaction) {
|
||||
final user = _getEffectiveUser(transaction);
|
||||
return HlcManager.forUser(user);
|
||||
return _hlcManagers.putIfAbsent(
|
||||
user.uuidUserId,
|
||||
() => HlcManager.forUser(user),
|
||||
);
|
||||
}
|
||||
|
||||
CrdtUser _getEffectiveUser(Transaction transaction) {
|
||||
|
|
@ -1004,7 +1053,7 @@ WHERE "id" IN (${_sqlLiteralList(rowIds)})
|
|||
if (persistentUserId == null) {
|
||||
throw StateError('No user ID found for transaction or persistent user ID.');
|
||||
}
|
||||
return CrdtUserManager.getCached(persistentUserId!);
|
||||
return _userManager.getCached(persistentUserId!);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ import 'package:meta/meta.dart';
|
|||
import 'package:serverpod_database/serverpod_database.dart';
|
||||
import 'package:uuid/uuid.dart';
|
||||
|
||||
import '../crdt/sync.dart';
|
||||
import 'database.dart';
|
||||
|
||||
/// Wraps a [DatabaseSession] to provide a [CrdtDatabase] as [DatabaseSession.db].
|
||||
|
|
@ -13,6 +14,12 @@ class CrdtDatabaseSession implements DatabaseSession {
|
|||
/// The list of tables to sync with CRDT.
|
||||
required List<Table> syncTables,
|
||||
|
||||
/// Maximum number of merge changes sent in one sync stream message.
|
||||
int syncBatchSize = CrdtSync.defaultSyncBatchSize,
|
||||
|
||||
/// Delay between continuous sync rounds.
|
||||
Duration continuousSyncInterval = CrdtSync.defaultContinuousSyncInterval,
|
||||
|
||||
/// The user ID to use for all CRDT operations. This should only be used for
|
||||
/// databases operating on the client side, where all data is for the same user.
|
||||
/// Otherwise, the user ID must be passed through the transaction.
|
||||
|
|
@ -20,6 +27,8 @@ class CrdtDatabaseSession implements DatabaseSession {
|
|||
}) : _db = CrdtDatabase(
|
||||
db,
|
||||
syncTables: syncTables,
|
||||
syncBatchSize: syncBatchSize,
|
||||
continuousSyncInterval: continuousSyncInterval,
|
||||
persistentUserId: persistentUserId,
|
||||
);
|
||||
|
||||
|
|
@ -30,6 +39,12 @@ class CrdtDatabaseSession implements DatabaseSession {
|
|||
/// The list of tables to sync with CRDT.
|
||||
required List<Table> syncTables,
|
||||
|
||||
/// Maximum number of merge changes sent in one sync stream message.
|
||||
int syncBatchSize = CrdtSync.defaultSyncBatchSize,
|
||||
|
||||
/// Delay between continuous sync rounds.
|
||||
Duration continuousSyncInterval = CrdtSync.defaultContinuousSyncInterval,
|
||||
|
||||
/// The user ID to use for all CRDT operations. This should only be used for
|
||||
/// databases operating on the client side, where all data is for the same user.
|
||||
/// Otherwise, the user ID must be passed through the transaction.
|
||||
|
|
@ -37,6 +52,8 @@ class CrdtDatabaseSession implements DatabaseSession {
|
|||
}) => CrdtDatabaseSession(
|
||||
session.db,
|
||||
syncTables: syncTables,
|
||||
syncBatchSize: syncBatchSize,
|
||||
continuousSyncInterval: continuousSyncInterval,
|
||||
persistentUserId: persistentUserId,
|
||||
);
|
||||
|
||||
|
|
@ -80,3 +97,16 @@ class BasicDatabaseSession implements DatabaseSession {
|
|||
extension DatabaseSessionExtension on Database {
|
||||
DatabaseSession get session => BasicDatabaseSession(this);
|
||||
}
|
||||
|
||||
/// Convenience access to a CRDT-aware database from a wrapped session.
|
||||
extension CrdtDatabaseAccess on DatabaseSession {
|
||||
/// Returns the wrapped [CrdtDatabase] for this session.
|
||||
CrdtDatabase get crdtDb {
|
||||
final database = db;
|
||||
if (database is CrdtDatabase) return database;
|
||||
throw StateError(
|
||||
'This database session is not wrapped with CrdtDatabaseSession. '
|
||||
'Use CrdtDatabaseSession.wraps(...) before accessing crdtDb.',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,12 +1,10 @@
|
|||
import 'package:serverpod_database/serverpod_database.dart';
|
||||
import 'package:serverpod_offline_sync_shared/serverpod_offline_sync_shared.dart';
|
||||
import 'package:serverpod_serialization/serverpod_serialization.dart';
|
||||
import 'package:uuid/uuid.dart';
|
||||
|
||||
import '../protocol/protocol.dart';
|
||||
|
||||
/// A manager for [Hlc] instances that is used to generate unique timestamps
|
||||
/// for CRDT operations. Will keep track of the HLC during the lifetime of the
|
||||
/// application.
|
||||
/// for CRDT operations.
|
||||
class HlcManager {
|
||||
HlcManager._(
|
||||
this.uuidUserId,
|
||||
|
|
@ -15,6 +13,16 @@ class HlcManager {
|
|||
this.lastHlc,
|
||||
);
|
||||
|
||||
/// Creates a new [HlcManager] for the current node of [user].
|
||||
factory HlcManager.forUser(CrdtUser user) {
|
||||
return HlcManager._(
|
||||
user.uuidUserId,
|
||||
user.id!,
|
||||
user.currentNodeId!,
|
||||
user.currentNode!.lastReceivedHlc ?? Hlc.zero(user.currentNode!.uuidNodeId),
|
||||
);
|
||||
}
|
||||
|
||||
/// The UUID of the user this manager is for.
|
||||
final UuidValue uuidUserId;
|
||||
|
||||
|
|
@ -30,64 +38,6 @@ class HlcManager {
|
|||
/// The last HLC timestamp for the current node.
|
||||
Hlc lastHlc;
|
||||
|
||||
/// Map of user ID to [HlcManager] instance of the current node for that user.
|
||||
static final Map<UuidValue, HlcManager> _instances = {};
|
||||
|
||||
/// Returns the [HlcManager] for the given node ID.
|
||||
///
|
||||
/// Will create a new [HlcManager] with [Hlc.zero] if no manager is found.
|
||||
static HlcManager forUser(CrdtUser user) {
|
||||
_instances[user.uuidUserId] ??= HlcManager._(
|
||||
user.uuidUserId,
|
||||
user.id!,
|
||||
user.currentNodeId!,
|
||||
user.currentNode!.lastReceivedHlc ?? Hlc.zero(user.currentNode!.uuidNodeId),
|
||||
);
|
||||
|
||||
return _instances[user.uuidUserId]!;
|
||||
}
|
||||
|
||||
/// Closes the [HlcManager] and updates the last received HLC for all nodes.
|
||||
///
|
||||
/// Should be called when the application is shutting down.
|
||||
static Future<void> close(DatabaseSession session) async {
|
||||
await session.db.transaction((transaction) async {
|
||||
final nodes = await CrdtNode.db.find(
|
||||
session,
|
||||
where: (t) => t.uuidNodeId.inSet(
|
||||
_instances.values.map((e) => e.uuidNodeId).toSet(),
|
||||
),
|
||||
transaction: transaction,
|
||||
lockMode: LockMode.forUpdate,
|
||||
lockBehavior: LockBehavior.wait,
|
||||
);
|
||||
|
||||
await CrdtNode.db.update(
|
||||
session,
|
||||
[
|
||||
for (final node in nodes)
|
||||
CrdtNode(
|
||||
id: node.id,
|
||||
userId: node.userId,
|
||||
lastReceivedHlc: _instances.values
|
||||
.firstWhere((e) => e.uuidNodeId == node.uuidNodeId)
|
||||
.lastHlc,
|
||||
),
|
||||
],
|
||||
columns: (t) => [t.lastReceivedHlc],
|
||||
transaction: transaction,
|
||||
);
|
||||
|
||||
_instances.clear();
|
||||
});
|
||||
}
|
||||
|
||||
/// Clears per-user HLC state when the database file was emptied (e.g. test
|
||||
/// cleanup). Call before recreating managers so state matches the store.
|
||||
static void reset() {
|
||||
_instances.clear();
|
||||
}
|
||||
|
||||
/// Returns the next HLC timestamp for the current node.
|
||||
Hlc increment() {
|
||||
lastHlc = lastHlc.increment();
|
||||
|
|
@ -98,4 +48,12 @@ class HlcManager {
|
|||
void merge(Hlc other) {
|
||||
lastHlc = lastHlc.merge(other);
|
||||
}
|
||||
|
||||
/// Converts this manager state to the persisted current-node model.
|
||||
CrdtNode getNode() => CrdtNode(
|
||||
id: normalizedNodeId,
|
||||
userId: normalizedUserId,
|
||||
uuidNodeId: uuidNodeId,
|
||||
lastReceivedHlc: lastHlc,
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,50 +5,48 @@ import '../protocol/protocol.dart';
|
|||
|
||||
/// A manager for [CrdtUser] instances.
|
||||
class CrdtUserManager {
|
||||
CrdtUserManager._();
|
||||
/// Creates a [CrdtUserManager] bound to a database session.
|
||||
CrdtUserManager(this._session);
|
||||
|
||||
/// Map of user ID to [CrdtUser] instance for that user.
|
||||
static final Map<UuidValue, CrdtUser> _instances = {};
|
||||
final DatabaseSession _session;
|
||||
|
||||
/// Returns the [CrdtUser] for the given user ID.
|
||||
///
|
||||
/// Will create a new [CrdtUser] if no user is found.
|
||||
static CrdtUser getCached(UuidValue uuidUserId) =>
|
||||
_instances[uuidUserId] ?? (throw StateError('User $uuidUserId not found.'));
|
||||
final Map<UuidValue, CrdtUser> _instances = {};
|
||||
|
||||
/// Clears the in-memory user cache. Used when the database is reset (e.g. tests)
|
||||
/// so [getOrCreate] loads fresh rows from the store.
|
||||
static void clearCache() {
|
||||
/// Returns the cached [CrdtUser] for the given user ID.
|
||||
CrdtUser getCached(UuidValue uuidUserId) =>
|
||||
_instances[uuidUserId] ??
|
||||
(throw StateError(
|
||||
'User $uuidUserId not found in cache. '
|
||||
'Ensure CrdtUserManager.getOrCreate() is called before getCached().',
|
||||
));
|
||||
|
||||
/// Clears the in-memory cache so state is reloaded from the store.
|
||||
void clearCache() {
|
||||
_instances.clear();
|
||||
}
|
||||
|
||||
/// Returns the [CrdtUser] for the given user ID.
|
||||
///
|
||||
/// Will create a new [CrdtUser] if no user is found.
|
||||
static Future<CrdtUser> getOrCreate(
|
||||
DatabaseSession session,
|
||||
UuidValue uuidUserId,
|
||||
) async {
|
||||
return _instances[uuidUserId] ??= await session.db.transaction(
|
||||
(transaction) => _getOrCreate(session, uuidUserId, transaction),
|
||||
Future<CrdtUser> getOrCreate(UuidValue uuidUserId) async {
|
||||
return _instances[uuidUserId] ??= await _session.db.transaction(
|
||||
(transaction) => _getOrCreate(uuidUserId, transaction),
|
||||
);
|
||||
}
|
||||
|
||||
static Future<CrdtUser> _getOrCreate(
|
||||
DatabaseSession session,
|
||||
Future<CrdtUser> _getOrCreate(
|
||||
UuidValue uuidUserId,
|
||||
Transaction transaction,
|
||||
) async {
|
||||
var user = await CrdtUser.db.findFirstRow(
|
||||
session,
|
||||
_session,
|
||||
where: (t) => t.uuidUserId.equals(uuidUserId),
|
||||
include: CrdtUser.include(currentNode: CrdtNode.include()),
|
||||
transaction: transaction,
|
||||
);
|
||||
|
||||
// User must be created.
|
||||
user ??= await CrdtUser.db.insertRow(
|
||||
session,
|
||||
_session,
|
||||
CrdtUser(uuidUserId: uuidUserId),
|
||||
transaction: transaction,
|
||||
);
|
||||
|
|
@ -57,15 +55,14 @@ class CrdtUserManager {
|
|||
return user;
|
||||
}
|
||||
|
||||
// User has no current node and must be created.
|
||||
final currentNode = await CrdtNode.db.insertRow(
|
||||
session,
|
||||
_session,
|
||||
CrdtNode(userId: user.id!),
|
||||
transaction: transaction,
|
||||
);
|
||||
|
||||
await CrdtUser.db.attachRow.currentNode(
|
||||
session,
|
||||
_session,
|
||||
user,
|
||||
currentNode,
|
||||
transaction: transaction,
|
||||
|
|
|
|||
8
packages/serverpod_offline_sync_client/lib/src/protocol/analysis_options.yaml
generated
Normal file
8
packages/serverpod_offline_sync_client/lib/src/protocol/analysis_options.yaml
generated
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
analyzer:
|
||||
errors:
|
||||
dead_code: ignore
|
||||
duplicate_ignore: ignore
|
||||
unnecessary_type_check: ignore
|
||||
|
||||
formatter:
|
||||
trailing_commas: preserve
|
||||
Loading…
Reference in a new issue