halkyon-days/serverpod-offline-sync
Watch
1
0
Fork
You've already forked serverpod-offline-sync
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

perf: Batch projection reads and engine writes (#157)

Browse source
This commit is contained in:
Marcelo Mendonça Soares 2026-10-03 00:27:56 -03:00 • committed by GitHub
commit e42075cdd9
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 1124 additions and 355 deletions
Download patch file Download diff file Expand all files Collapse all files

224
packages/serverpod_offline_sync/lib/src/database/merge.dart
View file

@ -1,68 +1,5 @@
part of 'recorder.dart';
/// Field metadata for merged inserts, waiting to be written as one batch.
///
/// Every query behind [CrdtForeignKeyProjector.recordInsertAttempts] already
/// takes a set of row ids, so a batch of inserts into one table costs the same
/// as one insert. Rows are grouped by node as well as table because the merge
/// cache stores each field with the node that authored it.
class _PendingInsertAttempts {
final Map<(String, int), ({Set<UuidValue> rowIds, CrdtNode node})> _byGroup = {};
final Map<String, ProjectionAttemptsByField> _attemptsByTable = {};
final Set<MergeRowKey> _rows = {};
bool get isEmpty => _byGroup.isEmpty;
/// Whether metadata for [rowKey] is still unwritten.
bool holds(MergeRowKey rowKey) => _rows.contains(rowKey);
void add({
required String tableName,
required UuidValue rowId,
required CrdtNode node,
required ProjectionAttemptsByField attempts,
}) {
_byGroup
.putIfAbsent(
(tableName, node.id!),
() => (rowIds: <UuidValue>{}, node: node),
)
.rowIds
.add(rowId);
_rows.add((tableName, rowId));
if (attempts.isEmpty) return;
_attemptsByTable
.putIfAbsent(tableName, () => <MergeFieldKey, ProjectionAttempt>{})
.addAll(attempts);
}
/// Empties the collector and returns what it held.
List<
({
String tableName,
Set<UuidValue> rowIds,
CrdtNode node,
ProjectionAttemptsByField attempts,
})
>
take() {
final groups = [
for (final MapEntry(key: key, value: group) in _byGroup.entries)
(
tableName: key.$1,
rowIds: group.rowIds,
node: group.node,
attempts:
_attemptsByTable[key.$1] ?? const <MergeFieldKey, ProjectionAttempt>{},
),
];
_byGroup.clear();
_attemptsByTable.clear();
_rows.clear();
return groups;
}
}
/// Adds merge-specific behavior to [CrdtMutationRecorder].
extension CrdtMergeRecorderExtension on CrdtMutationRecorder {
/// Merges remote CRDT changes into the current database.
@ -142,62 +79,61 @@ extension CrdtMergeRecorderExtension on CrdtMutationRecorder {
transaction,
);
// Field metadata for merged inserts is recorded per table and node rather
// than per row: every query behind it already takes a set of row ids.
final pendingAttempts = _PendingInsertAttempts();
final writes = _MergeWriteBatch(
session: _session,
projector: _foreignKeyProjector,
fields: context.fields,
transaction: transaction,
);
final deferredDeletes = <CrdtMergeDelete>[];
for (final operation in operations) {
if (!_context.isCrdtTrackedTableName(operation.tableName)) {
continue;
}
// A later operation on a row resolves its field metadata through the
// merge cache, so anything still pending for that row is written first.
if (pendingAttempts.holds((operation.tableName, operation.uuidRowId))) {
await _flushInsertAttempts(pendingAttempts, context, transaction);
}
await writes.apply(
operations.where(
(operation) => _context.isCrdtTrackedTableName(operation.tableName),
),
(operation) async {
switch (operation) {
case final CrdtMergeInsert insert:
await _applyMergeInsert(
insert,
nodesByUuid,
context,
authoredOverlays,
batch.plan,
writes,
presence,
transaction,
);
case final CrdtMergeUpdate update:
await _applyMergeUpdate(
update,
nodesByUuid,
context,
authoredOverlays,
writes,
transaction,
);
case final CrdtMergeDelete delete:
// A higher delete/restore generation can have an older HLC than a
// concurrent reinsertion. Bootstrap must retain it until this batch
// has inserted the row it belongs to.
if (!context.rows.containsKey((delete.tableName, delete.uuidRowId))) {
deferredDeletes.add(delete);
return;
}
await _applyMergeDelete(
delete,
nodesByUuid,
context,
transaction,
);
// A delete can hide the row, and a resurrecting one can bring it
// back; either way what was known about it no longer holds.
presence?.forget((delete.tableName, delete.uuidRowId));
}
},
);
switch (operation) {
case final CrdtMergeInsert insert:
await _applyMergeInsert(
insert,
nodesByUuid,
context,
authoredOverlays,
batch.plan,
pendingAttempts,
presence,
transaction,
);
case final CrdtMergeUpdate update:
await _applyMergeUpdate(
update,
nodesByUuid,
context,
authoredOverlays,
transaction,
);
case final CrdtMergeDelete delete:
// A higher delete/restore generation can have an older HLC than a
// concurrent reinsertion. Bootstrap must retain it until this batch
// has inserted the row it belongs to.
if (!context.rows.containsKey((delete.tableName, delete.uuidRowId))) {
deferredDeletes.add(delete);
continue;
}
await _applyMergeDelete(
delete,
nodesByUuid,
context,
transaction,
);
// A delete can hide the row, and a resurrecting one can bring it
// back; either way what was known about it no longer holds.
presence?.forget((delete.tableName, delete.uuidRowId));
}
}
await _flushInsertAttempts(pendingAttempts, context, transaction);
for (final delete in deferredDeletes) {
await _applyMergeDelete(delete, nodesByUuid, context, transaction);
}
@ -471,7 +407,7 @@ extension CrdtMergeRecorderExtension on CrdtMutationRecorder {
MergeContext context,
Map<MergeFieldKey, Object?> authoredOverlays,
ProjectionPlan? batchPlan,
_PendingInsertAttempts pendingAttempts,
_MergeWriteBatch writes,
CrdtForeignKeyPresenceCache? presence,
Transaction transaction,
) async {
@ -525,7 +461,7 @@ extension CrdtMergeRecorderExtension on CrdtMutationRecorder {
// same-space recovery), so same-merge field updates resolve ownership
// from the cache instead of re-reading the domain row.
context.domainOwners[rowKey] = (exists: true, spaceId: mergingSpaceId);
pendingAttempts.add(
writes.recordInsertAttempts(
tableName: insert.tableName,
rowId: insert.uuidRowId,
node: remoteNode,
@ -616,36 +552,12 @@ extension CrdtMergeRecorderExtension on CrdtMutationRecorder {
return (domain: plannedDomain, attempts: attempts);
}
/// Writes the field metadata collected for merged inserts, one call per
/// table and node.
///
/// Grouped by node because the merge cache pairs every field it stores with
/// the node that authored it, which is how the cached field resolves its own
/// clock.
Future<void> _flushInsertAttempts(
_PendingInsertAttempts pending,
MergeContext context,
Transaction transaction,
) async {
if (pending.isEmpty) return;
final groups = pending.take();
for (final group in groups) {
await _foreignKeyProjector.recordInsertAttempts(
group.tableName,
group.rowIds,
transaction,
group.attempts,
mergeCache: (fields: context.fields, node: group.node),
);
}
}
Future<void> _applyMergeUpdate(
CrdtMergeUpdate update,
Map<UuidValue, CrdtNode> remoteNodes,
MergeContext context,
Map<MergeFieldKey, Object?> authoredOverlays,
_MergeWriteBatch writes,
Transaction transaction,
) async {
final rowKey = (update.tableName, update.uuidRowId);
@ -673,6 +585,7 @@ extension CrdtMergeRecorderExtension on CrdtMutationRecorder {
remoteNode: remoteNode,
incomingHlc: update.hlc,
fields: context.fields,
writes: writes,
transaction: transaction,
);
@ -780,8 +693,9 @@ extension CrdtMergeRecorderExtension on CrdtMutationRecorder {
CrdtNode remoteNode,
Hlc incomingHlc,
CrdtDataField? currentField,
Transaction transaction,
) async {
Transaction transaction, {
_MergeWriteBatch? writes,
}) async {
// Projection can create field metadata for a row this merge context loaded
// before the row existed, so an unseen field may still be persisted. Adopt
// it instead of inserting a second row onto the row/column unique index.
@ -794,6 +708,8 @@ extension CrdtMergeRecorderExtension on CrdtMutationRecorder {
);
if (existingField == null) {
// Dependent metadata needs the field identity, so new fields cannot
// wait in the batch of existing field-clock updates.
final insertedField = await CrdtDataField.db.insertRow(
_session,
CrdtDataField(
@ -822,12 +738,16 @@ extension CrdtMergeRecorderExtension on CrdtMutationRecorder {
hlcCounter: incomingHlc.counter,
);
await CrdtDataField.db.update(
_session,
[updatedField],
columns: (t) => [t.nodeId, t.hlcDatetime, t.hlcCounter],
transaction: transaction,
);
if (writes != null) {
writes.recordFieldUpdate(updatedField);
} else {
await CrdtDataField.db.update(
_session,
[updatedField],
columns: (t) => [t.nodeId, t.hlcDatetime, t.hlcCounter],
transaction: transaction,
);
}
return updatedField;
}
@ -1018,6 +938,7 @@ extension CrdtMergeRecorderExtension on CrdtMutationRecorder {
required Map<MergeFieldKey, CrdtDataField> fields,
required Transaction transaction,
CrdtSchemaColumn? schemaColumn,
_MergeWriteBatch? writes,
}) async {
final resolvedSchemaColumn =
schemaColumn ?? _context.schemaColumn(tableName, columnName);
@ -1035,6 +956,7 @@ extension CrdtMergeRecorderExtension on CrdtMutationRecorder {
incomingHlc,
currentField,
transaction,
writes: writes,
);
return true;
}

35
packages/serverpod_offline_sync/lib/src/database/merge_utils/database_helpers.dart
View file

@ -94,7 +94,7 @@ TableRow<UuidValue?> withExplicitInsertNulls(
/// Keeps per-row default semantics when a batch contains projected nulls.
/// PostgreSQL builds a batch using the first row's table, so rows with different
/// null overrides must be written separately within the caller's transaction.
/// null overrides use separate consecutive batches in the caller's transaction.
@internal
Future<List<T>> insertWithExplicitNulls<T extends TableRow>(
Database database,
@ -112,15 +112,30 @@ Future<List<T>> insertWithExplicitNulls<T extends TableRow>(
noReturn: noReturn,
);
}
return [
for (final row in rows)
...(await database.insert<TableRow<UuidValue?>>(
[withExplicitInsertNulls(row, explicitNulls[row.id] ?? const {})],
transaction: transaction,
ignoreConflicts: ignoreConflicts,
noReturn: noReturn,
)).cast<T>(),
];
final inserted = <T>[];
var start = 0;
while (start < rows.length) {
final columns = explicitNulls[rows[start].id] ?? const <String>{};
var end = start + 1;
while (end < rows.length) {
final next = explicitNulls[rows[end].id] ?? const <String>{};
if (columns.length != next.length || !columns.containsAll(next)) break;
end++;
}
final result = await database.insert<TableRow<UuidValue?>>(
[
for (var index = start; index < end; index++)
withExplicitInsertNulls(rows[index], columns),
],
transaction: transaction,
ignoreConflicts: ignoreConflicts,
noReturn: noReturn,
);
inserted.addAll(result.cast<T>());
start = end;
}
return inserted;
}
class _ExplicitNullRow implements TableRow<UuidValue?> {

212
packages/serverpod_offline_sync/lib/src/database/merge_utils/foreign_key_projector.dart
View file

@ -11,6 +11,8 @@ import 'recorder_context.dart';
import 'types.dart';
import 'unique_resolver.dart';
part 'projection_reads.dart';
typedef _ProjectedForeignKeyRow = ({
MergeRowKey key,
CrdtDataRow crdtRow,
@ -880,10 +882,11 @@ class CrdtForeignKeyProjector {
Set<String>? seedTables,
Set<MergeRowKey>? seedRows,
}) async {
final rows = <MergeRowKey, _ProjectedForeignKeyRow>{};
final fieldIds = <MergeFieldKey, int>{};
final attemptedValues = <MergeFieldKey, CrdtDataAttemptedValue>{};
final fieldHlcs = <MergeFieldKey, Hlc>{};
final reader = _ProjectionReader(_context, transaction);
final rows = reader.rows;
final fieldIds = reader.fieldIds;
final attemptedValues = reader.attemptedValues;
final fieldHlcs = reader.fieldHlcs;
final tablesToLoad = _tablesToLoad(
seedTables: seedTables,
@ -897,18 +900,10 @@ class CrdtForeignKeyProjector {
);
if (seedRows == null) {
for (final tableName in tablesToLoad) {
await _loadTableRowsInto(
tableName: tableName,
rowIds: null,
columnNames: columnsByTable[tableName] ?? const <String>{},
rows: rows,
fieldIds: fieldIds,
attemptedValues: attemptedValues,
fieldHlcs: fieldHlcs,
transaction: transaction,
);
}
await reader.loadRows(
requestedRows: {for (final table in tablesToLoad) table: null},
columnsByTable: columnsByTable,
);
} else {
await _loadRowClosureInto(
tablesToLoad: tablesToLoad,
@ -916,11 +911,7 @@ class CrdtForeignKeyProjector {
pendingInserts: pendingInserts,
authoredOverlays: authoredOverlays,
columnsByTable: columnsByTable,
rows: rows,
fieldIds: fieldIds,
attemptedValues: attemptedValues,
fieldHlcs: fieldHlcs,
transaction: transaction,
reader: reader,
);
}
@ -1105,78 +1096,6 @@ class CrdtForeignKeyProjector {
return childRowsByAttempt;
}
/// Loads [rowIds] of [tableName], or every row of it when [rowIds] is null.
///
/// Returns the row ids that exist, so a closure pass can expand from them.
Future<Set<UuidValue>> _loadTableRowsInto({
required String tableName,
required Set<UuidValue>? rowIds,
required Set<String> columnNames,
required Map<MergeRowKey, _ProjectedForeignKeyRow> rows,
required Map<MergeFieldKey, int> fieldIds,
required Map<MergeFieldKey, CrdtDataAttemptedValue> attemptedValues,
required Map<MergeFieldKey, Hlc> fieldHlcs,
required Transaction transaction,
}) async {
if (rowIds != null && rowIds.isEmpty) return const {};
final (tableId, _) = _context.schema[tableName]!;
final userId = _context.hlcManagerFor(transaction).normalizedSpaceId;
final crdtRows = await CrdtDataRow.db.find(
_context.databaseSession,
where: (t) {
final inSpace = t.spaceId.equals(userId) & t.tblId.equals(tableId);
return rowIds == null ? inSpace : inSpace & t.uuidRowId.inSet(rowIds);
},
include: CrdtDataRow.include(
node: CrdtNode.include(),
deleted: CrdtDataDeleted.include(node: CrdtNode.include()),
),
orderBy: (t) => t.uuidRowId,
transaction: transaction,
);
if (crdtRows.isEmpty) return const {};
final loadedIds = {for (final row in crdtRows) row.uuidRowId};
final valuesByRowId = columnNames.isEmpty
? <UuidValue, Map<String, Object?>>{}
: await _context.readDomainColumnValues(
tableName,
loadedIds,
columnNames.toList(),
transaction,
);
for (final row in crdtRows) {
final key = (tableName, row.uuidRowId);
rows[key] = (
key: key,
crdtRow: row,
values: Map<String, Object?>.from(valuesByRowId[row.uuidRowId] ?? const {}),
);
}
if (columnNames.isNotEmpty) {
final loadedFields = await _loadFields(
tableName: tableName,
rowIds: loadedIds,
columnNames: columnNames,
transaction: transaction,
);
for (final field in loadedFields) {
final fieldKey = (tableName, field.row!.uuidRowId, field.column!.name);
fieldIds[fieldKey] = field.id!;
fieldHlcs[fieldKey] = field.hlc;
final attempted = field.attemptedValue;
if (attempted != null) {
attemptedValues[fieldKey] = attempted;
}
}
}
return loadedIds;
}
/// Loads the rows the seeds can reach instead of every row of their tables.
///
/// Defaults are loaded as evaluation context. Their implicit dependents join
@ -1199,12 +1118,13 @@ class CrdtForeignKeyProjector {
required List<PendingProjectionRow> pendingInserts,
required Map<MergeFieldKey, Object?> authoredOverlays,
required Map<String, Set<String>> columnsByTable,
required Map<MergeRowKey, _ProjectedForeignKeyRow> rows,
required Map<MergeFieldKey, int> fieldIds,
required Map<MergeFieldKey, CrdtDataAttemptedValue> attemptedValues,
required Map<MergeFieldKey, Hlc> fieldHlcs,
required Transaction transaction,
required _ProjectionReader reader,
}) async {
final transaction = reader._transaction;
final rows = reader.rows;
final attemptedValues = reader.attemptedValues;
final hasAttemptedClaims = await reader.hasAttemptedClaims(tablesToLoad);
final unwrittenValues = _unwrittenValues(pendingInserts, authoredOverlays);
final requested = <String, Set<UuidValue>>{};
@ -1245,20 +1165,10 @@ class CrdtForeignKeyProjector {
final wave = queued;
queued = <String, Set<UuidValue>>{};
final loadedNow = <String, Set<UuidValue>>{};
for (final MapEntry(key: tableName, value: ids) in wave.entries) {
final loaded = await _loadTableRowsInto(
tableName: tableName,
rowIds: ids,
columnNames: columnsByTable[tableName] ?? const <String>{},
rows: rows,
fieldIds: fieldIds,
attemptedValues: attemptedValues,
fieldHlcs: fieldHlcs,
transaction: transaction,
);
if (loaded.isNotEmpty) loadedNow[tableName] = loaded;
}
final loadedNow = await reader.loadRows(
requestedRows: wave,
columnsByTable: columnsByTable,
);
for (final MapEntry(key: tableName, value: ids) in frontier.entries) {
(loadedNow[tableName] ??= <UuidValue>{}).addAll(ids);
@ -1268,6 +1178,7 @@ class CrdtForeignKeyProjector {
await _expandRowClosure(
loadedNow: loadedNow,
hasAttemptedClaims: hasAttemptedClaims,
tablesToLoad: tablesToLoad,
rows: rows,
unwrittenValues: unwrittenValues,
@ -1535,6 +1446,7 @@ class CrdtForeignKeyProjector {
required Map<MergeRowKey, Map<String, Object?>> unwrittenValues,
required Map<MergeFieldKey, CrdtDataAttemptedValue> attemptedValues,
required void Function(String tableName, Iterable<UuidValue> ids) enqueue,
required bool hasAttemptedClaims,
required Transaction transaction,
}) async {
// Both the stored and the unwritten value matter: the walk has to reach the
@ -1553,14 +1465,19 @@ class CrdtForeignKeyProjector {
}..remove(null);
}
final claims = _ProjectionClaimReads(
_context,
transaction,
includeAttemptedClaims: hasAttemptedClaims,
);
for (final edge in _foreignKeys.edges) {
await _expandToChildren(
_expandToChildren(
edge: edge,
loadedNow: loadedNow,
tablesToLoad: tablesToLoad,
valuesFor: valuesFor,
enqueue: enqueue,
transaction: transaction,
collectClaims: claims.add,
);
await _expandToParents(
edge: edge,
@ -1574,27 +1491,27 @@ class CrdtForeignKeyProjector {
);
}
await _expandUniqueClaimants(
_expandUniqueClaimants(
loadedNow: loadedNow,
attemptedValues: attemptedValues,
valuesFor: valuesFor,
enqueue: enqueue,
transaction: transaction,
collectClaims: claims.add,
);
await claims.enqueueMatches(enqueue);
}
/// Enqueues the children of the parents loaded in this wave.
///
/// Hiding or restoring a parent decides its children's fate, so they join the
/// walk whether they still name it or only remember it as an attempt.
Future<void> _expandToChildren({
void _expandToChildren({
required ForeignKeyEdge edge,
required Map<String, Set<UuidValue>> loadedNow,
required Set<String> tablesToLoad,
required Set<Object?> Function(MergeRowKey, String) valuesFor,
required void Function(String tableName, Iterable<UuidValue> ids) enqueue,
required Transaction transaction,
}) async {
required void Function(String, Map<String, Set<Object?>>) collectClaims,
}) {
final parentIds = loadedNow[edge.parentTableName];
if (parentIds == null || parentIds.isEmpty) return;
if (!tablesToLoad.contains(edge.childTableName)) return;
@ -1611,12 +1528,7 @@ class CrdtForeignKeyProjector {
}
if (references.isEmpty) return;
await _enqueueRowsClaiming(
tableName: edge.childTableName,
valuesByColumn: {edge.childColumn: references},
enqueue: enqueue,
transaction: transaction,
);
collectClaims(edge.childTableName, {edge.childColumn: references});
}
/// Enqueues the parents of the children loaded in this wave.
@ -1686,13 +1598,12 @@ class CrdtForeignKeyProjector {
/// own unique index for it. A row that wants a value it does not hold is
/// already loaded, because holding a projection means holding an attempted
/// value.
Future<void> _expandUniqueClaimants({
void _expandUniqueClaimants({
required Map<String, Set<UuidValue>> loadedNow,
required Map<MergeFieldKey, CrdtDataAttemptedValue> attemptedValues,
required Set<Object?> Function(MergeRowKey, String) valuesFor,
required void Function(String tableName, Iterable<UuidValue> ids) enqueue,
required Transaction transaction,
}) async {
required void Function(String, Map<String, Set<Object?>>) collectClaims,
}) {
for (final MapEntry(key: tableName, value: rowIds) in loadedNow.entries) {
for (final uniqueIndex in _uniqueResolver.uniqueIndexesFor(tableName)) {
final claimedByColumn = <String, Set<Object?>>{};
@ -1713,46 +1624,11 @@ class CrdtForeignKeyProjector {
continue;
}
await _enqueueRowsClaiming(
tableName: tableName,
valuesByColumn: claimedByColumn,
enqueue: enqueue,
transaction: transaction,
);
collectClaims(tableName, claimedByColumn);
}
}
}
/// Enqueues every row of [tableName] that holds, or is owed, these values.
///
/// A row that was repaired or released away from a value no longer names it
/// in the domain column, so the attempted value is the only record left of
/// the claim it is waiting to take back.
Future<void> _enqueueRowsClaiming({
required String tableName,
required Map<String, Set<Object?>> valuesByColumn,
required void Function(String tableName, Iterable<UuidValue> ids) enqueue,
required Transaction transaction,
}) async {
enqueue(
tableName,
await _context.findDomainRowIdsWhereColumnsIn(
tableName: tableName,
valuesByColumn: valuesByColumn,
transaction: transaction,
),
);
enqueue(
tableName,
await _context.findRowIdsHoldingAttemptedValues(
tableName: tableName,
columnNames: valuesByColumn.keys.toSet(),
values: {for (final values in valuesByColumn.values) ...values},
transaction: transaction,
),
);
}
Future<List<CrdtDataField>> _loadFields({
required String tableName,
required Set<UuidValue> rowIds,

193
packages/serverpod_offline_sync/lib/src/database/merge_utils/projection_reads.dart Normal file
View file

@ -0,0 +1,193 @@
part of 'foreign_key_projector.dart';
/// Loads metadata for one projection read phase in one space and transaction.
/// Field identities are resolved through the scoped rows and registered columns,
/// so equal domain UUIDs in different tables remain distinct.
///
/// Its maps become mutable projection state. Complete loading before applying
/// pending inserts or authored overlays to them.
class _ProjectionReader {
_ProjectionReader(this._context, this._transaction);
final CrdtRecorderContext _context;
final Transaction _transaction;
final rows = <MergeRowKey, _ProjectedForeignKeyRow>{};
final fieldIds = <MergeFieldKey, int>{};
final attemptedValues = <MergeFieldKey, CrdtDataAttemptedValue>{};
final fieldHlcs = <MergeFieldKey, Hlc>{};
/// Whether this space and these tables hold any attempted claims.
/// A false result lets the closure skip attempted-claim queries, but is
/// valid only until the next write phase; each projection checks again.
Future<bool> hasAttemptedClaims(Set<String> tablesToLoad) async {
final spaceId = _context.hlcManagerFor(_transaction).normalizedSpaceId;
return tablesToLoad.isNotEmpty &&
await CrdtDataAttemptedValue.db.findFirstRow(
_context.databaseSession,
where: (t) =>
t.field.row.spaceId.equals(spaceId) &
t.field.row.tblId.inSet({
for (final table in tablesToLoad) _context.schema[table]!.$1,
}),
transaction: _transaction,
) !=
null;
}
/// Loads one closure wave, sharing metadata reads across its tables.
/// A null row-id set requests the whole table during a full rebuild.
Future<Map<String, Set<UuidValue>>> loadRows({
required Map<String, Set<UuidValue>?> requestedRows,
required Map<String, Set<String>> columnsByTable,
}) async {
final tablesById = {
for (final entry in requestedRows.entries)
if (entry.value == null || entry.value!.isNotEmpty)
_context.schema[entry.key]!.$1: entry.key,
};
if (tablesById.isEmpty) return {};
final spaceId = _context.hlcManagerFor(_transaction).normalizedSpaceId;
final crdtRows = await CrdtDataRow.db.find(
_context.databaseSession,
where: (t) =>
t.spaceId.equals(spaceId) &
tablesById.entries
.map((entry) {
final rowIds = requestedRows[entry.value];
final table = t.tblId.equals(entry.key);
return rowIds == null ? table : table & t.uuidRowId.inSet(rowIds);
})
.reduce((left, right) => left | right),
include: CrdtDataRow.include(
node: CrdtNode.include(),
deleted: CrdtDataDeleted.include(node: CrdtNode.include()),
),
orderBy: (t) => t.uuidRowId,
transaction: _transaction,
);
final metadataByTable = <String, List<CrdtDataRow>>{};
for (final row in crdtRows) {
(metadataByTable[tablesById[row.tblId]!] ??= []).add(row);
}
final loadedByTable = <String, Set<UuidValue>>{};
for (final tableName in requestedRows.keys) {
final tableRows = metadataByTable[tableName];
if (tableRows == null) continue;
final loadedIds = {for (final row in tableRows) row.uuidRowId};
loadedByTable[tableName] = loadedIds;
final columns = columnsByTable[tableName] ?? const <String>{};
final valuesByRowId = columns.isEmpty
? <UuidValue, Map<String, Object?>>{}
: await _context.readDomainColumnValues(
tableName,
loadedIds,
columns.toList(),
_transaction,
);
for (final row in tableRows) {
final key = (tableName, row.uuidRowId);
rows[key] = (
key: key,
crdtRow: row,
values: Map<String, Object?>.from(valuesByRowId[row.uuidRowId] ?? const {}),
);
}
}
final columnsById = <int, String>{};
final fieldRequests = <({Set<int> rows, Set<int> columns})>[];
for (final tableName in metadataByTable.keys) {
final columns = _context.schemaColumnIds(
tableName,
columnsByTable[tableName] ?? const <String>{},
);
if (columns.isEmpty) continue;
columnsById.addAll({for (final entry in columns.entries) entry.value: entry.key});
fieldRequests.add((
rows: {for (final row in metadataByTable[tableName]!) row.id!},
columns: columns.values.toSet(),
));
}
if (fieldRequests.isEmpty) return loadedByTable;
final rowKeysById = {
for (final row in crdtRows) row.id!: (tablesById[row.tblId]!, row.uuidRowId),
};
final fields = await CrdtDataField.db.find(
_context.databaseSession,
where: (t) => fieldRequests
.map((request) {
return t.rowId.inSet(request.rows) & t.columnId.inSet(request.columns);
})
.reduce((left, right) => left | right),
include: CrdtDataField.include(
node: CrdtNode.include(),
attemptedValue: CrdtDataAttemptedValue.include(),
),
transaction: _transaction,
);
for (final field in fields) {
final rowKey = rowKeysById[field.rowId]!;
final fieldKey = (rowKey.$1, rowKey.$2, columnsById[field.columnId]!);
fieldIds[fieldKey] = field.id!;
fieldHlcs[fieldKey] = field.hlc;
if (field.attemptedValue case final attempted?) {
attemptedValues[fieldKey] = attempted;
}
}
return loadedByTable;
}
}
/// Collects the required claim lookups for one closure wave by table.
/// Traversal selects the claims; this collector handles grouped execution.
/// Each composite claim remains independent when the queries are combined.
class _ProjectionClaimReads {
_ProjectionClaimReads(
this._context,
this._transaction, {
required this._includeAttemptedClaims,
});
final CrdtRecorderContext _context;
final Transaction _transaction;
final bool _includeAttemptedClaims;
final _byTable = <String, List<Map<String, Set<Object?>>>>{};
void add(String tableName, Map<String, Set<Object?>> claim) {
(_byTable[tableName] ??= []).add(claim);
}
Future<void> enqueueMatches(
void Function(String tableName, Iterable<UuidValue> ids) enqueue,
) async {
for (final MapEntry(key: tableName, value: claims) in _byTable.entries) {
enqueue(
tableName,
await _context.findDomainRowIdsWhereAnyColumnsIn(
tableName: tableName,
alternatives: claims,
transaction: _transaction,
),
);
if (!_includeAttemptedClaims) continue;
enqueue(
tableName,
await _context.findRowIdsHoldingAnyAttemptedValues(
tableName: tableName,
alternatives: [
for (final claim in claims)
(
columnNames: claim.keys.toSet(),
values: {for (final values in claim.values) ...values},
),
],
transaction: _transaction,
),
);
}
}
}

86
packages/serverpod_offline_sync/lib/src/database/merge_utils/recorder_context.dart
View file

@ -531,22 +531,43 @@ WHERE r."spaceId" = $spaceId AND r."tblId" = $childTableId
required Transaction transaction,
}) async {
if (columnNames.isEmpty || values.isEmpty) return const {};
return findRowIdsHoldingAnyAttemptedValues(
tableName: tableName,
alternatives: [(columnNames: columnNames, values: values)],
transaction: transaction,
);
}
/// Matches the union of claim lookups without mixing their columns and values.
Future<Set<UuidValue>> findRowIdsHoldingAnyAttemptedValues({
required String tableName,
required List<({Set<String> columnNames, Set<Object?> values})> alternatives,
required Transaction transaction,
}) async {
final (tableId, columns) = schema[tableName]!;
final columnIds = {
for (final columnName in columnNames) ?columns[columnName]?.id,
};
if (columnIds.isEmpty) return const {};
final predicates = <String>{};
for (final alternative in alternatives) {
final columnIds = {
for (final columnName in alternative.columnNames) ?columns[columnName]?.id,
};
if (columnIds.isEmpty) continue;
final encodedValues = {
for (final value in values)
if (value != null)
ValueEncoder.instance.encodeColumnValue(
CrdtDataAttemptedValue.t.value,
Protocol().dynamicFieldToJson(value),
),
};
if (encodedValues.isEmpty) return const {};
final encodedValues = {
for (final value in alternative.values)
if (value != null)
ValueEncoder.instance.encodeColumnValue(
CrdtDataAttemptedValue.t.value,
Protocol().dynamicFieldToJson(value),
),
};
if (encodedValues.isEmpty) continue;
predicates.add(
'(f."columnId" IN (${columnIds.join(', ')}) '
'AND a."value" IN (${encodedValues.join(', ')}))',
);
}
if (predicates.isEmpty) return const {};
final spaceId = hlcManagerFor(transaction).normalizedSpaceId;
final result = await database.unsafeQuery(
@ -557,8 +578,7 @@ JOIN "crdt_data_fields" f ON f."id" = a."fieldId"
JOIN "crdt_data_rows" r ON r."id" = f."rowId"
WHERE r."spaceId" = $spaceId
AND r."tblId" = $tableId
AND f."columnId" IN (${columnIds.join(', ')})
AND a."value" IN (${encodedValues.join(', ')})
AND (${predicates.join(' OR ')})
''',
transaction: transaction,
);
@ -618,22 +638,38 @@ WHERE r."spaceId" = $spaceId
required String tableName,
required Map<String, Set<Object?>> valuesByColumn,
required Transaction transaction,
}) async {
if (valuesByColumn.isEmpty) return const {};
if (valuesByColumn.values.any((values) => values.isEmpty)) return const {};
}) => findDomainRowIdsWhereAnyColumnsIn(
tableName: tableName,
alternatives: [valuesByColumn],
transaction: transaction,
);
final predicates = valuesByColumn.entries
.map(
(entry) =>
'"${entry.key.escapeIdentifier()}" IN (${entry.value.sqlLiteralList()})',
)
.join(') AND (');
/// Matches each composite claim independently and unions the matching row ids.
Future<Set<UuidValue>> findDomainRowIdsWhereAnyColumnsIn({
required String tableName,
required List<Map<String, Set<Object?>>> alternatives,
required Transaction transaction,
}) async {
final predicates = <String>{};
for (final valuesByColumn in alternatives) {
if (valuesByColumn.isEmpty) continue;
if (valuesByColumn.values.any((values) => values.isEmpty)) continue;
final claim = valuesByColumn.entries
.map(
(entry) =>
'"${entry.key.escapeIdentifier()}" IN (${entry.value.sqlLiteralList()})',
)
.join(') AND (');
predicates.add('(($claim))');
}
if (predicates.isEmpty) return const {};
final result = await database.unsafeQuery(
'''
SELECT "id"
FROM "${tableName.escapeIdentifier()}"
WHERE ($predicates)
WHERE ${predicates.join(' OR ')}
''',
transaction: transaction,
);

154
packages/serverpod_offline_sync/lib/src/database/merge_write_batch.dart Normal file
View file

@ -0,0 +1,154 @@
part of 'recorder.dart';
/// Owns pending metadata for one merge and the boundaries that make it readable.
/// Context fields advance immediately. Existing field writes wait until an
/// insert, delete, pending insert's next operation, or successful completion.
/// Insert attempts are flushed before another operation touches their row.
class _MergeWriteBatch {
_MergeWriteBatch({
required this._session,
required this._projector,
required this._fields,
required this._transaction,
});
final DatabaseSession _session;
final CrdtForeignKeyProjector _projector;
final Map<MergeFieldKey, CrdtDataField> _fields;
final Transaction _transaction;
final _insertAttempts = _PendingInsertAttempts();
final _fieldUpdates = <int, CrdtDataField>{};
/// Successful completion persists all metadata before projection can run.
/// Failure leaves rollback to the enclosing transaction, without flushing.
///
/// Update callbacks must use the merge context's current clocks and record
/// changes to existing fields through [recordFieldUpdate].
Future<void> apply(
Iterable<CrdtMergeChange> operations,
Future<void> Function(CrdtMergeChange operation) applyOperation,
) async {
for (final operation in operations) {
final needsInsertMetadata = _insertAttempts.holds((
operation.tableName,
operation.uuidRowId,
));
// Reinserts can advance field clocks immediately. Flush older queued
// updates first so the final flush cannot overwrite those newer clocks.
if (operation is! CrdtMergeUpdate || needsInsertMetadata) {
await _flushFieldUpdates();
}
if (needsInsertMetadata) {
await _flushInsertAttempts();
}
await applyOperation(operation);
}
await _flushFieldUpdates();
await _flushInsertAttempts();
}
void recordFieldUpdate(CrdtDataField field) {
_fieldUpdates[field.id!] = field;
}
void recordInsertAttempts({
required String tableName,
required UuidValue rowId,
required CrdtNode node,
required ProjectionAttemptsByField attempts,
}) {
_insertAttempts.add(
tableName: tableName,
rowId: rowId,
node: node,
attempts: attempts,
);
}
Future<void> _flushFieldUpdates() async {
if (_fieldUpdates.isEmpty) return;
await CrdtDataField.db.update(
_session,
_fieldUpdates.values.toList(),
columns: (t) => [t.nodeId, t.hlcDatetime, t.hlcCounter],
transaction: _transaction,
noReturn: true,
);
_fieldUpdates.clear();
}
Future<void> _flushInsertAttempts() async {
for (final group in _insertAttempts.take()) {
await _projector.recordInsertAttempts(
group.tableName,
group.rowIds,
_transaction,
group.attempts,
mergeCache: (fields: _fields, node: group.node),
);
}
}
}
/// Field metadata for merged inserts, waiting to be written as one batch.
///
/// Groups rows by table and author: cached field clocks need the node that
/// authored each group.
class _PendingInsertAttempts {
final Map<(String, int), ({Set<UuidValue> rowIds, CrdtNode node})> _byGroup = {};
final Map<String, ProjectionAttemptsByField> _attemptsByTable = {};
final Set<MergeRowKey> _rows = {};
/// Whether metadata for [rowKey] is still unwritten.
bool holds(MergeRowKey rowKey) => _rows.contains(rowKey);
void add({
required String tableName,
required UuidValue rowId,
required CrdtNode node,
required ProjectionAttemptsByField attempts,
}) {
_byGroup
.putIfAbsent(
(tableName, node.id!),
() => (rowIds: <UuidValue>{}, node: node),
)
.rowIds
.add(rowId);
_rows.add((tableName, rowId));
if (attempts.isEmpty) return;
_attemptsByTable
.putIfAbsent(tableName, () => <MergeFieldKey, ProjectionAttempt>{})
.addAll(attempts);
}
/// Empties the collector and returns what it held.
List<
({
String tableName,
Set<UuidValue> rowIds,
CrdtNode node,
ProjectionAttemptsByField attempts,
})
>
take() {
final groups = [
for (final MapEntry(key: key, value: group) in _byGroup.entries)
(
tableName: key.$1,
rowIds: group.rowIds,
node: group.node,
attempts:
_attemptsByTable[key.$1] ?? const <MergeFieldKey, ProjectionAttempt>{},
),
];
_byGroup.clear();
_attemptsByTable.clear();
_rows.clear();
return groups;
}
}

1
packages/serverpod_offline_sync/lib/src/database/recorder.dart
View file

@ -21,6 +21,7 @@ import 'schema.dart';
import 'unique_index_utils.dart';
part 'merge.dart';
part 'merge_write_batch.dart';
typedef _CrdtSchema = Map<String, (int, Map<String, CrdtSchemaColumn>)>;