halkyon-days/serverpod-offline-sync
Watch
1
0
Fork
You've already forked serverpod-offline-sync
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

perf: Batch projection reads and engine writes (#157)

Browse source
This commit is contained in:
Marcelo Mendonça Soares 2026-10-03 00:27:56 -03:00 • committed by GitHub
commit e42075cdd9
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 1124 additions and 355 deletions
Download patch file Download diff file Expand all files Collapse all files

224
packages/serverpod_offline_sync/lib/src/database/merge.dart
View file

@ -1,68 +1,5 @@
part of 'recorder.dart';
/// Field metadata for merged inserts, waiting to be written as one batch.
///
/// Every query behind [CrdtForeignKeyProjector.recordInsertAttempts] already
/// takes a set of row ids, so a batch of inserts into one table costs the same
/// as one insert. Rows are grouped by node as well as table because the merge
/// cache stores each field with the node that authored it.
class _PendingInsertAttempts {
final Map<(String, int), ({Set<UuidValue> rowIds, CrdtNode node})> _byGroup = {};
final Map<String, ProjectionAttemptsByField> _attemptsByTable = {};
final Set<MergeRowKey> _rows = {};
bool get isEmpty => _byGroup.isEmpty;
/// Whether metadata for [rowKey] is still unwritten.
bool holds(MergeRowKey rowKey) => _rows.contains(rowKey);
void add({
required String tableName,
required UuidValue rowId,
required CrdtNode node,
required ProjectionAttemptsByField attempts,
}) {
_byGroup
.putIfAbsent(
(tableName, node.id!),
() => (rowIds: <UuidValue>{}, node: node),
)
.rowIds
.add(rowId);
_rows.add((tableName, rowId));
if (attempts.isEmpty) return;
_attemptsByTable
.putIfAbsent(tableName, () => <MergeFieldKey, ProjectionAttempt>{})
.addAll(attempts);
}
/// Empties the collector and returns what it held.
List<
({
String tableName,
Set<UuidValue> rowIds,
CrdtNode node,
ProjectionAttemptsByField attempts,
})
>
take() {
final groups = [
for (final MapEntry(key: key, value: group) in _byGroup.entries)
(
tableName: key.$1,
rowIds: group.rowIds,
node: group.node,
attempts:
_attemptsByTable[key.$1] ?? const <MergeFieldKey, ProjectionAttempt>{},
),
];
_byGroup.clear();
_attemptsByTable.clear();
_rows.clear();
return groups;
}
}
/// Adds merge-specific behavior to [CrdtMutationRecorder].
extension CrdtMergeRecorderExtension on CrdtMutationRecorder {
/// Merges remote CRDT changes into the current database.
@ -142,62 +79,61 @@ extension CrdtMergeRecorderExtension on CrdtMutationRecorder {
transaction,
);
// Field metadata for merged inserts is recorded per table and node rather
// than per row: every query behind it already takes a set of row ids.
final pendingAttempts = _PendingInsertAttempts();
final writes = _MergeWriteBatch(
session: _session,
projector: _foreignKeyProjector,
fields: context.fields,
transaction: transaction,
);
final deferredDeletes = <CrdtMergeDelete>[];
for (final operation in operations) {
if (!_context.isCrdtTrackedTableName(operation.tableName)) {
continue;
}
// A later operation on a row resolves its field metadata through the
// merge cache, so anything still pending for that row is written first.
if (pendingAttempts.holds((operation.tableName, operation.uuidRowId))) {
await _flushInsertAttempts(pendingAttempts, context, transaction);
}
await writes.apply(
operations.where(
(operation) => _context.isCrdtTrackedTableName(operation.tableName),
),
(operation) async {
switch (operation) {
case final CrdtMergeInsert insert:
await _applyMergeInsert(
insert,
nodesByUuid,
context,
authoredOverlays,
batch.plan,
writes,
presence,
transaction,
);
case final CrdtMergeUpdate update:
await _applyMergeUpdate(
update,
nodesByUuid,
context,
authoredOverlays,
writes,
transaction,
);
case final CrdtMergeDelete delete:
// A higher delete/restore generation can have an older HLC than a
// concurrent reinsertion. Bootstrap must retain it until this batch
// has inserted the row it belongs to.
if (!context.rows.containsKey((delete.tableName, delete.uuidRowId))) {
deferredDeletes.add(delete);
return;
}
await _applyMergeDelete(
delete,
nodesByUuid,
context,
transaction,
);
// A delete can hide the row, and a resurrecting one can bring it
// back; either way what was known about it no longer holds.
presence?.forget((delete.tableName, delete.uuidRowId));
}
},
);
switch (operation) {
case final CrdtMergeInsert insert:
await _applyMergeInsert(
insert,
nodesByUuid,
context,
authoredOverlays,
batch.plan,
pendingAttempts,
presence,
transaction,
);
case final CrdtMergeUpdate update:
await _applyMergeUpdate(
update,
nodesByUuid,
context,
authoredOverlays,
transaction,
);
case final CrdtMergeDelete delete:
// A higher delete/restore generation can have an older HLC than a
// concurrent reinsertion. Bootstrap must retain it until this batch
// has inserted the row it belongs to.
if (!context.rows.containsKey((delete.tableName, delete.uuidRowId))) {
deferredDeletes.add(delete);
continue;
}
await _applyMergeDelete(
delete,
nodesByUuid,
context,
transaction,
);
// A delete can hide the row, and a resurrecting one can bring it
// back; either way what was known about it no longer holds.
presence?.forget((delete.tableName, delete.uuidRowId));
}
}
await _flushInsertAttempts(pendingAttempts, context, transaction);
for (final delete in deferredDeletes) {
await _applyMergeDelete(delete, nodesByUuid, context, transaction);
}
@ -471,7 +407,7 @@ extension CrdtMergeRecorderExtension on CrdtMutationRecorder {
MergeContext context,
Map<MergeFieldKey, Object?> authoredOverlays,
ProjectionPlan? batchPlan,
_PendingInsertAttempts pendingAttempts,
_MergeWriteBatch writes,
CrdtForeignKeyPresenceCache? presence,
Transaction transaction,
) async {
@ -525,7 +461,7 @@ extension CrdtMergeRecorderExtension on CrdtMutationRecorder {
// same-space recovery), so same-merge field updates resolve ownership
// from the cache instead of re-reading the domain row.
context.domainOwners[rowKey] = (exists: true, spaceId: mergingSpaceId);
pendingAttempts.add(
writes.recordInsertAttempts(
tableName: insert.tableName,
rowId: insert.uuidRowId,
node: remoteNode,
@ -616,36 +552,12 @@ extension CrdtMergeRecorderExtension on CrdtMutationRecorder {
return (domain: plannedDomain, attempts: attempts);
}
/// Writes the field metadata collected for merged inserts, one call per
/// table and node.
///
/// Grouped by node because the merge cache pairs every field it stores with
/// the node that authored it, which is how the cached field resolves its own
/// clock.
Future<void> _flushInsertAttempts(
_PendingInsertAttempts pending,
MergeContext context,
Transaction transaction,
) async {
if (pending.isEmpty) return;
final groups = pending.take();
for (final group in groups) {
await _foreignKeyProjector.recordInsertAttempts(
group.tableName,
group.rowIds,
transaction,
group.attempts,
mergeCache: (fields: context.fields, node: group.node),
);
}
}
Future<void> _applyMergeUpdate(
CrdtMergeUpdate update,
Map<UuidValue, CrdtNode> remoteNodes,
MergeContext context,
Map<MergeFieldKey, Object?> authoredOverlays,
_MergeWriteBatch writes,
Transaction transaction,
) async {
final rowKey = (update.tableName, update.uuidRowId);
@ -673,6 +585,7 @@ extension CrdtMergeRecorderExtension on CrdtMutationRecorder {
remoteNode: remoteNode,
incomingHlc: update.hlc,
fields: context.fields,
writes: writes,
transaction: transaction,
);
@ -780,8 +693,9 @@ extension CrdtMergeRecorderExtension on CrdtMutationRecorder {
CrdtNode remoteNode,
Hlc incomingHlc,
CrdtDataField? currentField,
Transaction transaction,
) async {
Transaction transaction, {
_MergeWriteBatch? writes,
}) async {
// Projection can create field metadata for a row this merge context loaded
// before the row existed, so an unseen field may still be persisted. Adopt
// it instead of inserting a second row onto the row/column unique index.
@ -794,6 +708,8 @@ extension CrdtMergeRecorderExtension on CrdtMutationRecorder {
);
if (existingField == null) {
// Dependent metadata needs the field identity, so new fields cannot
// wait in the batch of existing field-clock updates.
final insertedField = await CrdtDataField.db.insertRow(
_session,
CrdtDataField(
@ -822,12 +738,16 @@ extension CrdtMergeRecorderExtension on CrdtMutationRecorder {
hlcCounter: incomingHlc.counter,
);
await CrdtDataField.db.update(
_session,
[updatedField],
columns: (t) => [t.nodeId, t.hlcDatetime, t.hlcCounter],
transaction: transaction,
);
if (writes != null) {
writes.recordFieldUpdate(updatedField);
} else {
await CrdtDataField.db.update(
_session,
[updatedField],
columns: (t) => [t.nodeId, t.hlcDatetime, t.hlcCounter],
transaction: transaction,
);
}
return updatedField;
}
@ -1018,6 +938,7 @@ extension CrdtMergeRecorderExtension on CrdtMutationRecorder {
required Map<MergeFieldKey, CrdtDataField> fields,
required Transaction transaction,
CrdtSchemaColumn? schemaColumn,
_MergeWriteBatch? writes,
}) async {
final resolvedSchemaColumn =
schemaColumn ?? _context.schemaColumn(tableName, columnName);
@ -1035,6 +956,7 @@ extension CrdtMergeRecorderExtension on CrdtMutationRecorder {
incomingHlc,
currentField,
transaction,
writes: writes,
);
return true;
}