halkyon-days/serverpod-offline-sync
Watch
1
0
Fork
You've already forked serverpod-offline-sync
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

perf: Skip the projection passes for merged inserts that cannot change them

Browse source 
A merge that touched any foreign key ran the planning pass for its new
rows and then the full projection pass, whose closure follows foreign
keys both ways and so loads every row connected to the merged ones. In a
library whose tags join its parts, one merged rating, edit, tag or new
card loaded all of it: about 9.5 s on a device receiving it and 12 s on
the server merging a push, for 8,000 cards.

A batch that only inserts new rows (each once) and updates columns no
projection reads now leaves both passes out when its inserts are proven,
as local inserts are (canLeaveProjectionUnchangedByMergedInserts, on
localProjectionDependenciesAreStable): each row references its parents by
id, each parent visible or inserted earlier in the same batch, nothing
refers to the row yet (no row, no attempted value), and no unique claim
is contested. A parent inserted after its child in the batch would find
the child's reference held back, which only the pass restores, so it
counts only when earlier. Deletes, inserts onto rows there already and
anything unproven take the passes as before. The batch's parents are
still resolved together for the writes.

The new tests cover a child of a visible parent, a parent and child in
one batch (both through the shortcut), and a child of a merged delete
and a parent arriving after its held-back child (both through the
passes); dropping the parent or the reference check fails the matching
test. The core, server, integration and simulation suites pass, but for
two PostgreSQL tests that cannot start their database while another
embedded PostgreSQL runs, with or without this change.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Tizian 「ティツィアーン」 2026-10-17 16:51:28 +02:00
commit 9495f616c4
Signed by: raisondetredev
GPG key ID: 07F6FCF405488B64
3 changed files with 344 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

65
packages/serverpod_offline_sync/lib/src/database/merge.dart
View file

@ -36,7 +36,7 @@ extension CrdtMergeRecorderExtension on CrdtMutationRecorder {
// Nothing in this batch touches a foreign key or a unique index, so no
// projection decision can change: skip both passes instead of loading
// state for tables that cannot produce a candidate or a conflict.
final mayAffectProjection = _foreignKeyProjector.mergeOperationsMayAffectProjection(
var mayAffectProjection = _foreignKeyProjector.mergeOperationsMayAffectProjection(
operations,
);
final authoredOverlays = <MergeFieldKey, Object?>{};
@ -48,6 +48,14 @@ extension CrdtMergeRecorderExtension on CrdtMutationRecorder {
);
final nodesByUuid = remoteNodes.nodesByUuid;
final context = await _loadMergeContext(mergeSet, transaction);
// New rows whose parents are there and which nothing refers to yet (a
// rating, an edit, a tag given, a card added) can't change a projection
// decision either, though they carry foreign keys: proven so, neither
// pass runs, and the batch still resolves their parents in one go.
final unchangedByInserts =
mayAffectProjection &&
await _mergeLeavesProjectionUnchanged(operations, context, transaction);
if (unchangedByInserts) mayAffectProjection = false;
// Deletes carry no authored overlay but still change visibility, so the
// seed is every touched row, not just the ones with overlays.
@ -59,7 +67,7 @@ extension CrdtMergeRecorderExtension on CrdtMutationRecorder {
touchedRows.add((operation.tableName, operation.uuidRowId));
}
final batch = mayAffectProjection
final batch = mayAffectProjection || unchangedByInserts
? await _planBatchInserts(
operations,
nodesByUuid,
@ -67,6 +75,7 @@ extension CrdtMergeRecorderExtension on CrdtMutationRecorder {
touchedTables,
touchedRows,
transaction,
plan: mayAffectProjection,
)
: (plan: null, pending: const <PendingProjectionRow>[]);
// Every merged insert has to know whether the rows its foreign keys name
@ -362,8 +371,11 @@ extension CrdtMergeRecorderExtension on CrdtMutationRecorder {
MergeContext context,
Set<String> seedTables,
Set<MergeRowKey> seedRows,
Transaction transaction,
) async {
Transaction transaction, {
// Whether to plan them: without, only the rows are listed, for their
// parents to be resolved together.
bool plan = true,
}) async {
final pending = <PendingProjectionRow>[];
final seenRowKeys = <MergeRowKey>{};
@ -391,7 +403,7 @@ extension CrdtMergeRecorderExtension on CrdtMutationRecorder {
hidden: false,
));
}
if (pending.isEmpty) return (plan: null, pending: pending);
if (pending.isEmpty || !plan) return (plan: null, pending: pending);
return (
plan: await _foreignKeyProjector.project(
@ -404,6 +416,49 @@ extension CrdtMergeRecorderExtension on CrdtMutationRecorder {
);
}
/// Whether this batch leaves every projection decision as it is though it
/// touches foreign keys: it only inserts new rows, each once, and updates
/// columns no projection reads, and the inserts are proven
/// ([CrdtForeignKeyProjector.canLeaveProjectionUnchangedByMergedInserts]).
/// A delete, or an insert onto a row there already, takes the full pass.
Future<bool> _mergeLeavesProjectionUnchanged(
List<CrdtMergeChange> operations,
MergeContext context,
Transaction transaction,
) async {
final insertsByTable = <String, Map<UuidValue, Map<String, Object?>>>{};
final positions = <MergeRowKey, int>{};
for (final operation in operations) {
if (!_context.isCrdtTrackedTableName(operation.tableName)) continue;
switch (operation) {
case CrdtMergeDelete():
return false;
case final CrdtMergeUpdate update:
if (_foreignKeyProjector.needsProjection(update.tableName, {
update.columnName,
})) {
return false;
}
case final CrdtMergeInsert insert:
if (context.rows[(insert.tableName, insert.uuidRowId)] != null) {
return false;
}
final rows = insertsByTable[insert.tableName] ??= {};
if (rows.containsKey(insert.uuidRowId)) return false;
rows[insert.uuidRowId] = _sanitizeMergeRowData(
insert.tableName,
insert.databaseColumns,
);
positions[(insert.tableName, insert.uuidRowId)] = positions.length;
}
}
return _foreignKeyProjector.canLeaveProjectionUnchangedByMergedInserts(
insertsByTable,
positions,
transaction,
);
}
Future<void> _applyMergeInsert(
CrdtMergeInsert insert,
Map<UuidValue, CrdtNode> remoteNodes,

93
packages/serverpod_offline_sync/lib/src/database/merge_utils/foreign_key_projector.dart
View file

@ -756,6 +756,99 @@ class CrdtForeignKeyProjector {
);
}
/// Proves that merged inserts of new rows, [rowsByTable] (each row's
/// authored values by its id, by its table; [positions] says where each
/// comes in the batch), leave every projection decision as it is, as
/// [canLeaveLocalProjectionUnchanged] proves it for local inserts: each row
/// references its parents by id, each parent visible or inserted earlier in
/// the same merge (and proven alike; one inserted after its child would
/// find the child's reference held back, which only the pass restores),
/// nothing refers
/// to the row yet (no row, no attempted value), and no unique claim is
/// contested. The merge then writes them as authored, without the pass,
/// which would load every row connected to them: all of a library whose
/// tags join its parts.
Future<bool> canLeaveProjectionUnchangedByMergedInserts(
Map<String, Map<UuidValue, Map<String, Object?>>> rowsByTable,
Map<MergeRowKey, int> positions,
Transaction transaction,
) async {
for (final MapEntry(key: tableName, value: rows) in rowsByTable.entries) {
final ids = rows.keys.toSet();
final childEdges =
_foreignKeys.edgesByChildTable[tableName] ?? const <ForeignKeyEdge>[];
final parentEdges =
_foreignKeys.edgesByParentTable[tableName] ?? const <ForeignKeyEdge>[];
if (childEdges.any((edge) => edge.parentColumn != 'id') ||
parentEdges.any((edge) => edge.parentColumn != 'id')) {
return false;
}
final columns = _context.columnsByTableAndName[tableName];
Object? valueOf(Map<String, Object?> row, String column) =>
canonicalDomainValue(row[column], columns?[column]);
final parentsByTable = <String, Set<UuidValue>>{};
for (final edge in childEdges) {
for (final MapEntry(key: rowId, value: row) in rows.entries) {
final value = valueOf(row, edge.childColumn);
if (value == null) {
if (!edge.childNullable || edge.defaultValue != null) return false;
continue;
}
final id = tryUuidValue(value);
// A row of the same table inserted by the same merge would have
// to be proven first; the pass orders that.
if (id == null || (edge.parentTableName == tableName && ids.contains(id))) {
return false;
}
if (positions[(edge.parentTableName, id)] case final parentAt?) {
if (parentAt > positions[(tableName, rowId)]!) return false;
continue;
}
(parentsByTable[edge.parentTableName] ??= {}).add(id);
}
}
final referencingColumnsByTable = <String, Set<String>>{};
for (final edge in parentEdges) {
if (edge.action == ForeignKeyAction.setDefault &&
ids.contains(tryUuidValue(edge.defaultValue))) {
return false;
}
(referencingColumnsByTable[edge.childTableName] ??= {}).add(edge.childColumn);
}
final uniqueClaims = <Map<String, Set<Object?>>>[];
for (final index in _uniqueResolver.uniqueIndexesFor(tableName)) {
final tuples = <String>{};
final claims = <String, Set<Object?>>{};
for (final row in rows.values) {
final tuple = [
for (final column in index.indexedColumns) valueOf(row, column),
];
if (tuple.any((value) => value == null)) continue;
if (!tuples.add(tuple.map(canonicalProjectionValue).join('\x1f'))) {
return false;
}
for (final (i, column) in index.indexedColumns.indexed) {
(claims[column] ??= {}).add(tuple[i]);
}
}
if (claims.isNotEmpty) uniqueClaims.add(claims);
}
if (!await _context.localProjectionDependenciesAreStable(
tableName: tableName,
rowIds: ids,
persisted: false,
parentsByTable: parentsByTable,
referencingColumnsByTable: referencingColumnsByTable,
uniqueClaims: uniqueClaims,
transaction: transaction,
)) {
return false;
}
}
return true;
}
/// Proves that a local delete of [tableName] rows leaves nothing for the
/// projection pass to decide: the delete's own cascade
/// (`applyDeleteActions`, recursively) already hid everything it hides.

191
test/serverpod_offline_sync_test_server/test/integration/merge/merge_inserts_projection_test.dart Normal file
View file

@ -0,0 +1,191 @@
import 'package:serverpod/serverpod.dart';
import 'package:serverpod_offline_sync_server/serverpod_offline_sync_server.dart';
import 'package:serverpod_offline_sync_test_client/serverpod_offline_sync_test_client.dart';
import 'package:test/test.dart';
import '../test_tools/client_session.dart';
import '../test_tools/crdt_probes.dart';
// Merged inserts of new rows whose parents are visible, which nothing
// refers to yet, change no projection decision, and the merge leaves the
// projection pass out for them. Anything else still takes the pass: these
// cover both sides.
void main() {
initTestClientSession();
CrdtMergeInsert insertOf(TableRow row, String tableName) {
final node = const Uuid().v7obj();
return CrdtMergeInsert(
uuidSpaceId: testCrdtUserId,
tableName: tableName,
uuidRowId: row.id! as UuidValue,
uuidNodeId: node,
hlcDatetime: DateTime.now().toUtc(),
hlcCounter: 0,
data: row,
);
}
Future<void> merge(List<CrdtMergeChange> changes) =>
session.db.mergeChanges(changes, spaceId: testCrdtUserId);
group('Given a set-null middle stored here,', () {
late FkChainSetNullMiddle middle;
setUp(() async {
middle = await session.db.transactionForUser(
testCrdtUserId,
(tx) => FkChainSetNullMiddle.db.insertRow(
session,
FkChainSetNullMiddle(id: const Uuid().v7obj(), name: 'middle'),
transaction: tx,
),
);
});
group('when a remote insert of its cascade child is merged,', () {
late FkChainSetNullCascadeChild child;
setUp(() async {
child = FkChainSetNullCascadeChild(
id: const Uuid().v7obj(),
name: 'child',
setNullMiddleId: middle.id,
);
await merge([insertOf(child, FkChainSetNullCascadeChild.t.tableName)]);
});
test('then the child is there as authored, holding nothing back.', () async {
final visible = await FkChainSetNullCascadeChild.db.findById(
session,
child.id!,
);
expect(visible, isNotNull);
expect(visible!.setNullMiddleId, middle.id);
expect(
await attemptedValue(
rowId: child.id!,
columnName: FkChainSetNullCascadeChild.t.setNullMiddleId.columnName,
),
isNull,
);
});
});
group('when the middle is deleted by a merged remote delete and then a '
'remote insert of its cascade child is merged,', () {
late FkChainSetNullCascadeChild child;
setUp(() async {
await merge([
CrdtMergeDelete(
uuidSpaceId: testCrdtUserId,
tableName: FkChainSetNullMiddle.t.tableName,
uuidRowId: middle.id!,
uuidNodeId: const Uuid().v7obj(),
hlcDatetime: (await rowHlc(
middle.id!,
)).datetime.add(const Duration(milliseconds: 1)),
hlcCounter: 0,
clFlag: 2,
reason: CrdtDataDeletedReason.userDelete,
),
]);
child = FkChainSetNullCascadeChild(
id: const Uuid().v7obj(),
name: 'child of a deleted middle',
setNullMiddleId: middle.id,
);
await merge([insertOf(child, FkChainSetNullCascadeChild.t.tableName)]);
});
test('then the child is hidden with its middle.', () async {
expect(
await FkChainSetNullMiddle.db.findById(session, middle.id!),
isNull,
);
expect(
await FkChainSetNullCascadeChild.db.findById(session, child.id!),
isNull,
);
});
});
});
group('Given a set-null grandchild merged before its middle, its foreign '
'key held back,', () {
late FkChainSetNullMiddle middle;
late FkChainSetNullSetNullChild grandchild;
setUp(() async {
middle = FkChainSetNullMiddle(id: const Uuid().v7obj(), name: 'late');
grandchild = FkChainSetNullSetNullChild(
id: const Uuid().v7obj(),
name: 'early',
setNullMiddleId: middle.id,
);
await merge([
insertOf(grandchild, FkChainSetNullSetNullChild.t.tableName),
]);
final early = await FkChainSetNullSetNullChild.db.findById(
session,
grandchild.id!,
);
expect(early!.setNullMiddleId, isNull);
});
group("when the middle's insert is merged,", () {
setUp(() async {
await merge([insertOf(middle, FkChainSetNullMiddle.t.tableName)]);
});
test('then the grandchild points at it.', () async {
final visible = await FkChainSetNullSetNullChild.db.findById(
session,
grandchild.id!,
);
expect(visible!.setNullMiddleId, middle.id);
expect(
await attemptedValue(
rowId: grandchild.id!,
columnName: FkChainSetNullSetNullChild.t.setNullMiddleId.columnName,
),
isNull,
);
});
});
});
group('Given nothing stored,', () {
group('when a remote insert of a middle and its cascade child is merged '
'in one batch,', () {
late FkChainSetNullMiddle middle;
late FkChainSetNullCascadeChild child;
setUp(() async {
middle = FkChainSetNullMiddle(id: const Uuid().v7obj(), name: 'new');
child = FkChainSetNullCascadeChild(
id: const Uuid().v7obj(),
name: 'new child',
setNullMiddleId: middle.id,
);
await merge([
insertOf(middle, FkChainSetNullMiddle.t.tableName),
insertOf(child, FkChainSetNullCascadeChild.t.tableName),
]);
});
test('then both are there, the child pointing at the middle.', () async {
expect(
await FkChainSetNullMiddle.db.findById(session, middle.id!),
isNotNull,
);
final visible = await FkChainSetNullCascadeChild.db.findById(
session,
child.id!,
);
expect(visible!.setNullMiddleId, middle.id);
});
});
});
}