halkyon-days/serverpod-offline-sync
Watch
1
0
Fork
You've already forked serverpod-offline-sync
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

perf: Reduce redundant projection and metadata reads (#124)

Browse source
This commit is contained in:
Marcelo Mendonça Soares 2026-09-14 10:53:25 -03:00 • committed by GitHub
commit 4331daf661
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 924 additions and 95 deletions
Download patch file Download diff file Expand all files Collapse all files

77
packages/serverpod_offline_sync/lib/src/database/database.dart
View file

@ -201,6 +201,8 @@ class CrdtDatabase implements Database {
Transaction? transaction, {
required bool membershipWide,
}) async {
if (include == null && !_recorder.isCrdtTracked<T>()) return where;
final scopeIds = await _scopeIdsForQueries(
transaction,
membershipWide: membershipWide,
@ -429,18 +431,18 @@ class CrdtDatabase implements Database {
transaction,
(tx) async {
final prepared = _prepareRowsForInsert(rows, tx);
if (rows.isNotEmpty) {
final rowIds = {
for (final row in rows)
if (row.id case final UuidValue rowId) rowId,
};
await _recorder.projectCurrent(rows.first.table.tableName, rowIds, tx);
}
final values = _recorder.withForeignKeyInsertDefaults(prepared.rows);
final projectionUnchanged = await _recorder.prepareLocalUpsert(
values,
conflictColumns,
updateColumns,
tx,
);
// CRDT metadata needs the affected rows even when the public call uses
// noReturn, because inserted rows may have database-generated ids.
final result = await _delegate.upsert<T>(
_recorder.withForeignKeyInsertDefaults(prepared.rows),
values,
conflictColumns: conflictColumns,
updateColumns: updateColumns,
updateWhere: await _whereVisibleWithTombstone<T>(
@ -469,7 +471,12 @@ class CrdtDatabase implements Database {
for (final row in result)
if (row.id is! UuidValue || !insertedRowIds.contains(row.id)) row,
];
await _recorder.afterUpdate(updatedRows, updateColumns, tx);
await _recorder.afterUpdate(
updatedRows,
updateColumns,
tx,
projectionUnchanged: projectionUnchanged,
);
if (noReturn) return <T>[];
_stripStampedRows(result, prepared);
for (final row in reinsertedRows) {
@ -519,7 +526,7 @@ class CrdtDatabase implements Database {
transaction,
);
final reinsertedRows = await _delegate.update<T>(
plannedReinserts,
plannedReinserts.rows,
transaction: transaction,
);
await _recorder.afterReinsert(reinsertedRows, transaction);
@ -623,7 +630,7 @@ class CrdtDatabase implements Database {
(tx) async {
final plannedUpdates = await _recorder.planLocalUpdates(rows, columns, tx);
final updatedRows = [
for (final row in plannedUpdates)
for (final row in plannedUpdates.rows)
await _updateRowWithoutRecording(
row,
stripScopeId: _shouldStripReturnedScopeId(row, tx),
@ -632,7 +639,12 @@ class CrdtDatabase implements Database {
),
];
await _recorder.afterUpdate(updatedRows, columns, tx);
await _recorder.afterUpdate(
updatedRows,
columns,
tx,
projectionUnchanged: plannedUpdates.projectionUnchanged,
);
return noReturn ? <T>[] : updatedRows;
},
);
@ -652,13 +664,18 @@ class CrdtDatabase implements Database {
final plannedUpdates = await _recorder.planLocalUpdates([row], columns, tx);
final stripScopeId = _shouldStripReturnedScopeId(row, tx);
final updatedRow = await _updateRowWithoutRecording(
plannedUpdates.single,
plannedUpdates.rows.single,
stripScopeId: stripScopeId,
transaction: tx,
columns: columns,
);
await _recorder.afterUpdate([updatedRow], columns, tx);
await _recorder.afterUpdate(
[updatedRow],
columns,
tx,
projectionUnchanged: plannedUpdates.projectionUnchanged,
);
return updatedRow;
},
);
@ -980,9 +997,23 @@ class CrdtDatabase implements Database {
// On the server this is authoritative membership; on a persistent client it
// is the server-projected membership cache.
return _scopeIdsForUuids(
await CrdtScopeMembership.memberScopes(_delegate.session, userId),
);
final scopeGroups = await Future.wait<List<CrdtScope>>([
CrdtScope.db.find(
_delegate.session,
where: (t) => t.uuidScopeId.equals(userId),
),
CrdtScopeMember.db
.find(
_delegate.session,
where: (t) => t.userUuid.equals(userId),
include: CrdtScopeMember.include(scope: CrdtScope.include()),
)
.then((memberships) => [for (final member in memberships) member.scope!]),
]);
return {
for (final scopes in scopeGroups)
for (final scope in scopes) scope.id!,
}.toList();
}
List<int>? _actingScopeIdsForQueries(Transaction? transaction) {
@ -998,18 +1029,6 @@ class CrdtDatabase implements Database {
return _recorder.persistentUserId;
}
Future<List<int>> _scopeIdsForUuids(List<UuidValue> scopeUuids) async {
if (scopeUuids.isEmpty) return [];
final scopes = await CrdtScope.db.find(
_delegate.session,
where: (t) => t.uuidScopeId.inSet(scopeUuids.toSet()),
);
return [
for (final scope in scopes)
if (scope.id != null) scope.id!,
];
}
Future<UuidValue> _requireUserId(UuidValue? userId) async {
return userId ??
_recorder.persistentUserId ??

7
packages/serverpod_offline_sync/lib/src/database/merge_utils/foreign_key_graph.dart
View file

@ -155,6 +155,13 @@ class CrdtForeignKeyGraph {
return reached;
}
/// Components whose projection can load a default even when every seed is absent.
late final Set<String> tablesWithDefaultDependencies = {
for (final edge in edges)
if (edge.action == ForeignKeyAction.setDefault && edge.defaultValue != null)
...connectedTables([edge.childTableName]),
};
/// The child column names of foreign keys outgoing from [tableName].
///
/// When [columnNames] is provided, only edges whose child column is in the set

124
packages/serverpod_offline_sync/lib/src/database/merge_utils/foreign_key_projector.dart
View file

@ -636,6 +636,130 @@ class CrdtForeignKeyProjector {
};
}
/// Proves that an ordinary local write leaves the existing projection fixed.
///
/// Only new, unreferenced rows and additions to null FKs qualify. All parents
/// must already be live, no unique claim may be contested, and no authored
/// projection override may be pending on the table. Removing an edge, changing
/// a claim, restoring a row or creating a default target needs the full pass.
/// The proof is checked again after the write; it is never a transaction cache.
Future<bool> canLeaveLocalProjectionUnchanged<T extends TableRow>(
List<T> candidates,
Transaction transaction, {
required bool inserting,
bool persisted = true,
List<Column>? columns,
}) async {
if (candidates.isEmpty) return true;
final tableName = candidates.first.table.tableName;
if (!_context.isCrdtTrackedTableName(tableName)) return true;
if (candidates.any((row) => row.id is! UuidValue)) return false;
final ids = candidates.uuidRowIds;
if (ids.length != candidates.length) return false;
final childEdges = _foreignKeys.edgesByChildTable[tableName] ?? <ForeignKeyEdge>[];
final parentEdges =
_foreignKeys.edgesByParentTable[tableName] ?? <ForeignKeyEdge>[];
if (childEdges.any((edge) => edge.parentColumn != 'id')) return false;
if (inserting && parentEdges.any((edge) => edge.parentColumn != 'id')) return false;
final uniqueColumns = _uniqueResolver.uniqueColumnNamesFor(tableName);
final relevantColumns = {
for (final edge in childEdges) edge.childColumn,
for (final edge in parentEdges)
if (edge.parentColumn != 'id') edge.parentColumn,
...uniqueColumns,
};
final current = inserting
? <UuidValue, Map<String, Object?>>{}
: await _context.readDomainColumnValues(
tableName,
ids,
relevantColumns.toList(),
transaction,
);
final writtenColumns = columns?.map((column) => column.columnName).toSet();
final values = <UuidValue, Map<String, Object?>>{};
for (final candidate in candidates) {
final id = candidate.id as UuidValue;
final supplied = candidate.toJsonForDatabase() as Map;
final previous = current[id] ?? const <String, Object?>{};
final next = {
for (final column in relevantColumns)
column: canonicalDomainValue(
inserting || writtenColumns == null || writtenColumns.contains(column)
? supplied[column]
: previous[column],
),
};
if (!inserting) {
if (relevantColumns.isNotEmpty && !current.containsKey(id)) return false;
for (final column in relevantColumns) {
if (projectionValuesEqual(previous[column], next[column])) continue;
if (previous[column] != null ||
uniqueColumns.contains(column) ||
!childEdges.any((edge) => edge.childColumn == column) ||
parentEdges.any((edge) => edge.parentColumn == column)) {
return false;
}
}
}
values[id] = next;
}
final parentsByTable = <String, Set<UuidValue>>{};
for (final edge in childEdges) {
for (final row in values.values) {
final value = row[edge.childColumn];
if (value == null) {
if (!edge.childNullable || (!persisted && edge.defaultValue != null)) {
return false;
}
continue;
}
final id = tryUuidValue(value);
if (id == null || (edge.parentTableName == tableName && ids.contains(id))) {
return false;
}
(parentsByTable[edge.parentTableName] ??= {}).add(id);
}
}
final referencingColumnsByTable = <String, Set<String>>{};
if (inserting) {
for (final edge in parentEdges) {
if (edge.action == ForeignKeyAction.setDefault &&
ids.contains(tryUuidValue(edge.defaultValue))) {
return false;
}
(referencingColumnsByTable[edge.childTableName] ??= {}).add(edge.childColumn);
}
}
final uniqueClaims = <Map<String, Set<Object?>>>[];
for (final index in _uniqueResolver.uniqueIndexesFor(tableName)) {
final tuples = <String>{};
final claims = <String, Set<Object?>>{};
for (final row in values.values) {
final tuple = [for (final column in index.indexedColumns) row[column]];
if (tuple.any((value) => value == null)) continue;
if (!tuples.add(tuple.map(canonicalProjectionValue).join('\x1f'))) return false;
for (final column in index.indexedColumns) {
(claims[column] ??= {}).add(row[column]);
}
}
if (claims.isEmpty) continue;
uniqueClaims.add(claims);
}
return _context.localProjectionDependenciesAreStable(
tableName: tableName,
rowIds: ids,
persisted: persisted,
parentsByTable: parentsByTable,
referencingColumnsByTable: referencingColumnsByTable,
uniqueClaims: uniqueClaims,
transaction: transaction,
);
}
/// Whether a mutation of [tableName] may require FK or unique projection.
bool needsProjection(String tableName, Set<String>? columnNames) {
if (_foreignKeys.columnsMayAffectForeignKeys(tableName, columnNames)) {

159
packages/serverpod_offline_sync/lib/src/database/merge_utils/recorder_context.dart
View file

@ -148,6 +148,95 @@ class CrdtRecorderContext {
return crdtDataRows;
}
/// Checks local projection dependencies in one database snapshot.
Future<bool> localProjectionDependenciesAreStable({
required String tableName,
required Set<UuidValue> rowIds,
required bool persisted,
required Map<String, Set<UuidValue>> parentsByTable,
required Map<String, Set<String>> referencingColumnsByTable,
required List<Map<String, Set<Object?>>> uniqueClaims,
required Transaction transaction,
}) async {
final (tableId, _) = schema[tableName]!;
final scopeId = hlcManagerFor(transaction).normalizedScopeId;
final ids = rowIds.sqlLiteralList();
final rowPredicate =
'r."scopeId" = $scopeId AND r."tblId" = $tableId '
'AND r."uuidRowId" IN ($ids)';
final conditions = <String>[
if (persisted)
'''(SELECT COUNT(*) FROM "crdt_data_rows" r
LEFT JOIN "crdt_data_tombstone" d ON d."rowId" = r."id"
WHERE $rowPredicate AND ($_rowVisible)
AND (d."id" IS NULL OR d."clFlag" % 2 = 1)) = ${rowIds.length}'''
else
'NOT EXISTS (SELECT 1 FROM "crdt_data_rows" r WHERE $rowPredicate)',
// An attempted FK may own a unique fallback different from its authored value.
'''NOT EXISTS (
SELECT 1 FROM "crdt_data_attempted_value" a
JOIN "crdt_data_fields" f ON f."id" = a."fieldId"
JOIN "crdt_data_rows" r ON r."id" = f."rowId"
WHERE r."scopeId" = $scopeId AND r."tblId" = $tableId)''',
];
for (final MapEntry(key: parentTable, value: parentIds) in parentsByTable.entries) {
final (parentTableId, _) = schema[parentTable]!;
conditions.add('''(SELECT COUNT(*)
FROM "${parentTable.escapeIdentifier()}" p
JOIN "crdt_data_rows" r ON r."uuidRowId" = p."id"
AND r."tblId" = $parentTableId AND r."scopeId" = $scopeId
LEFT JOIN "crdt_data_tombstone" d ON d."rowId" = r."id"
WHERE p."scopeId" = $scopeId AND p."id" IN (${parentIds.sqlLiteralList()})
AND ($_rowVisible) AND (d."id" IS NULL OR d."clFlag" % 2 = 1)
) = ${parentIds.length}''');
}
final encodedIds = {
for (final id in rowIds)
ValueEncoder.instance.encodeColumnValue(
CrdtDataAttemptedValue.t.value,
Protocol().dynamicFieldToJson(id),
),
};
for (final MapEntry(key: childTable, value: columnNames)
in referencingColumnsByTable.entries) {
final predicates = [
for (final column in columnNames) '"${column.escapeIdentifier()}" IN ($ids)',
].join(' OR ');
conditions.add(
'NOT EXISTS (SELECT 1 FROM "${childTable.escapeIdentifier()}" '
'WHERE $predicates)',
);
final (childTableId, columns) = schema[childTable]!;
final columnIds = {
for (final name in columnNames) ?columns[name]?.id,
};
if (columnIds.isEmpty) continue;
conditions.add('''NOT EXISTS (
SELECT 1 FROM "crdt_data_attempted_value" a
JOIN "crdt_data_fields" f ON f."id" = a."fieldId"
JOIN "crdt_data_rows" r ON r."id" = f."rowId"
WHERE r."scopeId" = $scopeId AND r."tblId" = $childTableId
AND f."columnId" IN (${columnIds.join(', ')})
AND a."value" IN (${encodedIds.join(', ')}))''');
}
for (final claims in uniqueClaims) {
final predicates = [
for (final MapEntry(key: column, value: values) in claims.entries)
'("${column.escapeIdentifier()}" IN (${values.sqlLiteralList()}))',
].join(' AND ');
conditions.add(
'NOT EXISTS (SELECT 1 FROM "${tableName.escapeIdentifier()}" '
'WHERE $predicates AND "id" NOT IN ($ids))',
);
}
final result = await database.unsafeQuery(
'SELECT ${conditions.map((condition) => "($condition)").join(" AND ")}',
transaction: transaction,
);
final stable = result.single.first;
return stable == true || stable == 1;
}
Future<void> upsertCrdtFieldsForRows(
String tableName,
List<CrdtDataRow> crdtDataRows,
@ -157,66 +246,34 @@ class CrdtRecorderContext {
}) async {
if (crdtDataRows.isEmpty || schemaColumns.isEmpty) return;
final rowPks = crdtDataRows.map((r) => r.id!).toSet();
final columnPks = schemaColumns.map((c) => c.id!).toSet();
final existingFields = await CrdtDataField.db.find(
databaseSession,
where: (t) => t.rowId.inSet(rowPks) & t.columnId.inSet(columnPks),
transaction: transaction,
);
final fieldByRowAndColumn = {
for (final f in existingFields) (f.rowId, f.columnId): f,
};
final hlcManager = hlcManagerFor(transaction);
final toInsert = <CrdtDataField>[];
final toUpdate = <CrdtDataField>[];
final fields = <CrdtDataField>[];
for (final row in crdtDataRows) {
for (final schemaCol in schemaColumns) {
if (skippedFields.contains((tableName, row.uuidRowId, schemaCol.name))) {
continue;
}
final hlc = hlcManager.increment();
final existing = fieldByRowAndColumn[(row.id!, schemaCol.id!)];
if (existing == null) {
toInsert.add(
CrdtDataField(
rowId: row.id!,
columnId: schemaCol.id!,
nodeId: hlcManager.normalizedNodeId,
hlcDatetime: hlc.datetime,
hlcCounter: hlc.counter,
),
);
} else {
toUpdate.add(
existing.copyWith(
nodeId: hlcManager.normalizedNodeId,
hlcDatetime: hlc.datetime,
hlcCounter: hlc.counter,
),
);
}
fields.add(
CrdtDataField(
rowId: row.id!,
columnId: schemaCol.id!,
nodeId: hlcManager.normalizedNodeId,
hlcDatetime: hlc.datetime,
hlcCounter: hlc.counter,
),
);
}
}
if (toInsert.isNotEmpty) {
await CrdtDataField.db.insert(
databaseSession,
toInsert,
transaction: transaction,
);
}
if (toUpdate.isNotEmpty) {
await CrdtDataField.db.update(
databaseSession,
toUpdate,
transaction: transaction,
);
}
if (fields.isEmpty) return;
await CrdtDataField.db.upsert(
databaseSession,
fields,
conflictColumns: (t) => [t.rowId, t.columnId],
updateColumns: (t) => [t.nodeId, t.hlcDatetime, t.hlcCounter],
transaction: transaction,
noReturn: true,
);
}
Future<void> recordFieldsUpdatedByTable(

110
packages/serverpod_offline_sync/lib/src/database/recorder.dart
View file

@ -372,6 +372,42 @@ class CrdtMutationRecorder {
);
}
/// Returns whether primary-key upserts preserve the existing projection.
Future<bool> prepareLocalUpsert<T extends TableRow>(
List<T> rows,
List<Column> conflictColumns,
List<Column>? updateColumns,
Transaction transaction,
) async {
final tableName = rows.first.table.tableName;
final rowIds = {
for (final row in rows)
if (row.id case final UuidValue rowId) rowId,
};
if (conflictColumns.length == 1 &&
conflictColumns.single.columnName == 'id' &&
rowIds.length == rows.length) {
final stored = await _context.findCrdtRows(tableName, rowIds, transaction);
if (stored.isEmpty &&
!_foreignKeys.tablesWithDefaultDependencies.contains(tableName)) {
// The full pass has no persisted seed or default to load. Classification
// after the physical upsert still reads metadata again.
return false;
}
if (stored.length == rowIds.length &&
await _foreignKeyProjector.canLeaveLocalProjectionUnchanged(
rows,
transaction,
inserting: false,
columns: updateColumns,
)) {
return true;
}
}
await projectCurrent(tableName, rowIds, transaction);
return false;
}
/// Plans FK/unique projection for rows that are about to be inserted.
///
/// Releases hidden unique claims first and returns copies whose unique/FK
@ -392,6 +428,15 @@ class CrdtMutationRecorder {
return (rows: rows, attempts: unplanned);
}
if (await _foreignKeyProjector.canLeaveLocalProjectionUnchanged(
rows,
transaction,
inserting: true,
persisted: false,
)) {
return (rows: rows, attempts: unplanned);
}
final hlcManager = _context.hlcManagerFor(transaction);
final pendingHlc = hlcManager.peekNext();
final node = hlcManager.getNode();
@ -447,17 +492,29 @@ class CrdtMutationRecorder {
}
/// Plans FK/unique projection for rows that are about to be updated.
Future<List<T>> planLocalUpdates<T extends TableRow>(
Future<({List<T> rows, bool projectionUnchanged})>
planLocalUpdates<T extends TableRow>(
List<T> rows,
List<Column>? columns,
Transaction transaction,
) async {
if (rows.isEmpty) return rows;
if (rows.isEmpty) return (rows: rows, projectionUnchanged: true);
final tableName = rows.first.table.tableName;
if (!_context.isCrdtTrackedTableName(tableName)) return rows;
if (!_context.isCrdtTrackedTableName(tableName)) {
return (rows: rows, projectionUnchanged: true);
}
final columnNames = columns?.map((column) => column.columnName).toSet();
if (!_foreignKeyProjector.needsProjection(tableName, columnNames)) {
return rows;
return (rows: rows, projectionUnchanged: true);
}
if (await _foreignKeyProjector.canLeaveLocalProjectionUnchanged(
rows,
transaction,
inserting: false,
columns: columns,
)) {
return (rows: rows, projectionUnchanged: true);
}
final authored = {
@ -500,13 +557,16 @@ class CrdtMutationRecorder {
if (row.id case final UuidValue rowId) (tableName, rowId),
},
);
return [
for (final row in rows)
_withPlannedDomainValues(
row,
planned.domain[(tableName, row.id as UuidValue)],
),
];
return (
projectionUnchanged: false,
rows: [
for (final row in rows)
_withPlannedDomainValues(
row,
planned.domain[(tableName, row.id as UuidValue)],
),
],
);
}
Map<String, Object?> _authoredValuesFromRow<T extends TableRow>(
@ -538,7 +598,10 @@ class CrdtMutationRecorder {
data[column.fieldName] = planned[column.columnName];
}
}
final className = data.remove('__className__') as String?;
final className =
_session.db.serializationManager.getClassNameForObject(row) ??
data['__className__'] as String?;
data.remove('__className__');
if (className == null) {
throw StateError(
'Cannot apply planned domain values: ${row.runtimeType} has no class name.',
@ -613,7 +676,14 @@ class CrdtMutationRecorder {
transaction,
attempts,
);
await _maybeProject(tableName, rowIds, null, transaction);
if (_foreignKeyProjector.needsProjection(tableName, null) &&
!await _foreignKeyProjector.canLeaveLocalProjectionUnchanged(
insertedRows,
transaction,
inserting: true,
)) {
await _maybeProject(tableName, rowIds, null, transaction);
}
});
}
@ -667,8 +737,9 @@ class CrdtMutationRecorder {
Future<void> afterUpdate<T extends TableRow>(
List<T> updatedRows,
List<Column>? columns,
Transaction transaction,
) async {
Transaction transaction, {
bool projectionUnchanged = false,
}) async {
await _foreignKeyProjector.assertVisibleTargets(updatedRows, columns, transaction);
await _forTrackedRows(updatedRows, transaction, (
@ -696,6 +767,15 @@ class CrdtMutationRecorder {
transaction,
skippedFields: implicitForeignKeyRepairFields,
);
if (projectionUnchanged &&
await _foreignKeyProjector.canLeaveLocalProjectionUnchanged(
updatedRows,
transaction,
inserting: false,
columns: columns,
)) {
return;
}
final updatedColumnNames = columns?.map((column) => column.columnName).toSet();
await _maybeProject(
tableName,