halkyon-days/serverpod-offline-sync
Watch
1
0
Fork
You've already forked serverpod-offline-sync
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix: Stabilize benchmark sampling and expand storage reporting (#37)

Browse source 
Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: Marcelo Mendonça Soares <marcelo.me.soares@gmail.com>
Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>
This commit is contained in:
Copilot 2026-05-07 00:17:30 -03:00 • committed by GitHub
commit 0307094d58
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 347 additions and 86 deletions
Download patch file Download diff file Expand all files Collapse all files

35
benchmark/run.dart
View file

@ -34,7 +34,7 @@ Future<void> main(List<String> args) async {
crdtEnabled: false,
operation: operation,
rowCount: rowCount,
).report(),
).measure(),
skipProgress: runningInCI,
);
@ -45,8 +45,8 @@ Future<void> main(List<String> args) async {
crdtEnabled: true,
operation: operation,
rowCount: rowCount,
).report(),
validator: (result) => result.$1 > baselineResult.$1,
).measure(),
validator: (result) => result > baselineResult,
skipProgress: runningInCI,
);
@ -60,7 +60,7 @@ Future<void> main(List<String> args) async {
}
final spuriousBenchmarks = benchmarkResults.where(
(result) => result.baseline.$1 > result.crdt.$1,
(result) => result.baseline > result.crdt,
);
if (spuriousBenchmarks.isNotEmpty) {
@ -74,8 +74,8 @@ Future<void> main(List<String> args) async {
);
for (final result in spuriousBenchmarks) {
final baselineDelay = result.baseline.$1.toFormattedDuration();
final crdtDelay = result.crdt.$1.toFormattedDuration();
final baselineDelay = result.baseline.toFormattedDuration();
final crdtDelay = result.crdt.toFormattedDuration();
print(
' - ${result.operation.name.toUpperCase()}: $baselineDelay > $crdtDelay',
);
@ -89,8 +89,29 @@ Future<void> main(List<String> args) async {
runningInCI: runningInCI,
);
}
final baselineStorageResult = await runWithProgress(
'Running storage benchmark (baseline)',
() => TypesTableStorageBenchmark(
'storage (baseline)',
crdtEnabled: false,
rowCount: rowCount,
).reportStorage(),
skipProgress: runningInCI,
);
final crdtStorageResult = await runWithProgress(
'Running storage benchmark (CRDT)',
() => TypesTableStorageBenchmark(
'storage (CRDT)',
crdtEnabled: true,
rowCount: rowCount,
).reportStorage(),
skipProgress: runningInCI,
);
printStorageImpact(
benchmarkResults.first,
StorageBenchmarkResults(
baseline: baselineStorageResult,
crdt: crdtStorageResult,
),
rowCount: rowCount,
runningInCI: runningInCI,
);

271
benchmark/utils/benchmark.dart
View file

@ -13,6 +13,34 @@ import 'tables.dart';
enum Operation { insert, update, delete }
enum StorageStage { insert, update, delete }
class StorageBenchmarkRun {
const StorageBenchmarkRun({
required this.baseDatabaseSize,
required this.insertOnlySize,
required this.insertUpdateSize,
required this.insertUpdateDeleteSize,
});
final int baseDatabaseSize;
final int insertOnlySize;
final int insertUpdateSize;
final int insertUpdateDeleteSize;
int sizeFor(StorageStage stage) => switch (stage) {
StorageStage.insert => insertOnlySize,
StorageStage.update => insertUpdateSize,
StorageStage.delete => insertUpdateDeleteSize,
};
int previousSizeFor(StorageStage stage) => switch (stage) {
StorageStage.insert => 0,
StorageStage.update => insertOnlySize,
StorageStage.delete => insertUpdateSize,
};
}
/// Benchmarks plain SQLite writes vs CRDT-wrapped writes using the wide [Types]
/// row (same shape idea as legacy Drift `TableWithEveryColumnType`).
class TypesTableBenchmark extends AsyncBenchmarkBase {
@ -25,6 +53,9 @@ class TypesTableBenchmark extends AsyncBenchmarkBase {
/// Minimal blob column payload (shared; column stays non-null with zero bytes).
static final ByteData _emptyBlob = ByteData(0);
static const _warmupMillis = 100;
static const _measurementMillis = 2000;
static const updatedColumnsPerRow = 8;
final bool crdtEnabled;
final Operation operation;
@ -35,6 +66,7 @@ class TypesTableBenchmark extends AsyncBenchmarkBase {
late final File _dbFile;
late final ClientDatabaseSession _plainSession;
CrdtDatabaseSession? _crdtSession;
var _hasOpenSession = false;
final UuidValue _userId = const Uuid().v7obj();
@ -158,108 +190,142 @@ class TypesTableBenchmark extends AsyncBenchmarkBase {
@override
Future<void> setup() async {
_valueSeq = 0;
_lastDatabaseSize = 0;
_lastRowsCount = 0;
final dbPath = p.join(
Directory.systemTemp.path,
'offline_sync_benchmark_$name.db',
);
_dbFile = File(dbPath);
if (_dbFile.existsSync()) {
_dbFile.deleteSync();
}
_deleteDatabaseFiles(_dbFile);
_plainSession = await Client(_clientUrl).createSession(
_dbFile.path,
isDebugMode: true,
);
_hasOpenSession = true;
await _bootstrapDatabase();
}
Future<void> _prepareCycle() async {
if (_seededRows.isNotEmpty) {
await _deleteTypes(_seededRows);
_seededRows = [];
}
switch (operation) {
case Operation.insert:
_seededRows = [];
return;
case Operation.update:
_seededRows = await _insertTypes(rowCount);
return;
case Operation.delete:
_seededRows = [];
_seededRows = await _insertTypes(rowCount);
return;
}
}
@override
Future<void> warmup() async {}
Future<double> _measurePreparedCycles(int minimumMillis) async {
final minimumMicros = minimumMillis * 1000;
final watch = Stopwatch()..start();
var totalTimedMicros = 0.0;
var timedIterations = 0;
@override
Future<void> run() async {
switch (operation) {
case Operation.insert:
await _insertTypes(rowCount);
case Operation.update:
await _updateTypesInPlace();
case Operation.delete:
await _deleteTypes(_seededRows);
while (watch.elapsedMicroseconds < minimumMicros) {
await _prepareCycle();
final sw = Stopwatch()..start();
await run();
totalTimedMicros += sw.elapsedMicroseconds;
timedIterations++;
}
return totalTimedMicros / timedIterations;
}
Future<void> _prepareDeleteCycle() async {
await _bootstrapDatabase();
_seededRows = await _insertTypes(rowCount);
}
Future<void> _timedDeleteBatch() async {
await _deleteTypes(_seededRows);
}
@override
Future<double> measure() async {
if (operation == Operation.delete) {
await setup();
try {
await AsyncBenchmarkBase.measureFor(() async {}, 100);
const minimumMicros = 2000 * 1000;
final watch = Stopwatch()..start();
var totalTimedMicros = 0.0;
var timedIterations = 0;
while (watch.elapsedMicroseconds < minimumMicros) {
await _prepareDeleteCycle();
final sw = Stopwatch()..start();
await _timedDeleteBatch();
totalTimedMicros += sw.elapsedMicroseconds;
timedIterations++;
}
return totalTimedMicros / timedIterations;
} finally {
await teardown();
}
}
return super.measure();
}
@override
Future<(double, int)> report() async {
final averageMicroseconds = await measure();
return (averageMicroseconds, measureStorage());
}
@override
Future<void> teardown() async {
Future<void> _captureDatabaseSize() async {
try {
await _plainSession.db.unsafeExecute('PRAGMA wal_checkpoint(TRUNCATE)');
} on Exception {
// WAL checkpoint is best-effort (e.g. non-SQLite tests).
}
_lastDatabaseSize = _sqliteDbFootprintBytes(_dbFile);
_lastRowsCount = await Types.db.count(_plainSession);
}
Future<double> _measureAverageMicroseconds() async {
await _measurePreparedCycles(_warmupMillis);
return _measurePreparedCycles(_measurementMillis);
}
Future<void> _resetForStorageSample() async {
_valueSeq = 0;
_lastDatabaseSize = 0;
_lastRowsCount = 0;
_seededRows = [];
await clearUserTables(_plainSession);
await _plainSession.db.unsafeExecute('VACUUM');
CrdtUserManager.clearCache();
HlcManager.reset();
final wrapped = CrdtDatabaseSession.wraps(
_plainSession,
syncTables: benchmarkSyncTables,
);
await wrapped.db.initialize();
_crdtSession = crdtEnabled ? wrapped : null;
}
@override
Future<double> measure() async {
await setup();
try {
return await _measureAverageMicroseconds();
} finally {
await teardown();
}
}
@override
Future<(double, int)> report() async {
await setup();
try {
final averageMicroseconds = await _measureAverageMicroseconds();
await _resetForStorageSample();
await _prepareCycle();
await run();
await _captureDatabaseSize();
return (averageMicroseconds, measureStorage());
} finally {
await teardown();
}
}
@override
Future<void> run() async {
switch (operation) {
case Operation.insert:
_seededRows = await _insertTypes(rowCount);
_lastRowsCount = _seededRows.length;
return;
case Operation.update:
await _updateTypesInPlace();
_lastRowsCount = _seededRows.length;
return;
case Operation.delete:
await _deleteTypes(_seededRows);
_seededRows = [];
_lastRowsCount = 0;
return;
}
}
@override
Future<void> teardown() async {
if (!_hasOpenSession) {
return;
}
await _captureDatabaseSize();
await _plainSession.close();
if (_dbFile.existsSync()) {
_dbFile.deleteSync();
}
for (final companion in [
File('${_dbFile.path}-wal'),
File('${_dbFile.path}-shm'),
]) {
if (companion.existsSync()) {
companion.deleteSync();
}
}
_hasOpenSession = false;
_deleteDatabaseFiles(_dbFile);
}
int measureStorage() {
@ -279,6 +345,57 @@ class TypesTableBenchmark extends AsyncBenchmarkBase {
}
}
class TypesTableStorageBenchmark extends TypesTableBenchmark {
TypesTableStorageBenchmark(
super.name, {
required super.crdtEnabled,
required super.rowCount,
}) : super(operation: Operation.insert);
Future<StorageBenchmarkRun> reportStorage() async {
await setup();
try {
await _plainSession.db.unsafeExecute('VACUUM');
final baseDatabaseSize = await _captureCurrentDatabaseSize();
_seededRows = await _insertTypes(rowCount);
_lastRowsCount = _seededRows.length;
final insertOnlySize = await _captureNetDatabaseSize(baseDatabaseSize);
await _updateTypesInPlace();
_lastRowsCount = _seededRows.length;
final insertUpdateSize = await _captureNetDatabaseSize(baseDatabaseSize);
await _deleteTypes(_seededRows);
_seededRows = [];
_lastRowsCount = 0;
final insertUpdateDeleteSize = await _captureNetDatabaseSize(
baseDatabaseSize,
);
return StorageBenchmarkRun(
baseDatabaseSize: baseDatabaseSize,
insertOnlySize: insertOnlySize,
insertUpdateSize: insertUpdateSize,
insertUpdateDeleteSize: insertUpdateDeleteSize,
);
} finally {
await teardown();
}
}
Future<int> _captureCurrentDatabaseSize() async {
await _captureDatabaseSize();
return _lastDatabaseSize;
}
Future<int> _captureNetDatabaseSize(int baseDatabaseSize) async {
final currentDatabaseSize = await _captureCurrentDatabaseSize();
final netDatabaseSize = currentDatabaseSize - baseDatabaseSize;
return netDatabaseSize < 0 ? 0 : netDatabaseSize;
}
}
/// Bytes used by SQLite for [mainDb], including WAL sidecars (often omitted by
/// `mainDb.lengthSync()` alone).
int _sqliteDbFootprintBytes(File mainDb) {
@ -293,3 +410,17 @@ int _sqliteDbFootprintBytes(File mainDb) {
}
return total;
}
void _deleteDatabaseFiles(File mainDb) {
if (mainDb.existsSync()) {
mainDb.deleteSync();
}
for (final companion in [
File('${mainDb.path}-wal'),
File('${mainDb.path}-shm'),
]) {
if (companion.existsSync()) {
companion.deleteSync();
}
}
}

145
benchmark/utils/results.dart
View file

@ -12,8 +12,8 @@ void printPerformanceImpact(
'${runningInCI ? '```' : ''}'
'\n📊 ${benchmarkResults.operation.name.toUpperCase()} performance impact:',
);
final (baselineTime, _) = benchmarkResults.baseline;
final (crdtTime, _) = benchmarkResults.crdt;
final baselineTime = benchmarkResults.baseline;
final crdtTime = benchmarkResults.crdt;
final runDelayUs = crdtTime - baselineTime;
final slowdown = runDelayUs / baselineTime * 100;
@ -32,27 +32,75 @@ void printPerformanceImpact(
}
void printStorageImpact(
BenchmarkResults benchmarkResults, {
StorageBenchmarkResults benchmarkResults, {
required int rowCount,
bool runningInCI = false,
}) {
print('${runningInCI ? '```' : ''}\n💽 Storage impact:');
final (_, baselineSize) = benchmarkResults.baseline;
final (_, crdtSize) = benchmarkResults.crdt;
final stages = [
_StorageStageComparison(
title: 'INSERT only',
baselineSize: benchmarkResults.baseline.sizeFor(StorageStage.insert),
crdtSize: benchmarkResults.crdt.sizeFor(StorageStage.insert),
baselineIncrement: benchmarkResults.baseline.sizeFor(StorageStage.insert),
crdtIncrement: benchmarkResults.crdt.sizeFor(StorageStage.insert),
unitCount: rowCount,
unitLabel: 'inserted row',
),
_StorageStageComparison(
title: 'INSERT + UPDATE',
baselineSize: benchmarkResults.baseline.sizeFor(StorageStage.update),
crdtSize: benchmarkResults.crdt.sizeFor(StorageStage.update),
baselineIncrement:
benchmarkResults.baseline.sizeFor(StorageStage.update) -
benchmarkResults.baseline.previousSizeFor(StorageStage.update),
crdtIncrement:
benchmarkResults.crdt.sizeFor(StorageStage.update) -
benchmarkResults.crdt.previousSizeFor(StorageStage.update),
unitCount: rowCount * TypesTableBenchmark.updatedColumnsPerRow,
unitLabel: 'updated column',
),
_StorageStageComparison(
title: 'INSERT + UPDATE + DELETE',
baselineSize: benchmarkResults.baseline.sizeFor(StorageStage.delete),
crdtSize: benchmarkResults.crdt.sizeFor(StorageStage.delete),
baselineIncrement:
benchmarkResults.baseline.sizeFor(StorageStage.delete) -
benchmarkResults.baseline.previousSizeFor(StorageStage.delete),
crdtIncrement:
benchmarkResults.crdt.sizeFor(StorageStage.delete) -
benchmarkResults.crdt.previousSizeFor(StorageStage.delete),
unitCount: rowCount,
unitLabel: 'deleted row',
),
];
final storageIncrease = crdtSize - baselineSize;
final storageIncPercent = storageIncrease / baselineSize * 100;
final extraStoragePerRow = storageIncrease / rowCount;
final storageIncreaseVolume = storageIncPercent > 100
? '(x${formatter2.format(storageIncPercent / 100)})'
: '(+${(crdtSize - baselineSize).toFormattedStorageSize()})';
print('${runningInCI ? '```' : ''}\n💽 Storage impact (base footprint removed):');
print(
' Storage size: ${baselineSize.toFormattedStorageSize()} '
'--> ${crdtSize.toFormattedStorageSize()} $storageIncreaseVolume',
' Base database size: '
'${benchmarkResults.baseline.baseDatabaseSize.toFormattedStorageSize()} '
'--> ${benchmarkResults.crdt.baseDatabaseSize.toFormattedStorageSize()}',
);
print(' CRDT overhead: ${formatter2.format(storageIncPercent)}% increase');
for (final stage in stages) {
print(' ${stage.title}:');
print(
' Net storage: ${stage.baselineSize.toFormattedStorageSize()} '
'--> ${stage.crdtSize.toFormattedStorageSize()} '
'(${_formatSignedStorage(stage.extraBytes)})',
);
print(
' CRDT overhead: ${_formatPercentageIncrease(stage.overheadPercent)}',
);
print(
' Extra storage per ${stage.unitLabel}: '
'${_formatSignedStorage(stage.incrementalExtraBytesPerUnit)}',
);
}
print(
' Extra storage per row: ${extraStoragePerRow.toFormattedStorageSize()}',
' Storage overhead range: '
'${_formatPercentageValue(stages.first.overheadPercent)}% '
'--> ${_formatPercentageValue(stages.last.overheadPercent)}%',
);
if (runningInCI) print('```');
}
@ -65,6 +113,67 @@ class BenchmarkResults {
});
final Operation operation;
final (double, int) baseline;
final (double, int) crdt;
final double baseline;
final double crdt;
}
class StorageBenchmarkResults {
const StorageBenchmarkResults({
required this.baseline,
required this.crdt,
});
final StorageBenchmarkRun baseline;
final StorageBenchmarkRun crdt;
}
class _StorageStageComparison {
const _StorageStageComparison({
required this.title,
required this.baselineSize,
required this.crdtSize,
required this.baselineIncrement,
required this.crdtIncrement,
required this.unitCount,
required this.unitLabel,
});
final String title;
final int baselineSize;
final int crdtSize;
final int baselineIncrement;
final int crdtIncrement;
final int unitCount;
final String unitLabel;
int get extraBytes => crdtSize - baselineSize;
double? get overheadPercent =>
baselineSize == 0 ? null : extraBytes / baselineSize * 100;
double get incrementalExtraBytesPerUnit =>
(crdtIncrement - baselineIncrement) / unitCount;
}
String _formatSignedStorage(num size) {
if (size == 0) {
return '0 B';
}
final prefix = size > 0 ? '+' : '-';
return '$prefix${size.abs().toFormattedStorageSize()}';
}
String _formatPercentageIncrease(double? percent) {
if (percent == null) {
return 'n/a (baseline storage is zero)';
}
return '${formatter2.format(percent)}% increase';
}
String _formatPercentageValue(double? percent) {
if (percent == null) {
return 'n/a';
}
return formatter2.format(percent);
}