fix: Stabilize benchmark sampling and expand storage reporting (#37)
Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: Marcelo Mendonça Soares <marcelo.me.soares@gmail.com> Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>
This commit is contained in:
Copilot
GitHub
parent
3545bbba1d
commit
0307094d58
2 changed files with 347 additions and 86 deletions
|
|
@ -13,6 +13,34 @@ import 'tables.dart';
|
|||
|
||||
enum Operation { insert, update, delete }
|
||||
|
||||
enum StorageStage { insert, update, delete }
|
||||
|
||||
class StorageBenchmarkRun {
|
||||
const StorageBenchmarkRun({
|
||||
required this.baseDatabaseSize,
|
||||
required this.insertOnlySize,
|
||||
required this.insertUpdateSize,
|
||||
required this.insertUpdateDeleteSize,
|
||||
});
|
||||
|
||||
final int baseDatabaseSize;
|
||||
final int insertOnlySize;
|
||||
final int insertUpdateSize;
|
||||
final int insertUpdateDeleteSize;
|
||||
|
||||
int sizeFor(StorageStage stage) => switch (stage) {
|
||||
StorageStage.insert => insertOnlySize,
|
||||
StorageStage.update => insertUpdateSize,
|
||||
StorageStage.delete => insertUpdateDeleteSize,
|
||||
};
|
||||
|
||||
int previousSizeFor(StorageStage stage) => switch (stage) {
|
||||
StorageStage.insert => 0,
|
||||
StorageStage.update => insertOnlySize,
|
||||
StorageStage.delete => insertUpdateSize,
|
||||
};
|
||||
}
|
||||
|
||||
/// Benchmarks plain SQLite writes vs CRDT-wrapped writes using the wide [Types]
|
||||
/// row (same shape idea as legacy Drift `TableWithEveryColumnType`).
|
||||
class TypesTableBenchmark extends AsyncBenchmarkBase {
|
||||
|
|
@ -25,6 +53,9 @@ class TypesTableBenchmark extends AsyncBenchmarkBase {
|
|||
|
||||
/// Minimal blob column payload (shared; column stays non-null with zero bytes).
|
||||
static final ByteData _emptyBlob = ByteData(0);
|
||||
static const _warmupMillis = 100;
|
||||
static const _measurementMillis = 2000;
|
||||
static const updatedColumnsPerRow = 8;
|
||||
|
||||
final bool crdtEnabled;
|
||||
final Operation operation;
|
||||
|
|
@ -35,6 +66,7 @@ class TypesTableBenchmark extends AsyncBenchmarkBase {
|
|||
late final File _dbFile;
|
||||
late final ClientDatabaseSession _plainSession;
|
||||
CrdtDatabaseSession? _crdtSession;
|
||||
var _hasOpenSession = false;
|
||||
|
||||
final UuidValue _userId = const Uuid().v7obj();
|
||||
|
||||
|
|
@ -158,108 +190,142 @@ class TypesTableBenchmark extends AsyncBenchmarkBase {
|
|||
@override
|
||||
Future<void> setup() async {
|
||||
_valueSeq = 0;
|
||||
_lastDatabaseSize = 0;
|
||||
_lastRowsCount = 0;
|
||||
final dbPath = p.join(
|
||||
Directory.systemTemp.path,
|
||||
'offline_sync_benchmark_$name.db',
|
||||
);
|
||||
_dbFile = File(dbPath);
|
||||
if (_dbFile.existsSync()) {
|
||||
_dbFile.deleteSync();
|
||||
}
|
||||
_deleteDatabaseFiles(_dbFile);
|
||||
|
||||
_plainSession = await Client(_clientUrl).createSession(
|
||||
_dbFile.path,
|
||||
isDebugMode: true,
|
||||
);
|
||||
_hasOpenSession = true;
|
||||
|
||||
await _bootstrapDatabase();
|
||||
}
|
||||
|
||||
Future<void> _prepareCycle() async {
|
||||
if (_seededRows.isNotEmpty) {
|
||||
await _deleteTypes(_seededRows);
|
||||
_seededRows = [];
|
||||
}
|
||||
switch (operation) {
|
||||
case Operation.insert:
|
||||
_seededRows = [];
|
||||
return;
|
||||
case Operation.update:
|
||||
_seededRows = await _insertTypes(rowCount);
|
||||
return;
|
||||
case Operation.delete:
|
||||
_seededRows = [];
|
||||
_seededRows = await _insertTypes(rowCount);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
@override
|
||||
Future<void> warmup() async {}
|
||||
Future<double> _measurePreparedCycles(int minimumMillis) async {
|
||||
final minimumMicros = minimumMillis * 1000;
|
||||
final watch = Stopwatch()..start();
|
||||
var totalTimedMicros = 0.0;
|
||||
var timedIterations = 0;
|
||||
|
||||
@override
|
||||
Future<void> run() async {
|
||||
switch (operation) {
|
||||
case Operation.insert:
|
||||
await _insertTypes(rowCount);
|
||||
case Operation.update:
|
||||
await _updateTypesInPlace();
|
||||
case Operation.delete:
|
||||
await _deleteTypes(_seededRows);
|
||||
while (watch.elapsedMicroseconds < minimumMicros) {
|
||||
await _prepareCycle();
|
||||
final sw = Stopwatch()..start();
|
||||
await run();
|
||||
totalTimedMicros += sw.elapsedMicroseconds;
|
||||
timedIterations++;
|
||||
}
|
||||
|
||||
return totalTimedMicros / timedIterations;
|
||||
}
|
||||
|
||||
Future<void> _prepareDeleteCycle() async {
|
||||
await _bootstrapDatabase();
|
||||
_seededRows = await _insertTypes(rowCount);
|
||||
}
|
||||
|
||||
Future<void> _timedDeleteBatch() async {
|
||||
await _deleteTypes(_seededRows);
|
||||
}
|
||||
|
||||
@override
|
||||
Future<double> measure() async {
|
||||
if (operation == Operation.delete) {
|
||||
await setup();
|
||||
try {
|
||||
await AsyncBenchmarkBase.measureFor(() async {}, 100);
|
||||
const minimumMicros = 2000 * 1000;
|
||||
final watch = Stopwatch()..start();
|
||||
var totalTimedMicros = 0.0;
|
||||
var timedIterations = 0;
|
||||
while (watch.elapsedMicroseconds < minimumMicros) {
|
||||
await _prepareDeleteCycle();
|
||||
final sw = Stopwatch()..start();
|
||||
await _timedDeleteBatch();
|
||||
totalTimedMicros += sw.elapsedMicroseconds;
|
||||
timedIterations++;
|
||||
}
|
||||
return totalTimedMicros / timedIterations;
|
||||
} finally {
|
||||
await teardown();
|
||||
}
|
||||
}
|
||||
return super.measure();
|
||||
}
|
||||
|
||||
@override
|
||||
Future<(double, int)> report() async {
|
||||
final averageMicroseconds = await measure();
|
||||
return (averageMicroseconds, measureStorage());
|
||||
}
|
||||
|
||||
@override
|
||||
Future<void> teardown() async {
|
||||
Future<void> _captureDatabaseSize() async {
|
||||
try {
|
||||
await _plainSession.db.unsafeExecute('PRAGMA wal_checkpoint(TRUNCATE)');
|
||||
} on Exception {
|
||||
// WAL checkpoint is best-effort (e.g. non-SQLite tests).
|
||||
}
|
||||
_lastDatabaseSize = _sqliteDbFootprintBytes(_dbFile);
|
||||
_lastRowsCount = await Types.db.count(_plainSession);
|
||||
}
|
||||
|
||||
Future<double> _measureAverageMicroseconds() async {
|
||||
await _measurePreparedCycles(_warmupMillis);
|
||||
return _measurePreparedCycles(_measurementMillis);
|
||||
}
|
||||
|
||||
Future<void> _resetForStorageSample() async {
|
||||
_valueSeq = 0;
|
||||
_lastDatabaseSize = 0;
|
||||
_lastRowsCount = 0;
|
||||
_seededRows = [];
|
||||
await clearUserTables(_plainSession);
|
||||
await _plainSession.db.unsafeExecute('VACUUM');
|
||||
CrdtUserManager.clearCache();
|
||||
HlcManager.reset();
|
||||
final wrapped = CrdtDatabaseSession.wraps(
|
||||
_plainSession,
|
||||
syncTables: benchmarkSyncTables,
|
||||
);
|
||||
await wrapped.db.initialize();
|
||||
_crdtSession = crdtEnabled ? wrapped : null;
|
||||
}
|
||||
|
||||
@override
|
||||
Future<double> measure() async {
|
||||
await setup();
|
||||
try {
|
||||
return await _measureAverageMicroseconds();
|
||||
} finally {
|
||||
await teardown();
|
||||
}
|
||||
}
|
||||
|
||||
@override
|
||||
Future<(double, int)> report() async {
|
||||
await setup();
|
||||
try {
|
||||
final averageMicroseconds = await _measureAverageMicroseconds();
|
||||
await _resetForStorageSample();
|
||||
await _prepareCycle();
|
||||
await run();
|
||||
await _captureDatabaseSize();
|
||||
return (averageMicroseconds, measureStorage());
|
||||
} finally {
|
||||
await teardown();
|
||||
}
|
||||
}
|
||||
|
||||
@override
|
||||
Future<void> run() async {
|
||||
switch (operation) {
|
||||
case Operation.insert:
|
||||
_seededRows = await _insertTypes(rowCount);
|
||||
_lastRowsCount = _seededRows.length;
|
||||
return;
|
||||
case Operation.update:
|
||||
await _updateTypesInPlace();
|
||||
_lastRowsCount = _seededRows.length;
|
||||
return;
|
||||
case Operation.delete:
|
||||
await _deleteTypes(_seededRows);
|
||||
_seededRows = [];
|
||||
_lastRowsCount = 0;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
@override
|
||||
Future<void> teardown() async {
|
||||
if (!_hasOpenSession) {
|
||||
return;
|
||||
}
|
||||
await _captureDatabaseSize();
|
||||
await _plainSession.close();
|
||||
if (_dbFile.existsSync()) {
|
||||
_dbFile.deleteSync();
|
||||
}
|
||||
for (final companion in [
|
||||
File('${_dbFile.path}-wal'),
|
||||
File('${_dbFile.path}-shm'),
|
||||
]) {
|
||||
if (companion.existsSync()) {
|
||||
companion.deleteSync();
|
||||
}
|
||||
}
|
||||
_hasOpenSession = false;
|
||||
_deleteDatabaseFiles(_dbFile);
|
||||
}
|
||||
|
||||
int measureStorage() {
|
||||
|
|
@ -279,6 +345,57 @@ class TypesTableBenchmark extends AsyncBenchmarkBase {
|
|||
}
|
||||
}
|
||||
|
||||
class TypesTableStorageBenchmark extends TypesTableBenchmark {
|
||||
TypesTableStorageBenchmark(
|
||||
super.name, {
|
||||
required super.crdtEnabled,
|
||||
required super.rowCount,
|
||||
}) : super(operation: Operation.insert);
|
||||
|
||||
Future<StorageBenchmarkRun> reportStorage() async {
|
||||
await setup();
|
||||
try {
|
||||
await _plainSession.db.unsafeExecute('VACUUM');
|
||||
final baseDatabaseSize = await _captureCurrentDatabaseSize();
|
||||
|
||||
_seededRows = await _insertTypes(rowCount);
|
||||
_lastRowsCount = _seededRows.length;
|
||||
final insertOnlySize = await _captureNetDatabaseSize(baseDatabaseSize);
|
||||
|
||||
await _updateTypesInPlace();
|
||||
_lastRowsCount = _seededRows.length;
|
||||
final insertUpdateSize = await _captureNetDatabaseSize(baseDatabaseSize);
|
||||
|
||||
await _deleteTypes(_seededRows);
|
||||
_seededRows = [];
|
||||
_lastRowsCount = 0;
|
||||
final insertUpdateDeleteSize = await _captureNetDatabaseSize(
|
||||
baseDatabaseSize,
|
||||
);
|
||||
|
||||
return StorageBenchmarkRun(
|
||||
baseDatabaseSize: baseDatabaseSize,
|
||||
insertOnlySize: insertOnlySize,
|
||||
insertUpdateSize: insertUpdateSize,
|
||||
insertUpdateDeleteSize: insertUpdateDeleteSize,
|
||||
);
|
||||
} finally {
|
||||
await teardown();
|
||||
}
|
||||
}
|
||||
|
||||
Future<int> _captureCurrentDatabaseSize() async {
|
||||
await _captureDatabaseSize();
|
||||
return _lastDatabaseSize;
|
||||
}
|
||||
|
||||
Future<int> _captureNetDatabaseSize(int baseDatabaseSize) async {
|
||||
final currentDatabaseSize = await _captureCurrentDatabaseSize();
|
||||
final netDatabaseSize = currentDatabaseSize - baseDatabaseSize;
|
||||
return netDatabaseSize < 0 ? 0 : netDatabaseSize;
|
||||
}
|
||||
}
|
||||
|
||||
/// Bytes used by SQLite for [mainDb], including WAL sidecars (often omitted by
|
||||
/// `mainDb.lengthSync()` alone).
|
||||
int _sqliteDbFootprintBytes(File mainDb) {
|
||||
|
|
@ -293,3 +410,17 @@ int _sqliteDbFootprintBytes(File mainDb) {
|
|||
}
|
||||
return total;
|
||||
}
|
||||
|
||||
void _deleteDatabaseFiles(File mainDb) {
|
||||
if (mainDb.existsSync()) {
|
||||
mainDb.deleteSync();
|
||||
}
|
||||
for (final companion in [
|
||||
File('${mainDb.path}-wal'),
|
||||
File('${mainDb.path}-shm'),
|
||||
]) {
|
||||
if (companion.existsSync()) {
|
||||
companion.deleteSync();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -12,8 +12,8 @@ void printPerformanceImpact(
|
|||
'${runningInCI ? '```' : ''}'
|
||||
'\n📊 ${benchmarkResults.operation.name.toUpperCase()} performance impact:',
|
||||
);
|
||||
final (baselineTime, _) = benchmarkResults.baseline;
|
||||
final (crdtTime, _) = benchmarkResults.crdt;
|
||||
final baselineTime = benchmarkResults.baseline;
|
||||
final crdtTime = benchmarkResults.crdt;
|
||||
|
||||
final runDelayUs = crdtTime - baselineTime;
|
||||
final slowdown = runDelayUs / baselineTime * 100;
|
||||
|
|
@ -32,27 +32,75 @@ void printPerformanceImpact(
|
|||
}
|
||||
|
||||
void printStorageImpact(
|
||||
BenchmarkResults benchmarkResults, {
|
||||
StorageBenchmarkResults benchmarkResults, {
|
||||
required int rowCount,
|
||||
bool runningInCI = false,
|
||||
}) {
|
||||
print('${runningInCI ? '```' : ''}\n💽 Storage impact:');
|
||||
final (_, baselineSize) = benchmarkResults.baseline;
|
||||
final (_, crdtSize) = benchmarkResults.crdt;
|
||||
final stages = [
|
||||
_StorageStageComparison(
|
||||
title: 'INSERT only',
|
||||
baselineSize: benchmarkResults.baseline.sizeFor(StorageStage.insert),
|
||||
crdtSize: benchmarkResults.crdt.sizeFor(StorageStage.insert),
|
||||
baselineIncrement: benchmarkResults.baseline.sizeFor(StorageStage.insert),
|
||||
crdtIncrement: benchmarkResults.crdt.sizeFor(StorageStage.insert),
|
||||
unitCount: rowCount,
|
||||
unitLabel: 'inserted row',
|
||||
),
|
||||
_StorageStageComparison(
|
||||
title: 'INSERT + UPDATE',
|
||||
baselineSize: benchmarkResults.baseline.sizeFor(StorageStage.update),
|
||||
crdtSize: benchmarkResults.crdt.sizeFor(StorageStage.update),
|
||||
baselineIncrement:
|
||||
benchmarkResults.baseline.sizeFor(StorageStage.update) -
|
||||
benchmarkResults.baseline.previousSizeFor(StorageStage.update),
|
||||
crdtIncrement:
|
||||
benchmarkResults.crdt.sizeFor(StorageStage.update) -
|
||||
benchmarkResults.crdt.previousSizeFor(StorageStage.update),
|
||||
unitCount: rowCount * TypesTableBenchmark.updatedColumnsPerRow,
|
||||
unitLabel: 'updated column',
|
||||
),
|
||||
_StorageStageComparison(
|
||||
title: 'INSERT + UPDATE + DELETE',
|
||||
baselineSize: benchmarkResults.baseline.sizeFor(StorageStage.delete),
|
||||
crdtSize: benchmarkResults.crdt.sizeFor(StorageStage.delete),
|
||||
baselineIncrement:
|
||||
benchmarkResults.baseline.sizeFor(StorageStage.delete) -
|
||||
benchmarkResults.baseline.previousSizeFor(StorageStage.delete),
|
||||
crdtIncrement:
|
||||
benchmarkResults.crdt.sizeFor(StorageStage.delete) -
|
||||
benchmarkResults.crdt.previousSizeFor(StorageStage.delete),
|
||||
unitCount: rowCount,
|
||||
unitLabel: 'deleted row',
|
||||
),
|
||||
];
|
||||
|
||||
final storageIncrease = crdtSize - baselineSize;
|
||||
final storageIncPercent = storageIncrease / baselineSize * 100;
|
||||
final extraStoragePerRow = storageIncrease / rowCount;
|
||||
final storageIncreaseVolume = storageIncPercent > 100
|
||||
? '(x${formatter2.format(storageIncPercent / 100)})'
|
||||
: '(+${(crdtSize - baselineSize).toFormattedStorageSize()})';
|
||||
print('${runningInCI ? '```' : ''}\n💽 Storage impact (base footprint removed):');
|
||||
print(
|
||||
' Storage size: ${baselineSize.toFormattedStorageSize()} '
|
||||
'--> ${crdtSize.toFormattedStorageSize()} $storageIncreaseVolume',
|
||||
' Base database size: '
|
||||
'${benchmarkResults.baseline.baseDatabaseSize.toFormattedStorageSize()} '
|
||||
'--> ${benchmarkResults.crdt.baseDatabaseSize.toFormattedStorageSize()}',
|
||||
);
|
||||
print(' CRDT overhead: ${formatter2.format(storageIncPercent)}% increase');
|
||||
|
||||
for (final stage in stages) {
|
||||
print(' ${stage.title}:');
|
||||
print(
|
||||
' Net storage: ${stage.baselineSize.toFormattedStorageSize()} '
|
||||
'--> ${stage.crdtSize.toFormattedStorageSize()} '
|
||||
'(${_formatSignedStorage(stage.extraBytes)})',
|
||||
);
|
||||
print(
|
||||
' CRDT overhead: ${_formatPercentageIncrease(stage.overheadPercent)}',
|
||||
);
|
||||
print(
|
||||
' Extra storage per ${stage.unitLabel}: '
|
||||
'${_formatSignedStorage(stage.incrementalExtraBytesPerUnit)}',
|
||||
);
|
||||
}
|
||||
|
||||
print(
|
||||
' Extra storage per row: ${extraStoragePerRow.toFormattedStorageSize()}',
|
||||
' Storage overhead range: '
|
||||
'${_formatPercentageValue(stages.first.overheadPercent)}% '
|
||||
'--> ${_formatPercentageValue(stages.last.overheadPercent)}%',
|
||||
);
|
||||
if (runningInCI) print('```');
|
||||
}
|
||||
|
|
@ -65,6 +113,67 @@ class BenchmarkResults {
|
|||
});
|
||||
|
||||
final Operation operation;
|
||||
final (double, int) baseline;
|
||||
final (double, int) crdt;
|
||||
final double baseline;
|
||||
final double crdt;
|
||||
}
|
||||
|
||||
class StorageBenchmarkResults {
|
||||
const StorageBenchmarkResults({
|
||||
required this.baseline,
|
||||
required this.crdt,
|
||||
});
|
||||
|
||||
final StorageBenchmarkRun baseline;
|
||||
final StorageBenchmarkRun crdt;
|
||||
}
|
||||
|
||||
class _StorageStageComparison {
|
||||
const _StorageStageComparison({
|
||||
required this.title,
|
||||
required this.baselineSize,
|
||||
required this.crdtSize,
|
||||
required this.baselineIncrement,
|
||||
required this.crdtIncrement,
|
||||
required this.unitCount,
|
||||
required this.unitLabel,
|
||||
});
|
||||
|
||||
final String title;
|
||||
final int baselineSize;
|
||||
final int crdtSize;
|
||||
final int baselineIncrement;
|
||||
final int crdtIncrement;
|
||||
final int unitCount;
|
||||
final String unitLabel;
|
||||
|
||||
int get extraBytes => crdtSize - baselineSize;
|
||||
double? get overheadPercent =>
|
||||
baselineSize == 0 ? null : extraBytes / baselineSize * 100;
|
||||
double get incrementalExtraBytesPerUnit =>
|
||||
(crdtIncrement - baselineIncrement) / unitCount;
|
||||
}
|
||||
|
||||
String _formatSignedStorage(num size) {
|
||||
if (size == 0) {
|
||||
return '0 B';
|
||||
}
|
||||
|
||||
final prefix = size > 0 ? '+' : '-';
|
||||
return '$prefix${size.abs().toFormattedStorageSize()}';
|
||||
}
|
||||
|
||||
String _formatPercentageIncrease(double? percent) {
|
||||
if (percent == null) {
|
||||
return 'n/a (baseline storage is zero)';
|
||||
}
|
||||
|
||||
return '${formatter2.format(percent)}% increase';
|
||||
}
|
||||
|
||||
String _formatPercentageValue(double? percent) {
|
||||
if (percent == null) {
|
||||
return 'n/a';
|
||||
}
|
||||
|
||||
return formatter2.format(percent);
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue