fix: Stabilize benchmark sampling and expand storage reporting (#37)

Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: Marcelo Mendonça Soares <marcelo.me.soares@gmail.com>
Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>

benchmark/run.dart

@@ -34,7 +34,7 @@ Future<void> main(List<String> args) async {
         crdtEnabled: false,
         operation: operation,
         rowCount: rowCount,
-      ).report(),
+      ).measure(),
       skipProgress: runningInCI,
     );
 
@@ -45,8 +45,8 @@ Future<void> main(List<String> args) async {
         crdtEnabled: true,
         operation: operation,
         rowCount: rowCount,
-      ).report(),
+      ).measure(),
-      validator: (result) => result.$1 > baselineResult.$1,
+      validator: (result) => result > baselineResult,
       skipProgress: runningInCI,
     );
 
@@ -60,7 +60,7 @@ Future<void> main(List<String> args) async {
   }
 
   final spuriousBenchmarks = benchmarkResults.where(
-    (result) => result.baseline.$1 > result.crdt.$1,
+    (result) => result.baseline > result.crdt,
   );
 
   if (spuriousBenchmarks.isNotEmpty) {
@@ -74,8 +74,8 @@ Future<void> main(List<String> args) async {
     );
 
     for (final result in spuriousBenchmarks) {
-      final baselineDelay = result.baseline.$1.toFormattedDuration();
+      final baselineDelay = result.baseline.toFormattedDuration();
-      final crdtDelay = result.crdt.$1.toFormattedDuration();
+      final crdtDelay = result.crdt.toFormattedDuration();
       print(
         '  - ${result.operation.name.toUpperCase()}: $baselineDelay > $crdtDelay',
       );
@@ -89,8 +89,29 @@ Future<void> main(List<String> args) async {
       runningInCI: runningInCI,
     );
   }
+  final baselineStorageResult = await runWithProgress(
+    'Running storage benchmark (baseline)',
+    () => TypesTableStorageBenchmark(
+      'storage (baseline)',
+      crdtEnabled: false,
+      rowCount: rowCount,
+    ).reportStorage(),
+    skipProgress: runningInCI,
+  );
+  final crdtStorageResult = await runWithProgress(
+    'Running storage benchmark (CRDT)',
+    () => TypesTableStorageBenchmark(
+      'storage (CRDT)',
+      crdtEnabled: true,
+      rowCount: rowCount,
+    ).reportStorage(),
+    skipProgress: runningInCI,
+  );
   printStorageImpact(
-    benchmarkResults.first,
+    StorageBenchmarkResults(
+      baseline: baselineStorageResult,
+      crdt: crdtStorageResult,
+    ),
     rowCount: rowCount,
     runningInCI: runningInCI,
   );

benchmark/utils/benchmark.dart

@@ -13,6 +13,34 @@ import 'tables.dart';
 
 enum Operation { insert, update, delete }
 
+enum StorageStage { insert, update, delete }
+
+class StorageBenchmarkRun {
+  const StorageBenchmarkRun({
+    required this.baseDatabaseSize,
+    required this.insertOnlySize,
+    required this.insertUpdateSize,
+    required this.insertUpdateDeleteSize,
+  });
+
+  final int baseDatabaseSize;
+  final int insertOnlySize;
+  final int insertUpdateSize;
+  final int insertUpdateDeleteSize;
+
+  int sizeFor(StorageStage stage) => switch (stage) {
+    StorageStage.insert => insertOnlySize,
+    StorageStage.update => insertUpdateSize,
+    StorageStage.delete => insertUpdateDeleteSize,
+  };
+
+  int previousSizeFor(StorageStage stage) => switch (stage) {
+    StorageStage.insert => 0,
+    StorageStage.update => insertOnlySize,
+    StorageStage.delete => insertUpdateSize,
+  };
+}
+
 /// Benchmarks plain SQLite writes vs CRDT-wrapped writes using the wide [Types]
 /// row (same shape idea as legacy Drift `TableWithEveryColumnType`).
 class TypesTableBenchmark extends AsyncBenchmarkBase {
@@ -25,6 +53,9 @@ class TypesTableBenchmark extends AsyncBenchmarkBase {
 
   /// Minimal blob column payload (shared; column stays non-null with zero bytes).
   static final ByteData _emptyBlob = ByteData(0);
+  static const _warmupMillis = 100;
+  static const _measurementMillis = 2000;
+  static const updatedColumnsPerRow = 8;
 
   final bool crdtEnabled;
   final Operation operation;
@@ -35,6 +66,7 @@ class TypesTableBenchmark extends AsyncBenchmarkBase {
   late final File _dbFile;
   late final ClientDatabaseSession _plainSession;
   CrdtDatabaseSession? _crdtSession;
+  var _hasOpenSession = false;
 
   final UuidValue _userId = const Uuid().v7obj();
 
@@ -158,108 +190,142 @@ class TypesTableBenchmark extends AsyncBenchmarkBase {
   @override
   Future<void> setup() async {
     _valueSeq = 0;
+    _lastDatabaseSize = 0;
+    _lastRowsCount = 0;
     final dbPath = p.join(
       Directory.systemTemp.path,
       'offline_sync_benchmark_$name.db',
     );
     _dbFile = File(dbPath);
-    if (_dbFile.existsSync()) {
+    _deleteDatabaseFiles(_dbFile);
-      _dbFile.deleteSync();
-    }
 
     _plainSession = await Client(_clientUrl).createSession(
       _dbFile.path,
       isDebugMode: true,
     );
+    _hasOpenSession = true;
 
     await _bootstrapDatabase();
+  }
 
+  Future<void> _prepareCycle() async {
+    if (_seededRows.isNotEmpty) {
+      await _deleteTypes(_seededRows);
+      _seededRows = [];
+    }
     switch (operation) {
       case Operation.insert:
-        _seededRows = [];
+        return;
       case Operation.update:
         _seededRows = await _insertTypes(rowCount);
+        return;
       case Operation.delete:
-        _seededRows = [];
+        _seededRows = await _insertTypes(rowCount);
+        return;
     }
   }
 
-  @override
+  Future<double> _measurePreparedCycles(int minimumMillis) async {
-  Future<void> warmup() async {}
+    final minimumMicros = minimumMillis * 1000;
+    final watch = Stopwatch()..start();
+    var totalTimedMicros = 0.0;
+    var timedIterations = 0;
 
-  @override
+    while (watch.elapsedMicroseconds < minimumMicros) {
-  Future<void> run() async {
+      await _prepareCycle();
-    switch (operation) {
+      final sw = Stopwatch()..start();
-      case Operation.insert:
+      await run();
-        await _insertTypes(rowCount);
+      totalTimedMicros += sw.elapsedMicroseconds;
-      case Operation.update:
+      timedIterations++;
-        await _updateTypesInPlace();
-      case Operation.delete:
-        await _deleteTypes(_seededRows);
     }
+
+    return totalTimedMicros / timedIterations;
   }
 
-  Future<void> _prepareDeleteCycle() async {
+  Future<void> _captureDatabaseSize() async {
-    await _bootstrapDatabase();
-    _seededRows = await _insertTypes(rowCount);
-  }
-
-  Future<void> _timedDeleteBatch() async {
-    await _deleteTypes(_seededRows);
-  }
-
-  @override
-  Future<double> measure() async {
-    if (operation == Operation.delete) {
-      await setup();
-      try {
-        await AsyncBenchmarkBase.measureFor(() async {}, 100);
-        const minimumMicros = 2000 * 1000;
-        final watch = Stopwatch()..start();
-        var totalTimedMicros = 0.0;
-        var timedIterations = 0;
-        while (watch.elapsedMicroseconds < minimumMicros) {
-          await _prepareDeleteCycle();
-          final sw = Stopwatch()..start();
-          await _timedDeleteBatch();
-          totalTimedMicros += sw.elapsedMicroseconds;
-          timedIterations++;
-        }
-        return totalTimedMicros / timedIterations;
-      } finally {
-        await teardown();
-      }
-    }
-    return super.measure();
-  }
-
-  @override
-  Future<(double, int)> report() async {
-    final averageMicroseconds = await measure();
-    return (averageMicroseconds, measureStorage());
-  }
-
-  @override
-  Future<void> teardown() async {
     try {
       await _plainSession.db.unsafeExecute('PRAGMA wal_checkpoint(TRUNCATE)');
     } on Exception {
       // WAL checkpoint is best-effort (e.g. non-SQLite tests).
     }
     _lastDatabaseSize = _sqliteDbFootprintBytes(_dbFile);
-    _lastRowsCount = await Types.db.count(_plainSession);
+  }
+
+  Future<double> _measureAverageMicroseconds() async {
+    await _measurePreparedCycles(_warmupMillis);
+    return _measurePreparedCycles(_measurementMillis);
+  }
+
+  Future<void> _resetForStorageSample() async {
+    _valueSeq = 0;
+    _lastDatabaseSize = 0;
+    _lastRowsCount = 0;
+    _seededRows = [];
+    await clearUserTables(_plainSession);
+    await _plainSession.db.unsafeExecute('VACUUM');
+    CrdtUserManager.clearCache();
+    HlcManager.reset();
+    final wrapped = CrdtDatabaseSession.wraps(
+      _plainSession,
+      syncTables: benchmarkSyncTables,
+    );
+    await wrapped.db.initialize();
+    _crdtSession = crdtEnabled ? wrapped : null;
+  }
+
+  @override
+  Future<double> measure() async {
+    await setup();
+    try {
+      return await _measureAverageMicroseconds();
+    } finally {
+      await teardown();
+    }
+  }
+
+  @override
+  Future<(double, int)> report() async {
+    await setup();
+    try {
+      final averageMicroseconds = await _measureAverageMicroseconds();
+      await _resetForStorageSample();
+      await _prepareCycle();
+      await run();
+      await _captureDatabaseSize();
+      return (averageMicroseconds, measureStorage());
+    } finally {
+      await teardown();
+    }
+  }
+
+  @override
+  Future<void> run() async {
+    switch (operation) {
+      case Operation.insert:
+        _seededRows = await _insertTypes(rowCount);
+        _lastRowsCount = _seededRows.length;
+        return;
+      case Operation.update:
+        await _updateTypesInPlace();
+        _lastRowsCount = _seededRows.length;
+        return;
+      case Operation.delete:
+        await _deleteTypes(_seededRows);
+        _seededRows = [];
+        _lastRowsCount = 0;
+        return;
+    }
+  }
+
+  @override
+  Future<void> teardown() async {
+    if (!_hasOpenSession) {
+      return;
+    }
+    await _captureDatabaseSize();
     await _plainSession.close();
-    if (_dbFile.existsSync()) {
+    _hasOpenSession = false;
-      _dbFile.deleteSync();
+    _deleteDatabaseFiles(_dbFile);
-    }
-    for (final companion in [
-      File('${_dbFile.path}-wal'),
-      File('${_dbFile.path}-shm'),
-    ]) {
-      if (companion.existsSync()) {
-        companion.deleteSync();
-      }
-    }
   }
 
   int measureStorage() {
@@ -279,6 +345,57 @@ class TypesTableBenchmark extends AsyncBenchmarkBase {
   }
 }
 
+class TypesTableStorageBenchmark extends TypesTableBenchmark {
+  TypesTableStorageBenchmark(
+    super.name, {
+    required super.crdtEnabled,
+    required super.rowCount,
+  }) : super(operation: Operation.insert);
+
+  Future<StorageBenchmarkRun> reportStorage() async {
+    await setup();
+    try {
+      await _plainSession.db.unsafeExecute('VACUUM');
+      final baseDatabaseSize = await _captureCurrentDatabaseSize();
+
+      _seededRows = await _insertTypes(rowCount);
+      _lastRowsCount = _seededRows.length;
+      final insertOnlySize = await _captureNetDatabaseSize(baseDatabaseSize);
+
+      await _updateTypesInPlace();
+      _lastRowsCount = _seededRows.length;
+      final insertUpdateSize = await _captureNetDatabaseSize(baseDatabaseSize);
+
+      await _deleteTypes(_seededRows);
+      _seededRows = [];
+      _lastRowsCount = 0;
+      final insertUpdateDeleteSize = await _captureNetDatabaseSize(
+        baseDatabaseSize,
+      );
+
+      return StorageBenchmarkRun(
+        baseDatabaseSize: baseDatabaseSize,
+        insertOnlySize: insertOnlySize,
+        insertUpdateSize: insertUpdateSize,
+        insertUpdateDeleteSize: insertUpdateDeleteSize,
+      );
+    } finally {
+      await teardown();
+    }
+  }
+
+  Future<int> _captureCurrentDatabaseSize() async {
+    await _captureDatabaseSize();
+    return _lastDatabaseSize;
+  }
+
+  Future<int> _captureNetDatabaseSize(int baseDatabaseSize) async {
+    final currentDatabaseSize = await _captureCurrentDatabaseSize();
+    final netDatabaseSize = currentDatabaseSize - baseDatabaseSize;
+    return netDatabaseSize < 0 ? 0 : netDatabaseSize;
+  }
+}
+
 /// Bytes used by SQLite for [mainDb], including WAL sidecars (often omitted by
 /// `mainDb.lengthSync()` alone).
 int _sqliteDbFootprintBytes(File mainDb) {
@@ -293,3 +410,17 @@ int _sqliteDbFootprintBytes(File mainDb) {
   }
   return total;
 }
+
+void _deleteDatabaseFiles(File mainDb) {
+  if (mainDb.existsSync()) {
+    mainDb.deleteSync();
+  }
+  for (final companion in [
+    File('${mainDb.path}-wal'),
+    File('${mainDb.path}-shm'),
+  ]) {
+    if (companion.existsSync()) {
+      companion.deleteSync();
+    }
+  }
+}

benchmark/utils/results.dart

@@ -12,8 +12,8 @@ void printPerformanceImpact(
     '${runningInCI ? '```' : ''}'
     '\n📊 ${benchmarkResults.operation.name.toUpperCase()} performance impact:',
   );
-  final (baselineTime, _) = benchmarkResults.baseline;
+  final baselineTime = benchmarkResults.baseline;
-  final (crdtTime, _) = benchmarkResults.crdt;
+  final crdtTime = benchmarkResults.crdt;
 
   final runDelayUs = crdtTime - baselineTime;
   final slowdown = runDelayUs / baselineTime * 100;
@@ -32,27 +32,75 @@ void printPerformanceImpact(
 }
 
 void printStorageImpact(
-  BenchmarkResults benchmarkResults, {
+  StorageBenchmarkResults benchmarkResults, {
   required int rowCount,
   bool runningInCI = false,
 }) {
-  print('${runningInCI ? '```' : ''}\n💽 Storage impact:');
+  final stages = [
-  final (_, baselineSize) = benchmarkResults.baseline;
+    _StorageStageComparison(
-  final (_, crdtSize) = benchmarkResults.crdt;
+      title: 'INSERT only',
+      baselineSize: benchmarkResults.baseline.sizeFor(StorageStage.insert),
+      crdtSize: benchmarkResults.crdt.sizeFor(StorageStage.insert),
+      baselineIncrement: benchmarkResults.baseline.sizeFor(StorageStage.insert),
+      crdtIncrement: benchmarkResults.crdt.sizeFor(StorageStage.insert),
+      unitCount: rowCount,
+      unitLabel: 'inserted row',
+    ),
+    _StorageStageComparison(
+      title: 'INSERT + UPDATE',
+      baselineSize: benchmarkResults.baseline.sizeFor(StorageStage.update),
+      crdtSize: benchmarkResults.crdt.sizeFor(StorageStage.update),
+      baselineIncrement:
+          benchmarkResults.baseline.sizeFor(StorageStage.update) -
+          benchmarkResults.baseline.previousSizeFor(StorageStage.update),
+      crdtIncrement:
+          benchmarkResults.crdt.sizeFor(StorageStage.update) -
+          benchmarkResults.crdt.previousSizeFor(StorageStage.update),
+      unitCount: rowCount * TypesTableBenchmark.updatedColumnsPerRow,
+      unitLabel: 'updated column',
+    ),
+    _StorageStageComparison(
+      title: 'INSERT + UPDATE + DELETE',
+      baselineSize: benchmarkResults.baseline.sizeFor(StorageStage.delete),
+      crdtSize: benchmarkResults.crdt.sizeFor(StorageStage.delete),
+      baselineIncrement:
+          benchmarkResults.baseline.sizeFor(StorageStage.delete) -
+          benchmarkResults.baseline.previousSizeFor(StorageStage.delete),
+      crdtIncrement:
+          benchmarkResults.crdt.sizeFor(StorageStage.delete) -
+          benchmarkResults.crdt.previousSizeFor(StorageStage.delete),
+      unitCount: rowCount,
+      unitLabel: 'deleted row',
+    ),
+  ];
 
-  final storageIncrease = crdtSize - baselineSize;
+  print('${runningInCI ? '```' : ''}\n💽 Storage impact (base footprint removed):');
-  final storageIncPercent = storageIncrease / baselineSize * 100;
-  final extraStoragePerRow = storageIncrease / rowCount;
-  final storageIncreaseVolume = storageIncPercent > 100
-      ? '(x${formatter2.format(storageIncPercent / 100)})'
-      : '(+${(crdtSize - baselineSize).toFormattedStorageSize()})';
   print(
-    '  Storage size: ${baselineSize.toFormattedStorageSize()} '
+    '  Base database size: '
-    '--> ${crdtSize.toFormattedStorageSize()} $storageIncreaseVolume',
+    '${benchmarkResults.baseline.baseDatabaseSize.toFormattedStorageSize()} '
+    '--> ${benchmarkResults.crdt.baseDatabaseSize.toFormattedStorageSize()}',
   );
-  print('  CRDT overhead: ${formatter2.format(storageIncPercent)}% increase');
+
+  for (final stage in stages) {
+    print('  ${stage.title}:');
+    print(
+      '    Net storage: ${stage.baselineSize.toFormattedStorageSize()} '
+      '--> ${stage.crdtSize.toFormattedStorageSize()} '
+      '(${_formatSignedStorage(stage.extraBytes)})',
+    );
+    print(
+      '    CRDT overhead: ${_formatPercentageIncrease(stage.overheadPercent)}',
+    );
+    print(
+      '    Extra storage per ${stage.unitLabel}: '
+      '${_formatSignedStorage(stage.incrementalExtraBytesPerUnit)}',
+    );
+  }
+
   print(
-    '  Extra storage per row: ${extraStoragePerRow.toFormattedStorageSize()}',
+    '  Storage overhead range: '
+    '${_formatPercentageValue(stages.first.overheadPercent)}% '
+    '--> ${_formatPercentageValue(stages.last.overheadPercent)}%',
   );
   if (runningInCI) print('```');
 }
@@ -65,6 +113,67 @@ class BenchmarkResults {
   });
 
   final Operation operation;
-  final (double, int) baseline;
+  final double baseline;
-  final (double, int) crdt;
+  final double crdt;
+}
+
+class StorageBenchmarkResults {
+  const StorageBenchmarkResults({
+    required this.baseline,
+    required this.crdt,
+  });
+
+  final StorageBenchmarkRun baseline;
+  final StorageBenchmarkRun crdt;
+}
+
+class _StorageStageComparison {
+  const _StorageStageComparison({
+    required this.title,
+    required this.baselineSize,
+    required this.crdtSize,
+    required this.baselineIncrement,
+    required this.crdtIncrement,
+    required this.unitCount,
+    required this.unitLabel,
+  });
+
+  final String title;
+  final int baselineSize;
+  final int crdtSize;
+  final int baselineIncrement;
+  final int crdtIncrement;
+  final int unitCount;
+  final String unitLabel;
+
+  int get extraBytes => crdtSize - baselineSize;
+  double? get overheadPercent =>
+      baselineSize == 0 ? null : extraBytes / baselineSize * 100;
+  double get incrementalExtraBytesPerUnit =>
+      (crdtIncrement - baselineIncrement) / unitCount;
+}
+
+String _formatSignedStorage(num size) {
+  if (size == 0) {
+    return '0 B';
+  }
+
+  final prefix = size > 0 ? '+' : '-';
+  return '$prefix${size.abs().toFormattedStorageSize()}';
+}
+
+String _formatPercentageIncrease(double? percent) {
+  if (percent == null) {
+    return 'n/a (baseline storage is zero)';
+  }
+
+  return '${formatter2.format(percent)}% increase';
+}
+
+String _formatPercentageValue(double? percent) {
+  if (percent == null) {
+    return 'n/a';
+  }
+
+  return formatter2.format(percent);
 }