chore: Add benchmark for time and storage interference (#4)

.github/workflows/ci.yaml

@@ -16,6 +16,25 @@ concurrency:
   cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
 
 jobs:
+  generate:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Dart
+        uses: dart-lang/setup-dart@v1
+      - name: Run pub get
+        run: dart pub get
+      - name: Generate code
+        run: dart run build_runner build --delete-conflicting-outputs
+      - name: Upload generated code
+        uses: actions/upload-artifact@v4
+        with:
+          name: generated-code
+          path: |
+            **/*.g.dart
+            **/*.freezed.dart
+            lib/**/*.part.dart
+
   dart_format:
     name: dart format
     runs-on: ubuntu-latest
@@ -23,8 +42,6 @@ jobs:
       - uses: actions/checkout@v4
       - name: Set up Dart
         uses: dart-lang/setup-dart@v1
-        with:
-          sdk: stable
       - name: Run pub get
         run: dart pub get
       # Skip code generation for formatting check
@@ -32,22 +49,24 @@ jobs:
         run: dart format --output=none --set-exit-if-changed .
 
   dart_analyze:
+    needs: generate
     name: dart analyze
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
       - name: Set up Dart
         uses: dart-lang/setup-dart@v1
-        with:
-          sdk: stable
       - name: Run pub get
         run: dart pub get
-      - name: Generate code
+      - name: Download generated code
-        run: dart run build_runner build --delete-conflicting-outputs
+        uses: actions/download-artifact@v4
+        with:
+          name: generated-code
       - name: Run lint
         run: dart analyze .
 
   tests:
+    needs: generate
     name: dart test
     runs-on: ubuntu-latest
     steps:
@@ -58,12 +77,43 @@ jobs:
           sdk: stable
       - name: Run pub get
         run: dart pub get
-      - name: Generate code
+      - name: Download generated code
-        run: dart run build_runner build --delete-conflicting-outputs
+        uses: actions/download-artifact@v4
+        with:
+          name: generated-code
       - name: Run tests
         run: dart test
 
+  benchmarks:
+    needs: generate
+    name: run benchmarks
+    if: github.event_name == 'pull_request'
+    runs-on: ubuntu-latest
+    permissions:
+      pull-requests: write
+      contents: read
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Dart
+        uses: dart-lang/setup-dart@v1
+        with:
+          sdk: stable
+      - name: Run pub get
+        run: dart pub get
+      - name: Download generated code
+        uses: actions/download-artifact@v4
+        with:
+          name: generated-code
+      - name: Run benchmarks
+        run: dart benchmark/run.dart --ci > benchmark-results.txt
+      - name: Comment on PR
+        uses: thollander/actions-comment-pull-request@v3
+        with:
+          file-path: benchmark-results.txt
+          comment-tag: benchmark-results
+
   publish_check:
+    needs: generate
     name: dart publish check
     runs-on: ubuntu-latest
     steps:
@@ -74,7 +124,9 @@ jobs:
           sdk: stable
       - name: Run pub get
         run: dart pub get
-      - name: Generate code
+      - name: Download generated code
-        run: dart run build_runner build --delete-conflicting-outputs
+        uses: actions/download-artifact@v4
+        with:
+          name: generated-code
       - name: Run publish dry run
         run: dart pub publish --dry-run

analysis_options.yaml

@@ -62,7 +62,6 @@ linter:
     - noop_primitive_operations
     - null_check_on_nullable_type_parameter
     - omit_local_variable_types
-    - one_member_abstracts
     - only_throw_errors
     - package_prefixed_library_names
     - parameter_assignments

benchmark/README.md

@@ -0,0 +1,44 @@
+# Benchmarks
+
+This folder contains benchmarks to measure the performance impact of using the `drift_offline_sync` package on database operations.
+
+## What is Measured
+
+The benchmarks measure two key metrics:
+
+1. **Insert/Update/Delete Performance**: Time required to insert/update/delete
+   a large number of rows with many columns.
+2. **Storage Overhead**: Database file size after inserting a large number of
+   rows with many columns.
+
+The benchmarks are run in a separate isolate to avoid interference with the main
+process. A baseline benchmark is run first to establish a reference point and
+then a CRDT-enabled benchmark is run to measure the performance impact of using
+the `drift_offline_sync` package.
+
+## Running the Benchmarks
+
+To run the benchmarks:
+
+```bash
+dart benchmark/run.dart
+```
+
+## Benchmark Details
+
+### Table Structure
+
+The benchmarks use `TableWithEveryColumnType` which includes:
+- Boolean (`aBool`)
+- DateTime (`aDateTime`)
+- Text (`aText`)
+- Integer (`anInt`)
+- Int64 (`anInt64`)
+- Real/Double (`aReal`)
+- Blob (`aBlob`)
+- Integer Enum (`anIntEnum`)
+- Text with Converter (`aTextWithConverter`)
+- UUID (`aUuid`)
+
+This comprehensive table structure ensures the benchmarks reflect realistic use
+cases with various data types.

benchmark/run.dart

@@ -0,0 +1,93 @@
+import 'utils/benchmark.dart';
+import 'utils/conversion.dart';
+import 'utils/results.dart';
+import 'utils/runner.dart';
+
+Future<void> main(List<String> args) async {
+  final runningInCI = args.contains('--ci');
+  final rowCountArg = args.where((arg) => arg.startsWith('--rows=')).firstOrNull;
+  final rowCount = rowCountArg == null ? 1000 : int.parse(rowCountArg.substring(7));
+  final rowsString = formatter0.format(rowCount);
+
+  if (!runningInCI) print('\n${'=' * 60}');
+  print('## Performance Benchmark ($rowsString rows)\n');
+  print(
+    [
+      'This benchmark measures the performance impact of using CRDT ',
+      'synchronization on database operations.',
+    ].join(runningInCI ? '' : '\n'),
+  );
+  if (!runningInCI) print('-' * 60);
+
+  final benchmarkResults = <BenchmarkResults>[];
+
+  for (final operation in Operation.values) {
+    final baselineResult = await runWithProgress(
+      'Running ${operation.name} benchmark (baseline)',
+      CrdtBenchmark(
+        '${operation.name} (baseline)',
+        crdtEnabled: false,
+        operation: operation,
+        rowCount: rowCount,
+      ).report,
+      skipProgress: runningInCI,
+    );
+
+    final crdtResult = await runWithProgress(
+      'Running ${operation.name} benchmark (CRDT)',
+      CrdtBenchmark(
+        '${operation.name} (CRDT)',
+        crdtEnabled: true,
+        operation: operation,
+        rowCount: rowCount,
+      ).report,
+      validator: (result) => result.$1 > baselineResult.$1,
+      skipProgress: runningInCI,
+    );
+
+    benchmarkResults.add(
+      BenchmarkResults(
+        operation: operation,
+        baseline: baselineResult,
+        crdt: crdtResult,
+      ),
+    );
+  }
+
+  final spuriousBenchmarks = benchmarkResults.where(
+    (result) => result.baseline.$1 > result.crdt.$1,
+  );
+
+  if (spuriousBenchmarks.isNotEmpty) {
+    final leadingText = runningInCI ? '\n> ' : '\n';
+    print(
+      [
+        '$leadingText❌ Baseline time is greater than CRDT time. This is likely due ',
+        'to running with a very low number of rows ($rowsString). This is not ',
+        ' a valid benchmark and the time comparison should be ignored.',
+      ].join(runningInCI ? '' : '\n'),
+    );
+
+    for (final result in spuriousBenchmarks) {
+      final baselineDelay = result.baseline.$1.toFormattedDuration();
+      final crdtDelay = result.crdt.$1.toFormattedDuration();
+      print('  - ${result.operation.name.toUpperCase()}: $baselineDelay > $crdtDelay');
+    }
+  }
+
+  for (final result in benchmarkResults) {
+    printPerformanceImpact(
+      result,
+      rowCount: rowCount,
+      runningInCI: runningInCI,
+    );
+  }
+  printStorageImpact(
+    benchmarkResults.first,
+    rowCount: rowCount,
+    runningInCI: runningInCI,
+  );
+
+  print('\n${'-' * 60}');
+  print('✅ Benchmark complete!\n');
+}

benchmark/utils/benchmark.dart

@@ -0,0 +1,94 @@
+import 'dart:io';
+
+import 'package:benchmark_harness/benchmark_harness.dart';
+import 'package:drift_offline_sync/src/database/triggers.dart';
+import 'package:path/path.dart' as p;
+
+import 'database.dart';
+
+export 'package:drift_offline_sync/src/database/triggers.dart' show Operation;
+
+class CrdtBenchmark extends AsyncBenchmarkBase {
+  CrdtBenchmark(
+    super.name, {
+    required this.crdtEnabled,
+    required this.operation,
+    required this.rowCount,
+  });
+
+  final bool crdtEnabled;
+  final Operation operation;
+  final int rowCount;
+
+  late final CrdtBenchmarkDatabase _db;
+  late final File _dbFile;
+
+  int _lastDatabaseSize = 0;
+  int _lastRowsCount = 0;
+
+  @override
+  Future<void> setup() async {
+    final dbPath = p.join(Directory.systemTemp.path, 'benchmark_$name.db');
+    _dbFile = File(dbPath);
+    _db = CrdtBenchmarkDatabase.createDatabase(_dbFile, crdtEnabled: crdtEnabled);
+
+    // Force the migration to be applied during setup.
+    await _db.batchInsertTestRows(1);
+    await _db.deleteTestRows(1);
+    await _db.managers.tableWithEveryColumnType.delete();
+
+    if (operation != Operation.insert) {
+      await _db.batchInsertTestRows(rowCount);
+    }
+  }
+
+  /// The default warmup is to invoke the [run] method once, but this pollutes the
+  /// results by persisting data before the run.
+  @override
+  Future<void> warmup() async {}
+
+  @override
+  Future<void> run() async {
+    _lastDatabaseSize = 0;
+    _lastRowsCount = 0;
+    switch (operation) {
+      case Operation.insert:
+      case Operation.update:
+        await _db.upsertTestRows(rowCount);
+      case Operation.delete:
+        await _db.deleteTestRows(rowCount);
+    }
+  }
+
+  @override
+  Future<(double, int)> report() async {
+    final averageMicroseconds = await measure();
+    return (averageMicroseconds, measureStorage());
+  }
+
+  @override
+  Future<void> teardown() async {
+    _lastDatabaseSize = _dbFile.lengthSync();
+    _lastRowsCount = await _db.testRowsCount();
+    await _db.close();
+    if (_dbFile.existsSync()) {
+      _dbFile.deleteSync();
+    }
+  }
+
+  int measureStorage() {
+    if (operation == Operation.delete) {
+      return _lastDatabaseSize;
+    }
+
+    if (_lastDatabaseSize == 0) {
+      throw Exception('Last database size is not set.');
+    }
+    if (_lastRowsCount < rowCount) {
+      throw Exception('Rows count is $_lastRowsCount but expected at least $rowCount.');
+    }
+    // If the benchmark ran for more than one loop, it can contain a greater number
+    // of rows than the expected. We need to normalize it to the expected total.
+    return _lastDatabaseSize ~/ (_lastRowsCount / rowCount);
+  }
+}

benchmark/utils/conversion.dart

@@ -0,0 +1,48 @@
+import 'package:intl/intl.dart';
+
+final formatter0 = NumberFormat.decimalPatternDigits(decimalDigits: 0);
+final formatter2 = NumberFormat.decimalPatternDigits(decimalDigits: 2);
+final formatter3 = NumberFormat.decimalPatternDigits(decimalDigits: 3);
+
+extension FormattedStorageSize on num {
+  String toFormattedDuration() {
+    var time = this;
+    var unit = 1;
+
+    while (time.abs() > 1000 && unit < Duration.microsecondsPerSecond) {
+      time /= 1000;
+      unit *= 1000;
+    }
+    while (unit > Duration.microsecondsPerMinute && time.abs() > 60) {
+      time /= 60;
+      unit *= 60;
+    }
+    return '${formatter2.format(time)} ${switch (unit) {
+      1 => 'μs',
+      Duration.millisecondsPerSecond => 'ms',
+      Duration.microsecondsPerSecond => 's',
+      Duration.microsecondsPerMinute => 'min',
+      Duration.microsecondsPerHour => 'h',
+      _ => throw Exception('Unexpected time unit: $unit'),
+    }}';
+  }
+
+  String toFormattedStorageSize() {
+    var storageIncrease = toDouble();
+    var storageUnit = 1;
+
+    while (storageIncrease > 1024) {
+      storageIncrease /= 1024;
+      storageUnit *= 1024;
+    }
+
+    final storageUnitString = switch (storageUnit) {
+      1 => 'B',
+      1024 => 'KB',
+      const (1024 * 1024) => 'MB',
+      _ => throw Exception('Unexpected storage unit: $storageUnit'),
+    };
+
+    return '${formatter2.format(storageIncrease)} $storageUnitString';
+  }
+}

benchmark/utils/database.dart

@@ -0,0 +1,95 @@
+import 'dart:io';
+
+import 'package:drift/drift.dart';
+import 'package:drift/native.dart';
+import 'package:drift_offline_sync/drift_offline_sync.dart';
+import 'package:sqlite3/sqlite3.dart';
+import 'package:uuid/uuid.dart';
+
+import '../../test/utils/tables.dart';
+
+part 'database.g.dart';
+
+@DriftDatabase(tables: [TableWithEveryColumnType])
+class CrdtBenchmarkDatabase extends _$CrdtBenchmarkDatabase {
+  CrdtBenchmarkDatabase(super.e, {required this.crdtEnabled}) {
+    driftRuntimeOptions.dontWarnAboutMultipleDatabases = true;
+  }
+
+  static CrdtBenchmarkDatabase createDatabase(
+    File file, {
+    required bool crdtEnabled,
+  }) {
+    if (file.existsSync()) {
+      file.deleteSync();
+    }
+
+    return CrdtBenchmarkDatabase(
+      NativeDatabase.opened(
+        sqlite3.open(file.path),
+        setup: crdtEnabled ? registerHlcFunction : null,
+      ),
+      crdtEnabled: crdtEnabled,
+    );
+  }
+
+  final bool crdtEnabled;
+
+  @override
+  int get schemaVersion => 1;
+
+  @override
+  Migrator createMigrator() => !crdtEnabled
+      ? super.createMigrator()
+      : OfflineSyncMigrator(
+          this,
+          userId: const Uuid().v7(),
+          nodeId: const Uuid().v7(),
+          synchronizedTables: [tableWithEveryColumnType],
+        );
+
+  Future<void> batchInsertTestRows(int count) async {
+    final baseTimestamp = DateTime.now();
+    await managers.tableWithEveryColumnType.bulkCreate(
+      (_) => List.generate(count, (i) => _createTestRow(i, baseTimestamp)),
+    );
+  }
+
+  Future<void> upsertTestRows(int count) async {
+    final baseTimestamp = DateTime.now();
+    for (var i = 0; i < count; i++) {
+      await managers.tableWithEveryColumnType.create(
+        (_) => _createTestRow(i, baseTimestamp),
+        mode: InsertMode.insertOrReplace,
+      );
+    }
+  }
+
+  Future<void> deleteTestRows(int count) async {
+    for (var i = 0; i < count; i++) {
+      await managers.tableWithEveryColumnType
+          .filter((t) => t.id.equals(RowId(i)))
+          .delete();
+    }
+  }
+
+  Future<int> testRowsCount() async {
+    return managers.tableWithEveryColumnType.count();
+  }
+
+  TableWithEveryColumnTypeCompanion _createTestRow(int i, DateTime baseTimestamp) {
+    return TableWithEveryColumnTypeCompanion(
+      id: Value(RowId(i)),
+      aBool: Value(i.isEven),
+      aDateTime: Value(baseTimestamp.add(Duration(seconds: i))),
+      aText: Value('Text $i'),
+      anInt: Value(i),
+      anInt64: Value(BigInt.from(i)),
+      aReal: Value(i.toDouble()),
+      aBlob: Value(Uint8List.fromList([i % 256])),
+      anIntEnum: const Value(null),
+      aTextWithConverter: const Value(null),
+      aUuid: const Value(null),
+    );
+  }
+}

benchmark/utils/results.dart

@@ -0,0 +1,64 @@
+import 'benchmark.dart';
+import 'conversion.dart';
+
+void printPerformanceImpact(
+  BenchmarkResults benchmarkResults, {
+  required int rowCount,
+  bool runningInCI = false,
+}) {
+  print(
+    '${runningInCI ? '```' : ''}'
+    '\n📊 ${benchmarkResults.operation.name.toUpperCase()} performance impact:',
+  );
+  final (baselineTime, _) = benchmarkResults.baseline;
+  final (crdtTime, _) = benchmarkResults.crdt;
+
+  final runDelayUs = crdtTime - baselineTime;
+  final slowdown = runDelayUs / baselineTime * 100;
+
+  final baselineDelay = baselineTime.toFormattedDuration();
+  final crdtDelay = crdtTime.toFormattedDuration();
+  final runDelay = runDelayUs.toFormattedDuration();
+  final runDelayPerOperation = (runDelayUs / rowCount).toFormattedDuration();
+
+  print('  Time: $baselineDelay --> $crdtDelay ($runDelay)');
+  print('  CRDT overhead: ${formatter2.format(slowdown)}% slower');
+  print('  Delay per ${benchmarkResults.operation.name}: $runDelayPerOperation');
+  if (runningInCI) print('```');
+}
+
+void printStorageImpact(
+  BenchmarkResults benchmarkResults, {
+  required int rowCount,
+  bool runningInCI = false,
+}) {
+  print('${runningInCI ? '```' : ''}\n💽 Storage impact:');
+  final (_, baselineSize) = benchmarkResults.baseline;
+  final (_, crdtSize) = benchmarkResults.crdt;
+
+  final storageIncrease = crdtSize - baselineSize;
+  final storageIncPercent = storageIncrease / baselineSize * 100;
+  final extraStoragePerRow = storageIncrease / rowCount;
+  final storageIncreaseVolume = storageIncPercent > 100
+      ? '(x${formatter2.format(storageIncPercent / 100)})'
+      : '(+${(crdtSize - baselineSize).toFormattedStorageSize()})';
+  print(
+    '  Storage size: ${baselineSize.toFormattedStorageSize()} '
+    '--> ${crdtSize.toFormattedStorageSize()} $storageIncreaseVolume',
+  );
+  print('  CRDT overhead: ${formatter2.format(storageIncPercent)}% increase');
+  print('  Extra storage per row: ${extraStoragePerRow.toFormattedStorageSize()}');
+  if (runningInCI) print('```');
+}
+
+class BenchmarkResults {
+  const BenchmarkResults({
+    required this.operation,
+    required this.baseline,
+    required this.crdt,
+  });
+
+  final Operation operation;
+  final (double, int) baseline;
+  final (double, int) crdt;
+}

benchmark/utils/runner.dart

@@ -0,0 +1,39 @@
+import 'dart:io';
+import 'dart:isolate';
+
+import 'package:cli_tools/src/logger/helpers/progress.dart';
+
+Progress? _trackedAnimationInProgress;
+
+Future<T> runWithProgress<T>(
+  String message,
+  Future<T> Function() runner, {
+  bool Function(T result)? validator,
+  bool skipProgress = false,
+}) async {
+  if (skipProgress) {
+    return Isolate.run(runner);
+  }
+
+  _stopAnimationInProgress();
+  final progress = Progress(message, stdout);
+  _trackedAnimationInProgress = progress;
+  final result = await Isolate.run(runner);
+  _trackedAnimationInProgress = null;
+  if (validator?.call(result) ?? true) {
+    progress.complete();
+  } else {
+    progress.fail();
+  }
+  return result;
+}
+
+void _stopAnimationInProgress() {
+  if (_trackedAnimationInProgress != null) {
+    _trackedAnimationInProgress?.stopAnimation();
+    // Since animation modifies the current line we add a new line so that
+    // the next print doesn't end up on the same line.
+    stdout.write('\n');
+  }
+  _trackedAnimationInProgress = null;
+}

build.yaml

@@ -9,3 +9,9 @@ targets:
             dialects:
               - sqlite
               - postgres
+    sources:
+      include:
+        - $package$
+        - lib/**
+        - test/**
+        - benchmark/**

pubspec.yaml

@@ -14,9 +14,13 @@ dependencies:
   uuid: ^4.5.1
 
 dev_dependencies:
+  benchmark_harness: ^2.2.2
   build_runner: ^2.6.0
+  cli_tools: ^0.10.0
   drift_dev: ^2.28.0
+  intl: ^0.20.2
   lints: ^6.0.0
+  path: ^1.9.1
   sqlite3: any
   test: ^1.26.2
   test_descriptor: ^2.0.2