chore: Move the drift_offline_first to legacy in favor of Serverpod implementation (#32)

This commit is contained in:
Marcelo Mendonça Soares 2026-04-05 23:20:21 -03:00 • committed by Marcelo Soares
commit cbc472523b
No known key found for this signature in database
GPG key ID: CB2F09CE66E99EAD
20 changed files with 13575 additions and 284 deletions

View file

@ -0,0 +1,109 @@
import 'dart:async';
import 'package:meta/meta.dart';
import '../database/database.dart';
import '../hlc/hlc.dart';
/// Base class for CRDT implementations.
///
/// This class provides the basic functionality for a CRDT implementation.
/// It is not meant to be used directly, but rather to be extended by concrete
/// implementations.
abstract class CrdtBase {
/// Creates a new instance of [CrdtBase].
CrdtBase({required this.userId, required this.nodeId});
/// The user ID for the CRDT system.
final String userId;
/// The node ID of the CRDT.
final String nodeId;
/// The current canonical time from the database (in-SQLite HLC state).
Future<Hlc> getCanonicalTime();
/// Merge the [maxMergedHlc] into the canonical time.
Future<void> mergeCanonicalTime(Hlc maxMergedHlc);
/// Returns the last modified timestamp, optionally filtering for or against a
/// specific node id. Useful to get "modified since" timestamps for executing
/// synchronization.
Future<Hlc?> getLastModified({
String? onlyNodeId,
String? exceptNodeId,
});
/// Get a list of [CrdtDataEntry] with the changes to be merged.
///
/// Set the filtering parameters to to generate subsets:
/// [onlyNodeId] only records set by the specified node.
/// [exceptNodeId] only records not set by the specified node.
/// [modifiedAfter] records modified after the specified [Hlc].
Future<Iterable<CrdtDataEntry>> getChangeset({
String? onlyNodeId,
String? exceptNodeId,
Hlc? modifiedAfter,
});
/// Save the [changeset] to the database.
///
/// When saving, must be careful to only override values that are older than
/// the [Hlc] of the changeset entry. If there are any guardrails to be
/// applied to the saved data, they should be applied here.
///
/// The [receivedHlcs] is a list of HLCs, one for each external node. It must
/// be saved separately since it will be used to request updates from external
/// nodes on next syncs.
///
/// It is recommended to use a transaction to ensure the integrity of the
/// data and allow for rollback in case of errors.
///
/// Note that this method is only meant to be used internally by the [merge]
/// method and should not be called directly.
@protected
Future<void> saveChangeset(
Iterable<CrdtDataEntry> changeset,
Iterable<Hlc> receivedHlcs,
);
/// Merge [changeset] with the local dataset.
///
/// This operation must be done within a transaction to ensure that no other
/// operation modifies the data between the save, invariants preservation and
/// the merge of remote HLCs into the canonical time.
Future<void> merge(Iterable<CrdtDataEntry> changeset) async {
if (changeset.isEmpty) return;
final receivedHlcs = _extractLastReceivedHlcs(changeset);
await saveChangeset(changeset, receivedHlcs);
if (receivedHlcs.isEmpty) return;
await mergeCanonicalTime(receivedHlcs.max);
}
/// Extracts the last received HLCs from the changeset.
Iterable<Hlc> _extractLastReceivedHlcs(Iterable<CrdtDataEntry> changeset) {
final receivedHlcs = <String, Hlc>{};
for (final entry in changeset) {
if (entry.hlcTimestamp.nodeId == nodeId) continue;
receivedHlcs.addOrUpdate(entry.hlcTimestamp);
}
return receivedHlcs.values;
}
}
extension on Iterable<Hlc> {
Hlc get max => reduce((a, b) => a > b ? a : b);
}
extension on Map<String, Hlc> {
void addOrUpdate(Hlc hlc) {
final registeredHlc = this[hlc.nodeId];
if (registeredHlc == null) {
this[hlc.nodeId] = hlc;
} else if (registeredHlc < hlc) {
this[hlc.nodeId] = hlc;
}
}
}

View file

@ -0,0 +1,190 @@
import 'package:drift/drift.dart';
import '../database/database.dart';
import '../database/extensions/entry.dart';
import '../hlc/converter.dart';
import 'base.dart';
/// A CRDT implementation for offline-first synchronization.
class OfflineSyncCrdt extends CrdtBase {
/// Creates a new instance of [OfflineSyncCrdt].
///
/// The [nodeId] must be unique across the system.
OfflineSyncCrdt(this._db, {required super.userId, required super.nodeId});
final CrdtDatabase _db;
@override
Future<Hlc> getCanonicalTime() async {
final row = await _db.managers.crdtHlcStateTable
.filter((o) => o.userId.equals(userId))
.getSingleOrNull();
return row?.toHlc(nodeId) ?? Hlc.zero(nodeId);
}
@override
Future<void> mergeCanonicalTime(Hlc maxMergedHlc) async {
await _db.managers.crdtHlcStateTable.create(
(t) => t(
userId: userId,
lastTimestamp: maxMergedHlc.unixTimestamp,
counter: maxMergedHlc.counter,
),
mode: InsertMode.insertOrReplace,
onConflict: DoUpdate.withExcluded(
(old, excluded) => CrdtHlcStateTableCompanion.custom(
lastTimestamp: CaseWhenExpression(
// Combined with the `orElse`, this is the same as a max() function,
// but it works for all databases. Otherwise, we would need to check
// the database dialect to use either `MAX` or `GREATEST`.
cases: [
excluded.lastTimestamp
.isBiggerOrEqual(old.lastTimestamp)
.then(excluded.lastTimestamp),
],
orElse: old.lastTimestamp,
),
counter: CaseWhenExpression(
cases: [
// If one timestamp is greater, its counter should be persisted.
excluded.lastTimestamp
.isBiggerThan(old.lastTimestamp)
.then(excluded.counter),
// Otherwise, if the timestamps are equal, the counter should be
// the greater counter.
(excluded.lastTimestamp.equalsExp(old.lastTimestamp) &
excluded.counter.isBiggerThan(old.counter))
.then(excluded.counter),
],
orElse: old.counter,
),
),
),
);
}
@override
// TODO: Remove this method lately if it is not needed - as it seems it will.
Future<Hlc?> getLastModified({String? onlyNodeId, String? exceptNodeId}) async {
if (onlyNodeId != null && exceptNodeId != null) {
throw ArgumentError('Only one of onlyNodeId or exceptNodeId can be provided.');
}
// NOTE: Although `max` has a better performance for the general use case,
// when the column is indexed, `order by` with `limit 1` will be just as fast.
// The plus of using the order by approach is the native deserialization of
// the Hlc object by Drift.
final rowWithMaxLastSyncHlc = await _db.managers.crdtDataTable
.filter((o) {
if (onlyNodeId != null) {
return o.hlcTimestamp.column.contains(onlyNodeId);
}
if (exceptNodeId != null) {
return o.hlcTimestamp.column.contains(exceptNodeId).not();
}
return const Constant(true);
})
.orderBy((o) => o.hlcTimestamp.desc())
.limit(1)
.getSingleOrNull();
return rowWithMaxLastSyncHlc?.hlcTimestamp;
}
@override
Future<Iterable<CrdtDataEntry>> getChangeset({
String? onlyNodeId,
String? exceptNodeId,
Hlc? modifiedAfter,
}) async {
assert(onlyNodeId == null || exceptNodeId == null);
// Modified times use the local node id
modifiedAfter = modifiedAfter?.copyWith(nodeId: nodeId);
final query = _db.managers.crdtDataTable.filter((o) {
Expression<bool> condition = const Constant(true);
if (onlyNodeId != null) {
condition &= o.hlcTimestamp.column.contains(onlyNodeId);
}
if (exceptNodeId != null) {
condition &= o.hlcTimestamp.column.contains(exceptNodeId).not();
}
if (modifiedAfter != null) {
condition &= o.hlcTimestamp.column.isBiggerThanValue(
hlcConverter.toSql(modifiedAfter),
);
}
return condition;
});
return query.get();
}
@override
Future<void> saveChangeset(
Iterable<CrdtDataEntry> changeset,
Iterable<Hlc> receivedHlcs,
) async {
await _db.transaction(() async {
await _saveChangeset(changeset);
await _applyCompensationRules();
await _saveMergedHlcs(receivedHlcs);
});
}
/// Save the [changeset] to the database.
///
/// Should be called within a transaction. Must ensure that only newer values
/// are saved to the database.
Future<void> _saveChangeset(Iterable<CrdtDataEntry> changeset) async {
// Pure inserts; conflict resolution (newer HLC wins) is handled by the
// INSTEAD OF INSERT trigger on the CRDT data view.
return _db.managers.crdtDataTable.bulkCreate(
(_) => changeset.map((e) => e.toCompanion(false)),
mode: InsertMode.insert,
);
}
/// Applies compensation rules to the CRDT data table to preserve invariants.
///
/// If the any invariant fails to be preserved, a flag will be set on the
/// control table to prevent further merges until the issue is resolved by
/// the server and a repair merge is pushed to this node.
Future<void> _applyCompensationRules() async {
// TODO: Implement this.
}
/// Save the [mergedHlcs] to the database.
///
/// Should be called within a transaction after the changeset has been saved
/// and properly compensated.
Future<void> _saveMergedHlcs(Iterable<Hlc> mergedHlcs) async {
return _db.managers.crdtMergeHlcTable.bulkCreate(
(t) => mergedHlcs.map(
(e) => t(
userId: userId,
nodeId: e.nodeId,
lastReceivedHlc: e,
),
),
mode: InsertMode.insertOrReplace,
onConflict: DoUpdate.withExcluded(
(old, excluded) => CrdtMergeHlcTableCompanion.custom(
lastReceivedHlc: excluded.lastReceivedHlc,
),
where: (old, excluded) =>
old.lastReceivedHlc.isSmallerThan(excluded.lastReceivedHlc),
),
);
}
}
extension on Expression<bool> {
CaseWhen<bool, T> then<T extends Object>(Expression<T> value) => CaseWhen(
this,
then: value,
);
}

View file

@ -0,0 +1,129 @@
import 'package:drift/drift.dart';
import '../utils/sql_builder.dart';
import 'tables/control.dart';
import 'tables/data.dart';
import 'tables/hlc.dart';
import 'tables/merge.dart';
part 'database.g.dart';
/// The main database class for CRDT synchronization.
@DriftDatabase(
tables: [
CrdtDataTable,
CrdtNormalizedDataTable,
CrdtControlTable,
CrdtHlcStateTable,
CrdtMergeHlcTable,
],
)
class CrdtDatabase extends _$CrdtDatabase {
/// Creates a new instance of [CrdtDatabase] with the provided [executor].
CrdtDatabase(
this.userDatabase, {
required this.userId,
required this.nodeId,
required this.synchronizedTables,
}) : super(
userDatabase.executor.interceptWith(
_EnsureHlcInitialized(userDatabase),
),
);
/// The user database to synchronize with.
final GeneratedDatabase userDatabase;
/// The user ID for the CRDT system.
final String userId;
/// The node ID for the CRDT system.
final String nodeId;
/// Tables to be synchronized with CRDT.
final List<TableInfo> synchronizedTables;
/// The SQL builder for the CRDT data table.
late final sqlBuilder = CrdtDataSqlBuilder(this);
/// Whether the database is using the SQLite or Postgres dialect.
bool get isPostgres => executor.dialect == SqlDialect.postgres;
/// Not actually used, but required by the generated code. Will be ignored because
/// the [CrdtDatabase] always receives an already opened executor. No migrations
/// should ever be run on this database to avoid messing with the user schema.
@override
int schemaVersion = 1;
}
class _EnsureHlcInitialized extends QueryInterceptor {
_EnsureHlcInitialized(this.userDatabase);
final GeneratedDatabase userDatabase;
@override
Future<bool> ensureOpen(QueryExecutor executor, QueryExecutorUser user) async {
return executor.ensureOpen(_EnsureHlcInitializedUser(user, userDatabase));
}
}
class _EnsureHlcInitializedUser implements QueryExecutorUser {
_EnsureHlcInitializedUser(this.user, this.userDatabase);
final QueryExecutorUser user;
final GeneratedDatabase userDatabase;
/// This must be set to the [schemaVersion] of the [userDatabase] to ensure
/// that the `PRAGMA user_version` is correctly updated after migrations are
/// applied. Otherwise, the saved version will match the CRDT database version.
@override
int get schemaVersion => userDatabase.schemaVersion;
/// If this method is reached, it means that the CRDT database is being opened
/// before the user database. This method will ensure that the user database is
/// always opened first. When, on the opposite, the user database is opened before
/// the CRDT database, this method will never be called.
@override
Future<void> beforeOpen(QueryExecutor executor, OpeningDetails details) async {
final userDetails = OpeningDetails(
details.versionBefore,
userDatabase.schemaVersion,
);
await userDatabase.beforeOpen(executor, userDetails);
// This means an explicit opt-out from Drift managed migrations. Since the
// [userDatabase] will be migrated first, by using its [schemaVersion] on
// the [crdtDetails], the CRDT database will skip migrations entirely. If
// migrations are needed for the CRDT database in the future, they should
// be hooked manually on this method.
final crdtDetails = OpeningDetails(
userDatabase.schemaVersion,
userDatabase.schemaVersion,
);
await user.beforeOpen(executor, crdtDetails);
}
}
/// Extension methods for the [List<TableInfo>] class to find a synchronized table.
extension FindSynchronizedTable on List<TableInfo> {
/// Gets the table names of the synchronized tables.
Iterable<String> get tableNames => map((t) => t.actualTableName);
/// Gets the synchronized table information for the given table name.
///
/// If the table is not found, an [ArgumentError] is thrown.
TableInfo find<T extends Insertable>([String? tableName]) {
if (tableName == null && T == dynamic) {
throw ArgumentError('Table name is required when T is dynamic.');
}
return firstWhere(
(t) =>
(tableName == null && t is TableInfo<Table, T>) ||
t.actualTableName == tableName,
orElse: () => throw ArgumentError(
'Table ${tableName != null ? '"$tableName"' : 'for type "$T"'} not '
'found in synchronized tables.',
),
);
}
}

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,57 @@
import 'package:drift/drift.dart';
import '../../hlc/hlc.dart';
import '../database.dart';
/// Extensions for [CrdtDatabase] to convert data to [CrdtDataEntry].
extension CrdtDatabaseConvertExtensions on CrdtDatabase {
/// Converts the [Insertable] object to an [Iterable] of [CrdtDataEntry].
///
/// Note that the generated entries will have the same HLC timestamp for all
/// columns and will overwrite individual column updates if newer.
Iterable<CrdtDataEntry> convertToCrdtDataEntry<T extends Insertable>(
T data,
Hlc hlcTimestamp,
) {
final json = data.toJsonForDatabase(_generationContext);
final tableInfo = synchronizedTables.whereType<TableInfo<dynamic, T>>().single;
final rowIdValues = [for (final c in tableInfo.$primaryKey) json[c.$name]];
return tableInfo.$columns.map(
(c) => CrdtDataEntry(
userId: userId,
tblName: tableInfo.actualTableName,
columnName: c.$name,
rowId: rowIdValues.join(sqlBuilder.rowIdSeparator),
rawValue: json[c.$name]?.toDriftAny(),
hlcTimestamp: hlcTimestamp,
),
);
}
/// Gets a [GenerationContext] for the database.
GenerationContext get _generationContext => GenerationContext.fromDb(this);
}
extension on Insertable {
Map<String, Object?> toJsonForDatabase(GenerationContext context) {
final expressions = toColumns(false);
return {
for (final e in expressions.entries) e.key: e.value.extractValue(context),
};
}
}
extension on Expression {
Object? extractValue(GenerationContext context) {
final exp = this;
if (exp is! Variable) {
throw ArgumentError.value(exp, 'exp', 'Expected a Variable');
}
return exp.mapToSimpleValue(context);
}
}
extension on Object {
DriftAny? toDriftAny() => DriftAny(this);
}

View file

@ -0,0 +1,35 @@
import '../../hlc/hlc.dart';
import '../database.dart';
/// Extensions for the [CrdtHlcEntry] class.
extension CrdtHlcEntryExtensions on CrdtHlcEntry {
/// Returns the HLC. If the [nodeId] is not provided, it will use the [userId].
Hlc toHlc([String? nodeId]) => Hlc(
DateTime.fromMillisecondsSinceEpoch(lastTimestamp, isUtc: true),
counter,
nodeId ?? userId,
);
}
/// Extensions for the [CrdtDataEntry] class.
extension CrdtDataEntryUnwrapExtensions on CrdtDataEntry {
/// Returns the unwrapped raw value, if any.
Object? get unwrappedValue => rawValue?.rawSqlValue;
}
/// Extensions for an [Iterable] of [CrdtDataEntry].
extension CrdtDataEntryExtensions on Iterable<CrdtDataEntry> {
/// Gets the affected tables.
Iterable<String> get affectedTables => map((e) => e.tblName).toSet();
/// Gets the affected row IDs.
Iterable<String> get affectedRowIds => map((e) => e.rowId).toSet();
/// Gets the entries for a given table.
Iterable<CrdtDataEntry> forTable(String tableName) =>
where((e) => e.tblName == tableName);
/// Returns the entries with the HLC replaced by the [hlcTimestamp].
Iterable<CrdtDataEntry> withHlc(Hlc hlcTimestamp) =>
map((e) => e.copyWith(hlcTimestamp: hlcTimestamp));
}

View file

@ -0,0 +1,52 @@
import 'package:drift/drift.dart';
import '../database.dart';
/// Extensions for [CrdtDatabase] to extract data from the CRDT data table.
extension CrdtDatabaseExtractExtensions on CrdtDatabase {
/// Gets a single data entry from the CRDT data table.
Future<T?> getSingleFromCrdtData<T extends Insertable>(String rowId) async {
final data = await getFromCrdtData<T>([rowId]);
return data.firstOrNull;
}
/// Gets a list of data entries from the CRDT data table.
Future<Iterable<T>> getFromCrdtData<T extends Insertable>([
Iterable<String>? rowIds,
]) async {
final tableInfo = synchronizedTables.find<T>();
final tableName = tableInfo.actualTableName;
final crdtDataEntries = await managers.crdtDataTable.filter(
(o) {
var condition = o.tblName.equals(tableName);
if (rowIds != null) {
condition &= o.rowId.isIn(rowIds);
}
return condition;
},
).get();
return tableInfo.fromCrdtDataEntries<T>(crdtDataEntries);
}
}
/// Extensions for [TableInfo] to convert [CrdtDataEntry] to [Insertable].
extension ConvertCrdtDataEntry on TableInfo {
/// Converts [CrdtDataEntry] to [Insertable].
Future<Iterable<T>> fromCrdtDataEntries<T extends Insertable>(
Iterable<CrdtDataEntry> entries,
) async {
final foundRowIds = entries.map((e) => e.rowId).toSet();
return [
for (final rowId in foundRowIds)
await map({
for (final c in $columns)
c.$name: entries
.firstWhere((e) => e.rowId == rowId && e.columnName == c.$name)
.rawValue
?.rawSqlValue,
})
as T,
];
}
}

View file

@ -0,0 +1,136 @@
import 'package:drift/drift.dart';
import '../../hlc/hlc.dart';
import '../../utils/sql_builder.dart';
import '../database.dart';
import 'transactions.dart';
/// Extensions for [CrdtDatabase] to sync data to the target tables.
extension CrdtDatabaseSyncExtensions on CrdtDatabase {
/// Upserts rows in the target table from the CRDT data table.
///
/// This is the inverse operation of what CRDT triggers do. It reads from the
/// CRDT data table and reconstructs rows to insert/update in the target table.
/// Handles deletions and fields that are not changed in the filter criteria.
///
/// Parameters:
/// - [rowIdsPerTable]: Map of table names to row IDs to sync. If not provided,
/// all tables will be synchronized. Passing null for a table will synchronize
/// all rows in that table.
/// - [sinceHlc]: Optional HLC timestamp to filter by. The filter is inclusive,
/// meaning that rows with the exact same HLC timestamp will also be included.
Future<void> syncCrdtDataToTables({
Map<String, Iterable<String>?>? rowIdsPerTable,
Hlc? sinceHlc,
}) async {
return transactionWithDeferredConstraints(() async {
final tableNames = rowIdsPerTable?.keys ?? synchronizedTables.tableNames;
for (final tableName in tableNames) {
await _syncCrdtDataToTable(
tableName: tableName,
rowIds: rowIdsPerTable?[tableName],
sinceHlc: sinceHlc,
);
}
});
}
/// Syncs the CRDT data to the target table.
///
/// Must be called within a transaction with deferred constraints to ensure that
/// the foreign key constraints are not violated. The transaction is not created
/// by this method to allow grouping multiple table syncs into one transaction
/// to avoid nested transactions.
Future<void> _syncCrdtDataToTable({
required String tableName,
Iterable<String>? rowIds,
Hlc? sinceHlc,
}) async {
final deleteSql = _getDeleteFromCrdtDataSql(
tableName: tableName,
rowIds: rowIds,
sinceHlc: sinceHlc,
);
final upsertSql = _getSaveFromCrdtDataSql(
tableName: tableName,
rowIds: rowIds,
sinceHlc: sinceHlc,
);
await customStatement(deleteSql);
await customStatement(upsertSql);
}
/// Generates the SQL to delete rows in the target table from the CRDT data table.
String _getDeleteFromCrdtDataSql({
required String tableName,
Iterable<String>? rowIds,
Hlc? sinceHlc,
}) {
return '''
DELETE FROM "$tableName"
WHERE ${_getWhereClause(tableName, rowIds, sinceHlc, isDeleted: true)};''';
}
// TODO: Unify this with the SQL for the CRDT triggers.
/// Generates SQL to upsert rows in the target table from the CRDT data table.
String _getSaveFromCrdtDataSql({
required String tableName,
Iterable<String>? rowIds,
Hlc? sinceHlc,
}) {
final tableInfo = synchronizedTables.find(tableName);
final columnSelects = tableInfo.nonGenerated.map((column) {
final whereClause = 'WHERE c."column_name" = \'$column\'';
return 'MAX(c."raw_value") FILTER ($whereClause) AS "$column"';
});
final updateSets = tableInfo.nonPrimaryKeyColumns.map((column) {
final newField = 'EXCLUDED."$column"';
final originalField = '"$tableName"."$column"';
return '"$column" = COALESCE($newField, $originalField)';
});
final onConflict = updateSets.isEmpty
? 'DO NOTHING'
: 'DO UPDATE\nSET ${updateSets.join(',\n ')}';
return '''
INSERT INTO "$tableName" (
${tableInfo.nonGenerated.map((c) => '"$c"').join(', ')}
)
SELECT ${columnSelects.join(',\n ')}
FROM ${sqlBuilder.crdtDataTableName} AS c
WHERE ${_getWhereClause(tableName, rowIds, sinceHlc)}
GROUP BY c."row_id"
ON CONFLICT (${tableInfo.$primaryKey.quotedNames.join(', ')})
$onConflict;''';
}
String _getWhereClause(
String tableName,
Iterable<String>? rowIds,
Hlc? sinceHlc, {
bool isDeleted = false,
}) {
final whereConditions = <String>[
'"user_id" = \'$userId\'',
'"table_name" = \'$tableName\'',
if (rowIds != null && rowIds.isNotEmpty)
'row_id IN (${rowIds.map((id) => "'$id'").join(', ')})',
if (sinceHlc != null) '"hlc_timestamp" >= \'$sinceHlc\'',
if (isDeleted) '"${sqlBuilder.isDeletedColumnName}" = TRUE',
];
return whereConditions.join(' AND ');
}
}
extension on TableInfo {
Iterable<String> get nonGenerated =>
$columns.where((c) => c.generatedAs == null).map((c) => c.$name);
Iterable<String> get nonPrimaryKeyColumns =>
nonGenerated.where((c) => !$primaryKey.map((p) => p.$name).contains(c));
}

View file

@ -0,0 +1,39 @@
import '../database.dart';
import 'triggers.dart';
/// Extensions for [CrdtDatabase] to execute transactions.
extension CrdtDatabaseTransactionsExtensions on CrdtDatabase {
/// Executes the [operation] in a transaction with the constraints deferred.
///
/// On Postgres, this will result in deferring the constraints for the duration
/// of the operation. On SQLite, this will result in disabling the foreign keys
/// before the operation and restoring to the previous value afterwards.
Future<T> transactionWithDeferredConstraints<T>(
Future<T> Function() operation,
) async {
if (isPostgres) {
return transaction(
() => withTriggersDisabled(() async {
await customStatement('SET CONSTRAINTS ALL DEFERRED');
return operation();
}),
);
}
return _withForeignKeysDisabled(
() => transaction(() => withTriggersDisabled(operation)),
);
}
Future<T> _withForeignKeysDisabled<T>(Future<T> Function() operation) async {
const pragma = 'PRAGMA foreign_keys';
final originalValueResult = await customSelect(pragma).getSingle();
final originalValue = originalValueResult.read<int>('foreign_keys');
try {
await customStatement('$pragma = OFF');
return await operation();
} finally {
await customStatement('$pragma = $originalValue');
}
}
}

View file

@ -0,0 +1,32 @@
import 'package:drift/drift.dart';
import '../database.dart';
/// Extensions for [CrdtDatabase] to manage triggers.
extension CrdtDatabaseTriggersExtensions on CrdtDatabase {
/// Disables the CRDT triggers for the current node.
Future<void> disableTriggers() async {
await _updateTriggersStatus(false);
}
/// Enables the CRDT triggers for the current node.
Future<void> enableTriggers() async {
await _updateTriggersStatus(true);
}
/// Disables the CRDT triggers for the duration of the [operation].
Future<T> withTriggersDisabled<T>(Future<T> Function() operation) async {
await disableTriggers();
try {
return await operation();
} finally {
await enableTriggers();
}
}
Future<void> _updateTriggersStatus(bool enabled) async {
await managers.crdtControlTable
.filter((t) => t.userId.equals(userId) & t.nodeId.equals(nodeId))
.update((t) => t(crdtTriggersOn: Value(enabled)));
}
}

View file

@ -0,0 +1,57 @@
import 'package:drift/drift.dart';
import '../../hlc/converter.dart';
// Required for the generated code to know the HLC and converters.
export '../../hlc/converter.dart';
/// CRDT control table.
///
/// This table stores metadata about the CRDT synchronization state for each user.
///
@DataClassName('CrdtControlEntry')
class CrdtControlTable extends Table {
@override
String get tableName => '__crdt_control';
/// Identifier for the user or client.
late final userId = text()();
/// Node identifier for the client.
late final nodeId = text()();
/// Whether CRDT triggers are currently enabled for this user.
late final crdtTriggersOn = boolean().withDefault(const Constant(true))();
/// The schema version of the CRDT data.
late final schemaVersion = integer()();
/// The last HLC timestamp when the user synchronized.
late final lastSyncHlc = text().nullable().map(nullableHlcConverter)();
/// The last HLC timestamp when merged changes were applied.
late final lastApplyHlc = text().nullable().map(nullableHlcConverter)();
@override
Set<Column> get primaryKey => {userId, nodeId};
/// Query to insert a new entry into the CRDT control table.
///
/// Declared here to ensure the query is synchronized with the table.
static String getDefaultUpsertQuery({
required String userId,
required String nodeId,
required int schemaVersion,
}) {
return '''
INSERT INTO "__crdt_control" (
"user_id",
"node_id",
"crdt_triggers_on",
"schema_version"
) VALUES ('$userId', '$nodeId', TRUE, $schemaVersion)
ON CONFLICT ("user_id", "node_id") DO UPDATE SET
"crdt_triggers_on" = TRUE,
"schema_version" = $schemaVersion;''';
}
}

View file

@ -0,0 +1,97 @@
import 'package:drift/drift.dart';
import '../../hlc/converter.dart';
// Required for the generated code to know the HLC and converters.
export '../../hlc/converter.dart';
/// CRDT data table.
///
/// This table stores all CRDT required metadata to perform conflict resolution
/// and synchronization. The schema is designed to be generic enough to support
/// various data types and structures. It stores the data for all tables that are
/// registered for CRDT synchronization, simplifying the process of conflict
/// resolution and data merging.
///
/// This same schema will be used on a client and server, with the difference
/// that the client will have the `user_id` column with a single value, while
/// the server will have all users' data in the same table.
///
/// This table is used to interact with the CRDT data, but won't exist in the
/// database. On runtime, it will be replaced by a view with INSTEAD OF triggers
/// that will persist the data to the normalized data table.
@DataClassName('CrdtDataEntry')
class CrdtDataTable extends Table {
@override
String get tableName => '__crdt_data';
@override
bool get withoutRowId => true;
@override
bool get isStrict => true;
/// Identifier for the user or client that owns the data.
late final userId = text()();
/// Name of the table that data belongs to.
late final tblName = text().named('table_name')();
/// Name of the column this data belongs to.
late final columnName = text()();
/// Unique identifier for the row in the table.
late final rowId = text()();
/// Hybrid Logical Clock timestamp of the data update.
late final hlcTimestamp = text().map(hlcConverter)();
/// Raw value of the data.
late final rawValue = sqliteAny().nullable()();
@override
Set<Column> get primaryKey => {userId, tblName, columnName, rowId};
}
/// CRDT normalized data table.
///
/// This will be the actual table that will be used to store the data. See the
/// [CrdtDataTable] class to better understand the schema.
@DataClassName('CrdtNormalizedDataEntry')
class CrdtNormalizedDataTable extends Table {
@override
String get tableName => '__crdt_normalized_data';
@override
bool get withoutRowId => true;
@override
bool get isStrict => true;
/// Identifier for the user or client that owns the data.
late final userId = text()();
/// Name of the table that data belongs to.
late final tblName = text().named('table_name')();
/// Name of the column this data belongs to.
late final columnName = text()();
/// Unique identifier for the row in the table.
late final rowId = text()();
/// Hybrid Logical Clock timestamp component (milliseconds since epoch).
late final hlcTimestamp = integer()();
/// Hybrid Logical Clock counter component.
late final hlcCounter = integer()();
/// Hybrid Logical Clock node ID component.
late final hlcNodeId = text()();
/// Raw value of the data.
late final rawValue = sqliteAny().nullable()();
@override
Set<Column> get primaryKey => {userId, tblName, columnName, rowId};
}

View file

@ -0,0 +1,23 @@
import 'package:drift/drift.dart';
/// HLC state table.
///
/// Stores per-user last timestamp and counter for Hybrid Logical Clock state.
///
@DataClassName('CrdtHlcEntry')
class CrdtHlcStateTable extends Table {
@override
String get tableName => '__hlc_state';
/// Identifier for the user or client.
late final userId = text()();
/// Last HLC timestamp in milliseconds since epoch (logical time component).
late final lastTimestamp = integer()();
/// Counter for events at the same logical time (count component).
late final counter = integer()();
@override
Set<Column> get primaryKey => {userId};
}

View file

@ -0,0 +1,29 @@
import 'package:drift/drift.dart';
import '../../hlc/converter.dart';
// Required for the generated code to know the HLC and converters.
export '../../hlc/converter.dart';
/// CRDT merge HLC table.
///
/// This table stores the last HLC timestamp received from each node when merging
/// changes, so that the next merges can be performed incrementally.
///
@DataClassName('CrdtMergeHlcEntry')
class CrdtMergeHlcTable extends Table {
@override
String get tableName => '__crdt_merge_hlc';
/// Identifier for the user or client.
late final userId = text()();
/// Node identifier for the client.
late final nodeId = text()();
/// The last HLC timestamp received from this node when merging changes.
late final lastReceivedHlc = text().map(hlcConverter)();
@override
Set<Column> get primaryKey => {userId, nodeId};
}

View file

@ -0,0 +1,382 @@
import 'package:drift/drift.dart';
import '../migrator.dart';
import '../utils/sql_builder.dart';
/// The operation that was performed on a row.
// ignore: public_member_api_docs
enum Operation { insert, update, delete }
/// Provides the CRDT control trigger statements for the database.
abstract class OfflineSyncTriggers extends CrdtDataSqlBuilder {
/// Creates a new instance of [OfflineSyncTriggers].
OfflineSyncTriggers(
super.crdtDb, [
this.conflictingBias = ConflictingBias.deleteWins,
]);
/// The bias to use when conflicting operations are detected.
final ConflictingBias conflictingBias;
/// Wraps the provided SQL statement as a CREATE TRIGGER statement.
///
/// The received [innerSql] will be expected to go inside a BEGIN...END block.
String wrapAsCreateTriggerStatement({
required Operation operation,
required String tableName,
required String triggerName,
required String innerSql,
});
/// Gets the name of the trigger for a specific operation on a table.
String _getTriggerName(TableInfo<Table, Object?> table, Operation operation) =>
'${prefix}__${table.actualTableName}__${operation.name}';
/// Returns the SQL statement to insert a new row into the CRDT data table.
String getInsertStatement({
required TableInfo<Table, Object?> table,
required Operation operation,
});
/// Returns the SQL statements to create the triggers for the changelog table.
List<String> generateCreateTriggerStatements(TableInfo<Table, Object?> table) {
return [
for (final op in Operation.values)
wrapAsCreateTriggerStatement(
operation: op,
tableName: table.actualTableName,
triggerName: _getTriggerName(table, op),
innerSql: getInsertStatement(
table: table,
operation: op,
),
),
];
}
/// Returns extra statements to execute once before the triggers are created.
List<String> generateSetupStatements() => const [];
}
/// Provides the CRDT control trigger statements for the SQLite3 database.
class Sqlite3OfflineSyncTriggers extends OfflineSyncTriggers {
/// Creates a new instance of [Sqlite3OfflineSyncTriggers].
Sqlite3OfflineSyncTriggers(super.crdtDb, [super.conflictingBias]);
/// Name of the view used to advance HLC via INSERT (INSTEAD OF trigger).
static const hlcCanonicalViewName = '__hlc_canonical_view';
@override
String wrapAsCreateTriggerStatement({
required Operation operation,
required String tableName,
required String triggerName,
required String innerSql,
}) =>
'''
CREATE TRIGGER IF NOT EXISTS "$triggerName"
AFTER ${operation.name.toUpperCase()} ON "$tableName"
WHEN (
SELECT MAX("crdt_triggers_on")
FROM "__crdt_control"
WHERE ("user_id" = '${crdtDb.userId}')
AND ("node_id" = '${crdtDb.nodeId}')
) = TRUE
BEGIN
$innerSql
END;''';
@override
String getInsertStatement({
required TableInfo<Table, Object?> table,
required Operation operation,
}) {
final tableName = table.actualTableName;
final uniqueRowId = getUniqueRowId(
table,
tableAlias: operation == Operation.insert ? 'NEW' : 'OLD',
);
final columnNames = [
if (operation != Operation.delete) ...table.nonGenerated,
if (operation != Operation.update ||
conflictingBias == ConflictingBias.updateWins)
isDeletedColumnName,
];
final columnInserts = columnNames.map((c) {
final columnValue = c == isDeletedColumnName
? operation == Operation.delete
? 'TRUE'
: 'FALSE'
: 'NEW."$c"';
return ' SELECT\n${[
' \'$c\' AS "column_name"',
' $columnValue AS "raw_value"',
if (operation == Operation.update) ' NEW."$c" IS NOT OLD."$c" AS "value_changed"',
].join(',\n')}';
});
return '''
INSERT INTO "$hlcCanonicalViewName" ("user_id")
VALUES ('${crdtDb.userId}');
INSERT INTO "$crdtDataTableName" (
${crdtDataAllColumns.join(', ')}
)
SELECT
'${crdtDb.userId}' AS "user_id",
'$tableName' AS "table_name",
"column_name",
$uniqueRowId AS "row_id",
(
SELECT (${_getHlcJsonExprSql()})
FROM "$hlcStateTableName"
WHERE "user_id" = '${crdtDb.userId}'
) AS "hlc_timestamp",
raw_value
FROM (
${columnInserts.join('\n UNION ALL\n')}
)${operation == Operation.update ? '\n WHERE ("value_changed" = TRUE)' : ''};''';
}
/// Returns SQL fragment that formats (last_timestamp, counter, node_id) in the
/// same format as the `hlcConverter.toSql` method.
String _getHlcJsonExprSql() {
return "printf('%015d', \"last_timestamp\") || '-' || "
"printf('%04X', \"counter\") || '-' || "
"'${crdtDb.nodeId}'";
}
/// Returns SQL fragment that combines the split HLC columns (hlc_timestamp,
/// hlc_counter, hlc_node_id) back into a single text field.
String _getHlcCombineExprSql() {
return "printf('%015d', \"hlc_timestamp\") || '-' || "
"printf('%04X', \"hlc_counter\") || '-' || "
'"hlc_node_id"';
}
/// Returns SQL expression that converts a 4-character hex string to integer.
/// Used to parse the counter from the HLC text format.
String _hexToIntExpr(String hexString) {
// Convert each hex digit (0-9, A-F) to its decimal value and combine
return '''(
CASE SUBSTR($hexString, 1, 1)
WHEN '0' THEN 0 WHEN '1' THEN 1 WHEN '2' THEN 2 WHEN '3' THEN 3
WHEN '4' THEN 4 WHEN '5' THEN 5 WHEN '6' THEN 6 WHEN '7' THEN 7
WHEN '8' THEN 8 WHEN '9' THEN 9
WHEN 'A' THEN 10 WHEN 'B' THEN 11 WHEN 'C' THEN 12 WHEN 'D' THEN 13
WHEN 'E' THEN 14 WHEN 'F' THEN 15
WHEN 'a' THEN 10 WHEN 'b' THEN 11 WHEN 'c' THEN 12 WHEN 'd' THEN 13
WHEN 'e' THEN 14 WHEN 'f' THEN 15
ELSE 0
END * 4096 +
CASE SUBSTR($hexString, 2, 1)
WHEN '0' THEN 0 WHEN '1' THEN 1 WHEN '2' THEN 2 WHEN '3' THEN 3
WHEN '4' THEN 4 WHEN '5' THEN 5 WHEN '6' THEN 6 WHEN '7' THEN 7
WHEN '8' THEN 8 WHEN '9' THEN 9
WHEN 'A' THEN 10 WHEN 'B' THEN 11 WHEN 'C' THEN 12 WHEN 'D' THEN 13
WHEN 'E' THEN 14 WHEN 'F' THEN 15
WHEN 'a' THEN 10 WHEN 'b' THEN 11 WHEN 'c' THEN 12 WHEN 'd' THEN 13
WHEN 'e' THEN 14 WHEN 'f' THEN 15
ELSE 0
END * 256 +
CASE SUBSTR($hexString, 3, 1)
WHEN '0' THEN 0 WHEN '1' THEN 1 WHEN '2' THEN 2 WHEN '3' THEN 3
WHEN '4' THEN 4 WHEN '5' THEN 5 WHEN '6' THEN 6 WHEN '7' THEN 7
WHEN '8' THEN 8 WHEN '9' THEN 9
WHEN 'A' THEN 10 WHEN 'B' THEN 11 WHEN 'C' THEN 12 WHEN 'D' THEN 13
WHEN 'E' THEN 14 WHEN 'F' THEN 15
WHEN 'a' THEN 10 WHEN 'b' THEN 11 WHEN 'c' THEN 12 WHEN 'd' THEN 13
WHEN 'e' THEN 14 WHEN 'f' THEN 15
ELSE 0
END * 16 +
CASE SUBSTR($hexString, 4, 1)
WHEN '0' THEN 0 WHEN '1' THEN 1 WHEN '2' THEN 2 WHEN '3' THEN 3
WHEN '4' THEN 4 WHEN '5' THEN 5 WHEN '6' THEN 6 WHEN '7' THEN 7
WHEN '8' THEN 8 WHEN '9' THEN 9
WHEN 'A' THEN 10 WHEN 'B' THEN 11 WHEN 'C' THEN 12 WHEN 'D' THEN 13
WHEN 'E' THEN 14 WHEN 'F' THEN 15
WHEN 'a' THEN 10 WHEN 'b' THEN 11 WHEN 'c' THEN 12 WHEN 'd' THEN 13
WHEN 'e' THEN 14 WHEN 'f' THEN 15
ELSE 0
END
)''';
}
/// Returns SQL statements to create the CRDT data view and its INSTEAD OF
/// triggers, and the HLC canonical view with its INSTEAD OF trigger.
///
/// The CRDT data view exposes the crdt data table schema reading from the
/// normalized table. Conflict resolution (newer HLC wins) is done in the
/// INSTEAD OF INSERT trigger so that callers can use pure INSERT.
///
/// The INSTEAD OF trigger for the HLC canonical view will be used to update
/// the HLC on the HLC state table. This will be called from within trigger
/// statements for the CRDT data table.
@override
List<String> generateSetupStatements() => [
..._createCrdtDataViewSetupStatements(),
_createHlcCanonicalViewStatement(),
_createIncrementCanonicalHlcTriggerStatement(),
];
/// CRDT data view scope: view name, normalized table, and derived SQL
/// fragments. Returns the CREATE VIEW and the three INSTEAD OF triggers.
List<String> _createCrdtDataViewSetupStatements() {
return [
_createCrdtDataViewStatement(),
_createCrdtDataInsteadOfInsertTriggerStatement(),
_createCrdtDataInsteadOfUpdateTriggerStatement(),
_createCrdtDataInsteadOfDeleteTriggerStatement(),
];
}
String get _normalizedTableName => crdtDb.crdtNormalizedDataTable.actualTableName;
/// CREATE VIEW for the CRDT data view: combines the split HLC columns from
/// the normalized table back into a single hlc_timestamp text field.
String _createCrdtDataViewStatement() =>
'''
CREATE VIEW IF NOT EXISTS "$crdtDataTableName" AS
SELECT
"user_id",
"table_name",
"column_name",
"row_id",
${_getHlcCombineExprSql()} AS "hlc_timestamp",
"raw_value"
FROM "$_normalizedTableName";''';
/// INSTEAD OF INSERT on the CRDT data view: insert into the normalized table,
/// splitting the hlc_timestamp text into 3 columns, and on conflict update
/// only when the new row has a greater hlc_timestamp.
String _createCrdtDataInsteadOfInsertTriggerStatement() =>
'''
CREATE TRIGGER IF NOT EXISTS "${crdtDataTableName}__instead_of_insert"
INSTEAD OF INSERT ON "$crdtDataTableName"
BEGIN
INSERT INTO "$_normalizedTableName" (
"user_id", "table_name", "column_name", "row_id",
"hlc_timestamp", "hlc_counter", "hlc_node_id", "raw_value"
)
SELECT
NEW."user_id",
NEW."table_name",
NEW."column_name",
NEW."row_id",
CAST(SUBSTR(NEW."hlc_timestamp", 1, 15) AS INTEGER),
${_hexToIntExpr('SUBSTR(NEW."hlc_timestamp", 17, 4)')},
SUBSTR(NEW."hlc_timestamp", 22),
NEW."raw_value"
ON CONFLICT ("user_id", "table_name", "column_name", "row_id") DO UPDATE SET
"hlc_timestamp" = excluded."hlc_timestamp",
"hlc_counter" = excluded."hlc_counter",
"hlc_node_id" = excluded."hlc_node_id",
"raw_value" = excluded."raw_value"
WHERE "hlc_timestamp" < excluded."hlc_timestamp"
OR ("hlc_timestamp" = excluded."hlc_timestamp" AND "hlc_counter" < excluded."hlc_counter");
END;''';
/// INSTEAD OF UPDATE on the CRDT data view: apply the new values to the
/// matching row in the normalized table (by primary key), splitting the
/// hlc_timestamp text into 3 columns.
String _createCrdtDataInsteadOfUpdateTriggerStatement() =>
'''
CREATE TRIGGER IF NOT EXISTS "${crdtDataTableName}__instead_of_update"
INSTEAD OF UPDATE ON "$crdtDataTableName"
BEGIN
UPDATE "$_normalizedTableName"
SET
"user_id" = NEW."user_id",
"table_name" = NEW."table_name",
"column_name" = NEW."column_name",
"row_id" = NEW."row_id",
"hlc_timestamp" = CAST(SUBSTR(NEW."hlc_timestamp", 1, 15) AS INTEGER),
"hlc_counter" = ${_hexToIntExpr('SUBSTR(NEW."hlc_timestamp", 17, 4)')},
"hlc_node_id" = SUBSTR(NEW."hlc_timestamp", 22),
"raw_value" = NEW."raw_value"
WHERE "user_id" = OLD."user_id"
AND "table_name" = OLD."table_name"
AND "column_name" = OLD."column_name"
AND "row_id" = OLD."row_id";
END;''';
/// INSTEAD OF DELETE on the CRDT data view: delete the matching row from the
/// normalized table (by primary key).
String _createCrdtDataInsteadOfDeleteTriggerStatement() =>
'''
CREATE TRIGGER IF NOT EXISTS "${crdtDataTableName}__instead_of_delete"
INSTEAD OF DELETE ON "$crdtDataTableName"
BEGIN
DELETE FROM "$_normalizedTableName"
WHERE "user_id" = OLD."user_id"
AND "table_name" = OLD."table_name"
AND "column_name" = OLD."column_name"
AND "row_id" = OLD."row_id";
END;''';
/// CREATE VIEW for the HLC canonical view: an empty view (WHERE FALSE) used
/// only as a target for INSTEAD OF INSERT to advance the HLC.
String _createHlcCanonicalViewStatement() =>
'''
CREATE VIEW IF NOT EXISTS "$hlcCanonicalViewName" AS
SELECT CAST(NULL AS TEXT) AS "user_id"
WHERE FALSE;''';
/// INSTEAD OF INSERT on the HLC canonical view: validates clock and counter,
/// then upserts the HLC state row (advancing timestamp/counter on conflict).
String _createIncrementCanonicalHlcTriggerStatement() =>
'''
CREATE TRIGGER IF NOT EXISTS "increment_canonical_hlc"
INSTEAD OF INSERT ON "$hlcCanonicalViewName"
BEGIN
SELECT RAISE(ABORT, 'Clock drift exceeds 1 minute maximum.')
FROM "$hlcStateTableName"
WHERE "user_id" = NEW."user_id"
AND "last_timestamp" > (unixepoch('now','subsec') + 60) * 1000;
SELECT RAISE(ABORT, 'Timestamp counter overflow: 0xFFFF.')
FROM "$hlcStateTableName"
WHERE "user_id" = NEW."user_id"
AND "counter" = 0xFFFF;
INSERT INTO "$hlcStateTableName" ("user_id", "last_timestamp", "counter")
SELECT NEW."user_id", ts, 0
FROM (
SELECT CAST(unixepoch('now','subsec') * 1000 AS INTEGER) AS ts
)
WHERE TRUE
ON CONFLICT ("user_id") DO UPDATE SET
"last_timestamp" = MAX("last_timestamp", EXCLUDED."last_timestamp"),
"counter" = (
CASE
WHEN EXCLUDED."last_timestamp" > "last_timestamp" THEN 0
ELSE "counter" + 1
END
);
END;''';
}
/// Extension methods for the [GeneratedDatabase] class to get the CRDT trigger names.
extension GetDatabaseCrdtTriggers on GeneratedDatabase {
/// Returns the names of the CRDT triggers in the database.
Future<List<String>> getCrdtTriggers() async {
final result = await customSelect(
'''\
SELECT type, name, sql
FROM sqlite_master
WHERE (type = "trigger")
AND (name LIKE "%%__crdt__%%");''',
).get();
return result.map((row) => row.read<String>('name')).toList();
}
}
extension on TableInfo {
Iterable<String> get nonGenerated =>
$columns.where((c) => c.generatedAs == null).map((c) => c.$name);
}

View file

@ -0,0 +1,21 @@
import 'package:drift/drift.dart';
import 'hlc.dart';
export 'hlc.dart';
class _HlcConverter extends TypeConverter<Hlc, String> {
const _HlcConverter();
@override
Hlc fromSql(String fromDb) => Hlc.parse(fromDb);
@override
String toSql(Hlc value) => value.toJson(unixTimestamp: true);
}
/// Converter for [Hlc] to store it in the database as string.
const hlcConverter = _HlcConverter();
/// Null-aware converter for [Hlc] to handle nullable fields.
const nullableHlcConverter = NullAwareTypeConverter.wrap(hlcConverter);

View file

@ -0,0 +1,39 @@
/// Exception thrown when the clock drift exceeds the maximum allowed.
class ClockDriftException implements Exception {
/// Creates a new instance of [ClockDriftException].
ClockDriftException(DateTime dateTime, DateTime wallTime, this.maxDrift)
: drift = dateTime.difference(wallTime);
/// The duration of the clock drift.
final Duration drift;
/// The maximum allowed clock drift.
final Duration maxDrift;
@override
String toString() => 'Clock drift of $drift ms exceeds maximum ($maxDrift)';
}
/// Exception thrown when the timestamp counter overflows.
class OverflowException implements Exception {
/// Creates a new instance of [OverflowException].
OverflowException(this.counter);
/// The counter that overflowed.
final int counter;
@override
String toString() => 'Timestamp counter overflow: $counter';
}
/// Exception thrown when a duplicate node is detected.
class DuplicateNodeException implements Exception {
/// Creates a new instance of [DuplicateNodeException].
DuplicateNodeException(this.nodeId);
/// The node ID that is duplicated.
final String nodeId;
@override
String toString() => 'Duplicate node: $nodeId';
}

View file

@ -0,0 +1,160 @@
// This file includes code from the project: https://github.com/cachapa/crdt
// Licensed under Apache License 2.0: https://www.apache.org/licenses/LICENSE-2.0
// Copyright Daniel Cachapa.
//
// Modifications copyright 2026 Marcelo Soares
// Licensed under MIT License: https://opensource.org/licenses/MIT
import 'package:clock/clock.dart';
import 'exceptions.dart';
/// A Hybrid Logical Clock implementation that supports both ISO 8601 and Unix
/// timestamp formats. The datetime component has millisecond precision and is
/// always stored in UTC.
///
/// Although the original HLC paper uses microsecond precision, since it was
/// proved that the counter typically stays in single digits, millisecond
/// precision can be considered enough for the vast majority of applications.
///
/// The advantage of using milliseconds is that the timestamp can be generated
/// on any database (inside SQLite3 for example) and will occupy less space in
/// the database.
///
/// Inspiration: https://cse.buffalo.edu/tech-reports/2014-04.pdf
class Hlc implements Comparable<Hlc> {
/// Creates a new instance of [Hlc].
Hlc(DateTime dateTime, this.counter, this.nodeId)
: dateTime = dateTime.toUtcMillisecond(),
assert(counter <= _maxCounter);
/// Instantiates an Hlc at the beginning of time and space: January 1, 1970.
Hlc.zero(String nodeId) : this(DateTime.utc(1970), 0, nodeId);
/// Instantiates an Hlc using the wall clock.
Hlc.now(String nodeId) : this(DateTime.now(), 0, nodeId);
/// Parse an HLC string in the format `<ISO8601_date>-<counter>-<node_id>` or
/// `<unix_timestamp>-<counter>-<node_id>`.
factory Hlc.parse(String timestamp) {
final indexOfColon = timestamp.lastIndexOf(':');
final indexOfStart = indexOfColon > 0 && indexOfColon < 20 ? indexOfColon : 0;
final counterDash = timestamp.indexOf('-', indexOfStart);
final timestampPart = timestamp.substring(0, counterDash);
final nodeIdDash = timestamp.indexOf('-', counterDash + 1);
final counterPart = timestamp.substring(counterDash + 1, nodeIdDash);
final unixTimestamp = int.tryParse(timestampPart);
final datetime = unixTimestamp != null
? DateTime.fromMillisecondsSinceEpoch(unixTimestamp, isUtc: true)
: DateTime.tryParse(timestampPart);
if (datetime == null) {
throw FormatException('Invalid HLC string: $timestamp');
}
final counter = int.parse(counterPart, radix: 16);
final nodeId = timestamp.substring(nodeIdDash + 1);
return Hlc(datetime, counter, nodeId);
}
/// The date and time of the HLC.
final DateTime dateTime;
/// The counter of the HLC.
final int counter;
/// The node ID of the HLC.
final String nodeId;
/// The Unix timestamp of the HLC.
int get unixTimestamp => dateTime.millisecondsSinceEpoch;
static const _maxDrift = Duration(minutes: 1);
static const _maxCounter = 0xFFFF;
/// Increments the current timestamp for transmission to another system.
Hlc increment() {
final localWallTime = clock.now().toUtc();
final dateTimeNew = localWallTime.isAfter(dateTime) ? localWallTime : dateTime;
final counterNew = dateTimeNew == dateTime ? counter + 1 : 0;
if (dateTimeNew.difference(localWallTime) > const Duration(minutes: 1)) {
throw ClockDriftException(dateTimeNew, localWallTime, _maxDrift);
}
if (counterNew > _maxCounter) {
throw OverflowException(counterNew);
}
return Hlc(dateTimeNew, counterNew, nodeId);
}
/// Compares and validates a timestamp from a remote system with the local
/// timestamp to preserve monotonicity.
Hlc merge(Hlc remote) {
if (remote.dateTime.isBefore(dateTime) ||
(remote.dateTime.isAtSameMomentAs(dateTime) && remote.counter <= counter)) {
return this;
}
if (nodeId == remote.nodeId) {
throw DuplicateNodeException(nodeId);
}
final localWallTime = clock.now().toUtc();
if (remote.dateTime.difference(localWallTime) > _maxDrift) {
throw ClockDriftException(remote.dateTime, localWallTime, _maxDrift);
}
return remote.copyWith(nodeId: nodeId);
}
/// Create a copy of this object replacing the optional properties.
Hlc copyWith({DateTime? dateTime, int? counter, String? nodeId}) =>
Hlc(dateTime ?? this.dateTime, counter ?? this.counter, nodeId ?? this.nodeId);
/// Convenience method for easy json encoding.
String toJson({bool unixTimestamp = false}) => toString(unixTimestamp: unixTimestamp);
@override
String toString({bool unixTimestamp = false}) => [
if (unixTimestamp)
this.unixTimestamp.toString().padLeft(15, '0')
else
dateTime.toIso8601String(),
counter.toRadixString(16).toUpperCase().padLeft(4, '0'),
nodeId,
].join('-');
@override
int get hashCode => toString().hashCode;
@override
bool operator ==(Object other) => other is Hlc && compareTo(other) == 0;
/// Whether this [Hlc] is causally before the [other].
bool operator <(Object other) => other is Hlc && compareTo(other) < 0;
/// Whether this [Hlc] is causally before or equal to the [other].
bool operator <=(Object other) => this < other || this == other;
/// Whether this [Hlc] is causally after the [other].
bool operator >(Object other) => other is Hlc && compareTo(other) > 0;
/// Whether this [Hlc] is causally after or equal to the [other].
bool operator >=(Object other) => this > other || this == other;
@override
int compareTo(Hlc other) => dateTime.isAtSameMomentAs(other.dateTime)
? counter == other.counter
? nodeId.compareTo(other.nodeId)
: counter - other.counter
: dateTime.compareTo(other.dateTime);
}
extension on DateTime {
DateTime toUtcMillisecond() =>
DateTime.fromMillisecondsSinceEpoch(millisecondsSinceEpoch, isUtc: true);
}

View file

@ -0,0 +1,293 @@
import 'package:drift/drift.dart';
import 'crdt/crdt.dart';
import 'database/database.dart';
import 'database/tables/control.dart';
import 'database/tables/data.dart';
import 'database/triggers.dart';
/// The bias to use when conflicting operations are detected.
enum ConflictingBias {
/// Newer updates are overridden by deletes.
deleteWins,
/// Deletes are undone by newer conflicting updates.
updateWins,
}
/// Runs migrations declared by a [MigrationStrategy].
class OfflineSyncMigrator extends Migrator {
/// Used internally by drift when opening the database.
OfflineSyncMigrator(
super.database, {
required this.userId,
required this.nodeId,
this.conflictingBias = ConflictingBias.deleteWins,
/// Restrict tables to synchronize. If not provided, all tables will be considered.
List<TableInfo>? synchronizedTables,
/// Optionally specify tables to exclude from synchronization.
List<TableInfo>? excludeTables,
}) : synchronizedTables = synchronizedTables ?? database.allTables.toList() {
if (excludeTables != null) {
this.synchronizedTables.removeWhere(excludeTables.contains);
}
_validateSynchronizedTables();
}
/// The user ID for the CRDT system.
///
/// Note that this is the user ID, not the node ID. A user can have multiple nodes,
/// all sharing the same user ID so that they can synchronize with each other.
final String userId;
/// The node ID for the CRDT system.
///
/// A node is a unique identifier for a device or application instance.
final String nodeId;
/// Tables to be synchronized with CRDT.
final List<TableInfo> synchronizedTables;
/// The bias to use when conflicting operations are detected.
///
/// The default is [ConflictingBias.deleteWins], which is the most common case.
final ConflictingBias conflictingBias;
/// The database instance to apply the CRDT schema to.
late final crdtDb = CrdtDatabase(
database,
userId: userId,
nodeId: nodeId,
synchronizedTables: synchronizedTables,
);
/// The CRDT execution instance.
late final crdt = OfflineSyncCrdt(crdtDb, userId: userId, nodeId: nodeId);
/// The trigger creator for the CRDT system.
late final triggerCreator = switch (database.executor.dialect) {
SqlDialect.sqlite => Sqlite3OfflineSyncTriggers(crdtDb, conflictingBias),
// TODO: Implement the triggers for the Postgres dialect.
_ => throw UnsupportedError('Unsupported dialect: ${database.executor.dialect}'),
};
/// Migrations for tables that are part of the CRDT system.
final pendingMigrations = <String, CrdtTableMigration>{};
/// Eagerly validates that all synchronized tables are supported.
///
/// This prevents errors at runtime later if tables do not meet the requirements.
void _validateSynchronizedTables() {
for (final table in synchronizedTables) {
if (table.$primaryKey.isEmpty) {
throw ArgumentError(
'Table "${table.actualTableName}" has no primary key and is therefore not '
'supported to track changes to it. Either add a primary key to the table '
'or remove it from the synchronized tables list.',
);
}
}
}
/// Gets the migration for a table that is part of the CRDT system.
///
/// If the table is not in the list of synchronized tables, a new migration
/// will be created.
CrdtTableMigration _register(String tableName) {
return pendingMigrations.putIfAbsent(
tableName,
() => CrdtTableMigration(tableName),
);
}
/// Creates the CRDT-related entities in the database.
Future<void> createCrdtControl() async {
for (final table in crdtDb.allTables) {
if (table is CrdtDataTable) continue;
await createTable(table);
}
if (crdtDb.isPostgres) {
final crdtTable = crdtDb.crdtNormalizedDataTable;
await database.customStatement(
'ALTER TABLE ${crdtTable.actualTableName} '
// TODO: Fix this partitioning, since RANGE can not be used for discrete values.
'PARTITION BY RANGE (${crdtTable.userId.name}, ${crdtTable.tblName.name});',
);
// TODO: Alter all ON DELETE FK constraints from RESTRICT to NO ACTION.
// TODO: Alter all FK constraints to be deferrable initially deferred.
}
for (final statement in triggerCreator.generateSetupStatements()) {
await database.customStatement(statement);
}
for (final table in synchronizedTables) {
await _createTriggers(table);
}
// NOTE: Ensure that the CRDT control table always contains the default entry.
// This is needed to prevent triggers from failing due to missing entries.
// This row must be inserted using the custom statement since the database
// is still locked by the migrations.
await database.customStatement(
CrdtControlTable.getDefaultUpsertQuery(
userId: crdtDb.userId,
nodeId: crdtDb.nodeId,
schemaVersion: crdtDb.schemaVersion,
),
);
}
Future<void> _createTriggers(TableInfo table) async {
final triggers = triggerCreator.generateCreateTriggerStatements(table);
for (final trigger in triggers) {
await database.customStatement(trigger);
}
}
@override
Future<void> createAll() async {
await super.createAll();
await createCrdtControl();
}
@override
Future<void> createTable(TableInfo table) async {
// Tables that start with 'tmp_for_copy_' are used in alter table operations.
// We do not want to register them as created tables.
if (!table.actualTableName.startsWith('tmp_for_copy_')) {
_register(table.actualTableName).created = true;
}
await super.createTable(table);
if (synchronizedTables.contains(table)) {
await _createTriggers(table);
}
}
@override
Future<void> alterTable(TableMigration migration) async {
final table = migration.affectedTable;
pendingMigrations.putIfAbsent(
table.actualTableName,
() => CrdtTableMigration.from(migration),
);
final existingTriggers = await database.getCrdtTriggers();
for (final trigger in existingTriggers) {
await database.customStatement('DROP TRIGGER IF EXISTS $trigger;');
}
await super.alterTable(migration);
await _createTriggers(table);
}
@override
Future<void> drop(DatabaseSchemaEntity entity) async {
if (entity is TableInfo) {
_register(entity.actualTableName).dropped = true;
}
return super.drop(entity);
}
@override
Future<void> deleteTable(String name) async {
_register(name).dropped = true;
return super.deleteTable(name);
}
@override
Future<void> addColumn(TableInfo table, GeneratedColumn column) async {
_register(table.actualTableName).onColumn(column.$name).added = true;
return super.addColumn(table, column);
}
@override
Future<void> dropColumn(TableInfo table, String column) async {
_register(table.actualTableName).onColumn(column).dropped = true;
return super.dropColumn(table, column);
}
@override
Future<void> renameColumn(
TableInfo table,
String oldName,
GeneratedColumn column,
) async {
_register(table.actualTableName).onColumn(column.$name).oldName = oldName;
return super.renameColumn(table, oldName, column);
}
@override
Future<void> renameTable(TableInfo table, String oldName) async {
_register(table.actualTableName).oldName = oldName;
return super.renameTable(table, oldName);
}
}
/// A migration for a table that is part of the CRDT system.
class CrdtTableMigration {
/// Creates a new instance of [CrdtTableMigration].
CrdtTableMigration(this.tableName);
/// The table that is being migrated.
final String tableName;
/// Whether the table was created.
bool created = false;
/// Whether the table was dropped.
bool dropped = false;
/// The old name of the table, if the table was renamed.
String? oldName;
/// The columns that were added.
final columns = <String, CrdtColumnMigration>{};
/// Whether the table was modified.
bool get modified => oldName != null || columns.isNotEmpty;
/// Gets the migration for a column that is part of the CRDT system.
///
/// If the column is not in the list of synchronized columns, a new migration
/// will be created.
CrdtColumnMigration onColumn(String columnName) {
return columns.putIfAbsent(columnName, () => CrdtColumnMigration(columnName));
}
/// Creates a new instance of [CrdtTableMigration] from a [TableMigration].
static CrdtTableMigration from(TableMigration migration) {
final table = migration.affectedTable;
final tableMigration = CrdtTableMigration(table.actualTableName);
for (final column in migration.newColumns) {
// TODO: Handle the column transformer.
tableMigration.onColumn(column.name);
}
return tableMigration;
}
}
/// A migration for a table that is part of the CRDT system.
class CrdtColumnMigration {
/// Creates a new instance of [CrdtColumnMigration].
CrdtColumnMigration(this.columnName);
/// The column that is being migrated.
final String columnName;
/// Whether the column was added.
bool added = false;
/// Whether the column was dropped.
bool dropped = false;
/// The old name of the column, if the column was renamed.
String? oldName;
/// Whether the column type was modified.
bool changedType = false;
}

View file

@ -0,0 +1,62 @@
import 'package:drift/drift.dart';
import '../database/database.dart';
/// Provides the base building blocks for SQL statements to interact with the
/// CRDT data table.
class CrdtDataSqlBuilder {
/// Creates a new instance of [CrdtDataSqlBuilder].
const CrdtDataSqlBuilder(this.crdtDb);
/// The migrator instance to apply the CRDT schema to.
final CrdtDatabase crdtDb;
/// Prefix for all CRDT-related entities.
String get prefix => '__crdt';
/// Separator for joining primary key columns in the row ID.
String get rowIdSeparator => '||';
/// Name of the column that stores the information about whether the row is deleted.
String get isDeletedColumnName => '${prefix}_is_deleted';
/// Name of the CRDT data table.
String get crdtDataTableName => crdtDb.crdtDataTable.actualTableName;
/// Name of the HLC state table.
String get hlcStateTableName => crdtDb.crdtHlcStateTable.actualTableName;
/// The list of quoted column names in the CRDT data table.
Iterable<String> get crdtDataAllColumns => crdtDb.crdtDataTable.$columns.quotedNames;
/// The list of quoted primary key column names in the CRDT data table.
Iterable<String> get crdtDataPrimaryKeyColumns =>
crdtDb.crdtDataTable.$primaryKey.quotedNames;
/// Returns an expression that uniquely identifies a row in the table.
///
/// If the table has a single primary key column, the expression will be the
/// column value. If it has multiple primary key columns, the values for all
/// will be concatenated using the separator.
String getUniqueRowId(TableInfo table, {required String tableAlias}) {
if (table.$primaryKey.isEmpty) {
throw StateError(
'Table ${table.actualTableName} has no primary key and is therefore '
'not supported to track changes to it.',
);
}
if (table.$primaryKey.length == 1) {
return '$tableAlias."${table.$primaryKey.first.$name}"';
}
final columnNames = table.$primaryKey
.map((c) => '$tableAlias."${c.$name}"')
.join(', ');
return "CONCAT_WS('$rowIdSeparator', $columnNames)";
}
}
/// Extension methods for the [Iterable<GeneratedColumn>] class to get the column names with quotes.
extension GetColumnNamesWithQuotes on Iterable<GeneratedColumn> {
/// Returns the column names with quotes for safer SQL queries.
Iterable<String> get quotedNames => map((c) => '"${c.$name}"');
}