From ace841de737b7fdfef981db81f90bbb3ca2c5124 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcelo=20Mendon=C3=A7a=20Soares?= Date: Thu, 9 Jul 2026 01:16:49 -0300 Subject: [PATCH] chore: Transfer the package license to Serverpod (#99) --- .gitignore | 1 + LICENSE | 40 ++- README.md | 327 +++++++++++++++++- benchmark/pubspec.yaml | 2 +- docs/one-pager.md | 211 +++++++++++ example/offline_sync_demo/pubspec.yaml | 2 +- legacy/drift_offline_sync/pubspec.yaml | 2 +- .../serverpod_offline_sync_client/LICENSE | 27 ++ .../pubspec.yaml | 10 +- .../serverpod_offline_sync_server/LICENSE | 27 ++ .../pubspec.yaml | 12 +- .../serverpod_offline_sync_shared/LICENSE | 27 ++ .../pubspec.yaml | 6 +- pubspec.yaml | 2 +- .../pubspec.yaml | 6 +- .../pubspec.yaml | 8 +- 16 files changed, 653 insertions(+), 57 deletions(-) create mode 100644 docs/one-pager.md create mode 100644 packages/serverpod_offline_sync_client/LICENSE create mode 100644 packages/serverpod_offline_sync_server/LICENSE create mode 100644 packages/serverpod_offline_sync_shared/LICENSE diff --git a/.gitignore b/.gitignore index 16ce044..38375c2 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ pubspec.lock .dart_skills/ .cursor/skills/relic* .cursor/skills/serverpod* +temp/ # Coverage reports coverage/ diff --git a/LICENSE b/LICENSE index 22f6ce0..b183ba3 100644 --- a/LICENSE +++ b/LICENSE @@ -1,21 +1,27 @@ -MIT License +Copyright 2025 The Serverpod authors -Copyright (c) 2025 Marcelo Mendonça Soares +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. +3. Neither the name of the copyright holder nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +4. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. diff --git a/README.md b/README.md index 2d889f8..3a1c306 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ [![GitHub](https://img.shields.io/badge/GitHub-marcelomendoncasoares-181717.svg?style=flat&logo=github)](https://github.com/marcelomendoncasoares) [![Pub Package](https://img.shields.io/pub/v/serverpod_offline_sync.svg)](https://pub.dev/packages/serverpod_offline_sync) [![Dart SDK Version](https://badgen.net/pub/sdk-version/serverpod_offline_sync.svg)](https://pub.dev/packages/serverpod_offline_sync) -[![License: MIT](https://img.shields.io/badge/license-MIT-yellow.svg)](https://github.com/marcelomendoncasoares/serverpod_offline_sync/blob/main/LICENSE) +[![License: BSD 3-Clause](https://img.shields.io/badge/license-BSD_3--Clause-yellow.svg)](https://github.com/marcelomendoncasoares/serverpod_offline_sync/blob/main/LICENSE) [![CI](https://github.com/marcelomendoncasoares/serverpod_offline_sync/actions/workflows/ci.yaml/badge.svg?branch=main)](https://github.com/marcelomendoncasoares/serverpod_offline_sync/actions/workflows/ci.yaml) A plugin to build offline-first applications with Serverpod. Develop your app @@ -12,40 +12,337 @@ handle the sync process using CRDT (Conflict-free Replicated Data Type). Works out-of-the-box with Serverpod's existing APIs. - [Serverpod Offline Sync](#serverpod-offline-sync) + - [Overview](#overview) - [Features](#features) - [Getting started](#getting-started) + - [1. Add the package to the project](#1-add-the-package-to-the-project) + - [2. Prepare the synced tables](#2-prepare-the-synced-tables) + - [3. Configure the sync engine on the server](#3-configure-the-sync-engine-on-the-server) + - [4. Configure the sync engine on the client](#4-configure-the-sync-engine-on-the-client) + - [5. Wire the sync call on the client](#5-wire-the-sync-call-on-the-client) - [Usage](#usage) + - [Scopes and sharing](#scopes-and-sharing) + - [Data modeling limitations](#data-modeling-limitations) + - [How it works](#how-it-works) + - [Performance](#performance) + - [Example](#example) + - [Additional information](#additional-information) - [License](#license) --- +## Overview + +**Serverpod Offline Sync** turns an ordinary [Serverpod](https://serverpod.dev) +backend into an offline-first, collaborative system, without changing how you +write your app. It is a delta-CRDT engine that runs **the same code on the +client and server databases**, transparently and with minimal configuration. + +It sits nearly alone in the space of offline-first solutions, because most +others either use a non-relational database, require a new data API, or don't +preserve foreign-key/unique invariants while merging data. This package is built +on top of Serverpod, leveraging its ORM, server, and client to expose these +capabilities with almost nothing new to learn: you build a normal offline +Serverpod app (generated models, ordinary CRUD) and the plugin handles +replication automatically. + +The sync engine is a **black box that "just works"**: changes are tracked +atomically with each operation and you never intervene in conflict resolution. +Conflicts that need a human decision **bubble up as visible state** for the end +user to act on (e.g. a delete reverted by a concurrent constraint reappears, so +the user can simply delete it again). Migrations work normally, and deleted rows +never leave their tables — they are transparently hidden from the user. + +> The CRDT layer was inspired by the excellent work on +> [Synql](https://github.com/coast-team/synql) +> ([Ignat et al., DAIS 2024](https://inria.hal.science/hal-04969158v4/document)). +> This package goes beyond Synql by covering the full set of foreign-key +> `onDelete` actions, extending unique-constraint handling to deeply relational +> schemas, and shipping a production-ready implementation. + ## Features -TODO: List what your package can do. Maybe include GIFs, or videos. +- **Offline-first, no new API.** Read and write generated models with ordinary + Serverpod CRUD against a local database; the plugin replicates for you. +- **Same code on both ends.** Identical logic runs on the client's SQLite and + the server's Postgres, so there are no asymmetric merge results to reconcile. +- **Automatic conflict resolution.** Field-level merges ordered by hybrid + logical clocks, with a monotone causal-length tombstone governing row + existence so add/delete/restore can never oscillate. +- **Relational integrity preserved.** All foreign-key `onDelete` actions + (restrict / cascade / set null / set default) and unique constraints are + honored after every merge, as if the merged data had existed in a single + database at the time of each operation. +- **Conflicts surface to the user.** Resolutions that require a human decision + become visible state instead of silent data loss. +- **Scopes and sharing.** Data is isolated per scope; each user has a personal + scope and can be a member of any number of read-only or read-write shared + scopes. +- **One-shot and continuous sync.** The same stream-based protocol powers both + pull-to-refresh and near-real-time streaming. +- **Minimal overhead.** Metadata is normalized and never copies row data, + keeping the storage and performance penalty small (see [Performance](#performance)). ## Getting started -TODO: List prerequisites and provide or point to information on how to start -using the package. +Using the package on a project takes just five steps. After this setup, you +develop the app with normal repository calls against the wrapped `session`. + +> [!NOTE] +> This package is still in development and is not yet ready for production use. +> Although the package is functional and feature-complete, it will still pass +> through some refactors and breaking changes for a better integration on real +> Serverpod projects. + +### 1. Add the package to the project + +Since this package is still in development and not yet released to pub.dev, +you need to add it as a git reference to the project. On the client, add the following to the `pubspec.yaml` file: + +```yaml +dependencies: + serverpod_offline_sync_client: + git: https://github.com/marcelomendoncasoares/serverpod_offline_sync.git + path: packages/serverpod_offline_sync_client + ref: main +``` + +And, on the server's `pubspec.yaml`, add the following: + +```yaml +dependencies: + serverpod_offline_sync_server: + git: https://github.com/marcelomendoncasoares/serverpod_offline_sync.git + path: packages/serverpod_offline_sync_server + ref: main +``` + +Be mindful that installing the package as a git reference to `main` is subject +to sudden changes when resolving dependencies. If you want to use a specific +version, pin the commit hash on the `ref` field instead. + +After adding the dependencies, list the `serverpod_offline_sync` module on the +`generator.yaml` file: + +```yaml +modules: + serverpod_offline_sync: + nickname: offline_sync +``` + +The next call to `serverpod generate` and `serverpod create-migration` will +include the `offline_sync` module in the project. Note that the client tables +will only be present on the client's migration if at least one model on the +project is defined with `database: all` or `database: client`. + +### 2. Prepare the synced tables + +Each model you want to sync must be defined with `database: all` (so it exists +on both the client and the server), have a UUID primary key, and declare a +`scopeId` field. You mostly never set `scopeId` nor see its value on reads; it +is maintained by the CRDT sync layer. + +```yaml +class: Person +table: person +database: all +fields: + id: UuidValue?, defaultPersist=random_v7 + ### Owner scope of this row. Maintained by the CRDT sync layer. + ### The user never sets this field nor sees its value on reads. + scopeId: int?, relation(parent=crdt_scopes, onDelete=Cascade) + name: String +``` + +### 3. Configure the sync engine on the server + +On the server, the `databaseInterceptor` is what ensures that all `session.db` +objects are database instances that tracks operations for sync. + +```dart +final pod = Serverpod( + args, + Protocol(), + Endpoints(), + // Add the database interceptor. + databaseInterceptor: crdtDatabaseInterceptor, +); + +// Initialize the sync engine before pod.start() is called. +pod.initializeCrdtSync( + syncTables: [Person.t, Book.t, Author.t], +); +``` + +Don't forget to also configure the authentication following the regular +Serverpod auth setup. Authentication is required because the sync needs a +logged in user to synchronize data. + +### 4. Configure the sync engine on the client + +On the client, the regular `DatabaseSession` must be replaced by a wrapped +version that includes the sync engine. + +```dart +final session = CrdtDatabaseSession.wraps( + await client.createSession(databasePath, isDebugMode: kDebugMode), + syncTables: [Person.t, Book.t, Author.t], + persistentUserId: persistentUserId, +); + +// Initialize the sync engine. +await session.db.initialize(); +``` + +Then, store the `session` instance in the service locator. + +### 5. Wire the sync call on the client + +Be sure to authenticate the client before calling the sync methods. Otherwise, +the sync operation will fail with a `Unauthorized` error. + +```dart +// Push local changes and merge remote ones once (i.e. pull-to-refresh). +await client.crdt.syncOnce(session); + +// Or stream changes both ways until cancelled for a near-real-time sync. +final syncSession = client.crdt.syncContinuously(session); +``` ## Usage -TODO: Include short and useful examples for package users. Add longer examples -to `/example` folder. +Once wired up, use normal generated-model CRUD against the `session` instance — +`Person.db.insertRow(session, person)`, `session.db.find()`, and so on. +The sync layer tracks every operation atomically; you never touch conflict +resolution. -See the `/example` directory for a... +### Scopes and sharing -```dart -const like = 'sample'; +Data is isolated per **scope**. Each user has their own implicit personal scope +and can be a member of any number of shared scopes, with read-only or read-write +access. + +- **On the server**, wrap database operations in `session.db.transactionForUser` + to target the correct scope. +- **On the client**, set a `persistentUserId` to target the personal scope by + default, or use `transactionForUser` to target a shared scope. + +### Data modeling limitations + +Because of the nature of merge conflicts, the package imposes some data-modeling +limitations, enforced at runtime during initialization. Most are fundamental to +the design and can never be lifted. + +- Every synced table must: + - Exist on both the client and the server with `database: all`. + - Have a UUID primary key. + - Declare the field `scopeId: int?, relation(parent=crdt_scopes, onDelete=Cascade)`. +- Unique indexes must include `scopeId` together with the target columns. +- Global unique indexes are unsupported, except for FK-only indexes. +- Unique indexes are only supported with at least one + `String`/`UuidValue`/nullable column. +- All 1:1 relations must have the foreign-key column nullable (`optional` + relation). +- The only allowed non-synced-to-synced relation is `scopeId -> crdt_scopes.id`. +- All foreign-key relations must be `deferrable` (pending [serverpod#5338](https://github.com/serverpod/serverpod/issues/5338)). + +Respecting these limitations, all other database invariants — including +foreign-key actions — are preserved, with the exception of check constraints, +which Serverpod does not support either. + +## How it works + +- **Architecture.** The engine is a transparent layer between the application + and its database. Because identical code runs on the client's SQLite and the + server's Postgres, there are no asymmetric merge results to reconcile. + Migrations are generated symmetrically for both ends. The engine respects the + three mathematical properties a CRDT must hold to be consistent: idempotence, + associativity, and commutativity. +- **Metadata model.** Three metadata tables track change history *without copying + row data*: `crdt_data_rows` (insert + visibility), `crdt_data_fields` (update), + and `crdt_data_tombstone` (delete/restore). Hybrid logical clocks order + field-level values and break ties. A monotone causal-length tombstone governs + row existence, so add/delete/restore advance generations forward and can never + oscillate. +- **Visibility as a pure function.** Replicas never coordinate. Each merge + applies remote facts monotonically, then derives the visible database — + tombstone state, unique-conflict winners, and foreign-key repair — as a + deterministic projection computed to a fixed point. +- **Ownership & scopes.** A *scope* is the unit of replication and ownership; a + *user* is an authentication identity. Each row is globally identified by + `(table, rowId)` and owned by exactly one scope, so merging is always one chain + per row. Scopes can be purged for GDPR / account erasure via a + database-enforced cascade. +- **Sync protocol.** A session handshakes once to validate that both ends share + the same table schema. Per cycle, the server sends the authoritative scope set, + and every scope the user can access is synced (transferred all at once in + chunks, merged one at a time). The same stream-based implementation powers both + one-shot and continuous sync. + +For a deeper dive, see the design docs in [`docs/`](docs): +[row ownership](docs/row-ownership.md), +[foreign-key invariants](docs/foreign-key-invariants.md), +[unique-constraint invariants](docs/unique-constraint-invariants.md), and +[sync/non-sync relations](docs/sync-non-sync-relations.md). + +## Performance + +Since all operations are tracked, a CRDT layer can weigh heavily on storage and +speed. This package was built to keep that penalty minimal. The benchmark below +uses 1,000 rows to measure the impact against a baseline of the same operations +without the CRDT layer, both on SQLite (a fresh report is posted as a comment +on every pull request): + +``` +📊 SELECT performance impact: + Time: 12.62 ms --> 12.67 ms (48.50 μs) + CRDT overhead: 0.38% slower + +📊 INSERT performance impact: + Delay per insert: 231.04 μs + +📊 UPDATE performance impact: + Delay per update: 1.88 ms + +📊 DELETE performance impact: + Delay per delete: 408.75 μs + +💽 Storage impact (base footprint removed): + Storage overhead range: 62.07% --> 300.00% ``` - +## Example + +The bundled desktop demo shows the model directly: two independent replicas of +one account side by side, plus the server's merged truth. Drive each replica's +sync independently and watch them converge. It visualizes hidden / tombstoned +rows and every foreign-key `onDelete` action, with guided scenarios for the +restrict / cascade / set-null / set-default / unique-conflict cases. + +Run it by starting the test server — the app launches alongside it: + +```sh +cd test/serverpod_offline_sync_test_server +serverpod start +``` + +See [`example/offline_sync_demo`](example/offline_sync_demo) for details; the +entire integration lives in ~60 lines in +[`lib/src/offline_replica.dart`](example/offline_sync_demo/lib/src/offline_replica.dart). + +## Additional information + +The package is tested extensively following Serverpod's testing principles. The +vast majority of the tests live in the `serverpod_offline_sync_test_server` +package, whose server is shared with the `example/offline_sync_demo` project. ## License -This package is distributed under the MIT License. See the LICENSE file for -more information. +This package is distributed under the BSD 3-Clause License. See the LICENSE +file for more information. diff --git a/benchmark/pubspec.yaml b/benchmark/pubspec.yaml index 9aaa0ca..8e42691 100644 --- a/benchmark/pubspec.yaml +++ b/benchmark/pubspec.yaml @@ -11,7 +11,7 @@ dependencies: benchmark_harness: ^2.4.0 intl: ^0.20.2 path: ^1.9.1 - serverpod_database: 3.5.0-beta.10 + serverpod_database: 4.0.0-beta.0 serverpod_offline_sync_server: path: ../packages/serverpod_offline_sync_server serverpod_offline_sync_shared: diff --git a/docs/one-pager.md b/docs/one-pager.md new file mode 100644 index 0000000..bb3c466 --- /dev/null +++ b/docs/one-pager.md @@ -0,0 +1,211 @@ +# Serverpod Offline Sync — Product One-Pager + +*This is a one-pager that describes the product and its features, plus a roadmap for the transference process to Serverpod's monorepo.* + +## Overview + +**Serverpod Offline Sync** is a Dart plugin that turns an ordinary [Serverpod](https://serverpod.dev) backend into an offline-first, collaborative system, without changing how developers write their app. It is a delta-CRDT (Conflict-free Replicated Data Type) engine that runs **the same code on the client and server databases**, transparently and with minimal configuration. + +It sits nearly alone in the space of offline-first solutions, because most other solutions either use a non-relational database, require a new data API, or don't preserve foreign-key/unique invariants while merging data. The solution is built on top of the Serverpod framework, leveraging its ORM, server and client to expose such complex capabilities with minimal configuration. + +The work on this package was inspired by the excellent work on [Synql](https://github.com/coast-team/synql) ([Ignat et al., DAIS 2024](https://inria.hal.science/hal-04969158v4/document)), which provided the foundation for the CRDT layer. This package goes beyond `Synql` by covering the full set of foreign-key `onDelete` actions, extending unique-constraint handling to deeply relational schemas, and shipping a production-ready implementation. + +### The developer experience + +The main selling point of the package is that there is almost nothing to learn. The app is built +as a normal offline Serverpod app (generated models, ordinary CRUD) and the plugin handles replication automatically. + +The four steps below contain all required changes on a project to use the package: + +1. Prepare the synced tables by adding the `scopeId` field to the models. + ```yaml + class: Person + table: person + database: all + fields: + id: UuidValue?, defaultPersist=random_v7 + ### Owner scope of this row. Maintained by the CRDT sync layer. + ### The user never sets this field nor sees its value on reads. + scopeId: int?, relation(parent=crdt_scopes, onDelete=Cascade) + name: String + ``` + +2. Configure the sync engine on the server. + ```dart + final pod = Serverpod( + args, + Protocol(), + Endpoints(), + // Add the database interceptor. + databaseInterceptor: crdtDatabaseInterceptor, + ); + + // Initialize the sync engine before pod.start() is called. + pod.initializeCrdtSync( + syncTables: [Person.t, Book.t, Author.t], + ); + ``` + +3. Configure the sync engine on the client. + ```dart + final session = CrdtDatabaseSession.wraps( + await client.createSession(databasePath, isDebugMode: kDebugMode), + syncTables: [Person.t, Book.t, Author.t], + persistentUserId: persistentUserId, + ); + + // Initialize the sync engine. + // Then store the `session` instance in the service locator. + await session.db.initialize(); + ``` + +4. Wire the sync call on the client. + ```dart + // Push local changes and merge remote ones once (i.e. pull-to-refresh). + await client.crdt.syncOnce(session); + + // Or stream changes both ways until cancelled for a near-real-time sync. + final syncSession = client.crdt.syncContinuously(session); + ``` + +After this setup, the app can be developed using normal repository calls against the `session` instance. + +The sync engine is a **black box that "just works"**: changes are tracked atomically with each operation and the developer never intervenes in conflict resolution. Conflicts that need a human decision **bubble up as visible state** for the end user of the app to act on (e.g. a delete reverted by a concurrent constraint reappears, so the user can simply delete it again). Migrations work normally, and the same applies to deleted rows — which never leave their tables, but are transparently hidden from the user. + +Understanding the behavior is also straightforward: conflict resolution respects unique and relational constraints, mirroring what would be expected if the merged data existed in a single database at the time of each operation. + +Finally, the data is isolated per scope: each user has their own implicit personal scope, and can be a member of any number of shared scopes - with read-only or read-write access. On the server, database operations need to be wrapped in a special `session.db.transactionForUser` method to target the correct scope. On the client, it is possible to set a `persistentUserId` to target the personal scope by default, or use the `transactionForUser` to target a shared scope. + +#### Data modeling limitations + +Due to the nature of merge conflicts, the package imposes some limitations on the data modeling that are enforced at runtime during the initialization. Most of these limitations are fundamental to the design of the package and can never be lifted. + +- Every synced table must: + - Have a UUID primary key. + - Declare the field `scopeId: int?, relation(parent=crdt_scopes, onDelete=Cascade)`. +- Unique indexes must include `scopeId` together with the target columns. +- Global unique indexes are unsupported, except for FK-only indexes. +- Unique indexes are only supported with at least one `String`/`UuidValue`/nullable column. +- All 1:1 relations must have the foreign-key column nullable (`optional` relation). +- The only allowed non-synced-to-synced relation is `scopeId -> crdt_scopes.id`. +- All foreign-key relations must be `deferrable`* (see the technical debts section). + +Respecting these limitations, all other database invariants, including foreign-key actions, are preserved - with the exception of check constraints, which are also not supported by Serverpod. + +### The end-user experience + +The bundled desktop demo shows the model directly: two independent replicas of one account side by side, plus the server's merged truth. Drive each replica's sync independently and watch them converge under all common usage scenarios. It visualizes hidden / tombstoned rows and every foreign-key `onDelete` action, with guided scenarios for the restrict / cascade / set-null / set-default / unique-conflict cases. + +### Performance + +One common concern about offline-first solutions is the performance impact of the CRDT layer. Since all operations are tracked, it can weigh heavily on both storage and speed. This package was created with this concern in mind, achieving a minimal performance penalty. + +Below is one of the last performance reports of the package (available at each pull request as a comment). It uses 1,000 rows to measure the performance against a baseline of the same operations without the CRDT layer, both on SQLite: + +``` +📊 SELECT performance impact: + Time: 12.62 ms --> 12.67 ms (48.50 μs) + CRDT overhead: 0.38% slower +``` +``` +📊 INSERT performance impact: + Time: 251.01 ms --> 482.05 ms (231.04 ms) + CRDT overhead: 92.04% slower + Delay per insert: 231.04 μs +``` +``` +📊 UPDATE performance impact: + Time: 249.57 ms --> 2.13 s (1.88 s) + CRDT overhead: 753.54% slower + Delay per update: 1.88 ms +``` +``` +📊 DELETE performance impact: + Time: 11.53 ms --> 420.28 ms (408.75 ms) + CRDT overhead: 3,546.27% slower + Delay per delete: 408.75 μs +``` +``` +🔭 SELECT scope impact (100 extra users, 100,000 CRDT noise rows): + find all (1,000 rows): 9.91 ms --> 10.91 ms scoped (+10.07%) / 11.19 ms unscoped (+12.96%) + findById (per lookup): 295.04 μs --> 375.24 μs scoped (+27.19%) / 348.88 μs unscoped (+18.25%) +``` +``` +💽 Storage impact (base footprint removed): + Base database size: 540.00 KB --> 540.00 KB + INSERT only: + Net storage: 116.00 KB --> 188.00 KB (+72.00 KB) + CRDT overhead: 62.07% increase + Extra storage per inserted row: +73.73 B + INSERT + UPDATE: + Net storage: 128.00 KB --> 472.00 KB (+344.00 KB) + CRDT overhead: 268.75% increase + Extra storage per updated column: +34.82 B + INSERT + UPDATE + DELETE: + Net storage: 128.00 KB --> 512.00 KB (+384.00 KB) + CRDT overhead: 300.00% increase + Extra storage per deleted row: +40.96 B + Storage overhead range: 62.07% --> 300.00% +``` + +Despite the high percentage of slowdown in the insert/update/delete operations, the absolute impact is mostly in the microsecond range, making it barely noticeable in most cases - and way below a network delay if those operations were to be performed through an online endpoint. + +The storage impact is also minimal, reaching at most 3x the base storage size, which is perfectly acceptable for the benefits of offline-first speed and availability. + +A report similar to the one above is generated per pull request as a comment on the CI to track the performance evolution of the package. + +## Technical fundamentals + +**Architecture.** The engine is a transparent layer between the application and its database. Because identical code runs on the client's SQLite and the server's Postgres, there are no asymmetric merge results to reconcile. Migrations are also generated symmetrically for both ends, which makes the migration process result in the exact same data on both ends. The entire engine is built respecting the three mathematical properties that must hold for a CRDT to be consistent: idempotence, associativity, and commutativity. + +**Metadata model.** Three metadata tables track change history *without copying row data*: `crdt_data_rows` (insert + visibility), `crdt_data_fields` (update), and `crdt_data_tombstone` (delete/restore). HLCs are used to order field-level values and break ties. A monotone causal-length tombstone governs row existence, so add/delete/restore advance generations forward and can never oscillate. All metadata is normalized, with normalized IDs cached, to keep both the storage and performance impact minimal. + +**Visibility as a pure function.** Replicas never coordinate. Each merge applies remote facts monotonically, then derives the visible database — tombstone state, unique-conflict winners, and foreign-key repair — as a deterministic projection computed to a fixed point. This is the architecture introduced by *Synql* (Ignat et al., DAIS 2024), extended here to the full set of `ON DELETE` actions and to unique constraints over deeply relational schemas. + +**Ownership & scopes.** A *scope* is the unit of replication and ownership; a *user* is an authentication identity. Each row is globally identified by `(table, rowId)` and owned by exactly one scope (`scopeId`), so merging is always one chain per row. Sharing is a membership layer in front of scopes, not a change to row ownership. Scopes can be purged for GDPR / account erasure via a database-enforced cascade. Ownership collisions (the same `(table, rowId)` claimed by two scopes, which is mostly a malicious attack vector) fail the sync and are recorded in a durable `crdt_sync_integrity_violations` table. + +**Sync protocol.** A session handshakes once to validate both ends have the same table schema and ensure that a merge will reach the same result. Then, per cycle the server sends the authoritative scope set and all scopes that the user has access to are synced (data is transferred all at once, but merged one at a time). The data is transferred in chunks to avoid pressuring the bandwidth with too many small messages. This same stream-based implementation is used for both "one-shot" and "continuous" sync modes, so a one-shot sync with large amounts of data will be as performant as a continuous sync. It also means that all tests hold for both sync modes with minimal duplication. + +## Testing + +The package is tested extensively following the Serverpod testing principles. The vast majority of the tests live in the `serverpod_offline_sync_test_server` package, which is a Serverpod project that is used to test the package. Its server is also shared with the `example/offline_sync_demo` project, which is a real-world application that uses the package. + +## Roadmap + +The package is in a production-ready state, but it must naturally pass through a review process to be merged into the Serverpod monorepo. Before the review, some minor technical debts must be solved. + +### Technical debts + +- **Move the core of the implementation to a shared package instead of the client package**. Since the same code runs on the client and server, and requires table models, the first implementation was done in the client package. This means that users have to import a client package on their servers, crossing an isolation that is not common on Serverpod projects. To fix this, support for tables on shared models on Serverpod must be added, mapped on Serverpod's issue #5400. Then the core of the package must be moved to a shared package and all manually written SQL queries will be replaced by the Serverpod ORM. +- **Require that all foreign-key relations are declared deferrable**. Because the package does not store a copy of the data with the metadata, there are some scenarios in which the stored data might violate the causal order of the foreign-key constraints and a merge can fail irreversibly. To prevent this, all foreign-key constraints must be deferrable, which will be enabled by Serverpod's issue #5338. +- **Rename the `scope` name for data isolation to `space`**. This is a small refactor to avoid confusion with the `scope` keyword used in the Serverpod Auth modules. Using `space` is also clearer for usage on an application level. +- **Reorganize the package structure to facilitate the review**. The package is currently organized in a way that does not facilitate the review process. The `recorder.dart` file is extended through part files, sharing a too many private properties and methods. The resolver must be split into smaller, scoped files, as well as its unique and foreign-key resolver utils. + +### Post-integration UX improvements + +The package already requires minimal configuration from the developer, but it can be even smoother if the `serverpod_cli` is made aware of the offline sync capabilities. + +- **Add a fourth `sync` option for the `database` keyword to be used on models**. Models that use it will automatically receive the `scopeId` relation and be included in a generated list of synced tables on the server and client. This feature will enable all improvements below. +- **Simplify the `Serverpod` initialization by generating the class**. This is mapped on Serverpod's issue #4544 and will allow the CRDT database interceptor to be injected on the `Serverpod` constructor without the user having to call it manually whenever a `sync` model exists. +- **Move the restrictions that are runtime-enforced to a static analysis step**. The package enforces the limitations of the CRDT layer at runtime (during the initialization), but a better UX would be to apply the restrictions to `sync` models during static analysis of `generate`. +- **Inject the `serverpod_offline_sync` packages on the `pubspec.yaml` file**. This would remove the last manual step from the developer to make the solution work. Otherwise, the user will have to manually add the dependencies to their `pubspec.yaml` to generate the project. + +If these changes are made, all a user will have to do is to add the `sync` keyword to the `database` keyword on synced table models and the package will be completely wired up. + +Given that all these changes are on the `serverpod_cli`, it doesn't even require other internal packages to depend on the `serverpod_offline_sync` packages. + +### Review strategy + +Besides not being that big (~35k LOC for the packages, ~14k LOC for the tests), the codebase is dense on the logic. So the best way to review it is to read the performance reports first, then all tests, and finally the main files of the packages as needed. + +The design docs can also be used as a reference to understand the architecture and the implementation, as they were kept in sync throughout the development process. + +### Transference process + +The license transfer and transference process will be done following the steps below: + +1. A branch transferring the copyrights to the Serverpod organization will be merged into the main branch of the original repository. +2. The original repository with the updated license will be made public on GitHub by the author. This copy will be kept for historical purposes and as a portfolio of the author's work. +3. The package will be reviewed by the Serverpod team on the original repository to prepare the codebase for the transference. +4. After all changes originated from the review are merged into the main branch, a PR will be created to merge the packages on the Serverpod monorepo, preserving its commit history. +5. With the PR merged on Serverpod's monorepo, the original repository will be archived with a note about the new location of the package. diff --git a/example/offline_sync_demo/pubspec.yaml b/example/offline_sync_demo/pubspec.yaml index 804758e..c794a04 100644 --- a/example/offline_sync_demo/pubspec.yaml +++ b/example/offline_sync_demo/pubspec.yaml @@ -16,7 +16,7 @@ dependencies: http: ^1.5.0 path: ^1.9.1 path_provider: ^2.1.5 - serverpod_database: 3.5.0-beta.10 + serverpod_database: 4.0.0-beta.0 serverpod_offline_sync_client: path: ../../packages/serverpod_offline_sync_client serverpod_offline_sync_test_client: diff --git a/legacy/drift_offline_sync/pubspec.yaml b/legacy/drift_offline_sync/pubspec.yaml index 49fc3bb..4fbf880 100644 --- a/legacy/drift_offline_sync/pubspec.yaml +++ b/legacy/drift_offline_sync/pubspec.yaml @@ -1,6 +1,6 @@ name: drift_offline_sync description: A Drift plugin to build offline-first applications with CRDT support. -version: 0.0.1 +version: 4.0.0-beta.0 repository: https://github.com/marcelomendoncasoares/drift_offline_sync environment: diff --git a/packages/serverpod_offline_sync_client/LICENSE b/packages/serverpod_offline_sync_client/LICENSE new file mode 100644 index 0000000..b183ba3 --- /dev/null +++ b/packages/serverpod_offline_sync_client/LICENSE @@ -0,0 +1,27 @@ +Copyright 2025 The Serverpod authors + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +4. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. diff --git a/packages/serverpod_offline_sync_client/pubspec.yaml b/packages/serverpod_offline_sync_client/pubspec.yaml index cb2b336..310ebc2 100644 --- a/packages/serverpod_offline_sync_client/pubspec.yaml +++ b/packages/serverpod_offline_sync_client/pubspec.yaml @@ -1,6 +1,6 @@ name: serverpod_offline_sync_client +version: 4.0.0-beta.0 description: Client package for the Serverpod Offline Sync project. -version: 0.0.1 homepage: https://github.com/marcelomendoncasoares/serverpod_offline_sync environment: @@ -10,10 +10,10 @@ resolution: workspace dependencies: meta: ^1.18.0 - serverpod_client: 3.5.0-beta.10 - serverpod_database: 3.5.0-beta.10 - serverpod_offline_sync_shared: 0.0.1 - serverpod_serialization: 3.5.0-beta.10 + serverpod_client: 4.0.0-beta.0 + serverpod_database: 4.0.0-beta.0 + serverpod_offline_sync_shared: 4.0.0-beta.0 + serverpod_serialization: 4.0.0-beta.0 uuid: ^4.5.3 dev_dependencies: diff --git a/packages/serverpod_offline_sync_server/LICENSE b/packages/serverpod_offline_sync_server/LICENSE new file mode 100644 index 0000000..b183ba3 --- /dev/null +++ b/packages/serverpod_offline_sync_server/LICENSE @@ -0,0 +1,27 @@ +Copyright 2025 The Serverpod authors + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +4. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. diff --git a/packages/serverpod_offline_sync_server/pubspec.yaml b/packages/serverpod_offline_sync_server/pubspec.yaml index 0589423..046c937 100644 --- a/packages/serverpod_offline_sync_server/pubspec.yaml +++ b/packages/serverpod_offline_sync_server/pubspec.yaml @@ -1,6 +1,6 @@ name: serverpod_offline_sync_server +version: 4.0.0-beta.0 description: Server module for the Serverpod Offline Sync project. -version: 0.0.1 homepage: https://github.com/marcelomendoncasoares/serverpod_offline_sync environment: @@ -10,12 +10,12 @@ resolution: workspace dependencies: meta: ^1.16.0 - serverpod: 3.5.0-beta.10 - serverpod_database: 3.5.0-beta.10 - serverpod_offline_sync_client: 0.0.1 - serverpod_offline_sync_shared: 0.0.1 + serverpod: 4.0.0-beta.0 + serverpod_database: 4.0.0-beta.0 + serverpod_offline_sync_client: 4.0.0-beta.0 + serverpod_offline_sync_shared: 4.0.0-beta.0 dev_dependencies: lints: '>=3.0.0 <7.0.0' - serverpod_test: 3.5.0-beta.10 + serverpod_test: 4.0.0-beta.0 test: ^1.25.5 diff --git a/packages/serverpod_offline_sync_shared/LICENSE b/packages/serverpod_offline_sync_shared/LICENSE new file mode 100644 index 0000000..b183ba3 --- /dev/null +++ b/packages/serverpod_offline_sync_shared/LICENSE @@ -0,0 +1,27 @@ +Copyright 2025 The Serverpod authors + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +4. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. diff --git a/packages/serverpod_offline_sync_shared/pubspec.yaml b/packages/serverpod_offline_sync_shared/pubspec.yaml index 1908ce5..f9e30cb 100644 --- a/packages/serverpod_offline_sync_shared/pubspec.yaml +++ b/packages/serverpod_offline_sync_shared/pubspec.yaml @@ -1,6 +1,6 @@ name: serverpod_offline_sync_shared +version: 4.0.0-beta.0 description: Shared package for the Serverpod Offline Sync project. -version: 0.0.1 homepage: https://github.com/marcelomendoncasoares/serverpod_offline_sync environment: @@ -10,8 +10,8 @@ resolution: workspace dependencies: clock: ^1.1.2 - serverpod_database: 3.5.0-beta.10 - serverpod_serialization: 3.5.0-beta.10 + serverpod_database: 4.0.0-beta.0 + serverpod_serialization: 4.0.0-beta.0 uuid: ^4.5.3 dev_dependencies: diff --git a/pubspec.yaml b/pubspec.yaml index 8ee516d..b085c1d 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -14,7 +14,7 @@ workspace: dev_dependencies: melos: ^7.5.1 - serverpod_cli: 3.5.0-beta.10 + serverpod_cli: 4.0.0-beta.0 dependency_overrides: serverpod: diff --git a/test/serverpod_offline_sync_test_client/pubspec.yaml b/test/serverpod_offline_sync_test_client/pubspec.yaml index b68b528..a6605f3 100644 --- a/test/serverpod_offline_sync_test_client/pubspec.yaml +++ b/test/serverpod_offline_sync_test_client/pubspec.yaml @@ -8,8 +8,8 @@ environment: resolution: workspace dependencies: - serverpod_auth_core_client: 3.5.0-beta.10 - serverpod_client: 3.5.0-beta.10 - serverpod_database: 3.5.0-beta.10 + serverpod_auth_core_client: 4.0.0-beta.0 + serverpod_client: 4.0.0-beta.0 + serverpod_database: 4.0.0-beta.0 serverpod_offline_sync_client: path: ../../packages/serverpod_offline_sync_client diff --git a/test/serverpod_offline_sync_test_server/pubspec.yaml b/test/serverpod_offline_sync_test_server/pubspec.yaml index cf2feae..c71fc29 100644 --- a/test/serverpod_offline_sync_test_server/pubspec.yaml +++ b/test/serverpod_offline_sync_test_server/pubspec.yaml @@ -8,8 +8,8 @@ environment: resolution: workspace dependencies: - serverpod: 3.5.0-beta.10 - serverpod_auth_core_server: 3.5.0-beta.10 + serverpod: 4.0.0-beta.0 + serverpod_auth_core_server: 4.0.0-beta.0 serverpod_offline_sync_server: path: ../../packages/serverpod_offline_sync_server serverpod_offline_sync_shared: @@ -18,12 +18,12 @@ dependencies: dev_dependencies: lints: '>=3.0.0 <7.0.0' path: ^1.9.1 - serverpod_database: 3.5.0-beta.10 + serverpod_database: 4.0.0-beta.0 serverpod_offline_sync_client: path: ../../packages/serverpod_offline_sync_client serverpod_offline_sync_test_client: path: ../serverpod_offline_sync_test_client - serverpod_test: 3.5.0-beta.10 + serverpod_test: 4.0.0-beta.0 test: ^1.25.5 serverpod: