Custom checkpoints (#36)

README.md

@@ -14,6 +14,10 @@ This repository contains basic demonstrations in the `demos` folder.
 
   - This can be started from the repo root with `docker compose -f demos/nodejs/docker-compose.yaml up`
 
+- [Node.js (Postgres + Custom Write Checkpoints)](./demos/nodejs-custom-checkpoints/README.md)
+
+  - This can be started from the repo root with `docker compose -f demos/nodejs-custom-checkpoints/docker-compose.yaml up`
+
 - [Node.js (MongoDB)](./demos/nodejs-mongodb/README.md)
 
   - This can be started from the repo root with `docker compose -f demos/nodejs-mongodb/docker-compose.yaml up`

config/powersync.yaml

@@ -98,3 +98,8 @@ client_auth:
 
   # JWKS audience
   audience: ["powersync-dev", "powersync"]
+
+api:
+  tokens:
+    # These tokens are used for local admin API route authentication
+    - use_a_better_token_in_production

config/sync_rules.yaml

@@ -1,7 +1,11 @@
+# yaml-language-server: $schema=https://unpkg.com/@powersync/service-sync-rules@0.0.0-dev-20241022111143/schema/sync_rules.json
+#
 # See Documentation for more information:
 # https://docs.powersync.com/usage/sync-rules
+#
 # Note that changes to this file are not watched.
 # The service needs to be restarted for changes to take effect.
+
 bucket_definitions:
   global:
     data:

demos/nodejs-custom-checkpoints/.env

@@ -0,0 +1,23 @@
+# ==================== Postgres credentials ================================
+PG_DATABASE_NAME=postgres
+PG_DATABASE_PORT=5432
+PG_DATABASE_USER=postgres
+PG_DATABASE_PASSWORD=mypassword
+PS_DATA_SOURCE_URI=postgres://${PG_DATABASE_USER}:${PG_DATABASE_PASSWORD}@pg-db:${PG_DATABASE_PORT}/${PG_DATABASE_NAME}
+
+# ==================== Demo config =========================================
+DEMO_BACKEND_PORT=6060
+DEMO_BACKEND_DATABASE_TYPE=postgres
+DEMO_BACKEND_DATABASE_URI=${PS_DATA_SOURCE_URI}
+# The front-end demo application is accessible at this port on the host machine
+DEMO_CLIENT_PORT=3034
+PS_JWKS_URL=http://demo-backend:${DEMO_BACKEND_PORT}/api/auth/keys
+
+# These can be generated by following the instructions in the `key-generator` folder
+# A temporary key will be used if these are not specified
+DEMO_JWKS_PUBLIC_KEY=
+DEMO_JWKS_PRIVATE_KEY=
+
+# ==================== PowerSync variables ====================
+# The PowerSync API is accessible via this port
+PS_PORT=8080

demos/nodejs-custom-checkpoints/README.md

@@ -0,0 +1,29 @@
+# JavaScript PowerSync + Custom Write Checkpoints
+
+This is a demo for using custom Write Checkpoints with PowerSync. Custom Write Checkpoints eliminate flicker when uploads do not directly reflect in the backend source database. This feature is only available for customers on [Team and Enterprise](https://www.powersync.com/pricing) plans.
+
+Docs are available [here](https://docs.powersync.com/usage/use-case-examples/custom-write-checkpoints).
+
+## Running
+
+The `.env` file contains default configuration for the services. Reference this to connect to any services locally.
+
+Ensure you have authenticated with our Docker Image repository. Please reach out to support for an access token.
+
+```bash
+docker login container-registry@journeyapps.com -u user
+```
+
+This demo can be started by running the following in this demo directory
+
+```bash
+docker compose up
+```
+
+or in the root directory run
+
+```bash
+docker compose -f demos/nodejs-custom-checkpoints/docker-compose.yaml up
+```
+
+The frontend can be accessed at `http://localhost:3034` in a browser.

demos/nodejs-custom-checkpoints/config/powersync.yaml

@@ -0,0 +1,35 @@
+# yaml-language-server: $schema=../../../schema/schema.json
+telemetry:
+  # Opt out of reporting anonymized usage metrics to PowerSync telemetry service
+  disable_telemetry_sharing: false
+
+# Settings for source database replication
+replication:
+  connections:
+    - type: postgresql
+      # The PowerSync server container can access the Postgres DB via the DB's service name.
+      # In this case the hostname is pg-db
+
+      # The connection URI or individual parameters can be specified.
+      # Individual params take precedence over URI params
+      uri: !env PS_DATA_SOURCE_URI
+      sslmode: disable # 'verify-full' (default) or 'verify-ca' or 'disable'
+
+# Connection settings for sync bucket storage
+storage:
+  type: mongodb
+  uri: !env PS_MONGO_URI
+
+# The port which the PowerSync API server will listen on
+port: !env PS_PORT
+
+# Specify sync rules
+sync_rules:
+  path: sync_rules.yaml
+
+# Client (application end user) authentication settings
+client_auth:
+  # JWKS URIs can be specified here
+  jwks_uri: !env PS_JWKS_URL
+
+  audience: ["powersync-dev", "powersync"]

demos/nodejs-custom-checkpoints/config/sync_rules.yaml

@@ -0,0 +1,20 @@
+# yaml-language-server: $schema=https://unpkg.com/@powersync/service-sync-rules@0.0.0-dev-20241022111143/schema/sync_rules.json
+#
+# See Documentation for more information:
+# https://docs.powersync.com/usage/sync-rules
+#
+# Note that changes to this file are not watched.
+# The service needs to be restarted for changes to take effect.
+
+event_definitions:
+  # Note this event is only supported for customers on [Team and Enterprise](https://www.powersync.com/pricing) plans.
+  write_checkpoints:
+    payloads:
+      #  This defines where the replicated custom Write Checkpoints should be extracted from
+      - SELECT user_id, checkpoint, client_id FROM checkpoints
+
+bucket_definitions:
+  global:
+    data:
+      - select * from lists
+      - select * from todos

demos/nodejs-custom-checkpoints/docker-compose.yaml

@@ -0,0 +1,43 @@
+# Include syntax requires Docker compose > 2.20.3
+# https://docs.docker.com/compose/release-notes/#2203
+include:
+  # Creates a MongoDB replica set. This is used for internal and data storage
+  - path: ../../services/mongo.yaml
+
+services:
+  # Extend PowerSync with Mongo and Postgres healthchecks
+  powersync:
+    extends:
+      file: ../../services/powersync.yaml
+      service: powersync
+    depends_on:
+      mongo-rs-init:
+        condition: service_completed_successfully
+    # Custom Write Checkpoints are only available for customers on [Team and Enterprise](https://www.powersync.com/pricing) plans.
+    image: container-registry.journeyapps.com/powersync-service:next
+    volumes:
+      - ./config:/config
+
+  pg-db:
+    extends:
+      file: ../../services/postgres.yaml
+      service: pg-db
+    volumes:
+      - ./init-scripts:/docker-entrypoint-initdb.d
+
+  demo-client:
+    extends:
+      file: ../nodejs/ps-nodejs-demo.yaml
+      service: demo-client
+    build:
+      args:
+        VITE_CHECKPOINT_MODE: custom
+
+  demo-backend:
+    extends:
+      file: ../nodejs/ps-nodejs-demo.yaml
+      service: demo-backend
+
+volumes:
+  # Postgres data
+  pg_data:

demos/nodejs-custom-checkpoints/init-scripts/setup.sql

@@ -0,0 +1,40 @@
+-- TODO change this if changing the DB connection name
+\connect postgres;
+
+-- Create tables
+create table public.lists (
+    id uuid not null default gen_random_uuid (),
+    created_at timestamp with time zone not null default now(),
+    name text not null,
+    owner_id uuid not null,
+    constraint lists_pkey primary key (id)
+  );
+
+create table public.todos (
+    id uuid not null default gen_random_uuid (),
+    created_at timestamp with time zone not null default now(),
+    completed_at timestamp with time zone null,
+    description text not null,
+    completed boolean not null default false,
+    created_by uuid null,
+    completed_by uuid null,
+    list_id uuid not null,
+    photo_id uuid null,
+    constraint todos_pkey primary key (id)
+  );
+
+CREATE TABLE checkpoints (
+    user_id VARCHAR(255),
+    client_id VARCHAR(255),
+    checkpoint INTEGER,
+    PRIMARY KEY (user_id, client_id)
+);
+
+-- Creates some initial data to be synced
+INSERT INTO lists (id, name, owner_id) VALUES ('75f89104-d95a-4f16-8309-5363f1bb377a', 'Getting Started', gen_random_uuid()  );
+INSERT INTO todos(description, list_id, completed) VALUES ('Run services locally', '75f89104-d95a-4f16-8309-5363f1bb377a', true);
+INSERT INTO todos (description, list_id, completed) VALUES ('Create a todo here. Query the todos table via a Postgres connection. Your todo should be synced', '75f89104-d95a-4f16-8309-5363f1bb377a', false);
+
+-- Create publication for PowerSync
+-- Note that the `checkpoints` table is required in the publication
+create publication powersync for table lists, todos, checkpoints;

demos/nodejs-mongodb/config/powersync.yaml

@@ -1,4 +1,4 @@
-# yaml-language-server: $schema=../schema/schema.json
+# yaml-language-server: $schema=../../../schema/schema.json
 telemetry:
   # Opt out of reporting anonymized usage metrics to PowerSync telemetry service
   disable_telemetry_sharing: false
@@ -27,3 +27,8 @@ client_auth:
   jwks_uri: !env PS_JWKS_URL
 
   audience: ["powersync-dev", "powersync"]
+
+api:
+  tokens:
+    # These tokens are used for local admin API route authentication
+    - use_a_better_token_in_production

demos/nodejs-mongodb/config/sync_rules.yaml

@@ -1,7 +1,11 @@
+# yaml-language-server: $schema=https://unpkg.com/@powersync/service-sync-rules@0.0.0-dev-20241022111143/schema/sync_rules.json
+#
 # See Documentation for more information:
 # https://docs.powersync.com/usage/sync-rules
+#
 # Note that changes to this file are not watched.
 # The service needs to be restarted for changes to take effect.
+
 bucket_definitions:
   global:
     data:

demos/nodejs/demo-app/.env.template

@@ -2,3 +2,4 @@
 # Edit .env.local and enter your project details.
 VITE_BACKEND_URL=http://localhost:6060
 VITE_POWERSYNC_URL=http://localhost:8080
+VITE_CHECKPOINT_MODE=managed

demos/nodejs/demo-app/Dockerfile

@@ -3,6 +3,7 @@ FROM node:20
 
 ARG VITE_POWERSYNC_URL
 ARG VITE_BACKEND_URL
+ARG VITE_CHECKPOINT_MODE
 
 
 # Set the working directory inside the container