sync-service: Simplify configuration of the electric app (#2173)

Refactor app config to make most values optional. Except for connection_opts, all the config settings have defaults. When embedding electric inside another elixir app, our runtime.exs won't be sourced so the developer would be forced to unnecessarily set all these configuration values. Moves the default values from runtime.exs to the point of usage so we don't have defaults in two places.
electric-sql · Dec 17, 2024 · acb46e1 · acb46e1
1 parent d7e7c72
commit acb46e1
Show file tree

Hide file tree

Showing 10 changed files with 342 additions and 79 deletions.
diff --git a/packages/sync-service/README.md b/packages/sync-service/README.md
@@ -15,7 +15,7 @@ Electric Sync is powered by an Elixir-based application that connects to your Po
 
 For a quick setup and examples, refer to the [Quickstart guide](https://electric-sql.com/docs/quickstart).
 
-## Running
+## Running as a Standalone HTTP Endpoint
 
 Run Postgres:
 
@@ -36,3 +36,48 @@ Run the Elixir app:
 mix deps.get
 iex -S mix
 ```
+
+## Embedding into another Elixir Application
+
+Include `:electric` into your dependencies:
+
+    # mix.exs
+    defp deps do
+      [
+      {:electric, ">= 1.0.0-beta-1"}
+      ]
+    end
+
+Add the Postgres db connection configuration to your application's config.
+Electric accepts the same configuration format as
+[Ecto](https://hexdocs.pm/ecto/Ecto.html) (and
+[Postgrex](https://hexdocs.pm/postgrex/Postgrex.html#start_link/1)) so you can
+reuse that configuration if you want:
+
+    # config/*.exs
+    database_config = [
+      database: "ecto_simple",
+      username: "postgres",
+      password: "postgres",
+      hostname: "localhost"
+    ]
+    config :my_app, Repo, database_config
+
+    config :electric,
+      connection_opts: Electric.Utils.obfuscate_password(database_config)
+
+Or if you're getting your db connection from an environment variable, then you
+can use
+[`Electric.Config.parse_postgresql_uri!/1`](https://hexdocs.pm/electric/Electric.Config.html#parse_postgresql_uri!/1):
+
+    # config/*.exs
+    {:ok, database_config} = Electric.Config.parse_postgresql_uri(System.fetch_env!("DATABASE_URL"))
+
+    config :electric,
+      connection_opts: Electric.Utils.obfuscate_password(database_config)
+
+The Electric app will startup along with the rest of your Elixir app.
+
+Beyond the required database connection configuration there are a lot of other
+optional configuration parameters. See the [`Electric` docs for more
+information](https://hexdocs.pm/electric/Electric.html).
diff --git a/packages/sync-service/config/runtime.exs b/packages/sync-service/config/runtime.exs
@@ -1,15 +1,13 @@
 import Config
 import Dotenvy
 
-alias Electric.ConfigParser
-
 config :elixir, :time_zone_database, Tz.TimeZoneDatabase
 
 if config_env() in [:dev, :test] do
   source!([".env.#{config_env()}", ".env.#{config_env()}.local", System.get_env()])
 end
 
-config :logger, level: env!("ELECTRIC_LOG_LEVEL", &ConfigParser.parse_log_level!/1, :info)
+config :logger, level: env!("ELECTRIC_LOG_LEVEL", &Electric.Config.parse_log_level!/1, :info)
 
 config :logger, :default_formatter,
   # Doubled line breaks serve as long message boundaries
@@ -106,7 +104,7 @@ config :opentelemetry,
        local_parent_not_sampled: :always_off
      }}
 
-database_url_config = env!("DATABASE_URL", &ConfigParser.parse_postgresql_uri!/1)
+database_url_config = env!("DATABASE_URL", &Electric.Config.parse_postgresql_uri!/1)
 
 database_ipv6_config =
   env!("ELECTRIC_DATABASE_USE_IPV6", :boolean, false)
@@ -115,9 +113,9 @@ connection_opts = database_url_config ++ [ipv6: database_ipv6_config]
 
 config :electric, connection_opts: Electric.Utils.obfuscate_password(connection_opts)
 
-enable_integration_testing = env!("ELECTRIC_ENABLE_INTEGRATION_TESTING", :boolean, false)
-cache_max_age = env!("ELECTRIC_CACHE_MAX_AGE", :integer, 60)
-cache_stale_age = env!("ELECTRIC_CACHE_STALE_AGE", :integer, 60 * 5)
+enable_integration_testing? = env!("ELECTRIC_ENABLE_INTEGRATION_TESTING", :boolean, nil)
+cache_max_age = env!("ELECTRIC_CACHE_MAX_AGE", :integer, nil)
+cache_stale_age = env!("ELECTRIC_CACHE_STALE_AGE", :integer, nil)
 statsd_host = env!("ELECTRIC_STATSD_HOST", :string?, nil)
 
 storage_dir = env!("ELECTRIC_STORAGE_DIR", :string, "./persistent")
@@ -140,17 +138,12 @@ persistent_kv =
           raise Dotenvy.Error, message: "ELECTRIC_PERSISTENT_STATE must be one of: MEMORY, FILE"
       end
     end,
-    {Electric.PersistentKV.Filesystem, :new!, root: persistent_state_path}
+    nil
   )
 
-chunk_bytes_threshold =
-  env!(
-    "ELECTRIC_SHAPE_CHUNK_BYTES_THRESHOLD",
-    :integer,
-    Electric.ShapeCache.LogChunker.default_chunk_size_threshold()
-  )
+chunk_bytes_threshold = env!("ELECTRIC_SHAPE_CHUNK_BYTES_THRESHOLD", :integer, nil)
 
-{storage_mod, storage_opts} =
+storage =
   env!(
     "ELECTRIC_STORAGE",
     fn storage ->
@@ -172,7 +165,7 @@ chunk_bytes_threshold =
           raise Dotenvy.Error, message: "storage must be one of: MEMORY, FILE"
       end
     end,
-    {Electric.ShapeCache.FileStorage, storage_dir: shape_path}
+    nil
   )
 
 replication_stream_id =
@@ -185,34 +178,32 @@ replication_stream_id =
 
       parsed_id
     end,
-    "default"
+    nil
   )
 
-storage = {storage_mod, storage_opts}
-
 prometheus_port = env!("ELECTRIC_PROMETHEUS_PORT", :integer, nil)
 
 call_home_telemetry_url =
   env!(
     "ELECTRIC_TELEMETRY_URL",
-    &ConfigParser.parse_telemetry_url!/1,
-    "https://checkpoint.electric-sql.com"
+    &Electric.Config.parse_telemetry_url!/1,
+    nil
   )
 
 system_metrics_poll_interval =
   env!(
     "ELECTRIC_SYSTEM_METRICS_POLL_INTERVAL",
-    &ConfigParser.parse_human_readable_time!/1,
-    :timer.seconds(5)
+    &Electric.Config.parse_human_readable_time!/1,
+    nil
   )
 
 # The provided database id is relevant if you had used v0.8 and want to keep the storage
 # instead of having hanging files. We use a provided value as stack id, but nothing else.
-provided_database_id = env!("ELECTRIC_DATABASE_ID", :string, "single_stack")
+provided_database_id = env!("ELECTRIC_DATABASE_ID", :string, nil)
 
 config :electric,
   provided_database_id: provided_database_id,
-  allow_shape_deletion: enable_integration_testing,
+  allow_shape_deletion?: enable_integration_testing?,
   cache_max_age: cache_max_age,
   cache_stale_age: cache_stale_age,
   chunk_bytes_threshold: chunk_bytes_threshold,
@@ -223,10 +214,10 @@ config :electric,
   system_metrics_poll_interval: system_metrics_poll_interval,
   telemetry_statsd_host: statsd_host,
   prometheus_port: prometheus_port,
-  db_pool_size: env!("ELECTRIC_DB_POOL_SIZE", :integer, 20),
+  db_pool_size: env!("ELECTRIC_DB_POOL_SIZE", :integer, nil),
   replication_stream_id: replication_stream_id,
-  replication_slot_temporary?: env!("CLEANUP_REPLICATION_SLOTS_ON_SHUTDOWN", :boolean, false),
-  service_port: env!("ELECTRIC_PORT", :integer, 3000),
+  replication_slot_temporary?: env!("CLEANUP_REPLICATION_SLOTS_ON_SHUTDOWN", :boolean, nil),
+  service_port: env!("ELECTRIC_PORT", :integer, nil),
   storage: storage,
   persistent_kv: persistent_kv,
-  listen_on_ipv6?: env!("ELECTRIC_LISTEN_ON_IPV6", :boolean, false)
+  listen_on_ipv6?: env!("ELECTRIC_LISTEN_ON_IPV6", :boolean, nil)
diff --git a/packages/sync-service/lib/electric.ex b/packages/sync-service/lib/electric.ex
@@ -1,12 +1,129 @@
 defmodule Electric do
-  @doc """
-  Every electric cluster belongs to a particular console database instance
+  @connection_opts [
+    hostname: [type: :string, required: true, doc: "Server hostname"],
+    port: [type: :integer, required: true, doc: "Server port"],
+    database: [type: :string, required: true, doc: "Database"],
+    username: [type: :string, required: true, doc: "Username"],
+    password: [
+      type: {:fun, 0},
+      required: true,
+      doc:
+        "User password. To prevent leaking of the Pg password in logs and stack traces, you **must** wrap the password with a function." <>
+          " We provide `Electric.Utils.obfuscate_password/1` which will return the `connection_opts` with a wrapped password value.\n\n" <>
+          "    config :electric, connection_opts: Electric.Utils.obfuscate_password(connection_opts)"
+    ],
+    sslmode: [
+      type: {:in, [:disable, :allow, :prefer, :require]},
+      required: false,
+      default: :prefer,
+      doc:
+        "Connection SSL configuration. See https://www.postgresql.org/docs/current/libpq-ssl.html#LIBPQ-SSL-SSLMODE-STATEMENTS",
+      type_spec: quote(do: :disable | :allow | :prefer | :require)
+    ],
+    ipv6: [
+      type: :boolean,
+      required: false,
+      default: false,
+      doc: "Whether to use IPv6 for database connections"
+    ]
+  ]
+  opts_schema = NimbleOptions.new!(@connection_opts)
+
+  @type pg_connection_opts :: [unquote(NimbleOptions.option_typespec(opts_schema))]
+
+  default = fn key -> inspect(Electric.Config.default(key)) end
+
+  @moduledoc """
+
+  ## Configuration options
+
+  When embedding Electric, the following options are available:
+
+      config :electric,
+        connection_opts: nil
+        # Database
+        provided_database_id: #{default.(:provided_database_id)},
+        db_pool_size: #{default.(:db_pool_size)},
+        replication_stream_id: #{default.(:replication_stream_id)},
+        replication_slot_temporary?: #{default.(:replication_slot_temporary?)},
+        # HTTP API
+        service_port: #{default.(:service_port)},
+        allow_shape_deletion?: #{default.(:allow_shape_deletion?)},
+        cache_max_age: #{default.(:cache_max_age)},
+        cache_stale_age: #{default.(:cache_stale_age)},
+        chunk_bytes_threshold: #{default.(:chunk_bytes_threshold)},
+        listen_on_ipv6?: #{default.(:listen_on_ipv6?)},
+        # Storage
+        storage_dir: #{default.(:storage_dir)},
+        storage: #{default.(:storage)},
+        persistent_kv: #{default.(:persistent_kv)},
+        # Telemetry
+        instance_id: #{default.(:instance_id)},
+        telemetry_statsd_host: #{default.(:telemetry_statsd_host)},
+        prometheus_port: #{default.(:prometheus_port)},
+        call_home_telemetry?: #{default.(:call_home_telemetry?)},
+        telemetry_url: #{default.(:telemetry_url)},
+
+  Only the `connection_opts` are required.
+
+  ### Database
+
+  - `connection_opts` - **Required**
+     #{NimbleOptions.docs(opts_schema, nest_level: 1)}.
+  - `db_pool_size` - How many connections Electric opens as a pool for handling shape queries (default: `#{default.(:db_pool_size)}`)
+  - `replication_stream_id` - Suffix for the logical replication publication and slot name (default: `#{default.(:replication_stream_id)}`)
+
+  ### HTTP API
+
+  - `service_port` (`t:integer/0`) - Port that the [HTTP API](https://electric-sql.com/docs/api/http) is exposed on (default: `#{default.(:service_port)}`)
+  - `allow_shape_deletion?` (`t:boolean/0`) - Whether to allow deletion of Shapes via the HTTP API (default: `#{default.(:allow_shape_deletion?)}`)
+  - `cache_max_age` (`t:integer/0`) - Default `max-age` for the cache headers of the HTTP API in seconds (default: `#{default.(:cache_max_age)}`s)
+  - `cache_stale_age` (`t:integer/0`) - Default `stale-age` for the cache headers of the HTTP API in seconds (default: `#{default.(:cache_stale_age)}`s)
+  - `chunk_bytes_threshold` (`t:integer/0`) - Limit the maximum size in bytes of a shape log response,
+    to ensure they are cached by upstream caches. (default: `#{default.(:chunk_bytes_threshold)}` (10MiB)).
+  - `listen_on_ipv6?` (`t:boolean/0`) - Whether the HTTP API should listen on IPv6 as well as IPv4 (default: `#{default.(:listen_on_ipv6?)}`)
 
-  This is that database instance id
+  ### Storage
+
+  - `storage_dir` (`t:String.t/0`) - Path to root folder for storing data on the filesystem (default: `#{default.(:storage_dir)}`)
+  - `storage` (`t:Electric.ShapeCache.Storage.storage/0`) - Where to store shape logs. Must be a 2-tuple of `{module(),
+    term()}` where `module` points to an implementation of the
+    `Electric.ShapeCache.Storage` behaviour. (default: `#{default.(:storage)}`)
+  - `persistent_kv` (`t:Electric.PersistentKV.t/0`) - A mfa that when called constructs an implementation of
+    the `Electric.PersistentKV` behaviour, used to store system state (default: `#{default.(:persistent_kv)}`)
+
+  ### Telemetry
+
+  - `instance_id` (`t:binary/0`) - A unique identifier for the Electric instance. Set this to
+    enable tracking of instance usage metrics across restarts, otherwise will be
+    randomly generated at boot (default: a randomly generated UUID).
+  - `telemetry_statsd_host` (`t:String.t/0`) - If set, send telemetry data to the given StatsD reporting endpoint (default: `#{default.(:telemetry_statsd_host)}`)
+  - `prometheus_port` (`t:integer/0`) - If set, expose a prometheus reporter for telemetry data on the specified port (default: `#{default.(:prometheus_port)}`)
+  - `call_home_telemetry?` (`t:boolean/0`) - Allow [anonymous usage
+    data](https://electric-sql.com/docs/reference/telemetry#anonymous-usage-data)
+    about the instance being sent to a central checkpoint service (default: `true` for production)
+  - `telemetry_url` (`t:URI.t/0`) - Where to send the usage data (default: `#{default.(:telemetry_url)}`)
+
+  ### Deprecated
+
+  - `provided_database_id` (`t:binary/0`) - The provided database id is relevant if you had
+    used v0.8 and want to keep the storage instead of having hanging files. We
+    use a provided value as stack id, but nothing else.
+  """
+
+  require Logger
+
+  @doc false
+  def connection_opts_schema do
+    @connection_opts
+  end
+
+  @doc """
+  `instance_id` is used to track a particular server's telemetry metrics.
   """
   @spec instance_id() :: binary | no_return
   def instance_id do
-    Application.fetch_env!(:electric, :instance_id)
+    Electric.Config.fetch_env!(:instance_id)
   end
 
   @type relation :: {schema :: String.t(), table :: String.t()}