[HYD-178] Add multicorn support (#13)

* Get mysql build to run

* Enable mysql_fdw by default

* Replace Docker container with Docker Compose in acceptance tests

This slides in mysql container that is accessible from hydra

* Move http ext tests to main acceptance test cases

* Always run docker compose down to clean up networks

* Rename to `profileName`

* Add `mysql_fdw` tests to verify direct connection

* Only start all containers for acceptance tests

* Remove test docker-compose.yaml

* Remove unused import

* Bump `github.com/jackc/pgx` to consume bug fixes

* Validate mysql_fdw only return one row

* Remove vendor

* Use text/template

* Move `docker compose down` after kill

* Expand flags

* Fix bug that use `--profile` instead of `--project-name`

* Make `http` & `mysql_fdw` ext behavior consistent with `postgres_fdw`

* Disable preload
* Extensions are not loaded by default (require explicit `CREATE EXTENSION` statements)

* Multicorn build

* Get spilo build to work with multicorn

* Enable multicorn ext for spilo

* Add multicorn create foreign table acceptance test

* Do not enable `multicorn` ext by default

* Tag multicorn2 to v2.4

* Pin s3csv_fdw to the latest commit

Dockerfile

@@ -2,6 +2,8 @@
 
 FROM postgres_base
 
+ARG PYTHON_VERSION
+
 RUN set -eux; \
 	apt-get update; \
 	apt-get install -y --no-install-recommends \
@@ -10,15 +12,20 @@ RUN set -eux; \
         libcurl4-gnutls-dev \
         # mysql deps
         default-libmysqlclient-dev \
+        # multicorn deps
+        python${PYTHON_VERSION} \
+        python${PYTHON_VERSION}-dev \
 	; \
 	rm -rf /var/lib/apt/lists/*
 
-
 # columnar ext
 COPY --from=columnar /pg_ext /
 # mysql ext
 COPY --from=mysql /pg_ext /
 # http ext
 COPY --from=http /pg_ext /
+# multicorn ext
+COPY --from=multicorn /pg_ext /
+COPY --from=multicorn /python-dist-packages /usr/local/lib/python${PYTHON_VERSION}/dist-packages
 
 COPY files/postgres/docker-entrypoint-initdb.d /docker-entrypoint-initdb.d/

Dockerfile.spilo

@@ -2,26 +2,43 @@
 
 FROM spilo_base
 
+ARG PYTHON_VERSION
+
 RUN set -eux; \
-	apt-get update; \
-	apt-get install -y --no-install-recommends \
-        # http deps
-        ca-certificates \
-        libcurl4-gnutls-dev \
-        # mysql deps
-        default-libmysqlclient-dev \
+    apt-get update; \
+    apt-get install -y --no-install-recommends software-properties-common; \
+    # make the latest python available
+    add-apt-repository ppa:deadsnakes/ppa; \
+    apt-get update; \
+    apt-get install -y --no-install-recommends \
+    # http deps
+    ca-certificates \
+    libcurl4-gnutls-dev \
+    # mysql deps
+    default-libmysqlclient-dev \
+    # multicorn deps
+    python${PYTHON_VERSION} \
+    python${PYTHON_VERSION}-dev \
 	; \
-	rm -rf /var/lib/apt/lists/*
+    rm -rf /var/lib/apt/lists/*
 
+# columnar ext
 COPY --from=columnar_13 /pg_ext /
 COPY --from=columnar_14 /pg_ext /
 COPY files/spilo/postgres-appliance/scripts /scripts/
 
+# mysql ext
+COPY --from=mysql_13 /pg_ext /
+COPY --from=mysql_14 /pg_ext /
+
+# http ext
 COPY --from=http_13 /pg_ext /
 COPY --from=http_14 /pg_ext /
 
-COPY --from=mysql_13 /pg_ext /
-COPY --from=mysql_14 /pg_ext /
+# multicorn ext
+COPY --from=multicorn_13 /pg_ext /
+COPY --from=multicorn_14 /pg_ext /
+COPY --from=multicorn_14 /python-dist-packages /usr/local/lib/python${PYTHON_VERSION}/dist-packages
 
 ARG POSTGRES_BASE_VERSION
 # Default envs

acceptance/shared/cases.go

@@ -280,6 +280,63 @@ SELECT * FROM warehouse ORDER BY warehouse_id LIMIT 1;
 			}
 		},
 	},
+	{
+		Name: "multicorn ext available",
+		SQL: `
+SELECT count(1) FROM pg_available_extensions WHERE name = 'multicorn';
+			`,
+		Validate: func(t *testing.T, row pgx.Row) {
+			var count int
+			if err := row.Scan(&count); err != nil {
+				t.Fatal(err)
+			}
+
+			if want, got := 1, count; want != got {
+				t.Errorf("columnar ext should exist")
+			}
+		},
+	},
+	{
+		Name: "enable multicorn ext",
+		SQL: `
+CREATE EXTENSION multicorn;
+			`,
+	},
+	{
+		Name: "multicorn ext enabled",
+		SQL: `
+SELECT count(1) FROM pg_extension WHERE extname = 'multicorn';
+			`,
+		Validate: func(t *testing.T, row pgx.Row) {
+			var count int
+			if err := row.Scan(&count); err != nil {
+				t.Fatal(err)
+			}
+
+			if want, got := 1, count; want != got {
+				t.Errorf("columnar ext should exist")
+			}
+		},
+	},
+	{
+		Name: "create multicorn ext foreign table",
+		SQL: `
+CREATE SERVER multicorn_s3 FOREIGN DATA WRAPPER multicorn
+options (
+  wrapper 's3fdw.s3fdw.S3Fdw'
+);
+
+create foreign table s3 (
+  id int,
+  name text
+) server multicorn_s3 options (
+  aws_access_key 'FAKE',
+  aws_secret_key 'FAKE',
+  bucket 'test-bucket',
+  filename 'test.csv'
+);
+		`,
+	},
 }
 
 // These describe the shared setup and validation cases that occur to validate

docker-bake.hcl

@@ -22,6 +22,10 @@ variable "SPILO_POSTGRES_OLD_VERSIONS" {
   default = "13"
 }
 
+variable "PYTHON_VERSION" {
+  default = "3.9"
+}
+
 group "default" {
   targets = ["postgres", "spilo"]
 }
@@ -42,6 +46,11 @@ target "postgres" {
     columnar = "target:columnar_${POSTGRES_BASE_VERSION}"
     http = "target:http_${POSTGRES_BASE_VERSION}"
     mysql = "target:mysql_${POSTGRES_BASE_VERSION}"
+    multicorn = "target:multicorn_${POSTGRES_BASE_VERSION}"
+  }
+
+  args = {
+    PYTHON_VERSION = "${PYTHON_VERSION}"
   }
 
   tags = [
@@ -66,10 +75,13 @@ target "spilo" {
     http_14 = "target:http_14"
     mysql_13 = "target:mysql_13"
     mysql_14 = "target:mysql_14"
+    multicorn_13 = "target:multicorn_13"
+    multicorn_14 = "target:multicorn_14"
   }
 
   args = {
     POSTGRES_BASE_VERSION = "${SPILO_POSTGRES_VERSION}"
+    PYTHON_VERSION = "${PYTHON_VERSION}"
   }
 
   tags = [
@@ -168,6 +180,48 @@ target "mysql_14" {
   cache-from = ["type=local,src=tmp/bake_cache/mysql_14"]
 }
 
+target "multicorn" {
+  inherits = ["shared"]
+  context = "multicorn"
+  target = "output"
+
+  args = {
+    PYTHON_VERSION = "${PYTHON_VERSION}"
+    MULTICORN_TAG  = "v2.4"
+    S3CSV_FDW_COMMIT = "08ca3c082e2bfaa9ae60303fed67800c29a6fe6c"
+  }
+}
+
+target "multicorn_13" {
+  inherits = ["multicorn"]
+
+  contexts = {
+    postgres_base = "docker-image://postgres:13"
+  }
+
+  args = {
+    POSTGRES_BASE_VERSION = 13
+  }
+
+  cache-to = ["type=local,dest=tmp/bake_cache/multicorn_13"]
+  cache-from = ["type=local,src=tmp/bake_cache/multicorn_13"]
+}
+
+target "multicorn_14" {
+  inherits = ["multicorn"]
+
+  contexts = {
+    postgres_base = "docker-image://postgres:14"
+  }
+
+  args = {
+    POSTGRES_BASE_VERSION = 14
+  }
+
+  cache-to = ["type=local,dest=tmp/bake_cache/multicorn_14"]
+  cache-from = ["type=local,src=tmp/bake_cache/multicorn_14"]
+}
+
 target "columnar" {
   inherits = ["shared"]
   context = "columnar"

files/spilo/postgres-appliance/scripts/configure_spilo.py

@@ -312,7 +312,7 @@ def deep_update(a, b):
     bg_mon.history_buckets: 120
     pg_stat_statements.track_utility: 'off'
     extwlist.extensions: 'btree_gin,btree_gist,citext,extra_window_functions,first_last_agg,hll,\
-hstore,hypopg,intarray,ltree,pgcrypto,pgq,pgq_node,pg_trgm,postgres_fdw,http,mysql_fdw,tablefunc,uuid-ossp'
+hstore,hypopg,intarray,ltree,pgcrypto,pgq,pgq_node,pg_trgm,postgres_fdw,http,mysql_fdw,multicorn,tablefunc,uuid-ossp'
     extwlist.custom_path: /scripts
     cron.use_background_workers: 'on'
   pg_hba:

files/spilo/postgres-appliance/scripts/multicorn/after-create.sql

@@ -0,0 +1 @@
+GRANT USAGE ON FOREIGN DATA WRAPPER multicorn TO admin;

multicorn/Dockerfile

@@ -0,0 +1,55 @@
+#syntax=docker/dockerfile:1
+
+FROM postgres_base as setup
+
+ARG MYSQL_FDW_TAG
+ARG MULTICORN_TAG
+ARG PYTHON_VERSION
+ARG S3CSV_FDW_COMMIT
+ARG POSTGRES_BASE_VERSION=14
+
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install gnupg postgresql-common git -y
+RUN sh /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh -y
+RUN set -eux; \
+    export DEBIAN_FRONTEND=noninteractive && \
+    apt-get update; \
+    apt-get upgrade -y; \
+    apt-get install -y \
+      postgresql-${POSTGRES_BASE_VERSION} \
+      postgresql-server-dev-${POSTGRES_BASE_VERSION} \
+      git \
+      build-essential \
+      libreadline-dev \
+      zlib1g-dev \
+      wget \
+      flex \
+      bison \
+      libxml2-dev \
+      libxslt-dev \
+      libssl-dev \
+      libxml2-utils \
+      xsltproc \
+      python${PYTHON_VERSION} \
+      python${PYTHON_VERSION}-dev \
+      python3-pip
+
+# Update pip to the latest
+RUN wget https://bootstrap.pypa.io/get-pip.py && python3 get-pip.py && rm get-pip.py
+
+FROM setup as builder
+
+RUN git clone https://github.com/pgsql-io/multicorn2 && \
+  cd multicorn2 && \
+  git checkout ${MULTICORN_TAG} && \
+  DESTDIR=/pg_ext USE_PGXS=1 make && \
+  DESTDIR=/pg_ext USE_PGXS=1 make install
+
+# install runtime python deps
+RUN python3 -m pip install git+https://github.com/eligoenergy/s3csv_fdw.git@${S3CSV_FDW_COMMIT}
+
+FROM scratch as output
+
+ARG PYTHON_VERSION
+
+COPY --from=builder /pg_ext /pg_ext
+COPY --from=builder /usr/local/lib/python${PYTHON_VERSION}/dist-packages /python-dist-packages