diff --git a/Dockerfile b/Dockerfile index 899e5ac8..31f8da61 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,6 +2,8 @@ FROM postgres_base +ARG PYTHON_VERSION + RUN set -eux; \ apt-get update; \ apt-get install -y --no-install-recommends \ @@ -10,15 +12,20 @@ RUN set -eux; \ libcurl4-gnutls-dev \ # mysql deps default-libmysqlclient-dev \ + # multicorn deps + python${PYTHON_VERSION} \ + python${PYTHON_VERSION}-dev \ ; \ rm -rf /var/lib/apt/lists/* - # columnar ext COPY --from=columnar /pg_ext / # mysql ext COPY --from=mysql /pg_ext / # http ext COPY --from=http /pg_ext / +# multicorn ext +COPY --from=multicorn /pg_ext / +COPY --from=multicorn /python-dist-packages /usr/local/lib/python${PYTHON_VERSION}/dist-packages COPY files/postgres/docker-entrypoint-initdb.d /docker-entrypoint-initdb.d/ diff --git a/Dockerfile.spilo b/Dockerfile.spilo index 0318557a..53cf2870 100644 --- a/Dockerfile.spilo +++ b/Dockerfile.spilo @@ -2,26 +2,43 @@ FROM spilo_base +ARG PYTHON_VERSION + RUN set -eux; \ - apt-get update; \ - apt-get install -y --no-install-recommends \ - # http deps - ca-certificates \ - libcurl4-gnutls-dev \ - # mysql deps - default-libmysqlclient-dev \ + apt-get update; \ + apt-get install -y --no-install-recommends software-properties-common; \ + # make the latest python available + add-apt-repository ppa:deadsnakes/ppa; \ + apt-get update; \ + apt-get install -y --no-install-recommends \ + # http deps + ca-certificates \ + libcurl4-gnutls-dev \ + # mysql deps + default-libmysqlclient-dev \ + # multicorn deps + python${PYTHON_VERSION} \ + python${PYTHON_VERSION}-dev \ ; \ - rm -rf /var/lib/apt/lists/* + rm -rf /var/lib/apt/lists/* +# columnar ext COPY --from=columnar_13 /pg_ext / COPY --from=columnar_14 /pg_ext / COPY files/spilo/postgres-appliance/scripts /scripts/ +# mysql ext +COPY --from=mysql_13 /pg_ext / +COPY --from=mysql_14 /pg_ext / + +# http ext COPY --from=http_13 /pg_ext / COPY --from=http_14 /pg_ext / -COPY --from=mysql_13 /pg_ext / -COPY --from=mysql_14 /pg_ext / +# multicorn ext +COPY --from=multicorn_13 /pg_ext / +COPY --from=multicorn_14 /pg_ext / +COPY --from=multicorn_14 /python-dist-packages /usr/local/lib/python${PYTHON_VERSION}/dist-packages ARG POSTGRES_BASE_VERSION # Default envs diff --git a/acceptance/shared/cases.go b/acceptance/shared/cases.go index 97ed964a..6aafc938 100644 --- a/acceptance/shared/cases.go +++ b/acceptance/shared/cases.go @@ -280,6 +280,63 @@ SELECT * FROM warehouse ORDER BY warehouse_id LIMIT 1; } }, }, + { + Name: "multicorn ext available", + SQL: ` +SELECT count(1) FROM pg_available_extensions WHERE name = 'multicorn'; + `, + Validate: func(t *testing.T, row pgx.Row) { + var count int + if err := row.Scan(&count); err != nil { + t.Fatal(err) + } + + if want, got := 1, count; want != got { + t.Errorf("columnar ext should exist") + } + }, + }, + { + Name: "enable multicorn ext", + SQL: ` +CREATE EXTENSION multicorn; + `, + }, + { + Name: "multicorn ext enabled", + SQL: ` +SELECT count(1) FROM pg_extension WHERE extname = 'multicorn'; + `, + Validate: func(t *testing.T, row pgx.Row) { + var count int + if err := row.Scan(&count); err != nil { + t.Fatal(err) + } + + if want, got := 1, count; want != got { + t.Errorf("columnar ext should exist") + } + }, + }, + { + Name: "create multicorn ext foreign table", + SQL: ` +CREATE SERVER multicorn_s3 FOREIGN DATA WRAPPER multicorn +options ( + wrapper 's3fdw.s3fdw.S3Fdw' +); + +create foreign table s3 ( + id int, + name text +) server multicorn_s3 options ( + aws_access_key 'FAKE', + aws_secret_key 'FAKE', + bucket 'test-bucket', + filename 'test.csv' +); + `, + }, } // These describe the shared setup and validation cases that occur to validate diff --git a/docker-bake.hcl b/docker-bake.hcl index c594a78d..d83fc89c 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -22,6 +22,10 @@ variable "SPILO_POSTGRES_OLD_VERSIONS" { default = "13" } +variable "PYTHON_VERSION" { + default = "3.9" +} + group "default" { targets = ["postgres", "spilo"] } @@ -42,6 +46,11 @@ target "postgres" { columnar = "target:columnar_${POSTGRES_BASE_VERSION}" http = "target:http_${POSTGRES_BASE_VERSION}" mysql = "target:mysql_${POSTGRES_BASE_VERSION}" + multicorn = "target:multicorn_${POSTGRES_BASE_VERSION}" + } + + args = { + PYTHON_VERSION = "${PYTHON_VERSION}" } tags = [ @@ -66,10 +75,13 @@ target "spilo" { http_14 = "target:http_14" mysql_13 = "target:mysql_13" mysql_14 = "target:mysql_14" + multicorn_13 = "target:multicorn_13" + multicorn_14 = "target:multicorn_14" } args = { POSTGRES_BASE_VERSION = "${SPILO_POSTGRES_VERSION}" + PYTHON_VERSION = "${PYTHON_VERSION}" } tags = [ @@ -168,6 +180,48 @@ target "mysql_14" { cache-from = ["type=local,src=tmp/bake_cache/mysql_14"] } +target "multicorn" { + inherits = ["shared"] + context = "multicorn" + target = "output" + + args = { + PYTHON_VERSION = "${PYTHON_VERSION}" + MULTICORN_TAG = "v2.4" + S3CSV_FDW_COMMIT = "08ca3c082e2bfaa9ae60303fed67800c29a6fe6c" + } +} + +target "multicorn_13" { + inherits = ["multicorn"] + + contexts = { + postgres_base = "docker-image://postgres:13" + } + + args = { + POSTGRES_BASE_VERSION = 13 + } + + cache-to = ["type=local,dest=tmp/bake_cache/multicorn_13"] + cache-from = ["type=local,src=tmp/bake_cache/multicorn_13"] +} + +target "multicorn_14" { + inherits = ["multicorn"] + + contexts = { + postgres_base = "docker-image://postgres:14" + } + + args = { + POSTGRES_BASE_VERSION = 14 + } + + cache-to = ["type=local,dest=tmp/bake_cache/multicorn_14"] + cache-from = ["type=local,src=tmp/bake_cache/multicorn_14"] +} + target "columnar" { inherits = ["shared"] context = "columnar" diff --git a/files/spilo/postgres-appliance/scripts/configure_spilo.py b/files/spilo/postgres-appliance/scripts/configure_spilo.py index dc4bcc2a..bf428fdb 100755 --- a/files/spilo/postgres-appliance/scripts/configure_spilo.py +++ b/files/spilo/postgres-appliance/scripts/configure_spilo.py @@ -312,7 +312,7 @@ def deep_update(a, b): bg_mon.history_buckets: 120 pg_stat_statements.track_utility: 'off' extwlist.extensions: 'btree_gin,btree_gist,citext,extra_window_functions,first_last_agg,hll,\ -hstore,hypopg,intarray,ltree,pgcrypto,pgq,pgq_node,pg_trgm,postgres_fdw,http,mysql_fdw,tablefunc,uuid-ossp' +hstore,hypopg,intarray,ltree,pgcrypto,pgq,pgq_node,pg_trgm,postgres_fdw,http,mysql_fdw,multicorn,tablefunc,uuid-ossp' extwlist.custom_path: /scripts cron.use_background_workers: 'on' pg_hba: diff --git a/files/spilo/postgres-appliance/scripts/multicorn/after-create.sql b/files/spilo/postgres-appliance/scripts/multicorn/after-create.sql new file mode 100644 index 00000000..38af0bea --- /dev/null +++ b/files/spilo/postgres-appliance/scripts/multicorn/after-create.sql @@ -0,0 +1 @@ +GRANT USAGE ON FOREIGN DATA WRAPPER multicorn TO admin; diff --git a/multicorn/Dockerfile b/multicorn/Dockerfile new file mode 100644 index 00000000..5bc60281 --- /dev/null +++ b/multicorn/Dockerfile @@ -0,0 +1,55 @@ +#syntax=docker/dockerfile:1 + +FROM postgres_base as setup + +ARG MYSQL_FDW_TAG +ARG MULTICORN_TAG +ARG PYTHON_VERSION +ARG S3CSV_FDW_COMMIT +ARG POSTGRES_BASE_VERSION=14 + +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install gnupg postgresql-common git -y +RUN sh /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh -y +RUN set -eux; \ + export DEBIAN_FRONTEND=noninteractive && \ + apt-get update; \ + apt-get upgrade -y; \ + apt-get install -y \ + postgresql-${POSTGRES_BASE_VERSION} \ + postgresql-server-dev-${POSTGRES_BASE_VERSION} \ + git \ + build-essential \ + libreadline-dev \ + zlib1g-dev \ + wget \ + flex \ + bison \ + libxml2-dev \ + libxslt-dev \ + libssl-dev \ + libxml2-utils \ + xsltproc \ + python${PYTHON_VERSION} \ + python${PYTHON_VERSION}-dev \ + python3-pip + +# Update pip to the latest +RUN wget https://bootstrap.pypa.io/get-pip.py && python3 get-pip.py && rm get-pip.py + +FROM setup as builder + +RUN git clone https://github.com/pgsql-io/multicorn2 && \ + cd multicorn2 && \ + git checkout ${MULTICORN_TAG} && \ + DESTDIR=/pg_ext USE_PGXS=1 make && \ + DESTDIR=/pg_ext USE_PGXS=1 make install + +# install runtime python deps +RUN python3 -m pip install git+https://github.com/eligoenergy/s3csv_fdw.git@${S3CSV_FDW_COMMIT} + +FROM scratch as output + +ARG PYTHON_VERSION + +COPY --from=builder /pg_ext /pg_ext +COPY --from=builder /usr/local/lib/python${PYTHON_VERSION}/dist-packages /python-dist-packages