ci: split tests into native and wasm

.github/workflows/test_wrappers.yml

@@ -6,8 +6,11 @@ on:
       - main
 
 jobs:
-  test:
-    name: Run wrappers tests
+  # =============================================================
+  # Native wrappers test
+  # =============================================================
+  test_native:
+    name: Run native wrappers tests
     runs-on: ubuntu-20.04
 
     steps:
@@ -16,7 +19,62 @@ jobs:
 
     - name: Build docker images
       run: |
-        docker compose -f wrappers/.ci/docker-compose.yaml up -d
+        docker compose -f wrappers/.ci/docker-compose-native.yaml up -d
+
+    - uses: actions-rs/toolchain@v1
+      with:
+        toolchain: stable
+
+    - run: |
+        sudo apt remove -y postgres*
+        sudo apt-get install -y wget gnupg
+        sudo sh -c 'echo "deb https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
+        wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -
+        sudo apt-get update -y -qq --fix-missing
+        sudo apt-get install -y \
+          clang-10 \
+          llvm-10 \
+          clang \
+          gcc \
+          make \
+          build-essential \
+          libz-dev \
+          zlib1g-dev \
+          strace \
+          libssl-dev \
+          pkg-config \
+          postgresql-15 \
+          postgresql-server-dev-15
+        sudo chmod a+rwx `/usr/lib/postgresql/15/bin/pg_config --pkglibdir` `/usr/lib/postgresql/15/bin/pg_config --sharedir`/extension /var/run/postgresql/
+
+    - run: cargo install cargo-pgrx --version 0.11.3
+    - run: cargo pgrx init --pg15 /usr/lib/postgresql/15/bin/pg_config
+
+    - name: Format code
+      run: |
+        cd wrappers && cargo fmt --check
+
+    - name: Run clippy
+      run: |
+        cd wrappers && RUSTFLAGS="-D warnings" cargo clippy --all --tests --no-deps --features native_fdws,helloworld_fdw
+
+    - name: Perform test
+      run: cd wrappers && cargo pgrx test --features "native_fdws pg15"
+
+  # =============================================================
+  # Wasm wrappers test
+  # =============================================================
+  test_wasm:
+    name: Run Wasm wrappers tests
+    runs-on: ubuntu-20.04
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+
+    - name: Build docker images
+      run: |
+        docker compose -f wrappers/.ci/docker-compose-wasm.yaml up -d
 
     - uses: actions-rs/toolchain@v1
       with:
@@ -49,28 +107,17 @@ jobs:
     - run: cargo install cargo-component --version 0.13.2
     - run: rustup target add wasm32-unknown-unknown
 
-    - name: Build WASM FDWs
+    - name: Build Wasm FDWs
       run: |
-        cd wasm-wrappers/fdw
-        cargo component build --manifest-path ./helloworld_fdw/Cargo.toml --release --target wasm32-unknown-unknown
-        cargo component build --manifest-path ./paddle_fdw/Cargo.toml --release --target wasm32-unknown-unknown
-        cargo component build --manifest-path ./snowflake_fdw/Cargo.toml --release --target wasm32-unknown-unknown
+        find ./wasm-wrappers/fdw/ -name "Cargo.toml" -exec cargo component build --release --target wasm32-unknown-unknown --manifest-path {} \;
 
     - name: Format code
       run: |
-        cd wrappers && cargo fmt --check
-        cd ../wasm-wrappers/fdw
-        cargo fmt --manifest-path ./helloworld_fdw/Cargo.toml --check
-        cargo fmt --manifest-path ./paddle_fdw/Cargo.toml --check
-        cargo fmt --manifest-path ./snowflake_fdw/Cargo.toml --check
+        find ./wasm-wrappers/fdw/ -name "Cargo.toml" -exec cargo fmt --check --manifest-path {} \;
 
     - name: Run clippy
       run: |
-        cd wrappers && RUSTFLAGS="-D warnings" cargo clippy --all --tests --no-deps --features all_fdws,helloworld_fdw
-        cd ../wasm-wrappers/fdw
-        RUSTFLAGS="-D warnings" cargo clippy --manifest-path ./helloworld_fdw/Cargo.toml --all --tests --no-deps
-        RUSTFLAGS="-D warnings" cargo clippy --manifest-path ./paddle_fdw/Cargo.toml --all --tests --no-deps
-        RUSTFLAGS="-D warnings" cargo clippy --manifest-path ./snowflake_fdw/Cargo.toml --all --tests --no-deps
+        RUSTFLAGS="-D warnings" find ./wasm-wrappers/fdw/ -name "Cargo.toml" -exec cargo clippy --all --tests --no-deps --manifest-path {} \;
 
     - name: Perform test
-      run: cd wrappers && cargo pgrx test --features "all_fdws pg15"
+      run: cd wrappers && cargo pgrx test --features "wasm_fdw pg15"

wrappers/.ci/docker-compose-native.yaml

@@ -0,0 +1,144 @@
+services:
+
+  airtable:
+    container_name: airtable-local
+    build:
+      context: ../dockerfiles/airtable
+    ports:
+      - "8086:8086"
+    healthcheck:
+      test: curl --fail http://0.0.0.0:8086/baseID/table-foo || exit 1
+      interval: 10s
+      timeout: 5s
+      retries: 3
+
+  auth0:
+    container_name: auth0-local
+    build:
+      context: ../dockerfiles/auth0
+    ports:
+      - "3796:3796"
+    healthcheck:
+      test: curl --fail http://0.0.0.0:3796/ || exit 1
+      interval: 11s
+      timeout: 6s
+      retries: 3
+
+  clickhouse:
+    image: clickhouse/clickhouse-server
+    container_name: clickhouse-wrapped
+    ports:
+      - "9000:9000" # native interface
+      - "8123:8123" # http interface
+    healthcheck:
+      test: sleep 4 && wget --no-verbose --tries=1 --spider http://localhost:8123/?query=SELECT%201 || exit 1
+      interval: 10s
+      timeout: 5s
+      retries: 20
+
+  cognito:
+    image: jagregory/cognito-local
+    ports:
+      - "9229:9229"
+    volumes:
+      - ../dockerfiles/cognito/.cognito:/app/.cognito
+    healthcheck:
+      test: curl --fail http://0.0.0.0:9229/ || exit 1
+      interval: 11s
+      timeout: 6s
+      retries: 3
+
+  stripe:
+    image: stripe/stripe-mock:v0.144.0
+    container_name: stripe-mock
+    ports:
+      - "12111:12111"
+    healthcheck:
+      test: sleep 4
+      retries: 20
+
+  firebase:
+    image: andreysenov/firebase-tools:11.24.1-node-14-alpine
+    container_name: firebase-wrapped
+    command: firebase emulators:start --project supa --only auth,firestore --import=/baseline-data
+    volumes:
+      - ../dockerfiles/firebase/baseline-data:/baseline-data
+      - ../dockerfiles/firebase/firebase.json:/home/node/firebase.json
+    ports:
+      - "4000:4000" # UI
+      - "8080:8080" # Firestore
+      - "9099:9099" # Auth
+    healthcheck:
+      test: sleep 4 && wget --no-verbose --tries=1 --spider http://localhost:9099/ || exit 1
+      interval: 10s
+      timeout: 5s
+      retries: 30
+
+  bigquery:
+    container_name: bigquery-local
+    build:
+      context: ../..
+      dockerfile: ./wrappers/dockerfiles/bigquery/Dockerfile
+    volumes:
+      - ../dockerfiles/bigquery/data.yaml:/app/data.yaml
+    ports:
+      - "9111:9111" # REST
+      - "9060:9060" # gRPC
+    command: --project=test --dataset=dataset1 --data-from-yaml=/app/data.yaml --port=9111
+    healthcheck:
+      test: curl --fail http://0.0.0.0:9111/bigquery/v2/projects/test_project/datasets/test_dataset || exit 1
+      interval: 10s
+      timeout: 5s
+      retries: 3
+
+  s3:
+    image: localstack/localstack
+    container_name: s3-mock
+    volumes:
+      - ../dockerfiles/s3/test_data:/data
+      - ../dockerfiles/s3/init:/etc/localstack/init/ready.d
+    ports:
+      - "4566:4566"
+      - "4510-4559:4510-4559"
+    healthcheck:
+      test: curl --fail http://127.0.0.1:4566/_localstack/init || exit 1
+      interval: 10s
+      timeout: 5s
+      retries: 3
+
+  mssql:
+    image: mcr.microsoft.com/mssql/server:2019-latest
+    container_name: mssql-wrapped
+    environment:
+      ACCEPT_EULA: Y
+      MSSQL_SA_PASSWORD: Password1234_56
+    ports:
+      - "1433:1433"
+    healthcheck:
+      test: (echo >/dev/tcp/127.0.0.1/1433) &>/dev/null && echo "okay" || exit 1
+      interval: 10s
+      timeout: 5s
+      retries: 20
+
+  redis:
+    image: redis:6.2-alpine
+    container_name: redis-wrapped
+    ports:
+      - "6379:6379"
+    healthcheck:
+      test: (nc -vz localhost 6379) &>/dev/null && echo "okay" || exit 1
+      interval: 10s
+      timeout: 5s
+      retries: 20
+
+  notion:
+    container_name: notion-local
+    build:
+      context: ../dockerfiles/notion
+    ports:
+      - "4242:4242"
+    healthcheck:
+      test: curl --fail http://0.0.0.0:4242/ || exit 1
+      interval: 11s
+      timeout: 6s
+      retries: 3

wrappers/.ci/docker-compose-wasm.yaml

@@ -0,0 +1,13 @@
+services:
+
+  wasm:
+    container_name: wasm-local
+    build:
+      context: ../dockerfiles/wasm
+    ports:
+      - "8096:8096"
+    healthcheck:
+      test: curl --fail http://0.0.0.0:8096/paddle || exit 1
+      interval: 10s
+      timeout: 5s
+      retries: 3