setup CI for integration test

.ci/Dockerfile.client

@@ -0,0 +1,7 @@
+ARG PYTHON_VERSION=3.9
+FROM python:${PYTHON_VERSION}
+
+WORKDIR /code/opensearch-py-ml
+RUN python -m pip install nox
+
+COPY . .

.ci/cleanup.sh

@@ -0,0 +1,67 @@
+#!/usr/bin/env bash
+#
+# Shared cleanup routines between different steps
+#
+# Please source .ci/functions/imports.sh as a whole not just this file
+#
+# Version 1.0.0
+# - Initial version after refactor
+
+function cleanup_volume {
+  if [[ "$(docker volume ls -q -f name=$1)" ]]; then
+    echo -e "\033[34;1mINFO:\033[0m Removing volume $1\033[0m"
+    (docker volume rm "$1") || true
+  fi
+}
+function container_running {
+  if [[ "$(docker ps -q -f name=$1)" ]]; then
+    return 0;
+    else return 1;
+  fi
+}
+function cleanup_node {
+  if container_running "$1"; then
+    echo -e "\033[34;1mINFO:\033[0m Removing container $1\033[0m"
+    (docker container rm --force --volumes "$1") || true
+  fi
+  if [[ -n "$1" ]]; then
+    echo -e "\033[34;1mINFO:\033[0m Removing volume $1-rest-test-data\033[0m"
+    cleanup_volume "$1-rest-test-data"
+  fi
+}
+function cleanup_network {
+  if [[ "$(docker network ls -q -f name=$1)" ]]; then
+    echo -e "\033[34;1mINFO:\033[0m Removing network $1\033[0m"
+    (docker network rm "$1") || true
+  fi
+}
+
+function cleanup_trap {
+  status=$?
+  set +x
+  if [[ "$DETACH" != "true" ]]; then
+    echo -e "\033[34;1mINFO:\033[0m clean the network if not detached (start and exit)\033[0m"
+    cleanup_all_in_network "$1"
+  fi
+  # status is 0 or SIGINT
+  if [[ "$status" == "0" || "$status" == "130" ]]; then
+    echo -e "\n\033[32;1mSUCCESS run-tests\033[0m"
+    exit 0
+  else
+    echo -e "\n\033[31;1mFAILURE during run-tests\033[0m"
+    exit ${status}
+  fi
+};
+function cleanup_all_in_network {
+
+  if [[ -z "$(docker network ls -q -f name="^$1\$")" ]]; then
+    echo -e "\033[34;1mINFO:\033[0m $1 is already deleted\033[0m"
+    return 0
+  fi
+  containers=$(docker network inspect -f '{{ range $key, $value := .Containers }}{{ printf "%s\n" .Name}}{{ end }}' $1)
+  while read -r container; do
+    cleanup_node "$container"
+  done <<< "$containers"
+  cleanup_network $1
+  echo -e "\033[32;1mSUCCESS:\033[0m Cleaned up and exiting\033[0m"
+};

.ci/imports.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+#
+# Sets up all the common variables and imports relevant functions
+#
+# Version 1.0.1
+# - Initial version after refactor
+
+if [[ -z $opensearch_node_name ]]; then
+  # only set these once
+  set -euo pipefail
+  export TEST_SUITE=${TEST_SUITE-oss}
+  export RUNSCRIPTS=${RUNSCRIPTS-}
+  export DETACH=${DETACH-false}
+  export CLEANUP=${CLEANUP-false}
+  export OPENSEARCH_URL_EXTENSION=${OPENSEARCH_URL_EXTENSION-http}
+
+  export opensearch_node_name=instance
+  export opensearch_image=opensearchproject/opensearch
+
+  export opensearch_url=$OPENSEARCH_URL_EXTENSION://${opensearch_node_name}:9200
+  export external_opensearch_url=${opensearch_url/$opensearch_node_name/localhost}
+
+  export network_name=search-rest-test
+
+
+fi
+
+  export script_path=$(dirname $(realpath -s $0))
+  source $script_path/cleanup.sh
+  source $script_path/wait-for-container.sh
+  trap "cleanup_trap ${network_name}" EXIT
+
+
+if [[ "$CLEANUP" == "true" ]]; then
+  cleanup_all_in_network $network_name
+  exit 0
+fi
+
+echo -e "\033[34;1mINFO:\033[0m Creating network $network_name if it does not exist already \033[0m"
+docker network inspect "$network_name" > /dev/null 2>&1 || docker network create "$network_name"

.ci/opensearch/Dockerfile

@@ -0,0 +1,8 @@
+ARG OPENSEARCH_VERSION
+FROM opensearchproject/opensearch:$OPENSEARCH_VERSION
+
+ARG opensearch_path=/usr/share/opensearch
+ARG opensearch_yml=$opensearch_path/config/opensearch.yml
+
+ARG SECURE_INTEGRATION
+RUN if [ "$SECURE_INTEGRATION" != "true" ] ; then echo "plugins.security.disabled: true" >> $opensearch_yml; fi

.ci/run-opensearch.sh

@@ -0,0 +1,112 @@
+#!/usr/bin/env bash
+#
+# Launch one or more OpenSearch nodes via the Docker image,
+# to form a cluster suitable for running the REST API tests.
+#
+# Export the NUMBER_OF_NODES variable to start more than 1 node
+
+script_path=$(dirname $(realpath -s $0))
+source $script_path/imports.sh
+set -euo pipefail
+
+echo -e "\033[34;1mINFO:\033[0m Take down node if called twice with the same arguments (DETACH=true) or on separate terminals \033[0m"
+cleanup_node $opensearch_node_name
+
+master_node_name=${opensearch_node_name}
+cluster_name=search-py-ml-test
+
+declare -a volumes
+environment=($(cat <<-END
+  --env node.name=$opensearch_node_name
+  --env cluster.name=$cluster_name
+  --env discovery.type=single-node
+  --env discovery.seed_hosts=$master_node_name
+  --env cluster.routing.allocation.disk.threshold_enabled=false
+  --env bootstrap.memory_lock=true
+  --env node.attr.testattr=test
+  --env path.repo=/tmp
+  --env repositories.url.allowed_urls=http://snapshot.test*
+  --env action.destructive_requires_name=false
+END
+))
+
+if [[ "$SECURE_INTEGRATION" == "false" ]] && [[ "$CLUSTER" == "opensearch" ]] && [[ "$IS_UNRELEASED" == "false" ]]; then
+  security=($(cat <<-END
+    --env plugins.security.disabled=true
+END
+));
+else
+  security=($(cat <<-END
+    --env plugins.security.disabled=false
+END
+));
+fi
+
+NUMBER_OF_NODES=${NUMBER_OF_NODES-1}
+http_port=9200
+for (( i=0; i<$NUMBER_OF_NODES; i++, http_port++ )); do
+  node_name=${opensearch_node_name}$i
+  node_url=${external_opensearch_url/9200/${http_port}}$i
+  if [[ "$i" == "0" ]]; then node_name=$opensearch_node_name; fi
+  environment+=($(cat <<-END
+    --env node.name=$node_name
+END
+))
+  echo "$i: $http_port $node_url "
+  volume_name=${node_name}-rest-test-data
+  volumes+=($(cat <<-END
+    --volume $volume_name:/usr/share/opensearch/data${i}
+END
+))
+
+  # make sure we detach for all but the last node if DETACH=false (default) so all nodes are started
+  local_detach="true"
+  if [[ "$i" == "$((NUMBER_OF_NODES-1))" ]]; then local_detach=$DETACH; fi
+
+  set -x
+  healthcmd="curl -vvv -s --fail http://localhost:9200/_cluster/health || exit 1"
+  if [[ "$SECURE_INTEGRATION" == "true" ]]; then
+    healthcmd="curl -vvv -s --insecure -u admin:admin --fail https://localhost:9200/_cluster/health || exit 1"
+  fi
+
+  CLUSTER_TAG=$CLUSTER
+  if [[ "$IS_UNRELEASED" == "false" ]]; then
+    CLUSTER_TAG=$CLUSTER_TAG-secure-$SECURE_INTEGRATION
+    echo -e "\033[34;1mINFO: building $CLUSTER container\033[0m"
+    docker build \
+      --file=.ci/$CLUSTER/Dockerfile \
+      --build-arg SECURE_INTEGRATION=$SECURE_INTEGRATION \
+      --build-arg OPENSEARCH_VERSION=$OPENSEARCH_VERSION \
+      --tag=$CLUSTER_TAG \
+      .
+  else
+    CLUSTER_TAG=$CLUSTER_TAG:test
+  fi
+  echo -e "\033[34;1mINFO:\033[0m Starting container $node_name \033[0m"
+    docker run \
+      --name "$node_name" \
+      --network "$network_name" \
+      --env "ES_JAVA_OPTS=-Xms1g -Xmx1g" \
+      "${environment[@]}" \
+      "${volumes[@]}" \
+      "${security[@]}" \
+      --publish "$http_port":9200 \
+      --ulimit nofile=65536:65536 \
+      --ulimit memlock=-1:-1 \
+      --detach="$local_detach" \
+      --health-cmd="$(echo $healthcmd)" \
+      --health-interval=2s \
+      --health-retries=20 \
+      --health-timeout=2s \
+      --rm \
+      -d \
+      $CLUSTER_TAG;
+
+  set +x
+  if wait_for_container "$opensearch_node_name" "$network_name"; then
+    export node_url=$node_url
+    echo -e "\033[32;1mSUCCESS:\033[0m Running on: $node_url\033[0m"
+
+  fi
+
+done

.ci/run-repository.sh

@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+
+# Called by entry point `run-test` use this script to add your repository specific test commands
+# Once called opensearch is up and running and the following parameters are available to this script
+
+# OPENSEARCH_VERSION -- version e.g Major.Minor.Patch(-Prelease)
+# OPENSEARCH_URL -- The url at which opensearch is reachable
+# network_name -- The docker network name
+# NODE_NAME -- The docker container name also used as opensearch node name
+
+# When run in CI the test-matrix is used to define additional variables
+# TEST_SUITE -- defaults to `oss` in `run-tests`
+
+set -e
+
+echo -e "\033[34;1mINFO:\033[0m URL ${opensearch_url}\033[0m"
+echo -e "\033[34;1mINFO:\033[0m EXTERNAL OS URL ${external_opensearch_url}\033[0m"
+echo -e "\033[34;1mINFO:\033[0m VERSION ${OPENSEARCH_VERSION}\033[0m"
+echo -e "\033[34;1mINFO:\033[0m TEST_SUITE ${TEST_SUITE}\033[0m"
+echo -e "\033[34;1mINFO:\033[0m PYTHON_VERSION ${PYTHON_VERSION}\033[0m"
+echo -e "\033[34;1mINFO:\033[0m PYTHON_CONNECTION_CLASS ${PYTHON_CONNECTION_CLASS}\033[0m"
+echo -e "\033[34;1mINFO:\033[0m PANDAS_VERSION ${PANDAS_VERSION}\033[0m"
+
+echo -e "\033[1m>>>>> Build [opensearch-project/opensearch-py-ml container] >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m"
+
+docker build \
+       --file .ci/Dockerfile.client \
+       --tag opensearch-project/opensearch-py-ml \
+       --build-arg PYTHON_VERSION=${PYTHON_VERSION} \
+       .
+
+echo -e "\033[1m>>>>> Run [opensearch-project/opensearch-py-ml container] >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m"
+
+docker run \
+  --network=${network_name} \
+  --env "STACK_VERSION=${STACK_VERSION}" \
+  --env "OPENSEARCH_URL=${opensearch_url}" \
+  --env "OPENSEARCH_VERSION=${OPENSEARCH_VERSION}" \
+  --env "TEST_SUITE=${TEST_SUITE}" \
+  --env "PYTHON_CONNECTION_CLASS=${PYTHON_CONNECTION_CLASS}" \
+  --env "TEST_TYPE=server" \
+  --name opensearch-py-ml-test-runner \
+  --rm \
+  opensearch-project/opensearch-py-ml \
+  nox -s "test-${PYTHON_VERSION}(pandas_version='${PANDAS_VERSION}')"

.ci/run-tests

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+#
+# Entrypoint to run integration tests
+
+# Default environment variables
+export TEST_SUITE="${TEST_SUITE:=oss}"
+export PYTHON_VERSION="${PYTHON_VERSION:=3.9}"
+export PANDAS_VERSION=${PANDAS_VERSION-1.5.0}
+export PYTHON_CONNECTION_CLASS="${PYTHON_CONNECTION_CLASS:=Urllib3HttpConnection}"
+export CLUSTER="${1:-opensearch}"
+export SECURE_INTEGRATION="${2:-true}"
+export OPENSEARCH_VERSION="${3:-latest}"
+if [[ "$SECURE_INTEGRATION" == "true" ]]; then
+    export OPENSEARCH_URL_EXTENSION="https"
+else
+    export OPENSEARCH_URL_EXTENSION="http"
+fi
+
+export IS_UNRELEASED=false
+
+
+echo -e "\033[1m>>>>> Unreleased is $IS_UNRELEASED >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m"
+script_path=$(dirname $(realpath -s $0))
+echo -e $script_path
+
+source $script_path/imports.sh
+set -euo pipefail
+
+echo -e "\033[1m>>>>> Start server container >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m"
+DETACH=true bash $script_path/run-opensearch.sh
+
+if [[ -n "$RUNSCRIPTS" ]]; then
+  for RUNSCRIPT in ${RUNSCRIPTS//,/ } ; do
+    echo -e "\033[1m>>>>> Running run-$RUNSCRIPT.sh >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m"
+    CONTAINER_NAME=${RUNSCRIPT} \
+      DETACH=true \
+      bash $script_path/run-${RUNSCRIPT}.sh
+  done
+fi
+
+echo -e "\033[1m>>>>> Repository specific tests >>>>>>>>>>>>>>>>>>>>>>>>>>>>>\033[0m"
+bash $script_path/run-repository.sh

.ci/wait-for-container.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+#
+# Exposes a routine scripts can call to wait for a container if that container set up a health command
+#
+# Please source .ci/functions/imports.sh as a whole not just this file
+#
+# Version 1.0.1
+# - Initial version after refactor
+# - Make sure wait_for_contiainer is silent
+
+function wait_for_container {
+  set +x
+  until ! container_running "$1" || (container_running "$1" && [[ "$(docker inspect -f "{{.State.Health.Status}}" ${1})" != "starting" ]]); do
+    echo ""
+    docker inspect -f "{{range .State.Health.Log}}{{.Output}}{{end}}" ${1}
+    echo -e "\033[34;1mINFO:\033[0m waiting for node $1 to be up\033[0m"
+    sleep 2;
+  done;
+
+  # Always show logs if the container is running, this is very useful both on CI as well as while developing
+  if container_running $1; then
+    docker logs $1
+  fi
+
+  if ! container_running $1 || [[ "$(docker inspect -f "{{.State.Health.Status}}" ${1})" != "healthy" ]]; then
+    cleanup_all_in_network $2
+    echo
+    echo -e "\033[31;1mERROR:\033[0m Failed to start $1 in detached mode beyond health checks\033[0m"
+    echo -e "\033[31;1mERROR:\033[0m dumped the docker log before shutting the node down\033[0m"
+    return 1
+  else
+    echo
+    echo -e "\033[32;1mSUCCESS:\033[0m Detached and healthy: ${1} on docker network: ${network_name}\033[0m"
+    return 0
+  fi
+}

.dockerignore

@@ -2,3 +2,4 @@ docs/*
 example/*
 .git
 .nox
+venv

.github/workflows/integration.yml

@@ -0,0 +1,21 @@
+name: Integration tests
+
+on: [push, pull_request]
+
+jobs:
+  integration:
+    name: Integ
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        cluster: ["opensearch"]
+        secured: ["true"]
+        entry:
+          - { opensearch_version: 2.3.0 }
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Integ ${{ matrix.cluster }} secured=${{ matrix.secured }} version=${{matrix.entry.opensearch_version}}
+        run: "./.ci/run-tests ${{ matrix.cluster }} ${{ matrix.secured }} ${{ matrix.entry.opensearch_version }}"