Merge pull request #14 from naveentatikonda/main_rebase_develop_branch

rebase main branch with develop branch

Author: naveentatikonda

.ci/Dockerfile

@@ -0,0 +1,33 @@
+ARG LOGSTASH_VERSION
+FROM docker.elastic.co/logstash/logstash-oss:${LOGSTASH_VERSION}
+# install and enable password-less sudo for logstash user
+# allows modifying the system inside the container (using the .ci/setup.sh hook)
+# USER root
+# RUN if [ $(command -v apt-get) ]; then \
+#       apt-get install -y sudo && \
+#       gpasswd -a logstash sudo; \
+#     else \
+#       yum install -y sudo && \
+#       usermod -aG wheel logstash; \
+#     fi
+# RUN echo "logstash ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/logstash && \
+#     chmod 0440 /etc/sudoers.d/logstash
+USER logstash
+# whole . plugin code could be copied here but we only do that after bundle install,
+# to speedup incremental builds (locally one's mostly changing lib/ and spec/ files)
+COPY --chown=logstash:logstash Gemfile /usr/share/plugins/plugin/Gemfile
+COPY --chown=logstash:logstash *.gemspec VERSION* version* /usr/share/plugins/plugin/
+#COPY --chown=logstash:logstash Gemfile *.gemspec VERSION* version* /usr/share/plugins/plugin/
+RUN cp /usr/share/logstash/logstash-core/versions-gem-copy.yml /usr/share/logstash/versions.yml
+# NOTE: since 8.0 JDK is bundled as part of the LS distribution under $LS_HOME/jdk
+ENV PATH="${PATH}:/usr/share/logstash/vendor/jruby/bin:/usr/share/logstash/jdk/bin"
+ENV LOGSTASH_SOURCE="1"
+ARG INTEGRATION
+ENV INTEGRATION=$INTEGRATION
+RUN gem install bundler -v '< 2'
+WORKDIR /usr/share/plugins/plugin
+# COPY --chown=logstash:logstash .ci/* /usr/share/plugins/plugin/.ci/
+# RUN .ci/setup.sh
+RUN bundle install --with test ci
+COPY --chown=logstash:logstash . /usr/share/plugins/plugin/
+RUN bundle exec rake vendor

.ci/Dockerfile.elasticsearch

@@ -0,0 +1,6 @@
+#LS_JAVA_OPTS="-Xms256m -Xmx256m -XX:MaxMetaspaceSize=256m"
+# Common LS options (can be overridden from ENV e.g. in .travis.yml) :
+# - `-Xregexp.interruptible=true -Xcompile.invokedynamic=true -Xjit.threshold=0` LS base-line
+# - `-XX:+UseParallelGC` do not use G1 (default) on Java 11
+# - `-v -W1` print JRuby version but do not go verbose
+JRUBY_OPTS=-Xregexp.interruptible=true -Xcompile.invokedynamic=true -Xjit.threshold=0 -J-XX:+UseParallelGC -J-XX:+PrintCommandLineFlags -v -W1

.ci/docker-run.sh

@@ -4,20 +4,18 @@ set -ex
 export PATH=$BUILD_DIR/gradle/bin:$PATH
 
 wait_for_es() {
-  echo "Waiting for elasticsearch to respond..."
-  es_url="http://elasticsearch:9200"
+
+  SERVICE_URL="http://integration:9200"
   if [[ "$SECURE_INTEGRATION" == "true" ]]; then
-    es_url="https://elasticsearch:9200 -k"
+    SERVICE_URL="https://integration:9200 -k -u admin:admin"
   fi
-  count=120
-  while ! curl --silent $es_url && [[ $count -ne 0 ]]; do
+  count=10
+  while ! curl -s $SERVICE_URL >/dev/null && [[ $count -ne 0 ]]; do
     count=$(( $count - 1 ))
     [[ $count -eq 0 ]] && return 1
-    sleep 1
+    sleep 20
   done
-  echo "Elasticsearch is Up !"
-
-  return 0
+  echo $(curl -s $SERVICE_URL | python -c "import sys, json; print(json.load(sys.stdin)['version']['number'])")
 }
 
 if [[ "$INTEGRATION" != "true" ]]; then
@@ -28,6 +26,9 @@ else
   else
     extra_tag_args="--tag ~secure_integration --tag integration"
   fi
-  wait_for_es
-  bundle exec rspec -fd $extra_tag_args --tag es_version:$ELASTIC_STACK_VERSION spec/inputs/integration
+
+  echo "Waiting for cluster to respond..."
+  VERSION=$(wait_for_es)
+  echo "Cluster $VERSION is Up!"
+  bundle exec rspec -fd $extra_tag_args --tag es_version:$VERSION spec/inputs/integration
 fi

.ci/docker-setup.sh

@@ -0,0 +1,7 @@
+ARG OPENDISTRO_VERSION
+FROM amazon/opendistro-for-elasticsearch:${OPENDISTRO_VERSION}
+
+ARG es_path=/usr/share/elasticsearch
+
+ARG SECURE_INTEGRATION
+RUN if [ "$SECURE_INTEGRATION" != "true" ] ; then $es_path/bin/elasticsearch-plugin remove opendistro_security; fi

.ci/docker.env

@@ -7,21 +7,18 @@ services:
     environment:
         - INTEGRATION=${INTEGRATION:-false}
         - SECURE_INTEGRATION=${SECURE_INTEGRATION:-false}
-        - ELASTIC_STACK_VERSION=$ELASTIC_STACK_VERSION
-  elasticsearch:
+
+  integration:
     build:
-      context: ../
-      dockerfile: .ci/Dockerfile.elasticsearch
+      context: ../../
+      dockerfile: .ci/opendistro/Dockerfile.opendistro
       args:
-        - ELASTIC_STACK_VERSION=$ELASTIC_STACK_VERSION
+        - OPENDISTRO_VERSION=${OPENDISTRO_VERSION:-latest}
         - INTEGRATION=${INTEGRATION:-false}
         - SECURE_INTEGRATION=${SECURE_INTEGRATION:-false}
     environment:
-      #- ELASTIC_PASSWORD=$ELASTIC_PASSWORD a user is setup manually
-      - xpack.security.enabled=${SECURE_INTEGRATION:-false}
       - bootstrap.memory_lock=true
       - discovery.type=single-node
-      - ES_JAVA_OPTS=-Xms640m -Xmx640m
     tty: true
     ulimits:
       memlock:

.ci/logstash-run.sh

@@ -0,0 +1,16 @@
+version: '3'
+
+services:
+
+  logstash:
+    build:
+      context: ../../
+      dockerfile: .ci/Dockerfile
+      args:
+        - LOGSTASH_VERSION=${LOGSTASH_VERSION:-7.13.2}
+    command: .ci/run.sh
+    env_file: ../docker.env
+    environment:
+      - SPEC_OPTS
+      - LOG_LEVEL # devutils (>= 2.0.4) reads the ENV and sets LS logging
+    tty: true

.ci/opendistro/Dockerfile.opendistro

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+# This is intended to be run inside the docker container as the command of the docker-compose.
+set -ex
+
+cd .ci/opendistro
+
+docker-compose up --exit-code-from logstash