New folder structure for opensearch-build repo

Signed-off-by: Peter Zhu <zhujiaxi@amazon.com>

.gitignore

@@ -0,0 +1,12 @@
+Thumbs.db
+.DS_Store
+
+.idea/
+*.iml
+
+__pycache__
+.pytest_cache
+
+.gradle/
+.settings/
+out.txt

config/opensearch.yml

@@ -0,0 +1,7 @@
+cluster.name: docker-cluster
+
+# Bind to all interfaces because we don't know what IP address Docker will assign to us.
+network.host: 0.0.0.0
+
+# Setting network.host to a non-loopback address enables the annoying bootstrap checks. "Single-node" mode disables them again.
+discovery.type: single-node

config/opensearch_dashboards.yml

@@ -0,0 +1,27 @@
+# Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License").
+# You may not use this file except in compliance with the License.
+# A copy of the License is located at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# or in the "license" file accompanying this file. This file is distributed
+# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+# express or implied. See the License for the specific language governing
+# permissions and limitations under the License.
+
+# Description:
+# Default configuration for OpenSearch Dashboards
+
+opensearch.hosts: ["https://localhost:9200"]
+opensearch.ssl.verificationMode: none
+opensearch.username: "kibanaserver"
+opensearch.password: "kibanaserver"
+opensearch.requestHeadersWhitelist: [ authorization ]
+
+opensearch_security.multitenancy.enabled: true
+opensearch_security.multitenancy.tenants.preferred: ["Private", "Global"]
+opensearch_security.readonly_mode.roles: ["kibana_read_only"]
+# Use this setting if you are running kibana without https
+opensearch_security.cookie.secure: false

docker/build-image.sh

@@ -0,0 +1,103 @@
+#!/bin/bash
+
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+#
+# Modifications Copyright OpenSearch Contributors. See
+# GitHub history for details.
+
+
+# Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License").
+# You may not use this file except in compliance with the License.
+# A copy of the License is located at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# or in the "license" file accompanying this file. This file is distributed
+# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+# express or implied. See the License for the specific language governing
+# permissions and limitations under the License.
+
+
+set -e
+
+function usage() {
+    echo ""
+    echo "This script is used to build the OpenSearch Docker image. It prepares the files required by the Dockerfile in a temporary directory, then builds and tags the Docker image."
+    echo "--------------------------------------------------------------------------"
+    echo "Usage: $0 [args]"
+    echo ""
+    echo "Required arguments:"
+    echo -e "-v VERSION\tSpecify the OpenSearch version number that you are building, e.g. '1.0.0' or '1.0.0-beta1'. This will be used to label the Docker image. If you do not use the '-o' option then this tool will download a public OPENSEARCH release matching this version."
+    echo -e "-f DOCKERFILE\tSpecify the dockerfile full path, e.g. dockerfile/opensearch.al2.dockerfile."
+    echo -e "-p PRODUCT\tSpecify the product, e.g. opensearch or opensearch-dashboards, make sure this is the name of your .tgz defined in dockerfile."
+    echo ""
+    echo "Optional arguments:"
+    echo -e "-o FILENAME\tSpecify a local OPENSEARCH tarball. You still need to specify the version - this tool does not attempt to parse the filename."
+    echo -e "-h\t\tPrint this message."
+    echo "--------------------------------------------------------------------------"
+}
+
+while getopts ":ho:v:f:p:" arg; do
+    case $arg in
+        h)
+            usage
+            exit 1
+            ;;
+        o)
+            TARBALL=`realpath $OPTARG`
+            ;;
+        v)
+            VERSION=$OPTARG
+            ;;
+        f)
+            DOCKERFILE=$OPTARG
+            ;;
+        p)
+            PRODUCT=$OPTARG
+            ;;
+        :)
+            echo "-${OPTARG} requires an argument"
+            usage
+            exit 1
+            ;;
+        ?)
+            echo "Invalid option: -${arg}"
+            exit 1
+            ;;
+    esac
+done
+
+if [ -z "$VERSION" ] || [ -z "$DOCKERFILE" ] || [ -z "$PRODUCT" ]; then
+    echo "You must specify '-v VERSION', '-f DOCKERFILE', '-p PRODUCT'"
+    usage
+    exit 1
+fi
+
+echo $DOCKERFILE
+
+DIR=`mktemp -d`
+
+echo "Creating Docker workspace in $DIR"
+trap '{ echo Removing Docker workspace in "$DIR"; rm -rf -- "$DIR"; }' TERM INT EXIT
+
+if [ -z "$TARBALL" ]; then
+    # No tarball file specified so download one
+    URL="https://artifacts.opensearch.org/releases/bundle/${PRODUCT}/${VERSION}/${PRODUCT}-${VERSION}-linux-x64.tar.gz"
+    echo "Downloading ${PRODUCT} version ${VERSION} from ${URL}"
+    curl -f $URL -o $DIR/$PRODUCT.tgz || exit 1
+    ls -l $DIR
+else
+    cp -v $TARBALL $DIR/$PRODUCT.tgz
+fi
+
+cp -v ${PRODUCT}-config/* $DIR/
+
+docker build --build-arg VERSION=$VERSION --build-arg BUILD_DATE=`date -u +%Y-%m-%dT%H:%M:%SZ` -f $DOCKERFILE $DIR -t opensearchproject/$PRODUCT:$VERSION
+
+rm -rf $DIR

docker/dockerfiles/opensearch-dashboards.al2.dockerfile

@@ -0,0 +1,92 @@
+# SPDX-License-Identifier: Apache-2.0
+# 
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+# 
+# Modifications Copyright OpenSearch Contributors. See
+# GitHub history for details.
+
+
+# Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License").
+# You may not use this file except in compliance with the License.
+# A copy of the License is located at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# or in the "license" file accompanying this file. This file is distributed 
+# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 
+# express or implied. See the License for the specific language governing 
+# permissions and limitations under the License.
+
+
+# This dockerfile generates an AmazonLinux-based image containing an OpenSearch-Dashboards installation.
+# It assumes that the working directory contains four files: an OpenSearch-Dashboards tarball (opensearch-dashboards.tgz), opensearch_dashboards.yml, opensearch-dashboards-docker-entrypoint.sh, and example certs.
+# Build arguments:
+#   VERSION: Required. Used to label the image.
+#   BUILD_DATE: Required. Used to label the image. Should be in the form 'yyyy-mm-ddThh:mm:ssZ', i.e. a date-time from https://tools.ietf.org/html/rfc3339. The timestamp must be in UTC.
+#   UID: Optional. Specify the opensearch-dashboards userid. Defaults to 1000.
+#   GID: Optional. Specify the opensearch-dashboards groupid. Defaults to 1000.
+#   OPENSEARCH_DASHBOARDS_HOME: Optional. Specify the opensearch-dashboards root directory. Defaults to /usr/share/opensearch-dashboards.
+
+
+FROM amazonlinux:2 AS linux_x64_staging
+
+ARG UID=1000
+ARG GID=1000
+ARG OPENSEARCH_DASHBOARDS_HOME=/usr/share/opensearch-dashboards
+
+# Update packages
+# Install the tools we need: tar and gzip to unpack the OpenSearch tarball, and shadow-utils to give us `groupadd` and `useradd`.
+RUN yum update -y && yum install -y tar gzip shadow-utils && yum clean all
+
+# Create an opensearch-dashboards user, group, and directory
+RUN groupadd -g $GID opensearch-dashboards && \
+    adduser -u $UID -g $GID -d $OPENSEARCH_DASHBOARDS_HOME opensearch-dashboards && \
+    mkdir /tmp/opensearch-dashboards
+
+# Prepare working directory
+COPY opensearch-dashboards.tgz /tmp/opensearch-dashboards/opensearch-dashboards.tgz
+RUN tar -xzf /tmp/opensearch-dashboards/opensearch-dashboards.tgz -C $OPENSEARCH_DASHBOARDS_HOME --strip-components=1 && rm -rf /temp/opensearch-dashboards
+COPY opensearch-dashboards-docker-entrypoint.sh $OPENSEARCH_DASHBOARDS_HOME/
+COPY opensearch_dashboards.yml opensearch.example.org.* $OPENSEARCH_DASHBOARDS_HOME/config/
+
+# Copy working directory to the actual release docker images
+FROM amazonlinux:2
+
+ARG UID=1000
+ARG GID=1000
+ARG OPENSEARCH_DASHBOARDS_HOME=/usr/share/opensearch-dashboards
+
+COPY --from=linux_x64_staging $OPENSEARCH_DASHBOARDS_HOME $OPENSEARCH_DASHBOARDS_HOME
+
+# Setup OpenSearch-dashboards
+WORKDIR $OPENSEARCH_DASHBOARDS_HOME
+RUN yum update -y && yum install -y tar gzip shadow-utils && yum clean all
+RUN groupadd -g $GID opensearch-dashboards && \
+    adduser -u $UID -g $GID -d $OPENSEARCH_DASHBOARDS_HOME opensearch-dashboards
+RUN chown -R $UID:$GID $OPENSEARCH_DASHBOARDS_HOME
+
+# Change user
+USER $UID
+
+# Expose port
+EXPOSE 5601
+
+ARG VERSION
+ARG BUILD_DATE
+
+# Label
+LABEL org.label-schema.schema-version="1.0" \
+  org.label-schema.name="opensearch-dashboards" \
+  org.label-schema.version="$VERSION" \
+  org.label-schema.url="https://opensearch.org" \
+  org.label-schema.vcs-url="https://github.com/opensearch-project/OpenSearch-Dashboards" \
+  org.label-schema.license="Apache-2.0" \
+  org.label-schema.vendor="Amazon" \
+  org.label-schema.build-date="$BUILD_DATE"
+
+# CMD to run
+CMD ["./opensearch-dashboards-docker-entrypoint.sh"]

docker/dockerfiles/opensearch.al2.dockerfile

@@ -0,0 +1,97 @@
+# SPDX-License-Identifier: Apache-2.0
+# 
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+# 
+# Modifications Copyright OpenSearch Contributors. See
+# GitHub history for details.
+
+
+# Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License").
+# You may not use this file except in compliance with the License.
+# A copy of the License is located at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# or in the "license" file accompanying this file. This file is distributed 
+# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 
+# express or implied. See the License for the specific language governing 
+# permissions and limitations under the License.
+
+
+# This dockerfile generates an AmazonLinux-based image containing an OpenSearch installation.
+# It assumes that the working directory contains four files: an OpenSearch tarball (opensearch.tgz), log4j2.properties, opensearch.yml, opensearch-docker-entrypoint.sh, opensearch-onetime-setup.sh.
+# Build arguments:
+#   VERSION: Required. Used to label the image.
+#   BUILD_DATE: Required. Used to label the image. Should be in the form 'yyyy-mm-ddThh:mm:ssZ', i.e. a date-time from https://tools.ietf.org/html/rfc3339. The timestamp must be in UTC.
+#   UID: Optional. Specify the opensearch userid. Defaults to 1000.
+#   GID: Optional. Specify the opensearch groupid. Defaults to 1000.
+#   OPENSEARCH_HOME: Optional. Specify the opensearch root directory. Defaults to /usr/share/opensearch.
+#   PA_NAME: Optional. Specify the opensearch performance analyzer plugin folder name. Defaults to opensearch-performance-analyzer.
+
+
+FROM amazonlinux:2 AS linux_x64_staging
+
+ARG UID=1000
+ARG GID=1000
+ARG OPENSEARCH_HOME=/usr/share/opensearch
+ARG PA_NAME=opensearch-performance-analyzer
+
+# Update packages
+# Install the tools we need: tar and gzip to unpack the OpenSearch tarball, and shadow-utils to give us `groupadd` and `useradd`.
+RUN yum update -y && yum install -y tar gzip shadow-utils && yum clean all
+
+# Create an opensearch user, group, and directory
+RUN groupadd -g $GID opensearch && \
+    adduser -u $UID -g $GID -d $OPENSEARCH_HOME opensearch && \
+    mkdir /tmp/opensearch
+
+# Prepare working directory
+COPY opensearch.tgz /tmp/opensearch/opensearch.tgz
+RUN tar -xzf /tmp/opensearch/opensearch.tgz -C $OPENSEARCH_HOME --strip-components=1 && rm -rf /temp/opensearch
+COPY opensearch-docker-entrypoint.sh opensearch-onetime-setup.sh $OPENSEARCH_HOME/
+COPY log4j2.properties opensearch.yml $OPENSEARCH_HOME/config/
+COPY performance-analyzer.properties $OPENSEARCH_HOME/plugins/$PA_NAME/pa_config/
+
+# Copy working directory to the actual release docker images
+FROM amazonlinux:2
+
+ARG UID=1000
+ARG GID=1000
+ARG OPENSEARCH_HOME=/usr/share/opensearch
+ARG PA_NAME=opensearch-performance-analyzer
+
+COPY --from=linux_x64_staging $OPENSEARCH_HOME $OPENSEARCH_HOME
+
+# Setup OpenSearch
+WORKDIR $OPENSEARCH_HOME
+RUN yum update -y && yum install -y tar gzip shadow-utils && yum clean all
+RUN groupadd -g $GID opensearch && \
+    adduser -u $UID -g $GID -d $OPENSEARCH_HOME opensearch
+RUN ./opensearch-onetime-setup.sh && \
+    chown -R $UID:$GID $OPENSEARCH_HOME
+
+# Change user
+USER $UID
+
+# Expose ports for the opensearch service (9200 for HTTP and 9300 for internal transport) and performance analyzer (9600 for the agent and 9650 for the root cause analysis component)
+EXPOSE 9200 9300 9600 9650
+
+ARG VERSION
+ARG BUILD_DATE
+
+# Label
+LABEL org.label-schema.schema-version="1.0" \
+  org.label-schema.name="opensearch" \
+  org.label-schema.version="$VERSION" \
+  org.label-schema.url="https://opensearch.org" \
+  org.label-schema.vcs-url="https://github.com/OpenSearch" \
+  org.label-schema.license="Apache-2.0" \
+  org.label-schema.vendor="Amazon" \
+  org.label-schema.build-date="$BUILD_DATE"
+
+# CMD to run
+CMD ["./opensearch-docker-entrypoint.sh"]

docker/opensearch-config/log4j2.properties

@@ -0,0 +1,9 @@
+status = error
+
+appender.console.type = Console
+appender.console.name = console
+appender.console.layout.type = PatternLayout
+appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%n
+
+rootLogger.level = info
+rootLogger.appenderRef.console.ref = console