Creating dg-9 by @gurevichdmitry #738
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Create Environment | |
| run-name: Creating ${{ github.event.inputs.deployment_name }} by @${{ github.actor }} | |
| on: | |
| # Ability to execute on demand | |
| workflow_dispatch: | |
| inputs: | |
| deployment_name: | |
| type: string | |
| description: | | |
| Name with letters, numbers, hyphens; start with a letter. Max 20 chars. e.g., 'my-env-123' | |
| required: true | |
| serverless_mode: | |
| description: "Deploy a serverless project instead of an ESS deployment" | |
| type: boolean | |
| required: true | |
| default: false | |
| elk-stack-version: | |
| required: true | |
| description: "Stack version: For released version use 8.x.y, for BC use version with hash 8.x.y-hash, for SNAPSHOT use 8.x.y-SNAPSHOT" | |
| default: "8.13.3" | |
| type: string | |
| ess-region: | |
| required: true | |
| description: "Elastic Cloud deployment region" | |
| default: "gcp-us-west2" | |
| type: string | |
| docker-image-override: | |
| required: false | |
| description: "Provide the full Docker image path to override the default image (e.g. for testing BC/SNAPSHOT)" | |
| type: string | |
| run-sanity-tests: | |
| description: "Run sanity tests after provision" | |
| default: false | |
| type: boolean | |
| run-ui-sanity-tests: | |
| description: "Run UI sanity tests after provision" | |
| default: false | |
| type: boolean | |
| kibana_ref: | |
| description: "Kibana branch, tag, or commit SHA to check out the UI sanity tests from" | |
| required: false | |
| default: "main" | |
| type: string | |
| expiration_days: | |
| description: "Number of days until environment expiration" | |
| required: false | |
| default: 5 | |
| type: string | |
| ec-api-key: | |
| type: string | |
| description: "**Optional** By default, the environment will be created in our Cloud Security Organization. If you want to use your own cloud account, enter your Elastic Cloud API key." | |
| required: false | |
| workflow_call: | |
| inputs: | |
| deployment_name: | |
| description: Name of the deployment to create | |
| type: string | |
| required: true | |
| serverless_mode: | |
| description: "Deploy a serverless project instead of an ESS deployment" | |
| type: boolean | |
| required: true | |
| default: false | |
| elk-stack-version: | |
| required: true | |
| description: "Stack version: For released version use 8.x.y, for BC use version with hash 8.x.y-hash, for SNAPSHOT use 8.x.y-SNAPSHOT" | |
| default: "8.13.3" | |
| type: string | |
| ess-region: | |
| required: true | |
| description: "Elastic Cloud deployment region" | |
| default: "gcp-us-west2" | |
| type: string | |
| docker-image-override: | |
| required: false | |
| description: "Provide the full Docker image path to override the default image (e.g. for testing BC/SNAPSHOT)" | |
| type: string | |
| run-sanity-tests: | |
| description: "Run sanity tests after provision" | |
| default: false | |
| type: boolean | |
| run-ui-sanity-tests: | |
| description: "Run UI sanity tests after provision" | |
| default: false | |
| type: boolean | |
| kibana_ref: | |
| description: "Kibana branch, tag, or commit SHA to check out the UI sanity tests from" | |
| required: false | |
| default: "main" | |
| type: string | |
| expiration_days: | |
| description: "Number of days until environment expiration" | |
| required: false | |
| default: 5 | |
| type: string | |
| ec-api-key: | |
| type: string | |
| description: "**Optional** By default, the environment will be created in our Cloud Security Organization. If you want to use your own cloud account, enter your Elastic Cloud API key." | |
| required: false | |
| infra-type: | |
| description: "Type of infrastructure to create" | |
| type: string | |
| required: false | |
| default: "cis" | |
| outputs: | |
| s3-bucket: | |
| description: "Terraform state s3 bucket folder" | |
| value: ${{ jobs.Deploy.outputs.deploy-s3-bucket }} | |
| cnvm-stack-name: | |
| description: "AWS CNVM integration stack name" | |
| value: ${{ jobs.Deploy.outputs.aws-cnvm-stack-name }} | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_REGION: "eu-west-1" | |
| WORKING_DIR: deploy/test-environments | |
| INTEGRATIONS_SETUP_DIR: tests/integrations_setup | |
| AWS_DEFAULT_TAGS: "Key=division,Value=engineering Key=org,Value=security Key=team,Value=cloud-security-posture Key=project,Value=test-environments" | |
| GCP_ZONE: "us-central1-a" | |
| AZURE_DEFAULT_TAGS: "division=engineering org=security team=cloud-security-posture project=test-environments owner=${{ github.actor }}" | |
| TF_VAR_ec_api_key: ${{ secrets.EC_API_KEY }} | |
| TF_VAR_gcp_service_account_json: ${{ secrets.GCP_AGENT_CREDENTIALS }} | |
| jobs: | |
| Deploy: | |
| runs-on: ubuntu-20.04 | |
| timeout-minutes: 120 | |
| defaults: | |
| run: | |
| working-directory: ${{ env.WORKING_DIR }} | |
| env: | |
| TF_VAR_ess_region: ${{ inputs.ess-region }} | |
| DEPLOYMENT_NAME: ${{ inputs.deployment_name }} | |
| TF_VAR_serverless_mode: ${{ inputs.serverless_mode }} | |
| TEST_AGENTLESS: ${{ inputs.serverless_mode }} | |
| S3_BASE_BUCKET: "s3://tf-state-bucket-test-infra" | |
| S3_BUCKET_URL: "https://s3.console.aws.amazon.com/s3/buckets/tf-state-bucket-test-infra" | |
| DOCKER_IMAGE_OVERRIDE: ${{ inputs.docker-image-override }} | |
| CNVM_STACK_NAME: "${{ inputs.deployment_name }}-cnvm-sanity-test-stack" | |
| # Add "id-token" with the intended permissions. | |
| permissions: | |
| contents: 'read' | |
| id-token: 'write' | |
| outputs: | |
| deploy-s3-bucket: ${{ steps.upload-state.outputs.s3-bucket-folder }} | |
| aws-cnvm-stack-name: ${{ steps.upload-state.outputs.aws-cnvm-stack }} | |
| steps: | |
| - name: Check out the repo | |
| uses: actions/checkout@v4 | |
| - name: Init Hermit | |
| run: ./bin/hermit env -r >> $GITHUB_ENV | |
| working-directory: ./ | |
| - name: Check Deployment Name | |
| run: | | |
| deployment_name="${{ inputs.deployment_name }}" | |
| # Check length | |
| if [ ${#deployment_name} -gt 20 ]; then | |
| echo "error: Deployment name is too long (max 20 characters)" | |
| exit 1 | |
| fi | |
| # Check pattern required for cloud deployment | |
| if ! [[ $deployment_name =~ ^[a-z][-a-z0-9]*$ ]]; then | |
| echo "error: Deployment name doesn't match the required pattern [a-z][-a-z0-9]*" | |
| exit 1 | |
| fi | |
| - name: Mask Sensitive Data | |
| if: inputs.ec-api-key != '' | |
| run: | | |
| ec_api_key=$(jq -r '.inputs["ec-api-key"]' $GITHUB_EVENT_PATH) | |
| echo "::add-mask::$ec_api_key" | |
| echo "TF_VAR_ec_api_key=$ec_api_key" >> $GITHUB_ENV | |
| - name: Process Stack Version | |
| id: remove-commit-hash | |
| run: | | |
| # Extract the stack version | |
| stack_version="${{ inputs.elk-stack-version }}" | |
| echo "TF_VAR_stack_version=$stack_version" >> $GITHUB_ENV | |
| echo "STACK_VERSION=$stack_version" >> $GITHUB_ENV | |
| # Handle BC versions with commit hash (e.g. 8.11.0-1234567890) | |
| if [[ $stack_version =~ -[a-f0-9]+ ]]; then | |
| cleaned_version=$(echo $stack_version | awk -F"-" '{print $1}') | |
| # Versions with commit hash are not allowed for EC regular deployments and should be modified | |
| # EC module resource: | |
| # ec_deployment.deployment.version is required attribute and should be in format 8.x.y | 8.x.y-SNAPSHOT | |
| # Therefore, we need to modify the version in the env variable | |
| echo "TF_VAR_stack_version=$cleaned_version" >> $GITHUB_ENV | |
| # env variable STACK_VERSION is used in sanity tests for findings validation | |
| # findings are saved with version without commit hash | |
| # therefore, we need to modify the version in the env variable | |
| echo "STACK_VERSION=$cleaned_version" >> $GITHUB_ENV | |
| # TF_VAR_pin_version is used to override stack docker images | |
| # for BC versions with commit hash | |
| # This version will be used to override the docker images | |
| # elasticsearch.config.docker_image | |
| # kibana.config.docker_image | |
| # integrations_server.config.docker_image | |
| echo "TF_VAR_pin_version=$stack_version" >> $GITHUB_ENV | |
| fi | |
| - name: Init Enrollment Token | |
| run: | | |
| enrollment_token="init" | |
| echo "::add-mask::$enrollment_token" | |
| echo "ENROLLMENT_TOKEN=$enrollment_token" >> $GITHUB_ENV | |
| - name: Init Infra Type | |
| id: init-infra-type | |
| env: | |
| INPUT_INFRA_TYPE: ${{ inputs.infra-type }} | |
| run: | | |
| if [[ -z "${INPUT_INFRA_TYPE}" ]]; then | |
| echo "INFRA_TYPE=cis" >> $GITHUB_ENV | |
| else | |
| echo "INFRA_TYPE=$INPUT_INFRA_TYPE" >> $GITHUB_ENV | |
| fi | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.9' | |
| - name: Install Poetry | |
| run: | | |
| curl -sSL https://install.python-poetry.org | python3 - | |
| poetry --version | |
| - name: Install Fleet & Tests Dependencies | |
| id: fleet-and-tests-deps | |
| working-directory: ./tests | |
| run: | | |
| poetry install | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v4.0.1 | |
| with: | |
| aws-access-key-id: ${{ env.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ env.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: ${{ env.AWS_REGION }} | |
| - id: azure-auth | |
| name: Azure login | |
| uses: azure/login@v2 | |
| with: | |
| creds: ${{ secrets.AZURE_CREDENTIALS }} | |
| - id: google-auth | |
| name: Authenticate to Google Cloud | |
| uses: google-github-actions/auth@v2 | |
| with: | |
| workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }} | |
| service_account: ${{ secrets.GCP_SERVICE_ACCOUNT }} | |
| - name: Update Vars | |
| run: | | |
| echo "TF_VAR_gcp_project_id=$GCP_PROJECT" >> $GITHUB_ENV | |
| echo "TF_STATE_FOLDER=$(date +'%Y-%m-%d_%H-%M-%S')" >> $GITHUB_ENV | |
| - name: Deploy ELK Cloud Stack | |
| id: elk-stack | |
| uses: ./.github/actions/elk-stack | |
| with: | |
| deployment-name: ${{ env.DEPLOYMENT_NAME }} | |
| serverless-mode: ${{ env.TEST_AGENTLESS }} | |
| elk-stack-version: ${{ env.STACK_VERSION }} | |
| ess-region: ${{ env.TF_VAR_ess_region }} | |
| ec-api-key: ${{ env.TF_VAR_ec_api_key }} | |
| docker-image-version-override: ${{ env.TF_VAR_pin_version }} | |
| env-s3-bucket: "${{ env.S3_BASE_BUCKET }}/${{ env.DEPLOYMENT_NAME }}_${{ env.TF_STATE_FOLDER }}" | |
| tag-project: ${{ github.actor }} | |
| tag-owner: ${{ github.actor }} | |
| - name: Upload environment info | |
| id: upload-state | |
| if: always() | |
| env: | |
| S3_BUCKET: "${{ env.S3_BASE_BUCKET }}/${{ env.DEPLOYMENT_NAME }}_${{ env.TF_STATE_FOLDER }}" | |
| EXPIRATION_DAYS: ${{ inputs.expiration_days }} | |
| run: | | |
| echo "s3-bucket-folder=${S3_BUCKET}" >> $GITHUB_OUTPUT | |
| echo "aws-cnvm-stack=${CNVM_STACK_NAME}" >> $GITHUB_OUTPUT | |
| python3 ../../.ci/scripts/create_env_config.py | |
| aws s3 cp "./env_config.json" "${S3_BUCKET}/env_config.json" | |
| - name: Update Stack Vars | |
| env: | |
| STACK_ES_USER: ${{ steps.elk-stack.outputs.es-user }} | |
| STACK_ES_PASSWORD: ${{ steps.elk-stack.outputs.es-password }} | |
| STACK_KIBANA_URL: ${{ steps.elk-stack.outputs.kibana-url }} | |
| STACK_ES_URL: ${{ steps.elk-stack.outputs.es-url }} | |
| run: | | |
| echo "ES_USER=$STACK_ES_USER" >> $GITHUB_ENV | |
| echo "ES_PASSWORD=$STACK_ES_PASSWORD" >> $GITHUB_ENV | |
| echo "KIBANA_URL=$STACK_KIBANA_URL" >> $GITHUB_ENV | |
| echo "ES_URL=$STACK_ES_URL" >> $GITHUB_ENV | |
| - name: Summary | |
| if: success() | |
| run: | | |
| summary="Kibana URL: $KIBANA_URL" | |
| bucket_name="$S3_BASE_BUCKET" | |
| bucket_name="${bucket_name#s3://}" | |
| s3_bucket_link="[creds and keys](https://s3.console.aws.amazon.com/s3/buckets/$bucket_name)" | |
| summary=$(cat <<-EOF | |
| Kibana URL: [kibana]($KIBANA_URL) | |
| Environment Details: $s3_bucket_link | |
| EOF | |
| ) | |
| echo "$summary" >> $GITHUB_STEP_SUMMARY | |
| echo "$summary" # Print the summary to the workflow log | |
| - name: Deploy CDR Integrations | |
| id: cdr-integrations | |
| if: ${{ !cancelled() && steps.elk-stack.outcome == 'success' && env.INFRA_TYPE != 'cis' }} | |
| uses: ./.github/actions/cdr | |
| with: | |
| deployment-name: ${{ env.DEPLOYMENT_NAME }} | |
| aws-region: ${{ env.AWS_REGION }} | |
| gcp-project-id: ${{ env.GCP_PROJECT }} | |
| gcp-service-account-json: ${{ secrets.GCP_AGENT_CREDENTIALS }} | |
| aws-cloudtrail-s3-bucket: ${{ secrets.CLOUDTRAIL_S3 }} | |
| azure-eventhub-connection-string: ${{ secrets.AZURE_EVENTHUB_CONNECTION_STRING }} | |
| azure-storage-account-key: ${{ secrets.AZURE_STORAGE_ACCOUNT_KEY }} | |
| env-s3-bucket: "${{ env.S3_BASE_BUCKET }}/${{ env.DEPLOYMENT_NAME }}_${{ env.TF_STATE_FOLDER }}" | |
| es-user: ${{ steps.elk-stack.outputs.es-user }} | |
| es-password: ${{ steps.elk-stack.outputs.es-password }} | |
| kibana-url: ${{ steps.elk-stack.outputs.kibana-url }} | |
| elk-stack-version: ${{ env.STACK_VERSION }} | |
| azure-tags: ${{ env.AZURE_DEFAULT_TAGS }} | |
| tag-project: ${{ github.actor }} | |
| tag-owner: ${{ github.actor }} | |
| - name: Deploy CIS Integrations | |
| id: cis-integrations | |
| if: ${{ !cancelled() && steps.elk-stack.outcome == 'success' && env.INFRA_TYPE != 'cdr' }} | |
| uses: ./.github/actions/cis | |
| with: | |
| deployment-name: ${{ env.DEPLOYMENT_NAME }} | |
| cnvm-stack-name: ${{ env.CNVM_STACK_NAME }} | |
| cspm-gcp-zone: ${{ env.GCP_ZONE }} | |
| cspm-azure-creds: ${{ secrets.AZURE_CREDENTIALS }} | |
| cspm-azure-tags: ${{ env.AZURE_DEFAULT_TAGS }} | |
| stack-enrollment-token: ${{ env.ENROLLMENT_TOKEN }} | |
| env-s3-bucket: "${{ env.S3_BASE_BUCKET }}/${{ env.DEPLOYMENT_NAME }}_${{ env.TF_STATE_FOLDER }}" | |
| es-user: ${{ steps.elk-stack.outputs.es-user }} | |
| es-password: ${{ steps.elk-stack.outputs.es-password }} | |
| kibana-url: ${{ steps.elk-stack.outputs.kibana-url }} | |
| test-agentless: ${{ env.TEST_AGENTLESS }} | |
| tag-project: ${{ github.actor }} | |
| tag-owner: ${{ github.actor }} | |
| - name: Wait for agents to enroll | |
| id: wait-for-agents | |
| working-directory: ${{ env.INTEGRATIONS_SETUP_DIR }} | |
| run: | | |
| poetry run python ./agents_enrolled.py | |
| - name: Run Sanity checks | |
| if: ${{ success() && inputs.run-sanity-tests == true && env.INFRA_TYPE != 'cdr' }} | |
| working-directory: ./tests | |
| run: | | |
| poetry run pytest -m "sanity" --alluredir=./allure/results/ --clean-alluredir --maxfail=4 | |
| - name: Run UI Sanity checks (Kibana) | |
| uses: ./.github/actions/kibana-ftr | |
| if: ${{ success() && inputs.run-ui-sanity-tests == true && env.INFRA_TYPE != 'cdr' }} | |
| with: | |
| test_kibana_url: ${{ steps.elk-stack.outputs.test-kibana-url }} | |
| test_es_url: ${{ steps.elk-stack.outputs.test-es-url }} | |
| es_version: ${{ env.STACK_VERSION }} | |
| kibana_ref: ${{ inputs.kibana_ref }} | |
| - name: Create Slack Payload | |
| if: always() | |
| id: prepare-slack-data | |
| working-directory: ./ | |
| env: | |
| WORKFLOW: "${{ github.workflow }}" | |
| RUN_URL: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" | |
| GITHUB_ACTOR: "${{ github.actor }}" | |
| ESS_TYPE: ${{ inputs.serverless_mode }} | |
| JOB_STATUS: "${{ job.status }}" | |
| S3_BUCKET: "${{ env.S3_BUCKET_URL }}?region=${{ env.AWS_REGION }}&prefix=${{ env.DEPLOYMENT_NAME }}_${{ env.TF_STATE_FOLDER }}/" | |
| run: | | |
| python3 ./.ci/scripts/prepare_slack_data.py | |
| - name: Send Slack Notification | |
| uses: ./.github/actions/slack-notification | |
| if: always() | |
| continue-on-error: true | |
| with: | |
| vault-url: ${{ secrets.VAULT_ADDR }} | |
| vault-role-id: ${{ secrets.CSP_VAULT_ROLE_ID }} | |
| vault-secret-id: ${{ secrets.CSP_VAULT_SECRET_ID }} | |
| slack-payload: ${{ steps.prepare-slack-data.outputs.payload }} |