Merge pull request #693 from Altinity/24.8/cicd-fix/get_docker_from_secrets

24.8 Use credentials from secrets

Author: Enmk

.github/workflows/regression.yml

@@ -88,9 +88,9 @@ name: Regression test workflow - Release
 env:
   # Force the stdout and stderr streams to be unbuffered
   PYTHONUNBUFFERED: 1
-  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_REPORT_KEY_ID }}
-  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_REPORT_SECRET_ACCESS_KEY }}
-  AWS_DEFAULT_REGION: ${{ secrets.AWS_REPORT_REGION }}
+  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+  AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
   DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
   DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
   args: --test-to-end

.github/workflows/release_branches.yml

@@ -7,6 +7,11 @@ env:
   AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
   AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
   AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+  CLICKHOUSE_TEST_STAT_LOGIN: ${{ secrets.CLICKHOUSE_TEST_STAT_LOGIN }}
+  CLICKHOUSE_TEST_STAT_PASSWORD: ${{ secrets.CLICKHOUSE_TEST_STAT_PASSWORD }}
+  CLICKHOUSE_TEST_STAT_URL: ${{ secrets.CLICKHOUSE_TEST_STAT_URL }}
+  DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+  ROBOT_TOKEN: ${{ secrets.ROBOT_TOKEN }}
 
 on: # yamllint disable-line rule:truthy
   pull_request:
@@ -16,7 +21,7 @@ on: # yamllint disable-line rule:truthy
       - opened
     branches:
       # Anything/24.3 (e.g customizations/24.3.x)
-      - '**/24.3*'
+      - '**24.3*'
   release:
     types:
       - published

.github/workflows/reusable_build.yml

@@ -8,6 +8,11 @@ env:
   AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
   AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
   AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+  CLICKHOUSE_TEST_STAT_LOGIN: ${{ secrets.CLICKHOUSE_TEST_STAT_LOGIN }}
+  CLICKHOUSE_TEST_STAT_PASSWORD: ${{ secrets.CLICKHOUSE_TEST_STAT_PASSWORD }}
+  CLICKHOUSE_TEST_STAT_URL: ${{ secrets.CLICKHOUSE_TEST_STAT_URL }}
+  DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+  ROBOT_TOKEN: ${{ secrets.ROBOT_TOKEN }}
 
 name: Build ClickHouse
 'on':

.github/workflows/reusable_sign.yml

@@ -1,7 +1,4 @@
-### For the pure soul wishes to move it to another place
-# https://github.com/orgs/community/discussions/9050
-
-name: Testing workflow
+name: Sigining workflow
 'on':
   workflow_call:
     inputs:
@@ -63,6 +60,11 @@ env:
   AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
   AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
   AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+  CLICKHOUSE_TEST_STAT_LOGIN: ${{ secrets.CLICKHOUSE_TEST_STAT_LOGIN }}
+  CLICKHOUSE_TEST_STAT_PASSWORD: ${{ secrets.CLICKHOUSE_TEST_STAT_PASSWORD }}
+  CLICKHOUSE_TEST_STAT_URL: ${{ secrets.CLICKHOUSE_TEST_STAT_URL }}
+  DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+  ROBOT_TOKEN: ${{ secrets.ROBOT_TOKEN }}
 
 jobs:
   runner_labels_setup:

.github/workflows/reusable_test.yml

@@ -44,10 +44,25 @@ name: Testing workflow
         description: if given, it's passed to the environments
         required: false
       AWS_SECRET_ACCESS_KEY:
-        description: the access key to the aws param store.
+        description: the access key to the aws s3 bucket.
         required: true
       AWS_ACCESS_KEY_ID:
-        description: the access key id to the aws param store.
+        description: the access key id to the aws s3 bucket.
+        required: true
+      CLICKHOUSE_TEST_STAT_LOGIN:
+        description: username for ci db.
+        required: true
+      CLICKHOUSE_TEST_STAT_PASSWORD: 
+        description: password for ci db.
+        required: true
+      CLICKHOUSE_TEST_STAT_URL: 
+        description: url for ci db.
+        required: true
+      DOCKER_PASSWORD: 
+        description: token to upload docker images.
+        required: true
+      ROBOT_TOKEN:
+        description: token to update ci status.
         required: true
 
 env:
@@ -57,6 +72,11 @@ env:
   AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
   AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
   AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+  CLICKHOUSE_TEST_STAT_LOGIN: ${{ secrets.CLICKHOUSE_TEST_STAT_LOGIN }}
+  CLICKHOUSE_TEST_STAT_PASSWORD: ${{ secrets.CLICKHOUSE_TEST_STAT_PASSWORD }}
+  CLICKHOUSE_TEST_STAT_URL: ${{ secrets.CLICKHOUSE_TEST_STAT_URL }}
+  DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+  ROBOT_TOKEN: ${{ secrets.ROBOT_TOKEN }}
 
 jobs:
   runner_labels_setup:

docker/packager/packager

@@ -6,12 +6,13 @@ import os
 import subprocess
 import sys
 from pathlib import Path
-from typing import List, Optional
+from typing import Dict, List, Optional
 
 SCRIPT_PATH = Path(__file__).absolute()
 IMAGE_TYPE = "binary-builder"
 IMAGE_NAME = f"altinityinfra/{IMAGE_TYPE}"
-
+DEFAULT_TMP_PATH = SCRIPT_PATH.parent.absolute() / 'tmp'
+TEMP_PATH = Path(os.getenv("TEMP_PATH", DEFAULT_TMP_PATH))
 
 class BuildException(Exception):
     pass
@@ -82,9 +83,22 @@ def run_docker_image_with_env(
     ch_root: Path,
     cargo_cache_dir: Path,
     ccache_dir: Optional[Path],
+    aws_secrets : Optional[Dict[str,str]]
 ) -> None:
     output_dir.mkdir(parents=True, exist_ok=True)
     cargo_cache_dir.mkdir(parents=True, exist_ok=True)
+    extra_parts = ""
+
+    if aws_secrets:
+        # Pass AWS credentials via file rather than via env to avoid leaking secrets
+        env_part = {"AWS_CONFIG_FILE": "/home/clickhouse/.aws/credentials"}
+        host_aws_config_file_path = Path(TEMP_PATH) / 'aws_config'
+        with open(host_aws_config_file_path, 'wt') as f:
+            f.write("[default]")
+            for key, value in aws_secrets.items():
+                f.write(f"\n{key}={value}")
+
+        extra_parts = f"--volume={host_aws_config_file_path}:{env_part['AWS_CONFIG_FILE']}"
 
     env_part = " -e ".join(env_variables)
     if env_part:
@@ -107,6 +121,7 @@ def run_docker_image_with_env(
     cmd = (
         f"docker run --network=host --user={user} --rm {ccache_mount} "
         f"--volume={output_dir}:/output --volume={ch_root}:/build {env_part} "
+        f" {extra_parts} "
         f"--volume={cargo_cache_dir}:/rust/cargo/registry {interactive} {image_name}"
     )
 
@@ -130,11 +145,9 @@ def parse_env_variables(
     sanitizer: str,
     package_type: str,
     cache: str,
-    s3_access_key_id: str,
     s3_bucket: str,
     s3_directory: str,
     s3_rw_access: bool,
-    s3_secret_access_key: str,
     clang_tidy: bool,
     version: str,
     official: bool,
@@ -319,10 +332,6 @@ def parse_env_variables(
         result.append(f"SCCACHE_S3_KEY_PREFIX={sccache_dir}")
         if not s3_rw_access:
             result.append("SCCACHE_S3_NO_CREDENTIALS=true")
-        if s3_access_key_id:
-            result.append(f"AWS_ACCESS_KEY_ID={s3_access_key_id}")
-        if s3_secret_access_key:
-            result.append(f"AWS_SECRET_ACCESS_KEY={s3_secret_access_key}")
 
     if clang_tidy:
         # `CTCACHE_DIR` has the same purpose as the `CCACHE_DIR` above.
@@ -539,11 +548,9 @@ def main() -> None:
         args.sanitizer,
         args.package_type,
         args.cache,
-        args.s3_access_key_id,
         args.s3_bucket,
         args.s3_directory,
         args.s3_rw_access,
-        args.s3_secret_access_key,
         args.clang_tidy,
         args.version,
         args.official,
@@ -562,6 +569,10 @@ def main() -> None:
         ch_root,
         args.cargo_cache_dir,
         args.ccache_dir,
+        {
+            "aws_access_key_id" : args.s3_access_key_id,
+            "aws_secret_access_key" : args.s3_secret_access_key
+        }
     )
     logging.info("Output placed into %s", args.output_dir)
 

tests/ci/clickhouse_helper.py

@@ -8,7 +8,7 @@
 
 import requests
 
-from env_helper import GITHUB_REPOSITORY
+from env_helper import GITHUB_REPOSITORY, CLICKHOUSE_TEST_STAT_URL, CLICKHOUSE_TEST_STAT_PASSWORD, CLICKHOUSE_TEST_STAT_LOGIN
 from get_robot_token import get_parameter_from_ssm
 from pr_info import PRInfo
 from report import TestResults
@@ -27,12 +27,12 @@ def __init__(
         self, url: Optional[str] = None, auth: Optional[Dict[str, str]] = None
     ):
         if url is None:
-            url = get_parameter_from_ssm("clickhouse-test-stat-url")
+            url = CLICKHOUSE_TEST_STAT_URL
 
         self.url = url
         self.auth = auth or {
-            "X-ClickHouse-User": get_parameter_from_ssm("clickhouse-test-stat-login"),
-            "X-ClickHouse-Key": get_parameter_from_ssm("clickhouse-test-stat-password"),
+            "X-ClickHouse-User": CLICKHOUSE_TEST_STAT_LOGIN,
+            "X-ClickHouse-Key": CLICKHOUSE_TEST_STAT_PASSWORD,
         }
 
     @staticmethod

tests/ci/docker_images_helper.py

@@ -7,8 +7,7 @@
 from pathlib import Path
 from typing import Any, Dict, List, Optional
 
-from env_helper import ROOT_DIR, DOCKER_TAG
-from get_robot_token import get_parameter_from_ssm
+from env_helper import ROOT_DIR, DOCKER_TAG, DOCKER_PASSWORD
 
 IMAGES_FILE_PATH = Path("docker/images.json")
 
@@ -28,7 +27,7 @@ def docker_login(relogin: bool = True) -> None:
         logging.info('Doing docker login')
         subprocess.check_output(  # pylint: disable=unexpected-keyword-arg
             "docker login --username 'altinityinfra' --password-stdin",
-            input=get_parameter_from_ssm("dockerhub-password"),
+            input=DOCKER_PASSWORD,
             encoding="utf-8",
             shell=True,
         )

tests/ci/env_helper.py

@@ -38,6 +38,12 @@
     "{pr_or_release}/{commit}/{build_name}/{artifact}"
 )
 
+CLICKHOUSE_TEST_STAT_LOGIN = os.getenv("CLICKHOUSE_TEST_STAT_LOGIN")
+CLICKHOUSE_TEST_STAT_PASSWORD = os.getenv("CLICKHOUSE_TEST_STAT_PASSWORD")
+CLICKHOUSE_TEST_STAT_URL = os.getenv("CLICKHOUSE_TEST_STAT_URL")
+DOCKER_PASSWORD = os.getenv("DOCKER_PASSWORD")
+ROBOT_TOKEN = os.getenv("ROBOT_TOKEN")
+
 # These parameters are set only on demand, and only once
 _GITHUB_JOB_ID = ""
 _GITHUB_JOB_URL = ""

tests/ci/get_robot_token.py

@@ -9,6 +9,7 @@
 from github.GithubException import BadCredentialsException
 from github.NamedUser import NamedUser
 
+from env_helper import ROBOT_TOKEN
 
 @dataclass
 class Token:
@@ -52,20 +53,11 @@ def get_parameters_from_ssm(
 
     return results
 
-
-ROBOT_TOKEN = None  # type: Optional[Token]
-
 # NOTE(Arthur Passos): Original CI code uses the "_original" version of this method. Each robot token is rate limited
 # and the original implementation selects the "best one". To make it simpler and iterate faster,
 # we are using only one robot and keeping the method signature. In the future we might reconsider
 # having multiple robot tokens
-def get_best_robot_token(token_prefix_env_name="github_robot_token"):
-    # Re-use already fetched token (same as in get_best_robot_token_original)
-    # except here we assume it is always a string (since we use only one token and don't do token rotation)
-    global ROBOT_TOKEN
-    if ROBOT_TOKEN is not None:
-        return ROBOT_TOKEN
-    ROBOT_TOKEN = get_parameter_from_ssm(token_prefix_env_name)
+def get_best_robot_token():
     return ROBOT_TOKEN
 
 def get_best_robot_token_original(tokens_path: str = "/github-tokens") -> str: