Merge pull request ClickHouse#38580 from ClickHouse/artifactory-arm64

Push arm64 packages to artifactory and release assets

.github/workflows/release.yml

@@ -21,6 +21,9 @@ jobs:
         EOF
     - name: Check out repository code
       uses: actions/checkout@v2
+      with:
+        # Always use the most recent script version
+        ref: master
     - name: Download packages and push to Artifactory
       run: |
         rm -rf "$TEMP_PATH" && mkdir -p "$TEMP_PATH"

docs/_includes/install/tgz.sh

@@ -1,20 +1,34 @@
 LATEST_VERSION=$(curl -s https://packages.clickhouse.com/tgz/stable/ | \
     grep -Eo '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | sort -V -r | head -n 1)
 export LATEST_VERSION
-curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-common-static-$LATEST_VERSION.tgz"
-curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-common-static-dbg-$LATEST_VERSION.tgz"
-curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-server-$LATEST_VERSION.tgz"
-curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-client-$LATEST_VERSION.tgz"
 
-tar -xzvf "clickhouse-common-static-$LATEST_VERSION.tgz"
+case $(uname -m) in
+  x86_64) ARCH=amd64 ;;
+  aarch64) ARCH=arm64 ;;
+  *) echo "Unknown architecture $(uname -m)"; exit 1 ;;
+esac
+
+for PKG in clickhouse-common-static clickhouse-common-static-dbg clickhouse-server clickhouse-client
+do
+  curl -fO "https://packages.clickhouse.com/tgz/stable/$PKG-$LATEST_VERSION-${ARCH}.tgz" \
+    || curl -fO "https://packages.clickhouse.com/tgz/stable/$PKG-$LATEST_VERSION.tgz"
+done
+
+exit 0
+
+tar -xzvf "clickhouse-common-static-$LATEST_VERSION-${ARCH}.tgz" \
+  || tar -xzvf "clickhouse-common-static-$LATEST_VERSION.tgz"
 sudo "clickhouse-common-static-$LATEST_VERSION/install/doinst.sh"
 
-tar -xzvf "clickhouse-common-static-dbg-$LATEST_VERSION.tgz"
+tar -xzvf "clickhouse-common-static-dbg-$LATEST_VERSION-${ARCH}.tgz" \
+  || tar -xzvf "clickhouse-common-static-dbg-$LATEST_VERSION.tgz"
 sudo "clickhouse-common-static-dbg-$LATEST_VERSION/install/doinst.sh"
 
-tar -xzvf "clickhouse-server-$LATEST_VERSION.tgz"
+tar -xzvf "clickhouse-server-$LATEST_VERSION-${ARCH}.tgz" \
+  || tar -xzvf "clickhouse-server-$LATEST_VERSION.tgz"
 sudo "clickhouse-server-$LATEST_VERSION/install/doinst.sh"
 sudo /etc/init.d/clickhouse-server start
 
-tar -xzvf "clickhouse-client-$LATEST_VERSION.tgz"
+tar -xzvf "clickhouse-client-$LATEST_VERSION-${ARCH}.tgz" \
+  || tar -xzvf "clickhouse-client-$LATEST_VERSION.tgz"
 sudo "clickhouse-client-$LATEST_VERSION/install/doinst.sh"

docs/en/development/continuous-integration.md

@@ -123,12 +123,10 @@ Builds ClickHouse in various configurations for use in further steps. You have t
 - **Build log**: link to the building and files copying log, useful when build failed.
 - **Build time**.
 - **Artifacts**: build result files (with `XXX` being the server version e.g. `20.8.1.4344`).
-  - `clickhouse-client_XXX_all.deb`
+  - `clickhouse-client_XXX_amd64.deb`
   - `clickhouse-common-static-dbg_XXX[+asan, +msan, +ubsan, +tsan]_amd64.deb`
   - `clickhouse-common-staticXXX_amd64.deb`
-  - `clickhouse-server_XXX_all.deb`
-  - `clickhouse_XXX_amd64.buildinfo`
-  - `clickhouse_XXX_amd64.changes`
+  - `clickhouse-server_XXX_amd64.deb`
   - `clickhouse`: Main built binary.
   - `clickhouse-odbc-bridge`
   - `unit_tests_dbms`: GoogleTest binary with ClickHouse unit tests.

docs/en/getting-started/install.md

@@ -127,22 +127,36 @@ After that downloaded archives should be unpacked and installed with installatio
 LATEST_VERSION=$(curl -s https://packages.clickhouse.com/tgz/stable/ | \
     grep -Eo '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | sort -V -r | head -n 1)
 export LATEST_VERSION
-curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-common-static-$LATEST_VERSION-amd64.tgz"
-curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-common-static-dbg-$LATEST_VERSION-amd64.tgz"
-curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-server-$LATEST_VERSION-amd64.tgz"
-curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-client-$LATEST_VERSION-amd64.tgz"
 
-tar -xzvf "clickhouse-common-static-$LATEST_VERSION-amd64.tgz"
+case $(uname -m) in
+  x86_64) ARCH=amd64 ;;
+  aarch64) ARCH=arm64 ;;
+  *) echo "Unknown architecture $(uname -m)"; exit 1 ;;
+esac
+
+for PKG in clickhouse-common-static clickhouse-common-static-dbg clickhouse-server clickhouse-client
+do
+  curl -fO "https://packages.clickhouse.com/tgz/stable/$PKG-$LATEST_VERSION-${ARCH}.tgz" \
+    || curl -fO "https://packages.clickhouse.com/tgz/stable/$PKG-$LATEST_VERSION.tgz"
+done
+
+exit 0
+
+tar -xzvf "clickhouse-common-static-$LATEST_VERSION-${ARCH}.tgz" \
+  || tar -xzvf "clickhouse-common-static-$LATEST_VERSION.tgz"
 sudo "clickhouse-common-static-$LATEST_VERSION/install/doinst.sh"
 
-tar -xzvf "clickhouse-common-static-dbg-$LATEST_VERSION-amd64.tgz"
+tar -xzvf "clickhouse-common-static-dbg-$LATEST_VERSION-${ARCH}.tgz" \
+  || tar -xzvf "clickhouse-common-static-dbg-$LATEST_VERSION.tgz"
 sudo "clickhouse-common-static-dbg-$LATEST_VERSION/install/doinst.sh"
 
-tar -xzvf "clickhouse-server-$LATEST_VERSION-amd64.tgz"
+tar -xzvf "clickhouse-server-$LATEST_VERSION-${ARCH}.tgz" \
+  || tar -xzvf "clickhouse-server-$LATEST_VERSION.tgz"
 sudo "clickhouse-server-$LATEST_VERSION/install/doinst.sh"
 sudo /etc/init.d/clickhouse-server start
 
-tar -xzvf "clickhouse-client-$LATEST_VERSION-amd64.tgz"
+tar -xzvf "clickhouse-client-$LATEST_VERSION-${ARCH}.tgz" \
+  || tar -xzvf "clickhouse-client-$LATEST_VERSION.tgz"
 sudo "clickhouse-client-$LATEST_VERSION/install/doinst.sh"
 ```
 

docs/ru/getting-started/install.md

@@ -124,22 +124,36 @@ sudo yum install clickhouse-server clickhouse-client
 LATEST_VERSION=$(curl -s https://packages.clickhouse.com/tgz/stable/ | \
     grep -Eo '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | sort -V -r | head -n 1)
 export LATEST_VERSION
-curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-common-static-$LATEST_VERSION.tgz"
-curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-common-static-dbg-$LATEST_VERSION.tgz"
-curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-server-$LATEST_VERSION.tgz"
-curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-client-$LATEST_VERSION.tgz"
 
-tar -xzvf "clickhouse-common-static-$LATEST_VERSION.tgz"
+case $(uname -m) in
+  x86_64) ARCH=amd64 ;;
+  aarch64) ARCH=arm64 ;;
+  *) echo "Unknown architecture $(uname -m)"; exit 1 ;;
+esac
+
+for PKG in clickhouse-common-static clickhouse-common-static-dbg clickhouse-server clickhouse-client
+do
+  curl -fO "https://packages.clickhouse.com/tgz/stable/$PKG-$LATEST_VERSION-${ARCH}.tgz" \
+    || curl -fO "https://packages.clickhouse.com/tgz/stable/$PKG-$LATEST_VERSION.tgz"
+done
+
+exit 0
+
+tar -xzvf "clickhouse-common-static-$LATEST_VERSION-${ARCH}.tgz" \
+  || tar -xzvf "clickhouse-common-static-$LATEST_VERSION.tgz"
 sudo "clickhouse-common-static-$LATEST_VERSION/install/doinst.sh"
 
-tar -xzvf "clickhouse-common-static-dbg-$LATEST_VERSION.tgz"
+tar -xzvf "clickhouse-common-static-dbg-$LATEST_VERSION-${ARCH}.tgz" \
+  || tar -xzvf "clickhouse-common-static-dbg-$LATEST_VERSION.tgz"
 sudo "clickhouse-common-static-dbg-$LATEST_VERSION/install/doinst.sh"
 
-tar -xzvf "clickhouse-server-$LATEST_VERSION.tgz"
+tar -xzvf "clickhouse-server-$LATEST_VERSION-${ARCH}.tgz" \
+  || tar -xzvf "clickhouse-server-$LATEST_VERSION.tgz"
 sudo "clickhouse-server-$LATEST_VERSION/install/doinst.sh"
 sudo /etc/init.d/clickhouse-server start
 
-tar -xzvf "clickhouse-client-$LATEST_VERSION.tgz"
+tar -xzvf "clickhouse-client-$LATEST_VERSION-${ARCH}.tgz" \
+  || tar -xzvf "clickhouse-client-$LATEST_VERSION.tgz"
 sudo "clickhouse-client-$LATEST_VERSION/install/doinst.sh"
 ```
 

docs/zh/development/continuous-integration.md

@@ -86,13 +86,10 @@ git push
 - **Build log**: 链接到构建和文件复制日志, 当构建失败时很有用.
 - **Build time**.
 - **Artifacts**: 构建结果文件 (`XXX`是服务器版本, 比如`20.8.1.4344`).
-    - `clickhouse-client_XXX_all.deb`
+    - `clickhouse-client_XXX_amd64.deb`
     -` clickhouse-common-static-dbg_XXX[+asan, +msan, +ubsan, +tsan]_amd64.deb`
     - `clickhouse-common-staticXXX_amd64.deb`
-    - `clickhouse-server_XXX_all.deb`
-    - `clickhouse-test_XXX_all.deb`
-    - `clickhouse_XXX_amd64.buildinfo`
-    - `clickhouse_XXX_amd64.changes`
+    - `clickhouse-server_XXX_amd64.deb`
     - `clickhouse`: Main built binary.
     - `clickhouse-odbc-bridge`
     - `unit_tests_dbms`: 带有 ClickHouse 单元测试的 GoogleTest 二进制文件.

docs/zh/getting-started/install.md

@@ -121,22 +121,36 @@ sudo yum install clickhouse-server clickhouse-client
 LATEST_VERSION=$(curl -s https://packages.clickhouse.com/tgz/stable/ | \
     grep -Eo '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | sort -V -r | head -n 1)
 export LATEST_VERSION
-curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-common-static-$LATEST_VERSION.tgz"
-curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-common-static-dbg-$LATEST_VERSION.tgz"
-curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-server-$LATEST_VERSION.tgz"
-curl -O "https://packages.clickhouse.com/tgz/stable/clickhouse-client-$LATEST_VERSION.tgz"
 
-tar -xzvf "clickhouse-common-static-$LATEST_VERSION.tgz"
+case $(uname -m) in
+  x86_64) ARCH=amd64 ;;
+  aarch64) ARCH=arm64 ;;
+  *) echo "Unknown architecture $(uname -m)"; exit 1 ;;
+esac
+
+for PKG in clickhouse-common-static clickhouse-common-static-dbg clickhouse-server clickhouse-client
+do
+  curl -fO "https://packages.clickhouse.com/tgz/stable/$PKG-$LATEST_VERSION-${ARCH}.tgz" \
+    || curl -fO "https://packages.clickhouse.com/tgz/stable/$PKG-$LATEST_VERSION.tgz"
+done
+
+exit 0
+
+tar -xzvf "clickhouse-common-static-$LATEST_VERSION-${ARCH}.tgz" \
+  || tar -xzvf "clickhouse-common-static-$LATEST_VERSION.tgz"
 sudo "clickhouse-common-static-$LATEST_VERSION/install/doinst.sh"
 
-tar -xzvf "clickhouse-common-static-dbg-$LATEST_VERSION.tgz"
+tar -xzvf "clickhouse-common-static-dbg-$LATEST_VERSION-${ARCH}.tgz" \
+  || tar -xzvf "clickhouse-common-static-dbg-$LATEST_VERSION.tgz"
 sudo "clickhouse-common-static-dbg-$LATEST_VERSION/install/doinst.sh"
 
-tar -xzvf "clickhouse-server-$LATEST_VERSION.tgz"
+tar -xzvf "clickhouse-server-$LATEST_VERSION-${ARCH}.tgz" \
+  || tar -xzvf "clickhouse-server-$LATEST_VERSION.tgz"
 sudo "clickhouse-server-$LATEST_VERSION/install/doinst.sh"
 sudo /etc/init.d/clickhouse-server start
 
-tar -xzvf "clickhouse-client-$LATEST_VERSION.tgz"
+tar -xzvf "clickhouse-client-$LATEST_VERSION-${ARCH}.tgz" \
+  || tar -xzvf "clickhouse-client-$LATEST_VERSION.tgz"
 sudo "clickhouse-client-$LATEST_VERSION/install/doinst.sh"
 ```
 

packages/clickhouse-client.yaml

@@ -1,7 +1,7 @@
 # package sources should be placed in ${PWD}/root
 # nfpm should run from the same directory with a config
 name: "clickhouse-client"
-arch: "all"
+arch: "${DEB_ARCH}" # amd64, arm64
 platform: "linux"
 version: "${CLICKHOUSE_VERSION_STRING}"
 vendor: "ClickHouse Inc."

packages/clickhouse-server.yaml

@@ -1,7 +1,7 @@
 # package sources should be placed in ${PWD}/root
 # nfpm should run from the same directory with a config
 name: "clickhouse-server"
-arch: "all"
+arch: "${DEB_ARCH}" # amd64, arm64
 platform: "linux"
 version: "${CLICKHOUSE_VERSION_STRING}"
 vendor: "ClickHouse Inc."

tests/ci/build_download_helper.py

@@ -20,15 +20,17 @@ def get_with_retries(
     sleep: int = 3,
     **kwargs,
 ) -> requests.Response:
-    logging.info("Getting URL with %i and sleep %i in between: %s", retries, sleep, url)
+    logging.info(
+        "Getting URL with %i tries and sleep %i in between: %s", retries, sleep, url
+    )
     exc = None  # type: Optional[Exception]
-    for i in range(DOWNLOAD_RETRIES_COUNT):
+    for i in range(retries):
         try:
             response = requests.get(url, **kwargs)
             response.raise_for_status()
             break
         except Exception as e:
-            if i + 1 < DOWNLOAD_RETRIES_COUNT:
+            if i + 1 < retries:
                 logging.info("Exception '%s' while getting, retry %i", e, i + 1)
                 time.sleep(sleep)
 

tests/ci/download_previous_release.py

@@ -4,25 +4,24 @@
 import os
 import logging
 
-import requests
+import requests  # type: ignore
 
-from requests.adapters import HTTPAdapter
-from urllib3.util.retry import Retry
+from requests.adapters import HTTPAdapter  # type: ignore
+from urllib3.util.retry import Retry  # type: ignore
 
 CLICKHOUSE_TAGS_URL = "https://api.github.com/repos/ClickHouse/ClickHouse/tags"
 
-CLICKHOUSE_COMMON_STATIC_DOWNLOAD_URL = "https://github.com/ClickHouse/ClickHouse/releases/download/v{version}-{type}/clickhouse-common-static_{version}_amd64.deb"
-CLICKHOUSE_COMMON_STATIC_DBG_DOWNLOAD_URL = "https://github.com/ClickHouse/ClickHouse/releases/download/v{version}-{type}/clickhouse-common-static-dbg_{version}_amd64.deb"
-CLICKHOUSE_SERVER_DOWNLOAD_URL = "https://github.com/ClickHouse/ClickHouse/releases/download/v{version}-{type}/clickhouse-server_{version}_all.deb"
-CLICKHOUSE_CLIENT_DOWNLOAD_URL = "https://github.com/ClickHouse/ClickHouse/releases/download/v{version}-{type}/clickhouse-client_{version}_all.deb"
-
-
+DOWNLOAD_PREFIX = (
+    "https://github.com/ClickHouse/ClickHouse/releases/download/v{version}-{type}/"
+)
 CLICKHOUSE_COMMON_STATIC_PACKET_NAME = "clickhouse-common-static_{version}_amd64.deb"
 CLICKHOUSE_COMMON_STATIC_DBG_PACKET_NAME = (
     "clickhouse-common-static-dbg_{version}_amd64.deb"
 )
-CLICKHOUSE_SERVER_PACKET_NAME = "clickhouse-server_{version}_all.deb"
-CLICKHOUSE_CLIENT_PACKET_NAME = "clickhouse-client_{version}_all.deb"
+CLICKHOUSE_SERVER_PACKET_NAME = "clickhouse-server_{version}_amd64.deb"
+CLICKHOUSE_SERVER_PACKET_FALLBACK = "clickhouse-server_{version}_all.deb"
+CLICKHOUSE_CLIENT_PACKET_NAME = "clickhouse-client_{version}_amd64.deb"
+CLICKHOUSE_CLIENT_PACKET_FALLBACK = "clickhouse-client_{version}_all.deb"
 
 PACKETS_DIR = "previous_release_package_folder/"
 VERSION_PATTERN = r"((?:\d+\.)?(?:\d+\.)?(?:\d+\.)?\d+-[a-zA-Z]*)"
@@ -102,9 +101,10 @@ def download_packet(url, out_path, retries=10, backoff_factor=0.3):
     session.mount("http://", adapter)
     session.mount("https://", adapter)
     response = session.get(url)
-    print(url)
-    if response.ok:
-        open(out_path, "wb").write(response.content)
+    response.raise_for_status()
+    print(f"Download {url} to {out_path}")
+    with open(out_path, "wb") as fd:
+        fd.write(response.content)
 
 
 def download_packets(release, dest_path=PACKETS_DIR):
@@ -113,43 +113,31 @@ def download_packets(release, dest_path=PACKETS_DIR):
 
     logging.info("Will download %s", release)
 
-    download_packet(
-        CLICKHOUSE_COMMON_STATIC_DOWNLOAD_URL.format(
-            version=release.version, type=release.type
-        ),
-        out_path=os.path.join(
-            dest_path,
-            CLICKHOUSE_COMMON_STATIC_PACKET_NAME.format(version=release.version),
-        ),
-    )
-
-    download_packet(
-        CLICKHOUSE_COMMON_STATIC_DBG_DOWNLOAD_URL.format(
-            version=release.version, type=release.type
-        ),
-        out_path=os.path.join(
-            dest_path,
-            CLICKHOUSE_COMMON_STATIC_DBG_PACKET_NAME.format(version=release.version),
-        ),
-    )
-
-    download_packet(
-        CLICKHOUSE_SERVER_DOWNLOAD_URL.format(
-            version=release.version, type=release.type
-        ),
-        out_path=os.path.join(
-            dest_path, CLICKHOUSE_SERVER_PACKET_NAME.format(version=release.version)
-        ),
-    )
-
-    download_packet(
-        CLICKHOUSE_CLIENT_DOWNLOAD_URL.format(
-            version=release.version, type=release.type
-        ),
-        out_path=os.path.join(
-            dest_path, CLICKHOUSE_CLIENT_PACKET_NAME.format(version=release.version)
-        ),
-    )
+    def get_dest_path(pkg_name):
+        return os.path.join(dest_path, pkg_name)
+
+    for pkg in (
+        CLICKHOUSE_COMMON_STATIC_PACKET_NAME,
+        CLICKHOUSE_COMMON_STATIC_DBG_PACKET_NAME,
+    ):
+        url = (DOWNLOAD_PREFIX + pkg).format(version=release.version, type=release.type)
+        pkg_name = get_dest_path(pkg.format(version=release.version))
+        download_packet(url, pkg_name)
+
+    for pkg, fallback in (
+        (CLICKHOUSE_SERVER_PACKET_NAME, CLICKHOUSE_SERVER_PACKET_FALLBACK),
+        (CLICKHOUSE_CLIENT_PACKET_NAME, CLICKHOUSE_CLIENT_PACKET_FALLBACK),
+    ):
+        url = (DOWNLOAD_PREFIX + pkg).format(version=release.version, type=release.type)
+        pkg_name = get_dest_path(pkg.format(version=release.version))
+        try:
+            download_packet(url, pkg_name)
+        except Exception:
+            url = (DOWNLOAD_PREFIX + fallback).format(
+                version=release.version, type=release.type
+            )
+            pkg_name = get_dest_path(fallback.format(version=release.version))
+            download_packet(url, pkg_name)
 
 
 def download_previous_release(dest_path):