Skip to content

Commit

Permalink
ci: add native image builds for linux and macos (#690)
Browse files Browse the repository at this point in the history
* ci: add native-image builds and releases

* fixup! ci: add native-image builds and releases

* fixup! ci: add native-image builds and releases

* fixup! ci: add native-image builds and releases

* fixup! ci: add native-image builds and releases

* fixup! ci: add native-image builds and releases

* fixup! ci: add native-image builds and releases

* fixup! ci: add native-image builds and releases

* fixup! ci: add native-image builds and releases

* fixup! ci: add native-image builds and releases

* fixup! ci: add native-image builds and releases

* fixup! ci: add native-image builds and releases

* fixup! ci: add native-image builds and releases

* fixup! ci: add native-image builds and releases

* fixup! ci: add native-image builds and releases

* fixup! ci: add native-image builds and releases

* fixup! ci: add native-image builds and releases

* fixup! ci: add native-image builds and releases

* fixup! ci: add native-image builds and releases

* fixup! ci: add native-image builds and releases

* fixup! ci: add native-image builds and releases

* fixup! ci: add native-image builds and releases

* fixup! ci: add native-image builds and releases

* fixup! ci: add native-image builds and releases

* fixup! ci: add native-image builds and releases

* fixup! ci: add native-image builds and releases

* fixup! ci: add native-image builds and releases

* fixup! ci: add native-image builds and releases

* fixup! ci: add native-image builds and releases

* fixup! ci: add native-image builds and releases

* fixup! ci: add native-image builds and releases

* fixup! ci: add native-image builds and releases

* fixup! ci: add native-image builds and releases

* fixup! ci: add native-image builds and releases
  • Loading branch information
TimoKramer authored Sep 9, 2024
1 parent 099d8ba commit 7428d0a
Show file tree
Hide file tree
Showing 10 changed files with 338 additions and 100 deletions.
94 changes: 16 additions & 78 deletions .circleci/config.yml
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
version: 2.1

setup: true

orbs:
github-cli: circleci/github-cli@1.0
tools: replikativ/clj-tools@0
continuation: circleci/continuation@0.1.2

jobs:
setup:
Expand Down Expand Up @@ -65,46 +67,20 @@ jobs:
paths:
- /home/circleci/.m2
native-image:
machine:
image: default
resource_class: large
executor: tools/clojurecli
steps:
- attach_workspace:
at: /home/circleci
- run:
name: install graalvm
command: |
cd /home/circleci
wget https://github.com/graalvm/graalvm-ce-builds/releases/download/jdk-21.0.0/graalvm-community-jdk-21.0.0_linux-x64_bin.tar.gz
tar -xzf graalvm-community-jdk-21.0.0_linux-x64_bin.tar.gz
export PATH=/home/circleci/graalvm-community-openjdk-21+35.1/bin:$PATH
export JAVA_HOME=/home/circleci/graalvm-community-openjdk-21+35.1
- run:
name: install clojure
command: |
cd /home/circleci
curl -O https://download.clojure.org/install/linux-install-1.11.1.1165.sh
chmod +x linux-install-1.11.1.1165.sh
./linux-install-1.11.1.1165.sh --prefix /home/circleci/clojure
- run:
name: install babashka
command: |
cd /home/circleci
curl -sLO https://raw.githubusercontent.com/babashka/babashka/master/install
chmod +x install
./install --dir /home/circleci/bin
export PATH=/home/circleci/bin:$PATH
- run:
name: build native-image
name: Generate config
command: |
cd /home/circleci/replikativ
export PATH=/home/circleci/graalvm-community-openjdk-21+35.1/bin:/home/circleci/clojure/bin:$PATH
export JAVA_HOME=/home/circleci/graalvm-community-openjdk-21+35.1
bb ni-cli
- persist_to_workspace:
root: /home/circleci/
bb .circleci/scripts/gen_ci.clj > generated_config.yml
- continuation/continue:
configuration_path: generated_config.yml
- save_cache:
key: deps-{{ checksum "deps.edn" }}
paths:
- replikativ/dthk
- /home/circleci/.m2
persistent-set-test:
executor: tools/clojurecli
steps:
Expand Down Expand Up @@ -144,28 +120,6 @@ jobs:
key: deps-{{ checksum "deps.edn" }}
paths:
- /home/circleci/.m2
native-image-test:
executor: tools/clojurecli
steps:
- attach_workspace:
at: /home/circleci
- run:
name: Run native-image test
command: |
cd /home/circleci/replikativ
bb test native-image
no_output_timeout: 5m
bb-pod-test:
executor: tools/clojurecli
steps:
- attach_workspace:
at: /home/circleci
- run:
name: Run bb-pod test
command: |
cd /home/circleci/replikativ
bb test bb-pod
no_output_timeout: 5m
backward-compatibility-test:
executor: tools/clojurecli
steps:
Expand Down Expand Up @@ -218,20 +172,14 @@ jobs:
- run:
name: Deploy to clojars
command: bb clojars
release:
release-jar:
executor: tools/clojurecli
steps:
- attach_workspace:
at: /home/circleci
- run:
name: Release jar on GitHub
command: bb release jar
- run:
name: Release cli on GitHub
command: bb release native-cli
environment:
DTHK_PLATFORM: linux
DTHK_ARCH: x86_64

workflows:
build-test-and-deploy:
Expand All @@ -244,7 +192,9 @@ workflows:
requires:
- setup
- native-image:
context: dockerhub-deploy
context:
- dockerhub-deploy
- github-token
requires:
- setup
- format:
Expand All @@ -259,16 +209,6 @@ workflows:
context: dockerhub-deploy
requires:
- build
- native-image-test:
context: dockerhub-deploy
requires:
- build
- native-image
- bb-pod-test:
context: dockerhub-deploy
requires:
- build
- native-image
- backward-compatibility-test:
context: dockerhub-deploy
requires:
Expand All @@ -290,9 +230,8 @@ workflows:
- persistent-set-test
- hitchhiker-tree-test
- backward-compatibility-test
- native-image-test
- integration-test
- release:
- release-jar:
context:
- dockerhub-deploy
- github-token
Expand All @@ -304,5 +243,4 @@ workflows:
- persistent-set-test
- hitchhiker-tree-test
- backward-compatibility-test
- native-image-test
- integration-test
176 changes: 176 additions & 0 deletions .circleci/scripts/gen_ci.clj
Original file line number Diff line number Diff line change
@@ -0,0 +1,176 @@
(ns gen-ci
(:require
[babashka.tasks :as tasks]
[clj-yaml.core :as yaml]
[clojure.string :as str]
[flatland.ordered.map :refer [ordered-map]]))

(def graalvm-version "22.0.2")

(defn run
([cmd-name cmd]
(run cmd-name cmd nil))
([cmd-name cmd no-output-timeout]
(let [base {:run {:name cmd-name
:command cmd}}]
(if no-output-timeout
(assoc-in base [:run :no_output_timeout] no-output-timeout)
base))))

(defn make-graalvm-url [arch]
(let [myarch (case arch
"amd64" "x64"
"aarch64" "aarch64")]
(str "https://github.com/graalvm/graalvm-ce-builds/releases/download/jdk-"
graalvm-version
"/graalvm-community-jdk-"
graalvm-version
"_linux-"
myarch
"_bin.tar.gz")))

(defn build-native-image
[arch resource-class]
(let [cache-key (str arch "-deps-linux-{{ checksum \"deps.edn\" }}")
graalvm-url (make-graalvm-url arch)]
(ordered-map
:machine
{:image "ubuntu-2204:2023.10.1"
:resource_class resource-class}
:working_directory "/home/circleci/replikativ"
:environment {:GRAALVM_VERSION graalvm-version
:DTHK_PLATFORM "linux"
:DTHK_ARCH arch
:PATH "/bin:/home/circleci/graalvm/bin:/home/circleci/clojure/bin:/home/circleci/bin"
:JAVA_HOME "/home/circleci/graalvm/bin/java"}
:steps
[:checkout
{:restore_cache {:keys [cache-key]}}
(run "Install GraalVM"
(format "cd /home/circleci
/bin/wget -O graalvm.tar.gz %s
/bin/mkdir graalvm || true
/bin/tar -xzf graalvm.tar.gz --directory graalvm --strip-components 1
sudo update-alternatives --install /usr/bin/java java /home/circleci/graalvm/bin/java 0
sudo update-alternatives --install /usr/bin/javac javac /home/circleci/graalvm/bin/javac 0
sudo update-alternatives --set java /home/circleci/graalvm/bin/java
sudo update-alternatives --set javac /home/circleci/graalvm/bin/javac"
graalvm-url))
(run "Install Clojure"
"cd /home/circleci
/bin/curl -sLO https://download.clojure.org/install/linux-install-1.11.1.1165.sh
/bin/chmod +x linux-install-1.11.1.1165.sh
./linux-install-1.11.1.1165.sh --prefix /home/circleci/clojure")
(run "Install Babashka"
"cd /home/circleci
/bin/curl -sLO https://raw.githubusercontent.com/babashka/babashka/master/install
/bin/chmod +x install
./install --dir /home/circleci/bin")
(run "Build native image"
"cd /home/circleci/replikativ
bb ni-cli")
(run "Test native image"
"cd /home/circleci/replikativ
bb test native-image")
{:persist_to_workspace
{:root "/home/circleci/"
:paths ["replikativ/dthk"]}}
{:save_cache
{:paths ["~/.m2" "~/graalvm"]
:key cache-key}}])))

(defn release-native-image
[arch]
(let [cache-key (str arch "-deps-linux-{{ checksum \"deps.edn\" }}")]
(ordered-map
:executor "tools/clojurecli"
:working_directory "/home/circleci/replikativ"
:environment {:DTHK_PLATFORM "linux"
:DTHK_ARCH arch}
:steps
[:checkout
{:restore_cache {:keys [cache-key]}}
{:attach_workspace {:at "/home/circleci"}}
(run "Release native image"
"cd /home/circleci/replikativ
bb release native-image")
{:persist_to_workspace
{:root "/home/circleci/"
:paths ["replikativ/dthk"]}}
{:save_cache
{:paths ["~/.m2" "~/graalvm"]
:key cache-key}}])))

(defn make-config []
(ordered-map
:version 2.1
:orbs
{:tools "replikativ/clj-tools@0"}
:commands
{:setup-docker-buildx
{:steps
[{:run
{:name "Create multi-platform capable buildx builder"
:command
"docker run --privileged --rm tonistiigi/binfmt --install all\ndocker buildx create --name ci-builder --use"}}]}}
:jobs (ordered-map
:build-linux-amd64 (build-native-image "amd64" "large")
:build-linux-aarch64 (build-native-image "aarch64" "arm.large")
:release-linux-amd64 (release-native-image "amd64")
:release-linux-aarch64 (release-native-image "aarch64"))
:workflows (ordered-map
:version 2
:native-images
{:jobs ["build-linux-amd64"
"build-linux-aarch64"
{"release-linux-amd64"
{:context ["dockerhub-deploy"
"github-token"]
:filters {:branches {:only "main"}}
:requires ["build-linux-amd64"]}}
{"release-linux-aarch64"
{:context ["dockerhub-deploy"
"github-token"]
:filters {:branches {:only "main"}}
:requires ["build-linux-aarch64"]}}]})))

(def skip-config
{:skip-if-only [#"^doc\/.*"
#"^bb\/.*"
#"^dev\/.*"
#"^examples\/.*"
#"^test\/.*"
#".*.md$"]})

(defn get-changes
[]
(-> (tasks/shell {:out :string} "git diff --name-only HEAD~1")
(:out)
(str/split-lines)))

(defn irrelevant-change?
[change regexes]
(some? (some #(re-matches % change) regexes)))

(defn anything-relevant?
[change-set regexes]
(some? (some #(not (irrelevant-change? % regexes)) change-set)))

(defn main
[]
(let [{:keys [skip-if-only]} skip-config
changed-files (get-changes)]
(when (anything-relevant? changed-files skip-if-only)
(-> (make-config)
(yaml/generate-string :dumper-options {:flow-style :block})
println))))

(when (= *file* (System/getProperty "babashka.file"))
(main))

(comment
(def changed-files (get-changes))
(anything-relevant? changed-files (:skip-if-only skip-config))
(-> (make-config)
(yaml/generate-string :dumper-options {:flow-style :block})
println))
56 changes: 56 additions & 0 deletions .cirrus.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
macos_instance:
image: ghcr.io/cirruslabs/macos-monterey-base:latest

build-task:
env:
DTHK_PLATFORM: macos
DTHK_ARCH: aarch64
INSTALL_DIR: ${HOME}
GRAALVM_VERSION: 22.0.2
GRAALVM_SLUG: 9.1
GRAALVM_HOME: ${HOME}/graalvm-community-openjdk-${GRAALVM_VERSION}+${GRAALVM_SLUG}/Contents/Home/
GITHUB_TOKEN: ENCRYPTED[0ea782cbbd99c2486b0f55b8eefcf91606a0805faaa3f35e55ecb75268b0046b874c856753f528ba689a9cdaa11e47c2]
script: |
set -euo pipefail
# install babashka
curl -sLO https://raw.githubusercontent.com/babashka/babashka/master/install
chmod +x install
./install --dir ${HOME}/.local/bin
export PATH=${HOME}/.local/bin:$PATH
# install clojure
curl -L -O https://github.com/clojure/brew-install/releases/latest/download/posix-install.sh
chmod +x posix-install.sh
sudo ./posix-install.sh
# install graalvm
pushd "$INSTALL_DIR" >/dev/null
curl -L -O https://github.com/graalvm/graalvm-ce-builds/releases/download/jdk-${GRAALVM_VERSION}/graalvm-community-jdk-${GRAALVM_VERSION}_${DTHK_PLATFORM}-${DTHK_ARCH}_bin.tar.gz
sudo xattr -r -d com.apple.quarantine graalvm-community-jdk-${GRAALVM_VERSION}_${DTHK_PLATFORM}-${DTHK_ARCH}_bin.tar.gz
tar -xzf graalvm-community-jdk-${GRAALVM_VERSION}_${DTHK_PLATFORM}-${DTHK_ARCH}_bin.tar.gz
popd >/dev/null
# prepare
export PATH=${GRAALVM_HOME}/bin:${PATH}
export JAVA_HOME=${GRAALVM_HOME}
sudo /usr/sbin/softwareupdate --install-rosetta --agree-to-license
java -version
${GRAALVM_HOME}bin/native-image --version
echo $PATH
native-image --version
# compile
bb ni-cli
# test
file dthk
bb test native-image
bb test bb-pod
# upload artifact
if [[ $CIRRUS_RELEASE == "" ]]; then
bb release native-cli
fi
binaries_artifacts:
path: "dist/*"
Loading

0 comments on commit 7428d0a

Please sign in to comment.