From 7cf2fae772037e20d6687643dc6c902885648d4e Mon Sep 17 00:00:00 2001 From: krendelhoff2 Date: Tue, 8 Oct 2024 10:13:43 +0700 Subject: [PATCH] [TMP] --- .buildkite/pipeline-raw.yml | 177 +------------------------ docker/build/build-rust.sh | 2 +- docker/package/Dockerfile-ubuntu | 13 +- docker/package/package_generator.py | 1 - docker/package/scripts/build-binary.sh | 8 +- docker/vendor-rust-deps.py | 27 ++++ 6 files changed, 49 insertions(+), 179 deletions(-) create mode 100644 docker/vendor-rust-deps.py diff --git a/.buildkite/pipeline-raw.yml b/.buildkite/pipeline-raw.yml index 741eeddf8..74f0a8ac5 100644 --- a/.buildkite/pipeline-raw.yml +++ b/.buildkite/pipeline-raw.yml @@ -37,68 +37,10 @@ steps: - cd tests/bottle-hashes/ - ./test-hash-insert.sh - - label: build via nix - commands: - # NOTE: if this changes, remember to update the instructions in the nix/README.md too - - nix build . -o binaries - branches: "!master" - - label: check flake - commands: - - nix flake check -L - branches: "!master" - # nix builds are usually cached, so we don't care about only_changes setting for it - - - label: build-via-docker - # this step is used as a dependency, so we're defining 'key' explicitely - key: build-via-docker - commands: - - eval "$SET_VERSION" - - cd docker - - ./docker-static-build.sh - artifact_paths: - - ./docker/octez-* - only_changes: &static_binaries_changes_regexes - - docker/build/.*.sh - - docker/build/Dockerfile - - docker/docker-static-build.sh - - meta.json - - protocols.json - - - label: build-arm-via-docker - # this step is used as a dependency, so we're defining 'key' explicitely - key: build-arm-via-docker - commands: - - eval "$SET_VERSION" - - cd docker - - ./docker-static-build.sh - - > - for f in ./octez-*; do - mv "\$f" "\$f-arm64" - done - artifact_paths: - - ./docker/octez-* - agents: - queue: "arm64-darwin" - only_changes: *static_binaries_changes_regexes - - - label: test docker-built binaries - commands: - - buildkite-agent artifact download "docker/*" . --step "build-via-docker" - - chmod +x ./docker/* - - nix-build . -A binaries-test --no-out-link --arg path-to-binaries ./docker - branches: "!master" - depends_on: - - "build-via-docker" - only_changes: - - tests/tezos-binaries.nix - - tests/test_script.py - # files from nix/ are massively used in tests infrastructure - - nix/.* - - label: test deb source packages via docker commands: - eval "$SET_VERSION" - - nix develop .#docker-tezos-packages -c ./docker/build/ubuntu/build.py --type source + - nix develop .#docker-tezos-packages -c ./docker/build/ubuntu/build.py --type source -p tezos-baker-PtQenaB1 artifact_paths: - ./out/* branches: "!master" @@ -116,122 +58,9 @@ steps: - eval "$SET_VERSION" # Building all binary packages will take significant amount of time, so we build only one # in order to ensure package generation sanity - - nix develop .#docker-tezos-packages -c ./docker/build/ubuntu/build.py --type binary -p tezos-baker-PsParisC + - nix develop .#docker-tezos-packages -c ./docker/build/ubuntu/build.py --type binary -p tezos-baker-PtQenaB1 - rm -rf out # It takes much time to build binary package, so we do it only on master - branches: "master" - timeout_in_minutes: 240 - only_changes: *ubuntu_native_packaging_changes_regexes - - - label: test rpm source packages via docker - commands: - - eval "$SET_VERSION" - - nix develop .#docker-tezos-packages -c ./docker/build/fedora/build.py --type source - artifact_paths: - - ./out/* branches: "!master" - timeout_in_minutes: 60 - only_changes: &fedora_native_packaging_changes_regexes - - docker/package/.* - - docker/build/fedora/build.py - - docker/build/util/build.py - - docker/baking/.* - - meta.json - - protocols.json - - - label: test rpm binary packages via docker - commands: - - eval "$SET_VERSION" - # Building all binary packages will take significant amount of time, so we build only one - # in order to ensure package generation sanity - - nix develop .#docker-tezos-packages -c ./docker/build/fedora/build.py --type binary -p tezos-baker-PsParisC - - rm -rf out - # It takes much time to build binary package, so we do it only on master - branches: "master" - timeout_in_minutes: 180 - only_changes: *fedora_native_packaging_changes_regexes - - - label: build deb packages with static binaries - key: build-static-deb - depends_on: - - "build-via-docker" - commands: - - eval "$SET_VERSION" - - buildkite-agent artifact download "docker/octez-*" . --step build-via-docker - - nix develop .#docker-tezos-packages -c ./docker/build/ubuntu/build.py --type binary --binaries-dir docker --distributions focal - artifact_paths: - - ./out/* + timeout_in_minutes: 240 only_changes: *ubuntu_native_packaging_changes_regexes - - - label: test gen_systemd_service_file.py script - commands: - - eval "$SET_VERSION" - - nix develop .#autorelease -c ./gen_systemd_service_file.py tezos-node - branches: "!master" - only_changes: - - gen_systemd_service_file.py - - docker/package/.* - - - label: test bundled systemd services - # Used in .buildkite/hooks/pre-exit - key: test-systemd-services - depends_on: - - "build-static-deb" - only_changes: - - :<< *ubuntu_native_packaging_changes_regexes - - tests/systemd/.* - agents: - queue: "default" - commands: - - buildkite-agent artifact download "out/*~focal_amd64.deb" . --step build-static-deb - - cd tests/systemd - - nix shell ../..#legacyPackages.x86_64-linux.vagrant ../..#legacyPackages.x86_64-linux.curl -c vagrant --packages-directory=../../out up --provider=libvirt - retry: - automatic: - limit: 3 - - - label: create auto release/pre-release - key: auto-release - commands: - - mkdir binaries - - mkdir arm-binaries - - buildkite-agent artifact download "docker/*" binaries --step "build-via-docker" - - buildkite-agent artifact download "docker/*" arm-binaries --step "build-arm-via-docker" - - ls binaries - - nix develop .#autorelease -c ./scripts/autorelease.sh "$BUILDKITE_MESSAGE" - branches: master - depends_on: - - "build-via-docker" - - "build-arm-via-docker" - only_changes: - - scripts/autorelease.sh - - scripts/shell.nix - # files from 'nix/' directory are used in the autorelease script - - nix/.* - - tezos-release.nix - - release.nix - - # Update mirror repositories on master if brew formulae have been updated or bottle hashes added - - label: update stable mirror repository - if: | - build.branch == "master" && - ( build.message =~ /^Merge pull request .* from serokell\/auto\/v[0-9]+\.[0-9]+-release/ || - build.message =~ /^Merge pull request .* from serokell\/auto\/update-brew-formulae-v[.0-9]+-[0-9]+/ ) - depends_on: - - "auto-release" - env: - MIRROR_REPO: "git@github.com:serokell/tezos-packaging-stable.git" - commands: &update_mirror - - git pull origin "$BUILDKITE_BRANCH:$BUILDKITE_BRANCH" - - git push --mirror "$$MIRROR_REPO" - - - label: update RC mirror repository - if: | - build.branch == "master" && - (build.message =~ /^Merge pull request .* from serokell\/auto\/v[0-9]+\.[0-9]+-(rc|beta).*-release/ || - build.message =~ /^Merge pull request .* from serokell\/auto\/update-brew-formulae-v[.0-9]+-(rc|beta).*/) - depends_on: - - "auto-release" - env: - MIRROR_REPO: "git@github.com:serokell/tezos-packaging-rc.git" - commands: *update_mirror diff --git a/docker/build/build-rust.sh b/docker/build/build-rust.sh index b11224d4b..bf0c7feb9 100755 --- a/docker/build/build-rust.sh +++ b/docker/build/build-rust.sh @@ -7,4 +7,4 @@ set -euo pipefail wget https://sh.rustup.rs/rustup-init.sh chmod +x rustup-init.sh -./rustup-init.sh --profile minimal --default-toolchain 1.71.1 -y +./rustup-init.sh --profile minimal --default-toolchain 1.78.0 -y diff --git a/docker/package/Dockerfile-ubuntu b/docker/package/Dockerfile-ubuntu index 77ce4f488..4f5c41601 100644 --- a/docker/package/Dockerfile-ubuntu +++ b/docker/package/Dockerfile-ubuntu @@ -18,8 +18,7 @@ RUN if [ "$dist" = "focal" ]; then apt-get install -y dh-systemd; fi RUN install -m 0755 /usr/bin/python3 /usr/bin/builder -RUN if [ "$dist" != "noble"]; then add-apt-repository ppa:ubuntu-mozilla-security/rust-next -y && apt-get update -y; fi -RUN apt-get -y install cargo opam +RUN apt-get -y install opam ENV USER dockerbuilder RUN useradd dockerbuilder @@ -31,6 +30,16 @@ WORKDIR /tezos-packaging/docker/sources COPY docker/fetch_tezos_sources.py . RUN python3 fetch_tezos_sources.py +#COPY docker/build/build-rust.sh . +#RUN bash build-rust.sh +#RUN cp -r ${HOME}/.cargo .cargo + +RUN apt-get -y install cargo-1.78 + +COPY docker/vendor-rust-deps.py . + +RUN python3 vendor-rust-deps.py + WORKDIR /tezos-packaging/docker COPY meta.json /tezos-packaging/meta.json COPY protocols.json /tezos-packaging/protocols.json diff --git a/docker/package/package_generator.py b/docker/package/package_generator.py index 5b658d213..4dde248cc 100755 --- a/docker/package/package_generator.py +++ b/docker/package/package_generator.py @@ -112,7 +112,6 @@ def get_build_deps(binaries_dir): "unzip", "rsync", "gcc", - "cargo", "opam", "git", "autoconf", diff --git a/docker/package/scripts/build-binary.sh b/docker/package/scripts/build-binary.sh index 6279e828e..61f57c65f 100755 --- a/docker/package/scripts/build-binary.sh +++ b/docker/package/scripts/build-binary.sh @@ -19,7 +19,9 @@ cd tezos echo "******* Creating switch *******" -. scripts/version.sh +ocaml_version='' + +source scripts/version.sh opam switch create . --repositories=tezos "ocaml-base-compiler.$ocaml_version" --no-install @@ -27,8 +29,12 @@ export OPAMSWITCH="$PWD" opam repository remove default > /dev/null 2>&1 eval "$(opam env)" + OPAMASSUMEDEPEXTS=true opam install conf-rust conf-rust-2021 +# FIXME another dirty hack +sed -i 's|cargo|cargo-1.78|' src/rust_deps/build.sh + export CFLAGS="-fPIC ${CFLAGS:-}" OPAMASSUMEDEPEXTS=true opam install opam/virtual/octez-deps.opam.locked --deps-only --criteria="-notuptodate,-changed,-removed" diff --git a/docker/vendor-rust-deps.py b/docker/vendor-rust-deps.py new file mode 100644 index 000000000..aec772737 --- /dev/null +++ b/docker/vendor-rust-deps.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2024 Oxhead Alpha +# SPDX-License-Identifier: LicenseRef-MIT-OA + +import subprocess +import os + +os.chdir("tezos/src/rust_deps") +subprocess.run(["cargo-1.78", "vendor"], check=True) + +with open(".cargo/config.toml", "a") as f: + f.write( + """ +[source.crates-io] +replace-with = "vendored-sources" + +[source.vendored-sources] +directory = "vendor" + """ + ) + +# FIXME dirty hack to make dune copy vendor directory to the build directory +with open("dune", "r") as f: + dune = f.read() + +with open("dune", "w") as f: + f.write(dune.replace("(source_tree .cargo)", "(source_tree .cargo) (source_tree vendor)"))