Skip to content

Commit

Permalink
Move turbo outside of build for docker swc builds (#54035)
Browse files Browse the repository at this point in the history 
removes usage of turbo and uses the default node version inside the
image.
  • Loading branch information
@ijjk
ijjk authored Aug 15, 2023
1 parent cd95fe4 commit bdc3a8e
Show file tree
Hide file tree
Showing 2 changed files with 68 additions and 29 deletions.
65 changes: 36 additions & 29 deletions .github/workflows/build_and_deploy.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -150,17 +150,14 @@ jobs:
set -e &&
apt update &&
apt install -y pkg-config xz-utils &&
wget https://nodejs.org/dist/v18.12.0/node-v18.12.0-linux-x64.tar.xz -O node.tar.xz &&
tar -xf node.tar.xz &&
cd node-* && cp -r ./{bin,include,lib,share} /usr/local/ && cd .. &&
rustup toolchain install "${RUST_TOOLCHAIN}" &&
rustup default "${RUST_TOOLCHAIN}" &&
rustup target add x86_64-unknown-linux-gnu &&
npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}" && corepack enable && pnpm -v && node -v &&
npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" &&
unset CC_x86_64_unknown_linux_gnu && unset CC &&
turbo run build-native-release --remote-cache-timeout 90 --summarize -- --target x86_64-unknown-linux-gnu &&
strip packages/next-swc/native/next-swc.*.node &&
objdump -T packages/next-swc/native/next-swc.*.node | grep GLIBC_
cd packages/next-swc && npm run build-native-release -- --target x86_64-unknown-linux-gnu &&
strip native/next-swc.*.node &&
objdump -T native/next-swc.*.node | grep GLIBC_
- host:
- 'self-hosted'
Expand All @@ -172,16 +169,14 @@ jobs:
docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:stable-2022-10-24-alpine
build: >-
set -e &&
rm -rfv /usr/local/bin/{node,nodejs,npm,npx,corepack} &&
apk update && apk upgrade &&
apk add --no-cache libc6-compat pkgconfig nodejs-current npm &&
rm -rfv /usr/local/bin/{node,nodejs,npm,npx} &&
apk update &&
apk add --no-cache libc6-compat pkgconfig &&
rustup toolchain install "${RUST_TOOLCHAIN}" &&
rustup default "${RUST_TOOLCHAIN}" &&
rustup target add x86_64-unknown-linux-musl &&
npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}" && corepack enable && pnpm -v && node -v &&
turbo run build-native-release --remote-cache-timeout 90 --summarize -- --target x86_64-unknown-linux-musl &&
strip packages/next-swc/native/next-swc.*.node
npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" &&
cd packages/next-swc && npm run build-native-release -- --target x86_64-unknown-linux-musl &&
strip native/next-swc.*.node
- host:
- 'self-hosted'
Expand All @@ -195,18 +190,15 @@ jobs:
set -e &&
apt update &&
apt install -y pkg-config xz-utils &&
wget https://nodejs.org/dist/v18.12.0/node-v18.12.0-linux-x64.tar.xz -O node.tar.xz &&
tar -xf node.tar.xz &&
cd node-* && cp -r ./{bin,include,lib,share} /usr/local/ && cd .. &&
export JEMALLOC_SYS_WITH_LG_PAGE=16 &&
rustup toolchain install "${RUST_TOOLCHAIN}" &&
rustup default "${RUST_TOOLCHAIN}" &&
rustup target add aarch64-unknown-linux-gnu &&
npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}" && corepack enable &&
npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" &&
export CC_aarch64_unknown_linux_gnu=/usr/aarch64-unknown-linux-gnu/bin/aarch64-unknown-linux-gnu-gcc &&
turbo run build-native-release --remote-cache-timeout 90 --summarize -- --target aarch64-unknown-linux-gnu --features plugin,rustls-tls,tracing/release_max_level_info &&
llvm-strip -x packages/next-swc/native/next-swc.*.node &&
objdump -T packages/next-swc/native/next-swc.*.node | grep GLIBC_
cd packages/next-swc && npm run build-native-release -- --target aarch64-unknown-linux-gnu --features plugin,rustls-tls,tracing/release_max_level_info &&
llvm-strip -x native/next-swc.*.node &&
objdump -T native/next-swc.*.node | grep GLIBC_
- host:
- 'self-hosted'
Expand All @@ -218,16 +210,15 @@ jobs:
docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:stable-2022-10-24-alpine
build: >-
set -e &&
rm -rfv /usr/local/bin/{node,nodejs,npm,npx,corepack} &&
apk update && apk upgrade &&
apk add --no-cache libc6-compat pkgconfig nodejs-current npm &&
apk update &&
apk add --no-cache libc6-compat pkgconfig &&
export JEMALLOC_SYS_WITH_LG_PAGE=16 &&
npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}" && corepack enable &&
npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" &&
rustup toolchain install "${RUST_TOOLCHAIN}" &&
rustup default "${RUST_TOOLCHAIN}" &&
rustup target add aarch64-unknown-linux-musl &&
turbo run build-native-release --remote-cache-timeout 90 --summarize -- --target aarch64-unknown-linux-musl &&
llvm-strip -x packages/next-swc/native/next-swc.*.node
cd packages/next-swc && npm run build-native-release -- --target aarch64-unknown-linux-musl &&
llvm-strip -x native/next-swc.*.node
name: stable - ${{ matrix.settings.target }} - node@16
runs-on: ${{ matrix.settings.host }}
Expand Down Expand Up @@ -269,17 +260,33 @@ jobs:
run: ${{ matrix.settings.setup }}
if: ${{ matrix.settings.setup }}

- name: Build in docker
# we only need custom caching for docker builds
# as they are on an older Node.js version and have
# issues with turbo caching
- name: pull build cache
if: ${{ matrix.settings.docker }}
run: node ./scripts/pull-turbo-cache.js ${{ matrix.settings.target }}

- name: check build exists
if: ${{ matrix.settings.docker }}
run: if [ -f packages/next-swc/native/next-swc.*.node ]; then echo "BUILD_EXISTS=yes" >> $GITHUB_OUTPUT; else echo "BUILD_EXISTS=no" >> $GITHUB_OUTPUT; fi
id: build-exists

- name: Build in docker
if: ${{ matrix.settings.docker && steps.build-exists.outputs.BUILD_EXISTS == 'no' }}
run: docker run -v "/var/run/docker.sock":"/var/run/docker.sock" -e RUST_TOOLCHAIN -e RUST_BACKTRACE -e NAPI_CLI_VERSION -e CARGO_TERM_COLOR -e CARGO_INCREMENTAL -e CARGO_PROFILE_RELEASE_LTO -e CARGO_REGISTRIES_CRATES_IO_PROTOCOL -e TURBO_API -e TURBO_TEAM -e TURBO_TOKEN -e TURBO_VERSION -e TURBO_REMOTE_ONLY -v ${{ env.HOME }}/.cargo/git:/root/.cargo/git -v ${{ env.HOME }}/.cargo/registry:/root/.cargo/registry -v ${{ github.workspace }}:/build -w /build --entrypoint=bash ${{ matrix.settings.docker }} -c "${{ matrix.settings.build }}"

- name: cache build
if: ${{ matrix.settings.docker && steps.build-exists.outputs.BUILD_EXISTS == 'no' }}
run: pnpm turbo run cache-build-native --force -- ${{ matrix.settings.target }}

- name: 'Build'
run: ${{ matrix.settings.build }}
if: ${{ !matrix.settings.docker }}

- name: 'check build cache status'
id: check-did-build
run: if [[ ! -z $(ls .turbo/runs) ]]; then echo "DID_BUILD=yup" >> $GITHUB_OUTPUT; fi
run: if [[ ! -z $(ls packages/next-swc/native) ]]; then echo "DID_BUILD=yup" >> $GITHUB_OUTPUT; fi

# Trying to upload metrics for the Turbopack to datadog's CI pipeline execution
- name: 'Collect turbopack build metrics'
Expand Down
32 changes: 32 additions & 0 deletions scripts/pull-turbo-cache.js
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
#!/usr/bin/env node
// @ts-check

const { execSync } = require('child_process')

;(async function () {
const target = process.argv[process.argv.length - 1]

const turboResult = execSync(
`pnpm turbo run cache-build-native --dry=json -- ${target}`
).toString()

const turboData = JSON.parse(turboResult)

const task = turboData.tasks.find((t) => t.command !== '<NONEXISTENT>')

if (!task) {
console.warn(`Failed to find related turbo task`, turboResult)
return
}

// pull cache if it was available
if (task.cache.local || task.cache.remote) {
const pullResult = execSync(
`pnpm turbo run cache-build-native -- ${target}`
).toString()
console.log(pullResult)
} else {
console.warn(`No turbo cache was available, continuing...`)
console.warn(task)
}
})()

0 comments on commit bdc3a8e

Please sign in to comment.