From 513bace2d66b88f751f86715ed7f3c176aed731e Mon Sep 17 00:00:00 2001 From: Daniel Lockyer Date: Sat, 23 Dec 2023 14:24:13 +0100 Subject: [PATCH] Replaced `@mapbox/node-pre-gyp` in favor of `prebuild` + `prebuild-install` fixes https://github.com/TryGhost/node-sqlite3/issues/1641 fixes https://github.com/TryGhost/node-sqlite3/issues/1721 fixes https://github.com/TryGhost/node-sqlite3/issues/1714 fixes https://github.com/TryGhost/node-sqlite3/issues/1713 fixes https://github.com/TryGhost/node-sqlite3/issues/1700 fixes https://github.com/TryGhost/node-sqlite3/issues/1704 - `@mapbox/node-pre-gyp` is effectively unmaintained [1] as has a few bugs which our users keep running into - it seems the prebuilt binary world has moved in favor of prebuild + it's various other forms - one option would be to use prebuildify to bundle all binaries into the package, but that's a step too far removed from the current situation for now - instead, we can use prebuild-install to download the binaries, and `prebuild` to build + upload the binaries - this means we can remove node-pre-gyp and fix a bunch of issues! [1]: https://github.com/mapbox/node-pre-gyp/issues/657 --- .github/workflows/ci.yml | 25 ++++++++++--------------- .gitignore | 1 + README.md | 8 +++----- binding.gyp | 14 ++------------ lib/sqlite3-binding.js | 6 +----- package.json | 17 +++++++---------- tools/BinaryBuilder.Dockerfile | 11 ++--------- 7 files changed, 26 insertions(+), 56 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7d3e06837..9c7dd6cfe 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -82,40 +82,35 @@ jobs: echo "CFLAGS=${CFLAGS:-} -include ../src/gcc-preinclude.h" >> $GITHUB_ENV echo "CXXFLAGS=${CXXFLAGS:-} -include ../src/gcc-preinclude.h" >> $GITHUB_ENV - - name: Configure build - run: yarn node-pre-gyp configure --target_arch=${{ env.TARGET }} - - name: Build binaries - run: yarn node-pre-gyp build --target_arch=${{ env.TARGET }} + run: yarn prebuild -a ${{ env.TARGET }} - name: Print binary info if: contains(matrix.os, 'ubuntu') run: | - ldd lib/binding/*/node_sqlite3.node + ldd build/**/node_sqlite3.node echo "---" - nm lib/binding/*/node_sqlite3.node | grep "GLIBC_" | c++filt || true + nm build/**/node_sqlite3.node | grep "GLIBC_" | c++filt || true echo "---" - file lib/binding/napi-v*/* + file build/**/node_sqlite3.node - name: Run tests run: yarn test - - name: Package prebuilt binaries - run: yarn node-pre-gyp package --target_arch=${{ env.TARGET }} - - name: Upload binaries to commit artifacts uses: actions/upload-artifact@v3 if: matrix.node == 18 with: name: prebuilt-binaries - path: build/stage/*/* + path: prebuilds/* retention-days: 7 - name: Upload binaries to GitHub Release - run: yarn node-pre-gyp-github publish + run: yarn upload if: matrix.node == 18 && startsWith(github.ref, 'refs/tags/') env: - NODE_PRE_GYP_GITHUB_TOKEN: ${{ github.token }} + GITHUB_TOKEN: ${{ github.token }} + build-qemu: runs-on: ubuntu-latest if: github.event_name == 'workflow_dispatch' || startsWith(github.ref, 'refs/tags/') @@ -167,7 +162,7 @@ jobs: retention-days: 7 - name: Upload binaries to GitHub Release - run: yarn install --ignore-scripts && yarn node-pre-gyp-github publish + run: yarn install --ignore-scripts && yarn upload if: matrix.node == 18 && startsWith(github.ref, 'refs/tags/') env: - NODE_PRE_GYP_GITHUB_TOKEN: ${{ github.token }} + GITHUB_TOKEN: ${{ github.token }} diff --git a/.gitignore b/.gitignore index 3a26475cb..108a59ae0 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,4 @@ setup.sh *.tgz package-lock.json yarn.lock +prebuilds diff --git a/README.md b/README.md index c94309f5a..cd8516f6d 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ yarn add sqlite3 `sqlite3` v5+ was rewritten to use [Node-API](https://nodejs.org/api/n-api.html) so prebuilt binaries do not need to be built for specific Node versions. `sqlite3` currently builds for both Node-API v3 and v6. Check the [Node-API version matrix](https://nodejs.org/api/n-api.html#node-api-version-matrix) to ensure your Node version supports one of these. The prebuilt binaries should be supported on Node v10+. -The module uses [node-pre-gyp](https://github.com/mapbox/node-pre-gyp) to download the prebuilt binary for your platform, if it exists. These binaries are hosted on GitHub Releases for `sqlite3` versions above 5.0.2, and they are hosted on S3 otherwise. The following targets are currently provided: +The module uses [`prebuild-install`](https://github.com/prebuild/prebuild-install) to download the prebuilt binary for your platform, if it exists. These binaries are hosted on GitHub Releases for `sqlite3` versions above 5.0.2, and they are hosted on S3 otherwise. The following targets are currently provided: Format: `napi-v{napi_build_version}-{platform}-{libc}-{arch}` @@ -56,11 +56,11 @@ Format: `napi-v{napi_build_version}-{platform}-{libc}-{arch}` * `napi-v6-win32-unknown-ia32` * `napi-v6-win32-unknown-x64` -Unfortunately, [node-pre-gyp](https://github.com/mapbox/node-pre-gyp) cannot differentiate between `armv6` and `armv7`, and instead uses `arm` as the `{arch}`. Until that is fixed, you will still need to install `sqlite3` from [source](#source-install). +Unfortunately, [prebuild](https://github.com/prebuild/prebuild/issues/174) cannot differentiate between `armv6` and `armv7`, and instead uses `arm` as the `{arch}`. Until that is fixed, you will still need to install `sqlite3` from [source](#source-install). Support for other platforms and architectures may be added in the future if CI supports building on them. -If your environment isn't supported, it'll use `node-gyp` to build SQLite but you will need to install a C++ compiler and linker. +If your environment isn't supported, it'll use `node-gyp` to build SQLite, but you will need to install a C++ compiler and linker. ### Other ways to install @@ -148,8 +148,6 @@ NODE_WEBKIT_VERSION="0.8.6" # see latest version at https://github.com/rogerwang npm install sqlite3 --build-from-source --runtime=node-webkit --target_arch=ia32 --target=$(NODE_WEBKIT_VERSION) ``` -This command internally calls out to [`node-pre-gyp`](https://github.com/mapbox/node-pre-gyp) which itself calls out to [`nw-gyp`](https://github.com/rogerwang/nw-gyp) when the `--runtime=node-webkit` option is passed. - You can also run this command from within a `sqlite3` checkout: ```bash diff --git a/binding.gyp b/binding.gyp index 20d418b3c..a8fccd0c4 100644 --- a/binding.gyp +++ b/binding.gyp @@ -2,7 +2,8 @@ "includes": [ "deps/common-sqlite.gypi" ], "variables": { "sqlite%":"internal", - "sqlite_libname%":"sqlite3" + "sqlite_libname%":"sqlite3", + "module_name": "node_sqlite3", }, "targets": [ { @@ -52,17 +53,6 @@ "src/statement.cc" ], "defines": [ "NAPI_VERSION=<(napi_build_version)", "NAPI_DISABLE_CPP_EXCEPTIONS=1" ] - }, - { - "target_name": "action_after_build", - "type": "none", - "dependencies": [ "<(module_name)" ], - "copies": [ - { - "files": [ "<(PRODUCT_DIR)/<(module_name).node" ], - "destination": "<(module_path)" - } - ] } ] } diff --git a/lib/sqlite3-binding.js b/lib/sqlite3-binding.js index 65f01546e..98d1b91b7 100644 --- a/lib/sqlite3-binding.js +++ b/lib/sqlite3-binding.js @@ -1,5 +1 @@ -const binary = require('@mapbox/node-pre-gyp'); -const path = require('path'); -const binding_path = binary.find(path.resolve(path.join(__dirname,'../package.json'))); -const binding = require(binding_path); -module.exports = exports = binding; +module.exports = require('bindings')('node_sqlite3.node'); diff --git a/package.json b/package.json index 6c26284ae..fcd4e7f62 100644 --- a/package.json +++ b/package.json @@ -8,11 +8,6 @@ "url": "https://mapbox.com/" }, "binary": { - "module_name": "node_sqlite3", - "module_path": "./lib/binding/napi-v{napi_build_version}-{platform}-{libc}-{arch}", - "host": "https://github.com/TryGhost/node-sqlite3/releases/download/", - "remote_path": "v{version}", - "package_name": "napi-v{napi_build_version}-{platform}-{libc}-{arch}.tar.gz", "napi_versions": [ 3, 6 @@ -49,14 +44,15 @@ "url": "https://github.com/TryGhost/node-sqlite3.git" }, "dependencies": { - "@mapbox/node-pre-gyp": "^1.0.0", + "bindings": "^1.5.0", "node-addon-api": "^4.2.0", + "prebuild-install": "^7.1.1", "tar": "^6.1.11" }, "devDependencies": { "eslint": "8.56.0", "mocha": "10.2.0", - "node-pre-gyp-github": "1.4.4" + "prebuild": "12.1.0" }, "peerDependencies": { "node-gyp": "8.x" @@ -70,9 +66,10 @@ "node-gyp": "8.x" }, "scripts": { - "build": "node-pre-gyp build", - "build:debug": "node-pre-gyp build --debug", - "install": "node-pre-gyp install --fallback-to-build", + "install": "prebuild-install -r napi || node-gyp rebuild", + "prebuild": "prebuild --runtime napi --all --verbose", + "rebuild": "node-gyp rebuild", + "upload": "prebuild --upload-all ${GITHUB_TOKEN}", "test": "node test/support/createdb.js && mocha -R spec --timeout 480000" }, "license": "BSD-3-Clause", diff --git a/tools/BinaryBuilder.Dockerfile b/tools/BinaryBuilder.Dockerfile index 457c60845..2e089b31d 100644 --- a/tools/BinaryBuilder.Dockerfile +++ b/tools/BinaryBuilder.Dockerfile @@ -12,19 +12,12 @@ WORKDIR /usr/src/build COPY . . RUN npm install --ignore-scripts -# Workaround for https://github.com/mapbox/node-pre-gyp/issues/644 -RUN cd node_modules/\@mapbox/node-pre-gyp \ - && npm install fs-extra@10.0.1 \ - && sed -i -e s/\'fs/\'fs-extra/ -e s/fs\.renameSync/fs.moveSync/ ./lib/util/napi.js - ENV CFLAGS="${CFLAGS:-} -include ../src/gcc-preinclude.h" ENV CXXFLAGS="${CXXFLAGS:-} -include ../src/gcc-preinclude.h" -RUN npx node-pre-gyp configure -RUN npx node-pre-gyp build +RUN npm run prebuild -RUN if case $VARIANT in "alpine"*) false;; *) true;; esac; then ldd lib/binding/*/node_sqlite3.node; nm lib/binding/*/node_sqlite3.node | grep \"GLIBC_\" | c++filt || true ; fi +RUN if case $VARIANT in "alpine"*) false;; *) true;; esac; then ldd build/**/node_sqlite3.node; nm build/**/node_sqlite3.node | grep \"GLIBC_\" | c++filt || true ; fi RUN npm run test -RUN npx node-pre-gyp package CMD ["sh"]