Fix source distribution build and upgrade CI baselines

.circleci/config.yml

@@ -8,7 +8,7 @@ macos_env: &macos_env
 
 linux_env: &linux_env
   machine:
-    image: ubuntu-2004:202201-02
+    image: ubuntu-2204:current
 
 orbs:
   win: circleci/windows@5.0.0
@@ -29,24 +29,39 @@ commands:
             equal: ["ON", << parameters.use_kenlm >>]
           steps:
             - run:
-                name: "Install KenLM Build Dependencies"
+                name: "Install Build Dependencies (KenLM)"
                 # TODO: remove boost once https://github.com/kpu/kenlm/pull/418 is merged
                 command: |
                   sudo apt -y install libboost-program-options-dev libboost-system-dev libboost-thread-dev \
                   libboost-test-dev liblzma-dev libbz2-dev zlib1g-dev
   install_macos_build_dependencies:
+    parameters:
+      use_kenlm:
+        type: string
     steps:
       - run:
           name: "Install Build Dependencies"
-          # TODO: remove boost once https://github.com/kpu/kenlm/pull/418 is merged
-          command: |
-            brew install cmake googletest boost zlib bzip2 lzip
+          command: brew install googletest cmake
+      - when:
+          condition:
+            equal: ["ON", << parameters.use_kenlm >>]
+          steps:
+            - run:
+                name: "Install Build Dependencies (KenLM)"
+                # TODO: remove boost once https://github.com/kpu/kenlm/pull/418 is merged
+                command: brew install boost zlib bzip2 lzip
   install_msvc_build_dependencies:
+    parameters:
+      use_kenlm:
+        type: string
     steps:
       - run:
           name: "Install Build Dependencies"
           command: |
-            choco install cmake python3 -y
+            choco install python3 cmake -y
+      - run:
+          name: "Install Build Dependencies"
+          command: choco install cmake -y
       # windows needs a path modification
       - run:
           name: "Set PATH to find CMake"
@@ -148,26 +163,33 @@ commands:
         type: string
         default: "ON"
     steps:
-      - run:
-          name: "Install Python bindings dependencies"
-          command: pip install wheel numpy cmake
       - when:
           condition:
             equal: ["ON", << parameters.use_kenlm >>]
           steps:
             - run:
                 name: "Install KenLM"
                 command: |
-                  pip install git+https://github.com/kpu/kenlm.git
+                  pip install -v git+https://github.com/kpu/kenlm.git
+      - run:
+          name: "Install Python bindings build dependencies"
+          command: |
+            pip install setuptools wheel
       - run:
           name: "Install Python Bindings"
-          command: USE_KENLM=<< parameters.use_kenlm >> pip install -v .
+          # Because of KenLM, use --no-build-isolation
+          command: |
+            USE_KENLM=<< parameters.use_kenlm >> pip install -v . --no-build-isolation
   run_python_tests:
     parameters:
       use_kenlm:
         type: string
         default: "ON"
     steps:
+      - run:
+          name: "Install test dependencies"
+          command: |
+            pip install -r bindings/python/test/requirements.txt
       - run:
           name: "Run Python Binding Tests"
           command: |
@@ -224,7 +246,7 @@ commands:
           name: Compute wheel version
           command: |
             pip install packaging
-            echo "$(python bindings/python/compute_version.py)" > BUILD_VERSION.txt
+            python bindings/python/compute_version.py > BUILD_VERSION.txt
             echo "Building version $(cat BUILD_VERSION.txt)"
       - run:
           name: Build wheels
@@ -333,7 +355,8 @@ jobs:
     shell: /bin/bash -eux -o pipefail
     steps:
       - checkout
-      - install_macos_build_dependencies
+      - install_macos_build_dependencies:
+          use_kenlm: << parameters.use_kenlm >>
       - when:
           condition:
             and:
@@ -360,7 +383,8 @@ jobs:
     shell: /bin/bash -eux -o pipefail
     steps:
       - checkout
-      - install_macos_build_dependencies
+      - install_macos_build_dependencies:
+          use_kenlm: << parameters.use_kenlm >>
       - setup_python_virtualenv:
           platform: "macos"
       - install_python_bindings:
@@ -382,7 +406,8 @@ jobs:
       size: 2xlarge
     steps:
       - checkout
-      - install_msvc_build_dependencies
+      - install_msvc_build_dependencies:
+          use_kenlm: "OFF"
       - build_flashlight_text:
           platform: "windows"
           use_kenlm: "OFF"
@@ -397,7 +422,8 @@ jobs:
       shell: bash.exe
     steps:
       - checkout
-      - install_msvc_build_dependencies
+      - install_msvc_build_dependencies:
+          use_kenlm: << parameters.use_kenlm >>
       - setup_python_virtualenv:
           platform: "windows"
       - install_python_bindings:
@@ -479,14 +505,14 @@ workflows:
     jobs:
       - ubuntu_20_gcc_9:
           build_shared_libs: "ON"
-          name: "Ubuntu 20.04 (<< matrix.resource_class >>) gcc-9 kenlm=<< matrix.use_kenlm >> standalone=<< matrix.build_standalone >>"
+          name: "Ubuntu 22.04 (<< matrix.resource_class >>) gcc-9 kenlm=<< matrix.use_kenlm >> standalone=<< matrix.build_standalone >>"
           matrix:
             parameters:
               resource_class: [xlarge, arm.xlarge] # x64, arm64
               use_kenlm: ["ON", "OFF"]
               build_standalone: ["ON", "OFF"]
       - ubuntu_20_gcc_9:
-          name: "Ubuntu 20.04 (xlarge) gcc-9 static + KenLM"
+          name: "Ubuntu 22.04 (xlarge) gcc-9 static + KenLM"
           build_shared_libs: "OFF"
           resource_class: xlarge
           use_kenlm: "ON"
@@ -502,7 +528,7 @@ workflows:
             parameters:
               resource_class: [xlarge, arm.xlarge]
               use_kenlm: ["ON", "OFF"]
-          name: "Ubuntu 20.04 (<< matrix.resource_class >>) gcc-9 Python kenlm=<< matrix.use_kenlm >>"
+          name: "Ubuntu 22.04 (<< matrix.resource_class >>) gcc-9 Python kenlm=<< matrix.use_kenlm >>"
       - macos_clang:
           matrix:
             parameters:
@@ -527,12 +553,12 @@ workflows:
   build-wheels:
     jobs:
       - linux_arm64_wheels:
-          name: "Build Python Wheels | Build wheels on ubuntu-20.04 arm64"
+          name: "Build Python Wheels | Build wheels on ubuntu-22.04 arm64"
       - macos_arm64_wheels:
           name: "Build Python Wheels | Build wheels on macOS-14 arm64"
       - upload_wheels_pypi:
           requires:
-            - "Build Python Wheels | Build wheels on ubuntu-20.04 arm64"
+            - "Build Python Wheels | Build wheels on ubuntu-22.04 arm64"
             - "Build Python Wheels | Build wheels on macOS-14 arm64"
           filters:
             branches:

.github/workflows/wheels.yml

@@ -8,10 +8,10 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ubuntu-20.04, windows-2022, macOS-12]
+        os: [ubuntu-22.04, windows-2022, macOS-12]
     if: (github.event_name == 'push' && github.ref == 'refs/heads/main') || github.event_name == 'pull_request'
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - uses: actions/setup-python@v4
@@ -36,15 +36,16 @@ jobs:
           CIBW_BEFORE_BUILD: pip install -v git+https://github.com/kpu/kenlm.git
           CIBW_PRERELEASE_PYTHONS: 0
           CIBW_BUILD_VERBOSITY: 1
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         with:
           path: ./wheelhouse/*.whl
+          name: dist-${{ matrix.os }}
 
   build_sdist:
     name: Build source distribution
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
@@ -54,9 +55,27 @@ jobs:
       - name: Build sdist
         run: pipx run build --sdist
 
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         with:
           path: dist/*.tar.gz
+          name: dist
+
+  install_from_sdist:
+    needs: [build_sdist]
+    name: Install from source distribution
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          pattern: dist*
+          merge-multiple: true
+          path: dist
+
+      - name: Install KenLM
+        run: pip install -v git+https://github.com/kpu/kenlm.git
+
+      - name: Install from sdist (with KenLM)
+        run: ls && pip install -v dist/*.tar.gz
 
   upload_pypi:
     needs: [build_wheels, build_sdist]
@@ -66,14 +85,13 @@ jobs:
     # in the future: publish on github releases:
     # if: github.event_name == 'release' && github.event.action == 'published'
     steps:
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         with:
-          # unpacks default artifact into dist/
-          # name required to create an extra parent dir
-          name: artifact
+          pattern: dist*
+          merge-multiple: true
           path: dist
 
-      - uses: pypa/gh-action-pypi-publish@v1.6.4
+      - uses: pypa/gh-action-pypi-publish@v1.8.12
         with:
           verbose: true
           password: ${{ secrets.PYPI_PASSWORD }}

bindings/python/test/requirements.txt

@@ -0,0 +1 @@
+numpy

bindings/python/version.txt

@@ -1 +1 @@
-0.0.6
+0.0.7

setup.py

@@ -26,6 +26,12 @@
 PACKAGE_DIR = "bindings/python"
 ARTIFACTS_DIR = os.path.join(PACKAGE_DIR, "flashlight/lib/text")
 BUILD_VERSION_PATH = Path(os.path.join(THIS_DIR, "BUILD_VERSION.txt"))
+VERSION_TXT_PATH = Path(os.path.join(THIS_DIR, PACKAGE_DIR, "version.txt"))
+VERSION_PY_PATH = Path(
+    os.path.join(THIS_DIR, PACKAGE_DIR, "flashlight", "lib", "text", "version.py")
+)
+
+CMAKE_MINIMUM_VERSION = (3, 18)  # 3.18
 
 
 # Environment variables:
@@ -121,7 +127,7 @@ def run(self):
 
         cmake_version = re.search(r"version\s*([\d.]+)", out.decode().lower()).group(1)
         cmake_version_tuple = tuple([int(v) for v in cmake_version.split(".")])
-        if cmake_version_tuple < (3, 18):
+        if cmake_version_tuple < CMAKE_MINIMUM_VERSION:
             raise RuntimeError(
                 f"CMake >= 3.18 is required to build flashlight-text; found {cmake_version}"
             )
@@ -186,11 +192,19 @@ def main():
         version = os.getenv("BUILD_VERSION")
     elif BUILD_VERSION_PATH.is_file():
         version = BUILD_VERSION_PATH.read_text().strip()
-    else:
-        version_txt = os.path.join(THIS_DIR, PACKAGE_DIR, "version.txt")
-        with open(version_txt) as f:
+    elif VERSION_PY_PATH.is_file():
+        # see if version.py is already written (as is the case with a source distribution)
+        from bindings.python.flashlight.lib.text.version import __version__ as version
+    elif VERSION_TXT_PATH.is_file():
+        with open(VERSION_TXT_PATH) as f:
             version = f.readline().strip()
         version += get_local_version_suffix()
+    else:
+        raise RuntimeError(
+            "Could not find version.txt, BUILD_VERSION.txt, or version.py. "
+            "Please run `python bindings/python/compute_version.py` to "
+            "generate the version file."
+        )
 
     write_version_file(version)