Implement tests with tox-conda (#92)

* added configuration for tox-conda

* run tests with tox-conda

* data file locations and test skips

* make tests run in workflow

* add pytest-cov

* removed codestyle env typo

* added codestyle to envlist

* tox-conda version for codestyle

* added python version to codestyle workflow

* codestyle on 3.10 specifically

* flake8 fixes

* do not install everything for codestyle

* added temp fix for flaky sklearn install

* modified installation and testing instructions

* update action versions

* remove inapplicable parts of astropy cookie cutter, mention conda env in INSTALL.rst

* add make to conda deps

.github/workflows/codestyle.yml

@@ -8,15 +8,19 @@ jobs:
   flake8:
     runs-on: ubuntu-latest
     steps:
-    - name: Checkout Repository
-      uses: actions/checkout@v2
-    - name: Install Python 3.x
-      uses: actions/setup-python@v2
-      with:
-        python-version: 3.x
-    - name: Install Dependencies
-      run: |
-        pip install tox
-    - name: Check Code Style
-      run: |
-        tox -e codestyle
+      - name: Checkout repository
+        uses: actions/checkout@v2
+      - name: Install Conda w/ Python 3.10
+        uses: conda-incubator/setup-miniconda@v2
+        with:
+          auto-activate-base: false
+          python-version: '3.10'
+          channels: conda-forge
+      - name: Install Dependencies
+        shell: bash -el {0}
+        run: |
+          pip install tox-conda flake8
+      - name: Check Code Style
+        shell: bash -el {0}
+        run: |
+          tox -e codestyle

.github/workflows/testing.yml

@@ -3,66 +3,46 @@ name: Testing
 on: [push, pull_request]
 
 jobs:
-  build:
-
+  test:
+    name: ${{ matrix.name }}
     runs-on: ${{ matrix.os }}
-    defaults:
-      run:
-        shell: bash -l {0}    
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, macos-latest]
-        python-version: [3.8, 3.9]
-    steps:
-    - uses: actions/checkout@v2
-
-    - uses: conda-incubator/setup-miniconda@v2
-      with:
-        python-version: ${{ matrix.py }}
-        auto-update-conda: true
-        channels: conda-forge,defaults
-        channel-priority: strict
-        show-channel-urls: true
-
-    - name: install reqs
-      run: |
-        conda install pip compilers pytest pytest-cov pyccl cython
-        pip install cobaya
+        include:
 
-    - name: install extra reqs
-      run: |
-        pip install cosmopower
+          - name: latest supported versions
+            os: ubuntu-latest
+            python-version: '3.10'
+            toxenv: py310-test-all-latest-cov
 
-      env:  
-        MATRIX_OS: ${{ matrix.os }}
+          - name: oldest supported versions
+            os: ubuntu-latest
+            python-version: '3.7'
+            toxenv: py37-test-oldest
 
-    - name: Install classy
-      run: |
-        if [[ ${MATRIX_OS} == "macos-latest" ]]; then
-          . ci_scripts/install_class_osx.sh
-        else
-          . ci_scripts/install_class_linux.sh
-        fi
-        python -c "import classy; print(classy)"
-      env:
-        MATRIX_OS: ${{ matrix.os }}
+          - name: macOS 11
+            os: macos-11
+            python-version: '3.10'
+            toxenv: py310-test-latest
 
-    - name: install
-      run: |
-        pip install .
-
-    - name: install test data
-      run: |
-        export COBAYA_PACKAGES_PATH="../packages"
-        cobaya-install planck_2018_highl_plik.TTTEEE_lite_native
-
-    - name: Unit tests
-      run: |
-        pytest -vv soliket --cov=soliket --cov-report=xml:coverage.xml --cov-config=setup.cfg
-      env:
-        PYTEST_ADDOPTS: "--color=yes"
-
-    - name: Report Coverage (codecov)
-      if: matrix.os == 'ubuntu-latest'
-      uses: codecov/codecov-action@v2
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+      - name: Install Conda w/ Python ${{ matrix.python-version }}
+        uses: conda-incubator/setup-miniconda@v2
+        with:
+          auto-activate-base: false
+          python-version: ${{ matrix.python-version }}
+          channels: conda-forge
+      - name: Install Dependencies
+        shell: bash -el {0}
+        run: |
+          pip install tox-conda
+      - name: Run Tests
+        shell: bash -el {0}
+        run: |
+           tox -e ${{ matrix.toxenv }} ${{ matrix.toxargs }} -- ${{ matrix.toxposargs }}
+      - if: contains(matrix.toxenv, '-cov')
+        name: Report Coverage
+        uses: codecov/codecov-action@v3

INSTALL.rst

@@ -1,6 +1,13 @@
 INSTALL AND RUN COBAYA+SOLIKET
 ==============================
 
+Using conda
+-----------
+
+We have provided a conda environment defined in `soliket-tests.yml <https://github.com/simonsobs/SOLikeT/blob/master/soliket-tests.yml>`_ which provides easy set up of a virtual envrinoment with all the dependencies installed in order to run SOLikeT and its tests on multiple platforms (explicitly tested for ubuntu and MacOS-11).
+
+If you wish to install it using your own system tools some useful information is provided below.
+
 On your own laptop/virtual machine
 ----------------------------------
 

README.md

@@ -7,31 +7,66 @@ A centralized package for likelihood and theory implementations for SO.
 
 ## Installation
 
-For a set of detailed instructions, please see [here](INSTALL.rst).
+For a set of detailed instructions for different machines (e.g. NERSC), please see [here](INSTALL.rst).
+
+To install SOLikeT we expect that you have the following system-level tools:
+  - python>=3.7,<3.11
+  - pip
+  - compilers (c, cxx, fortran)
+  - cmake
+  - swig
+  - gsl
+  - fftw
+  - cython
+  - mpi4py
+
+A convenient way to obtain these things (along with the python dependencies listed in requirements.txt) is through using the conda environment in soliket-tests.yml. This conda environment is the one we use for running tests.
+
+You can then install SOLikeT in the usual way with pip:
 
 ```
 git clone https://github.com/simonsobs/soliket
 cd soliket
 pip install -e .
 ```
-You will also need to either run
+
+## Running Tests
+
+Running tests
+
+There are (at least) two reasons you might want to run tests:
+
+1. To see if tests you have written when developing SOLikeT are valid and will pass the Continuous Integration (CI) tests which we require for merging on github.
+
+If you are using conda, the easiest way to run tests (and the way we run them) is to use tox-conda
 ```
-pip install camb
+pip install tox-conda
+tox -e test
 ```
-or, for a fuller cobaya install:
+
+This will create a fresh virtual environment replicating the one which is used for CI then run the tests (i.e. without touching your current environment). Note that any args after a '--' string will be passed to pytest, so
+
 ```
-cobaya-install cosmo -p /your/path/to/cobaya/packages
+tox -e test -- -k my_new_module
 ```
-To run tests, you will also need the original LAT_MFlike package:
+
+will only run tests which have names containing the string 'my_new_model', and 
+
 ```
-pip install git+https://github.com/simonsobs/lat_mflike
+tox -e test -- -pdb
 ```
-Then, you can run tests with 
+
+will start a pdb debug instance when (sorry, _if_) a test fails.
+
+2. Check SOLikeT is working as intended in an environment of your own specification.
+
+For this you need to make sure all of the above system-level and python dependencies are working correctly, then run:
 ```
-pip install pytest
-pytest -v .
+pytest -v soliket
 ```
 
+Good luck!
+
 Please raise an issue if you have trouble installing or any of the tests fail.
 
 ## Contains

requirements.txt

@@ -0,0 +1,20 @@
+numpy
+scipy
+pandas
+scikit-learn
+pyyaml
+py-bobyqa
+packaging
+tqdm
+portalocker
+dill
+fuzzywuzzy
+astropy
+camb
+cosmopower
+getdist
+cobaya
+pyccl
+sacc
+fgspectra @ git+https://github.com/simonsobs/fgspectra@act_sz_x_cib#egg=fgspectra
+mflike @ git+https://github.com/simonsobs/lat_mflike@master

setup.cfg

@@ -1,8 +1,42 @@
+[metadata]
+name = soliket
+author = simonsobs
+version = 0.1.dev
+description = "SO Likelihoods & Theories"
+long_description = file: README.rst
+url = https://github.com/simonsobs/SOLikeT/
+edit_on_github = True
+
+[options]
+zip_safe = False
+packages = find:
+python_requires = >=3.7
+setup_requires = setuptools_scm
+install_requires =
+    numpy
+    scipy
+    pandas
+    scikit-learn
+    astropy
+    camb
+    cosmopower
+    getdist
+    cobaya
+    pyccl
+    sacc
+    fgspectra @ git+https://github.com/simonsobs/fgspectra@act_sz_x_cib#egg=fgspectra
+    mflike @ git+https://github.com/simonsobs/lat_mflike@master
+
+[options.package_data]
+soliket = *.yaml,*.bibtex,clusters/data/*,clusters/data/selFn_equD56/*,lensing/data/*.txt,mflike/*.yaml,tests/*.yaml,data/xcorr_simulated/*.txt
+testpaths = "soliket"
+text_file_format = rst
+
 [flake8]
 select = E713,E704,E703,E714,E741,E10,E11,E20,E22,E23,E25,E27,E301,E302,E304,E9,
 		 F405,F406,F5,F6,F7,F8,W1,W2,W3,W6,E501
 max-line-length = 90
-exclude = .tox,build
+exclude = .tox,build,cobaya_packages,test,.eggs
 
 [coverage:run]
 omit =

setup.py

@@ -1,36 +1,41 @@
-# example cobaya-compliant SO likelihood package;
-# adapted from github.com/cobayasampler/example_external_likelihood
+#!/usr/bin/env python
+# Licensed under a 3-clause BSD style license - see LICENSE.rst
+
+# NOTE: The configuration for the package, including the name, version, and
+# other information are set in the setup.cfg file.
+
+import os
+import sys
 
 from setuptools import setup
 
-setup(
-    name="soliket",
-    version="0.0",
-    description="SO Likelihoods & Theories",
-    zip_safe=False,
-    packages=["soliket", "soliket.tests", "soliket.clusters"],
-    package_data={
-        "soliket": [
-            "*.yaml",
-            "*.bibtex",
-            # "data/simulated*/*.txt",
-            "clusters/data/*",
-            "clusters/data/selFn_equD56/*",
-            "lensing/data/*.txt",
-        ]
-    },
-    install_requires=[
-        "astropy",
-        "scikit-learn",
-        "cobaya",
-        "sacc",
-        "pyccl",
-        "fgspectra @ git+https://github.com/simonsobs/fgspectra@act_sz_x_cib#egg=fgspectra", # noqa E501
-        "mflike @ git+https://github.com/simonsobs/lat_mflike@master"
-    ],
-    extras_requires=[
-        "cosmopower"
-    ],
-    test_suite="soliket.tests",
-    include_package_data=True,
-)
+
+# First provide helpful messages if contributors try and run legacy commands
+# for tests or docs.
+
+TEST_HELP = """
+Note: running tests is no longer done using 'python setup.py test'. Instead
+you will need to run:
+    tox -e test
+If you don't already have tox-conda installed, you can install it with:
+    pip install tox-conda
+If you only want to run part of the test suite, you can also pass pytest
+args through directly following a '--':
+    tox -e test -- -k name_of_my_test
+For more information, see:
+  https://github.com/simonsobs/SOLikeT#running-tests
+"""
+
+VERSION_TEMPLATE = """
+# Note that we need to fall back to the hard-coded version if either
+# setuptools_scm can't be imported or setuptools_scm can't determine the
+# version, so we catch the generic 'Exception'.
+try:
+    from setuptools_scm import get_version
+    version = get_version(root='..', relative_to=__file__)
+except Exception:
+    version = '{version}'
+""".lstrip()
+
+setup(use_scm_version={'write_to': os.path.join('.', 'version.py'),
+                       'write_to_template': VERSION_TEMPLATE})

soliket-tests.yml

@@ -0,0 +1,18 @@
+name: soliket-tests
+channels:
+  - conda-forge
+  - nodefaults
+dependencies:
+  - python>=3.7,<3.11
+  - pip
+  - pytest-cov
+  - compilers
+  - make
+  - cmake
+  - swig
+  - gsl
+  - fftw
+  - cython
+  - mpi4py
+  - pip:
+      - -r requirements.txt