Merge pull request #146 from automl/development

Version 0.3.0

Author: TheEimer

.flake8

@@ -10,11 +10,13 @@ on:
   push:
     branches:
       - main
+      - development
 
   # Trigger on a open/push to a PR targeting one of these branches
   pull_request:
     branches:
       - main
+      - development
 
 env:
   name: DACBench
@@ -42,13 +44,13 @@ jobs:
         make doc
 
     - name: Pull latest gh-pages
-      if: contains(github.ref, 'main') && github.event_name == 'push'
+      if: github.event_name == 'push'
       run: |
         cd ..
         git clone https://github.com/${{ github.repository }}.git --branch gh-pages --single-branch gh-pages
 
     - name: Copy new docs into gh-pages
-      if: contains(github.ref, 'main') && github.event_name == 'push'
+      if: github.event_name == 'push'
       run: |
         branch_name=${GITHUB_REF##*/}
         cd ../gh-pages
@@ -57,7 +59,7 @@ jobs:
 
 
     - name: Push to gh-pages
-      if: contains(github.ref, 'main') && github.event_name == 'push'
+      if: github.event_name == 'push'
       run: |
         last_commit=$(git log --pretty=format:"%an: %s")
         cd ../gh-pages

.github/workflows/docs.yml

@@ -6,33 +6,29 @@ on:
 
   # When a push occurs on either of these branches
   push:
-    branches-ignore:
-        - '**'
-    # branches:
-    #   - main
-    #   - development
+    branches:
+      - main
+      - development
 
   # When a push occurs on a PR that targets these branches
   pull_request:
-    branches-ignore:
-      - '**'
-    # branches:
-    #   - main
-    #   - development
+    branches:
+      - main
+      - development
 
 jobs:
   run-all-files:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
       with:
         submodules: recursive
 
     - name: Setup Python 3.10
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
-        python-version: 3.10
+        python-version: "3.10"
 
     - name: Install pre-commit
       run: |

.github/workflows/main.yml

@@ -1,58 +1,35 @@
 name: tests
 on:
+  workflow_dispatch:
   push:
     branches:
       - main
+      - development
   pull_request:
     branches:
       - main
 
 jobs:
   build:
-    runs-on: ubuntu-18.04
+    runs-on: "ubuntu-latest"
     steps:
-      - uses: actions/checkout@v2
-        with:
-          submodules: recursive
-      - name: Cache MNIST
-        id: cache-mnist
-        uses: actions/cache@v2
-        with:
-          path: ./data/MNIST
-          key: mnist
-      - name: Cache Fast-downward build
-        id: cache-fast-downward-build
-        uses: actions/cache@v2
-        with:
-          path: ./dacbench/envs/rl-plan/fast-downward
-          key: fast-downward-build
-      - name: Install Python 3
+      - uses: actions/checkout@v3
+      - name: Set up Python 
         uses: actions/setup-python@v4
         with:
           python-version: '3.10'
-      - name: Install dependencies
+      - name: Install uv
         run: |
-          python -m pip install --upgrade pip
-          pip install -e.[dev,all]
-      - name: Build fast-downward
-        run: ./dacbench/envs/rl-plan/fast-downward/build.py
-      - name: Run tests with pytest
-        run: coverage run -m pytest --html=test-report.html
-      - name: Run coverage
+          python -m pip install uv
+          uv venv
+      - name: Activate virtualenv
         run: |
-          coverage report
-          coverage html
-      - name: Archive code coverage results
-        uses: actions/upload-artifact@v2
-        if: ${{ always() }}
-        with:
-          name: code-coverage-report
-          path: coverage_report
-      - name: Archive test report
-        uses: actions/upload-artifact@v2
-        if: ${{ always() }}
-        with:
-          name: test-report
-          path: |
-            test-report.html
-            assets
+          . .venv/bin/activate
+          which python
+          echo PATH=$PATH >> $GITHUB_ENV
+      - name: Install dependencies
+        run:  |
+          git submodule update --init --recursive
+          uv pip install -e ".[dev, docs, all, example]"
+      - name: Run tests with pytest
+        run: /home/runner/work/DACBench/DACBench/.venv/bin/python -m pytest tests

.github/workflows/pre_commit.yml

@@ -16,5 +16,15 @@ port_*.txt
 tests/data/
 test/*.json
 
-# pycharm specific stuff
+# pycharm/vscode specific stuff
 .idea
+.vscode/
+
+#datasets for sgd
+*MNIST/raw*
+*raw*
+*cifar-10-batches-py*
+*cifar*.tar.gz
+
+# venvs
+*.lock*

.github/workflows/pytest.yml

@@ -1,6 +1,3 @@
 [submodule "dacbench/envs/rl-plan"]
 	path = dacbench/envs/rl-plan
 	url = https://github.com/speckdavid/rl-plan.git
-[submodule "docs/_themes/sphinx_rtd_theme"]
-	path = docs/_themes/sphinx_rtd_theme
-	url = https://github.com/readthedocs/sphinx_rtd_theme.git

.gitignore

@@ -1,11 +1,10 @@
 repos:
--   repo: https://github.com/ambv/black
-    rev: stable
-    hooks:
-    - id: black
-      language_version: python3.10
-      exclude: dacbench/envs/fast-downward
--   repo: https://gitlab.com/pycqa/flake8
-    rev: 3.8.4
-    hooks:
-    - id: flake8
+- repo: https://github.com/astral-sh/ruff-pre-commit
+  # Ruff version.
+  rev: v0.2.2
+  hooks:
+    # Run the linter.
+    - id: ruff
+      args: [ --fix ]
+    # Run the formatter.
+    - id: ruff-format

.gitmodules

@@ -1,3 +1,40 @@
+# 0.3.0
+
+### Instance Specification
+Instances for most benchmarks now have a dataclass Type instead of being a simple list. Additionally, the corresponding datasets are saved as proper csv files with headers now. This should make anything relating to instances much more convenient for the user. The benchmarks in question are:
+- CMA-ES
+- SGD
+- Luby
+- ToySGD
+- Function Approximation
+
+### Unifying CMA-ES Versions
+Instead of having two different versions of CMA-ES, we now have a single environment which covers both step size adaption and algorithm variant selection of CMA-ES (formerly ModCMA). By changing the configuration space, the user can select which hyperparameters to adapt. This change includes a switch to the newer "ioh" package, meaning an increased amount of target functions could be interfaced in principle. Anything outside of BBOB will need to be loaded separately from the "read_instance_set" function of the benchmark, however.
+
+### Unifying Function Approximation Tasks
+To reduce the number of separate classes to maintain for simple function approximation, the Sigmoid variations and the Geometric environment have been fused into the FunctionApproximation Benchmark. There are several options for functions to approximate, as in Sigmoid it is possible to use a discrete space and you can add importance weights for the dimensions. The "get_benchmark" method will still provide the original Sigmoid configurations. Apart from that, this new environment should provide all functionality of the previous environments, just with a simpler path to getting there.
+
+### Re-implementing SGD Benchmark
+The original SGD benchmark was complex and error prone, lacking important features like compatibility with torchhub. Therefore this has been re-implemented, mostly based on the existing competition version. The code is simpler now and compatible with a larger selection of models and optimizers.
+
+### Deprecating FastDownward
+The FastDownward version compatible with the benchmark cannot be used with any Ubuntu version after 16.x - which is fairly old by now. For reference, it is not possible to even build a container for this version of FastDownward on th GitHub servers. Since there is no option to update the planner version without updating the environment and this is tied to significant domain knowledge, FastDownward will now be deprecated. This means there is no testing, the benchmark will not be updated and it will not be listed as an official benchmark any longer. If someone is familiar enough with FastDownward to facilitate an update, please notify us, we'd love to continue this benchmark!
+
+### Updating Instance Sets
+Most instance sets have been updated due to the instance specification change. The Sigmoid ones have been preserved, the rest has been updated. Sampling options for all benchmarks are included in the "instance_set" directory, however.
+
+### Logger Update
+The logs have been a bit hard to read & work with. We flattened them by removing timestamps, hopefully making them easier to work with.
+
+### Dependency Upgrades
+The dependencies have been upgraded to their current highest possible version.
+
+### Simplify Examples
+Some examples had a lot of extra dependencies. We removed many of these for now - this means less explicit RL examples, but if you want to use an RL library, DACBench will plug in like any other env, so you should read their documentation anyway.
+
+### Including Instance Files in PyPI 
+There was a persistent configuration mistake which prevented the instance sets to be included in the PyPI installation - this should now be fixed and all instances sets from the repository come with the PyPI package.
+
 # 0.2.0
 
 ### Interface Update