Add a release test suite (#437)

valentin-seitz · Valentin Seitz · preCICE Tests VM · web-flow · commit a1d46c445dd3 · 2024-01-23T22:21:18.000+01:00
Co-authored-by: Valentin Seitz &lt;valentin.seitz@tum.de&gt;
Co-authored-by: preCICE Tests VM &lt;tests@precice.org&gt;
Co-authored-by: Gerasimos Chourdakis &lt;chourdak@in.tum.de&gt;
diff --git a/elastic-tube-1d/reference-data/fluid-cpp_solid-cpp.tar.gz b/elastic-tube-1d/reference-data/fluid-cpp_solid-cpp.tar.gz
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d7a857fcd5ce4b624c6ee45f698d3777ada63973499776b8d6edd927377502bb
-size 345364
+oid sha256:1649e17e9f1faebbe1dd4206bc6986382bfe3d471ba1bb0837e0265c2f64af12
+size 345437
diff --git a/elastic-tube-1d/reference-data/fluid-cpp_solid-python.tar.gz b/elastic-tube-1d/reference-data/fluid-cpp_solid-python.tar.gz
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b51e59cb65eaf303683e43f2337e3b97e9f94f840ac94f290e99f976929e5477
-size 705114
+oid sha256:949a6fbc5847293d33b0877af5d7895824661320ee4ba1301e3d5de9c6b47717
+size 705273
diff --git a/elastic-tube-1d/reference-data/fluid-python_solid-python.tar.gz b/elastic-tube-1d/reference-data/fluid-python_solid-python.tar.gz
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8dc48beb874a14236278fddd90528e66a27f6ae5270f74c69b1d2323e5d1bdcd
-size 705322
+oid sha256:3c295096dd92814d659dd9d54be75aa0ea40e0687f6b1d39ac00deb2a3a2b3b4
+size 705390
diff --git a/elastic-tube-1d/reference_results.metadata b/elastic-tube-1d/reference_results.metadata
@@ -11,9 +11,9 @@ We also include some information on the machine used to generate them
 
 | name | time | sha256 |
 |------|------|-------|
-| fluid-python_solid-python.tar.gz | 2023-12-14 16:20:26 | 8dc48beb874a14236278fddd90528e66a27f6ae5270f74c69b1d2323e5d1bdcd |
-| fluid-cpp_solid-python.tar.gz | 2023-12-14 16:20:26 | b51e59cb65eaf303683e43f2337e3b97e9f94f840ac94f290e99f976929e5477 |
-| fluid-cpp_solid-cpp.tar.gz | 2023-12-14 16:20:26 | d7a857fcd5ce4b624c6ee45f698d3777ada63973499776b8d6edd927377502bb |
+| fluid-python_solid-python.tar.gz | 2024-01-11 20:52:11 | 3c295096dd92814d659dd9d54be75aa0ea40e0687f6b1d39ac00deb2a3a2b3b4 |
+| fluid-cpp_solid-python.tar.gz | 2024-01-11 20:52:11 | 949a6fbc5847293d33b0877af5d7895824661320ee4ba1301e3d5de9c6b47717 |
+| fluid-cpp_solid-cpp.tar.gz | 2024-01-11 20:52:11 | 1649e17e9f1faebbe1dd4206bc6986382bfe3d471ba1bb0837e0265c2f64af12 |
 
 ## List of arguments used to generate the files
 
@@ -32,7 +32,7 @@ We also include some information on the machine used to generate them
 
 ### uname -a
 
-Linux precice-tests 5.15.0-89-generic #99-Ubuntu SMP Mon Oct 30 20:42:41 UTC 2023 x86_64 x86_64 x86_64 GNU/Linux
+Linux precice-tests 5.15.0-91-generic #101-Ubuntu SMP Tue Nov 14 13:30:08 UTC 2023 x86_64 x86_64 x86_64 GNU/Linux
 
 
 ### lscpu
diff --git a/flow-over-heated-plate/reference-results/fluid-openfoam_solid-fenics.tar.gz b/flow-over-heated-plate/reference-results/fluid-openfoam_solid-fenics.tar.gz
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6e204c5ef1ed001981d0b432c12fd93920ecef053fdac644c98118323eb25d91
-size 778409
+oid sha256:ec027fafb234e7e6277a781c96d0b2a6d5048f912bb106d174b6d8817b3a49fd
+size 778502
diff --git a/flow-over-heated-plate/reference-results/fluid-openfoam_solid-nutils.tar.gz b/flow-over-heated-plate/reference-results/fluid-openfoam_solid-nutils.tar.gz
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bb432426dc3a737fd79704a0baf5dab405478cdc4ee160b12520c8f8368c5231
-size 533356
+oid sha256:6bd5abefb23c900951768dec0ae38cc0be14f0344422560888ebe52481575869
+size 533177
diff --git a/flow-over-heated-plate/reference-results/fluid-openfoam_solid-openfoam.tar.gz b/flow-over-heated-plate/reference-results/fluid-openfoam_solid-openfoam.tar.gz
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cb0212291630004bdc9acf0cdbaed4e952fb1155cc4175f22bac714fb16f5130
-size 499028
+oid sha256:844a54543a5c81ca03c1f7ffb0739cf5125357237bcdb79b60ab957b919e425d
+size 498870
diff --git a/flow-over-heated-plate/reference_results.metadata b/flow-over-heated-plate/reference_results.metadata
@@ -11,9 +11,9 @@ We also include some information on the machine used to generate them
 
 | name | time | sha256 |
 |------|------|-------|
-| fluid-openfoam_solid-nutils.tar.gz | 2023-12-14 16:20:26 | bb432426dc3a737fd79704a0baf5dab405478cdc4ee160b12520c8f8368c5231 |
-| fluid-openfoam_solid-fenics.tar.gz | 2023-12-14 16:20:26 | 6e204c5ef1ed001981d0b432c12fd93920ecef053fdac644c98118323eb25d91 |
-| fluid-openfoam_solid-openfoam.tar.gz | 2023-12-14 16:20:26 | cb0212291630004bdc9acf0cdbaed4e952fb1155cc4175f22bac714fb16f5130 |
+| fluid-openfoam_solid-fenics.tar.gz | 2024-01-11 20:52:11 | ec027fafb234e7e6277a781c96d0b2a6d5048f912bb106d174b6d8817b3a49fd |
+| fluid-openfoam_solid-nutils.tar.gz | 2024-01-11 20:52:11 | 6bd5abefb23c900951768dec0ae38cc0be14f0344422560888ebe52481575869 |
+| fluid-openfoam_solid-openfoam.tar.gz | 2024-01-11 20:52:11 | 844a54543a5c81ca03c1f7ffb0739cf5125357237bcdb79b60ab957b919e425d |
 
 ## List of arguments used to generate the files
 
@@ -32,7 +32,7 @@ We also include some information on the machine used to generate them
 
 ### uname -a
 
-Linux precice-tests 5.15.0-89-generic #99-Ubuntu SMP Mon Oct 30 20:42:41 UTC 2023 x86_64 x86_64 x86_64 GNU/Linux
+Linux precice-tests 5.15.0-91-generic #101-Ubuntu SMP Tue Nov 14 13:30:08 UTC 2023 x86_64 x86_64 x86_64 GNU/Linux
 
 
 ### lscpu
diff --git a/perpendicular-flap/reference-results/fluid-openfoam_solid-calculix.tar.gz b/perpendicular-flap/reference-results/fluid-openfoam_solid-calculix.tar.gz
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6a2eb8eaa81763f2d3b353984deac5c260ecb206f90af93635ccf84e8e4fadb2
-size 13560146
+oid sha256:5ea071efe0147a4cfe13d612bd1fc1d2e5efcee7cdbcaf1e5b086895dc30af66
+size 13560221
diff --git a/perpendicular-flap/reference_results.metadata b/perpendicular-flap/reference_results.metadata
@@ -11,7 +11,7 @@ We also include some information on the machine used to generate them
 
 | name | time | sha256 |
 |------|------|-------|
-| fluid-openfoam_solid-calculix.tar.gz | 2023-12-14 16:20:26 | 6a2eb8eaa81763f2d3b353984deac5c260ecb206f90af93635ccf84e8e4fadb2 |
+| fluid-openfoam_solid-calculix.tar.gz | 2024-01-11 20:52:11 | 5ea071efe0147a4cfe13d612bd1fc1d2e5efcee7cdbcaf1e5b086895dc30af66 |
 
 ## List of arguments used to generate the files
 
@@ -30,7 +30,7 @@ We also include some information on the machine used to generate them
 
 ### uname -a
 
-Linux precice-tests 5.15.0-89-generic #99-Ubuntu SMP Mon Oct 30 20:42:41 UTC 2023 x86_64 x86_64 x86_64 GNU/Linux
+Linux precice-tests 5.15.0-91-generic #101-Ubuntu SMP Tue Nov 14 13:30:08 UTC 2023 x86_64 x86_64 x86_64 GNU/Linux
 
 
 ### lscpu
diff --git a/tools/tests/README.md b/tools/tests/README.md
@@ -10,19 +10,18 @@ The tutorials repository hosts cases that need multiple components from the preC
 
 ## Running the system tests
 
-**! Warning: still under development !**
-
 The main workflow for the user is executing the `systemtests.py` script. Depending on the options given to the script, it reads in the respective metadata files and generates `docker-compose.yaml` files that can start a fully-defined coupled simulation.
 
 ### Running specific test suites
 
 To test a certain test-suite defined in `tests.yaml`, use:
 
 ```bash
-python3 systemtests.py --suites=openfoam-adapter-release,<someothersuite>
+python3 systemtests.py --suites=release_test,<someothersuite>
 ```
 
-To discover all tests, use `python print_test_suites.py`.
+To discover all tests, use `python print_test_suites.py`. The `release_test` is meant to cover all tests that need to be executed before a preCICE distribution release. Smaller test suites focus on testing specific components (e.g., only running tutorials involving specific adapters).
+
 To be able to fill in the right case tuple into the `tests.yaml`, you can use the `python3 print_case_combinations.py` script.
 
 ## Running the system tests on GitHub Actions
diff --git a/tools/tests/tests.yaml b/tools/tests/tests.yaml
@@ -51,3 +51,25 @@ test_suites:
           - fluid-cpp
           - solid-python
         reference_result: ./elastic-tube-1d/reference-data/fluid-cpp_solid-python.tar.gz
+  release_test:
+    tutorials:
+      - path: elastic-tube-1d
+        case_combination:
+          - fluid-cpp
+          - solid-python
+        reference_result: ./elastic-tube-1d/reference-data/fluid-cpp_solid-python.tar.gz
+      - path: flow-over-heated-plate
+        case_combination:
+          - fluid-openfoam
+          - solid-nutils
+        reference_result: ./flow-over-heated-plate/reference-results/fluid-openfoam_solid-nutils.tar.gz
+      - path: flow-over-heated-plate
+        case_combination:
+          - fluid-openfoam
+          - solid-fenics
+        reference_result: ./flow-over-heated-plate/reference-results/fluid-openfoam_solid-fenics.tar.gz
+      - path: flow-over-heated-plate
+        case_combination:
+          - fluid-openfoam
+          - solid-openfoam
+        reference_result: ./flow-over-heated-plate/reference-results/fluid-openfoam_solid-openfoam.tar.gz
