From cf121aa293cda66cdf6fa6c4acd804de7917486c Mon Sep 17 00:00:00 2001 From: Nicki Skafte Detlefsen Date: Tue, 30 Jan 2024 13:13:24 +0100 Subject: [PATCH 1/9] Investigate doctest not being skipped (#2328) * warning print * try with resume download * kill * no parallel * token --------- Co-authored-by: Jirka --- .azure/gpu-unittests.yml | 9 +++++---- .../functional/multimodal/clip_iqa.py | 8 ++++---- .../functional/multimodal/clip_score.py | 8 ++++---- src/torchmetrics/functional/text/bert.py | 8 ++++---- src/torchmetrics/multimodal/clip_iqa.py | 8 ++++---- src/torchmetrics/multimodal/clip_score.py | 8 ++++---- src/torchmetrics/text/bert.py | 11 ++++++----- src/torchmetrics/utilities/checks.py | 15 ++++++++------- 8 files changed, 39 insertions(+), 36 deletions(-) diff --git a/.azure/gpu-unittests.yml b/.azure/gpu-unittests.yml index b743c10f406..5175830e4c5 100644 --- a/.azure/gpu-unittests.yml +++ b/.azure/gpu-unittests.yml @@ -42,6 +42,7 @@ jobs: # these two caches assume to run repetitively on the same set of machines # see: https://github.com/microsoft/azure-pipelines-agent/issues/4113#issuecomment-1439241481 TORCH_HOME: "/var/tmp/torch" + TOKENIZERS_PARALLELISM: "false" HF_HOME: "/var/tmp/huggingface" PIP_CACHE_DIR: "/var/tmp/pip" # MKL_THREADING_LAYER: "GNU" @@ -113,11 +114,11 @@ jobs: displayName: "Show caches" - bash: | - python -m pytest torchmetrics --cov=torchmetrics \ - --timeout=240 --durations=50 \ - --numprocesses=5 --dist=loadfile + python -m pytest torchmetrics -s --cov=torchmetrics \ + --timeout=240 --durations=50 + # --numprocesses=5 --dist=loadfile env: - DOCTEST_DOWNLOAD_TIMEOUT: "240" + DOCTEST_DOWNLOAD_TIMEOUT: "180" SKIP_SLOW_DOCTEST: "1" workingDirectory: src displayName: "DocTesting" diff --git a/src/torchmetrics/functional/multimodal/clip_iqa.py b/src/torchmetrics/functional/multimodal/clip_iqa.py index ef2f354f1fc..92df16935e9 100644 --- a/src/torchmetrics/functional/multimodal/clip_iqa.py +++ b/src/torchmetrics/functional/multimodal/clip_iqa.py @@ -28,11 +28,11 @@ from transformers import CLIPModel as _CLIPModel from transformers import CLIPProcessor as _CLIPProcessor - def _download_clip() -> None: - _CLIPModel.from_pretrained("openai/clip-vit-base-patch16") - _CLIPProcessor.from_pretrained("openai/clip-vit-base-patch16") + def _download_clip_for_iqa_metric() -> None: + _CLIPModel.from_pretrained("openai/clip-vit-base-patch16", resume_download=True) + _CLIPProcessor.from_pretrained("openai/clip-vit-base-patch16", resume_download=True) - if not _try_proceed_with_timeout(_download_clip): + if not _try_proceed_with_timeout(_download_clip_for_iqa_metric): __doctest_skip__ = ["clip_image_quality_assessment"] else: __doctest_skip__ = ["clip_image_quality_assessment"] diff --git a/src/torchmetrics/functional/multimodal/clip_score.py b/src/torchmetrics/functional/multimodal/clip_score.py index 9070f4d8d6a..bc0df92bd63 100644 --- a/src/torchmetrics/functional/multimodal/clip_score.py +++ b/src/torchmetrics/functional/multimodal/clip_score.py @@ -29,11 +29,11 @@ from transformers import CLIPModel as _CLIPModel from transformers import CLIPProcessor as _CLIPProcessor - def _download_clip() -> None: - _CLIPModel.from_pretrained("openai/clip-vit-large-patch14") - _CLIPProcessor.from_pretrained("openai/clip-vit-large-patch14") + def _download_clip_for_clip_score() -> None: + _CLIPModel.from_pretrained("openai/clip-vit-large-patch14", resume_download=True) + _CLIPProcessor.from_pretrained("openai/clip-vit-large-patch14", resume_download=True) - if not _try_proceed_with_timeout(_download_clip): + if not _try_proceed_with_timeout(_download_clip_for_clip_score): __doctest_skip__ = ["clip_score"] else: __doctest_skip__ = ["clip_score"] diff --git a/src/torchmetrics/functional/text/bert.py b/src/torchmetrics/functional/text/bert.py index 601d7c81453..3623ff548cc 100644 --- a/src/torchmetrics/functional/text/bert.py +++ b/src/torchmetrics/functional/text/bert.py @@ -39,12 +39,12 @@ if _TRANSFORMERS_GREATER_EQUAL_4_4: from transformers import AutoModel, AutoTokenizer - def _download_model() -> None: + def _download_model_for_bert_score() -> None: """Download intensive operations.""" - AutoTokenizer.from_pretrained(_DEFAULT_MODEL) - AutoModel.from_pretrained(_DEFAULT_MODEL) + AutoTokenizer.from_pretrained(_DEFAULT_MODEL, resume_download=True) + AutoModel.from_pretrained(_DEFAULT_MODEL, resume_download=True) - if _SKIP_SLOW_DOCTEST and not _try_proceed_with_timeout(_download_model): + if _SKIP_SLOW_DOCTEST and not _try_proceed_with_timeout(_download_model_for_bert_score): __doctest_skip__ = ["bert_score"] else: __doctest_skip__ = ["bert_score"] diff --git a/src/torchmetrics/multimodal/clip_iqa.py b/src/torchmetrics/multimodal/clip_iqa.py index 5d0bd4866e1..a9b0623ce5e 100644 --- a/src/torchmetrics/multimodal/clip_iqa.py +++ b/src/torchmetrics/multimodal/clip_iqa.py @@ -43,11 +43,11 @@ from transformers import CLIPModel as _CLIPModel from transformers import CLIPProcessor as _CLIPProcessor - def _download_clip() -> None: - _CLIPModel.from_pretrained("openai/clip-vit-large-patch14") - _CLIPProcessor.from_pretrained("openai/clip-vit-large-patch14") + def _download_clip_iqa_metric() -> None: + _CLIPModel.from_pretrained("openai/clip-vit-large-patch14", resume_download=True) + _CLIPProcessor.from_pretrained("openai/clip-vit-large-patch14", resume_download=True) - if not _try_proceed_with_timeout(_download_clip): + if not _try_proceed_with_timeout(_download_clip_iqa_metric): __doctest_skip__ = ["CLIPImageQualityAssessment", "CLIPImageQualityAssessment.plot"] else: __doctest_skip__ = ["CLIPImageQualityAssessment", "CLIPImageQualityAssessment.plot"] diff --git a/src/torchmetrics/multimodal/clip_score.py b/src/torchmetrics/multimodal/clip_score.py index cd133c87e44..eb432e0be16 100644 --- a/src/torchmetrics/multimodal/clip_score.py +++ b/src/torchmetrics/multimodal/clip_score.py @@ -30,11 +30,11 @@ from transformers import CLIPModel as _CLIPModel from transformers import CLIPProcessor as _CLIPProcessor - def _download_clip() -> None: - _CLIPModel.from_pretrained("openai/clip-vit-large-patch14") - _CLIPProcessor.from_pretrained("openai/clip-vit-large-patch14") + def _download_clip_for_clip_score() -> None: + _CLIPModel.from_pretrained("openai/clip-vit-large-patch14", resume_download=True) + _CLIPProcessor.from_pretrained("openai/clip-vit-large-patch14", resume_download=True) - if not _try_proceed_with_timeout(_download_clip): + if not _try_proceed_with_timeout(_download_clip_for_clip_score): __doctest_skip__ = ["CLIPScore", "CLIPScore.plot"] else: __doctest_skip__ = ["CLIPScore", "CLIPScore.plot"] diff --git a/src/torchmetrics/text/bert.py b/src/torchmetrics/text/bert.py index 83d2cd8c1a1..53031d09913 100644 --- a/src/torchmetrics/text/bert.py +++ b/src/torchmetrics/text/bert.py @@ -35,12 +35,12 @@ if _SKIP_SLOW_DOCTEST and _TRANSFORMERS_GREATER_EQUAL_4_4: from transformers import AutoModel, AutoTokenizer - def _download_model() -> None: + def _download_model_for_bert_score() -> None: """Download intensive operations.""" - AutoTokenizer.from_pretrained(_DEFAULT_MODEL) - AutoModel.from_pretrained(_DEFAULT_MODEL) + AutoTokenizer.from_pretrained(_DEFAULT_MODEL, resume_download=True) + AutoModel.from_pretrained(_DEFAULT_MODEL, resume_download=True) - if not _try_proceed_with_timeout(_download_model): + if not _try_proceed_with_timeout(_download_model_for_bert_score): __doctest_skip__ = ["BERTScore", "BERTScore.plot"] else: __doctest_skip__ = ["BERTScore", "BERTScore.plot"] @@ -187,7 +187,8 @@ def __init__( " `transformers` model is used." f" It will use the default recommended model - {_DEFAULT_MODEL!r}." ) - self.tokenizer = AutoTokenizer.from_pretrained(self.model_name_or_path) + resume_download = self.model_name_or_path == _DEFAULT_MODEL + self.tokenizer = AutoTokenizer.from_pretrained(self.model_name_or_path, resume_download=resume_download) self.user_tokenizer = False self.add_state("preds_input_ids", [], dist_reduce_fx="cat") diff --git a/src/torchmetrics/utilities/checks.py b/src/torchmetrics/utilities/checks.py index 96455064c3a..7d7d67784f5 100644 --- a/src/torchmetrics/utilities/checks.py +++ b/src/torchmetrics/utilities/checks.py @@ -11,9 +11,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import logging import multiprocessing import os +import sys from functools import partial from time import perf_counter from typing import Any, Callable, Dict, Mapping, Optional, Sequence, Tuple, no_type_check @@ -766,7 +766,7 @@ def is_overridden(method_name: str, instance: object, parent: object) -> bool: def _try_proceed_with_timeout(fn: Callable, timeout: int = _DOCTEST_DOWNLOAD_TIMEOUT) -> bool: """Check if a certain function is taking too long to execute. - Function will only be executed if running inside a doctest context. Currently does not support Windows. + Function will only be executed if running inside a doctest context. Currently, does not support Windows. Args: fn: function to check @@ -778,7 +778,8 @@ def _try_proceed_with_timeout(fn: Callable, timeout: int = _DOCTEST_DOWNLOAD_TIM """ # source: https://stackoverflow.com/a/14924210/4521646 proc = multiprocessing.Process(target=fn) - logging.debug(f"try to run `{fn.__name__}` for {timeout}s...") + + print(f"Trying to run `{fn.__name__}` for {timeout}s...", file=sys.stderr) proc.start() # Wait for N seconds or until process finishes proc.join(timeout) @@ -786,10 +787,10 @@ def _try_proceed_with_timeout(fn: Callable, timeout: int = _DOCTEST_DOWNLOAD_TIM if not proc.is_alive(): return True - logging.warning(f"running `{fn.__name__}`... let's kill it...") + print(f"`{fn.__name__}` did not complete with {timeout}, killing process and returning False", file=sys.stderr) # Terminate - may not work if process is stuck for good - proc.terminate() + # proc.terminate() + # proc.join() # OR Kill - will work for sure, no chance for process to finish nicely however - # p.kill() - proc.join() + proc.kill() return False From a424a705c94449ebb0b2f785a61590ec58780230 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 2 Feb 2024 00:31:48 +0100 Subject: [PATCH 2/9] build(deps): bump codecov/codecov-action from 3 to 4 (#2341) Bumps [codecov/codecov-action](https://github.com/codecov/codecov-action) from 3 to 4. - [Release notes](https://github.com/codecov/codecov-action/releases) - [Changelog](https://github.com/codecov/codecov-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/codecov/codecov-action/compare/v3...v4) --- updated-dependencies: - dependency-name: codecov/codecov-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index f7c975de86e..9940bd84b13 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -177,7 +177,7 @@ jobs: coverage report - name: Upload coverage to Codecov - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@v4 with: token: ${{ secrets.CODECOV_TOKEN }} file: tests/coverage.xml From 30b9fb88ce0edb3b117b70c5c121f0f768fed490 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 2 Feb 2024 00:32:47 +0100 Subject: [PATCH 3/9] build(deps): bump juliangruber/sleep-action from 1 to 2 (#2340) Bumps [juliangruber/sleep-action](https://github.com/juliangruber/sleep-action) from 1 to 2. - [Release notes](https://github.com/juliangruber/sleep-action/releases) - [Commits](https://github.com/juliangruber/sleep-action/compare/v1...v2) --- updated-dependencies: - dependency-name: juliangruber/sleep-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/publish-pkg.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish-pkg.yml b/.github/workflows/publish-pkg.yml index 7e1793509e5..d9cfa786de9 100644 --- a/.github/workflows/publish-pkg.yml +++ b/.github/workflows/publish-pkg.yml @@ -78,7 +78,7 @@ jobs: needs: publish-pypi-test runs-on: ubuntu-latest steps: - - uses: juliangruber/sleep-action@v1 + - uses: juliangruber/sleep-action@v2 with: time: 5m From ca759b40198c97ba11d1ad07459379676eb0aab7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 2 Feb 2024 00:46:19 +0100 Subject: [PATCH 4/9] build(deps): bump peter-evans/create-or-update-comment from 3.1.0 to 4.0.0 (#2343) build(deps): bump peter-evans/create-or-update-comment Bumps [peter-evans/create-or-update-comment](https://github.com/peter-evans/create-or-update-comment) from 3.1.0 to 4.0.0. - [Release notes](https://github.com/peter-evans/create-or-update-comment/releases) - [Commits](https://github.com/peter-evans/create-or-update-comment/compare/v3.1.0...v4.0.0) --- updated-dependencies: - dependency-name: peter-evans/create-or-update-comment dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/cmd-help.yml | 2 +- .github/workflows/cmd-rebase.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/cmd-help.yml b/.github/workflows/cmd-help.yml index 5b0d4b6f9f0..4507cecd312 100644 --- a/.github/workflows/cmd-help.yml +++ b/.github/workflows/cmd-help.yml @@ -8,7 +8,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Update with comment - uses: peter-evans/create-or-update-comment@v3.1.0 + uses: peter-evans/create-or-update-comment@v4.0.0 with: token: ${{ secrets.PAT_GHOST }} reaction-token: ${{ secrets.PAT_GHOST }} diff --git a/.github/workflows/cmd-rebase.yml b/.github/workflows/cmd-rebase.yml index 59c0c46338f..c7388aba9e7 100644 --- a/.github/workflows/cmd-rebase.yml +++ b/.github/workflows/cmd-rebase.yml @@ -30,7 +30,7 @@ jobs: git push --force-with-lease - name: Update comment - uses: peter-evans/create-or-update-comment@v3.1.0 + uses: peter-evans/create-or-update-comment@v4.0.0 with: token: ${{ secrets.PAT_GHOST }} repository: ${{ github.event.client_payload.github.payload.repository.full_name }} @@ -43,7 +43,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Update comment - uses: peter-evans/create-or-update-comment@v3.1.0 + uses: peter-evans/create-or-update-comment@v4.0.0 with: token: ${{ secrets.PAT_GHOST }} repository: ${{ github.event.client_payload.github.payload.repository.full_name }} From 331560e3ccb68e7c845559eedf956b3482d3dfd3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 2 Feb 2024 00:50:56 +0100 Subject: [PATCH 5/9] build(deps): bump peter-evans/slash-command-dispatch from 3.0.2 to 4.0.0 (#2344) Bumps [peter-evans/slash-command-dispatch](https://github.com/peter-evans/slash-command-dispatch) from 3.0.2 to 4.0.0. - [Release notes](https://github.com/peter-evans/slash-command-dispatch/releases) - [Commits](https://github.com/peter-evans/slash-command-dispatch/compare/v3.0.2...v4.0.0) --- updated-dependencies: - dependency-name: peter-evans/slash-command-dispatch dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/slash-cmd-dispatch.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/slash-cmd-dispatch.yml b/.github/workflows/slash-cmd-dispatch.yml index 8b115747e5d..12e042871bf 100644 --- a/.github/workflows/slash-cmd-dispatch.yml +++ b/.github/workflows/slash-cmd-dispatch.yml @@ -8,7 +8,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Slash Command Dispatch - uses: peter-evans/slash-command-dispatch@v3.0.2 + uses: peter-evans/slash-command-dispatch@v4.0.0 with: token: ${{ secrets.PAT_GHOST }} reaction-token: ${{ secrets.PAT_GHOST }} From 548e30243999805e666c8805f856da1ecd3d1fac Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 2 Feb 2024 00:52:06 +0100 Subject: [PATCH 6/9] build(deps): bump actions/setup-python from 4 to 5 (#2342) Bumps [actions/setup-python](https://github.com/actions/setup-python) from 4 to 5. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v4...v5) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci-integrate.yml | 2 +- .github/workflows/ci-tests.yml | 2 +- .github/workflows/docs-build.yml | 2 +- .github/workflows/focus-diff.yml | 2 +- .github/workflows/publish-pkg.yml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci-integrate.yml b/.github/workflows/ci-integrate.yml index 523df5598cd..245ecc08ac5 100644 --- a/.github/workflows/ci-integrate.yml +++ b/.github/workflows/ci-integrate.yml @@ -46,7 +46,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 9940bd84b13..60ead38b3b6 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -64,7 +64,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/docs-build.yml b/.github/workflows/docs-build.yml index a34fab9a73e..5751a98784c 100644 --- a/.github/workflows/docs-build.yml +++ b/.github/workflows/docs-build.yml @@ -36,7 +36,7 @@ jobs: timeout-minutes: 30 steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: "3.9" diff --git a/.github/workflows/focus-diff.yml b/.github/workflows/focus-diff.yml index 4b374199d69..246eff24c1a 100644 --- a/.github/workflows/focus-diff.yml +++ b/.github/workflows/focus-diff.yml @@ -17,7 +17,7 @@ jobs: focus: ${{ steps.diff-domains.outputs.focus }} steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 #with: # python-version: 3.8 diff --git a/.github/workflows/publish-pkg.yml b/.github/workflows/publish-pkg.yml index d9cfa786de9..c556bfbe1c7 100644 --- a/.github/workflows/publish-pkg.yml +++ b/.github/workflows/publish-pkg.yml @@ -18,7 +18,7 @@ jobs: timeout-minutes: 10 steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: 3.8 From 72cb852074b38f73aecf7f83cec3e1584f46e488 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 2 Feb 2024 00:53:24 +0100 Subject: [PATCH 7/9] build(deps): bump coverage from 7.4.0 to 7.4.1 in /requirements (#2334) Bumps [coverage](https://github.com/nedbat/coveragepy) from 7.4.0 to 7.4.1. - [Release notes](https://github.com/nedbat/coveragepy/releases) - [Changelog](https://github.com/nedbat/coveragepy/blob/master/CHANGES.rst) - [Commits](https://github.com/nedbat/coveragepy/compare/7.4.0...7.4.1) --- updated-dependencies: - dependency-name: coverage dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements/_tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/_tests.txt b/requirements/_tests.txt index 243db7b501d..19dd63253f1 100644 --- a/requirements/_tests.txt +++ b/requirements/_tests.txt @@ -1,7 +1,7 @@ # NOTE: the upper bound for the package version is only set for CI stability, and it is dropped while installing this package # in case you want to preserve/enforce restrictions on the latest compatible version, add "strict" as an in-line comment -coverage ==7.4.0 +coverage ==7.4.1 pytest ==7.4.4 pytest-cov ==4.1.0 pytest-doctestplus ==1.1.0 From a7424fd8b9776a4fb1917751c84aaeb9478624a3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 2 Feb 2024 15:45:38 +0100 Subject: [PATCH 8/9] build(deps): bump pytest-xdist from 3.3.1 to 3.5.0 in /requirements (#2308) Bumps [pytest-xdist](https://github.com/pytest-dev/pytest-xdist) from 3.3.1 to 3.5.0. - [Release notes](https://github.com/pytest-dev/pytest-xdist/releases) - [Changelog](https://github.com/pytest-dev/pytest-xdist/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest-xdist/compare/v3.3.1...v3.5.0) --- updated-dependencies: - dependency-name: pytest-xdist dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Jirka Borovec <6035284+Borda@users.noreply.github.com> --- requirements/_tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/_tests.txt b/requirements/_tests.txt index 19dd63253f1..7a58207fb16 100644 --- a/requirements/_tests.txt +++ b/requirements/_tests.txt @@ -7,7 +7,7 @@ pytest-cov ==4.1.0 pytest-doctestplus ==1.1.0 pytest-rerunfailures ==13.0 pytest-timeout ==2.2.0 -pytest-xdist ==3.3.1 +pytest-xdist ==3.5.0 phmdoctest ==1.4.0 psutil <5.10.0 From b187bfd60531234de756afd661500963bbedceca Mon Sep 17 00:00:00 2001 From: Jirka Borovec <6035284+Borda@users.noreply.github.com> Date: Tue, 6 Feb 2024 09:50:05 +0100 Subject: [PATCH 9/9] ci: redo HF cashing (#2350) * ci: redo HF cashing * show py-tree * drop CUDA_LAUNCH_BLOCKING * missing torch.use_deterministic_algorithms(False) * reruns 3 --- .azure/gpu-integrations.yml | 4 +- .azure/gpu-unittests.yml | 19 ++++---- .github/actions/pull-caches/action.yml | 49 +++++++++------------ .github/actions/push-caches/action.yml | 33 +++++--------- tests/unittests/utilities/test_utilities.py | 3 +- 5 files changed, 44 insertions(+), 64 deletions(-) diff --git a/.azure/gpu-integrations.yml b/.azure/gpu-integrations.yml index 78511d22254..0d48f9f57ff 100644 --- a/.azure/gpu-integrations.yml +++ b/.azure/gpu-integrations.yml @@ -33,7 +33,9 @@ jobs: DEVICES: $( python -c 'name = "$(Agent.Name)" ; gpus = name.split("_")[-1] if "_" in name else "0,1"; print(gpus)' ) # these two caches assume to run repetitively on the same set of machines TORCH_HOME: "/var/tmp/torch" - HF_HOME: "/var/tmp/huggingface" + TRANSFORMERS_CACHE: "/var/tmp/hf/transformers" + HF_HOME: "/var/tmp/hf/home" + HF_HUB_CACHE: "/var/tmp/hf/hub" PIP_CACHE_DIR: "/var/tmp/pip" container: image: "$(docker-image)" diff --git a/.azure/gpu-unittests.yml b/.azure/gpu-unittests.yml index 5175830e4c5..85eda36371c 100644 --- a/.azure/gpu-unittests.yml +++ b/.azure/gpu-unittests.yml @@ -43,7 +43,9 @@ jobs: # see: https://github.com/microsoft/azure-pipelines-agent/issues/4113#issuecomment-1439241481 TORCH_HOME: "/var/tmp/torch" TOKENIZERS_PARALLELISM: "false" - HF_HOME: "/var/tmp/huggingface" + TRANSFORMERS_CACHE: "/var/tmp/hf/transformers" + HF_HOME: "/var/tmp/hf/home" + HF_HUB_CACHE: "/var/tmp/hf/hub" PIP_CACHE_DIR: "/var/tmp/pip" # MKL_THREADING_LAYER: "GNU" MKL_SERVICE_FORCE_INTEL: 1 @@ -105,12 +107,9 @@ jobs: displayName: "Sanity check" - bash: | - printf "cache location: $(TORCH_HOME)\n" - mkdir -p $(TORCH_HOME) # in case cache was void - ls -lh $(TORCH_HOME) - printf "cache location: $(HF_HOME)\n" - mkdir -p $(HF_HOME) # in case cache was void - ls -lh $(HF_HOME) + pip install -q py-tree + py-tree /var/tmp/torch + py-tree /var/tmp/hf displayName: "Show caches" - bash: | @@ -133,9 +132,8 @@ jobs: - bash: | python -m pytest unittests -v \ -m "not DDP" --numprocesses=5 --dist=loadfile \ - --cov=torchmetrics --timeout=240 --durations=500 - env: - CUDA_LAUNCH_BLOCKING: "1" + --cov=torchmetrics --timeout=240 --durations=200 \ + --reruns 3 --reruns-delay 1 workingDirectory: tests displayName: "UnitTesting common" @@ -145,7 +143,6 @@ jobs: --cov=torchmetrics --timeout=240 --durations=500 env: USE_PYTEST_POOL: "1" - CUDA_LAUNCH_BLOCKING: "1" workingDirectory: tests displayName: "UnitTesting DDP" diff --git a/.github/actions/pull-caches/action.yml b/.github/actions/pull-caches/action.yml index c8074e64356..7e3071e8a23 100644 --- a/.github/actions/pull-caches/action.yml +++ b/.github/actions/pull-caches/action.yml @@ -14,6 +14,10 @@ inputs: description: location of local PyPI cache required: false default: "_ci-cache_PyPI" + pypi-key: + description: cache restore/dump key + required: false + default: "pypi-packages" runs: using: "composite" @@ -38,18 +42,22 @@ runs: - name: Define caches id: cache_dirs run: | - torch_cache=$(python -c "import os ; print(os.path.join(os.getcwd(), '_ci-cache_pytorch'))") - echo "TORCH_HOME=$torch_cache" >> $GITHUB_ENV - hf_cache=$(python -c "import os ; print(os.path.join(os.getcwd(), '_ci-cache_huggingface'))") - echo "HF_HOME=$hf_cache" >> $GITHUB_ENV + cache_dir=$(python -c "import os ; print(os.path.join(os.getcwd(), '_ci-cache'))") + echo "CACHES_DIR=${cache_dir}" >> $GITHUB_ENV + dir_sep=$(python -c "import os ; print(os.path.sep)") + echo "TORCH_HOME=${cache_dir}${dir_sep}torch" >> $GITHUB_ENV + echo "TRANSFORMERS_CACHE=${cache_dir}${dir_sep}transformers" >> $GITHUB_ENV + echo "HF_HOME=${cache_dir}${dir_sep}hf-home" >> $GITHUB_ENV + echo "HF_HUB_CACHE=${cache_dir}${dir_sep}hf-hub" >> $GITHUB_ENV shell: bash - name: Cache pip continue-on-error: true uses: actions/cache/restore@v3 with: + enableCrossOsArchive: true path: ${{ inputs.pypi-dir }} - key: pypi-packages + key: ${{ inputs.pypi-key }} - name: Restored Packages run: | @@ -58,32 +66,17 @@ runs: ls -lh ${{ inputs.pypi-dir }} shell: bash - - name: Cache Torch - continue-on-error: true - uses: actions/cache/restore@v3 - with: - path: ${{ env.TORCH_HOME }} - key: cache-pytorch - - - name: Restored PT - if: ${{ runner.os == 'Linux' }} - run: | - mkdir -p $TORCH_HOME - printf "list $TORCH_HOME:\n" - sudo apt install -q -y tree - tree -h $TORCH_HOME - shell: bash - - - name: Cache HF + - name: Cache Torch & HF continue-on-error: true uses: actions/cache/restore@v3 with: - path: ${{ env.HF_HOME }} - key: cache-transformers + enableCrossOsArchive: true + path: ${{ env.CACHES_DIR }} + key: ci-caches - - name: Restored HF + - name: Restored Torch & HF run: | - mkdir -p $HF_HOME - printf "list $HF_HOME:\n" - ls -lh $HF_HOME + mkdir -p $CACHES_DIR + pip install -q py-tree + py-tree $CACHES_DIR shell: bash diff --git a/.github/actions/push-caches/action.yml b/.github/actions/push-caches/action.yml index c8cbdfedaa8..3215cf1a767 100644 --- a/.github/actions/push-caches/action.yml +++ b/.github/actions/push-caches/action.yml @@ -47,9 +47,10 @@ runs: ls -lh .pip-wheels shell: bash - - name: Cache pull + - name: Cache pull packages uses: actions/cache/restore@v3 with: + enableCrossOsArchive: true path: ${{ inputs.pypi-dir }} key: ${{ inputs.pypi-key }} @@ -69,36 +70,22 @@ runs: if: ${{ steps.wheels-diff.outputs.count-new != 0 }} shell: bash - - name: Cache push + - name: Cache push packages if: ${{ steps.wheels-diff.outputs.count-new != 0 }} uses: actions/cache/save@v3 with: + enableCrossOsArchive: true path: ${{ inputs.pypi-dir }} key: ${{ inputs.pypi-key }} - - name: Post PT - if: ${{ runner.os == 'Linux' }} - run: | - printf "list $TORCH_HOME:\n" - tree -h $TORCH_HOME - shell: bash - - - name: Cache Torch - continue-on-error: true - uses: actions/cache/save@v3 - with: - path: ${{ env.TORCH_HOME }} - key: cache-pytorch - - - name: Post HF - run: | - printf "list $HF_HOME:\n" - ls -lh $HF_HOME + - name: Post Torch & HF + run: py-tree $CACHES_DIR shell: bash - - name: Cache HF + - name: Cache Torch & HF continue-on-error: true uses: actions/cache/save@v3 with: - path: ${{ env.HF_HOME }} - key: cache-transformers + enableCrossOsArchive: true + path: ${{ env.CACHES_DIR }} + key: ci-caches diff --git a/tests/unittests/utilities/test_utilities.py b/tests/unittests/utilities/test_utilities.py index 48c22139dbd..9cffa7699c6 100644 --- a/tests/unittests/utilities/test_utilities.py +++ b/tests/unittests/utilities/test_utilities.py @@ -186,7 +186,7 @@ def test_recursive_allclose(inputs, expected): def test_cumsum_still_not_supported(): """Make sure that cumsum on gpu and deterministic mode still fails. - If this test begins to passes, it means newer Pytorch versions support this and we can drop internal support. + If this test begins to pass, it means newer Pytorch versions support this and we can drop internal support. """ torch.use_deterministic_algorithms(True) @@ -207,6 +207,7 @@ def test_custom_cumsum(): res = _cumsum(x, dim=0).cpu() else: res = _cumsum(x, dim=0).cpu() + torch.use_deterministic_algorithms(False) res2 = np.cumsum(x.cpu(), axis=0) assert torch.allclose(res, res2)