From 95d23843cf9c4b83524ade2ec683052d1ea2b2f9 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Tue, 1 Aug 2023 17:02:52 -0400 Subject: [PATCH] action, conftest: initial xfail support (#95) * action, conftest: initial xfail support Signed-off-by: William Woodruff * action: fix --skip-signing behavior Signed-off-by: William Woodruff * action: remove unused helpers Signed-off-by: William Woodruff * conformance: test xfail temporarily Signed-off-by: William Woodruff * Revert "conformance: test xfail temporarily" This reverts commit 25ffed863670224af0cedaf3bae33c31c7e71f05. * treewide: lintage, fixup lint tools Signed-off-by: William Woodruff --------- Signed-off-by: William Woodruff --- Makefile | 3 +-- action.py | 15 ++--------- action.yml | 5 ++++ dev-requirements.txt | 3 +-- pyproject.toml | 6 +++++ sigstore-python-conformance | 4 +-- test/client.py | 8 ++---- test/conftest.py | 49 +++++++++++++++++++---------------- test/test_bundle.py | 8 ++---- test/test_signature_verify.py | 12 +++------ 10 files changed, 50 insertions(+), 63 deletions(-) create mode 100644 pyproject.toml diff --git a/Makefile b/Makefile index 6ccc8d8..0b43d5e 100644 --- a/Makefile +++ b/Makefile @@ -18,6 +18,5 @@ dev: env/pyvenv.cfg .PHONY: lint lint: env/pyvenv.cfg $(ALL_PY_SRCS) ./env/bin/python -m black $(ALL_PY_SRCS) - ./env/bin/python -m isort $(ALL_PY_SRCS) - ./env/bin/python -m flake8 --max-line-length 100 $(ALL_PY_SRCS) + ./env/bin/python -m ruff --fix $(ALL_PY_SRCS) ./env/bin/python -m mypy action.py test/ diff --git a/action.py b/action.py index 4dd7205..6e214d4 100755 --- a/action.py +++ b/action.py @@ -12,9 +12,7 @@ _SUMMARY = Path(os.getenv("GITHUB_STEP_SUMMARY")).open("a") # type: ignore _RENDER_SUMMARY = os.getenv("GHA_SIGSTORE_CONFORMANCE_SUMMARY", "true") == "true" -_DEBUG = ( - os.getenv("GHA_SIGSTORE_CONFORMANCE_INTERNAL_BE_CAREFUL_DEBUG", "false") != "false" -) +_DEBUG = os.getenv("GHA_SIGSTORE_CONFORMANCE_INTERNAL_BE_CAREFUL_DEBUG", "false") != "false" _ACTION_PATH = Path(os.getenv("GITHUB_ACTION_PATH")) # type: ignore @@ -28,19 +26,10 @@ def _debug(msg): print(f"\033[93mDEBUG: {msg}\033[0m", file=sys.stderr) -def _log(msg): - print(msg, file=sys.stderr) - - def _sigstore_conformance(*args) -> int: return pytest.main([str(_ACTION_PATH / "test"), *args]) -def _fatal_help(msg): - print(f"::error::❌ {msg}") - sys.exit(1) - - sigstore_conformance_args = [] if _DEBUG: @@ -50,7 +39,7 @@ def _fatal_help(msg): if entrypoint: sigstore_conformance_args.extend(["--entrypoint", entrypoint]) -skip_signing = os.getenv("GHA_SIGSTORE_CONFORMANCE_SKIP_SIGNING") +skip_signing = os.getenv("GHA_SIGSTORE_CONFORMANCE_SKIP_SIGNING", "false").lower() == "true" if skip_signing: sigstore_conformance_args.extend(["--skip-signing"]) diff --git a/action.yml b/action.yml index 8c0b68d..38a9d32 100644 --- a/action.yml +++ b/action.yml @@ -15,6 +15,10 @@ inputs: description: "skip tests that involve signing (default false)" required: false default: "false" + xfail: + description: "one or more tests that are expected to fail, whitespace-separated" + required: false + default: "" runs: using: "composite" @@ -33,5 +37,6 @@ runs: GHA_SIGSTORE_CONFORMANCE_ENTRYPOINT: "${{ inputs.entrypoint }}" GHA_SIGSTORE_CONFORMANCE_INTERNAL_BE_CAREFUL_DEBUG: "${{ inputs.internal-be-careful-debug }}" GHA_SIGSTORE_CONFORMANCE_SKIP_SIGNING: "${{ inputs.skip-signing }}" + GHA_SIGSTORE_CONFORMANCE_XFAIL: "${{ inputs.xfail }}" GHA_SIGSTORE_GITHUB_TOKEN: "${{ github.token }}" shell: bash diff --git a/dev-requirements.txt b/dev-requirements.txt index e3da308..803441f 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,5 +1,4 @@ -flake8 -isort +ruff black mypy types-requests diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..9a4b3a5 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,6 @@ +[tool.black] +line-length = 100 + +[tool.ruff] +line-length = 100 +select = ["E", "F", "I", "W", "UP"] diff --git a/sigstore-python-conformance b/sigstore-python-conformance index 3dcd085..427fd97 100755 --- a/sigstore-python-conformance +++ b/sigstore-python-conformance @@ -26,9 +26,7 @@ if subcmd in SUBCMD_REPLACEMENTS: fixed_args[0] = SUBCMD_REPLACEMENTS[subcmd] # Replace incompatible flags. -fixed_args = [ - ARG_REPLACEMENTS[arg] if arg in ARG_REPLACEMENTS else arg for arg in fixed_args -] +fixed_args = [ARG_REPLACEMENTS[arg] if arg in ARG_REPLACEMENTS else arg for arg in fixed_args] # Fix-up the subcommand: the conformance suite uses `verify`, but # `sigstore` requires `verify identity` for identity based verifications. diff --git a/test/client.py b/test/client.py index e148544..37b6de3 100644 --- a/test/client.py +++ b/test/client.py @@ -172,9 +172,7 @@ def _sign_for_sigcrt( ) @sign.register - def _sign_for_bundle( - self, materials: BundleMaterials, artifact: os.PathLike - ) -> None: + def _sign_for_bundle(self, materials: BundleMaterials, artifact: os.PathLike) -> None: """ Sign an artifact with the Sigstore client, producing a bundle. @@ -236,9 +234,7 @@ def _verify_for_sigcrt( ) @verify.register - def _verify_for_bundle( - self, materials: BundleMaterials, artifact: os.PathLike - ) -> None: + def _verify_for_bundle(self, materials: BundleMaterials, artifact: os.PathLike) -> None: """ Verify an artifact given a bundle with the Sigstore client. diff --git a/test/conftest.py b/test/conftest.py index f6bb628..131bebd 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -2,21 +2,26 @@ import shutil import tempfile import time +from collections.abc import Callable from datetime import datetime, timedelta from io import BytesIO from pathlib import Path -from typing import Callable, Optional, Tuple, TypeVar +from typing import TypeVar from zipfile import ZipFile -import pytest # type: ignore +import pytest import requests -from .client import (BundleMaterials, SignatureCertificateMaterials, - SigstoreClient, VerificationMaterials) +from .client import ( + BundleMaterials, + SignatureCertificateMaterials, + SigstoreClient, + VerificationMaterials, +) _M = TypeVar("_M", bound=VerificationMaterials) -_MakeMaterialsByType = Callable[[str, _M], Tuple[Path, _M]] -_MakeMaterials = Callable[[str], Tuple[Path, VerificationMaterials]] +_MakeMaterialsByType = Callable[[str, _M], tuple[Path, _M]] +_MakeMaterials = Callable[[str], tuple[Path, VerificationMaterials]] _OIDC_BEACON_API_URL = ( "https://api.github.com/repos/sigstore-conformance/extremely-dangerous-public-oidc-beacon/" @@ -24,12 +29,14 @@ ) _OIDC_BEACON_WORKFLOW_ID = 55399612 +_XFAIL_LIST = os.getenv("GHA_SIGSTORE_CONFORMANCE_XFAIL", "").split() + class OidcTokenError(Exception): pass -def pytest_addoption(parser): +def pytest_addoption(parser) -> None: """ Add the `--entrypoint`, `--github-token`, and `--skip-signing` flags to the `pytest` CLI. @@ -57,19 +64,15 @@ def pytest_addoption(parser): def pytest_runtest_setup(item): if "signing" in item.keywords and item.config.getoption("--skip-signing"): - pytest.skip( - "skipping test that requires signing support due to `--skip-signing` flag" - ) + pytest.skip("skipping test that requires signing support due to `--skip-signing` flag") def pytest_configure(config): - config.addinivalue_line( - "markers", "signing: mark test as requiring signing functionality" - ) + config.addinivalue_line("markers", "signing: mark test as requiring signing functionality") @pytest.fixture -def identity_token(pytestconfig): +def identity_token(pytestconfig) -> str: gh_token = pytestconfig.getoption("--github-token") session = requests.Session() headers = { @@ -78,13 +81,11 @@ def identity_token(pytestconfig): "Authorization": f"Bearer {gh_token}", } - workflow_time: Optional[datetime] = None + workflow_time: datetime | None = None run_id: str # We need a token that was generated in the last 5 minutes. Keep checking until we find one. - while workflow_time is None or datetime.now() - workflow_time >= timedelta( - minutes=5 - ): + while workflow_time is None or datetime.now() - workflow_time >= timedelta(minutes=5): # If there's a lot of traffic in the GitHub Actions cron queue, we might not have a valid # token to use. In that case, wait for 30 seconds and try again. if workflow_time is not None: @@ -110,9 +111,7 @@ def identity_token(pytestconfig): continue run_id = workflow_run["id"] - workflow_time = datetime.strptime( - workflow_run["run_started_at"], "%Y-%m-%dT%H:%M:%SZ" - ) + workflow_time = datetime.strptime(workflow_run["run_started_at"], "%Y-%m-%dT%H:%M:%SZ") resp = session.get( url=_OIDC_BEACON_API_URL + f"/runs/{run_id}/artifacts", @@ -166,7 +165,7 @@ def make_materials_by_type() -> _MakeMaterialsByType: def _make_materials_by_type( input_name: str, cls: VerificationMaterials - ) -> Tuple[Path, VerificationMaterials]: + ) -> tuple[Path, VerificationMaterials]: input_path = Path(input_name) output = cls.from_input(input_path) @@ -208,3 +207,9 @@ def workspace(): yield Path(workspace.name) workspace.cleanup() + + +@pytest.fixture(autouse=True) +def conformance_xfail(request): + if request.node.originalname in _XFAIL_LIST: + request.node.add_marker(pytest.mark.xfail(reason="skipped by suite runner", strict=True)) diff --git a/test/test_bundle.py b/test/test_bundle.py index e84d291..aa5f30a 100644 --- a/test/test_bundle.py +++ b/test/test_bundle.py @@ -7,9 +7,7 @@ from sigstore_protobuf_specs.dev.sigstore.bundle.v1 import Bundle -def test_verify( - client: SigstoreClient, make_materials_by_type: _MakeMaterialsByType -) -> None: +def test_verify(client: SigstoreClient, make_materials_by_type: _MakeMaterialsByType) -> None: """ Test the happy path of verification """ @@ -29,9 +27,7 @@ def test_verify_rejects_root( """ materials: BundleMaterials - input_path, materials = make_materials_by_type( - "has_root_in_chain.txt", BundleMaterials - ) + input_path, materials = make_materials_by_type("has_root_in_chain.txt", BundleMaterials) with pytest.raises(ClientFail): client.verify(materials, input_path) diff --git a/test/test_signature_verify.py b/test/test_signature_verify.py index 41c7099..302e37f 100644 --- a/test/test_signature_verify.py +++ b/test/test_signature_verify.py @@ -34,9 +34,7 @@ def test_verify_empty(client: SigstoreClient, make_materials: _MakeMaterials) -> @pytest.mark.signing -def test_verify_mismatch( - client: SigstoreClient, make_materials: _MakeMaterials -) -> None: +def test_verify_mismatch(client: SigstoreClient, make_materials: _MakeMaterials) -> None: """ Tests that verification fails with mismatching artifacts, certificates and signatures. @@ -72,12 +70,8 @@ def test_verify_sigcrt( Test cases for the signature+certificate flow: empty sigs/crts and mismatched sigs/crts. """ - a_artifact_path, a_materials = make_materials_by_type( - "a.txt", SignatureCertificateMaterials - ) - b_artifact_path, b_materials = make_materials_by_type( - "b.txt", SignatureCertificateMaterials - ) + a_artifact_path, a_materials = make_materials_by_type("a.txt", SignatureCertificateMaterials) + b_artifact_path, b_materials = make_materials_by_type("b.txt", SignatureCertificateMaterials) # Sign a.txt, b.txt. client.sign(a_materials, a_artifact_path)