action, conftest: initial xfail support (#95)

* action, conftest: initial xfail support

Signed-off-by: William Woodruff <william@trailofbits.com>

* action: fix --skip-signing behavior

Signed-off-by: William Woodruff <william@trailofbits.com>

* action: remove unused helpers

Signed-off-by: William Woodruff <william@trailofbits.com>

* conformance: test xfail temporarily

Signed-off-by: William Woodruff <william@trailofbits.com>

* Revert "conformance: test xfail temporarily"

This reverts commit 25ffed8.

* treewide: lintage, fixup lint tools

Signed-off-by: William Woodruff <william@trailofbits.com>

---------

Signed-off-by: William Woodruff <william@trailofbits.com>

Makefile

@@ -18,6 +18,5 @@ dev: env/pyvenv.cfg
 .PHONY: lint
 lint: env/pyvenv.cfg $(ALL_PY_SRCS)
 	./env/bin/python -m black $(ALL_PY_SRCS)
-	./env/bin/python -m isort $(ALL_PY_SRCS)
-	./env/bin/python -m flake8 --max-line-length 100 $(ALL_PY_SRCS)
+	./env/bin/python -m ruff --fix $(ALL_PY_SRCS)
 	./env/bin/python -m mypy action.py test/

action.py

@@ -12,9 +12,7 @@
 
 _SUMMARY = Path(os.getenv("GITHUB_STEP_SUMMARY")).open("a")  # type: ignore
 _RENDER_SUMMARY = os.getenv("GHA_SIGSTORE_CONFORMANCE_SUMMARY", "true") == "true"
-_DEBUG = (
-    os.getenv("GHA_SIGSTORE_CONFORMANCE_INTERNAL_BE_CAREFUL_DEBUG", "false") != "false"
-)
+_DEBUG = os.getenv("GHA_SIGSTORE_CONFORMANCE_INTERNAL_BE_CAREFUL_DEBUG", "false") != "false"
 _ACTION_PATH = Path(os.getenv("GITHUB_ACTION_PATH"))  # type: ignore
 
 
@@ -28,19 +26,10 @@ def _debug(msg):
         print(f"\033[93mDEBUG: {msg}\033[0m", file=sys.stderr)
 
 
-def _log(msg):
-    print(msg, file=sys.stderr)
-
-
 def _sigstore_conformance(*args) -> int:
     return pytest.main([str(_ACTION_PATH / "test"), *args])
 
 
-def _fatal_help(msg):
-    print(f"::error::❌ {msg}")
-    sys.exit(1)
-
-
 sigstore_conformance_args = []
 
 if _DEBUG:
@@ -50,7 +39,7 @@ def _fatal_help(msg):
 if entrypoint:
     sigstore_conformance_args.extend(["--entrypoint", entrypoint])
 
-skip_signing = os.getenv("GHA_SIGSTORE_CONFORMANCE_SKIP_SIGNING")
+skip_signing = os.getenv("GHA_SIGSTORE_CONFORMANCE_SKIP_SIGNING", "false").lower() == "true"
 if skip_signing:
     sigstore_conformance_args.extend(["--skip-signing"])
 

action.yml

@@ -15,6 +15,10 @@ inputs:
     description: "skip tests that involve signing (default false)"
     required: false
     default: "false"
+  xfail:
+    description: "one or more tests that are expected to fail, whitespace-separated"
+    required: false
+    default: ""
 
 runs:
   using: "composite"
@@ -33,5 +37,6 @@ runs:
         GHA_SIGSTORE_CONFORMANCE_ENTRYPOINT: "${{ inputs.entrypoint }}"
         GHA_SIGSTORE_CONFORMANCE_INTERNAL_BE_CAREFUL_DEBUG: "${{ inputs.internal-be-careful-debug }}"
         GHA_SIGSTORE_CONFORMANCE_SKIP_SIGNING: "${{ inputs.skip-signing }}"
+        GHA_SIGSTORE_CONFORMANCE_XFAIL: "${{ inputs.xfail }}"
         GHA_SIGSTORE_GITHUB_TOKEN: "${{ github.token }}"
       shell: bash

dev-requirements.txt

@@ -1,5 +1,4 @@
-flake8
-isort
+ruff
 black
 mypy
 types-requests

pyproject.toml

@@ -0,0 +1,6 @@
+[tool.black]
+line-length = 100
+
+[tool.ruff]
+line-length = 100
+select = ["E", "F", "I", "W", "UP"]

sigstore-python-conformance

@@ -26,9 +26,7 @@ if subcmd in SUBCMD_REPLACEMENTS:
     fixed_args[0] = SUBCMD_REPLACEMENTS[subcmd]
 
 # Replace incompatible flags.
-fixed_args = [
-    ARG_REPLACEMENTS[arg] if arg in ARG_REPLACEMENTS else arg for arg in fixed_args
-]
+fixed_args = [ARG_REPLACEMENTS[arg] if arg in ARG_REPLACEMENTS else arg for arg in fixed_args]
 
 # Fix-up the subcommand: the conformance suite uses `verify`, but
 # `sigstore` requires `verify identity` for identity based verifications.

test/client.py

@@ -172,9 +172,7 @@ def _sign_for_sigcrt(
         )
 
     @sign.register
-    def _sign_for_bundle(
-        self, materials: BundleMaterials, artifact: os.PathLike
-    ) -> None:
+    def _sign_for_bundle(self, materials: BundleMaterials, artifact: os.PathLike) -> None:
         """
         Sign an artifact with the Sigstore client, producing a bundle.
 
@@ -236,9 +234,7 @@ def _verify_for_sigcrt(
         )
 
     @verify.register
-    def _verify_for_bundle(
-        self, materials: BundleMaterials, artifact: os.PathLike
-    ) -> None:
+    def _verify_for_bundle(self, materials: BundleMaterials, artifact: os.PathLike) -> None:
         """
         Verify an artifact given a bundle with the Sigstore client.
 

test/conftest.py

@@ -2,34 +2,41 @@
 import shutil
 import tempfile
 import time
+from collections.abc import Callable
 from datetime import datetime, timedelta
 from io import BytesIO
 from pathlib import Path
-from typing import Callable, Optional, Tuple, TypeVar
+from typing import TypeVar
 from zipfile import ZipFile
 
-import pytest  # type: ignore
+import pytest
 import requests
 
-from .client import (BundleMaterials, SignatureCertificateMaterials,
-                     SigstoreClient, VerificationMaterials)
+from .client import (
+    BundleMaterials,
+    SignatureCertificateMaterials,
+    SigstoreClient,
+    VerificationMaterials,
+)
 
 _M = TypeVar("_M", bound=VerificationMaterials)
-_MakeMaterialsByType = Callable[[str, _M], Tuple[Path, _M]]
-_MakeMaterials = Callable[[str], Tuple[Path, VerificationMaterials]]
+_MakeMaterialsByType = Callable[[str, _M], tuple[Path, _M]]
+_MakeMaterials = Callable[[str], tuple[Path, VerificationMaterials]]
 
 _OIDC_BEACON_API_URL = (
     "https://api.github.com/repos/sigstore-conformance/extremely-dangerous-public-oidc-beacon/"
     "actions"
 )
 _OIDC_BEACON_WORKFLOW_ID = 55399612
 
+_XFAIL_LIST = os.getenv("GHA_SIGSTORE_CONFORMANCE_XFAIL", "").split()
+
 
 class OidcTokenError(Exception):
     pass
 
 
-def pytest_addoption(parser):
+def pytest_addoption(parser) -> None:
     """
     Add the `--entrypoint`, `--github-token`, and `--skip-signing` flags to
     the `pytest` CLI.
@@ -57,19 +64,15 @@ def pytest_addoption(parser):
 
 def pytest_runtest_setup(item):
     if "signing" in item.keywords and item.config.getoption("--skip-signing"):
-        pytest.skip(
-            "skipping test that requires signing support due to `--skip-signing` flag"
-        )
+        pytest.skip("skipping test that requires signing support due to `--skip-signing` flag")
 
 
 def pytest_configure(config):
-    config.addinivalue_line(
-        "markers", "signing: mark test as requiring signing functionality"
-    )
+    config.addinivalue_line("markers", "signing: mark test as requiring signing functionality")
 
 
 @pytest.fixture
-def identity_token(pytestconfig):
+def identity_token(pytestconfig) -> str:
     gh_token = pytestconfig.getoption("--github-token")
     session = requests.Session()
     headers = {
@@ -78,13 +81,11 @@ def identity_token(pytestconfig):
         "Authorization": f"Bearer {gh_token}",
     }
 
-    workflow_time: Optional[datetime] = None
+    workflow_time: datetime | None = None
     run_id: str
 
     # We need a token that was generated in the last 5 minutes. Keep checking until we find one.
-    while workflow_time is None or datetime.now() - workflow_time >= timedelta(
-        minutes=5
-    ):
+    while workflow_time is None or datetime.now() - workflow_time >= timedelta(minutes=5):
         # If there's a lot of traffic in the GitHub Actions cron queue, we might not have a valid
         # token to use. In that case, wait for 30 seconds and try again.
         if workflow_time is not None:
@@ -110,9 +111,7 @@ def identity_token(pytestconfig):
             continue
 
         run_id = workflow_run["id"]
-        workflow_time = datetime.strptime(
-            workflow_run["run_started_at"], "%Y-%m-%dT%H:%M:%SZ"
-        )
+        workflow_time = datetime.strptime(workflow_run["run_started_at"], "%Y-%m-%dT%H:%M:%SZ")
 
     resp = session.get(
         url=_OIDC_BEACON_API_URL + f"/runs/{run_id}/artifacts",
@@ -166,7 +165,7 @@ def make_materials_by_type() -> _MakeMaterialsByType:
 
     def _make_materials_by_type(
         input_name: str, cls: VerificationMaterials
-    ) -> Tuple[Path, VerificationMaterials]:
+    ) -> tuple[Path, VerificationMaterials]:
         input_path = Path(input_name)
         output = cls.from_input(input_path)
 
@@ -208,3 +207,9 @@ def workspace():
 
     yield Path(workspace.name)
     workspace.cleanup()
+
+
+@pytest.fixture(autouse=True)
+def conformance_xfail(request):
+    if request.node.originalname in _XFAIL_LIST:
+        request.node.add_marker(pytest.mark.xfail(reason="skipped by suite runner", strict=True))

test/test_bundle.py

@@ -7,9 +7,7 @@
 from sigstore_protobuf_specs.dev.sigstore.bundle.v1 import Bundle
 
 
-def test_verify(
-    client: SigstoreClient, make_materials_by_type: _MakeMaterialsByType
-) -> None:
+def test_verify(client: SigstoreClient, make_materials_by_type: _MakeMaterialsByType) -> None:
     """
     Test the happy path of verification
     """
@@ -29,9 +27,7 @@ def test_verify_rejects_root(
     """
 
     materials: BundleMaterials
-    input_path, materials = make_materials_by_type(
-        "has_root_in_chain.txt", BundleMaterials
-    )
+    input_path, materials = make_materials_by_type("has_root_in_chain.txt", BundleMaterials)
 
     with pytest.raises(ClientFail):
         client.verify(materials, input_path)

test/test_signature_verify.py

@@ -34,9 +34,7 @@ def test_verify_empty(client: SigstoreClient, make_materials: _MakeMaterials) ->
 
 
 @pytest.mark.signing
-def test_verify_mismatch(
-    client: SigstoreClient, make_materials: _MakeMaterials
-) -> None:
+def test_verify_mismatch(client: SigstoreClient, make_materials: _MakeMaterials) -> None:
     """
     Tests that verification fails with mismatching artifacts, certificates and
     signatures.
@@ -72,12 +70,8 @@ def test_verify_sigcrt(
     Test cases for the signature+certificate flow: empty sigs/crts and
     mismatched sigs/crts.
     """
-    a_artifact_path, a_materials = make_materials_by_type(
-        "a.txt", SignatureCertificateMaterials
-    )
-    b_artifact_path, b_materials = make_materials_by_type(
-        "b.txt", SignatureCertificateMaterials
-    )
+    a_artifact_path, a_materials = make_materials_by_type("a.txt", SignatureCertificateMaterials)
+    b_artifact_path, b_materials = make_materials_by_type("b.txt", SignatureCertificateMaterials)
 
     # Sign a.txt, b.txt.
     client.sign(a_materials, a_artifact_path)