pin integration test dependencies, refactor constants in tests (#155)

requirements-integration.in

@@ -1,3 +1,5 @@
+# This is required due to the abstraction of cos integration
+charmed-kubeflow-chisme>=0.4.0
 # Pinning to <4.0 due to compatibility with the 3.1 controller version
 juju<4.0
 lightkube

requirements-integration.txt

@@ -8,6 +8,8 @@ anyio==4.0.0
     # via httpcore
 asttokens==2.4.0
     # via stack-data
+attrs==23.2.0
+    # via jsonschema
 backcall==0.2.0
     # via ipython
 bcrypt==4.0.1
@@ -24,6 +26,8 @@ cffi==1.15.1
     # via
     #   cryptography
     #   pynacl
+charmed-kubeflow-chisme==0.4.0
+    # via -r requirements-integration.in
 charset-normalizer==3.2.0
     # via requests
 cryptography==41.0.3
@@ -32,6 +36,8 @@ decorator==5.1.1
     # via
     #   ipdb
     #   ipython
+deepdiff==6.2.1
+    # via charmed-kubeflow-chisme
 exceptiongroup==1.1.3
     # via
     #   anyio
@@ -53,6 +59,8 @@ idna==3.4
     #   anyio
     #   httpx
     #   requests
+importlib-resources==6.4.0
+    # via jsonschema
 iniconfig==2.0.0
     # via pytest
 ipdb==0.13.13
@@ -62,15 +70,22 @@ ipython==8.12.2
 jedi==0.19.0
     # via ipython
 jinja2==3.1.2
-    # via pytest-operator
+    # via
+    #   charmed-kubeflow-chisme
+    #   pytest-operator
+jsonschema==4.17.3
+    # via serialized-data-interface
 juju==3.2.2
     # via
     #   -r requirements-integration.in
+    #   charmed-kubeflow-chisme
     #   pytest-operator
 kubernetes==27.2.0
     # via juju
 lightkube==0.14.0
-    # via -r requirements-integration.in
+    # via
+    #   -r requirements-integration.in
+    #   charmed-kubeflow-chisme
 lightkube-models==1.28.1.4
     # via lightkube
 macaroonbakery==1.3.1
@@ -85,6 +100,12 @@ oauthlib==3.2.2
     # via
     #   kubernetes
     #   requests-oauthlib
+ops==2.14.0
+    # via
+    #   charmed-kubeflow-chisme
+    #   serialized-data-interface
+ordered-set==4.1.0
+    # via deepdiff
 packaging==23.1
     # via pytest
 paramiko==2.12.0
@@ -95,6 +116,8 @@ pexpect==4.8.0
     # via ipython
 pickleshare==0.7.5
     # via ipython
+pkgutil-resolve-name==1.3.10
+    # via jsonschema
 pluggy==1.3.0
     # via pytest
 prompt-toolkit==3.0.39
@@ -129,6 +152,8 @@ pyrfc3339==1.1
     # via
     #   juju
     #   macaroonbakery
+pyrsistent==0.20.0
+    # via jsonschema
 pytest==7.4.2
     # via
     #   -r requirements-integration.in
@@ -148,18 +173,27 @@ pyyaml==6.0.1
     #   juju
     #   kubernetes
     #   lightkube
+    #   ops
     #   pytest-operator
+    #   serialized-data-interface
 requests==2.31.0
     # via
     #   -r requirements-integration.in
     #   hvac
     #   kubernetes
     #   macaroonbakery
     #   requests-oauthlib
+    #   serialized-data-interface
 requests-oauthlib==1.3.1
     # via kubernetes
 rsa==4.9
     # via google-auth
+ruamel-yaml==0.18.6
+    # via charmed-kubeflow-chisme
+ruamel-yaml-clib==0.2.8
+    # via ruamel-yaml
+serialized-data-interface==0.7.0
+    # via charmed-kubeflow-chisme
 six==1.16.0
     # via
     #   asttokens
@@ -177,7 +211,9 @@ sniffio==1.3.0
 stack-data==0.6.2
     # via ipython
 tenacity==8.2.3
-    # via -r requirements-integration.in
+    # via
+    #   -r requirements-integration.in
+    #   charmed-kubeflow-chisme
 tomli==2.0.1
     # via
     #   ipdb
@@ -202,6 +238,10 @@ urllib3==1.26.16
 wcwidth==0.2.6
     # via prompt-toolkit
 websocket-client==1.6.2
-    # via kubernetes
+    # via
+    #   kubernetes
+    #   ops
 websockets==8.1
     # via juju
+zipp==3.19.2
+    # via importlib-resources

tests/integration/test_charm.py

@@ -2,17 +2,21 @@
 # See LICENSE file for licensing details.
 
 import glob
-import json
 import logging
 from pathlib import Path
 
 import lightkube
 import lightkube.codecs
 import lightkube.generic_resource
 import pytest
-import requests
 import tenacity
 import yaml
+from charmed_kubeflow_chisme.testing import (
+    assert_alert_rules,
+    assert_metrics_endpoint,
+    deploy_and_assert_grafana_agent,
+    get_alert_rules,
+)
 from lightkube.resources.apiextensions_v1 import CustomResourceDefinition
 from lightkube.resources.rbac_authorization_v1 import ClusterRole
 from pytest_operator.plugin import OpsTest
@@ -22,6 +26,9 @@
 METADATA = yaml.safe_load(Path("./metadata.yaml").read_text())
 APP_NAME = "training-operator"
 CHARM_LOCATION = None
+APP_PREVIOUS_CHANNEL = "1.7/stable"
+METRICS_PATH = "/metrics"
+METRICS_PORT = 8080
 
 
 @pytest.mark.abort_on_fail
@@ -46,6 +53,9 @@ async def test_build_and_deploy(ops_test: OpsTest):
     global CHARM_LOCATION
     CHARM_LOCATION = charm_under_test
 
+    # Deploy grafana-agent for COS integration tests
+    await deploy_and_assert_grafana_agent(ops_test.model, APP_NAME, metrics=True)
+
 
 def lightkube_create_global_resources() -> dict:
     """Returns a dict with GenericNamespacedResource as value for each CRD key."""
@@ -128,78 +138,25 @@ def assert_job_status_running_success():
     assert_job_status_running_success()
 
 
-async def test_prometheus_grafana_integration(ops_test: OpsTest):
-    """Deploy prometheus, grafana and required relations, then test the metrics."""
-    prometheus = "prometheus-k8s"
-    grafana = "grafana-k8s"
-    prometheus_scrape = "prometheus-scrape-config-k8s"
-    scrape_config = {"scrape_interval": "30s"}
-
-    # Deploy and relate prometheus
-    # FIXME: Unpin revision once https://github.com/canonical/bundle-kubeflow/issues/688 is closed
-    await ops_test.juju(
-        "deploy",
-        prometheus,
-        "--channel",
-        "latest/edge",
-        "--revision",
-        "137",
-        "--trust",
-        check=True,
-    )
-    # FIXME: Unpin revision once https://github.com/canonical/bundle-kubeflow/issues/690 is closed
-    await ops_test.juju(
-        "deploy",
-        grafana,
-        "--channel",
-        "latest/edge",
-        "--revision",
-        "89",
-        "--trust",
-        check=True,
-    )
-    await ops_test.model.deploy(prometheus_scrape, channel="latest/beta", config=scrape_config)
-
-    await ops_test.model.add_relation(APP_NAME, prometheus_scrape)
-    await ops_test.model.add_relation(
-        f"{prometheus}:grafana-dashboard", f"{grafana}:grafana-dashboard"
-    )
-    await ops_test.model.add_relation(
-        f"{APP_NAME}:grafana-dashboard", f"{grafana}:grafana-dashboard"
-    )
-    await ops_test.model.add_relation(
-        f"{prometheus}:metrics-endpoint", f"{prometheus_scrape}:metrics-endpoint"
-    )
+async def test_alert_rules(ops_test: OpsTest):
+    """Test check charm alert rules and rules defined in relation data bag."""
+    app = ops_test.model.applications[APP_NAME]
+    alert_rules = get_alert_rules()
+    logger.info("found alert_rules: %s", alert_rules)
+    for attempt in retry_for_5_attempts:
+        await assert_alert_rules(app, alert_rules)
 
-    await ops_test.model.wait_for_idle(status="active", timeout=60 * 20)
 
-    status = await ops_test.model.get_status()
-    prometheus_unit_ip = status["applications"][prometheus]["units"][f"{prometheus}/0"]["address"]
-    logger.info(f"Prometheus available at http://{prometheus_unit_ip}:9090")
+async def test_metrics_enpoint(ops_test: OpsTest):
+    """Test metrics_endpoints are defined in relation data bag and their accessibility.
 
+    This function gets all the metrics_endpoints from the relation data bag, checks if
+    they are available from the grafana-agent-k8s charm and finally compares them with the
+    ones provided to the function.
+    """
+    app = ops_test.model.applications[APP_NAME]
     for attempt in retry_for_5_attempts:
-        logger.info(
-            f"Testing prometheus deployment (attempt " f"{attempt.retry_state.attempt_number})"
-        )
-        with attempt:
-            r = requests.get(
-                f"http://{prometheus_unit_ip}:9090/api/v1/query?"
-                f'query=up{{juju_application="{APP_NAME}"}}'
-            )
-            response = json.loads(r.content.decode("utf-8"))
-            response_status = response["status"]
-            logger.info(f"Response status is {response_status}")
-            assert response_status == "success"
-
-            # Assert the unit is available by checking the query result
-            # The data is presented as a list [1707357912.349, '1'], where the
-            # first value is a timestamp and the second value is the state of the unit
-            # 1 means available, 0 means unavailable
-            assert response["data"]["result"][0]["value"][1] == "1"
-
-            response_metric = response["data"]["result"][0]["metric"]
-            assert response_metric["juju_application"] == APP_NAME
-            assert response_metric["juju_model"] == ops_test.model_name
+        await assert_metrics_endpoint(app, metrics_port=METRICS_PORT, metrics_path=METRICS_PATH)
 
 
 # Helper to retry calling a function over 30 seconds or 5 attempts
@@ -245,7 +202,7 @@ async def test_upgrade(ops_test: OpsTest):
     """
 
     # deploy stable version of the charm
-    await ops_test.model.deploy(entity_url=APP_NAME, channel="1.5/stable", trust=True)
+    await ops_test.model.deploy(entity_url=APP_NAME, channel=APP_PREVIOUS_CHANNEL, trust=True)
     await ops_test.model.wait_for_idle(
         apps=[APP_NAME], status="active", raise_on_blocked=True, timeout=60 * 10
     )

tests/integration/test_charm_with_profile.py

@@ -30,6 +30,13 @@
 METADATA = yaml.safe_load(Path("./metadata.yaml").read_text())
 APP_NAME = "training-operator"
 
+KUBEFLOW_ROLES = "kubeflow-roles"
+KUBEFLOW_ROLES_CHANNEL = "latest/edge"
+KUBEFLOW_ROLES_TRUST = True
+KUBEFLOW_PROFILES = "kubeflow-profiles"
+KUBEFLOW_PROFILES_CHANNEL = "latest/edge"
+KUBEFLOW_PROFILES_TRUST = True
+
 log = logging.getLogger(__name__)
 
 
@@ -46,14 +53,14 @@ async def test_build_and_deploy(ops_test: OpsTest):
 
     # Deploy kubeflow-roles and kubeflow-profiles to create a Profile
     await ops_test.model.deploy(
-        entity_url="kubeflow-roles",
-        channel="latest/edge",
-        trust=True,
+        entity_url=KUBEFLOW_ROLES,
+        channel=KUBEFLOW_ROLES_CHANNEL,
+        trust=KUBEFLOW_ROLES_TRUST,
     )
     await ops_test.model.deploy(
-        entity_url="kubeflow-profiles",
-        channel="latest/edge",
-        trust=True,
+        entity_url=KUBEFLOW_PROFILES,
+        channel=KUBEFLOW_PROFILES_CHANNEL,
+        trust=KUBEFLOW_PROFILES_TRUST,
     )
 
     await ops_test.model.wait_for_idle(