Merge branch 'main' into utilities_stress_stability

.version_information

@@ -1 +1 @@
-v2.11-beta1+fall2024
+v2.11-beta3+fall2024

alembic/versions/e2c185af1226_pics_v2_support.py

@@ -0,0 +1,70 @@
+"""pics_v2_support
+
+Revision ID: e2c185af1226
+Revises: 9df8004ad9bb
+Create Date: 2024-06-19 11:46:15.158526
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "e2c185af1226"
+down_revision = "9df8004ad9bb"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column(
+        "testrunexecution",
+        sa.Column("certification_mode", sa.Boolean(), nullable=True, default=False),
+    )
+    op.add_column(
+        "testsuiteexecution",
+        sa.Column("mandatory", sa.Boolean(), nullable=True, default=False),
+    )
+
+    op.add_column(
+        "testsuitemetadata",
+        sa.Column("mandatory", sa.Boolean(), nullable=True, default=False),
+    )
+    op.add_column(
+        "testcasemetadata",
+        sa.Column("mandatory", sa.Boolean(), nullable=True, default=False),
+    )
+
+    op.execute("UPDATE testrunexecution SET certification_mode = false")
+    op.execute("UPDATE testsuiteexecution SET mandatory = false")
+    op.execute("UPDATE testsuitemetadata SET mandatory = false")
+    op.execute("UPDATE testcasemetadata SET mandatory = false")
+
+    op.alter_column(
+        "testrunexecution",
+        "certification_mode",
+        existing_type=sa.Boolean(),
+        nullable=False,
+    )
+    op.alter_column(
+        "testsuiteexecution", "mandatory", existing_type=sa.Boolean(), nullable=False
+    )
+    op.alter_column(
+        "testsuitemetadata",
+        sa.Column("mandatory", nullable=False, default=False),
+    )
+    op.alter_column(
+        "testcasemetadata",
+        sa.Column("mandatory", nullable=False, default=False),
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column("testrunexecution", "certification_mode")
+    op.drop_column("testsuiteexecution", "mandatory")
+    op.drop_column("testsuitemetadata", "mandatory")
+    op.drop_column("testcasemetadata", "mandatory")
+    # ### end Alembic commands ###

app/api/api_v1/endpoints/test_run_executions.py

@@ -81,11 +81,13 @@ def create_test_run_execution(
     db: Session = Depends(get_db),
     test_run_execution_in: schemas.TestRunExecutionCreate,
     selected_tests: schemas.TestSelection,
+    certification_mode: bool = False,
 ) -> TestRunExecution:
     """Create a new test run execution."""
 
     # TODO: Remove test_run_config completely from the project
     test_run_execution_in.test_run_config_id = None
+    test_run_execution_in.certification_mode = certification_mode
 
     test_run_execution = crud.test_run_execution.create(
         db=db, obj_in=test_run_execution_in, selected_tests=selected_tests
@@ -263,6 +265,7 @@ def repeat_test_run_execution(
     test_run_execution_in.description = execution_to_repeat.description
     test_run_execution_in.project_id = execution_to_repeat.project_id
     test_run_execution_in.operator_id = execution_to_repeat.operator_id
+    test_run_execution_in.certification_mode = execution_to_repeat.certification_mode
     # TODO: Remove test_run_config completely from the project
     test_run_execution_in.test_run_config_id = None
 

app/container_manager/container_manager.py

@@ -122,4 +122,4 @@ async def __container_started(self, container: Container) -> None:
             await asyncio.sleep(sleep_interval)
 
 
-container_manager = ContainerManager()
+container_manager: ContainerManager = ContainerManager()

app/crud/crud_test_run_execution.py

@@ -164,6 +164,25 @@ def __load_stats(
 
         return result
 
+    def __sort_selected_tests(
+        self, selected_tests: List[TestSuiteExecution]
+    ) -> List[TestSuiteExecution]:
+        """Sorts the selected tests, make the mandatories test cases the first to be
+        returned."""
+        sorted_selected_tests = []
+
+        # First add the mandatories test cases
+        for suite in selected_tests:
+            if suite.mandatory:
+                sorted_selected_tests.append(suite)
+
+        # Add the remaining test cases
+        for suite in selected_tests:
+            if not suite.mandatory:
+                sorted_selected_tests.append(suite)
+
+        return sorted_selected_tests
+
     def create(
         self,
         db: Session,
@@ -196,7 +215,9 @@ def create(
             )
         )
 
-        test_run_execution.test_suite_executions.extend(test_suites)
+        # Sorting test_suite according to mandatories suites
+        test_suites_sorted = self.__sort_selected_tests(test_suites)
+        test_run_execution.test_suite_executions.extend(test_suites_sorted)
 
         db.commit()
         db.refresh(test_run_execution)

app/models/test_case_metadata.py

@@ -37,6 +37,7 @@ class TestCaseMetadata(Base):
     description: Mapped[str] = mapped_column(Text, nullable=False)
     version: Mapped[str] = mapped_column(nullable=False)
     source_hash: Mapped[str] = mapped_column(VARCHAR(64), nullable=False, index=True)
+    mandatory: Mapped[bool] = mapped_column(default=False, nullable=False)
 
     created_at: Mapped[datetime] = mapped_column(default=datetime.now, nullable=False)
 

app/models/test_enums.py

@@ -24,5 +24,5 @@ class TestStateEnum(str, Enum):
     PASSED = "passed"  # Test Passed with no issued
     FAILED = "failed"  # Test Failed
     ERROR = "error"  # Test Error due to tool setup or environment
-    NOT_APPLICABLE = "not_applicable"  # TODO: Do we need this for full cert runs?
+    NOT_APPLICABLE = "not_applicable"  # Test is not applicable - e.g. PICS mismatch
     CANCELLED = "cancelled"

app/models/test_run_execution.py

@@ -50,6 +50,7 @@ class TestRunExecution(Base):
     completed_at: Mapped[Optional[datetime]]
     archived_at: Mapped[Optional[datetime]]
     imported_at: Mapped[Optional[datetime]]
+    certification_mode: Mapped[bool] = mapped_column(default=False, nullable=False)
 
     description: Mapped[Optional[str]] = mapped_column(default=None, nullable=True)
 

app/models/test_suite_execution.py

@@ -38,6 +38,7 @@ class TestSuiteExecution(Base):
     public_id: Mapped[str] = mapped_column(nullable=False)
     execution_index: Mapped[int] = mapped_column(nullable=False)
     collection_id: Mapped[str] = mapped_column(nullable=False)
+    mandatory: Mapped[bool] = mapped_column(default=False, nullable=False)
 
     state: Mapped[TestStateEnum] = mapped_column(
         Enum(TestStateEnum), nullable=False, default=TestStateEnum.PENDING

app/models/test_suite_metadata.py

@@ -35,6 +35,7 @@ class TestSuiteMetadata(Base):
     description: Mapped[str] = mapped_column(Text, nullable=False)
     version: Mapped[str] = mapped_column(nullable=False)
     source_hash: Mapped[str] = mapped_column(VARCHAR(64), nullable=False, index=True)
+    mandatory: Mapped[bool] = mapped_column(default=False, nullable=False)
 
     created_at: Mapped[datetime] = mapped_column(default=datetime.now)
 

app/pics_applicable_test_cases.py

@@ -13,9 +13,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+from typing import Dict
+
 from loguru import logger
 
 from app.schemas.pics import PICS, PICSApplicableTestCases
+from app.test_engine.models.test_declarations import TestCollectionDeclaration
 from app.test_engine.test_script_manager import test_script_manager
 
 
@@ -29,7 +32,7 @@ def applicable_test_cases_list(pics: PICS) -> PICSApplicableTestCases:
         PICSApplicableTestCases: List of test cases that are applicable
           for this Project
     """
-    applicable_tests: set = set()
+    applicable_tests: list = []
 
     if len(pics.clusters) == 0:
         # If the user has not uploaded any PICS
@@ -40,15 +43,37 @@ def applicable_test_cases_list(pics: PICS) -> PICSApplicableTestCases:
     test_collections = test_script_manager.test_collections
     enabled_pics = set([item.number for item in pics.all_enabled_items()])
 
-    for test_collection in test_collections.values():
-        for test_suite in test_collection.test_suites.values():
-            for test_case in test_suite.test_cases.values():
-                if len(test_case.pics) == 0:
-                    # Test cases without pics required are always applicable
-                    applicable_tests.add(test_case.metadata["title"])
-                elif len(test_case.pics) > 0:
-                    if test_case.pics.issubset(enabled_pics):
-                        applicable_tests.add(test_case.metadata["title"])
+    applicable_mandatories_tests = __applicable_test_cases(
+        test_collections, enabled_pics, True
+    )
+    applicable_remaining_tests = __applicable_test_cases(
+        test_collections, enabled_pics, False
+    )
+
+    # Add first the mandatories test cases
+    applicable_tests.extend(applicable_mandatories_tests)
+    # Add the remaining test cases
+    applicable_tests.extend(applicable_remaining_tests)
 
     logger.debug(f"Applicable test cases: {applicable_tests}")
     return PICSApplicableTestCases(test_cases=applicable_tests)
+
+
+def __applicable_test_cases(
+    test_collections: Dict[str, TestCollectionDeclaration],
+    enabled_pics: set[str],
+    mandatory: bool,
+) -> list:
+    applicable_tests: list = []
+
+    for test_collection in test_collections.values():
+        if test_collection.mandatory == mandatory:
+            for test_suite in test_collection.test_suites.values():
+                for test_case in test_suite.test_cases.values():
+                    if len(test_case.pics) == 0:
+                        # Test cases without pics required are always applicable
+                        applicable_tests.append(test_case.metadata["title"])
+                    elif len(test_case.pics) > 0:
+                        if test_case.pics.issubset(enabled_pics):
+                            applicable_tests.append(test_case.metadata["title"])
+    return applicable_tests

app/schemas/pics.py

@@ -38,7 +38,7 @@ def all_enabled_items(self) -> list[PICSItem]:
 
 
 class PICSApplicableTestCases(BaseModel):
-    test_cases: set[str]
+    test_cases: list[str]
 
 
 class PICSError(Exception):

app/schemas/test_case_metadata.py

@@ -26,6 +26,7 @@ class TestCaseMetadataBase(BaseModel):
     description: str
     version: str
     source_hash: str
+    mandatory: bool = False
 
     class Config:
         orm_mode = True

app/schemas/test_collections.py

@@ -23,6 +23,7 @@ class TestMetadata(BaseModel):
     version: str
     title: str
     description: str
+    mandatory: bool = False
 
 
 class TestCase(BaseModel):

app/schemas/test_run_execution.py

@@ -39,6 +39,7 @@ class TestRunExecutionBase(BaseModel):
 
     title: str
     description: Optional[str]
+    certification_mode: bool = False
 
 
 # Base + properties that represent relationhips

app/schemas/test_suite_execution.py

@@ -31,6 +31,7 @@ class TestSuiteExecutionBase(BaseModel):
     public_id: str
     execution_index: int
     collection_id: str
+    mandatory: bool = False
 
 
 # Properties shared by models stored in DB

app/schemas/test_suite_metadata.py

@@ -26,6 +26,7 @@ class TestSuiteMetadataBase(BaseModel):
     description: str
     version: str
     source_hash: str
+    mandatory: bool = False
 
     class Config:
         orm_mode = True

app/test_engine/models/test_case.py

@@ -14,11 +14,10 @@
 # limitations under the License.
 #
 from asyncio import CancelledError
-from typing import Any, List, Union, cast
+from typing import Any, List, Union
 
 from app.models import Project, TestCaseExecution
 from app.models.test_enums import TestStateEnum
-from app.schemas.test_environment_config import TestEnvironmentConfig
 from app.test_engine.logger import test_engine_logger as logger
 from app.test_engine.models.utils import LogSeparator
 from app.test_engine.test_observable import TestObservable
@@ -48,12 +47,10 @@ def default_test_parameters(cls) -> dict[str, Any]:
 
     @property
     def test_parameters(self) -> dict[str, Any]:
-        config_dict = cast(dict, self.config)
-
         test_parameters = self.default_test_parameters()
 
-        if config_dict and config_dict.get("test_parameters"):
-            test_parameters |= config_dict.get("test_parameters")  # type: ignore
+        if self.config and self.config.get("test_parameters"):
+            test_parameters |= self.config.get("test_parameters")  # type: ignore
 
         return test_parameters
 
@@ -82,7 +79,7 @@ def project(self) -> Project:
         return self.test_case_execution.test_suite_execution.test_run_execution.project
 
     @property
-    def config(self) -> TestEnvironmentConfig:
+    def config(self) -> dict:
         return self.project.config
 
     @property
@@ -131,6 +128,11 @@ def __compute_state(self) -> TestStateEnum:
         if self.errors is not None and len(self.errors) > 0:
             return TestStateEnum.ERROR
 
+        # Test cases that have already been marked as not applicable should not
+        # change state
+        if self.state == TestStateEnum.NOT_APPLICABLE:
+            return TestStateEnum.NOT_APPLICABLE
+
         # Note: These loops cannot be easily coalesced as we need to iterate through
         # and assign Test Case State in order.
         if self.any_steps_with_state(TestStateEnum.CANCELLED):
@@ -148,10 +150,14 @@ def __compute_state(self) -> TestStateEnum:
         return TestStateEnum.PASSED
 
     def any_steps_with_state(self, state: TestStateEnum) -> bool:
-        return any(ts for ts in self.test_steps if ts.state == state)
+        return any(ts.state == state for ts in self.test_steps)
 
     def completed(self) -> bool:
-        return self.state not in [TestStateEnum.PENDING, TestStateEnum.EXECUTING]
+        return self.state not in [
+            TestStateEnum.PENDING,
+            TestStateEnum.EXECUTING,
+            TestStateEnum.NOT_APPLICABLE,
+        ]
 
     def __cancel_remaning_test_steps(self) -> None:
         for step in self.test_steps:
@@ -172,7 +178,9 @@ def mark_as_completed(self) -> None:
         if self.completed():
             return
         self.state = self.__compute_state()
-        logger.info(f"Test Case Completed[{self.state.name}]: {self.metadata['title']}")
+        logger.info(
+            f"Test Case Completed [{self.state.name}]: {self.metadata['title']}"
+        )
         self.__print_log_separator()
 
     def mark_as_executing(self) -> None:
@@ -270,6 +278,10 @@ def mark_step_failure(self, msg: Union[str, Exception]) -> None:
 
         self.current_test_step.append_failure(message)
 
+    def mark_as_not_applicable(self) -> None:
+        self.state = TestStateEnum.NOT_APPLICABLE
+        logger.warning(f"Test Case Not Applicable: {self.metadata['public_id']}")
+
     def next_step(self) -> None:
         if self.current_test_step_index + 1 >= len(self.test_steps):
             return