Add optional recommendations for informational tests on pilot devices

framework/python/src/common/session.py

@@ -394,6 +394,7 @@ def add_test_result(self, result):
         if len(result.description) != 0:
           test_result.description = result.description
 
+        # Add recommendations if provided
         if result.recommendations is not None:
           test_result.recommendations = result.recommendations
 
@@ -404,7 +405,19 @@ def add_test_result(self, result):
 
           # Any informational test should always report informational
           if test_result.required_result == 'Informational':
-            test_result.result = TestResult.INFORMATIONAL
+
+            # Set test result to informational
+            if result.result in [
+              TestResult.NON_COMPLIANT,
+              TestResult.COMPLIANT,
+              TestResult.INFORMATIONAL
+            ]:
+              test_result.result = TestResult.INFORMATIONAL
+            else:
+              test_result.result = result.result
+
+            # Copy any test recommendations to optional
+            test_result.optional_recommendations = result.recommendations
 
             # Remove recommendations from informational tests
             test_result.recommendations = None

framework/python/src/common/testreport.py

@@ -120,6 +120,10 @@ def to_json(self):
       if test.recommendations is not None and len(test.recommendations) > 0:
         test_dict['recommendations'] = test.recommendations
 
+      if (test.optional_recommendations is not None
+        and len(test.optional_recommendations) > 0):
+        test_dict['optional_recommendations'] = test.optional_recommendations
+
       test_results.append(test_dict)
 
     report_json['tests'] = {'total': self._total_tests,
@@ -165,8 +169,16 @@ def from_json(self, json_file):
         expected_behavior=test_result['expected_behavior'],
         required_result=test_result['required_result'],
         result=test_result['result'])
+
+      # Add test recommendations
       if 'recommendations' in test_result:
         test_case.recommendations = test_result['recommendations']
+
+      # Add optional test recommendations
+      if 'optional_recommendations' in test_result:
+        test_case.optional_recommendations = test_result[
+          'optional_recommendations']
+
       self.add_test(test_case)
 
   # Create a pdf file in memory and return the bytes
@@ -187,32 +199,43 @@ def to_html(self):
                             encoding='UTF-8'
                             ) as template_file:
       template = Template(template_file.read())
+
+    # Load styles
     with open(os.path.join(report_resource_dir,
                            TEST_REPORT_STYLES),
                            'r',
                            encoding='UTF-8'
                            ) as style_file:
       styles = style_file.read()
+
+    # Load Testrun logo to base64
     with open(test_run_img_file, 'rb') as f:
       logo = base64.b64encode(f.read()).decode('utf-8')
     json_data=self.to_json()
 
     # Convert the timestamp strings to datetime objects
     start_time = datetime.strptime(json_data['started'], '%Y-%m-%d %H:%M:%S')
     end_time = datetime.strptime(json_data['finished'], '%Y-%m-%d %H:%M:%S')
+
     # Calculate the duration
     duration = end_time - start_time
 
+    # Calculate number of successful tests
     successful_tests = 0
     for test in json_data['tests']['results']:
       if test['result'] != 'Error':
         successful_tests += 1
 
+    # Obtain the steps to resolve
     steps_to_resolve = self._get_steps_to_resolve(json_data)
 
+    # Obtain optional recommendations
+    optional_steps_to_resolve = self._get_optional_steps_to_resolve(json_data)
+
     module_reports = self._get_module_pages()
     pages_num = self._pages_num(json_data)
     total_pages = pages_num + len(module_reports)
+
     if len(steps_to_resolve) > 0:
       total_pages += 1
 
@@ -228,6 +251,7 @@ def to_html(self):
                            total_tests=self._total_tests,
                            test_results=json_data['tests']['results'],
                            steps_to_resolve=steps_to_resolve,
+                           optional_steps_to_resolve=optional_steps_to_resolve,
                            module_reports=module_reports,
                            pages_num=pages_num,
                            total_pages=total_pages,
@@ -284,6 +308,16 @@ def _get_steps_to_resolve(self, json_data):
 
     return tests_with_recommendations
 
+  def _get_optional_steps_to_resolve(self, json_data):
+    tests_with_recommendations = []
+
+    # Collect all tests with recommendations
+    for test in json_data['tests']['results']:
+      if 'optional_recommendations' in test:
+        tests_with_recommendations.append(test)
+
+    return tests_with_recommendations
+
   def _get_module_pages(self):
     content_max_size = 913
 

framework/python/src/test_orc/test_case.py

@@ -27,23 +27,23 @@ class TestCase:  # pylint: disable=too-few-public-methods,too-many-instance-attr
   required_result: str = "Recommended"
   result: str = TestResult.NON_COMPLIANT
   recommendations: list = field(default_factory=lambda: [])
+  optional_recommendations: list = field(default_factory=lambda: [])
 
   def to_dict(self):
 
+    test_dict = {
+      "name": self.name,
+      "description": self.description,
+      "expected_behavior": self.expected_behavior,
+      "required_result": self.required_result,
+      "result": self.result
+    }
+
     if self.recommendations is not None and len(self.recommendations) > 0:
-      return {
-        "name": self.name,
-        "description": self.description,
-        "expected_behavior": self.expected_behavior,
-        "required_result": self.required_result,
-        "result": self.result,
-        "recommendations": self.recommendations
-      }
-
-    return {
-        "name": self.name,
-        "description": self.description,
-        "expected_behavior": self.expected_behavior,
-        "required_result": self.required_result,
-        "result": self.result
-      }
+      test_dict["recommendations"] = self.recommendations
+
+    if (self.optional_recommendations is not None
+      and len(self.optional_recommendations) > 0):
+      test_dict["optional_recommendations"] = self.optional_recommendations
+
+    return test_dict

framework/python/src/test_orc/test_orchestrator.py

@@ -138,7 +138,6 @@ def run_test_modules(self):
 
         # Set the required result from the correct test pack
         required_result = test_pack.get_required_result(test.name)
-        LOGGER.debug(f"Required result for {test.name} is {required_result}")
 
         test_copy.required_result = required_result
 
@@ -166,7 +165,11 @@ def run_test_modules(self):
       return TestrunStatus.CANCELLED
 
     report = TestReport()
-    report.from_json(self._generate_report())
+
+    generated_report_json = self._generate_report()
+    print("Generated report:")
+    print(generated_report_json)
+    report.from_json(generated_report_json)
     report.add_module_reports(self.get_session().get_module_reports())
     device.add_report(report)