fix(pylint,pytest): detect failures and error message associated with…

… the task (kubeflow#818)

* fix(pylint): detect failures and erros associated with the task

* do not break if neither failure or failure element doesn't have message attribute

* send xml files to artifacts folder to get detected by spyglass

* parse all the tests to xml

* path for pytest gets added from generate_xml function

* Revert "path for pytest gets added from generate_xml function"

This reverts commit 8be433afb7f93fa9017dad1a7870d9929c4d4e7b.

* add only: path for pytest gets added from generate_xml function

acm-repos/kf-ci-v1/namespaces/auto-deploy/tekton.dev_v1alpha1_pipeline_py-unittests.yaml

@@ -23,7 +23,7 @@ spec:
   - name: py-lint
     params:
     - name: artifacts-gcs
-      value: $(params.artifacts-gcs)
+      value: $(params.artifacts-gcs)/artifacts/junit_py-lint
     - name: lint-src-dir
       value: $(params.lint-src-dir)
     resources:
@@ -35,7 +35,7 @@ spec:
   - name: py-test
     params:
     - name: artifacts-gcs
-      value: $(params.artifacts-gcs)
+      value: $(params.artifacts-gcs)/artifacts
     - name: test-src-dir
       value: $(params.test-src-dir)
     resources:

acm-repos/kf-ci-v1/namespaces/kf-ci/tekton.dev_v1alpha1_pipeline_py-unittests.yaml

@@ -23,7 +23,7 @@ spec:
   - name: py-lint
     params:
     - name: artifacts-gcs
-      value: $(params.artifacts-gcs)
+      value: $(params.artifacts-gcs)/artifacts/junit_py-lint
     - name: lint-src-dir
       value: $(params.lint-src-dir)
     resources:
@@ -35,7 +35,7 @@ spec:
   - name: py-test
     params:
     - name: artifacts-gcs
-      value: $(params.artifacts-gcs)
+      value: $(params.artifacts-gcs)/artifacts
     - name: test-src-dir
       value: $(params.test-src-dir)
     resources:

py/kubeflow/testing/tekton_client.py

@@ -405,12 +405,22 @@ def junit_parse_and_upload(artifacts_dir, output_gcs):
 
         for testcase in root:
           testname = testcase.attrib.get("name", "unknown-test")
+          # Detect error and failures from testcase
+          failed_num += int(testcase.attrib.get(
+            "errors", "0")) +int(testcase.attrib.get("failures", "0"))
           has_failure = False
           for failure in testcase:
             has_failure = True
-            logging.error("%s has failure: %s",
-                          testname,
-                          failure.attrib.get("message", "message not found"))
+            failure_element = failure.find('failure')
+            if failure_element is not None:
+              logging.error("%s has failure: %s",
+                            testname,
+                            failure_element.attrib.get("message", "message not found"))
+            else:
+              logging.error("%s has failure: %s",
+                            testname,
+                            failure.attrib.get("message", "message not found"))
+
           if not has_failure:
             logging.info("%s has passed all the tests.", testname)
 

py/kubeflow/testing/test_helper.py

@@ -53,12 +53,8 @@ def generate_xml(self):
     # want one underscore after junit.
     output_file = os.path.join(self.artifacts_dir,
                                "junit_" + self.name.replace("_", "-") + ".xml")
-    # junit_xml produces a list of test suites, but gubernator
-    # only parses a single test suite. So here we generate
-    # the xml using junit-xml and only output the first test
-    # suite in our output file
     xml_out = junit_xml.TestSuite.to_xml_string([self])
-    first_test_suite = ET.fromstring(xml_out)[0]
+    first_test_suite = ET.fromstring(xml_out)
     logging.info("Writing file: %s", output_file)
     ET.ElementTree(first_test_suite).write(output_file)
 

tekton/templates/pipelines/py-unittests-pipeline.yaml

@@ -25,7 +25,7 @@ spec:
   - name: py-lint
     params:
     - name: artifacts-gcs
-      value: "$(params.artifacts-gcs)"
+      value: $(params.artifacts-gcs)/artifacts/junit_py-lint
     - name: lint-src-dir
       value: "$(params.lint-src-dir)"
     resources:
@@ -37,7 +37,7 @@ spec:
   - name: py-test
     params:
     - name: artifacts-gcs
-      value: "$(params.artifacts-gcs)"
+      value: $(params.artifacts-gcs)/artifacts
     - name: test-src-dir
       value: "$(params.test-src-dir)"
     resources: