diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index 342aece0..2b68ecee 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -82,6 +82,26 @@ jobs: python -m pip install --upgrade --force --no-cache-dir pip pip install --force --no-cache-dir -r python/requirements.txt pip install --force --no-cache-dir -r python/test/requirements.txt -c python/test/constraints.txt + pip freeze | sort + + - name: Update expectation files + id: changes + continue-on-error: true + run: | + python/test/files/update_expectations.sh + git status + + if [[ ! git diff --exit-code ]] || [[ $(git ls-files -o --exclude-standard | wc -l) -gt 0 ]] + then + zip changes.zip $(git diff --name-only) $(git ls-files -o --exclude-standard) + exit 1 + fi + - name: Upload changed expectation files + if: steps.changes.outcome == 'failure' + uses: actions/upload-artifact@v3 + with: + name: Changed expectations + path: changed-expectations.zip - name: PyTest env: diff --git a/python/test/files/nunit/nunit3/jenkins/NUnit-sec1752-file.exception b/python/test/files/nunit/nunit3/jenkins/NUnit-sec1752-file.exception new file mode 100644 index 00000000..2ced6ac7 --- /dev/null +++ b/python/test/files/nunit/nunit3/jenkins/NUnit-sec1752-file.exception @@ -0,0 +1 @@ +ParseError: file='files/nunit/nunit3/jenkins/NUnit-sec1752-file.xml', message='Failure to process entity xxe, line 17, column 51 (NUnit-sec1752-file.xml, line 17)', line=None, column=None, exception=XMLSyntaxError('Failure to process entity xxe, line 17, column 51') \ No newline at end of file diff --git a/python/test/files/nunit/nunit3/jenkins/NUnit-sec1752-https.exception b/python/test/files/nunit/nunit3/jenkins/NUnit-sec1752-https.exception new file mode 100644 index 00000000..0b10953e --- /dev/null +++ b/python/test/files/nunit/nunit3/jenkins/NUnit-sec1752-https.exception @@ -0,0 +1 @@ +ParseError: file='files/nunit/nunit3/jenkins/NUnit-sec1752-https.xml', message='Failure to process entity xxe, line 17, column 51 (NUnit-sec1752-https.xml, line 17)', line=None, column=None, exception=XMLSyntaxError('Failure to process entity xxe, line 17, column 51') \ No newline at end of file diff --git a/python/test/test_action_script.py b/python/test/test_action_script.py index 0dde29ea..cd9f810b 100644 --- a/python/test/test_action_script.py +++ b/python/test/test_action_script.py @@ -1,15 +1,16 @@ import io -import re import json import logging import os import pathlib +import re import sys import tempfile import unittest from typing import Optional, Union, List, Type import mock +from packaging.version import Version from publish import pull_request_build_mode_merge, fail_on_mode_failures, fail_on_mode_errors, \ fail_on_mode_nothing, comment_modes, comment_mode_always, \ @@ -809,35 +810,57 @@ def test_parse_files(self): self.assertEqual([], gha.method_calls) self.assertEqual(66, actual.files) - self.assertEqual(6, len(actual.errors)) - self.assertEqual(357, actual.suites) - self.assertEqual(1928, actual.suite_tests) - self.assertEqual(106, actual.suite_skipped) - self.assertEqual(225, actual.suite_failures) - self.assertEqual(8, actual.suite_errors) - self.assertEqual(3964, actual.suite_time) - self.assertEqual(1916, len(actual.cases)) + if Version(sys.version.split(' ')[0]) >= Version('3.10.0') and sys.platform.startswith('darwin'): + # on macOS and Python 3.10 we see one particular error + self.assertEqual(8, len(actual.errors)) + self.assertEqual(355, actual.suites) + self.assertEqual(1924, actual.suite_tests) + self.assertEqual(106, actual.suite_skipped) + self.assertEqual(223, actual.suite_failures) + self.assertEqual(8, actual.suite_errors) + self.assertEqual(3964, actual.suite_time) + self.assertEqual(1912, len(actual.cases)) + else: + self.assertEqual(6, len(actual.errors)) + self.assertEqual(357, actual.suites) + self.assertEqual(1928, actual.suite_tests) + self.assertEqual(106, actual.suite_skipped) + self.assertEqual(225, actual.suite_failures) + self.assertEqual(8, actual.suite_errors) + self.assertEqual(3964, actual.suite_time) + self.assertEqual(1916, len(actual.cases)) self.assertEqual('commit', actual.commit) with io.StringIO() as string: gha = GithubAction(file=string) with mock.patch('publish.github_action.logger') as m: log_parse_errors(actual.errors, gha) + self.maxDiff = None + expected = [ + "::error::lxml.etree.XMLSyntaxError: Start tag expected, '<' not found, line 1, column 1", + "::error file=non-xml.xml::Error processing result file: Start tag expected, '<' not found, line 1, column 1 (non-xml.xml, line 1)", + "::error::Exception: File is empty.", + "::error file=empty.xml::Error processing result file: File is empty.", + "::error::lxml.etree.XMLSyntaxError: Premature end of data in tag skipped line 9, line 11, column 22", + "::error file=corrupt-xml.xml::Error processing result file: Premature end of data in tag skipped line 9, line 11, column 22 (corrupt-xml.xml, line 11)", + "::error::junitparser.junitparser.JUnitXmlError: Invalid format.", + "::error file=non-junit.xml::Error processing result file: Invalid format.", + "::error::lxml.etree.XMLSyntaxError: Char 0x0 out of allowed range, line 33, column 16", + "::error file=NUnit-issue17521.xml::Error processing result file: Char 0x0 out of allowed range, line 33, column 16 (NUnit-issue17521.xml, line 33)", + "::error::lxml.etree.XMLSyntaxError: attributes construct error, line 5, column 109", + "::error file=NUnit-issue47367.xml::Error processing result file: attributes construct error, line 5, column 109 (NUnit-issue47367.xml, line 5)" + ] + if Version(sys.version.split(' ')[0]) >= Version('3.10.0') and sys.platform.startswith('darwin'): + expected.extend([ + '::error::lxml.etree.XMLSyntaxError: Failure to process entity xxe, line 17, column 51', + '::error file=NUnit-sec1752-file.xml::Error processing result file: Failure to process entity xxe, line 17, column 51 (NUnit-sec1752-file.xml, line 17)', + '::error::lxml.etree.XMLSyntaxError: Failure to process entity xxe, line 17, column 51', + '::error file=NUnit-sec1752-https.xml::Error processing result file: Failure to process entity xxe, line 17, column 51 (NUnit-sec1752-https.xml, line 17)' + ]) self.assertEqual( - sorted([ - "::error::lxml.etree.XMLSyntaxError: Start tag expected, '<' not found, line 1, column 1", - "::error file=non-xml.xml::Error processing result file: Start tag expected, '<' not found, line 1, column 1 (non-xml.xml, line 1)", - "::error::Exception: File is empty.", - "::error file=empty.xml::Error processing result file: File is empty.", - "::error::lxml.etree.XMLSyntaxError: Premature end of data in tag skipped line 9, line 11, column 22", - "::error file=corrupt-xml.xml::Error processing result file: Premature end of data in tag skipped line 9, line 11, column 22 (corrupt-xml.xml, line 11)", - "::error::junitparser.junitparser.JUnitXmlError: Invalid format.", - "::error file=non-junit.xml::Error processing result file: Invalid format.", - "::error::lxml.etree.XMLSyntaxError: Char 0x0 out of allowed range, line 33, column 16", - "::error file=NUnit-issue17521.xml::Error processing result file: Char 0x0 out of allowed range, line 33, column 16 (NUnit-issue17521.xml, line 33)", - "::error::lxml.etree.XMLSyntaxError: attributes construct error, line 5, column 109", - "::error file=NUnit-issue47367.xml::Error processing result file: attributes construct error, line 5, column 109 (NUnit-issue47367.xml, line 5)" - ]), sorted([re.sub(r'file=.*[/\\]', 'file=', re.sub(r'[(]file:.*/', '(', line)) for line in string.getvalue().split(os.linesep) if line]) + sorted(expected), + sorted([re.sub(r'file=.*[/\\]', 'file=', re.sub(r'[(]file:.*/', '(', line)) + for line in string.getvalue().split(os.linesep) if line]) ) # self.assertEqual([], m.method_calls) diff --git a/python/test/test_junit.py b/python/test/test_junit.py index c06b9808..d7552df7 100644 --- a/python/test/test_junit.py +++ b/python/test/test_junit.py @@ -1,9 +1,9 @@ import dataclasses +import os import pathlib import re import sys import unittest -from distutils.version import LooseVersion from glob import glob from typing import Optional, List @@ -11,6 +11,7 @@ import prettyprinter as pp from junitparser import JUnitXml, Element from lxml import etree +from packaging.version import Version sys.path.append(str(pathlib.Path(__file__).resolve().parent.parent)) sys.path.append(str(pathlib.Path(__file__).resolve().parent)) @@ -58,6 +59,8 @@ def parse_file(filename) -> JUnitTreeOrParseError: @staticmethod def assert_expectation(test, actual, filename): + if not os.path.exists(filename): + test.fail(f'file does not exist: {filename}, expected content: {actual}') with open(filename, 'r', encoding='utf-8') as r: expected = r.read() test.assertEqual(expected, actual) @@ -159,7 +162,7 @@ def test_junitparser_locale(self): junit = JUnitXml.fromfile(str(test_files_path / 'pytest' / 'junit.spark.integration.1.xml')) self.assertAlmostEqual(162.933, junit.time, 3) - @unittest.skipIf(LooseVersion(junitparser.version) < LooseVersion('2.0.0'), + @unittest.skipIf(Version(junitparser.version) < Version('2.0.0'), 'multiple results per test case not supported by junitparser') def test_parse_junit_xml_file_with_multiple_results(self): junit = process_junit_xml_elems(parse_junit_xml_files([str(test_files_path / 'junit.multiresult.xml')]))