ci: fix collect multi-dut test case with markers issue

While collecting, we stop registering the new plugin.
Otherwise the new created plugin will override the one we passed

also run the tests inside idf. the behavior is different.

conftest.py

@@ -1,14 +1,13 @@
 # SPDX-FileCopyrightText: 2021-2024 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
 # pylint: disable=W0621  # redefined-outer-name
-
+#
 # IDF is using [pytest](https://github.com/pytest-dev/pytest) and
 # [pytest-embedded plugin](https://github.com/espressif/pytest-embedded) as its test framework.
-
+#
 # if you found any bug or have any question,
 # please report to https://github.com/espressif/pytest-embedded/issues
 # or discuss at https://github.com/espressif/pytest-embedded/discussions
-
 import os
 import sys
 
@@ -432,11 +431,12 @@ def pytest_configure(config: Config) -> None:
         for f in glob.glob(os.path.join(IDF_PATH, app_info_filepattern)):
             apps.extend(import_apps_from_txt(f))
 
-    config.stash[IDF_PYTEST_EMBEDDED_KEY] = IdfPytestEmbedded(
-        target=target,
-        apps=apps,
-    )
-    config.pluginmanager.register(config.stash[IDF_PYTEST_EMBEDDED_KEY])
+    if '--collect-only' not in config.invocation_params.args:
+        config.stash[IDF_PYTEST_EMBEDDED_KEY] = IdfPytestEmbedded(
+            target=target,
+            apps=apps,
+        )
+        config.pluginmanager.register(config.stash[IDF_PYTEST_EMBEDDED_KEY])
 
 
 def pytest_unconfigure(config: Config) -> None:

pytest.ini

@@ -13,6 +13,7 @@ addopts =
   --logfile-extension ".txt"
   --check-duplicates y
   --ignore-glob */managed_components/*
+  --ignore pytest_embedded_log
 
 # ignore DeprecationWarning
 filterwarnings =

tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml

@@ -82,6 +82,5 @@
       --parallel-count ${CI_NODE_TOTAL:-1}
       --parallel-index ${CI_NODE_INDEX:-1}
       ${PYTEST_EXTRA_FLAGS}
-      --app-info-filepattern "list_job_*.txt"
   after_script:
     - python tools/ci/artifacts_handler.py upload --type logs junit_reports

tools/ci/idf_pytest/plugin.py

@@ -1,6 +1,5 @@
 # SPDX-FileCopyrightText: 2023-2024 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
-
 import os
 import typing as t
 from collections import defaultdict
@@ -16,11 +15,20 @@
 from idf_build_apps.constants import BuildStatus
 from pytest_embedded import Dut
 from pytest_embedded.plugin import parse_multi_dut_args
-from pytest_embedded.utils import find_by_suffix, to_list
-from pytest_ignore_test_results.ignore_results import ChildCase, ChildCasesStashKey
-
-from .constants import DEFAULT_SDKCONFIG, PREVIEW_TARGETS, SUPPORTED_TARGETS, CollectMode, PytestApp, PytestCase
-from .utils import comma_sep_str_to_list, format_case_id, merge_junit_files
+from pytest_embedded.utils import find_by_suffix
+from pytest_embedded.utils import to_list
+from pytest_ignore_test_results.ignore_results import ChildCase
+from pytest_ignore_test_results.ignore_results import ChildCasesStashKey
+
+from .constants import CollectMode
+from .constants import DEFAULT_SDKCONFIG
+from .constants import PREVIEW_TARGETS
+from .constants import PytestApp
+from .constants import PytestCase
+from .constants import SUPPORTED_TARGETS
+from .utils import comma_sep_str_to_list
+from .utils import format_case_id
+from .utils import merge_junit_files
 
 IDF_PYTEST_EMBEDDED_KEY = pytest.StashKey['IdfPytestEmbedded']()
 ITEM_FAILED_CASES_KEY = pytest.StashKey[list]()
@@ -43,9 +51,10 @@ def __init__(
         apps: t.Optional[t.List[App]] = None,
     ):
         if isinstance(target, str):
-            self.target = sorted(comma_sep_str_to_list(target))
+            # sequence also matters
+            self.target = comma_sep_str_to_list(target)
         else:
-            self.target = sorted(target)
+            self.target = target
 
         if not self.target:
             raise ValueError('`target` should not be empty')

tools/ci/idf_pytest/tests/conftest.py

@@ -1,10 +1,12 @@
-# SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2023-2024 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
-
 import os
 import sys
+import tempfile
 from pathlib import Path
 
+import pytest
+
 tools_ci_dir = os.path.join(os.path.dirname(__file__), '..', '..')
 if tools_ci_dir not in sys.path:
     sys.path.append(tools_ci_dir)
@@ -13,6 +15,8 @@
 if tools_dir not in sys.path:
     sys.path.append(tools_dir)
 
+from idf_ci_utils import IDF_PATH  # noqa: E402
+
 
 def create_project(name: str, folder: Path) -> Path:
     p = folder / name
@@ -46,3 +50,10 @@ def create_project(name: str, folder: Path) -> Path:
         )
 
     return p
+
+
+@pytest.fixture
+def tmp_path() -> Path:
+    os.makedirs(os.path.join(IDF_PATH, 'pytest_embedded_log'), exist_ok=True)
+
+    return Path(tempfile.mkdtemp(prefix=os.path.join(IDF_PATH, 'pytest_embedded_log') + os.sep))

tools/ci/idf_pytest/tests/test_get_all_apps.py

@@ -1,9 +1,9 @@
 # SPDX-FileCopyrightText: 2023-2024 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
-
 from pathlib import Path
 
-from idf_pytest.script import SUPPORTED_TARGETS, get_all_apps
+from idf_pytest.script import get_all_apps
+from idf_pytest.script import SUPPORTED_TARGETS
 
 from conftest import create_project
 
@@ -38,6 +38,26 @@ def test_foo(dut):
     assert len(non_test_related_apps) == 2 * len(SUPPORTED_TARGETS) - 2
 
 
+def test_get_all_apps_multi_dut_with_markers_test_script(tmp_path: Path) -> None:
+    create_project('foo', tmp_path)
+
+    (tmp_path / 'foo' / 'pytest_get_all_apps_multi_dut_with_markers_test_script.py').write_text(
+        """import pytest
+
+@pytest.mark.esp32
+@pytest.mark.parametrize('count', [2, 3], indirect=True)
+def test_foo(dut):
+    pass
+""",
+        encoding='utf-8',
+    )
+
+    test_related_apps, non_test_related_apps = get_all_apps([str(tmp_path)], target='all')
+
+    assert len(test_related_apps) == 1
+    assert len(non_test_related_apps) == len(SUPPORTED_TARGETS) - 1
+
+
 def test_get_all_apps_multi_dut_test_script(tmp_path: Path) -> None:
     create_project('foo', tmp_path)
     with open(tmp_path / 'foo' / 'pytest_get_all_apps_multi_dut_test_script.py', 'w') as fw:

tools/ci/idf_pytest/tests/test_get_pytest_cases.py

@@ -1,6 +1,5 @@
-# SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2023-2024 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
-
 from pathlib import Path
 
 from idf_pytest.constants import CollectMode
@@ -45,11 +44,14 @@ def test_get_pytest_cases_single_specific(tmp_path: Path) -> None:
 def test_get_pytest_cases_multi_specific(tmp_path: Path) -> None:
     script = tmp_path / 'pytest_get_pytest_cases_multi_specific.py'
     script.write_text(TEMPLATE_SCRIPT)
-    cases = get_pytest_cases([str(tmp_path)], 'esp32s3,esp32s2, esp32s2')
+    cases = get_pytest_cases([str(tmp_path)], 'esp32s2,esp32s2, esp32s3')
 
     assert len(cases) == 1
     assert cases[0].targets == ['esp32s2', 'esp32s2', 'esp32s3']
 
+    cases = get_pytest_cases([str(tmp_path)], 'esp32s3,esp32s2,esp32s2')  # order matters
+    assert len(cases) == 0
+
 
 def test_get_pytest_cases_multi_all(tmp_path: Path) -> None:
     script = tmp_path / 'pytest_get_pytest_cases_multi_all.py'