build: update dependencies and process

Even though some packages are 'safer' to update,
we cannot rely on having the CI discovering it on
random integrations. Pinning the remaining packages
and ordering them a bit.

The update of the 'build' package (to create wheels)
required the replacement of 'build.pep517' by
the new 'pyproject_hooks' module, and other
modification to the wheel artifacts.

The removed dependencies are currently placed at the tool level
requirements.txt so it's not like they are not needed anymore.

Test and CI scripts were adapted in order to rely on the 'dist'
directory rather than the 'dist_new' one, removing the old
step of creating the wheels with 'setup.py bdist_wheel'.

The entry points (console scripts) that we used to have
in the 'setup.py' were moved to the 'pyproject.toml' in order
to advance towards not relying on the setup.py file.

Flake8 issues were addressed in the different files
that this patch modified.

Change-Id: I83480c1920206e11fcb9a45264b6beaf6f8b686b
Pick-to: 6.6
Reviewed-by: Friedemann Kleint <Friedemann.Kleint@qt.io>

coin_build_instructions.py

@@ -12,8 +12,8 @@
 from build_scripts.options import has_option, option_value
 from build_scripts.utils import (expand_clang_variables, get_ci_qtpaths_path,
                                  get_qtci_virtualEnv,
-                                 parse_cmake_conf_assignments_by_key, remove_tree,
-                                 run_instruction)
+                                 parse_cmake_conf_assignments_by_key,
+                                 remove_tree, run_instruction)
 
 log.setLevel(logging.INFO)
 
@@ -27,20 +27,22 @@
 CI_ENV_AGENT_DIR = option_value("agentdir")
 CI_COMPILER = option_value("compiler")
 CI_USE_SCCACHE = option_value("compiler-launcher")
-CI_INTEGRATION_ID = option_value("coinIntegrationId") or str(calendar.timegm(datetime.datetime.now().timetuple()))
+CI_INTEGRATION_ID = option_value("coinIntegrationId") or str(
+    calendar.timegm(datetime.datetime.now().timetuple())
+)
 CI_FEATURES = []
 _ci_features = option_value("features")
 if _ci_features is not None:
-    for f in _ci_features.split(', '):
+    for f in _ci_features.split(", "):
         CI_FEATURES.append(f)
 CI_RELEASE_CONF = has_option("packaging")
 CI_TEST_PHASE = option_value("phase")
-if CI_TEST_PHASE not in ["ALL", "BUILD", "WHEEL"]:
+if CI_TEST_PHASE not in ["ALL", "BUILD"]:
     CI_TEST_PHASE = "ALL"
 
 
 def get_current_script_path():
-    """ Returns the absolute path containing this script. """
+    """Returns the absolute path containing this script."""
     try:
         this_file = __file__
     except NameError:
@@ -64,8 +66,8 @@ def is_snapshot_build():
     pyside_project_dir = os.path.join(setup_script_dir, "sources", "pyside6")
 
     d = parse_cmake_conf_assignments_by_key(pyside_project_dir)
-    release_version_type = d.get('pyside_PRE_RELEASE_VERSION_TYPE')
-    pre_release_version = d.get('pyside_PRE_RELEASE_VERSION')
+    release_version_type = d.get("pyside_PRE_RELEASE_VERSION_TYPE")
+    pre_release_version = d.get("pyside_PRE_RELEASE_VERSION")
     if pre_release_version and release_version_type:
         return True
     return False
@@ -75,7 +77,9 @@ def call_setup(python_ver, phase):
     print("call_setup")
     print("python_ver", python_ver)
     print("phase", phase)
-    _pExe, _env, env_pip, env_python = get_qtci_virtualEnv(python_ver, CI_HOST_OS, CI_HOST_ARCH, CI_TARGET_ARCH)
+    _pExe, _env, env_pip, env_python = get_qtci_virtualEnv(
+        python_ver, CI_HOST_OS, CI_HOST_ARCH, CI_TARGET_ARCH
+    )
 
     if phase in ["BUILD"]:
         remove_tree(_env, True)
@@ -84,7 +88,10 @@ def call_setup(python_ver, phase):
         python3 = "python3"
         if sys.platform == "win32":
             python3 = os.path.join(os.getenv("PYTHON3_PATH"), "python.exe")
-        run_instruction([python3, "-m", "pip", "install", "--user", "virtualenv==20.7.2"], "Failed to pin virtualenv")
+        run_instruction(
+            [python3, "-m", "pip", "install", "--user", "virtualenv==20.7.2"],
+            "Failed to pin virtualenv",
+        )
         # installing to user base might not be in PATH by default.
         env_path = os.path.join(site.USER_BASE, "bin")
         v_env = os.path.join(env_path, "virtualenv")
@@ -99,17 +106,17 @@ def call_setup(python_ver, phase):
             v_env = "virtualenv"
         run_instruction([str(v_env), "-p", str(_pExe), str(_env)], "Failed to create virtualenv")
         # When the 'python_ver' variable is empty, we are using Python 2
-        # Pip is always upgraded when CI template is provisioned, upgrading it in later phase may cause perm issue
-        run_instruction([str(env_pip), "install", "-r", "requirements.txt"], "Failed to install dependencies")
+        # Pip is always upgraded when CI template is provisioned,
+        # upgrading it in later phase may cause perm issue
+        run_instruction(
+            [str(env_pip), "install", "-r", "requirements.txt"], "Failed to install dependencies"
+        )
 
     cmd = [env_python, "-u", "setup.py"]
     if phase in ["BUILD"]:
         cmd += ["build", "--standalone", "--unity"]
-    elif phase in ["WHEEL"] or CI_RELEASE_CONF:
-        cmd += ["bdist_wheel", "--reuse-build", "--standalone", "--skip-cmake", "--skip-make-install", "--only-package"]
 
-    cmd += ["--build-tests",
-            "--log-level=verbose"]
+    cmd += ["--build-tests", "--log-level=verbose"]
 
     if CI_TARGET_ARCH == "X86_64-ARM64":
         cmd += ["--macos-arch='x86_64;arm64'"]
@@ -137,20 +144,20 @@ def call_setup(python_ver, phase):
     env = os.environ
     run_instruction(cmd, "Failed to run setup.py for build", initial_env=env)
 
-if __name__ == "__main__":
 
+if __name__ == "__main__":
     # Remove some environment variables that impact cmake
-    arch = '32' if CI_TARGET_ARCH == 'X86' else '64'
+    arch = "32" if CI_TARGET_ARCH == "X86" else "64"
     expand_clang_variables(arch)
-    for env_var in ['CC', 'CXX']:
+    for env_var in ["CC", "CXX"]:
         if os.environ.get(env_var):
             del os.environ[env_var]
     python_ver = "3"
-    if CI_TARGET_OS in ["Linux"] and CI_HOST_ARCH !="aarch64":
+    if CI_TARGET_OS in ["Linux"] and CI_HOST_ARCH != "aarch64":
         python_ver = "3.11"
     wheel_package_dir = "qfpa-p3.6"
     if CI_TARGET_OS in ["Windows"]:
-        if (os.environ.get('HOST_OSVERSION_COIN')).startswith('windows_10'):
+        if (os.environ.get("HOST_OSVERSION_COIN")).startswith("windows_10"):
             python_ver = "3.10.0"
         else:
             python_ver = "3.8.1"
@@ -160,8 +167,4 @@ def call_setup(python_ver, phase):
     if os.environ.get("QTEST_ENVIRONMENT") == "ci" and sys.platform == "win32":
         signing_dir = str(os.environ.get("PYSIDE_SIGNING_DIR"))
         print("Check for signing dir " + signing_dir)
-        assert(os.path.isdir(signing_dir))
-    if CI_TEST_PHASE in ["ALL", "WHEEL"] and sys.platform != "win32":
-        # "Old" Windows wheels won't be signed anyway so there is no need to
-        # create those, so that we don't accidentally release those.
-        call_setup(python_ver, "WHEEL")
+        assert os.path.isdir(signing_dir)

coin_test_instructions.py

@@ -6,7 +6,7 @@
 import sys
 
 from build_scripts.log import log
-from build_scripts.options import has_option, log, option_value
+from build_scripts.options import has_option, option_value
 from build_scripts.utils import (expand_clang_variables, get_ci_qmake_path,
                                  get_qtci_virtualEnv, remove_tree, run_instruction)
 
@@ -30,7 +30,8 @@
 
 
 def call_testrunner(python_ver, buildnro):
-    _pExe, _env, env_pip, env_python = get_qtci_virtualEnv(python_ver, CI_HOST_OS, CI_HOST_ARCH, CI_TARGET_ARCH)
+    _pExe, _env, env_pip, env_python = get_qtci_virtualEnv(python_ver, CI_HOST_OS, CI_HOST_ARCH,
+                                                           CI_TARGET_ARCH)
     remove_tree(_env, True)
     # Pinning the virtualenv before creating one
     # Use pip3 if possible while pip seems to install the virtualenv to wrong dir in some OS
@@ -42,9 +43,11 @@ def call_testrunner(python_ver, buildnro):
     if CI_HOST_OS == "MacOS" and CI_HOST_ARCH == "ARM64":
         v_env = "virtualenv"
         run_instruction([str(v_env), "-p", str(_pExe), str(_env)], "Failed to create virtualenv")
-        run_instruction([env_pip, "install", "-r", "requirements.txt"], "Failed to install dependencies")
+        run_instruction([env_pip, "install", "-r", "requirements.txt"],
+                        "Failed to install dependencies")
     else:
-        run_instruction([python3, "-m", "pip", "install", "--user", "virtualenv==20.7.2"], "Failed to pin virtualenv")
+        run_instruction([python3, "-m", "pip", "install", "--user", "virtualenv==20.7.2"],
+                        "Failed to pin virtualenv")
         # installing to user base might not be in PATH by default.
         env_path = os.path.join(site.USER_BASE, "bin")
         v_env = os.path.join(env_path, "virtualenv")
@@ -59,8 +62,10 @@ def call_testrunner(python_ver, buildnro):
             v_env = "virtualenv"
         run_instruction([str(v_env), "-p", str(_pExe), str(_env)], "Failed to create virtualenv")
         # When the 'python_ver' variable is empty, we are using Python 2
-        # Pip is always upgraded when CI template is provisioned, upgrading it in later phase may cause perm issue
-        run_instruction([env_pip, "install", "-r", "requirements.txt"], "Failed to install dependencies")
+        # Pip is always upgraded when CI template is provisioned,
+        # upgrading it in later phase may cause perm issue
+        run_instruction([env_pip, "install", "-r", "requirements.txt"],
+                        "Failed to install dependencies")
         # Install distro to replace missing platform.linux_distribution() in python3.8
         run_instruction([env_pip, "install", "distro"], "Failed to install distro")
 
@@ -73,18 +78,8 @@ def call_testrunner(python_ver, buildnro):
     # Try to install built wheels, and build some buildable examples.
     if CI_RELEASE_CONF:
         wheel_tester_path = os.path.join("testing", "wheel_tester.py")
-        # We create wheels differently in Qt CI with Windows and there are no "old" wheels
-        if CI_HOST_OS != "Windows":
-            # Run the test for the old set of wheels
-            cmd = [env_python, wheel_tester_path, qmake_path]
-            run_instruction(cmd, "Error while running wheel_tester.py on old wheels")
-
-            # Uninstalling the other wheels
-            run_instruction([env_pip, "uninstall", "shiboken6", "shiboken6_generator", "pyside6", "-y"],
-                            "Failed to uninstall old wheels")
-
         # Run the test for the new set of wheels
-        cmd = [env_python, wheel_tester_path, qmake_path, "--wheels-dir=dist_new", "--new"]
+        cmd = [env_python, wheel_tester_path, qmake_path, "--wheels-dir=dist", "--new"]
         run_instruction(cmd, "Error while running wheel_tester.py on new wheels")
 
 

create_wheels.py

@@ -12,6 +12,7 @@
 from typing import List, Optional, Tuple
 
 import build  # type: ignore
+import pyproject_hooks
 from build_scripts.wheel_files import (ModuleData,  # type: ignore
                                        set_pyside_package_path,
                                        wheel_files_pyside_addons,
@@ -20,6 +21,7 @@
 
 
 PACKAGE_FOR_WHEELS = "package_for_wheels"
+PYSIDE_DESCRIPTION = "Python bindings for the Qt cross-platform application and UI framework"
 
 
 @dataclass
@@ -136,6 +138,11 @@ def generate_pyproject_toml(artifacts: Path, setup: SetupData) -> str:
 
     _tag = get_platform_tag()
 
+    _console_scripts = ""
+    if setup.console_scripts:
+        _formatted_console_scripts = "\n".join(setup.console_scripts)
+        _console_scripts = f"[project.scripts]\n{_formatted_console_scripts}"
+
     with open(artifacts / "pyproject.toml.base") as f:
         content = (
             f.read()
@@ -144,6 +151,7 @@ def generate_pyproject_toml(artifacts: Path, setup: SetupData) -> str:
             .replace("PROJECT_DESCRIPTION", f'"{setup.description}"')
             .replace("PROJECT_README", f'"{setup.readme}"')
             .replace("PROJECT_TAG", f'"{_tag}"')
+            .replace("PROJECT_SCRIPTS", _console_scripts)
         )
 
     return content
@@ -187,7 +195,6 @@ def generate_setup_py(artifacts: Path, setup: SetupData):
             name=_name,
             fake_ext=fext,
             install=install_requires,
-            console_scripts={"console_scripts": setup.console_scripts},
         )
 
     return content
@@ -200,8 +207,8 @@ def wheel_shiboken_generator(package_path: Path) -> Tuple[SetupData, None]:
         description="Python/C++ bindings generator",
         readme="README.shiboken6-generator.md",
         console_scripts=[
-            "shiboken6 = shiboken6_generator.scripts.shiboken_tool:main",
-            "shiboken6-genpyi = shiboken6_generator.scripts.shiboken_tool:genpyi",
+            'shiboken6 = "shiboken6_generator.scripts.shiboken_tool:main"',
+            'shiboken6-genpyi = "shiboken6_generator.scripts.shiboken_tool:genpyi"',
         ],
     )
 
@@ -229,16 +236,16 @@ def wheel_pyside6_essentials(package_path: Path) -> Tuple[SetupData, List[Module
     # Also, the tool should not exist in any other platform than Linux
     _console_scripts = []
     if ("android_deploy" in _pyside_tools) and sys.platform.startswith("linux"):
-        _console_scripts = ["pyside6-android-deploy = PySide6.scripts.pyside_tool:android_deploy"]
+        _console_scripts = ['pyside6-android-deploy = "PySide6.scripts.pyside_tool:android_deploy"']
     _pyside_tools.remove("android_deploy")
 
-    _console_scripts.extend([f"pyside6-{tool} = PySide6.scripts.pyside_tool:{tool}"
+    _console_scripts.extend([f'pyside6-{tool} = "PySide6.scripts.pyside_tool:{tool}"'
                             for tool in _pyside_tools])
 
     setup = SetupData(
         name="PySide6_Essentials",
         version=get_version_from_package("PySide6", package_path),  # we use 'PySide6' here
-        description="Python bindings for the Qt cross-platform application and UI framework (Essentials)",
+        description=f"{PYSIDE_DESCRIPTION} (Essentials)",
         readme="README.pyside6_essentials.md",
         console_scripts=_console_scripts
     )
@@ -252,7 +259,7 @@ def wheel_pyside6_addons(package_path: Path) -> Tuple[SetupData, List[ModuleData
     setup = SetupData(
         name="PySide6_Addons",
         version=get_version_from_package("PySide6", package_path),  # we use 'PySide6' here
-        description="Python bindings for the Qt cross-platform application and UI framework (Addons)",
+        description=f"{PYSIDE_DESCRIPTION} (Addons)",
         readme="README.pyside6_addons.md",
         console_scripts=[],
     )
@@ -266,7 +273,7 @@ def wheel_pyside6(package_path: Path) -> Tuple[SetupData, Optional[List[ModuleDa
     setup = SetupData(
         name="PySide6",
         version=get_version_from_package("PySide6", package_path),
-        description="Python bindings for the Qt cross-platform application and UI framework",
+        description=PYSIDE_DESCRIPTION,
         readme="README.pyside6.md",
         console_scripts=[],
     )
@@ -423,18 +430,18 @@ def get_build_directory(options: Namespace):
         # 5. call the build module to create the wheel
         print("-- Creating wheels")
         if not verbose:
-            _runner = build.pep517.wrappers.quiet_subprocess_runner
+            _runner = pyproject_hooks.quiet_subprocess_runner
         else:
-            _runner = build.pep517.wrappers.default_subprocess_runner
+            _runner = pyproject_hooks.default_subprocess_runner
         builder = build.ProjectBuilder(package_path, runner=_runner)
-        builder.build("wheel", "dist_new")
+        builder.build("wheel", "dist")
 
         # 6. Copy wheels back
-        print("-- Copying wheels to dist_new/")
-        dist_path = Path("dist_new")
+        print("-- Copying wheels to dist/")
+        dist_path = Path("dist")
         if not dist_path.is_dir():
             dist_path.mkdir()
-        for wheel in Path(package_path / "dist_new").glob("*.whl"):
+        for wheel in Path(package_path / "dist").glob("*.whl"):
             copy(wheel, dist_path / wheel.name)
 
         # 7. Remove leftover files

requirements.txt

@@ -1,12 +1,16 @@
-setuptools==67.8.0
-packaging
-build==0.7
-six
-wheel>=0.35
+# Build dependencies
+setuptools==69.0.3
+packaging==23.2
+build==1.0.3
+wheel==0.42.0
+distro==1.9.0; sys_platform == 'linux'
+patchelf==0.17.2; sys_platform == 'linux'
+# 1.24.4 is the last version that supports Python 3.8
+numpy<1.25; python_version < '3.9'
+numpy==1.26.3; python_version >= '3.9'
+
+# For examples
 PyOpenGL
+
+# For tests
 pyinstaller==3.6; platform_machine != 'aarch64'
-numpy
-distro; sys_platform == 'linux'
-patchelf==0.15; sys_platform == 'linux'
-tqdm; sys_platform == 'linux'
-gitpython; sys_platform == 'linux'

testing/wheel_tester.py

@@ -35,9 +35,9 @@
 setup_script_dir = os.path.abspath(os.path.join(this_dir, ".."))
 sys.path.append(setup_script_dir)
 
-from build_scripts.utils import (find_files_using_glob, find_glob_in_path,
+from build_scripts.utils import (find_files_using_glob, find_glob_in_path,  # noqa: E402
                                  remove_tree, run_process, run_process_output)
-from build_scripts.log import log
+from build_scripts.log import log  # noqa: E402
 
 log.setLevel(logging.DEBUG)
 
@@ -83,7 +83,8 @@ def package_prefix_names():
     # Note: shiboken6_generator is not needed for compile_using_nuitka,
     # but building modules with cmake needs it.
     if NEW_WHEELS:
-        return ["shiboken6", "shiboken6_generator", "PySide6_Essentials", "PySide6_Addons", "PySide6"]
+        return ["shiboken6", "shiboken6_generator", "PySide6_Essentials", "PySide6_Addons",
+                "PySide6"]
     else:
         return ["shiboken6", "shiboken6_generator", "PySide6"]
 
@@ -360,7 +361,7 @@ def try_build_examples():
             for modname in modules:
                 # PYSIDE-1735: pyi files are no longer compatible with Python.
                 # XXX Maybe add a test with Mypy here?
-                pass # execute_script(src_path / f"{modname}.pyi")
+                pass  # execute_script(src_path / f"{modname}.pyi")
 
 
 def run_wheel_tests(install_wheels, wheels_dir_name):
@@ -392,7 +393,8 @@ def run_wheel_tests(install_wheels, wheels_dir_name):
     )
     parser.add_argument("--qmake", type=str, help="Path to qmake")
     parser.add_argument("--cmake", type=str, help="Path to cmake")
-    parser.add_argument("--wheels-dir", type=str, help="Path to where the wheels are", default="dist")
+    parser.add_argument("--wheels-dir", type=str, help="Path to where the wheels are",
+                        default="dist")
     parser.add_argument("--new", action="store_true", help="Option to test new wheels")
     options = parser.parse_args()
     QMAKE_PATH = find_executable("qmake", options.qmake)

wheel_artifacts/pyproject.toml.base

@@ -50,12 +50,11 @@ Repository = "https://code.qt.io/cgit/pyside/pyside-setup.git/"
 Changelog = "https://code.qt.io/cgit/pyside/pyside-setup.git/tree/doc/changelogs"
 Tracker = "https://bugreports.qt.io/projects/PYSIDE"
 
+PROJECT_SCRIPTS
+
 [tool.distutils.bdist_wheel]
 py_limited_api = "cp38"
 plat_name = PROJECT_TAG
 
-[tool.setuptools.packages]
-find = {}
-
 [tool.setuptools.dynamic]
 version = {attr = PROJECT_VERSION}