From e968e98f11af004645b53caed421eafb393d0d89 Mon Sep 17 00:00:00 2001 From: Nathaniel van Diepen Date: Wed, 2 Aug 2023 13:49:42 -0600 Subject: [PATCH] Update to python 3.10 (#684) * Upgrade black to remove need for typed-ast * Simplify workflow and install wheel * Upgrade to python 3.10 * Update documentation --- .github/actions/setup/action.yml | 5 +++++ .github/workflows/pr.yml | 8 -------- .github/workflows/stable.yml | 4 ---- .github/workflows/testing.yml | 4 ---- docs/building.md | 2 +- requirements.txt | 13 ++++++------- scripts/toltec/bash.py | 1 + scripts/toltec/builder.py | 6 ++++-- scripts/toltec/graphlib.py | 1 + scripts/toltec/ipk.py | 1 + scripts/toltec/recipe.py | 1 + scripts/toltec/repo.py | 8 +++++++- scripts/toltec/version.py | 8 ++++++++ 13 files changed, 35 insertions(+), 27 deletions(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 003825831..f10600cf6 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -67,6 +67,10 @@ runs: sudo chown root:root shellcheck sudo mv shellcheck "$install_dir" fi + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: '3.10' - name: Cache Python environment uses: actions/cache@v3 id: cache-python @@ -80,5 +84,6 @@ runs: run: | if [[ "$CACHE_HIT" != 'true' ]]; then python -m pip install --upgrade pip + pip install wheel pip install -r requirements.txt fi diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 49ebeb5c7..34bc2ca28 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -8,10 +8,6 @@ jobs: steps: - name: Checkout the Git repository uses: actions/checkout@v3 - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: '3.8' - name: Setup Toltec dependencies uses: ./.github/actions/setup - name: Check formatting @@ -25,10 +21,6 @@ jobs: steps: - name: Checkout the Git repository uses: actions/checkout@v3 - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: '3.8' - name: Setup Toltec dependencies uses: ./.github/actions/setup - name: Build packages diff --git a/.github/workflows/stable.yml b/.github/workflows/stable.yml index c53b6edfc..e4d880017 100644 --- a/.github/workflows/stable.yml +++ b/.github/workflows/stable.yml @@ -10,10 +10,6 @@ jobs: steps: - name: Checkout the Git repository uses: actions/checkout@v3 - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: '3.8' - name: Setup Toltec dependencies uses: ./.github/actions/setup - name: Build packages diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 8c961e419..883718745 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -10,10 +10,6 @@ jobs: steps: - name: Checkout the Git repository uses: actions/checkout@v3 - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: '3.8' - name: Setup Toltec dependencies uses: ./.github/actions/setup - name: Build packages diff --git a/docs/building.md b/docs/building.md index 38d95e91b..6696d363a 100644 --- a/docs/building.md +++ b/docs/building.md @@ -18,7 +18,7 @@ Before running the build, make sure you have all the required dependencies: * Docker * bsdtar -* Python ⩾ 3.8 +* Python 3.10 You’ll also need all the Python modules listed in [requirements.txt](../requirements.txt) (install them by running `pip install --user -r requirements.txt` or using a [virtual environment](https://docs.python.org/3/tutorial/venv.html)). diff --git a/requirements.txt b/requirements.txt index b18273a33..a36680a62 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,9 @@ appdirs==1.4.4 -astroid==2.4.2 -black==20.8b1 +astroid==2.15.4 +black==23.3.0 certifi==2020.12.5 chardet==4.0.0 -click==7.1.2 +click==8.1.3 docker==4.4.1 idna==2.10 isort==5.7.0 @@ -13,16 +13,15 @@ MarkupSafe==1.1.1 mccabe==0.6.1 mypy==0.790 mypy-extensions==0.4.3 -pathspec==0.8.1 +pathspec==0.11.1 pyelftools==0.27 -pylint==2.6.0 +pylint==2.17.4 python-dateutil==2.8.1 regex==2020.11.13 requests==2.25.1 six==1.15.0 toml==0.10.2 -typed-ast==1.4.2 -typing-extensions==3.7.4.3 +typing-extensions==4.5.0 urllib3==1.26.2 websocket-client==0.57.0 wrapt==1.12.1 diff --git a/scripts/toltec/bash.py b/scripts/toltec/bash.py index cb194e253..b763dd18c 100644 --- a/scripts/toltec/bash.py +++ b/scripts/toltec/bash.py @@ -334,6 +334,7 @@ def run_script(variables: Variables, script: str) -> LogGenerator: :returns: generator yielding output lines from the script :raises ScriptError: if the script exits with a non-zero code """ + # pylint: disable-next=consider-using-with process = subprocess.Popen( ["/usr/bin/env", "bash"], stdin=subprocess.PIPE, diff --git a/scripts/toltec/builder.py b/scripts/toltec/builder.py index 6a5401747..27253eef2 100644 --- a/scripts/toltec/builder.py +++ b/scripts/toltec/builder.py @@ -96,6 +96,7 @@ def __init__(self, work_dir: str, repo_dir: str) -> None: self.context: Dict[str, str] = {} self.adapter = BuildContextAdapter(logger, self.context) + # pylint: disable-next=unspecified-encoding with open(install_lib_path, "r") as file: for line in file: if not line.strip().startswith("#"): @@ -201,7 +202,7 @@ def _fetch_sources( ) else: # Fetch source file from the network - req = requests.get(source.url) + req = requests.get(source.url, timeout=5) if req.status_code != 200: raise BuildError( @@ -215,7 +216,7 @@ def _fetch_sources( # Verify checksum file_sha = util.file_sha256(local_path) - if source.checksum != "SKIP" and file_sha != source.checksum: + if source.checksum not in ("SKIP", source.checksum): raise BuildError( f"Invalid checksum for source file {source.url}:\n" f" expected {source.checksum}\n" @@ -381,6 +382,7 @@ def _postprocessing(self, recipe: Recipe, src_dir: str) -> None: script = [] mount_src = "/src" + # pylint: disable-next=unnecessary-lambda-assignment docker_file_path = lambda file_path: shlex.quote( os.path.join(mount_src, os.path.relpath(file_path, src_dir)) ) diff --git a/scripts/toltec/graphlib.py b/scripts/toltec/graphlib.py index 2a303928c..12c12446d 100644 --- a/scripts/toltec/graphlib.py +++ b/scripts/toltec/graphlib.py @@ -225,6 +225,7 @@ def _find_cycle(self) -> Optional[List[_T]]: seen = set() node2stacki: MutableMapping[_T, int] = {} + # pylint: disable-next=consider-using-dict-items for node in n2i: if node in seen: continue diff --git a/scripts/toltec/ipk.py b/scripts/toltec/ipk.py index 78678bd0c..4a064ba55 100644 --- a/scripts/toltec/ipk.py +++ b/scripts/toltec/ipk.py @@ -22,6 +22,7 @@ def _targz_open(fileobj: IO[bytes], epoch: int) -> tarfile.TarFile: ) try: + # pylint: disable-next=consider-using-with archive = tarfile.TarFile( mode="w", fileobj=gzipobj, # type:ignore diff --git a/scripts/toltec/recipe.py b/scripts/toltec/recipe.py index e462be5ec..fc1cf9ae0 100644 --- a/scripts/toltec/recipe.py +++ b/scripts/toltec/recipe.py @@ -41,6 +41,7 @@ def from_file(path: str) -> "GenericRecipe": :returns: loaded recipe """ name = os.path.basename(path) + # pylint: disable-next=unspecified-encoding with open(os.path.join(path, "package"), "r") as recipe: return GenericRecipe(name, path, recipe.read()) diff --git a/scripts/toltec/repo.py b/scripts/toltec/repo.py index c99fe3988..af9c1ad28 100644 --- a/scripts/toltec/repo.py +++ b/scripts/toltec/repo.py @@ -24,6 +24,8 @@ class PackageStatus(Enum): """Possible existence statuses of a built package.""" + # pylint: disable=invalid-name + # The package already existed in the local filesystem before the build AlreadyExists = auto() @@ -33,6 +35,8 @@ class PackageStatus(Enum): # The package is missing both from the local filesystem and the remote repo Missing = auto() + # pylint: enable=invalid-name + GroupedPackages = Dict[PackageStatus, Dict[str, Dict[str, List[Package]]]] @@ -132,7 +136,7 @@ def fetch_package( remote_path = os.path.join(remote, filename) - req = requests.get(remote_path) + req = requests.get(remote_path, timeout=5) if req.status_code != 200: return PackageStatus.Missing @@ -208,6 +212,7 @@ def make_index(self) -> None: index_path = os.path.join(arch_dir, "Packages") index_gzip_path = os.path.join(arch_dir, "Packages.gz") + # pylint: disable-next=unspecified-encoding with open(index_path, "w") as index_file: with gzip.open(index_gzip_path, "wt") as index_gzip_file: for generic_recipe in self.generic_recipes.values(): @@ -258,5 +263,6 @@ def make_listing(self) -> None: listing_path = os.path.join(self.repo_dir, "index.html") template = templating.env.get_template("listing.html") + # pylint: disable-next=unspecified-encoding with open(listing_path, "w") as listing_file: listing_file.write(template.render(sections=sections)) diff --git a/scripts/toltec/version.py b/scripts/toltec/version.py index 67dcb46f4..91c143e5a 100644 --- a/scripts/toltec/version.py +++ b/scripts/toltec/version.py @@ -16,12 +16,16 @@ class VersionComparator(Enum): """Operators used to compare two version numbers.""" + # pylint: disable=invalid-name + LowerThan = "<<" LowerThanOrEqual = "<=" Equal = "=" GreaterThanOrEqual = ">=" GreaterThan = ">>" + # pylint: enable=invalid-name + class InvalidVersionError(Exception): """Raised when parsing of an invalid version is attempted.""" @@ -100,6 +104,8 @@ def __repr__(self) -> str: class DependencyKind(Enum): """Kinds of dependencies that may be requested by a package.""" + # pylint: disable=invalid-name + # Dependency installed in the system used to build a package # (e.g., a Debian package) Build = "build" @@ -107,6 +113,8 @@ class DependencyKind(Enum): # (e.g., another Entware or Toltec package) Host = "host" + # pylint: enable=invalid-name + class InvalidDependencyError(Exception): """Raised when parsing an invalid dependency specification."""