feat: add --strict option to fail on exising files (gorilla-co#113)

* feat: require python 3.8 or greater

* refactor: move arguments to config dataclasses

* feat: add --strict option to fail on exising files

* feat: install mypy and add type annotations

* chore: bump version to 1.2.0

* test: add test case without --strict

Author: mdwint

.github/workflows/tests.yml

@@ -7,7 +7,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11']
+        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
 
     steps:
     - uses: actions/checkout@v2

CHANGELOG.md

@@ -6,6 +6,17 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/).
 
 
+## 1.2.0 - 2023-12-30
+
+### Added
+
+- `--strict` option to fail when trying to upload existing files.
+
+### Changed
+
+- Require Python 3.8 or greater.
+
+
 ## 1.1.1 - 2023-02-20
 
 ### Fixed

poetry.lock

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "s3pypi"
-version = "1.1.1"
+version = "1.2.0"
 description = "CLI for creating a Python Package Repository in an S3 bucket"
 authors = [
     "Matteo De Wint <matteo@gorilla.co>",
@@ -11,28 +11,19 @@ authors = [
 s3pypi = "s3pypi.__main__:main"
 
 [tool.poetry.dependencies]
-boto3 = "^1.26.32"
-python = "^3.7"
+boto3 = "^1.34.11"
+boto3-stubs = {extras = ["s3"], version = "^1.34.11"}
+python = "^3.8"
 
-[tool.poetry.dev-dependencies]
-black = "^22.12.0"
-flake8 = "^5.0.0"
-isort = "^5.11.3"
-moto = "^4.0.12"
-pytest = "^7.2.0"
-pytest-cov = "^4.0.0"
-
-[tool.black]
-exclude = '''
-      \.eggs
-    | \.git
-    | \.mypy_cache
-    | \.tox
-    | \.venv
-    | _build
-    | build
-    | dist
-'''
+[tool.poetry.group.dev.dependencies]
+black = "^23.12.1"
+bump2version = "^1.0.1"
+flake8 = "^5.0.4"
+isort = "^5.13.2"
+moto = "^4.2.12"
+mypy = "^1.8.0"
+pytest = "^7.4.3"
+pytest-cov = "^4.1.0"
 
 [build-system]
 requires = ["poetry>=0.12"]

pyproject.toml

@@ -1,2 +1,2 @@
 __prog__ = "s3pypi"
-__version__ = "1.1.1"
+__version__ = "1.2.0"

s3pypi/__init__.py

@@ -1,8 +1,8 @@
 from __future__ import print_function
 
-import argparse
 import logging
 import sys
+from argparse import ArgumentParser
 from pathlib import Path
 from typing import Dict
 
@@ -16,8 +16,8 @@ def string_dict(text: str) -> Dict[str, str]:
     return dict(tuple(item.strip().split("=", 1)) for item in text.split(","))  # type: ignore
 
 
-def get_arg_parser():
-    p = argparse.ArgumentParser(prog=__prog__)
+def build_arg_parser() -> ArgumentParser:
+    p = ArgumentParser(prog=__prog__)
     p.add_argument(
         "dist",
         nargs="+",
@@ -33,6 +33,7 @@ def get_arg_parser():
     p.add_argument(
         "--s3-put-args",
         type=string_dict,
+        default={},
         help=(
             "Optional extra arguments to S3 PutObject calls. Example: "
             "'ServerSideEncryption=aws:kms,SSEKMSKeyId=1234...'"
@@ -67,18 +68,48 @@ def get_arg_parser():
         action="store_true",
         help="Don't use authentication when communicating with S3.",
     )
-    p.add_argument("-f", "--force", action="store_true", help="Overwrite files.")
+
+    g = p.add_mutually_exclusive_group()
+    g.add_argument(
+        "--strict",
+        action="store_true",
+        help="Fail when trying to upload existing files.",
+    )
+    g.add_argument(
+        "-f", "--force", action="store_true", help="Overwrite existing files."
+    )
+
     p.add_argument("-v", "--verbose", action="store_true", help="Verbose output.")
     p.add_argument("-V", "--version", action="version", version=__version__)
     return p
 
 
-def main(*args):
-    kwargs = vars(get_arg_parser().parse_args(args or sys.argv[1:]))
-    log.setLevel(logging.DEBUG if kwargs.pop("verbose") else logging.INFO)
+def main(*raw_args: str) -> None:
+    args = build_arg_parser().parse_args(raw_args or sys.argv[1:])
+    log.setLevel(logging.DEBUG if args.verbose else logging.INFO)
+
+    cfg = core.Config(
+        dist=args.dist,
+        s3=core.S3Config(
+            bucket=args.bucket,
+            prefix=args.prefix,
+            endpoint_url=args.s3_endpoint_url,
+            put_kwargs=args.s3_put_args,
+            unsafe_s3_website=args.unsafe_s3_website,
+            no_sign_request=args.no_sign_request,
+        ),
+        strict=args.strict,
+        force=args.force,
+        lock_indexes=args.lock_indexes,
+        put_root_index=args.put_root_index,
+        profile=args.profile,
+        region=args.region,
+    )
+    if args.acl:
+        cfg.s3.put_kwargs["ACL"] = args.acl
 
     try:
-        core.upload_packages(**kwargs)
+        core.upload_packages(cfg)
     except core.S3PyPiError as e:
         sys.exit(f"ERROR: {e}")
 

s3pypi/__main__.py

@@ -15,13 +15,25 @@
 from s3pypi.exceptions import S3PyPiError
 from s3pypi.index import Hash
 from s3pypi.locking import DummyLocker, DynamoDBLocker
-from s3pypi.storage import S3Storage
+from s3pypi.storage import S3Config, S3Storage
 
 log = logging.getLogger(__prog__)
 
 PackageMetadata = email.message.Message
 
 
+@dataclass
+class Config:
+    dist: List[Path]
+    s3: S3Config
+    strict: bool = False
+    force: bool = False
+    lock_indexes: bool = False
+    put_root_index: bool = False
+    profile: Optional[str] = None
+    region: Optional[str] = None
+
+
 @dataclass
 class Distribution:
     name: str
@@ -33,26 +45,18 @@ def normalize_package_name(name: str) -> str:
     return re.sub(r"[-_.]+", "-", name.lower())
 
 
-def upload_packages(
-    dist: List[Path],
-    bucket: str,
-    force: bool = False,
-    lock_indexes: bool = False,
-    put_root_index: bool = False,
-    profile: Optional[str] = None,
-    region: Optional[str] = None,
-    **kwargs,
-):
-    session = boto3.Session(profile_name=profile, region_name=region)
-    storage = S3Storage(session, bucket, **kwargs)
+def upload_packages(cfg: Config) -> None:
+    session = boto3.Session(profile_name=cfg.profile, region_name=cfg.region)
+    storage = S3Storage(session, cfg.s3)
     lock = (
-        DynamoDBLocker(session, table=f"{bucket}-locks")
-        if lock_indexes
+        DynamoDBLocker(session, table=f"{cfg.s3.bucket}-locks")
+        if cfg.lock_indexes
         else DummyLocker()
     )
 
-    distributions = parse_distributions(dist)
+    distributions = parse_distributions(cfg.dist)
     get_name = attrgetter("name")
+    existing_files = []
 
     for name, group in groupby(sorted(distributions, key=get_name), get_name):
         directory = normalize_package_name(name)
@@ -62,7 +66,8 @@ def upload_packages(
             for distr in group:
                 filename = distr.local_path.name
 
-                if not force and filename in index.filenames:
+                if not cfg.force and filename in index.filenames:
+                    existing_files.append(filename)
                     msg = "%s already exists! (use --force to overwrite)"
                     log.warning(msg, filename)
                 else:
@@ -72,11 +77,14 @@ def upload_packages(
 
             storage.put_index(directory, index)
 
-    if put_root_index:
+    if cfg.put_root_index:
         with lock(storage.root):
             index = storage.build_root_index()
             storage.put_index(storage.root, index)
 
+    if cfg.strict and existing_files:
+        raise S3PyPiError(f"Found {len(existing_files)} existing files on S3")
+
 
 def parse_distribution(path: Path) -> Distribution:
     extensions = (".whl", ".tar.gz", ".tar.bz2", ".tar.xz", ".zip")

s3pypi/core.py

@@ -5,6 +5,7 @@
 import logging
 import time
 from contextlib import contextmanager
+from typing import Iterator
 
 import boto3
 
@@ -15,7 +16,7 @@
 
 class Locker(abc.ABC):
     @contextmanager
-    def __call__(self, key: str):
+    def __call__(self, key: str) -> Iterator[None]:
         lock_id = hashlib.sha1(key.encode()).hexdigest()
         self._lock(lock_id)
         try:
@@ -24,16 +25,16 @@ def __call__(self, key: str):
             self._unlock(lock_id)
 
     @abc.abstractmethod
-    def _lock(self, lock_id: str):
+    def _lock(self, lock_id: str) -> None:
         ...
 
     @abc.abstractmethod
-    def _unlock(self, lock_id: str):
+    def _unlock(self, lock_id: str) -> None:
         ...
 
 
 class DummyLocker(Locker):
-    def _lock(self, lock_id: str):
+    def _lock(self, lock_id: str) -> None:
         pass
 
     _unlock = _lock
@@ -54,7 +55,7 @@ def __init__(
         self.max_attempts = max_attempts
         self.caller_id = session.client("sts").get_caller_identity()["Arn"]
 
-    def _lock(self, lock_id: str):
+    def _lock(self, lock_id: str) -> None:
         for attempt in range(1, self.max_attempts + 1):
             now = dt.datetime.now(dt.timezone.utc)
             try:
@@ -76,7 +77,7 @@ def _lock(self, lock_id: str):
         item = self.table.get_item(Key={"LockID": lock_id})["Item"]
         raise DynamoDBLockTimeoutError(self.table.name, item)
 
-    def _unlock(self, lock_id: str):
+    def _unlock(self, lock_id: str) -> None:
         self.table.delete_item(Key={"LockID": lock_id})