Added s3upload script to upload single file using S3URL environ

CHANGELOG.md

@@ -6,7 +6,18 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html), 
 as of 0.8.2
 
-## [0.8.2]
+## [0.8.3] - 2022-06-03
+
+### Added
+
+- `s3upload` script to upload single file using `S3URL` environment variable
+- `kiwixstorage.__version__` now available as well
+
+### Changed
+
+- Default progress output when humanfriendly is not present now adds “ bytes” suffix.
+
+## [0.8.2] - 2022-05-18
 
 ### Changed
 

setup.py

@@ -35,7 +35,10 @@ def read(*names, **kwargs):
     zip_safe=False,
     include_package_data=True,
     entry_points={
-        "console_scripts": ["s3_test_url=kiwixstorage.test_credentials:test_url"]
+        "console_scripts": [
+            "s3_test_url=kiwixstorage.test_credentials:test_url",
+            "s3upload=kiwixstorage.upload:upload_file",
+        ]
     },
     classifiers=[
         "Development Status :: 4 - Beta",

src/kiwixstorage/VERSION

@@ -1 +1 @@
-0.8.2
+0.8.3

src/kiwixstorage/__init__.py

@@ -34,9 +34,11 @@
 except ImportError:
 
     def format_size(num_bytes, keep_width=False, binary=False):
-        return str(num_bytes)
+        return f"{num_bytes} bytes"
 
 
+with open(pathlib.Path(__file__).parent / "VERSION", "r") as fh:
+    __version__ = fh.read().strip()
 logger = logging.getLogger(__name__)
 
 
@@ -815,7 +817,7 @@ def calc_etag(fpath, partsize):
         def md5sum(fpath):
             digest = hashlib.md5()  # nosec
             with open(fpath, "rb") as fh:
-                for chunk in iter(lambda: fh.read(2 ** 20 * 8), b""):
+                for chunk in iter(lambda: fh.read(2**20 * 8), b""):
                     digest.update(chunk)
             return digest.hexdigest()
 

src/kiwixstorage/upload.py

@@ -0,0 +1,114 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# vim: ai ts=4 sts=4 et sw=4 nu
+
+""" Simple script to upload a file to a bucket, reusing S3_URL environ
+
+    Optional dependencies:
+    - humanfriendly
+    - progressbar2 """
+
+import argparse
+import os
+import pathlib
+import sys
+
+from kiwixstorage import KiwixStorage, __version__, format_size
+
+try:
+    import progressbar
+except ImportError:
+    progressbar = None
+
+NAME = "s3upload"
+FULLNAME = f"{NAME} v{__version__}"
+
+
+class CustomProgressBar:
+    def __init__(self, total: int = None):
+        widgets = [
+            " [",
+            progressbar.Timer(),
+            "] ",
+            progressbar.DataSize(),
+            progressbar.Bar(),
+            progressbar.AdaptiveTransferSpeed(),
+            " (",
+            progressbar.ETA(),
+            ") ",
+        ]
+        self.bar = progressbar.ProgressBar(max_value=total, widgets=widgets)
+        self.seen_so_far = 0
+
+    def callback(self, bytes_amount: int):
+        self.seen_so_far += bytes_amount
+        self.bar.update(self.seen_so_far)
+
+
+def do_upload_file(url: str, fpath: pathlib.Path, key: str = None):
+    if not fpath.exists():
+        raise IOError(f"{fpath} missing.")
+    fsize = fpath.stat().st_size
+    if not key:
+        key = fpath.name
+
+    s3 = KiwixStorage(url)
+    dest = f"s3://{s3.url.netloc}/{s3.bucket_name}"
+
+    if s3.has_object(key):
+        raise ValueError(f"Key `{key}` already exists at {dest}. Specify another one.")
+    print(f"Uploading {fpath.name} ({format_size(fsize)}) to {dest}/{key}")
+
+    progress = CustomProgressBar(fsize).callback if progressbar else True
+    s3.upload_file(fpath=fpath, key=key, progress=progress)
+
+
+def upload_file():
+    parser = argparse.ArgumentParser(
+        prog="s3upload",
+        description="KiwixStorage-based S3 single file uploader",
+    )
+
+    parser.add_argument(
+        help="File to upload",
+        dest="fpath",
+    )
+
+    parser.add_argument(
+        "--key",
+        help="Key to upload to. Defaults to fpath's name",
+        default=None,
+        dest="key",
+    )
+
+    parser.add_argument(
+        "--url",
+        help="S3 URL with credentials and bucketName. "
+        "Defaults to `S3URL` environment variable",
+        default=os.getenv("S3URL"),
+        dest="url",
+    )
+
+    parser.add_argument(
+        "--version",
+        help="Display builder script version and exit",
+        action="version",
+        version=FULLNAME,
+    )
+
+    parser.set_defaults(url=os.getenv("S3URL"))
+    kwargs = dict(parser.parse_args()._get_kwargs())
+    kwargs["fpath"] = pathlib.Path(kwargs["fpath"]).expanduser().resolve()
+
+    if not kwargs.get("url"):
+        parser.error("the following arguments are required: --url")
+
+    try:
+        sys.exit(do_upload_file(**kwargs))
+    except Exception as exc:
+        print(f"FAILED. An error occurred: {exc}")
+        raise SystemExit(1)
+
+
+if __name__ == "__main__":
+    upload_file()