add support to BaseClient for blob upload

 * change BaseClientV2._http_response to accept bindata which is passed
directly to requests
 * add _Manifest.from_file to support loading a manifest from a file
then calling put_manifest
 * add BaseClientV2.put_blob based on implementation in python-dxf

Author: Matthew Stoltenberg

docker_registry_client/_BaseClient.py

@@ -1,10 +1,18 @@
 import logging
-from requests import get, put, delete
+from requests import codes, get, head, post, put, delete
 from requests.exceptions import HTTPError
 import json
 from .AuthorizationService import AuthorizationService
+from .digest import docker_digest
 from .manifest import sign as sign_manifest
 
+try:
+    import urllib.parse as urlparse
+    from urllib.parse import urlencode
+except ImportError:
+    from urllib import urlencode
+    import urlparse
+
 # urllib3 throws some ssl warnings with older versions of python
 #   they're probably ok for the registry client to ignore
 import warnings
@@ -139,6 +147,14 @@ def __init__(self, content, type, digest):
         self._type = type
         self._digest = digest
 
+    @classmethod
+    def from_file(cls, fpath=None, fobj=None):
+        digest = docker_digest(fpath, fobj)
+        if fobj is None:
+            fobj = open(fpath, 'rb')
+        return cls(json.loads(fobj.read().decode()),
+                   'application/json', digest)
+
 
 BASE_CONTENT_TYPE = 'application/vnd.docker.distribution.manifest'
 
@@ -147,6 +163,7 @@ class BaseClientV2(CommonBaseClient):
     LIST_TAGS = '/v2/{name}/tags/list'
     MANIFEST = '/v2/{name}/manifests/{reference}'
     BLOB = '/v2/{name}/blobs/{digest}'
+    BLOB_UPLOAD = '/v2/{name}/blobs/uploads/'
     schema_1_signed = BASE_CONTENT_TYPE + '.v1+prettyjws'
     schema_1 = BASE_CONTENT_TYPE + '.v1+json'
     schema_2 = BASE_CONTENT_TYPE + '.v2+json'
@@ -208,6 +225,31 @@ def put_manifest(self, name, reference, manifest):
             name=name, reference=reference,
         )
 
+    def put_blob(self, name, fpath=None, fobj=None):
+        self.auth.desired_scope = 'repository:%s:*' % name
+        digest = docker_digest(fpath, fobj)
+        try:
+            self._http_call(self.BLOB, head,
+                            name=name, digest=digest)
+            return digest
+        except HTTPError as exc:
+            if exc.response.status_code != codes.not_found:
+                raise
+        if fobj is None:
+            fobj = open(fpath, 'rb')
+        resp = self._http_response(self.BLOB_UPLOAD, post,
+                                   name=name)
+        parts = list(urlparse.urlparse(resp.headers['Location']))
+        query = urlparse.parse_qs(parts[4])
+        query.update({'digest': digest})
+        parts[0] = ''  # scheme
+        parts[1] = ''  # netloc
+        parts[4] = urlencode(query, True)
+        self._http_call(urlparse.urlunparse(parts),
+                        put, bindata=fobj,
+                        name=name, digest=digest,
+                        content_type='application/octet-stream')
+
     def delete_manifest(self, name, digest):
         self.auth.desired_scope = 'repository:%s:*' % name
         return self._http_call(self.MANIFEST, delete,
@@ -227,7 +269,7 @@ def _cache_manifest_digest(self, name, reference, response=None):
         self._manifest_digests[(name, reference)] = untrusted_digest
 
     def _http_response(self, url, method, data=None, content_type=None,
-                       schema=None, **kwargs):
+                       bindata=None, schema=None, **kwargs):
         """url -> full target url
            method -> method from requests
            data -> request body
@@ -258,6 +300,8 @@ def _http_response(self, url, method, data=None, content_type=None,
 
         if data and not content_type:
             data = json.dumps(data)
+        if bindata:
+            data = bindata
 
         path = url.format(**kwargs)
         logger.debug("%s %s", method.__name__.upper(), path)

docker_registry_client/digest.py

@@ -0,0 +1,14 @@
+import hashlib
+
+
+def docker_digest(fpath=None, fobj=None, prepend=True):
+    if fobj is None:
+        fobj = open(fpath, 'rb')
+    hasher = hashlib.sha256()
+    for chunk in iter(lambda: fobj.read(8192), b''):
+        hasher.update(chunk)
+    fobj.seek(0)
+    retval = hasher.hexdigest()
+    if prepend:
+        retval = 'sha256:' + retval
+    return retval

tests/test_base_client.py

@@ -1,6 +1,9 @@
 from __future__ import absolute_import
+from io import BytesIO
 
-from docker_registry_client._BaseClient import BaseClientV1, BaseClientV2
+from docker_registry_client._BaseClient import (
+    BaseClientV1, BaseClientV2, _Manifest
+)
 from drc_test_utils.mock_registry import (
     mock_v1_registry, mock_v2_registry, TEST_NAME, TEST_TAG,
 )
@@ -21,3 +24,14 @@ def test_get_manifest_and_digest(self):
         url = mock_v2_registry()
         manifest, digest = BaseClientV2(url).get_manifest_and_digest(TEST_NAME,
                                                                      TEST_TAG)
+
+
+class TestManifest(object):
+    FAKE_MANIFEST = b'{ "schemaVersion": 2 }'
+    FAKE_DIGEST = ('sha256:0467ad45d6957ca671e3d219aa5965d4'
+                   '7f8621823a80c8e76174bcc1b9a225fd')
+
+    def test_fromfile(self):
+        fobj = BytesIO(self.FAKE_MANIFEST)
+        manifest = _Manifest.from_file(fobj=fobj)
+        assert manifest._digest == self.FAKE_DIGEST

tox.ini

@@ -7,6 +7,7 @@ deps =
     docker-py==1.10.6
     flexmock==0.10.2
     pytest==3.0.5
+    ecdsa==0.13
 
 [testenv:lint]
 deps =