v2.6.8

Author: chenjing.cjcj

CHANGELOG.rst

@@ -1,5 +1,11 @@
 TOS SDK for Python 版本记录
 ===========================
+Version 2.6.8
+-------------
+- 增加：BucketTagging相关接口
+- 改变：重试时重新生成签名
+- 改变：签名只签必要的Header
+
 Version 2.6.7
 -------------
 - 修复：pre_signed_post_signature部分参数为可选

LICENSE

@@ -19,7 +19,6 @@ def test_generate_presigned_url(self):
         with mock.patch('datetime.datetime', new=datetime_mock):
             tos_cli = tos.TosClient(tos.Auth('ak', 'sk', 'beijing'), 'tos-cn-beijing.volces.com')
             url = tos_cli.generate_presigned_url(Method='GET', Bucket='bkt', Key='key', ExpiresIn=86400)
-            print(url)
             self.assertEqual(url,
                              'https://bkt.tos-cn-beijing.volces.com/key?X-Tos-Algorithm=TOS4-HMAC-SHA256&X-Tos-Creden'
                              'tial=ak%2F20210101%2Fbeijing%2Ftos%2Frequest&X-Tos-Date=20210101T000000Z&X-Tos-Expires=8'
@@ -31,14 +30,14 @@ def test_generate_presigned_url(self):
             self.assertEqual(url,
                              'https://bkt.tos-cn-beijing.volces.com/key?X-Tos-Algorithm=TOS4-HMAC-SHA256&X-Tos-Credentia'
                              'l=ak%2F20210101%2Fbeijing%2Ftos%2Frequest&X-Tos-Date=20210101T000000Z&X-Tos-Expires=36'
-                             '00&X-Tos-SignedHeaders=host&X-Tos-Security-Token=sts&X-Tos-Signature=3041fb481e31ec25f'
+                             '00&X-Tos-Security-Token=sts&X-Tos-SignedHeaders=host&X-Tos-Signature=3041fb481e31ec25f'
                              'e7a1be44cafe25caf1cd97228710ee440b2ca1bd49bc563')
 
             url = tos_cli.generate_presigned_url(Method='PUT', Bucket='bkt', Key='key', Params={'acl': ''})
             self.assertEqual(url,
                              'https://bkt.tos-cn-beijing.volces.com/key?acl&X-Tos-Algorithm=TOS4-HMAC-SHA256&X-Tos-C'
                              'redential=ak%2F20210101%2Fbeijing%2Ftos%2Frequest&X-Tos-Date=20210101T000000Z&X-Tos-Exp'
-                             'ires=3600&X-Tos-SignedHeaders=host&X-Tos-Security-Token=sts&X-Tos-Signature=51c89070206'
+                             'ires=3600&X-Tos-Security-Token=sts&X-Tos-SignedHeaders=host&X-Tos-Signature=51c89070206'
                              'dd438fd8be1ed201a77335af80e33cad5fd408841590b99aa93d3')
 
     def test_client2_presigned_url(self):
@@ -70,20 +69,19 @@ def test_client2_presigned_url(self):
             self.assertEqual(url.signed_url,
                              'https://bkt.tos-cn-beijing.volces.com/key?acl&X-Tos-Algorithm=TOS4-HMAC-SHA256&X-Tos-C'
                              'redential=ak%2F20210101%2Fbeijing%2Ftos%2Frequest&X-Tos-Date=20210101T000000Z&X-Tos-Exp'
-                             'ires=3600&X-Tos-SignedHeaders=host&X-Tos-Security-Token=sts&X-Tos-Signature=51c89070206'
+                             'ires=3600&X-Tos-Security-Token=sts&X-Tos-SignedHeaders=host&X-Tos-Signature=51c89070206'
                              'dd438fd8be1ed201a77335af80e33cad5fd408841590b99aa93d3')
 
             url = tos_cli.pre_signed_url(http_method=HttpMethodType.Http_Method_Put, bucket='bkt', key='key')
             self.assertEqual(url.signed_url,
                              'https://bkt.tos-cn-beijing.volces.com/key?X-Tos-Algorithm=TOS4-HMAC-SHA256&X-Tos-Credentia'
                              'l=ak%2F20210101%2Fbeijing%2Ftos%2Frequest&X-Tos-Date=20210101T000000Z&X-Tos-Expires=36'
-                             '00&X-Tos-SignedHeaders=host&X-Tos-Security-Token=sts&X-Tos-Signature=3041fb481e31ec25f'
+                             '00&X-Tos-Security-Token=sts&X-Tos-SignedHeaders=host&X-Tos-Signature=3041fb481e31ec25f'
                              'e7a1be44cafe25caf1cd97228710ee440b2ca1bd49bc563')
             url = tos_cli.pre_signed_url(http_method=HttpMethodType.Http_Method_Put, bucket='bkt', key='key',
                                          expires=1234, header={'contentLength': 1000})
             self.assertTrue('1234' in url.signed_url)
-            self.assertTrue('contentlength' in url.signed_url)
-            url.signed_header['contentLength'] = 1000
+            self.assertTrue('contentlength' not in url.signed_url)
 
             self.anonymousCli = tos.TosClientV2(ak='', sk='', endpoint='tos-cn-beijing.volces.com', region='beijing')
             url = self.anonymousCli.pre_signed_url(http_method=HttpMethodType.Http_Method_Put, bucket='bkt', key='key',

tests/test_auth.py

@@ -1,10 +1,10 @@
 # -*- coding: utf-8 -*-
-import http
+import datetime
 import os
 import time
 import unittest
 from io import StringIO
-import http.client as httplib
+from unittest import mock
 import requests
 from requests.exceptions import RetryError
 from urllib3.exceptions import NewConnectionError
@@ -15,11 +15,13 @@
     convert_az_redundancy_type, PermissionType, convert_permission_type, GranteeType, convert_grantee_type, \
     convert_canned_type, CannedType, RedirectType, convert_redirect_type, StatusType, convert_status_type, \
     StorageClassInheritDirectiveType, convert_storage_class_inherit_directive_type, VersioningStatusType, \
-    convert_versioning_status_type, ProtocolType, convert_protocol_type, CertStatus, convert_cert_status
+    convert_versioning_status_type, ProtocolType, convert_protocol_type, CertStatus, convert_cert_status, \
+    StaticCredentialsProvider
+from tos.auth import CredentialProviderAuth
 from tos.checkpoint import CancelHook
-from tos.clientv2 import _handler_retry_policy, _is_wrapper_data
+from tos.clientv2 import _handler_retry_policy, _is_wrapper_data, _signed_req
 from tos.exceptions import TosServerError, CancelNotWithAbortError, CancelWithAbortError
-from tos.http import Response
+from tos.http import Response, Request
 from tos.utils import SizeAdapter
 
 
@@ -226,6 +228,27 @@ def test_convert_enum_type(self):
             assert t == convert_cert_status(t.value)
         assert CertStatus.Cert_Unknown == convert_cert_status('test')
 
+    def test_sign_req(self):
+        auth = CredentialProviderAuth(StaticCredentialsProvider('ak', 'sk', 'sts'), 'region')
+        host = 'zzz.com'
+        headers = {'content-type': 'application/json', 'Host': host}
+        params = {'versionId': 'test'}
+
+        req = Request('GET', 'http://zzz.com/key', 'key', 'zzz.com',
+                      params=params,
+                      headers=headers)
+
+        datetime_mock = mock.Mock(wraps=datetime.datetime)
+        datetime_mock.utcnow.return_value = datetime.datetime(2021, 1, 1)
+        with mock.patch('datetime.datetime', new=datetime_mock):
+            req = _signed_req(auth, req, host)
+            self.assertEqual(req.headers['Authorization'],
+                             'TOS4-HMAC-SHA256 Credential=ak/20210101/region/tos/request, '
+                             'SignedHeaders=content-type;host;x-tos-date;x-tos-security-token, Signature=ca8eb8987663e61e740bc5be9078d6d4f716990573a56c86a6602d42faf3af7e')
+            self.assertEqual(req.headers['Host'], host)
+            self.assertEqual(req.headers['x-tos-date'], '20210101T000000Z')
+            self.assertEqual(req.headers['x-tos-security-token'], 'sts')
+
 
 class args(object):
     def __init__(self, body, method, fun_name, client_exp, server_exp, want, expect_data=None):

tests/test_base_func.py

@@ -717,6 +717,30 @@ def test_put_bucket_real_time_log(self):
         with self.assertRaises(TosServerError):
             self.client2.get_bucket_real_time_log(bucket_name)
 
+    def test_bucket_tagging(self):
+        bucket_name = self.bucket_name  # + '-bucket-tagging'
+        self.client.create_bucket(bucket_name)
+        self.bucket_delete.append(bucket_name)
+        tag_set = [Tag(
+            key='1',
+            value='2'
+        ), Tag(
+            key='3',
+            value='4'
+        )]
+        self.client.put_bucket_tagging(bucket_name, tag_set)
+        out = self.client.get_bucket_tagging(bucket_name)
+        self.assertIsNotNone(out.request_id)
+        self.assertEqual(len(out.tag_set), 2)
+        self.assertEqual(out.tag_set[0].key, tag_set[0].key)
+        self.assertEqual(out.tag_set[0].value, tag_set[0].value)
+        self.assertEqual(out.tag_set[1].key, tag_set[1].key)
+        self.assertEqual(out.tag_set[1].value, tag_set[1].value)
+        delete_out = self.client.delete_bucket_tagging(bucket_name)
+        self.assertIsNotNone(delete_out.request_id)
+        with self.assertRaises(TosServerError):
+            self.client.get_bucket_tagging(bucket_name)
+
     def retry_assert(self, func):
         for i in range(5):
             if func():

tests/test_v2_bucket.py

@@ -179,8 +179,12 @@ def test_put_with_empty_content(self):
             content = content_io
             content.read()
             key = self.random_key()
-        self.client.put_object(bucket_name, key)
         conn.close()
+        file_name = self.random_filename()
+        file = open(file_name, 'w')
+        file.close()
+        self.client.put_object_from_file(bucket_name, key, file_path=file_name)
+        os.remove(file_name)
 
     def test_put_with_illegal_name(self):
         bucket_name = self.bucket_name + '-put-object-with-illegal-name'

tests/test_v2_object.py

@@ -1 +1 @@
-__version__ = 'v2.6.7'
+__version__ = 'v2.6.8'

tos/__version__.py

@@ -9,7 +9,7 @@
 
 import pytz
 
-from .consts import DATE_FORMAT, UNSIGNED_PAYLOAD, LAST_MODIFY_TIME_DATE_FORMAT
+from .consts import DATE_FORMAT, UNSIGNED_PAYLOAD, LAST_MODIFY_TIME_DATE_FORMAT, SIGNATURE_QUERY_LOWER, V4_PREFIX
 from .credential import FederationCredentials, StaticCredentialsProvider
 from .exceptions import TosClientError
 from .models2 import PreSignedPostSignatureOutPut, ContentLengthRange
@@ -21,37 +21,47 @@
 def _canonical_query_string_params(params):
     results = []
     for param in sorted(params):
+        if param.lower() == SIGNATURE_QUERY_LOWER:
+            continue
         value = str(params[param])
         results.append('%s=%s' % (quote(param, safe='-_.~'),
                                   quote(value, safe='-_.~')))
     cqs = '&'.join(results)
     return cqs
 
 
-def _signed_headers(headers):
-    hl = sorted(headers.items(), key=lambda d: d[0].lower())
-    vl = []
-    for v in hl:
-        vl.append(v[0].lower())
-    return ';'.join(vl)
+def _need_signed_headers(key, is_signing_query):
+    return (key == "content-type" and not is_signing_query) or key.startswith(V4_PREFIX) or key == 'host'
+
+
+def _get_signed_headers(headers, is_signing_query=False):
+    signed_headers = {}
+    for k, v in headers.items():
+        k = k.lower()
+        if _need_signed_headers(k, is_signing_query):
+            signed_headers[k] = v
+    return sorted(signed_headers.items(), key=lambda d: d[0].lower())
+
+
+def _get_signed_header_key(signed_headers):
+    return ';'.join([v[0] for v in signed_headers])
 
 
 def _canonical_headers(headers):
-    hl = sorted(headers.items(), key=lambda d: d[0].lower())
     s = ''
-    for val in hl:
+    for val in headers:
         if isinstance(val[1], list):
             tlist = sorted(val[1])
             for v in tlist:
-                s += val[0] + ':' + v + '\n'
+                s += val[0].lower() + ':' + v + '\n'
         else:
             s += val[0].lower() + ':' + str(val[1]) + '\n'
     return s
 
 
-def _canonical_request(req):
+def _canonical_request(req, signed_headers):
     cr = [req.method.upper(), quote(req.path, safe='/~'), _canonical_query_string_params(req.params),
-          _canonical_headers(req.headers), _signed_headers(req.headers)]
+          _canonical_headers(signed_headers), _get_signed_header_key(signed_headers)]
     if req.headers.get('x-tos-content-sha256'):
         cr.append(req.headers['x-tos-content-sha256'])
     else:
@@ -134,8 +144,9 @@ def sign_request(self, req):
         req.headers['Date'] = date
         req.headers['x-tos-date'] = date
 
-        signature = self._make_signature(req=req, date=date)
-        req.headers['Authorization'] = self._inject_signature_to_request(req, signature, date)
+        signed_headers = _get_signed_headers(req.headers)
+        signature = self._make_signature(req=req, date=date, signed_headers=signed_headers)
+        req.headers['Authorization'] = self._inject_signature_to_request(signature, date, signed_headers)
 
     def sign_url(self, req, expires):
         if expires is None:
@@ -147,11 +158,11 @@ def sign_url(self, req, expires):
         req.params['X-Tos-Credential'] = self._credential(date)
         req.params['X-Tos-Date'] = date
         req.params['X-Tos-Expires'] = expires
-        req.params['X-Tos-SignedHeaders'] = _signed_headers(req.headers)
-
         if self.credential.get_security_token():
             req.params["X-Tos-Security-Token"] = self.credential.get_security_token()
-        req.params['X-Tos-Signature'] = self._make_signature(req=req, date=date)
+        signed_headers = _get_signed_headers(req.headers, True)
+        req.params['X-Tos-SignedHeaders'] = _get_signed_header_key(signed_headers)
+        req.params['X-Tos-Signature'] = self._make_signature(req=req, date=date, signed_headers=signed_headers)
 
         return req.url + '?' + '&'.join(_param_to_quoted_query(k, v) for k, v in req.params.items())
 
@@ -190,9 +201,9 @@ def x_tos_post_sign(self, expires: int, conditions: []):
 
         return '&'.join(_param_to_quoted_query(k, v) for k, v in params.items())
 
-    def _make_signature(self, date, req=None, string_to_sign=None):
+    def _make_signature(self, date, req=None, string_to_sign=None, signed_headers=None):
         if not string_to_sign:
-            canonical_request = _canonical_request(req)
+            canonical_request = _canonical_request(req, signed_headers)
             logger.debug("pre-request: canonical_request:\n%s", canonical_request)
             string_to_sign = self._string_to_sign(canonical_request, date)
         logger.debug("pre-request: string_to_sign:\n%s", string_to_sign)
@@ -209,9 +220,9 @@ def _make_x_tos_policy_signature(self, date, params):
         logger.debug("pre-request: signature:\n%s", signature)
         return signature
 
-    def _inject_signature_to_request(self, req, signature, date):
+    def _inject_signature_to_request(self, signature, date, signed_headers):
         results = ['TOS4-HMAC-SHA256 Credential=%s' % self._credential(date),
-                   'SignedHeaders=%s' % _signed_headers(req.headers), 'Signature=%s' % signature]
+                   'SignedHeaders=%s' % _get_signed_header_key(signed_headers), 'Signature=%s' % signature]
         return ', '.join(results)
 
     def _string_to_sign(self, canonical_request, date):