Pull request batch2: Batch of my different pull requests

README

@@ -63,49 +63,50 @@ Amazon S3 pricing explained
 ---------------------------
 At the time of this writing the costs of using S3 are (in USD):
 
-$0.15 per GB per month of storage space used
+$0.03 per GB per month of storage space used
 
 plus
 
-$0.10 per GB - all data uploaded
+$0.00 per GB - all data uploaded
 
 plus
 
-$0.18 per GB - first 10 TB / month data downloaded
-$0.16 per GB - next 40 TB / month data downloaded
-$0.13 per GB - data downloaded / month over 50 TB
+$0.00 per GB - first 1GB / month data downloaded
+$0.12 per GB - up to 10 TB / month data downloaded
+$0.09 per GB - next 40 TB / month data downloaded
+$0.07 per GB - data downloaded / month over 50 TB
 
 plus
 
-$0.01 per 1,000 PUT or LIST requests
-$0.01 per 10,000 GET and all other requests
+$0.005 per 1,000 PUT or LIST requests
+$0.004 per 10,000 GET and all other requests
 
 If for instance on 1st of January you upload 2GB of 
 photos in JPEG from your holiday in New Zealand, at the 
-end of January you will be charged $0.30 for using 2GB of
-storage space for a month, $0.20 for uploading 2GB
+end of January you will be charged $0.06 for using 2GB of
+storage space for a month, $0.0 for uploading 2GB
 of data, and a few cents for requests. 
-That comes to slightly over $0.50 for a complete backup 
+That comes to slightly over $0.06 for a complete backup 
 of your precious holiday pictures.
 
 In February you don't touch it. Your data are still on S3 
-servers so you pay $0.30 for those two gigabytes, but not
+servers so you pay $0.06 for those two gigabytes, but not
 a single cent will be charged for any transfer. That comes 
-to $0.30 as an ongoing cost of your backup. Not too bad.
+to $0.06 as an ongoing cost of your backup. Not too bad.
 
 In March you allow anonymous read access to some of your
 pictures and your friends download, say, 500MB of them. 
 As the files are owned by you, you are responsible for the 
 costs incurred. That means at the end of March you'll be 
-charged $0.30 for storage plus $0.09 for the download traffic 
+charged $0.06 for storage plus $0.06 for the download traffic 
 generated by your friends.
 
 There is no minimum monthly contract or a setup fee. What 
 you use is what you pay for. At the beginning my bill used
 to be like US$0.03 or even nil.
 
 That's the pricing model of Amazon S3 in a nutshell. Check
-Amazon S3 homepage at http://aws.amazon.com/s3 for more 
+Amazon S3 homepage at http://aws.amazon.com/s3/pricing/ for more 
 details.
 
 Needless to say that all these money are charged by Amazon 
@@ -367,4 +368,4 @@ the Free Software Foundation; either version 2 of the License, or
 This program is distributed in the hope that it will be useful,
 but WITHOUT ANY WARRANTY; without even the implied warranty of
 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
+GNU General Public License for more details.

S3/ACL.py

@@ -159,11 +159,11 @@ def grant(self, name, permission):
         grantee.name = name
         grantee.permission = permission
 
-        if  name.find('@') > -1:
+        if  '@' in name:
             grantee.name = grantee.name.lower()
             grantee.xsi_type = "AmazonCustomerByEmail"
             grantee.tag = "EmailAddress"
-        elif name.find('http://acs.amazonaws.com/groups/') > -1:
+        elif 'http://acs.amazonaws.com/groups/' in name:
             grantee.xsi_type = "Group"
             grantee.tag = "URI"
         else:

S3/Exceptions.py

@@ -6,6 +6,7 @@
 
 from Utils import getTreeFromXml, unicodise, deunicodise
 from logging import debug, info, warning, error
+import ExitCodes
 
 try:
     import xml.etree.ElementTree as ET
@@ -64,6 +65,26 @@ def __unicode__(self):
             retval += (u": %s" % self.info["Message"])
         return retval
 
+    def get_error_code(self):
+        if self.status in [301, 307]:
+            return ExitCodes.EX_SERVERMOVED
+        elif self.status in [400, 405, 411, 416, 501]:
+            return ExitCodes.EX_SERVERERROR
+        elif self.status == 403:
+            return ExitCodes.EX_ACCESSDENIED
+        elif self.status == 404:
+            return ExitCodes.EX_NOTFOUND
+        elif self.status == 409:
+            return ExitCodes.EX_CONFLICT
+        elif self.status == 412:
+            return ExitCodes.EX_PRECONDITION
+        elif self.status == 500:
+            return ExitCodes.EX_SOFTWARE
+        elif self.status == 503:
+            return ExitCodes.EX_SERVICE
+        else:
+            return ExitCodes.EX_SOFTWARE
+
     @staticmethod
     def parse_error_xml(tree):
         info = {}

S3/ExitCodes.py

@@ -1,16 +1,22 @@
 # patterned on /usr/include/sysexits.h
 
-EX_OK         = 0
-EX_GENERAL    = 1
-EX_SOMEFAILED = 2    # some parts of the command succeeded, while others failed
-EX_USAGE      = 64   # The command was used incorrectly (e.g. bad command line syntax)
-EX_SOFTWARE   = 70   # internal software error (e.g. S3 error of unknown specificity)
-EX_OSERR      = 71   # system error (e.g. out of memory)
-EX_OSFILE     = 72   # OS error (e.g. invalid Python version)
-EX_IOERR      = 74   # An error occurred while doing I/O on some file.
-EX_TEMPFAIL   = 75   # temporary failure (S3DownloadError or similar, retry later)
-EX_NOPERM     = 77   # Insufficient permissions to perform the operation on S3  
-EX_CONFIG     = 78   # Configuration file error
-_EX_SIGNAL    = 128
-_EX_SIGINT    = 2
-EX_BREAK      = _EX_SIGNAL + _EX_SIGINT # Control-C (KeyboardInterrupt raised)
+EX_OK               = 0
+EX_GENERAL          = 1
+EX_PARTIAL          = 2    # some parts of the command succeeded, while others failed
+EX_SERVERMOVED      = 10   # 301: Moved permanantly & 307: Moved temp
+EX_SERVERERROR      = 11   # 400, 405, 411, 416, 501: Bad request
+EX_NOTFOUND         = 12   # 404: Not found
+EX_CONFLICT         = 13   # 409: Conflict (ex: bucket error)
+EX_PRECONDITION     = 14   # 412: Precondition failed
+EX_SERVICE          = 15   # 503: Service not available or slow down
+EX_USAGE            = 64   # The command was used incorrectly (e.g. bad command line syntax)
+EX_SOFTWARE         = 70   # internal software error (e.g. S3 error of unknown specificity)
+EX_OSERR            = 71   # system error (e.g. out of memory)
+EX_OSFILE           = 72   # OS error (e.g. invalid Python version)
+EX_IOERR            = 74   # An error occurred while doing I/O on some file.
+EX_TEMPFAIL         = 75   # temporary failure (S3DownloadError or similar, retry later)
+EX_ACCESSDENIED     = 77   # Insufficient permissions to perform the operation on S3
+EX_CONFIG           = 78   # Configuration file error
+_EX_SIGNAL          = 128
+_EX_SIGINT          = 2
+EX_BREAK            = _EX_SIGNAL + _EX_SIGINT # Control-C (KeyboardInterrupt raised)

S3/FileLists.py

@@ -381,14 +381,14 @@ def _get_filelist_remote(remote_uri, recursive = True):
             rem_list[key] = {
                 'size' : int(object['Size']),
                 'timestamp' : dateS3toUnix(object['LastModified']), ## Sadly it's upload time, not our lastmod time :-(
-                'md5' : object['ETag'][1:-1],
+                'md5' : object['ETag'].strip('"\''),
                 'object_key' : object['Key'],
                 'object_uri_str' : object_uri_str,
                 'base_uri' : remote_uri,
                 'dev' : None,
                 'inode' : None,
             }
-            if rem_list[key]['md5'].find("-") > 0: # always get it for multipart uploads
+            if '-' in rem_list[key]['md5']: # always get it for multipart uploads
                 _get_remote_attribs(S3Uri(object_uri_str), rem_list[key])
             md5 = rem_list[key]['md5']
             rem_list.record_md5(key, md5)
@@ -478,7 +478,7 @@ def _compare(src_list, dst_lst, src_remote, dst_remote, file):
         compare_md5 = 'md5' in cfg.sync_checks
         # Multipart-uploaded files don't have a valid md5 sum - it ends with "...-nn"
         if compare_md5:
-            if (src_remote == True and src_list[file]['md5'].find("-") >= 0) or (dst_remote == True and dst_list[file]['md5'].find("-") >= 0):
+            if (src_remote == True and '-' in src_list[file]['md5']) or (dst_remote == True and '-' in dst_list[file]['md5']):
                 compare_md5 = False
                 info(u"disabled md5 check for %s" % file)
         if attribs_match and compare_md5:

S3/MultiPart.py

@@ -147,7 +147,7 @@ def upload_part(self, seq, offset, chunk_size, labels, buffer = '', remote_statu
         if remote_status is not None:
             if int(remote_status['size']) == chunk_size:
                 checksum = calculateChecksum(buffer, self.file, offset, chunk_size, self.s3.config.send_chunk)
-                remote_checksum = remote_status['checksum'].strip('"')
+                remote_checksum = remote_status['checksum'].strip('"\'')
                 if remote_checksum == checksum:
                     warning("MultiPart: size and md5sum match for %s part %d, skipping." % (self.uri, seq))
                     self.parts[seq] = remote_status['checksum']

S3/S3.py

@@ -61,7 +61,7 @@ def mime_magic_file(file):
             return magic_.file(file)
 
 except ImportError, e:
-    if str(e).find("magic") >= 0:
+    if 'magic' in str(e):
         magic_message = "Module python-magic is not available."
     else:
         magic_message = "Module python-magic can't be used (%s)." % e.message
@@ -208,7 +208,7 @@ def __init__(self, config):
         self.config = config
 
     def get_hostname(self, bucket):
-        if bucket and check_bucket_name_dns_conformity(bucket):
+        if bucket and check_bucket_name_dns_support(self.config.host_bucket, bucket):
             if self.redir_map.has_key(bucket):
                 host = self.redir_map[bucket]
             else:
@@ -222,7 +222,7 @@ def set_hostname(self, bucket, redir_hostname):
         self.redir_map[bucket] = redir_hostname
 
     def format_uri(self, resource):
-        if resource['bucket'] and not check_bucket_name_dns_conformity(resource['bucket']):
+        if resource['bucket'] and not check_bucket_name_dns_support(self.config.host_bucket, resource['bucket']):
             uri = "/%s%s" % (resource['bucket'], resource['uri'])
         else:
             uri = resource['uri']
@@ -439,7 +439,7 @@ def _expiration_set(self, uri):
         return (request, body)
 
     def add_encoding(self, filename, content_type):
-        if content_type.find("charset=") != -1:
+        if 'charset=' in content_type:
            return False
         exts = self.config.add_encoding_exts.split(',')
         if exts[0]=='':
@@ -528,7 +528,7 @@ def object_put(self, filename, uri, extra_headers = None, extra_label = ""):
 
             if info is not None:
                 remote_size = int(info['headers']['content-length'])
-                remote_checksum = info['headers']['etag'].strip('"')
+                remote_checksum = info['headers']['etag'].strip('"\'')
                 if size == remote_size:
                     checksum = calculateChecksum('', file, 0, size, self.config.send_chunk)
                     if remote_checksum == checksum:
@@ -579,7 +579,8 @@ def compose_batch_del_xml(bucket, key_list):
         request_body = compose_batch_del_xml(bucket, batch)
         md5_hash = md5()
         md5_hash.update(request_body)
-        headers = {'content-md5': base64.b64encode(md5_hash.digest())}
+        headers = {'content-md5': base64.b64encode(md5_hash.digest()),
+                   'content-type': 'application/xml'}
         request = self.create_request("BATCH_DELETE", bucket = bucket, extra = '?delete', headers = headers)
         response = self.send_request(request, request_body)
         return response
@@ -652,10 +653,13 @@ def get_acl(self, uri):
         return acl
 
     def set_acl(self, uri, acl):
+        headers = {'content-type': 'application/xml'}
         if uri.has_object():
-            request = self.create_request("OBJECT_PUT", uri = uri, extra = "?acl")
+            request = self.create_request("OBJECT_PUT", uri = uri, extra = "?acl",
+                                          headers = headers)
         else:
-            request = self.create_request("BUCKET_CREATE", bucket = uri.bucket(), extra = "?acl")
+            request = self.create_request("BUCKET_CREATE", bucket = uri.bucket(), extra = "?acl",
+                                          headers = headers)
 
         body = str(acl)
         debug(u"set_acl(%s): acl-xml: %s" % (uri, body))
@@ -1170,7 +1174,7 @@ def recv_file(self, request, stream, labels, start_position = 0, retries = _max_
             except KeyError:
                 pass
 
-        response["md5match"] = md5_hash.find(response["md5"]) >= 0
+        response["md5match"] = response["md5"] in md5_hash
         response["elapsed"] = timestamp_end - timestamp_start
         response["size"] = current_position
         response["speed"] = response["elapsed"] and float(response["size"]) / response["elapsed"] or float(-1)

S3/S3Uri.py

@@ -78,7 +78,7 @@ def uri(self):
         return u"/".join([u"s3:/", self._bucket, self._object])
 
     def is_dns_compatible(self):
-        return check_bucket_name_dns_conformity(self._bucket)
+        return check_bucket_name_dns_support(Config.Config().host_bucket, self._bucket)
 
     def public_url(self):
         if self.is_dns_compatible():

S3/Utils.py

@@ -447,6 +447,20 @@ def check_bucket_name_dns_conformity(bucket):
         return False
 __all__.append("check_bucket_name_dns_conformity")
 
+def check_bucket_name_dns_support(bucket_host, bucket_name):
+    """
+    Check whether either the host_bucket support buckets and
+    either bucket name is dns compatible
+    """
+    if "%(bucket)s" not in bucket_host:
+        return False
+
+    try:
+        return check_bucket_name(bucket_name, dns_strict = True)
+    except Exceptions.ParameterError:
+        return False
+__all__.append("check_bucket_name_dns_support")
+
 def getBucketFromHostname(hostname):
     """
     bucket, success = getBucketFromHostname(hostname)

run-tests.py

@@ -90,7 +90,7 @@
 def test(label, cmd_args = [], retcode = 0, must_find = [], must_not_find = [], must_find_re = [], must_not_find_re = []):
     def command_output():
         print "----"
-        print " ".join([arg.find(" ")>=0 and "'%s'" % arg or arg for arg in cmd_args])
+        print " ".join([" " in arg and "'%s'" % arg or arg for arg in cmd_args])
         print "----"
         print stdout
         print "----"
@@ -245,7 +245,7 @@ def test_copy(label, src_file, dst_file):
             print "Bucket prefix option must explicitly supply a bucket name prefix"
             sys.exit(0)
         continue
-    if arg.find("..") >= 0:
+    if ".." in arg:
         range_idx = arg.find("..")
         range_start = arg[:range_idx] or 0
         range_end = arg[range_idx+2:] or 999