s3tools · mdomsch · Mar 16, 2015 · Mar 6, 2015 · Mar 10, 2015 · Mar 11, 2015
diff --git a/S3/ACL.py b/S3/ACL.py
@@ -1,3 +1,5 @@
+# -*- coding: utf-8 -*-
+
 ## Amazon S3 - Access Control List representation
 ## Author: Michal Ludvig <michal@logix.cz>
 ##         http://www.logix.cz/michal

diff --git a/S3/AccessLog.py b/S3/AccessLog.py
@@ -1,3 +1,5 @@
+# -*- coding: utf-8 -*-
+
 ## Amazon S3 - Access Control List representation
 ## Author: Michal Ludvig <michal@logix.cz>
 ##         http://www.logix.cz/michal
@@ -44,7 +46,7 @@ def enableLogging(self, target_prefix_uri):
     def targetPrefix(self):
         if self.isLoggingEnabled():
             el = self.tree.find(".//LoggingEnabled")
-            target_prefix = "s3://%s/%s" % (
+            target_prefix = u"s3://%s/%s" % (
                 self.tree.find(".//LoggingEnabled//TargetBucket").text,
                 self.tree.find(".//LoggingEnabled//TargetPrefix").text)
             return S3Uri.S3Uri(target_prefix)
@@ -81,7 +83,7 @@ def __str__(self):
     from S3Uri import S3Uri
     log = AccessLog()
     print log
-    log.enableLogging(S3Uri("s3://targetbucket/prefix/log-"))
+    log.enableLogging(S3Uri(u"s3://targetbucket/prefix/log-"))
     print log
     log.setAclPublic(True)
     print log

diff --git a/S3/BidirMap.py b/S3/BidirMap.py
@@ -1,3 +1,5 @@
+# -*- coding: utf-8 -*-
+
 ## Amazon S3 manager
 ## Author: Michal Ludvig <michal@logix.cz>
 ##         http://www.logix.cz/michal

diff --git a/S3/CloudFront.py b/S3/CloudFront.py
@@ -1,3 +1,5 @@
+# -*- coding: utf-8 -*-
+
 ## Amazon CloudFront support
 ## Author: Michal Ludvig <michal@logix.cz>
 ##         http://www.logix.cz/michal
@@ -19,7 +21,7 @@
 from S3 import S3
 from Config import Config
 from Exceptions import *
-from Utils import getTreeFromXml, appendXmlTextNode, getDictFromTree, dateS3toPython, getBucketFromHostname, getHostnameFromBucket
+from Utils import getTreeFromXml, appendXmlTextNode, getDictFromTree, dateS3toPython, getBucketFromHostname, getHostnameFromBucket, deunicodise
 from Crypto import sign_string_v2
 from S3Uri import S3Uri, S3UriS3
 from FileLists import fetch_remote_list
@@ -65,7 +67,7 @@ def parse(self, tree):
             self.info['CNAME'] = [self.info['CNAME']]
 
     def uri(self):
-        return S3Uri("cf://%s" % self.info['Id'])
+        return S3Uri(u"cf://%s" % self.info['Id'])
 
 class DistributionList(object):
     ## Example:
@@ -121,7 +123,7 @@ def parse(self, tree):
         self.info['DistributionConfig'] = DistributionConfig(tree = tree.find(".//DistributionConfig"))
 
     def uri(self):
-        return S3Uri("cf://%s" % self.info['Id'])
+        return S3Uri(u"cf://%s" % self.info['Id'])
 
 class DistributionConfig(object):
     ## Example:
@@ -169,7 +171,7 @@ def parse(self, tree):
             logging_dict['Bucket'], success = getBucketFromHostname(logging_dict['Bucket'])
             if not success:
                 warning("Logging to unparsable bucket name: %s" % logging_dict['Bucket'])
-            self.info['Logging'] = S3UriS3("s3://%(Bucket)s/%(Prefix)s" % logging_dict)
+            self.info['Logging'] = S3UriS3(u"s3://%(Bucket)s/%(Prefix)s" % logging_dict)
         else:
             self.info['Logging'] = None
 
@@ -451,8 +453,8 @@ def InvalidateObjects(self, uri, paths, default_index_file, invalidate_default_i
         if len(paths) > 999:
             try:
                 tmp_filename = Utils.mktmpfile()
-                f = open(tmp_filename, "w")
-                f.write("\n".join(paths)+"\n")
+                f = open(deunicodise(tmp_filename), "w")
+                f.write(deunicodise("\n".join(paths)+"\n"))
                 f.close()
                 warning("Request to invalidate %d paths (max 999 supported)" % len(paths))
                 warning("All the paths are now saved in: %s" % tmp_filename)

diff --git a/S3/Config.py b/S3/Config.py
@@ -1,3 +1,5 @@
+# -*- coding: utf-8 -*-
+
 ## Amazon S3 manager
 ## Author: Michal Ludvig <michal@logix.cz>
 ##         http://www.logix.cz/michal

diff --git a/S3/ConnMan.py b/S3/ConnMan.py
@@ -1,3 +1,5 @@
+# -*- coding: utf-8 -*-
+
 ## Amazon S3 manager
 ## Author: Michal Ludvig <michal@logix.cz>
 ##         http://www.logix.cz/michal

diff --git a/S3/Crypto.py b/S3/Crypto.py
@@ -1,3 +1,5 @@
+# -*- coding: utf-8 -*-
+
 ## Amazon S3 manager
 ## Author: Michal Ludvig <michal@logix.cz>
 ##         http://www.logix.cz/michal
@@ -10,9 +12,8 @@
 
 import Config
 from logging import debug
-import Utils
+from Utils import encode_to_s3, time_to_epoch, deunicodise
 
-import os
 import datetime
 import urllib
 
@@ -55,7 +56,7 @@ def sign_url_v2(url_to_sign, expiry):
 
 def sign_url_base_v2(**parms):
     """Shared implementation of sign_url methods. Takes a hash of 'bucket', 'object' and 'expiry' as args."""
-    parms['expiry']=Utils.time_to_epoch(parms['expiry'])
+    parms['expiry']=time_to_epoch(parms['expiry'])
     parms['access_key']=Config.Config().access_key
     parms['host_base']=Config.Config().host_base
     debug("Expiry interpreted as epoch time %s", parms['expiry'])
@@ -66,10 +67,10 @@ def sign_url_base_v2(**parms):
     return "http://%(bucket)s.%(host_base)s/%(object)s?AWSAccessKeyId=%(access_key)s&Expires=%(expiry)d&Signature=%(sig)s" % parms
 
 def sign(key, msg):
-    return hmac.new(key, msg.encode('utf-8'), sha256).digest()
+    return hmac.new(key, encode_to_s3(msg), sha256).digest()
 
 def getSignatureKey(key, dateStamp, regionName, serviceName):
-    kDate = sign(('AWS4' + key).encode('utf-8'), dateStamp)
+    kDate = sign(encode_to_s3('AWS4' + key), dateStamp)
     kRegion = sign(kDate, regionName)
     kService = sign(kRegion, serviceName)
     kSigning = sign(kService, 'aws4_request')
@@ -127,7 +128,7 @@ def sign_string_v4(method='GET', host='', canonical_uri='/', params={}, region='
     credential_scope = datestamp + '/' + region + '/' + service + '/' + 'aws4_request'
     string_to_sign = algorithm + '\n' +  amzdate + '\n' +  credential_scope + '\n' +  sha256(canonical_request).hexdigest()
     signing_key = getSignatureKey(secret_key, datestamp, region, service)
-    signature = hmac.new(signing_key, (string_to_sign).encode('utf-8'), sha256).hexdigest()
+    signature = hmac.new(signing_key, encode_to_s3(string_to_sign), sha256).hexdigest()
     authorization_header = algorithm + ' ' + 'Credential=' + access_key + '/' + credential_scope + ',' +  'SignedHeaders=' + signed_headers + ',' + 'Signature=' + signature
     headers = dict(cur_headers.items() + {'x-amz-date':amzdate, 'Authorization':authorization_header, 'x-amz-content-sha256': payload_hash}.items())
     debug("signature-v4 headers: %s" % headers)
@@ -146,7 +147,7 @@ def checksum_sha256_file(filename, offset=0, size=None):
     except:
         # fallback to Crypto SHA256 module
         hash = sha256.new()
-    with open(filename,'rb') as f:
+    with open(deunicodise(filename),'rb') as f:
         if size is None:
             for chunk in iter(lambda: f.read(8192), b''):
                 hash.update(chunk)

diff --git a/S3/Exceptions.py b/S3/Exceptions.py
@@ -1,3 +1,5 @@
+# -*- coding: utf-8 -*-
+
 ## Amazon S3 manager - Exceptions library
 ## Author: Michal Ludvig <michal@logix.cz>
 ##         http://www.logix.cz/michal

diff --git a/S3/ExitCodes.py b/S3/ExitCodes.py
@@ -1,3 +1,5 @@
+# -*- coding: utf-8 -*-
+
 # patterned on /usr/include/sysexits.h
 
 EX_OK               = 0

diff --git a/S3/FileDict.py b/S3/FileDict.py
@@ -1,3 +1,5 @@
+# -*- coding: utf-8 -*-
+
 ## Amazon S3 manager
 ## Author: Michal Ludvig <michal@logix.cz>
 ##         http://www.logix.cz/michal