expand S3 upload for bucket names, prefixes, acl; sha1, sha256

setup.py

@@ -27,7 +27,7 @@
   - SVG2PNG conversion (Inkscape-based)
   - Amazon Web Service (S3 Delta Uploads etc)
   - Google Closure (JavaScript optimization)
-  - File utils (GZip etc)
+  - File utils (GZip, MD5, SHA1, SHA256, etc)
 
 License: Apache 2.0
 
@@ -36,7 +36,7 @@
 
 setup (
    name = 'taschenmesser',
-   version = '0.0.4',
+   version = '0.0.5',
    description = 'Taschenmesser, a toolbelt with plugins for SCons',
    long_description = LONGSDESC,
    license = 'Apache License 2.0',

taschenmesser/aws.py

@@ -19,6 +19,9 @@
 __all__ = ['exists', 'generate']
 
 
+import hashlib
+
+
 def exists(env):
    try:
       import boto
@@ -38,14 +41,27 @@ def generate(env):
    from boto.s3.connection import S3Connection
    from boto.s3.key import Key
 
+
    def s3_uploader(target, source, env):
       """
       SCons builder for Amazon S3 upload.
       """
+
+      def s3_upload_percent_cb(complete, total):
+         if total > 0:
+            sys.stdout.write("%d %%\n" % round(100. * float(complete) / float(total)))
+            sys.stdout.flush()
+
+      ## the bucket, bucket prefix and object ACLs come from env
+      ##
+      s3_bucket_name = env['S3_BUCKET']
+      s3_bucket_prefix = env.get('S3_BUCKET_PREFIX', None)
+      s3_object_acl = env.get('S3_OBJECT_ACL', 'public-read')
+
       ## S3 connection and bucket to upload to
       ##
       s3 = S3Connection()
-      bucket = s3.get_bucket("autobahn")
+      bucket = s3.get_bucket(s3_bucket_name)
 
       ## compute MD5s of artifacts to upload
       ##
@@ -59,7 +75,7 @@ def s3_uploader(target, source, env):
       ##
       uploads = []
       for s in source:
-         key = bucket.lookup("js/%s" % s.name)
+         key = bucket.lookup("%s%s" % (s3_bucket_prefix if s3_bucket_prefix else '', s.name))
          if not key or key.etag.replace('"', '') != checksums[s.name]:
             uploads.append(s)
          else:
@@ -69,23 +85,24 @@ def s3_uploader(target, source, env):
       ##
       for u in uploads:
          print "Uploading %s to S3 .." % u.name
-         key = Key(bucket, "js/%s" % u.name)
-         ##
+         key = Key(bucket, "%s%s" % (s3_bucket_prefix if s3_bucket_prefix else '', u.name))
+
          ## Do special stuff for "*.jgz". Note that "set_metadata"
          ## must be set before uploading!
          ##
          if os.path.splitext(u.name)[1].lower() == ".jgz":
-            ## override default chosen by S3 ..
+            ## override defaults chosen by AWS S3 ..
             key.set_metadata('Content-Type', 'application/x-javascript')
             key.set_metadata('Content-Encoding', 'gzip')
-         key.set_contents_from_filename(u.path)
-         key.set_acl('public-read')
+
+         key.set_contents_from_filename(u.path, cb = s3_upload_percent_cb, num_cb = 100)
+         key.set_acl(s3_object_acl)
 
       ## revisit uploaded stuff and get MD5s
       ##
       checksumsS3 = {}
       for s in source:
-         key = bucket.lookup("js/%s" % s.name)
+         key = bucket.lookup("%s%s" % (s3_bucket_prefix if s3_bucket_prefix else '', s.name))
          md5 = key.etag.replace('"', '')
          checksumsS3[s.name] = md5
       checksumsS3String = ''.join(["MD5 (%s) = %s\n" % c for c in checksumsS3.items()])
@@ -99,15 +116,15 @@ def s3_uploader(target, source, env):
 
    def checksumsMD5(target, source, env):
       """
-      SCons builder for computing a fingerprint file for artifacts.
+      SCons builder for computing a MD5 fingerprint file for artifacts.
       """
       checksums = {}
       for s in source:
          key = Key(s.name)
          md5 = key.compute_md5(open(s.path, "rb"))[0]
          checksums[s.name] = md5
 
-      ## MD5 (autobahn.js) = d1ff7ad2c5c4cf0d652566cbc78476ea
+      ## MD5 (autobahn.js) = 9f26d4774ce6ebafd32e75b68b22a526
       ##
       checksumsString = ''.join(["MD5 (%s) = %s\n" % c for c in checksums.items()])
 
@@ -116,5 +133,47 @@ def checksumsMD5(target, source, env):
       f.close()
 
 
+   def checksumsSHA1(target, source, env):
+      """
+      SCons builder for computing a SHA1 fingerprint file for artifacts.
+      """
+      checksums = {}
+      for s in source:
+         m = hashlib.sha1()
+         m.update(open(s.path, "rb").read())
+         fp = m.hexdigest()
+         checksums[s.name] = fp
+
+      ## SHA1 (autobahn.js) = 82e4e4961b8c68189f57a1c4b57b3953d5234850
+      ##
+      checksumsString = ''.join(["SHA1 (%s) = %s\n" % c for c in checksums.items()])
+
+      f = open(target[0].path, 'wb')
+      f.write(checksumsString)
+      f.close()
+
+
+   def checksumsSHA256(target, source, env):
+      """
+      SCons builder for computing a SHA256 fingerprint file for artifacts.
+      """
+      checksums = {}
+      for s in source:
+         m = hashlib.sha256()
+         m.update(open(s.path, "rb").read())
+         fp = m.hexdigest()
+         checksums[s.name] = fp
+
+      ## SHA256 (autobahn.js) = be266f59ff09214f4afe610c0c15abc10b86e96e82e943749efacfb7f8d72dd0
+      ##
+      checksumsString = ''.join(["SHA256 (%s) = %s\n" % c for c in checksums.items()])
+
+      f = open(target[0].path, 'wb')
+      f.write(checksumsString)
+      f.close()
+
+
    env.Append(BUILDERS = {'S3': Builder(action = s3_uploader),
-                          'MD5': Builder(action = checksumsMD5)})
+                          'MD5': Builder(action = checksumsMD5),
+                          'SHA1': Builder(action = checksumsSHA1),
+                          'SHA256': Builder(action = checksumsSHA256)})