Skip to content

ENH: S3 access using instance role #2621

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Jul 23, 2018
Merged

ENH: S3 access using instance role #2621

Changes from 4 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
d0f29dc
avoid much string manip
anibalsolon Jun 19, 2018
ad93735
Try to connect to S3 using IAM Roles
anibalsolon Jun 19, 2018
aab85cb
import botocore
anibalsolon Jun 19, 2018
502b498
fix string condition, move bounded-wannabe method as function
anibalsolon Jul 23, 2018
5abe1c9
privatize function
anibalsolon Jul 23, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
77 changes: 48 additions & 29 deletions nipype/interfaces/io.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,35 @@ def add_traits(base, names, trait_type=None):
return base


def get_head_bucket(s3_resource, bucket_name):
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Minor quibble: I would put a _ prefix on the function still, just to mark it as a private function that external code shouldn't rely on.

""" Try to get the header info of a bucket, in order to
check if it exists and its permissions
"""

import botocore

# Try fetch the bucket with the name argument
try:
s3_resource.meta.client.head_bucket(Bucket=bucket_name)
except botocore.exceptions.ClientError as exc:
error_code = int(exc.response['Error']['Code'])
if error_code == 403:
err_msg = 'Access to bucket: %s is denied; check credentials'\
% bucket_name
raise Exception(err_msg)
elif error_code == 404:
err_msg = 'Bucket: %s does not exist; check spelling and try '\
'again' % bucket_name
raise Exception(err_msg)
else:
err_msg = 'Unable to connect to bucket: %s. Error message:\n%s'\
% (bucket_name, exc)
except Exception as exc:
err_msg = 'Unable to connect to bucket: %s. Error message:\n%s'\
% (bucket_name, exc)
raise Exception(err_msg)


class IOBase(BaseInterface):
def _run_interface(self, runtime):
return runtime
Expand Down Expand Up @@ -536,43 +565,35 @@ def _fetch_bucket(self, bucket_name):
session = boto3.session.Session(
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key)
s3_resource = session.resource('s3', use_ssl=True)

# Otherwise, connect anonymously
else:
iflogger.info('Connecting to AWS: %s anonymously...', bucket_name)
iflogger.info('Connecting to S3 bucket: %s with IAM role...',
bucket_name)

# Lean on AWS environment / IAM role authentication and authorization
session = boto3.session.Session()
s3_resource = session.resource('s3', use_ssl=True)
s3_resource.meta.client.meta.events.register(
'choose-signer.s3.*', botocore.handlers.disable_signing)

# Explicitly declare a secure SSL connection for bucket object
bucket = s3_resource.Bucket(bucket_name)
s3_resource = session.resource('s3', use_ssl=True)

# And try fetch the bucket with the name argument
try:
s3_resource.meta.client.head_bucket(Bucket=bucket_name)
except botocore.exceptions.ClientError as exc:
error_code = int(exc.response['Error']['Code'])
if error_code == 403:
err_msg = 'Access to bucket: %s is denied; check credentials'\
% bucket_name
raise Exception(err_msg)
elif error_code == 404:
err_msg = 'Bucket: %s does not exist; check spelling and try '\
'again' % bucket_name
raise Exception(err_msg)
else:
err_msg = 'Unable to connect to bucket: %s. Error message:\n%s'\
% (bucket_name, exc)
get_head_bucket(s3_resource, bucket_name)
except Exception as exc:
err_msg = 'Unable to connect to bucket: %s. Error message:\n%s'\
% (bucket_name, exc)
raise Exception(err_msg)

# Try to connect anonymously
s3_resource.meta.client.meta.events.register(
'choose-signer.s3.*', botocore.handlers.disable_signing)

iflogger.info('Connecting to AWS: %s anonymously...', bucket_name)
get_head_bucket(s3_resource, bucket_name)

# Explicitly declare a secure SSL connection for bucket object
bucket = s3_resource.Bucket(bucket_name)

# Return the bucket
return bucket


# Send up to S3 method
def _upload_to_s3(self, bucket, src, dst):
'''
Expand All @@ -592,10 +613,8 @@ def _upload_to_s3(self, bucket, src, dst):
s3_prefix = s3_str + bucket.name

# Explicitly lower-case the "s3"
if dst.lower().startswith(s3_str):
dst_sp = dst.split('/')
dst_sp[0] = dst_sp[0].lower()
dst = '/'.join(dst_sp)
if dst[:len(s3_str)].lower() == s3_str:
dst = s3_str + dst[len(s3_str):]

# If src is a directory, collect files (this assumes dst is a dir too)
if os.path.isdir(src):
Expand Down