Skip to content

Commit 61fe4ee

Browse files
authored
Downloading from AWS using boto3
Utility to help me remember the next time I've download stuff from AWS coz what horrible documentation the library has :-/
1 parent de82def commit 61fe4ee

File tree

1 file changed

+27
-0
lines changed

1 file changed

+27
-0
lines changed

Utilities/download_files_from_aws

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
import boto3
2+
import os
3+
4+
# Never do this unless you're using it to validate data on a trusted system.
5+
# This kind of value setting is extremely irresponsible.
6+
os.environ['AWS_DEFAULT_REGION']="region"
7+
os.environ["AWS_SECRET_ACCESS_KEY"]="aws_secret_access_key"
8+
os.environ["AWS_ACCESS_KEY_ID"]="aws_access_key_id"
9+
os.environ['BUCKET_NAME']='bucket_name'
10+
# I've behaved very irresponsibly
11+
12+
cloud_storage_path = '<bucket_name>/<sub_folder_path>'
13+
14+
def download_objects_from_folder(prefix,dest_folder):
15+
try:
16+
s3_ = boto3.resource('s3')
17+
bucket_ = s3_.Bucket(os.environ['BUCKET_NAME'])
18+
objects = my_bucket.objects.filter(Prefix=prefix)
19+
print("Objects = {}".format(objects))
20+
for obj in objects:
21+
path, filename = os.path.split(obj.key)
22+
print(filename)
23+
bucket_.download_file(obj.key, os.path.join(dest_folder,str(filename) ))
24+
except Exception as e:
25+
print(e)
26+
27+
list_of_contents = download_objects_from_folder(os.path.join(cloud_storage_path, "some_prefix"),"/tmp/some_destination_folder_that_exists")

0 commit comments

Comments
 (0)