S3 clean up

Author: Saul Frank

src/dataplane/DataStorage/s3/test_cities.csv renamed to filename

@@ -1,68 +1,35 @@
-""" Host: {name}.sharepoint.com 
-TenantID: Directory or TenantID as per Azure portal
-ClientID: Azure Client ID or Application ID.
-Secret: Azure secret for client ID.
-SiteName: Name of the site to be looked up <- in url e.g https://{name}.sharepoint.com/sites/DataplanePython
+""" 
+S3Client: Client for s3
+S3FilePath: /tmp/source.xlsx (needs UploadMethod="File")
+LocalFilePath: /General/hello.xlxs (where download method is File)
+Download Method: File or Object
 FilePath: /General/hello.xlxs
-ProxyUse: Whether to use a proxy, true or false
-ProxyUrl: Proxy endpoint to use
-ProxyMethod: https or http, default https
 """
 
-def s3_download(Bucket, AccessKey, SecretKey, S3FilePath, DownloadMethod="Object", LocalFilePath="", ProxyUse=False, ProxyUrl="", ProxyMethod="https", EndPointUrl=None):
+def s3_download(S3Client, Bucket,  S3FilePath, DownloadMethod="Object", LocalFilePath=""):
 
-    import boto3
-    from botocore.exceptions import ClientError
     from datetime import datetime
     from io import BytesIO
 
     # Start the timer
     start = datetime.now()
-
-    # Connect to S3 Bucket
-
-    if ProxyUse:
-        proxies = {ProxyMethod: ProxyUrl}
-    else:
-        proxies = {}
 
-    s3_conn = boto3.resource(
-        "s3",
-        endpoint_url = EndPointUrl,
-        aws_access_key_id=AccessKey,
-        aws_secret_access_key=SecretKey,
-        aws_session_token=None,
-        config=boto3.session.Config(
-            signature_version='s3v4',
-            proxies=proxies
-        ),
-    )
+    # Download S3 file content to file - uses multi threaded download
+    # https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html#S3.Client.download_fileobj
 
-    bucket_conn = s3_conn.Bucket(Bucket)
+    if DownloadMethod == "File":
+        if LocalFilePath == "":
+            duration = datetime.now() - start
+            return {"result":"Fail", "duration": str(duration), "reason":"File method requires a local file file path"} 
 
-    try:
-        _ = bucket_conn.creation_date
-    except ClientError as e:
-        duration = datetime.now() - start
-        return {
-            "result":"Fail",
-            "reason":"Auth fail",
-            "duration": str(duration),
-            "status": None,
-            "error": e
-        }
-    
-    # Download S3 file content
-    r=BytesIO()
-    print(S3FilePath[1:])
-    bucket_conn.download_fileobj(S3FilePath[1:], r)
+        with open(LocalFilePath, 'wb') as data:
+            S3Client.download_fileobj(Bucket=Bucket, Key=S3FilePath, Fileobj=data)
 
-    if DownloadMethod == "File":
-        with open(LocalFilePath, 'wb') as f:
-            r.seek(0)
-            f.write(r.read())
         duration = datetime.now() - start
-        return {"result":"OK", "duration": str(duration), "status": None, "FilePath": LocalFilePath} 
+        return {"result":"OK", "duration": str(duration), "FilePath": LocalFilePath} 
+
+    # Download S3 file content to object
+    objectGet = S3Client.get_object(Bucket=Bucket, Key=S3FilePath, ChecksumMode='ENABLED')["Body"].read()
 
     duration = datetime.now() - start
-    return {"result":"OK", "duration": str(duration), "status": None, "content": r} 
+    return {"result":"OK", "duration": str(duration), "content": objectGet} 

src/dataplane/DataStorage/s3/s3_download.py

@@ -1,71 +1,28 @@
 """
-AccessKey: S3 Access Key.
-SecretKey: S3 Secret Key.
+S3Client: Client for s3
 SourceFilePath: /tmp/source.xlsx (needs UploadMethod="File")
 TargetFilePath: /General/hello.xlxs
-UploadMethod
-ProxyUse: Whether to use a proxy, true or false
-ProxyUrl: Proxy endpoint to use
-ProxyMethod: https or http, default https
-EndPointUrl: Custom endpoint URL (eg. for minio)
+UploadMethod: Object or File
 """
 
-def s3_upload(Bucket, AccessKey, SecretKey, TargetFilePath, SourceFilePath="/tmp/default.txt", UploadMethod="Object", UploadObject="", ProxyUse=False, ProxyUrl="", ProxyMethod="https", EndPointUrl=None):
+def s3_upload(S3Client, Bucket, TargetFilePath, SourceFilePath="/tmp/default.txt", UploadMethod="Object", UploadObject=""):
 
-
-    import boto3
-    from botocore.config import Config
-    from botocore.exceptions import ClientError
     from datetime import datetime
     from io import BytesIO
-    import os
 
     start = datetime.now()
 
     # Connect to S3 Bucket
 
-    if ProxyUse:
-        proxies = {ProxyMethod: ProxyUrl}
-    else:
-        proxies = {}
-
-    s3_conn = boto3.resource(
-        "s3",
-        endpoint_url = EndPointUrl,
-        aws_access_key_id=AccessKey,
-        aws_secret_access_key=SecretKey,
-        aws_session_token=None,
-        config=boto3.session.Config(
-            signature_version='s3v4',
-            proxies=proxies
-        ),
-    )
-
-    bucket_conn = s3_conn.Bucket(Bucket)
-
-    try:
-        _ = bucket_conn.creation_date
-    except ClientError as e:
-        duration = datetime.now() - start
-        return {
-            "result":"Fail",
-            "reason":"Auth fail",
-            "duration": str(duration),
-            "status": None,
-            "error": e
-        }
-
     # ====== Obtain the file from disk ======
     if UploadMethod == "File":
-        UploadObject = open(SourceFilePath, 'rb')
-    else:
-        temp_file = BytesIO()
-        temp_file.write(UploadObject)
-        temp_file.seek(0)
-        UploadObject = temp_file
+        with open(SourceFilePath, 'rb') as data:
+            S3Client.upload_fileobj(Fileobj=data, Bucket=Bucket, Key=TargetFilePath)
+            duration = datetime.now() - start
+            return {"result":"OK", "duration": str(duration), "Path":TargetFilePath}
 
     # ====== Upload file using boto3 upload_file ======
-    bucket_conn.upload_fileobj(UploadObject, TargetFilePath)
+    S3Client.upload_fileobj(Fileobj=BytesIO(UploadObject), Bucket=Bucket, Key=TargetFilePath)
 
     duration = datetime.now() - start
 

src/dataplane/DataStorage/s3/s3_upload.py

@@ -1,10 +1,12 @@
 
 import os
-from dataplane.DataStorage.s3.s3_upload import s3_upload
-from dataplane.DataStorage.s3.s3_download import s3_download
+from .s3_upload import s3_upload
+from .s3_download import s3_download
 from nanoid import generate
 import os
 from dotenv import load_dotenv
+import boto3
+from botocore.client import Config
 
 def test_s3():
 
@@ -15,11 +17,21 @@ def test_s3():
 
     # Sharepoint connection
     load_dotenv()
-
     RUN_ID = os.environ["DP_RUNID"]
-    BUCKET = os.getenv('BUCKET')
-    ACCESS_KEY = os.getenv('ACCESS_KEY')
-    SECRET_KEY = os.getenv('SECRET_KEY')
+
+        # S3 connection
+    S3Connect = boto3.client(
+        's3',
+        endpoint_url="http://minio:9000",
+        aws_access_key_id="admin",
+        aws_secret_access_key="hello123",
+        config=Config(signature_version='s3v4'),
+        region_name='us-east-1'
+
+    )
+
+    bucket = "dataplanebucket"
+
     CURRENT_DIRECTORY = os.path.realpath(os.path.dirname(__file__))
 
     if os.path.exists(CURRENT_DIRECTORY+"/test_cities_delete.csv"):
@@ -29,28 +41,23 @@ def test_s3():
     # s3_upload(Bucket, AccessKey, SecretKey, TargetFilePath, SourceFilePath="/tmp/default.txt", UploadMethod="Object", UploadObject="", ProxyUse=False, ProxyUrl="", ProxyMethod="https", EndPointUrl=None)
     print(CURRENT_DIRECTORY)
     # Store the data with key hello - run id will be attached
-    rs = s3_upload(Bucket=BUCKET,
-    AccessKey=ACCESS_KEY, 
-    SecretKey=SECRET_KEY,
-    TargetFilePath=f"/General/myfile {RUN_ID}.csv",
-    SourceFilePath=CURRENT_DIRECTORY+"/test_cities.csv",
-    UploadMethod="File",
-    EndPointUrl="http://minio:9000"
+    rs = s3_upload(Bucket=bucket,
+    S3Client=S3Connect,
+    TargetFilePath=f"/s3test/myfile {RUN_ID}.csv",
+    SourceFilePath=CURRENT_DIRECTORY+"/test_s3_cities.csv",
+    UploadMethod="File"
     )
     print(rs)
     assert rs["result"]=="OK"
 
 
     # ---------- RETRIEVE PARQUET FROM S3 ------------
 
-    rs = s3_download(Bucket=BUCKET, 
-    AccessKey=ACCESS_KEY, 
-    SecretKey=SECRET_KEY,
-    S3FilePath=f"/General/myfile {RUN_ID}.csv",
+    rs = s3_download(Bucket=bucket,
+    S3Client=S3Connect,
+    S3FilePath=f"/s3test/myfile {RUN_ID}.csv",
     LocalFilePath=CURRENT_DIRECTORY+"/test_cities_delete.csv",
-    DownloadMethod="File",
-    ProxyUse=False, ProxyUrl="", ProxyMethod="https",
-    EndPointUrl='http://minio:9000'
+    DownloadMethod="File"
     )
     print(rs)
     assert rs["result"]=="OK"

src/dataplane/DataStorage/s3/test_s3.py

@@ -0,0 +1,14 @@
+"LatD", "LatM", "LatS", "NS", "LonD", "LonM", "LonS", "EW", "City", "State"
+   41,    5,   59, "N",     80,   39,    0, "W", "Youngstown", OH
+   42,   52,   48, "N",     97,   23,   23, "W", "Yankton", SD
+   46,   35,   59, "N",    120,   30,   36, "W", "Yakima", WA
+   42,   16,   12, "N",     71,   48,    0, "W", "Worcester", MA
+   43,   37,   48, "N",     89,   46,   11, "W", "Wisconsin Dells", WI
+   36,    5,   59, "N",     80,   15,    0, "W", "Winston-Salem", NC
+   49,   52,   48, "N",     97,    9,    0, "W", "Winnipeg", MB
+   39,   11,   23, "N",     78,    9,   36, "W", "Winchester", VA
+   34,   14,   24, "N",     77,   55,   11, "W", "Wilmington", NC
+   39,   45,    0, "N",     75,   33,    0, "W", "Wilmington", DE
+   48,    9,    0, "N",    103,   37,   12, "W", "Williston", ND
+   41,   15,    0, "N",     77,    0,    0, "W", "Williamsport", PA
+   37,   40,   48, "N",     82,   16,   47, "W", "Williamson", WV