update naming convention for transfers

Author: Saul Frank

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "dataplane"
-version = "0.0.12"
+version = "0.0.13"
 authors = [
   { name="Saul Frank" }, {name ="Dataplane, Inc."}
 ]

src/dataplane/Microsoft/Teams/webhook_send.py

@@ -1,8 +1,3 @@
-import requests
-import os
-from datetime import datetime
-import json
-
 """ Url: Webhook url generated by Teams
 Message: Json message to send
 ProxyUse: Whether to use a proxy, true or false
@@ -11,6 +6,9 @@
 """
 def teams_webhook_send(Url, Message, ProxyUse=False, ProxyUrl="", ProxyMethod="https"):
 
+    import requests
+    from datetime import datetime
+
     # Start the timer
     start  = datetime.now()
 

src/dataplane/__init__.py

@@ -1,11 +1,11 @@
-from dataplane.pipelinerun.data_persist.redis_store import (
-    pipeline_redis_store,
-    pipeline_redis_get,
+from dataplane.pipelinerun.data_persist.pandas_redis_store import (
+    pipeline_pandas_redis_store,
+    pipeline_pandas_redis_get,
 )
 
-from dataplane.pipelinerun.data_persist.s3_store import (
-    pipeline_s3_get,
-    pipeline_s3_store,
+from dataplane.pipelinerun.data_persist.pandas_s3_store import (
+    pipeline_pandas_s3_get,
+    pipeline_pandas_s3_store,
 )
 
 from dataplane.hello import (
@@ -24,10 +24,10 @@
     "hello",
 
     # Pipeline transfers
-    "pipeline_redis_store", 
-    "pipeline_redis_get", 
-    "pipeline_s3_get", 
-    "pipeline_s3_store",
+    "pipeline_pandas_redis_store", 
+    "pipeline_pandas_redis_get", 
+    "pipeline_pandas_s3_get", 
+    "pipeline_pandas_s3_store",
 
     # Microsoft connectors
     "teams_webhook_send",

src/dataplane/pipelinerun/data_persist/redis_store.py renamed to src/dataplane/pipelinerun/data_persist/pandas_redis_store.py

@@ -17,7 +17,7 @@ def RedisCheck(r):
 Expire: Expires the data if true.
 ExpireDuration: If expires is true, how much time to expire. Default 15 mins
 """
-def pipeline_redis_store(StoreKey, DataFrame, Redis, Expire=True, ExpireDuration=timedelta(minutes=15)):
+def pipeline_pandas_redis_store(StoreKey, DataFrame, Redis, Expire=True, ExpireDuration=timedelta(minutes=15)):
 
     import os
     import io
@@ -50,7 +50,7 @@ def pipeline_redis_store(StoreKey, DataFrame, Redis, Expire=True, ExpireDuration
 StoreKey: is the key to look up for retrieval (set with RedisStore). 
 Redis: e.g. Redis = redis.Redis(host='redis-service', port=6379, db=0)
 """
-def pipeline_redis_get(StoreKey, Redis):
+def pipeline_pandas_redis_get(StoreKey, Redis):
 
     import os
     import io

src/dataplane/pipelinerun/data_persist/s3_store.py renamed to src/dataplane/pipelinerun/data_persist/pandas_s3_store.py

@@ -10,7 +10,7 @@
 Expire: Expires the data if true.
 ExpireDuration: If expires is true, how much time to expire. Default 15 mins
 """
-def pipeline_s3_store(StoreKey, DataFrame, S3Client, Bucket, Expire=True, ExpireDuration=timedelta(days=30)):
+def pipeline_pandas_s3_store(StoreKey, DataFrame, S3Client, Bucket, Expire=True, ExpireDuration=timedelta(days=30)):
 
     import os
     from io import BytesIO
@@ -35,7 +35,7 @@ def pipeline_s3_store(StoreKey, DataFrame, S3Client, Bucket, Expire=True, Expire
 S3Client: e.g. S3Client = boto3.client(...)
 Bucket: Name of the s3 bucket
 """
-def pipeline_s3_get(StoreKey, S3Client, Bucket):
+def pipeline_pandas_s3_get(StoreKey, S3Client, Bucket):
 
     import os
     from io import BytesIO

src/dataplane/pipelinerun/data_persist/test_redis.py

@@ -1,7 +1,7 @@
 
 import os
-from .redis_store import pipeline_redis_store
-from .redis_store import pipeline_redis_get
+from .pandas_redis_store import pipeline_pandas_redis_store
+from .pandas_redis_store import pipeline_pandas_redis_get
 import redis
 from datetime import timedelta
 from nanoid import generate
@@ -29,14 +29,14 @@ def test_redis_store():
     # ---------- STORE PARQUET TO REDIS ------------
 
     # Store the data with key hello - run id will be attached
-    rs = pipeline_redis_store(StoreKey="hello", DataFrame=df, Redis=redisConnect, Expire=True, ExpireDuration=timedelta(minutes=15))
+    rs = pipeline_pandas_redis_store(StoreKey="hello", DataFrame=df, Redis=redisConnect, Expire=True, ExpireDuration=timedelta(minutes=15))
     print(rs)
     assert rs["result"]=="OK"
 
     # ---------- RETRIEVE PARQUET FROM REDIS ------------
 
     # Get the data
-    rsget = pipeline_redis_get(StoreKey="hello", Redis=redisConnect)
+    rsget = pipeline_pandas_redis_get(StoreKey="hello", Redis=redisConnect)
     print(rsget)
     df = rsget["dataframe"]
     print(df.shape[0])

src/dataplane/pipelinerun/data_persist/test_s3_store.py

@@ -1,7 +1,7 @@
 
 import os
-from .s3_store import pipeline_s3_store
-from .s3_store import pipeline_s3_get
+from .pandas_s3_store import pipeline_pandas_s3_store
+from .pandas_s3_store import pipeline_pandas_s3_get
 import boto3
 from botocore.client import Config
 from nanoid import generate
@@ -38,14 +38,14 @@ def test_s3_store():
     # ---------- STORE PARQUET TO S3 ------------
 
     # Store the data with key hello - run id will be attached
-    rs = pipeline_s3_store(StoreKey="s3me", DataFrame=df, S3Client=S3Connect, Bucket=bucket, Expire=False)
+    rs = pipeline_pandas_s3_store(StoreKey="s3me", DataFrame=df, S3Client=S3Connect, Bucket=bucket, Expire=False)
     print(rs)
     assert rs["result"]=="OK"
 
     # ---------- RETRIEVE PARQUET FROM S3 ------------
 
     # Get the data
-    rsget = pipeline_s3_get(StoreKey="s3me", S3Client=S3Connect, Bucket=bucket)
+    rsget = pipeline_pandas_s3_get(StoreKey="s3me", S3Client=S3Connect, Bucket=bucket)
     print(rsget)
     df = rsget["dataframe"]
     print(df.shape[0])