Merge branch 'main' into pr/2

Author: Saul Frank

import_try.py

@@ -6,8 +6,6 @@
 
 duration = datetime.now() - start
 
-print(hello())
+hello()
 
 print(duration)
-
-

notes-distribute.md

@@ -16,6 +16,6 @@ python3 -m twine upload dist/*
 
 4. Testing impport times - lazy loading:
 ```shell
-pip install dataplane==0.0.11
+pip install dataplane==0.0.12
 python -v import_try.py
 ```

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "dataplane"
-version = "0.0.11"
+version = "0.0.13"
 authors = [
   { name="Saul Frank" }, {name ="Dataplane, Inc."}
 ]

src/dataplane/Microsoft/Teams/webhook_send.py

@@ -1,8 +1,3 @@
-import requests
-import os
-from datetime import datetime
-import json
-
 """ Url: Webhook url generated by Teams
 Message: Json message to send
 ProxyUse: Whether to use a proxy, true or false
@@ -11,6 +6,9 @@
 """
 def teams_webhook_send(Url, Message, ProxyUse=False, ProxyUrl="", ProxyMethod="https"):
 
+    import requests
+    from datetime import datetime
+
     # Start the timer
     start  = datetime.now()
 

src/dataplane/__init__.py

@@ -1,11 +1,11 @@
-from dataplane.pipelinerun.data_persist.redis_store import (
-    pipeline_redis_store,
-    pipeline_redis_get,
+from dataplane.pipelinerun.data_persist.pandas_redis_store import (
+    pipeline_pandas_redis_store,
+    pipeline_pandas_redis_get,
 )
 
-from dataplane.pipelinerun.data_persist.s3_store import (
-    pipeline_s3_get,
-    pipeline_s3_store,
+from dataplane.pipelinerun.data_persist.pandas_s3_store import (
+    pipeline_pandas_s3_get,
+    pipeline_pandas_s3_store,
 )
 
 from dataplane.hello import (
@@ -24,10 +24,10 @@
     "hello",
 
     # Pipeline transfers
-    "pipeline_redis_store", 
-    "pipeline_redis_get", 
-    "pipeline_s3_get", 
-    "pipeline_s3_store",
+    "pipeline_pandas_redis_store", 
+    "pipeline_pandas_redis_get", 
+    "pipeline_pandas_s3_get", 
+    "pipeline_pandas_s3_store",
 
     # Microsoft connectors
     "teams_webhook_send",

src/dataplane/pipelinerun/data_persist/redis_store.py renamed to src/dataplane/pipelinerun/data_persist/pandas_redis_store.py

@@ -1,9 +1,9 @@
-import os
-import io
-from datetime import datetime, timedelta
-import redis
+from datetime import timedelta
 
 def RedisCheck(r):
+
+    import redis
+
     try:
         r.ping()
     except (redis.exceptions.ConnectionError, ConnectionRefusedError):
@@ -17,7 +17,11 @@ def RedisCheck(r):
 Expire: Expires the data if true.
 ExpireDuration: If expires is true, how much time to expire. Default 15 mins
 """
-def pipeline_redis_store(StoreKey, DataFrame, Redis, Expire=True, ExpireDuration=timedelta(minutes=15)):
+def pipeline_pandas_redis_store(StoreKey, DataFrame, Redis, Expire=True, ExpireDuration=timedelta(minutes=15)):
+
+    import os
+    import io
+    from datetime import datetime
 
     # Start the timer
     start  = datetime.now()
@@ -46,7 +50,11 @@ def pipeline_redis_store(StoreKey, DataFrame, Redis, Expire=True, ExpireDuration
 StoreKey: is the key to look up for retrieval (set with RedisStore). 
 Redis: e.g. Redis = redis.Redis(host='redis-service', port=6379, db=0)
 """
-def pipeline_redis_get(StoreKey, Redis):
+def pipeline_pandas_redis_get(StoreKey, Redis):
+
+    import os
+    import io
+    from datetime import datetime
 
     # Start the timer
     start  = datetime.now()

src/dataplane/pipelinerun/data_persist/s3_store.py renamed to src/dataplane/pipelinerun/data_persist/pandas_s3_store.py

@@ -1,9 +1,7 @@
 
 
 # import requests
-import os
-from io import BytesIO
-from datetime import datetime, timedelta
+from datetime import timedelta
 
 """ StoreKey: is the key to look up for retrieval later on. 
 S3Client: e.g. S3Client = boto3.client(...)
@@ -12,7 +10,11 @@
 Expire: Expires the data if true.
 ExpireDuration: If expires is true, how much time to expire. Default 15 mins
 """
-def pipeline_s3_store(StoreKey, DataFrame, S3Client, Bucket, Expire=True, ExpireDuration=timedelta(days=30)):
+def pipeline_pandas_s3_store(StoreKey, DataFrame, S3Client, Bucket, Expire=True, ExpireDuration=timedelta(days=30)):
+
+    import os
+    from io import BytesIO
+    from datetime import datetime
 
     # Start the timer
     start  = datetime.now()
@@ -33,7 +35,11 @@ def pipeline_s3_store(StoreKey, DataFrame, S3Client, Bucket, Expire=True, Expire
 S3Client: e.g. S3Client = boto3.client(...)
 Bucket: Name of the s3 bucket
 """
-def pipeline_s3_get(StoreKey, S3Client, Bucket):
+def pipeline_pandas_s3_get(StoreKey, S3Client, Bucket):
+
+    import os
+    from io import BytesIO
+    from datetime import datetime
 
     # Start the timer
     start  = datetime.now()

src/dataplane/pipelinerun/data_persist/test_redis.py

@@ -1,7 +1,7 @@
 
 import os
-from .redis_store import pipeline_redis_store
-from .redis_store import pipeline_redis_get
+from .pandas_redis_store import pipeline_pandas_redis_store
+from .pandas_redis_store import pipeline_pandas_redis_get
 import redis
 from datetime import timedelta
 from nanoid import generate
@@ -29,14 +29,14 @@ def test_redis_store():
     # ---------- STORE PARQUET TO REDIS ------------
 
     # Store the data with key hello - run id will be attached
-    rs = pipeline_redis_store(StoreKey="hello", DataFrame=df, Redis=redisConnect, Expire=True, ExpireDuration=timedelta(minutes=15))
+    rs = pipeline_pandas_redis_store(StoreKey="hello", DataFrame=df, Redis=redisConnect, Expire=True, ExpireDuration=timedelta(minutes=15))
     print(rs)
     assert rs["result"]=="OK"
 
     # ---------- RETRIEVE PARQUET FROM REDIS ------------
 
     # Get the data
-    rsget = pipeline_redis_get(StoreKey="hello", Redis=redisConnect)
+    rsget = pipeline_pandas_redis_get(StoreKey="hello", Redis=redisConnect)
     print(rsget)
     df = rsget["dataframe"]
     print(df.shape[0])

src/dataplane/pipelinerun/data_persist/test_s3_store.py

@@ -1,7 +1,7 @@
 
 import os
-from .s3_store import pipeline_s3_store
-from .s3_store import pipeline_s3_get
+from .pandas_s3_store import pipeline_pandas_s3_store
+from .pandas_s3_store import pipeline_pandas_s3_get
 import boto3
 from botocore.client import Config
 from nanoid import generate
@@ -38,14 +38,14 @@ def test_s3_store():
     # ---------- STORE PARQUET TO S3 ------------
 
     # Store the data with key hello - run id will be attached
-    rs = pipeline_s3_store(StoreKey="s3me", DataFrame=df, S3Client=S3Connect, Bucket=bucket, Expire=False)
+    rs = pipeline_pandas_s3_store(StoreKey="s3me", DataFrame=df, S3Client=S3Connect, Bucket=bucket, Expire=False)
     print(rs)
     assert rs["result"]=="OK"
 
     # ---------- RETRIEVE PARQUET FROM S3 ------------
 
     # Get the data
-    rsget = pipeline_s3_get(StoreKey="s3me", S3Client=S3Connect, Bucket=bucket)
+    rsget = pipeline_pandas_s3_get(StoreKey="s3me", S3Client=S3Connect, Bucket=bucket)
     print(rsget)
     df = rsget["dataframe"]
     print(df.shape[0])