hub sdk initial working

Author: davidbuniat

.dockerignore

@@ -1,2 +1,6 @@
 ./dask-worker-space/
-./cache
+./cache
+./data
+./dist
+./.pytest_cache
+./hub.egg-info

docker-compose.yaml

@@ -1,15 +1,24 @@
 version: '2'
 
 services:
-  test-gpu:
-     build:
-       context: . 
-     volumes:
-        - ./:/workspace/
-     command: bash -c "
+  test:
+    build:
+      context: . 
+    volumes:
+      - ./:/workspace/
+    command: bash -c "
            cd /workspace
            && pytest ./"
-          
+
+  test-gpu:
+    build:
+      context: . 
+    volumes:
+      - ./:/workspace/
+    command: bash -c "
+          cd /workspace
+          && pytest ./"
+
   py3:
     build:
       context: .

examples/basic.py

@@ -1,25 +1,11 @@
 import hub
 import numpy as np
-import sys, os, time, random, uuid, itertools, json, traceback, io
 
-# Create
-conn = hub.s3(
-    'waymo-dataset-upload', 
-    aws_creds_filepath='.creds/aws.json'
-    ).connect()
+images = hub.tensor.from_array(np.zeros((4, 512, 512)))
+labels = hub.tensor.from_array(np.zeros((4, 512, 512)))
 
+# ds = hub.dataset.from_tensors({"images": images, "labels": labels})
+# ds.store("basic")
 
-x = conn.array_create(
-    shape = (50000, 250, 250, 3),
-    chunk=(4, 250, 250, 3),
-    name= os.path.join('test', f'{int(time.time())}'),
-    dtype='uint8',
-)
-
-# Upload
-x[0] = np.ones((250, 250, 3), dtype='uint8')
-
-# Download
-print(x[0].mean())
-print(x[1].mean())
-print(x.shape, x[1].shape)
+ds = hub.load("test/test3")
+print(ds["images"][0].compute())

hub/datasets/cifar10.py examples/cifar10.pyhub/datasets/cifar10.py renamed to examples/cifar10.py

@@ -41,6 +41,7 @@ def main():
     )
     print(images.shape, fine_labels.shape, coarse_labels.shape)
     Image.fromarray(images[1000].transpose(1, 2, 0)).save("./data/image.png")
+
     images_t = tensor.from_array(images)
     fine_labels_t = tensor.from_array(fine_labels)
     coarse_labels_t = tensor.from_array(coarse_labels)

hub/datasets/cifar100.py examples/cifar100.pyhub/datasets/cifar100.py renamed to examples/cifar100.py

@@ -47,10 +47,14 @@ def load_mnist(dataset="training", digits=np.arange(10), path=".", size=60000):
 def main():
     parser = argparse.ArgumentParser()
     parser.add_argument(
-        "dataset_path", type=str, help="Path to cifar dataset", default="./data/mnist",
+        "-d",
+        "--dataset_path",
+        type=str,
+        help="Path to cifar dataset",
+        default="./data/mnist",
     )
     parser.add_argument(
-        "output_name", type=str, help="Dataset output name", default="mnist",
+        "-o", "--output_name", type=str, help="Dataset output name", default="mnist",
     )
     args = parser.parse_args()
     files = ["training", "testing"]
@@ -62,7 +66,7 @@ def main():
     # images = images.reshape((len(images), 3, 32, 32))
     labels = np.concatenate([np.array(d["labels"], dtype="int8") for d in dicts])
     print(images.shape, labels.shape)
-    Image.fromarray(images[1000]).save("./data/image.png")
+    # Image.fromarray(images[1000]).save("./data/image.png")
     images_t = tensor.from_array(images)
     labels_t = tensor.from_array(labels)
     # coarse_labels_t = tensor.from_array(coarse_labels)