first commit

Author: davidbuniat

.gitignore

@@ -0,0 +1,125 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+.terraform
+terraform.tfstate
+terraform.tfstate.backup
+.secrets
+node/.secrets
+examples/real/*
+node/*.pem
+
+# C extensions
+*.so
+.DS_Store
+.clusters
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+hyper/cluster/terraform/asg/.ssh/
+hyper/cluster/terraform/asg/terraform.tfstate
+hyper/cluster/terraform/asg/terraform.tfstate.backup
+
+.kube
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+
+hyper.sublime-*
+.kube-temp
+
+hyper_search/
+yolo/
+logs/

Dockerfile

@@ -0,0 +1,3 @@
+from python:3
+ADD ./ /workspace
+RUN pip install -e /workspace 

README.md

@@ -0,0 +1,22 @@
+# Meta arrays
+
+Minimum viable product
+
+1. Setup
+Install, then provide AWS credentials and bucket name
+```sh
+pip3 install -e .
+meta configure
+```
+
+2. Create an array
+```python
+mnist = meta.array((50000, 28, 28, 1), name="username/mnist:v1", dtype='float32')
+mnist[0, :] = np.random.random((1, 28, 28, 1)).astype('float32')
+```
+
+3. Load an array
+```python
+mnist = meta.load(name='username/mnist:v1')
+print(mnist[0,0,0,0])
+```

docker-compose.yaml

@@ -0,0 +1,21 @@
+version: '2'
+
+services:
+  meta:
+    build:
+      context: . 
+    volumes:
+       - ./:/workspace/
+    command: bash -c "
+          cd /workspace
+          && meta configure
+          && python3 example.py"
+  prod:
+    image: python:3
+    volumes:
+       - ./:/workspace/
+    command: bash -c "
+          cd /workspace
+          && ls
+          && pip3 install meta-array-beta
+          && meta configure"

example.py

@@ -0,0 +1,28 @@
+import meta
+import numpy as np
+
+vol = meta.load(name='imagenet/image:val')[400:600]
+a = (vol.mean(axis=(1,2,3)) == 0).sum()
+print(vol.mean(axis=(1,2,3)) == 0)
+exit()
+# Currently only support for up to 4D images
+mnist = meta.array((50000, 28, 28, 1), name="jason/mnist:v2", dtype='float32')
+mnist[0, :] = np.random.random((1, 28, 28, 1)).astype('float32')
+
+#mnist[0,:,:,:] = np.random.random((1,100,1)).astype('float32')
+#print(mnist[0])
+#print(mnist.dtype)
+
+
+print(mnist[0,0,0,0])
+# TODO load
+mnist = meta.load(name='jason/mnist:v1')
+
+print(mnist[0].shape)
+print(mnist[0,0,0,0])
+#print(mnist[0,:].shape)
+
+# - Upload ImageNet (Read only) 
+
+
+

meta/__init__.py

@@ -0,0 +1,16 @@
+import meta
+from meta.marray.array import array, load
+import click
+import sys
+from meta.log import configure_logger
+from .cli.auth import configure
+
+@click.group()
+@click.option('-v', '--verbose', count=True, help='Devel debugging')
+def cli(verbose):
+    configure_logger(verbose)
+
+def add_commands(cli):
+    cli.add_command(configure)
+
+add_commands(cli)

meta/cli/__init__.py

@@ -0,0 +1,38 @@
+import click
+from meta.log import logger
+from meta.utils.store_control import StoreControlClient
+from meta.utils.verify import Verify
+
+@click.command()
+@click.option('--username', '-u', default=None, help='Your AWS access key')
+@click.option('--password', '-p', default=None, help='Your AWS secret key')
+@click.option('--bucket',   '-b', default=None, help='Desired bucket name')
+def configure(username, password, bucket):
+    """ Logs in to Meta"""
+    logger.info("Please log in using your AWS credentials.")
+    if not username:
+        logger.debug("Prompting for Access Key")
+        username = click.prompt('AWS Access Key ID', type=str, hide_input=False)
+    access_key = username.strip()
+
+    if not password:
+        logger.debug("Prompting for Secret Key")
+        password = click.prompt('AWS Secret Access Key', type=str, hide_input=False)
+    secret_key = password.strip()
+
+    if not bucket:
+        logger.debug("Prompting for bucket name")
+        bucket = click.prompt('Bucket Name (e.g. company-name)', type=str, hide_input=False)
+    bucket = bucket.strip()
+    
+    success, creds = Verify(access_key, secret_key).verify_aws(bucket)
+    if success:
+        StoreControlClient().save_config(creds)
+        logger.info("Login Successful.")
+    else:
+        logger.error("Login error, please try again")
+        
+def logout():
+    """ Logs out of Snark AI"""
+    # TODO remove scripts
+    TokenManager.purge_token()

meta/cli/auth.py

@@ -0,0 +1,6 @@
+import os
+
+curr_path = os.path.dirname(os.path.abspath(__file__))
+
+TOKEN_FILE_PATH = os.path.expanduser("~/.meta/token")
+CLOUDVOLUME_PATH = os.path.expanduser("~/.cloudvolume/secrets/aws-secret.json")

meta/config.py

@@ -0,0 +1,10 @@
+import logging
+import sys
+
+logger = logging.getLogger('meta')
+def configure_logger(debug):
+    log_level = logging.DEBUG if debug == 1 else logging.INFO
+    logging.basicConfig(format='%(message)s',
+                        level=log_level,
+                        stream=sys.stdout)
+