Revisiting Cloud Deployment

.travis.yml

@@ -15,9 +15,9 @@ before_script:
 script:
 - if test -z "$AWS_ACCESS_KEY_ID"; then echo "AWS ACCESS KEY empty"; fi;
 - if test -z "$AWS_SECRET_ACCESS_KEY"; then echo "AWS SECRET empty"; fi;
-- bash ./environment/create-aws-creds.sh
-- stat ./environment/aws_config
-- docker build --pull --cache-from "$IMAGE_NAME" --tag "$IMAGE_NAME" --file environment/Dockerfile
+- bash ./deployment/environment/create-aws-creds.sh
+- stat ./deployment/environment/aws_config
+- docker build --pull --cache-from "$IMAGE_NAME" --tag "$IMAGE_NAME" --file deployment/sagemaker/Dockerfile
   .
 after_script:
 - docker images

deployment/development/Dockerfile

@@ -0,0 +1,20 @@
+FROM jupyter/datascience-notebook
+
+USER root 
+
+RUN apt-get -q update && \
+    apt-get -q -y install gdal-bin proj-bin && \
+    apt-get clean && \ 
+    rm -rf /var/lib/apt/ /var/cache/apt/ /var/cache/debconf/
+
+USER $NB_USER
+
+COPY ./deployment/environment/pytorch_p36.yml /tmp/
+RUN conda env update -q --file /tmp/pytorch_p36.yml && \
+  fix-permissions $CONDA_DIR && \
+  fix-permissions /home/$NB_USER
+
+SHELL ["/bin/bash", "-c"]
+RUN echo "source activate pytorch_p36" > ~/.bashrc
+ENV PATH /opt/conda/envs/pytorch_p36/bin:$PATH
+

deployment/environment/all_deps_raw.txt

@@ -0,0 +1,283 @@
+__main__:
+  argparse
+  preprocess
+aftertrain-checkpoint:
+  boto3
+  click
+  datetime
+  os
+  s3fs
+  subprocess
+  tarfile
+  tempfile
+compare-metrics:
+  ast
+  click
+  hashlib
+  json
+  matplotlib
+  numpy
+  os
+  rasterio
+  seaborn
+  shutil
+  sklearn
+  subprocess
+compare-metrics-checkpoint:
+  ast
+  click
+  hashlib
+  json
+  matplotlib
+  numpy
+  os
+  rasterio
+  shutil
+  sklearn
+  subprocess
+conf:
+
+file_utils:
+  boto3
+get_images:
+  argparse
+  datetime
+  fnmatch
+  functools
+  geopandas
+  json
+  multiprocessing
+  os
+  pandas
+  planet_utils.download
+  planet_utils.search
+  shapely.geometry
+  smart_open
+  subprocess
+  unittest
+  yaspin
+  zipfile
+gt_pre:
+  argparse
+  geopandas
+  json
+  numpy
+  os
+  raster_utils
+  rasterio
+  rasterio.features
+  rasterio.transform
+  shapely.geometry
+  shapely.ops
+  unittest
+  yaspin
+loss:
+  numpy
+  torch
+  torch.nn
+  utils
+planet_utils:
+  geopandas
+  itertools
+  json
+  multiprocessing
+  os
+  pandas
+  planet
+  requests
+  retrying
+  shapely.geometry
+  smart_open
+  tqdm
+process-planet-order:
+  argparse
+  boto3
+  pprint
+  s3fs
+raster_average:
+  click
+  matplotlib
+  numpy
+  rasterio
+  seaborn
+raster_utils:
+  numpy
+  rasterio
+  rasterio.warp
+robosat_pink:
+  PIL
+  boto3
+  colorsys
+  copy
+  csv
+  cv2
+  functools
+  glob
+  hashlib
+  io
+  itertools
+  json
+  math
+  mercantile
+  numpy
+  os
+  pandas
+  pathlib
+  pyproj
+  random
+  rasterio
+  rasterio.session
+  re
+  robosat_pink.tiles
+  rtree.index
+  s3fs
+  shapely
+  sklearn
+  sys
+  toml
+  torch
+  torch.utils
+  webcolors
+robosat_pink.features:
+  cv2
+  numpy
+  robosat_pink.tiles
+robosat_pink.losses:
+  torch
+robosat_pink.models:
+  torch
+  torchvision
+robosat_pink.osm:
+  geojson
+  math
+  osmium
+  robosat_pink.osm.core
+  shapely
+  sys
+robosat_pink.tools:
+  PIL
+  albumentations
+  argparse
+  boto3
+  collections
+  concurrent.futures
+  csv
+  glob
+  importlib
+  io
+  json
+  math
+  mercantile
+  numpy
+  numpy.random
+  os
+  pathlib
+  pkgutil
+  pprint
+  rasterio
+  rasterio.crs
+  rasterio.enums
+  rasterio.features
+  rasterio.transform
+  rasterio.vrt
+  rasterio.warp
+  re
+  requests
+  robosat_pink
+  robosat_pink.colors
+  robosat_pink.config
+  robosat_pink.datasets
+  robosat_pink.logs
+  robosat_pink.metrics
+  robosat_pink.tiles
+  robosat_pink.transforms
+  robosat_pink.web_ui
+  s3fs
+  shutil
+  sklearn.model_selection
+  supermercado
+  sys
+  time
+  torch
+  torch.backends
+  torch.optim
+  torch.utils.data
+  torchvision.transforms
+  tqdm
+scalebar-checkpoint:
+  cartopy
+  numpy
+stats:
+
+test:
+  preprocess.gt_pre
+  unittest
+test_deploy:
+
+test_rasterize:
+  PIL
+  json
+  mercantile
+  numpy
+  robosat_pink.tools.rasterize
+  unittest
+tests:
+  mercantile
+  robosat_pink.datasets
+  robosat_pink.tiles
+  robosat_pink.transforms
+  torch
+  unittest
+thumbs:
+  PIL
+  io
+  json
+  math
+  matplotlib
+  os
+  pandas
+  requests
+  sys
+  tqdm
+tile:
+  argparse
+  boto3
+  concurrent.futures
+  functools
+  geopandas
+  json
+  mercantile
+  numpy
+  os
+  pandas
+  raster_utils
+  rasterio
+  rio_tiler.utils
+  s3fs
+  shapely.geometry
+  supermercado
+  time
+utils:
+  datetime
+  json
+  numpy
+  pathlib
+  random
+  torch
+  tqdm
+valid_polygons:
+  argparse
+  numpy
+  os
+  rasterio
+  rasterio.features
+  subprocess
+  sys
+  tempfile
+validate_cloud_optimized_geotiff:
+  osgeo
+  sys
+validation:
+  numpy
+  torch
+  utils

deployment/environment/create-aws-creds.sh

@@ -0,0 +1,6 @@
+KEY=$AWS_ACCESS_KEY_ID
+SECRET=$AWS_SECRET_ACCESS_KEY
+PROFILE="esip"
+
+printf "[profile $PROFILE]\nregion = us-west-2 "> ./deployment/environment/aws_config
+printf "[$PROFILE]\naws_access_key_id=$KEY\naws_secret_access_key=$SECRET" > ./deployment/environment/aws_creds