diff --git a/Dockerfile b/Dockerfile index d4212b2..202e3fc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,11 +1,10 @@ -FROM remotepixel/amazonlinux-gdal:2.4.2 +FROM remotepixel/amazonlinux:gdal3.0-py3.7-cogeo -RUN pip3 install pip -U +WORKDIR /tmp -ENV PACKAGE_PREFIX /tmp/python +ENV PYTHONUSERBASE=/var/task COPY setup.py setup.py COPY lambda_tiler/ lambda_tiler/ -# Install dependencies -RUN CFLAGS="--std=c99" pip3 install . --no-binary numpy,rasterio -t $PACKAGE_PREFIX -U +RUN pip install . --user diff --git a/Makefile b/Makefile index 6e90cff..5d59402 100644 --- a/Makefile +++ b/Makefile @@ -7,8 +7,6 @@ package: -w /tmp \ --volume $(shell pwd)/bin:/tmp/bin \ --volume $(shell pwd)/:/local \ - --env PACKAGE_TMP=/tmp/package \ - --env PACKAGE_PATH=/local/package.zip \ -itd lambdatiler:latest \ bash docker exec -it lambdatiler bash '/tmp/bin/package.sh' diff --git a/README.md b/README.md index bb55a97..4056231 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ $ docker-compose run --rm package $ docker-compose run --rm tests ``` -Note: Docker image from https://github.com/RemotePixel/amazonlinux-gdal +Note: Docker image from https://github.com/RemotePixel/amazonlinux #### Deploy to AWS diff --git a/bin/package.sh b/bin/package.sh index 275b02d..dd3d265 100755 --- a/bin/package.sh +++ b/bin/package.sh @@ -2,32 +2,14 @@ echo "-----------------------" echo "Creating lambda package" echo "-----------------------" -echo -#echo "Remove useless python files" -#find $PACKAGE_PREFIX -name "*-info" -type d -exec rm -rdf {} + - -echo "Remove lambda python packages" -rm -rdf $PACKAGE_PREFIX/boto3/ \ - && rm -rdf $PACKAGE_PREFIX/botocore/ \ - && rm -rdf $PACKAGE_PREFIX/docutils/ \ - && rm -rdf $PACKAGE_PREFIX/dateutil/ \ - && rm -rdf $PACKAGE_PREFIX/jmespath/ \ - && rm -rdf $PACKAGE_PREFIX/s3transfer/ \ - && rm -rdf $PACKAGE_PREFIX/numpy/doc/ - echo "Remove uncompiled python scripts" -find $PACKAGE_PREFIX -type f -name '*.pyc' | while read f; do n=$(echo $f | sed 's/__pycache__\///' | sed 's/.cpython-36//'); cp $f $n; done; -find $PACKAGE_PREFIX -type d -a -name '__pycache__' -print0 | xargs -0 rm -rf -find $PACKAGE_PREFIX -type f -a -name '*.py' -print0 | xargs -0 rm -f - -echo "Strip shared libraries" -cd $PREFIX && find lib -name \*.so\* -exec strip {} \; -cd $PREFIX && find lib64 -name \*.so\* -exec strip {} \; +# Leave module precompiles for faster Lambda startup +cd ${PYTHONUSERBASE}/lib/python3.7/site-packages/ +find . -type f -name '*.pyc' | while read f; do n=$(echo $f | sed 's/__pycache__\///' | sed 's/.cpython-[2-3][0-9]//'); cp $f $n; done; +find . -type d -a -name '__pycache__' -print0 | xargs -0 rm -rf +find . -type f -a -name '*.py' -print0 | xargs -0 rm -f echo "Create archive" -cd $PACKAGE_PREFIX && zip -r9q /tmp/package.zip * -cd $PREFIX && zip -r9q --symlinks /tmp/package.zip lib/*.so* -cd $PREFIX && zip -r9q --symlinks /tmp/package.zip lib64/*.so* -cd $PREFIX && zip -r9q /tmp/package.zip share +zip -r9q /tmp/package.zip * cp /tmp/package.zip /local/package.zip \ No newline at end of file diff --git a/lambda_tiler/__init__.py b/lambda_tiler/__init__.py index e29d30c..627f1f6 100644 --- a/lambda_tiler/__init__.py +++ b/lambda_tiler/__init__.py @@ -1,3 +1,5 @@ """Lambda-tiler.""" -__version__ = "2.1.1" +import pkg_resources + +version = pkg_resources.get_distribution(__package__).version diff --git a/lambda_tiler/handler.py b/lambda_tiler/handler.py index 6c9d3fb..97453cc 100644 --- a/lambda_tiler/handler.py +++ b/lambda_tiler/handler.py @@ -1,12 +1,12 @@ """app.main: handle request for lambda-tiler.""" -from typing import Dict, Tuple, Union -from typing.io import BinaryIO +from typing import BinaryIO, Dict, Tuple, Union import os import re import json import urllib +from io import BytesIO import numpy @@ -83,7 +83,7 @@ class TilerError(Exception): ) def viewer_handler(url: str, **kwargs: Dict) -> Tuple[str, str, str]: """Handle Viewer requests.""" - qs = [f"{k}={v}" for k, v in kwargs.items()] + qs = urllib.parse.urlencode(list(kwargs.items())) if qs: qs = "&".join(qs) else: @@ -128,7 +128,7 @@ def tilejson_handler(url: str, tile_format: str = "png", **kwargs: Dict): with rasterio.open(url) as src_dst: bounds = warp.transform_bounds( - *[src_dst.crs, "epsg:4326"] + list(src_dst.bounds), densify_pts=21 + src_dst.crs, "epsg:4326", *src_dst.bounds, densify_pts=21 ) center = [(bounds[0] + bounds[2]) / 2, (bounds[1] + bounds[3]) / 2] minzoom, maxzoom = get_zooms(src_dst) @@ -171,12 +171,12 @@ def metadata_handler( url: str, pmin: Union[str, float] = 2.0, pmax: Union[str, float] = 98.0, - nodata: Union[str, float, int] = None, - indexes: Union[str, Tuple, int] = None, - overview_level: Union[str, int] = None, + nodata: Union[str, float, int, None] = None, + indexes: Union[str, Tuple, int, None] = None, + overview_level: Union[str, int, None] = None, max_size: Union[str, int] = 1024, histogram_bins: Union[str, int] = 20, - histogram_range: Union[str, int] = None, + histogram_range: Union[str, int, None] = None, ) -> Tuple[str, str, str]: """Handle /metadata requests.""" pmin = float(pmin) if isinstance(pmin, str) else pmin @@ -291,15 +291,11 @@ def tile_handler( if color_map: color_map = get_colormap(color_map, format="gdal") - if ext == "jpg": - driver = "jpeg" - elif ext == "tif": - driver = "GTiff" - else: - driver = ext - + driver = "jpeg" if ext == "jpg" else ext options = img_profiles.get(driver, {}) - if driver == "GTiff": + if ext == "tif": + ext = "tiff" + driver = "GTiff" mercator_tile = mercantile.Tile(x=x, y=y, z=z) bounds = mercantile.xy_bounds(mercator_tile) w, s, e, n = bounds @@ -308,11 +304,19 @@ def tile_handler( dtype=rtile.dtype, crs={"init": "EPSG:3857"}, transform=dst_transform ) - return ( - "OK", - f"image/{ext}", - array_to_image(rtile, rmask, img_format=driver, color_map=color_map, **options), - ) + if ext == "npy": + sio = BytesIO() + numpy.save(sio, (rtile, mask)) + sio.seek(0) + return ("OK", "application/x-binary", sio.getvalue()) + else: + return ( + "OK", + f"image/{ext}", + array_to_image( + rtile, rmask, img_format=driver, color_map=color_map, **options + ), + ) @APP.route("/favicon.ico", methods=["GET"], cors=True, tag=["other"]) diff --git a/serverless.yml b/serverless.yml index 7848679..c536873 100644 --- a/serverless.yml +++ b/serverless.yml @@ -2,26 +2,26 @@ service: lambda-tiler provider: name: aws - runtime: python3.6 - stage: production + runtime: python3.7 + stage: ${opt:stage, 'production'} region: ${opt:region, 'us-east-1'} # Add optional bucket deployement # deploymentBucket: ${opt:bucket} - environment: - PYTHONWARNINGS: ignore - GDAL_DATA: /var/task/share/gdal - GDAL_CACHEMAX: 512 - VSI_CACHE: TRUE - VSI_CACHE_SIZE: 536870912 CPL_TMPDIR: /tmp + GDAL_CACHEMAX: 512 + GDAL_DATA: /opt/share/gdal + GDAL_DISABLE_READDIR_ON_OPEN: EMPTY_DIR GDAL_HTTP_MERGE_CONSECUTIVE_RANGES: YES GDAL_HTTP_MULTIPLEX: YES GDAL_HTTP_VERSION: 2 - GDAL_DISABLE_READDIR_ON_OPEN: EMPTY_DIR - MAX_THREADS: 20 + MAX_THREADS: 50 + PROJ_LIB: /opt/share/proj + PYTHONWARNINGS: ignore + VSI_CACHE: TRUE + VSI_CACHE_SIZE: 536870912 # Add IAM Role statements to allow the tiler to access files # iamRoleStatements: @@ -44,6 +44,8 @@ functions: handler: lambda_tiler.handler.APP memorySize: 1536 timeout: 20 + layers: + - arn:aws:lambda:${self:provider.region}:524387336408:layer:gdal30-py37-cogeo:8 events: - http: path: /{proxy+} diff --git a/setup.py b/setup.py index b882a60..d75330a 100644 --- a/setup.py +++ b/setup.py @@ -1,17 +1,9 @@ -"""Setup remotepixel-tiler""" +"""Setup lambda-tiler.""" from setuptools import setup, find_packages -with open("lambda_tiler/__init__.py") as f: - for line in f: - if line.find("__version__") >= 0: - version = line.split("=")[1].strip() - version = version.strip('"') - version = version.strip("'") - continue - # Runtime requirements. -inst_reqs = ["rio-tiler~=1.2", "lambda-proxy~=4.1.4", "rio-color"] +inst_reqs = ["lambda-proxy~=5.0", "rio-tiler", "rio-color"] extra_reqs = { "test": ["mock", "pytest", "pytest-cov"], @@ -20,7 +12,7 @@ setup( name="lambda-tiler", - version=version, + version="3.0.0", description=u"""""", long_description=u"", python_requires=">=3", diff --git a/tests/test_api.py b/tests/test_api.py index 54d8978..eb40754 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -429,7 +429,7 @@ def test_API_tilesMock(tiler, event): assert res["body"] assert res["isBase64Encoded"] headers = res["headers"] - assert headers["Content-Type"] == "image/tif" + assert headers["Content-Type"] == "image/tiff" kwargs = tiler.call_args[1] assert kwargs["tilesize"] == 256 vars = tiler.call_args[0]