v1.8.3

VERSION

@@ -1 +1 @@
-1.8.2
+1.8.3

docker/Dockerfile

@@ -1,5 +1,5 @@
 # Container for packages that need to be built from source but have massive dev dependencies.
-FROM python:3.10.8-slim-bullseye as builder
+FROM python:3.9.16-slim-bullseye as builder
 RUN set -e && \
     apt-get update && \
     apt-get install -y --no-install-recommends \
@@ -11,7 +11,7 @@ RUN pip config set global.disable-pip-version-check true && \
     pip wheel --wheel-dir=/root/wheels regex==2022.10.31
 
 # The actual container.
-FROM python:3.10.8-slim-bullseye
+FROM python:3.9.16-slim-bullseye
 RUN set -e && \
     apt-get update && \
     apt-get install -y --no-install-recommends \

docker/uwsgi.ini

@@ -18,3 +18,9 @@ manage-script-name = true
 die-on-term = true
 
 buffer-size = 65535
+
+max-requests = 10000
+max-worker-lifetime = 3600
+worker-reload-mercy = 20
+reload-on-rss = 512
+reload-on-as = 512

docs/changelog.md

@@ -3,6 +3,13 @@
 This is a list of changes to Open Topo Data between each release.
 
 
+## Version 1.8.3 (7 Feb 2023)
+
+* Fix memory leak ([#68](https://github.com/ajnisbet/opentopodata/issues/68))
+* Fix invalid file error message ([#70](https://github.com/ajnisbet/opentopodata/issues/70))
+* Downgrade to python 3.9 and rasterio 1.2.10
+
+
 ## Version 1.8.2 (7 Nov 2022)
 
 * Fix broken Docker image ([#66](https://github.com/ajnisbet/opentopodata/issues/66))

docs/datasets/srtm.md

@@ -21,7 +21,7 @@ SRTM has coverage from -60 to 60 degrees latitude. The dataset is released in 1
 
 
 
-## Adding 30m SRTM to Open Topo Data
+## Downloading 30m SRTM
 
 Make a new folder for the dataset:
 
@@ -33,6 +33,25 @@ Download the files from [USGS](https://e4ftl01.cr.usgs.gov/MEASURES/) into `./da
 
 You want the `xxxxxxx.SRTMGL1.hgt.zip` files. To make downloading a bit easier, here's a list of the 14,297 URLs: [srtm30m_urls.txt](/datasets/srtm30m_urls.txt).
 
+If those scripts aren't working for you, an [@SamDurand](https://github.com/SamDurand) ([#70](https://github.com/ajnisbet/opentopodata/issues/70)) had success with logging into Earthdata in your browser, then automating the browser to download the files:
+
+
+```python
+import webbrowser
+import time
+
+with open("srtm30m_urls.txt", "r") as f:
+    url_list = f.read().split("\n")
+
+for i, url in enumerate(url_list):
+    webbrowser.open_new_tab(url)
+    if i % 100 == 0:
+        time.sleep(5) # pause 5s every 100 it to avoid rate limiting.
+```
+
+## Adding 30m SRTM to Open Topo Data
+
+
 
 Create a `config.yaml` file:
 

opentopodata/backend.py

@@ -239,7 +239,6 @@ def get_elevation(lats, lons, datasets, interpolation="nearest", nodata_value=No
     # Check
     points = [_Point(lat, lon, idx) for idx, (lat, lon) in enumerate(zip(lats, lons))]
     for dataset in datasets:
-
         # Only check points that have no point yet. Can exit early if
         # there's no unqueried points.
         dataset_points = [p for p in points if p.elevation is None]

opentopodata/config.py

@@ -265,9 +265,14 @@ def from_config(cls, name, path=None, **kwargs):
             )
 
         # Unable to identify dataset type.
+        invalid_files = [
+            fn for fn, is_valid in zip(all_filenames, is_srtm_raster) if not is_valid
+        ]
+        is_srtm_raster = np.array(is_srtm_raster, dtype=bool)
         msg = f"Unknown dataset type for '{name}'. Dataset should either be a single file,"
         msg += " or split into tiles with the lower-left corner coord in the filename like 'N20W120'."
-        msg += f" Unrecognised filename: '{all_filenames[np.argmin(is_srtm_raster)]}'."
+        if invalid_files:
+            msg += f" Unrecognised filename: '{invalid_files[0]}'."
         raise ConfigError(msg)
 
     @abc.abstractmethod

opentopodata/utils.py

@@ -137,7 +137,6 @@ def sample_points_on_path(path_lats, path_lons, n_samples):
     # distance along line.
     points = []
     for point_distance in point_distances:
-
         # Find start.
         i_start = np.argwhere(point_distance >= path_distances_cum)[:, 0][-1]
 

requirements.in

@@ -1,15 +1,14 @@
 black
-Flask
+Flask>=2.2.2
 flask-caching
 geographiclib
 numpy
 pip-tools
 polyline
 pylibmc
 pyproj
-pylibmc
 pytest
 pytest-cov
 PyYAML
-rasterio
+rasterio==1.2.10
 requests

requirements.txt

@@ -1,27 +1,27 @@
 #
-# This file is autogenerated by pip-compile with python 3.10
-# To update, run:
+# This file is autogenerated by pip-compile with Python 3.9
+# by the following command:
 #
 #    pip-compile requirements.in
 #
-affine==2.3.1
+affine==2.4.0
     # via rasterio
-attrs==22.1.0
+attrs==22.2.0
     # via
     #   pytest
     #   rasterio
-black==22.10.0
+black==23.1.0
     # via -r requirements.in
-build==0.9.0
+build==0.10.0
     # via pip-tools
 cachelib==0.9.0
     # via flask-caching
-certifi==2022.9.24
+certifi==2022.12.7
     # via
     #   pyproj
     #   rasterio
     #   requests
-charset-normalizer==2.1.1
+charset-normalizer==3.0.1
     # via requests
 click==8.1.3
     # via
@@ -35,75 +35,74 @@ click-plugins==1.1.1
     # via rasterio
 cligj==0.7.2
     # via rasterio
-coverage[toml]==6.5.0
+coverage[toml]==7.1.0
     # via pytest-cov
-exceptiongroup==1.0.1
+exceptiongroup==1.1.0
     # via pytest
 flask==2.2.2
     # via
     #   -r requirements.in
     #   flask-caching
-flask-caching==2.0.1
+flask-caching==2.0.2
     # via -r requirements.in
 geographiclib==2.0
     # via -r requirements.in
 idna==3.4
     # via requests
-iniconfig==1.1.1
+importlib-metadata==6.0.0
+    # via flask
+iniconfig==2.0.0
     # via pytest
 itsdangerous==2.1.2
     # via flask
 jinja2==3.1.2
     # via flask
-markupsafe==2.1.1
+markupsafe==2.1.2
     # via
     #   jinja2
     #   werkzeug
-mypy-extensions==0.4.3
+mypy-extensions==1.0.0
     # via black
-numpy==1.23.4
+numpy==1.24.2
     # via
     #   -r requirements.in
     #   rasterio
     #   snuggs
-packaging==21.3
+packaging==23.0
     # via
+    #   black
     #   build
     #   pytest
-pathspec==0.10.1
+pathspec==0.11.0
     # via black
-pep517==0.13.0
-    # via build
-pip-tools==6.9.0
+pip-tools==6.12.2
     # via -r requirements.in
-platformdirs==2.5.3
+platformdirs==3.0.0
     # via black
 pluggy==1.0.0
     # via pytest
-polyline==1.4.0
+polyline==2.0.0
     # via -r requirements.in
 pylibmc==1.6.3
     # via -r requirements.in
 pyparsing==3.0.9
-    # via
-    #   packaging
-    #   snuggs
-pyproj==3.4.0
+    # via snuggs
+pyproj==3.4.1
     # via -r requirements.in
-pytest==7.2.0
+pyproject-hooks==1.0.0
+    # via build
+pytest==7.2.1
     # via
     #   -r requirements.in
     #   pytest-cov
 pytest-cov==4.0.0
     # via -r requirements.in
 pyyaml==6.0
     # via -r requirements.in
-rasterio==1.3.3
+rasterio==1.2.10
     # via -r requirements.in
-requests==2.28.1
+requests==2.28.2
     # via -r requirements.in
-six==1.16.0
-    # via polyline
 snuggs==1.4.7
     # via rasterio
 tomli==2.0.1
@@ -112,12 +111,16 @@ tomli==2.0.1
     #   build
     #   coverage
     #   pytest
-urllib3==1.26.12
+typing-extensions==4.4.0
+    # via black
+urllib3==1.26.14
     # via requests
 werkzeug==2.2.2
     # via flask
-wheel==0.38.2
+wheel==0.38.4
     # via pip-tools
+zipp==3.12.1
+    # via importlib-metadata
 
 # The following packages are considered to be unsafe in a requirements file:
 # pip

tests/test_api.py

@@ -20,6 +20,7 @@
 ETOPO1_DATASET_NAME = "etopo1deg"
 MAX_N_POINTS = 100
 
+
 # Mock changing config.
 @pytest.fixture
 def patch_config():