From 305e45337f58275838a208bb6a5e708ee2d63af6 Mon Sep 17 00:00:00 2001 From: Andrew Huang Date: Tue, 27 Jun 2023 14:09:26 -0400 Subject: [PATCH] Convert to markdown --- doc/user_guide/Using_Features_Offline.ipynb | 132 ---------------- doc/user_guide/Using_Features_Offline.md | 114 +++++++++++++ doc/user_guide/Using_WMTS_Offline.ipynb | 167 -------------------- doc/user_guide/Using_WMTS_Offline.md | 149 +++++++++++++++++ 4 files changed, 263 insertions(+), 299 deletions(-) delete mode 100644 doc/user_guide/Using_Features_Offline.ipynb create mode 100644 doc/user_guide/Using_Features_Offline.md delete mode 100644 doc/user_guide/Using_WMTS_Offline.ipynb create mode 100644 doc/user_guide/Using_WMTS_Offline.md diff --git a/doc/user_guide/Using_Features_Offline.ipynb b/doc/user_guide/Using_Features_Offline.ipynb deleted file mode 100644 index bac44961..00000000 --- a/doc/user_guide/Using_Features_Offline.ipynb +++ /dev/null @@ -1,132 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Using Features Offline\n", - "\n", - "## Creating Environment\n", - "\n", - "Under the hood, GeoViews features simply wrap ``cartopy`` features, so it's a matter of properly\n", - "configuring ``cartopy`` ahead of time.\n", - "\n", - "1. Create a new cartopy environment (or use an existing one):\n", - "\n", - " ```bash\n", - " conda create -n cartopy_env python=3.10\n", - " ```\n", - "\n", - "2. Install the required packages:\n", - "\n", - " ```bash\n", - " conda install -c conda-forge geoviews cartopy cartopy_offlinedata\n", - " ```\n", - "\n", - " Or if you have an environment already, you may just need [`cartopy_offlinedata`](https://anaconda.org/conda-forge/cartopy_offlinedata):\n", - "\n", - " ```bash\n", - " conda install -c conda-forge cartopy_offlinedata\n", - " ```\n", - "\n", - "## Verifying Setup\n", - "\n", - "Now, we will verify that the shapefiles are available offline.\n", - "\n", - "1. Ensure offline shapefiles were downloaded:\n", - "\n", - " ```python\n", - " from pathlib import Path\n", - " import cartopy\n", - "\n", - " data_dir = Path(cartopy.config[\"pre_existing_data_dir\"])\n", - " shapefiles = data_dir / \"shapefiles\" / \"natural_earth\" / \"cultural\"\n", - " list(shapefiles.glob(\"*\"))\n", - " ```\n", - "\n", - "2. Test GeoViews offline (toggle internet off):\n", - "\n", - " ```python\n", - " import geoviews as gv\n", - " from bokeh.resources import INLINE\n", - "\n", - " gv.extension(\"bokeh\")\n", - "\n", - " coastline = gv.feature.coastline()\n", - " borders = gv.feature.borders()\n", - " world = (coastline * borders).opts(global_extent=True)\n", - "\n", - " gv.save(world, \"world.html\", resources=INLINE)\n", - " ```\n", - "\n", - " Please ensure to set [`resources=INLINE`](https://docs.bokeh.org/en/latest/docs/reference/resources.html#bokeh.resources.INLINE) if the machine you're using is completely\n", - " offline and you intend to view the output on that machine.\n", - " Failure to do so will result in the HTML file appearing empty when opened.\n", - "\n", - "## Changing Directory\n", - "\n", - "If you wish to change the default data directory, follow these steps.\n", - "\n", - "1. Create a new directory and move the data:\n", - "\n", - " ```python\n", - " from pathlib import Path\n", - " import cartopy\n", - "\n", - " new_data_dir = Path(\"~/.cartopy\").expanduser()\n", - " new_data_dir.mkdir(exist_ok=True)\n", - "\n", - " data_dir = Path(cartopy.config[\"pre_existing_data_dir\"])\n", - " data_dir.rename(new_data_dir / \"cartopy\")\n", - " ```\n", - "\n", - "2. Point to the new data directory within the script:\n", - "\n", - " ```python\n", - " from pathlib import Path\n", - "\n", - " import cartopy\n", - " import geoviews as gv\n", - " from bokeh.resources import INLINE\n", - "\n", - " cartopy.config[\"pre_existing_data_dir\"] = str(Path(\"~/.cartopy/cartopy\").expanduser())\n", - "\n", - " gv.extension(\"bokeh\")\n", - "\n", - " coastline = gv.feature.coastline()\n", - " borders = gv.feature.borders()\n", - " world = (coastline * borders).opts(global_extent=True)\n", - "\n", - " gv.save(world, \"world.html\", resources=INLINE)\n", - " ```\n", - "\n", - "3. Or set an environment variable ``CARTOPY_DATA_DIR``:\n", - "\n", - " For sh:\n", - " ```bash\n", - " export CARTOPY_DATA_DIR=\"$HOME/.cartopy/cartopy\"\n", - " ```\n", - "\n", - " For powershell:\n", - " ```powershell\n", - " $env:CARTOPY_DATA_DIR = \"$HOME/.cartopy/cartopy\"\n", - " ```\n", - "\n", - " For cmd:\n", - " ```cmd\n", - " set CARTOPY_DATA_DIR=%USERPROFILE%\\.cartopy\\cartopy\n", - " ```\n", - "\n", - " Please note using tilde (``~``) in the environment variable will not work." - ] - } - ], - "metadata": { - "language_info": { - "name": "python", - "pygments_lexer": "ipython3" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/doc/user_guide/Using_Features_Offline.md b/doc/user_guide/Using_Features_Offline.md new file mode 100644 index 00000000..51e59b05 --- /dev/null +++ b/doc/user_guide/Using_Features_Offline.md @@ -0,0 +1,114 @@ +# Using Features Offline + +## Creating Environment + +Under the hood, GeoViews features simply wrap ``cartopy`` features, so it's a matter of properly +configuring ``cartopy`` ahead of time. + +1. Create a new cartopy environment (or use an existing one): + + ```bash + conda create -n cartopy_env python=3.10 + ``` + +2. Install the required packages: + + ```bash + conda install -c conda-forge geoviews cartopy cartopy_offlinedata + ``` + + Or if you have an environment already, you may just need [`cartopy_offlinedata`](https://anaconda.org/conda-forge/cartopy_offlinedata): + + ```bash + conda install -c conda-forge cartopy_offlinedata + ``` + +## Verifying Setup + +Now, we will verify that the shapefiles are available offline. + +1. Ensure offline shapefiles were downloaded: + + ```python + from pathlib import Path + import cartopy + + data_dir = Path(cartopy.config["pre_existing_data_dir"]) + shapefiles = data_dir / "shapefiles" / "natural_earth" / "cultural" + list(shapefiles.glob("*")) + ``` + +2. Test GeoViews offline (toggle internet off): + + ```python + import geoviews as gv + from bokeh.resources import INLINE + + gv.extension("bokeh") + + coastline = gv.feature.coastline() + borders = gv.feature.borders() + world = (coastline * borders).opts(global_extent=True) + + gv.save(world, "world.html", resources=INLINE) + ``` + + Please ensure to set [`resources=INLINE`](https://docs.bokeh.org/en/latest/docs/reference/resources.html#bokeh.resources.INLINE) if the machine you're using is completely + offline and you intend to view the output on that machine. + Failure to do so will result in the HTML file appearing empty when opened. + +## Changing Directory + +If you wish to change the default data directory, follow these steps. + +1. Create a new directory and move the data: + + ```python + from pathlib import Path + import cartopy + + new_data_dir = Path("~/.cartopy").expanduser() + new_data_dir.mkdir(exist_ok=True) + + data_dir = Path(cartopy.config["pre_existing_data_dir"]) + data_dir.rename(new_data_dir / "cartopy") + ``` + +2. Point to the new data directory within the script: + + ```python + from pathlib import Path + + import cartopy + import geoviews as gv + from bokeh.resources import INLINE + + cartopy.config["pre_existing_data_dir"] = str(Path("~/.cartopy/cartopy").expanduser()) + + gv.extension("bokeh") + + coastline = gv.feature.coastline() + borders = gv.feature.borders() + world = (coastline * borders).opts(global_extent=True) + + gv.save(world, "world.html", resources=INLINE) + ``` + +3. Or set an environment variable ``CARTOPY_DATA_DIR``: + + For sh: + ```bash + export CARTOPY_DATA_DIR="$HOME/.cartopy/cartopy" + ``` + + For powershell: + ```powershell + $env:CARTOPY_DATA_DIR = "$HOME/.cartopy/cartopy" + ``` + + For cmd: + ```cmd + set CARTOPY_DATA_DIR=%USERPROFILE%\.cartopy\cartopy + ``` + + Please note using tilde (``~``) in the environment variable will not work. diff --git a/doc/user_guide/Using_WMTS_Offline.ipynb b/doc/user_guide/Using_WMTS_Offline.ipynb deleted file mode 100644 index 3f66d23f..00000000 --- a/doc/user_guide/Using_WMTS_Offline.ipynb +++ /dev/null @@ -1,167 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Using WMTS Offline\n", - "\n", - "## Caching the Tiles\n", - "\n", - "Web map tile services simply provide tiled images for a given target domain request. So to use them offline, you simply need to copy these images from their server to a preferred local mirror and point to that local mirror.\n", - "\n", - "However, attempting to determine the corresponding tiles to specific target domains can be a daunting task--thankfully, Cartopy provides utilities that can assist you with this task.\n", - "\n", - "When Cartopy is invoked for a given target domain, it retrieves and stores the relevant map tiles in a NumPy, binary file format `.npy`.\n", - "\n", - "```python\n", - "from pathlib import Path\n", - "\n", - "import cartopy.crs as ccrs\n", - "import cartopy.io.img_tiles as cimgt\n", - "import numpy as np\n", - "from PIL import Image\n", - "from shapely import box\n", - "\n", - "\n", - "def cache_tiles(\n", - " tile_source,\n", - " max_target_z=1,\n", - " x_bounds=(-180, 180),\n", - " y_bounds=(-90, 90),\n", - " cache_dir=\"tiles\",\n", - "):\n", - " \"\"\"\n", - " Caches map tiles within specified bounds from a given tile source.\n", - "\n", - " Args:\n", - " tile_source (str or cartopy.io.img_tiles.Tiles): The tile source to use for caching.\n", - " It can be a string specifying a built-in tile source, or an instance of a custom tile source class.\n", - " max_target_z (int, optional): The maximum zoom level to cache. Defaults to 1.\n", - " x_bounds (tuple, optional): The longitudinal bounds of the tiles to cache. Defaults to (-180, 180).\n", - " y_bounds (tuple, optional): The latitudinal bounds of the tiles to cache. Defaults to (-90, 90).\n", - " cache_dir (str, optional): The directory to store the cached tiles. Defaults to \"tiles\".\n", - "\n", - " Returns:\n", - " pathlib.Path: The path to the cache directory.\n", - " \"\"\"\n", - " if not isinstance(tile_source, cimgt.GoogleWTS):\n", - " tile_source = getattr(cimgt, tile_source)\n", - " tiles = tile_source(cache=cache_dir)\n", - "\n", - " bbox = ccrs.GOOGLE_MERCATOR.transform_points(\n", - " ccrs.PlateCarree(), x=np.array(x_bounds), y=np.array(y_bounds)\n", - " )[:, :-1].flatten() # drop Z, then convert to x0, y0, x1, y1\n", - " target_domain = box(*bbox)\n", - "\n", - " for target_z in range(max_target_z):\n", - " tiles.image_for_domain(target_domain, target_z)\n", - " return Path(cache_dir) / tile_source.__name__\n", - "```\n", - "\n", - "As an example, to cache OpenStreetMaps tiles, you can simply call the provided function, ensuring that you specify a maximum zoom level (`max_target_z`).\n", - "\n", - "```python\n", - "cache_dir = cache_tiles(\"OSM\", max_target_z=6)\n", - "```\n", - "\n", - "WARNING: When working with higher zoom levels, it is **highly recommended** to specify the `x_bounds` and `y_bounds` parameters to your region of interest. This is crucial to prevent potential issues such as rate limiting or, in extreme cases, *being banned*.\n", - "\n", - "As the zoom level increases, the time required for downloading and caching the tiles grows exponentially due to the increasing number of fine-grained tiles that need to be retrieved. By setting appropriate boundaries, you can effectively manage the download process and mitigate the risk of encountering problems related to excessive requests.\n", - "\n", - "Here is a table illustrating the number of tiles for *global extents* at different zoom levels:\n", - "\n", - "```\n", - "z=0: 1 tile (entire world)\n", - "z=1: 4 tiles\n", - "z=2: 16 tiles\n", - "z=3: 64 tiles\n", - "z=4: 256 tiles\n", - "z=5: 1,024 tiles\n", - "z=6: 4,096 tiles\n", - "z=7: 16,384 tiles\n", - "z=8: 65,536 tiles\n", - "z=9: 262,144 tiles\n", - "z=10: 1,048,576 tiles\n", - "z=11: 4,194,304 tiles\n", - "z=12: 16,777,216 tiles\n", - "z=13: 67,108,864 tiles\n", - "z=14: 268,435,456 tiles\n", - "z=15: 1,073,741,824 tiles\n", - "z=16: 4,294,967,296 tiles\n", - "z=17: 17,179,869,184 tiles\n", - "z=18: 68,719,476,736 tiles\n", - "z=19: 274,877,906,944 tiles\n", - "z=20: 1,099,511,627,776 tiles\n", - "```\n", - "\n", - "## Converting to PNG\n", - "\n", - "Since GeoViews lacks support for reading cached NumPy binary files, an additional step is required to:\n", - "\n", - "1. convert them to PNG format\n", - "2. update their directories\n", - "3. build a format string containing \"{X}/{Y}/{Z}\" (or similar variations)\n", - "\n", - "Fortunately, this process only involves a straightforward loop that performs minimal processing on each file.\n", - "\n", - "```python\n", - "def convert_tiles_cache(cache_dir):\n", - " \"\"\"\n", - " Converts cached tiles from numpy format to PNG format.\n", - "\n", - " Args:\n", - " cache_dir (str): The directory containing the cached tiles in numpy format.\n", - "\n", - " Returns:\n", - " str: The format string representing the converted PNG tiles.\n", - " \"\"\"\n", - " for np_path in Path(cache_dir).rglob(\"*.npy\"):\n", - " img = Image.fromarray(np.load(np_path))\n", - " img_path = Path(str(np_path.with_suffix(\".png\")).replace(\"_\", \"/\"))\n", - " img_path.parent.mkdir(parents=True, exist_ok=True)\n", - " img.save(img_path)\n", - "\n", - " tiles_fmt = str(cache_dir / \"{X}\" / \"{Y}\" / \"{Z}.png\")\n", - " return tiles_fmt\n", - "```\n", - "\n", - "```python\n", - "tiles_fmt = convert_tiles_cache(cache_dir)\n", - "```\n", - "\n", - "## Testing Locally\n", - "\n", - "Now, all that's left is passing that generated tiles format string into `gv.WMTS`.\n", - "\n", - "```python\n", - "import geoviews as gv\n", - "\n", - "gv.extension(\"bokeh\")\n", - "\n", - "gv.WMTS(tiles_dir).opts(global_extent=True)\n", - "```\n", - "\n", - "Please keep in mind that when reaching higher zoom levels beyond the cached max_target_z, you might encounter a blank map.\n", - "\n", - "To avoid this issue, it is essential to set the `max_zoom` option to the same value as `max_target_z`.\n", - "\n", - "```python\n", - "import geoviews as gv\n", - "\n", - "gv.extension(\"bokeh\")\n", - "\n", - "gv.WMTS(tiles_dir).opts(global_extent=True, max_zoom=6)\n", - "```" - ] - } - ], - "metadata": { - "language_info": { - "name": "python", - "pygments_lexer": "ipython3" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/doc/user_guide/Using_WMTS_Offline.md b/doc/user_guide/Using_WMTS_Offline.md new file mode 100644 index 00000000..e7af2c19 --- /dev/null +++ b/doc/user_guide/Using_WMTS_Offline.md @@ -0,0 +1,149 @@ +# Using WMTS Offline + +## Caching the Tiles + +Web map tile services simply provide tiled images for a given target domain request. So to use them offline, you simply need to copy these images from their server to a preferred local mirror and point to that local mirror. + +However, attempting to determine the corresponding tiles to specific target domains can be a daunting task--thankfully, Cartopy provides utilities that can assist you with this task. + +When Cartopy is invoked for a given target domain, it retrieves and stores the relevant map tiles in a NumPy, binary file format `.npy`. + +```python +from pathlib import Path + +import cartopy.crs as ccrs +import cartopy.io.img_tiles as cimgt +import numpy as np +from PIL import Image +from shapely import box + + +def cache_tiles( + tile_source, + max_target_z=1, + x_bounds=(-180, 180), + y_bounds=(-90, 90), + cache_dir="tiles", +): + """ + Caches map tiles within specified bounds from a given tile source. + + Args: + tile_source (str or cartopy.io.img_tiles.Tiles): The tile source to use for caching. + It can be a string specifying a built-in tile source, or an instance of a custom tile source class. + max_target_z (int, optional): The maximum zoom level to cache. Defaults to 1. + x_bounds (tuple, optional): The longitudinal bounds of the tiles to cache. Defaults to (-180, 180). + y_bounds (tuple, optional): The latitudinal bounds of the tiles to cache. Defaults to (-90, 90). + cache_dir (str, optional): The directory to store the cached tiles. Defaults to "tiles". + + Returns: + pathlib.Path: The path to the cache directory. + """ + if not isinstance(tile_source, cimgt.GoogleWTS): + tile_source = getattr(cimgt, tile_source) + tiles = tile_source(cache=cache_dir) + + bbox = ccrs.GOOGLE_MERCATOR.transform_points( + ccrs.PlateCarree(), x=np.array(x_bounds), y=np.array(y_bounds) + )[:, :-1].flatten() # drop Z, then convert to x0, y0, x1, y1 + target_domain = box(*bbox) + + for target_z in range(max_target_z): + tiles.image_for_domain(target_domain, target_z) + return Path(cache_dir) / tile_source.__name__ +``` + +As an example, to cache OpenStreetMaps tiles, you can simply call the provided function, ensuring that you specify a maximum zoom level (`max_target_z`). + +```python +cache_dir = cache_tiles("OSM", max_target_z=6) +``` + +WARNING: When working with higher zoom levels, it is **highly recommended** to specify the `x_bounds` and `y_bounds` parameters to your region of interest. This is crucial to prevent potential issues such as rate limiting or, in extreme cases, *being banned*. + +As the zoom level increases, the time required for downloading and caching the tiles grows exponentially due to the increasing number of fine-grained tiles that need to be retrieved. By setting appropriate boundaries, you can effectively manage the download process and mitigate the risk of encountering problems related to excessive requests. + +Here is a table illustrating the number of tiles for *global extents* at different zoom levels: + +``` +z=0: 1 tile (entire world) +z=1: 4 tiles +z=2: 16 tiles +z=3: 64 tiles +z=4: 256 tiles +z=5: 1,024 tiles +z=6: 4,096 tiles +z=7: 16,384 tiles +z=8: 65,536 tiles +z=9: 262,144 tiles +z=10: 1,048,576 tiles +z=11: 4,194,304 tiles +z=12: 16,777,216 tiles +z=13: 67,108,864 tiles +z=14: 268,435,456 tiles +z=15: 1,073,741,824 tiles +z=16: 4,294,967,296 tiles +z=17: 17,179,869,184 tiles +z=18: 68,719,476,736 tiles +z=19: 274,877,906,944 tiles +z=20: 1,099,511,627,776 tiles +``` + +## Converting to PNG + +Since GeoViews lacks support for reading cached NumPy binary files, an additional step is required to: + +1. convert them to PNG format +2. update their directories +3. build a format string containing "{X}/{Y}/{Z}" (or similar variations) + +Fortunately, this process only involves a straightforward loop that performs minimal processing on each file. + +```python +def convert_tiles_cache(cache_dir): + """ + Converts cached tiles from numpy format to PNG format. + + Args: + cache_dir (str): The directory containing the cached tiles in numpy format. + + Returns: + str: The format string representing the converted PNG tiles. + """ + for np_path in Path(cache_dir).rglob("*.npy"): + img = Image.fromarray(np.load(np_path)) + img_path = Path(str(np_path.with_suffix(".png")).replace("_", "/")) + img_path.parent.mkdir(parents=True, exist_ok=True) + img.save(img_path) + + tiles_fmt = str(cache_dir / "{X}" / "{Y}" / "{Z}.png") + return tiles_fmt +``` + +```python +tiles_fmt = convert_tiles_cache(cache_dir) +``` + +## Testing Locally + +Now, all that's left is passing that generated tiles format string into `gv.WMTS`. + +```python +import geoviews as gv + +gv.extension("bokeh") + +gv.WMTS(tiles_dir).opts(global_extent=True) +``` + +Please keep in mind that when reaching higher zoom levels beyond the cached max_target_z, you might encounter a blank map. + +To avoid this issue, it is essential to set the `max_zoom` option to the same value as `max_target_z`. + +```python +import geoviews as gv + +gv.extension("bokeh") + +gv.WMTS(tiles_dir).opts(global_extent=True, max_zoom=6) +```