From 817471bfd9df1fecfd4e65bed62dabbfba55a01e Mon Sep 17 00:00:00 2001 From: Filipe Fernandes Date: Fri, 17 Feb 2023 11:15:30 -0300 Subject: [PATCH 1/7] make pypi gha a bit more generic --- .github/workflows/pypi.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml index 0f12b656..43457a0f 100644 --- a/.github/workflows/pypi.yml +++ b/.github/workflows/pypi.yml @@ -41,12 +41,12 @@ jobs: - name: Test wheels run: | - cd dist && python -m pip install erddapy*.whl + cd dist && python -m pip install *.whl python -m twine check * - name: Publish a Python distribution to PyPI - if: ${{ github.event_name == 'release' }} - uses: pypa/gh-action-pypi-publish@v1.6.4 + if: success() && github.event_name == 'release' + uses: pypa/gh-action-pypi-publish@release/v1 with: user: __token__ password: ${{ secrets.PYPI_PASSWORD }} From 560d74c65c3741e8a61214f45b81614a6e3277a9 Mon Sep 17 00:00:00 2001 From: Filipe Fernandes Date: Fri, 17 Feb 2023 12:16:41 -0300 Subject: [PATCH 2/7] use a fair server --- notebooks/01a-griddap.ipynb | 152 +++++++++++------------------------- 1 file changed, 46 insertions(+), 106 deletions(-) diff --git a/notebooks/01a-griddap.ipynb b/notebooks/01a-griddap.ipynb index 07566662..f382a57b 100644 --- a/notebooks/01a-griddap.ipynb +++ b/notebooks/01a-griddap.ipynb @@ -6,13 +6,13 @@ "source": [ "# Griddap\n", "\n", - "Erddapy can access gridded datasets, using the server-side subsetting of griddap\n", - "or the OPeNDAP response, to download only the parts of a dataset that the user\n", - "requires.\n", + "Erddapy can access gridded datasets,\n", + "using the server-side subsetting of griddap or the OPeNDAP response,\n", + "to download only the parts of a dataset that the user requires.\n", "\n", - "In our example we will use a Region of Interest (ROI) to extract data within its\n", - "bounds. First we need to read the ROI with `geopandas`. Let's use the South\n", - "Atlantic Ocean basin from Natural Earth as our ROI.\n" + "In our example we will use a Region of Interest (ROI) to extract data within its bounds.\n", + "First we need to read the ROI with `geopandas`.\n", + "Let's use the South Atlantic Ocean basin from Natural Earth as our ROI." ] }, { @@ -24,6 +24,7 @@ "import geopandas\n", "import pooch\n", "\n", + "\n", "url = \"https://naturalearth.s3.amazonaws.com/4.1.1/50m_physical/ne_50m_geography_marine_polys.zip\"\n", "fname = pooch.retrieve(\n", " url,\n", @@ -57,23 +58,22 @@ "from erddapy import ERDDAP\n", "\n", "e = ERDDAP(\n", - " server=\"CSWC\", # CoastWatch West Coast Node\n", - " protocol=\"griddap\", # Protocol for gridded datasets\n", + " server=\"https://pae-paha.pacioos.hawaii.edu/erddap\",\n", + " protocol=\"griddap\",\n", ")\n", "\n", - "e.dataset_id = \"jplAvisoSshMon_LonPM180\" # AVISO Model Output, obs4MIPs NASA-JPL, Global, 1 Degree" + "e.dataset_id = \"etopo5_lon180\"" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "CAVEAT: Note that ERDDAP can serve gridded data with longitudes in the\n", - "0–360 format or -180–180. You must inspect the dataset and modify\n", - "your query accordingly.\n", + "CAVEAT: Note that ERDDAP can serve gridded data with longitudes in the 0–360 format or -180–180.\n", + "The user must inspect the dataset and modify your query accordingly.\n", "\n", - "Information on the griddap dataset is fetched with `griddap_initialize`. This\n", - "fills in the `variables` and `constraints` properties for that dataset.\n" + "Information on the griddap dataset is fetched with `griddap_initialize`.\n", + "This fills in the `variables` and `constraints` properties for that dataset." ] }, { @@ -96,12 +96,11 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "The default behaviour is to use erddap standard subsetting: return all variables\n", - "at the most recent timestep and every point of the remaining dimensions.\n", + "The default behaviour is to use erddap standard subsetting to return all variables at the most recent timestep and every point of the remaining dimensions.\n", "\n", - "This can result in large datasets, the values of the constraints can be changed,\n", - "and variables dropped before data set is downloaded. Here we will download only\n", - "one variable from that list.\n" + "This can result in large data requests!\n", + "However, the values of the constraints can be changed and variables dropped before data set is downloaded.\n", + "Here we will download only one variable from that list." ] }, { @@ -119,8 +118,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "We will reduce the dataset a bit further by requesting only the data that is\n", - "inside the bounding box of the South Atlantic.\n" + "And we will reduce the dataset a bit further by requesting only the data that is inside the bounding box of the South Atlantic." ] }, { @@ -142,35 +140,12 @@ "e.constraints" ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "print(f\"\\nconstraints for download:\\n\\n{json.dumps(e.constraints, indent=1)}\")" - ] - }, { "cell_type": "markdown", "metadata": {}, "source": [ - "Once the query is prepared we can download the data into an `xarray.Dataset`\n", - "object. (We can also download it in a `pandas.DataFrame` or `iris.Cube`\n", - "objects.)\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from urllib.parse import unquote_plus\n", - "\n", - "url = \"https://coastwatch.pfeg.noaa.gov/erddap/griddap/jplAvisoSshMon_LonPM180.nc?sshag%5B(2010-12-16T12:00:00Z):1:(2010-12-16T12:00:00Z)%5D%5B(-60.53346241642455):1:(0.03286652261984102)%5D%5B(-69.09208207871731):1:(19.63485354989288)%5D,nObs%5B(2010-12-16T12:00:00Z):1:(2010-12-16T12:00:00Z)%5D%5B(-60.53346241642455):1:(0.03286652261984102)%5D%5B(-69.09208207871731):1:(19.63485354989288)%5D\"\n", - "\n", - "unquote_plus(url)" + "Note the new longitude/latitude limits in the query above.\n", + "New can download the data into an `xarray.Dataset` object." ] }, { @@ -183,6 +158,7 @@ "source": [ "%%time\n", "\n", + "\n", "ds = e.to_xarray()" ] }, @@ -190,8 +166,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Once downloaded, data can be quickly visualised with xarray's inbuilt plotting\n", - "functionality\n" + "Once downloaded data can be quickly visualised with xarray's inbuilt plotting\n", + "functionality." ] }, { @@ -205,18 +181,16 @@ "\n", "\n", "fig, ax = plt.subplots(subplot_kw={\"projection\": ccrs.PlateCarree()})\n", - "ds[\"sshag\"].plot(ax=ax)\n", - "ax.coastlines()" + "ds[\"ROSE\"].plot(ax=ax)\n", + "ax.coastlines();" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "Note that we did not extract the exact ROI but instead we downloaded what is\n", - "inside its bounds. We can refine the data selection using region mask and\n", - "download strictly what is inside th ROI. The `regionmask` module can created\n", - "from the `geopandas` object.\n" + "Note that we did not extract the exact ROI but instead we downloaded what is inside a square defined by its bounds.\n", + "We can refine the data selection using region mask and download strictly what is inside th ROI." ] }, { @@ -229,7 +203,7 @@ "\n", "\n", "region = regionmask.from_geopandas(SA, name=name)\n", - "region.plot()" + "region.plot();" ] }, { @@ -247,30 +221,23 @@ "\n", "\n", "fig, ax = plt.subplots(subplot_kw={\"projection\": ccrs.PlateCarree()})\n", - "ds[\"sshag\"].sel(time=\"2010\").where(mask == region.numbers[0]).plot(ax=ax)\n", - "ax.coastlines()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The main difference is that now we did not load data from rivers plumes that are\n", - "not part of the ROI. (Check the Río de la Plata area in both plots.)\n" + "ds[\"ROSE\"].where(mask == region.numbers[0]).plot(ax=ax)\n", + "ax.coastlines();" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ + "Now we have data only for the Altantic Ocean, no land data in our plot.\n", + "\n", "### Subset after the request with OPeNDAP\n", "\n", "ERDDAP server-side subsetting can be avoided by specifying the OPeNDAP protocol.\n", - "This is a good choice if you intend to use a full dataset or subset it after the\n", - "request.\n", + "This is a good choice if you intend to use a full dataset or subset it after the request.\n", "\n", "Note that most OPeNDAP clients will eagerly download only the coordinates,\n", - "making a post request subset almost as fast as serve-side subset.\n" + "making a post request subset almost as fast as serve-side subset." ] }, { @@ -280,11 +247,12 @@ "outputs": [], "source": [ "e = ERDDAP(\n", - " server=\"CSWC\", # CoastWatch West Coast Node\n", + " server=\"https://pae-paha.pacioos.hawaii.edu/erddap\",\n", " protocol=\"griddap\",\n", " response=\"opendap\",\n", ")\n", - "e.dataset_id = \"jplAquariusSSS3MonthV5\" # Aquarius Sea Surface Salinity, L3 SMI, Version 5, 1.0°, Global," + "\n", + "e.dataset_id = \"etopo5_lon180\"" ] }, { @@ -309,7 +277,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Let's take a quick look at the data from 2015.\n" + "Let's take a quick look at the data.\n" ] }, { @@ -318,12 +286,13 @@ "metadata": {}, "outputs": [], "source": [ + "%%time\n", + "\n", "projection = ccrs.PlateCarree()\n", "\n", "fig, ax = plt.subplots(subplot_kw={\"projection\": projection})\n", - "sss = ds[\"sss\"].sel(time=\"2015\")\n", - "sss.plot(ax=ax)\n", - "ax.coastlines()" + "ds[\"ROSE\"].plot(ax=ax)\n", + "ax.coastlines();" ] }, { @@ -341,37 +310,8 @@ "\n", "\n", "fig, ax = plt.subplots(subplot_kw={\"projection\": ccrs.PlateCarree()})\n", - "ds[\"sss\"].sel(time=\"2015\").where(mask == region.numbers[0]).plot(ax=ax)\n", - "ax.coastlines()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Thanks to `regionmask` we can go a few steps further with the ROI and compute\n", - "statistics. Let's calculate the mean salinity time-series for the South Atlantic\n", - "and the mean spatial salinity for the time coverage.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "sasal = ds.groupby(mask).mean()\n", - "sasal[\"sss\"].plot(marker=\"o\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "sasal = ds.groupby(mask).mean(dim=\"time\")\n", - "sasal[\"sss\"].plot()" + "ds[\"ROSE\"].where(mask == region.numbers[0]).plot(ax=ax)\n", + "ax.coastlines();" ] } ], @@ -391,7 +331,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.4" + "version": "3.11.0" } }, "nbformat": 4, From d62f1466520bb56a4f3e0dddbbe7a65c3c3aa18a Mon Sep 17 00:00:00 2001 From: Filipe Fernandes Date: Fri, 17 Feb 2023 12:30:31 -0300 Subject: [PATCH 3/7] remove decode_times=False --- notebooks/02-extras.ipynb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/notebooks/02-extras.ipynb b/notebooks/02-extras.ipynb index 451ed766..1fb0a2ab 100644 --- a/notebooks/02-extras.ipynb +++ b/notebooks/02-extras.ipynb @@ -80,7 +80,7 @@ "metadata": {}, "outputs": [], "source": [ - "ds = e.to_xarray(decode_times=False)\n", + "ds = e.to_xarray()\n", "\n", "ds" ] @@ -149,7 +149,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.6" + "version": "3.11.0" } }, "nbformat": 4, From 88465b94bdfa77079b3eff1ad73ba3a0bef65a62 Mon Sep 17 00:00:00 2001 From: Filipe Fernandes Date: Mon, 20 Feb 2023 16:23:28 -0300 Subject: [PATCH 4/7] change to a better server --- notebooks/00-quick_intro.ipynb | 56 +++++++++++++--------------------- 1 file changed, 22 insertions(+), 34 deletions(-) diff --git a/notebooks/00-quick_intro.ipynb b/notebooks/00-quick_intro.ipynb index 00111ed2..9a7021f1 100644 --- a/notebooks/00-quick_intro.ipynb +++ b/notebooks/00-quick_intro.ipynb @@ -16,16 +16,12 @@ "\n", "```shell\n", "pip install erddapy\n", - "```\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "First we need to instantiate the ERDDAP URL constructor for a server. In this\n", - "example we will use\n", - "[https://upwell.pfeg.noaa.gov/erddap](https://upwell.pfeg.noaa.gov/erddap/index.html).\n" + "```\n", + "\n", + "\n", + "First we need to instantiate the ERDDAP URL constructor for a server.\n", + "In this example we will use\n", + "[https://erddap.ifremer.fr/erddap](https://erddap.ifremer.fr/erddap/index.html)." ] }, { @@ -38,7 +34,7 @@ "\n", "\n", "e = ERDDAP(\n", - " server=\"UAF\", # NOAA UAF (Unified Access Framework)\n", + " server=\"ifremer\",\n", " protocol=\"tabledap\",\n", " response=\"csv\",\n", ")" @@ -59,15 +55,15 @@ "metadata": {}, "outputs": [], "source": [ - "e.dataset_id = \"scrippsGliders\"\n", + "e.dataset_id = \"OceanGlidersGDACTrajectories\"\n", "\n", "e.variables = [\n", - " \"depth\",\n", + " \"time\",\n", " \"latitude\",\n", " \"longitude\",\n", - " \"salinity\",\n", - " \"temperature\",\n", - " \"time\",\n", + " \"PRES\",\n", + " \"PSAL\",\n", + " \"TEMP\",\n", "]\n", "\n", "e.constraints = {\n", @@ -101,17 +97,13 @@ "constraints = {\n", " \"time>=\": \"now-14days\",\n", " \"time<=\": \"now-7days\",\n", - " \"latitude>=\": \"min(latitude)+5\",\n", - " \"latitude<=\": \"max(latitude)-5\",\n", - " \"longitude>=\": \"min(longitude)+5\",\n", " \"longitude<=\": \"min(longitude)+10\",\n", - " \"depth>=\": \"min(depth)+5\",\n", - " \"depth<=\": \"max(depth)-40\",\n", + " \"longitude>=\": \"min(longitude)\",\n", "}\n", "\n", "\n", "url = e.get_download_url(\n", - " response=\"html\",\n", + " response=\"htmlTable\",\n", " constraints=constraints,\n", ")\n", "\n", @@ -133,14 +125,10 @@ "outputs": [], "source": [ "constraints = {\n", - " \"time>=\": \"2021-09-11T23:26:55Z\",\n", - " \"time<=\": \"2021-09-18T23:26:55Z\",\n", - " \"latitude>=\": -7.52,\n", - " \"latitude<=\": 46.67,\n", - " \"longitude>=\": -121.48,\n", - " \"longitude<=\": -116.48,\n", - " \"depth>=\": -3.82,\n", - " \"depth<=\": 1001.59,\n", + " \"time>=\": \"2023-02-06T19:12:32Z\",\n", + " \"time<=\": \"2023-02-13T19:12:32Z\",\n", + " \"longitude>=\": -149.60,\n", + " \"longitude<=\": -139.60,\n", "}\n", "\n", "url = e.get_download_url(\n", @@ -155,7 +143,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "We can search all datasets with a set of constraints by setting `dataset_id` to `\"allDatasets\"`. Note that these variables are different than the ones available at the individual dataset level. For a reference of the possible variables to query all datasets see the `/erddap//allDatasets.html` page, like [this one](https://upwell.pfeg.noaa.gov/erddap/tabledap/allDatasets.html) for the UAF ERDDAP server." + "We can search all datasets with a set of constraints by setting `dataset_id` to `\"allDatasets\"`. Note that these variables are different than the ones available at the individual dataset level. For a reference of the possible variables to query all datasets see the `/erddap//allDatasets.html` page, like [this one](https://erddap.ifremer.fr/erddap/tabledap/allDatasets.html) for the Ifremer ERDDAP server." ] }, { @@ -177,9 +165,9 @@ " \"maxLongitude\",\n", "]\n", "\n", + "\n", "e.constraints = {\n", - " \"minTime>=\": \"2010-01-01T00:00:00Z\",\n", - " \"maxTime<=\": \"2011-01-01T00:00:00Z\",\n", + " \"minTime>=\": \"1997-01-01T00:00:00Z\",\n", "}\n", "\n", "\n", @@ -222,7 +210,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.7" + "version": "3.11.0" } }, "nbformat": 4, From 29e607d3058a0e4c6838eadf1a1047e8d12f7c16 Mon Sep 17 00:00:00 2001 From: Filipe Fernandes Date: Mon, 20 Feb 2023 16:26:59 -0300 Subject: [PATCH 5/7] ignore variable spelling --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6d0452da..5246c023 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -63,7 +63,7 @@ repos: .*\.yaml )$ args: - - --quiet-level=2 + - --ignore-words-list=pres - repo: https://github.com/asottile/pyupgrade rev: v3.3.1 From 5456c77b446b0f590234f23785896593cdf88305 Mon Sep 17 00:00:00 2001 From: Filipe Fernandes Date: Mon, 20 Feb 2023 16:36:41 -0300 Subject: [PATCH 6/7] we don't need the version here --- .github/workflows/deploy-docs.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/deploy-docs.yml b/.github/workflows/deploy-docs.yml index 56602d5a..beb9bf6e 100644 --- a/.github/workflows/deploy-docs.yml +++ b/.github/workflows/deploy-docs.yml @@ -32,10 +32,6 @@ jobs: micromamba activate TEST python -m pip install -e . --no-deps --force-reinstall - - name: Get the version - id: get_version - run: echo ::set-output name=VERSION::$(python setup.py --version) - - name: Build documentation shell: bash -l {0} run: | From db33501763d83b8a545862380e129ab5ed115716 Mon Sep 17 00:00:00 2001 From: Filipe Fernandes Date: Mon, 20 Feb 2023 16:48:29 -0300 Subject: [PATCH 7/7] another server change --- notebooks/03-advanced_search.ipynb | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/notebooks/03-advanced_search.ipynb b/notebooks/03-advanced_search.ipynb index 0c4a5b05..48439259 100644 --- a/notebooks/03-advanced_search.ipynb +++ b/notebooks/03-advanced_search.ipynb @@ -19,7 +19,7 @@ "from erddapy import ERDDAP\n", "\n", "\n", - "e = ERDDAP(server=\"https://upwell.pfeg.noaa.gov/erddap\", protocol=\"griddap\")" + "e = ERDDAP(server=\"https://pae-paha.pacioos.hawaii.edu/erddap\", protocol=\"griddap\")" ] }, { @@ -37,7 +37,7 @@ "source": [ "import pandas as pd\n", "\n", - "search_for = \"HFRadar\"\n", + "search_for = \"etopo\"\n", "\n", "url = e.get_search_url(search_for=search_for, response=\"csv\")\n", "\n", @@ -57,7 +57,7 @@ "metadata": {}, "outputs": [], "source": [ - "search_for = \"HFRadar 2km\"\n", + "search_for = \"etopo5\"\n", "\n", "url = e.get_search_url(search_for=search_for, response=\"csv\")\n", "\n", @@ -77,7 +77,7 @@ "metadata": {}, "outputs": [], "source": [ - "search_for = \"HFRadar -EXPERIMENTAL\"\n", + "search_for = \"etopo5 -lon180\"\n", "\n", "url = e.get_search_url(search_for=search_for, response=\"csv\")\n", "\n", @@ -99,7 +99,7 @@ }, "outputs": [], "source": [ - "search_for = \"wind speed\"\n", + "search_for = \"ocean bathymetry\"\n", "\n", "url = e.get_search_url(search_for=search_for, response=\"csv\")\n", "\n", @@ -120,7 +120,7 @@ "metadata": {}, "outputs": [], "source": [ - "search_for = '\"wind speed\"'\n", + "search_for = '\"ocean bathymetry\"'\n", "\n", "url = e.get_search_url(search_for=search_for, response=\"csv\")\n", "\n", @@ -131,10 +131,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Slightly less but still quite a lot!\n", - "\n", "Another common search operation would be to search multiple servers instead of\n", - "only one. In erddapy we can achieve that with `search_servers`:\n" + "only one. In erddapy we can achieve that with `search_servers`:" ] }, { @@ -245,7 +243,7 @@ }, "gist_id": "3f0f25b13ade0c64c84607bd92903d1b", "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -259,7 +257,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.4" + "version": "3.11.0" } }, "nbformat": 4,