Merge pull request #9 from ClimateImpactLab/feature/update-results

update to SLIIDERS v1.2

Author: bolliger32

HISTORY.rst

@@ -1,11 +1,12 @@
 History
 =======
 
-v1.2 (unreleased)
--------------------
+v1.2.0
+------
 * Point `data-acquisition.ipynb` to updated Zenodo deposit that fixes the dtype of `subsets` variable in `diaz2016_inputs_raw.zarr.zip` to be bool rather than int8
 * Variable name bugfix in `data-acquisition.ipynb`
 * Add netcdf versions of SLIIDERS and the pyCIAM results to `upload-zenodo.ipynb`
+* Update results in Zenodo record to use SLIIDERS v1.2
 
 v1.1.2
 ------

notebooks/models/create-surge-lookup-tables.ipynb

@@ -30,8 +30,23 @@
   {
    "cell_type": "code",
    "execution_count": 1,
+   "id": "6b183aa3-366a-4f5f-bb95-5ef4bf48d438",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "%load_ext autoreload\n",
+    "%autoreload 2"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
    "id": "cd4b2928-65a0-4733-9c71-fbefa85590be",
-   "metadata": {},
+   "metadata": {
+    "tags": []
+   },
    "outputs": [],
    "source": [
     "import sys\n",
@@ -41,16 +56,26 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": 3,
    "id": "afd03880-8e46-4d81-867d-bfb6505ea788",
-   "metadata": {},
+   "metadata": {
+    "tags": []
+   },
    "outputs": [
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "/srv/conda/envs/notebook/lib/python3.9/site-packages/dask_gateway/client.py:21: FutureWarning: format_bytes is deprecated and will be removed in a future release. Please use dask.utils.format_bytes instead.\n",
-      "  from distributed.utils import LoopRunner, format_bytes\n"
+      "/srv/conda/envs/notebook/lib/python3.10/site-packages/google/cloud/storage/transfer_manager.py:30: UserWarning: The module `transfer_manager` is a preview feature. Functionality and API may change. This warning will be removed in a future release.\n",
+      "  warnings.warn(\n",
+      "/home/jovyan/git-repos/pyCIAM-public/notebooks/models/../shared.py:3: UserWarning: Shapely 2.0 is installed, but because PyGEOS is also installed, GeoPandas will still use PyGEOS by default for now. To force to use and test Shapely 2.0, you have to set the environment variable USE_PYGEOS=0. You can do this before starting the Python process, or in your code before importing geopandas:\n",
+      "\n",
+      "import os\n",
+      "os.environ['USE_PYGEOS'] = '0'\n",
+      "import geopandas\n",
+      "\n",
+      "In a future release, GeoPandas will switch to using Shapely by default. If you are using PyGEOS directly (calling PyGEOS functions on geometries from GeoPandas), this will then stop working and you are encouraged to migrate from PyGEOS to Shapely 2.0 (https://shapely.readthedocs.io/en/latest/migration_pygeos.html).\n",
+      "  import geopandas as gpd\n"
      ]
     }
    ],
@@ -75,7 +100,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 4,
    "id": "47979c48-7fff-49b4-b445-ea6bf746fa37",
    "metadata": {
     "tags": [
@@ -86,18 +111,20 @@
    "source": [
     "# When running on larger/scalable dask cluster, may wish to specify number of workers\n",
     "# Default is LocalCluster which will use the number of CPUs available on local machine\n",
-    "# N_WORKERS_MIN = 7\n",
-    "# N_WORKERS_MAX = 700\n",
+    "N_WORKERS_MIN = 7\n",
+    "N_WORKERS_MAX = 700\n",
     "SEG_CHUNKSIZE = 5\n",
     "\n",
     "PARAMS = pd.read_json(PATH_PARAMS)[\"values\"]"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": 5,
    "id": "6104bced",
-   "metadata": {},
+   "metadata": {
+    "tags": []
+   },
    "outputs": [],
    "source": [
     "DMF_I = getattr(damage_funcs, PARAMS.dmf + \"_i\")\n",
@@ -106,70 +133,31 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": 6,
    "id": "dbb04b4a-14b8-4403-ad33-88bfe71bd8fc",
-   "metadata": {},
+   "metadata": {
+    "tags": []
+   },
    "outputs": [
     {
      "data": {
-      "text/html": [
-       "<div>\n",
-       "    <div style=\"width: 24px; height: 24px; background-color: #e1e1e1; border: 3px solid #9D9D9D; border-radius: 5px; position: absolute;\"> </div>\n",
-       "    <div style=\"margin-left: 48px;\">\n",
-       "        <h3 style=\"margin-bottom: 0px;\">Client</h3>\n",
-       "        <p style=\"color: #9D9D9D; margin-bottom: 0px;\">Client-f992a2c6-c6d1-11ed-862e-665816eadc72</p>\n",
-       "        <table style=\"width: 100%; text-align: left;\">\n",
-       "\n",
-       "        <tr>\n",
-       "        \n",
-       "            <td style=\"text-align: left;\"><strong>Connection method:</strong> Cluster object</td>\n",
-       "            <td style=\"text-align: left;\"><strong>Cluster type:</strong> dask_gateway.GatewayCluster</td>\n",
-       "        \n",
-       "        </tr>\n",
-       "\n",
-       "        \n",
-       "            <tr>\n",
-       "                <td style=\"text-align: left;\">\n",
-       "                    <strong>Dashboard: </strong> <a href=\"/services/dask-gateway/clusters/daskhub-dev.e0f56d1ca0ae4a42b9352ad6b7204454/status\" target=\"_blank\">/services/dask-gateway/clusters/daskhub-dev.e0f56d1ca0ae4a42b9352ad6b7204454/status</a>\n",
-       "                </td>\n",
-       "                <td style=\"text-align: left;\"></td>\n",
-       "            </tr>\n",
-       "        \n",
-       "\n",
-       "        </table>\n",
-       "\n",
-       "        \n",
-       "            <details>\n",
-       "            <summary style=\"margin-bottom: 20px;\"><h3 style=\"display: inline;\">Cluster Info</h3></summary>\n",
-       "            <div style='background-color: #f2f2f2; display: inline-block; padding: 10px; border: 1px solid #999999;'>\n",
-       "  <h3>GatewayCluster</h3>\n",
-       "  <ul>\n",
-       "    <li><b>Name: </b>daskhub-dev.e0f56d1ca0ae4a42b9352ad6b7204454\n",
-       "    <li><b>Dashboard: </b><a href='/services/dask-gateway/clusters/daskhub-dev.e0f56d1ca0ae4a42b9352ad6b7204454/status' target='_blank'>/services/dask-gateway/clusters/daskhub-dev.e0f56d1ca0ae4a42b9352ad6b7204454/status</a>\n",
-       "  </ul>\n",
-       "</div>\n",
-       "\n",
-       "            </details>\n",
-       "        \n",
-       "\n",
-       "    </div>\n",
-       "</div>"
-      ],
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "fcadb94377b4430d9c2964ac9c808c2a",
+       "version_major": 2,
+       "version_minor": 0
+      },
       "text/plain": [
-       "<Client: 'tls://10.4.102.2:8786' processes=0 threads=0, memory=0 B>"
+       "VBox(children=(HTML(value='<h2>GatewayCluster</h2>'), HBox(children=(HTML(value='\\n<div>\\n<style scoped>\\n    …"
       ]
      },
-     "execution_count": 5,
      "metadata": {},
-     "output_type": "execute_result"
+     "output_type": "display_data"
     }
    ],
    "source": [
-    "client = start_dask_cluster(\n",
-    "    # n_workers_min=N_WORKERS_MIN,\n",
-    "    # n_workers_max=N_WORKERS_MAX,\n",
-    ")\n",
-    "client"
+    "client, cluster = start_dask_cluster()\n",
+    "cluster.adapt(minimum=N_WORKERS_MIN, maximum=N_WORKERS_MAX)\n",
+    "cluster"
    ]
   },
   {
@@ -182,7 +170,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": 7,
    "id": "1e823775-0d12-4e52-93d1-7e3586913e65",
    "metadata": {
     "tags": []
@@ -205,7 +193,7 @@
     "        start_year=PARAMS.model_start,\n",
     "        slr_0_years=PARAMS.slr_0_year,\n",
     "        client=client,\n",
-    "        # client_kwargs={\"batch_size\": N_WORKERS_MAX},\n",
+    "        client_kwargs={\"batch_size\": N_WORKERS_MAX},\n",
     "        force_overwrite=True,\n",
     "        seg_chunksize=SEG_CHUNKSIZE,\n",
     "        mc_dim=\"quantile\",\n",
@@ -223,9 +211,11 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": 11,
    "id": "f941a509-7035-4105-8b79-88a97e759737",
-   "metadata": {},
+   "metadata": {
+    "tags": []
+   },
    "outputs": [],
    "source": [
     "# ensure completion and close cluster\n",
@@ -237,23 +227,25 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": 12,
    "id": "15563792-1ba0-435a-9981-e2b15eaa8dcd",
-   "metadata": {},
+   "metadata": {
+    "tags": []
+   },
    "outputs": [
     {
      "data": {
       "text/plain": [
        "(None, None)"
       ]
      },
-     "execution_count": 10,
+     "execution_count": 12,
      "metadata": {},
      "output_type": "execute_result"
     }
    ],
    "source": [
-    "client.cluster.close(), client.close()"
+    "cluster.close(), client.close()"
    ]
   }
  ],
@@ -273,7 +265,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.10"
+   "version": "3.10.10"
   },
   "widgets": {
    "application/vnd.jupyter.widget-state+json": {