Skip to content

Commit

Permalink
📦 Tile the new Arthern snow accumulation data with other rasters
Browse files Browse the repository at this point in the history 
Finally we have Arthern Accumulation, and at a spatial resolution of 1000m just like BEDMAP2! Putting it down as conditional input W3, after REMA W1 and MEASURES Ice Velocity W2. We are down from 2499 to 2347 tiles due to the new grounding line restriction from #147, though this subset should be scientifically better. Time to spin up our GPUs for some neural network training!
  • Loading branch information
@weiji14
weiji14 committed May 29, 2019
1 parent 1bfca80 commit 63b3ef1
Show file tree
Hide file tree
Showing 2 changed files with 76 additions and 33 deletions.
95 changes: 62 additions & 33 deletions data_prep.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1084,7 +1084,7 @@
{
"data": {
"text/plain": [
"<matplotlib.axes._subplots.AxesSubplot at 0x7f89ce6eb940>"
"<matplotlib.axes._subplots.AxesSubplot at 0x7f40de201a90>"
]
},
"execution_count": 21,
Expand Down Expand Up @@ -1144,7 +1144,7 @@
},
{
"cell_type": "code",
"execution_count": 21,
"execution_count": 23,
"metadata": {
"lines_to_next_cell": 2
},
Expand Down Expand Up @@ -1247,7 +1247,7 @@
},
{
"cell_type": "code",
"execution_count": 22,
"execution_count": 24,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -1269,21 +1269,21 @@
},
{
"cell_type": "code",
"execution_count": 23,
"execution_count": 25,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Tiling: highres/2010tr.nc\n",
"Tiling: highres/201x_Antarctica_Basler.nc\n",
"Tiling: highres/20xx_Antarctica_DC8.nc\n",
"Tiling: highres/20xx_Antarctica_TO.nc\n",
"Tiling: highres/WISE_ISODYN_RadarByFlight.nc\n",
"Tiling: highres/bed_WGS84_grid.nc\n",
"Tiling: highres/istarxx.nc\n",
"(2499, 1, 32, 32) float32\n"
"Tiling: highres/2010tr.nc ... done!\n",
"Tiling: highres/201x_Antarctica_Basler.nc ... done!\n",
"Tiling: highres/20xx_Antarctica_DC8.nc ... done!\n",
"Tiling: highres/20xx_Antarctica_TO.nc ... done!\n",
"Tiling: highres/WISE_ISODYN_RadarByFlight.nc ... done!\n",
"Tiling: highres/bed_WGS84_grid.nc ... done!\n",
"Tiling: highres/istarxx.nc ... done!\n",
"(2347, 1, 32, 32) float32\n"
]
}
],
Expand All @@ -1305,15 +1305,15 @@
},
{
"cell_type": "code",
"execution_count": 24,
"execution_count": 26,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Tiling: lowres/bedmap2_bed.tif\n",
"(2499, 1, 10, 10) float32\n"
"Tiling: lowres/bedmap2_bed.tif ... done!\n",
"(2347, 1, 10, 10) float32\n"
]
}
],
Expand All @@ -1333,15 +1333,15 @@
},
{
"cell_type": "code",
"execution_count": 25,
"execution_count": 27,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Tiling: misc/REMA_100m_dem.tif\n",
"(2499, 1, 100, 100) float32\n"
"Tiling: misc/REMA_100m_dem.tif ... done!\n",
"(2347, 1, 100, 100) float32\n"
]
}
],
Expand All @@ -1357,7 +1357,7 @@
},
{
"cell_type": "code",
"execution_count": 26,
"execution_count": 28,
"metadata": {},
"outputs": [
{
Expand All @@ -1379,15 +1379,15 @@
},
{
"cell_type": "code",
"execution_count": 27,
"execution_count": 29,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Tiling: misc/MEaSUREs_IceFlowSpeed_450m.tif ... done!\n",
"(2499, 1, 20, 20) float32\n"
"(2347, 1, 20, 20) float32\n"
]
}
],
Expand All @@ -1402,6 +1402,29 @@
"print(measuresiceflow.shape, measuresiceflow.dtype)"
]
},
{
"cell_type": "code",
"execution_count": 30,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Tiling: misc/Arthern_accumulation_bedmap2_grid1.tif ... done!\n",
"(2347, 1, 10, 10) float32\n"
]
}
],
"source": [
"accumulation = selective_tile(\n",
" filepath=\"misc/Arthern_accumulation_bedmap2_grid1.tif\",\n",
" window_bounds=window_bounds_concat,\n",
" padding=1000,\n",
")\n",
"print(accumulation.shape, accumulation.dtype)"
]
},
{
"cell_type": "markdown",
"metadata": {},
Expand All @@ -1419,13 +1442,14 @@
},
{
"cell_type": "code",
"execution_count": 27,
"execution_count": 31,
"metadata": {},
"outputs": [],
"source": [
"os.makedirs(name=\"model/train\", exist_ok=True)\n",
"np.save(file=\"model/train/W1_data.npy\", arr=rema)\n",
"np.save(file=\"model/train/W2_data.npy\", arr=measuresiceflow)\n",
"np.save(file=\"model/train/W3_data.npy\", arr=accumulation)\n",
"np.save(file=\"model/train/X_data.npy\", arr=lores)\n",
"np.save(file=\"model/train/Y_data.npy\", arr=hires)"
]
Expand All @@ -1441,7 +1465,7 @@
},
{
"cell_type": "code",
"execution_count": 28,
"execution_count": 32,
"metadata": {},
"outputs": [
{
Expand All @@ -1457,7 +1481,7 @@
"name": "stdin",
"output_type": "stream",
"text": [
"Enter the code from the webpage: eyJpZCI6ICIyOWI4YzUyNS1lZmM1LTQ5NTItOGQ4Yy03NzQyYTg1YmI1MmEiLCAiY29kZSI6ICI2NjkyMzQxMC03ZDk0LTRlZjQtYjQ2NS03MWRhZDU1NTkxYzMifQ==\n"
"Enter the code from the webpage: eyJpZCI6ICIyOWI4YzUyNS1lZmM1LTQ5NTItOGQ4Yy03NzQyYTg1YmI1MmEiLCAiY29kZSI6ICI0MTZlOWUxNS02MWI1LTQ1MWYtOTU3My0xNmMwMzAxNjYxZmEifQ==\n"
]
}
],
Expand All @@ -1467,13 +1491,14 @@
},
{
"cell_type": "code",
"execution_count": 29,
"execution_count": 33,
"metadata": {},
"outputs": [],
"source": [
"# Tiled datasets for training neural network\n",
"quilt.build(package=\"weiji14/deepbedmap/model/train/W1_data\", path=rema)\n",
"quilt.build(package=\"weiji14/deepbedmap/model/train/W2_data\", path=measuresiceflow)\n",
"quilt.build(package=\"weiji14/deepbedmap/model/train/W3_data\", path=accumulation)\n",
"quilt.build(package=\"weiji14/deepbedmap/model/train/X_data\", path=lores)\n",
"quilt.build(package=\"weiji14/deepbedmap/model/train/Y_data\", path=hires)\n",
"\n",
Expand All @@ -1491,12 +1516,16 @@
"quilt.build(\n",
" package=\"weiji14/deepbedmap/misc/MEaSUREs_IceFlowSpeed_450m\",\n",
" path=\"misc/MEaSUREs_IceFlowSpeed_450m.tif\",\n",
")\n",
"quilt.build(\n",
" package=\"weiji14/deepbedmap/misc/Arthern_accumulation_bedmap2_grid1\",\n",
" path=\"misc/Arthern_accumulation_bedmap2_grid1.tif\",\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 30,
"execution_count": 34,
"metadata": {},
"outputs": [
{
Expand All @@ -1510,41 +1539,41 @@
"name": "stderr",
"output_type": "stream",
"text": [
" 0%| | 0.00/6.48G [00:00<?, ?B/s]"
" 0%| | 0.00/6.74G [00:00<?, ?B/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Uploading 11 fragments (6481462251 bytes)...\n"
"Uploading 13 fragments (6739537494 bytes)...\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
" 26%|██▌ | 1.69G/6.48G [00:01<1:29:17, 893kB/s]"
" 94%|█████████▍| 6.37G/6.74G [00:01<6:35:49, 15.7kB/s] "
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Fragment 1f66fe557ce079c063597f0b04d15862f67af2c9dd4f286801851e0c71f0e869 already uploaded; skipping.\n",
"Fragment f750893861a1a268c8ffe0ba7db36c933223bbf5fcbb786ecef3f052b20f9b8a already uploaded; skipping.\n",
"Fragment f1f660d1287225c30b8b2cbf2a727283d807a1ee443153519cbf407a08937965 already uploaded; skipping.\n",
"Fragment f750893861a1a268c8ffe0ba7db36c933223bbf5fcbb786ecef3f052b20f9b8a already uploaded; skipping.\n",
"Fragment 1f66fe557ce079c063597f0b04d15862f67af2c9dd4f286801851e0c71f0e869 already uploaded; skipping.\n",
"Fragment ca9c41a8dd56097e40865d2e65c65d299c22fc17608ddb6c604c532a69936307 already uploaded; skipping.\n",
"Fragment 28e2ca7656d61b0bc7f8f8c1db41914023e0cab1634e0ee645f38a87d894b416 already uploaded; skipping.\n",
"Fragment 4a4efc3a84204c3d67887e8d7fa1186467b51e696451f2832ebbea3ca491c8a8 already uploaded; skipping.\n",
"Fragment ca9c41a8dd56097e40865d2e65c65d299c22fc17608ddb6c604c532a69936307 already uploaded; skipping.\n",
"Fragment 80c9fa41ccc69be1d2cd4a367d56168321d1079e7260a1996089810db25172f6 already uploaded; skipping.\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|██████████| 6.48G/6.48G [00:10<00:00, 637MB/s] \n"
"100%|██████████| 6.74G/6.74G [00:16<00:00, 418MB/s] \n"
]
},
{
Expand Down
14 changes: 14 additions & 0 deletions data_prep.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -714,6 +714,14 @@ def selective_tile(
)
print(measuresiceflow.shape, measuresiceflow.dtype)

# %%
accumulation = selective_tile(
filepath="misc/Arthern_accumulation_bedmap2_grid1.tif",
window_bounds=window_bounds_concat,
padding=1000,
)
print(accumulation.shape, accumulation.dtype)

# %% [markdown]
# ## 4. Save the arrays
#
Expand All @@ -729,6 +737,7 @@ def selective_tile(
os.makedirs(name="model/train", exist_ok=True)
np.save(file="model/train/W1_data.npy", arr=rema)
np.save(file="model/train/W2_data.npy", arr=measuresiceflow)
np.save(file="model/train/W3_data.npy", arr=accumulation)
np.save(file="model/train/X_data.npy", arr=lores)
np.save(file="model/train/Y_data.npy", arr=hires)

Expand All @@ -744,6 +753,7 @@ def selective_tile(
# Tiled datasets for training neural network
quilt.build(package="weiji14/deepbedmap/model/train/W1_data", path=rema)
quilt.build(package="weiji14/deepbedmap/model/train/W2_data", path=measuresiceflow)
quilt.build(package="weiji14/deepbedmap/model/train/W3_data", path=accumulation)
quilt.build(package="weiji14/deepbedmap/model/train/X_data", path=lores)
quilt.build(package="weiji14/deepbedmap/model/train/Y_data", path=hires)

Expand All @@ -762,6 +772,10 @@ def selective_tile(
package="weiji14/deepbedmap/misc/MEaSUREs_IceFlowSpeed_450m",
path="misc/MEaSUREs_IceFlowSpeed_450m.tif",
)
quilt.build(
package="weiji14/deepbedmap/misc/Arthern_accumulation_bedmap2_grid1",
path="misc/Arthern_accumulation_bedmap2_grid1.tif",
)

# %%
quilt.push(package="weiji14/deepbedmap", is_public=True)

0 comments on commit 63b3ef1

Please sign in to comment.