Skip to content

Commit dffaad2

Browse files
ray-chewray-chew
committed
(#15) Fresh HPC install
1 parent ad1a4cc commit dffaad2

File tree

5 files changed

+240
-10
lines changed

5 files changed

+240
-10
lines changed

‎.gitignore‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,3 +23,7 @@ manuscript/*
2323
first_revision/*
2424
outputs/*
2525
local_archive/*
26+
27+
# Local configuration (never commit!)
28+
pycsa/local_paths.py
29+
setup_paths_local.sh

‎pycsa/local_paths.py.template‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
"""
2+
Template for local paths configuration.
3+
4+
To use:
5+
1. Copy this file to local_paths.py: cp local_paths.py.template local_paths.py
6+
2. Edit local_paths.py with your actual paths
7+
3. Never commit local_paths.py (it's in .gitignore)
8+
9+
Environment variables (optional):
10+
You can also set these as environment variables:
11+
- SPEC_APPX_DATA_DIR: Base directory for project data
12+
- SPEC_APPX_OUTPUT_DIR: Output directory
13+
- SPEC_APPX_MERIT_DIR: MERIT data directory
14+
- SPEC_APPX_REMA_DIR: REMA data directory
15+
- SPEC_APPX_ETOPO_DIR: ETOPO data directory
16+
"""
17+
18+
import os
19+
from pathlib import Path
20+
from pycsa import var
21+
22+
paths = var.obj()
23+
24+
# Get base directories from environment or use defaults
25+
data_dir = os.getenv('SPEC_APPX_DATA_DIR', '/path/to/data')
26+
output_dir = os.getenv('SPEC_APPX_OUTPUT_DIR', '/path/to/outputs')
27+
merit_dir = os.getenv('SPEC_APPX_MERIT_DIR', '/path/to/MERIT')
28+
rema_dir = os.getenv('SPEC_APPX_REMA_DIR', '/path/to/REMA')
29+
etopo_dir = os.getenv('SPEC_APPX_ETOPO_DIR', '/path/to/etopo_15s')
30+
31+
# Project data paths
32+
paths.compact_grid = os.path.join(data_dir, "icon_compact.nc")
33+
paths.compact_topo = os.path.join(data_dir, "topo_compact.nc")
34+
paths.icon_grid = os.path.join(data_dir, "icon_grid_0012_R02B04_G_linked.nc")
35+
36+
# Output path
37+
paths.output = os.path.join(output_dir, "global_run/")
38+
39+
# External data sources
40+
paths.merit = merit_dir
41+
paths.rema = rema_dir
42+
paths.etopo = etopo_dir

‎pyproject.toml‎

Lines changed: 9 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -3,17 +3,16 @@ name = "pyCSA"
33
version = "0.95.1"
44

55
dependencies = [
6-
"Cartopy==0.21.1",
7-
"h5py==3.9.0",
8-
"ipython==8.12.3",
9-
"matplotlib==3.7.2",
10-
"netCDF4==1.6.5",
6+
"Cartopy==0.25.0",
7+
"h5py==3.15.1",
8+
"matplotlib==3.10.7",
9+
"netCDF4==1.7.3",
1110
"noise==1.2.2",
12-
"numba==0.57.1",
13-
"numpy==1.24.3",
14-
"pandas==2.0.3",
15-
"scikit_learn==1.3.0",
16-
"scipy==1.12.0",
11+
"numba==0.62.1",
12+
"numpy==2.2.6",
13+
"pandas==2.3.3",
14+
"scipy==1.15.3",
15+
"tqdm>=4.66.0",
1716
]
1817

1918
[project.optional-dependencies]

‎scripts/download_etopo_15s.sh‎

Lines changed: 149 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,149 @@
1+
#!/bin/bash
2+
# Script to download ETOPO 2022 15 arc-second elevation data
3+
# Data source: NOAA National Centers for Environmental Information
4+
# DOI: 10.25921/fd45-gt74
5+
6+
set -e # Exit on error
7+
8+
# Configuration
9+
# NOTE: Use 15s_surface_elev_netcdf for GLOBAL coverage (288 tiles)
10+
# Use 15s_bed_elev_netcdf for POLAR bedrock only (60 tiles, high latitudes only)
11+
DATA_TYPE="${ETOPO_DATA_TYPE:-surface}" # 'surface' or 'bed'
12+
13+
if [ "$DATA_TYPE" = "bed" ]; then
14+
BASE_URL="https://www.ngdc.noaa.gov/thredds/fileServer/global/ETOPO2022/15s/15s_bed_elev_netcdf"
15+
FILE_SUFFIX="bed"
16+
else
17+
BASE_URL="https://www.ngdc.noaa.gov/thredds/fileServer/global/ETOPO2022/15s/15s_surface_elev_netcdf"
18+
FILE_SUFFIX="surface"
19+
fi
20+
21+
OUTPUT_DIR="${1:-./data/etopo_15s}"
22+
23+
# Create output directory
24+
mkdir -p "$OUTPUT_DIR"
25+
26+
echo "Downloading ETOPO 2022 15 arc-second data to: $OUTPUT_DIR"
27+
echo "Data type: $DATA_TYPE (files will end with _${FILE_SUFFIX}.nc)"
28+
echo "=========================================================="
29+
30+
# Option 1: Download ALL tiles (WARNING: This is a LOT of data - 288 tiles!)
31+
# Uncomment the section below to download everything
32+
33+
download_all_tiles() {
34+
echo "Downloading all 288 tiles (this will take a while and use significant disk space)..."
35+
36+
# Northern Hemisphere - High Latitude (N60-N90)
37+
for lat in N60 N75 N90; do
38+
case $lat in
39+
N60)
40+
# N60 tiles
41+
for lon in W180 W165 W150 W135 W120 W105 W090 W075 W060 W045 W030 W015 \
42+
E000 E015 E030 E045 E060 E075 E090 E105 E120 E135 E150 E165; do
43+
wget -c -P "$OUTPUT_DIR" "${BASE_URL}/ETOPO_2022_v1_15s_${lat}${lon}_${FILE_SUFFIX}.nc"
44+
done
45+
;;
46+
N75)
47+
# N75 tiles
48+
for lon in W180 W165 W150 W135 W120 W105 W090 W075 W060 W045 W030 W015 \
49+
E000 E015 E030 E045 E060 E075 E090 E105 E120 E135 E150 E165; do
50+
wget -c -P "$OUTPUT_DIR" "${BASE_URL}/ETOPO_2022_v1_15s_${lat}${lon}_${FILE_SUFFIX}.nc"
51+
done
52+
;;
53+
N90)
54+
# N90 tiles (polar cap - fewer tiles)
55+
for lon in W180 W165 W150 W135 W120 W105 W090 W075 W060 W045 W030 W015 E000; do
56+
wget -c -P "$OUTPUT_DIR" "${BASE_URL}/ETOPO_2022_v1_15s_${lat}${lon}_${FILE_SUFFIX}.nc"
57+
done
58+
;;
59+
esac
60+
done
61+
62+
# Mid Latitudes (N45, N30, N15, N00)
63+
for lat in N45 N30 N15 N00; do
64+
for lon in W180 W165 W150 W135 W120 W105 W090 W075 W060 W045 W030 W015 \
65+
E000 E015 E030 E045 E060 E075 E090 E105 E120 E135 E150 E165; do
66+
wget -c -P "$OUTPUT_DIR" "${BASE_URL}/ETOPO_2022_v1_15s_${lat}${lon}_${FILE_SUFFIX}.nc"
67+
done
68+
done
69+
70+
# Southern Hemisphere (S15, S30, S45, S60, S75)
71+
for lat in S15 S30 S45 S60 S75; do
72+
for lon in W180 W165 W150 W135 W120 W105 W090 W075 W060 W045 W030 W015 \
73+
E000 E015 E030 E045 E060 E075 E090 E105 E120 E135 E150 E165; do
74+
wget -c -P "$OUTPUT_DIR" "${BASE_URL}/ETOPO_2022_v1_15s_${lat}${lon}_${FILE_SUFFIX}.nc"
75+
done
76+
done
77+
}
78+
79+
# Option 2: Download specific region (RECOMMENDED)
80+
download_specific_region() {
81+
echo "Downloading specific region..."
82+
echo "Edit this function to specify your region of interest"
83+
84+
# Example: Download Europe and North Atlantic region
85+
# Modify the lat/lon ranges below for your area of interest
86+
87+
# European region example (45N-75N, 15W-45E)
88+
for lat in N45 N60 N75; do
89+
for lon in W015 E000 E015 E030 E045; do
90+
echo "Downloading tile: ${lat}${lon}"
91+
wget -c -P "$OUTPUT_DIR" "${BASE_URL}/ETOPO_2022_v1_15s_${lat}${lon}_${FILE_SUFFIX}.nc"
92+
done
93+
done
94+
}
95+
96+
# Option 3: Download specific tiles by name
97+
download_specific_tiles() {
98+
# Add specific tile names to this array
99+
local tiles=(
100+
"N45W120" # US West Coast
101+
"N45W105" # US Rockies
102+
"N45E000" # Western Europe
103+
# Add more tiles as needed
104+
)
105+
106+
echo "Downloading ${#tiles[@]} specific tiles..."
107+
for tile in "${tiles[@]}"; do
108+
echo "Downloading tile: $tile"
109+
wget -c -P "$OUTPUT_DIR" "${BASE_URL}/ETOPO_2022_v1_15s_${tile}_${FILE_SUFFIX}.nc"
110+
done
111+
}
112+
113+
# Main execution
114+
echo ""
115+
echo "Choose download option:"
116+
echo "1) Download all tiles (WARNING: ~50-100 GB total)"
117+
echo "2) Download specific region (recommended - edit script first)"
118+
echo "3) Download specific tiles (edit script to list tiles)"
119+
echo ""
120+
read -p "Enter choice [1-3]: " choice
121+
122+
case $choice in
123+
1)
124+
read -p "Are you sure? This will download ~288 files. (yes/no): " confirm
125+
if [ "$confirm" = "yes" ]; then
126+
download_all_tiles
127+
else
128+
echo "Download cancelled."
129+
exit 0
130+
fi
131+
;;
132+
2)
133+
download_specific_region
134+
;;
135+
3)
136+
download_specific_tiles
137+
;;
138+
*)
139+
echo "Invalid choice. Exiting."
140+
exit 1
141+
;;
142+
esac
143+
144+
echo ""
145+
echo "Download complete!"
146+
echo "Files saved to: $OUTPUT_DIR"
147+
echo ""
148+
echo "To use these files, update your paths in setup_paths.sh or local_paths.py"
149+
echo " export SPEC_APPX_ETOPO_DIR=\"$OUTPUT_DIR\""

‎setup_paths.sh‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
#!/bin/bash
2+
# Setup script for local paths
3+
# Usage: source setup_paths.sh
4+
5+
# Detect if we're on HPC or local machine
6+
if [[ -n "$SLURM_JOB_ID" ]] || [[ -n "$PBS_JOBID" ]] || [[ $(hostname) == *"hpc"* ]]; then
7+
echo "Detected HPC environment"
8+
export SPEC_APPX_ENV="HPC"
9+
10+
# HPC paths - UPDATE THESE FOR YOUR HPC
11+
export SPEC_APPX_DATA_DIR="${HOME}/pyCSA/data"
12+
export SPEC_APPX_OUTPUT_DIR="${HOME}/pyCSA/outputs"
13+
export SPEC_APPX_MERIT_DIR="${HOME}/pyCSA/data/MERIT"
14+
export SPEC_APPX_REMA_DIR="${HOME}/pyCSA/data/REMA"
15+
export SPEC_APPX_ETOPO_DIR="${HOME}/pyCSA/data/etopo_15s"
16+
else
17+
echo "Detected local environment"
18+
export SPEC_APPX_ENV="LOCAL"
19+
20+
# Local paths - UPDATE THESE FOR YOUR LOCAL MACHINE
21+
export SPEC_APPX_DATA_DIR="${HOME}/pyCSA/data"
22+
export SPEC_APPX_OUTPUT_DIR="${HOME}/pyCSA/outputs"
23+
export SPEC_APPX_MERIT_DIR="${HOME}/pyCSA/data/MERIT"
24+
export SPEC_APPX_REMA_DIR="${HOME}/pyCSA/data/REMA"
25+
export SPEC_APPX_ETOPO_DIR="${HOME}/pyCSA/data/etopo_15s"
26+
fi
27+
28+
echo "Environment: $SPEC_APPX_ENV"
29+
echo "Data directory: $SPEC_APPX_DATA_DIR"
30+
echo "Output directory: $SPEC_APPX_OUTPUT_DIR"
31+
32+
# Create local_paths.py if it doesn't exist
33+
if [ ! -f "pycsa/local_paths.py" ]; then
34+
echo "Creating pycsa/local_paths.py from template..."
35+
cp pycsa/local_paths.py.template pycsa/local_paths.py
36+
fi

0 commit comments

Comments
 (0)