Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update GitHub action to run V4 #695

Merged
merged 10 commits into from
Nov 15, 2023
23 changes: 21 additions & 2 deletions .github/workflows/ubuntu-latest.yml
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,26 @@ jobs:
run: |
./compiler.sh no-e

- name: Run Test
- name: Run V3 Test
run: |
cd test/LowerColorado_TX
python -m nwm_routing -f -V3 test_AnA.yaml
python -m nwm_routing -f -V3 test_AnA.yaml
cd ../..

- name: Run V4 Test on NHD
run: |
cd test/LowerColorado_TX
python -m nwm_routing -f -V4 test_AnA_V4_NHD.yaml
cd ../..

- name: Run V4 Test on HYFeature
run: |
cd test/LowerColorado_TX_v4
python -m nwm_routing -f -V4 test_AnA_V4_HYFeature.yaml
cd ../..

- name: Run V4 Test on HYFeature without DA
run: |
cd test/LowerColorado_TX_v4
python -m nwm_routing -f -V4 test_AnA_V4_HYFeature_noDA.yaml
cd ../..
80 changes: 77 additions & 3 deletions src/troute-network/troute/HYFeaturesNetwork.py
Original file line number Diff line number Diff line change
Expand Up @@ -559,10 +559,9 @@ def preprocess_data_assimilation(self, network):
if usgs_da:
self._waterbody_types_df.loc[self._usgs_lake_gage_crosswalk.index,'reservoir_type'] = 2
if rfc_da:
#FIXME: Temporary fix, read in predefined rfc lake gage crosswalk file for rfc reservoirs.
#FIXME: Temporary fix, load predefined rfc lake gage crosswalk info for rfc reservoirs.
# Replace relevant waterbody_types as type 4.
temp_rfc_file = Path(__file__).parent / 'rfc_lake_gage_crosswalk.csv'
rfc_lake_gage_crosswalk = pd.read_csv(temp_rfc_file)
rfc_lake_gage_crosswalk = get_rfc_lake_gage_crosswalk().reset_index()
self._rfc_lake_gage_crosswalk = rfc_lake_gage_crosswalk[rfc_lake_gage_crosswalk['rfc_lake_id'].isin(self.waterbody_dataframe.index)].set_index('rfc_lake_id')
self._waterbody_types_df.loc[self._rfc_lake_gage_crosswalk.index,'reservoir_type'] = 4
else:
Expand Down Expand Up @@ -886,3 +885,78 @@ def replace_waterbodies_connections(connections, waterbodies):
new_conn[n] = connections[n]

return new_conn, link_lake

def get_rfc_lake_gage_crosswalk():
"""
Temporary solution for loading rfc gage-lake crosswalk information. This prevents having to
read this info from a file (e.g., a .csv file). This will not be needed once this information
is added to the hydrofabric.
"""
rfc_dict = {'rfc_gage_id': {347987: 'KNFC1', 361273: 'IRGC1', 371555: 'HEYO2', 389804: 'CNLO2',
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If this will eventually be an external resource, could we make the temporary fix also an external file and import(e.g., from rfc_lake_gage_crosswalk import rfc_dict), rather than baking the dictionary into this file? It would be easy for this to linger for a while.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These changes have been made.

402142: 'TENO2', 481656: 'NRMO2', 591880: 'HGLO2', 617546: 'PCLO2', 630011: 'BKDO2',
664014: 'HLZU1', 681831: 'FTCO2', 746373: 'PRRK1', 846266: 'BIRO2', 1111211: 'TBLT2',
1148575: 'SKCT2', 1158647: 'JSPT2', 1230577: 'WFRC2', 1233515: 'CBGC2', 1233595: 'WCKC2',
1267710: 'FLWT2', 1267898: 'LART2', 1267968: 'BNBT2', 1269548: 'GPET2', 1291638: 'LVNT2',
1304891: 'BPRT2', 1311881: 'BGMC2', 1312051: 'BLRC2', 1332558: 'TRBC2', 1336910: 'MDCC2',
1445978: 'BDWT2', 1492358: 'LVDT2', 1637751: 'CKDT2', 1748473: 'DWNN6', 2529647: 'SLKM7',
2567012: 'LLET2', 2571688: 'BLNT2', 2613174: 'CNNN6', 2781993: 'WHSC1', 2992128: 'HILK1',
3140356: 'FOSO2', 3169144: 'MTHT2', 3174852: 'RRGC2', 3180118: 'VEGC2', 3192800: 'VIVW4',
3210503: 'RLAT2', 3247342: 'GBRN6', 3251185: 'CRFC2', 3252975: 'BMDC2', 3252989: 'MPSC2',
3318110: 'BNGM1', 3588682: 'SMCT2', 3730511: 'CLIK1', 3745418: 'GLLA4', 3745478: 'DIEA4',
3745526: 'DQDA4', 3806779: 'YGOP1', 3906255: 'SFSU1', 3950979: 'CMCC1', 4132531: 'DAWT2',
4451216: 'LPTT2', 4460822: 'LBRM7', 4467951: 'BEAI1', 4558474: 'BBOI1', 4739601: 'CCDP1',
4875263: 'ELLU1', 4876219: 'JOVU1', 4876263: 'MLSU1', 5278062: 'LCRT2', 5278224: 'LMTT2',
5278562: 'LMVT2', 5292739: 'LFKT2', 5489185: 'HAWT2', 5490089: 'LSWT2', 5494328: 'LMGT2',
5495308: 'LPPT2', 5506926: 'LSFT2', 5511586: 'LCLT2', 5525777: 'HCRT2', 5531274: 'ACTT2',
5541638: 'SMKT2', 5569731: 'SOMT2', 5575089: 'LLST2', 5632093: 'JBTT2', 5635571: 'CLRT2',
5635591: 'CCRT2', 5670421: 'GNGT2', 5670445: 'GGLT2', 5702167: 'TBRT2', 5721267: 'BLKT2',
5722027: 'EVST2', 5734861: 'LBWT2', 5740630: 'LKCT2', 5753298: 'BCRT2', 5756886: 'BUDT2',
6043008: 'WSLO2', 6127281: 'SCIR1', 6479321: 'CTRG1', 6495106: 'CMMG1', 6495140: 'CVLG1',
6597410: 'SAYI4', 6719717: 'STDM1', 6873355: 'GHDP1', 7330556: 'WLSK1', 7387259: 'PTXM7',
7666880: 'CLRM7', 7801259: 'BMTA4', 7827790: 'NMBA4', 7897413: 'BLZI1', 7989989: 'BLBC1',
7990259: 'SGEC1', 7993809: 'EPRC1', 8005383: 'INVC1', 8060079: 'NBBC1', 8060131: 'HLEC1',
8210941: 'NACC1', 8245384: 'LLKC1', 8271433: 'WSDC1', 8306696: 'LAMC1', 8327942: 'BKLT2',
8341399: 'WAGL1', 8347705: 'MYST2', 8357843: 'WRLO2', 8674019: 'PTTV2', 8914219: 'BPRC1',
8932970: 'BCVC1', 8932974: 'PSRC1', 8932994: 'DNRC1', 9051313: 'WCDP1', 9250140: 'WLKN7',
10022260: 'GUUU1', 10038598: 'GRZU1', 10089124: 'CRDU1', 10091588: 'ECWU1', 10114004: 'PLKK1',
10273702: 'OPDU1', 10276260: 'ECCU1', 10327875: 'UTLU1', 10361596: 'WDRF1', 10375440: 'DCRU1',
10742110: 'TOPN2', 10814778: 'MYSU1', 10834778: 'MDLT2', 10922417: 'TIOP1', 11758154: 'GPDN1',
11964599: 'STIU1', 11965847: 'STAU1', 11975087: 'LAAU1', 12076080: 'ORDC1', 12996298: 'SHDP1',
13152353: 'MSQO1', 13152651: 'MLPO1', 13155459: 'KITO1', 13732800: 'SYOT2', 13756271: 'ALTO2',
13811164: 'FSLO2', 14930103: 'SCSC1', 15012277: 'CFWC1', 15021939: 'NIMC1', 15078462: 'BKDL1',
15231972: 'BMDA4', 15252778: 'ARKM6', 17014523: 'FRVC2', 17034145: 'LPVC2', 17065952: 'NHGC1',
17080041: 'LNGC1', 17080601: 'CHVC1', 17080669: 'HETC1', 17539821: 'CRVI4', 17568947: 'PYMC1',
17573833: 'LKPC1', 17609317: 'CCHC1', 17693795: 'ANDC1', 17693799: 'LEXC1', 17693805: 'CADC1',
17693823: 'COYC1', 18014672: 'GRNM6', 18300278: 'BIGW4', 18352167: 'GBFW4', 18880944: 'MTTK1',
19040443: 'REPN1', 19566308: 'CCRA1', 19568056: 'UBRA1', 19626258: 'LBRA1', 19667261: 'OCCT1',
19668381: 'BRDG1', 19674973: 'HADT1', 19925795: 'BCEA4', 19980072: 'MRIT2', 20079044: 'EDRK1',
20332350: 'ELPC1', 20334374: 'SVIC1', 20342929: 'HAWC1', 20438850: 'VDBA3', 20476542: 'SMDA3',
20584396: 'LYMA3', 20603248: 'MVDN2', 20603252: 'PCON2', 20917521: 'MLBK1', 20927390: 'CNGK1',
20943110: 'ECLK1', 20953830: 'TRLK1', 20971700: 'KEYO2', 21000069: 'GSPO2', 21038107: 'KAWO2',
21160241: 'CHNK1', 21318615: 'GCDA3', 21383092: 'BWAA3', 21454020: 'MRFA1', 21514748: 'FLLK1',
21606713: 'MSWC1', 21638156: 'CLBA1', 21772239: 'MFDO2', 21783714: 'CPLO2', 21784832: 'HULO2',
21795481: 'BIGK1', 21980955: 'CPNN6', 22000120: 'DGDA4', 22050039: 'PFTC1', 22071698: 'LKSA3',
22294608: 'HDLN6', 22670080: 'ISAC1', 22697680: 'NADA4', 22741995: 'HIKN6', 22971534: 'HHWM8',
23807386: 'SCOO3', 23927716: 'LOSO3', 23979340: 'HAHW1', 24136807: 'CLEW1', 24161747: 'DRBI1',
24518496: 'PRVO3', 120051895: 'FALT2', 120051930: 'OCEU1', 120051933: 'EAMT2', 120051934: 'GPVT2',
120051936: 'TRNT2', 120051967: 'MCWW4', 120052259: 'NFDA4', 120052268: 'STVC3', 120052275: 'AMIT2',
120052363: 'CTDT2', 120052624: 'TMDC1', 120052757: 'CBDN2', 120052904: 'RSVA3', 120052975: 'TYLA1',
120053021: 'LEWT2', 120053026: 'FRHT2', 120053033: 'LCTT2', 120053035: 'HSJT2', 120053199: 'GBYT2',
120053201: 'PCTT2', 120053204: 'WTYT2', 120053324: 'RADI4', 120053340: 'PNTT2', 120053359: 'PSMT2',
120053388: 'STIT2', 120053389: 'SAGT2', 120053393: 'MSDT2', 120053440: 'BCHP1', 120053442: 'SNRC1',
120053459: 'JCKV2', 120053476: 'STPC1', 120053784: 'TAHC1', 120053836: 'FRAC1', 120053896: 'LBUL1',
120053952: 'HORA3', 130951632: 'LBEC1', 166195943: 'RKWM1', 166248016: 'BRWU1', 166414857: 'ALAT2',
166737717: 'NUDN7', 166755060: 'NHPN7', 166758703: 'WETG1', 166758723: 'FOGG1', 166759840: 'HRTG1',
166890766: 'SWRN6', 166899002: 'KNZP1', 166899099: 'TYGW2', 166899115: 'BRWO1', 166912876: 'FONN7',
166997463: 'DUGT1', 166997474: 'SHDT1', 166997475: 'FPHT1', 166997499: 'WTGT1', 166997501: 'CRKT1',
166997561: 'NRST1', 166997584: 'CHAN7', 166997585: 'NOTG1', 166997625: 'NRMT1', 167121028: 'PELI4',
167166315: 'WPPM7', 167166404: 'ENDM6', 167182317: 'SRDM6', 167182354: 'LCAL1', 167266519: 'KANK1',
167267891: 'MLVK1', 167267899: 'STXM7', 167297255: 'JRLK1', 167299818: 'BSGA4', 167299819: 'GRRA4',
167299904: 'OOLO2', 167299912: 'PENO2', 167299918: 'GIBO2', 167300241: 'MTNO2', 167300255: 'DSNT2',
167300271: 'ARBO2', 167300345: 'AHDA4', 167300374: 'CRLL1', 167484077: 'CHDS1', 167484295: 'CMDP1',
167484347: 'LHDP1', 167484418: 'SWJW2', 167484743: 'MGCO2', 167496465: 'KERV2', 167800866: 'MLFK1',
167801019: 'EUFO2', 941070128: 'SKLO2'}}
rfc_df = pd.DataFrame().from_dict(rfc_dict)
rfc_df.index.name = 'rfc_lake_id'

return rfc_df
2 changes: 1 addition & 1 deletion src/troute-nwm/src/nwm_routing/__main__.py
Original file line number Diff line number Diff line change
Expand Up @@ -216,7 +216,7 @@ def main_v04(argv):
data_assimilation.update_after_compute(run_results, dt*nts)

# TODO move the conditional call to write_lite_restart to nwm_output_generator.
if output_parameters['lite_restart'] is not None:
if output_parameters.get('lite_restart',{}).get('lite_restart_output_directory', None):
nhd_io.write_lite_restart(
network.q0,
network._waterbody_df,
Expand Down
38 changes: 19 additions & 19 deletions test/LowerColorado_TX/test_AnA.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -81,23 +81,23 @@ compute_parameters:
reservoir_persistence_usace : True
gage_lakeID_crosswalk_file : domain/reservoir_index_AnA.nc
#--------------------------------------------------------------------------------
output_parameters:
#----------
test_output: output/lcr_flowveldepth.pkl
lite_restart:
#----------
lite_restart_output_directory: restart/
chrtout_output:
#----------
wrf_hydro_channel_output_source_folder: channel_forcing/
chanobs_output:
#----------
chanobs_output_directory: output/
chanobs_filepath : lcr_chanobs.nc
lakeout_output: lakeout/
stream_output:
stream_output_directory: output/
stream_output_time: 1 #[hr]
stream_output_type: '.nc' #please select only between netcdf '.nc' or '.csv' or '.pkl'
stream_output_internal_frequency: 60 #[min] it should be order of 5 minutes. For instance if you want to output every hour put 60
# output_parameters:
# #----------
# test_output: output/lcr_flowveldepth.pkl
# lite_restart:
# #----------
# lite_restart_output_directory: restart/
# chrtout_output:
# #----------
# wrf_hydro_channel_output_source_folder: channel_forcing/
# chanobs_output:
# #----------
# chanobs_output_directory: output/
# chanobs_filepath : lcr_chanobs.nc
# lakeout_output: lakeout/
# stream_output:
# stream_output_directory: output/
# stream_output_time: 1 #[hr]
# stream_output_type: '.nc' #please select only between netcdf '.nc' or '.csv' or '.pkl'
# stream_output_internal_frequency: 60 #[min] it should be order of 5 minutes. For instance if you want to output every hour put 60

38 changes: 19 additions & 19 deletions test/LowerColorado_TX/test_AnA_V4_NHD.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -105,24 +105,24 @@ compute_parameters:
reservoir_rfc_forecasts_lookback_hours : 48

#--------------------------------------------------------------------------------
output_parameters:
#----------
test_output: output/lcr_flowveldepth.pkl
lite_restart:
#----------
lite_restart_output_directory: restart/
chrtout_output:
#----------
wrf_hydro_channel_output_source_folder: channel_forcing/
chanobs_output:
#----------
chanobs_output_directory: output/
chanobs_filepath : lcr_chanobs.nc
lakeout_output: lakeout/
stream_output:
stream_output_directory: output/
stream_output_time: 1 #[hr]
stream_output_type: '.nc' #please select only between netcdf '.nc' or '.csv' or '.pkl'
stream_output_internal_frequency: 30 #[min] it should be order of 5 minutes. For instance if you want to output every hour put 60
# output_parameters:
# #----------
# test_output: output/lcr_flowveldepth.pkl
# lite_restart:
# #----------
# lite_restart_output_directory: restart/
# chrtout_output:
# #----------
# wrf_hydro_channel_output_source_folder: channel_forcing/
# chanobs_output:
# #----------
# chanobs_output_directory: output/
# chanobs_filepath : lcr_chanobs.nc
# lakeout_output: lakeout/
# stream_output:
# stream_output_directory: output/
# stream_output_time: 1 #[hr]
# stream_output_type: '.nc' #please select only between netcdf '.nc' or '.csv' or '.pkl'
# stream_output_internal_frequency: 30 #[min] it should be order of 5 minutes. For instance if you want to output every hour put 60


26 changes: 13 additions & 13 deletions test/LowerColorado_TX_v4/test_AnA_V4_HYFeature.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ network_topology_parameters:
columns:
key: 'id'
downstream: 'toid'
dx : 'lengthkm'
dx : 'length_m'
n : 'n'
ncc : 'nCC'
s0 : 'So'
Expand Down Expand Up @@ -82,15 +82,15 @@ compute_parameters:
reservoir_rfc_forecasts_lookback_hours : 28
reservoir_rfc_forecasts_offset_hours : 28
#--------------------------------------------------------------------------------
output_parameters:
#----------
test_output : output/lcr_flowveldepth.pkl
lite_restart:
#----------
lite_restart_output_directory: restart/
lakeout_output: lakeout/
stream_output :
stream_output_directory: output/
stream_output_time: 1 #[hr]
stream_output_type: '.pkl' #please select only between netcdf '.nc' or '.csv' or '.pkl'
stream_output_internal_frequency: 60 #[min] it should be order of 5 minutes. For instance if you want to output every hour put 60
# output_parameters:
# #----------
# test_output : output/lcr_flowveldepth.pkl
# lite_restart:
# #----------
# lite_restart_output_directory: restart/
# lakeout_output: lakeout/
# stream_output :
# stream_output_directory: output/
# stream_output_time: 1 #[hr]
# stream_output_type: '.nc' #please select only between netcdf '.nc' or '.csv' or '.pkl'
# stream_output_internal_frequency: 60 #[min] it should be order of 5 minutes. For instance if you want to output every hour put 60
96 changes: 96 additions & 0 deletions test/LowerColorado_TX_v4/test_AnA_V4_HYFeature_noDA.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
# $ python3 -m nwm_routing -f -V4 test_AnA_V4_HYFeature.yaml
#--------------------------------------------------------------------------------
log_parameters:
#----------
showtiming: True
log_level : DEBUG
#--------------------------------------------------------------------------------
network_topology_parameters:
#----------
supernetwork_parameters:
#----------
geo_file_path: domain/LowerColorado_NGENv20.gpkg
columns:
key: 'id'
downstream: 'toid'
dx : 'length_m'
n : 'n'
ncc : 'nCC'
s0 : 'So'
bw : 'BtmWdth'
waterbody : 'rl_NHDWaterbodyComID'
gages : 'rl_gages'
tw : 'TopWdth'
twcc : 'TopWdthCC'
musk : 'MusK'
musx : 'MusX'
cs : 'ChSlp'
alt: 'alt'
waterbody_parameters:
#----------
break_network_at_waterbodies: True
#--------------------------------------------------------------------------------
compute_parameters:
#----------
parallel_compute_method: by-subnetwork-jit-clustered # serial
compute_kernel : V02-structured
assume_short_ts : True
subnetwork_target_size : 10000
cpu_pool : 36
restart_parameters:
#----------
start_datetime : 2023-04-02_00:00
lite_channel_restart_file : #restart/RESTART.2020082600_DOMAIN1
lite_waterbody_restart_file : #restart/waterbody_restart_202006011200
hybrid_parameters:
run_hybrid_routing: False
diffusive_domain : #domain/coastal_domain_subset.yaml
use_natl_xsections: False
topobathy_domain : #domain/final_diffusive_natural_xs.nc
run_refactored_network: False
refactored_domain: #domain/refactored_coastal_domain_subset.yaml
refactored_topobathy_domain: #domain/refac_final_diffusive_natural_xs.nc
coastal_boundary_domain: #domain/coastal_boundary_domain.yaml
forcing_parameters:
#----------
qts_subdivisions : 12
dt : 300 # [sec]
qlat_input_folder : channel_forcing
qlat_file_pattern_filter : "*.CHRTOUT_DOMAIN1.csv"
coastal_boundary_input_file : #boundary_forcing
nts : 288 # 288 for 1day; 2592 for 9 days
max_loop_size : 24 # [hr]
data_assimilation_parameters:
#----------
usgs_timeslices_folder : usgs_timeslices/
usace_timeslices_folder : usace_timelices/
streamflow_da:
#----------
streamflow_nudging : False
diffusive_streamflow_nudging : False
lastobs_output_folder : lastobs/
reservoir_da:
#----------
reservoir_persistence_da:
#----------
reservoir_persistence_usgs : False
reservoir_persistence_usace : False
reservoir_rfc_da:
#----------
reservoir_rfc_forecasts : False
reservoir_rfc_forecasts_time_series_path: rfc_timeseries/
reservoir_rfc_forecasts_lookback_hours : 28
reservoir_rfc_forecasts_offset_hours : 28
#--------------------------------------------------------------------------------
# output_parameters:
# #----------
# test_output : output/lcr_flowveldepth.pkl
# lite_restart:
# #----------
# lite_restart_output_directory: restart/
# lakeout_output: lakeout/
# stream_output :
# stream_output_directory: output/
# stream_output_time: 1 #[hr]
# stream_output_type: '.nc' #please select only between netcdf '.nc' or '.csv' or '.pkl'
# stream_output_internal_frequency: 60 #[min] it should be order of 5 minutes. For instance if you want to output every hour put 60