save in one file

src/troute-config/troute/config/output_parameters.py

@@ -88,7 +88,7 @@ def validate_stream_output_internal_frequency(cls, value, values):
         if value is not None:
             if value % 5 != 0:
                 raise ValueError("stream_output_internal_frequency must be a multiple of 5.")
-            if value / 60 > values['stream_output_time']:
+            if values.get('stream_output_time') != -1 and value / 60 > values['stream_output_time']:
                 raise ValueError("stream_output_internal_frequency should be less than or equal to stream_output_time in minutes.")
         return value
 

src/troute-network/troute/nhd_io.py

@@ -2169,42 +2169,64 @@ def write_flowveldepth(
     empty_ids = list(set(flowveldepth.index).difference(set(nudge_df.index)))
     empty_df = pd.DataFrame(index=empty_ids, columns=nudge_df.columns).fillna(-9999.0)
     nudge_df = pd.concat([nudge_df, empty_df]).loc[flowveldepth.index]
-
-    ts_per_file = stream_output_timediff*60//stream_output_internal_frequency
-
-    num_files = flowveldepth.shape[1]//3*dt//(stream_output_timediff*60*60)
-    if num_files==0:
-        num_files=1
-
     file_name_time = t0
     jobs = []
-    for _ in range(num_files):
+    import pdb;pdb.set_trace()
+    if stream_output_timediff > 0:
+        ts_per_file = stream_output_timediff*60//stream_output_internal_frequency
+
+        num_files = flowveldepth.shape[1]//3*dt//(stream_output_timediff*60*60)
+        if num_files==0:
+            num_files=1
+
+        for _ in range(num_files):
+            filename = 'troute_output_' + file_name_time.strftime('%Y%m%d%H%M') + stream_output_type
+            args = (stream_output_directory,filename,
+                    flow.iloc[:,0:ts_per_file],
+                    velocity.iloc[:,0:ts_per_file],
+                    depth.iloc[:,0:ts_per_file],
+                    nudge_df.iloc[:,0:ts_per_file],
+                    timestamps_sec[0:ts_per_file],t0)
+            if stream_output_type == '.nc':
+                if cpu_pool > 1 & num_files > 1:
+                    jobs.append(delayed(write_flowveldepth_netcdf)(*args))
+                else:
+                    write_flowveldepth_netcdf(*args)
+            else:
+                if cpu_pool > 1 & num_files > 1:
+                    jobs.append(delayed(write_flowveldepth_csv_pkl)(*args))
+                else:
+                    write_flowveldepth_csv_pkl(*args)
+
+            flow = flow.iloc[:,ts_per_file:]
+            velocity = velocity.iloc[:,ts_per_file:]
+            depth = depth.iloc[:,ts_per_file:]
+            nudge_df = nudge_df.iloc[:,ts_per_file:]
+            timestamps_sec = timestamps_sec[ts_per_file:]
+            file_name_time = file_name_time + timedelta(hours=stream_output_timediff)
+
+    elif stream_output_timediff == -1:
+
         filename = 'troute_output_' + file_name_time.strftime('%Y%m%d%H%M') + stream_output_type
         args = (stream_output_directory,filename,
-                flow.iloc[:,0:ts_per_file],
-                velocity.iloc[:,0:ts_per_file],
-                depth.iloc[:,0:ts_per_file],
-                nudge_df.iloc[:,0:ts_per_file],
-                timestamps_sec[0:ts_per_file],t0)
+                flow.iloc,
+                velocity.iloc,
+                depth.iloc,
+                nudge_df.iloc,
+                timestamps_sec,
+                t0)
         if stream_output_type == '.nc':
-            if cpu_pool > 1 & num_files > 1:
+            if cpu_pool > 1:
                 jobs.append(delayed(write_flowveldepth_netcdf)(*args))
             else:
                 write_flowveldepth_netcdf(*args)
         else:
-            if cpu_pool > 1 & num_files > 1:
+            if cpu_pool > 1:
                 jobs.append(delayed(write_flowveldepth_csv_pkl)(*args))
             else:
                 write_flowveldepth_csv_pkl(*args)
 
-        flow = flow.iloc[:,ts_per_file:]
-        velocity = velocity.iloc[:,ts_per_file:]
-        depth = depth.iloc[:,ts_per_file:]
-        nudge_df = nudge_df.iloc[:,ts_per_file:]
-        timestamps_sec = timestamps_sec[ts_per_file:]
-        file_name_time = file_name_time + timedelta(hours=stream_output_timediff)
-
-    if cpu_pool > 1 & num_files > 1:
+    if cpu_pool > 1:
         try:
             # Execute all jobs in parallel
             with Parallel(n_jobs=cpu_pool) as parallel:

test/LowerColorado_TX_v4/test_AnA_V4_HYFeature.yaml

@@ -131,7 +131,7 @@ compute_parameters:
 #      lastobs_output: lastobs/
 #     stream_output : 
 #         stream_output_directory: output/
-#         stream_output_time: 1 #[hr]
+#         stream_output_time: 1 #[hr] ** Consider `stream_output_time = -1` means save everything in one file **
 #         stream_output_type: '.nc' #please select only between netcdf '.nc' or '.csv' or '.pkl'
 #         stream_output_internal_frequency: 60 #[min] it should be order of 5 minutes. For instance if you want to output every hour put 60