Test mth5 add phoenix

.github/workflows/tests.yml

@@ -18,7 +18,7 @@ jobs:
       fail-fast: false
       matrix:
         os: ["ubuntu-latest"]
-        python-version: [3.8, 3.7, 3.6] 
+        python-version: [3.6, 3.7, 3.8] 
 
     steps:
     - uses: actions/checkout@v2
@@ -35,7 +35,7 @@ jobs:
         conda install -c conda-forge pytest pytest-cov certifi">=2017.4.17" pandoc
         pip install -r requirements-dev.txt
         pip install git+https://github.com/kujaku11/mt_metadata.git 
-        pip install git+https://github.com/kujaku11/mth5.git #@branch_name
+        pip install git+https://github.com/kujaku11/mth5.git@add_phoenix
 
     - name: Install Our Package
       run: |

aurora/pipelines/process_mth5.py

@@ -297,15 +297,24 @@ def populate_dataset_df(i_dec_level, config, dataset_df):
 
             # APPLY TIMING CORRECTIONS HERE
     else:
+        print(f"DECIMATION LEVEL {i_dec_level}")
         # See Note 1 top of module
         # See Note 2 top of module
         for i, row in dataset_df.iterrows():
             run_xrts = row["run_dataarray"].to_dataset("channel")
             input_dict = {"run": row["run"], "mvts": run_xrts}
             run_dict = prototype_decimate(config.decimation, input_dict)
+            print("PACKING RUN")
+            print("HERE is teh dtype of the zero-decimation level run column")
+            type(dataset_df.iloc[0].run)
+            print("ANd the new one")
+            print(type(run_dict["run"]))
+            import pandas as pd
+
+            print(pd.__version__)
             dataset_df["run"].at[i] = run_dict["run"]
             dataset_df["run_dataarray"].at[i] = run_dict["mvts"].to_array("channel")
-
+    print("OK OK OK")
     return dataset_df
 
 
@@ -399,7 +408,9 @@ def process_mth5(
     for i_dec_level, dec_level_config in enumerate(processing_config.decimations):
         dataset_df = populate_dataset_df(i_dec_level, dec_level_config, dataset_df)
         # ANY MERGING OF RUNS IN TIME DOMAIN WOULD GO HERE
-
+        print("DATASET DF POPULATED")
+        print("DATASET DF POPULATED")
+        print("DATASET DF POPULATED")
         # TFK 1: get clock-zero from data if needed
         if dec_level_config.window.clock_zero_type == "data start":
             dec_level_config.window.clock_zero = str(dataset_df.start.min())

aurora/pipelines/time_series_helpers.py

@@ -352,7 +352,6 @@ def prototype_decimate(config, run_run_ts):
     run_obj = run_run_ts["run"]
     run_xrts = run_run_ts["mvts"]
     run_obj.metadata.sample_rate = config.sample_rate
-
     slicer = slice(None, None, int(config.factor))  # decimation.factor
     downsampled_time_axis = run_xrts.time.data[slicer]
 
@@ -371,5 +370,8 @@ def prototype_decimate(config, run_run_ts):
 
     xr_ds = xr_da.to_dataset("channel")
     result = {"run": run_obj, "mvts": xr_ds}
-
+    print("RUN OBJECT")
+    print(run_obj)
+    print("XRDS")
+    print(xr_ds)
     return result

aurora/test_utils/synthetic/make_mth5_from_asc.py

@@ -20,9 +20,15 @@
 
 from aurora.config.metadata.channel_nomenclature import ChannelNomenclature
 from aurora.test_utils.synthetic.synthetic_station_config import make_filters
-from aurora.test_utils.synthetic.synthetic_station_config import make_station_01
-from aurora.test_utils.synthetic.synthetic_station_config import make_station_02
-from aurora.test_utils.synthetic.synthetic_station_config import make_station_03
+from aurora.test_utils.synthetic.synthetic_station_config import (
+    make_station_01,
+)
+from aurora.test_utils.synthetic.synthetic_station_config import (
+    make_station_02,
+)
+from aurora.test_utils.synthetic.synthetic_station_config import (
+    make_station_03,
+)
 
 np.random.seed(0)
 
@@ -134,7 +140,9 @@ def create_mth5_synthetic_file(
             # add noise
             for col in run.channels:
                 if run.noise_scalars[col]:
-                    df[col] += run.noise_scalars[col] * np.random.randn(len(df))
+                    df[col] += run.noise_scalars[col] * np.random.randn(
+                        len(df)
+                    )
 
             # add nan
             if add_nan_values:
@@ -173,7 +181,9 @@ def create_mth5_synthetic_file(
 
 
 def create_test1_h5(file_version="0.1.0", channel_nomenclature="default"):
-    station_01_params = make_station_01(channel_nomenclature=channel_nomenclature)
+    station_01_params = make_station_01(
+        channel_nomenclature=channel_nomenclature
+    )
     mth5_path = station_01_params.mth5_path  # DATA_PATH.joinpath("test1.h5")
     mth5_path = create_mth5_synthetic_file(
         [
@@ -188,7 +198,9 @@ def create_test1_h5(file_version="0.1.0", channel_nomenclature="default"):
 
 
 def create_test2_h5(file_version="0.1.0", channel_nomenclature="default"):
-    station_02_params = make_station_02(channel_nomenclature=channel_nomenclature)
+    station_02_params = make_station_02(
+        channel_nomenclature=channel_nomenclature
+    )
     mth5_path = station_02_params.mth5_path
     mth5_path = create_mth5_synthetic_file(
         [
@@ -201,8 +213,12 @@ def create_test2_h5(file_version="0.1.0", channel_nomenclature="default"):
     return mth5_path
 
 
-def create_test1_h5_with_nan(file_version="0.1.0", channel_nomenclature="default"):
-    station_01_params = make_station_01(channel_nomenclature=channel_nomenclature)
+def create_test1_h5_with_nan(
+    file_version="0.1.0", channel_nomenclature="default"
+):
+    station_01_params = make_station_01(
+        channel_nomenclature=channel_nomenclature
+    )
     mth5_path = station_01_params.mth5_path  # DATA_PATH.joinpath("test1.h5")
     mth5_path = create_mth5_synthetic_file(
         [
@@ -217,10 +233,16 @@ def create_test1_h5_with_nan(file_version="0.1.0", channel_nomenclature="default
 
 
 def create_test12rr_h5(file_version="0.1.0", channel_nomenclature="default"):
-    station_01_params = make_station_01(channel_nomenclature=channel_nomenclature)
-    station_02_params = make_station_02(channel_nomenclature=channel_nomenclature)
+    station_01_params = make_station_01(
+        channel_nomenclature=channel_nomenclature
+    )
+    station_02_params = make_station_02(
+        channel_nomenclature=channel_nomenclature
+    )
     station_params = [station_01_params, station_02_params]
-    mth5_path = station_01_params.mth5_path.__str__().replace("test1.h5", "test12rr.h5")
+    mth5_path = station_01_params.mth5_path.__str__().replace(
+        "test1.h5", "test12rr.h5"
+    )
     mth5_path = create_mth5_synthetic_file(
         station_params,
         mth5_path,

aurora/test_utils/synthetic/rms_helpers.py

@@ -44,10 +44,10 @@ def get_expected_rms_misfit(test_case_id, emtf_version=None):
             expected_rms_misfit["rho"]["yx"] = 3.658614
             expected_rms_misfit["phi"]["yx"] = 0.844645
         elif emtf_version == "matlab":
-            expected_rms_misfit["rho"]["xy"] = 2.713543
-            expected_rms_misfit["phi"]["xy"] = 0.784464
-            expected_rms_misfit["rho"]["yx"] = 3.74120
-            expected_rms_misfit["phi"]["yx"] = 1.375335
+            expected_rms_misfit["rho"]["xy"] = 2.706098
+            expected_rms_misfit["phi"]["xy"] = 0.784228
+            expected_rms_misfit["rho"]["yx"] = 3.745279
+            expected_rms_misfit["phi"]["yx"] = 1.374937
 
     elif test_case_id == "test2r1":
         expected_rms_misfit["rho"]["xy"] = 3.971313
@@ -80,6 +80,22 @@ def assert_rms_misfit_ok(
     expected_rms_phi = expected_rms_misfit["phi"][xy_or_yx]
     print(f"expected_rms_rho_{xy_or_yx} {expected_rms_rho}")
     print(f"expected_rms_phi_{xy_or_yx} {expected_rms_phi}")
-    assert np.isclose(rho_rms_aurora - expected_rms_rho, 0, atol=rho_tol)
-    assert np.isclose(phi_rms_aurora - expected_rms_phi, 0, atol=phi_tol)
+    if not np.isclose(rho_rms_aurora - expected_rms_rho, 0, atol=rho_tol):
+        print("==== AURORA ====\n")
+        print(rho_rms_aurora)
+        print("==== EXPECTED ====\n")
+        print(expected_rms_rho)
+        print("==== DIFFERENCE ====\n")
+        print(rho_rms_aurora - expected_rms_rho)
+        raise AssertionError("Expected misfit for resistivity is not correct")
+
+    if not np.isclose(phi_rms_aurora - expected_rms_phi, 0, atol=rho_tol):
+        print("==== AURORA ====\n")
+        print(phi_rms_aurora)
+        print("==== EXPECTED ====\n")
+        print(expected_rms_phi)
+        print("==== DIFFERENCE ====\n")
+        print(phi_rms_aurora - expected_rms_phi)
+        raise AssertionError("Expected misfit for phase is not correct")
+
     return

aurora/time_series/frequency_band.py

@@ -1,7 +1,7 @@
 import numpy as np
 import pandas as pd
 
-from aurora.sandbox.io_helpers.emtf_band_setup import EMTFBandSetupFile
+# from aurora.sandbox.io_helpers.emtf_band_setup import EMTFBandSetupFile
 
 
 class FrequencyBand(pd.Interval):

setup.py

@@ -18,7 +18,7 @@
     "numpy",
     "numba",
     "obspy",
-    "pandas",
+    "pandas<1.5",
     "scipy",
     "xarray",
     "fortranformat",