Sync 1.1 and testing

.gitignore

@@ -23,3 +23,6 @@ Examples/reprocessDataServer.py
 *.stats
 
 newsletter.py
+
+.DS_Store
+.vscode/

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "tests/opencap-test-data"]
+	path = tests/opencap-test-data
+	url = https://github.com/stanfordnmbl/opencap-test-data.git

defaults.py

@@ -0,0 +1 @@
+DEFAULT_SYNC_VER = '1.0'

main.py

@@ -22,14 +22,15 @@
 from utilsChecker import calcExtrinsicsFromVideo
 from utilsChecker import isCheckerboardUpsideDown
 from utilsChecker import autoSelectExtrinsicSolution
-from utilsChecker import synchronizeVideos
 from utilsChecker import triangulateMultiviewVideo
 from utilsChecker import writeTRCfrom3DKeypoints
 from utilsChecker import popNeutralPoseImages
 from utilsChecker import rotateIntrinsics
+from utilsSync import synchronizeVideos
 from utilsDetector  import runPoseDetector
 from utilsAugmenter import augmentTRC
 from utilsOpenSim import runScaleTool, getScaleTimeRange, runIKTool, generateVisualizerJson
+from defaults import DEFAULT_SYNC_VER
 
 def main(sessionName, trialName, trial_id, cameras_to_use=['all'],
          intrinsicsFinalFolder='Deployed', isDocker=False,
@@ -42,7 +43,7 @@ def main(sessionName, trialName, trial_id, cameras_to_use=['all'],
          dataDir=None, overwriteAugmenterModel=False,
          filter_frequency='default', overwriteFilterFrequency=False,
          scaling_setup='upright_standing_pose', overwriteScalingSetup=False,
-         overwriteCamerasToUse=False):
+         overwriteCamerasToUse=False, syncVer=None,):
 
     # %% High-level settings.
     # Camera calibration.
@@ -131,6 +132,10 @@ def main(sessionName, trialName, trial_id, cameras_to_use=['all'],
     else:
         camerasToUse = cameras_to_use
 
+    # We'll use syncVer if provided to this function. If not, try to use one 
+    # from sessionMetadata, otherwise use the default one.
+    syncVer = syncVer or sessionMetadata.get('sync_ver', DEFAULT_SYNC_VER)
+
     # %% Paths to pose detector folder for local testing.
     if poseDetector == 'OpenPose':
         poseDetectorDirectory = getOpenPoseDirectory(isDocker)
@@ -392,7 +397,8 @@ def main(sessionName, trialName, trial_id, cameras_to_use=['all'],
                     filtFreqs=filtFreqs, confidenceThreshold=0.4,
                     imageBasedTracker=False, cams2Use=camerasToUse_c, 
                     poseDetector=poseDetector, trialName=trialName,
-                    resolutionPoseDetection=resolutionPoseDetection))
+                    resolutionPoseDetection=resolutionPoseDetection,
+                    syncVer=syncVer))
         except Exception as e:
             if len(e.args) == 2: # specific exception
                 raise Exception(e.args[0], e.args[1])

tests/test_api.py

@@ -1,78 +1,79 @@
 import logging
+import os
+import sys
 import pytest
-import requests
-from unittest.mock import patch, Mock, ANY
+from unittest.mock import patch, Mock
 from http.client import HTTPMessage
 
+thisDir = os.path.dirname(os.path.realpath(__file__))
+repoDir = os.path.abspath(os.path.join(thisDir,'../'))
+sys.path.append(repoDir)
 from utils import makeRequestWithRetry
 
-class TestMakeRequestWithRetry:
-    logging.getLogger('urllib3').setLevel(logging.DEBUG)
+logging.getLogger('urllib3').setLevel(logging.DEBUG)
 
-    @patch("requests.Session.request")
-    def test_get(self, mock_response):
-        status_code = 200
-        mock_response.return_value.status_code = status_code
+@patch("requests.Session.request")
+def test_get(mock_response):
+    status_code = 200
+    mock_response.return_value.status_code = status_code
 
-        response = makeRequestWithRetry('GET', 'https://test.com', retries=2)
-        assert response.status_code == status_code
-        mock_response.assert_called_once_with('GET', 'https://test.com',
-                                              headers=None,
-                                              data=None,
-                                              params=None,
-                                              files=None)
+    response = makeRequestWithRetry('GET', 'https://test.com', retries=2)
+    assert response.status_code == status_code
+    mock_response.assert_called_once_with('GET', 'https://test.com',
+                                          headers=None,
+                                          data=None,
+                                          params=None,
+                                          files=None)
 
-    @patch("requests.Session.request")
-    def test_put(self, mock_response):
-        status_code = 201
-        mock_response.return_value.status_code = status_code
-
-        data = {
-            "key1": "value1",
-            "key2": "value2"
-        }
+@patch("requests.Session.request")
+def test_put(mock_response):
+    status_code = 201
+    mock_response.return_value.status_code = status_code
 
-        params = {
-            "param1": "value1"
-        }
-
-        response = makeRequestWithRetry('POST', 
-                                        'https://test.com',
-                                        data=data,
-                                        headers={"Authorization": "my_token"},
-                                        params=params,
-                                        retries=2)
-
-        assert response.status_code == status_code
-        mock_response.assert_called_once_with('POST', 
-                                             'https://test.com',
-                                             data=data,
-                                             headers={"Authorization": "my_token"},
-                                             params=params,
-                                             files=None)
+    data = {
+        "key1": "value1",
+        "key2": "value2"
+    }
 
-    @patch("urllib3.connectionpool.HTTPConnectionPool._get_conn")
-    def test_success_after_retries(self, mock_response):
-        mock_response.return_value.getresponse.side_effect = [
-            Mock(status=500, msg=HTTPMessage()),
-            Mock(status=502, msg=HTTPMessage()),
-            Mock(status=200, msg=HTTPMessage()),
-            Mock(status=429, msg=HTTPMessage()),
-        ]
+    params = {
+        "param1": "value1"
+    }
 
-        response = makeRequestWithRetry('GET', 
-                                        'https://test.com', 
-                                        retries=5, 
-                                        backoff_factor=0.1)
+    response = makeRequestWithRetry('POST',
+                                    'https://test.com',
+                                    data=data,
+                                    headers={"Authorization": "my_token"},
+                                    params=params,
+                                    retries=2)
 
-        assert response.status_code == 200
-        assert mock_response.call_count == 3
+    assert response.status_code == status_code
+    mock_response.assert_called_once_with('POST',
+                                         'https://test.com',
+                                         data=data,
+                                         headers={"Authorization": "my_token"},
+                                         params=params,
+                                         files=None)
 
-    # comment out test since httpbin can be unstable and we don't want to rely
-    # on it for tests. uncomment and see debug log to see retry attempts
-    '''def test_httpbin(self):
-        response = makeRequestWithRetry('GET', 
-                                        'https://httpbin.org/status/500', 
-                                        retries=4, 
-                                        backoff_factor=0.1)
-    '''
+@patch("urllib3.connectionpool.HTTPConnectionPool._get_conn")
+def test_success_after_retries(mock_response):
+    mock_response.return_value.getresponse.side_effect = [
+        Mock(status=500, msg=HTTPMessage()),
+        Mock(status=502, msg=HTTPMessage()),
+        Mock(status=200, msg=HTTPMessage()),
+        Mock(status=429, msg=HTTPMessage()),
+    ]
+
+    response = makeRequestWithRetry('GET',
+                                    'https://test.com',
+                                    retries=5,
+                                    backoff_factor=0.1)
+
+    assert response.status_code == 200
+    assert mock_response.call_count == 3
+
+# The httpbin test remains commented out for stability reasons
+# def test_httpbin():
+#     response = makeRequestWithRetry('GET',
+#                                     'https://httpbin.org/status/500',
+#                                     retries=4,
+#                                     backoff_factor=0.1)

tests/test_main.py

@@ -0,0 +1,147 @@
+import logging
+import os
+import sys
+import numpy as np
+import pandas as pd
+import pytest
+
+thisDir = os.path.dirname(os.path.realpath(__file__))
+repoDir = os.path.abspath(os.path.join(thisDir, '../'))
+sys.path.append(repoDir)
+from main import main
+
+# Helper functions to load and compare TRC and MOT files
+def load_trc(file, num_metadata_lines=5):
+    with open(file, 'r') as f:
+        lines = f.readlines()
+        metadata = lines[:num_metadata_lines]
+    df = pd.read_csv(file, sep='\t', skiprows=num_metadata_lines + 1, header=None)
+    return df, metadata
+
+def load_mot(file, num_metadata_lines=10):
+    with open(file, 'r') as f:
+        lines = f.readlines()
+        metadata = lines[:num_metadata_lines]
+    df = pd.read_csv(file, sep='\t', skiprows=num_metadata_lines)
+    return df, metadata
+
+
+def calc_rmse(series1, series2):
+    return np.sqrt(((series1 - series2) ** 2).mean())
+
+def compare_mot(output_mot_df, ref_mot_df, t0, tf):
+    '''Function to compare MOT dataframes within a time range [t0, tf].
+    We use the specific time range to analyze the range with the motion
+    of interest. In particular, the arm raise can create larger differences
+    on single frames.
+
+    - Time column is checked for equality (IK is frame-by-frame).
+    - Translation error is checked within 2 mm max per frame, RMSE within 
+      1 mm.
+    - Rotation error for wrist pronation/supination (coordinates pro_sup_r
+      and pro_sup_l) are checked within 5.0 degrees max per frame, RMSE
+      within 1.0 degrees.
+    - Rotation error for all other coordinates are tighter and checked 
+      within 2.5 degrees max per frame, RMSE within 0.5 degrees.
+    '''
+    output_mot_df_slice = output_mot_df[(output_mot_df['time'] >= t0) & (output_mot_df['time'] <= tf)]
+    ref_mot_df_slice = ref_mot_df[(ref_mot_df['time'] >= t0) & (ref_mot_df['time'] <= tf)]
+    for col in ref_mot_df.columns:
+        # time column should be equal since IK is frame-by-frame
+        if col == 'time':
+            pd.testing.assert_series_equal(output_mot_df[col], ref_mot_df[col])
+
+        # check translational within 2 mm max error, rmse within 1 mm
+        elif any(substr in col for substr in ['tx', 'ty', 'tz']):
+            pd.testing.assert_series_equal(
+                output_mot_df_slice[col], ref_mot_df_slice[col], atol=0.002
+            )
+            rmse = calc_rmse(output_mot_df_slice[col], ref_mot_df_slice[col])
+            assert rmse <= 0.001
+
+        elif 'pro_sup' in col:
+            pd.testing.assert_series_equal(
+                output_mot_df_slice[col], ref_mot_df_slice[col], atol=5.0
+            )
+            rmse = calc_rmse(output_mot_df_slice[col], ref_mot_df_slice[col])
+            assert rmse <= 1.0
+
+        # check rotational within 2.5 degrees max error, rmse within 0.5 degrees
+        else:
+            pd.testing.assert_series_equal(
+                output_mot_df_slice[col], ref_mot_df_slice[col], atol=2.5
+            )
+            rmse = calc_rmse(output_mot_df_slice[col], ref_mot_df_slice[col])
+            assert rmse <= 0.5
+
+# End to end tests with different sync methods (hand, gait, general).
+# Also check that syncVer updates with main changes.
+# Note: no pose detection, uses pre-scaled opensim model
+@pytest.mark.parametrize("syncVer", ['1.0', '1.1'])
+@pytest.mark.parametrize("trialName, t0, tf", [
+    ('squats-with-arm-raise', 5.0, 10.0),
+    ('squats', 3.0, 8.0),
+    ('walk', 1.0, 5.0),
+])
+def test_main(trialName, t0, tf, syncVer, caplog):
+    caplog.set_level(logging.INFO)
+
+    sessionName = 'sync_2-cameras'
+    trialID = trialName
+    dataDir = os.path.join(thisDir, 'opencap-test-data')
+    main(
+        sessionName,
+        trialName,
+        trialID,
+        dataDir=dataDir,
+        genericFolderNames=True,
+        poseDetector='hrnet',
+        syncVer=syncVer,
+    )
+    assert f"Synchronizing Keypoints using version {syncVer}" in caplog.text
+
+    # Compare marker data
+    output_trc = os.path.join(dataDir,
+        'Data',
+        sessionName,
+        'MarkerData',
+        'PostAugmentation',
+        f'{trialName}.trc',
+    )
+    ref_trc = os.path.join(
+        dataDir,
+        'Data',
+        sessionName,
+        'OutputReference',
+        f'{trialName}.trc',
+    )
+    output_trc_df, _ = load_trc(output_trc)
+    ref_trc_df, _ = load_trc(ref_trc)
+    pd.testing.assert_frame_equal(
+        output_trc_df, ref_trc_df, check_exact=False, atol=1e-3
+    )
+
+    # Compare IK data
+    output_mot = os.path.join(
+        dataDir,
+        'Data',
+        sessionName,
+        'OpenSimData',
+        'Kinematics',
+        f'{trialName}.mot',
+    )
+    ref_mot = os.path.join(
+        dataDir,
+        'Data',
+        sessionName,
+        'OutputReference',
+        f'{trialName}.mot',
+    )
+    output_mot_df, _ = load_mot(output_mot)
+    ref_mot_df, _ = load_mot(ref_mot)
+    pd.testing.assert_index_equal(output_mot_df.columns, ref_mot_df.columns)
+    compare_mot(output_mot_df, ref_mot_df, t0, tf)
+
+# TODO: calibration and neutral
+# TODO: > 2 cameras
+# TODO: augmenter versions