Fix validation for output parameters columns in the condition table (#161)

Parameters introduced via `observableFormula` or `noiseFormula` are allowed to occur as condition table columns. This was not handled correctly by `petablint`. 

Parameters introduced via `observableFormula` or `noiseFormula` are only required in the parameter table if they are not already overridden for *all* conditions. This was not handled correctly by `petablint`.

Author: dweindl

petab/lint.py

@@ -85,12 +85,16 @@ def assert_no_leading_trailing_whitespace(
 
 
 def check_condition_df(
-        df: pd.DataFrame, model: Optional[Model] = None) -> None:
+        df: pd.DataFrame,
+        model: Optional[Model] = None,
+        observable_df: Optional[pd.DataFrame] = None
+) -> None:
     """Run sanity checks on PEtab condition table
 
     Arguments:
         df: PEtab condition DataFrame
         model: Model for additional checking of parameter IDs
+        observable_df: PEtab observables DataFrame
 
     Raises:
         AssertionError: in case of problems
@@ -119,6 +123,9 @@ def check_condition_df(
 
     if model is not None:
         allowed_cols = set(model.get_valid_ids_for_condition_table())
+        if observable_df is not None:
+            allowed_cols |= set(petab.get_output_parameters(
+                model=model, observable_df=observable_df))
         for column_name in df.columns:
             if column_name != CONDITION_NAME \
                     and column_name not in allowed_cols:
@@ -795,7 +802,8 @@ def lint_problem(problem: 'petab.Problem') -> bool:
     if problem.condition_df is not None:
         logger.info("Checking condition table...")
         try:
-            check_condition_df(problem.condition_df, problem.model)
+            check_condition_df(problem.condition_df, problem.model,
+                               problem.observable_df)
         except AssertionError as e:
             logger.error(e)
             errors_occurred = True

petab/parameters.py

@@ -265,6 +265,13 @@ def append_overrides(overrides):
         if not model.has_entity_with_id(p):
             parameter_ids[p] = None
 
+    # remove parameters that occur in the condition table and are overridden
+    #  for ALL conditions
+    for p in condition_df.columns[~condition_df.isnull().any()]:
+        try:
+            del parameter_ids[p]
+        except KeyError:
+            pass
     return parameter_ids.keys()
 
 

tests/test_lint.py

@@ -419,7 +419,14 @@ def test_check_condition_df():
     with pytest.raises(AssertionError):
         lint.check_condition_df(condition_df, model)
 
-    # fix:
+    # fix by adding output parameter
+    observable_df = pd.DataFrame({
+        OBSERVABLE_ID: ["obs1"],
+        OBSERVABLE_FORMULA: ["p1"],
+    })
+    lint.check_condition_df(condition_df, model, observable_df)
+
+    # fix by adding parameter
     ss_model.addParameter('p1', 1.0)
     lint.check_condition_df(condition_df, model)
 
@@ -452,7 +459,7 @@ def test_check_ids():
 
 
 def test_check_parameter_df():
-    """Check parameters.normalize_parameter_df."""
+    """Check lint.check_parameter_df."""
 
     parameter_df = pd.DataFrame({
         PARAMETER_ID: ['par0', 'par1', 'par2'],