v2: handle merged observable.observableTransformation observable.nois… (#376)

* v2: handle merged observable.observableTransformation observable.noiseDistribution

Update to changes in PEtab v2 draft, see PEtab-dev/PEtab#619.

Closes #375.

* upconversion

Author: dweindl

petab/v2/C.py

@@ -147,8 +147,6 @@
 OBSERVABLE_FORMULA = "observableFormula"
 #: Noise formula column in the observable table
 NOISE_FORMULA = "noiseFormula"
-#: Observable transformation column in the observable table
-OBSERVABLE_TRANSFORMATION = "observableTransformation"
 #: Noise distribution column in the observable table
 NOISE_DISTRIBUTION = "noiseDistribution"
 
@@ -162,7 +160,6 @@
 #: Optional columns of observable table
 OBSERVABLE_DF_OPTIONAL_COLS = [
     OBSERVABLE_NAME,
-    OBSERVABLE_TRANSFORMATION,
     NOISE_DISTRIBUTION,
 ]
 
@@ -181,8 +178,6 @@
 LOG = "log"
 #: Logarithmic base 10 transformation
 LOG10 = "log10"
-#: Supported observable transformations
-OBSERVABLE_TRANSFORMATIONS = [LIN, LOG, LOG10]
 
 
 # NOISE MODELS
@@ -232,7 +227,7 @@
 
 
 #: Supported noise distributions
-NOISE_MODELS = [NORMAL, LAPLACE]
+NOISE_DISTRIBUTIONS = [NORMAL, LAPLACE, LOG_NORMAL, LOG_LAPLACE]
 
 
 # VISUALIZATION

petab/v2/core.py

@@ -32,7 +32,6 @@
 __all__ = [
     "Observable",
     "ObservableTable",
-    "ObservableTransformation",
     "NoiseDistribution",
     "Change",
     "Condition",
@@ -87,20 +86,6 @@ def _valid_petab_id(v: str) -> str:
     return v
 
 
-class ObservableTransformation(str, Enum):
-    """Observable transformation types.
-
-    Observable transformations as used in the PEtab observables table.
-    """
-
-    #: No transformation
-    LIN = C.LIN
-    #: Logarithmic transformation (natural logarithm)
-    LOG = C.LOG
-    #: Logarithmic transformation (base 10)
-    LOG10 = C.LOG10
-
-
 class ParameterScale(str, Enum):
     """Parameter scales.
 
@@ -122,6 +107,10 @@ class NoiseDistribution(str, Enum):
     NORMAL = C.NORMAL
     #: Laplace distribution
     LAPLACE = C.LAPLACE
+    #: Log-normal distribution
+    LOG_NORMAL = C.LOG_NORMAL
+    #: Log-Laplace distribution
+    LOG_LAPLACE = C.LOG_LAPLACE
 
 
 class PriorDistribution(str, Enum):
@@ -173,10 +162,6 @@ class Observable(BaseModel):
     name: str | None = Field(alias=C.OBSERVABLE_NAME, default=None)
     #: Observable formula.
     formula: sp.Basic | None = Field(alias=C.OBSERVABLE_FORMULA, default=None)
-    #: Observable transformation.
-    transformation: ObservableTransformation = Field(
-        alias=C.OBSERVABLE_TRANSFORMATION, default=ObservableTransformation.LIN
-    )
     #: Noise formula.
     noise_formula: sp.Basic | None = Field(alias=C.NOISE_FORMULA, default=None)
     #: Noise distribution.
@@ -193,9 +178,7 @@ class Observable(BaseModel):
         "name",
         "formula",
         "noise_formula",
-        "noise_formula",
         "noise_distribution",
-        "transformation",
         mode="before",
     )
     @classmethod

petab/v2/lint.py

@@ -326,12 +326,12 @@ class CheckPosLogMeasurements(ValidationTask):
     log-transformation are positive."""
 
     def run(self, problem: Problem) -> ValidationIssue | None:
-        from .core import ObservableTransformation as ot
+        from .core import NoiseDistribution as nd
 
         log_observables = {
             o.id
             for o in problem.observable_table.observables
-            if o.transformation in [ot.LOG, ot.LOG10]
+            if o.noise_distribution in [nd.LOG_NORMAL, nd.LOG_LAPLACE]
         }
         if log_observables:
             for m in problem.measurement_table.measurements:

petab/v2/petab1to2.py

@@ -104,14 +104,23 @@ def petab_files_1to2(yaml_config: Path | str, output_dir: Path | str):
     # sub-problems
     for problem_config in new_yaml_config.problems:
         # copy files that don't need conversion
-        #  (models, observables, visualizations)
+        #  (models, visualizations)
         for file in chain(
-            problem_config.observable_files,
             (model.location for model in problem_config.model_files.values()),
             problem_config.visualization_files,
         ):
             _copy_file(get_src_path(file), Path(get_dest_path(file)))
 
+        # Update observable table
+        for observable_file in problem_config.observable_files:
+            observable_df = v1.get_observable_df(get_src_path(observable_file))
+            observable_df = v1v2_observable_df(
+                observable_df,
+            )
+            v2.write_observable_df(
+                observable_df, get_dest_path(observable_file)
+            )
+
         # Update condition table
         for condition_file in problem_config.condition_files:
             condition_df = v1.get_condition_df(get_src_path(condition_file))
@@ -339,3 +348,48 @@ def v1v2_condition_df(
         )
 
     return condition_df
+
+
+def v1v2_observable_df(observable_df: pd.DataFrame) -> pd.DataFrame:
+    """Convert observable table from petab v1 to v2.
+
+    Perform all updates that can be done solely on the observable table:
+    * drop observableTransformation, update noiseDistribution
+    """
+    df = observable_df.copy().reset_index()
+
+    # drop observableTransformation, update noiseDistribution
+    #  if there is no observableTransformation, no need to update
+    if v1.C.OBSERVABLE_TRANSFORMATION in df.columns:
+        df[v1.C.OBSERVABLE_TRANSFORMATION] = df[
+            v1.C.OBSERVABLE_TRANSFORMATION
+        ].fillna(v1.C.LIN)
+
+        if v1.C.NOISE_DISTRIBUTION in df:
+            df[v1.C.NOISE_DISTRIBUTION] = df[v1.C.NOISE_DISTRIBUTION].fillna(
+                v1.C.NORMAL
+            )
+        else:
+            df[v1.C.NOISE_DISTRIBUTION] = v1.C.NORMAL
+
+        # merge observableTransformation into noiseDistribution
+        def update_noise_dist(row):
+            dist = row.get(v1.C.NOISE_DISTRIBUTION)
+            trans = row.get(v1.C.OBSERVABLE_TRANSFORMATION)
+
+            if trans == v1.C.LIN:
+                new_dist = dist
+            else:
+                new_dist = f"{trans}-{dist}"
+
+            if new_dist not in v2.C.NOISE_DISTRIBUTIONS:
+                raise NotImplementedError(
+                    f"Noise distribution `{new_dist}' for "
+                    f"observable `{row[v1.C.OBSERVABLE_ID]}'"
+                    f" is not supported in PEtab v2."
+                )
+
+        df[v2.C.NOISE_DISTRIBUTION] = df.apply(update_noise_dist, axis=1)
+        df.drop(columns=[v1.C.OBSERVABLE_TRANSFORMATION], inplace=True)
+
+    return df

petab/v2/problem.py

@@ -903,7 +903,6 @@ def add_observable(
         formula: str,
         noise_formula: str | float | int = None,
         noise_distribution: str = None,
-        transform: str = None,
         name: str = None,
         **kwargs,
     ):
@@ -914,7 +913,6 @@ def add_observable(
             formula: The observable formula
             noise_formula: The noise formula
             noise_distribution: The noise distribution
-            transform: The observable transformation
             name: The observable name
             kwargs: additional columns/values to add to the observable table
 
@@ -929,8 +927,7 @@ def add_observable(
             record[NOISE_FORMULA] = noise_formula
         if noise_distribution is not None:
             record[NOISE_DISTRIBUTION] = noise_distribution
-        if transform is not None:
-            record[OBSERVABLE_TRANSFORMATION] = transform
+
         record.update(kwargs)
 
         self.observable_table += core.Observable(**record)

tests/v2/test_conversion.py

@@ -40,6 +40,9 @@ def test_benchmark_collection(problem_id):
         pytest.skip("Too slow. Re-enable once we are faster.")
 
     yaml_path = benchmark_models_petab.get_problem_yaml_path(problem_id)
-    problem = petab1to2(yaml_path)
+    try:
+        problem = petab1to2(yaml_path)
+    except NotImplementedError as e:
+        pytest.skip(str(e))
     assert isinstance(problem, Problem)
     assert len(problem.measurement_table.measurements)