Align RoboCasa and BEHAVIOR adapter contracts

Author: haozhezju

adapter/protocol.py

@@ -185,14 +185,20 @@ def _depth_range(depth_pixels: Any) -> tuple[float, float]:
 
 @dataclass(slots=True)
 class CameraFrame:
-    """RGBD camera frame in JSON-serializable form for the initial bridge."""
+    """RGBD camera frame in JSON-serializable form for the initial bridge.
+
+    ``role`` is an optional backend-neutral semantic hint such as
+    ``scene_primary`` or ``wrist_primary``.  ``frame_id`` remains the stable
+    backend identifier and is never rewritten to emulate another simulator.
+    """
 
     frame_id: str
     rgb: list[list[list[int]]]
     depth: list[list[float]] | None = None
     intrinsics: JsonDict = field(default_factory=dict)
     extrinsics: JsonDict = field(default_factory=dict)
     timestamp_s: float | None = None
+    role: str = ""
 
     def to_dict(self) -> JsonDict:
         """Convert to a plain JSON-serialisable dict."""
@@ -205,6 +211,8 @@ def to_dict(self) -> JsonDict:
         }
         if self.timestamp_s is not None:
             d["timestamp_s"] = self.timestamp_s
+        if self.role:
+            d["role"] = self.role
         return d
 
     @classmethod
@@ -233,6 +241,7 @@ def from_dict(cls, d: dict, *, frame_id: str = "") -> CameraFrame:
             intrinsics=_ensure_plain_dict(d.get("intrinsics")),
             extrinsics=_ensure_plain_dict(d.get("extrinsics")),
             timestamp_s=d.get("timestamp_s"),
+            role=str(d.get("role") or ""),
         )
 
     def to_mcp_dict(self) -> JsonDict:
@@ -249,59 +258,50 @@ def to_mcp_dict(self) -> JsonDict:
 
             depth_m = depth_uint16 / 1000.0
 
+        Newly normalized OpenCV packets make that wire representation
+        machine-readable with ``depth_encoding="uint16_png"`` and
+        ``depth_scale=1000.0`` (encoded units per metre).  Legacy packets
+        intentionally retain their existing shape for reproducibility.
+
         Values fall within ``[znear, zfar]`` from ``intrinsics`` (metric
         clip planes, in metres).
 
-        **Extrinsics convention** (MuJoCo: MetaWorld / LIBERO).  The dict is
-        self-describing — read the tags rather than assuming a layout:
+        **Extrinsics convention.**  Every dict is self-describing: consumers
+        must read ``camera_frame`` instead of inferring a convention from the
+        simulator name or matrix shape.
+
+        Agent-facing RoboCasa and BEHAVIOR adapters normalize the pose to the
+        same **OpenCV optical** frame used by their returned RGB, metric depth,
+        and intrinsics (+X right, +Y down, +Z forward):
 
         * ``matrix_layout`` — ``"row_major"``
         * ``frame_transform`` — ``"camera_to_world"``
-        * ``camera_frame`` — ``"opengl"`` (camera looks along local **-Z**)
+        * ``camera_frame`` — ``"opencv"``
+        * ``image_origin`` — ``"top_left"``
         * ``pos`` — ``[x, y, z]`` camera position in **world** coordinates
-          (metres), NOT relative to the end-effector
+          (metres), not relative to the end-effector
         * ``mat`` — 3×3 rotation matrix, **camera → world**, flattened
-          **row-major**: ``[m00, m01, m02, m10, m11, m12, m20, m21, m22]``.
-          So ``np.array(mat).reshape(3, 3)`` gives the rotation directly.
-
-          **Columns** = camera-local axes expressed in world::
-
-              col 0 = camera X (right) in world
-              col 1 = camera Y (up) in world
-              col 2 = camera Z (forward) in world
-
-          **Rows** = world axes expressed in camera-local::
+          row-major. ``camera_to_world`` also carries the equivalent 4×4
+          homogeneous matrix.
+        * ``normalized_from`` / ``raw_camera_convention`` — optional debug
+          provenance for the renderer frame normalized by the adapter.
 
-              row 0 = world X in camera
-              row 1 = world Y in camera
-              row 2 = world Z in camera
-
-          Transformation formulas (``R = np.array(mat).reshape(3, 3)``)::
-
-              p_world = R @ p_cam + pos      # camera → world
-              p_cam = R.T @ (p_world - pos)  # world → camera
-
-          The camera looks along **-Z** locally, so the look direction
-          in world coordinates is ``-col2`` (i.e. ``-R[:, 2]``).
-
-        **ManiSkill** (SAPIEN): ``pos`` + ``quat_xyzw`` (reordered from
-        SAPIEN's native wxyz), ``frame_transform="camera_to_world"``,
-        ``camera_frame="ros"`` (camera looks along local **+X**, +Z up).
-
-        **Pixel → world (deprojection).**  Pinhole deprojection yields a
-        point in the **OpenCV optical** frame (X right, Y down, Z forward);
-        convert it into the camera's native frame *before* rotating::
+        For these normalized packets, pinhole deprojection is direct::
 
             x = (u - cx) * d / fx
             y = (v - cy) * d / fy
             p_opencv = np.array([x, y, d])
-            # optical -> camera-native (per ``camera_frame``):
-            #   "opengl" (MuJoCo): p_cam = diag(1, -1, -1) @ p_opencv
-            #   "ros" (ManiSkill): p_cam = np.array([d, -x, -y])
-            p_world = R @ p_cam + pos
-
-        The optical->native flip is mandatory and backend-specific; a
-        correct round-trip recovers object centres to ~2-3 cm.
+            R = np.array(mat).reshape(3, 3)
+            p_world = R @ p_opencv + pos
+
+        LIBERO is deliberately kept on its existing reproducible v1 packet:
+        ``camera_frame="opengl"``, where local +Z is the renderer's backward
+        axis and the camera looks along -Z.  MetaWorld and other legacy
+        MuJoCo adapters currently use the same form.  For those packets,
+        convert the OpenCV point with ``diag(1, -1, -1)`` before applying
+        ``R``.  ManiSkill remains self-described as ``camera_frame="ros"``.
+        The generic ``camera_pose_to_world`` tool accepts these legacy forms;
+        no consumer should guess from the backend name.
         """
         d: JsonDict = {
             "frame_id": self.frame_id,
@@ -314,6 +314,8 @@ def to_mcp_dict(self) -> JsonDict:
         }
         if self.timestamp_s is not None:
             d["timestamp_s"] = self.timestamp_s
+        if self.role:
+            d["role"] = self.role
 
         # RGB → base64 PNG
         if self.rgb:
@@ -326,9 +328,9 @@ def to_mcp_dict(self) -> JsonDict:
             except Exception:
                 pass
 
-        # Depth → base64 PNG (16-bit, scaled to 0–65535)
-        # Also record depth_min/depth_max so the consumer can reconstruct
-        # absolute depth:  depth_m = dmin + (pixel / 65535) * (dmax - dmin)
+        # Depth → uint16 PNG in fixed millimetres. depth_min/depth_max remain
+        # informational scene-range hints; reconstruction is always
+        # depth_m = pixel / 1000.0.
         if self.depth:
             try:
                 _dmin, _dmax = _depth_range(self.depth)
@@ -337,6 +339,16 @@ def to_mcp_dict(self) -> JsonDict:
                     d["depth_base64"] = denc
                     d["depth_min"] = _dmin
                     d["depth_max"] = _dmax
+                    if (
+                        self.extrinsics.get("camera_frame") == "opencv"
+                        and self.extrinsics.get("normalized_from")
+                    ):
+                        # Additive metadata for the new canonical camera
+                        # contract.  Do not alter LIBERO's legacy OpenGL
+                        # packet, whose exact wire shape is reproducibility
+                        # sensitive.
+                        d["depth_encoding"] = "uint16_png"
+                        d["depth_scale"] = 1000.0
                     if not d["width"]:
                         d["width"] = dw
                         d["height"] = dh

agent/runtime/image_artifacts.py

@@ -33,6 +33,7 @@ class ImageArtifact:
     source_field: str
     format: str = "png"
     frame_id: str = ""
+    role: str = ""
     width: int | None = None
     height: int | None = None
     byte_size: int = 0
@@ -48,6 +49,8 @@ def to_dict(self) -> JsonDict:
         }
         if self.frame_id:
             payload["frame_id"] = self.frame_id
+        if self.role:
+            payload["role"] = self.role
         if self.width is not None:
             payload["width"] = self.width
         if self.height is not None:
@@ -151,6 +154,7 @@ def _materialize_dict(
     images: list[ImageArtifact],
 ) -> JsonDict:
     frame_id = str(payload.get("frame_id") or payload.get("camera") or "")
+    role = str(payload.get("role") or "")
     if not frame_id and path_parts:
         frame_id = path_parts[-1]
 
@@ -172,6 +176,7 @@ def _materialize_dict(
             source_field=source_field,
             format=fmt,
             frame_id=frame_id,
+            role=role,
             width=_optional_int(payload.get("width")),
             height=_optional_int(payload.get("height")),
             byte_size=len(data),

agent/runtime/memory.py

@@ -1408,22 +1408,35 @@ def _capture_sam3_selection_state(self, action: EnvAction) -> None:
             parameters = details.get("parameters")
             if not isinstance(parameters, dict):
                 parameters = {}
+            source_camera_role = str(
+                outputs.get("source_camera_role")
+                or details.get("source_camera_role")
+                or ""
+            )
+            source_frame_id = (
+                outputs.get("frame_id")
+                or outputs.get("source_frame_id")
+                or (
+                    details.get("source_frame_id")
+                    if source_camera_role
+                    else None
+                )
+                or parameters.get("frame_id")
+            )
             base = {
                 "result_id": result_id,
                 "target_prompt": outputs.get("prompt") or parameters.get("prompt"),
                 "source_image": outputs.get("source_image") or parameters.get("image"),
-                "frame_id": (
-                    outputs.get("frame_id")
-                    or outputs.get("source_frame_id")
-                    or parameters.get("frame_id")
-                ),
+                "frame_id": source_frame_id,
                 "ranking": outputs.get("ranking") or "score_descending",
                 "candidate_count": len(candidates),
                 "candidates": candidates,
                 "selection_bundle": dict(selection_bundle),
                 "segmentation_mode": outputs.get("segmentation_mode"),
                 "scene_epoch": self.scene_epoch(),
             }
+            if source_camera_role:
+                base["camera_role"] = source_camera_role
             self.facts.pop(REFERENCE_LOCALIZATION_FAILURE_KEY, None)
             asset_reference = self.target_asset_reference()
             if isinstance(asset_reference, dict):
@@ -6360,7 +6373,16 @@ def _camera_packet_from_payload(
         "anygrasp_intrinsics": dict(normalized_intrinsics),
         "extrinsics": dict(extrinsics),
     }
-    for field_name in ("width", "height", "depth_min", "depth_max"):
+    role = camera.get("role")
+    if isinstance(role, str) and role:
+        packet["role"] = role
+    for field_name in (
+        "width",
+        "height",
+        "depth_min",
+        "depth_max",
+        "depth_encoding",
+    ):
         if field_name in camera:
             packet[field_name] = camera[field_name]
     if depth_scale is not None:
@@ -6745,6 +6767,7 @@ def summarize_memory_artifact(artifact: JsonDict) -> JsonDict:
             "mcp_server_url",
             "dashboard_url",
             "frame_id",
+            "role",
             "rgb_path",
             "depth_path",
             "width",