How to tansfer the the coordinate system to the blender's coordinate system

[yjb6]

Thanks for the great work. Now I have the glb file and want to transform and scale it in blender to match the rendering result in G-objaverse. But I failed, the json file provided in G-objaverse seems can't be used directly in Blender when doing transform in Blender. How can I get the righe transform parameters? thanks a lot.

[lingtengqiu]

I saw the original scaling scripts that demonstrate the following operation:

def normalize_scene(mesh):
    bounds = mesh.bounding_box.bounds
    bounds_min = bounds[0]
    bounds_max = bounds[1]
    scale_tmp = max((bounds_max[0] - bounds_min[0]), (bounds_max[1]-bounds_min[1]))
    scale_tmp = max((bounds_max[2] - bounds_min[2]), scale_tmp)
    scale_tmp = 0.9 / scale_tmp

    offset = -(bounds_max + bounds_min) / 2
    return scale_tmp, offset


# pesudo code.
mesh = trimesh.load("xxx.glb", process=False)
scale, offset = normalize_scene(mesh)
new_mesh = scale * (mesh.verts + offset)

Then you can use our providing intrinsic and extrinsic parameters to blender's coordinates.

[lingtengqiu]

I will give you an example to demonstrate how to transfer *.glb to Pytorch3D coordiante. Then, you can simply transfer pytorch3D coordinates to blender coordinate system following this issues and this documents

def generate_obja_camera(view_dir, num_views=24, stride=1, start_idx=0):
    view_dir = Path(view_dir)
    R_lst = []
    T_lst = []
    scale_lst = []

    for view_idx in range(0, num_views, stride):
        view_json = view_dir / f"{view_idx:05d}" / f"{view_idx:05d}.json"
        with open(view_json, "r") as fp:
            cam_json = json.load(fp)

        cam_pos = np.array(cam_json["origin"], dtype=np.float32)
        cam_pos = cam_pos[[0, 2, 1]] * np.array([1, 1, -1], dtype=np.float32)  # unity to pytorch3d?

        pytorch3d_R = look_at_rotation(torch.from_numpy(cam_pos[None, :]))
        # camera.set_intrin_params(img_h, img_w, focal, max(img_w, img_h))
        pytorch3d_T = -pytorch3d_R[0].T @ torch.from_numpy(cam_pos).to(pytorch3d_R)

        R_lst.append(pytorch3d_R)
        T_lst.append(pytorch3d_T)
        # scale_lst.append(torch.from_numpy(scale[None, :]).to(pytorch3d_R))

    assert cam_json["x_fov"] == cam_json["y_fov"]
    fov = cam_json["x_fov"]
    R_lst = R_lst[start_idx:] + R_lst[:start_idx]
    T_lst = T_lst[start_idx:] + T_lst[:start_idx]
    R_lst.append(R_lst[0])
    T_lst.append(T_lst[0])
    # scale_lst.append(scale_lst[0])
    Rs = torch.cat(R_lst, dim=0)
    Ts = torch.stack(T_lst, dim=0)
    # scales = torch.cat(scale_lst, dim=0)
    mesh_scale = cam_json["scale"][0]
    mesh_offset = torch.from_numpy(np.array(cam_json["offset"], dtype=np.float32)).to(pytorch3d_R)
    # mesh_offset = mesh_offset[[0, 2, 1]]

    return Rs, Ts, fov, mesh_scale, mesh_offset


def normalize_scene(mesh):
    bounds = mesh.bounding_box.bounds
    bounds_min = bounds[0]
    bounds_max = bounds[1]
    scale_tmp = max((bounds_max[0] - bounds_min[0]), (bounds_max[1]-bounds_min[1]))
    scale_tmp = max((bounds_max[2] - bounds_min[2]), scale_tmp)
    scale_tmp = 0.9 / scale_tmp

    offset = -(bounds_max + bounds_min) / 2
    return scale_tmp, offset

# Rs: Rotation Matrix, Ts: Translation Matrix, Fov
Rs, Ts, fov, mesh_scale, mesh_offset = generate_obja_camera(image_dir, start_idx=start_idx)

    
mesh = trimesh.load(mesh_path)
scale_tmp, offset = normalize_scene(mesh)
offset = torch.from_numpy(offset).to(mesh_offset)
mesh.export("./tmp.obj")

# for rednering
# following the pytorch3D or blender.
.......

[yjb6]

Thank you for your assistance, but I am still confused.

For certain cases (e.g., Gobjda: 458441, obja: ae857bd7749f46fd8829be990788b809.glb), the scale computed from the normalize_scene function differs from the scale provided in the JSON file.

def normalize_scene(mesh):
    bounds = mesh.bounding_box.bounds
    bounds_min = bounds[0]
    bounds_max = bounds[1]
    scale_tmp = max((bounds_max[0] - bounds_min[0]), (bounds_max[1] - bounds_min[1]))
    scale_tmp = max((bounds_max[2] - bounds_min[2]), scale_tmp)
    scale_tmp = 0.9 / scale_tmp

    offset = -(bounds_max + bounds_min) / 2
    return scale_tmp, offset

# Pseudo code.
mesh = trimesh.load("C:\project\datasets\obja_glb/ae857bd7749f46fd8829be990788b809.glb", force="mesh")
scale, offset = normalize_scene(mesh)
print(scale, offset)

Output:

0.4256730844402861 [ 1.2111664e-08 -5.0454730e-01  6.8111748e-01]

However, the scale and offset provided in the JSON file are:

When I attempt to use the computed scale and offset to transform the glb file and compare it with the point cloud (pcd) fused from the depth map, there is a misalignment. The black points represent the pcd fused from the depth map, and the noise points can be ignored.

Why does this happen? How can I obtain the correctly transformed mesh to align with the fused pcd and the rendered RGB/depth?

Thank you very much. The code for fusing the depth map is as follows:

`   folder ="../datasets/458441/campos_512_v4"
    def depth_2_pcd(depths, c2ws, intrinsic):
        '''
        depths:[V,H,W]
        c2w: [V,4,4]
        intrinsic: [3,3]
        '''
        desired_num_points = 2048

        intrinsic = o3d.camera.PinholeCameraIntrinsic(width=depths.shape[2], height=depths.shape[1],
                                                    fx=intrinsic[0, 0], fy=intrinsic[1, 1],
                                                    cx=intrinsic[0, 2], cy=intrinsic[1, 2])

        xyz_list = []
        merged_pcd = o3d.geometry.PointCloud()

        for i in range(depths.shape[0]):
            depth = depths[i]
            c2w = c2ws[i]
            depth= np.ascontiguousarray(depth, dtype=np.float32)
            depth_image = o3d.geometry.Image(depth)

            pcd = o3d.geometry.PointCloud.create_from_depth_image(depth_image, intrinsic)
            points = np.asarray(pcd.points)
            points[:,2] = -points[:,2] #open3d 是基于opencv的相机坐标系，所以要进行变换
            points[:,1] = -points[:,1]

            # print(depth,points)
            pcd.points = o3d.utility.Vector3dVector(points)
            # o3d.visualization.draw_geometries([pcd])
            # o3d.io.write_point_cloud(f"pcd_{i}.ply", pcd)

            pcd_transformed = pcd.transform(c2w.astype(np.float64))
            # o3d.io.write_point_cloud(f"pcd{i}.ply", pcd)
            merged_pcd += pcd_transformed


        xyz = np.asarray(merged_pcd.points)

        return xyz

    def read_dnormal(normald_path, cond_pos):
        cond_cam_dis = np.linalg.norm(cond_pos, 2)

        near = 0.867 #sqrt(3) * 0.5
        near_distance = cond_cam_dis - near

        normald = cv2.imread(normald_path, cv2.IMREAD_UNCHANGED).astype(np.float32)
        depth = normald[...,3:]

        depth[depth<near_distance] = 0

        return depth

    def read_camera_matrix_single(json_file):
        print(json_file)
        with open(json_file, 'r', encoding='utf8') as reader:
            json_content = json.load(reader)

        # NOTE that different from unity2blender experiments.
        camera_matrix = np.eye(4)
        camera_matrix[:3, 0] = np.array(json_content['x'])
        camera_matrix[:3, 1] = -np.array(json_content['y'])
        camera_matrix[:3, 2] = -np.array(json_content['z'])
        camera_matrix[:3, 3] = np.array(json_content['origin'])


        # '''
        # camera_matrix = np.eye(4)
        # camera_matrix[:3, 0] = np.array(json_content['x'])
        # camera_matrix[:3, 1] = np.array(json_content['y'])
        # camera_matrix[:3, 2] = np.array(json_content['z'])
        # camera_matrix[:3, 3] = np.array(json_content['origin'])
        # print(camera_matrix)
        # '''

        return camera_matrix

    def get_intr(target_im):
        h, w = target_im.shape[:2]

        fx = fy = 1422.222
        res_raw = 1024
        f_x = f_y = fx * h / res_raw
        K = torch.tensor([f_x, 0, w / 2, 0, f_y, h / 2, 0, 0, 1]).reshape(3, 3)
        # print("intr: ", K)
        return K
    images = []
    masks = []
    cam_poses = []
    depths = []
    vid_cnt = 0


    for name in os.listdir(folder):
        depth_filename = os.path.join(folder,name, name+"_nd.exr")
        json_name = os.path.join(folder, name,name+".json")

        print(json_name)
        c2w = read_camera_matrix_single(json_name)
        # print(c2w)
        depth = read_dnormal(depth_filename, c2w)

        cam_poses.append(c2w)
        depths.append(depth)
        intrinsic = get_intr(depth).numpy()
    cam_poses = np.stack(cam_poses, axis=0) # [V, 4, 4]
    depths = np.stack(depths, axis=0)



    pcd_list = []

    pcd = depth_2_pcd(depths, cam_poses, intrinsic)
    pcd_fuse = o3d.geometry.PointCloud()
    pcd_fuse.points = o3d.utility.Vector3dVector(pcd)

    o3d.io.write_point_cloud("fuse.ply", pcd_fuse)
`
```python

[lingtengqiu]

the offset key is not used in JSON file; this is a bug for rendering system. Please use (normalize)[https://github.com//issues/53#issuecomment-2366150631] to get scale and offset

[yjb6]

Thank you for your prompt response. However, when I use the scale and offset computed from normalize_scene, there is a misalignment between the transformed object and the point cloud fused from the depth map, as illustrated in the example I provided in https://github.com/modelscope/richdreamer/issues/53#issuecomment-2370701004. Why does this occur?