Skip to content

Commit 9cc6893

Browse files
angli66angli66
committed
add manualtest for StereoDepthSensor
1 parent 9d03572 commit 9cc6893

File tree

7 files changed

+282
-137
lines changed

7 files changed

+282
-137
lines changed

‎docker_install_39.sh

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
#!/bin/bash
2+
rm -rf sapien.egg-info
3+
rm -f wheelhouse/*.whl
4+
if [ -z ${VERSION} ]
5+
then
6+
echo VERSION variable is not specified
7+
VERSION=2.0.0.dev$(date +"%Y%m%d")
8+
echo VERSION defaults to ${VERSION}
9+
sleep 3
10+
fi
11+
12+
echo ${VERSION} > python/VERSION
13+
echo __version__=\"${VERSION}\" > python/py_package/version.py
14+
15+
docker run -v `pwd`:/workspace/SAPIEN -it --rm \
16+
-u $(id -u ${USER}):$(id -g ${USER}) \
17+
fxiangucsd/sapien-build-env:latest bash -c "cd /workspace/SAPIEN && ./build.sh 39"
18+
19+
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
20+
21+
cd wheelhouse
22+
pip3 uninstall -y sapien
23+
pip3 install *
24+
25+
cd /tmp && rm stubs -rf && pybind11-stubgen sapien.core --ignore-invalid all
26+
cp /tmp/stubs/sapien/core-stubs/__init__.pyi $DIR/python/py_package/core
27+
cp -r /tmp/stubs/sapien/core-stubs/pysapien $DIR/python/py_package/core

‎manualtest/rt_sensor_cuda.py renamed to ‎manualtest/rt_stereo_sensor.py

Lines changed: 42 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -1,32 +1,13 @@
11
import sapien.core as sapien
22
import numpy as np
33
import transforms3d.euler
4-
from sapien.core import Pose
5-
from sapien.sensor import ActiveLightSensorCUDA
64
import PIL.Image as im
75
import matplotlib.pyplot as plt
8-
import time
9-
10-
from tqdm import trange
11-
12-
def main():
13-
sim = sapien.Engine()
14-
15-
sapien.render_config.camera_shader_dir = "../vulkan_shader/rt"
16-
sapien.render_config.viewer_shader_dir = "../vulkan_shader/rt"
17-
sapien.render_config.rt_samples_per_pixel = 32
18-
sapien.render_config.rt_max_path_depth = 8
19-
sapien.render_config.rt_use_denoiser = True
20-
renderer = sapien.VulkanRenderer()
21-
22-
# config = sapien.KuafuConfig()
23-
# config.spp = 32
24-
# config.max_bounces = 8
25-
# config.use_denoiser = True
26-
# renderer = sapien.KuafuRenderer(config)
27-
28-
sim.set_renderer(renderer)
6+
import open3d as o3d
7+
from sapien.core import Pose
8+
from sapien.sensor import StereoDepthSensor, StereoDepthSensorConfig
299

10+
def build_scene(sim, renderer):
3011
scene_config = sapien.SceneConfig()
3112
scene = sim.create_scene(scene_config)
3213

@@ -53,7 +34,6 @@ def main():
5334
material.base_color = [1.0, 1.0, 1.0, 1.0]
5435
material.roughness = 0.3
5536
material.metallic = 0.0
56-
# material.metallic = 0.1
5737
builder.add_sphere_visual(radius=0.07, material=material)
5838
builder.add_sphere_collision(radius=0.07)
5939
sphere2 = builder.build()
@@ -87,51 +67,51 @@ def main():
8767
builder.add_visual_from_file(
8868
'../3rd_party/kuafu/resources/models/suzanne.dae', scale=[0.1, 0.1, 0.1], material=material)
8969
builder.add_box_collision(half_size=[0.1, 0.1, 0.1])
90-
box = builder.build()
91-
box.set_pose(Pose(p=[0.15, -0.25, 0.1], q=transforms3d.euler.euler2quat(0, 0, -1)))
70+
monkey = builder.build()
71+
monkey.set_pose(Pose(p=[0.15, -0.25, 0.1], q=transforms3d.euler.euler2quat(0, 0, -1)))
9272

9373
scene.set_ambient_light([0.3, 0.3, 0.3])
9474
scene.add_directional_light([0, 0.5, -1], color=[3.0, 3.0, 3.0])
9575

96-
sensor_cuda = ActiveLightSensorCUDA(
97-
'sensor', renderer, scene, sensor_type='d415')
98-
sensor_cuda.depth_sensor.set_penalties(p1_penalty=7, p2_penalty=86)
99-
sensor_cuda.depth_sensor.set_uniqueness_ratio(15)
76+
return scene
10077

101-
sensor_cuda.set_pose(
78+
def main():
79+
sim = sapien.Engine()
80+
81+
renderer = sapien.SapienRenderer()
82+
sim.set_renderer(renderer)
83+
sapien.render_config.camera_shader_dir = "rt"
84+
sapien.render_config.rt_samples_per_pixel = 32
85+
sapien.render_config.rt_max_path_depth = 8
86+
sapien.render_config.rt_use_denoiser = True
87+
88+
scene = build_scene(sim, renderer)
89+
90+
sensor_config = StereoDepthSensorConfig()
91+
sensor = StereoDepthSensor('sensor', scene, sensor_config)
92+
sensor.set_pose(
10293
Pose([-0.28, -0.28, 0.46], [0.8876263, -0.135299, 0.3266407, 0.2951603]))
10394

104-
# scene.step()
105-
# scene.update_render()
106-
# sensor_cuda.take_picture()
107-
# depth_cuda = sensor_cuda.get_depth()
108-
109-
# rgb = sensor_cuda.get_rgb()
110-
# im.fromarray((rgb * 255).astype(np.uint8)).show()
111-
# start = time.process_time()
112-
# depth_cuda = sensor_cuda.get_depth()
113-
# print("Runtime of get_depth() for gpu sensor: ", time.process_time() - start)
114-
# plt.figure()
115-
# plt.imshow(depth_cuda)
116-
# plt.figure()
117-
# ir = sensor_cuda.get_ir()
118-
# print(np.sum(np.isnan(ir[0])))
119-
# plt.imshow(ir[0])
120-
# plt.show()
121-
122-
123-
from sapien.utils import Viewer
124-
viewer = Viewer(renderer)
125-
viewer.set_scene(scene)
126-
viewer.set_camera_xyz(x=-1, y=0, z=0.5)
127-
viewer.set_camera_rpy(r=0, p=-np.arctan2(2, 4), y=0)
128-
129-
while not viewer.closed:
130-
scene.step()
131-
scene.update_render()
132-
viewer.render()
133-
tex = viewer.window.get_float_texture("Color")
134-
print(np.sum(np.isnan(tex)))
95+
scene.step()
96+
scene.update_render()
97+
sensor.take_picture()
98+
sensor.compute_depth()
99+
depth = sensor.get_depth()
100+
101+
rgb = sensor.get_rgb()
102+
im.fromarray((rgb * 255).astype(np.uint8)).show()
103+
104+
ir_l, ir_r = sensor.get_ir()
105+
im.fromarray((ir_l * 255).astype(np.uint8)).show()
106+
im.fromarray((ir_r * 255).astype(np.uint8)).show()
107+
108+
plt.imshow(depth)
109+
plt.show()
110+
111+
pc = sensor.get_pointcloud(frame='world', with_rgb=True)
112+
pc1 = o3d.geometry.PointCloud(o3d.utility.Vector3dVector(pc[..., :3]))
113+
pc1.colors = o3d.utility.Vector3dVector(pc[..., 3:])
114+
o3d.visualization.draw_geometries([pc1])
135115

136116

137117
main()

‎manualtest/rt_stereo_sensor2.py

Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,128 @@
1+
import sapien.core as sapien
2+
import numpy as np
3+
import transforms3d.euler
4+
import PIL.Image as im
5+
import matplotlib.pyplot as plt
6+
import open3d as o3d
7+
import torch
8+
from sapien.core import Pose
9+
from sapien.sensor import StereoDepthSensor, StereoDepthSensorConfig
10+
from tqdm import tqdm
11+
12+
def build_scene(sim, renderer):
13+
scene_config = sapien.SceneConfig()
14+
scene = sim.create_scene(scene_config)
15+
16+
ground_material = renderer.create_material()
17+
ground_material.base_color = np.array([202, 164, 114, 256]) / 256
18+
ground_material.specular = 0.5
19+
scene.add_ground(0, render_material=ground_material)
20+
scene.set_timestep(1 / 240)
21+
22+
builder = scene.create_actor_builder()
23+
material = renderer.create_material()
24+
material.base_color = [0.2, 0.2, 0.8, 1.0]
25+
material.roughness = 0.5
26+
material.metallic = 0.0
27+
builder.add_sphere_visual(radius=0.06, material=material)
28+
builder.add_sphere_collision(radius=0.06)
29+
sphere1 = builder.build()
30+
sphere1.set_pose(Pose(p=[-0.05, 0.05, 0.06]))
31+
32+
builder = scene.create_actor_builder()
33+
material = renderer.create_material()
34+
material.ior = 1.2
35+
material.transmission = 1.0
36+
material.base_color = [1.0, 1.0, 1.0, 1.0]
37+
material.roughness = 0.3
38+
material.metallic = 0.0
39+
builder.add_sphere_visual(radius=0.07, material=material)
40+
builder.add_sphere_collision(radius=0.07)
41+
sphere2 = builder.build()
42+
sphere2.set_pose(Pose(p=[0.05, -0.05, 0.07]))
43+
44+
builder = scene.create_actor_builder()
45+
material = renderer.create_material()
46+
material.base_color = [0.8, 0.7, 0.1, 1.0]
47+
material.roughness = 0.01
48+
material.metallic = 0.95
49+
builder.add_capsule_visual(radius=0.02, half_length=0.1, material=material)
50+
builder.add_capsule_collision(radius=0.02, half_length=0.1)
51+
cap = builder.build()
52+
cap.set_pose(Pose(p=[0.15, -0.01, 0.01], q=transforms3d.euler.euler2quat(0, 0, -0.7)))
53+
54+
builder = scene.create_actor_builder()
55+
material = renderer.create_material()
56+
material.base_color = [0.8, 0.2, 0.2, 1.0]
57+
material.roughness = 0.01
58+
material.metallic = 1.0
59+
builder.add_box_visual(half_size=[0.09, 0.09, 0.09], material=material)
60+
builder.add_box_collision(half_size=[0.09, 0.09, 0.09])
61+
box = builder.build()
62+
box.set_pose(Pose(p=[0.05, 0.17, 0.09]))
63+
64+
builder = scene.create_actor_builder()
65+
material = renderer.create_material()
66+
material.base_color = [0.9, 0.6, 0.5, 1.0]
67+
material.roughness = 0.0
68+
material.metallic = 1.0
69+
builder.add_visual_from_file(
70+
'../3rd_party/kuafu/resources/models/suzanne.dae', scale=[0.1, 0.1, 0.1], material=material)
71+
builder.add_box_collision(half_size=[0.1, 0.1, 0.1])
72+
monkey = builder.build()
73+
monkey.set_pose(Pose(p=[0.15, -0.25, 0.1], q=transforms3d.euler.euler2quat(0, 0, -1)))
74+
75+
scene.set_ambient_light([0.3, 0.3, 0.3])
76+
scene.add_directional_light([0, 0.5, -1], color=[3.0, 3.0, 3.0])
77+
78+
return scene
79+
80+
def main():
81+
sim = sapien.Engine()
82+
83+
renderer = sapien.SapienRenderer()
84+
sim.set_renderer(renderer)
85+
sapien.render_config.camera_shader_dir = "rt"
86+
sapien.render_config.rt_samples_per_pixel = 8
87+
sapien.render_config.rt_max_path_depth = 8
88+
sapien.render_config.rt_use_denoiser = True
89+
90+
scene = build_scene(sim, renderer)
91+
92+
sensor_config = StereoDepthSensorConfig()
93+
sensor_config.rgb_resolution = (480, 270)
94+
sensor_config.rgb_intrinsic = np.array([
95+
[345., 0., 240.],
96+
[0., 345., 135.],
97+
[0., 0., 1.]
98+
])
99+
sensor_config.ir_resolution = (640, 360)
100+
sensor_config.ir_intrinsic = np.array([
101+
[460., 0., 320.],
102+
[0., 460., 180.],
103+
[0., 0., 1.]
104+
])
105+
sensor_config.max_disp = 96 # Max disparity for stereo matching
106+
sensor = StereoDepthSensor('sensor', scene, sensor_config)
107+
sensor.set_pose(
108+
Pose([-0.28, -0.28, 0.46], [0.8876263, -0.135299, 0.3266407, 0.2951603]))
109+
110+
# Warm up
111+
scene.step()
112+
scene.update_render()
113+
sensor.take_picture()
114+
sensor.compute_depth()
115+
116+
for _ in tqdm(range(100)):
117+
scene.step()
118+
scene.update_render()
119+
sensor.take_picture(infrared_only=True)
120+
sensor.compute_depth()
121+
torch.utils.dlpack.from_dlpack(sensor.get_depth_dl_tensor())
122+
123+
depth = sensor.get_depth()
124+
plt.imshow(depth)
125+
plt.show()
126+
127+
128+
main()

‎python/py_package/core/__init__.pyi

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@ from sapien.core.pysapien import RenderServer
6262
from sapien.core.pysapien import RenderServerBuffer
6363
from sapien.core.pysapien import RenderShape
6464
from sapien.core.pysapien import RenderTexture
65+
from sapien.core.pysapien import SapienRenderer
6566
from sapien.core.pysapien import Scene
6667
from sapien.core.pysapien import SceneConfig
6768
from sapien.core.pysapien import SceneMultistepCallback
@@ -74,7 +75,6 @@ from sapien.core.pysapien import URDFLoader
7475
from sapien.core.pysapien import VisualRecord
7576
from sapien.core.pysapien import VulkanParticleBody
7677
from sapien.core.pysapien import VulkanRenderMesh
77-
from sapien.core.pysapien import VulkanRenderer
7878
from sapien.core.pysapien import VulkanRigidbody
7979
from sapien.core.pysapien import VulkanScene
8080
from sapien.core.pysapien import VulkanWindow
@@ -152,6 +152,7 @@ __all__ = [
152152
"RenderServerBuffer",
153153
"RenderShape",
154154
"RenderTexture",
155+
"SapienRenderer",
155156
"Scene",
156157
"SceneConfig",
157158
"SceneMultistepCallback",
@@ -187,6 +188,8 @@ __all__ = [
187188

188189
class KuafuRenderer(pysapien.IPxrRenderer):
189190
pass
191+
class VulkanRenderer(pysapien.SapienRenderer, pysapien.IPxrRenderer):
192+
pass
190193
def add_profiler_event(name: str) -> None:
191194
pass
192195
def get_global_render_config() -> pysapien.RenderConfig:

0 commit comments

Comments
 (0)