merge pull request #1 from kotleni/update-alvr-sdk

update alvr sdk + disable foveation encoding

ALVR/alvr/client_core/Cargo.toml

@@ -23,10 +23,13 @@ alvr_sockets.workspace = true
 app_dirs2 = "2"
 bincode = "1"
 glyph_brush_layout = "0.2"
+jni = "0.21"
+local-ip-address = "0.5"
+mdns-sd = "0.10"
 rand = "0.8"
 serde = "1"
 serde_json = "1"
-jni = "0.21"
+whoami = "1"
 
 [target.'cfg(target_os = "android")'.dependencies]
 android_logger = "0.13"
@@ -40,10 +43,7 @@ ndk-sys = "0.5"
 oboe = "0.5" # todo: remove once AudioThread shutdown crash is fixed
 
 [target.'cfg(not(target_os = "android"))'.dependencies]
-env_logger = "0.10"
-
-[target.'cfg(not(any(target_os = "android", target_os = "macos")))'.dependencies]
-local-ip-address = "0.5"
+env_logger = "0.11"
 
 [build-dependencies]
 bindgen = "0.69"

ALVR/alvr/client_core/src/c_api.rs

@@ -1,6 +1,6 @@
 use crate::{
     opengl::{self, RenderViewInput},
-    ClientCoreEvent,
+    storage, ClientCoreEvent,
 };
 use alvr_common::{
     debug, error,
@@ -10,7 +10,7 @@ use alvr_common::{
     parking_lot::Mutex,
     warn, DeviceMotion, Fov, Pose,
 };
-use alvr_packets::{ButtonEntry, ButtonValue, Tracking};
+use alvr_packets::{ButtonEntry, ButtonValue, FaceData, Tracking};
 use alvr_session::{CodecType, FoveatedEncodingConfig};
 use std::{
     collections::VecDeque,
@@ -25,9 +25,11 @@ struct ReconstructedNal {
     timestamp_ns: u64,
     data: Vec<u8>,
 }
+
+static HUD_MESSAGE: Lazy<Mutex<String>> = Lazy::new(|| Mutex::new("".into()));
+static SETTINGS: Lazy<Mutex<String>> = Lazy::new(|| Mutex::new("".into()));
 static NAL_QUEUE: Lazy<Mutex<VecDeque<ReconstructedNal>>> =
     Lazy::new(|| Mutex::new(VecDeque::new()));
-static HUD_MESSAGE: Lazy<Mutex<String>> = Lazy::new(|| Mutex::new("".into()));
 
 #[repr(u8)]
 pub enum AlvrCodec {
@@ -43,13 +45,7 @@ pub enum AlvrEvent {
         view_width: u32,
         view_height: u32,
         refresh_rate_hint: f32,
-        enable_foveation: bool,
-        foveation_center_size_x: f32,
-        foveation_center_size_y: f32,
-        foveation_center_shift_x: f32,
-        foveation_center_shift_y: f32,
-        foveation_edge_ratio_x: f32,
-        foveation_edge_ratio_y: f32,
+        enable_foveated_encoding: bool,
     },
     StreamingStopped,
     Haptics {
@@ -58,7 +54,8 @@ pub enum AlvrEvent {
         frequency: f32,
         amplitude: f32,
     },
-    CreateDecoder {
+    // Note: All subsequent DecoderConfig events should be ignored until reconnection
+    DecoderConfig {
         codec: AlvrCodec,
     },
     FrameReady,
@@ -84,10 +81,16 @@ pub struct AlvrQuat {
 
 #[repr(C)]
 #[derive(Clone, Default)]
-pub struct AlvrDeviceMotion {
-    device_id: u64,
+pub struct AlvrPose {
     orientation: AlvrQuat,
     position: [f32; 3],
+}
+
+#[repr(C)]
+#[derive(Clone, Default)]
+pub struct AlvrDeviceMotion {
+    device_id: u64,
+    pose: AlvrPose,
     linear_velocity: [f32; 3],
     angular_velocity: [f32; 3],
 }
@@ -130,30 +133,60 @@ pub unsafe extern "C" fn alvr_log_time(tag: *const c_char) {
     error!("[ALVR NATIVE] {tag}: {:?}", Instant::now());
 }
 
+fn string_to_c_str(buffer: *mut c_char, value: &str) -> u64 {
+    let cstring = CString::new(value).unwrap();
+    if !buffer.is_null() {
+        unsafe {
+            ptr::copy_nonoverlapping(cstring.as_ptr(), buffer, cstring.as_bytes_with_nul().len());
+        }
+    }
+
+    cstring.as_bytes_with_nul().len() as u64
+}
+
+#[no_mangle]
+pub extern "C" fn alvr_mdns_service(service_buffer: *mut c_char) -> u64 {
+    string_to_c_str(service_buffer, alvr_sockets::MDNS_SERVICE_TYPE)
+}
+
+// To make sure the value is correct, call after alvr_initialize()
+#[no_mangle]
+pub extern "C" fn alvr_hostname(hostname_buffer: *mut c_char) -> u64 {
+    string_to_c_str(hostname_buffer, &storage::Config::load().hostname)
+}
+
+// To make sure the value is correct, call after alvr_initialize()
+#[no_mangle]
+pub extern "C" fn alvr_protocol_id(protocol_buffer: *mut c_char) -> u64 {
+    string_to_c_str(protocol_buffer, &storage::Config::load().protocol_id)
+}
+
 /// On non-Android platforms, java_vm and constext should be null.
 /// NB: context must be thread safe.
 #[allow(unused_variables)]
 #[no_mangle]
 pub unsafe extern "C" fn alvr_initialize(
     java_vm: *mut c_void,
     context: *mut c_void,
-    recommended_view_width: u32,
-    recommended_view_height: u32,
+    default_view_width: u32,
+    default_view_height: u32,
     refresh_rates: *const f32,
     refresh_rates_count: i32,
+    supports_foveated_encoding: bool,
     external_decoder: bool,
 ) {
     #[cfg(target_os = "android")]
     ndk_context::initialize_android_context(java_vm, context);
 
-    let recommended_view_resolution = UVec2::new(recommended_view_width, recommended_view_height);
+    let default_view_resolution = UVec2::new(default_view_width, default_view_height);
 
     let supported_refresh_rates =
         slice::from_raw_parts(refresh_rates, refresh_rates_count as _).to_vec();
 
     crate::initialize(
-        recommended_view_resolution,
+        default_view_resolution,
         supported_refresh_rates,
+        supports_foveated_encoding,
         external_decoder,
     );
 }
@@ -187,34 +220,16 @@ pub extern "C" fn alvr_poll_event(out_event: *mut AlvrEvent) -> bool {
                 AlvrEvent::HudMessageUpdated
             }
             ClientCoreEvent::StreamingStarted {
-                view_resolution,
-                refresh_rate_hint,
                 settings,
+                negotiated_config,
             } => {
-                let foveated_encoding = settings.video.foveated_encoding.as_option();
+                *SETTINGS.lock() = serde_json::to_string(&settings).unwrap();
+
                 AlvrEvent::StreamingStarted {
-                    view_width: view_resolution.x,
-                    view_height: view_resolution.y,
-                    refresh_rate_hint,
-                    enable_foveation: foveated_encoding.is_some(),
-                    foveation_center_size_x: foveated_encoding
-                        .map(|f| f.center_size_x)
-                        .unwrap_or_default(),
-                    foveation_center_size_y: foveated_encoding
-                        .map(|f| f.center_size_y)
-                        .unwrap_or_default(),
-                    foveation_center_shift_x: foveated_encoding
-                        .map(|f| f.center_shift_x)
-                        .unwrap_or_default(),
-                    foveation_center_shift_y: foveated_encoding
-                        .map(|f| f.center_shift_y)
-                        .unwrap_or_default(),
-                    foveation_edge_ratio_x: foveated_encoding
-                        .map(|f| f.edge_ratio_x)
-                        .unwrap_or_default(),
-                    foveation_edge_ratio_y: foveated_encoding
-                        .map(|f| f.edge_ratio_y)
-                        .unwrap_or_default(),
+                    view_width: negotiated_config.view_resolution.x,
+                    view_height: negotiated_config.view_resolution.y,
+                    refresh_rate_hint: negotiated_config.refresh_rate_hint,
+                    enable_foveated_encoding: negotiated_config.enable_foveated_encoding,
                 }
             }
             ClientCoreEvent::StreamingStopped => AlvrEvent::StreamingStopped,
@@ -229,13 +244,13 @@ pub extern "C" fn alvr_poll_event(out_event: *mut AlvrEvent) -> bool {
                 frequency,
                 amplitude,
             },
-            ClientCoreEvent::MaybeCreateDecoder { codec, config_nal } => {
+            ClientCoreEvent::DecoderConfig { codec, config_nal } => {
                 NAL_QUEUE.lock().push_back(ReconstructedNal {
                     timestamp_ns: 0,
                     data: config_nal,
                 });
 
-                AlvrEvent::CreateDecoder {
+                AlvrEvent::DecoderConfig {
                     codec: match codec {
                         CodecType::H264 => AlvrCodec::H264,
                         CodecType::Hevc => AlvrCodec::Hevc,
@@ -261,6 +276,12 @@ pub extern "C" fn alvr_poll_event(out_event: *mut AlvrEvent) -> bool {
     }
 }
 
+// Settings will be updated after receiving StreamingStarted event
+#[no_mangle]
+pub extern "C" fn alvr_get_settings_json(buffer: *mut c_char) -> u64 {
+    string_to_c_str(buffer, &SETTINGS.lock())
+}
+
 /// Call only with external decoder
 /// Returns the number of bytes of the next nal, or 0 if there are no nals ready.
 /// If out_nal or out_timestamp_ns is null, no nal is dequeued. Use to get the nal allocation size.
@@ -347,11 +368,19 @@ pub extern "C" fn alvr_send_button(path_id: u64, value: AlvrButtonValue) {
     crate::send_buttons(vec![ButtonEntry { path_id, value }]);
 }
 
+/// hand_skeleton:
+/// * outer ptr: array of 2 (can be null);
+/// * inner ptr: array of 26 (can be null if hand is absent)
+/// eye_gazes:
+/// * outer ptr: array of 2 (can be null);
+/// * inner ptr: pose (can be null if eye gaze is absent)
 #[no_mangle]
 pub extern "C" fn alvr_send_tracking(
     target_timestamp_ns: u64,
     device_motions: *const AlvrDeviceMotion,
     device_motions_count: u64,
+    hand_skeletons: *const *const AlvrPose,
+    eye_gazes: *const *const AlvrPose,
 ) {
     fn from_capi_quat(quat: AlvrQuat) -> Quat {
         Quat::from_xyzw(quat.x, quat.y, quat.z, quat.w)
@@ -373,8 +402,8 @@ pub extern "C" fn alvr_send_tracking(
                 motion.device_id,
                 DeviceMotion {
                     pose: Pose {
-                        orientation: from_capi_quat(motion.orientation),
-                        position: Vec3::from_slice(&motion.position),
+                        orientation: from_capi_quat(motion.pose.orientation),
+                        position: Vec3::from_slice(&motion.pose.position),
                     },
                     linear_velocity: Vec3::from_slice(&motion.linear_velocity),
                     angular_velocity: Vec3::from_slice(&motion.angular_velocity),
@@ -383,10 +412,66 @@ pub extern "C" fn alvr_send_tracking(
         })
         .collect::<Vec<_>>();
 
+    let hand_skeletons = if !hand_skeletons.is_null() {
+        let hand_skeletons = unsafe { slice::from_raw_parts(hand_skeletons, 2) };
+        let hand_skeletons = hand_skeletons
+            .iter()
+            .map(|&hand_skeleton| {
+                if !hand_skeleton.is_null() {
+                    let hand_skeleton = unsafe { slice::from_raw_parts(hand_skeleton, 26) };
+
+                    let mut array = [Pose::default(); 26];
+
+                    for (pose, capi_pose) in array.iter_mut().zip(hand_skeleton.iter()) {
+                        *pose = Pose {
+                            orientation: from_capi_quat(capi_pose.orientation),
+                            position: Vec3::from_slice(&capi_pose.position),
+                        };
+                    }
+
+                    Some(array)
+                } else {
+                    None
+                }
+            })
+            .collect::<Vec<_>>();
+
+        [hand_skeletons[0], hand_skeletons[1]]
+    } else {
+        [None, None]
+    };
+
+    let eye_gazes = if !eye_gazes.is_null() {
+        let eye_gazes = unsafe { slice::from_raw_parts(eye_gazes, 2) };
+        let eye_gazes = eye_gazes
+            .iter()
+            .map(|&eye_gaze| {
+                if !eye_gaze.is_null() {
+                    let eye_gaze = unsafe { &*eye_gaze };
+
+                    Some(Pose {
+                        orientation: from_capi_quat(eye_gaze.orientation),
+                        position: Vec3::from_slice(&eye_gaze.position),
+                    })
+                } else {
+                    None
+                }
+            })
+            .collect::<Vec<_>>();
+
+        [eye_gazes[0], eye_gazes[1]]
+    } else {
+        [None, None]
+    };
+
     let tracking = Tracking {
         target_timestamp: Duration::from_nanos(target_timestamp_ns),
         device_motions,
-        ..Default::default()
+        hand_skeletons,
+        face_data: FaceData {
+            eye_gazes,
+            ..Default::default()
+        },
     };
 
     crate::send_tracking(tracking);
@@ -528,6 +613,7 @@ pub unsafe extern "C" fn alvr_start_stream_opengl(config: AlvrStreamConfig) {
     let swapchain_textures =
         convert_swapchain_array(config.swapchain_textures, config.swapchain_length);
     let foveated_encoding = config.enable_foveation.then_some(FoveatedEncodingConfig {
+        force_enable: true,
         center_size_x: config.foveation_center_size_x,
         center_size_y: config.foveation_center_size_y,
         center_shift_x: config.foveation_center_shift_x,