Skip to content

Commit ee03026

Browse files
cloudwebrtchiroshihoriegraszka22
committed
Simulcast support for iOS/Android.
Simulcast support for iOS SDK (#4) Support for simulcast in Android SDK (#3) include simulcast headers for mac also (#10) Fix simulcast using hardware encoder on Android (#48) Co-authored-by: Hiroshi Horie <548776+hiroshihorie@users.noreply.github.com> Co-authored-by: Angelika Serwa <angelika.serwa@gmail.com>
1 parent 272127d commit ee03026

File tree

12 files changed

+275
-12
lines changed

12 files changed

+275
-12
lines changed

‎sdk/BUILD.gn

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -741,6 +741,7 @@ if (is_ios || is_mac) {
741741
]
742742

743743
deps = [
744+
":simulcast",
744745
":base_objc",
745746
":native_video",
746747
":videocodec_objc",
@@ -842,6 +843,22 @@ if (is_ios || is_mac) {
842843
]
843844
}
844845

846+
rtc_library("simulcast") {
847+
sources = [
848+
"objc/components/video_codec/RTCVideoEncoderFactorySimulcast.h",
849+
"objc/components/video_codec/RTCVideoEncoderFactorySimulcast.mm",
850+
"objc/api/video_codec/RTCVideoEncoderSimulcast.h",
851+
"objc/api/video_codec/RTCVideoEncoderSimulcast.mm",
852+
]
853+
854+
deps = [
855+
":base_objc",
856+
":wrapped_native_codec_objc",
857+
"../media:rtc_media_base",
858+
"../media:rtc_simulcast_encoder_adapter",
859+
]
860+
}
861+
845862
rtc_library("mediaconstraints_objc") {
846863
configs += [ "..:no_global_constructors" ]
847864
sources = [
@@ -1360,6 +1377,9 @@ if (is_ios || is_mac) {
13601377
"objc/api/video_codec/RTCVideoEncoderAV1.h",
13611378
"objc/api/video_frame_buffer/RTCNativeI420Buffer.h",
13621379
"objc/api/video_frame_buffer/RTCNativeMutableI420Buffer.h",
1380+
# Added for Simulcast support
1381+
"objc/components/video_codec/RTCVideoEncoderFactorySimulcast.h",
1382+
"objc/api/video_codec/RTCVideoEncoderSimulcast.h",
13631383
]
13641384

13651385
if (!build_with_chromium) {
@@ -1510,6 +1530,9 @@ if (is_ios || is_mac) {
15101530
"objc/components/video_codec/RTCVideoEncoderH264.h",
15111531
"objc/components/video_frame_buffer/RTCCVPixelBuffer.h",
15121532
"objc/helpers/RTCDispatcher.h",
1533+
# Added for Simulcast support
1534+
"objc/components/video_codec/RTCVideoEncoderFactorySimulcast.h",
1535+
"objc/api/video_codec/RTCVideoEncoderSimulcast.h",
15131536
]
15141537
if (!build_with_chromium) {
15151538
sources += [

‎sdk/android/BUILD.gn

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -522,6 +522,8 @@ if (is_android) {
522522
sources = [
523523
"api/org/webrtc/SoftwareVideoDecoderFactory.java",
524524
"api/org/webrtc/SoftwareVideoEncoderFactory.java",
525+
"api/org/webrtc/SimulcastVideoEncoder.java",
526+
"api/org/webrtc/SimulcastVideoEncoderFactory.java",
525527
]
526528

527529
deps = [
@@ -884,6 +886,21 @@ if (current_os == "linux" || is_android) {
884886
]
885887
}
886888

889+
rtc_library("simulcast_jni") {
890+
visibility = [ "*" ]
891+
allow_poison = [ "software_video_codecs" ]
892+
sources = [
893+
"src/jni/simulcast_video_encoder.cc",
894+
"src/jni/simulcast_video_encoder.h"
895+
]
896+
deps = [
897+
":base_jni",
898+
":video_jni",
899+
":native_api_codecs",
900+
"../../media:rtc_simulcast_encoder_adapter"
901+
]
902+
}
903+
887904
rtc_library("swcodecs_jni") {
888905
visibility = [ "*" ]
889906
allow_poison = [ "software_video_codecs" ]
@@ -897,6 +914,7 @@ if (current_os == "linux" || is_android) {
897914
":libvpx_vp8_jni",
898915
":libvpx_vp9_jni",
899916
":native_api_jni",
917+
":simulcast_jni",
900918
":video_jni",
901919
"../../api/video_codecs:builtin_video_decoder_factory",
902920
"../../api/video_codecs:builtin_video_encoder_factory",

‎sdk/android/api/org/webrtc/SimulcastVideoEncoder.java

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
package org.webrtc;
2+
3+
public class SimulcastVideoEncoder extends WrappedNativeVideoEncoder {
4+
5+
static native long nativeCreateEncoder(VideoEncoderFactory primary, VideoEncoderFactory fallback, VideoCodecInfo info);
6+
7+
VideoEncoderFactory primary;
8+
VideoEncoderFactory fallback;
9+
VideoCodecInfo info;
10+
11+
public SimulcastVideoEncoder(VideoEncoderFactory primary, VideoEncoderFactory fallback, VideoCodecInfo info) {
12+
this.primary = primary;
13+
this.fallback = fallback;
14+
this.info = info;
15+
}
16+
17+
@Override
18+
public long createNativeVideoEncoder() {
19+
return nativeCreateEncoder(primary, fallback, info);
20+
}
21+
22+
@Override
23+
public boolean isHardwareEncoder() {
24+
return false;
25+
}
26+
27+
}
28+

‎sdk/android/api/org/webrtc/SimulcastVideoEncoderFactory.java

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
/*
2+
* Copyright 2017 The WebRTC project authors. All Rights Reserved.
3+
*
4+
* Use of this source code is governed by a BSD-style license
5+
* that can be found in the LICENSE file in the root of the source
6+
* tree. An additional intellectual property rights grant can be found
7+
* in the file PATENTS. All contributing project authors may
8+
* be found in the AUTHORS file in the root of the source tree.
9+
*/
10+
11+
package org.webrtc;
12+
13+
import androidx.annotation.Nullable;
14+
import java.util.ArrayList;
15+
import java.util.HashMap;
16+
import java.util.List;
17+
import java.util.Arrays;
18+
19+
public class SimulcastVideoEncoderFactory implements VideoEncoderFactory {
20+
21+
VideoEncoderFactory primary;
22+
VideoEncoderFactory fallback;
23+
24+
public SimulcastVideoEncoderFactory(VideoEncoderFactory primary, VideoEncoderFactory fallback) {
25+
this.primary = primary;
26+
this.fallback = fallback;
27+
}
28+
29+
@Nullable
30+
@Override
31+
public VideoEncoder createEncoder(VideoCodecInfo info) {
32+
return new SimulcastVideoEncoder(primary, fallback, info);
33+
}
34+
35+
@Override
36+
public VideoCodecInfo[] getSupportedCodecs() {
37+
List<VideoCodecInfo> codecs = new ArrayList<VideoCodecInfo>();
38+
codecs.addAll(Arrays.asList(primary.getSupportedCodecs()));
39+
codecs.addAll(Arrays.asList(fallback.getSupportedCodecs()));
40+
return codecs.toArray(new VideoCodecInfo[codecs.size()]);
41+
}
42+
43+
}

‎sdk/android/src/java/org/webrtc/HardwareVideoEncoder.java

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -45,8 +45,8 @@ class HardwareVideoEncoder implements VideoEncoder {
4545
private static final int MEDIA_CODEC_RELEASE_TIMEOUT_MS = 5000;
4646
private static final int DEQUEUE_OUTPUT_BUFFER_TIMEOUT_US = 100000;
4747

48-
// Size of the input frames should be multiple of 16 for the H/W encoder.
49-
private static final int REQUIRED_RESOLUTION_ALIGNMENT = 16;
48+
// Size of the input frames should be multiple of 2 for the H/W encoder.
49+
private static final int REQUIRED_RESOLUTION_ALIGNMENT = 2;
5050

5151
/**
5252
* Keeps track of the number of output buffers that have been passed down the pipeline and not yet
@@ -210,6 +210,11 @@ public VideoCodecStatus initEncode(Settings settings, Callback callback) {
210210
this.callback = callback;
211211
automaticResizeOn = settings.automaticResizeOn;
212212

213+
if (settings.width % REQUIRED_RESOLUTION_ALIGNMENT != 0
214+
|| settings.height % REQUIRED_RESOLUTION_ALIGNMENT != 0) {
215+
Logging.e(TAG, "MediaCodec requires 2x2 alignment.");
216+
return VideoCodecStatus.ERR_SIZE;
217+
}
213218
this.width = settings.width;
214219
this.height = settings.height;
215220
useSurfaceMode = canUseSurface();
@@ -533,6 +538,12 @@ private VideoCodecStatus resetCodec(int newWidth, int newHeight, boolean newUseS
533538
if (status != VideoCodecStatus.OK) {
534539
return status;
535540
}
541+
542+
if (newWidth % REQUIRED_RESOLUTION_ALIGNMENT != 0
543+
|| newHeight % REQUIRED_RESOLUTION_ALIGNMENT != 0) {
544+
Logging.e(TAG, "MediaCodec requires 2x2 alignment.");
545+
return VideoCodecStatus.ERR_SIZE;
546+
}
536547
width = newWidth;
537548
height = newHeight;
538549
useSurfaceMode = newUseSurfaceMode;

‎sdk/android/src/jni/simulcast_video_encoder.cc

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
#include <jni.h>
2+
3+
#include "sdk/android/src/jni/jni_helpers.h"
4+
#include "sdk/android/src/jni/video_encoder_factory_wrapper.h"
5+
#include "sdk/android/src/jni/video_codec_info.h"
6+
#include "sdk/android/native_api/codecs/wrapper.h"
7+
#include "media/engine/simulcast_encoder_adapter.h"
8+
#include "rtc_base/logging.h"
9+
10+
using namespace webrtc;
11+
using namespace webrtc::jni;
12+
13+
#ifdef __cplusplus
14+
extern "C" {
15+
#endif
16+
17+
// (VideoEncoderFactory primary, VideoEncoderFactory fallback, VideoCodecInfo info)
18+
JNIEXPORT jlong JNICALL Java_org_webrtc_SimulcastVideoEncoder_nativeCreateEncoder(JNIEnv *env, jclass klass, jobject primary, jobject fallback, jobject info) {
19+
RTC_LOG(LS_INFO) << "Create simulcast video encoder";
20+
JavaParamRef<jobject> info_ref(info);
21+
SdpVideoFormat format = VideoCodecInfoToSdpVideoFormat(env, info_ref);
22+
23+
// TODO: 影響は軽微だが、リークする可能性があるので将来的に修正したい
24+
// https://github.com/shiguredo-webrtc-build/webrtc-build/pull/16#pullrequestreview-600675795
25+
return NativeToJavaPointer(std::make_unique<SimulcastEncoderAdapter>(
26+
JavaToNativeVideoEncoderFactory(env, primary).release(),
27+
JavaToNativeVideoEncoderFactory(env, fallback).release(),
28+
format).release());
29+
}
30+
31+
32+
#ifdef __cplusplus
33+
}
34+
#endif

‎sdk/android/src/jni/simulcast_video_encoder.h

Lines changed: 22 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎sdk/objc/api/video_codec/RTCVideoEncoderSimulcast.h

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
#import "RTCMacros.h"
2+
#import "RTCVideoEncoder.h"
3+
#import "RTCVideoEncoderFactory.h"
4+
#import "RTCVideoCodecInfo.h"
5+
6+
RTC_OBJC_EXPORT
7+
@interface RTC_OBJC_TYPE (RTCVideoEncoderSimulcast) : NSObject
8+
9+
+ (id<RTC_OBJC_TYPE(RTCVideoEncoder)>)simulcastEncoderWithPrimary:(id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)>)primary
10+
fallback:(id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)>)fallback
11+
videoCodecInfo:(RTC_OBJC_TYPE(RTCVideoCodecInfo) *)videoCodecInfo;
12+
13+
@end

‎sdk/objc/api/video_codec/RTCVideoEncoderSimulcast.mm

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
#import <Foundation/Foundation.h>
2+
3+
#import "RTCMacros.h"
4+
#import "RTCVideoEncoderSimulcast.h"
5+
#import "RTCWrappedNativeVideoEncoder.h"
6+
#import "api/peerconnection/RTCVideoCodecInfo+Private.h"
7+
8+
#include "native/api/video_encoder_factory.h"
9+
#include "media/engine/simulcast_encoder_adapter.h"
10+
11+
@implementation RTC_OBJC_TYPE (RTCVideoEncoderSimulcast)
12+
13+
+ (id<RTC_OBJC_TYPE(RTCVideoEncoder)>)simulcastEncoderWithPrimary:(id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)>)primary
14+
fallback:(id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)>)fallback
15+
videoCodecInfo:(RTC_OBJC_TYPE(RTCVideoCodecInfo) *)videoCodecInfo {
16+
auto nativePrimary = webrtc::ObjCToNativeVideoEncoderFactory(primary);
17+
auto nativeFallback = webrtc::ObjCToNativeVideoEncoderFactory(fallback);
18+
auto nativeFormat = [videoCodecInfo nativeSdpVideoFormat];
19+
return [[RTC_OBJC_TYPE(RTCWrappedNativeVideoEncoder) alloc]
20+
initWithNativeEncoder: std::make_unique<webrtc::SimulcastEncoderAdapter>(
21+
nativePrimary.release(),
22+
nativeFallback.release(),
23+
std::move(nativeFormat))];
24+
}
25+
26+
@end

‎sdk/objc/components/video_codec/RTCVideoEncoderFactorySimulcast.h

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
#import <Foundation/Foundation.h>
2+
3+
#import "RTCMacros.h"
4+
#import "RTCVideoEncoderFactory.h"
5+
6+
NS_ASSUME_NONNULL_BEGIN
7+
8+
RTC_OBJC_EXPORT
9+
@interface RTC_OBJC_TYPE (RTCVideoEncoderFactorySimulcast) : NSObject <RTC_OBJC_TYPE(RTCVideoEncoderFactory)>
10+
11+
- (instancetype)initWithPrimary:(id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)>)primary
12+
fallback:(id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)>)fallback;
13+
14+
@end
15+
16+
NS_ASSUME_NONNULL_END

0 commit comments

Comments
 (0)