Skip to content

Commit 3182d06

Browse files
committed
AudioConverter implementation for remixing and resampling
1 parent af5f6ca commit 3182d06

File tree

6 files changed

+560
-1
lines changed

6 files changed

+560
-1
lines changed

webrtc-jni/src/main/cpp/include/JNI_AudioConverter.h

+37
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
/*
2+
* Copyright 2021 Alex Andres
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
#ifndef JNI_WEBRTC_MEDIA_AUDIO_CONVERTER_H_
18+
#define JNI_WEBRTC_MEDIA_AUDIO_CONVERTER_H_
19+
20+
#include "JavaClass.h"
21+
#include "JavaRef.h"
22+
23+
#include <jni.h>
24+
25+
#include "rtc_base/constructor_magic.h"
26+
27+
namespace jni
28+
{
29+
class AudioConverter
30+
{
31+
public:
32+
static std::unique_ptr<AudioConverter> create(size_t srcFrames, size_t srcChannels, size_t dstFrames, size_t dstChannels);
33+
34+
virtual ~AudioConverter() = default;
35+
36+
virtual void convert(const int16_t * src, size_t srcSize, int16_t * dst, size_t dstSize) = 0;
37+
38+
size_t getSrcChannels() const { return srcChannels; }
39+
size_t getSrcFrames() const { return srcFrames; }
40+
size_t getDstChannels() const { return dstChannels; }
41+
size_t getDstFrames() const { return dstFrames; }
42+
43+
protected:
44+
AudioConverter();
45+
AudioConverter(size_t srcFrames, size_t srcChannels, size_t dstFrames, size_t dstChannels);
46+
47+
void checkSizes(size_t srcSize, size_t dstCapacity) const;
48+
49+
const size_t srcFrames;
50+
const size_t srcChannels;
51+
const size_t dstFrames;
52+
const size_t dstChannels;
53+
54+
private:
55+
RTC_DISALLOW_COPY_AND_ASSIGN(AudioConverter);
56+
};
57+
}
58+
59+
#endif
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
/*
2+
* Copyright 2021 Alex Andres
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
#include "JNI_AudioConverter.h"
18+
#include "media/audio/AudioConverter.h"
19+
#include "JavaUtils.h"
20+
21+
JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_audio_AudioConverter_convertInternal
22+
(JNIEnv * env, jobject caller, jbyteArray src, jint nSrcSamples, jbyteArray dst, jint nDstSamples)
23+
{
24+
jni::AudioConverter * converter = GetHandle<jni::AudioConverter>(env, caller);
25+
CHECK_HANDLE(converter);
26+
27+
jboolean isDstCopy = JNI_FALSE;
28+
29+
jbyte * srcPtr = env->GetByteArrayElements(src, nullptr);
30+
jbyte * dstPtr = env->GetByteArrayElements(dst, &isDstCopy);
31+
32+
converter->convert(reinterpret_cast<const int16_t *>(srcPtr), nSrcSamples, reinterpret_cast<int16_t *>(dstPtr), nDstSamples);
33+
34+
if (isDstCopy == JNI_TRUE) {
35+
jsize dstLength = env->GetArrayLength(dst);
36+
37+
env->SetByteArrayRegion(dst, 0, dstLength, dstPtr);
38+
}
39+
40+
env->ReleaseByteArrayElements(src, srcPtr, JNI_ABORT);
41+
env->ReleaseByteArrayElements(dst, dstPtr, JNI_ABORT);
42+
}
43+
44+
JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_audio_AudioConverter_dispose
45+
(JNIEnv * env, jobject caller)
46+
{
47+
jni::AudioConverter * converter = GetHandle<jni::AudioConverter>(env, caller);
48+
CHECK_HANDLE(converter);
49+
50+
SetHandle<std::nullptr_t>(env, caller, nullptr);
51+
52+
delete converter;
53+
}
54+
55+
JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_audio_AudioConverter_initialize
56+
(JNIEnv * env, jobject caller, jint srcSampleRate, jint srcChannels, jint dstSampleRate, jint dstChannels)
57+
{
58+
// 10 ms frames
59+
size_t srcFrames = srcSampleRate / 100;
60+
size_t dstFrames = dstSampleRate / 100;
61+
62+
jni::AudioConverter * converter = jni::AudioConverter::create(srcFrames, srcChannels, dstFrames, dstChannels).release();
63+
64+
SetHandle(env, caller, converter);
65+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,200 @@
1+
/*
2+
* Copyright 2021 Alex Andres
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
#include "media/audio/AudioConverter.h"
18+
19+
#include <memory>
20+
21+
#include "audio/utility/channel_mixer.h"
22+
#include "common_audio/resampler/include/push_resampler.h"
23+
24+
namespace jni
25+
{
26+
class CopyConverter : public AudioConverter {
27+
public:
28+
CopyConverter(size_t srcFrames, size_t srcChannels, size_t dstFrames, size_t dstChannels)
29+
: AudioConverter(srcFrames, srcChannels, dstFrames, dstChannels)
30+
{
31+
}
32+
33+
~CopyConverter() override
34+
{
35+
}
36+
37+
void convert(const int16_t * src, size_t srcSize, int16_t * dst, size_t dstSize) override {
38+
checkSizes(srcSize, dstSize);
39+
40+
if (src != dst) {
41+
std::memcpy(dst, src, dstFrames * dstChannels * sizeof(int16_t));
42+
}
43+
}
44+
};
45+
46+
47+
class ChannelConverter : public AudioConverter {
48+
public:
49+
ChannelConverter(size_t srcFrames, size_t srcChannels, size_t dstFrames, size_t dstChannels)
50+
: AudioConverter(srcFrames, srcChannels, dstFrames, dstChannels)
51+
{
52+
webrtc::ChannelLayout srcLayout = webrtc::GuessChannelLayout(static_cast<int>(srcChannels));
53+
webrtc::ChannelLayout dstLayout = webrtc::GuessChannelLayout(static_cast<int>(dstChannels));
54+
55+
mixer = std::make_unique<webrtc::ChannelMixer>(srcLayout, dstLayout);
56+
frame = std::make_unique<webrtc::AudioFrame>();
57+
58+
frame->samples_per_channel_ = srcFrames;
59+
frame->num_channels_ = srcChannels;
60+
}
61+
62+
~ChannelConverter() override
63+
{
64+
}
65+
66+
void convert(const int16_t * src, size_t srcSize, int16_t * dst, size_t dstSize) override {
67+
checkSizes(srcSize, dstSize);
68+
69+
std::memcpy(frame->mutable_data(), src, srcFrames * srcChannels * sizeof(int16_t));
70+
71+
mixer->Transform(frame.get());
72+
73+
std::memcpy(dst, frame->data(), dstFrames * dstChannels * sizeof(int16_t));
74+
}
75+
76+
private:
77+
std::unique_ptr<webrtc::ChannelMixer> mixer;
78+
std::unique_ptr<webrtc::AudioFrame> frame;
79+
};
80+
81+
82+
class ResampleConverter : public AudioConverter {
83+
public:
84+
ResampleConverter(size_t srcFrames, size_t srcChannels, size_t dstFrames, size_t dstChannels)
85+
: AudioConverter(srcFrames, srcChannels, dstFrames, dstChannels)
86+
{
87+
resampler = std::make_unique<webrtc::PushResampler<int16_t>>();
88+
resampler->InitializeIfNeeded(static_cast<int>(srcFrames * 100), static_cast<int>(dstFrames * 100), srcChannels);
89+
}
90+
91+
~ResampleConverter() override
92+
{
93+
}
94+
95+
void convert(const int16_t * src, size_t srcSize, int16_t * dst, size_t dstSize) override {
96+
checkSizes(srcSize, dstSize);
97+
98+
resampler->Resample(src, srcSize, dst, dstSize);
99+
}
100+
101+
private:
102+
std::unique_ptr<webrtc::PushResampler<int16_t>> resampler;
103+
};
104+
105+
106+
class CompositionConverter : public AudioConverter {
107+
public:
108+
explicit CompositionConverter(std::vector<std::unique_ptr<AudioConverter>> converters_)
109+
: converters(std::move(converters_))
110+
{
111+
RTC_CHECK_GE(converters.size(), 2);
112+
113+
// We need an intermediate buffer after every converter.
114+
for (auto it = converters.begin(); it != converters.end() - 1; ++it) {
115+
buffers.push_back(std::vector<int16_t>((*it)->getDstFrames() * (*it)->getDstChannels(), 0));
116+
}
117+
}
118+
119+
~CompositionConverter() override
120+
{
121+
}
122+
123+
void convert(const int16_t * src, size_t srcSize, int16_t * dst, size_t dstSize) override {
124+
converters.front()->convert(src, srcSize, buffers.front().data(), buffers.front().size());
125+
126+
for (size_t i = 2; i < converters.size(); ++i) {
127+
auto & src_buffer = buffers[i - 2];
128+
auto & dst_buffer = buffers[i - 1];
129+
130+
converters[i]->convert(src_buffer.data(), src_buffer.size(), dst_buffer.data(), dst_buffer.size());
131+
}
132+
133+
converters.back()->convert(buffers.back().data(), buffers.back().size(), dst, dstSize);
134+
}
135+
136+
private:
137+
std::vector<std::unique_ptr<AudioConverter>> converters;
138+
std::vector<std::vector<int16_t>> buffers;
139+
};
140+
141+
142+
std::unique_ptr<AudioConverter> AudioConverter::create(size_t srcFrames, size_t srcChannels, size_t dstFrames, size_t dstChannels)
143+
{
144+
std::unique_ptr<AudioConverter> converter;
145+
146+
if (srcChannels > dstChannels) {
147+
if (srcFrames != dstFrames) {
148+
std::vector<std::unique_ptr<AudioConverter>> converters;
149+
converters.push_back(std::unique_ptr<AudioConverter>(new ChannelConverter(srcFrames, srcChannels, srcFrames, dstChannels)));
150+
converters.push_back(std::unique_ptr<AudioConverter>(new ResampleConverter(srcFrames, dstChannels, dstFrames, dstChannels)));
151+
152+
converter.reset(new CompositionConverter(std::move(converters)));
153+
}
154+
else {
155+
converter.reset(new ChannelConverter(srcFrames, srcChannels, dstFrames, dstChannels));
156+
}
157+
}
158+
else if (srcChannels < dstChannels) {
159+
if (srcFrames != dstFrames) {
160+
std::vector<std::unique_ptr<AudioConverter>> converters;
161+
converters.push_back(std::unique_ptr<AudioConverter>(new ResampleConverter(srcFrames, srcChannels, dstFrames, srcChannels)));
162+
converters.push_back(std::unique_ptr<AudioConverter>(new ChannelConverter(dstFrames, srcChannels, dstFrames, dstChannels)));
163+
164+
converter.reset(new CompositionConverter(std::move(converters)));
165+
}
166+
else {
167+
converter.reset(new ChannelConverter(srcFrames, srcChannels, dstFrames, dstChannels));
168+
}
169+
}
170+
else if (srcFrames != dstFrames) {
171+
converter.reset(new ResampleConverter(srcFrames, srcChannels, dstFrames, dstChannels));
172+
}
173+
else {
174+
converter.reset(new CopyConverter(srcFrames, srcChannels, dstFrames, dstChannels));
175+
}
176+
177+
return converter;
178+
}
179+
180+
AudioConverter::AudioConverter() :
181+
srcFrames(0),
182+
srcChannels(0),
183+
dstFrames(0),
184+
dstChannels(0)
185+
{
186+
}
187+
188+
AudioConverter::AudioConverter(size_t srcFrames, size_t srcChannels, size_t dstFrames, size_t dstChannels) :
189+
srcFrames(srcFrames),
190+
srcChannels(srcChannels),
191+
dstFrames(dstFrames),
192+
dstChannels(dstChannels)
193+
{
194+
}
195+
196+
void AudioConverter::checkSizes(size_t srcSize, size_t dstCapacity) const {
197+
RTC_CHECK_EQ(srcSize, srcChannels * srcFrames);
198+
RTC_CHECK_GE(dstCapacity, dstChannels * dstFrames);
199+
}
200+
}

0 commit comments

Comments
 (0)