Allow audio system to handle synchronized low-latency audio I/O

As an important part of WebAudio/WebRTC integration, we need to be able to process and analyse
live audio.  This change adds the ability to our audio system for handling synchronized audio
input and output in the same callback (same thread) which is important for good performance
and low-latency.

As a part of this change, the audio IPC system now takes an optional |input_channels| argument
in the CreateStream() message and associated browser-side code in AudioRendererHost::OnCreateStream(), etc.
|input_channels| will be 0 during normal operation of audio output (and no input).
But when synchronized audio I/O is needed, then a non-zero value can be passed in here.
The |params| passed in represents both the input and output format, particularly the
frames_per_buffer() and sample_rate().

AudioRendererSink now has an new InitializeIO() method which will allow the use of
synchronized I/O with the |input_channels| argument.  AudioRendererSink::RenderCallback
now has a new RenderIO() which will be called instead of Render() in the case where a
non-zero value is passed in for |input_channels|.

BUG=none
TEST=none
(manual testing on early Mac OS X and Windows audio back-ends)

Review URL: https://chromiumcodereview.appspot.com/10830268

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@156234 0039d316-1c4b-4281-b951-d872f2087c98

chrome/app/generated_resources.grd

@@ -6059,6 +6059,12 @@ Keep your key file in a safe place. You will need it to create new versions of y
       <message name="IDS_FLAGS_ENABLE_POINTER_LOCK_DESCRIPTION" desc="Description for the flag to enable the mouse lock feature.">
         Web pages may capture the mouse pointer and remove it from the user's control. The mouse movement data is directed solely to the web application. Users can escape by pressing the 'Esc' key.
       </message>
+      <message name="IDS_FLAGS_ENABLE_WEBAUDIO_INPUT_NAME" desc="Title for the flag to enable the webaudio input feature.">
+        Web Audio Input
+      </message>
+      <message name="IDS_FLAGS_ENABLE_WEBAUDIO_INPUT_DESCRIPTION" desc="Title for the flag to enable the webaudio input feature.">
+        Enables live audio input using getUserMedia() and the Web Audio API.
+      </message>
       <message name="IDS_FLAGS_DISABLE_WEBSITE_SETTINGS_NAME" desc="Title for the flag to enable the website settings feature.">
         Disable the Website Settings UI
       </message>

chrome/browser/DEPS

@@ -10,6 +10,7 @@ include_rules = [
   "+google_update",
   "+grit",  # For generated headers
   "+installer_util_strings",  # For generated headers
+  "+media/base",  # For media switches
   "+policy",  # For generated headers and source
   "+ppapi/c",  # For various types.
   "+ppapi/proxy",

chrome/browser/about_flags.cc

@@ -21,6 +21,7 @@
 #include "chrome/common/pref_names.h"
 #include "content/public/browser/user_metrics.h"
 #include "grit/generated_resources.h"
+#include "media/base/media_switches.h"
 #include "ui/app_list/app_list_switches.h"
 #include "ui/base/l10n/l10n_util.h"
 #include "ui/base/ui_base_switches.h"
@@ -915,6 +916,13 @@ const Experiment kExperiments[] = {
     kOsAll,
     SINGLE_VALUE_TYPE(switches::kDisableWebsiteSettings),
   },
+  {
+    "enable-webaudio-input",
+    IDS_FLAGS_ENABLE_WEBAUDIO_INPUT_NAME,
+    IDS_FLAGS_ENABLE_WEBAUDIO_INPUT_DESCRIPTION,
+    kOsAll,
+    SINGLE_VALUE_TYPE(switches::kEnableWebAudioInput),
+  },
   {
     "enable-contacts",
     IDS_FLAGS_ENABLE_CONTACTS_NAME,

content/browser/renderer_host/media/audio_renderer_host.cc

@@ -18,6 +18,7 @@
 
 using content::BrowserMessageFilter;
 using content::BrowserThread;
+using media::AudioBus;
 
 AudioRendererHost::AudioEntry::AudioEntry()
     : stream_id(0),
@@ -194,7 +195,7 @@ bool AudioRendererHost::OnMessageReceived(const IPC::Message& message,
 }
 
 void AudioRendererHost::OnCreateStream(
-    int stream_id, const media::AudioParameters& params) {
+    int stream_id, const media::AudioParameters& params, int input_channels) {
   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO));
   DCHECK(LookupById(stream_id) == NULL);
 
@@ -204,11 +205,28 @@ void AudioRendererHost::OnCreateStream(
   DCHECK_LE(buffer_size,
             static_cast<uint32>(media::limits::kMaxPacketSizeInBytes));
 
+  DCHECK_GE(input_channels, 0);
+  DCHECK_LT(input_channels, media::limits::kMaxChannels);
+
+  // Calculate output and input memory size.
+  int output_memory_size = AudioBus::CalculateMemorySize(audio_params);
+  DCHECK_GT(output_memory_size, 0);
+
+  int frames = audio_params.frames_per_buffer();
+  int input_memory_size =
+      AudioBus::CalculateMemorySize(input_channels, frames);
+
+  DCHECK_GE(input_memory_size, 0);
+
   scoped_ptr<AudioEntry> entry(new AudioEntry());
 
   // Create the shared memory and share with the renderer process.
+  // For synchronized I/O (if input_channels > 0) then we allocate
+  // extra memory after the output data for the input data.
+  uint32 io_buffer_size = output_memory_size + input_memory_size;
+
   uint32 shared_memory_size =
-      media::TotalSharedMemorySizeInBytes(buffer_size);
+      media::TotalSharedMemorySizeInBytes(io_buffer_size);
   if (!entry->shared_memory.CreateAndMapAnonymous(shared_memory_size)) {
     // If creation of shared memory failed then send an error message.
     SendErrorMessage(stream_id);
@@ -217,7 +235,7 @@ void AudioRendererHost::OnCreateStream(
 
   // Create sync reader and try to initialize it.
   scoped_ptr<AudioSyncReader> reader(
-      new AudioSyncReader(&entry->shared_memory, params));
+      new AudioSyncReader(&entry->shared_memory, params, input_channels));
 
   if (!reader->Init()) {
     SendErrorMessage(stream_id);

content/browser/renderer_host/media/audio_renderer_host.h

@@ -122,7 +122,9 @@ class CONTENT_EXPORT AudioRendererHost
   // Creates an audio output stream with the specified format. If this call is
   // successful this object would keep an internal entry of the stream for the
   // required properties.
-  void OnCreateStream(int stream_id, const media::AudioParameters& params);
+  void OnCreateStream(int stream_id,
+                      const media::AudioParameters& params,
+                      int input_channels);
 
   // Play the audio stream referenced by |stream_id|.
   void OnPlayStream(int stream_id);

content/browser/renderer_host/media/audio_renderer_host_unittest.cc

@@ -207,7 +207,7 @@ class AudioRendererHostTest : public testing::Test {
 
     // Send a create stream message to the audio output stream and wait until
     // we receive the created message.
-    host_->OnCreateStream(kStreamId, params);
+    host_->OnCreateStream(kStreamId, params, 0);
     message_loop_->Run();
   }
 

content/browser/renderer_host/media/audio_sync_reader.cc

@@ -17,12 +17,25 @@
 const int kMinIntervalBetweenReadCallsInMs = 10;
 #endif
 
+using media::AudioBus;
+
 AudioSyncReader::AudioSyncReader(base::SharedMemory* shared_memory,
-                                 const media::AudioParameters& params)
-    : shared_memory_(shared_memory) {
+                                 const media::AudioParameters& params,
+                                 int input_channels)
+    : shared_memory_(shared_memory),
+      input_channels_(input_channels) {
   packet_size_ = media::PacketSizeInBytes(shared_memory_->created_size());
-  DCHECK_EQ(packet_size_, media::AudioBus::CalculateMemorySize(params));
-  audio_bus_ = media::AudioBus::WrapMemory(params, shared_memory->memory());
+  DCHECK_EQ(packet_size_, AudioBus::CalculateMemorySize(params));
+  output_bus_ = AudioBus::WrapMemory(params, shared_memory->memory());
+
+  if (input_channels_ > 0) {
+    // The input storage is after the output storage.
+    int output_memory_size = AudioBus::CalculateMemorySize(params);
+    int frames = params.frames_per_buffer();
+    char* input_data =
+        static_cast<char*>(shared_memory_->memory()) + output_memory_size;
+    input_bus_ = AudioBus::WrapMemory(input_channels_, frames, input_data);
+  }
 }
 
 AudioSyncReader::~AudioSyncReader() {
@@ -45,7 +58,7 @@ void AudioSyncReader::UpdatePendingBytes(uint32 bytes) {
   }
 }
 
-int AudioSyncReader::Read(media::AudioBus* audio_bus) {
+int AudioSyncReader::Read(AudioBus* source, AudioBus* dest) {
 #if defined(OS_WIN)
   // HACK: yield if reader is called too often.
   // Problem is lack of synchronization between host and renderer. We cannot be
@@ -62,6 +75,14 @@ int AudioSyncReader::Read(media::AudioBus* audio_bus) {
   previous_call_time_ = base::Time::Now();
 #endif
 
+  // Copy optional synchronized live audio input for consumption by renderer
+  // process.
+  if (source && input_bus_.get()) {
+    DCHECK_EQ(source->channels(), input_bus_->channels());
+    DCHECK_LE(source->frames(), input_bus_->frames());
+    source->CopyTo(input_bus_.get());
+  }
+
   // Retrieve the actual number of bytes available from the shared memory.  If
   // the renderer has not completed rendering this value will be invalid (still
   // the marker stored in UpdatePendingBytes() above) and must be sanitized.
@@ -75,20 +96,21 @@ int AudioSyncReader::Read(media::AudioBus* audio_bus) {
   // value for a couple reasons.  One, it might still be the unknown data size
   // marker.  Two, shared memory comes from a potentially untrusted source.
   int frames =
-      size / (sizeof(*audio_bus_->channel(0)) * audio_bus_->channels());
+      size / (sizeof(*output_bus_->channel(0)) * output_bus_->channels());
   if (frames < 0)
     frames = 0;
-  else if (frames > audio_bus_->frames())
-    frames = audio_bus_->frames();
+  else if (frames > output_bus_->frames())
+    frames = output_bus_->frames();
 
   // Copy data from the shared memory into the caller's AudioBus.
-  audio_bus_->CopyTo(audio_bus);
+  output_bus_->CopyTo(dest);
 
   // Zero out any unfilled frames in the destination bus.
-  audio_bus->ZeroFramesPartial(frames, audio_bus->frames() - frames);
+  dest->ZeroFramesPartial(frames, dest->frames() - frames);
 
-  // Zero out the entire buffer.
-  memset(shared_memory_->memory(), 0, packet_size_);
+  // Zero out the entire output buffer to avoid stuttering/repeating-buffers
+  // in the anomalous case if the renderer is unable to keep up with real-time.
+  output_bus_->Zero();
 
   // Store unknown length of data into buffer, in case renderer does not store
   // the length itself. It also helps in decision if we need to yield.

content/browser/renderer_host/media/audio_sync_reader.h

@@ -24,13 +24,14 @@ class SharedMemory;
 class AudioSyncReader : public media::AudioOutputController::SyncReader {
  public:
   AudioSyncReader(base::SharedMemory* shared_memory,
-                  const media::AudioParameters& params);
+                  const media::AudioParameters& params,
+                  int input_channels);
 
   virtual ~AudioSyncReader();
 
   // media::AudioOutputController::SyncReader implementations.
   virtual void UpdatePendingBytes(uint32 bytes) OVERRIDE;
-  virtual int Read(media::AudioBus* audio_bus) OVERRIDE;
+  virtual int Read(media::AudioBus* source, media::AudioBus* dest) OVERRIDE;
   virtual void Close() OVERRIDE;
   virtual bool DataReady() OVERRIDE;
 
@@ -46,6 +47,9 @@ class AudioSyncReader : public media::AudioOutputController::SyncReader {
   base::SharedMemory* shared_memory_;
   base::Time previous_call_time_;
 
+  // Number of input channels for synchronized I/O.
+  int input_channels_;
+
   // Socket for transmitting audio data.
   scoped_ptr<base::CancelableSyncSocket> socket_;
 
@@ -54,7 +58,10 @@ class AudioSyncReader : public media::AudioOutputController::SyncReader {
   scoped_ptr<base::CancelableSyncSocket> foreign_socket_;
 
   // Shared memory wrapper used for transferring audio data to Read() callers.
-  scoped_ptr<media::AudioBus> audio_bus_;
+  scoped_ptr<media::AudioBus> output_bus_;
+
+  // Shared memory wrapper used for transferring audio data from Read() callers.
+  scoped_ptr<media::AudioBus> input_bus_;
 
   // Maximum amount of audio data which can be transferred in one Read() call.
   int packet_size_;

content/browser/renderer_host/render_process_host_impl.cc

@@ -769,6 +769,7 @@ void RenderProcessHostImpl::PropagateBrowserCommandLineToRenderer(
 #else
     switches::kDisableWebAudio,
 #endif
+    switches::kEnableWebAudioInput,
     switches::kDisableWebSockets,
     switches::kDomAutomationController,
     switches::kEnableAccessibilityLogging,

content/common/media/audio_messages.h

@@ -85,9 +85,10 @@ IPC_MESSAGE_CONTROL2(AudioInputMsg_NotifyDeviceStarted,
 // Messages sent from the renderer to the browser.
 
 // Request that got sent to browser for creating an audio output stream
-IPC_MESSAGE_CONTROL2(AudioHostMsg_CreateStream,
+IPC_MESSAGE_CONTROL3(AudioHostMsg_CreateStream,
                      int /* stream_id */,
-                     media::AudioParameters /* params */)
+                     media::AudioParameters, /* params */
+                     int /* input_channels */)
 
 // Request that got sent to browser for creating an audio input stream
 IPC_MESSAGE_CONTROL4(AudioInputHostMsg_CreateStream,

content/renderer/media/audio_message_filter.cc

@@ -35,8 +35,9 @@ void AudioMessageFilter::RemoveDelegate(int id) {
 }
 
 void AudioMessageFilter::CreateStream(int stream_id,
-                                      const media::AudioParameters& params) {
-  Send(new AudioHostMsg_CreateStream(stream_id, params));
+                                      const media::AudioParameters& params,
+                                      int input_channels) {
+  Send(new AudioHostMsg_CreateStream(stream_id, params, input_channels));
 }
 
 void AudioMessageFilter::PlayStream(int stream_id) {

content/renderer/media/audio_message_filter.h

@@ -32,7 +32,7 @@ class CONTENT_EXPORT AudioMessageFilter
   virtual int AddDelegate(media::AudioOutputIPCDelegate* delegate) OVERRIDE;
   virtual void RemoveDelegate(int id) OVERRIDE;
   virtual void CreateStream(int stream_id,
-      const media::AudioParameters& params) OVERRIDE;
+      const media::AudioParameters& params, int input_channels) OVERRIDE;
   virtual void PlayStream(int stream_id) OVERRIDE;
   virtual void PauseStream(int stream_id) OVERRIDE;
   virtual void FlushStream(int stream_id) OVERRIDE;

content/renderer/media/renderer_webaudiodevice_impl.cc

@@ -4,8 +4,10 @@
 
 #include "content/renderer/media/renderer_webaudiodevice_impl.h"
 
+#include "base/command_line.h"
 #include "base/logging.h"
 #include "content/renderer/media/audio_device_factory.h"
+#include "media/base/media_switches.h"
 
 using content::AudioDeviceFactory;
 using WebKit::WebAudioDevice;
@@ -17,7 +19,17 @@ RendererWebAudioDeviceImpl::RendererWebAudioDeviceImpl(
     : is_running_(false),
       client_callback_(callback) {
   audio_device_ = AudioDeviceFactory::NewOutputDevice();
-  audio_device_->Initialize(params, this);
+
+  // TODO(crogers): remove once we properly handle input device selection.
+  // https://code.google.com/p/chromium/issues/detail?id=147327
+  if (CommandLine::ForCurrentProcess()->HasSwitch(
+      switches::kEnableWebAudioInput)) {
+    // TODO(crogers): support more than hard-coded stereo:
+    // https://code.google.com/p/chromium/issues/detail?id=147326
+    audio_device_->InitializeIO(params, 2, this);
+  } else {
+    audio_device_->Initialize(params, this);
+  }
 }
 
 RendererWebAudioDeviceImpl::~RendererWebAudioDeviceImpl() {
@@ -42,20 +54,35 @@ double RendererWebAudioDeviceImpl::sampleRate() {
   return 44100.0;
 }
 
-int RendererWebAudioDeviceImpl::Render(media::AudioBus* audio_bus,
+int RendererWebAudioDeviceImpl::Render(media::AudioBus* dest,
                                        int audio_delay_milliseconds) {
-  // Make the client callback to get rendered audio.
+  RenderIO(NULL, dest, audio_delay_milliseconds);
+  return dest->frames();
+}
+
+void RendererWebAudioDeviceImpl::RenderIO(media::AudioBus* source,
+                                          media::AudioBus* dest,
+                                          int audio_delay_milliseconds) {
+  // Make the client callback for an I/O cycle.
   DCHECK(client_callback_);
   if (client_callback_) {
-    // Wrap the pointers using WebVector.
+    // Wrap the input pointers using WebVector.
+    size_t input_channels =
+        source ? static_cast<size_t>(source->channels()) : 0;
+    WebVector<float*> web_audio_input_data(input_channels);
+    for (size_t i = 0; i < input_channels; ++i)
+      web_audio_input_data[i] = source->channel(i);
+
+    // Wrap the output pointers using WebVector.
     WebVector<float*> web_audio_data(
-        static_cast<size_t>(audio_bus->channels()));
-    for (int i = 0; i < audio_bus->channels(); ++i)
-      web_audio_data[i] = audio_bus->channel(i);
+        static_cast<size_t>(dest->channels()));
+    for (int i = 0; i < dest->channels(); ++i)
+      web_audio_data[i] = dest->channel(i);
 
-    client_callback_->render(web_audio_data, audio_bus->frames());
+    client_callback_->render(web_audio_input_data,
+                             web_audio_data,
+                             dest->frames());
   }
-  return audio_bus->frames();
 }
 
 void RendererWebAudioDeviceImpl::OnRenderError() {

content/renderer/media/renderer_webaudiodevice_impl.h

@@ -24,8 +24,13 @@ class RendererWebAudioDeviceImpl
   virtual double sampleRate();
 
   // AudioRendererSink::RenderCallback implementation.
-  virtual int Render(media::AudioBus* audio_bus,
+  virtual int Render(media::AudioBus* dest,
                      int audio_delay_milliseconds) OVERRIDE;
+
+  virtual void RenderIO(media::AudioBus* source,
+                        media::AudioBus* dest,
+                        int audio_delay_milliseconds) OVERRIDE;
+
   virtual void OnRenderError() OVERRIDE;
 
  private:

content/renderer/pepper/pepper_platform_audio_output_impl.cc

@@ -152,7 +152,7 @@ bool PepperPlatformAudioOutputImpl::Initialize(
 void PepperPlatformAudioOutputImpl::InitializeOnIOThread(
     const media::AudioParameters& params) {
   stream_id_ = ipc_->AddDelegate(this);
-  ipc_->CreateStream(stream_id_, params);
+  ipc_->CreateStream(stream_id_, params, 0);
 }
 
 void PepperPlatformAudioOutputImpl::StartPlaybackOnIOThread() {

media/audio/audio_device_thread.cc

@@ -177,8 +177,10 @@ void AudioDeviceThread::Thread::Run() {
 
 AudioDeviceThread::Callback::Callback(
     const AudioParameters& audio_parameters,
+    int input_channels,
     base::SharedMemoryHandle memory, int memory_length)
     : audio_parameters_(audio_parameters),
+      input_channels_(input_channels),
       samples_per_ms_(audio_parameters.sample_rate() / 1000),
       bytes_per_ms_(audio_parameters.channels() *
                     (audio_parameters_.bits_per_sample() / 8) *