Adding key press detection in the browser process.

It works like this on the browser side: A new object UserInputMonitor is created on BrowserMainLoop and passed to AudioInputRendererHost to pass to AudioInputController. AudioInputController::DoRecord calls UserInputMonitor::AddKeyStrokeListener --> UserInputMonitor listens to system key events (only implemented on Linux) --> AudioInputController::OnKeyPressed is called and sets key_pressed_ --> When AudioInputController::OnData called, it writes key_pressed_ to shared memory along with the audio data buffer. On the renderer side a new param "key_pressed" is added through the code path of passing the flag to the webrtc voice engine. This CL includes all these changes except the implementation of UserInputMonitor for Windows and Mac. The impl of UserInputMonitor is mostly copied from remoting/host/local_input_monitor_linux.cc BUG= Committed: https://src.chromium.org/viewvc/chrome?view=rev&revision=217768 Review URL: https://chromiumcodereview.appspot.com/21183002 git-svn-id: svn://svn.chromium.org/chrome/trunk/src@217844 0039d316-1c4b-4281-b951-d872f2087c98
hylschwy · Aug 15, 2013 · 61f697f · 61f697f
1 parent 92be5a4
commit 61f697f
Show file tree

Hide file tree

Showing 37 changed files with 931 additions and 159 deletions.
diff --git a/content/browser/browser_main_loop.cc b/content/browser/browser_main_loop.cc
@@ -52,6 +52,7 @@
 #include "crypto/nss_util.h"
 #include "media/audio/audio_manager.h"
 #include "media/base/media.h"
+#include "media/base/user_input_monitor.h"
 #include "media/midi/midi_manager.h"
 #include "net/base/network_change_notifier.h"
 #include "net/socket/client_socket_factory.h"
@@ -872,6 +873,14 @@ int BrowserMainLoop::BrowserThreadsStarted() {
         audio_manager_.get(), media_stream_manager_.get()));
   }
 
+  {
+    TRACE_EVENT0(
+        "startup",
+        "BrowserMainLoop::BrowserThreadsStarted::InitUserInputMonitor");
+    user_input_monitor_ = media::UserInputMonitor::Create(
+        io_thread_->message_loop_proxy(), main_thread_->message_loop_proxy());
+  }
+
   // Alert the clipboard class to which threads are allowed to access the
   // clipboard:
   std::vector<base::PlatformThreadId> allowed_clipboard_threads;

diff --git a/content/browser/browser_main_loop.h b/content/browser/browser_main_loop.h
@@ -26,6 +26,7 @@ class TraceMemoryController;
 namespace media {
 class AudioManager;
 class MIDIManager;
+class UserInputMonitor;
 }  // namespace media
 
 namespace net {
@@ -86,6 +87,9 @@ class CONTENT_EXPORT BrowserMainLoop {
   MediaStreamManager* media_stream_manager() const {
     return media_stream_manager_.get();
   }
+  media::UserInputMonitor* user_input_monitor() const {
+    return user_input_monitor_.get();
+  }
   media::MIDIManager* midi_manager() const { return midi_manager_.get(); }
   base::Thread* indexed_db_thread() const { return indexed_db_thread_.get(); }
 
@@ -122,6 +126,8 @@ class CONTENT_EXPORT BrowserMainLoop {
   scoped_ptr<base::PowerMonitor> power_monitor_;
   scoped_ptr<base::HighResolutionTimerManager> hi_res_timer_manager_;
   scoped_ptr<net::NetworkChangeNotifier> network_change_notifier_;
+  // user_input_monitor_ has to outlive audio_manager_, so declared first.
+  scoped_ptr<media::UserInputMonitor> user_input_monitor_;
   scoped_ptr<media::AudioManager> audio_manager_;
   scoped_ptr<media::MIDIManager> midi_manager_;
   scoped_ptr<AudioMirroringManager> audio_mirroring_manager_;

diff --git a/content/browser/renderer_host/media/audio_input_renderer_host.cc b/content/browser/renderer_host/media/audio_input_renderer_host.cc
@@ -51,11 +51,12 @@ AudioInputRendererHost::AudioEntry::~AudioEntry() {}
 AudioInputRendererHost::AudioInputRendererHost(
     media::AudioManager* audio_manager,
     MediaStreamManager* media_stream_manager,
-    AudioMirroringManager* audio_mirroring_manager)
+    AudioMirroringManager* audio_mirroring_manager,
+    media::UserInputMonitor* user_input_monitor)
     : audio_manager_(audio_manager),
       media_stream_manager_(media_stream_manager),
-      audio_mirroring_manager_(audio_mirroring_manager) {
-}
+      audio_mirroring_manager_(audio_mirroring_manager),
+      user_input_monitor_(user_input_monitor) {}
 
 AudioInputRendererHost::~AudioInputRendererHost() {
   DCHECK(audio_entries_.empty());
@@ -272,20 +273,23 @@ void AudioInputRendererHost::OnCreateStream(
     entry->controller = media::AudioInputController::CreateForStream(
         audio_manager_->GetMessageLoop(),
         this,
-        WebContentsAudioInputStream::Create(
-            device_id, audio_params, audio_manager_->GetWorkerLoop(),
-            audio_mirroring_manager_),
-        entry->writer.get());
+        WebContentsAudioInputStream::Create(device_id,
+                                            audio_params,
+                                            audio_manager_->GetWorkerLoop(),
+                                            audio_mirroring_manager_),
+        entry->writer.get(),
+        user_input_monitor_);
   } else {
     // TODO(henrika): replace CreateLowLatency() with Create() as soon
     // as satish has ensured that Speech Input also uses the default low-
     // latency path. See crbug.com/112472 for details.
-    entry->controller = media::AudioInputController::CreateLowLatency(
-        audio_manager_,
-        this,
-        audio_params,
-        device_id,
-        entry->writer.get());
+    entry->controller =
+        media::AudioInputController::CreateLowLatency(audio_manager_,
+                                                      this,
+                                                      audio_params,
+                                                      device_id,
+                                                      entry->writer.get(),
+                                                      user_input_monitor_);
   }
 
   if (!entry->controller.get()) {

diff --git a/content/browser/renderer_host/media/audio_input_renderer_host.h b/content/browser/renderer_host/media/audio_input_renderer_host.h
@@ -44,6 +44,7 @@
 namespace media {
 class AudioManager;
 class AudioParameters;
+class UserInputMonitor;
 }
 
 namespace content {
@@ -55,10 +56,11 @@ class CONTENT_EXPORT AudioInputRendererHost
       public media::AudioInputController::EventHandler {
  public:
   // Called from UI thread from the owner of this object.
-  AudioInputRendererHost(
-      media::AudioManager* audio_manager,
-      MediaStreamManager* media_stream_manager,
-      AudioMirroringManager* audio_mirroring_manager);
+  // |user_input_monitor| is used for typing detection and can be NULL.
+  AudioInputRendererHost(media::AudioManager* audio_manager,
+                         MediaStreamManager* media_stream_manager,
+                         AudioMirroringManager* audio_mirroring_manager,
+                         media::UserInputMonitor* user_input_monitor);
 
   // BrowserMessageFilter implementation.
   virtual void OnChannelClosing() OVERRIDE;
@@ -154,6 +156,9 @@ class CONTENT_EXPORT AudioInputRendererHost
   // A map of stream IDs to audio sources.
   AudioEntryMap audio_entries_;
 
+  // Raw pointer of the UserInputMonitor.
+  media::UserInputMonitor* user_input_monitor_;
+
   DISALLOW_COPY_AND_ASSIGN(AudioInputRendererHost);
 };
 

diff --git a/content/browser/renderer_host/media/audio_input_sync_writer.cc b/content/browser/renderer_host/media/audio_input_sync_writer.cc
@@ -29,14 +29,17 @@ void AudioInputSyncWriter::UpdateRecordedBytes(uint32 bytes) {
   socket_->Send(&bytes, sizeof(bytes));
 }
 
-uint32 AudioInputSyncWriter::Write(
-    const void* data, uint32 size, double volume) {
+uint32 AudioInputSyncWriter::Write(const void* data,
+                                   uint32 size,
+                                   double volume,
+                                   bool key_pressed) {
   uint8* ptr = static_cast<uint8*>(shared_memory_->memory());
   ptr += current_segment_id_ * shared_memory_segment_size_;
   media::AudioInputBuffer* buffer =
       reinterpret_cast<media::AudioInputBuffer*>(ptr);
   buffer->params.volume = volume;
   buffer->params.size = size;
+  buffer->params.key_pressed = key_pressed;
   memcpy(buffer->audio, data, size);
 
   if (++current_segment_id_ >= shared_memory_segment_count_)

diff --git a/content/browser/renderer_host/media/audio_input_sync_writer.h b/content/browser/renderer_host/media/audio_input_sync_writer.h
@@ -28,7 +28,10 @@ class AudioInputSyncWriter : public media::AudioInputController::SyncWriter {
 
   // media::AudioOutputController::SyncWriter implementation.
   virtual void UpdateRecordedBytes(uint32 bytes) OVERRIDE;
-  virtual uint32 Write(const void* data, uint32 size, double volume) OVERRIDE;
+  virtual uint32 Write(const void* data,
+                       uint32 size,
+                       double volume,
+                       bool key_pressed) OVERRIDE;
   virtual void Close() OVERRIDE;
 
   bool Init();

diff --git a/content/browser/renderer_host/render_process_host_impl.cc b/content/browser/renderer_host/render_process_host_impl.cc
@@ -617,11 +617,14 @@ void RenderProcessHostImpl::CreateMessageFilters() {
   channel_->AddFilter(new AudioInputRendererHost(
       audio_manager,
       media_stream_manager,
-      BrowserMainLoop::GetInstance()->audio_mirroring_manager()));
+      BrowserMainLoop::GetInstance()->audio_mirroring_manager(),
+      BrowserMainLoop::GetInstance()->user_input_monitor()));
   channel_->AddFilter(new AudioRendererHost(
-      GetID(), audio_manager,
+      GetID(),
+      audio_manager,
       BrowserMainLoop::GetInstance()->audio_mirroring_manager(),
-      media_internals, media_stream_manager));
+      media_internals,
+      media_stream_manager));
   channel_->AddFilter(
       new MIDIHost(BrowserMainLoop::GetInstance()->midi_manager()));
   channel_->AddFilter(new MIDIDispatcherHost(GetID(), browser_context));

diff --git a/content/browser/speech/speech_recognizer_impl.cc b/content/browser/speech/speech_recognizer_impl.cc
@@ -564,7 +564,7 @@ SpeechRecognizerImpl::StartRecording(const FSMEventArgs&) {
       new OnDataConverter(input_parameters, output_parameters));
 
   audio_controller_ = AudioInputController::Create(
-      audio_manager, this, input_parameters, device_id_);
+      audio_manager, this, input_parameters, device_id_, NULL);
 
   if (!audio_controller_.get()) {
     return Abort(SpeechRecognitionError(SPEECH_RECOGNITION_ERROR_AUDIO));

diff --git a/content/renderer/media/webaudio_capturer_source.cc b/content/renderer/media/webaudio_capturer_source.cc
@@ -96,7 +96,7 @@ void WebAudioCapturerSource::consumeAudio(
   int capture_frames = params_.frames_per_buffer();
   while (fifo_->frames() >= capture_frames) {
     fifo_->Consume(capture_bus_.get(), 0, capture_frames);
-    callback_->Capture(capture_bus_.get(), 0, 1.0);
+    callback_->Capture(capture_bus_.get(), 0, 1.0, false);
   }
 }
 

diff --git a/content/renderer/media/webrtc_audio_capturer.cc b/content/renderer/media/webrtc_audio_capturer.cc
@@ -102,14 +102,16 @@ class WebRtcAudioCapturer::TrackOwner
                    int number_of_channels,
                    int number_of_frames,
                    int audio_delay_milliseconds,
-                   int volume) {
+                   int volume,
+                   bool key_pressed) {
     base::AutoLock lock(lock_);
     if (delegate_) {
       delegate_->CaptureData(audio_data,
                              number_of_channels,
                              number_of_frames,
                              audio_delay_milliseconds,
-                             volume);
+                             volume,
+                             key_pressed);
     }
   }
 
@@ -424,10 +426,11 @@ void WebRtcAudioCapturer::SetAutomaticGainControl(bool enable) {
 
 void WebRtcAudioCapturer::Capture(media::AudioBus* audio_source,
                                   int audio_delay_milliseconds,
-                                  double volume) {
-  // This callback is driven by AudioInputDevice::AudioThreadCallback if
-  // |source_| is AudioInputDevice, otherwise it is driven by client's
-  // CaptureCallback.
+                                  double volume,
+                                  bool key_pressed) {
+// This callback is driven by AudioInputDevice::AudioThreadCallback if
+// |source_| is AudioInputDevice, otherwise it is driven by client's
+// CaptureCallback.
 #if defined(OS_WIN) || defined(OS_MACOSX)
   DCHECK_LE(volume, 1.0);
 #elif defined(OS_LINUX) || defined(OS_OPENBSD)
@@ -471,8 +474,11 @@ void WebRtcAudioCapturer::Capture(media::AudioBus* audio_source,
        it != tracks.end();
        ++it) {
     (*it)->CaptureData(buffer_ref_while_calling->buffer(),
-                       audio_source->channels(), audio_source->frames(),
-                       audio_delay_milliseconds, volume_);
+                       audio_source->channels(),
+                       audio_source->frames(),
+                       audio_delay_milliseconds,
+                       volume,
+                       key_pressed);
   }
 }
 

diff --git a/content/renderer/media/webrtc_audio_capturer.h b/content/renderer/media/webrtc_audio_capturer.h
@@ -108,7 +108,8 @@ class CONTENT_EXPORT WebRtcAudioCapturer
   // Called on the AudioInputDevice audio thread.
   virtual void Capture(media::AudioBus* audio_source,
                        int audio_delay_milliseconds,
-                       double volume) OVERRIDE;
+                       double volume,
+                       bool key_pressed) OVERRIDE;
   virtual void OnCaptureError() OVERRIDE;
 
   // Reconfigures the capturer with a new buffer size and capture parameters.

diff --git a/content/renderer/media/webrtc_audio_capturer_sink_owner.cc b/content/renderer/media/webrtc_audio_capturer_sink_owner.cc
@@ -18,13 +18,19 @@ int WebRtcAudioCapturerSinkOwner::CaptureData(const std::vector<int>& channels,
                                               int number_of_frames,
                                               int audio_delay_milliseconds,
                                               int current_volume,
-                                              bool need_audio_processing) {
+                                              bool need_audio_processing,
+                                              bool key_pressed) {
   base::AutoLock lock(lock_);
   if (delegate_) {
-    return delegate_->CaptureData(channels, audio_data, sample_rate,
-                                  number_of_channels, number_of_frames,
-                                  audio_delay_milliseconds, current_volume,
-                                  need_audio_processing);
+    delegate_->CaptureData(channels,
+                           audio_data,
+                           sample_rate,
+                           number_of_channels,
+                           number_of_frames,
+                           audio_delay_milliseconds,
+                           current_volume,
+                           need_audio_processing,
+                           key_pressed);
   }
 
   return 0;

diff --git a/content/renderer/media/webrtc_audio_capturer_sink_owner.h b/content/renderer/media/webrtc_audio_capturer_sink_owner.h
@@ -31,7 +31,8 @@ class WebRtcAudioCapturerSinkOwner
                           int number_of_frames,
                           int audio_delay_milliseconds,
                           int current_volume,
-                          bool need_audio_processing) OVERRIDE;
+                          bool need_audio_processing,
+                          bool key_pressed) OVERRIDE;
 
   virtual void SetCaptureFormat(const media::AudioParameters& params) OVERRIDE;
 

diff --git a/content/renderer/media/webrtc_audio_device_impl.cc b/content/renderer/media/webrtc_audio_device_impl.cc
@@ -59,7 +59,8 @@ int WebRtcAudioDeviceImpl::CaptureData(const std::vector<int>& channels,
                                        int number_of_frames,
                                        int audio_delay_milliseconds,
                                        int current_volume,
-                                       bool need_audio_processing) {
+                                       bool need_audio_processing,
+                                       bool key_pressed) {
   int total_delay_ms = 0;
   {
     base::AutoLock auto_lock(lock_);
@@ -75,11 +76,9 @@ int WebRtcAudioDeviceImpl::CaptureData(const std::vector<int>& channels,
   // Write audio samples in blocks of 10 milliseconds to the registered
   // webrtc::AudioTransport sink. Keep writing until our internal byte
   // buffer is empty.
-  // TODO(niklase): Wire up the key press detection.
   const int16* audio_buffer = audio_data;
   const int samples_per_10_msec = (sample_rate / 100);
   int accumulated_audio_samples = 0;
-  bool key_pressed = false;
   uint32_t new_volume = 0;
   while (accumulated_audio_samples < number_of_frames) {
     // Deliver 10ms of recorded 16-bit linear PCM audio.

diff --git a/content/renderer/media/webrtc_audio_device_impl.h b/content/renderer/media/webrtc_audio_device_impl.h
@@ -223,7 +223,8 @@ class WebRtcAudioCapturerSink {
                           int number_of_frames,
                           int audio_delay_milliseconds,
                           int current_volume,
-                          bool need_audio_processing) = 0;
+                          bool need_audio_processing,
+                          bool key_pressed) = 0;
 
   // Set the format for the capture audio parameters.
   virtual void SetCaptureFormat(const media::AudioParameters& params) = 0;
@@ -339,7 +340,8 @@ class CONTENT_EXPORT WebRtcAudioDeviceImpl
                           int number_of_frames,
                           int audio_delay_milliseconds,
                           int current_volume,
-                          bool need_audio_processing) OVERRIDE;
+                          bool need_audio_processing,
+                          bool key_pressed) OVERRIDE;
 
   // Called on the main render thread.
   virtual void SetCaptureFormat(const media::AudioParameters& params) OVERRIDE;

diff --git a/content/renderer/media/webrtc_audio_device_unittest.cc b/content/renderer/media/webrtc_audio_device_unittest.cc
@@ -221,7 +221,8 @@ class MockWebRtcAudioCapturerSink : public WebRtcAudioCapturerSink {
                           int number_of_frames,
                           int audio_delay_milliseconds,
                           int current_volume,
-                          bool need_audio_processing) OVERRIDE {
+                          bool need_audio_processing,
+                          bool key_pressed) OVERRIDE {
     // Signal that a callback has been received.
     event_->Signal();
     return 0;
@@ -381,8 +382,13 @@ int RunWebRtcLoopbackTimeTest(media::AudioManager* manager,
     capturer_sink->CaptureData(
         voe_channels,
         reinterpret_cast<int16*>(capture_data.get() + input_packet_size * j),
-        params.sample_rate(), params.channels(), params.frames_per_buffer(),
-        kHardwareLatencyInMs, 1.0, enable_apm);
+        params.sample_rate(),
+        params.channels(),
+        params.frames_per_buffer(),
+        kHardwareLatencyInMs,
+        1.0,
+        enable_apm,
+        false);
 
     // Receiving data from WebRtc.
     renderer_source->RenderData(

diff --git a/content/renderer/media/webrtc_local_audio_renderer.cc b/content/renderer/media/webrtc_local_audio_renderer.cc
@@ -52,13 +52,14 @@ void WebRtcLocalAudioRenderer::OnRenderError() {
 
 // content::WebRtcAudioCapturerSink implementation
 int WebRtcLocalAudioRenderer::CaptureData(const std::vector<int>& channels,
-                                           const int16* audio_data,
-                                           int sample_rate,
-                                           int number_of_channels,
-                                           int number_of_frames,
-                                           int audio_delay_milliseconds,
-                                           int current_volume,
-                                           bool need_audio_processing) {
+                                          const int16* audio_data,
+                                          int sample_rate,
+                                          int number_of_channels,
+                                          int number_of_frames,
+                                          int audio_delay_milliseconds,
+                                          int current_volume,
+                                          bool need_audio_processing,
+                                          bool key_pressed) {
   TRACE_EVENT0("audio", "WebRtcLocalAudioRenderer::CaptureData");
   base::AutoLock auto_lock(thread_lock_);
 

diff --git a/content/renderer/media/webrtc_local_audio_renderer.h b/content/renderer/media/webrtc_local_audio_renderer.h
@@ -78,7 +78,8 @@ class CONTENT_EXPORT WebRtcLocalAudioRenderer
                           int number_of_frames,
                           int audio_delay_milliseconds,
                           int current_volume,
-                          bool need_audio_processing) OVERRIDE;
+                          bool need_audio_processing,
+                          bool key_pressed) OVERRIDE;
 
   // Can be called on different user thread.
   virtual void SetCaptureFormat(const media::AudioParameters& params) OVERRIDE;