From 3ffb2fde1c3f1ce446f0fe6508367884f14803da Mon Sep 17 00:00:00 2001 From: itsmattkc Date: Wed, 14 Apr 2021 15:25:38 +1000 Subject: [PATCH] audiomonitor: use mipmapped waveform Since we have these now, may as well use them. This should improve general playback performance since the audio monitor will be able to iterate far fewer --- app/audio/audiomanager.h | 3 + app/audio/audiovisualwaveform.cpp | 37 +++++++--- app/audio/audiovisualwaveform.h | 4 + app/widget/audiomonitor/audiomonitor.cpp | 94 +++++++++++++++--------- app/widget/audiomonitor/audiomonitor.h | 11 ++- app/widget/viewer/audiowaveformview.h | 5 ++ app/widget/viewer/viewer.cpp | 2 + 7 files changed, 111 insertions(+), 45 deletions(-) diff --git a/app/audio/audiomanager.h b/app/audio/audiomanager.h index 3e159d2552..d6fbae2327 100644 --- a/app/audio/audiomanager.h +++ b/app/audio/audiomanager.h @@ -27,6 +27,7 @@ #include #include +#include "audiovisualwaveform.h" #include "common/define.h" #include "outputmanager.h" #include "render/audioparams.h" @@ -94,6 +95,8 @@ class AudioManager : public QObject void OutputDeviceStarted(AudioPlaybackCache* cache, qint64 offset, int playback_speed); + void OutputWaveformStarted(const AudioVisualWaveform* waveform, const rational &start, int playback_speed); + void AudioParamsChanged(const AudioParams& params); void OutputPushed(const QByteArray& data); diff --git a/app/audio/audiovisualwaveform.cpp b/app/audio/audiovisualwaveform.cpp index 2a5201a271..1bf307dda8 100644 --- a/app/audio/audiovisualwaveform.cpp +++ b/app/audio/audiovisualwaveform.cpp @@ -203,6 +203,19 @@ void AudioVisualWaveform::Shift(const rational &from, const rational &to) } } +AudioVisualWaveform::Sample AudioVisualWaveform::GetSummaryFromTime(const rational &start, const rational &length) const +{ + // Find mipmap that requries + auto using_mipmap = GetMipmapForScale(length.flipped().toDouble()); + + double rate_dbl = using_mipmap->first.toDouble(); + + int start_sample = time_to_samples(start, rate_dbl); + int sample_length = time_to_samples(length, rate_dbl); + + return ReSumSamples(&using_mipmap->second.constData()[start_sample], sample_length, channels_); +} + AudioVisualWaveform::Sample AudioVisualWaveform::SumSamples(const float *samples, int nb_samples, int nb_channels) { return SumSamplesInternal(samples, nb_samples, nb_channels); @@ -285,15 +298,7 @@ void AudioVisualWaveform::DrawWaveform(QPainter *painter, const QRect& rect, con return; } - // Find largest mipmap for this scale (or the largest if we don't find one sufficient) - auto using_mipmap = samples.mipmapped_data_.cend(); - using_mipmap--; - for (auto it=samples.mipmapped_data_.cbegin(); it!=samples.mipmapped_data_.cend(); it++) { - if (it->first.toDouble() >= scale) { - using_mipmap = it; - break; - } - } + auto using_mipmap = samples.GetMipmapForScale(scale); rational rate = using_mipmap->first; double rate_dbl = rate.toDouble(); @@ -350,6 +355,20 @@ int AudioVisualWaveform::time_to_samples(const double &time, double sample_rate) return qFloor(time * sample_rate) * channels_; } +std::map::const_iterator AudioVisualWaveform::GetMipmapForScale(double scale) const +{ + // Find largest mipmap for this scale (or the largest if we don't find one sufficient) + auto using_mipmap = mipmapped_data_.cend(); + using_mipmap--; + for (auto it=mipmapped_data_.cbegin(); it!=mipmapped_data_.cend(); it++) { + if (it->first.toDouble() >= scale) { + using_mipmap = it; + break; + } + } + return using_mipmap; +} + template AudioVisualWaveform::Sample AudioVisualWaveform::SumSamplesInternal(const T *samples, int nb_samples, int nb_channels) { diff --git a/app/audio/audiovisualwaveform.h b/app/audio/audiovisualwaveform.h index 447e4fd848..d85760d392 100644 --- a/app/audio/audiovisualwaveform.h +++ b/app/audio/audiovisualwaveform.h @@ -85,6 +85,8 @@ class AudioVisualWaveform { void Shift(const rational& from, const rational& to); + Sample GetSummaryFromTime(const rational& start, const rational& length) const; + static Sample SumSamples(const float* samples, int nb_samples, int nb_channels); static Sample SumSamples(SampleBufferPtr samples, int start_index, int length); @@ -108,6 +110,8 @@ class AudioVisualWaveform { int time_to_samples(const rational& time, double sample_rate) const; int time_to_samples(const double& time, double sample_rate) const; + std::map::const_iterator GetMipmapForScale(double scale) const; + int channels_; std::map mipmapped_data_; diff --git a/app/widget/audiomonitor/audiomonitor.cpp b/app/widget/audiomonitor/audiomonitor.cpp index 22ba0eb584..0ba3f24755 100644 --- a/app/widget/audiomonitor/audiomonitor.cpp +++ b/app/widget/audiomonitor/audiomonitor.cpp @@ -36,11 +36,12 @@ const int kMaximumSmoothness = 8; AudioMonitor::AudioMonitor(QWidget *parent) : QOpenGLWidget(parent), file_(nullptr), + waveform_(nullptr), cached_channels_(0) { values_.resize(kMaximumSmoothness); - connect(AudioManager::instance(), &AudioManager::OutputDeviceStarted, this, &AudioMonitor::OutputDeviceSet); + connect(AudioManager::instance(), &AudioManager::OutputWaveformStarted, this, &AudioMonitor::OutputAudioVisualWaveformSet); connect(AudioManager::instance(), &AudioManager::OutputPushed, this, &AudioMonitor::OutputPushed); connect(AudioManager::instance(), &AudioManager::AudioParamsChanged, this, &AudioMonitor::SetParams); connect(AudioManager::instance(), &AudioManager::Stopped, this, &AudioMonitor::Stop); @@ -84,6 +85,7 @@ void AudioMonitor::Stop() { delete file_; file_ = nullptr; + waveform_ = nullptr; } void AudioMonitor::OutputPushed(const QByteArray &d) @@ -97,6 +99,20 @@ void AudioMonitor::OutputPushed(const QByteArray &d) SetUpdateLoop(true); } +void AudioMonitor::OutputAudioVisualWaveformSet(const AudioVisualWaveform *waveform, const rational &start, int playback_speed) +{ + Stop(); + + waveform_ = waveform; + waveform_time_ = start; + + playback_speed_ = playback_speed; + + last_time_ = QDateTime::currentMSecsSinceEpoch(); + + SetUpdateLoop(true); +} + void AudioMonitor::SetUpdateLoop(bool e) { if (e) { @@ -211,8 +227,24 @@ void AudioMonitor::paintGL() QVector v(params_.channel_count(), 0); - if (file_) { - UpdateValuesFromFile(v); + if (file_ || waveform_) { + // Determines how many milliseconds have passed since last update + qint64 current_time = QDateTime::currentMSecsSinceEpoch(); + qint64 delta_time = current_time - last_time_; + int abs_speed = qAbs(playback_speed_); + + // Multiply by speed if the speed is not 1 + if (abs_speed != 1) { + delta_time *= abs_speed; + } + + if (file_) { + UpdateValuesFromFile(v, delta_time); + } else if (waveform_) { + UpdateValuesFromWaveform(v, delta_time); + } + + last_time_ = current_time; } PushValue(v); @@ -254,7 +286,7 @@ void AudioMonitor::paintGL() } } - if (all_zeroes && !file_) { + if (all_zeroes && !file_ && !waveform_) { // Optimize by disabling the update loop SetUpdateLoop(false); } @@ -266,20 +298,10 @@ void AudioMonitor::mousePressEvent(QMouseEvent *) update(); } -void AudioMonitor::UpdateValuesFromFile(QVector& v) +void AudioMonitor::UpdateValuesFromFile(QVector& v, qint64 delta_time) { - // Determines how many milliseconds have passed since last update - qint64 current_time = QDateTime::currentMSecsSinceEpoch(); - qint64 time_passed = current_time - last_time_; - int abs_speed = qAbs(playback_speed_); - - // Multiply by speed if the speed is not 1 - if (abs_speed != 1) { - time_passed *= abs_speed; - } - // Convert ms to float seconds and determine how many bytes that is - qint64 bytes_to_read = params_.time_to_bytes(static_cast(time_passed) * 0.001); + qint64 bytes_to_read = params_.time_to_bytes(static_cast(delta_time) * 0.001); if (playback_speed_ < 0) { // If reversing, jump back by the amount of bytes we're going to read @@ -297,31 +319,35 @@ void AudioMonitor::UpdateValuesFromFile(QVector& v) file_->seek(file_->pos() - bytes_to_read); } - // If speed is not 1, transform it here - if (abs_speed != 1) { - int sample_sz = params_.samples_to_bytes(1); - int in_nb_samples = params_.bytes_to_samples(b.size()); - int out_nb_samples = in_nb_samples / abs_speed; - QByteArray speed_adjusted(out_nb_samples * sample_sz, Qt::Uninitialized); - - for (int i=0;i &v, qint64 delta_time) +{ + // Delta time is provided in milliseconds, so we convert to seconds in rational + rational length(delta_time, 1000); - BytesToSampleSummary(b, v); + AudioVisualWaveform::Sample sum = waveform_->GetSummaryFromTime(waveform_time_, length); - last_time_ = current_time; + for (int i=0; i v.at(output_index)) { + v[output_index] = max; + } + } + + waveform_time_ += length; } void AudioMonitor::PushValue(const QVector &v) { - values_.removeFirst(); - values_.append(v); + int lim = values_.size()-1; + for (int i=0; i &v) diff --git a/app/widget/audiomonitor/audiomonitor.h b/app/widget/audiomonitor/audiomonitor.h index e9189b3d81..daeb5cec2c 100644 --- a/app/widget/audiomonitor/audiomonitor.h +++ b/app/widget/audiomonitor/audiomonitor.h @@ -25,6 +25,7 @@ #include #include +#include "audio/audiovisualwaveform.h" #include "common/define.h" #include "render/audioparams.h" #include "render/audioplaybackcache.h" @@ -46,8 +47,9 @@ public slots: void OutputPushed(const QByteArray& d); + void OutputAudioVisualWaveformSet(const AudioVisualWaveform *waveform, const rational& start, int playback_speed); + protected: - //virtual void paintEvent(QPaintEvent* event) override; virtual void paintGL() override; virtual void mousePressEvent(QMouseEvent* event) override; @@ -55,7 +57,9 @@ public slots: private: void SetUpdateLoop(bool e); - void UpdateValuesFromFile(QVector &v); + void UpdateValuesFromFile(QVector &v, qint64 delta_time); + + void UpdateValuesFromWaveform(QVector &v, qint64 delta_time); void PushValue(const QVector& v); @@ -68,6 +72,9 @@ public slots: QIODevice* file_; qint64 last_time_; + const AudioVisualWaveform* waveform_; + rational waveform_time_; + int playback_speed_; QVector< QVector > values_; diff --git a/app/widget/viewer/audiowaveformview.h b/app/widget/viewer/audiowaveformview.h index 097ff1e050..18e59eb7fb 100644 --- a/app/widget/viewer/audiowaveformview.h +++ b/app/widget/viewer/audiowaveformview.h @@ -41,6 +41,11 @@ class AudioWaveformView : public SeekableWidget void SetViewer(AudioPlaybackCache *playback); + const AudioVisualWaveform* waveform() const + { + return &waveform_; + } + protected: virtual void paintEvent(QPaintEvent* event) override; diff --git a/app/widget/viewer/viewer.cpp b/app/widget/viewer/viewer.cpp index 7924a6699a..2a236c55f1 100644 --- a/app/widget/viewer/viewer.cpp +++ b/app/widget/viewer/viewer.cpp @@ -412,6 +412,8 @@ void ViewerWidget::StartAudioOutput() AudioManager::instance()->StartOutput(audio_cache, audio_cache->GetParameters().time_to_bytes(GetTime()), playback_speed_); + emit AudioManager::instance()->OutputWaveformStarted(waveform_view_->waveform(), + GetTime(), playback_speed_); } }