diff --git a/components/Screen.qml b/components/Screen.qml index 8f848f9..63d0836 100644 --- a/components/Screen.qml +++ b/components/Screen.qml @@ -12,6 +12,8 @@ Rectangle { height = _height; } + + //Brukes dette? Image { id: liveImage anchors.fill: parent diff --git a/handlers/audioplaybackhandler.cpp b/handlers/audioplaybackhandler.cpp index f6351c0..8c5cc25 100644 --- a/handlers/audioplaybackhandler.cpp +++ b/handlers/audioplaybackhandler.cpp @@ -1,22 +1,14 @@ #include "audioplaybackhandler.h" -AudioPlaybackHandler::AudioPlaybackHandler(std::mutex* _writeLock, QByteArray* buffer, int bufferSize, - QObject *parent) : QObject(parent) +AudioPlaybackHandler::AudioPlaybackHandler(std::mutex* _writeLock, QByteArray* buffer, size_t bufferSize, + QObject *parent) : Playback(_writeLock, buffer, bufferSize, parent) { - // mLastVideoPacketTime = lastVideoPacketTime; - //mLastAudioPacketTime = lastAudioPacketTime; - mBufferSize = bufferSize; - // mSocketHandler = _socketHandler; - initAudio(parent); - mStruct = new mBufferAndLockStruct(); - mStruct->buffer = buffer; - mStruct->writeLock = _writeLock; } AudioPlaybackHandler::~AudioPlaybackHandler() { - delete mStruct; + } void AudioPlaybackHandler::initAudio(QObject *parent) @@ -38,282 +30,194 @@ void AudioPlaybackHandler::initAudio(QObject *parent) mpOut = mpAudio->start(); } -/* -void PlaybackHandler::changeSpeaker() -{ - //todo -} -*/ - -/** - * Custom readPacket function for av_read_frame and av_open_input. - * Will read and remove bytes from the buffer found in the struct - * and copy them to buf. - * @param buf_size int how many bytes to read from the buffer - * @param[out] buf uint8_t* bytes to send back to ffmpeg - * @param opaque void* pointer set by avio_alloc_context - * @return buf_size int - */ -int AudioPlaybackHandler::customReadPacket(void *opaque, uint8_t *buf, int buf_size) + + +void AudioPlaybackHandler::start() { - //qDebug() << buf_size; - mBufferAndLockStruct *s = reinterpret_cast(opaque); + int error = 0; + AVFormatContext *inputFormatContext = avformat_alloc_context(); + Q_ASSERT(inputFormatContext); - //buf_size = FFMIN(buf_size, s->socketHandler->mBuffer.size()); + uint8_t *avioContextBuffer = reinterpret_cast(av_malloc(mBufferSize)); + Q_ASSERT(avioContextBuffer); + AVIOContext *avioContext = avio_alloc_context(avioContextBuffer, static_cast(mBufferSize), 0, mStruct, &customReadPacket, nullptr, nullptr); + Q_ASSERT(avioContext); - while (s->buffer->size() <= buf_size) + inputFormatContext->pb = avioContext; + error = avformat_open_input(&inputFormatContext, nullptr, nullptr, nullptr); + if(error < 0) { - //int ms = 5; - //struct timespec ts = { ms / 1000, (ms % 1000) * 1000 * 1000 }; - //qDebug() << "sleeping"; - //nanosleep(&ts, NULL); + char* errbuff = (char *)malloc((1000)*sizeof(char)); + av_strerror(error,errbuff,1000); + qDebug() << "AVformat open input UDP stream failed" << errbuff; + exit(1); + } + //ret = avformat_find_stream_info(fmt_ctx, nullptr); + if(error < 0) + { + char* errbuff = (char *)malloc((1000)*sizeof(char)); + av_strerror(error,errbuff,1000); + qDebug() << "AVFormat find udp stream failed" << errbuff; + exit(1); } - s->writeLock->lock(); - - QByteArray tempBuffer = QByteArray(s->buffer->data(), buf_size); - s->buffer->remove(0,buf_size); - - s->writeLock->unlock(); - //qDebug() << " buffer after removal: " << s->socketHandler->mBuffer.size(); - + qDebug() << "Dumping audioplayback format"; + av_dump_format(inputFormatContext, 0, NULL, 0); - memcpy(buf, tempBuffer.constData(), buf_size); + AVStream * audio_stream = nullptr; + for (uint i=0; i < inputFormatContext->nb_streams; ++i) { + auto st = inputFormatContext->streams[i]; + //Debug() << st->id << st->index << st->start_time << st->duration << st->codecpar->codec_type; + if(st->codecpar->codec_type == AVMEDIA_TYPE_AUDIO) + { + audio_stream = st; + } + } - //mSenderId = something; - //qDebug() << "Reading packet"; - return buf_size; -} + AVCodecParameters *audioStreamCodecParameters = audio_stream->codecpar; + AVCodec* audioDecoderCodec = avcodec_find_decoder(audioStreamCodecParameters->codec_id); + AVCodecContext *audioDecoderCodecContext = avcodec_alloc_context3(audioDecoderCodec); + error = avcodec_parameters_to_context(audioDecoderCodecContext, audio_stream->codecpar); + Q_ASSERT(error>=0); + error = avcodec_open2(audioDecoderCodecContext, audioDecoderCodec, nullptr); + Q_ASSERT(error>=0); + + //Hardcoded sample because the audio_stream codecpar does not contain samplerate + audioDecoderCodecContext->sample_rate = 48000; + audioDecoderCodecContext->channels = 2; + + // To initalize libao for playback + ao_initialize(); + + int driver = ao_default_driver_id(); + + // The format of the decoded PCM samples + ao_sample_format sample_format; + sample_format.bits = 16; + sample_format.channels = 2; + sample_format.rate = 48000; //Must match input with 48000 or audio stream delay keeps increasing + sample_format.byte_format = AO_FMT_NATIVE; + sample_format.matrix = 0; + + ao_device* device = ao_open_live(driver, &sample_format, NULL); + + + // qDebug() << audio_stream->codecpar->sample_rate; + //qDebug() << audioDecoderCodecContext->sample_rate; + SwrContext *resample_context = NULL; + resample_context = swr_alloc_set_opts(NULL, + av_get_default_channel_layout(2), + AV_SAMPLE_FMT_S16, + audioDecoderCodecContext->sample_rate, + av_get_default_channel_layout(audioDecoderCodecContext->channels), + audioDecoderCodecContext->sample_fmt, + audioDecoderCodecContext->sample_rate, + 0, NULL); + + + //qDebug() << resample_context; + if (!(resample_context)) { + fprintf(stderr,"Unable to allocate resampler context\n"); + exit(-1); + //return AVERROR(ENOMEM); + } -void AudioPlaybackHandler::start() -{ + // Open the resampler - //int ms = 10000; - // struct timespec ts = { ms / 1000, (ms % 1000) * 1000 * 1000 }; - //nanosleep(&ts, NULL); + if ((error = swr_init(resample_context)) < 0) { + fprintf(stderr,"Unable to open resampler context: "); + swr_free(&resample_context); + exit(-1); + } - AVFormatContext *fmt_ctx = nullptr; - AVIOContext *avio_ctx = nullptr; - uint8_t /**buffer = nullptr, */*avio_ctx_buffer = nullptr; - size_t /*buffer_size = 0,*/ avio_ctx_buffer_size = mBufferSize; + AVFrame* frame = av_frame_alloc(); + AVPacket packet; + AVFrame* resampled = 0; - int ret = 0; - fmt_ctx = avformat_alloc_context(); - Q_ASSERT(fmt_ctx); + while (!mStopPlayback) { + error = av_read_frame(inputFormatContext,&packet); + if(error < 0) + { + char* errbuff = (char *)malloc((1000)*sizeof(char)); + av_strerror(error,errbuff,1000); + qDebug() << "Failed av_read_frame: code " << error << " meaning: " << errbuff; + int ms = 1000; + struct timespec ts = { ms / 1000, (ms % 1000) * 1000 * 1000 }; + nanosleep(&ts, NULL); + continue; + } + //qDebug() << "stream: " << packet.stream_index << " mvideostream: " << mVideoStreamIndex; - avio_ctx_buffer = reinterpret_cast(av_malloc(avio_ctx_buffer_size)); - Q_ASSERT(avio_ctx_buffer); - avio_ctx = avio_alloc_context(avio_ctx_buffer, static_cast(avio_ctx_buffer_size), 0, mStruct, &customReadPacket, nullptr, nullptr); - Q_ASSERT(avio_ctx); - fmt_ctx->pb = avio_ctx; - ret = avformat_open_input(&fmt_ctx, nullptr, nullptr, nullptr); - if(ret < 0) + error = avcodec_send_packet(audioDecoderCodecContext, &packet); + if (error == AVERROR_EOF || error == AVERROR(EOF)) { - char* errbuff = (char *)malloc((1000)*sizeof(char)); - av_strerror(ret,errbuff,1000); - qDebug() << "AVformat open input UDP stream failed" << errbuff; - exit(1); + qDebug() << "send packet sleep"; + int ms = 1000; + struct timespec ts = { ms / 1000, (ms % 1000) * 1000 * 1000 }; + nanosleep(&ts, NULL); + continue; } - //ret = avformat_find_stream_info(fmt_ctx, nullptr); - if(ret < 0) + else if(error < 0) { char* errbuff = (char *)malloc((1000)*sizeof(char)); - av_strerror(ret,errbuff,1000); - qDebug() << "AVFormat find udp stream failed" << errbuff; + av_strerror(error,errbuff,1000); + qDebug() << "Failed udp input avcodec_send_packet: code "<nb_streams; ++i) { - auto st = fmt_ctx->streams[i]; - //Debug() << st->id << st->index << st->start_time << st->duration << st->codecpar->codec_type; - if(st->codecpar->codec_type == AVMEDIA_TYPE_AUDIO) - { - audio_stream = st; - mAudioStreamIndex = i; - } + if (!resampled) + { + resampled = av_frame_alloc(); } + resampled->channel_layout = av_get_default_channel_layout(2); + resampled->sample_rate = audioDecoderCodecContext->sample_rate; + resampled->format = AV_SAMPLE_FMT_S16; - int err; - AVCodecContext *audioDecoderCodecContext; - ao_device* device; - - - AVCodecParameters *audioStreamCodecParameters = audio_stream->codecpar; - AVCodec* audioDecoderCodec = avcodec_find_decoder(audioStreamCodecParameters->codec_id); - audioDecoderCodecContext = avcodec_alloc_context3(audioDecoderCodec); - err = avcodec_parameters_to_context(audioDecoderCodecContext, audio_stream->codecpar); - Q_ASSERT(err>=0); - err = avcodec_open2(audioDecoderCodecContext, audioDecoderCodec, nullptr); - Q_ASSERT(err>=0); - - //Hardcoded sample because the audio_stream codecpar does not contain samplerate - audioDecoderCodecContext->sample_rate = 48000; - audioDecoderCodecContext->channels = 2; - - // To initalize libao for playback - ao_initialize(); - - int driver = ao_default_driver_id(); - - // The format of the decoded PCM samples - ao_sample_format sample_format; - sample_format.bits = 16; - sample_format.channels = 2; - sample_format.rate = 48000; //Must match input with 48000 or audio stream delay keeps increasing - sample_format.byte_format = AO_FMT_NATIVE; - sample_format.matrix = 0; - - device = ao_open_live(driver, &sample_format, NULL); - - - // qDebug() << audio_stream->codecpar->sample_rate; - //qDebug() << audioDecoderCodecContext->sample_rate; - SwrContext *resample_context = NULL; - resample_context = swr_alloc_set_opts(NULL, - av_get_default_channel_layout(2), - AV_SAMPLE_FMT_S16, - audioDecoderCodecContext->sample_rate, - av_get_default_channel_layout(audioDecoderCodecContext->channels), - audioDecoderCodecContext->sample_fmt, - audioDecoderCodecContext->sample_rate, - 0, NULL); - - - //qDebug() << resample_context; - if (!(resample_context)) { - fprintf(stderr,"Unable to allocate resampler context\n"); - exit(-1); - //return AVERROR(ENOMEM); + if ((error = swr_convert_frame(resample_context, resampled, frame)) < 0) + { + char* errbuff = (char *)malloc((1000)*sizeof(char)); + av_strerror(error,errbuff,1000); + qDebug() << "Failed playbackhandler swr_convert_frame: code "<data[0], resampled->linesize[0]); + ao_play(device, (char*)resampled->extended_data[0], + av_samples_get_buffer_size(resampled->linesize, + resampled->channels, + resampled->nb_samples, + (AVSampleFormat)resampled->format, + 0)); } + av_frame_unref(resampled); + av_frame_unref(frame); + av_packet_unref(&packet); - AVFrame* frame = av_frame_alloc(); - AVPacket packet; - AVFrame* resampled = 0; - - while (1) { - ret = av_read_frame(fmt_ctx,&packet); - if(ret < 0) - { - char* errbuff = (char *)malloc((1000)*sizeof(char)); - av_strerror(ret,errbuff,1000); - qDebug() << "Failed av_read_frame: code " << ret << " meaning: " << errbuff; - int ms = 1000; - struct timespec ts = { ms / 1000, (ms % 1000) * 1000 * 1000 }; - nanosleep(&ts, NULL); - continue; - } - //qDebug() << "stream: " << packet.stream_index << " mvideostream: " << mVideoStreamIndex; - - if(packet.stream_index == mAudioStreamIndex) - { - ret = avcodec_send_packet(audioDecoderCodecContext, &packet); - if (ret == AVERROR_EOF || ret == AVERROR(EOF)) - { - qDebug() << "send packet sleep"; - int ms = 1000; - struct timespec ts = { ms / 1000, (ms % 1000) * 1000 * 1000 }; - nanosleep(&ts, NULL); - continue; - } - else if(ret < 0) - { - char* errbuff = (char *)malloc((1000)*sizeof(char)); - av_strerror(ret,errbuff,1000); - qDebug() << "Failed udp input avcodec_send_packet: code "<pts; - //qDebug() << "AudioPacketTime: " << *mLastAudioPacketTime; - //sync(); - - if (!resampled) - { - resampled = av_frame_alloc(); - } - - resampled->channel_layout = av_get_default_channel_layout(2); - resampled->sample_rate = audioDecoderCodecContext->sample_rate; - resampled->format = AV_SAMPLE_FMT_S16; - - if ((ret = swr_convert_frame(resample_context, resampled, frame)) < 0) - { - char* errbuff = (char *)malloc((1000)*sizeof(char)); - av_strerror(ret,errbuff,1000); - qDebug() << "Failed playbackhandler swr_convert_frame: code "<data[0], resampled->linesize[0]); - ao_play(device, (char*)resampled->extended_data[0], - av_samples_get_buffer_size(resampled->linesize, - resampled->channels, - resampled->nb_samples, - (AVSampleFormat)resampled->format, - 0)); - } - av_frame_unref(resampled); - av_frame_unref(frame); - } - else - { - qDebug() << "Vi er fucked"; - } - - - av_frame_unref(frame); - av_packet_unref(&packet); + } - } + swr_free(&resample_context); + avformat_close_input(&inputFormatContext); + avcodec_free_context(&audioDecoderCodecContext); + ao_close(device); } - -/*void AudioPlaybackHandler::sync() -{ - if(*mLastAudioPacketTime != -1) - { - int diff = *mLastVideoPacketTime - *mLastAudioPacketTime; - if(diff < 0) - { - //int ms = abs(diff)/1000; - int ms = 5; - - //qDebug() << "Audio Sleeping: " << ms; - - struct timespec ts = { ms / 1000, (ms % 1000) * 1000 * 1000 }; - nanosleep(&ts, NULL); - } - } -}*/ diff --git a/handlers/audioplaybackhandler.h b/handlers/audioplaybackhandler.h index 88cd3cf..3719870 100644 --- a/handlers/audioplaybackhandler.h +++ b/handlers/audioplaybackhandler.h @@ -11,35 +11,22 @@ extern "C" { #include "libswresample/swresample.h" } #include "handlers/imagehandler.h" +#include "playback.h" -class AudioPlaybackHandler : public QObject +class AudioPlaybackHandler : public Playback { Q_OBJECT public: AudioPlaybackHandler(std::mutex* writeLock, QByteArray* buffer, - int bufferSize, QObject *parent = nullptr); + size_t bufferSize, QObject *parent = nullptr); ~AudioPlaybackHandler(); - void getStream(); - static int customReadPacket(void *opaque, uint8_t *buf, int buf_size); void start(); - int decodeAndPlay(); - int mAudioStreamIndex = -1; private: - int mVectorIndex; - struct mBufferAndLockStruct { - QByteArray* buffer; - std::mutex* writeLock; - }; - int mBufferSize; - mBufferAndLockStruct* mStruct; void initAudio(QObject *parent); - QByteArray mBuffer; QAudioFormat mAudioFormat; QAudioOutput* mpAudio; QIODevice* mpOut; - uint8_t mSenderId = 0; //Value set to 0 just for testing. - ImageHandler* mImageHandler; -signals: + }; #endif // PLAYBACKHANDLER_H diff --git a/handlers/imagehandler.cpp b/handlers/imagehandler.cpp index 1ea5f12..36986cf 100644 --- a/handlers/imagehandler.cpp +++ b/handlers/imagehandler.cpp @@ -11,7 +11,7 @@ ImageHandler::ImageHandler(Settings* settings) : QQuickImageProvider(QQuickImage QImage ImageHandler::requestImage(const QString &id, QSize *size, const QSize &requestedSize) { - int index = 0; + uint8_t index = 0; QStringList onlyId = id.split("="); if(onlyId.size() >= 2) { @@ -19,13 +19,19 @@ QImage ImageHandler::requestImage(const QString &id, QSize *size, const QSize &r if(idIndex.size() >= 2) { - index = idIndex[1].toInt(); - if(index > 0) - { - //qDebug() << index; - } + index = idIndex[1].toUInt(); } } + //This means the screen for the user, which is stored in numeric_value_max in the map + if(index==0) + { + index = std::numeric_limits::max(); + } + else + { + //All other participants are located in the map at their screen index-1 + index--; + } QImage result = mImageMap[index].first; if(result.isNull()) @@ -101,10 +107,6 @@ void ImageHandler::updatePeerDisplayName(uint8_t index, QString displayName) void ImageHandler::updateImage(const QImage &image, uint8_t index) { imgLock.lock(); - if(index > 0) - { - //qDebug() << index; - } if(mImageMap[index].first != image) { mImageMap[index].first = image; @@ -169,7 +171,7 @@ void ImageHandler::readImage(AVCodecContext* codecContext, AVFrame* frame, uint8 if(codecContext == nullptr) { - emit updateImage(generateGenericImage(mSettings->getDisplayName()), 0); + updateImage(generateGenericImage(mSettings->getDisplayName()), std::numeric_limits::max()); return; } @@ -206,7 +208,7 @@ void ImageHandler::readImage(AVCodecContext* codecContext, AVFrame* frame, uint8 sws_freeContext(imgConvertCtx); //av_frame_free(&frameRGB); - emit updateImage(img, index); + updateImage(img, index); //av_frame_unref(frameRGB); //av_frame_free(&frameRGB); //delete frameRGB; diff --git a/handlers/inputstreamhandler.cpp b/handlers/inputstreamhandler.cpp index 1ce1eea..e7fa0a4 100644 --- a/handlers/inputstreamhandler.cpp +++ b/handlers/inputstreamhandler.cpp @@ -147,12 +147,18 @@ void InputStreamHandler::handleHeader(QByteArray data) /** * When recieving a TCP request with a header from a unknown streamId, * we need to create new buffers, mutex and playbackhandlers for both video and audio. - * @param streamId QString to add to mStreamIdVector + * @param streamId QString to add to mStreamIdVector, used to identify packets and datagrams * @param index int to add a peer to ImageHandler + * @param displayName QString to display on GUI */ void InputStreamHandler::addStreamToVector(int index, QString streamId, QString displayName) { qDebug() << "Adding streamId: " << streamId; + if(index >= std::numeric_limits::max()) + { + qDebug() << "We currently do not allow for more than 255 participants in a room" << Q_FUNC_INFO; + return; + } QByteArray* tempVideoHeaderBuffer = new QByteArray(); mVideoHeaderVector.push_back(tempVideoHeaderBuffer); QByteArray* tempAudioBuffer = new QByteArray(); @@ -164,7 +170,8 @@ void InputStreamHandler::addStreamToVector(int index, QString streamId, QString mAudioMutexVector.push_back(tempAudioLock); mVideoMutexVector.push_back(tempVideoLock); mAudioPlaybackHandlerVector.push_back(new AudioPlaybackHandler(tempAudioLock, tempAudioBuffer, mBufferSize)); - mVideoPlaybackHandlerVector.push_back(new VideoPlaybackHandler(tempVideoLock, mImageHandler, tempVideoHeaderBuffer, tempVideoBuffer, mBufferSize, (index + 1))); + mVideoPlaybackHandlerVector.push_back(new VideoPlaybackHandler(tempVideoLock, tempVideoBuffer, mBufferSize, + mImageHandler, index)); mStreamIdVector.push_back(streamId); mAudioPlaybackStartedVector.push_back(false); mVideoPlaybackStartedVector.push_back(false); diff --git a/handlers/outputstreamhandler.cpp b/handlers/outputstreamhandler.cpp index 67ce2c1..df261a4 100644 --- a/handlers/outputstreamhandler.cpp +++ b/handlers/outputstreamhandler.cpp @@ -1,7 +1,7 @@ #include "outputstreamhandler.h" -OutputStreamHandler::OutputStreamHandler(ImageHandler* _imageHandler, UdpSocketHandler* _socketHandler, int bufferSize, Settings* settings, TcpSocketHandler* tcpSocketHandler, QObject *parent) : QObject(parent) +OutputStreamHandler::OutputStreamHandler(ImageHandler* _imageHandler, UdpSocketHandler* _socketHandler, size_t bufferSize, Settings* settings, TcpSocketHandler* tcpSocketHandler, QObject *parent) : QObject(parent) { mSettings = settings; mBufferSize = bufferSize; @@ -91,7 +91,8 @@ void OutputStreamHandler::grabVideoHeader() { qDebug() << "Creating new VideoHandler"; mVideoHandler = new VideoHandler(mVideoDevice, &mUDPSendDatagramMutexLock, - mTime, mImageHandler, mSocketHandler,mBufferSize, mTcpSocketHandler); + mTime, mImageHandler, mSocketHandler, + mBufferSize, mTcpSocketHandler); } qDebug() << "Init videoHandler"; mVideoHandler->init(); @@ -112,7 +113,7 @@ int OutputStreamHandler::enableAudio() qDebug() << "creating mAudioHandler"; //int64_t time = av_gettime(); mAudioHandler = new AudioHandler(mAudioDevice, &mUDPSendDatagramMutexLock, - mTime, mSocketHandler,mBufferSize); + mTime, mSocketHandler, mBufferSize); } int error = mAudioHandler->init(); @@ -150,7 +151,8 @@ int OutputStreamHandler::enableVideo() qDebug() << "new videohandler"; //int64_t time = av_gettime(); mVideoHandler = new VideoHandler(mVideoDevice, &mUDPSendDatagramMutexLock, - mTime, mImageHandler, mSocketHandler, mBufferSize, mTcpSocketHandler); + mTime, mImageHandler, mSocketHandler, + mBufferSize, mTcpSocketHandler); } int error = mVideoHandler->init(); diff --git a/handlers/outputstreamhandler.h b/handlers/outputstreamhandler.h index 294a9ff..4a0f619 100644 --- a/handlers/outputstreamhandler.h +++ b/handlers/outputstreamhandler.h @@ -12,7 +12,7 @@ class OutputStreamHandler : public QObject { Q_OBJECT public: - OutputStreamHandler(ImageHandler* _imageHandler, UdpSocketHandler* _socketHandler, int buffer_size, Settings* settings, TcpSocketHandler* tcpSocketHandler, QObject *parent = nullptr); + OutputStreamHandler(ImageHandler* _imageHandler, UdpSocketHandler* _socketHandler, size_t buffer_size, Settings* settings, TcpSocketHandler* tcpSocketHandler, QObject *parent = nullptr); VideoHandler* mVideoHandler = nullptr; AudioHandler* mAudioHandler = nullptr; Q_INVOKABLE void disableAudio(); @@ -35,7 +35,7 @@ class OutputStreamHandler : public QObject ImageHandler* mImageHandler; bool mAudioEnabled = true; bool mVideoEnabled = true; - int mBufferSize; + size_t mBufferSize; std::mutex mUDPSendDatagramMutexLock; QString mAudioDevice; QString mVideoDevice; diff --git a/handlers/sessionhandler.cpp b/handlers/sessionhandler.cpp index 401809f..11b6618 100644 --- a/handlers/sessionhandler.cpp +++ b/handlers/sessionhandler.cpp @@ -118,7 +118,7 @@ bool SessionHandler::joinSession(QString _roomId, QString _roomPassword) qDebug() << "Adding peer with display name" << mSettings->getDisplayName(); //TODO maybe more intensive to find numeric_limit in map compared to 0? - uint_8 userIndex = std::numeric_limits::max(); + uint8_t userIndex = std::numeric_limits::max(); mImageHandler->addPeer(userIndex, mSettings->getDisplayName()); diff --git a/handlers/videohandler.cpp b/handlers/videohandler.cpp index b3efc80..9486f61 100644 --- a/handlers/videohandler.cpp +++ b/handlers/videohandler.cpp @@ -294,7 +294,7 @@ void VideoHandler::grabFrames() scaledFrame->pts = pts; pts += ifmt_ctx->streams[0]->time_base.den/ifmt_ctx->streams[0]->r_frame_rate.num; } - imageHandler->readImage(outputVideoCodecContext, scaledFrame, 0); + imageHandler->readImage(outputVideoCodecContext, scaledFrame, std::numeric_limits::max()); ret = avcodec_send_frame(outputVideoCodecContext, scaledFrame); if(ret < 0) { @@ -318,7 +318,7 @@ void VideoHandler::grabFrames() videoFrame->pts = pts; pts += ifmt_ctx->streams[0]->time_base.den/ifmt_ctx->streams[0]->r_frame_rate.num; } - imageHandler->readImage(outputVideoCodecContext, videoFrame, 0); + imageHandler->readImage(outputVideoCodecContext, videoFrame, std::numeric_limits::max()); ret = avcodec_send_frame(outputVideoCodecContext, videoFrame); //av_frame_free(&videoFrame); if(ret < 0) diff --git a/handlers/videoplaybackhandler.cpp b/handlers/videoplaybackhandler.cpp index 4e16631..e6c2b15 100644 --- a/handlers/videoplaybackhandler.cpp +++ b/handlers/videoplaybackhandler.cpp @@ -1,265 +1,154 @@ #include "videoplaybackhandler.h" -VideoPlaybackHandler::VideoPlaybackHandler(std::mutex* _writeLock,ImageHandler* _imageHandler, QByteArray* headerBuffer, - QByteArray* buffer, int bufferSize, int index, - //int64_t* lastVideoPacketTime, int64_t* lastAudioPacketTime, - QObject *parent) : QObject(parent) +VideoPlaybackHandler::VideoPlaybackHandler(std::mutex* _writeLock, QByteArray* buffer, + size_t bufferSize,ImageHandler* _imageHandler,int index, + QObject *parent) : Playback(_writeLock, buffer, bufferSize, parent) { mIndex = index; - mBufferSize = bufferSize; - mImageHandler = _imageHandler; - mStruct = new mBufferAndLockStruct(); - mStruct->buffer = buffer; - mStruct->writeLock = _writeLock; - //Lagt til disse for å kunne få header via tcp - mStruct->headerReceived = new bool(false); - mStruct->headerBuffer = headerBuffer; - + imageHandler = _imageHandler; } VideoPlaybackHandler::~VideoPlaybackHandler() -{ - delete mStruct; -} - -int VideoPlaybackHandler::read_packet(void *opaque, uint8_t *buf, int buf_size) { - // qDebug() << "Inne i read packet"; +} - mBufferAndLockStruct *s = reinterpret_cast(opaque); - //Kaller server for å få header via tcp - QByteArray tempBuffer; +void VideoPlaybackHandler::start() +{ + int error = 0; + AVFormatContext *inputFormatContext = avformat_alloc_context(); + Q_ASSERT(inputFormatContext); - if(0)//!(*s->headerReceived)) - { - //while(s->headerBuffer->size() <= buf_size); - buf_size = FFMIN(buf_size, s->headerBuffer->size()); - if(!buf_size) - { - qDebug() << AVERROR_EXIT; - qDebug() << AVERROR_EOF; - return AVERROR_EXIT; - } - s->writeLock->lock(); - //tempBuffer = QByteArray(s->headerBuffer->data(), buf_size); - qDebug() << s->headerBuffer->length(); - qDebug() << buf_size; - qDebug() << "HeaderBuffer: " << tempBuffer; + uint8_t *avioContextBuffer = reinterpret_cast(av_malloc(mBufferSize)); + Q_ASSERT(avioContextBuffer); - s->headerBuffer->remove(0, buf_size); + AVIOContext *avioContext = avio_alloc_context(avioContextBuffer, static_cast(mBufferSize), 0, mStruct, &customReadPacket, nullptr, nullptr); + Q_ASSERT(avioContext); - s->writeLock->unlock(); + inputFormatContext->pb = avioContext; - memcpy(buf, tempBuffer.constData(), buf_size); + error = avformat_open_input(&inputFormatContext, nullptr, nullptr, nullptr); + qDebug() << "HEADER RECEIVED" << Q_FUNC_INFO; - //mSenderId = something; - //qDebug() << "Reading packet"; - qDebug() << "Returning buf_size" << buf_size; - return buf_size; - } - else + if(error < 0) { - //buf_size = FFMIN(buf_size, s->socketHandler->mBuffer.size()); + char* errbuff = (char *)malloc((1000)*sizeof(char)); + av_strerror(error,errbuff,1000); + qDebug() << "AVformat open input UDP stream failed" << errbuff; + exit(1); + } - while (s->buffer->size() <= buf_size) + //Funker ikke med denne linja, men litt rart det funker uten? + //error = avformat_find_stream_info(fmt_ctx, nullptr); + /*if(error < 0) { - /*int ms = 50; - struct timespec ts = { ms / 1000, (ms % 1000) * 1000 * 1000 }; - qDebug() << "sleeping"; - nanosleep(&ts, NULL);*/ - //qDebug() << "sleeping"; - - } - - - s->writeLock->lock(); - - tempBuffer = QByteArray(s->buffer->data(), buf_size); - s->buffer->remove(0,buf_size); - //qDebug() << " buffer after removal: " << s->socketHandler->mBuffer.size(); - - s->writeLock->unlock(); - - memcpy(buf, tempBuffer.constData(), buf_size); + char* errbuff = (char *)malloc((1000)*sizeof(char)); + av_strerror(error,errbuff,1000); + qDebug() << "AVFormat find udp stream failed" << errbuff; + exit(1); + }*/ - //mSenderId = something; - return buf_size; + qDebug() << "Dumping videoplayback format"; + av_dump_format(inputFormatContext, 0, NULL, 0); + AVStream* video_stream = nullptr; + for (uint i=0; i < inputFormatContext->nb_streams; ++i) { + auto st = inputFormatContext->streams[i]; + //Debug() << st->id << st->index << st->start_time << st->duration << st->codecpar->codec_type; + if (st->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) + { + video_stream = st; + } } -} - -void VideoPlaybackHandler::start() -{ - - AVFormatContext *fmt_ctx = nullptr; - AVIOContext *avio_ctx = nullptr; - uint8_t /**buffer = nullptr,*/ *avio_ctx_buffer = nullptr; - size_t /*buffer_size = 0,*/ avio_ctx_buffer_size = mBufferSize; - - int ret = 0; - fmt_ctx = avformat_alloc_context(); - Q_ASSERT(fmt_ctx); + int err; + //Q_ASSERT(video_stream); + AVCodecContext *videoDecoderCodecContext; + if(video_stream) + { + // video_stream->codec->flags |= AV_CODEC_FLAG_GLOBAL_HEADER; + + AVCodecParameters *videoStreamCodecParameters = video_stream->codecpar; + AVCodec* videoDecoderCodec = avcodec_find_decoder(videoStreamCodecParameters->codec_id); + videoDecoderCodecContext = avcodec_alloc_context3(videoDecoderCodec); + videoDecoderCodecContext->pix_fmt = AV_PIX_FMT_YUV420P; + err = avcodec_open2(videoDecoderCodecContext, videoDecoderCodec, nullptr); + Q_ASSERT(err>=0); + qDebug() << "codec name: " << videoDecoderCodec->name<< " codec id " << videoDecoderCodec->id; + qDebug() << "codecpar width" << videoStreamCodecParameters->width <<" h: "<< videoStreamCodecParameters->height << " format: "<< videoStreamCodecParameters->format<< " pix fmt: " << videoDecoderCodecContext->pix_fmt; + + AVFrame *frameRGB = av_frame_alloc(); + frameRGB->format = AV_PIX_FMT_RGB24; + frameRGB->width = videoStreamCodecParameters->width; + frameRGB->height = videoStreamCodecParameters->height; + err = av_frame_get_buffer(frameRGB, 0); + Q_ASSERT(err == 0); + } - avio_ctx_buffer = reinterpret_cast(av_malloc(avio_ctx_buffer_size)); - Q_ASSERT(avio_ctx_buffer); - avio_ctx = avio_alloc_context(avio_ctx_buffer, static_cast(avio_ctx_buffer_size), 0, mStruct, &read_packet, nullptr, nullptr); - Q_ASSERT(avio_ctx); + AVFrame* frame = av_frame_alloc(); + AVPacket packet; + //AVFrame* resampled = 0; - fmt_ctx->pb = avio_ctx; - ret = avformat_open_input(&fmt_ctx, nullptr, nullptr, nullptr); - qDebug() << "HEADER RECEIVED"; - *mStruct->headerReceived = true; - if(ret < 0) + while (!mStopPlayback) { + //qDebug() << "About to call av read frame"; + //av_read_frame(fmt_ctx, NULL); + error = av_read_frame(inputFormatContext,&packet); + //qDebug() << "AVREADFRAME: " << ret; + if(error < 0) { char* errbuff = (char *)malloc((1000)*sizeof(char)); - av_strerror(ret,errbuff,1000); - qDebug() << "AVformat open input UDP stream failed" << errbuff; - exit(1); + av_strerror(error,errbuff,1000); + qDebug() << "Failed av_read_frame in videoplaybackhandler: code " << error << " meaning: " << errbuff; + //int ms = 1000; + //struct timespec ts = { ms / 1000, (ms % 1000) * 1000 * 1000 }; + //nanosleep(&ts, NULL); + continue; } + //qDebug() << "stream: " << packet.stream_index << " mvideostream: " << mVideoStreamIndex; - //Funker ikke med denne linja, men litt rart det funker uten? - //ret = avformat_find_stream_info(fmt_ctx, nullptr); - if(ret < 0) + //Decode and send to ImageHandler + //qDebug() << "packet dts VideoPlaybackHandler: " << packet.dts; + //qDebug() << "packet pts VideoPlaybackHandler: " << packet.pts; + error = avcodec_send_packet(videoDecoderCodecContext, &packet); + if (error == AVERROR_EOF || error == AVERROR(EOF)) + { + qDebug() << "send packet sleep"; + int ms = 1000; + struct timespec ts = { ms / 1000, (ms % 1000) * 1000 * 1000 }; + nanosleep(&ts, NULL); + continue; + } + else if(error < 0) { char* errbuff = (char *)malloc((1000)*sizeof(char)); - av_strerror(ret,errbuff,1000); - qDebug() << "AVFormat find udp stream failed" << errbuff; + av_strerror(error,errbuff,1000); + qDebug() << "Failed udp input avcodec_send_packet: code "<nb_streams; ++i) { - auto st = fmt_ctx->streams[i]; - //Debug() << st->id << st->index << st->start_time << st->duration << st->codecpar->codec_type; - if (st->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) - { - video_stream = st; - mVideoStreamIndex = i; - } } - - int err; - //Q_ASSERT(video_stream); - AVCodecContext *videoDecoderCodecContext; - if(video_stream) - { - // video_stream->codec->flags |= AV_CODEC_FLAG_GLOBAL_HEADER; - - AVCodecParameters *videoStreamCodecParameters = video_stream->codecpar; - AVCodec* videoDecoderCodec = avcodec_find_decoder(videoStreamCodecParameters->codec_id); - videoDecoderCodecContext = avcodec_alloc_context3(videoDecoderCodec); - videoDecoderCodecContext->pix_fmt = AV_PIX_FMT_YUV420P; - err = avcodec_open2(videoDecoderCodecContext, videoDecoderCodec, nullptr); - Q_ASSERT(err>=0); - qDebug() << "codec name: " << videoDecoderCodec->name<< " codec id " << videoDecoderCodec->id; - qDebug() << "codecpar width" << videoStreamCodecParameters->width <<" h: "<< videoStreamCodecParameters->height << " format: "<< videoStreamCodecParameters->format<< " pix fmt: " << videoDecoderCodecContext->pix_fmt; - - AVFrame *frameRGB = av_frame_alloc(); - frameRGB->format = AV_PIX_FMT_RGB24; - frameRGB->width = videoStreamCodecParameters->width; - frameRGB->height = videoStreamCodecParameters->height; - err = av_frame_get_buffer(frameRGB, 0); - Q_ASSERT(err == 0); + error = avcodec_receive_frame(videoDecoderCodecContext, frame); + if (error == AVERROR(EAGAIN) || error == AVERROR_EOF){ + //skipped_frames++; + qDebug() << "Skipped a Frame VideoPlaybackHandler"; + continue; } - - AVFrame* frame = av_frame_alloc(); - AVPacket packet; - //AVFrame* resampled = 0; - - //Mulig vi må legge til en stop variabel her, slik at ting ikke krasher når vi forlater en rom. - while (1) { - //qDebug() << "About to call av read frame"; - //av_read_frame(fmt_ctx, NULL); - ret = av_read_frame(fmt_ctx,&packet); - //qDebug() << "AVREADFRAME: " << ret; - if(ret < 0) - { - char* errbuff = (char *)malloc((1000)*sizeof(char)); - av_strerror(ret,errbuff,1000); - qDebug() << "Failed av_read_frame in videoplaybackhandler: code " << ret << " meaning: " << errbuff; - //int ms = 1000; - //struct timespec ts = { ms / 1000, (ms % 1000) * 1000 * 1000 }; - //nanosleep(&ts, NULL); - continue; - } - //qDebug() << "stream: " << packet.stream_index << " mvideostream: " << mVideoStreamIndex; - if(packet.stream_index == mVideoStreamIndex) - { - //Decode and send to ImageHandler - //qDebug() << "packet dts VideoPlaybackHandler: " << packet.dts; - //qDebug() << "packet pts VideoPlaybackHandler: " << packet.pts; - ret = avcodec_send_packet(videoDecoderCodecContext, &packet); - if (ret == AVERROR_EOF || ret == AVERROR(EOF)) - { - qDebug() << "send packet sleep"; - int ms = 1000; - struct timespec ts = { ms / 1000, (ms % 1000) * 1000 * 1000 }; - nanosleep(&ts, NULL); - continue; - } - else if(ret < 0) - { - char* errbuff = (char *)malloc((1000)*sizeof(char)); - av_strerror(ret,errbuff,1000); - qDebug() << "Failed udp input avcodec_send_packet: code "<pts; - //qDebug() << "VideoPacketTime: " << *mLastVideoPacketTime; - //sync(); - - //qDebug() << frame->data[0]; - //qDebug() << mIndex; - // qDebug() << "Sending image to imageHandler"; - mImageHandler->readImage(videoDecoderCodecContext, frame, mIndex); - } - else - { - qDebug() << "Vi er fucked"; - } - - av_frame_unref(frame); - av_packet_unref(&packet); + else if (error < 0) { + char* errbuff = (char *)malloc((1000)*sizeof(char)); + av_strerror(error,errbuff,1000); + qDebug() << "Failed avcodec_receive_frame: code "<readImage(videoDecoderCodecContext, frame, mIndex); -/*void VideoPlaybackHandler::sync() -{ - if(*mLastAudioPacketTime != -1) - { - int diff = *mLastAudioPacketTime - *mLastVideoPacketTime; - if(diff < 0) - { - //int ms = abs(diff)/1000; - int ms = 5; - // qDebug() << "Video Sleeping: " << ms; - struct timespec ts = { ms / 1000, (ms % 1000) * 1000 * 1000 }; - nanosleep(&ts, NULL); - } + av_frame_unref(frame); + av_packet_unref(&packet); } -}*/ + avformat_close_input(&inputFormatContext); + avcodec_free_context(&videoDecoderCodecContext); + +} diff --git a/handlers/videoplaybackhandler.h b/handlers/videoplaybackhandler.h index bda0f03..433e70e 100644 --- a/handlers/videoplaybackhandler.h +++ b/handlers/videoplaybackhandler.h @@ -6,34 +6,20 @@ #include #include "handlers/imagehandler.h" #include +#include -class VideoPlaybackHandler : public QObject +class VideoPlaybackHandler : public Playback { Q_OBJECT public: - VideoPlaybackHandler(std::mutex* writeLock, ImageHandler* imageHandler, QByteArray* headerBuffer, - QByteArray* buffer, int bufferSize, int index, QObject *parent = nullptr); + VideoPlaybackHandler(std::mutex* writeLock, QByteArray* buffer, + size_t bufferSize, ImageHandler* imageHandler, + int index, QObject *parent = nullptr); ~VideoPlaybackHandler(); - void getStream(); - static int read_packet(void *opaque, uint8_t *buf, int buf_size); void start(); - int mVideoStreamIndex = -1; private: int mIndex; - struct mBufferAndLockStruct { - QByteArray* buffer; - std::mutex* writeLock; - bool* headerReceived; - QByteArray* headerBuffer; - - }; - int mBufferSize; - mBufferAndLockStruct* mStruct; - QByteArray mBuffer; - QIODevice* mpOut; - uint8_t mSenderId = 0; //Value set to 0 just for testing. - ImageHandler* mImageHandler; -signals: + ImageHandler* imageHandler; }; #endif // PLAYBACKHANDLER_H diff --git a/playback.cpp b/playback.cpp new file mode 100644 index 0000000..904a636 --- /dev/null +++ b/playback.cpp @@ -0,0 +1,51 @@ +#include "playback.h" + +Playback::Playback(std::mutex* _writeLock, QByteArray* buffer, + size_t bufferSize, QObject *parent) : QObject(parent) +{ + mBufferSize = bufferSize; + mStruct = new mBufferAndLockStruct(); + mStruct->buffer = buffer; + mStruct->writeLock = _writeLock; +} +Playback::~Playback() +{ + delete mStruct; +} + +void Playback::stop() +{ + mStopPlayback = true; +} + +/** + * Custom readPacket function for av_read_frame and av_open_input. + * Will read and remove bytes from the buffer found in the struct + * and copy them to buf. + * @param buf_size int how many bytes to read from the buffer + * @param[out] buf uint8_t* bytes to send back to ffmpeg + * @param opaque void* pointer set by avio_alloc_context + * @return buf_size int + */ +int Playback::customReadPacket(void *opaque, uint8_t *buf, int buf_size) +{ + mBufferAndLockStruct *s = reinterpret_cast(opaque); + while (s->buffer->size() <= buf_size) + { + //int ms = 5; + //struct timespec ts = { ms / 1000, (ms % 1000) * 1000 * 1000 }; + //qDebug() << "sleeping"; + //nanosleep(&ts, NULL); + } + + s->writeLock->lock(); + QByteArray tempBuffer = QByteArray(s->buffer->data(), buf_size); + s->buffer->remove(0,buf_size); + s->writeLock->unlock(); + + + memcpy(buf, tempBuffer.constData(), buf_size); + + //Since return value is fixed it will never stop reading, should not be a problem for us? + return buf_size; +} diff --git a/playback.h b/playback.h new file mode 100644 index 0000000..ac8ae9f --- /dev/null +++ b/playback.h @@ -0,0 +1,25 @@ +#ifndef PLAYBACK_H +#define PLAYBACK_H + +#include +#include +class Playback : public QObject +{ + Q_OBJECT +public: + explicit Playback(std::mutex* _writeLock, QByteArray* buffer, size_t bufferSize, QObject *parent = nullptr); + virtual void start(){}; + void stop(); + ~Playback(); +protected: + static int customReadPacket(void *opaque, uint8_t *buf, int buf_size); + struct mBufferAndLockStruct { + QByteArray* buffer; + std::mutex* writeLock; + }; + size_t mBufferSize; + mBufferAndLockStruct* mStruct; + bool mStopPlayback = false; +}; + +#endif // PLAYBACK_H diff --git a/qZoom-Client.pro b/qZoom-Client.pro index b55b147..e423d9c 100644 --- a/qZoom-Client.pro +++ b/qZoom-Client.pro @@ -27,6 +27,7 @@ SOURCES += \ handlers/imagehandler.cpp \ handlers/inputstreamhandler.cpp \ main.cpp \ + playback.cpp \ settings.cpp \ handlers/udpsockethandler.cpp \ handlers/tcpserverhandler.cpp \ @@ -63,6 +64,7 @@ HEADERS += \ handlers/userhandler.h \ handlers/imagehandler.h \ handlers/inputstreamhandler.h \ + playback.h \ settings.h \ handlers/udpsockethandler.h \ handlers/tcpserverhandler.h \ diff --git a/view/session.qml b/view/session.qml index 0e515a6..53553db 100644 --- a/view/session.qml +++ b/view/session.qml @@ -66,6 +66,7 @@ Rectangle { color: "#161637" Image { + id: liveImage anchors.fill: parent property bool counter: false @@ -94,7 +95,7 @@ Rectangle { Item { Timer { interval: 41; running: true; repeat: true - onTriggered: liveImage.reload(); + onTriggered:liveImage.reload(); } } }