Fix rate control for AMD cards using VAAPI

src/platform/common.h

@@ -27,6 +27,8 @@ struct sockaddr;
 struct AVFrame;
 struct AVBufferRef;
 struct AVHWFramesContext;
+struct AVCodecContext;
+struct AVDictionary;
 
 // Forward declarations of boost classes to avoid having to include boost headers
 // here, which results in issues with Windows.h and WinSock2.h include order.
@@ -398,6 +400,13 @@ namespace platf {
     virtual void
     init_hwframes(AVHWFramesContext *frames) {};
 
+    /**
+     * @brief Provides a hook for allow platform-specific code to adjust codec options.
+     * @note Implementations may set or modify codec options prior to codec initialization.
+     */
+    virtual void
+    init_codec_options(AVCodecContext *ctx, AVDictionary *options) {};
+
     /**
      * @brief Prepare to derive a context.
      * @note Implementations may make modifications required before context derivation

src/platform/linux/vaapi.cpp

@@ -129,6 +129,16 @@ namespace va {
       return 0;
     }
 
+    void
+    init_codec_options(AVCodecContext *ctx, AVDictionary *options) override {
+      // Don't set the RC buffer size when using H.264 on Intel GPUs. It causes
+      // major encoding quality degradation.
+      auto vendor = vaQueryVendorString(va_display);
+      if (ctx->codec_id != AV_CODEC_ID_H264 || (vendor && !strstr(vendor, "Intel"))) {
+        ctx->rc_buffer_size = ctx->bit_rate * ctx->framerate.den / ctx->framerate.num;
+      }
+    }
+
     int
     set_frame(AVFrame *frame, AVBufferRef *hw_frames_ctx_buf) override {
       this->hwframe.reset(frame);

src/video.cpp

@@ -859,6 +859,7 @@ namespace video {
       std::make_optional<encoder_t::option_t>("qp"s, &config::video.qp),
       "h264_vaapi"s,
     },
+    // RC buffer size will be set in platform code if supported
     LIMITED_GOP_SIZE | PARALLEL_ENCODING | SINGLE_SLICE_ONLY | NO_RC_BUF_LIMIT
   };
 #endif
@@ -1610,6 +1611,9 @@ namespace video {
         return nullptr;
       }
 
+      // Allow the encoding device a final opportunity to set/unset or override any options
+      encode_device->init_codec_options(ctx.get(), options);
+
       if (auto status = avcodec_open2(ctx.get(), codec, &options)) {
         char err_str[AV_ERROR_MAX_STRING_SIZE] { 0 };