#800 b-frames support:

* add "encoding.video_b_frames" list of encodings capability, only enabled for "h264" for now
* pass "b-frames" flag to video encoders
* use the b-frames flag in x264 encoder: compress_image now returns flags to indicate that some frames are delayed
* when we see the delayed frames flag, schedule a timer to ensure we flush the encoder, encoders that support delayed frames must implement flush()
* workaround a bug in x264 which causes invalid frame data if we flush the encoder after the first (IDR) frame: do a full frame refresh instead
* client now ignores empty image data if the delayed flag is set

git-svn-id: https://xpra.org/svn/Xpra/trunk@12858 3bb7dfac-3a0b-4e04-842a-767bc560f471

src/xpra/client/ui_client_base.py

@@ -1457,6 +1457,7 @@ def make_hello(self):
             #TODO: check for csc support (swscale only?)
             "video_reinit"              : True,
             "video_scaling"             : True,
+            "video_b_frames"            : ["h264"],     #only tested with dec_avcodec2
             "webp_leaks"                : False,
             "transparency"              : self.has_transparency(),
             "rgb24zlib"                 : True,

src/xpra/client/window_backing_base.py

@@ -365,14 +365,14 @@ def paint_with_video_decoder(self, decoder_module, coding, img_data, x, y, width
                 message = "window %s is already gone!" % self.wid
                 log(message)
                 fire_paint_callbacks(callbacks, -1, message)
-                return  False
+                return
             enc_width, enc_height = options.intpair("scaled_size", (width, height))
             input_colorspace = options.strget("csc")
             if not input_colorspace:
                 message = "csc mode is missing from the video options!"
                 log.error(message)
                 fire_paint_callbacks(callbacks, False, message)
-                return  False
+                return
             #do we need a prep step for decoders that cannot handle the input_colorspace directly?
             decoder_colorspaces = decoder_module.get_input_colorspaces(coding)
             assert input_colorspace in decoder_colorspaces, "decoder does not support %s for %s" % (input_colorspace, coding)
@@ -404,15 +404,19 @@ def paint_with_video_decoder(self, decoder_module, coding, img_data, x, y, width
 
             img = vd.decompress_image(img_data, options)
             if not img:
-                fire_paint_callbacks(callbacks, False, "video decoder %s failed to decode %i bytes of %s data" % (vd.get_type(), len(img_data), coding))
-                log.error("Error: decode failed on %s bytes of %s data", len(img_data), coding)
-                log.error(" %sx%s pixels using %s", width, height, vd.get_type())
-                log.error(" decoding options=%s", options)
-                return False
+                if options.get("delayed", 0)>0:
+                    #there are further frames queued up,
+                    #and this frame references those, so assume all is well:
+                    fire_paint_callbacks(callbacks, True)
+                else:
+                    fire_paint_callbacks(callbacks, False, "video decoder %s failed to decode %i bytes of %s data" % (vd.get_type(), len(img_data), coding))
+                    log.error("Error: decode failed on %s bytes of %s data", len(img_data), coding)
+                    log.error(" %sx%s pixels using %s", width, height, vd.get_type())
+                    log.error(" decoding options=%s", options)
+                return
             self.do_video_paint(img, x, y, enc_width, enc_height, width, height, options, callbacks)
         if self._backing is None:
             self.close_decoder(True)
-        return  False
 
     def do_video_paint(self, img, x, y, enc_width, enc_height, width, height, options, callbacks):
         #try 24 bit first (paint_rgb24), then 32 bit (paint_rgb32):

src/xpra/codecs/codec_checks.py

@@ -187,10 +187,17 @@ def do_testencoding(encoder_module, encoding, W, H, full=False, limit_w=TEST_LIM
         for cs_out in encoder_module.get_output_colorspaces(encoding, cs_in):
             e = encoder_module.Encoder()
             try:
-                e.init_context(W, H, cs_in, [cs_out], encoding, 0, 100, (1,1), {})
+                options = {"b-frames" : True}
+                e.init_context(W, H, cs_in, [cs_out], encoding, 0, 100, (1,1), options)
                 image = make_test_image(cs_in, W, H)
                 data, meta = e.compress_image(image)
+                if data is None:
+                    delayed = meta.get("delayed", 0)
+                    assert delayed>0, "data is empty and there are no delayed frames!"
+                    #now we should get one:
+                    data, meta = e.flush(delayed)
                 del image
+                assert data is not None, "None data for %s using %s encoding with %s / %s" % (encoder_module.get_type(), encoding, cs_in, cs_out)
                 assert len(data)>0, "no compressed data for %s using %s encoding with %s / %s" % (encoder_module.get_type(), encoding, cs_in, cs_out)
                 assert meta is not None, "missing metadata for %s using %s encoding with %s / %s" % (encoder_module.get_type(), encoding, cs_in, cs_out)
                 #print("test_encoder: %s.compress_image(%s)=%s" % (encoder_module.get_type(), image, (data, meta)))

src/xpra/codecs/dec_avcodec2/decoder.pyx

@@ -237,7 +237,7 @@ cdef class AVFrameWrapper:
         #we must have freed it!
         assert self.frame==NULL and self.avctx==NULL, "frame was freed by both, but not actually freed!"
 
-    def __str__(self):
+    def __repr__(self):
         if self.frame==NULL:
             return "AVFrameWrapper(NULL)"
         return "AVFrameWrapper(%#x)" % <unsigned long> self.frame
@@ -260,8 +260,8 @@ class AVImageWrapper(ImageWrapper):
         when the image is freed, or once we have made a copy of the pixels.
     """
 
-    def __repr__(self):                          #@DuplicatedSignature
-        return ImageWrapper.__repr__(self)+"-(%s)" % self.av_frame
+    def _cn(self):                          #@DuplicatedSignature
+        return "AVImageWrapper-%s" % self.av_frame
 
     def free(self):                             #@DuplicatedSignature
         log("AVImageWrapper.free()")
@@ -476,6 +476,16 @@ cdef class Decoder:
     def get_type(self):                             #@DuplicatedSignature
         return "avcodec"
 
+    def log_error(self, int buf_len, err, options={}):
+        log.error("avcodec error decoding %i bytes of %s data:", buf_len, self.encoding)
+        log.error(" %s", err)
+        log.error(" frame %i", self.frames)
+        if options:
+            log.error(" options=%s", options)
+        log.error(" decoder state:")
+        for k,v in self.get_info().items():
+            log.error("  %s = %s", k, v)
+
     def decompress_image(self, input, options):
         cdef unsigned char * padded_buf = NULL
         cdef const unsigned char * buf = NULL
@@ -516,23 +526,25 @@ cdef class Decoder:
         if len<0:
             av_frame_unref(self.av_frame)
             log("%s.decompress_image(%s:%s, %s) avcodec_decode_video2 failure: %s", self, type(input), buf_len, options, self.av_error_str(len))
-            log.error("avcodec_decode_video2 %s decoding failure:", self.encoding)
-            log.error(" %s", self.av_error_str(len))
+            self.log_error(buf_len, self.av_error_str(len), options)
             return None
         if len==0:
             av_frame_unref(self.av_frame)
             log("%s.decompress_image(%s:%s, %s) avcodec_decode_video2 failed to decode the stream", self, type(input), buf_len, options)
-            log.error("avcodec_decode_video2 %s decoding failure - no stream", self.encoding)
+            self.log_error(buf_len, "no stream", options)
             return None
-
+        if got_picture==0:
+            d = options.get("delayed", 0)
+            if d>0:
+                log("avcodec_decode_video2 %i delayed pictures", d)
+                return None
+            self.log_error(buf_len, "no picture", options)
+            return None
+
+        log("avcodec_decode_video2 returned %i, got_picture=%i", len, got_picture)
         if self.actual_pix_fmt!=self.av_frame.format:
             if self.av_frame.format==-1:
-                log.error("avcodec error decoding %i bytes of %s data", buf_len, self.encoding)
-                log.error(" frame %i", self.frames)
-                log.error(" options=%s", options)
-                log.error(" decoder state:")
-                for k,v in self.get_info().items():
-                    log.error("  %s = %s", k, v)
+                self.log_error(buf_len, "unknown format returned")
                 return None
             self.actual_pix_fmt = self.av_frame.format
             if self.actual_pix_fmt not in ENUM_TO_FORMAT:
@@ -584,7 +596,8 @@ cdef class Decoder:
         #add to weakref list after cleaning it up:
         self.weakref_images = [x for x in self.weakref_images if x() is not None]
         self.weakref_images.append(weakref.ref(img))
-        log("%s.decompress_image(%s:%s, %s)=%s", self, type(input), buf_len, options, img)
+        log("%s:", self)
+        log("decompress_image(%s:%s, %s)=%s", type(input), buf_len, options, img)
         return img
 
 

src/xpra/codecs/enc_x264/encoder.pyx

@@ -87,6 +87,7 @@ cdef extern from "x264.h":
 
     #frame type
     int X264_TYPE_AUTO              # Let x264 choose the right type
+    int X264_TYPE_KEYFRAME
     int X264_TYPE_IDR
     int X264_TYPE_I
     int X264_TYPE_P
@@ -210,7 +211,7 @@ cdef extern from "x264.h":
         x264_sei_t extra_sei#In: arbitrary user SEI (e.g subtitles, AFDs)
         void *opaque        #private user data. copied from input to output frames.
 
-    void x264_picture_init(x264_picture_t *pic)
+    void x264_picture_init(x264_picture_t *pic) nogil
 
     int x264_param_default_preset(x264_param_t *param, const char *preset, const char *tune)
     int x264_param_apply_profile(x264_param_t *param, const char *profile)
@@ -221,6 +222,8 @@ cdef extern from "x264.h":
     void x264_encoder_close(x264_t *context)
 
     int x264_encoder_encode(x264_t *context, x264_nal_t **pp_nal, int *pi_nal, x264_picture_t *pic_in, x264_picture_t *pic_out ) nogil
+    int x264_encoder_delayed_frames(x264_t *)
+    int x264_encoder_maximum_delayed_frames(x264_t *h)
 
 
 cdef set_f_rf(x264_param_t *param, float q):
@@ -255,6 +258,7 @@ SLICE_TYPES = {
     X264_TYPE_P     : "P",
     X264_TYPE_BREF  : "BREF",
     X264_TYPE_B     : "B",
+    X264_TYPE_KEYFRAME  : "KEYFRAME",
     }
 
 NAL_TYPES = {
@@ -409,6 +413,7 @@ cdef class Encoder:
     cdef int preset
     cdef int quality
     cdef int speed
+    cdef int b_frames
     cdef unsigned long long bytes_in
     cdef unsigned long long bytes_out
     cdef object last_frame_times
@@ -417,7 +422,7 @@ cdef class Encoder:
 
     cdef object __weakref__
 
-    def init_context(self, int width, int height, src_format, dst_formats, encoding, int quality, int speed, scaling, options):    #@DuplicatedSignature
+    def init_context(self, int width, int height, src_format, dst_formats, encoding, int quality, int speed, scaling, options={}):
         global COLORSPACE_FORMATS, generation
         cs_info = COLORSPACE_FORMATS.get(src_format)
         assert cs_info is not None, "invalid source format: %s, must be one of: %s" % (src_format, COLORSPACE_FORMATS.keys())
@@ -431,6 +436,7 @@ cdef class Encoder:
         self.preset = get_preset_for_speed(speed)
         self.src_format = src_format
         self.colorspace = cs_info[0]
+        self.b_frames = options.get("b-frames", False)
         self.frames = 0
         self.last_frame_times = deque(maxlen=200)
         self.time = 0
@@ -469,12 +475,15 @@ cdef class Encoder:
         param.b_intra_refresh = 0   #no intra refresh
         param.b_open_gop = 1        #allow open gop
         param.b_opencl = self.opencl
+        param.i_bframe = self.b_frames
+
         #param.p_log_private =
         x264_param_apply_profile(&param, self.profile)
         param.pf_log = <void *> X264_log
         param.i_log_level = LOG_LEVEL
         self.context = x264_encoder_open(&param)
-        log("x264 context=%#x, %7s %4ix%-4i opencl=%s", <unsigned long> self.context, self.src_format, self.width, self.height, bool(self.opencl))
+        cdef int maxd = x264_encoder_maximum_delayed_frames(self.context)
+        log("x264 context=%#x, %7s %4ix%-4i opencl=%s, b-frames=%i, max delayed frames=%i", <unsigned long> self.context, self.src_format, self.width, self.height, bool(self.opencl), self.b_frames, maxd)
         assert self.context!=NULL,  "context initialization failed for format %s" % self.src_format
 
     def clean(self):                        #@DuplicatedSignature
@@ -576,36 +585,20 @@ cdef class Encoder:
 
 
     def compress_image(self, image, int quality=-1, int speed=-1, options={}):
-        cdef x264_nal_t *nals = NULL
-        cdef int i_nals = 0
-        cdef x264_picture_t pic_out
         cdef x264_picture_t pic_in
-        cdef int frame_size = 0
-
         cdef uint8_t *pic_buf
         cdef Py_ssize_t pic_buf_len = 0
         cdef char *out
 
         cdef int i                        #@DuplicatedSignature
-        start = time.time()
 
         if self.frames==0:
             self.first_frame_timestamp = image.get_timestamp()
 
-        if speed>=0:
-            self.set_encoding_speed(speed)
-        else:
-            speed = self.speed
-        if quality>=0:
-            self.set_encoding_quality(quality)
-        else:
-            quality = self.quality
-        assert self.context!=NULL
         pixels = image.get_pixels()
         istrides = image.get_rowstride()
         assert pixels, "failed to get pixels from %s" % image
 
-        x264_picture_init(&pic_out)
         x264_picture_init(&pic_in)
 
         if self.src_format.find("RGB")>=0 or self.src_format.find("BGR")>=0:
@@ -627,17 +620,45 @@ cdef class Encoder:
         pic_in.img.i_csp = self.colorspace
         pic_in.img.i_plane = 3
         pic_in.i_pts = image.get_timestamp()-self.first_frame_timestamp
+        return self.do_compress_image(&pic_in, quality, speed)
+
+    cdef do_compress_image(self, x264_picture_t *pic_in, int quality=-1, int speed=-1):
+        cdef x264_nal_t *nals = NULL
+        cdef int i_nals = 0
+        cdef x264_picture_t pic_out
+        cdef int frame_size = 0
+        assert self.context!=NULL
+        start = time.time()
+
+        if speed>=0:
+            self.set_encoding_speed(speed)
+        else:
+            speed = self.speed
+        if quality>=0:
+            self.set_encoding_quality(quality)
+        else:
+            quality = self.quality
 
         with nogil:
-            frame_size = x264_encoder_encode(self.context, &nals, &i_nals, &pic_in, &pic_out)
+            x264_picture_init(&pic_out)
+            frame_size = x264_encoder_encode(self.context, &nals, &i_nals, pic_in, &pic_out)
         if frame_size < 0:
             log.error("x264 encoding error: frame_size is invalid!")
             return None
+        cdef int delayed = x264_encoder_delayed_frames(self.context)
+        if i_nals==0:
+            if self.b_frames and delayed>0:
+                log("x264 encode %i delayed frames after %i", delayed, self.frames)
+                return None, {
+                              "delayed" : delayed,
+                              "frame"   : self.frames,
+                              }
+            raise Exception("x264_encoder_encode produced no data!")
         slice_type = SLICE_TYPES.get(pic_out.i_type, pic_out.i_type)
-        log("x264 encode frame %i as %4s slice with %i nals, total %7i bytes", self.frames, slice_type, i_nals, frame_size)
+        log("x264 encode frame %i as %4s slice with %i nals, total %7i bytes, keyframe=%s, delayed=%i", self.frames, slice_type, i_nals, frame_size, pic_out.b_keyframe, delayed)
         if LOG_NALS:
             for i in range(i_nals):
-                log.info(" nal %s priority:%10s, type:%10s, payload=%#x, payload size=%#x",
+                log.info(" nal %s priority:%10s, type:%10s, payload=%#x, payload size=%i",
                          i, NAL_PRIORITIES.get(nals[i].i_ref_idc, nals[i].i_ref_idc), NAL_TYPES.get(nals[i].i_type, nals[i].i_type), <unsigned long> nals[i].p_payload, nals[i].i_payload)
             #log.info("x264 nal %s: %s", i, (<char *>nals[i].p_payload)[:64])
         out = <char *>nals[0].p_payload
@@ -650,6 +671,8 @@ cdef class Encoder:
                 "quality"   : max(0, min(100, quality)),
                 "speed"     : max(0, min(100, speed)),
                 "type"      : slice_type}
+        if delayed>0:
+            client_options["delayed"] = delayed
         #accounting:
         end = time.time()
         self.time += end-start
@@ -659,7 +682,21 @@ cdef class Encoder:
         if self.file and frame_size>0:
             self.file.write(cdata)
             self.file.flush()
-        return  cdata, client_options
+        return cdata, client_options
+
+    def flush(self, int frame_no):
+        if self.frames>frame_no or self.context==NULL:
+            return None, {}
+        cdef int i = x264_encoder_delayed_frames(self.context)
+        log("x264 flush(%i) %i delayed frames", frame_no, i)
+        if i<=0:
+            return None, {}
+        cdef x264_nal_t *nals = NULL
+        cdef int i_nals = 0
+        cdef x264_picture_t pic_out
+        cdef int frame_size = 0
+        x264_picture_init(&pic_out)
+        return self.do_compress_image(NULL)
 
 
     def set_encoding_speed(self, int pct):