-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathdecoder.py
More file actions
5267 lines (4631 loc) · 206 KB
/
Copy pathdecoder.py
File metadata and controls
5267 lines (4631 loc) · 206 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
# h264/decoder/decoder.py
"""H.264 Baseline profile decoder.
Main decoder orchestration that ties together:
- NAL unit parsing (bitstream/)
- SPS/PPS parsing (parameters/)
- Slice header parsing (slice/)
- Macroblock reconstruction (reconstruct/)
- Inter prediction (inter/)
- Color conversion (color/)
H.264 Spec Reference:
- Section 7: Syntax and semantics
- Section 8: Decoding process
This decoder supports:
- Baseline profile (no CABAC, no B-frames)
- I-slices and P-slices
- 4:2:0 chroma format
"""
import logging
import os
from dataclasses import dataclass, field
from typing import Optional, Tuple
import numpy as np
from bitstream import (
NALUnit,
NALUnitType,
BitReader,
extract_nal_units,
iter_nal_units,
)
from parameters import SPS, PPS, parse_sps, parse_pps
from parameters.sps import _is_high_profile
from slice import SliceHeader, SliceType, parse_slice_header
from reconstruct import decode_macroblock, MacroblockData
from reconstruct.macroblock import (
decode_cbp_inter,
decode_chroma_dc,
decode_chroma_ac,
build_chroma_residual,
get_luma_neighbor_nz,
BLOCK_SCAN_ORDER,
)
from entropy import decode_residual_block, calculate_nC, ZIGZAG_4x4
from dequant import dequant_4x4, get_chroma_qp
from dequant.dequant import dequant_8x8
from transform import idct_4x4
from transform.idct_8x8 import idct_8x8
from color import ycbcr_to_rgb, ColorMatrix
from color.chroma_format import monochrome_to_rgb, ycbcr_422_to_rgb, ycbcr_444_to_rgb
from inter.reference import ReferenceFrame, ReferenceFrameBuffer
from decoder.poc import POCCalculator
from decoder.mmco import MMCOProcessor
from inter.mv_prediction import (
MVCache,
predict_mv_16x16,
predict_mv_16x8,
predict_mv_8x16,
predict_mv_8x8,
predict_mv_partition,
)
from inter.p_macroblock import parse_p_mb_type, parse_sub_mb_type, PMacroblockInfo
from inter.p_reconstruct import (
reconstruct_p_skip,
reconstruct_p_skip_weighted,
reconstruct_p_16x16,
reconstruct_p_16x16_weighted,
reconstruct_p_16x8,
reconstruct_p_16x8_weighted,
reconstruct_p_8x16,
reconstruct_p_8x16_weighted,
reconstruct_p_8x8,
reconstruct_p_8x8_weighted,
reconstruct_p_8x8_sub,
apply_chroma_prediction,
apply_inter_prediction,
)
from inter.b_macroblock import (
parse_b_mb_type,
get_b_skip_info,
get_partition_pred_modes,
parse_b_sub_mb_type,
)
from inter.b_reconstruct import (
reconstruct_b_skip,
reconstruct_b_l0_16x16,
reconstruct_b_l1_16x16,
reconstruct_b_bi_16x16,
reconstruct_b_direct_16x16,
reconstruct_b_16x8,
reconstruct_b_8x16,
reconstruct_b_8x8,
)
from entropy.cabac_arith import CABACDecoder
from entropy.cabac_context import init_context_models
from entropy.cabac_macroblock import (
decode_macroblock_layer_cabac,
decode_end_of_slice_flag_cabac,
)
from entropy.cabac_residual import decode_residual_block_cabac
logger = logging.getLogger(__name__)
@dataclass
class DecoderStats:
concealed_mb_count: int = 0
@dataclass
class DecodedFrame:
"""Represents a decoded video frame.
Attributes:
frame_num: H.264 frame number
poc: Picture order count
luma: Y plane (H x W), uint8
cb: Cb plane (H/2 x W/2), uint8
cr: Cr plane (H/2 x W/2), uint8
width: Frame width in pixels
height: Frame height in pixels
chroma_format_idc: Chroma format ID (0 = monochrome, 1 = 4:2:0, 2 = 4:2:2, 3 = 4:4:4)
"""
frame_num: int
poc: int
luma: np.ndarray
cb: Optional[np.ndarray]
cr: Optional[np.ndarray]
width: int
height: int
chroma_format_idc: int = 1
@property
def shape(self) -> tuple:
"""Return (height, width) of luma plane."""
return (self.height, self.width)
def to_rgb(self, color_matrix: ColorMatrix = ColorMatrix.BT601) -> np.ndarray:
"""Convert YUV frame to RGB.
Args:
color_matrix: Color standard (BT601 or BT709)
Returns:
RGB array (H x W x 3), uint8
"""
if self.chroma_format_idc == 0 or self.cb is None or self.cr is None:
return monochrome_to_rgb(self.luma)
if self.chroma_format_idc == 1:
return ycbcr_to_rgb(self.luma, self.cb, self.cr, color_matrix=color_matrix)
if self.chroma_format_idc == 2:
return ycbcr_422_to_rgb(self.luma, self.cb, self.cr, color_matrix=color_matrix)
if self.chroma_format_idc == 3:
return ycbcr_444_to_rgb(self.luma, self.cb, self.cr, color_matrix=color_matrix)
raise ValueError(f"Invalid chroma_format_idc: {self.chroma_format_idc}")
@dataclass
class DecoderState:
"""Internal decoder state.
Tracks SPS/PPS parameter sets and current decoding context.
"""
sps_dict: dict = field(default_factory=dict) # id -> SPS
pps_dict: dict = field(default_factory=dict) # id -> PPS
current_sps: Optional[SPS] = None
current_pps: Optional[PPS] = None
chroma_format_idc: int = 1
# Frame buffers for reconstruction
frame_luma: Optional[np.ndarray] = None
frame_cb: Optional[np.ndarray] = None
frame_cr: Optional[np.ndarray] = None
# Non-zero coefficient counts for CAVLC context
nz_counts: Optional[np.ndarray] = None
# Intra 4x4 prediction modes per MB for cross-MB MPM computation
# Shape: (num_mbs, 16) - 16 modes per MB, -1 for non-I4x4 MBs
intra_modes: Optional[np.ndarray] = None
# Reference frame buffer for inter prediction
ref_buffer: Optional[ReferenceFrameBuffer] = None
# MV cache for current frame (L0 and L1 for B-frames)
mv_cache: Optional[MVCache] = None
mv_cache_l1: Optional[MVCache] = None
# Current macroblock QP (updated by mb_qp_delta)
current_mb_qp: int = 26
# Per-MB info for deblocking filter
mb_types: Optional[np.ndarray] = None # MB type per MB
mb_coeffs: Optional[np.ndarray] = None # Has non-zero coeffs per 4x4 block
mb_cbps: Optional[np.ndarray] = None # CBP per MB for CABAC context
mb_qps: Optional[np.ndarray] = None # Per-MB QP values
mb_chroma_modes: Optional[np.ndarray] = None # Chroma pred mode per MB for CABAC
mb_qp_deltas: Optional[np.ndarray] = None # QP delta per MB for CABAC context
# Per-MB MVD values per 4x4 block for CABAC MVD context (H.264 9.3.3.1.1.7)
mb_mvds_l0: Optional[np.ndarray] = None # Shape: (mb_count, 4, 4, 2)
mb_mvds_l1: Optional[np.ndarray] = None # Shape: (mb_count, 4, 4, 2)
# Slice tracking for multiple slice support
mb_slice_ids: Optional[np.ndarray] = None # Which slice each MB belongs to
current_slice_id: int = 0 # Current slice being decoded
# B-frame reference lists
l0_list: list = field(default_factory=list) # L0 reference list (past frames)
l1_list: list = field(default_factory=list) # L1 reference list (future frames)
# POC tracking
prev_poc_msb: int = 0
prev_poc_lsb: int = 0
# CABAC context models (None = not initialized)
cabac_contexts: Optional[list] = None
def get_sps(self, sps_id: int) -> SPS:
"""Get SPS by ID."""
if sps_id not in self.sps_dict:
raise ValueError(f"SPS {sps_id} not found")
return self.sps_dict[sps_id]
def get_pps(self, pps_id: int) -> PPS:
"""Get PPS by ID."""
if pps_id not in self.pps_dict:
raise ValueError(f"PPS {pps_id} not found")
return self.pps_dict[pps_id]
def apply_sps(self, sps: SPS) -> None:
self.sps_dict[sps.seq_parameter_set_id] = sps
self.current_sps = sps
self.chroma_format_idc = getattr(sps, "chroma_format_idc", 1)
def allocate_frame_buffers(self, sps: SPS) -> None:
"""Allocate frame buffers based on SPS dimensions."""
height = sps.frame_height_in_mbs * 16
width = sps.pic_width_in_mbs * 16
self.frame_luma = np.zeros((height, width), dtype=np.uint8)
chroma_format_idc = getattr(sps, "chroma_format_idc", 1)
self.chroma_format_idc = chroma_format_idc
if chroma_format_idc == 0:
cb_shape = (0, 0)
elif chroma_format_idc == 1:
cb_shape = (height // 2, width // 2)
elif chroma_format_idc == 2:
cb_shape = (height, width // 2)
elif chroma_format_idc == 3:
cb_shape = (height, width)
else:
raise ValueError(f"Invalid chroma_format_idc: {chroma_format_idc}")
self.frame_cb = np.zeros(cb_shape, dtype=np.uint8)
self.frame_cr = np.zeros(cb_shape, dtype=np.uint8)
# Stored per MB for context calculation
mb_count = sps.frame_height_in_mbs * sps.pic_width_in_mbs
if chroma_format_idc == 0:
blocks_per_mb = 16
elif chroma_format_idc in (1, 2):
blocks_per_mb = 24
else: # chroma_format_idc == 3
blocks_per_mb = 48
self.nz_counts = np.zeros((mb_count, blocks_per_mb), dtype=np.int32)
# Intra 4x4 prediction modes: initialized to DC (2) like JM reference.
# I_16x16 and non-intra MBs keep this default value.
# Only genuinely unavailable neighbors (outside picture) use -1.
self.intra_modes = np.full((mb_count, 16), 2, dtype=np.int32)
# Initialize reference frame buffer
max_refs = sps.max_num_ref_frames if hasattr(sps, 'max_num_ref_frames') else 4
self.ref_buffer = ReferenceFrameBuffer(max_frames=max(1, max_refs))
# Per-MB info for deblocking
self.mb_types = np.zeros(mb_count, dtype=np.int32)
self.mb_coeffs = np.zeros((mb_count, blocks_per_mb), dtype=bool)
self.mb_qps = np.zeros(mb_count, dtype=np.int32)
# CBP per MB for CABAC neighbor context
self.mb_cbps = np.zeros(mb_count, dtype=np.int32)
# Chroma pred mode per MB for CABAC neighbor context
self.mb_chroma_modes = np.zeros(mb_count, dtype=np.int32)
# QP delta per MB for CABAC context
self.mb_qp_deltas = np.zeros(mb_count, dtype=np.int32)
# DC coded_block_flag per MB for CABAC context
# [0]=luma DC, [1]=Cb DC, [2]=Cr DC
self.mb_dc_cbf = np.zeros((mb_count, 3), dtype=np.int32)
# MB skip flags for CABAC skip context derivation
self.mb_skip_flags = np.zeros(mb_count, dtype=np.int32)
# Raw CABAC mb_type per MB (preserved for CABAC context derivation).
# Unlike mb_types (overwritten to 99 for deblocking), this keeps the
# original decoded value so neighbor lookups get correct condTermFlags.
self.cabac_mb_types = np.zeros(mb_count, dtype=np.int32)
# Per-MB MVD values per 4x4 block for CABAC MVD context derivation
# Shape: (mb_count, 4, 4, 2) → [mb_idx, blk_row, blk_col, comp]
self.mb_mvds_l0 = np.zeros((mb_count, 4, 4, 2), dtype=np.int32)
self.mb_mvds_l1 = np.zeros((mb_count, 4, 4, 2), dtype=np.int32)
# Per-MB ref_idx per 4x4 block for CABAC ref_idx context derivation
# Shape: (mb_count, 4, 4) → ref_idx for partition covering each block
# 0 = ref_idx 0 or intra (both give condTermFlag=0)
self.mb_ref_idx_l0 = np.zeros((mb_count, 4, 4), dtype=np.int8)
self.mb_ref_idx_l1 = np.zeros((mb_count, 4, 4), dtype=np.int8)
# Per-MB transform_size_8x8_flag for CABAC context derivation
self.mb_transform_8x8_flags = np.zeros(mb_count, dtype=np.int32)
# Slice tracking
self.mb_slice_ids = np.zeros(mb_count, dtype=np.int32)
self.current_slice_id = 0
logger.debug(
f"Allocated frame buffers: {width}x{height} "
f"({sps.pic_width_in_mbs}x{sps.frame_height_in_mbs} MBs)"
)
def init_mv_cache(self, sps: SPS) -> None:
"""Initialize MV cache for a new frame."""
self.mv_cache = MVCache(
width_in_mbs=sps.pic_width_in_mbs,
height_in_mbs=sps.frame_height_in_mbs
)
self.mv_cache_l1 = MVCache(
width_in_mbs=sps.pic_width_in_mbs,
height_in_mbs=sps.frame_height_in_mbs
)
class H264Decoder:
"""H.264 Baseline profile decoder.
Decodes I-slices from Annex B bitstreams.
Example:
decoder = H264Decoder()
for frame in decoder.decode_file("video.264"):
rgb = frame.to_rgb()
# Display or save rgb array
"""
def __init__(self, deblocking_enabled: bool = True, aso_enabled: bool = False):
"""Initialize decoder.
Args:
deblocking_enabled: Whether to apply deblocking filter (default True)
"""
self.state = DecoderState()
self.deblocking_enabled = deblocking_enabled
self.aso_enabled = aso_enabled
self.chroma_format_idc = 1
self.poc_calculator = POCCalculator()
self.error_resilience = False
self.on_concealment = None
self.stats = DecoderStats()
self.mmco_processor = MMCOProcessor()
self._mmco = self.mmco_processor
self._trace_mb = None
self._trace_pixel = None
self._trace_pre_mb = None
trace_mb = os.environ.get("H264_TRACE_MB")
if trace_mb:
try:
parts = [p.strip() for p in trace_mb.split(",")]
if len(parts) == 3:
self._trace_mb = (int(parts[0]), int(parts[1]), int(parts[2]))
except Exception:
self._trace_mb = None
trace_pixel = os.environ.get("H264_TRACE_PIXEL")
if trace_pixel:
try:
parts = [p.strip() for p in trace_pixel.split(",")]
if len(parts) == 2:
self._trace_pixel = (int(parts[0]), int(parts[1]))
except Exception:
self._trace_pixel = None
def _process_dec_ref_pic_marking(self, slice_header: SliceHeader, nal: NALUnit) -> None:
marking = getattr(slice_header, "dec_ref_pic_marking", None)
if nal.nal_ref_idc == 0 or marking is None:
return
if nal.nal_unit_type == NALUnitType.SLICE_IDR:
self.mmco_processor.process_idr(marking)
def _apply_dec_ref_pic_marking(
self,
slice_header: SliceHeader,
nal: NALUnit,
ref_frame: Optional[ReferenceFrame] = None,
) -> None:
marking = getattr(slice_header, "dec_ref_pic_marking", None)
if nal.nal_ref_idc == 0:
self.mmco_processor.process_for_non_reference()
return
if ref_frame is None or marking is None:
return
if nal.nal_unit_type == NALUnitType.SLICE_IDR:
self.mmco_processor.process_idr(marking, idr_frame=ref_frame)
return
sps = getattr(self.state, "current_sps", None)
max_frame_num = getattr(sps, "max_frame_num", None)
current_poc = self._calculate_poc(slice_header, sps) if sps else 0
self.mmco_processor.process_non_idr(
marking,
current_frame=ref_frame,
current_frame_num=slice_header.frame_num,
max_frame_num=max_frame_num,
current_poc=current_poc,
)
def _apply_mmco_to_ref_buffer(
self,
marking: 'DecRefPicMarking',
current_frame_num: int,
sps: 'SPS',
) -> None:
"""Apply MMCO operations to the reference buffer.
Currently supports MMCO 1 (mark short-term as unused).
H.264 Spec: Section 8.2.5.4
"""
ops = getattr(marking, 'memory_management_control_operations', [])
diffs = getattr(marking, 'difference_of_pic_nums_minus1', [])
max_frame_num = getattr(sps, 'max_frame_num', 256)
diff_idx = 0
for op in ops:
if op == 0:
break
if op == 1:
# Mark short-term reference as unused
if diff_idx < len(diffs):
diff = diffs[diff_idx] + 1
diff_idx += 1
pic_num = (current_frame_num - diff) % max_frame_num
self.state.ref_buffer.remove_by_frame_num(pic_num)
logger.debug(
f"MMCO 1: marked frame_num={pic_num} as unused "
f"(current_fn={current_frame_num}, diff={diff})"
)
elif op == 5:
# Reset all reference pictures
self.state.ref_buffer.clear()
logger.debug("MMCO 5: cleared all reference pictures")
else:
# MMCO 2,3,4,6: long-term operations (skip for now)
logger.debug(f"MMCO {op}: not yet implemented, skipping")
if op in (1, 3) and diff_idx < len(diffs):
diff_idx += 1
def conceal_macroblock(self, *args, **kwargs):
from decoder.error_concealment import conceal_macroblock as _conceal_macroblock
result = _conceal_macroblock(*args, **kwargs)
try:
self.stats.concealed_mb_count += 1
except Exception:
pass
cb = getattr(self, "on_concealment", None)
if callable(cb):
try:
cb(result)
except Exception:
pass
return result
def _detect_aso_mode(self) -> bool:
return False
def configure_chroma_format(self, chroma_format_idc: int) -> None:
chroma_format_idc = int(chroma_format_idc)
if chroma_format_idc not in (0, 1, 2, 3):
raise ValueError(f"Invalid chroma_format_idc: {chroma_format_idc}")
self.chroma_format_idc = chroma_format_idc
self.state.chroma_format_idc = chroma_format_idc
def decode_file(self, path: str):
"""Decode H.264 file and yield frames.
Supports both raw Annex B (.264, .h264) and MP4 container files.
Args:
path: Path to H.264 bitstream or MP4 file.
Yields:
DecodedFrame objects
"""
with open(path, "rb") as f:
data = f.read()
# Auto-detect MP4: ftyp box signature at offset 4
if len(data) >= 8 and data[4:8] == b'ftyp':
from container.mp4 import extract_h264_from_mp4
data = extract_h264_from_mp4(data)
yield from self.decode_bytes(data)
def decode_bytes(self, data: bytes):
"""Decode H.264 bitstream bytes and yield frames.
Args:
data: Annex B bitstream bytes
Yields:
DecodedFrame objects
"""
for nal in iter_nal_units(data):
try:
frame = self._process_nal(nal)
except Exception as e:
if not getattr(self, "error_resilience", False):
raise
logger.warning(
f"Error processing NAL type={getattr(nal, 'nal_unit_type', None)} "
f"at pos={getattr(nal, 'start_position', None)}: {e}"
)
continue
if frame is not None:
yield frame
def _process_nal(self, nal: NALUnit) -> Optional[DecodedFrame]:
"""Process a single NAL unit.
Args:
nal: NAL unit to process
Returns:
DecodedFrame if this NAL completed a frame, None otherwise
"""
logger.debug(f"Processing NAL: type={nal.nal_unit_type}, size={len(nal.rbsp)}")
if nal.nal_unit_type == NALUnitType.SPS:
self._process_sps(nal)
return None
elif nal.nal_unit_type == NALUnitType.PPS:
self._process_pps(nal)
return None
elif nal.nal_unit_type in (NALUnitType.SLICE_IDR, NALUnitType.SLICE_NON_IDR):
return self._decode_slice(nal)
else:
logger.debug(f"Skipping NAL type {nal.nal_unit_type}")
return None
def _process_sps(self, nal: NALUnit) -> None:
"""Parse and store SPS."""
try:
sps = parse_sps(nal.rbsp) # parse_sps expects raw bytes
except Exception as e:
logger.warning(f"Failed to parse SPS (truncated or malformed): {e}")
return
self.state.sps_dict[sps.seq_parameter_set_id] = sps
logger.info(
f"Parsed SPS {sps.seq_parameter_set_id}: "
f"{sps.pic_width_in_mbs * 16}x{sps.frame_height_in_mbs * 16}"
)
def _process_pps(self, nal: NALUnit) -> None:
"""Parse and store PPS."""
try:
# Determine if any SPS is High profile for PPS extension parsing
is_hp = any(
_is_high_profile(sps.profile_idc)
for sps in self.state.sps_dict.values()
)
pps = parse_pps(nal.rbsp, is_high_profile=is_hp)
except Exception as e:
logger.warning(f"Failed to parse PPS (truncated or malformed): {e}")
return
self.state.pps_dict[pps.pic_parameter_set_id] = pps
logger.info(
f"Parsed PPS {pps.pic_parameter_set_id}: "
f"entropy_mode={pps.entropy_coding_mode_flag}"
)
def _decode_slice(self, nal: NALUnit) -> Optional[DecodedFrame]:
"""Decode a slice NAL unit.
Args:
nal: Slice NAL unit
Returns:
DecodedFrame if slice completes a frame
"""
# Need SPS/PPS to parse slice header
if not self.state.sps_dict or not self.state.pps_dict:
logger.warning("No SPS/PPS available, skipping slice")
return None
# Peek at first_mb_in_slice / slice_type / pic_parameter_set_id
# to select the correct PPS for this slice.
try:
probe_reader = BitReader(nal.rbsp)
probe_reader.read_ue() # first_mb_in_slice
probe_reader.read_ue() # slice_type
pps_id = probe_reader.read_ue()
except Exception as exc:
logger.warning(f"Failed to probe slice header PPS id: {exc}")
return None
try:
pps = self.state.get_pps(pps_id)
except Exception as exc:
logger.warning(f"PPS {pps_id} not found for slice: {exc}")
return None
sps = self.state.get_sps(pps.seq_parameter_set_id)
# Parse slice header (expects raw bytes)
slice_header = parse_slice_header(
nal.rbsp, sps, pps, nal.nal_unit_type, nal.nal_ref_idc
)
logger.debug(
f"Slice header: type={slice_header.slice_type}, "
f"frame_num={slice_header.frame_num}"
)
# Check slice type
is_i_slice = slice_header.is_i_slice
is_p_slice = slice_header.is_p_slice if hasattr(slice_header, 'is_p_slice') else False
is_b_slice = slice_header.is_b_slice if hasattr(slice_header, 'is_b_slice') else False
if not is_i_slice and not is_p_slice and not is_b_slice:
logger.warning(f"Skipping unsupported slice: {slice_header.slice_type_name}")
return None
# P/B-slices need reference frames
if is_p_slice or is_b_slice:
if self.state.ref_buffer is None or len(self.state.ref_buffer) == 0:
logger.warning(f"{'P' if is_p_slice else 'B'}-slice but no reference frames available")
return None
# Clear reference buffer on IDR (H.264 8.2.5.1)
if nal.nal_unit_type == NALUnitType.SLICE_IDR:
if self.state.ref_buffer is not None:
self.state.ref_buffer.clear()
logger.debug("Cleared reference buffer on IDR")
# Update current parameter sets
self.state.current_sps = sps
self.state.current_pps = pps
# Allocate frame buffers if needed
if self.state.frame_luma is None:
self.state.allocate_frame_buffers(sps)
# Calculate slice QP
slice_qp = 26 + pps.pic_init_qp_minus26 + slice_header.slice_qp_delta
logger.debug(f"Slice QP: {slice_qp}")
# Calculate POC (works for all poc_type values)
current_poc = self._calculate_poc(slice_header, sps)
# Initialize MV cache for P/B-slices
if is_p_slice or is_b_slice:
self.state.init_mv_cache(sps)
# Create BitReader for slice data (positioned after slice header)
reader = BitReader(nal.rbsp)
reader.position = slice_header.header_bit_size
# Check entropy coding mode
use_cabac = getattr(pps, 'entropy_coding_mode_flag', 0) == 1
# For CABAC, align to byte boundary (skip cabac_alignment_one_bit + zeros)
if use_cabac:
reader.byte_align()
# Decode macroblocks
if use_cabac:
# CABAC entropy coding
self._decode_slice_cabac(
reader, slice_header, sps, pps, slice_qp,
is_i_slice, is_p_slice, is_b_slice
)
elif is_i_slice:
self._decode_slice_data(reader, slice_header, sps, pps, slice_qp)
elif is_p_slice:
self._decode_p_slice_data(reader, slice_header, sps, pps, slice_qp)
else: # B-slice
self._decode_b_slice_data(reader, slice_header, sps, pps, slice_qp)
# Apply deblocking filter (in-place on frame buffers)
if self.deblocking_enabled and slice_header.deblocking_enabled:
self._deblock_frame(slice_header, sps, pps)
trace_pre_mb = getattr(self, "_trace_pre_mb", None)
if trace_pre_mb is not None:
try:
t_frame_num, t_mb_x, t_mb_y, pre_luma = trace_pre_mb
if t_frame_num == slice_header.frame_num:
ly, lx = t_mb_y * 16, t_mb_x * 16
post_luma = self.state.frame_luma[ly:ly + 16, lx:lx + 16]
d = np.abs(post_luma.astype(np.int16) - pre_luma.astype(np.int16))
logger.warning(
"TRACE MB post-deblock frame_num=%s MB=(%s,%s) max_delta=%s mean_delta=%.6f",
t_frame_num,
t_mb_x,
t_mb_y,
int(d.max()),
float(d.mean()),
)
trace_pixel = getattr(self, "_trace_pixel", None)
if trace_pixel is not None:
px, py = trace_pixel
if lx <= px < lx + 16 and ly <= py < ly + 16:
rx, ry = px - lx, py - ly
logger.warning(
"TRACE pixel post-deblock frame_num=%s (x,y)=(%s,%s) rel=(%s,%s) pre=%s post=%s",
t_frame_num,
px,
py,
rx,
ry,
int(pre_luma[ry, rx]),
int(post_luma[ry, rx]),
)
except Exception:
pass
# Return completed frame
# For simplicity, assume each slice is a complete frame
out_luma = self.state.frame_luma
out_cb = self.state.frame_cb
out_cr = self.state.frame_cr
out_width = sps.pic_width_in_mbs * 16
out_height = sps.frame_height_in_mbs * 16
if getattr(sps, "frame_cropping_flag", False):
chroma_format_idc = getattr(sps, "chroma_format_idc", 1)
separate_colour_plane_flag = getattr(
sps, "separate_colour_plane_flag", False
)
if separate_colour_plane_flag or chroma_format_idc == 0:
sub_w, sub_h = 1, 1
elif chroma_format_idc == 1:
sub_w, sub_h = 2, 2
elif chroma_format_idc == 2:
sub_w, sub_h = 2, 1
else: # chroma_format_idc == 3
sub_w, sub_h = 1, 1
if separate_colour_plane_flag or chroma_format_idc == 0:
crop_unit_x = 1
crop_unit_y = 2 - (1 if sps.frame_mbs_only_flag else 0)
else:
crop_unit_x = sub_w
crop_unit_y = sub_h * (2 - (1 if sps.frame_mbs_only_flag else 0))
crop_left = int(sps.frame_crop_left_offset * crop_unit_x)
crop_right = int(sps.frame_crop_right_offset * crop_unit_x)
crop_top = int(sps.frame_crop_top_offset * crop_unit_y)
crop_bottom = int(sps.frame_crop_bottom_offset * crop_unit_y)
out_luma = out_luma[
crop_top : out_height - crop_bottom,
crop_left : out_width - crop_right,
]
out_height, out_width = out_luma.shape
if chroma_format_idc != 0 and out_cb is not None and out_cr is not None:
crop_left_c = crop_left // sub_w
crop_right_c = crop_right // sub_w
crop_top_c = crop_top // sub_h
crop_bottom_c = crop_bottom // sub_h
out_cb = out_cb[
crop_top_c : out_cb.shape[0] - crop_bottom_c,
crop_left_c : out_cb.shape[1] - crop_right_c,
]
out_cr = out_cr[
crop_top_c : out_cr.shape[0] - crop_bottom_c,
crop_left_c : out_cr.shape[1] - crop_right_c,
]
frame = DecodedFrame(
frame_num=slice_header.frame_num,
poc=current_poc,
luma=out_luma.copy(),
cb=out_cb.copy() if out_cb is not None else None,
cr=out_cr.copy() if out_cr is not None else None,
width=out_width,
height=out_height,
chroma_format_idc=getattr(sps, "chroma_format_idc", 1),
)
# Add to reference buffer if this is a reference frame
if nal.nal_ref_idc > 0:
# Apply MMCO before adding (H.264 8.2.5)
marking = getattr(slice_header, 'dec_ref_pic_marking', None)
if marking is not None and getattr(marking, 'adaptive_ref_pic_marking_mode_flag', False):
self._apply_mmco_to_ref_buffer(
marking, slice_header.frame_num, sps
)
ref_frame = ReferenceFrame(
luma=self.state.frame_luma.copy(),
cb=self.state.frame_cb.copy(),
cr=self.state.frame_cr.copy(),
frame_num=slice_header.frame_num,
poc=current_poc,
)
self._store_mvs_in_ref(ref_frame)
self.state.ref_buffer.add_frame(ref_frame)
logger.debug(f"Added frame {slice_header.frame_num} to reference buffer")
return frame
def _decode_slice_data(
self,
reader: BitReader,
slice_header: SliceHeader,
sps: SPS,
pps: PPS,
slice_qp: int,
) -> None:
"""Decode slice data (macroblocks).
Processes macroblocks in raster scan order.
Args:
reader: BitReader positioned at slice data
slice_header: Parsed slice header
sps: Active SPS
pps: Active PPS
slice_qp: Slice-level QP
"""
mb_width = sps.pic_width_in_mbs
mb_height = sps.frame_height_in_mbs
total_mbs = mb_width * mb_height
logger.debug(f"Decoding {total_mbs} macroblocks ({mb_width}x{mb_height})")
if slice_header.first_mb_in_slice == 0 and self.state.intra_modes is not None:
# Reset intra 4x4 prediction modes at the start of a new frame.
# Prevents stale modes from previous frames influencing MPM.
self.state.intra_modes[:] = 2
current_qp = slice_qp
for mb_idx in range(slice_header.first_mb_in_slice, total_mbs):
mb_x = mb_idx % mb_width
mb_y = mb_idx // mb_width
# Check for RBSP trailing bits (H.264 spec 7.2)
if not reader.more_rbsp_data():
logger.warning(f"Reached RBSP trailing bits after MB ({mb_x-1 if mb_x > 0 else mb_width-1}, "
f"{mb_y-1 if mb_x == 0 else mb_y}), stopping slice decode "
f"({mb_idx}/{total_mbs} MBs decoded)")
break
# Get neighbor information for prediction
neighbors = self._get_mb_neighbors(mb_x, mb_y, mb_width)
try:
# Decode single macroblock
mb_data = decode_macroblock(
reader=reader,
sps=sps,
pps=pps,
slice_qp=current_qp,
mb_x=mb_x,
mb_y=mb_y,
frame_luma=self.state.frame_luma,
frame_cb=self.state.frame_cb,
frame_cr=self.state.frame_cr,
is_i_slice=True,
frame_nz_counts=self.state.nz_counts,
frame_width_mbs=mb_width,
frame_intra_modes=self.state.intra_modes,
)
# Store non-zero counts for CAVLC context
self.state.nz_counts[mb_idx] = mb_data.nz_counts
# Store MB type for neighbor reference and diagnostics
if self.state.mb_types is not None:
self.state.mb_types[mb_idx] = mb_data.mb_type
# Update running QP for next MB (H.264: QP is cumulative)
current_qp = (current_qp + mb_data.mb_qp_delta + 52) % 52
# Store per-MB QP for deblocking filter
if self.state.mb_qps is not None:
self.state.mb_qps[mb_idx] = current_qp
# Store per-block coefficient flags for deblocking
if self.state.mb_coeffs is not None:
self.state.mb_coeffs[mb_idx, :16] = (
mb_data.nz_counts[:16] > 0
)
logger.debug(f"Decoded MB ({mb_x}, {mb_y}): type={mb_data.mb_type}")
except ValueError as e:
# Check if it's an invalid mb_type error (likely RBSP trailing bits)
if "Invalid I_16x16 mb_type" in str(e) or "Invalid I_4x4 mb_type" in str(e):
logger.warning(f"Invalid mb_type at MB ({mb_x}, {mb_y}), likely RBSP trailing bits. "
f"Stopping slice decode ({mb_idx}/{total_mbs} MBs decoded)")
break
else:
logger.error(f"Error decoding MB ({mb_x}, {mb_y}): {e}")
raise
except Exception as e:
logger.error(f"Error decoding MB ({mb_x}, {mb_y}): {e}")
raise
def _get_mb_neighbors(
self, mb_x: int, mb_y: int, mb_width: int
) -> dict:
"""Get neighbor macroblock data for prediction.
Args:
mb_x: Macroblock X position
mb_y: Macroblock Y position
mb_width: Frame width in macroblocks
Returns:
Dictionary with neighbor information
"""
neighbors = {
"top_available": mb_y > 0,
"left_available": mb_x > 0,
"top_left_available": mb_x > 0 and mb_y > 0,
"top_right_available": mb_y > 0 and mb_x < mb_width - 1,
}
# Extract neighbor pixels from frame buffers
if neighbors["top_available"] and self.state.frame_luma is not None:
top_row = mb_y * 16 - 1
left_col = mb_x * 16
neighbors["top_luma"] = self.state.frame_luma[
top_row, left_col : left_col + 16
]
else:
neighbors["top_luma"] = None
if neighbors["left_available"] and self.state.frame_luma is not None:
top_row = mb_y * 16
left_col = mb_x * 16 - 1
neighbors["left_luma"] = self.state.frame_luma[
top_row : top_row + 16, left_col
]
else:
neighbors["left_luma"] = None
return neighbors
def _decode_p_slice_data(
self,
reader: BitReader,
slice_header: SliceHeader,
sps: SPS,
pps: PPS,
slice_qp: int,
) -> None:
"""Decode P-slice data (macroblocks with inter prediction).
P-slices can contain:
- P_Skip macroblocks (mb_skip_run)
- P_L0_16x16, P_L0_L0_16x8, P_L0_L0_8x16, P_8x8 macroblocks
- I-macroblocks (intra in P-slice)
Args:
reader: BitReader positioned at slice data
slice_header: Parsed slice header
sps: Active SPS
pps: Active PPS
slice_qp: Slice-level QP
"""
mb_width = sps.pic_width_in_mbs
mb_height = sps.frame_height_in_mbs
total_mbs = mb_width * mb_height
logger.debug(f"Decoding P-slice: {total_mbs} macroblocks")
# Apply reference list reordering for P-slices (H.264 8.2.4)
max_frame_num = getattr(sps, 'max_frame_num', 256)
mod_l0 = getattr(slice_header, 'ref_pic_list_modification_l0', None)
num_l0_active = getattr(slice_header, 'num_ref_idx_l0_active_minus1', 0) + 1