diff --git a/library/rtsp/src/main/java/com/google/android/exoplayer2/source/rtsp/reader/RtpVp8Reader.java b/library/rtsp/src/main/java/com/google/android/exoplayer2/source/rtsp/reader/RtpVp8Reader.java index 8663ef9c711..5067caceaff 100644 --- a/library/rtsp/src/main/java/com/google/android/exoplayer2/source/rtsp/reader/RtpVp8Reader.java +++ b/library/rtsp/src/main/java/com/google/android/exoplayer2/source/rtsp/reader/RtpVp8Reader.java @@ -15,6 +15,8 @@ */ package com.google.android.exoplayer2.source.rtsp.reader; +import static com.google.android.exoplayer2.util.Assertions.checkNotNull; +import static com.google.android.exoplayer2.util.Assertions.checkState; import static com.google.android.exoplayer2.util.Assertions.checkStateNotNull; import com.google.android.exoplayer2.C; @@ -51,6 +53,8 @@ /** The combined size of a sample that is fragmented into multiple RTP packets. */ private int fragmentedSampleSizeBytes; + private long fragmentedSampleTimeUs; + private long startTimeOffsetUs; /** * Whether the first packet of one VP8 frame is received. A VP8 frame can be split into two RTP @@ -67,6 +71,7 @@ public RtpVp8Reader(RtpPayloadFormat payloadFormat) { firstReceivedTimestamp = C.TIME_UNSET; previousSequenceNumber = C.INDEX_UNSET; fragmentedSampleSizeBytes = C.LENGTH_UNSET; + fragmentedSampleTimeUs = C.TIME_UNSET; // The start time offset must be 0 until the first seek. startTimeOffsetUs = 0; gotFirstPacketOfVp8Frame = false; @@ -81,7 +86,10 @@ public void createTracks(ExtractorOutput extractorOutput, int trackId) { } @Override - public void onReceivingFirstPacket(long timestamp, int sequenceNumber) {} + public void onReceivingFirstPacket(long timestamp, int sequenceNumber) { + checkState(firstReceivedTimestamp == C.TIME_UNSET); + firstReceivedTimestamp = timestamp; + } @Override public void consume( @@ -113,21 +121,16 @@ public void consume( int fragmentSize = data.bytesLeft(); trackOutput.sampleData(data, fragmentSize); - fragmentedSampleSizeBytes += fragmentSize; + if (fragmentedSampleSizeBytes == C.LENGTH_UNSET) { + fragmentedSampleSizeBytes = fragmentSize; + } else { + fragmentedSampleSizeBytes += fragmentSize; + } + + fragmentedSampleTimeUs = toSampleUs(startTimeOffsetUs, timestamp, firstReceivedTimestamp); if (rtpMarker) { - if (firstReceivedTimestamp == C.TIME_UNSET) { - firstReceivedTimestamp = timestamp; - } - long timeUs = toSampleUs(startTimeOffsetUs, timestamp, firstReceivedTimestamp); - trackOutput.sampleMetadata( - timeUs, - isKeyFrame ? C.BUFFER_FLAG_KEY_FRAME : 0, - fragmentedSampleSizeBytes, - /* offset= */ 0, - /* cryptoData= */ null); - fragmentedSampleSizeBytes = C.LENGTH_UNSET; - gotFirstPacketOfVp8Frame = false; + outputSampleMetadataForFragmentedPackets(); } previousSequenceNumber = sequenceNumber; } @@ -147,18 +150,18 @@ public void seek(long nextRtpTimestamp, long timeUs) { private boolean validateVp8Descriptor(ParsableByteArray payload, int packetSequenceNumber) { // VP8 Payload Descriptor is defined in RFC7741 Section 4.2. int header = payload.readUnsignedByte(); - if (!gotFirstPacketOfVp8Frame) { - // TODO(b/198620566) Consider using ParsableBitArray. - // For start of VP8 partition S=1 and PID=0 as per RFC7741 Section 4.2. - if ((header & 0x10) != 0x1 || (header & 0x07) != 0) { - Log.w(TAG, "RTP packet is not the start of a new VP8 partition, skipping."); - return false; + // TODO(b/198620566) Consider using ParsableBitArray. + // For start of VP8 partition S=1 and PID=0 as per RFC7741 Section 4.2. + if ((header & 0x10) == 0x10 && (header & 0x07) == 0) { + if (gotFirstPacketOfVp8Frame && fragmentedSampleSizeBytes > 0) { + // Received new VP8 fragment, output data of previous fragment to decoder. + outputSampleMetadataForFragmentedPackets(); } gotFirstPacketOfVp8Frame = true; - } else { + } else if (gotFirstPacketOfVp8Frame) { // Check that this packet is in the sequence of the previous packet. int expectedSequenceNumber = RtpPacket.getNextSequenceNumber(previousSequenceNumber); - if (packetSequenceNumber != expectedSequenceNumber) { + if (packetSequenceNumber < expectedSequenceNumber) { Log.w( TAG, Util.formatInvariant( @@ -167,6 +170,9 @@ private boolean validateVp8Descriptor(ParsableByteArray payload, int packetSeque expectedSequenceNumber, packetSequenceNumber)); return false; } + } else { + Log.w(TAG, "RTP packet is not the start of a new VP8 partition, skipping."); + return false; } // Check if optional X header is present. @@ -195,6 +201,24 @@ private boolean validateVp8Descriptor(ParsableByteArray payload, int packetSeque return true; } + /** + * Outputs sample metadata of the received fragmented packets. + * + *

Call this method only after receiving an end of a VP8 partition. + */ + private void outputSampleMetadataForFragmentedPackets() { + checkNotNull(trackOutput) + .sampleMetadata( + fragmentedSampleTimeUs, + isKeyFrame ? C.BUFFER_FLAG_KEY_FRAME : 0, + fragmentedSampleSizeBytes, + /* offset= */ 0, + /* cryptoData= */ null); + fragmentedSampleSizeBytes = 0; + fragmentedSampleTimeUs = C.TIME_UNSET; + gotFirstPacketOfVp8Frame = false; + } + private static long toSampleUs( long startTimeOffsetUs, long rtpTimestamp, long firstReceivedRtpTimestamp) { return startTimeOffsetUs diff --git a/library/rtsp/src/test/java/com/google/android/exoplayer2/source/rtsp/reader/RtpVp8ReaderTest.java b/library/rtsp/src/test/java/com/google/android/exoplayer2/source/rtsp/reader/RtpVp8ReaderTest.java new file mode 100644 index 00000000000..243ea743d8a --- /dev/null +++ b/library/rtsp/src/test/java/com/google/android/exoplayer2/source/rtsp/reader/RtpVp8ReaderTest.java @@ -0,0 +1,203 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.exoplayer2.source.rtsp.reader; + +import static com.google.android.exoplayer2.util.Util.getBytesFromHexString; +import static com.google.common.truth.Truth.assertThat; + +import androidx.test.ext.junit.runners.AndroidJUnit4; +import com.google.android.exoplayer2.C; +import com.google.android.exoplayer2.Format; +import com.google.android.exoplayer2.source.rtsp.RtpPacket; +import com.google.android.exoplayer2.source.rtsp.RtpPayloadFormat; +import com.google.android.exoplayer2.testutil.FakeExtractorOutput; +import com.google.android.exoplayer2.testutil.FakeTrackOutput; +import com.google.android.exoplayer2.util.MimeTypes; +import com.google.android.exoplayer2.util.ParsableByteArray; +import com.google.android.exoplayer2.util.Util; +import com.google.common.collect.ImmutableMap; +import com.google.common.primitives.Bytes; +import java.util.Arrays; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; + +/** Unit test for {@link RtpVp8Reader}. */ +@RunWith(AndroidJUnit4.class) +public final class RtpVp8ReaderTest { + + /** VP9 uses a 90 KHz media clock (RFC7741 Section 4.1). */ + private static final long MEDIA_CLOCK_FREQUENCY = 90_000; + + private static final byte[] PARTITION_1 = getBytesFromHexString("000102030405060708090A0B0C0D0E"); + // 000102030405060708090A + private static final byte[] PARTITION_1_FRAGMENT_1 = + Arrays.copyOf(PARTITION_1, /* newLength= */ 11); + // 0B0C0D0E + private static final byte[] PARTITION_1_FRAGMENT_2 = + Arrays.copyOfRange(PARTITION_1, /* from= */ 11, /* to= */ 15); + private static final long PARTITION_1_RTP_TIMESTAMP = 2599168056L; + private static final RtpPacket PACKET_PARTITION_1_FRAGMENT_1 = + new RtpPacket.Builder() + .setTimestamp(PARTITION_1_RTP_TIMESTAMP) + .setSequenceNumber(40289) + .setMarker(false) + .setPayloadData(Bytes.concat(getBytesFromHexString("10"), PARTITION_1_FRAGMENT_1)) + .build(); + private static final RtpPacket PACKET_PARTITION_1_FRAGMENT_2 = + new RtpPacket.Builder() + .setTimestamp(PARTITION_1_RTP_TIMESTAMP) + .setSequenceNumber(40290) + .setMarker(false) + .setPayloadData(Bytes.concat(getBytesFromHexString("00"), PARTITION_1_FRAGMENT_2)) + .build(); + + private static final byte[] PARTITION_2 = getBytesFromHexString("0D0C0B0A09080706050403020100"); + // 0D0C0B0A090807060504 + private static final byte[] PARTITION_2_FRAGMENT_1 = + Arrays.copyOf(PARTITION_2, /* newLength= */ 10); + // 03020100 + private static final byte[] PARTITION_2_FRAGMENT_2 = + Arrays.copyOfRange(PARTITION_2, /* from= */ 10, /* to= */ 14); + private static final long PARTITION_2_RTP_TIMESTAMP = 2599168344L; + private static final RtpPacket PACKET_PARTITION_2_FRAGMENT_1 = + new RtpPacket.Builder() + .setTimestamp(PARTITION_2_RTP_TIMESTAMP) + .setSequenceNumber(40291) + .setMarker(false) + .setPayloadData(Bytes.concat(getBytesFromHexString("10"), PARTITION_2_FRAGMENT_1)) + .build(); + private static final RtpPacket PACKET_PARTITION_2_FRAGMENT_2 = + new RtpPacket.Builder() + .setTimestamp(PARTITION_2_RTP_TIMESTAMP) + .setSequenceNumber(40292) + .setMarker(true) + .setPayloadData( + Bytes.concat( + getBytesFromHexString("80"), + // Optional header. + getBytesFromHexString("D6AA953961"), + PARTITION_2_FRAGMENT_2)) + .build(); + private static final long PARTITION_2_PRESENTATION_TIMESTAMP_US = + Util.scaleLargeTimestamp( + (PARTITION_2_RTP_TIMESTAMP - PARTITION_1_RTP_TIMESTAMP), + /* multiplier= */ C.MICROS_PER_SECOND, + /* divisor= */ MEDIA_CLOCK_FREQUENCY); + + private FakeExtractorOutput extractorOutput; + + @Before + public void setUp() { + extractorOutput = + new FakeExtractorOutput( + (id, type) -> new FakeTrackOutput(/* deduplicateConsecutiveFormats= */ true)); + } + + @Test + public void consume_validPackets() { + RtpVp8Reader vp8Reader = createVp8Reader(); + + vp8Reader.createTracks(extractorOutput, /* trackId= */ 0); + vp8Reader.onReceivingFirstPacket( + PACKET_PARTITION_1_FRAGMENT_1.timestamp, PACKET_PARTITION_1_FRAGMENT_1.sequenceNumber); + consume(vp8Reader, PACKET_PARTITION_1_FRAGMENT_1); + consume(vp8Reader, PACKET_PARTITION_1_FRAGMENT_2); + consume(vp8Reader, PACKET_PARTITION_2_FRAGMENT_1); + consume(vp8Reader, PACKET_PARTITION_2_FRAGMENT_2); + + FakeTrackOutput trackOutput = extractorOutput.trackOutputs.get(0); + assertThat(trackOutput.getSampleCount()).isEqualTo(2); + assertThat(trackOutput.getSampleData(0)).isEqualTo(PARTITION_1); + assertThat(trackOutput.getSampleTimeUs(0)).isEqualTo(0); + assertThat(trackOutput.getSampleData(1)).isEqualTo(PARTITION_2); + assertThat(trackOutput.getSampleTimeUs(1)).isEqualTo(PARTITION_2_PRESENTATION_TIMESTAMP_US); + } + + @Test + public void consume_fragmentedFrameMissingFirstFragment() { + RtpVp8Reader vp8Reader = createVp8Reader(); + + vp8Reader.createTracks(extractorOutput, /* trackId= */ 0); + // First packet timing information is transmitted over RTSP, not RTP. + vp8Reader.onReceivingFirstPacket( + PACKET_PARTITION_1_FRAGMENT_1.timestamp, PACKET_PARTITION_1_FRAGMENT_1.sequenceNumber); + consume(vp8Reader, PACKET_PARTITION_1_FRAGMENT_2); + consume(vp8Reader, PACKET_PARTITION_2_FRAGMENT_1); + consume(vp8Reader, PACKET_PARTITION_2_FRAGMENT_2); + + FakeTrackOutput trackOutput = extractorOutput.trackOutputs.get(0); + assertThat(trackOutput.getSampleCount()).isEqualTo(1); + assertThat(trackOutput.getSampleData(0)).isEqualTo(PARTITION_2); + assertThat(trackOutput.getSampleTimeUs(0)).isEqualTo(PARTITION_2_PRESENTATION_TIMESTAMP_US); + } + + @Test + public void consume_fragmentedFrameMissingBoundaryFragment() { + RtpVp8Reader vp8Reader = createVp8Reader(); + + vp8Reader.createTracks(extractorOutput, /* trackId= */ 0); + vp8Reader.onReceivingFirstPacket( + PACKET_PARTITION_1_FRAGMENT_1.timestamp, PACKET_PARTITION_1_FRAGMENT_1.sequenceNumber); + consume(vp8Reader, PACKET_PARTITION_1_FRAGMENT_1); + consume(vp8Reader, PACKET_PARTITION_2_FRAGMENT_1); + consume(vp8Reader, PACKET_PARTITION_2_FRAGMENT_2); + + FakeTrackOutput trackOutput = extractorOutput.trackOutputs.get(0); + assertThat(trackOutput.getSampleCount()).isEqualTo(2); + assertThat(trackOutput.getSampleData(0)).isEqualTo(PARTITION_1_FRAGMENT_1); + assertThat(trackOutput.getSampleTimeUs(0)).isEqualTo(0); + assertThat(trackOutput.getSampleData(1)).isEqualTo(PARTITION_2); + assertThat(trackOutput.getSampleTimeUs(1)).isEqualTo(PARTITION_2_PRESENTATION_TIMESTAMP_US); + } + + @Test + public void consume_outOfOrderFragmentedFrame() { + RtpVp8Reader vp8Reader = createVp8Reader(); + + vp8Reader.createTracks(extractorOutput, /* trackId= */ 0); + vp8Reader.onReceivingFirstPacket( + PACKET_PARTITION_1_FRAGMENT_1.timestamp, PACKET_PARTITION_1_FRAGMENT_1.sequenceNumber); + consume(vp8Reader, PACKET_PARTITION_1_FRAGMENT_1); + consume(vp8Reader, PACKET_PARTITION_2_FRAGMENT_1); + consume(vp8Reader, PACKET_PARTITION_1_FRAGMENT_2); + consume(vp8Reader, PACKET_PARTITION_2_FRAGMENT_2); + + FakeTrackOutput trackOutput = extractorOutput.trackOutputs.get(0); + assertThat(trackOutput.getSampleCount()).isEqualTo(2); + assertThat(trackOutput.getSampleData(0)).isEqualTo(PARTITION_1_FRAGMENT_1); + assertThat(trackOutput.getSampleTimeUs(0)).isEqualTo(0); + assertThat(trackOutput.getSampleData(1)).isEqualTo(PARTITION_2); + assertThat(trackOutput.getSampleTimeUs(1)).isEqualTo(PARTITION_2_PRESENTATION_TIMESTAMP_US); + } + + private static RtpVp8Reader createVp8Reader() { + return new RtpVp8Reader( + new RtpPayloadFormat( + new Format.Builder().setSampleMimeType(MimeTypes.VIDEO_VP8).build(), + /* rtpPayloadType= */ 96, + /* clockRate= */ (int) MEDIA_CLOCK_FREQUENCY, + /* fmtpParameters= */ ImmutableMap.of())); + } + + private static void consume(RtpVp8Reader vp8Reader, RtpPacket rtpPacket) { + vp8Reader.consume( + new ParsableByteArray(rtpPacket.payloadData), + rtpPacket.timestamp, + rtpPacket.sequenceNumber, + rtpPacket.marker); + } +}