Skip to content

Commit

Permalink
Fix DASH content type parsing
Browse files Browse the repository at this point in the history 
The DASH parser was not always correctly deducing the content type.

For unspecified content types, the type can be deduced from the MIME
type.  For example, video/mp4 is video, and audio/webm is audio.

For text, things are a little more complicated.  Text types do not
always start with text/.  In particular, embedded text, such as VTT
in MP4, have a MIME type that starts with application/mp4.

To deal with that, if we see an unknown type, we ask TextEngine if it
supports it.  If so, we deduce that the content type should be text.

This check against TextEngine was only happening for MIME types
specified on the Representation, but not on AdaptationSet.  This
replaces a weaker deduction in the general frame parser with the same
TextEngine check we were using elsewhere.

Furthermore, Player mishandled the content types it passed to
AbrManager.  AbrManager will only choose audio, video, and text
streams.  It ignores all other types.  Player, meanwhile, threw a
confusing error if AbrManager failed to choose some of the types
passed to it.  Therefore, Player should only pass audio, video, and
text types to AbrManager.

This fixes both issues and adds new unit tests for both.

Closes shaka-project#631

Change-Id: Ib1311d37d00c3989367fd066f66e1eba85652e40
  • Loading branch information
@joeyparrish
joeyparrish committed Dec 14, 2016
1 parent be0d4b5 commit 7dc98a7
Show file tree
Hide file tree
Showing 6 changed files with 147 additions and 32 deletions.
47 changes: 30 additions & 17 deletions lib/dash/dash_parser.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ goog.require('shaka.util.LanguageUtils');
goog.require('shaka.util.MapUtils');
goog.require('shaka.util.Mp4Parser');
goog.require('shaka.util.MultiMap');
goog.require('shaka.util.StreamUtils');
goog.require('shaka.util.StringUtils');
goog.require('shaka.util.XmlUtils');

Expand Down Expand Up @@ -817,21 +818,8 @@ shaka.dash.DashParser.prototype.parseAdaptationSet_ = function(context, elem) {
// Guess the AdaptationSet's content type.
var mimeType = streams[0].mimeType;
var codecs = streams[0].codecs;
var fullMimeType = mimeType;
if (codecs) {
fullMimeType += '; codecs="' + codecs + '"';
}

if (shaka.media.TextEngine.isTypeSupported(fullMimeType)) {
// If it's supported by TextEngine, it's definitely text.
// We don't check MediaSourceEngine, because that would report support
// for platform-supported video and audio types as well.
context.adaptationSet.contentType = 'text';
} else {
// Otherwise, just split the MIME type. This handles video and audio
// types well.
context.adaptationSet.contentType = mimeType.split('/')[0];
}
context.adaptationSet.contentType =
shaka.dash.DashParser.guessContentType_(mimeType, codecs);
}

return {
Expand Down Expand Up @@ -1034,11 +1022,12 @@ shaka.dash.DashParser.prototype.createFrame_ = function(

var contentType = elem.getAttribute('contentType') || parent.contentType;
var mimeType = elem.getAttribute('mimeType') || parent.mimeType;
var codecs = elem.getAttribute('codecs') || parent.codecs;
var frameRate = XmlUtils.parseAttr(elem, 'frameRate',
evalDivision) || parent.frameRate;

if (!contentType) {
contentType = mimeType.split('/')[0];
contentType = shaka.dash.DashParser.guessContentType_(mimeType, codecs);
}

return {
Expand All @@ -1051,7 +1040,7 @@ shaka.dash.DashParser.prototype.createFrame_ = function(
height: XmlUtils.parseAttr(elem, 'height', parseNumber) || parent.height,
contentType: contentType,
mimeType: mimeType,
codecs: elem.getAttribute('codecs') || parent.codecs,
codecs: codecs,
frameRate: frameRate,
id: elem.getAttribute('id')
};
Expand Down Expand Up @@ -1383,6 +1372,30 @@ shaka.dash.DashParser.prototype.emsgResponseFilter_ = function(type, response) {
};


/**
* Guess the content type based on MIME type and codecs.
*
* @param {string} mimeType
* @param {string} codecs
* @return {string}
* @private
*/
shaka.dash.DashParser.guessContentType_ = function(mimeType, codecs) {
var fullMimeType = shaka.util.StreamUtils.getFullMimeType(mimeType, codecs);

if (shaka.media.TextEngine.isTypeSupported(fullMimeType)) {
// If it's supported by TextEngine, it's definitely text.
// We don't check MediaSourceEngine, because that would report support
// for platform-supported video and audio types as well.
return 'text';
}

// Otherwise, just split the MIME type. This handles video and audio
// types well.
return mimeType.split('/')[0];
};


/** @const {number} */
shaka.dash.DashParser.BOX_TYPE_EMSG = 0x656D7367;

Expand Down
9 changes: 5 additions & 4 deletions lib/media/streaming_engine.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ goog.require('shaka.util.Functional');
goog.require('shaka.util.IDestroyable');
goog.require('shaka.util.MapUtils');
goog.require('shaka.util.PublicPromise');
goog.require('shaka.util.StreamUtils');



Expand Down Expand Up @@ -419,8 +420,8 @@ shaka.media.StreamingEngine.prototype.switch = function(
if (contentType == 'text') {
// Mime types are allowed to change for text streams.
// Reinitialize the text parser.
var fullMimeType = stream.mimeType +
(stream.codecs ? '; codecs="' + stream.codecs + '"' : '');
var fullMimeType = shaka.util.StreamUtils.getFullMimeType(
stream.mimeType, stream.codecs);
this.mediaSourceEngine_.reinitText(fullMimeType);
}

Expand Down Expand Up @@ -555,8 +556,8 @@ shaka.media.StreamingEngine.prototype.initStreams_ = function(streamsByType) {

// Init MediaSourceEngine.
var typeConfig = MapUtils.map(streamsByType, function(stream) {
return stream.mimeType +
(stream.codecs ? '; codecs="' + stream.codecs + '"' : '');
return shaka.util.StreamUtils.getFullMimeType(
stream.mimeType, stream.codecs);
});

this.mediaSourceEngine_.init(typeConfig,
Expand Down
19 changes: 17 additions & 2 deletions lib/player.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1430,8 +1430,23 @@ shaka.Player.prototype.chooseStreams_ =
var needsUpdate = {};

if (opt_chooseAll) {
// Choose new streams for all types.
needsUpdate = streamSetsByType;
// Choose new streams for all recognized types.

// When MIME types starting with 'application/' were incorrectly turned
// into StreamSets of type 'application', we fed these to AbrManager.
// AbrManager ignored them, and this caused us to fail with a very
// confusing RESTRICTIONS_CANNOT_BE_MET error.

// It is important that we only feed AbrManager things we can expect it to
// deal with. That way, we can later check that AbrManager chose streams
// for all the types we requested.

var recognizedTypes = ['video', 'audio', 'text'];
recognizedTypes.forEach(function(type) {
if (type in streamSetsByType) {
needsUpdate[type] = streamSetsByType[type];
}
});
} else {
// Check if any of the active streams is no longer available
// or is using the wrong language.
Expand Down
22 changes: 18 additions & 4 deletions lib/util/stream_utils.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,10 +112,8 @@ shaka.util.StreamUtils.filterPeriod = function(

for (var j = 0; j < streamSet.streams.length; ++j) {
var stream = streamSet.streams[j];
var fullMimeType = stream.mimeType;
if (stream.codecs) {
fullMimeType += '; codecs="' + stream.codecs + '"';
}
var fullMimeType = shaka.util.StreamUtils.getFullMimeType(
stream.mimeType, stream.codecs);

if (!shaka.media.MediaSourceEngine.isTypeSupported(fullMimeType)) {
// Remove streams that cannot be played by the platform.
Expand Down Expand Up @@ -364,3 +362,19 @@ shaka.util.StreamUtils.getHighestResolution = function(streamSet) {

return highestRes;
};


/**
* Takes a MIME type and optional codecs string and produces the full MIME type.
*
* @param {string} mimeType
* @param {string=} opt_codecs
* @return {string}
*/
shaka.util.StreamUtils.getFullMimeType = function(mimeType, opt_codecs) {
var fullMimeType = mimeType;
if (opt_codecs) {
fullMimeType += '; codecs="' + opt_codecs + '"';
}
return fullMimeType;
};
31 changes: 31 additions & 0 deletions test/dash/dash_parser_manifest_unit.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -935,4 +935,35 @@ describe('DashParser Manifest', function() {
expect(manifest.periods[0].streamSets[0].streams.length).toBe(1);
}).catch(fail).then(done);
});

it('sets contentType to text for embedded text mime types', function(done) {
// One MIME type for embedded TTML, one for embedded WebVTT.
// One MIME type specified on AdaptationSet, on one Representation.
var manifestText = [
'<MPD minBufferTime="PT75S">',
' <Period id="1" duration="PT30S">',
' <AdaptationSet id="1" mimeType="application/mp4" codecs="stpp">',
' <Representation>',
' <SegmentTemplate media="1.mp4" duration="1" />',
' </Representation>',
' </AdaptationSet>',
' <AdaptationSet id="2">',
' <Representation mimeType="application/mp4" codecs="wvtt">',
' <SegmentTemplate media="2.mp4" duration="1" />',
' </Representation>',
' </AdaptationSet>',
' </Period>',
'</MPD>'
].join('\n');

fakeNetEngine.setResponseMapAsText({'dummy://foo': manifestText});
parser.start('dummy://foo', fakeNetEngine, filterPeriod, fail, onEventSpy)
.then(function(manifest) {
expect(manifest.periods.length).toBe(1);
expect(manifest.periods[0].streamSets.length).toBe(2);
// At one time, these came out as 'application' rather than 'text'.
expect(manifest.periods[0].streamSets[0].type).toBe('text');
expect(manifest.periods[0].streamSets[1].type).toBe('text');
}).catch(fail).then(done);
});
});
51 changes: 46 additions & 5 deletions test/player_unit.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ describe('Player', function() {
var logErrorSpy;
var logWarnSpy;
var manifest;
var onError;
var player;
var networkingEngine;
var streamingEngine;
Expand Down Expand Up @@ -81,6 +82,12 @@ describe('Player', function() {

abrManager = new shaka.test.FakeAbrManager();
player.configure({abr: {manager: abrManager}});

onError = jasmine.createSpy('error event');
onError.and.callFake(function(event) {
fail(event.detail);
});
player.addEventListener('error', onError);
});

afterEach(function(done) {
Expand Down Expand Up @@ -1297,7 +1304,6 @@ describe('Player', function() {
});

it('issues error if no streams are playable', function() {
var onError = jasmine.createSpy('error event');
onError.and.callFake(function(e) {
var error = e.detail;
shaka.test.Util.expectToEqualError(
Expand All @@ -1306,7 +1312,6 @@ describe('Player', function() {
shaka.util.Error.Category.MANIFEST,
shaka.util.Error.Code.RESTRICTIONS_CANNOT_BE_MET));
});
player.addEventListener('error', onError);

player.configure(
{restrictions: {maxAudioBandwidth: 0, maxVideoBandwidth: 0}});
Expand Down Expand Up @@ -1351,9 +1356,10 @@ describe('Player', function() {
var timeline = new shaka.media.PresentationTimeline(300, 0);
timeline.setStatic(false);
manifest = new shaka.test.ManifestGenerator()
.setTimeline(timeline)
.addPeriod(0)
.build();
.setTimeline(timeline)
.addPeriod(0)
.addStreamSet('video').addStream(1)
.build();
goog.asserts.assert(manifest, 'manifest must be non-null');
var parser = new shaka.test.FakeManifestParser(manifest);
var factory = function() { return parser; };
Expand All @@ -1380,6 +1386,41 @@ describe('Player', function() {
}).then(done);
});

it('does not error on unknown contentTypes', function(done) {
manifest = new shaka.test.ManifestGenerator()
.addPeriod(0)
.addStreamSet('audio')
.addStream(1)
.addStreamSet('video')
.addStream(2)
.addStreamSet('application')
.addStream(3).mime('application/mp4', 'wvtt')
.build();
var parser = new shaka.test.FakeManifestParser(manifest);
var factory = function() { return parser; };

abrManager.chooseStreams.and.callFake(function(streamSetsByType) {
// Emulate the real AbrManager and ignore strange content types.
// Only return 'audio' and 'video', in this case.
return {
'audio': streamSetsByType['audio'].streams[0],
'video': streamSetsByType['video'].streams[0]
};
});

player.load('', 0, factory).catch(fail).then(function() {
// For this test to be valid, the 'application' stream set must not be
// filtered out as unsupported. There should be 3 sets:
expect(manifest.periods[0].streamSets.length).toBe(3);
// Now ask AbrManager to choose streams.
chooseStreams();
// Expect that streams were chosen and no error was dispatched.
expect(abrManager.chooseStreams).toHaveBeenCalled();
expect(onError).not.toHaveBeenCalled();
done();
});
});

/**
* Choose streams for the given period.
*
Expand Down

0 comments on commit 7dc98a7

Please sign in to comment.