From 6b4b7e26df973fe2ba62f5c02cebce977935f47e Mon Sep 17 00:00:00 2001 From: Alex Barstow Date: Mon, 18 Nov 2024 16:35:38 -0500 Subject: [PATCH] fix: issues with live playback timing (#1553) --- src/playlist-controller.js | 140 ++++++++++++--------- src/segment-loader.js | 1 + src/util/media-sequence-sync.js | 25 +++- test/playlist-controller.test.js | 170 ++++++++++++++------------ test/util/media-sequence-sync.test.js | 109 +++++++++++++++++ test/videojs-http-streaming.test.js | 5 +- 6 files changed, 307 insertions(+), 143 deletions(-) create mode 100644 test/util/media-sequence-sync.test.js diff --git a/src/playlist-controller.js b/src/playlist-controller.js index 6f4f97130..1c996e155 100644 --- a/src/playlist-controller.js +++ b/src/playlist-controller.js @@ -1642,9 +1642,73 @@ export class PlaylistController extends videojs.EventTarget { return this.seekable_; } - onSyncInfoUpdate_() { - let audioSeekable; + getSeekableRange_(playlistLoader, mediaType) { + const media = playlistLoader.media(); + + if (!media) { + return null; + } + + const mediaSequenceSync = this.syncController_.getMediaSequenceSync(mediaType); + + if (mediaSequenceSync && mediaSequenceSync.isReliable) { + const start = mediaSequenceSync.start; + const end = mediaSequenceSync.end; + + if (!isFinite(start) || !isFinite(end)) { + return null; + } + + const liveEdgeDelay = Vhs.Playlist.liveEdgeDelay(this.mainPlaylistLoader_.main, media); + + // Make sure our seekable end is not negative + const calculatedEnd = Math.max(0, end - liveEdgeDelay); + + if (calculatedEnd < start) { + return null; + } + + return createTimeRanges([[start, calculatedEnd]]); + } + + const expired = this.syncController_.getExpiredTime(media, this.duration()); + + if (expired === null) { + return null; + } + + const seekable = Vhs.Playlist.seekable( + media, + expired, + Vhs.Playlist.liveEdgeDelay(this.mainPlaylistLoader_.main, media) + ); + + return seekable.length ? seekable : null; + } + + computeFinalSeekable_(mainSeekable, audioSeekable) { + if (!audioSeekable) { + return mainSeekable; + } + + const mainStart = mainSeekable.start(0); + const mainEnd = mainSeekable.end(0); + const audioStart = audioSeekable.start(0); + const audioEnd = audioSeekable.end(0); + if (audioStart > mainEnd || mainStart > audioEnd) { + // Seekables are far apart, rely on main + return mainSeekable; + } + + // Return the overlapping seekable range + return createTimeRanges([[ + Math.max(mainStart, audioStart), + Math.min(mainEnd, audioEnd) + ]]); + } + + onSyncInfoUpdate_() { // TODO check for creation of both source buffers before updating seekable // // A fix was made to this function where a check for @@ -1668,87 +1732,45 @@ export class PlaylistController extends videojs.EventTarget { return; } - let media = this.mainPlaylistLoader_.media(); - - if (!media) { - return; - } - - let expired = this.syncController_.getExpiredTime(media, this.duration()); + const mainSeekable = this.getSeekableRange_(this.mainPlaylistLoader_, 'main'); - if (expired === null) { - // not enough information to update seekable + if (!mainSeekable) { return; } - const main = this.mainPlaylistLoader_.main; - const mainSeekable = Vhs.Playlist.seekable( - media, - expired, - Vhs.Playlist.liveEdgeDelay(main, media) - ); - - if (mainSeekable.length === 0) { - return; - } + let audioSeekable; if (this.mediaTypes_.AUDIO.activePlaylistLoader) { - media = this.mediaTypes_.AUDIO.activePlaylistLoader.media(); - expired = this.syncController_.getExpiredTime(media, this.duration()); - - if (expired === null) { - return; - } + audioSeekable = this.getSeekableRange_(this.mediaTypes_.AUDIO.activePlaylistLoader, 'audio'); - audioSeekable = Vhs.Playlist.seekable( - media, - expired, - Vhs.Playlist.liveEdgeDelay(main, media) - ); - - if (audioSeekable.length === 0) { + if (!audioSeekable) { return; } } - let oldEnd; - let oldStart; + const oldSeekable = this.seekable_; - if (this.seekable_ && this.seekable_.length) { - oldEnd = this.seekable_.end(0); - oldStart = this.seekable_.start(0); - } + this.seekable_ = this.computeFinalSeekable_(mainSeekable, audioSeekable); - if (!audioSeekable) { - // seekable has been calculated based on buffering video data so it - // can be returned directly - this.seekable_ = mainSeekable; - } else if (audioSeekable.start(0) > mainSeekable.end(0) || - mainSeekable.start(0) > audioSeekable.end(0)) { - // seekables are pretty far off, rely on main - this.seekable_ = mainSeekable; - } else { - this.seekable_ = createTimeRanges([[ - (audioSeekable.start(0) > mainSeekable.start(0)) ? audioSeekable.start(0) : - mainSeekable.start(0), - (audioSeekable.end(0) < mainSeekable.end(0)) ? audioSeekable.end(0) : - mainSeekable.end(0) - ]]); + if (!this.seekable_) { + return; } - // seekable is the same as last time - if (this.seekable_ && this.seekable_.length) { - if (this.seekable_.end(0) === oldEnd && this.seekable_.start(0) === oldStart) { + if (oldSeekable && oldSeekable.length && this.seekable_.length) { + if (oldSeekable.start(0) === this.seekable_.start(0) && + oldSeekable.end(0) === this.seekable_.end(0)) { + // Seekable range hasn't changed return; } } this.logger_(`seekable updated [${Ranges.printableRange(this.seekable_)}]`); + const metadata = { seekableRanges: this.seekable_ }; - this.trigger({type: 'seekablerangeschanged', metadata}); + this.trigger({ type: 'seekablerangeschanged', metadata }); this.tech_.trigger('seekablechanged'); } diff --git a/src/segment-loader.js b/src/segment-loader.js index 877209e78..d1339452a 100644 --- a/src/segment-loader.js +++ b/src/segment-loader.js @@ -1117,6 +1117,7 @@ export default class SegmentLoader extends videojs.EventTarget { if (!newPlaylist) { return; } + const oldPlaylist = this.playlist_; const segmentInfo = this.pendingSegment_; diff --git a/src/util/media-sequence-sync.js b/src/util/media-sequence-sync.js index 21aeb6eb9..5c9959276 100644 --- a/src/util/media-sequence-sync.js +++ b/src/util/media-sequence-sync.js @@ -132,7 +132,7 @@ export class MediaSequenceSync { return this.updateStorage_( segments, mediaSequence, - this.calculateBaseTime_(mediaSequence, currentTime) + this.calculateBaseTime_(mediaSequence, segments, currentTime) ); } @@ -228,7 +228,7 @@ export class MediaSequenceSync { this.diagnostics_ = newDiagnostics; } - calculateBaseTime_(mediaSequence, fallback) { + calculateBaseTime_(mediaSequence, segments, fallback) { if (!this.storage_.size) { // Initial setup flow. return 0; @@ -239,6 +239,23 @@ export class MediaSequenceSync { return this.storage_.get(mediaSequence).segmentSyncInfo.start; } + const minMediaSequenceFromStorage = Math.min(...this.storage_.keys()); + + // This case captures a race condition that can occur if we switch to a new media playlist that is out of date + // and still has an older Media Sequence. If this occurs, we extrapolate backwards to get the base time. + if (mediaSequence < minMediaSequenceFromStorage) { + const mediaSequenceDiff = minMediaSequenceFromStorage - mediaSequence; + let baseTime = this.storage_.get(minMediaSequenceFromStorage).segmentSyncInfo.start; + + for (let i = 0; i < mediaSequenceDiff; i++) { + const segment = segments[i]; + + baseTime -= segment.duration; + } + + return baseTime; + } + // Fallback flow. // There is a gap between last recorded playlist and a new one received. return fallback; @@ -256,7 +273,7 @@ export class DependantMediaSequenceSync extends MediaSequenceSync { this.parent_ = parent; } - calculateBaseTime_(mediaSequence, fallback) { + calculateBaseTime_(mediaSequence, segments, fallback) { if (!this.storage_.size) { const info = this.parent_.getSyncInfoForMediaSequence(mediaSequence); @@ -267,6 +284,6 @@ export class DependantMediaSequenceSync extends MediaSequenceSync { return 0; } - return super.calculateBaseTime_(mediaSequence, fallback); + return super.calculateBaseTime_(mediaSequence, segments, fallback); } } diff --git a/test/playlist-controller.test.js b/test/playlist-controller.test.js index 0f555d556..597e9eb54 100644 --- a/test/playlist-controller.test.js +++ b/test/playlist-controller.test.js @@ -2493,129 +2493,141 @@ QUnit.test( ); QUnit.test( - 'seekable uses the intersection of alternate audio and combined tracks', + 'seekable uses the intersection of alternate audio and combined tracks with MediaSequenceSync', function(assert) { - const origSeekable = Playlist.seekable; const pc = this.playlistController; const mainMedia = {}; const audioMedia = {}; - let mainTimeRanges = []; - let audioTimeRanges = []; + // mock mainPlaylistLoader_ and media this.playlistController.mainPlaylistLoader_.main = {}; this.playlistController.mainPlaylistLoader_.media = () => mainMedia; - this.playlistController.syncController_.getExpiredTime = () => 0; - Playlist.seekable = (media) => { - if (media === mainMedia) { - return createTimeRanges(mainTimeRanges); + // mock SyncController and MediaSequenceSync instances + const mainMediaSequenceSync = { + isReliable: true, + start: 0, + end: 10 + }; + + const audioMediaSequenceSync = { + isReliable: true, + start: 0, + end: 10 + }; + + this.playlistController.syncController_.getMediaSequenceSync = (type) => { + if (type === 'main') { + return mainMediaSequenceSync; + } + + if (type === 'audio') { + return audioMediaSequenceSync; } - return createTimeRanges(audioTimeRanges); + + return null; }; - timeRangesEqual(pc.seekable(), createTimeRanges(), 'empty when main empty'); - mainTimeRanges = [[0, 10]]; + // helper function to set the start and end for main and audio + const setSyncInfo = (mainStart, mainEnd, audioStart, audioEnd) => { + mainMediaSequenceSync.start = mainStart; + mainMediaSequenceSync.end = mainEnd; + audioMediaSequenceSync.start = audioStart; + audioMediaSequenceSync.end = audioEnd; + }; + + // Test cases + // No audio loader, only main + pc.mediaTypes_.AUDIO.activePlaylistLoader = null; + setSyncInfo(0, 10); pc.seekable_ = createTimeRanges(); pc.onSyncInfoUpdate_(); timeRangesEqual(pc.seekable(), createTimeRanges([[0, 10]]), 'main when no audio'); + // Both main and audio have the same range pc.mediaTypes_.AUDIO.activePlaylistLoader = { media: () => audioMedia, - dispose() {}, - expired_: 0 + dispose() { } }; - mainTimeRanges = []; - pc.seekable_ = createTimeRanges(); - pc.onSyncInfoUpdate_(); - - timeRangesEqual(pc.seekable(), createTimeRanges(), 'empty when both empty'); - mainTimeRanges = [[0, 10]]; - pc.seekable_ = createTimeRanges(); - pc.onSyncInfoUpdate_(); - timeRangesEqual(pc.seekable(), createTimeRanges(), 'empty when audio empty'); - mainTimeRanges = []; - audioTimeRanges = [[0, 10]]; - pc.seekable_ = createTimeRanges(); - pc.onSyncInfoUpdate_(); - timeRangesEqual(pc.seekable(), createTimeRanges(), 'empty when main empty'); - mainTimeRanges = [[0, 10]]; - audioTimeRanges = [[0, 10]]; + setSyncInfo(0, 10, 0, 10); pc.seekable_ = createTimeRanges(); pc.onSyncInfoUpdate_(); timeRangesEqual(pc.seekable(), createTimeRanges([[0, 10]]), 'ranges equal'); - mainTimeRanges = [[5, 10]]; + + // Main starts later than audio + setSyncInfo(5, 10, 0, 10); pc.seekable_ = createTimeRanges(); pc.onSyncInfoUpdate_(); timeRangesEqual(pc.seekable(), createTimeRanges([[5, 10]]), 'main later start'); - mainTimeRanges = [[0, 10]]; - audioTimeRanges = [[5, 10]]; + + // Audio starts later than main + setSyncInfo(0, 10, 5, 10); pc.seekable_ = createTimeRanges(); pc.onSyncInfoUpdate_(); timeRangesEqual(pc.seekable(), createTimeRanges([[5, 10]]), 'audio later start'); - mainTimeRanges = [[0, 9]]; - audioTimeRanges = [[0, 10]]; + + // Main ends earlier than audio + setSyncInfo(0, 9, 0, 10); pc.seekable_ = createTimeRanges(); pc.onSyncInfoUpdate_(); timeRangesEqual(pc.seekable(), createTimeRanges([[0, 9]]), 'main earlier end'); - mainTimeRanges = [[0, 10]]; - audioTimeRanges = [[0, 9]]; + + // Audio ends earlier than main + setSyncInfo(0, 10, 0, 9); pc.seekable_ = createTimeRanges(); pc.onSyncInfoUpdate_(); timeRangesEqual(pc.seekable(), createTimeRanges([[0, 9]]), 'audio earlier end'); - mainTimeRanges = [[1, 10]]; - audioTimeRanges = [[0, 9]]; - pc.seekable_ = createTimeRanges(); - pc.onSyncInfoUpdate_(); - timeRangesEqual( - pc.seekable(), - createTimeRanges([[1, 9]]), - 'main later start, audio earlier end' - ); - mainTimeRanges = [[0, 9]]; - audioTimeRanges = [[1, 10]]; + + // Main starts and ends within audio range + setSyncInfo(1, 9, 0, 10); pc.seekable_ = createTimeRanges(); pc.onSyncInfoUpdate_(); - timeRangesEqual( - pc.seekable(), - createTimeRanges([[1, 9]]), - 'audio later start, main earlier end' - ); - mainTimeRanges = [[2, 9]]; + timeRangesEqual(pc.seekable(), createTimeRanges([[1, 9]]), 'main within audio'); + + // Audio starts and ends within main range + setSyncInfo(0, 10, 1, 9); pc.seekable_ = createTimeRanges(); pc.onSyncInfoUpdate_(); - timeRangesEqual( - pc.seekable(), - createTimeRanges([[2, 9]]), - 'main later start, main earlier end' - ); - mainTimeRanges = [[1, 10]]; - audioTimeRanges = [[2, 9]]; + timeRangesEqual(pc.seekable(), createTimeRanges([[1, 9]]), 'audio within main'); + + // No intersection, audio later than main + setSyncInfo(1, 10, 11, 20); pc.seekable_ = createTimeRanges(); pc.onSyncInfoUpdate_(); - timeRangesEqual( - pc.seekable(), - createTimeRanges([[2, 9]]), - 'audio later start, audio earlier end' - ); - mainTimeRanges = [[1, 10]]; - audioTimeRanges = [[11, 20]]; + // Should default to main seekable + timeRangesEqual(pc.seekable(), createTimeRanges([[1, 10]]), 'no intersection, audio later'); + + // No intersection, main later than audio + setSyncInfo(11, 20, 1, 10); pc.seekable_ = createTimeRanges(); pc.onSyncInfoUpdate_(); - timeRangesEqual( - pc.seekable(), - createTimeRanges([[1, 10]]), - 'no intersection, audio later' - ); - mainTimeRanges = [[11, 20]]; - audioTimeRanges = [[1, 10]]; + // Should default to main seekable + timeRangesEqual(pc.seekable(), createTimeRanges([[11, 20]]), 'no intersection, main later'); + + // MediaSequenceSync not reliable, fallback to expired time seekable calculation + mainMediaSequenceSync.isReliable = false; + audioMediaSequenceSync.isReliable = false; + + // Mock getExpiredTime and Playlist.seekable + this.playlistController.syncController_.getExpiredTime = (media) => 0; + + const origSeekable = Playlist.seekable; + + Playlist.seekable = (media) => { + if (media === mainMedia) { + return createTimeRanges([[0, 10]]); + } + if (media === audioMedia) { + return createTimeRanges([[0, 10]]); + } + return createTimeRanges(); + }; + pc.seekable_ = createTimeRanges(); pc.onSyncInfoUpdate_(); - timeRangesEqual( - pc.seekable(), - createTimeRanges([[11, 20]]), - 'no intersection, main later' - ); + timeRangesEqual(pc.seekable(), createTimeRanges([[0, 10]]), 'fallback to expired time seekable calculation'); + // Restore original Playlist.seekable Playlist.seekable = origSeekable; } ); diff --git a/test/util/media-sequence-sync.test.js b/test/util/media-sequence-sync.test.js new file mode 100644 index 000000000..5e5354eca --- /dev/null +++ b/test/util/media-sequence-sync.test.js @@ -0,0 +1,109 @@ +import QUnit from 'qunit'; +import { MediaSequenceSync } from '../../src/util/media-sequence-sync'; + +QUnit.module('MediaSequenceSync: update', function(hooks) { + let mediaSequenceSync; + + hooks.beforeEach(function() { + mediaSequenceSync = new MediaSequenceSync(); + }); + + QUnit.test('update calculates correct base time based on mediaSequence of new playlist', function(assert) { + const initialMediaSequence = 10; + const initialSegments = [ + // Segment 10 with duration 5 + { duration: 5 }, + // Segment 11 with duration 6 + { duration: 6 }, + // Segment 12 with duration 7 + { duration: 7 } + ]; + + // Initial update with starting playlist + mediaSequenceSync.update( + { + mediaSequence: initialMediaSequence, + segments: initialSegments + }, + // Current time, value is used for fallback and not significant here + 20 + ); + + // Confirm that the initial update set the correct start and end times + assert.strictEqual( + mediaSequenceSync.start, + 0, + 'The start time is set to the initial value of 0.' + ); + + // Confirm the end time is the correct sum of the segment durations + // = 18 + const expectedInitialEndTime = 0 + 5 + 6 + 7; + + assert.strictEqual( + mediaSequenceSync.end, + expectedInitialEndTime, + 'The end time is calculated correctly after the initial update.' + ); + + // New playlist with higher mediaSequence + let newMediaSequence = 11; + let newSegments = [ + // Segment 11 with duration 4 + { duration: 4 }, + // Segment 12 with duration 5 + { duration: 5 }, + // Segment 13 with duration 6 + { duration: 6 } + ]; + + // Update with the new playlist + mediaSequenceSync.update( + { + mediaSequence: newMediaSequence, + segments: newSegments + }, + 30 + ); + + // Segment 10 with duration 5 has fallen off the start of the playlist + let expectedStartTime = 5; + + assert.strictEqual( + mediaSequenceSync.start, + expectedStartTime, + 'The base time is calculated correctly when a new playlist with a higher mediaSequence is loaded.' + ); + + // New playlist with lower mediaSequence + newMediaSequence = 10; + newSegments = [ + // Segment 10 with duration 5 + { duration: 5 }, + // Segment 11 with duration 6 + { duration: 6 }, + // Segment 12 with duration 7 + { duration: 7 } + ]; + + // Update with the new playlist + mediaSequenceSync.update( + { + mediaSequence: newMediaSequence, + segments: newSegments + }, + 40 + ); + + // Expected base time is calculated by extrapolating backwards: + // Segment 11 start time: 5 + // Segment 10 start time: Segment 11 start time (5) - Segment 10 duration (5) = 0 + expectedStartTime = 0; + + assert.strictEqual( + mediaSequenceSync.start, + expectedStartTime, + 'The base time is calculated correctly when a new playlist with a lower mediaSequence is loaded.' + ); + }); +}); diff --git a/test/videojs-http-streaming.test.js b/test/videojs-http-streaming.test.js index 027ca75b1..5fea187cc 100644 --- a/test/videojs-http-streaming.test.js +++ b/test/videojs-http-streaming.test.js @@ -2488,7 +2488,7 @@ QUnit.test('live playlist starts with correct currentTime value', function(asser }); QUnit.test( - 'estimates seekable ranges for live streams that have been paused for a long time', + 'estimates seekable ranges for live streams that have been paused for a long time and unreliable MediaSequenceSync', function(assert) { this.player.src({ src: 'http://example.com/manifest/liveStart30sBefore.m3u8', @@ -2505,6 +2505,9 @@ QUnit.test( mediaSequence: 130, time: 80 }; + this.player.tech_.vhs.playlistController_.syncController_.getMediaSequenceSync = () => { + return { isReliable: false }; + }; this.player.tech_.vhs.playlistController_.onSyncInfoUpdate_(); assert.equal( this.player.seekable().start(0),