diff --git a/JitsiConference.js b/JitsiConference.js index 4592ac5d29..7af40982a7 100644 --- a/JitsiConference.js +++ b/JitsiConference.js @@ -11,7 +11,6 @@ import JitsiTrackError from './JitsiTrackError'; import * as JitsiTrackErrors from './JitsiTrackErrors'; import * as JitsiTrackEvents from './JitsiTrackEvents'; import authenticateAndUpgradeRole from './authenticateAndUpgradeRole'; -import { CodecSelection } from './modules/RTC/CodecSelection'; import RTC from './modules/RTC/RTC'; import { SS_DEFAULT_FRAME_RATE } from './modules/RTC/ScreenObtainer'; import browser from './modules/browser'; @@ -28,8 +27,7 @@ import E2ePing from './modules/e2eping/e2eping'; import Jvb121EventGenerator from './modules/event/Jvb121EventGenerator'; import FeatureFlags from './modules/flags/FeatureFlags'; import { LiteModeContext } from './modules/litemode/LiteModeContext'; -import ReceiveVideoController from './modules/qualitycontrol/ReceiveVideoController'; -import SendVideoController from './modules/qualitycontrol/SendVideoController'; +import QualityController from './modules/qualitycontrol/QualityController'; import RecordingManager from './modules/recording/RecordingManager'; import Settings from './modules/settings/Settings'; import AudioOutputProblemDetector from './modules/statistics/AudioOutputProblemDetector'; @@ -383,31 +381,6 @@ JitsiConference.prototype._init = function(options = {}) { const { config } = this.options; - // Get the codec preference settings from config.js. - const codecSettings = { - jvb: { - preferenceOrder: browser.isMobileDevice() && config.videoQuality?.mobileCodecPreferenceOrder - ? config.videoQuality.mobileCodecPreferenceOrder - : config.videoQuality?.codecPreferenceOrder, - disabledCodec: _getCodecMimeType(config.videoQuality?.disabledCodec), - preferredCodec: _getCodecMimeType(config.videoQuality?.preferredCodec), - screenshareCodec: browser.isMobileDevice() - ? _getCodecMimeType(config.videoQuality?.mobileScreenshareCodec) - : _getCodecMimeType(config.videoQuality?.screenshareCodec) - }, - p2p: { - preferenceOrder: browser.isMobileDevice() && config.p2p?.mobileCodecPreferenceOrder - ? config.p2p.mobileCodecPreferenceOrder - : config.p2p?.codecPreferenceOrder, - disabledCodec: _getCodecMimeType(config.p2p?.disabledCodec), - preferredCodec: _getCodecMimeType(config.p2p?.preferredCodec), - screenshareCodec: browser.isMobileDevice() - ? _getCodecMimeType(config.p2p?.mobileScreenshareCodec) - : _getCodecMimeType(config.p2p?.screenshareCodec) - } - }; - - this.codecSelection = new CodecSelection(this, codecSettings); this._statsCurrentId = config.statisticsId ? config.statisticsId : Settings.callStatsUserName; this.room = this.xmpp.createRoom( this.options.name, { @@ -475,8 +448,34 @@ JitsiConference.prototype._init = function(options = {}) { this._registerRtcListeners(this.rtc); } - this.receiveVideoController = new ReceiveVideoController(this, this.rtc); - this.sendVideoController = new SendVideoController(this, this.rtc); + // Get the codec preference settings from config.js. + const codecSettings = { + jvb: { + preferenceOrder: browser.isMobileDevice() && config.videoQuality?.mobileCodecPreferenceOrder + ? config.videoQuality.mobileCodecPreferenceOrder + : config.videoQuality?.codecPreferenceOrder, + disabledCodec: _getCodecMimeType(config.videoQuality?.disabledCodec), + preferredCodec: _getCodecMimeType(config.videoQuality?.preferredCodec), + screenshareCodec: browser.isMobileDevice() + ? _getCodecMimeType(config.videoQuality?.mobileScreenshareCodec) + : _getCodecMimeType(config.videoQuality?.screenshareCodec) + }, + p2p: { + preferenceOrder: browser.isMobileDevice() && config.p2p?.mobileCodecPreferenceOrder + ? config.p2p.mobileCodecPreferenceOrder + : config.p2p?.codecPreferenceOrder, + disabledCodec: _getCodecMimeType(config.p2p?.disabledCodec), + preferredCodec: _getCodecMimeType(config.p2p?.preferredCodec), + screenshareCodec: browser.isMobileDevice() + ? _getCodecMimeType(config.p2p?.mobileScreenshareCodec) + : _getCodecMimeType(config.p2p?.screenshareCodec) + } + }; + + this.qualityController = new QualityController( + this, + codecSettings, + config.videoQuality?.enableAdaptiveMode); if (!this.statistics) { this.statistics = new Statistics(this, { @@ -574,7 +573,7 @@ JitsiConference.prototype._init = function(options = {}) { } // Publish the codec preference to presence. - this.setLocalParticipantProperty('codecList', this.codecSelection.getCodecPreferenceList('jvb')); + this.setLocalParticipantProperty('codecList', this.qualityController.codecController.getCodecPreferenceList('jvb')); // Set transcription language presence extension. // In case the language config is undefined or has the default value that the transcriber uses @@ -1576,7 +1575,7 @@ JitsiConference.prototype.unlock = function() { * @returns {number} */ JitsiConference.prototype.getLastN = function() { - return this.receiveVideoController.getLastN(); + return this.qualityController.receiveVideoController.getLastN(); }; /** @@ -1604,7 +1603,7 @@ JitsiConference.prototype.setLastN = function(lastN) { if (n < -1) { throw new RangeError('lastN cannot be smaller than -1'); } - this.receiveVideoController.setLastN(n); + this.qualityController.receiveVideoController.setLastN(n); // If the P2P session is not fully established yet, we wait until it gets established. if (this.p2pJingleSession) { @@ -2232,8 +2231,8 @@ JitsiConference.prototype._acceptJvbIncomingCall = function(jingleSession, jingl ...this.options.config, codecSettings: { mediaType: MediaType.VIDEO, - codecList: this.codecSelection.getCodecPreferenceList('jvb'), - screenshareCodec: this.codecSelection.getScreenshareCodec('jvb') + codecList: this.qualityController.codecController.getCodecPreferenceList('jvb'), + screenshareCodec: this.qualityController.codecController.getScreenshareCodec('jvb') }, enableInsertableStreams: this.isE2EEEnabled() || FeatureFlags.isRunInLiteModeEnabled() }); @@ -2943,8 +2942,8 @@ JitsiConference.prototype._acceptP2PIncomingCall = function(jingleSession, jingl ...this.options.config, codecSettings: { mediaType: MediaType.VIDEO, - codecList: this.codecSelection.getCodecPreferenceList('p2p'), - screenshareCodec: this.codecSelection.getScreenshareCodec('p2p') + codecList: this.qualityController.codecController.getCodecPreferenceList('p2p'), + screenshareCodec: this.qualityController.codecController.getScreenshareCodec('p2p') }, enableInsertableStreams: this.isE2EEEnabled() || FeatureFlags.isRunInLiteModeEnabled() }); @@ -3299,8 +3298,8 @@ JitsiConference.prototype._startP2PSession = function(remoteJid) { ...this.options.config, codecSettings: { mediaType: MediaType.VIDEO, - codecList: this.codecSelection.getCodecPreferenceList('p2p'), - screenshareCodec: this.codecSelection.getScreenshareCodec('p2p') + codecList: this.qualityController.codecController.getCodecPreferenceList('p2p'), + screenshareCodec: this.qualityController.codecController.getScreenshareCodec('p2p') }, enableInsertableStreams: this.isE2EEEnabled() || FeatureFlags.isRunInLiteModeEnabled() }); @@ -3702,7 +3701,7 @@ JitsiConference.prototype.sendFaceLandmarks = function(payload) { * Where A, B and C are source-names of the remote tracks that are being requested from the bridge. */ JitsiConference.prototype.setReceiverConstraints = function(videoConstraints) { - this.receiveVideoController.setReceiverConstraints(videoConstraints); + this.qualityController.receiveVideoController.setReceiverConstraints(videoConstraints); }; /** @@ -3711,7 +3710,7 @@ JitsiConference.prototype.setReceiverConstraints = function(videoConstraints) { * @param {Number} assumedBandwidthBps - The bandwidth value expressed in bits per second. */ JitsiConference.prototype.setAssumedBandwidthBps = function(assumedBandwidthBps) { - this.receiveVideoController.setAssumedBandwidthBps(assumedBandwidthBps); + this.qualityController.receiveVideoController.setAssumedBandwidthBps(assumedBandwidthBps); }; /** @@ -3723,7 +3722,7 @@ JitsiConference.prototype.setAssumedBandwidthBps = function(assumedBandwidthBps) * @returns {void} */ JitsiConference.prototype.setReceiverVideoConstraint = function(maxFrameHeight) { - this.receiveVideoController.setPreferredReceiveMaxFrameHeight(maxFrameHeight); + this.qualityController.receiveVideoController.setPreferredReceiveMaxFrameHeight(maxFrameHeight); }; /** @@ -3734,7 +3733,7 @@ JitsiConference.prototype.setReceiverVideoConstraint = function(maxFrameHeight) * successful and rejected otherwise. */ JitsiConference.prototype.setSenderVideoConstraint = function(maxFrameHeight) { - return this.sendVideoController.setPreferredSendMaxFrameHeight(maxFrameHeight); + return this.qualityController.sendVideoController.setPreferredSendMaxFrameHeight(maxFrameHeight); }; /** diff --git a/modules/RTC/MockClasses.js b/modules/RTC/MockClasses.js index b69ef1c9e4..374045fb4c 100644 --- a/modules/RTC/MockClasses.js +++ b/modules/RTC/MockClasses.js @@ -1,5 +1,7 @@ import transform from 'sdp-transform'; +import Listenable from '../util/Listenable'; + /* eslint-disable no-empty-function */ /* eslint-disable max-len */ @@ -230,7 +232,7 @@ export class MockPeerConnection { /** * Mock {@link RTC} - add things as needed, but only things useful for all tests. */ -export class MockRTC { +export class MockRTC extends Listenable { /** * {@link RTC.createPeerConnection}. * diff --git a/modules/RTC/TPCUtils.js b/modules/RTC/TPCUtils.js index 4cfd4f8012..e5994f517f 100644 --- a/modules/RTC/TPCUtils.js +++ b/modules/RTC/TPCUtils.js @@ -192,7 +192,7 @@ export class TPCUtils { if (localTrack.isAudioTrack()) { return [ { active: this.pc.audioTransferActive } ]; } - const codec = this.pc.getConfiguredVideoCodec(); + const codec = this.pc.getConfiguredVideoCodec(localTrack); if (this.pc.isSpatialScalabilityOn()) { return this._getVideoStreamEncodings(localTrack, codec); diff --git a/modules/RTC/TraceablePeerConnection.js b/modules/RTC/TraceablePeerConnection.js index d5e924251b..ad2eed4082 100644 --- a/modules/RTC/TraceablePeerConnection.js +++ b/modules/RTC/TraceablePeerConnection.js @@ -648,10 +648,12 @@ TraceablePeerConnection.prototype.getAudioLevels = function(speakerList = []) { /** * Checks if the browser is currently doing true simulcast where in three different media streams are being sent to the * bridge. Currently this happens always for VP8 and only if simulcast is enabled for VP9/AV1/H264. + * + * @param {JitsiLocalTrack} localTrack - The local video track. * @returns {boolean} */ -TraceablePeerConnection.prototype.doesTrueSimulcast = function() { - const currentCodec = this.getConfiguredVideoCodec(); +TraceablePeerConnection.prototype.doesTrueSimulcast = function(localTrack) { + const currentCodec = this.getConfiguredVideoCodec(localTrack); return this.isSpatialScalabilityOn() && this.tpcUtils.isRunningInSimulcastMode(currentCodec); }; @@ -804,10 +806,11 @@ TraceablePeerConnection.prototype.getRemoteSourceInfoByParticipant = function(id /** * Returns the target bitrates configured for the local video source. * + * @param {JitsiLocalTrack} - The local video track. * @returns {Object} */ -TraceablePeerConnection.prototype.getTargetVideoBitrates = function() { - const currentCodec = this.getConfiguredVideoCodec(); +TraceablePeerConnection.prototype.getTargetVideoBitrates = function(localTrack) { + const currentCodec = this.getConfiguredVideoCodec(localTrack); return this.tpcUtils.codecSettings[currentCodec].maxBitratesVideo; }; @@ -1596,17 +1599,17 @@ TraceablePeerConnection.prototype._assertTrackBelongs = function( }; /** - * Returns the codec that is configured on the client as the preferred video codec. - * This takes into account the current order of codecs in the local description sdp. + * Returns the codec that is configured on the client as the preferred video codec for the given local video track. + * + * @param {JitsiLocalTrack} localTrack - The local video track. + * @returns {CodecMimeType} The codec that is set as the preferred codec for the given local video track. * - * @returns {CodecMimeType} The codec that is set as the preferred codec to receive - * video in the local SDP. */ -TraceablePeerConnection.prototype.getConfiguredVideoCodec = function() { - const localVideoTrack = this.getLocalVideoTracks()[0]; +TraceablePeerConnection.prototype.getConfiguredVideoCodec = function(localTrack) { + const localVideoTrack = localTrack ?? this.getLocalVideoTracks()[0]; + const rtpSender = this.findSenderForTrack(localVideoTrack.getTrack()); - if (this.usesCodecSelectionAPI() && localVideoTrack) { - const rtpSender = this.findSenderForTrack(localVideoTrack.getTrack()); + if (this.usesCodecSelectionAPI() && rtpSender) { const { codecs } = rtpSender.getParameters(); return codecs[0].mimeType.split('/')[1].toLowerCase(); @@ -1619,7 +1622,8 @@ TraceablePeerConnection.prototype.getConfiguredVideoCodec = function() { return defaultCodec; } const parsedSdp = transform.parse(sdp); - const mLine = parsedSdp.media.find(m => m.type === MediaType.VIDEO); + const mLine = parsedSdp.media + .find(m => m.mid.toString() === this._localTrackTransceiverMids.get(localVideoTrack.rtcId)); const payload = mLine.payloads.split(' ')[0]; const { codec } = mLine.rtp.find(rtp => rtp.payload === Number(payload)); @@ -1680,18 +1684,21 @@ TraceablePeerConnection.prototype.setDesktopSharingFrameRate = function(maxFps) /** * Sets the codec preference on the peerconnection. The codec preference goes into effect when - * the next renegotiation happens. + * the next renegotiation happens for older clients that do not support the codec selection API. * - * @param {CodecMimeType} preferredCodec the preferred codec. - * @param {CodecMimeType} disabledCodec the codec that needs to be disabled. + * @param {Array} codecList - Preferred codecs for video. + * @param {CodecMimeType} screenshareCodec - The preferred codec for screenshare. * @returns {void} */ -TraceablePeerConnection.prototype.setVideoCodecs = function(codecList) { +TraceablePeerConnection.prototype.setVideoCodecs = function(codecList, screenshareCodec) { if (!this.codecSettings || !codecList?.length) { return; } this.codecSettings.codecList = codecList; + if (screenshareCodec) { + this.codecSettings.screenshareCodec = screenshareCodec; + } if (this.usesCodecSelectionAPI()) { this.configureVideoSenderEncodings(); @@ -1755,9 +1762,7 @@ TraceablePeerConnection.prototype.findReceiverForTrack = function(track) { * was found. */ TraceablePeerConnection.prototype.findSenderForTrack = function(track) { - if (this.peerconnection.getSenders) { - return this.peerconnection.getSenders().find(s => s.track === track); - } + return this.peerconnection.getSenders().find(s => s.track === track); }; /** diff --git a/modules/RTC/CodecSelection.js b/modules/qualitycontrol/CodecSelection.js similarity index 62% rename from modules/RTC/CodecSelection.js rename to modules/qualitycontrol/CodecSelection.js index afe275feda..5d09c09b32 100644 --- a/modules/RTC/CodecSelection.js +++ b/modules/qualitycontrol/CodecSelection.js @@ -1,9 +1,10 @@ import { getLogger } from '@jitsi/logger'; -import * as JitsiConferenceEvents from '../../JitsiConferenceEvents'; import { CodecMimeType } from '../../service/RTC/CodecMimeType'; import { MediaType } from '../../service/RTC/MediaType'; +import { VIDEO_CODECS_BY_COMPLEXITY } from '../../service/RTC/StandardVideoSettings'; +import { VideoType } from '../../service/RTC/VideoType'; import browser from '../browser'; const logger = getLogger(__filename); @@ -93,26 +94,10 @@ export class CodecSelection { this.codecPreferenceOrder[connectionType] = selectedOrder; // Set the preferred screenshare codec. - if (screenshareCodec && supportedCodecs.has(screenshareCodec)) { - this.screenshareCodec[connectionType] = screenshareCodec; + if (screenshareCodec && supportedCodecs.has(screenshareCodec.toLowerCase())) { + this.screenshareCodec[connectionType] = screenshareCodec.toLowerCase(); } } - - this.conference.on( - JitsiConferenceEvents._MEDIA_SESSION_STARTED, - session => this._selectPreferredCodec(session)); - this.conference.on( - JitsiConferenceEvents.CONFERENCE_VISITOR_CODECS_CHANGED, - codecList => this._updateVisitorCodecs(codecList)); - this.conference.on( - JitsiConferenceEvents.USER_JOINED, - () => this._selectPreferredCodec()); - this.conference.on( - JitsiConferenceEvents.USER_LEFT, - () => this._selectPreferredCodec()); - this.conference.on( - JitsiConferenceEvents.ENCODE_TIME_STATS_RECEIVED, - (tpc, stats) => this._processEncodeTimeStats(tpc, stats)); } /** @@ -138,81 +123,23 @@ export class CodecSelection { } /** - * Processes the encode time stats received for all the local video sources. + * Returns the current codec preference order for the given connection type. * - * @param {TraceablePeerConnection} tpc - the peerconnection for which stats were gathered. - * @param {Object} stats - the encode time stats for local video sources. - * @returns {void} + * @param {String} connectionType The media connection type, 'p2p' or 'jvb'. + * @returns {Array} */ - _processEncodeTimeStats(tpc, stats) { - const activeSession = this.conference.getActiveMediaSession(); - - // Process stats only for the active media session. - if (activeSession.peerconnection !== tpc) { - return; - } - - const statsPerTrack = new Map(); - - for (const ssrc of stats.keys()) { - const { codec, encodeTime, qualityLimitationReason, resolution, timestamp } = stats.get(ssrc); - const track = tpc.getTrackBySSRC(ssrc); - let existingStats = statsPerTrack.get(track.rtcId); - const encodeResolution = Math.min(resolution.height, resolution.width); - const ssrcStats = { - encodeResolution, - encodeTime, - qualityLimitationReason - }; - - if (existingStats) { - existingStats.codec = codec; - existingStats.timestamp = timestamp; - existingStats.trackStats.push(ssrcStats); - } else { - existingStats = { - codec, - timestamp, - trackStats: [ ssrcStats ] - }; - - statsPerTrack.set(track.rtcId, existingStats); - } - } - - // Aggregate the stats for multiple simulcast streams with different SSRCs but for the same video stream. - for (const trackId of statsPerTrack.keys()) { - const { codec, timestamp, trackStats } = statsPerTrack.get(trackId); - const totalEncodeTime = trackStats - .map(stat => stat.encodeTime) - .reduce((totalValue, currentValue) => totalValue + currentValue, 0); - const avgEncodeTime = totalEncodeTime / trackStats.length; - const { qualityLimitationReason = 'none' } - = trackStats.find(stat => stat.qualityLimitationReason !== 'none') ?? {}; - const encodeResolution = trackStats - .map(stat => stat.encodeResolution) - .reduce((resolution, currentValue) => Math.max(resolution, currentValue), 0); - const localTrack = this.conference.getLocalVideoTracks().find(t => t.rtcId === trackId); - - const exisitingStats = this.encodeTimeStats.get(trackId); - const sourceStats = { - avgEncodeTime, - codec, - encodeResolution, - qualityLimitationReason, - localTrack, - timestamp - }; - - if (exisitingStats) { - exisitingStats.push(sourceStats); - } else { - this.encodeTimeStats.set(trackId, [ sourceStats ]); - } + getCodecPreferenceList(connectionType) { + return this.codecPreferenceOrder[connectionType]; + } - logger.debug(`Encode stats for ${localTrack}: codec=${codec}, time=${avgEncodeTime},` - + `resolution=${encodeResolution}, qualityLimitationReason=${qualityLimitationReason}`); - } + /** + * Returns the preferred screenshare codec for the given connection type. + * + * @param {String} connectionType The media connection type, 'p2p' or 'jvb'. + * @returns CodecMimeType + */ + getScreenshareCodec(connectionType) { + return this.screenshareCodec[connectionType]; } /** @@ -221,17 +148,17 @@ export class CodecSelection { * * @param {JingleSessionPC} mediaSession session for which the codec selection has to be made. */ - _selectPreferredCodec(mediaSession) { + selectPreferredCodec(mediaSession) { const session = mediaSession ? mediaSession : this.conference.jvbJingleSession; if (!session) { return; } - const isJvbSession = session === this.conference.jvbJingleSession; - let localPreferredCodecOrder = isJvbSession ? this.codecPreferenceOrder.jvb : this.codecPreferenceOrder.p2p; + + let localPreferredCodecOrder = this.codecPreferenceOrder.jvb; // E2EE is curently supported only for VP8 codec. - if (this.conference.isE2EEEnabled() && isJvbSession) { + if (this.conference.isE2EEEnabled()) { localPreferredCodecOrder = [ CodecMimeType.VP8 ]; } @@ -254,17 +181,13 @@ export class CodecSelection { const selectedCodecOrder = localPreferredCodecOrder.reduce((acc, localCodec) => { let codecNotSupportedByRemote = false; - // Ignore remote codecs for p2p since only the JVB codec preferences are published in presence. - // For p2p, we rely on the codec order present in the remote offer/answer. - if (!session.isP2P) { - // Remove any codecs that are not supported by any of the remote endpoints. The order of the supported - // codecs locally however will remain the same since we want to support asymmetric codecs. - for (const remoteCodecs of remoteCodecsPerParticipant) { - // Ignore remote participants that do not publish codec preference in presence (transcriber). - if (remoteCodecs.length) { - codecNotSupportedByRemote = codecNotSupportedByRemote - || !remoteCodecs.find(participantCodec => participantCodec === localCodec); - } + // Remove any codecs that are not supported by any of the remote endpoints. The order of the supported + // codecs locally however will remain the same since we want to support asymmetric codecs. + for (const remoteCodecs of remoteCodecsPerParticipant) { + // Ignore remote participants that do not publish codec preference in presence (transcriber). + if (remoteCodecs.length) { + codecNotSupportedByRemote = codecNotSupportedByRemote + || !remoteCodecs.find(participantCodec => participantCodec === localCodec); } } @@ -281,7 +204,46 @@ export class CodecSelection { return; } - session.setVideoCodecs(selectedCodecOrder); + session.setVideoCodecs(selectedCodecOrder, this.screenshareCodec?.jvb); + } + + /** + * Changes the codec preference order. + * + * @param {JitsiLocalTrack} localTrack - The local video track. + * @param {CodecMimeType} codec - The codec used for encoding the given local video track. + * @returns boolean - Returns true if the codec order has been updated, false otherwise. + */ + changeCodecPreferenceOrder(localTrack, codec) { + const session = this.conference.getActiveMediaSession(); + const connectionType = session.isP2P ? 'p2p' : 'jvb'; + const codecOrder = this.codecPreferenceOrder[connectionType]; + const videoType = localTrack.getVideoType(); + const codecsByVideoType = VIDEO_CODECS_BY_COMPLEXITY[videoType] + .filter(val => Boolean(codecOrder.find(supportedCodec => supportedCodec === val))); + const codecIndex = codecsByVideoType.findIndex(val => val === codec.toLowerCase()); + + // Do nothing if we are using the lowest complexity codec already. + if (codecIndex === codecsByVideoType.length - 1) { + return false; + } + + const newCodec = codecsByVideoType[codecIndex + 1]; + + if (videoType === VideoType.CAMERA) { + const idx = codecOrder.findIndex(val => val === newCodec); + + codecOrder.splice(idx, 1); + codecOrder.unshift(newCodec); + logger.info(`QualityController - switching camera codec to ${newCodec} because of cpu restriction`); + } else { + this.screenshareCodec[connectionType] = newCodec; + logger.info(`QualityController - switching screenshare codec to ${newCodec} because of cpu restriction`); + } + + this.selectPreferredCodec(session); + + return true; } /** @@ -290,32 +252,12 @@ export class CodecSelection { * @param {Array} codecList - visitor codecs. * @returns {void} */ - _updateVisitorCodecs(codecList) { + updateVisitorCodecs(codecList) { if (this.visitorCodecs === codecList) { return; } this.visitorCodecs = codecList; - this._selectPreferredCodec(); - } - - /** - * Returns the current codec preference order for the given connection type. - * - * @param {String} connectionType The media connection type, 'p2p' or 'jvb'. - * @returns {Array} - */ - getCodecPreferenceList(connectionType) { - return this.codecPreferenceOrder[connectionType]; - } - - /** - * Returns the preferred screenshare codec for the given connection type. - * - * @param {String} connectionType The media connection type, 'p2p' or 'jvb'. - * @returns CodecMimeType - */ - getScreenshareCodec(connectionType) { - return this.screenshareCodec[connectionType]; + this.selectPreferredCodec(); } } diff --git a/modules/RTC/CodecSelection.spec.js b/modules/qualitycontrol/CodecSelection.spec.js similarity index 59% rename from modules/RTC/CodecSelection.spec.js rename to modules/qualitycontrol/CodecSelection.spec.js index 7a38344335..720fa0cb2d 100644 --- a/modules/RTC/CodecSelection.spec.js +++ b/modules/qualitycontrol/CodecSelection.spec.js @@ -1,10 +1,11 @@ -import * as JitsiConferenceEvents from '../../JitsiConferenceEvents.ts'; -import Listenable from '../util/Listenable.js'; -import JingleSessionPC from '../xmpp/JingleSessionPC.js'; -import { MockChatRoom, MockStropheConnection } from '../xmpp/MockClasses.js'; +import * as JitsiConferenceEvents from '../../JitsiConferenceEvents'; +import { MockRTC, MockSignalingLayerImpl } from '../RTC/MockClasses'; +import Listenable from '../util/Listenable'; +import { nextTick } from '../util/TestUtils'; +import JingleSessionPC from '../xmpp/JingleSessionPC'; +import { MockChatRoom, MockStropheConnection } from '../xmpp/MockClasses'; -import { CodecSelection } from './CodecSelection.js'; -import { MockRTC, MockSignalingLayerImpl } from './MockClasses.js'; +import QualityController from './QualityController'; /** * MockParticipant @@ -26,6 +27,29 @@ class MockParticipant { } } +/** + * MockLocalTrack + */ +class MockLocalTrack { + /** + * Constructor + * @param {number} resolution + * @param {string} videoType + */ + constructor(resolution, videoType) { + this.maxEnabledResolution = resolution; + this.videoType = videoType; + } + + /** + * Returns the video type of the mock local track. + * @returns {string} + */ + getVideoType() { + return this.videoType; + } +} + /** * MockConference */ @@ -57,6 +81,14 @@ class MockConference extends Listenable { this.eventEmitter.emit(JitsiConferenceEvents.USER_JOINED); } + /** + * Returns the active media session. + * @returns {JingleSessionPC} + */ + getActiveMediaSession() { + return this.jvbJingleSession; + } + /** * Returns the list of participants. * @returns Array @@ -86,12 +118,12 @@ class MockConference extends Listenable { describe('Codec Selection', () => { /* eslint-disable-next-line no-unused-vars */ - let codecSelection; + let qualityController; let conference; let connection; let jingleSession; let options; - let participant1, participant2; + let participant1, participant2, participant3; let rtc; const SID = 'sid12345'; @@ -116,17 +148,19 @@ describe('Codec Selection', () => { /* Signaling layer */ conference._signalingLayer, /* options */ { }); conference.jvbJingleSession = jingleSession; + conference.rtc = rtc; }); describe('when codec preference list is used in config.js', () => { beforeEach(() => { options = { jvb: { - preferenceOrder: [ 'VP9', 'VP8', 'H264' ] + preferenceOrder: [ 'VP9', 'VP8', 'H264' ], + screenshareCodec: 'VP9' } }; - codecSelection = new CodecSelection(conference, options); + qualityController = new QualityController(conference, options); spyOn(jingleSession, 'setVideoCodecs'); }); @@ -141,7 +175,7 @@ describe('Codec Selection', () => { participant2 = new MockParticipant('remote-2'); conference.addParticipant(participant2, [ 'vp8' ]); - expect(jingleSession.setVideoCodecs).toHaveBeenCalledWith([ 'vp8' ]); + expect(jingleSession.setVideoCodecs).toHaveBeenCalledWith([ 'vp8' ], 'vp9'); // Make p2 leave the call conference.removeParticipant(participant2); @@ -153,13 +187,13 @@ describe('Codec Selection', () => { participant1 = new MockParticipant('remote-1'); conference.addParticipant(participant1, null, 'vp8'); - expect(jingleSession.setVideoCodecs).toHaveBeenCalledWith([ 'vp8' ]); + expect(jingleSession.setVideoCodecs).toHaveBeenCalledWith([ 'vp8' ], 'vp9'); // Add a third user (newer) to the call. participant2 = new MockParticipant('remote-2'); conference.addParticipant(participant2, [ 'vp9', 'vp8' ]); - expect(jingleSession.setVideoCodecs).toHaveBeenCalledWith([ 'vp8' ]); + expect(jingleSession.setVideoCodecs).toHaveBeenCalledWith([ 'vp8' ], 'vp9'); // Make p1 leave the call conference.removeParticipant(participant1); @@ -176,7 +210,7 @@ describe('Codec Selection', () => { } }; - codecSelection = new CodecSelection(conference, options); + qualityController = new QualityController(conference, options); spyOn(jingleSession, 'setVideoCodecs'); }); @@ -191,7 +225,7 @@ describe('Codec Selection', () => { participant2 = new MockParticipant('remote-2'); conference.addParticipant(participant2, [ 'vp8' ]); - expect(jingleSession.setVideoCodecs).toHaveBeenCalledWith([ 'vp8' ]); + expect(jingleSession.setVideoCodecs).toHaveBeenCalledWith([ 'vp8' ], undefined); // Make p2 leave the call conference.removeParticipant(participant2); @@ -203,7 +237,7 @@ describe('Codec Selection', () => { participant1 = new MockParticipant('remote-1'); conference.addParticipant(participant1, [ 'h264', 'vp8' ]); - expect(jingleSession.setVideoCodecs).toHaveBeenCalledWith([ 'vp8' ]); + expect(jingleSession.setVideoCodecs).toHaveBeenCalledWith([ 'vp8' ], undefined); }); it('and remote endpoints use the old codec selection logic (RN)', () => { @@ -211,17 +245,86 @@ describe('Codec Selection', () => { participant1 = new MockParticipant('remote-1'); conference.addParticipant(participant1, null, 'vp8'); - expect(jingleSession.setVideoCodecs).toHaveBeenCalledWith([ 'vp8' ]); + expect(jingleSession.setVideoCodecs).toHaveBeenCalledWith([ 'vp8' ], undefined); // Add a third user (newer) to the call. participant2 = new MockParticipant('remote-2'); conference.addParticipant(participant2, [ 'vp9', 'vp8', 'h264' ]); - expect(jingleSession.setVideoCodecs).toHaveBeenCalledWith([ 'vp8' ]); + expect(jingleSession.setVideoCodecs).toHaveBeenCalledWith([ 'vp8' ], undefined); // Make p1 leave the call conference.removeParticipant(participant1); expect(jingleSession.setVideoCodecs).toHaveBeenCalledTimes(3); }); }); + + describe('when codec switching is triggered based on outbound-rtp stats', () => { + beforeEach(() => { + options = { + jvb: { + preferenceOrder: [ 'AV1', 'VP9', 'VP8' ] + } + }; + jasmine.clock().install(); + qualityController = new QualityController(conference, options); + spyOn(jingleSession, 'setVideoCodecs'); + }); + + afterEach(() => { + jasmine.clock().uninstall(); + }); + + it('and encode resolution is limited by cpu for camera tracks', () => { + const localTrack = new MockLocalTrack(720, 'camera'); + + participant1 = new MockParticipant('remote-1'); + conference.addParticipant(participant1, [ 'av1', 'vp9', 'vp8' ]); + expect(jingleSession.setVideoCodecs).toHaveBeenCalledWith([ 'av1', 'vp9', 'vp8' ], undefined); + + participant2 = new MockParticipant('remote-2'); + conference.addParticipant(participant2, [ 'av1', 'vp9', 'vp8' ]); + expect(jingleSession.setVideoCodecs).toHaveBeenCalledWith([ 'av1', 'vp9', 'vp8' ], undefined); + + qualityController.codecController.changeCodecPreferenceOrder(localTrack, 'av1'); + + return nextTick(121000).then(() => { + expect(jingleSession.setVideoCodecs).toHaveBeenCalledWith([ 'vp9', 'av1', 'vp8' ], undefined); + }) + .then(() => { + participant3 = new MockParticipant('remote-3'); + conference.addParticipant(participant3, [ 'av1', 'vp9', 'vp8' ]); + + // Expect the local endpoint to continue sending VP9. + expect(jingleSession.setVideoCodecs).toHaveBeenCalledWith([ 'vp9', 'av1', 'vp8' ], undefined); + }); + }); + + it('and does not change codec if the current codec is already the lowest complexity codec', () => { + const localTrack = new MockLocalTrack(720, 'camera'); + + qualityController.codecController.codecPreferenceOrder.jvb = [ 'vp8', 'vp9', 'av1' ]; + + participant1 = new MockParticipant('remote-1'); + conference.addParticipant(participant1, [ 'av1', 'vp9', 'vp8' ]); + expect(jingleSession.setVideoCodecs).toHaveBeenCalledWith([ 'vp8', 'vp9', 'av1' ], undefined); + + participant2 = new MockParticipant('remote-2'); + conference.addParticipant(participant2, [ 'av1', 'vp9', 'vp8' ]); + expect(jingleSession.setVideoCodecs).toHaveBeenCalledWith([ 'vp8', 'vp9', 'av1' ], undefined); + + qualityController.codecController.changeCodecPreferenceOrder(localTrack, 'vp8'); + + return nextTick(121000).then(() => { + expect(jingleSession.setVideoCodecs).toHaveBeenCalledWith([ 'vp8', 'vp9', 'av1' ], undefined); + }) + .then(() => { + participant3 = new MockParticipant('remote-3'); + conference.addParticipant(participant3, [ 'av1', 'vp9', 'vp8' ]); + + // Expect the local endpoint to continue sending VP9. + expect(jingleSession.setVideoCodecs).toHaveBeenCalledWith([ 'vp8', 'vp9', 'av1' ], undefined); + }); + }); + }); }); diff --git a/modules/qualitycontrol/QualityController.ts b/modules/qualitycontrol/QualityController.ts new file mode 100644 index 0000000000..de95554652 --- /dev/null +++ b/modules/qualitycontrol/QualityController.ts @@ -0,0 +1,365 @@ +import { getLogger } from '@jitsi/logger'; + +import JitsiConference from "../../JitsiConference"; +import { JitsiConferenceEvents } from "../../JitsiConferenceEvents"; +import { CodecMimeType } from "../../service/RTC/CodecMimeType"; +import RTCEvents from "../../service/RTC/RTCEvents"; +import JitsiLocalTrack from "../RTC/JitsiLocalTrack"; +import TraceablePeerConnection from "../RTC/TraceablePeerConnection"; +import JingleSessionPC from "../xmpp/JingleSessionPC"; +import { CodecSelection } from "./CodecSelection"; +import ReceiveVideoController from "./ReceiveVideoController"; +import SendVideoController from "./SendVideoController"; +import { VIDEO_CODECS_BY_COMPLEXITY, VIDEO_QUALITY_LEVELS } from '../../service/RTC/StandardVideoSettings'; + +const logger = getLogger(__filename); + +// Period for which the client will wait for the cpu limitation flag to be reset in the peerconnection stats before it +// attempts to rectify the situation by attempting a codec switch. +const LIMITED_BY_CPU_TIMEOUT = 60000; + +// The min. value that lastN will be set to while trying to fix video qaulity issues. +const MIN_LAST_N = 3; + +enum QualityLimitationReason { + BANDWIDTH = 'bandwidth', + CPU = 'cpu', + NONE = 'none' +}; + +interface IResolution { + height: number; + width: number; +} + +interface IOutboundRtpStats { + codec: CodecMimeType; + encodeTime: number; + qualityLimitationReason: QualityLimitationReason; + resolution: IResolution; + timestamp: number; +} + +interface ISourceStats { + avgEncodeTime: number; + codec: CodecMimeType; + encodeResolution: number; + localTrack: JitsiLocalTrack; + qualityLimitationReason: QualityLimitationReason; + timestamp: number; +}; + +interface ITrackStats { + encodeResolution: number + encodeTime: number; + qualityLimitationReason: QualityLimitationReason; +} + +interface IVideoConstraints { + maxHeight: number; + sourceName: string; +} + +class FixedSizeArray { + private _data: ISourceStats[]; + private _maxSize: number; + + constructor(size: number) { + this._maxSize = size; + this._data = []; + } + + add(item: ISourceStats): void { + if (this._data.length >= this._maxSize) { + this._data.shift(); + } + this._data.push(item); + } + + get(index: number): ISourceStats | undefined { + if (index < 0 || index >= this._data.length) { + throw new Error("Index out of bounds"); + } + return this._data[index]; + } + + size(): number { + return this._data.length; + } +} + +/** + * QualityController class that is responsible for maintaining optimal video quality experience on the local endpoint + * by controlling the codec, encode resolution and receive resolution of the remote video streams. It also makes + * adjustments based on the outbound and inbound rtp stream stats reported by the underlying peer connection. + */ +export default class QualityController { + private _codecController: CodecSelection; + private _conference: JitsiConference; + private _enableAdaptiveMode: boolean; + private _encodeTimeStats: Map; + private _limitedByCpuTimeout: number | undefined; + private _receiveVideoController: ReceiveVideoController; + private _sendVideoController: SendVideoController; + + /** + * + * @param {JitsiConference} conference - The JitsiConference instance. + * @param {Object} codecSettings - Codec settings for jvb and p2p connections passed to config.js. + * @param {boolean} enableAdaptiveMode - Whether the client needs to make runtime adjustments for better video + * quality. + */ + constructor(conference: JitsiConference, codecSettings: any, enableAdaptiveMode: boolean) { + this._conference = conference; + this._codecController = new CodecSelection(conference, codecSettings); + this._enableAdaptiveMode = enableAdaptiveMode; + this._receiveVideoController = new ReceiveVideoController(conference); + this._sendVideoController = new SendVideoController(conference); + this._encodeTimeStats = new Map(); + + this._conference.on( + JitsiConferenceEvents._MEDIA_SESSION_STARTED, + (session: JingleSessionPC) => { + this._codecController.selectPreferredCodec(session); + this._receiveVideoController.onMediaSessionStarted(session); + this._sendVideoController.onMediaSessionStarted(session); + }); + this._conference.on( + JitsiConferenceEvents._MEDIA_SESSION_ACTIVE_CHANGED, + () => this._sendVideoController.configureConstraintsForLocalSources()); + this._conference.on( + JitsiConferenceEvents.CONFERENCE_VISITOR_CODECS_CHANGED, + (codecList: CodecMimeType[]) => this._codecController.updateVisitorCodecs(codecList)); + this._conference.on( + JitsiConferenceEvents.USER_JOINED, + () => this._codecController.selectPreferredCodec(this._conference.jvbJingleSession)); + this._conference.on( + JitsiConferenceEvents.USER_LEFT, + () => this._codecController.selectPreferredCodec(this._conference.jvbJingleSession)); + this._conference.rtc.on( + RTCEvents.SENDER_VIDEO_CONSTRAINTS_CHANGED, + (videoConstraints: IVideoConstraints) => this._sendVideoController.onSenderConstraintsReceived(videoConstraints)); + this._conference.on( + JitsiConferenceEvents.ENCODE_TIME_STATS_RECEIVED, + (tpc: TraceablePeerConnection, stats: Map) => this._processOutboundRtpStats(tpc, stats)); + } + + /** + * Adjusts the lastN value so that fewer remote video sources are received from the bridge in an attempt to improve + * encode resolution of the outbound video streams. + * @returns boolean - Returns true if the lastN was lowered, false otherwise. + */ + _maybeLowerLastN(): boolean { + const lastN = this.receiveVideoController.getLastN(); + const videoStreamsReceived = this._conference.getForwardedSources().length; + + if (lastN !== -1 && lastN <= MIN_LAST_N) { + return false; + } + + let newLastN = videoStreamsReceived / 2; + + if (lastN === -1 || newLastN < MIN_LAST_N) { + newLastN = MIN_LAST_N; + } + + logger.info(`QualityController - setting lastN=${newLastN}`); + this.receiveVideoController.setLastN(newLastN); + + return true; + } + + /** + * Adjusts the requested resolution for remote video sources by updating the receiver constraints in an attempt to + * improve the encode resolution of the outbound video streams. + * @return {void} + */ + _maybeLowerReceiveResolution(): void { + const currentConstraints = this.receiveVideoController.getCurrentReceiverConstraints(); + const individualConstraints = currentConstraints.constraints; + let maxHeight = 0; + + if (individualConstraints && Object.keys(individualConstraints).length) { + for (const value of Object.values(individualConstraints)) { + const v: any = value; + maxHeight = Math.max(maxHeight, v.maxHeight); + } + } + + const currentLevel = VIDEO_QUALITY_LEVELS.findIndex(lvl => lvl.height <= maxHeight); + + // Do not lower the resolution to less than 180p. + if (VIDEO_QUALITY_LEVELS[currentLevel].height === 180) { + return; + } + + this.receiveVideoController.setPreferredReceiveMaxFrameHeight(VIDEO_QUALITY_LEVELS[currentLevel + 1].height); + } + + /** + * Updates the codec preference order for the local endpoint on the active media session and switches the video + * codec if needed. + * + * @param {string} trackId - The track ID of the local video track for which stats have been captured. + * @returns {boolean} - Returns true if video codec was changed. + */ + _maybeSwitchVideoCodec(trackId: string): boolean { + const stats = this._encodeTimeStats.get(trackId); + const { codec, encodeResolution, localTrack } = stats.get(stats.size() - 1); + const codecsByVideoType = VIDEO_CODECS_BY_COMPLEXITY[localTrack.getVideoType()]; + const codecIndex = codecsByVideoType.findIndex(val => val === codec.toLowerCase()); + + // Do nothing if the encoder is using the lowest complexity codec already. + if (codecIndex === codecsByVideoType.length - 1) { + return false; + } + + if (!this._limitedByCpuTimeout) { + this._limitedByCpuTimeout = window.setTimeout(() => { + this._limitedByCpuTimeout = undefined; + const updatedStats = this._encodeTimeStats.get(trackId); + const latestSourceStats: ISourceStats = updatedStats.get(updatedStats.size() - 1); + + // If the encoder is still limited by CPU, switch to a lower complexity codec. + if (latestSourceStats.qualityLimitationReason === QualityLimitationReason.CPU + || encodeResolution < Math.min(localTrack.maxEnabledResolution, localTrack.getCaptureResolution())) { + return this.codecController.changeCodecPreferenceOrder(localTrack, codec) + } + }, LIMITED_BY_CPU_TIMEOUT); + } + + return false; + } + + /** + * Processes the outbound RTP stream stats as reported by the WebRTC peerconnection and makes runtime adjustments + * to the client for better quality experience if the adaptive mode is enabled. + * + * @param {TraceablePeerConnection} tpc - The underlying WebRTC peerconnection where stats have been captured. + * @param {Map} stats - Outbound-rtp stream stats per SSRC. + * @returns void + */ + _processOutboundRtpStats(tpc: TraceablePeerConnection, stats: Map): void { + const activeSession = this._conference.getActiveMediaSession(); + + // Process stats only for the active media session. + if (activeSession.peerconnection !== tpc) { + return; + } + + const statsPerTrack = new Map(); + + for (const ssrc of stats.keys()) { + const { codec, encodeTime, qualityLimitationReason, resolution, timestamp } = stats.get(ssrc); + const track = tpc.getTrackBySSRC(ssrc); + let existingStats = statsPerTrack.get(track.rtcId); + const encodeResolution = Math.min(resolution.height, resolution.width); + const ssrcStats = { + encodeResolution, + encodeTime, + qualityLimitationReason + }; + + if (existingStats) { + existingStats.codec = codec; + existingStats.timestamp = timestamp; + existingStats.trackStats.push(ssrcStats); + } else { + existingStats = { + codec, + timestamp, + trackStats: [ ssrcStats ] + }; + + statsPerTrack.set(track.rtcId, existingStats); + } + } + + // Aggregate the stats for multiple simulcast streams with different SSRCs but for the same video stream. + for (const trackId of statsPerTrack.keys()) { + const { codec, timestamp, trackStats } = statsPerTrack.get(trackId); + const totalEncodeTime = trackStats + .map((stat: ITrackStats) => stat.encodeTime) + .reduce((totalValue: number, currentValue: number) => totalValue + currentValue, 0); + const avgEncodeTime: number = totalEncodeTime / trackStats.length; + const { qualityLimitationReason = QualityLimitationReason.NONE } + = trackStats + .find((stat: ITrackStats) => stat.qualityLimitationReason !== QualityLimitationReason.NONE) ?? {}; + const encodeResolution: number = trackStats + .map((stat: ITrackStats) => stat.encodeResolution) + .reduce((resolution: number, currentValue: number) => Math.max(resolution, currentValue), 0); + const localTrack = this._conference.getLocalVideoTracks().find(t => t.rtcId === trackId); + + const exisitingStats: FixedSizeArray = this._encodeTimeStats.get(trackId); + const sourceStats = { + avgEncodeTime, + codec, + encodeResolution, + qualityLimitationReason, + localTrack, + timestamp + }; + + if (exisitingStats) { + exisitingStats.add(sourceStats); + } else { + // Save stats for only the last 5 mins. + const data = new FixedSizeArray(300); + + data.add(sourceStats); + this._encodeTimeStats.set(trackId, data); + } + + logger.debug(`Encode stats for ${localTrack}: codec=${codec}, time=${avgEncodeTime},` + + `resolution=${encodeResolution}, qualityLimitationReason=${qualityLimitationReason}`); + + // Do not attempt run time adjustments if the adaptive mode is disabled. + if (!this._enableAdaptiveMode) { + return; + } + + if (encodeResolution === localTrack.maxEnabledResolution) { + if (this._limitedByCpuTimeout) { + window.clearTimeout(this._limitedByCpuTimeout); + this._limitedByCpuTimeout = undefined; + } + + return; + } + + // Do nothing if the limitation reason is bandwidth since the browser will dynamically adapt the outbound + // resolution based on available uplink bandwith. Otherwise, + // 1. Switch the codec to the lowest complexity one incrementally. + // 2. Switch to a lower lastN value, cutting the receive videos by half in every iteration until + // LAST_N_THRESHOLD is reached. + // 3. Lower the receive resolution of individual streams up to 180p. + if (qualityLimitationReason === QualityLimitationReason.CPU + || (encodeResolution < Math.min(localTrack.maxEnabledResolution, localTrack.getCaptureResolution()) + && qualityLimitationReason !== QualityLimitationReason.BANDWIDTH)) { + const codecSwitched = this._maybeSwitchVideoCodec(trackId); + + if (!codecSwitched && !this._limitedByCpuTimeout) { + this.receiveVideoController.setLastNLimitedByCpu(true); + const lastNChanged = this._maybeLowerLastN(); + + if (!lastNChanged) { + this.receiveVideoController.setReceiveResolutionLimitedByCpu(true); + this._maybeLowerReceiveResolution(); + } + } + } + } + } + + get codecController() { + return this._codecController; + } + + get receiveVideoController() { + return this._receiveVideoController; + } + + get sendVideoController() { + return this._sendVideoController; + } +} diff --git a/modules/qualitycontrol/ReceiveVideoController.js b/modules/qualitycontrol/ReceiveVideoController.js index 60307029bd..161c352abf 100644 --- a/modules/qualitycontrol/ReceiveVideoController.js +++ b/modules/qualitycontrol/ReceiveVideoController.js @@ -1,7 +1,6 @@ import { getLogger } from '@jitsi/logger'; import isEqual from 'lodash.isequal'; -import * as JitsiConferenceEvents from '../../JitsiConferenceEvents'; import { MediaType } from '../../service/RTC/MediaType'; const logger = getLogger(__filename); @@ -9,132 +8,6 @@ const MAX_HEIGHT = 2160; const LASTN_UNLIMITED = -1; const ASSUMED_BANDWIDTH_BPS = -1; -/** - * This class translates the legacy signaling format between the client and the bridge (that affects bandwidth - * allocation) to the new format described here https://github.com/jitsi/jitsi-videobridge/blob/master/doc/allocation.md - */ -class ReceiverVideoConstraints { - /** - * Creates a new instance. - * @param {Object} options - The instance options: - * - lastN: Number of videos to be requested from the bridge. - * - assumedBandwidthBps: Number of bps to be requested from the bridge. - */ - constructor(options) { - const { lastN, assumedBandwidthBps } = options; - - // The number of videos requested from the bridge. - this._lastN = lastN ?? LASTN_UNLIMITED; - - // The number representing the maximum video height the local client should receive from the bridge/peer. - this._maxFrameHeight = MAX_HEIGHT; - - // The number representing the assumed count of bps the local client should receive from the bridge. - this._assumedBandwidthBps = assumedBandwidthBps ?? ASSUMED_BANDWIDTH_BPS; - - this._receiverVideoConstraints = { - assumedBandwidthBps: this._assumedBandwidthBps, - constraints: {}, - defaultConstraints: { 'maxHeight': this._maxFrameHeight }, - lastN: this._lastN - }; - } - - /** - * Returns the receiver video constraints that need to be sent on the bridge channel or to the remote peer. - */ - get constraints() { - this._receiverVideoConstraints.assumedBandwidthBps = this._assumedBandwidthBps; - this._receiverVideoConstraints.lastN = this._lastN; - const individualConstraints = this._receiverVideoConstraints.constraints; - - if (individualConstraints && Object.keys(individualConstraints).length) { - /* eslint-disable no-unused-vars */ - for (const [ key, value ] of Object.entries(individualConstraints)) { - value.maxHeight = this._maxFrameHeight; - } - } else { - this._receiverVideoConstraints.defaultConstraints = { 'maxHeight': this._maxFrameHeight }; - } - - return this._receiverVideoConstraints; - } - - - /** - * Updates the assumed bandwidth bps of the ReceiverVideoConstraints sent to the bridge. - * - * @param {number} assumedBandwidthBps - * @requires {boolean} Returns true if the the value has been updated, false otherwise. - */ - updateAssumedBandwidthBps(assumedBandwidthBps) { - const changed = this._assumedBandwidthBps !== assumedBandwidthBps; - - if (changed) { - this._assumedBandwidthBps = assumedBandwidthBps; - logger.debug(`Updating receive assumedBandwidthBps: ${assumedBandwidthBps}`); - } - - return changed; - } - - /** - * Updates the lastN field of the ReceiverVideoConstraints sent to the bridge. - * - * @param {number} value - * @returns {boolean} Returns true if the the value has been updated, false otherwise. - */ - updateLastN(value) { - const changed = this._lastN !== value; - - if (changed) { - this._lastN = value; - logger.debug(`Updating ReceiverVideoConstraints lastN(${value})`); - } - - return changed; - } - - /** - * Updates the resolution (height requested) in the contraints field of the ReceiverVideoConstraints - * sent to the bridge. - * - * @param {number} maxFrameHeight - * @requires {boolean} Returns true if the the value has been updated, false otherwise. - */ - updateReceiveResolution(maxFrameHeight) { - const changed = this._maxFrameHeight !== maxFrameHeight; - - if (changed) { - this._maxFrameHeight = maxFrameHeight; - logger.debug(`Updating receive maxFrameHeight: ${maxFrameHeight}`); - } - - return changed; - } - - /** - * Updates the receiver constraints sent to the bridge. - * - * @param {Object} videoConstraints - * @returns {boolean} Returns true if the the value has been updated, false otherwise. - */ - updateReceiverVideoConstraints(videoConstraints) { - const changed = !isEqual(this._receiverVideoConstraints, videoConstraints); - - if (changed) { - this._receiverVideoConstraints = videoConstraints; - - if (videoConstraints.defaultConstraints?.maxHeight) { - this.updateReceiveResolution(videoConstraints.defaultConstraints.maxHeight); - } - logger.debug(`Updating ReceiverVideoConstraints ${JSON.stringify(videoConstraints)}`); - } - - return changed; - } -} - /** * This class manages the receive video contraints for a given {@link JitsiConference}. These constraints are * determined by the application based on how the remote video streams need to be displayed. This class is responsible @@ -146,11 +19,10 @@ export default class ReceiveVideoController { * * @param {JitsiConference} conference the conference instance for which the new instance will be managing * the receive video quality constraints. - * @param {RTC} rtc the rtc instance which is responsible for initializing the bridge channel. */ - constructor(conference, rtc) { + constructor(conference) { this._conference = conference; - this._rtc = rtc; + this._rtc = conference.rtc; const { config } = conference.options; // The number of videos requested from the bridge, -1 represents unlimited or all available videos. @@ -171,15 +43,14 @@ export default class ReceiveVideoController { */ this._assumedBandwidthBps = ASSUMED_BANDWIDTH_BPS; + this._lastNLimitedByCpu = false; + this._receiveResolutionLimitedByCpu = false; + // The default receiver video constraints. - this._receiverVideoConstraints = new ReceiverVideoConstraints({ - lastN: this._lastN, - assumedBandwidthBps: this._assumedBandwidthBps - }); - - this._conference.on( - JitsiConferenceEvents._MEDIA_SESSION_STARTED, - session => this._onMediaSessionStarted(session)); + this._receiverVideoConstraints = { + assumedBandwidthBps: this._assumedBandwidthBps, + lastN: this._lastN + }; } /** @@ -202,18 +73,19 @@ export default class ReceiveVideoController { } /** - * Handles the {@link JitsiConferenceEvents.MEDIA_SESSION_STARTED}, that is when the conference creates new media - * session. The preferred receive frameHeight is applied on the media session. + * Updates the source based constraints based on the maxHeight set. * - * @param {JingleSessionPC} mediaSession - the started media session. * @returns {void} - * @private */ - _onMediaSessionStarted(mediaSession) { - if (mediaSession.isP2P) { - mediaSession.setReceiverVideoConstraint(this._getDefaultSourceReceiverConstraints(mediaSession)); + _updateIndividualConstraints() { + const individualConstraints = this._receiverVideoConstraints.constraints; + + if (individualConstraints && Object.keys(individualConstraints).length) { + for (const value of Object.values(individualConstraints)) { + value.maxHeight = Math.min(value.maxHeight, this._maxFrameHeight); + } } else { - this._rtc.setReceiverVideoConstraints(this._receiverVideoConstraints.constraints); + this._receiverVideoConstraints.defaultConstraints = { 'maxHeight': this._maxFrameHeight }; } } @@ -226,6 +98,29 @@ export default class ReceiveVideoController { return this._lastN; } + /** + * Handles the {@link JitsiConferenceEvents.MEDIA_SESSION_STARTED}, that is when the conference creates new media + * session. The preferred receive frameHeight is applied on the media session. + * + * @param {JingleSessionPC} mediaSession - the started media session. + * @returns {void} + */ + onMediaSessionStarted(mediaSession) { + if (mediaSession.isP2P) { + mediaSession.setReceiverVideoConstraint(this._getDefaultSourceReceiverConstraints(mediaSession)); + } else { + this._rtc.setReceiverVideoConstraints(this._receiverVideoConstraints); + } + } + + /** + * Returns the last set of receiver constraints that were set on the bridge channel. + * @returns + */ + getCurrentReceiverConstraints() { + return this._receiverVideoConstraints; + } + /** * Sets the assumed bandwidth bps the local participant should receive from remote participants. * @@ -233,8 +128,9 @@ export default class ReceiveVideoController { * @returns {void} */ setAssumedBandwidthBps(assumedBandwidthBps) { - if (this._receiverVideoConstraints.updateAssumedBandwidthBps(assumedBandwidthBps)) { - this._rtc.setReceiverVideoConstraints(this._receiverVideoConstraints.constraints); + if (this._receiverVideoConstraints.assumedBandwidthBps !== assumedBandwidthBps) { + this._receiverVideoConstraints.assumedBandwidthBps = assumedBandwidthBps; + this._rtc.setReceiverVideoConstraints(this._receiverVideoConstraints); } } @@ -248,9 +144,21 @@ export default class ReceiveVideoController { setLastN(value) { if (this._lastN !== value) { this._lastN = value; - if (this._receiverVideoConstraints.updateLastN(value)) { - this._rtc.setReceiverVideoConstraints(this._receiverVideoConstraints.constraints); - } + this._receiverVideoConstraints.lastN = value; + this._rtc.setReceiverVideoConstraints(this._receiverVideoConstraints); + } + } + + /** + * Updates the lastNLimitedByCpu field. + * + * @param {boolean} enabled + * @returns {void} + */ + setLastNLimitedByCpu(enabled) { + if (this._lastNLimitedByCpu !== enabled) { + this._lastNLimitedByCpu = enabled; + logger.info(`ReceiveVideoController - Setting the _lastNLimitedByCpu flag to ${enabled}`); } } @@ -266,8 +174,9 @@ export default class ReceiveVideoController { for (const session of this._conference.getMediaSessions()) { if (session.isP2P) { session.setReceiverVideoConstraint(this._getDefaultSourceReceiverConstraints(session, maxFrameHeight)); - } else if (this._receiverVideoConstraints.updateReceiveResolution(maxFrameHeight)) { - this._rtc.setReceiverVideoConstraints(this._receiverVideoConstraints.constraints); + } else { + this._updateIndividualConstraints(); + this._rtc.setReceiverVideoConstraints(this._receiverVideoConstraints); } } } @@ -281,6 +190,7 @@ export default class ReceiveVideoController { if (!constraints) { return; } + const isEndpointsFormat = Object.keys(constraints).includes('onStageEndpoints', 'selectedEndpoints'); if (isEndpointsFormat) { @@ -288,22 +198,26 @@ export default class ReceiveVideoController { '"onStageEndpoints" and "selectedEndpoints" are not supported when sourceNameSignaling is enabled.' ); } - const constraintsChanged = this._receiverVideoConstraints.updateReceiverVideoConstraints(constraints); + const constraintsChanged = !isEqual(this._receiverVideoConstraints, constraints); + + if (constraintsChanged || this._lastNLimitedByCpu || this._receiveResolutionLimitedByCpu) { + this._receiverVideoConstraints = constraints; - if (constraintsChanged) { this._assumedBandwidthBps = constraints.assumedBandwidthBps ?? this._assumedBandwidthBps; - this._lastN = constraints.lastN ?? this._lastN; + this._lastN = constraints.lastN && !this._lastNLimitedByCpu ? constraints.lastN : this._lastN; + this._receiverVideoConstraints.lastN = this._lastN; + this._receiveResolutionLimitedByCpu && this._updateIndividualConstraints(); // Send the contraints on the bridge channel. - this._rtc.setReceiverVideoConstraints(constraints); + this._rtc.setReceiverVideoConstraints(this._receiverVideoConstraints); const p2pSession = this._conference.getMediaSessions().find(session => session.isP2P); - if (!p2pSession || !constraints.constraints) { + if (!p2pSession || !this._receiverVideoConstraints.constraints) { return; } - const mappedConstraints = Array.from(Object.entries(constraints.constraints)) + const mappedConstraints = Array.from(Object.entries(this._receiverVideoConstraints.constraints)) .map(constraint => { constraint[1] = constraint[1].maxHeight; @@ -316,4 +230,17 @@ export default class ReceiveVideoController { p2pSession.setReceiverVideoConstraint(this._sourceReceiverConstraints); } } + + /** + * Updates the receivedResolutioLimitedByCpu field. + * + * @param {booem} enabled + * @return {void} + */ + setReceiveResolutionLimitedByCpu(enabled) { + if (this._receiveResolutionLimitedByCpu !== enabled) { + this._receiveResolutionLimitedByCpu = enabled; + logger.info(`ReceiveVideoController - Setting the _receiveResolutionLimitedByCpu flag to ${enabled}`); + } + } } diff --git a/modules/qualitycontrol/ReceiveVideoController.spec.js b/modules/qualitycontrol/ReceiveVideoController.spec.js index 486746e0bc..c1b9d2f770 100644 --- a/modules/qualitycontrol/ReceiveVideoController.spec.js +++ b/modules/qualitycontrol/ReceiveVideoController.spec.js @@ -53,7 +53,8 @@ describe('ReceiveVideoController', () => { beforeEach(() => { conference = new MockConference(); rtc = new MockRTC(); - receiveVideoController = new ReceiveVideoController(conference, rtc); + conference.rtc = rtc; + receiveVideoController = new ReceiveVideoController(conference); }); describe('when sourceNameSignaling is enabled', () => { diff --git a/modules/qualitycontrol/SendVideoController.js b/modules/qualitycontrol/SendVideoController.js index 2c3618ed3d..22b4b90dff 100644 --- a/modules/qualitycontrol/SendVideoController.js +++ b/modules/qualitycontrol/SendVideoController.js @@ -1,7 +1,5 @@ import { getLogger } from '@jitsi/logger'; -import * as JitsiConferenceEvents from '../../JitsiConferenceEvents'; -import RTCEvents from '../../service/RTC/RTCEvents'; import MediaSessionEvents from '../xmpp/MediaSessionEvents'; const logger = getLogger(__filename); @@ -20,85 +18,16 @@ export default class SendVideoController { * * @param {JitsiConference} conference - the conference instance for which the new instance will be managing * the send video quality constraints. - * @param {RTC} rtc - the rtc instance that is responsible for sending the messages on the bridge channel. */ - constructor(conference, rtc) { + constructor(conference) { this._conference = conference; this._preferredSendMaxFrameHeight = MAX_LOCAL_RESOLUTION; - this._rtc = rtc; /** * Source name based sender constraints. * @type {Map}; */ - this._sourceSenderConstraints = new Map(); - this._conference.on( - JitsiConferenceEvents._MEDIA_SESSION_STARTED, - session => this._onMediaSessionStarted(session)); - this._conference.on( - JitsiConferenceEvents._MEDIA_SESSION_ACTIVE_CHANGED, - () => this._configureConstraintsForLocalSources()); - this._rtc.on( - RTCEvents.SENDER_VIDEO_CONSTRAINTS_CHANGED, - videoConstraints => this._onSenderConstraintsReceived(videoConstraints)); - } - - /** - * Configures the video encodings on the local sources when a media connection is established or becomes active. - * - * @returns {Promise} - * @private - */ - _configureConstraintsForLocalSources() { - for (const track of this._rtc.getLocalVideoTracks()) { - const sourceName = track.getSourceName(); - - sourceName && this._propagateSendMaxFrameHeight(sourceName); - } - } - - /** - * Handles the {@link JitsiConferenceEvents.MEDIA_SESSION_STARTED}, that is when the conference creates new media - * session. It doesn't mean it's already active though. For example the JVB connection may be created after - * the conference has entered the p2p mode already. - * - * @param {JingleSessionPC} mediaSession - the started media session. - * @private - */ - _onMediaSessionStarted(mediaSession) { - mediaSession.addListener( - MediaSessionEvents.REMOTE_SOURCE_CONSTRAINTS_CHANGED, - (session, sourceConstraints) => { - session === this._conference.getActiveMediaSession() - && sourceConstraints.forEach(constraint => this._onSenderConstraintsReceived(constraint)); - }); - } - - /** - * Propagates the video constraints if they have changed. - * - * @param {Object} videoConstraints - The sender video constraints received from the bridge. - * @returns {Promise} - * @private - */ - _onSenderConstraintsReceived(videoConstraints) { - const { maxHeight, sourceName } = videoConstraints; - const localVideoTracks = this._conference.getLocalVideoTracks() ?? []; - - for (const track of localVideoTracks) { - // Propagate the sender constraint only if it has changed. - if (track.getSourceName() === sourceName - && this._sourceSenderConstraints.get(sourceName) !== maxHeight) { - this._sourceSenderConstraints.set( - sourceName, - maxHeight === -1 - ? Math.min(MAX_LOCAL_RESOLUTION, this._preferredSendMaxFrameHeight) - : maxHeight); - logger.debug(`Sender constraints for source:${sourceName} changed to maxHeight:${maxHeight}`); - this._propagateSendMaxFrameHeight(sourceName); - } - } } /** @@ -151,6 +80,60 @@ export default class SendVideoController { return this._preferredSendMaxFrameHeight; } + /** + * Configures the video encodings on the local sources when a media connection is established or becomes active. + * + * @returns {void} + */ + configureConstraintsForLocalSources() { + for (const track of this._conference.getLocalVideoTracks()) { + const sourceName = track.getSourceName(); + + sourceName && this._propagateSendMaxFrameHeight(sourceName); + } + } + + /** + * Handles the {@link JitsiConferenceEvents.MEDIA_SESSION_STARTED}, that is when the conference creates new media + * session. It doesn't mean it's already active though. For example the JVB connection may be created after + * the conference has entered the p2p mode already. + * + * @param {JingleSessionPC} mediaSession - the started media session. + */ + onMediaSessionStarted(mediaSession) { + mediaSession.addListener( + MediaSessionEvents.REMOTE_SOURCE_CONSTRAINTS_CHANGED, + (session, sourceConstraints) => { + session === this._conference.getActiveMediaSession() + && sourceConstraints.forEach(constraint => this.onSenderConstraintsReceived(constraint)); + }); + } + + /** + * Propagates the video constraints if they have changed. + * + * @param {Object} videoConstraints - The sender video constraints received from the bridge. + * @returns {Promise} + */ + onSenderConstraintsReceived(videoConstraints) { + const { maxHeight, sourceName } = videoConstraints; + const localVideoTracks = this._conference.getLocalVideoTracks() ?? []; + + for (const track of localVideoTracks) { + // Propagate the sender constraint only if it has changed. + if (track.getSourceName() === sourceName + && this._sourceSenderConstraints.get(sourceName) !== maxHeight) { + this._sourceSenderConstraints.set( + sourceName, + maxHeight === -1 + ? Math.min(MAX_LOCAL_RESOLUTION, this._preferredSendMaxFrameHeight) + : maxHeight); + logger.debug(`Sender constraints for source:${sourceName} changed to maxHeight:${maxHeight}`); + this._propagateSendMaxFrameHeight(sourceName); + } + } + } + /** * Sets local preference for max send video frame height. * diff --git a/modules/statistics/RTPStatsCollector.js b/modules/statistics/RTPStatsCollector.js index b7ddd524eb..ef278afd4b 100644 --- a/modules/statistics/RTPStatsCollector.js +++ b/modules/statistics/RTPStatsCollector.js @@ -334,7 +334,7 @@ StatsCollector.prototype._processAndEmitReport = function() { // calculated based on the outbound-rtp streams that are currently active for the simulcast case. // However for the SVC case, there will be only 1 "outbound-rtp" stream which will have the correct // send resolution width and height. - if (track.isLocal() && !browser.supportsTrackBasedStats() && this.peerconnection.doesTrueSimulcast()) { + if (track.isLocal() && !browser.supportsTrackBasedStats() && this.peerconnection.doesTrueSimulcast(track)) { const localSsrcs = this.peerconnection.getLocalVideoSSRCs(track); for (const localSsrc of localSsrcs) { diff --git a/modules/xmpp/JingleSessionPC.js b/modules/xmpp/JingleSessionPC.js index 8f78630d5c..dd163c71aa 100644 --- a/modules/xmpp/JingleSessionPC.js +++ b/modules/xmpp/JingleSessionPC.js @@ -1177,13 +1177,13 @@ export default class JingleSessionPC extends JingleSession { * Updates the codecs on the peerconnection and initiates a renegotiation for the * new codec config to take effect. * - * @param {Array} codecList the preferred codecs. + * @param {Array} codecList - Preferred codecs for video. + * @param {CodecMimeType} screenshareCodec - The preferred screenshare codec. */ - setVideoCodecs(codecList) { - + setVideoCodecs(codecList, screenshareCodec) { if (this._assertNotEnded()) { - logger.info(`${this} setVideoCodecs: ${codecList}`); - this.peerconnection.setVideoCodecs(codecList); + logger.info(`${this} setVideoCodecs: codecList=${codecList}, screenshareCodec=${screenshareCodec}`); + this.peerconnection.setVideoCodecs(codecList, screenshareCodec); // Browser throws an error when H.264 is set on the encodings. Therefore, munge the SDP when H.264 needs to // be selected. diff --git a/service/RTC/StandardVideoSettings.ts b/service/RTC/StandardVideoSettings.ts index 2dcd02fff3..b0c04b47a4 100644 --- a/service/RTC/StandardVideoSettings.ts +++ b/service/RTC/StandardVideoSettings.ts @@ -1,4 +1,5 @@ import browser from '../../modules/browser'; +import { CodecMimeType } from './CodecMimeType'; // Default simulcast encodings config. export const SIM_LAYERS = [ @@ -74,6 +75,19 @@ export const STANDARD_CODEC_SETTINGS = { } }; +export const VIDEO_CODECS_BY_COMPLEXITY = { + 'camera' : [ + CodecMimeType.AV1, + CodecMimeType.VP9, + CodecMimeType.VP8 + ], + 'desktop' : [ + CodecMimeType.VP9, + CodecMimeType.VP8, + CodecMimeType.AV1 + ] +}; + /** * Standard video resolutions and the corresponding quality level that will be picked for the given resolution. * For quality levels: