Skip to content

Commit

Permalink
feat(quality) Add a QualityController class for runtime adjustments. (#…
Browse files Browse the repository at this point in the history 
…2542)

* feat(quality) Add a QualityController class for runtime adjustments.
Make run time adjustments to the client when adaptive mode is enabled.

* feat: Update lastN and receive resolution to improve quality.

* squash: Address review comments

* squash: Add more logging and address review comments.
  • Loading branch information
@jallamsetty1
jallamsetty1 authored Jul 12, 2024
1 parent 0a2b079 commit a9b6dd7
Show file tree
Hide file tree
Showing 13 changed files with 791 additions and 450 deletions.
85 changes: 42 additions & 43 deletions JitsiConference.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ import JitsiTrackError from './JitsiTrackError';
import * as JitsiTrackErrors from './JitsiTrackErrors';
import * as JitsiTrackEvents from './JitsiTrackEvents';
import authenticateAndUpgradeRole from './authenticateAndUpgradeRole';
import { CodecSelection } from './modules/RTC/CodecSelection';
import RTC from './modules/RTC/RTC';
import { SS_DEFAULT_FRAME_RATE } from './modules/RTC/ScreenObtainer';
import browser from './modules/browser';
Expand All @@ -28,8 +27,7 @@ import E2ePing from './modules/e2eping/e2eping';
import Jvb121EventGenerator from './modules/event/Jvb121EventGenerator';
import FeatureFlags from './modules/flags/FeatureFlags';
import { LiteModeContext } from './modules/litemode/LiteModeContext';
import ReceiveVideoController from './modules/qualitycontrol/ReceiveVideoController';
import SendVideoController from './modules/qualitycontrol/SendVideoController';
import QualityController from './modules/qualitycontrol/QualityController';
import RecordingManager from './modules/recording/RecordingManager';
import Settings from './modules/settings/Settings';
import AudioOutputProblemDetector from './modules/statistics/AudioOutputProblemDetector';
Expand Down Expand Up @@ -383,31 +381,6 @@ JitsiConference.prototype._init = function(options = {}) {

const { config } = this.options;

// Get the codec preference settings from config.js.
const codecSettings = {
jvb: {
preferenceOrder: browser.isMobileDevice() && config.videoQuality?.mobileCodecPreferenceOrder
? config.videoQuality.mobileCodecPreferenceOrder
: config.videoQuality?.codecPreferenceOrder,
disabledCodec: _getCodecMimeType(config.videoQuality?.disabledCodec),
preferredCodec: _getCodecMimeType(config.videoQuality?.preferredCodec),
screenshareCodec: browser.isMobileDevice()
? _getCodecMimeType(config.videoQuality?.mobileScreenshareCodec)
: _getCodecMimeType(config.videoQuality?.screenshareCodec)
},
p2p: {
preferenceOrder: browser.isMobileDevice() && config.p2p?.mobileCodecPreferenceOrder
? config.p2p.mobileCodecPreferenceOrder
: config.p2p?.codecPreferenceOrder,
disabledCodec: _getCodecMimeType(config.p2p?.disabledCodec),
preferredCodec: _getCodecMimeType(config.p2p?.preferredCodec),
screenshareCodec: browser.isMobileDevice()
? _getCodecMimeType(config.p2p?.mobileScreenshareCodec)
: _getCodecMimeType(config.p2p?.screenshareCodec)
}
};

this.codecSelection = new CodecSelection(this, codecSettings);
this._statsCurrentId = config.statisticsId ? config.statisticsId : Settings.callStatsUserName;
this.room = this.xmpp.createRoom(
this.options.name, {
Expand Down Expand Up @@ -475,8 +448,34 @@ JitsiConference.prototype._init = function(options = {}) {
this._registerRtcListeners(this.rtc);
}

this.receiveVideoController = new ReceiveVideoController(this, this.rtc);
this.sendVideoController = new SendVideoController(this, this.rtc);
// Get the codec preference settings from config.js.
const codecSettings = {
jvb: {
preferenceOrder: browser.isMobileDevice() && config.videoQuality?.mobileCodecPreferenceOrder
? config.videoQuality.mobileCodecPreferenceOrder
: config.videoQuality?.codecPreferenceOrder,
disabledCodec: _getCodecMimeType(config.videoQuality?.disabledCodec),
preferredCodec: _getCodecMimeType(config.videoQuality?.preferredCodec),
screenshareCodec: browser.isMobileDevice()
? _getCodecMimeType(config.videoQuality?.mobileScreenshareCodec)
: _getCodecMimeType(config.videoQuality?.screenshareCodec)
},
p2p: {
preferenceOrder: browser.isMobileDevice() && config.p2p?.mobileCodecPreferenceOrder
? config.p2p.mobileCodecPreferenceOrder
: config.p2p?.codecPreferenceOrder,
disabledCodec: _getCodecMimeType(config.p2p?.disabledCodec),
preferredCodec: _getCodecMimeType(config.p2p?.preferredCodec),
screenshareCodec: browser.isMobileDevice()
? _getCodecMimeType(config.p2p?.mobileScreenshareCodec)
: _getCodecMimeType(config.p2p?.screenshareCodec)
}
};

this.qualityController = new QualityController(
this,
codecSettings,
config.videoQuality?.enableAdaptiveMode);

if (!this.statistics) {
this.statistics = new Statistics(this, {
Expand Down Expand Up @@ -574,7 +573,7 @@ JitsiConference.prototype._init = function(options = {}) {
}

// Publish the codec preference to presence.
this.setLocalParticipantProperty('codecList', this.codecSelection.getCodecPreferenceList('jvb'));
this.setLocalParticipantProperty('codecList', this.qualityController.codecController.getCodecPreferenceList('jvb'));

// Set transcription language presence extension.
// In case the language config is undefined or has the default value that the transcriber uses
Expand Down Expand Up @@ -1576,7 +1575,7 @@ JitsiConference.prototype.unlock = function() {
* @returns {number}
*/
JitsiConference.prototype.getLastN = function() {
return this.receiveVideoController.getLastN();
return this.qualityController.receiveVideoController.getLastN();
};

/**
Expand Down Expand Up @@ -1604,7 +1603,7 @@ JitsiConference.prototype.setLastN = function(lastN) {
if (n < -1) {
throw new RangeError('lastN cannot be smaller than -1');
}
this.receiveVideoController.setLastN(n);
this.qualityController.receiveVideoController.setLastN(n);

// If the P2P session is not fully established yet, we wait until it gets established.
if (this.p2pJingleSession) {
Expand Down Expand Up @@ -2232,8 +2231,8 @@ JitsiConference.prototype._acceptJvbIncomingCall = function(jingleSession, jingl
...this.options.config,
codecSettings: {
mediaType: MediaType.VIDEO,
codecList: this.codecSelection.getCodecPreferenceList('jvb'),
screenshareCodec: this.codecSelection.getScreenshareCodec('jvb')
codecList: this.qualityController.codecController.getCodecPreferenceList('jvb'),
screenshareCodec: this.qualityController.codecController.getScreenshareCodec('jvb')
},
enableInsertableStreams: this.isE2EEEnabled() || FeatureFlags.isRunInLiteModeEnabled()
});
Expand Down Expand Up @@ -2943,8 +2942,8 @@ JitsiConference.prototype._acceptP2PIncomingCall = function(jingleSession, jingl
...this.options.config,
codecSettings: {
mediaType: MediaType.VIDEO,
codecList: this.codecSelection.getCodecPreferenceList('p2p'),
screenshareCodec: this.codecSelection.getScreenshareCodec('p2p')
codecList: this.qualityController.codecController.getCodecPreferenceList('p2p'),
screenshareCodec: this.qualityController.codecController.getScreenshareCodec('p2p')
},
enableInsertableStreams: this.isE2EEEnabled() || FeatureFlags.isRunInLiteModeEnabled()
});
Expand Down Expand Up @@ -3299,8 +3298,8 @@ JitsiConference.prototype._startP2PSession = function(remoteJid) {
...this.options.config,
codecSettings: {
mediaType: MediaType.VIDEO,
codecList: this.codecSelection.getCodecPreferenceList('p2p'),
screenshareCodec: this.codecSelection.getScreenshareCodec('p2p')
codecList: this.qualityController.codecController.getCodecPreferenceList('p2p'),
screenshareCodec: this.qualityController.codecController.getScreenshareCodec('p2p')
},
enableInsertableStreams: this.isE2EEEnabled() || FeatureFlags.isRunInLiteModeEnabled()
});
Expand Down Expand Up @@ -3702,7 +3701,7 @@ JitsiConference.prototype.sendFaceLandmarks = function(payload) {
* Where A, B and C are source-names of the remote tracks that are being requested from the bridge.
*/
JitsiConference.prototype.setReceiverConstraints = function(videoConstraints) {
this.receiveVideoController.setReceiverConstraints(videoConstraints);
this.qualityController.receiveVideoController.setReceiverConstraints(videoConstraints);
};

/**
Expand All @@ -3711,7 +3710,7 @@ JitsiConference.prototype.setReceiverConstraints = function(videoConstraints) {
* @param {Number} assumedBandwidthBps - The bandwidth value expressed in bits per second.
*/
JitsiConference.prototype.setAssumedBandwidthBps = function(assumedBandwidthBps) {
this.receiveVideoController.setAssumedBandwidthBps(assumedBandwidthBps);
this.qualityController.receiveVideoController.setAssumedBandwidthBps(assumedBandwidthBps);
};

/**
Expand All @@ -3723,7 +3722,7 @@ JitsiConference.prototype.setAssumedBandwidthBps = function(assumedBandwidthBps)
* @returns {void}
*/
JitsiConference.prototype.setReceiverVideoConstraint = function(maxFrameHeight) {
this.receiveVideoController.setPreferredReceiveMaxFrameHeight(maxFrameHeight);
this.qualityController.receiveVideoController.setPreferredReceiveMaxFrameHeight(maxFrameHeight);
};

/**
Expand All @@ -3734,7 +3733,7 @@ JitsiConference.prototype.setReceiverVideoConstraint = function(maxFrameHeight)
* successful and rejected otherwise.
*/
JitsiConference.prototype.setSenderVideoConstraint = function(maxFrameHeight) {
return this.sendVideoController.setPreferredSendMaxFrameHeight(maxFrameHeight);
return this.qualityController.sendVideoController.setPreferredSendMaxFrameHeight(maxFrameHeight);
};

/**
Expand Down
4 changes: 3 additions & 1 deletion modules/RTC/MockClasses.js
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
import transform from 'sdp-transform';

import Listenable from '../util/Listenable';

/* eslint-disable no-empty-function */
/* eslint-disable max-len */

Expand Down Expand Up @@ -230,7 +232,7 @@ export class MockPeerConnection {
/**
* Mock {@link RTC} - add things as needed, but only things useful for all tests.
*/
export class MockRTC {
export class MockRTC extends Listenable {
/**
* {@link RTC.createPeerConnection}.
*
Expand Down
2 changes: 1 addition & 1 deletion modules/RTC/TPCUtils.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -192,7 +192,7 @@ export class TPCUtils {
if (localTrack.isAudioTrack()) {
return [ { active: this.pc.audioTransferActive } ];
}
const codec = this.pc.getConfiguredVideoCodec();
const codec = this.pc.getConfiguredVideoCodec(localTrack);

if (this.pc.isSpatialScalabilityOn()) {
return this._getVideoStreamEncodings(localTrack, codec);
Expand Down
45 changes: 25 additions & 20 deletions modules/RTC/TraceablePeerConnection.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -648,10 +648,12 @@ TraceablePeerConnection.prototype.getAudioLevels = function(speakerList = []) {
/**
* Checks if the browser is currently doing true simulcast where in three different media streams are being sent to the
* bridge. Currently this happens always for VP8 and only if simulcast is enabled for VP9/AV1/H264.
*
* @param {JitsiLocalTrack} localTrack - The local video track.
* @returns {boolean}
*/
TraceablePeerConnection.prototype.doesTrueSimulcast = function() {
const currentCodec = this.getConfiguredVideoCodec();
TraceablePeerConnection.prototype.doesTrueSimulcast = function(localTrack) {
const currentCodec = this.getConfiguredVideoCodec(localTrack);

return this.isSpatialScalabilityOn() && this.tpcUtils.isRunningInSimulcastMode(currentCodec);
};
Expand Down Expand Up @@ -804,10 +806,11 @@ TraceablePeerConnection.prototype.getRemoteSourceInfoByParticipant = function(id
/**
* Returns the target bitrates configured for the local video source.
*
* @param {JitsiLocalTrack} - The local video track.
* @returns {Object}
*/
TraceablePeerConnection.prototype.getTargetVideoBitrates = function() {
const currentCodec = this.getConfiguredVideoCodec();
TraceablePeerConnection.prototype.getTargetVideoBitrates = function(localTrack) {
const currentCodec = this.getConfiguredVideoCodec(localTrack);

return this.tpcUtils.codecSettings[currentCodec].maxBitratesVideo;
};
Expand Down Expand Up @@ -1596,17 +1599,17 @@ TraceablePeerConnection.prototype._assertTrackBelongs = function(
};

/**
* Returns the codec that is configured on the client as the preferred video codec.
* This takes into account the current order of codecs in the local description sdp.
* Returns the codec that is configured on the client as the preferred video codec for the given local video track.
*
* @param {JitsiLocalTrack} localTrack - The local video track.
* @returns {CodecMimeType} The codec that is set as the preferred codec for the given local video track.
*
* @returns {CodecMimeType} The codec that is set as the preferred codec to receive
* video in the local SDP.
*/
TraceablePeerConnection.prototype.getConfiguredVideoCodec = function() {
const localVideoTrack = this.getLocalVideoTracks()[0];
TraceablePeerConnection.prototype.getConfiguredVideoCodec = function(localTrack) {
const localVideoTrack = localTrack ?? this.getLocalVideoTracks()[0];
const rtpSender = this.findSenderForTrack(localVideoTrack.getTrack());

if (this.usesCodecSelectionAPI() && localVideoTrack) {
const rtpSender = this.findSenderForTrack(localVideoTrack.getTrack());
if (this.usesCodecSelectionAPI() && rtpSender) {
const { codecs } = rtpSender.getParameters();

return codecs[0].mimeType.split('/')[1].toLowerCase();
Expand All @@ -1619,7 +1622,8 @@ TraceablePeerConnection.prototype.getConfiguredVideoCodec = function() {
return defaultCodec;
}
const parsedSdp = transform.parse(sdp);
const mLine = parsedSdp.media.find(m => m.type === MediaType.VIDEO);
const mLine = parsedSdp.media
.find(m => m.mid.toString() === this._localTrackTransceiverMids.get(localVideoTrack.rtcId));
const payload = mLine.payloads.split(' ')[0];
const { codec } = mLine.rtp.find(rtp => rtp.payload === Number(payload));

Expand Down Expand Up @@ -1680,18 +1684,21 @@ TraceablePeerConnection.prototype.setDesktopSharingFrameRate = function(maxFps)

/**
* Sets the codec preference on the peerconnection. The codec preference goes into effect when
* the next renegotiation happens.
* the next renegotiation happens for older clients that do not support the codec selection API.
*
* @param {CodecMimeType} preferredCodec the preferred codec.
* @param {CodecMimeType} disabledCodec the codec that needs to be disabled.
* @param {Array<CodecMimeType>} codecList - Preferred codecs for video.
* @param {CodecMimeType} screenshareCodec - The preferred codec for screenshare.
* @returns {void}
*/
TraceablePeerConnection.prototype.setVideoCodecs = function(codecList) {
TraceablePeerConnection.prototype.setVideoCodecs = function(codecList, screenshareCodec) {
if (!this.codecSettings || !codecList?.length) {
return;
}

this.codecSettings.codecList = codecList;
if (screenshareCodec) {
this.codecSettings.screenshareCodec = screenshareCodec;
}

if (this.usesCodecSelectionAPI()) {
this.configureVideoSenderEncodings();
Expand Down Expand Up @@ -1755,9 +1762,7 @@ TraceablePeerConnection.prototype.findReceiverForTrack = function(track) {
* was found.
*/
TraceablePeerConnection.prototype.findSenderForTrack = function(track) {
if (this.peerconnection.getSenders) {
return this.peerconnection.getSenders().find(s => s.track === track);
}
return this.peerconnection.getSenders().find(s => s.track === track);
};

/**
Expand Down
Loading

0 comments on commit a9b6dd7

Please sign in to comment.