Merge branch 'master' into silent-participant-status

JitsiConference.js

@@ -390,14 +390,20 @@ JitsiConference.prototype._init = function(options = {}) {
                 ? config.videoQuality.mobileCodecPreferenceOrder
                 : config.videoQuality?.codecPreferenceOrder,
             disabledCodec: _getCodecMimeType(config.videoQuality?.disabledCodec),
-            preferredCodec: _getCodecMimeType(config.videoQuality?.preferredCodec)
+            preferredCodec: _getCodecMimeType(config.videoQuality?.preferredCodec),
+            screenshareCodec: browser.isMobileDevice()
+                ? _getCodecMimeType(config.videoQuality?.mobileScreenshareCodec)
+                : _getCodecMimeType(config.videoQuality?.screenshareCodec)
         },
         p2p: {
             preferenceOrder: browser.isMobileDevice() && config.p2p?.mobileCodecPreferenceOrder
                 ? config.p2p.mobileCodecPreferenceOrder
                 : config.p2p?.codecPreferenceOrder,
             disabledCodec: _getCodecMimeType(config.p2p?.disabledCodec),
-            preferredCodec: _getCodecMimeType(config.p2p?.preferredCodec)
+            preferredCodec: _getCodecMimeType(config.p2p?.preferredCodec),
+            screenshareCodec: browser.isMobileDevice()
+                ? _getCodecMimeType(config.p2p?.mobileScreenshareCodec)
+                : _getCodecMimeType(config.p2p?.screenshareCodec)
         }
     };
 
@@ -2226,7 +2232,8 @@ JitsiConference.prototype._acceptJvbIncomingCall = function(jingleSession, jingl
                 ...this.options.config,
                 codecSettings: {
                     mediaType: MediaType.VIDEO,
-                    codecList: this.codecSelection.getCodecPreferenceList('jvb')
+                    codecList: this.codecSelection.getCodecPreferenceList('jvb'),
+                    screenshareCodec: this.codecSelection.getScreenshareCodec('jvb')
                 },
                 enableInsertableStreams: this.isE2EEEnabled() || FeatureFlags.isRunInLiteModeEnabled()
             });
@@ -2936,7 +2943,8 @@ JitsiConference.prototype._acceptP2PIncomingCall = function(jingleSession, jingl
             ...this.options.config,
             codecSettings: {
                 mediaType: MediaType.VIDEO,
-                codecList: this.codecSelection.getCodecPreferenceList('p2p')
+                codecList: this.codecSelection.getCodecPreferenceList('p2p'),
+                screenshareCodec: this.codecSelection.getScreenshareCodec('p2p')
             },
             enableInsertableStreams: this.isE2EEEnabled() || FeatureFlags.isRunInLiteModeEnabled()
         });
@@ -3291,7 +3299,8 @@ JitsiConference.prototype._startP2PSession = function(remoteJid) {
             ...this.options.config,
             codecSettings: {
                 mediaType: MediaType.VIDEO,
-                codecList: this.codecSelection.getCodecPreferenceList('p2p')
+                codecList: this.codecSelection.getCodecPreferenceList('p2p'),
+                screenshareCodec: this.codecSelection.getScreenshareCodec('p2p')
             },
             enableInsertableStreams: this.isE2EEEnabled() || FeatureFlags.isRunInLiteModeEnabled()
         });

JitsiConferenceEventManager.js

@@ -737,6 +737,11 @@ JitsiConferenceEventManager.prototype.setupStatisticsListeners = function() {
             JitsiConferenceEvents.BEFORE_STATISTICS_DISPOSED);
     });
 
+    conference.statistics.addEncodeTimeStatsListener((tpc, stats) => {
+        conference.eventEmitter.emit(
+            JitsiConferenceEvents.ENCODE_TIME_STATS_RECEIVED, tpc, stats);
+    });
+
     // if we are in startSilent mode we will not be sending/receiving so nothing to detect
     if (!conference.options.config.startSilent) {
         conference.statistics.addByteSentStatsListener((tpc, stats) => {

JitsiConferenceEvents.spec.ts

@@ -27,6 +27,7 @@ describe( "/JitsiConferenceEvents members", () => {
         E2EE_VERIFICATION_AVAILABLE,
         E2EE_VERIFICATION_READY,
         E2EE_VERIFICATION_COMPLETED,
+        ENCODE_TIME_STATS_RECEIVED,
         ENDPOINT_MESSAGE_RECEIVED,
         ENDPOINT_STATS_RECEIVED,
         JVB121_STATUS,
@@ -168,6 +169,7 @@ describe( "/JitsiConferenceEvents members", () => {
         expect( BREAKOUT_ROOMS_UPDATED ).toBe( 'conference.breakout-rooms.updated' );
         expect( METADATA_UPDATED ).toBe( 'conference.metadata.updated' );
         expect( SILENT_STATUS_CHANGED ).toBe( 'conference.silentStatusChanged' );
+        expect( ENCODE_TIME_STATS_RECEIVED ).toBe( 'conference.encode_time_stats_received' );
 
         expect( JitsiConferenceEvents ).toBeDefined();
 
@@ -190,6 +192,7 @@ describe( "/JitsiConferenceEvents members", () => {
         expect( JitsiConferenceEvents.DOMINANT_SPEAKER_CHANGED ).toBe( 'conference.dominantSpeaker' );
         expect( JitsiConferenceEvents.CONFERENCE_CREATED_TIMESTAMP ).toBe( 'conference.createdTimestamp' );
         expect( JitsiConferenceEvents.DTMF_SUPPORT_CHANGED ).toBe( 'conference.dtmfSupportChanged' );
+        expect( JitsiConferenceEvents.ENCODE_TIME_STATS_RECEIVED ).toBe( 'conference.encode_time_stats_received' );
         expect( JitsiConferenceEvents.ENDPOINT_MESSAGE_RECEIVED ).toBe( 'conference.endpoint_message_received' );
         expect( JitsiConferenceEvents.ENDPOINT_STATS_RECEIVED ).toBe( 'conference.endpoint_stats_received' );
         expect( JitsiConferenceEvents.JVB121_STATUS ).toBe( 'conference.jvb121Status' );

JitsiConferenceEvents.ts

@@ -193,6 +193,11 @@ export enum JitsiConferenceEvents {
 
     E2EE_VERIFICATION_READY = 'conference.e2ee.verification.ready',
 
+    /**
+     * Indicates that the encode time stats for the local video sources has been received.
+     */
+    ENCODE_TIME_STATS_RECEIVED = 'conference.encode_time_stats_received',
+
     /**
      * Indicates that a message from another participant is received on data
      * channel.
@@ -523,6 +528,7 @@ export const DTMF_SUPPORT_CHANGED = JitsiConferenceEvents.DTMF_SUPPORT_CHANGED;
 export const E2EE_VERIFICATION_AVAILABLE = JitsiConferenceEvents.E2EE_VERIFICATION_AVAILABLE;
 export const E2EE_VERIFICATION_COMPLETED = JitsiConferenceEvents.E2EE_VERIFICATION_COMPLETED;
 export const E2EE_VERIFICATION_READY = JitsiConferenceEvents.E2EE_VERIFICATION_READY;
+export const ENCODE_TIME_STATS_RECEIVED = JitsiConferenceEvents.ENCODE_TIME_STATS_RECEIVED;
 export const ENDPOINT_MESSAGE_RECEIVED = JitsiConferenceEvents.ENDPOINT_MESSAGE_RECEIVED;
 export const ENDPOINT_STATS_RECEIVED = JitsiConferenceEvents.ENDPOINT_STATS_RECEIVED;
 export const FORWARDED_SOURCES_CHANGED = JitsiConferenceEvents.FORWARDED_SOURCES_CHANGED;

JitsiConnectionErrors.spec.ts

@@ -5,6 +5,7 @@ import * as exported from "./JitsiConnectionErrors";
 describe( "/JitsiConnectionErrors members", () => {
     const {
         CONNECTION_DROPPED_ERROR,
+        NOT_LIVE_ERROR,
         OTHER_ERROR,
         PASSWORD_REQUIRED,
         SERVER_ERROR,
@@ -21,6 +22,7 @@ describe( "/JitsiConnectionErrors members", () => {
         expect( JitsiConnectionErrors ).toBeDefined();
 
         expect( JitsiConnectionErrors.CONNECTION_DROPPED_ERROR ).toBe( 'connection.droppedError' );
+        expect( JitsiConnectionErrors.NOT_LIVE_ERROR ).toBe( 'connection.notLiveError' );
         expect( JitsiConnectionErrors.OTHER_ERROR ).toBe( 'connection.otherError' );
         expect( JitsiConnectionErrors.PASSWORD_REQUIRED ).toBe( 'connection.passwordRequired' );
         expect( JitsiConnectionErrors.SERVER_ERROR ).toBe( 'connection.serverError' );
@@ -30,4 +32,4 @@ describe( "/JitsiConnectionErrors members", () => {
         const keys = Object.keys( others );
         expect( keys ).withContext( `Extra members: ${ keys.join( ", " ) }` ).toEqual( [] );
     } );
-} );
+} );

JitsiConnectionErrors.ts

@@ -16,6 +16,11 @@ export enum JitsiConnectionErrors {
      */
     CONNECTION_DROPPED_ERROR = 'connection.droppedError',
 
+    /**
+     * Not ready error. When the conference error is not ready according to jicofo.
+     */
+    NOT_LIVE_ERROR = 'connection.notLiveError',
+
     /**
      * Not specified errors.
      */
@@ -35,6 +40,7 @@ export enum JitsiConnectionErrors {
 
 // exported for backward compatibility
 export const CONNECTION_DROPPED_ERROR = JitsiConnectionErrors.CONNECTION_DROPPED_ERROR;
+export const NOT_LIVE_ERROR = JitsiConnectionErrors.NOT_LIVE_ERROR;
 export const OTHER_ERROR = JitsiConnectionErrors.OTHER_ERROR;
 export const PASSWORD_REQUIRED = JitsiConnectionErrors.PASSWORD_REQUIRED;
 export const SERVER_ERROR = JitsiConnectionErrors.SERVER_ERROR;

modules/RTC/CodecSelection.js

@@ -30,14 +30,16 @@ export class CodecSelection {
      * @param {string} options.p2p settings (codec list, preferred and disabled) for the p2p connection.
      */
     constructor(conference, options) {
+        this.codecPreferenceOrder = {};
         this.conference = conference;
+        this.encodeTimeStats = new Map();
         this.options = options;
-        this.codecPreferenceOrder = {};
+        this.screenshareCodec = {};
         this.visitorCodecs = [];
 
         for (const connectionType of Object.keys(options)) {
             // eslint-disable-next-line prefer-const
-            let { disabledCodec, preferredCodec, preferenceOrder } = options[connectionType];
+            let { disabledCodec, preferredCodec, preferenceOrder, screenshareCodec } = options[connectionType];
             const supportedCodecs = new Set(this._getSupportedVideoCodecs(connectionType));
 
             // Default preference codec order when no codec preferences are set in config.js
@@ -89,6 +91,11 @@ export class CodecSelection {
 
             logger.info(`Codec preference order for ${connectionType} connection is ${selectedOrder}`);
             this.codecPreferenceOrder[connectionType] = selectedOrder;
+
+            // Set the preferred screenshare codec.
+            if (screenshareCodec && supportedCodecs.has(screenshareCodec)) {
+                this.screenshareCodec[connectionType] = screenshareCodec;
+            }
         }
 
         this.conference.on(
@@ -103,6 +110,9 @@ export class CodecSelection {
         this.conference.on(
             JitsiConferenceEvents.USER_LEFT,
             () => this._selectPreferredCodec());
+        this.conference.on(
+            JitsiConferenceEvents.ENCODE_TIME_STATS_RECEIVED,
+            (tpc, stats) => this._processEncodeTimeStats(tpc, stats));
     }
 
     /**
@@ -127,6 +137,84 @@ export class CodecSelection {
         return supportedCodecs;
     }
 
+    /**
+     * Processes the encode time stats received for all the local video sources.
+     *
+     * @param {TraceablePeerConnection} tpc - the peerconnection for which stats were gathered.
+     * @param {Object} stats - the encode time stats for local video sources.
+     * @returns {void}
+     */
+    _processEncodeTimeStats(tpc, stats) {
+        const activeSession = this.conference.getActiveMediaSession();
+
+        // Process stats only for the active media session.
+        if (activeSession.peerconnection !== tpc) {
+            return;
+        }
+
+        const statsPerTrack = new Map();
+
+        for (const ssrc of stats.keys()) {
+            const { codec, encodeTime, qualityLimitationReason, resolution, timestamp } = stats.get(ssrc);
+            const track = tpc.getTrackBySSRC(ssrc);
+            let existingStats = statsPerTrack.get(track.rtcId);
+            const encodeResolution = Math.min(resolution.height, resolution.width);
+            const ssrcStats = {
+                encodeResolution,
+                encodeTime,
+                qualityLimitationReason
+            };
+
+            if (existingStats) {
+                existingStats.codec = codec;
+                existingStats.timestamp = timestamp;
+                existingStats.trackStats.push(ssrcStats);
+            } else {
+                existingStats = {
+                    codec,
+                    timestamp,
+                    trackStats: [ ssrcStats ]
+                };
+
+                statsPerTrack.set(track.rtcId, existingStats);
+            }
+        }
+
+        // Aggregate the stats for multiple simulcast streams with different SSRCs but for the same video stream.
+        for (const trackId of statsPerTrack.keys()) {
+            const { codec, timestamp, trackStats } = statsPerTrack.get(trackId);
+            const totalEncodeTime = trackStats
+                .map(stat => stat.encodeTime)
+                .reduce((totalValue, currentValue) => totalValue + currentValue, 0);
+            const avgEncodeTime = totalEncodeTime / trackStats.length;
+            const { qualityLimitationReason = 'none' }
+                = trackStats.find(stat => stat.qualityLimitationReason !== 'none') ?? {};
+            const encodeResolution = trackStats
+                .map(stat => stat.encodeResolution)
+                .reduce((resolution, currentValue) => Math.max(resolution, currentValue), 0);
+            const localTrack = this.conference.getLocalVideoTracks().find(t => t.rtcId === trackId);
+
+            const exisitingStats = this.encodeTimeStats.get(trackId);
+            const sourceStats = {
+                avgEncodeTime,
+                codec,
+                encodeResolution,
+                qualityLimitationReason,
+                localTrack,
+                timestamp
+            };
+
+            if (exisitingStats) {
+                exisitingStats.push(sourceStats);
+            } else {
+                this.encodeTimeStats.set(trackId, [ sourceStats ]);
+            }
+
+            logger.debug(`Encode stats for ${localTrack}: codec=${codec}, time=${avgEncodeTime},`
+                + `resolution=${encodeResolution}, qualityLimitationReason=${qualityLimitationReason}`);
+        }
+    }
+
     /**
      * Sets the codec on the media session based on the codec preference order configured in config.js and the supported
      * codecs published by the remote participants in their presence.
@@ -139,9 +227,7 @@ export class CodecSelection {
         if (!session) {
             return;
         }
-        const currentCodecOrder = session.peerconnection.getConfiguredVideoCodecs();
         const isJvbSession = session === this.conference.jvbJingleSession;
-
         let localPreferredCodecOrder = isJvbSession ? this.codecPreferenceOrder.jvb : this.codecPreferenceOrder.p2p;
 
         // E2EE is curently supported only for VP8 codec.
@@ -195,10 +281,7 @@ export class CodecSelection {
             return;
         }
 
-        // Reconfigure the codecs on the media session.
-        if (!selectedCodecOrder.every((val, index) => val === currentCodecOrder[index])) {
-            session.setVideoCodecs(selectedCodecOrder);
-        }
+        session.setVideoCodecs(selectedCodecOrder);
     }
 
     /**
@@ -225,4 +308,14 @@ export class CodecSelection {
     getCodecPreferenceList(connectionType) {
         return this.codecPreferenceOrder[connectionType];
     }
+
+    /**
+     * Returns the preferred screenshare codec for the given connection type.
+     *
+     * @param {String} connectionType The media connection type, 'p2p' or 'jvb'.
+     * @returns CodecMimeType
+     */
+    getScreenshareCodec(connectionType) {
+        return this.screenshareCodec[connectionType];
+    }
 }

modules/RTC/CodecSelection.spec.js

@@ -135,7 +135,7 @@ describe('Codec Selection', () => {
             participant1 = new MockParticipant('remote-1');
             conference.addParticipant(participant1, [ 'vp9', 'vp8' ]);
 
-            expect(jingleSession.setVideoCodecs).toHaveBeenCalledTimes(0);
+            expect(jingleSession.setVideoCodecs).toHaveBeenCalledTimes(1);
 
             // Add a third user joining the call with a subset of codecs.
             participant2 = new MockParticipant('remote-2');
@@ -145,7 +145,7 @@ describe('Codec Selection', () => {
 
             // Make p2 leave the call
             conference.removeParticipant(participant2);
-            expect(jingleSession.setVideoCodecs).toHaveBeenCalledTimes(1);
+            expect(jingleSession.setVideoCodecs).toHaveBeenCalledTimes(3);
         });
 
         it('and remote endpoints use the old codec selection logic (RN)', () => {
@@ -163,7 +163,7 @@ describe('Codec Selection', () => {
 
             // Make p1 leave the call
             conference.removeParticipant(participant1);
-            expect(jingleSession.setVideoCodecs).toHaveBeenCalledTimes(2);
+            expect(jingleSession.setVideoCodecs).toHaveBeenCalledTimes(3);
         });
     });
 
@@ -185,7 +185,7 @@ describe('Codec Selection', () => {
             participant1 = new MockParticipant('remote-1');
             conference.addParticipant(participant1, [ 'vp9', 'vp8', 'h264' ]);
 
-            expect(jingleSession.setVideoCodecs).toHaveBeenCalledTimes(0);
+            expect(jingleSession.setVideoCodecs).toHaveBeenCalledTimes(1);
 
             // Add a third user joining the call with a subset of codecs.
             participant2 = new MockParticipant('remote-2');
@@ -195,7 +195,7 @@ describe('Codec Selection', () => {
 
             // Make p2 leave the call
             conference.removeParticipant(participant2);
-            expect(jingleSession.setVideoCodecs).toHaveBeenCalledTimes(1);
+            expect(jingleSession.setVideoCodecs).toHaveBeenCalledTimes(3);
         });
 
         it('and remote endpoint prefers a codec that is locally disabled', () => {
@@ -221,7 +221,7 @@ describe('Codec Selection', () => {
 
             // Make p1 leave the call
             conference.removeParticipant(participant1);
-            expect(jingleSession.setVideoCodecs).toHaveBeenCalledTimes(2);
+            expect(jingleSession.setVideoCodecs).toHaveBeenCalledTimes(3);
         });
     });
 });