diff --git a/protractor.saucelabs.config.js b/protractor.saucelabs.config.js index 02b7a1190..deaaaa7ec 100644 --- a/protractor.saucelabs.config.js +++ b/protractor.saucelabs.config.js @@ -9,28 +9,25 @@ exports.config = { sauceBuild: process.env.GITHUB_RUN_NUMBER, // See https://wiki.saucelabs.com/display/DOCS/Platform+Configurator#/ multiCapabilities: [ + // Old browser+OS platforms for non-web-vitals tests newSaucelabsCapability('internet explorer', '11.103', 'Windows 10'), newSaucelabsCapability('MicrosoftEdge', '14.14393', 'Windows 10'), - newSaucelabsCapability('MicrosoftEdge', '79.0', 'Windows 10'), - newSaucelabsCapability('MicrosoftEdge', '80.0', 'Windows 10'), - newSaucelabsCapability('MicrosoftEdge', '96.0', 'Windows 10'), newSaucelabsCapability('safari', '9.0', 'OS X 10.11'), newSaucelabsCapability('safari', '10.1', 'macOS 10.12'), newSaucelabsCapability('safari', '11.0', 'macOS 10.12'), newSaucelabsCapability('safari', '11.1', 'macOS 10.13'), - newSaucelabsCapability('firefox', '58.0', 'Windows 11'), newSaucelabsCapability('firefox', '78.0', 'Windows 7'), - newSaucelabsCapability('firefox', '89.0', 'Windows 11'), - newSaucelabsCapability('firefox', '122.0', 'Windows 11'), - newSaucelabsCapability('firefox', '123.0', 'Windows 11'), + newSaucelabsCapability('firefox', '58.0', 'Windows 11'), newSaucelabsCapability('chrome', '67.0', 'Windows 10'), newSaucelabsCapability('chrome', '54.0', 'OS X 10.11'), newSaucelabsCapability('chrome', '65.0', 'OS X 10.11'), - newSaucelabsCapability('chrome', '76.0', 'Windows 10'), - newSaucelabsCapability('chrome', '77.0', 'Windows 10'), - newSaucelabsCapability('chrome', '78.0', 'Windows 10'), - newSaucelabsCapability('chrome', '79.0', 'Windows 10'), - newSaucelabsCapability('chrome', '96.0', 'Windows 10') + + // Specific supported browser+OS for web-vitals tests + newSaucelabsCapability('chrome', '80.0', 'Windows 10', true), + newSaucelabsCapability('chrome', '83.0', 'OS X 10.11', true), + newSaucelabsCapability('MicrosoftEdge', '80.0', 'Windows 10', true), + newSaucelabsCapability('firefox', '123.0', 'Windows 7', true), + newSaucelabsCapability('firefox', '123.0', 'Windows 11', true) ], // Do not allow parallel test execution. Makes the test execution a lot // slower, but the setup simpler. @@ -40,13 +37,17 @@ exports.config = { } }; -function newSaucelabsCapability(browserName, version, platform) { +function newSaucelabsCapability(browserName, version, platform, isWebVitalsTest = false) { return { browserName, version, platform, - name: 'weasel e2e', + shardTestFiles: true, // allows specs to be executed in parallel. + maxInstances: 2, // total number of specs that can be run at once. + name: isWebVitalsTest ? 'weasel e2e - web vitals' : 'weasel e2e', 'tunnel-identifier': 'github-action-tunnel', - build: process.env.GITHUB_RUN_NUMBER + build: process.env.GITHUB_RUN_NUMBER, + specs: isWebVitalsTest ? ['test/e2e/12_webvitalsAsCustomEvent/*.spec.js'] : ['test/e2e/**/*.spec.js'], + exclude: isWebVitalsTest ? [] : ['test/e2e/12_webvitalsAsCustomEvent/*.spec.js'] }; } diff --git a/test/e2e/12_webvitalsAsCustomEvent/webvitalsAsCustomEvent.spec.js b/test/e2e/12_webvitalsAsCustomEvent/webvitalsAsCustomEvent.spec.js index 8a7dd3b59..94488a0f6 100644 --- a/test/e2e/12_webvitalsAsCustomEvent/webvitalsAsCustomEvent.spec.js +++ b/test/e2e/12_webvitalsAsCustomEvent/webvitalsAsCustomEvent.spec.js @@ -9,90 +9,91 @@ describe('12_webvitalsAsCustomEvent', () => { registerBaseHooks(); describe('webvitalsAsCustomEvent', () => { + beforeEach(async () => { + // webvital CLS does not work if following another test, so restart browser to cleanup all context + await restartBrowser(); + + await browser.get(getE2ETestBaseUrl('12_webvitalsAsCustomEvent/webvitalsAsCustomEvent')); + // wait for webvital metrics + await browser.sleep(5000); + await performUserActions(); + + async function performUserActions() { + await element(by.id('searchInput')).sendKeys('hello'); + await element(by.id('searchButton')).click(); + await browser.sleep(1000); + await element(by.id('button3')).click(); + await browser.sleep(1000); + await element(by.id('button2')).click(); + await browser.sleep(1000); + + const currentHandle = browser.getWindowHandle(); + await element(by.id('open-blank-tab')).click(); + await browser.sleep(3000); + await browser.switchTo().window(currentHandle); + await browser.sleep(3000); + } + }); // isLCPTestApplicable function isLCPTestApplicable(capabilities) { const version = Number(capabilities.version); return ( (capabilities.browserName === 'chrome' && version > 77) || (capabilities.browserName === 'MicrosoftEdge' && version > 79) || - (capabilities.browserName === 'firefox' && version > 122) + (capabilities.browserName === 'Firefox' && version > 122) ); } - beforeEach(() => { - // webvital CLS does not work if following another test, so restart browser to cleanup all context - restartBrowser(); + fit('must report web-vitals as custom events', async () => { + const capabilities = await getCapabilities(); - browser.get(getE2ETestBaseUrl('12_webvitalsAsCustomEvent/webvitalsAsCustomEvent')); - // wait for webvital metrics - browser.sleep(5000); - element(by.id('searchInput')).sendKeys('hello'); - element(by.id('searchButton')).click(); - browser.sleep(1000); - element(by.id('button3')).click(); - browser.sleep(1000); - element(by.id('button2')).click(); - browser.sleep(1000); - - const currentHandle = browser.getWindowHandle(); - element(by.id('open-blank-tab')).click(); - browser.sleep(3000); - browser.switchTo().window(currentHandle); - browser.sleep(3000); - }); + if (!isLCPTestApplicable(capabilities)) { + return true; + } + await retry(async () => { + const beacons = await getBeacons(); + const pageLoadBeacon = expectOneMatching(beacons, beacon => { + cexpect(beacon.ty).to.equal('pl'); + }); - it('must report web-vitals as custom events', () => { - return getCapabilities().then(capabilities => { - if (!isLCPTestApplicable(capabilities)) { - return true; + if (isLCPTestApplicable(capabilities)) { + expectOneMatching(beacons, beacon => { + cexpect(beacon.ty).to.equal('cus'); + cexpect(beacon.ts).to.be.a('string'); + cexpect(parseFloat(beacon.d)).to.be.above(3000); + cexpect(beacon.n).to.equal('instana-webvitals-LCP'); + cexpect(beacon.l).to.be.a('string'); + cexpect(beacon.pl).to.equal(pageLoadBeacon.t); + cexpect(beacon.m_id).to.match(/^v\d+(-\d+)+$/); + }); } - return retry(() => { - return getBeacons().then(beacons => { - const pageLoadBeacon = expectOneMatching(beacons, beacon => { - cexpect(beacon.ty).to.equal('pl'); - }); - // LCP test: Run for chrome > 77, MicrosoftEdge > 79, firefox > 122 - if (isLCPTestApplicable(capabilities)) { - expectOneMatching(beacons, beacon => { - cexpect(beacon.ty).to.equal('cus'); - cexpect(beacon.ts).to.be.a('string'); - cexpect(parseFloat(beacon.d)).to.be.above(3000); - cexpect(beacon.n).to.equal('instana-webvitals-LCP'); - cexpect(beacon.l).to.be.a('string'); - cexpect(beacon.pl).to.equal(pageLoadBeacon.t); - cexpect(beacon.m_id).to.match(/^v\d+(-\d+)+$/); - }); - } + // expectOneMatching(beacons, beacon => { + // cexpect(beacon.ty).to.equal('cus'); + // cexpect(beacon.ts).to.be.a('string'); + // cexpect(beacon.n).to.equal('instana-webvitals-FID'); + // cexpect(beacon.l).to.be.a('string'); + // cexpect(beacon.pl).to.equal(pageLoadBeacon.t); + // cexpect(beacon.m_id).to.match(/^v\d+(-\d+)+$/); + // }); - // expectOneMatching(beacons, beacon => { - // cexpect(beacon.ty).to.equal('cus'); - // cexpect(beacon.ts).to.be.a('string'); - // cexpect(beacon.n).to.equal('instana-webvitals-FID'); - // cexpect(beacon.l).to.be.a('string'); - // cexpect(beacon.pl).to.equal(pageLoadBeacon.t); - // cexpect(beacon.m_id).to.match(/^v\d+(-\d+)+$/); - // }); + // expectOneMatching(beacons, beacon => { + // cexpect(beacon.ty).to.equal('cus'); + // cexpect(beacon.ts).to.be.a('string'); + // cexpect(beacon.n).to.equal('instana-webvitals-CLS'); + // cexpect(beacon.l).to.be.a('string'); + // cexpect(beacon.pl).to.equal(pageLoadBeacon.t); + // cexpect(beacon.m_id).to.match(/^v\d+(-\d+)+$/); + // }); - // expectOneMatching(beacons, beacon => { - // cexpect(beacon.ty).to.equal('cus'); - // cexpect(beacon.ts).to.be.a('string'); - // cexpect(beacon.n).to.equal('instana-webvitals-CLS'); - // cexpect(beacon.l).to.be.a('string'); - // cexpect(beacon.pl).to.equal(pageLoadBeacon.t); - // cexpect(beacon.m_id).to.match(/^v\d+(-\d+)+$/); - // }); - - // expectOneMatching(beacons, beacon => { - // cexpect(beacon.ty).to.equal('cus'); - // cexpect(beacon.ts).to.be.a('string'); - // cexpect(beacon.n).to.equal('instana-webvitals-INP'); - // cexpect(beacon.l).to.be.a('string'); - // cexpect(beacon.pl).to.equal(pageLoadBeacon.t); - // cexpect(beacon.m_id).to.match(/^v\d+(-\d+)+$/); - // }); - }); - }); + // expectOneMatching(beacons, beacon => { + // cexpect(beacon.ty).to.equal('cus'); + // cexpect(beacon.ts).to.be.a('string'); + // cexpect(beacon.n).to.equal('instana-webvitals-INP'); + // cexpect(beacon.l).to.be.a('string'); + // cexpect(beacon.pl).to.equal(pageLoadBeacon.t); + // cexpect(beacon.m_id).to.match(/^v\d+(-\d+)+$/); + // }); }); }); });