Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add testing support for toggling native fallback and configuration specific failure expectations #127

Merged
merged 2 commits into from
Apr 25, 2017
Merged

Add testing support for toggling native fallback and configuration specific failure expectations #127

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
55ee858
testEVERYTHINGAAAA
alancutter Apr 19, 2017
939ec79
lint
alancutter Apr 20, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 13 additions & 5 deletions Gruntfile.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -206,6 +206,7 @@ module.exports = function(grunt) {
}

function filterTests(testFiles) {
console.assert(testFiles.length > 0);
if (!testFilter) {
return testFiles;
}
Expand All @@ -226,18 +227,20 @@ module.exports = function(grunt) {
karmaConfig.files = ['test/karma-mocha-setup.js'].concat(config.src, testFiles);
});
}
function runWebPlatformTests() {
function runWebPlatformTests(withNativeFallback) {
var testFiles = filterTests(grunt.file.expand(config.webPlatformTests));
if (testFiles.length == 0) {
return Promise.resolve(true);
}

console.info('Running web-platform-tests/web-animations tests...');
var withOrWithout = withNativeFallback ? 'with' : 'without';
console.info('Running web-platform-tests/web-animations tests for target ' + task.target + ' ' + withOrWithout + ' native fallback...');
return runKarma(function(karmaConfig) {
configCallback(karmaConfig);
karmaConfig.client.testharnessTests = require('./test/web-platform-tests-expectations.js');
karmaConfig.client.testharnessTests.testURLList = testFiles;
karmaConfig.proxies['/base/polyfill.js'] = '/base/' + task.target + '.min.js';
karmaConfig.client.testharnessTests.target = task.target;
karmaConfig.client.testharnessTests.withNativeFallback = withNativeFallback;
karmaConfig.files.push('test/karma-testharness-adapter.js');
var servedFiles = [
'test/web-platform-tests/resources/**',
Expand All @@ -256,10 +259,15 @@ module.exports = function(grunt) {
}

var polyfillTestsPassed = false;
var webPlatformTestsWithFallbackPassed = false;
var webPlatformTestsWithoutFallbackPassed = false;
runPolyfillTests().then(success => {
polyfillTestsPassed = success;
}).then(runWebPlatformTests).then(webPlatformTestsPassed => {
done(polyfillTestsPassed && webPlatformTestsPassed);
}).then(() => runWebPlatformTests(true)).then(success => {
webPlatformTestsWithFallbackPassed = success;
}).then(() => runWebPlatformTests(false)).then(success => {
webPlatformTestsWithoutFallbackPassed = success;
done(polyfillTestsPassed && webPlatformTestsWithFallbackPassed && webPlatformTestsWithoutFallbackPassed);
}).catch(function(error) {
console.error(error);
done(false);
Expand Down
67 changes: 54 additions & 13 deletions test/karma-testharness-adapter.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,12 @@
// Behaves like JSON.stringify() except only for strings and outputs strings with single quotes
// instead of double quotes.
// This so we can paste test results as expectations while keeping our linter happy.
function stringify(string) {
return '\'' + string.replace(/\\/g, '\\\\').replace(/\n/g, '\\n').replace(/'/g, '\\\'') + '\'';
function stringify(x) {
if (typeof x == 'string') {
return '\'' + x.replace(/\\/g, '\\\\').replace(/\n/g, '\\n').replace(/'/g, '\\\'') + '\'';
}
console.assert(typeof x != 'object');
return x.toString();
}

function checkExpectations(testURL, passes, failures, expectedFailures) {
Expand Down Expand Up @@ -98,10 +102,12 @@

function checkConfig(config) {
var requiredProperties = {
target: '<String name of polyfill target being tested>',
testURLList: '<Array of test URLs>',
skip: '<Object mapping skipped test URLs to the reason for skipping>',
expectedFailures: '<Object mapping test URLs to expected inner test failures>',
failureConfigurations: '<Array of objects mapping test configuration to test URLs to expected inner test failures>',
flakyTestIndicator: '<String used in expectedFailures to indicate flaky test (pass/fail)>',
withNativeFallback: '<Boolean to indicate whether the native browser fallback should be included>',
};
var errorMessage = '';
if (!config) {
Expand All @@ -125,29 +131,64 @@
return true;
}

var filteredConfigurationAttributes = ['target', 'withNativeFallback'];

// Serialises the failures suitable for pasting into expectedFailures: {} in web-platform-tests-expectations.js
function formatFailures(failures) {
function formatFailures(config, failures) {
var result = ' {\n';

result += ' configuration: {\n';
filteredConfigurationAttributes.forEach(function(attribute) {
result += ' ' + attribute + ': ' + stringify(config[attribute]) + ',\n';
});
result += ' },\n';

result += ' failures: {\n';
var testURLs = Object.keys(failures);
testURLs.sort();
return testURLs.map(function(testURL) {
result += testURLs.map(function(testURL) {
var tests = Object.keys(failures[testURL]);
tests.sort();
return (
' ' + stringify(testURL) + ': {\n' +
' ' + stringify(testURL) + ': {\n' +
tests.map(function(test) {
return (
' ' + stringify(test) + ':\n' +
' ' + stringify(failures[testURL][test]) + ',\n');
' ' + stringify(test) + ':\n' +
' ' + stringify(failures[testURL][test]) + ',\n');
}).join('\n') +
' },\n');
' },\n');
}).join('\n');
result += ' },\n';

result += ' },\n';
return result;
}

function getExpectedFailures(config, testURL) {
var result = {};
config.failureConfigurations.forEach(function(failureConfiguration) {
var configFilter = failureConfiguration.configuration;
var filterMatches = filteredConfigurationAttributes.every(function(attribute) {
console.assert(attribute in config);
console.assert(attribute in configFilter);
return configFilter[attribute] == null || config[attribute] == configFilter[attribute];
});
if (!filterMatches) {
return;
}
var testURLFailures = failureConfiguration.failures[testURL] || [];
for (var testName in testURLFailures) {
result[testName] = testURLFailures[testName];
}
});
return result;
}

function runRemainingTests(remainingTestURLs, config, testNameDiv, iframe, outputFailures) {
if (remainingTestURLs.length == 0) {
karma.complete();
window.failures = outputFailures;
window.formattedFailures = formatFailures(outputFailures);
window.formattedFailures = formatFailures(config, outputFailures);
return;
}

Expand All @@ -167,11 +208,11 @@
// parent window and call it once testharness.js has loaded.
window.onTestharnessLoaded = function(innerWindow) {
innerWindow.add_completion_callback(function(results) {
var expectations = config.expectedFailures[testURL];
var expectedFailures = getExpectedFailures(config, testURL);
var failures = {};
var passes = {};
results.forEach(function(result) {
if (expectations && expectations[result.name] == config.flakyTestIndicator) {
if (expectedFailures && expectedFailures[result.name] == config.flakyTestIndicator) {
failures[result.name] = config.flakyTestIndicator;
return;
}
Expand All @@ -188,7 +229,7 @@
outputFailures[testURL] = failures;
}

karma.result(checkExpectations(testURL, passes, failures, expectations));
karma.result(checkExpectations(testURL, passes, failures, expectedFailures));
runRemainingTests(remainingTestURLs.slice(1), config, testNameDiv, iframe, outputFailures);
});
};
Expand Down
17 changes: 7 additions & 10 deletions test/resources/testharnessreport.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -407,17 +407,14 @@ function dump_test_results(tests, status) {

/* BEGIN WEB ANIMATIONS POLYFILL EXTRAS */

// Disable native Web Animations fallback.
// TODO(alancutter): Make this configurable in testing targets.
Element.prototype.animate = null;

// The path /base/polyfill.js is expected to be a proxy for the appropriate polyfill script configured in Karma.
document.write('<script src="/base/polyfill.js"></script>');
if (window.parent && parent.window.onTestharnessLoaded) {
parent.window.onTestharnessLoaded(window);
} else {
metadata_generator.setup();
// This must be run under a Karma server via "grunt test".
var config = window.parent.__karma__.config.testharnessTests;
if (!config.withNativeFallback) {
// Disable native Web Animations fallback.
Element.prototype.animate = null;
}
document.write('<script src="/base/' + config.target + '.min.js"></script>');
parent.window.onTestharnessLoaded(window);

/* END WEB ANIMATIONS POLYFILL EXTRAS */

Expand Down
Loading