Skip to content

Commit

Permalink
Merge pull request #127 from alancutter/testEVERYTHINGAAAA
Browse files Browse the repository at this point in the history 
Add testing support for toggling native fallback and configuration specific failure expectations
  • Loading branch information
@ewilligers
ewilligers authored Apr 25, 2017
2 parents ef2aa01 + 939ec79 commit c576378
Show file tree
Hide file tree
Showing 4 changed files with 2,340 additions and 939 deletions.
18 changes: 13 additions & 5 deletions Gruntfile.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -206,6 +206,7 @@ module.exports = function(grunt) {
}

function filterTests(testFiles) {
console.assert(testFiles.length > 0);
if (!testFilter) {
return testFiles;
}
Expand All @@ -226,18 +227,20 @@ module.exports = function(grunt) {
karmaConfig.files = ['test/karma-mocha-setup.js'].concat(config.src, testFiles);
});
}
function runWebPlatformTests() {
function runWebPlatformTests(withNativeFallback) {
var testFiles = filterTests(grunt.file.expand(config.webPlatformTests));
if (testFiles.length == 0) {
return Promise.resolve(true);
}

console.info('Running web-platform-tests/web-animations tests...');
var withOrWithout = withNativeFallback ? 'with' : 'without';
console.info('Running web-platform-tests/web-animations tests for target ' + task.target + ' ' + withOrWithout + ' native fallback...');
return runKarma(function(karmaConfig) {
configCallback(karmaConfig);
karmaConfig.client.testharnessTests = require('./test/web-platform-tests-expectations.js');
karmaConfig.client.testharnessTests.testURLList = testFiles;
karmaConfig.proxies['/base/polyfill.js'] = '/base/' + task.target + '.min.js';
karmaConfig.client.testharnessTests.target = task.target;
karmaConfig.client.testharnessTests.withNativeFallback = withNativeFallback;
karmaConfig.files.push('test/karma-testharness-adapter.js');
var servedFiles = [
'test/web-platform-tests/resources/**',
Expand All @@ -256,10 +259,15 @@ module.exports = function(grunt) {
}

var polyfillTestsPassed = false;
var webPlatformTestsWithFallbackPassed = false;
var webPlatformTestsWithoutFallbackPassed = false;
runPolyfillTests().then(success => {
polyfillTestsPassed = success;
}).then(runWebPlatformTests).then(webPlatformTestsPassed => {
done(polyfillTestsPassed && webPlatformTestsPassed);
}).then(() => runWebPlatformTests(true)).then(success => {
webPlatformTestsWithFallbackPassed = success;
}).then(() => runWebPlatformTests(false)).then(success => {
webPlatformTestsWithoutFallbackPassed = success;
done(polyfillTestsPassed && webPlatformTestsWithFallbackPassed && webPlatformTestsWithoutFallbackPassed);
}).catch(function(error) {
console.error(error);
done(false);
Expand Down
67 changes: 54 additions & 13 deletions test/karma-testharness-adapter.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,12 @@
// Behaves like JSON.stringify() except only for strings and outputs strings with single quotes
// instead of double quotes.
// This so we can paste test results as expectations while keeping our linter happy.
function stringify(string) {
return '\'' + string.replace(/\\/g, '\\\\').replace(/\n/g, '\\n').replace(/'/g, '\\\'') + '\'';
function stringify(x) {
if (typeof x == 'string') {
return '\'' + x.replace(/\\/g, '\\\\').replace(/\n/g, '\\n').replace(/'/g, '\\\'') + '\'';
}
console.assert(typeof x != 'object');
return x.toString();
}

function checkExpectations(testURL, passes, failures, expectedFailures) {
Expand Down Expand Up @@ -98,10 +102,12 @@

function checkConfig(config) {
var requiredProperties = {
target: '<String name of polyfill target being tested>',
testURLList: '<Array of test URLs>',
skip: '<Object mapping skipped test URLs to the reason for skipping>',
expectedFailures: '<Object mapping test URLs to expected inner test failures>',
failureConfigurations: '<Array of objects mapping test configuration to test URLs to expected inner test failures>',
flakyTestIndicator: '<String used in expectedFailures to indicate flaky test (pass/fail)>',
withNativeFallback: '<Boolean to indicate whether the native browser fallback should be included>',
};
var errorMessage = '';
if (!config) {
Expand All @@ -125,29 +131,64 @@
return true;
}

var filteredConfigurationAttributes = ['target', 'withNativeFallback'];

// Serialises the failures suitable for pasting into expectedFailures: {} in web-platform-tests-expectations.js
function formatFailures(failures) {
function formatFailures(config, failures) {
var result = ' {\n';

result += ' configuration: {\n';
filteredConfigurationAttributes.forEach(function(attribute) {
result += ' ' + attribute + ': ' + stringify(config[attribute]) + ',\n';
});
result += ' },\n';

result += ' failures: {\n';
var testURLs = Object.keys(failures);
testURLs.sort();
return testURLs.map(function(testURL) {
result += testURLs.map(function(testURL) {
var tests = Object.keys(failures[testURL]);
tests.sort();
return (
' ' + stringify(testURL) + ': {\n' +
' ' + stringify(testURL) + ': {\n' +
tests.map(function(test) {
return (
' ' + stringify(test) + ':\n' +
' ' + stringify(failures[testURL][test]) + ',\n');
' ' + stringify(test) + ':\n' +
' ' + stringify(failures[testURL][test]) + ',\n');
}).join('\n') +
' },\n');
' },\n');
}).join('\n');
result += ' },\n';

result += ' },\n';
return result;
}

function getExpectedFailures(config, testURL) {
var result = {};
config.failureConfigurations.forEach(function(failureConfiguration) {
var configFilter = failureConfiguration.configuration;
var filterMatches = filteredConfigurationAttributes.every(function(attribute) {
console.assert(attribute in config);
console.assert(attribute in configFilter);
return configFilter[attribute] == null || config[attribute] == configFilter[attribute];
});
if (!filterMatches) {
return;
}
var testURLFailures = failureConfiguration.failures[testURL] || [];
for (var testName in testURLFailures) {
result[testName] = testURLFailures[testName];
}
});
return result;
}

function runRemainingTests(remainingTestURLs, config, testNameDiv, iframe, outputFailures) {
if (remainingTestURLs.length == 0) {
karma.complete();
window.failures = outputFailures;
window.formattedFailures = formatFailures(outputFailures);
window.formattedFailures = formatFailures(config, outputFailures);
return;
}

Expand All @@ -167,11 +208,11 @@
// parent window and call it once testharness.js has loaded.
window.onTestharnessLoaded = function(innerWindow) {
innerWindow.add_completion_callback(function(results) {
var expectations = config.expectedFailures[testURL];
var expectedFailures = getExpectedFailures(config, testURL);
var failures = {};
var passes = {};
results.forEach(function(result) {
if (expectations && expectations[result.name] == config.flakyTestIndicator) {
if (expectedFailures && expectedFailures[result.name] == config.flakyTestIndicator) {
failures[result.name] = config.flakyTestIndicator;
return;
}
Expand All @@ -188,7 +229,7 @@
outputFailures[testURL] = failures;
}

karma.result(checkExpectations(testURL, passes, failures, expectations));
karma.result(checkExpectations(testURL, passes, failures, expectedFailures));
runRemainingTests(remainingTestURLs.slice(1), config, testNameDiv, iframe, outputFailures);
});
};
Expand Down
17 changes: 7 additions & 10 deletions test/resources/testharnessreport.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -407,17 +407,14 @@ function dump_test_results(tests, status) {

/* BEGIN WEB ANIMATIONS POLYFILL EXTRAS */

// Disable native Web Animations fallback.
// TODO(alancutter): Make this configurable in testing targets.
Element.prototype.animate = null;

// The path /base/polyfill.js is expected to be a proxy for the appropriate polyfill script configured in Karma.
document.write('<script src="/base/polyfill.js"></script>');
if (window.parent && parent.window.onTestharnessLoaded) {
parent.window.onTestharnessLoaded(window);
} else {
metadata_generator.setup();
// This must be run under a Karma server via "grunt test".
var config = window.parent.__karma__.config.testharnessTests;
if (!config.withNativeFallback) {
// Disable native Web Animations fallback.
Element.prototype.animate = null;
}
document.write('<script src="/base/' + config.target + '.min.js"></script>');
parent.window.onTestharnessLoaded(window);

/* END WEB ANIMATIONS POLYFILL EXTRAS */

Expand Down
Loading

0 comments on commit c576378

Please sign in to comment.