From 4949cc2f9661596582ed7998c296a74975df0d87 Mon Sep 17 00:00:00 2001 From: Tami Takamiya Date: Tue, 2 Apr 2024 16:05:12 -0400 Subject: [PATCH] Re-enable extension_development_path option and add coverage support --- .gitignore | 3 +- package-lock.json | 178 ++++++++++++++++++++++++- package.json | 1 + packages/extester/package.json | 1 + packages/extester/src/browser.ts | 6 +- packages/extester/src/cli.ts | 11 +- packages/extester/src/extester.ts | 10 +- packages/extester/src/suite/runner.ts | 14 +- packages/extester/src/util/codeUtil.ts | 28 +++- packages/extester/src/util/coverage.ts | 84 ++++++++++++ tests/test-project/package.json | 3 +- 11 files changed, 322 insertions(+), 17 deletions(-) create mode 100644 packages/extester/src/util/coverage.ts diff --git a/.gitignore b/.gitignore index 1ee798d1d..59be38e3f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ out/ node_modules/ -.npmrc \ No newline at end of file +.npmrc +tests/test-project/coverage/ diff --git a/package-lock.json b/package-lock.json index ceafd4c40..3cdf818cd 100644 --- a/package-lock.json +++ b/package-lock.json @@ -144,6 +144,11 @@ "node": ">=4" } }, + "node_modules/@bcoe/v8-coverage": { + "version": "0.2.3", + "resolved": "https://registry.npmjs.org/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz", + "integrity": "sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==" + }, "node_modules/@eslint-community/eslint-utils": { "version": "4.4.0", "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.4.0.tgz", @@ -375,6 +380,14 @@ "url": "https://github.com/chalk/wrap-ansi?sponsor=1" } }, + "node_modules/@istanbuljs/schema": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/@istanbuljs/schema/-/schema-0.1.3.tgz", + "integrity": "sha512-ZXRY4jNvVgSVQ8DL3LTcakaAtXwTVUxE81hslsyD2AtoXW/wVob10HkOJ1X/pAlcI7D+2YoZKg5do8G/w6RYgA==", + "engines": { + "node": ">=8" + } + }, "node_modules/@jest/schemas": { "version": "29.6.3", "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-29.6.3.tgz", @@ -387,6 +400,28 @@ "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.4.15", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.15.tgz", + "integrity": "sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg==" + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.25", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.25.tgz", + "integrity": "sha512-vNk6aEwybGtawWmy/PzwnGDOjCkLWSD2wqvjGGAgOAwCGWySYXfYoxt00IJkTF+8Lb57DwOb3Aa0o9CApepiYQ==", + "dependencies": { + "@jridgewell/resolve-uri": "^3.1.0", + "@jridgewell/sourcemap-codec": "^1.4.14" + } + }, "node_modules/@lerna/create": { "version": "8.1.2", "resolved": "https://registry.npmjs.org/@lerna/create/-/create-8.1.2.tgz", @@ -1557,6 +1592,11 @@ "resolved": "https://registry.npmjs.org/@types/http-cache-semantics/-/http-cache-semantics-4.0.4.tgz", "integrity": "sha512-1m0bIFVc7eJWyve9S0RnuRgcQqF/Xd5QsUZAZeQFr1Q3/p9JWoQQEqmVy+DPTNpGXwhgIetAoYF8JSc33q29QA==" }, + "node_modules/@types/istanbul-lib-coverage": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.6.tgz", + "integrity": "sha512-2QF/t/auWm0lsy8XtKVPG19v3sSOQlJe/YHZgfjb/KBBHOGSV+J2q/S671rcq9uTBrLAXmZpqJiaQbMT+zNU1w==" + }, "node_modules/@types/js-yaml": { "version": "4.0.9", "resolved": "https://registry.npmjs.org/@types/js-yaml/-/js-yaml-4.0.9.tgz", @@ -2501,6 +2541,30 @@ "node": ">=12.17" } }, + "node_modules/c8": { + "version": "9.1.0", + "resolved": "https://registry.npmjs.org/c8/-/c8-9.1.0.tgz", + "integrity": "sha512-mBWcT5iqNir1zIkzSPyI3NCR9EZCVI3WUD+AVO17MVWTSFNyUueXE82qTeampNtTr+ilN/5Ua3j24LgbCKjDVg==", + "dependencies": { + "@bcoe/v8-coverage": "^0.2.3", + "@istanbuljs/schema": "^0.1.3", + "find-up": "^5.0.0", + "foreground-child": "^3.1.1", + "istanbul-lib-coverage": "^3.2.0", + "istanbul-lib-report": "^3.0.1", + "istanbul-reports": "^3.1.6", + "test-exclude": "^6.0.0", + "v8-to-istanbul": "^9.0.0", + "yargs": "^17.7.2", + "yargs-parser": "^21.1.1" + }, + "bin": { + "c8": "bin/c8.js" + }, + "engines": { + "node": ">=14.14.0" + } + }, "node_modules/cacache": { "version": "18.0.2", "resolved": "https://registry.npmjs.org/cacache/-/cacache-18.0.2.tgz", @@ -3014,7 +3078,6 @@ "version": "8.0.1", "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", - "dev": true, "dependencies": { "string-width": "^4.2.0", "strip-ansi": "^6.0.1", @@ -3028,7 +3091,6 @@ "version": "7.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", - "dev": true, "dependencies": { "ansi-styles": "^4.0.0", "string-width": "^4.1.0", @@ -3298,6 +3360,11 @@ "node": ">=14" } }, + "node_modules/convert-source-map": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", + "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==" + }, "node_modules/core-util-is": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", @@ -4972,6 +5039,11 @@ "node": ">=14" } }, + "node_modules/html-escaper": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.2.tgz", + "integrity": "sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==" + }, "node_modules/htmlparser2": { "version": "8.0.2", "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-8.0.2.tgz", @@ -5556,6 +5628,39 @@ "node": ">=0.10.0" } }, + "node_modules/istanbul-lib-coverage": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/istanbul-lib-coverage/-/istanbul-lib-coverage-3.2.2.tgz", + "integrity": "sha512-O8dpsF+r0WV/8MNRKfnmrtCWhuKjxrq2w+jpzBL5UZKTi2LeVWnWOmWRxFlesJONmc+wLAGvKQZEOanko0LFTg==", + "engines": { + "node": ">=8" + } + }, + "node_modules/istanbul-lib-report": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/istanbul-lib-report/-/istanbul-lib-report-3.0.1.tgz", + "integrity": "sha512-GCfE1mtsHGOELCU8e/Z7YWzpmybrx/+dSTfLrvY8qRmaY6zXTKWn6WQIjaAFw069icm6GVMNkgu0NzI4iPZUNw==", + "dependencies": { + "istanbul-lib-coverage": "^3.0.0", + "make-dir": "^4.0.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/istanbul-reports": { + "version": "3.1.7", + "resolved": "https://registry.npmjs.org/istanbul-reports/-/istanbul-reports-3.1.7.tgz", + "integrity": "sha512-BewmUXImeuRk2YY0PVbxgKAysvhRPUQE0h5QRM++nVWyubKGV0l8qQ5op8+B2DOmwSe63Jivj0BjkPQVf8fP5g==", + "dependencies": { + "html-escaper": "^2.0.0", + "istanbul-lib-report": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/jackspeak": { "version": "2.3.6", "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-2.3.6.tgz", @@ -6312,7 +6417,6 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-4.0.0.tgz", "integrity": "sha512-hXdUTZYIVOt1Ex//jAQi+wTZZpUpwBj/0QsOzqegb3rGMMeJiSEu5xLHnYfBrRV4RH2+OCSOO95Is/7x1WJ4bw==", - "dev": true, "dependencies": { "semver": "^7.5.3" }, @@ -10160,6 +10264,58 @@ "node": ">=4" } }, + "node_modules/test-exclude": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/test-exclude/-/test-exclude-6.0.0.tgz", + "integrity": "sha512-cAGWPIyOHU6zlmg88jwm7VRyXnMN7iV68OGAbYDk/Mh/xC/pzVPlQtY6ngoIH/5/tciuhGfvESU8GrHrcxD56w==", + "dependencies": { + "@istanbuljs/schema": "^0.1.2", + "glob": "^7.1.4", + "minimatch": "^3.0.4" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/test-exclude/node_modules/brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/test-exclude/node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/test-exclude/node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, "node_modules/text-extensions": { "version": "1.9.0", "resolved": "https://registry.npmjs.org/text-extensions/-/text-extensions-1.9.0.tgz", @@ -10673,6 +10829,19 @@ "uuid": "dist/bin/uuid" } }, + "node_modules/v8-to-istanbul": { + "version": "9.2.0", + "resolved": "https://registry.npmjs.org/v8-to-istanbul/-/v8-to-istanbul-9.2.0.tgz", + "integrity": "sha512-/EH/sDgxU2eGxajKdwLCDmQ4FWq+kpi3uCmBGpw1xJtnAxEjlD8j8PEiGWpCIMIs3ciNAgH0d3TTJiUkYzyZjA==", + "dependencies": { + "@jridgewell/trace-mapping": "^0.3.12", + "@types/istanbul-lib-coverage": "^2.0.1", + "convert-source-map": "^2.0.0" + }, + "engines": { + "node": ">=10.12.0" + } + }, "node_modules/validate-npm-package-license": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/validate-npm-package-license/-/validate-npm-package-license-3.0.4.tgz", @@ -10966,7 +11135,6 @@ "version": "17.7.2", "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", - "dev": true, "dependencies": { "cliui": "^8.0.1", "escalade": "^3.1.1", @@ -10984,7 +11152,6 @@ "version": "21.1.1", "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", - "dev": true, "engines": { "node": ">=12" } @@ -11070,6 +11237,7 @@ "@vscode-extension-tester/locators": "^1.0.0", "@vscode-extension-tester/page-objects": "^1.0.0", "@vscode/vsce": "^2.24.0", + "c8": "^9.1.0", "commander": "^12.0.0", "compare-versions": "^6.1.0", "fs-extra": "^11.2.0", diff --git a/package.json b/package.json index c8d151e22..89427a1d9 100644 --- a/package.json +++ b/package.json @@ -16,6 +16,7 @@ "build": "npx lerna run build", "build:changed": "npx lerna run build --since main", "test": "npm run ui-test --workspace=extester-test", + "coverage": "npm run ui-coverage --workspace=extester-test", "test:build": "npm run build && npm install --workspace=extester-test && npm test" }, "workspaces": [ diff --git a/packages/extester/package.json b/packages/extester/package.json index b3b27b9b0..02b144e3d 100644 --- a/packages/extester/package.json +++ b/packages/extester/package.json @@ -54,6 +54,7 @@ "@vscode/vsce": "^2.24.0", "@vscode-extension-tester/locators": "^1.0.0", "@vscode-extension-tester/page-objects": "^1.0.0", + "c8": "^9.1.0", "commander": "^12.0.0", "compare-versions": "^6.1.0", "fs-extra": "^11.2.0", diff --git a/packages/extester/src/browser.ts b/packages/extester/src/browser.ts index 6a6be97ef..206282fb0 100644 --- a/packages/extester/src/browser.ts +++ b/packages/extester/src/browser.ts @@ -15,6 +15,7 @@ export class VSBrowser { static readonly browserName = 'vscode'; private storagePath: string; private extensionsFolder: string | undefined; + private extensionDevelopmentPath: string | undefined; private customSettings: Object; private _driver!: WebDriver; private codeVersion: string; @@ -25,6 +26,7 @@ export class VSBrowser { constructor(codeVersion: string, releaseType: ReleaseQuality, customSettings: Object = {}, logLevel: logging.Level = logging.Level.INFO) { this.storagePath = process.env.TEST_RESOURCES ? process.env.TEST_RESOURCES : path.resolve(DEFAULT_STORAGE_FOLDER); this.extensionsFolder = process.env.EXTENSIONS_FOLDER ? process.env.EXTENSIONS_FOLDER : undefined; + this.extensionDevelopmentPath = process.env.EXTENSION_DEV_PATH ? process.env.EXTENSION_DEV_PATH : undefined; this.customSettings = customSettings; this.codeVersion = codeVersion; this.releaseType = releaseType; @@ -76,6 +78,8 @@ export class VSBrowser { fs.copyFileSync(path.resolve(__dirname, '..', '..', 'resources', 'state.vscdb'), path.join(userSettings, 'globalStorage', 'state.vscdb')); } args.push(`--extensionDevelopmentPath=${process.cwd()}`); + } else if(this.extensionDevelopmentPath) { + args.push(`--extensionDevelopmentPath=${this.extensionDevelopmentPath}`); } let options = new Options().setChromeBinaryPath(codePath).addArguments(...args) as any; @@ -187,7 +191,7 @@ export class VSBrowser { return; } - const code = new CodeUtil(this.storagePath, this.releaseType, this.extensionsFolder); + const code = new CodeUtil(this.storagePath, this.releaseType, this.extensionsFolder, this.extensionDevelopmentPath); code.open(...paths); await new Promise(res => setTimeout(res, 3000)); await this.waitForWorkbench(); diff --git a/packages/extester/src/cli.ts b/packages/extester/src/cli.ts index 94cb58229..81e169cb4 100644 --- a/packages/extester/src/cli.ts +++ b/packages/extester/src/cli.ts @@ -64,8 +64,9 @@ program.command('setup-tests') .option('-t, --type ', 'Type of VS Code release (stable/insider)') .option('-y, --yarn', 'Use yarn to build the extension via vsce instead of npm', false) .option('-i, --install_dependencies', 'Automatically install extensions your extension depends on', false) + .option('-d, --extension_development_path ', 'VSCode will use the unpackaged extension source from this directory') .action(withErrors(async (cmd) => { - const extest = new ExTester(cmd.storage, codeStream(cmd.type), cmd.extensions_dir); + const extest = new ExTester(cmd.storage, codeStream(cmd.type), cmd.extensions_dir, cmd.extension_development_path); await extest.setupRequirements({vscodeVersion: cmd.code_version, useYarn: cmd.yarn, installDependencies: cmd.install_dependencies}); })); @@ -80,9 +81,11 @@ program.command('run-tests ') .option('-m, --mocha_config ', 'Path to Mocha configuration file') .option('-l, --log_level ', 'Log messages from webdriver with a given level', 'Info') .option('-f, --offline', 'Attempt to run without internet connection, make sure to have all requirements downloaded', false) + .option('-d, --extension_development_path ', 'VSCode will use the unpackaged extension source from this directory') + .option('-C, --coverage', 'Enable code coverage using c8') .option('-r, --open_resource ', 'Open resources in VS Code. Multiple files and folders can be specified.') .action(withErrors(async (testFiles, cmd) => { - const extest = new ExTester(cmd.storage, codeStream(cmd.type), cmd.extensions_dir); + const extest = new ExTester(cmd.storage, codeStream(cmd.type), cmd.extensions_dir, cmd.extension_development_path, cmd.coverage); await extest.runTests(testFiles, {vscodeVersion: cmd.code_version, settings: cmd.code_settings, cleanup: cmd.uninstall_extension, config: cmd.mocha_config, logLevel: cmd.log_level, offline: cmd.offline, resources: cmd.open_resource ?? []}); })); @@ -99,9 +102,11 @@ program.command('setup-and-run ') .option('-i, --install_dependencies', 'Automatically install extensions your extension depends on', false) .option('-l, --log_level ', 'Log messages from webdriver with a given level', 'Info') .option('-f, --offline', 'Attempt to run without internet connection, make sure to have all requirements downloaded', false) + .option('-d, --extension_development_path ', 'VSCode will use the unpackaged extension source from this directory') + .option('-C, --coverage', 'Enable code coverage using c8') .option('-r, --open_resource ', 'Open resources in VS Code. Multiple files and folders can be specified.') .action(withErrors(async (testFiles, cmd) => { - const extest = new ExTester(cmd.storage, codeStream(cmd.type), cmd.extensions_dir); + const extest = new ExTester(cmd.storage, codeStream(cmd.type), cmd.extensions_dir, cmd.extension_development_path, cmd.coverage); await extest.setupAndRunTests(testFiles, cmd.code_version, {useYarn: cmd.yarn, installDependencies: cmd.install_dependencies}, {settings: cmd.code_settings, cleanup: cmd.uninstall_extension, config: cmd.mocha_config, logLevel: cmd.log_level, resources: cmd.open_resource ?? []}); })); diff --git a/packages/extester/src/extester.ts b/packages/extester/src/extester.ts index d2b1690db..9649a5481 100644 --- a/packages/extester/src/extester.ts +++ b/packages/extester/src/extester.ts @@ -44,8 +44,8 @@ export class ExTester { private code: CodeUtil; private chrome: DriverUtil; - constructor(storageFolder: string = DEFAULT_STORAGE_FOLDER, releaseType: ReleaseQuality = ReleaseQuality.Stable, extensionsDir?: string) { - this.code = new CodeUtil(storageFolder, releaseType, extensionsDir); + constructor(storageFolder: string = DEFAULT_STORAGE_FOLDER, releaseType: ReleaseQuality = ReleaseQuality.Stable, extensionsDir?: string, extensionDevPath?: string, coverage?: boolean) { + this.code = new CodeUtil(storageFolder, releaseType, extensionsDir, extensionDevPath, coverage); this.chrome = new DriverUtil(storageFolder); if (process.versions.node.slice(0, 2) > NODEJS_VERSION_MAX) { @@ -133,7 +133,11 @@ export class ExTester { console.log(`Attempting with ChromeDriver ${actualChromeVersion} anyway. Tests may experience issues due to version mismatch.`); } } - await this.installVsix({useYarn}); + if (this.code.extensionDevPath) { + await this.code.packageExtension(useYarn); + } else { + await this.installVsix({useYarn}); + } if (installDependencies && !offline) { this.code.installDependencies(); } diff --git a/packages/extester/src/suite/runner.ts b/packages/extester/src/suite/runner.ts index 4cd9157a6..01290aa84 100644 --- a/packages/extester/src/suite/runner.ts +++ b/packages/extester/src/suite/runner.ts @@ -10,6 +10,7 @@ import * as yaml from 'js-yaml'; import sanitize from 'sanitize-filename'; import { logging } from 'selenium-webdriver'; import * as os from 'os'; +import { Coverage } from '../util/coverage'; /** * Mocha runner wrapper @@ -43,6 +44,7 @@ export class VSRunner { return new Promise(resolve => { let self = this; let browser: VSBrowser = new VSBrowser(this.codeVersion, this.releaseType, this.customSettings, logLevel); + let coverage: Coverage | undefined; const testFiles = new Set(); for (const pattern of testFilesPattern) { @@ -65,6 +67,12 @@ export class VSRunner { this.mocha.suite.beforeAll(async function () { this.timeout(180000); + + if (code.coverageEnabled) { + coverage = new Coverage(); + process.env.NODE_V8_COVERAGE = coverage?.targetDir; + } + const start = Date.now(); const binPath = process.platform === 'darwin' ? await self.createShortcut(code.getCodeFolder(), self.tmpLink) : self.chromeBin; await browser.start(binPath); @@ -87,8 +95,10 @@ export class VSRunner { } } } - - code.uninstallExtension(self.cleanup); + if (!code.extensionDevPath) { + code.uninstallExtension(self.cleanup); + } + await coverage?.write(); }); this.mocha.run((failures) => { diff --git a/packages/extester/src/util/codeUtil.ts b/packages/extester/src/util/codeUtil.ts index 2d8184766..ff0c30d71 100644 --- a/packages/extester/src/util/codeUtil.ts +++ b/packages/extester/src/util/codeUtil.ts @@ -28,6 +28,8 @@ export interface RunOptions { logLevel?: logging.Level; /** try to perform all setup without internet connection, needs all requirements pre-downloaded manually */ offline?: boolean; + /** path to unbundled extension sources */ + extensionDevelopmentPath?: string; /** list of resources to be opened by VS Code */ resources: string[]; } @@ -55,19 +57,28 @@ export class CodeUtil { private cliEnv!: string; private availableVersions: string[]; private extensionsFolder: string | undefined; + private extensionDevelopmentPath: string | undefined; + private coverage: boolean | undefined; /** * Create an instance of code handler * @param folder Path to folder where all the artifacts will be stored. * @param extensionsFolder Path to use as extensions directory by VS Code */ - constructor(folder: string = DEFAULT_STORAGE_FOLDER, type: ReleaseQuality = ReleaseQuality.Stable, extensionsFolder?: string) { + constructor(folder: string = DEFAULT_STORAGE_FOLDER, type: ReleaseQuality = ReleaseQuality.Stable, extensionsFolder?: string, extensionDevelopmentPath?: string, coverage?: boolean) { this.availableVersions = []; this.downloadPlatform = this.getPlatform(); this.downloadFolder = path.resolve(folder); this.extensionsFolder = extensionsFolder ? path.resolve(extensionsFolder) : undefined; + this.extensionDevelopmentPath = extensionDevelopmentPath ? path.resolve(extensionDevelopmentPath) : undefined; + this.coverage = coverage; this.releaseType = type; + // If code coverage is enabled, but extensionDevelopmentPath is not, assume that extensionDevelopmentPath is the current directory. + if (this.coverage && !this.extensionDevelopmentPath) { + this.extensionDevelopmentPath = path.resolve("."); + } + if (type === ReleaseQuality.Stable) { this.codeFolder = path.join(this.downloadFolder, (process.platform === 'darwin') ? 'Visual Studio Code.app' : `VSCode-${this.downloadPlatform}`); @@ -257,6 +268,7 @@ export class CodeUtil { process.env = finalEnv; process.env.TEST_RESOURCES = this.downloadFolder; process.env.EXTENSIONS_FOLDER = this.extensionsFolder; + process.env.EXTENSION_DEV_PATH = this.extensionDevelopmentPath; const runner = new VSRunner(this.executablePath, literalVersion, this.parseSettings(runOptions.settings ?? DEFAULT_RUN_OPTIONS.settings), runOptions.cleanup, this.releaseType, runOptions.config); return await runner.runTests(testFilesPattern, this, runOptions.logLevel, runOptions.resources); } @@ -336,6 +348,20 @@ export class CodeUtil { return this.codeFolder; } + /** + * Getter for extension development path + */ + get extensionDevPath() { + return this.extensionDevelopmentPath; + } + + /** + * Getter for coverage enablement option + */ + get coverageEnabled() { + return this.coverage; + } + /** * Check if given version is available in the given stream */ diff --git a/packages/extester/src/util/coverage.ts b/packages/extester/src/util/coverage.ts new file mode 100644 index 000000000..565dfae95 --- /dev/null +++ b/packages/extester/src/util/coverage.ts @@ -0,0 +1,84 @@ +/*--------------------------------------------------------- + * Copyright (C) Microsoft Corporation. All rights reserved. + *--------------------------------------------------------*/ + +/*--------------------------------------------------------- + * Modified for vscode-extension-tester + *--------------------------------------------------------*/ + +import { Report } from 'c8'; +import { randomUUID } from 'crypto'; +import { promises as fs, mkdirSync } from 'fs'; +import { tmpdir } from 'os'; +import { join } from 'path'; + +/** + * Manages collecting coverage data from test runs. All runs, regardless of + * platform, expect coverage data given in the V8 coverage format. We then + * use c8 to convert it to the common Istanbul format and represent it with + * a variety of reporters. + */ +export class Coverage { + public readonly targetDir = join(tmpdir(), `vsc-coverage-${randomUUID()}`); + + constructor() { + mkdirSync(this.targetDir, { recursive: true }); + } + + public async write() { + try { + const report = new Report({ + "reporter": ["text", "html"], + "all": false, + "excludeNodeModules": true, + "include": [], + "exclude": [ + "coverage/**", + "packages/*/test{,s}/**", + "**/*.d.ts", + "test{,s}/**", + "test{,-*}.{js,cjs,mjs,ts,tsx,jsx}", + "**/*{.,-}test.{js,cjs,mjs,ts,tsx,jsx}", + "**/__tests__/**", + "**/{ava,babel,nyc}.config.{js,cjs,mjs}", + "**/jest.config.{js,cjs,mjs,ts}", + "**/{karma,rollup,webpack}.config.js", + "**/.{eslint,mocha}rc.{js,cjs}" + ], + "extension": [ + ".js", + ".cjs", + ".mjs", + ".ts", + ".tsx", + ".jsx" + ], + "excludeAfterRemap": false, + "skipFull": false, + "tempDirectory": this.targetDir, + "resolve": "", + "omitRelative": true, + "allowExternal": false, + }); + + // A hacky fix due to an outstanding bug in Istanbul's exclusion testing + // code: its subdirectory checks are case-sensitive on Windows, but file + // URIs might have mixed casing. + // + // Setting `relativePath: false` on the exclude bypasses this code path. + // + // https://github.com/istanbuljs/test-exclude/issues/43 + // https://github.com/istanbuljs/test-exclude/blob/a5b1d07584109f5f553ccef97de64c6cbfca4764/index.js#L91 + (report as any).exclude.relativePath = false; + + await report.run(); + } catch (e) { + // throw new CliExpectedError( + throw new Error( + `Coverage report generated failed, please file an issue with original reports located in ${this.targetDir}:\n\n${e}`, + ); + } + + await fs.rm(this.targetDir, { recursive: true, force: true }); + } +} diff --git a/tests/test-project/package.json b/tests/test-project/package.json index cf4fd89fd..6e0aae3e7 100644 --- a/tests/test-project/package.json +++ b/tests/test-project/package.json @@ -178,7 +178,8 @@ "compile": "tsc -p ./ && npm run lint", "lint": "eslint src --ext .ts", "cb-init": "echo hello_ExTester | clipboard", - "ui-test": "npm run cb-init && extest setup-and-run './out/test/cli/order-3.test.js' './out/test/cli/order-2.test.js' './out/test/cli/order-1.test.js' './out/test/**/*.test.js' './out/test/system/clipboard.test.js' -u -i -r . -e ./test-extensions" + "ui-test": "npm run cb-init && extest setup-and-run './out/test/cli/order-3.test.js' './out/test/cli/order-2.test.js' './out/test/cli/order-1.test.js' './out/test/**/*.test.js' './out/test/system/clipboard.test.js' -u -i -r . -e ./test-extensions", + "ui-coverage": "npm run cb-init && extest setup-and-run './out/test/cli/order-3.test.js' './out/test/cli/order-2.test.js' './out/test/cli/order-1.test.js' './out/test/**/*.test.js' './out/test/system/clipboard.test.js' -u -i -r . -e ./test-extensions --coverage" }, "devDependencies": { "@types/chai": "^4.3.14",