diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml new file mode 100644 index 00000000..2c1dfa82 --- /dev/null +++ b/.github/workflows/cd.yml @@ -0,0 +1,29 @@ +name: Continuous Deploiment Package and Publish +on: + release: + types: [published] +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + # Setup .npmrc file to publish to npm + - uses: actions/setup-node@v2 + with: + node-version: "14.x" + registry-url: "https://registry.npmjs.org" + # Defaults to the user or organization that owns the workflow file + scope: "@octocat" + - run: yarn + - run: yarn build + - run: yarn test + env: + TESTRAIL_DOMAIN: ${{ secrets.DOMAIN }} + TESTRAIL_USERNAME: ${{ secrets.USERNAME }} + TESTRAIL_PASSWORD: ${{ secrets.PASSWORD }} + TESTRAIL_PROJECTID: 1 + TESTRAIL_SUITEID: 1 + TESTRAIL_ASSIGNEDTOID: 1 + - run: yarn publish --access public + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..b64db455 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,28 @@ +name: Continuous Integration Checking +on: + pull_request: + types: [opened, ready_for_review, review_requested, synchronize] + push: + branches: ["master", "develop"] +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + # Setup .npmrc file to publish to npm + - uses: actions/setup-node@v2 + with: + node-version: "14.x" + registry-url: "https://registry.npmjs.org" + # Defaults to the user or organization that owns the workflow file + scope: "@octocat" + - run: yarn + - run: yarn build + - run: yarn test + env: + TESTRAIL_DOMAIN: ${{ secrets.DOMAIN }} + TESTRAIL_USERNAME: ${{ secrets.USERNAME }} + TESTRAIL_PASSWORD: ${{ secrets.PASSWORD }} + TESTRAIL_PROJECTID: 1 + TESTRAIL_SUITEID: 1 + TESTRAIL_ASSIGNEDTOID: 1 \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..d8f94703 --- /dev/null +++ b/.gitignore @@ -0,0 +1,12 @@ +node_modules +node-debug* +*.log +.c9/ +*.iml +.idea/ +*.sh +dist +.env +*.tgz +yarn.lock +package-lock.json diff --git a/.npmignore b/.npmignore new file mode 100644 index 00000000..7357c032 --- /dev/null +++ b/.npmignore @@ -0,0 +1,12 @@ +.github +.idea/ +src/ +dist/test +*.iml +tsconfig.json +*.map +*.lock +package-lock.json +.prettier* +.env +*.tgz \ No newline at end of file diff --git a/.npmrc b/.npmrc new file mode 100644 index 00000000..214c29d1 --- /dev/null +++ b/.npmrc @@ -0,0 +1 @@ +registry=https://registry.npmjs.org/ diff --git a/.prettierignore b/.prettierignore new file mode 100644 index 00000000..650b3597 --- /dev/null +++ b/.prettierignore @@ -0,0 +1,2 @@ +dist/ +node_modules/* diff --git a/.prettierrc b/.prettierrc new file mode 100644 index 00000000..e69de29b diff --git a/README.md b/README.md new file mode 100644 index 00000000..80964c31 --- /dev/null +++ b/README.md @@ -0,0 +1,86 @@ +#Testrail Reporter for Mocha + +[![npm version](https://badge.fury.io/js/mocha-testrail-reporter.svg)](https://badge.fury.io/js/mocha-testrail-reporter) + +Pushes test results into Testrail system. + +> **NOTE** : Version 2.0.x is backward compatible with v1 but has been updated to latest dependencies. The V2 choice is to ensure that existing users are not affected! + +## Installation + +```shell +$ npm install mocha-testrail-reporter --save-dev +``` + +## Usage +Ensure that your testrail installation API is enabled and generate your API keys. See http://docs.gurock.com/ + +Run mocha with `mocha-testrail-reporter`: + +```shell +$ mocha test --reporter mocha-testrail-reporter --reporter-options domain=instance.testrail.net,username=test@example.com,password=12345678,projectId=1,suiteId=1 +``` + +or use a mocha.options file +```shell +mocha --opts mocha-testrail.opts build/test +--recursive +--reporter mocha-testrail-reporter +--reporter-options domain=instance.testrail.net,username=test@example.com,password=12345678,projectId=1,suiteId=1 +--no-exit +``` + + +Mark your mocha test names with ID of Testrail test cases. Ensure that your case ids are well distinct from test descriptions. + +```Javascript +it("C123 C124 Authenticate with invalid user", () => {}) +it("Authenticate a valid user C321", () => {}) +``` + +Only passed or failed tests will be published. Skipped or pending tests will not be published resulting in a "Pending" status in testrail test run. + +## Options + +**domain**: *string* domain name of your Testrail instance (e.g. for a hosted instance instance.testrail.net) + +**username**: *string* user under which the test run will be created (e.g. jenkins or ci) + +**password**: *string* password or API token for user + +**projectId**: *number* projet number with which the tests are associated + +**suiteId**: *number* suite number with which the tests are associated + +**assignedToId**: *number* (optional) user id which will be assigned failed tests + +## Build and test locally + +### Setup testrail test server + +- Start a new TestRail trial from https://www.gurock.com/testrail/test-management/ +- Enable API under https://XXX.testrail.io/index.php?/admin/site_settings&tab=8 +- Add a new project (Use multiple test suites to manage cases) +- Add a new test suite (ids: 1) +- Add at least 4 test cases (ids: C1, C2, C3, C4, etc) +- Once setup, set your environment variables - recommend using .env file in the root folder + - TESTRAIL_DOMAIN=XXX.testrail.io + - TESTRAIL_USERNAME=XXX + - TESTRAIL_PASSWORD=XXX + - TESTRAIL_PROJECTID=1 + - TESTRAIL_SUITEID=1 + - TESTRAIL_ASSIGNEDTOID=1 +- Execute the build and test commands (unit and integration tests) +- Execute the e2e test (requires build and .env file) + +### Build and test +``` +npm run clean +npm run build +npm run test +``` + +## References +- http://mochajs.org/#mochaopts +- https://github.com/mochajs/mocha/wiki/Third-party-reporters +- http://docs.gurock.com/testrail-api2/start diff --git a/e2e/.gitignore b/e2e/.gitignore new file mode 100644 index 00000000..1b6ef2b1 --- /dev/null +++ b/e2e/.gitignore @@ -0,0 +1,3 @@ +package-lock.json +yarn.lock +node_modules \ No newline at end of file diff --git a/e2e/e2e.spec.js b/e2e/e2e.spec.js new file mode 100644 index 00000000..c46c8108 --- /dev/null +++ b/e2e/e2e.spec.js @@ -0,0 +1,7 @@ +describe("E2E Testrail Mocha Reporter", () => { + it("C1 Test case 1", () => {}); + it("C2 Test case 1", () => {}); + it("C3 Test case 1", () => { + throw new Error("Failed test"); + }); +}); diff --git a/e2e/package.json b/e2e/package.json new file mode 100644 index 00000000..339aec3f --- /dev/null +++ b/e2e/package.json @@ -0,0 +1,16 @@ +{ + "name": "e2e", + "version": "1.0.0", + "description": "E2E test for mocha-testrail-reporter", + "main": "index.js", + "author": "Pierre Awaragi", + "license": "MIT", + "private": true, + "scripts": { + "test": ". ../.env && mocha e2e.spec.js --reporter mocha-testrail-reporter --reporter-options domain=$DOMAIN,username=$USERNAME,password=$PASSWORD,projectId=$PROJECTID,suiteId=$SUITEID" + }, + "devDependencies": { + "mocha": "^9.2.2", + "mocha-testrail-reporter": "file:.." + } +} diff --git a/index.js b/index.js new file mode 100644 index 00000000..1b7332c2 --- /dev/null +++ b/index.js @@ -0,0 +1 @@ +module.exports = require("./dist/lib/mocha-testrail-reporter").MochaTestRailReporter \ No newline at end of file diff --git a/package.json b/package.json new file mode 100644 index 00000000..7039476e --- /dev/null +++ b/package.json @@ -0,0 +1,50 @@ +{ + "name": "mocha-testrail-reporter", + "version": "2.0.5", + "description": "A Testrail reporter for mocha utilising TestRail API", + "main": "index.js", + "private": false, + "keywords": [ + "mocha", + "testrail", + "reporter" + ], + "author": { + "name": "Pierre Awaragi", + "email": "pierre@awaragi.com" + }, + "license": "MIT", + "readmeFilename": "README.md", + "repository": { + "type": "git", + "url": "https://github.com/awaragi/mocha-testrail-reporter.git" + }, + "bugs": { + "url": "https://github.com/awaragi/mocha-testrail-reporter/issues" + }, + "scripts": { + "build": "tsc", + "clean": "rimraf dist", + "test": "mocha dist/test", + "test:ts": "mocha -r ts-node/register src/test/*", + "format": "prettier --check ./**/* --write" + }, + "dependencies": { + "btoa": "^1.1.2", + "unirest": "^0.6.0" + }, + "peerDependencies": { + "mocha": "9.2.2" + }, + "devDependencies": { + "@types/chai": "^4.3.0", + "@types/mocha": "^9.1.0", + "@types/node": "^12.20.47", + "chai": "^4.3.6", + "mocha": "^9.2.2", + "prettier": "^2.6.0", + "rimraf": "^3.0.2", + "ts-node": "^10.7.0", + "typescript": "^4.6.2" + } +} diff --git a/src/lib/mocha-testrail-reporter.ts b/src/lib/mocha-testrail-reporter.ts new file mode 100644 index 00000000..d917c7e9 --- /dev/null +++ b/src/lib/mocha-testrail-reporter.ts @@ -0,0 +1,113 @@ +import { reporters } from "mocha"; +import { TestRail } from "./testrail"; +import { titleToCaseIds } from "./shared"; +import { Status, TestRailOptions, TestRailResult } from "./testrail.interface"; + +export class MochaTestRailReporter extends reporters.Spec { + private results: TestRailResult[] = []; + private passes: number = 0; + private fails: number = 0; + private pending: number = 0; + private out: string[] = []; + + constructor(runner: any, options: any) { + super(runner); + + let reporterOptions: TestRailOptions = ( + options.reporterOptions + ); + + // validate options + ["domain", "username", "password", "projectId", "suiteId"].forEach( + (option) => MochaTestRailReporter.validate(reporterOptions, option) + ); + + runner.on("start", () => {}); + + runner.on("suite", () => {}); + + runner.on("suite end", () => {}); + + runner.on("pending", (test) => { + this.pending++; + this.out.push(test.fullTitle() + ": pending"); + }); + + runner.on("pass", (test) => { + this.passes++; + this.out.push(test.fullTitle() + ": pass"); + let caseIds = titleToCaseIds(test.title); + if (caseIds.length > 0) { + if (test.speed === "fast") { + let results = caseIds.map((caseId) => { + return { + case_id: caseId, + status_id: Status.Passed, + comment: test.title, + }; + }); + this.results.push(...results); + } else { + let results = caseIds.map((caseId) => { + return { + case_id: caseId, + status_id: Status.Passed, + comment: `${test.title} (${test.duration}ms)`, + }; + }); + this.results.push(...results); + } + } + }); + + runner.on("fail", (test) => { + this.fails++; + this.out.push(test.fullTitle() + ": fail"); + let caseIds = titleToCaseIds(test.title); + if (caseIds.length > 0) { + let results = caseIds.map((caseId) => { + return { + case_id: caseId, + status_id: Status.Failed, + comment: `${test.title} +${test.err}`, + }; + }); + this.results.push(...results); + } + }); + + runner.on("end", () => { + if (this.results.length == 0) { + console.warn( + "No testcases were matched. Ensure that your tests are declared correctly and matches TCxxx" + ); + } + let executionDateTime = new Date().toISOString(); + let total = this.passes + this.fails + this.pending; + let name = `Automated test run ${executionDateTime}`; + let description = `Automated test run executed on ${executionDateTime} +Execution summary: +Passes: ${this.passes} +Fails: ${this.fails} +Pending: ${this.pending} +Total: ${total} + +Execution details: +${this.out.join("\n")} +`; + new TestRail(reporterOptions).publish(name, description, this.results); + }); + } + + private static validate(options: TestRailOptions, name: string) { + if (options == null) { + throw new Error("Missing --reporter-options in mocha.opts"); + } + if (options[name] == null) { + throw new Error( + `Missing ${name} value. Please update --reporter-options in mocha.opts` + ); + } + } +} diff --git a/src/lib/shared.ts b/src/lib/shared.ts new file mode 100644 index 00000000..174486db --- /dev/null +++ b/src/lib/shared.ts @@ -0,0 +1,16 @@ +/** + * Search for all applicable test cases + * @param title + * @returns {any} + */ +export function titleToCaseIds(title: string): number[] { + let caseIds: number[] = []; + + let testCaseIdRegExp: RegExp = /\bT?C(\d+)\b/g; + let m; + while ((m = testCaseIdRegExp.exec(title)) !== null) { + let caseId = parseInt(m[1]); + caseIds.push(caseId); + } + return caseIds; +} diff --git a/src/lib/testrail.interface.ts b/src/lib/testrail.interface.ts new file mode 100644 index 00000000..ee64db29 --- /dev/null +++ b/src/lib/testrail.interface.ts @@ -0,0 +1,46 @@ +export interface TestRailOptions { + domain: string; + username: string; + password: string; + projectId: string; + suiteId: string; + assignedToId?: string; +} + +export enum Status { + Passed = 1, + Blocked = 2, + Untested = 3, + Retest = 4, + Failed = 5, +} + +export interface TestRailResult { + case_id: number; + status_id: Status; + comment?: String; +} + +export interface TestRailCase { + id: number; + title: string; + section_id: number; + template_id: number; + type_id: number; + priority_id: number; + milestone_id?: number; + refs?: string; + created_by: number; + created_on: number; + updated_by: number; + updated_on: number; + estimate?: string; + estimate_forecast?: string; + suite_id: number; + custom_preconds?: string; + custom_steps?: string; + custom_expected?: string; + custom_steps_separated?: string; + custom_mission?: string; + custom_goals?: string; +} diff --git a/src/lib/testrail.ts b/src/lib/testrail.ts new file mode 100644 index 00000000..9dd487eb --- /dev/null +++ b/src/lib/testrail.ts @@ -0,0 +1,137 @@ +import request = require("unirest"); +import { TestRailOptions, TestRailResult } from "./testrail.interface"; + +/** + * TestRail basic API wrapper + */ +export class TestRail { + private base: String; + + constructor(private options: TestRailOptions) { + // check if all required options are specified + ["username", "password", "domain", "projectId", "projectId"].forEach( + (option) => { + if (!options[option]) { + throw new Error(`Missing required option ${option}`); + } + } + ); + + // compute base url + this.base = `https://${options.domain}/index.php`; + } + + private _post(api: String, body: any, callback: Function, error?: Function) { + request("POST", this.base) + .query(`/api/v2/${api}`) + .headers({ + "content-type": "application/json", + }) + .type("json") + .send(body) + .auth(this.options.username, this.options.password) + .end((res) => { + if (res.error) { + console.log("Error: %s", JSON.stringify(res.body)); + if (error) { + error(res.error); + } else { + throw new Error(res.error); + } + } + callback(res.body); + }); + } + + private _get(api: String, callback: Function, error?: Function): void { + request("GET", this.base) + .query(`/api/v2/${api}`) + .headers({ + "content-type": "application/json", + }) + .type("json") + .auth(this.options.username, this.options.password) + .end((res) => { + if (res.error) { + console.log("Error: %s", JSON.stringify(res.body)); + if (error) { + error(res.error); + } else { + throw new Error(res.error); + } + } + callback(res.body); + }); + } + + /** + * Fetchs test cases from projet/suite based on filtering criteria (optional) + * @param {{[p: string]: number[]}} filters + * @param {Function} callback + */ + public fetchCases( + filters?: { [key: string]: number[] }, + callback?: Function + ): void { + let filter = ""; + if (filters) { + for (let key in filters) { + if (filters.hasOwnProperty(key)) { + filter += "&" + key + "=" + filters[key].join(","); + } + } + } + + this._get( + `get_cases/${this.options.projectId}&suite_id=${this.options.suiteId}${filter}`, + (body) => { + if (callback) { + callback(body.cases); + } + } + ); + } + + /** + * Publishes results of execution of an automated test run + * @param {string} name + * @param {string} description + * @param {TestRailResult[]} results + * @param {Function} callback + */ + public publish( + name: string, + description: string, + results: TestRailResult[], + callback?: Function + ): void { + console.log(`Publishing ${results.length} test result(s) to ${this.base}`); + + this._post( + `add_run/${this.options.projectId}`, + { + suite_id: this.options.suiteId, + name: name, + description: description, + assignedto_id: this.options.assignedToId, + include_all: true, + }, + (body) => { + const runId = body.id; + console.log(`Results published to ${this.base}?/runs/view/${runId}`); + this._post( + `add_results_for_cases/${runId}`, + { + results: results, + }, + (body) => { + // execute callback if specified + if (callback) { + callback(body); + } + } + ); + } + ); + } +} diff --git a/src/test/shared.spec.ts b/src/test/shared.spec.ts new file mode 100644 index 00000000..5bcd73f3 --- /dev/null +++ b/src/test/shared.spec.ts @@ -0,0 +1,38 @@ +import * as chai from "chai"; +chai.should(); + +import { titleToCaseIds } from "../lib/shared"; + +describe("Shared functions", () => { + describe("titleToCaseIds", () => { + it("Single test case id present", () => { + let caseIds = titleToCaseIds("C123 Test title"); + caseIds.length.should.be.equals(1); + caseIds[0].should.be.equals(123); + + caseIds = titleToCaseIds("Execution of C123 Test title"); + caseIds.length.should.be.equals(1); + caseIds[0].should.be.equals(123); + }); + it("Multiple test case ids present", () => { + let caseIds = titleToCaseIds("Execution C321 C123 Test title"); + caseIds.length.should.be.equals(2); + caseIds[0].should.be.equals(321); + caseIds[1].should.be.equals(123); + }); + it("No test case ids present", () => { + let caseIds = titleToCaseIds("Execution Test title"); + caseIds.length.should.be.equals(0); + }); + }); + + describe("Misc tests", () => { + it("String join", () => { + let out: string[] = []; + out.push("Test 1: fail"); + out.push("Test 2: pass"); + out.join("\n").should.be.equals(`Test 1: fail +Test 2: pass`); + }); + }); +}); diff --git a/src/test/testrail.spec.ts b/src/test/testrail.spec.ts new file mode 100644 index 00000000..3ab20e5c --- /dev/null +++ b/src/test/testrail.spec.ts @@ -0,0 +1,46 @@ +import { TestRail } from "../lib/testrail"; +import { TestRailCase, Status } from "../lib/testrail.interface"; + +describe("TestRail API", () => { + it("Publish test run", (done) => { + let testRail = new TestRail({ + domain: process.env.TESTRAIL_DOMAIN, + username: process.env.TESTRAIL_USERNAME, + password: process.env.TESTRAIL_PASSWORD, + projectId: process.env.TESTRAIL_PROJECTID, + suiteId: process.env.TESTRAIL_SUITEID, + assignedToId: process.env.TESTRAIL_ASSIGNEDTOID, + }); + + testRail.fetchCases({}, (cases: TestRailCase[]) => { + console.log(cases); + cases.forEach((value) => { + console.log(value.id, "-", value.title); + }); + }); + + testRail.publish( + "Unit Test of mocha-testrail-reporter", + "Unit Test of mocha-testrail-reporter", + [ + { + case_id: 1, + status_id: Status.Passed, + comment: "Passing....", + }, + { + case_id: 2, + status_id: Status.Passed, + }, + { + case_id: 3, + status_id: Status.Failed, + comment: "Failure....", + }, + ], + () => { + done(); + } + ); + }); +}); diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 00000000..23a2bd23 --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,19 @@ +{ + "compilerOptions": { + "target": "ES6", + "module": "commonjs", + "moduleResolution": "node", + "sourceMap": true, + "emitDecoratorMetadata": true, + "experimentalDecorators": true, + "removeComments": false, + "noImplicitAny": false, + "pretty": true, + "outDir": "dist", + "typeRoots": ["node_modules/@types"] + }, + "exclude": [ + "node_modules", + "dist" + ] +} \ No newline at end of file