Merge branch 'master' into templatetag

CONTRIBUTING.md

@@ -1,51 +1,3 @@
-# Contributing
+# Contributing to Screwdriver
 
-Thank you for considering contributing! There are many ways you can help.
-
-## Issues
-
-File an issue if you think you've found a bug. Be sure to describe
-
-1. How can it be reproduced?
-2. What did you expect?
-3. What actually occurred?
-4. Version, platform, etc. if possibly relevant.
-
-## Docs
-
-Documentation, READMEs, and examples are extremely important. Please help improve them and if you find a typo or notice a problem, please send a fix or say something.
-
-## Submitting Patches
-
-Patches for fixes, features, and improvements are accepted through pull requests.
-
-* Write good commit messages, in the present tense! (Add X, not Added X). Short title, blank line, bullet points if needed. Capitalize the first letter of the title or bullet item. No punctuation in the title.
-* Code must pass lint and style checks.
-* All external methods must be documented.
-* Include tests to improve coverage and prevent regressions.
-* Squash changes into a single commit per feature/fix. Ask if you're unsure how to discretize your work.
-
-Please ask before embarking on a large improvement so you're not disappointed if it does not align with the goals of the project or owner(s).
-
-## Commit message format
-
-We use [semantic-release](https://www.npmjs.com/package/semantic-release), which requires commit messages to be in this specific format: `<type>(<scope>): <subject>`
-
-* Types:
-  * feat (feature)
-  * fix (bug fix)
-  * docs (documentation)
-  * style (formatting, missing semi colons, …)
-  * refactor
-  * test (when adding missing tests)
-  * chore (maintain)
-* Scope: anything that specifies the scope of the commit. Can be blank or `*`
-* Subject: description of the commit. For **breaking changes** that require major version bump, add `BREAKING CHANGE` to the commit message.
-
-**Examples commit messages:**
-* Bug fix: `fix: Remove extra space`
-* Breaking change: `feat(scm): Support new scm plugin. BREAKING CHANGE: github no longer works`
-
-## Feature Requests
-
-Make the case for a feature via an issue with a good title. The feature should be discussed and given a target inclusion milestone or closed.
+Have a look at our guidelines, as well as pointers on where to start making changes, in our official [documentation](http://docs.screwdriver.cd/about/contributing/).

README.md

@@ -82,10 +82,11 @@ All the possible environment variables are [defined here](config/custom-environm
 
 ## Plugins
 
-This API comes preloaded with 8 (eight) resources:
+This API comes preloaded with 9 (nine) resources:
 
  - [auth](plugins/auth/README.md)
  - [builds](plugins/builds/README.md)
+ - [collections](plugins/collections/README.md)
  - [events](plugins/events/README.md)
  - [jobs](plugins/jobs/README.md)
  - [pipelines](plugins/pipelines/README.md)
@@ -121,7 +122,7 @@ Fork `functional-*` repositories to your organization from [screwdriver-cd-test]
 Add `.func_config` to the root of the Screwdriver API folder with your username, github token, access key, host, and organization for test:
 ```
 GIT_TOKEN=YOUR-GITHUB-TOKEN
-ACCESS_KEY=YOUR-ACCESS-KEY
+API_TOKEN=YOUR-API-TOKEN
 SD_API=YOUR-LOCAL-API-HOST
 TEST_ORG=OUR-TEST-ORGANIZATION
 TEST_USERNAME=YOUR-GITHUB-USERNAME
@@ -134,9 +135,7 @@ PROTOCOL=PROTOCOL-FOR-SD-API // e.g.PROTOCOL=http; by default it is https
 Set the environment variables:
 
 ```bash
-$ export TEST_USERNAME=YOUR-GITHUB-USERNAME
-$ export GIT_TOKEN=YOUR-GITHUB-TOKEN
-$ export ACCESS_KEY=YOUR-ACCESS-KEY
+$ export API_TOKEN=YOUR-API-TOKEN
 $ export SD_API=YOUR-LOCAL-API-HOST
 $ export TEST_ORG=YOUR-TEST-ORGANIZATION
 $ export PROTOCOL=PROTOCAL-FOR-SD-API

bin/server

@@ -110,6 +110,9 @@ const eventFactory = Models.EventFactory.getInstance({
     datastore,
     scm
 });
+const collectionFactory = Models.CollectionFactory.getInstance({
+    datastore
+});
 
 // @TODO run setup for SCM and Executor
 datastore.setup()
@@ -128,6 +131,7 @@ datastore.setup()
         secretFactory,
         tokenFactory,
         eventFactory,
+        collectionFactory,
         builds: {
             ecosystem
         },

config/custom-environment-variables.yaml

@@ -10,12 +10,6 @@ auth:
     # Generate one by running
     # $ openssl rsa -in jwt.pem -pubout -out jwt.pub
     jwtPublicKey: SECRET_JWT_PUBLIC_KEY
-    # The access token to use on behalf of a user to access the API. Used as
-    # an alternative to the OAuth flow
-    temporaryAccessKey: SECRET_ACCESS_KEY
-    # The user name associated with the temproary access token. Used as a
-    # means of functionally testing the API
-    temporaryAccessUser: SECRET_ACCESS_USER
     # A password used for encrypting session data.
     # **Needs to be minimum 32 characters**
     cookiePassword: SECRET_COOKIE_PASSWORD

config/default.yaml

@@ -20,10 +20,6 @@ auth:
     # A password used for encrypting stored secrets.
     # **Needs to be minimum 32 characters**
     encryptionPassword: WOW-ANOTHER-MORE-INSECURE-PASSWORD!!!
-    # A single access token used as an alternative to the Oauth login flow
-    temporaryAccessKey: someAccessTokenThatIsTemporaryForUsingInTheMeantime
-    # User name that is associated with the temporary access key
-    temporaryAccessUser: quality_automation
     # A flag to set if the server is running over https.
     # Used as a flag for the OAuth flow
     https: false

features/scripts/create-test-user.js

@@ -0,0 +1,84 @@
+#!/usr/bin/env node
+/**
+ * Make a user in the database and generate an access token for it
+ * @param {String} username     Username for the new user
+ * @param {String} gitToken     Git access token for the user
+ */
+
+'use strict';
+
+/* eslint-disable import/no-dynamic-require */
+/* eslint-disable no-console */
+
+// Make sure script is being called correctly
+if (process.argv.length !== 4) {
+    console.log('Usage: npm run create-test-user -- $username $git-token');
+
+    return 1;
+}
+
+const username = process.argv[2];
+const gitToken = process.argv[3];
+
+const config = require('config');
+const hoek = require('hoek');
+
+// Setup Authentication
+const authConfig = config.get('auth');
+
+// Setup HTTPd
+const httpdConfig = config.get('httpd');
+
+// Special urls for things like the UI
+const ecosystem = config.get('ecosystem');
+
+ecosystem.api = httpdConfig.uri;
+
+// Setup Datastore
+const datastoreConfig = config.get('datastore');
+const DatastorePlugin = require(`screwdriver-datastore-${datastoreConfig.plugin}`);
+const datastore = new DatastorePlugin(hoek.applyToDefaults({ ecosystem },
+    (datastoreConfig[datastoreConfig.plugin] || {})));
+
+// Source Code Plugin
+const scmConfig = config.get('scm');
+const ScmPlugin = require(`screwdriver-scm-${scmConfig.plugin}`);
+const scm = new ScmPlugin(hoek.applyToDefaults({ ecosystem },
+    (scmConfig[scmConfig.plugin] || {})));
+
+authConfig.scm = scm;
+
+// Setup Model Factories
+const Models = require('screwdriver-models');
+const userFactory = Models.UserFactory.getInstance({
+    datastore,
+    scm,
+    password: authConfig.encryptionPassword
+});
+const tokenFactory = Models.TokenFactory.getInstance({
+    datastore
+});
+
+// Setup datastore and create test user
+return datastore.setup()
+    .then(() => userFactory.get({ username }))
+    .then((model) => {
+        if (!model) {
+            return userFactory.create({
+                username,
+                token: gitToken
+            });
+        }
+
+        return model.sealToken(gitToken)
+          .then((token) => {
+              model.token = token;
+
+              return model.update();
+          });
+    })
+    .then(testUser => tokenFactory.create({
+        name: 'Functional test token',
+        userId: testUser.id
+    }))
+    .then(token => console.log(`Token created for user ${username}: ${token.value}`));

features/scripts/readme.md

@@ -0,0 +1,15 @@
+## Scripts
+
+Scripts used for setting up functional tests
+
+### Creating a test user
+
+The script `create-test-user` will create a user in your database. Run this script with a username and SCM access token, and the script will create the user and print an API token for use in tests to `stdout`.
+
+```bash
+$ npm run create-test-user -- test-user git-access-token
+```
+
+Put the token generated by the script in your `.func_config` file (or set it as an environment variable) as `API_TOKEN`.
+
+In addition, you will have to remove the `temporaryAccessUser` and `temporaryAccessKey` from your `screwdriver/config/local.yaml`.

features/step_definitions/authorization.js

@@ -18,11 +18,11 @@ defineSupportCode(({ Before, Given, Then }) => {
     Given(/^an existing pipeline with that repository$/, () => null);
 
     Given(/^"([^"]*)" is logged in$/, function step(user) {
-        if (!(this.accessKey)) {
+        if (!(this.apiToken)) {
             throw new Error('insufficient set up, missing access key');
         }
 
-        return this.getJwt(this.accessKey).then((response) => {
+        return this.getJwt(this.apiToken).then((response) => {
             const accessToken = response.body.token;
             const decodedToken = jwt.decode(accessToken);
 

features/step_definitions/git-flow.js

@@ -23,7 +23,7 @@ defineSupportCode(({ Before, Given, When, Then }) => {
 
         return request({  // TODO : perform this in the before-hook for all func tests
             method: 'GET',
-            url: `${this.instance}/${this.namespace}/auth/token?access_key=${this.accessKey}`,
+            url: `${this.instance}/${this.namespace}/auth/token?api_token=${this.apiToken}`,
             followAllRedirects: true,
             json: true
         }).then((response) => {

features/step_definitions/sd-step.js

@@ -24,7 +24,7 @@ defineSupportCode(({ Before, Given, When, Then }) => {
 
     Given(/^an existing pipeline with (.*) image and (.*) package$/,
         { timeout: TIMEOUT }, function step(image, pkg) {
-            return this.getJwt(this.accessKey)
+            return this.getJwt(this.apiToken)
                 .then((response) => {
                     this.jwt = response.body.token;
                     this.expectedImage = image;

features/step_definitions/templates.js

@@ -28,7 +28,7 @@ defineSupportCode(({ Given, When, Then }) => {
     });
 
     When(/^they submit it to the API$/, function step() {
-        return this.getJwt(this.accessKey)
+        return this.getJwt(this.apiToken)
         .then((response) => {
             const jwt = response.body.token;
 

features/support/world.js

@@ -39,7 +39,7 @@ function promiseToWait(timeToWait) {
  * @return {Promise}
  */
 function ensurePipelineExists(config) {
-    return this.getJwt(this.accessKey)
+    return this.getJwt(this.apiToken)
         .then((response) => {
             this.jwt = response.body.token;
 
@@ -95,20 +95,20 @@ function CustomWorld({ attach, parameters }) {
     this.parameters = parameters;
     env(path.join(__dirname, '../../.func_config'), { raise: false });
     this.gitToken = process.env.GIT_TOKEN;
-    this.accessKey = process.env.ACCESS_KEY;
+    this.apiToken = process.env.API_TOKEN;
     this.protocol = process.env.PROTOCOL || 'https';
     this.instance = `${this.protocol}://${process.env.SD_API}`;
     this.testOrg = process.env.TEST_ORG;
     this.username = process.env.TEST_USERNAME;
     this.scmContext = process.env.TEST_SCM_CONTEXT;
     this.namespace = 'v4';
     this.promiseToWait = time => promiseToWait(time);
-    this.getJwt = accessKey =>
+    this.getJwt = apiToken =>
         request({
             followAllRedirects: true,
             json: true,
             method: 'GET',
-            url: `${this.instance}/${this.namespace}/auth/token?access_key=${accessKey}`
+            url: `${this.instance}/${this.namespace}/auth/token?api_token=${apiToken}`
         });
     this.waitForBuild = buildID =>
         requestretry({
@@ -119,15 +119,15 @@ function CustomWorld({ attach, parameters }) {
             retryStrategy: buildRetryStrategy,
             json: true
         });
-    this.loginWithToken = accessKey =>
+    this.loginWithToken = apiToken =>
         request({
             uri: `${this.instance}/${this.namespace}/auth/logout`,
             method: 'POST',
             auth: {
                 bearer: this.jwt
             }
         // Actual login is accomplished through getJwt
-        }).then(() => this.getJwt(accessKey).then((response) => {
+        }).then(() => this.getJwt(apiToken).then((response) => {
             this.loginResponse = response;
         }));
     this.ensurePipelineExists = ensurePipelineExists;

lib/registerPlugins.js

@@ -41,6 +41,7 @@ function registerResourcePlugins(server, config, callback) {
     const plugins = [
         'auth',
         'builds',
+        'collections',
         'events',
         'jobs',
         'pipelines',

lib/server.js

@@ -21,11 +21,17 @@ function prettyPrintErrors(request, reply) {
             request.log(['server', 'error'], stack);
         }
 
-        return reply({
+        const response = {
             statusCode,
             error: errName,
             message: errMessage
-        }).code(statusCode);
+        };
+
+        if (err.data) {
+            response.data = err.data;
+        }
+
+        return reply(response).code(statusCode);
     }
 
     return reply.continue();
@@ -47,7 +53,9 @@ function prettyPrintErrors(request, reply) {
  * @param  {Factory}     config.userFactory         User Factory instance
  * @param  {Factory}     config.buildFactory        Build Factory instance
  * @param  {Factory}     config.secretFactory       Secret Factory instance
+ * @param  {Factory}     config.tokenFactory        Token Factory instance
  * @param  {Factory}     config.eventFactory        Event Factory instance
+ * @param  {Factory}     config.collectionFactory   Collection Factory instance
  * @param  {Function}    callback                   Callback to invoke when server has started.
  * @return {http.Server}                            A listener: NodeJS http.Server object
  */
@@ -88,6 +96,7 @@ module.exports = (config) => {
         secretFactory: config.secretFactory,
         tokenFactory: config.tokenFactory,
         eventFactory: config.eventFactory,
+        collectionFactory: config.collectionFactory,
         ecosystem: config.ecosystem
     };
 

package.json

@@ -7,7 +7,8 @@
     "pretest": "eslint .",
     "test": "jenkins-mocha --recursive --timeout 3000",
     "start": "./bin/server",
-    "functional": "cucumber-js --format=pretty --tags 'not @ignore'"
+    "functional": "cucumber-js --format=pretty --tags 'not @ignore'",
+    "create-test-user": "node ./features/scripts/create-test-user.js"
   },
   "bin": {
     "screwdriver-api": "./bin/server"
@@ -83,13 +84,13 @@
     "screwdriver-artifact-bookend": "^1.0.1",
     "screwdriver-build-bookend": "^2.0.1",
     "screwdriver-config-parser": "^3.3.0",
-    "screwdriver-data-schema": "^16.9.0",
+    "screwdriver-data-schema": "^16.14.1",
     "screwdriver-datastore-sequelize": "^2.0.0",
     "screwdriver-executor-docker": "^2.2.2",
     "screwdriver-executor-k8s": "^10.3.3",
     "screwdriver-executor-k8s-vm": "^1.0.0",
     "screwdriver-executor-router": "^1.0.0",
-    "screwdriver-models": "^22.2.2",
+    "screwdriver-models": "^22.5.0",
     "screwdriver-notifications-email": "^1.1.2",
     "screwdriver-scm-bitbucket": "^2.8.1",
     "screwdriver-scm-github": "^4.8.2",