From cbf3dc2255f0b731a76d73369ecb6c9cf310ef61 Mon Sep 17 00:00:00 2001 From: Kalin Chernev Date: Wed, 20 Feb 2019 14:31:25 +0200 Subject: [PATCH 1/3] chore(drone): automate deployments - EUBFR-146 --- .drone.yml | 19 +++++ docker-compose.yml | 33 ++++++++ docs/AUTOMATION.md | 102 +++++++++++++++++++++++ tools/eubfr-cli/bin/eubfr-cli-content.js | 15 +++- 4 files changed, 166 insertions(+), 3 deletions(-) create mode 100644 docker-compose.yml create mode 100644 docs/AUTOMATION.md diff --git a/.drone.yml b/.drone.yml index 79f4865fc..7f7300322 100644 --- a/.drone.yml +++ b/.drone.yml @@ -49,6 +49,25 @@ pipeline: secrets: [GH_TOKEN] commands: - ./scripts/deployment/deploy-documentation.sh + when: + event: [push] + branch: master + + deploy: + image: node:8.10 + secrets: + [ + eubfr_env, + eubfr_stage, + aws_access_key_id, + aws_secret_access_key, + eubfr_aws_region, + eubfr_content_repository, + ] + commands: + - npx serverless config credentials --provider aws --key $${AWS_ACCESS_KEY_ID} --secret $${AWS_SECRET_ACCESS_KEY} + - yarn deploy when: event: [push] branch: master + diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 000000000..709b1972f --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,33 @@ +version: '2' + +services: + drone-server: + image: drone/drone:0.8 + + ports: + - 80:8000 + - 9000 + volumes: + - /var/lib/drone:/var/lib/drone/ + restart: always + environment: + - DRONE_OPEN=true + - DRONE_HOST=${DRONE_HOST} + - DRONE_GITLAB=true + - DRONE_GITLAB_CLIENT=${DRONE_GITLAB_CLIENT} + - DRONE_GITLAB_SECRET=${DRONE_GITLAB_SECRET} + - DRONE_GITLAB_URL=https://gitlab.com + - DRONE_SECRET=${DRONE_SECRET} + + drone-agent: + image: drone/agent:0.8 + + command: agent + restart: always + depends_on: + - drone-server + volumes: + - /var/run/docker.sock:/var/run/docker.sock + environment: + - DRONE_SERVER=drone-server:9000 + - DRONE_SECRET=${DRONE_SECRET} diff --git a/docs/AUTOMATION.md b/docs/AUTOMATION.md new file mode 100644 index 000000000..dea69221a --- /dev/null +++ b/docs/AUTOMATION.md @@ -0,0 +1,102 @@ +# Automation + +Automation is achieved with [Drone](https://docs.drone.io/). The `.drone.yml` file describes the pipeline, whereas `docker-compose.yml` provides the means for development and testing of the automation infrastructure to a given machine. + +## Requirements + +Current configuration file requires the following tools for working locally with the automation system: + +- [Drone 0.8](https://0-8-0.docs.drone.io/) +- [Docker](https://www.docker.com/) +- [Docker Compose](https://docs.docker.com/compose/) +- [Gitlab Application](https://docs.drone.io/intro/gitlab/single-machine/) +- [ngrok](https://ngrok.com/) + +You could choose a different instrument for exposing your local server and port to the external world (instead of `ngrok`), and you could also use [Github application](https://docs.drone.io/intro/github/single-machine/) integration, for which you will need to also [modify the configuration file](https://0-8-0.docs.drone.io/install-for-github/) accordingly. + +## Environment variables + +Here's a list of the environment variables used by the setup, for which you will need to prepare and set values for: + +- `DRONE_HOST`: set to the address given by `ngrok`, i.e. `http://ada4e47d.ngrok.io` +- `DRONE_GITLAB_CLIENT`: set to value taken from Gitlab's application `Application ID` +- `DRONE_GITLAB_SECRET`: set to value taken from Gitlab's application `Secret` +- `DRONE_SECRET`: set to any value of your choice + +For the EUBFR CLI to operate correctly, please also specify the following variables: + +- `EUBFR_ENV` +- `EUBFR_STAGE` +- `AWS_ACCESS_KEY_ID` +- `AWS_SECRET_ACCESS_KEY` +- `EUBFR_AWS_REGION` +- `EUBFR_CONTENT_REPOSITORY` + +For detailed explanations about these, please refer to [this guide](https://github.com/ec-europa/eubfr-data-lake/blob/master/tools/eubfr-cli/docs/ENVIRONMENT_VARIABLES.md). + +## Setup + +Here's a brief list of steps you need to take in order to start the automation infrastructure locally. + +### Share local protocol and port + +Simply run: + +``` +$ ngrok http 80 +``` + +This will start sharing your local `http` protocol via port 80. You will receive an address like `http://ada4e47d.ngrok.io` + +This is necessary in order to enable integration between locally-run Drone server and external Oauth2 providers such as Gitlab or Github. + +### Create Oauth2 application + +Go to [Gitlab applications console](https://gitlab.com/oauth/applications) and create a new item. Name is not important. + +Set `Redirect URI` to be `http://ada4e47d.ngrok.io/authorize` giving the application `api` and `read_user` permissions in `Scopes`. + +When the application is created, save the values of `Application ID` and `Secret`. + +### Start Drone + +Create a file `.env` in the root folder. Inside, set the appropriate values: + +``` +DRONE_HOST=http://ada4e47d.ngrok.io +DRONE_GITLAB_CLIENT=value of Application ID +DRONE_GITLAB_SECRET=value of Secret +DRONE_SECRET=value of your preference +``` + +The, run the following: + +```sh +$ docker-compose up +``` + +When the server is running, open `http://ada4e47d.ngrok.io` in your browser and authorize the application. + +When authorized, [activate the project](https://0-8-0.docs.drone.io/getting-started/) in the web UI. + +This activate is necessary for you to be able to configure the secrets and make use of the hooks attached for changes to trigger builds in the Drone automation system. + +### Secrets + +From the web console of your project, i.e. `http://ada4e47d.ngrok.io/you/eubfr-data-lake/settings/secrets`, set the following: + +- `aws_access_key_id` +- `aws_secret_access_key` +- `eubfr_aws_region` - optional +- `eubfr_content_repository` - optional +- `eubfr_env` +- `eubfr_stage` +- `eubfr_username` - optional + +Values for these secrets come from the same reference as if you'd seek for the values of the environment variables with the same names in uppercase. (refer to upper section about the environment variables) + +### Checking results + +At this point, you can either push changes to the remote origin of Gitlab's repository, or run the pipeline via the Drone agent. + +Results are available at: `http://ada4e47d.ngrok.io/you/eubfr-data-lake` diff --git a/tools/eubfr-cli/bin/eubfr-cli-content.js b/tools/eubfr-cli/bin/eubfr-cli-content.js index 7532b4906..f4983e515 100755 --- a/tools/eubfr-cli/bin/eubfr-cli-content.js +++ b/tools/eubfr-cli/bin/eubfr-cli-content.js @@ -113,9 +113,18 @@ program if (producerIsSet) { console.log(`Only files for ${producer} will be downloaded.`); - console.log( - 'Please consider setting EUBFR_USERNAME in order to narrow down deployment and upload operations to this producer for the other operations as well.' - ); + if (!process.env.EUBFR_USERNAME) { + console.log( + 'Please consider setting EUBFR_USERNAME in order to narrow down deployment and upload operations to this producer for the other operations as well.' + ); + } + if (process.env.EUBFR_USERNAME !== producer) { + console.log( + `You are downloading content for ${producer}, but EUBFR_USERNAME is ${ + process.env.EUBFR_USERNAME + }. This could lead to issues among content-related CLI commands.` + ); + } } if (options.confirm) { From 71c0c3133a6b6c79fd13cdbf96947700502c0ffd Mon Sep 17 00:00:00 2001 From: Yannick Huard Date: Tue, 23 Apr 2019 10:43:08 +0300 Subject: [PATCH 2/3] Update docs/AUTOMATION.md Co-Authored-By: kalinchernev --- docs/AUTOMATION.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/AUTOMATION.md b/docs/AUTOMATION.md index dea69221a..e326871c4 100644 --- a/docs/AUTOMATION.md +++ b/docs/AUTOMATION.md @@ -69,7 +69,7 @@ DRONE_GITLAB_SECRET=value of Secret DRONE_SECRET=value of your preference ``` -The, run the following: +Then, run the following: ```sh $ docker-compose up From a55639d83598615994b51376b2beb55a7da10657 Mon Sep 17 00:00:00 2001 From: Yannick Huard Date: Tue, 23 Apr 2019 10:43:18 +0300 Subject: [PATCH 3/3] Update docs/AUTOMATION.md Co-Authored-By: kalinchernev --- docs/AUTOMATION.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/AUTOMATION.md b/docs/AUTOMATION.md index e326871c4..7428ce126 100644 --- a/docs/AUTOMATION.md +++ b/docs/AUTOMATION.md @@ -79,7 +79,7 @@ When the server is running, open `http://ada4e47d.ngrok.io` in your browser and When authorized, [activate the project](https://0-8-0.docs.drone.io/getting-started/) in the web UI. -This activate is necessary for you to be able to configure the secrets and make use of the hooks attached for changes to trigger builds in the Drone automation system. +This activation is necessary for you to be able to configure the secrets and make use of the hooks attached for changes to trigger builds in the Drone automation system. ### Secrets