diff --git a/CHANGELOG b/CHANGELOG index 5ec4c3909..5b33642b9 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -5,6 +5,19 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) ## [Unreleased] +## [v1.38.5] - 2019-04-09 +### Added +- Support for CSS themes + +### Changed +- The CI/CD order to now build docker images in CodeBuild, upload them + to DockerHub and then pull them down in the packer instance. Updated docs. +- Assert TravisCI Python version in advance of change of Travis default to 3.6 + +### Fixed +- Dashboard error on docker spinup + + ## [v1.38.4] - 2019-04-08 ### Fixed - Docker image tagging for git version tag builds @@ -64,7 +77,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Added checks on sending SQS messages to only accept intra-account messages - Improved docker performance and disk space requirements -[Unreleased]: https://github.com/mozilla/MozDef/compare/v1.38.4...HEAD +[Unreleased]: https://github.com/mozilla/MozDef/compare/v1.38.5...HEAD +[v1.38.5]: https://github.com/mozilla/MozDef/compare/v1.38.4...v1.38.5 [v1.38.4]: https://github.com/mozilla/MozDef/compare/v1.38.3...v1.38.4 [v1.38.3]: https://github.com/mozilla/MozDef/compare/v1.38.2...v1.38.3 [v1.38.2]: https://github.com/mozilla/MozDef/compare/v1.38.1...v1.38.2 diff --git a/Makefile b/Makefile index a88a9db7c..0a926bf2a 100644 --- a/Makefile +++ b/Makefile @@ -143,7 +143,3 @@ rebuild: clean build-from-cwd .PHONY: new-alert new-alert: ## Create an example alert and working alert unit test python tests/alert_templater.py - -.PHONY: set-version-and-fetch-docker-container -set-version-and-fetch-docker-container: build-from-cwd tag-images # Lock the release of MozDef by pulling the docker containers on AMI build and caching replace all instances of latest in the compose override with the BRANCH - sed -i s/latest/$(BRANCH)/g docker/compose/docker-compose-cloudy-mozdef.yml diff --git a/cloudy_mozdef/ci/deploy b/cloudy_mozdef/ci/deploy index a5a2e347b..ed82a06da 100644 --- a/cloudy_mozdef/ci/deploy +++ b/cloudy_mozdef/ci/deploy @@ -15,20 +15,20 @@ echo "It's dangerous to go alone. Take one of these: <%%%%|==========>" # Then again we probably do not need to run the test suite here because it has been run three times to get the code here. # echo "Tests complete. -echo "Processing webhook event for ${CODEBUILD_WEBHOOK_TRIGGER}." +echo "Processing webhook event for '${CODEBUILD_WEBHOOK_TRIGGER}'." if [[ "branch/master" == "$CODEBUILD_WEBHOOK_TRIGGER" \ || "$CODEBUILD_WEBHOOK_TRIGGER" =~ ^tag\/v[0-9]+\.[0-9]+\.[0-9]+(\-(prod|pre|testing))?$ ]]; then echo "Building a release" echo "C|_| This may take a bit. Might as well grab a coffee." + BRANCH="`echo $CODEBUILD_WEBHOOK_TRIGGER | cut -d '/' -f2`" make build-from-cwd + make hub-login + make BRANCH=${BRANCH} docker-push-tagged cd cloudy_mozdef - BRANCH="`echo $CODEBUILD_WEBHOOK_TRIGGER | cut -d '/' -f2`" make BRANCH=${BRANCH} packer-build-github make BRANCH=${BRANCH} publish-versioned-templates cd .. - make hub-login - make BRANCH=${BRANCH} docker-push-tagged fi echo "End build of the MozDef codebase." diff --git a/cloudy_mozdef/packer/packer.json b/cloudy_mozdef/packer/packer.json index 58bb59ab5..27f39dbe2 100644 --- a/cloudy_mozdef/packer/packer.json +++ b/cloudy_mozdef/packer/packer.json @@ -1,72 +1,69 @@ { "variables": { - "aws_access_key": "{{env `AWS_ACCESS_KEY_ID`}}", - "aws_secret_key": "{{env `AWS_SECRET_ACCESS_KEY`}}", - "aws_security_token": "{{env `AWS_SESSION_TOKEN`}}" + "aws_access_key": "{{env `AWS_ACCESS_KEY_ID`}}", + "aws_secret_key": "{{env `AWS_SECRET_ACCESS_KEY`}}", + "aws_security_token": "{{env `AWS_SESSION_TOKEN`}}" }, - "builders": [{ - "type": "amazon-ebs", - "region": "us-west-2", - "access_key": "{{user `aws_access_key`}}", - "secret_key": "{{user `aws_secret_key`}}", - "token": "{{user `aws_security_token`}}", - "source_ami": "ami-0d1000aff9a9bad89", - "instance_type": "t2.large", - "ssh_pty" : "true", - "ssh_username": "ec2-user", - "ami_name": "mozdef_{{timestamp}}", - "launch_block_device_mappings": [ - { - "delete_on_termination": true, - "device_name": "/dev/xvda", - "volume_size": 14 + "builders": [ + { + "type": "amazon-ebs", + "region": "us-west-2", + "access_key": "{{user `aws_access_key`}}", + "secret_key": "{{user `aws_secret_key`}}", + "token": "{{user `aws_security_token`}}", + "source_ami": "ami-0d1000aff9a9bad89", + "instance_type": "t2.large", + "ssh_pty": "true", + "ssh_username": "ec2-user", + "ami_name": "mozdef_{{timestamp}}", + "launch_block_device_mappings": [ + { + "delete_on_termination": true, + "device_name": "/dev/xvda", + "volume_size": 14 + } + ], + "ami_description": "An automated build of MozDef triggered via the makefile.", + "ami_groups": [ + "all" + ], + "run_tags": { + "app": "packer-builder-mozdef" + }, + "run_volume_tags": { + "app": "packer-builder-mozdef" + }, + "snapshot_tags": { + "app": "packer-builder-mozdef" + }, + "tags": { + "github:Branch": "{{ user `github_branch`}}", + "buildTimestamp": "{{timestamp}}", + "app": "mozdef" } - ], - "ami_description": "An automated build of MozDef triggered via the makefile.", - "ami_groups": [ - "all" - ], - "run_tags": { - "app": "packer-builder-mozdef" - }, - "run_volume_tags": { - "app": "packer-builder-mozdef" - }, - "snapshot_tags": { - "app": "packer-builder-mozdef" - }, - "tags": { - "github:Branch": "{{ user `github_branch`}}", - "buildTimestamp": "{{timestamp}}", - "app": "mozdef" } - }], - "provisioners": [ - { "type": "shell", + ], + "provisioners": [ + { + "type": "shell", "inline": [ + "set -e", "sudo yum update -y", "sudo yum makecache fast", - "sudo yum install -y glibc-devel gcc libstdc++ libffi-devel zlib-devel make ", - "sudo yum install -y mysql-devel python python-devel python-pip", - "sudo yum install -y git", - "sudo yum install -y docker", - "sudo yum install -y python3", - "sudo pip install virtualenv ", - "sudo pip install docker-compose", + "sudo yum install -y glibc-devel gcc libstdc++ libffi-devel zlib-devel make mysql-devel python python-devel python-pip git docker python3", + "sudo pip install virtualenv docker-compose", "sudo systemctl enable docker", "sudo systemctl start docker", - "sudo mkdir -p /opt/mozdef/", + "sudo mkdir --verbose --parents /opt/mozdef/", "sudo git clone https://github.com/mozilla/MozDef /opt/mozdef", - "cd /opt/mozdef && sudo git checkout {{ user `github_branch`}}", - "cd /opt/mozdef && sudo git rev-parse HEAD", - "cd /opt/mozdef && sudo touch docker/compose/cloudy_mozdef.env docker/compose/rabbitmq.env docker/compose/cloudy_mozdef_mq_cloudtrail.env docker/compose/cloudy_mozdef_mq_sns_sqs.env docker/compose/cloudy_mozdef_kibana.env", - "cd /opt/mozdef && sudo make BRANCH={{ user `github_branch`}} set-version-and-fetch-docker-container", - "cd /opt/mozdef && sudo docker-compose -f docker/compose/docker-compose-cloudy-mozdef.yml -p mozdef pull", - "rm -rf /home/ec2-user/.ssh/authorized_keys", - "rm -rf /home/ec2-user/.ssh/known_hosts", - "sudo rm -rf /tmp/*", - "sudo rm -rf /home/ec2-user/.bash_history", - "sudo rm -rf /root/.ssh" - ]} - ] + "cd /opt/mozdef", + "sudo git checkout {{ user `github_branch`}}", + "sudo git rev-parse HEAD", + "sudo touch docker/compose/cloudy_mozdef.env docker/compose/rabbitmq.env docker/compose/cloudy_mozdef_mq_cloudtrail.env docker/compose/cloudy_mozdef_mq_sns_sqs.env docker/compose/cloudy_mozdef_kibana.env", + "sudo sed --in-place s/latest/{{ user `github_branch`}}/g docker/compose/docker-compose-cloudy-mozdef.yml", + "sudo docker-compose --file docker/compose/docker-compose-cloudy-mozdef.yml --project-name mozdef pull", + "sudo rm --recursive --force --verbose /tmp/* /home/ec2-user/.bash_history /root/.ssh /home/ec2-user/.ssh/known_hosts /home/ec2-user/.ssh/authorized_keys" + ] + } + ] } diff --git a/docker/compose/mozdef_meteor/Dockerfile b/docker/compose/mozdef_meteor/Dockerfile index 39ede7264..7fd6a6733 100644 --- a/docker/compose/mozdef_meteor/Dockerfile +++ b/docker/compose/mozdef_meteor/Dockerfile @@ -46,7 +46,8 @@ RUN mkdir -p /opt/mozdef/envs/meteor/mozdef RUN if [ "${METEOR_BUILD}" = "YES" ]; then \ cd /opt/mozdef/envs/mozdef/meteor && \ meteor npm install && \ - meteor build --server localhost:3002 --directory /opt/mozdef/envs/meteor/mozdef && \ + echo "Starting meteor build" && \ + time meteor build --server localhost:3002 --directory /opt/mozdef/envs/meteor/mozdef && \ cp -r /opt/mozdef/envs/mozdef/meteor/node_modules /opt/mozdef/envs/meteor/mozdef/node_modules &&\ cd /opt/mozdef/envs/meteor/mozdef/bundle/programs/server && \ npm install ;\ diff --git a/docs/source/cicd.rst b/docs/source/cicd.rst index eed94fcad..95155aeb5 100644 --- a/docs/source/cicd.rst +++ b/docs/source/cicd.rst @@ -78,9 +78,9 @@ _________________________________________ The Build Sequence __________________ -* A branch is merged into `master` in the GitHub repo -* GitHub emits a webhook event to AWS CodeBuild indicating that a commit was - pushed to `master` +* A branch is merged into `master` in the GitHub repo or a version git tag is + applied to a commit +* GitHub emits a webhook event to AWS CodeBuild indicating this * AWS CodeBuild reads the `buildspec.yml `_ file to know what to do @@ -98,8 +98,28 @@ __________________ target of the `Makefile` which calls `docker-compose build` on the `docker-compose.yml `_ file, building the docker images in the AWS CodeBuild environment. These are - built so they can be uploaded later to DockerHub for use by developers and - the community. + built both so they can be consumed later in the build by packer and also + for use by developers and the community. +* `deploy` then calls the + `docker-push-tagged `_ + make target which calls + + * the tag-images_ + make target which calls the + `cloudy_mozdef/ci/docker_tag_or_push tag `_ + script which applies a docker image tag to the local image that was just + built by AWS CodeBuild. + * the + `hub-tagged `_ + make target which calls the + `cloudy_mozdef/ci/docker_tag_or_push push `_ + script which + + * Uploads the local image that was just built by AWS CodeBuild to DockerHub. + If the branch being built is `master` then the image is uploaded both with + a tag of `master` as well as with a tag of `latest` + * If the branch being built is from a version tag (e.g. `v1.2.3`) then the + image is uploaded with only that version tag applied * The `deploy` script next calls the `packer-build-github `_ make target in the @@ -124,31 +144,14 @@ __________________ * Within this ec2 instance, packer `clones the MozDef GitHub repo and checks out the branch that triggered this build `_ - * packer calls the `set-version-and-fetch-docker-container `_ - target of the `Makefile` which - - * Calls the - `build-from-cwd `_ - target of the `Makefile` which calls `docker-compose build` on the - `docker-compose.yml `_ - file, building the docker images in the packer ec2 environment - * Calls the tag-images_ - make target which calls the `cloudy_mozdef/ci/docker_tag_or_push tag `_ - script. - - * This applies a docker image tag to the local image that was just built - by packer. This tag will be referenced by the - `docker-compose-cloudy-mozdef.yml` file when the MozDef ec2 instance is - launched from the AMI. - - * Replaces all instances of the word `latest` in the - `docker-compose-cloudy-mozdef.yml `_ - file with either the branch `master` or the version tag (e.g. `v1.2.3`) - + * packer replaces all instances of the word `latest` in the + `docker-compose-cloudy-mozdef.yml `_ + file with either the branch `master` or the version tag (e.g. `v1.2.3`) * packer runs `docker-compose pull` on the `docker-compose-cloudy-mozdef.yml `_ - file to pull down any remaining non MozDef container images that weren't - just built in preceding packer steps + file to pull down both the docker images that were just built by AWS + CodeBuild and uploaded to Dockerhub as well as other non MozDef docker + images * After packer completes executing the steps laid out in `packer.json` inside the ec2 instance, it generates an AMI from that instance and continues with @@ -168,26 +171,5 @@ __________________ * uploads the CloudFormation templates to S3 in a directory either called `master` or the tag version that was built (e.g. `v1.2.3`) -* `deploy` then calls the - `docker-push-tagged `_ - make target which calls - - * the tag-images_ - make target which calls the - `cloudy_mozdef/ci/docker_tag_or_push tag `_ - script which applies a docker image tag to the local image that was just - built by AWS CodeBuild. - * the - `hub-tagged `_ - make target which calls the - `cloudy_mozdef/ci/docker_tag_or_push push `_ - script which - - * Uploads the local image that was just built by AWS CodeBuild to DockerHub. - If the branch being built is `master` then the image is uploaded both with - a tag of `master` as well as with a tag of `latest` - * If the branch being built is from a version tag (e.g. `v1.2.3`) then the - image is uploaded with only that version tag applied - .. _docker/compose/docker-compose-tests.yml: https://github.com/mozilla/MozDef/blob/master/docker/compose/docker-compose-tests.yml .. _tag-images: https://github.com/mozilla/MozDef/blob/cfeafb77f9d4d4d8df02117a0ffca0ec9379a7d5/Makefile#L109-L110