Merge pull request #28 from OnionIoT/feature/pipeline-automation

Feature/pipeline automation

.github/workflows/development-pipeline-deletion-event.yml

@@ -0,0 +1,91 @@
+# This pipeline create a new pipeline in AWS CodePipeline if the branch follows the desired pattern defined in BRANCH_PATTERN.
+name: "Delete branch and delete AWS CodePipeline" 
+
+on:
+  delete:
+    branches:
+      - 'openwrt-2*'
+
+env:
+  BRANCH_PATTERN: openwrt-2[0-9]\.[0-9]{2}
+  TERRAFORM_VERSION: 1.7.4
+  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+
+jobs:
+  branch_created:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+
+      - name: Get branch name
+        run: | 
+            echo "BRANCH_NAME=${{ github.event.ref }}" >> $GITHUB_ENV
+      
+      - name: Verify branch name
+        run: |
+          if [[ ! "${BRANCH_NAME}" =~ $BRANCH_PATTERN ]]; then
+            echo "Branch name doesn't match the pattern."
+            echo "VALID_BRANCH=false" >> $GITHUB_ENV
+          else
+            echo "New branch created $GITHUB_REF"
+            echo "VALID_BRANCH=true" >> $GITHUB_ENV
+          fi
+
+      - name: Set up Terraform
+        if: env.VALID_BRANCH == 'true'
+        uses: hashicorp/setup-terraform@v3
+        with:
+          terraform_version: $TERRAFORM_VERSION
+
+      - name: Terraform Init 
+        if: env.VALID_BRANCH == 'true'
+        run: |
+          cd .terraform/pipeline
+          terraform init
+
+      - name: Terraform Select Workspace
+        if: env.VALID_BRANCH == 'true'
+        run: |
+          cd .terraform/pipeline
+          terraform workspace select $BRANCH_NAME
+          echo "Current Workspace is $(terraform workspace show)"
+
+      - name: Check Terraform Workspace
+        if: env.VALID_BRANCH == 'true'
+        run: |
+          cd .terraform/pipeline
+          if [ -z "$(terraform state list)" ]; then
+            echo "TF_IS_EMPTY=true"
+            echo "Workspace Terraform is empty"
+          else
+            echo "TF_IS_EMPTY=false"
+            echo "Workspace Terraform is not empty"
+          fi
+
+      - name: Terraform Update Bucket Force Deletion
+        if: env.VALID_BRANCH == 'true' && env.TF_IS_EMPTY == 'false'
+        run: |
+          cd .terraform/pipeline
+          sed -i "/stage_vars = {/a \ \ \"\${{ env.BRANCH_NAME }}\" = {\n \ \ \ branch = \"\${{ env.BRANCH_NAME }}\"\n \ \ }" terraform.tfvars
+          sed -i '/^resource "aws_s3_bucket" "codepipeline_bucket" {/a force_destroy = true' main.tf
+          terraform apply -auto-approve -var="buildspec_file_name=development-buildspec.yml"
+      
+      - name: Terraform Destroy && env.TF_IS_EMPTY == 'false'
+        if: env.VALID_BRANCH == 'true'
+        run: |
+          cd .terraform/pipeline
+          terraform destroy -auto-approve
+
+      - name: Terraform Delete Workspace
+        if: env.VALID_BRANCH == 'true'
+        run: |
+          cd .terraform/pipeline
+          terraform workspace select default
+          terraform workspace delete $BRANCH_NAME
+          echo "Workspace $BRANCH_NAME deleted"
+
+      - name: NOTHING TO DO
+        if: env.VALID_BRANCH == 'false'
+        run: echo "THIS BRANCH DOES NOT CREATE A NEW AWS CODEPIPELINE"

.github/workflows/development-pipeline.yml

@@ -0,0 +1,78 @@
+# This pipeline create a new pipeline in AWS CodePipeline if the branch follows the desired pattern defined in BRANCH_PATTERN.
+name: "New Branch create AWS CodePipeline" 
+
+on:
+  create:
+    branches:
+      - 'openwrt-2*'
+
+env:
+  BRANCH_PATTERN: openwrt-2[0-9]\.[0-9]{2}
+  TERRAFORM_VERSION: 1.7.4
+  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+
+jobs:
+  branch_created:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+        with:
+          ref: devops
+
+      - name: Get branch name
+        run: | 
+            echo "BRANCH_NAME=${{ github.event.ref }}" >> $GITHUB_ENV
+      
+      - name: Verify branch name
+        run: |
+          if [[ ! "${BRANCH_NAME}" =~ $BRANCH_PATTERN ]]; then
+            echo "Branch name doesn't match the pattern."
+            echo "VALID_BRANCH=false" >> $GITHUB_ENV
+          else
+            echo "New branch created $GITHUB_REF"
+            echo "VALID_BRANCH=true" >> $GITHUB_ENV
+          fi
+
+      - name: Set up Terraform
+        if: env.VALID_BRANCH == 'true'
+        uses: hashicorp/setup-terraform@v3
+        with:
+          terraform_version: $TERRAFORM_VERSION
+
+      - name: Terraform Init 
+        if: env.VALID_BRANCH == 'true'
+        run: |
+          cd .terraform/pipeline
+          terraform init
+
+      - name: Terraform New Workspace
+        if: env.VALID_BRANCH == 'true'
+        run: |
+          cd .terraform/pipeline
+          terraform workspace new $BRANCH_NAME
+          echo "Current Workspace is $(terraform workspace show)"
+
+      - name: Terraform Validate
+        if: env.VALID_BRANCH == 'true'
+        run: |
+          cd .terraform/pipeline
+          terraform validate
+
+      - name: Terraform Plan
+        if: env.VALID_BRANCH == 'true'
+        run: |
+          cd .terraform/pipeline
+          sed -i "/stage_vars = {/a \ \ \"\${{ env.BRANCH_NAME }}\" = {\n \ \ \ branch = \"\${{ env.BRANCH_NAME }}\"\n \ \ }" terraform.tfvars
+          terraform plan -var="buildspec_file_name=development-buildspec.yml"
+
+      - name: Terraform Apply
+        if: env.VALID_BRANCH == 'true'
+        run: |
+          cd .terraform/pipeline
+          terraform apply -auto-approve -var="buildspec_file_name=development-buildspec.yml"
+
+      - name: NOTHING TO DO
+        if: env.VALID_BRANCH == 'false'
+        run: echo "THIS BRANCH DOES NOT CREATE A NEW AWS CODEPIPELINE"

.terraform/pipeline/build-action.tf

@@ -24,7 +24,7 @@ module "build_action" {
   secrets                    = local.codebuild_shared_secrets
   build_step                 = "build"
   compute_type               = "BUILD_GENERAL1_LARGE"
-  buildspec_file             = "buildspec.yml"
+  buildspec_file             = var.buildspec_file_name
   cache_bucket               = aws_s3_bucket.codepipeline_bucket.bucket
   is_privileged_mode         = true
 }

.terraform/pipeline/main.tf

@@ -1,32 +1,34 @@
 
 data "aws_caller_identity" "current" {}
 
+data "aws_codestarconnections_connection" "github_connection" {
+  name = "openwrt-buildsystem-git-devops"
+}
+
 locals {
   stage      = terraform.workspace
   stage_vars = var.stage_vars[local.stage]
+
+  stage_parts   = can(split("-", local.stage)) ? split("-", local.stage) : [local.stage]
+  stage_suffix  = length(local.stage_parts) > 1 ? local.stage_parts[1] : local.stage
+
   tags = {
     ProjectName = var.project_name
     Stage       = local.stage
     Scope       = "pipeline"
   }
 
-
   tf_codebuild_env_vars = {
     stage         = local.stage
     REGION        = var.region
     OUTPUT_BUCKET = var.deployment_bucket
+    RELEASE_VERSION = stage_suffix
   }
 
   codebuild_shared_secrets = {
   }
 }
 
-resource "aws_codestarconnections_connection" "github_connection" {
-  name          = "${var.project_name}-git-${local.stage}"
-  provider_type = "GitHub"
-}
-
-
 resource "aws_s3_bucket" "codepipeline_bucket" {
   bucket = "devops-${var.project_name}-artifacts-${local.stage}"
   tags   = local.tags
@@ -54,7 +56,7 @@ resource "aws_codepipeline" "codepipeline" {
       output_artifacts = ["source_output"]
 
       configuration = {
-        ConnectionArn        = aws_codestarconnections_connection.github_connection.arn
+        ConnectionArn        = data.aws_codestarconnections_connection.github_connection.arn
         FullRepositoryId     = var.repository
         BranchName           = local.stage_vars.branch
         DetectChanges        = true
@@ -76,14 +78,11 @@ resource "aws_codepipeline" "codepipeline" {
       output_artifacts = ["build_output"]
       version          = "1"
 
-
       configuration = {
         ProjectName   = module.build_action.aws_codebuild_project
         PrimarySource = "source_output"
       }
     }
   }
-
-
 }
 

.terraform/pipeline/modules/codebuild/main.tf

@@ -8,7 +8,7 @@ resource "aws_cloudwatch_log_group" "step_log_group" {
 }
 
 resource "aws_codebuild_project" "step_build_project" {
-  name          = "${var.project_name}-codebuild-${var.build_step}-${var.stage}"
+  name          = "${var.project_name}-codebuild-${var.build_step}-${replace(var.stage, ".", "")}"
   description   = var.step_description
   service_role  = aws_iam_role.service_role.arn
   tags          = var.tags

.terraform/pipeline/output.tf

@@ -1,13 +1,12 @@
 locals {
-  code_start_connection_id = split("/", aws_codestarconnections_connection.github_connection.id)[1]
+  code_start_connection_id = split("/", data.aws_codestarconnections_connection.github_connection.id)[1]
 }
 
 output "Github_connection_url" {
   value       = "https://${var.region}.console.aws.amazon.com/codesuite/settings/${data.aws_caller_identity.current.account_id}/${var.region}/connections/${local.code_start_connection_id}"
   description = "Connection url for code star to link Code pipeline to Github app"
 }
 
-
 output "artifacts_bucket" {
   value       = aws_s3_bucket.codepipeline_bucket.bucket
   description = "Bucket name for artifacts. (used also for cache)"

.terraform/pipeline/terraform.tfvars

@@ -1,8 +1,10 @@
 project_name      = "openwrt-buildsystem"
 region            = "us-east-1"
 deployment_bucket = "downloads.onioniot.com"
+repository        = "OnionIoT/openwrt-buildsystem-wrapper"
 
-repository = "OnionIoT/openwrt-buildsystem-wrapper"
+# Set the buildspec file name. Options include 'development-buildspec.yml' for development or 'buildspec.yml' for production.
+buildspec_file_name = "buildspec.yml"
 
 stage_vars = {
   prod = {

.terraform/pipeline/variables.tf

@@ -18,11 +18,13 @@ variable "repository" {
   description = "Source Github repository containing files and buildspec"
 }
 
+variable "buildspec_file_name" {
+  type        = string
+  description = "The name of the buildspec file for correct development or production builds."
+}
+
+
 variable "stage_vars" {
   description = "Stage Specific Variables"
-  type = map(
-    object({
-      branch = string
-    })
-  )
+  type        = map(object({ branch = string }))
 }

buildspec.yml

@@ -38,12 +38,10 @@ phases:
 
       - cd "$ROOT_PATH/openwrt/bin/packages/mipsel_24kc"
       - aws s3 cp --recursive . "s3://$OUTPUT_BUCKET/releases/$RELEASE_VERSION/packages/mipsel_24kc/"
-      - aws s3 cp --recursive . "s3://$OUTPUT_BUCKET/builds/$RELEASE_VERSION/$TAG_NAME/packages/mipsel_24kc/"
 
       - echo "Copying Binaries"
       - cd "$ROOT_PATH/openwrt/bin/targets/ramips/mt76x8"
       - aws s3 cp --recursive . "s3://$OUTPUT_BUCKET/releases/$RELEASE_VERSION/targets/ramips/mt76x8/"
-      - aws s3 cp --recursive . "s3://$OUTPUT_BUCKET/builds/$RELEASE_VERSION/$TAG_NAME/targets/ramips/mt76x8/"
 
 cache:
   paths:

development-buildspec.yml

@@ -0,0 +1,54 @@
+#  Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#  SPDX-License-Identifier: MIT-0
+version: 0.2
+env:
+  variables:
+    FORCE_UNSAFE_CONFIGURE: 1
+    ROOT_PATH: "/openwrt"
+phases:
+  install:
+    commands:
+      - echo "Installing dependencies"
+      - apt-get update
+      - apt -y install build-essential clang flex bison g++ gawk gcc-multilib g++-multilib gettext git libncurses5-dev libssl-dev python3-distutils rsync unzip zlib1g-dev file wget python3
+      - apt -y install --upgrade awscli
+      # remove pyenv from path to use the default system installation.
+      - export PATH=$(echo $PATH | sed 's@/root/.pyenv/shims:/root/.pyenv/bin:@@g')
+  pre_build:
+    commands:
+      - mkdir -p $ROOT_PATH
+      - cp -r $CODEBUILD_SRC_DIR/. $ROOT_PATH
+      - mkdir -p $ROOT_PATH/keys
+      - aws s3 cp s3://onion-build/openwrt/openwrt21.key $ROOT_PATH/keys/key-build
+      - aws s3 cp s3://onion-build/openwrt/openwrt21.pub $ROOT_PATH/keys/key-build.pub
+
+  build:
+    commands:
+      - cd $ROOT_PATH
+      - bash build.sh
+  post_build:
+    commands:
+      - if [ "$CODEBUILD_BUILD_SUCCEEDING" = "0" ]; then echo ">>> VERBOSE DEBUG BUILD"; bash build.sh -V ; exit 1; fi
+
+      - cd $ROOT_PATH
+      - BUILD_DATE=$(date +%Y%m%d%H%M%S)
+
+      - echo "Copying Packages"
+      - cd "$ROOT_PATH/openwrt/bin/packages/mipsel_24kc"
+      - aws s3 cp --recursive . "s3://$OUTPUT_BUCKET/builds/$RELEASE_VERSION/$BUILD_DATE/packages/mipsel_24kc/"
+
+      - echo "Copying Binaries"
+      - cd "$ROOT_PATH/openwrt/bin/targets/ramips/mt76x8"
+      - aws s3 cp --recursive . "s3://$OUTPUT_BUCKET/builds/$RELEASE_VERSION/$BUILD_DATE/targets/ramips/mt76x8/"
+
+cache:
+  paths:
+    - /var/cache/apt/**/*
+    - /var/lib/apt/lists/**/*
+
+artifacts:
+  type: zip
+  files:
+    - "packages/mipsel_24kc/**/*"
+    - "targets/ramips/mt76x8/**/*"
+  base-directory: "$ROOT_PATH/openwrt/bin"