build.yml

name: Build, Test, Deploy

on:
  push:
    branches:
      - main
  pull_request:
    branches:
      - main

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  check_changes:
    runs-on: ubuntu-latest
    permissions:
      contents: read
      pull-requests: read
    outputs:
      sdk_node: ${{ steps.filter.outputs.sdk_node }}
      sdk_dotnet: ${{ steps.filter.outputs.sdk_dotnet }}
      sdk_go: ${{ steps.filter.outputs.sdk_go }}
      sdk_react: ${{ steps.filter.outputs.sdk_react }}
      cli: ${{ steps.filter.outputs.cli }}
      control_plane: ${{ steps.filter.outputs.control_plane }}
      app: ${{ steps.filter.outputs.app }}
      # Deploy steps (Docker build / CFN) if either control-plane or app changed
      deploy: ${{ steps.filter.outputs.deploy }}
      load_tests: ${{ steps.filter.outputs.load_tests }}
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: Filter changed files
        uses: dorny/paths-filter@v3
        id: filter
        with:
          filters: |
            sdk_node:
              - 'sdk-node/**'
            sdk_dotnet:
              - 'sdk-dotnet/**'
            sdk_go:
              - 'sdk-go/**'
            sdk_react:
              - 'sdk-react/**'
            cli:
              - 'cli/**'
            control_plane:
              - 'control-plane/**'
            app:
              - 'app/**'
            load_tests:
              - 'load-tests/**'
            deploy:
              - 'control-plane/**'
              - 'app/**'

  build-control-plane:
    needs: check_changes
    if: ${{ needs.check_changes.outputs.control_plane == 'true' }}
    runs-on: ubuntu-latest
    permissions:
      contents: read
    defaults:
      run:
        working-directory: control-plane
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
      - name: Set up Node.js
        uses: actions/setup-node@v4
        with:
          node-version: 20
          cache: "npm"
          cache-dependency-path: control-plane/package-lock.json
      - name: Install dependencies
        run: npm ci
      - name: Build package
        run: npm run build

  test-control-plane:
    runs-on: ubuntu-latest
    needs: [check_changes, build-control-plane]
    if: ${{ needs.check_changes.outputs.control_plane == 'true' }}
    defaults:
      run:
        working-directory: control-plane
    services:
      redis:
        image: redis
        ports:
          - 6379:6379
        options: >-
          --health-cmd "redis-cli ping"
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
      postgres:
        image: pgvector/pgvector:pg16
        env:
          POSTGRES_PASSWORD: postgres
        options: >-
          --health-cmd pg_isready
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
        ports:
          - 5432:5432
    steps:
      - name: Checkout code
        uses: actions/checkout@v2
      - name: Set up Node.js
        uses: actions/setup-node@v4
        with:
          node-version: 20
          cache: "npm"
          cache-dependency-path: control-plane/package-lock.json
      - name: Install dependencies
        run: npm ci
      - name: Run tests
        run: |
          set -eo pipefail
          npm run test
          if [[ "$(git rev-parse --abbrev-ref HEAD)" == "main" ]]; then
            npm run test:ai
          fi
        env:
          REDIS_URL: "redis://localhost:6379"
          DATABASE_URL: "postgres://postgres:postgres@localhost:5432/postgres"
          DATABASE_SSL_DISABLED: "1"
          JWKS_URL: ${{ secrets.TEST_JWKS_URL }}
          AWS_ACCESS_KEY_ID: ${{ secrets.TEST_AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.TEST_AWS_SECRET_ACCESS_KEY }}
          AWS_DEFAULT_REGION: "ap-southeast-2"
          BEDROCK_AVAILABLE: "true"
          SQS_RUN_PROCESS_QUEUE_URL: "PLACEHOLDER"
          SQS_RUN_GENERATE_NAME_QUEUE_URL: "PLACEHOLDER"
          SQS_LEARNING_INGEST_QUEUE_URL: "PLACEHOLDER"
          SQS_CUSTOMER_TELEMETRY_QUEUE_URL: "PLACEHOLDER"
          SQS_EXTERNAL_TOOL_CALL_QUEUE_URL: "PLACEHOLDER"

  build-app:
    needs: check_changes
    if: ${{ needs.check_changes.outputs.app == 'true' }}
    runs-on: ubuntu-latest
    permissions:
      contents: read
    defaults:
      run:
        working-directory: app
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
      - name: Set up Node.js
        uses: actions/setup-node@v4
        with:
          node-version: 20
          cache: "npm"
          cache-dependency-path: app/package-lock.json
      - name: Install dependencies
        run: npm ci
      - name: Build package
        run: npm run build
    env:
      NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY: ${{ secrets.CLOUD_PROD_CLERK_PUBLISHABLE_KEY }}

  build-node:
    needs: check_changes
    if: ${{ needs.check_changes.outputs.sdk_node == 'true' }}
    runs-on: ubuntu-latest
    permissions:
      contents: read
    defaults:
      run:
        working-directory: sdk-node
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
      - name: Set up Node.js
        uses: actions/setup-node@v4
        with:
          node-version: 20
          cache: "npm"
          cache-dependency-path: sdk-node/package-lock.json
      - name: Install dependencies
        run: npm ci
      - name: Build package
        run: npm run build

  test-node:
    needs: [check_changes, build-node]
    if: ${{ needs.check_changes.outputs.sdk_node == 'true' }}
    runs-on: ubuntu-latest
    permissions:
      contents: read
    defaults:
      run:
        working-directory: sdk-node
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
      - name: Set up Node.js ${{ matrix.node-version }}
        uses: actions/setup-node@v4
        with:
          node-version: ${{ matrix.node-version }}
          cache: "npm"
          cache-dependency-path: sdk-node/package-lock.json
      - name: Install dependencies
        run: npm ci
      - name: Run tests
        run: npm run test
        env:
          INFERABLE_TEST_API_ENDPOINT: "https://api.inferable.ai"
          INFERABLE_TEST_CLUSTER_ID: ${{ secrets.INFERABLE_TEST_CLUSTER_ID }}
          INFERABLE_TEST_API_SECRET: ${{ secrets.INFERABLE_TEST_API_SECRET }}

  build-cli:
    needs: check_changes
    if: ${{ needs.check_changes.outputs.cli == 'true' }}
    runs-on: ubuntu-latest
    permissions:
      contents: read
    defaults:
      run:
        working-directory: cli
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
      - name: Set up Node.js
        uses: actions/setup-node@v4
        with:
          node-version: 20
          cache: "npm"
          cache-dependency-path: cli/package-lock.json
      - name: Install dependencies
        run: npm ci
      - name: Build package
        run: npm run build

  test-cli:
    needs: [check_changes, build-cli]
    if: ${{ needs.check_changes.outputs.cli == 'true' }}
    runs-on: ubuntu-latest
    permissions:
      contents: read
    defaults:
      run:
        working-directory: cli
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
      - name: Set up Node.js ${{ matrix.node-version }}
        uses: actions/setup-node@v4
        with:
          node-version: ${{ matrix.node-version }}
          cache: "npm"
          cache-dependency-path: cli/package-lock.json
      - name: Install dependencies
        run: npm ci
      - name: Run tests
        run: npm run test


  build-dotnet:
    needs: check_changes
    if: ${{ needs.check_changes.outputs.sdk_dotnet == 'true' }}
    runs-on: windows-latest
    permissions:
      contents: read
    defaults:
      run:
        working-directory: sdk-dotnet
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
      - name: Set up .NET
        uses: actions/setup-dotnet@v4
        with:
          dotnet-version: "8.0.x"
      - name: Restore dependencies
        run: dotnet restore
      - name: Build
        run: dotnet build --configuration Release --no-restore

  test-dotnet:
    needs: [check_changes, build-dotnet]
    if: ${{ needs.check_changes.outputs.sdk_dotnet == 'true' }}
    runs-on: windows-latest
    permissions:
      contents: read
    defaults:
      run:
        working-directory: sdk-dotnet
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
      - name: Set up .NET
        uses: actions/setup-dotnet@v4
        with:
          dotnet-version: "8.0.x"
      - name: Restore dependencies
        run: dotnet restore
      - name: Test
        run: dotnet test --no-restore
        env:
          INFERABLE_TEST_API_ENDPOINT: "https://api.inferable.ai"
          INFERABLE_TEST_CLUSTER_ID: ${{ secrets.INFERABLE_TEST_CLUSTER_ID }}
          INFERABLE_TEST_API_SECRET: ${{ secrets.INFERABLE_TEST_API_SECRET }}

  build-go:
    needs: check_changes
    if: ${{ needs.check_changes.outputs.sdk_go == 'true' }}
    runs-on: ubuntu-latest
    permissions:
      contents: read
    defaults:
      run:
        working-directory: sdk-go
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
      - name: Set up Go
        uses: actions/setup-go@v5
        with:
          go-version: "1.22"
      - name: Check formatting
        run: |
          if [ "$(gofmt -l . | wc -l)" -gt 0 ]; then
            echo "The following files are not formatted correctly:"
            gofmt -l .
            exit 1
          fi
      - name: Get dependencies
        run: go mod download
      - name: Build
        run: go build -v ./...

  test-go:
    needs: [check_changes, build-go]
    if: ${{ needs.check_changes.outputs.sdk_go == 'true' }}
    runs-on: ubuntu-latest
    permissions:
      contents: read
    defaults:
      run:
        working-directory: sdk-go
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
      - name: Set up Go
        uses: actions/setup-go@v5
        with:
          go-version: "1.22"
      - name: Get dependencies
        run: go mod download
      - name: Test
        run: go test -v ./...
        env:
          INFERABLE_TEST_API_ENDPOINT: "https://api.inferable.ai"
          INFERABLE_TEST_CLUSTER_ID: ${{ secrets.INFERABLE_TEST_CLUSTER_ID }}
          INFERABLE_TEST_API_SECRET: ${{ secrets.INFERABLE_TEST_API_SECRET }}

  build-react:
    needs: check_changes
    if: ${{ needs.check_changes.outputs.sdk_react == 'true' }}
    runs-on: ubuntu-latest
    permissions:
      contents: read
    defaults:
      run:
        working-directory: sdk-react
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
      - name: Set up Node.js
        uses: actions/setup-node@v4
        with:
          node-version: 20
          cache: "npm"
          cache-dependency-path: sdk-react/package-lock.json
      - name: Install dependencies
        run: npm ci
      - name: Build package
        run: npm run build

  test-react:
    needs: [check_changes, build-react]
    if: ${{ needs.check_changes.outputs.sdk_react == 'true' }}
    runs-on: ubuntu-latest
    permissions:
      contents: read
    defaults:
      run:
        working-directory: sdk-react
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
      - name: Set up Node.js
        uses: actions/setup-node@v4
        with:
          node-version: 20
          cache: "npm"
          cache-dependency-path: sdk-react/package-lock.json
      - name: Install dependencies
        run: npm ci
      - name: Run tests
        run: npm test

  validate-load-tests:
    needs: [check_changes]
    runs-on: ubuntu-latest
    if: ${{ needs.check_changes.outputs.load_tests == 'true' }}
    defaults:
      run:
        working-directory: load-tests
    env:
      INFERABLE_TEST_CLUSTER_ID: ${{ secrets.INFERABLE_TEST_CLUSTER_ID }}
      INFERABLE_TEST_API_SECRET: ${{ secrets.INFERABLE_TEST_API_SECRET }}
    steps:
      - uses: actions/checkout@v4
      - name: Install dependencies
        run: npm ci
        # Run machine:start backgrounded
      - run: npm run machine:start &
      - uses: grafana/setup-k6-action@v1
      - uses: grafana/run-k6-action@v1
        with:
          path: ./load-tests/script.js

  build-load-test-machine-image:
    runs-on: ubuntu-latest
    needs: check_changes
    permissions:
      id-token: write
    if: ${{ needs.check_changes.outputs.load_tests == 'true' }}
    #if: ${{ needs.check_changes.outputs.load_tests == 'true' && github.ref == 'refs/heads/main' }}
    steps:
      - name: Checkout code
        uses: actions/checkout@v2

      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: ${{ secrets.AWS_ROLE_ARN }}
          aws-region: us-east-1

      - name: Login to Amazon ECR
        id: login-ecr
        uses: aws-actions/amazon-ecr-login@v1

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v2

      - name: Build App Docker Image
        env:
          IMAGE_TAG: ${{ github.sha }}
          ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
          ECR_REPOSITORY: inferable-load-test-machine
        run: |
          cd load-tests
          VERSION=${{ github.sha }}
          SHORT_VERSION=$(echo ${{ github.sha }} | cut -c 1-6)
          docker buildx build \
          --file Dockerfile.machine \
          --push \
          --cache-to mode=min,image-manifest=true,oci-mediatypes=true,type=registry,ref=$ECR_REGISTRY/$ECR_REPOSITORY:cache \
          --cache-from type=registry,ref=$ECR_REGISTRY/$ECR_REPOSITORY:cache \
          --build-arg="VERSION=$VERSION" \
          --build-arg="SHORT_VERSION=$SHORT_VERSION" \
          -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG -t $ECR_REGISTRY/$ECR_REPOSITORY:latest ./

  build-app-image:
    runs-on: ubuntu-latest
    needs: check_changes
    if: ${{ needs.check_changes.outputs.deploy == 'true' && github.ref == 'refs/heads/main' }}
    permissions:
      id-token: write
    steps:
      - name: Checkout code
        uses: actions/checkout@v2

      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: ${{ secrets.AWS_ROLE_ARN }}
          aws-region: us-east-1

      - name: Login to Amazon ECR
        id: login-ecr
        uses: aws-actions/amazon-ecr-login@v1

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v2

      - name: Build App Docker Image
        env:
          IMAGE_TAG: ${{ github.sha }}
          ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
          ECR_REPOSITORY: inferable-app
        run: |
          cd app
          VERSION=${{ github.sha }}
          SHORT_VERSION=$(echo ${{ github.sha }} | cut -c 1-6)
          docker buildx build \
          --target run \
          --push \
          --cache-to mode=min,image-manifest=true,oci-mediatypes=true,type=registry,ref=$ECR_REGISTRY/$ECR_REPOSITORY:cache \
          --cache-from type=registry,ref=$ECR_REGISTRY/$ECR_REPOSITORY:cache \
          --build-arg="VERSION=$VERSION" \
          --build-arg="SHORT_VERSION=$SHORT_VERSION" \
          --build-arg="NEXT_PUBLIC_INFERABLE_API_URL=${{ vars.API_URL }}" \
          --build-arg="NEXT_PUBLIC_APP_URL=${{ vars.APP_URL }}" \
          --build-arg="NEXT_PUBLIC_CLERK_PUBLISHABLE_KEY=${{ secrets.CLOUD_PROD_CLERK_PUBLISHABLE_KEY }}" \
          --build-arg="NEXT_PUBLIC_HYPERDX_API_KEY=${{ secrets.CLOUD_PROD_HYPERDX_API_KEY }}" \
          --build-arg="NEXT_PUBLIC_POSTHOG_KEY=${{ secrets.CLOUD_PROD_POSTHOG_KEY }}" \
          -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG -t $ECR_REGISTRY/$ECR_REPOSITORY:latest ./

  build-control-plane-image:
    runs-on: ubuntu-latest
    needs: check_changes
    if: ${{ needs.check_changes.outputs.deploy == 'true' && github.ref == 'refs/heads/main' }}
    permissions:
      id-token: write
    steps:
      - name: Checkout code
        uses: actions/checkout@v2

      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: ${{ secrets.AWS_ROLE_ARN }}
          aws-region: us-east-1

      - name: Login to Amazon ECR
        id: login-ecr
        uses: aws-actions/amazon-ecr-login@v1

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v2

      - name: Build Control Plane Docker Image
        env:
          IMAGE_TAG: ${{ github.sha }}
          ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
          ECR_REPOSITORY: inferable-api
          DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
          DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
        run: |
          cd control-plane
          VERSION=${{ github.sha }}
          SHORT_VERSION=$(echo ${{ github.sha }} | cut -c 1-6)
          docker buildx build \
          --target run \
          --push \
          --cache-to mode=min,image-manifest=true,oci-mediatypes=true,type=registry,ref=$ECR_REGISTRY/$ECR_REPOSITORY:cache \
          --cache-from type=registry,ref=$ECR_REGISTRY/$ECR_REPOSITORY:cache \
          --build-arg="VERSION=$VERSION" \
          --build-arg="SHORT_VERSION=$SHORT_VERSION" \
          -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG -t $ECR_REGISTRY/$ECR_REPOSITORY:latest ./
          echo "Pushing Control Plane Image to Docker Hub"
          docker login -u $DOCKERHUB_USERNAME -p $DOCKERHUB_PASSWORD
          docker pull $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG
          docker tag $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG $DOCKERHUB_USERNAME/control-plane:$IMAGE_TAG
          docker tag $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG $DOCKERHUB_USERNAME/control-plane:latest
          docker push $DOCKERHUB_USERNAME/control-plane:$IMAGE_TAG
          docker push $DOCKERHUB_USERNAME/control-plane:latest

  deploy-cloud:
    runs-on: ubuntu-latest
    if: ${{ needs.check_changes.outputs.deploy == 'true' && github.ref == 'refs/heads/main' }}
    permissions:
      id-token: write
    needs:
      [check_changes, build-app-image, build-control-plane-image]
    steps:
    - name: Checkout code
      uses: actions/checkout@v2
    - name: Configure AWS credentials
      uses: aws-actions/configure-aws-credentials@v4
      with:
        role-to-assume: ${{ secrets.AWS_ROLE_ARN }}
        aws-region: us-east-1
    - name: Notify start deploy to Rollbar
      uses: rollbar/github-deploy-action@2.1.1
      id: rollbar_pre_deploy
      with:
        environment: 'production'
        version: ${{ github.sha }}
        status: 'started'
      env:
        ROLLBAR_ACCESS_TOKEN: ${{ secrets.CLOUD_ROLLBAR_ACCESS_TOKEN }}
        ROLLBAR_USERNAME: ${{ github.actor }}
    - name: Deploy to AWS CloudFormation
      uses: aws-actions/aws-cloudformation-github-deploy@v1
      with:
        name: "prod-inferable"
        role-arn: ${{ secrets.CLOUD_AWS_CFN_ROLE_ARN }}
        template: ${{ secrets.CLOUD_AWS_CFN_TEMPLATE }}
        tags: "Environment=prod"
        capabilities: "CAPABILITY_NAMED_IAM,CAPABILITY_IAM"
        parameter-overrides: >-
          Environment=prod,
          ApiImageTag=${{ github.sha }},
          AppImageTag=${{ github.sha }}
    - name: Notify finish deploy to Rollbar
      uses: rollbar/github-deploy-action@2.1.1
      id: rollbar_post_deploy
      with:
        environment: 'production'
        version: ${{ github.sha }}
        status: 'succeeded'
      env:
        ROLLBAR_ACCESS_TOKEN: ${{ secrets.CLOUD_ROLLBAR_ACCESS_TOKEN }}
        ROLLBAR_USERNAME: ${{ github.actor }}
        DEPLOY_ID: ${{ steps.rollbar_pre_deploy.outputs.deploy_id }}