Skip to content

Commit

Permalink
add initial ci pipleline to run on every commit, pr and merge to main
Browse files Browse the repository at this point in the history
galxy25 committed Mar 24, 2023

Verified

This commit was signed with the committer’s verified signature.
LunarX Gibran Chevalley
1 parent 37490a8 commit 8b9fb60
Showing 13 changed files with 170 additions and 15 deletions.
1 change: 1 addition & 0 deletions .env
Original file line number Diff line number Diff line change
@@ -30,6 +30,7 @@ PROXY_HOST_DEBUG_PORT=2345

##### E2E Testing Config
TEST_PROXY_SERVICE_EVM_RPC_URL=http://localhost:7777
TEST_PROXY_SERVICE_EVM_RPC_HOSTNAME=localhost:7777
TEST_PROXY_SERVICE_EVM_RPC_DATA_URL=http://localhost:7778
TEST_PROXY_BACKEND_EVM_RPC_HOST_URL=http://localhost:8545
TEST_DATABASE_ENDPOINT_URL=localhost:5432
7 changes: 7 additions & 0 deletions .github/workflows/ci-commit.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
name: Continuous Integration (Commit)
on:
push:
# run per commit ci checks against this commit
jobs:
lint:
uses: ./.github/workflows/ci-lint.yml
31 changes: 31 additions & 0 deletions .github/workflows/ci-default.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
name: Continuous Integration (Default Checks)

on:
workflow_call:
jobs:
build:
runs-on: ubuntu-latest
steps:
- name: checkout repo from current commit
uses: actions/checkout@v3
- name: Set up Go
uses: actions/setup-go@v3
with:
go-version: '1.20'
check-latest: true
cache: true
- name: build application binary
run: make install
unit-test:
runs-on: ubuntu-latest
steps:
- name: checkout repo from current commit
uses: actions/checkout@v3
- name: Set up Go
uses: actions/setup-go@v3
with:
go-version: '1.20'
check-latest: true
cache: true
- name: run unit tests
run: make unit-test
39 changes: 39 additions & 0 deletions .github/workflows/ci-e2e-tests.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
name: Continuous Integration (E2E Testing Checks)

on:
workflow_call:
jobs:
e2e-test:
runs-on: ubuntu-latest
steps:
- name: checkout repo from current commit
uses: actions/checkout@v3
- name: set up Go
uses: actions/setup-go@v3
with:
go-version: "1.20"
check-latest: true
cache: true
- name: pull pre-built images
run: sudo docker compose -f ci.docker-compose.yml pull
# In this step, this action saves a list of existing images,
# the cache is created without them in the post run.
# It also restores the cache if it exists.
- name: cache docker images
uses: satackey/[email protected]
# Ignore the failure of a step and avoid terminating the job.
continue-on-error: true
- name: build and start proxy service and it's dependencies
run: sudo docker compose -f ci.docker-compose.yml up -d
- name: wait for proxy service to be running
run: bash ${GITHUB_WORKSPACE}/scripts/wait-for-proxy-service-running.sh
env:
PROXY_CONTAINER_PORT: 7777
- name: run e2e tests
run: make e2e-test
- name: print proxy service logs
run: sudo docker compose -f ci.docker-compose.yml logs proxy
# because we especially want the logs if the test(s) fail 😅
if: always()
# Finally, "Post Run jpribyl/[email protected]",
# which is the process of saving the cache, will be executed.
14 changes: 14 additions & 0 deletions .github/workflows/ci-lint.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
name: Lint Checks
on:
workflow_call:
# run per commit lint checks against this commit
jobs:
golangci-lint:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@master
- name: golangci-lint
uses: reviewdog/action-golangci-lint@v2
with:
github_token: ${{ secrets.github_token }}
reporter: github-pr-review
13 changes: 13 additions & 0 deletions .github/workflows/ci-main.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
name: Continuous Integration (Kava Proxy Service Main)
on:
push:
# run CI on any push to the main branch
branches:
- main
jobs:
# run per commit ci checks against master branch
lint-checks:
uses: ./.github/workflows/ci-lint.yml
# run default ci checks against master branch
default-checks:
uses: ./.github/workflows/ci-default.yml
13 changes: 13 additions & 0 deletions .github/workflows/ci-pr.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
name: Continuous Integration (PR)
on:
pull_request:
# run CI on pull requests to main or a release branch
branches:
- main
- 'releases/**'
# run default ci checks against current PR
jobs:
# default:
# uses: ./.github/workflows/ci-default.yml
e2e-tests:
uses: ./.github/workflows/ci-e2e-tests.yml
18 changes: 9 additions & 9 deletions Makefile
Original file line number Diff line number Diff line change
@@ -51,29 +51,29 @@ test: unit-test e2e-test
.PHONY: up
# start dockerized versions of the service and it's dependencies
up:
docker-compose up -d
docker compose up -d

.PHONY: down
# stop the service and it's dependencies
down:
docker-compose down
docker compose down

.PHONY: restart
# restart just the service (useful for picking up new environment variable values)
restart:
docker-compose up -d proxy --force-recreate
docker compose up -d proxy --force-recreate

.PHONY: reset
# wipe state and restart the service and all it's dependencies
reset: lint
docker-compose up -d --build --remove-orphans --renew-anon-volumes --force-recreate
docker compose up -d --build --remove-orphans --renew-anon-volumes --force-recreate

.PHONY: refresh

# rebuild from latest local sources and restart just the service containers
# (preserving any volume state such as database tables & rows)
refresh:
docker-compose up -d proxy --build --force-recreate
docker compose up -d proxy --build --force-recreate

# poll kava service status endpoint until it doesn't error
.PHONY: ready
@@ -87,22 +87,22 @@ ready:
# or one
# make logs S=proxy
logs:
docker-compose logs ${S} -f
docker compose logs ${S} -f

.PHONY: debug-proxy
# attach the dlv debugger to the running service and connect to the dlv debugger
debug-proxy:
docker-compose exec -d proxy dlv attach 1 --listen=:${PROXY_CONTAINER_DEBUG_PORT} --headless --api-version=2 --log && \
docker compose exec -d proxy dlv attach 1 --listen=:${PROXY_CONTAINER_DEBUG_PORT} --headless --api-version=2 --log && \
dlv connect :${PROXY_HOST_DEBUG_PORT}

.PHONY: debug-database
# open a connection to the postgres database for debugging it's state
# https://www.postgresql.org/docs/current/app-psql.html
debug-database:
docker-compose exec postgres psql -U ${DATABASE_USERNAME} -d ${DATABASE_NAME}
docker compose exec postgres psql -U ${DATABASE_USERNAME} -d ${DATABASE_NAME}

.PHONY: debug-cache
# open a connection to the redis service for debugging it's state
# https://redis.io/docs/ui/cli/
debug-cache:
docker-compose exec redis redis-cli
docker compose exec redis redis-cli
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
@@ -17,8 +17,8 @@ Many aspects of the service are configurable via environment variables:
- ERROR

- `PROXY_BACKEND_HOST_URL_MAP` - comma delimited list of `HOSTNAME_TO_PROXY>BACKEND_PROXY_SERVER_FOR_HOSTNAME` controls what backend server the proxy service will proxy a request to based on the hostname of the request. `,` is used as a separator between entries in the map, `>` is used as a separator within an entry to delimit between the hostname to proxy for and the backend to proxy to. At least one entry must be present. The same backend can be used as the proxy for multiple hostnames by creating one entry for each of the hostnames to proxy for. Example value:
> PROXY_BACKEND_HOST_URL_MAP=evm.app.internal.testnet.us-east.production.kava.io>https://evmrpc.internal.testnet.proxy.kava.io,evm.data.internal.testnet.us-east.production.kava.io>https://evmrpcdata.internal.testnet.proxy.kava.io

> PROXY_BACKEND_HOST_URL_MAP=evm.app.internal.testnet.us-east.production.kava.io>https://evmrpc.internal.testnet.proxy.kava.io,evm.data.internal.testnet.us-east.production.kava.io>https://evmrpcdata.internal.testnet.proxy.kava.io
### Logging

34 changes: 34 additions & 0 deletions ci.docker-compose.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
---
services:
# run postgres for proxy service to store observability metrics
postgres:
image: postgres:15
env_file: .env
ports:
- "${POSTGRES_HOST_PORT}:${POSTGRES_CONTAINER_PORT}"
expose:
- "${POSTGRES_CONTAINER_PORT}"
# run redis for proxy service to cache responses
redis:
image: 'bitnami/redis:latest'
env_file: .env
ports:
- "${REDIS_HOST_PORT}:${REDIS_CONTAINER_PORT}"
expose:
- "${REDIS_CONTAINER_PORT}"
# run proxy service to observe, route, and scale requests to kava api endpoints
proxy:
build:
dockerfile: local.Dockerfile
env_file: .env
environment:
# use public testnet as backend origin server to avoid having
# to self-host a beefy Github Action runner
# to build and run a kava node each execution
PROXY_BACKEND_HOST_URL_MAP: localhost:7777>https://evmrpc.internal.testnet.proxy.kava.io,localhost:7778>https://evmrpcdata.internal.testnet.proxy.kava.io
ports:
- "${PROXY_HOST_PORT}:${PROXY_CONTAINER_PORT}"
- "${PROXY_CONTAINER_EVM_RPC_DATA_PORT}:${PROXY_CONTAINER_PORT}"
- "${PROXY_HOST_DEBUG_PORT}:${PROXY_CONTAINER_DEBUG_PORT}"
cap_add:
- SYS_PTRACE # Allows for attaching debugger to process in this container
6 changes: 5 additions & 1 deletion main.go
Original file line number Diff line number Diff line change
@@ -43,5 +43,9 @@ func main() {
serviceLogger.Panic().Msg(fmt.Sprintf("%v", errors.Unwrap(err)))
}

service.Run()
finalErr := service.Run()

if finalErr != nil {
serviceLogger.Debug().Msg(fmt.Sprintf("service stopped with error %s", finalErr))
}
}
2 changes: 1 addition & 1 deletion scripts/wait-for-proxy-service-running.sh
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
#!/bin/bash
set -x

until curl -f localhost:"${PROXY_CONTAINER_PORT}/healthcheck"
until curl -f http://localhost:"${PROXY_CONTAINER_PORT}/healthcheck"
do
echo "waiting for proxy service to be running"
sleep 0.5
5 changes: 2 additions & 3 deletions service/middleware.go
Original file line number Diff line number Diff line change
@@ -5,7 +5,6 @@ import (
"context"
"fmt"
"io"
"io/ioutil"
"net/http"
"net/http/httputil"
"strings"
@@ -70,7 +69,7 @@ func createRequestLoggingMiddleware(h http.HandlerFunc, serviceLogger *logging.S

var err error

rawBody, err = ioutil.ReadAll(body)
rawBody, err = io.ReadAll(body)

if err != nil {
serviceLogger.Debug().Msg(fmt.Sprintf("error %s reading request body %s", err, body))
@@ -81,7 +80,7 @@ func createRequestLoggingMiddleware(h http.HandlerFunc, serviceLogger *logging.S
}

// Repopulate the request body for the ultimate consumer of this request
r.Body = ioutil.NopCloser(&rawBodyBuffer)
r.Body = io.NopCloser(&rawBodyBuffer)
}

decodedRequest, err := decode.DecodeEVMRPCRequest(rawBody)

0 comments on commit 8b9fb60

Please sign in to comment.