From 21901ff9d655e757f0493ce99dd62b3ac50aa204 Mon Sep 17 00:00:00 2001
From: Levi Schoen <levi.schoen@kava.io>
Date: Thu, 2 Nov 2023 06:18:45 -0700
Subject: [PATCH] ensure metric collection happens async of request handling
 (#55)

add docs on how to hotfix & debug service
---
 DEVELOPMENT.md        | 45 +++++++++++++++++++++++++++++++++++++++++++
 main.go               |  2 ++
 service/middleware.go | 30 +++++++++++++++--------------
 3 files changed, 63 insertions(+), 14 deletions(-)

diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md
index 6a610f7..06e54e9 100644
--- a/DEVELOPMENT.md
+++ b/DEVELOPMENT.md
@@ -103,6 +103,7 @@ For details on the local E2E setup, see [the `docker` directory](./docker/README
 The Continuous Integration (CI) for this repo is setup to run a local proxy service with database & redis, but configures the service to use public testnet urls for the kava requests. This allows for testing the proxy service against a production-like environment (requests are routed to public testnet).
 
 You can emulate the CI configuration in your local environment:
+
 ```bash
 make ci-setup
 ```
@@ -280,6 +281,50 @@ If the service is deployed on AWS ECS, to force ECS to start a new instance of t
 AWS_PROFILE=production aws ecs update-service --cluster kava-internal-testnet-proxy-service --service kava-internal-testnet-proxy-service --force-new-deployment
 ```
 
+### Hotfix flow
+
+1. Make changes to code
+
+2. Build and push updated image to deployment repo
+
+```bash
+# login before pushing to a Dockerhub repo
+# create a personal access token first https://docs.docker.com/docker-hub/access-tokens/
+docker login -u kavaops
+
+# build and push image, you can customize the tag here
+docker buildx build -f ./production.Dockerfile  --platform linux/amd64,linux/arm64 --push -t kava/kava-proxy-service:latest .
+
+# see below for if you need to update service config to use a custom tag
+AWS_PROFILE=production aws ecs update-service --cluster kava-internal-testnet-proxy-service --service kava-internal-testnet-proxy-service --force-new-deployment
+```
+
+3. If you need to update values used to configure the service (such as what tag the service should run with)
+
+```bash
+cd /infrastructure/terraform/product/production/us-east-1/kava-internal-testnet-proxy-service/service
+
+AWS_PROFILE=production terragrunt apply
+```
+
+4. Verify
+
+Run e2e and manual tests against the endpoint as needed (note that any E2E tests that involve checking the database or redis will fail when run against a production environment), for example to run the e2e tests against internal testnet
+
+```bash
+# update this value in .env
+TEST_PROXY_SERVICE_EVM_RPC_URL=https://evm.data.internal.testnet.us-east.production.kava.io
+```
+
+```bash
+# run e2e tests (filtered down using regex pattern `p`)
+make it p="TestE2ETestProxyReturnsNonZero
+LatestBlockHeader"
+
+# run curl commands against endpoint hot fix is deployed to
+time curl -H "Content-Type: application/json" --data '{"jsonrpc":"2.0","method":"eth_getLogs","params":[{"topics":["0x1602ce2aba92f09827c6a342020908249036cb9863e6895041095afdce392a5e"]}],"id":74}' https://evm.data.internal.testnet.us-east.production.kava.io
+```
+
 ## Feedback
 
 For suggesting changes or reporting issues, please open a Github Issue.
diff --git a/main.go b/main.go
index a213682..62733b2 100644
--- a/main.go
+++ b/main.go
@@ -135,6 +135,8 @@ func main() {
 	// create the main proxy service
 	service, err := service.New(serviceContext, serviceConfig, &serviceLogger)
 
+	serviceLogger.Trace().Msg(fmt.Sprintf("service initialized: %+v", service))
+
 	if err != nil {
 		serviceLogger.Panic().Msg(fmt.Sprintf("%v", errors.Unwrap(err)))
 	}
diff --git a/service/middleware.go b/service/middleware.go
index b673729..a91d364 100644
--- a/service/middleware.go
+++ b/service/middleware.go
@@ -474,20 +474,22 @@ func createAfterProxyFinalizer(service *ProxyService, config config.Config) http
 			CacheHit:                    isCached,
 		}
 
-		// save metric to database
-		err = metric.Save(context.Background(), service.Database.DB)
+		// save metric to database async
+		go func() {
+			// using background context so save won't be terminated when request finishes
+			err = metric.Save(context.Background(), service.Database.DB)
 
-		if err != nil {
-			// TODO: consider only logging
-			//  if it's not due to connection exhaustion, e.g.
-			// FATAL: remaining connection slots are reserved for non-replication
-			// superuser connections; SQLState: 53300
-			// OR
-			// FATAL: sorry, too many clients already; SQLState: 53300
-			service.ServiceLogger.Error().Msg(fmt.Sprintf("error %s saving metric %+v using database %+v", err, metric, service.Database))
-			return
-		}
-
-		service.ServiceLogger.Trace().Msg("created request metric")
+			if err != nil {
+				// TODO: consider only logging
+				//  if it's not due to connection exhaustion, e.g.
+				// FATAL: remaining connection slots are reserved for non-replication
+				// superuser connections; SQLState: 53300
+				// OR
+				// FATAL: sorry, too many clients already; SQLState: 53300
+				service.ServiceLogger.Error().Msg(fmt.Sprintf("error %s saving metric %+v using database %+v", err, metric, service.Database))
+				return
+			}
+			service.ServiceLogger.Trace().Msg("created request metric")
+		}()
 	}
 }