feat: add interface to support different types of databases

DEVELOPMENT.md

@@ -131,7 +131,7 @@ make it p=".*Eth_getBlockByNumberRequest"
 
 ## Migrations
 
-On startup the proxy service will run any SQL based migration in the [migrations folder](./clients/database/migrations) that haven't already been run against the database being used.
+On startup the proxy service will run any SQL based migration in the [migrations folder](clients/database/postgres/migrations) that haven't already been run against the database being used.
 
 For lower level details on how the migration process works consult [these docs](https://bun.uptrace.dev/guide/migrations.html).
 
@@ -144,7 +144,7 @@ $ date '+%Y%m%d%H%M%S'
 > 20230306182227
 ```
 
-Add new SQL file with commands to run in the new migration (add/delete/modify tables and or indices) in the in the [migrations folder](./clients/database/migrations)
+Add new SQL file with commands to run in the new migration (add/delete/modify tables and or indices) in the in the [migrations folder](clients/database/postgres/migrations)
 
 ### Running migrations
 

architecture/ARCHITECTURE.md

@@ -20,7 +20,7 @@ The proxy functionality provides the foundation for all other proxy service feat
 
 ![API Observability Worfklow Conceptual Overview](./images/observability_workflow_conceptual.jpg)
 
-For every request that is proxied by the proxy service, a [request metric](../decode/evm_rpc.go) is created and stored in a [postgres table](../clients/database/migrations/20230306182203_add_proxied_request_metrics_table.up.sql) that can be aggregated with other request metrics over a time range to answer ad hoc questions such as:
+For every request that is proxied by the proxy service, a [request metric](../decode/evm_rpc.go) is created and stored in a [postgres table](../clients/database/postgres/migrations/20230306182203_add_proxied_request_metrics_table.up.sql) that can be aggregated with other request metrics over a time range to answer ad hoc questions such as:
 
 - what methods take the longest time?
 - what methods are called the most frequently?

architecture/MIGRATIONS.MD

@@ -14,7 +14,7 @@ Setting an environment variable named `RUN_DATABASE_MIGRATIONS` to true will cau
 
 ### Migration Format
 
-New migration files must be placed in the [migrations directory](../clients/database/migrations/), have a unique name, and start with a timestamp in the below format:
+New migration files must be placed in the [migrations directory](../clients/database/postgres/migrations/), have a unique name, and start with a timestamp in the below format:
 
 ```bash
 $ date '+%Y%m%d%H%M%S'

clients/database/empty/database.go

@@ -0,0 +1,40 @@
+package empty
+
+import (
+	"context"
+	"github.com/kava-labs/kava-proxy-service/clients/database"
+)
+
+type Empty struct{}
+
+func New() *Empty {
+	return &Empty{}
+}
+
+func (e *Empty) SaveProxiedRequestMetric(ctx context.Context, metric *database.ProxiedRequestMetric) error {
+	return nil
+}
+
+func (e *Empty) ListProxiedRequestMetricsWithPagination(ctx context.Context, cursor int64, limit int) ([]*database.ProxiedRequestMetric, int64, error) {
+	return []*database.ProxiedRequestMetric{}, 0, nil
+}
+
+func (e *Empty) CountAttachedProxiedRequestMetricPartitions(ctx context.Context) (int64, error) {
+	return 0, nil
+}
+
+func (e *Empty) GetLastCreatedAttachedProxiedRequestMetricsPartitionName(ctx context.Context) (string, error) {
+	return "", nil
+}
+
+func (e *Empty) DeleteProxiedRequestMetricsOlderThanNDays(ctx context.Context, n int64) error {
+	return nil
+}
+
+func (e *Empty) HealthCheck() error {
+	return nil
+}
+
+func (e *Empty) Partition(prefillPeriodDays int) error {
+	return nil
+}

clients/database/interface.go

@@ -1,11 +1,34 @@
 package database
 
-import "context"
+import (
+	"context"
+	"time"
+)
 
 type MetricsDatabase interface {
-	SaveProxiedRequestMetric(ctx context.Context, prm *ProxiedRequestMetric) error
-	ListProxiedRequestMetricsWithPagination(ctx context.Context, cursor int64, limit int) ([]ProxiedRequestMetric, int64, error)
+	SaveProxiedRequestMetric(ctx context.Context, metric *ProxiedRequestMetric) error
+	ListProxiedRequestMetricsWithPagination(ctx context.Context, cursor int64, limit int) ([]*ProxiedRequestMetric, int64, error)
 	CountAttachedProxiedRequestMetricPartitions(ctx context.Context) (int64, error)
 	GetLastCreatedAttachedProxiedRequestMetricsPartitionName(ctx context.Context) (string, error)
-	DeleteProxiedRequestMetricsOlderThanNDays(ctx context.Context, days int) error
+	DeleteProxiedRequestMetricsOlderThanNDays(ctx context.Context, n int64) error
+
+	HealthCheck() error
+	Partition(prefillPeriodDays int) error
+}
+
+type ProxiedRequestMetric struct {
+	ID                          int64
+	MethodName                  string
+	BlockNumber                 *int64
+	ResponseLatencyMilliseconds int64
+	Hostname                    string
+	RequestIP                   string
+	RequestTime                 time.Time
+	UserAgent                   *string
+	Referer                     *string
+	Origin                      *string
+	ResponseBackend             string
+	ResponseBackendRoute        string
+	CacheHit                    bool
+	PartOfBatch                 bool
 }

clients/database/postgres.go → clients/database/postgres/database.go

@@ -1,4 +1,4 @@
-package database
+package postgres
 
 import (
 	"crypto/tls"
@@ -13,12 +13,9 @@ import (
 	"github.com/uptrace/bun/extra/bundebug"
 )
 
-// PostgresDatabaseConfig contains values for creating a
+// DatabaseConfig contains values for creating a
 // new connection to a postgres database
-type PostgresDatabaseConfig struct {
-	// DatabaseDisabled is used to disable the database, and it won't be used at all. All operations will be skipped.
-	DatabaseDisabled bool
-
+type DatabaseConfig struct {
 	DatabaseName                     string
 	DatabaseEndpointURL              string
 	DatabaseUsername                 string
@@ -34,21 +31,15 @@ type PostgresDatabaseConfig struct {
 	Logger                           *logging.ServiceLogger
 }
 
-// PostgresClient wraps a connection to a postgres database
-type PostgresClient struct {
-	isDisabled bool
-	*bun.DB
+// Client wraps a connection to a postgres database
+type Client struct {
+	db     *bun.DB
+	logger *logging.ServiceLogger
 }
 
-// NewPostgresClient returns a new connection to the specified
+// NewClient returns a new connection to the specified
 // postgres data and error (if any)
-func NewPostgresClient(config PostgresDatabaseConfig) (PostgresClient, error) {
-	if config.DatabaseDisabled {
-		return PostgresClient{
-			isDisabled: true,
-		}, nil
-	}
-
+func NewClient(config DatabaseConfig) (Client, error) {
 	// configure postgres database connection options
 	var pgOptions *pgdriver.Connector
 
@@ -96,17 +87,14 @@ func NewPostgresClient(config PostgresDatabaseConfig) (PostgresClient, error) {
 		db.AddQueryHook(bundebug.NewQueryHook(bundebug.WithVerbose(true)))
 	}
 
-	return PostgresClient{
-		DB: db,
+	return Client{
+		db:     db,
+		logger: config.Logger,
 	}, nil
 }
 
 // HealthCheck returns an error if the database can not
 // be connected to and queried, nil otherwise
-func (pg *PostgresClient) HealthCheck() error {
-	if pg.isDisabled {
-		return nil
-	}
-
-	return pg.Ping()
+func (c *Client) HealthCheck() error {
+	return c.db.Ping()
 }

clients/database/postgres/database_test.go

@@ -0,0 +1,18 @@
+package postgres
+
+import (
+	"github.com/stretchr/testify/require"
+	"testing"
+)
+
+func TestDisabledDBCreation(t *testing.T) {
+	config := DatabaseConfig{}
+	_, err := NewClient(config)
+	require.Error(t, err)
+}
+
+func TestHealthcheckNoDatabase(t *testing.T) {
+	config := DatabaseConfig{}
+	_, err := NewClient(config)
+	require.Error(t, err)
+}

clients/database/database.go → clients/database/postgres/migrate.go

@@ -1,12 +1,11 @@
-package database
+package postgres
 
 import (
 	"context"
 	"fmt"
 	"time"
 
 	"github.com/kava-labs/kava-proxy-service/logging"
-	"github.com/uptrace/bun"
 	"github.com/uptrace/bun/migrate"
 )
 
@@ -15,12 +14,9 @@ import (
 // returning error (if any) and a list of migrations that have been
 // run and any that were not
 // If db is nil, returns empty slice and nil error, as there is no database to migrate.
-func Migrate(ctx context.Context, db *bun.DB, migrations migrate.Migrations, logger *logging.ServiceLogger) (*migrate.MigrationSlice, error) {
-	if db == nil {
-		return &migrate.MigrationSlice{}, nil
-	}
+func (c *Client) Migrate(ctx context.Context, migrations migrate.Migrations, logger *logging.ServiceLogger) (*migrate.MigrationSlice, error) {
 	// set up migration config
-	migrator := migrate.NewMigrator(db, &migrations)
+	migrator := migrate.NewMigrator(c.db, &migrations)
 
 	// create / verify tables used to tack migrations
 	err := migrator.Init(ctx)

clients/database/postgres/migrate_test.go

@@ -0,0 +1,15 @@
+package postgres
+
+import (
+	"context"
+	"github.com/stretchr/testify/require"
+	"github.com/uptrace/bun/migrate"
+	"testing"
+)
+
+func TestMigrateNoDatabase(t *testing.T) {
+	db := &Client{}
+
+	_, err := db.Migrate(context.Background(), migrate.Migrations{}, nil)
+	require.Error(t, err)
+}

clients/database/postgres/partition_test.go

@@ -0,0 +1,55 @@
+package postgres
+
+import (
+	"github.com/kava-labs/kava-proxy-service/config"
+	"github.com/stretchr/testify/assert"
+	"testing"
+	"time"
+)
+
+func TestUnitTestPartitionsForPeriodReturnsExpectedNumPartitionsWhenPrefillPeriodIsNotContainedInCurrentMonth(t *testing.T) {
+	// prepare
+
+	// pick a date in the middle of a month
+	startFrom := time.Date(1989, 5, 20, 12, 0, 0, 0, time.UTC)
+
+	// set prefill period to more then days remaining in month
+	// from above date
+	daysToPrefill := 21
+
+	// execute
+	actualPartitionsForPeriod, err := PartitionsForPeriod(startFrom, daysToPrefill)
+
+	// assert
+	assert.Nil(t, err)
+	assert.Equal(t, daysToPrefill, len(actualPartitionsForPeriod))
+}
+
+func TestUnitTestPartitionsForPeriodReturnsErrWhenTooManyPrefillDays(t *testing.T) {
+	// prepare
+	daysToPrefill := config.MaxMetricPartitioningPrefillPeriodDays + 1
+
+	// execute
+	_, err := PartitionsForPeriod(time.Now(), daysToPrefill)
+
+	// assert
+	assert.NotNil(t, err)
+}
+
+func TestUnitTestPartitionsForPeriodReturnsExpectedNumPartitionsWhenPrefillPeriodIsContainedInCurrentMonth(t *testing.T) {
+	// prepare
+
+	// pick a date in the middle of a month
+	startFrom := time.Date(1989, 5, 11, 12, 0, 0, 0, time.UTC)
+
+	// set prefill period to less then days remaining in month
+	// from above date
+	daysToPrefill := 3
+
+	// execute
+	actualPartitionsForPeriod, err := PartitionsForPeriod(startFrom, daysToPrefill)
+
+	// assert
+	assert.Nil(t, err)
+	assert.Equal(t, daysToPrefill, len(actualPartitionsForPeriod))
+}