Skip to content

Commit

Permalink
add config and routine init logic for metric pruning
Browse files Browse the repository at this point in the history
  • Loading branch information
@galxy25
galxy25 committed Oct 27, 2023
1 parent 1407348 commit cbb9e50
Show file tree
Hide file tree
Showing 4 changed files with 214 additions and 83 deletions.
14 changes: 14 additions & 0 deletions .env
View file
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,20 @@ METRIC_PARTITIONING_ROUTINE_DELAY_FIRST_RUN_SECONDS=10
METRIC_PARTITIONINING_PREFILL_PERIOD_DAYS=7
# Used by `ready` script to ensure metric partitions have been created.
MINIMUM_REQUIRED_PARTITIONS=30
# Whether metric pruning routines should run on the configured interval, defaults to true
METRIC_PRUNING_ENABLED=true
# How frequenlty metric pruning routines should run
# defaults to 1 day
METRIC_PRUNING_ROUTINE_INTERVAL_SECONDS=10
# how long (after the proxy service starts) it will wait
# to run the first iteration of the metric pruning routine
METRIC_PRUNING_ROUTINE_DELAY_FIRST_RUN_SECONDS=5
# The maximum number of days of request metrics that should
# be preserved starting from
# the end of the current day each time
# the pruning routine runs
# all metrics for earlier days will be truncated
METRIC_PRUNING_MAX_REQUEST_METRICS_HISTORY_DAYS=1
# CACHE_ENABLED specifies if cache should be enabled. By default cache is disabled.
CACHE_ENABLED=true
# REDIS_ENDPOINT_URL is an url of redis
Expand Down
180 changes: 97 additions & 83 deletions config/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,39 +13,42 @@ import (
)

type Config struct {
ProxyServicePort string
LogLevel string
ProxyBackendHostURLMapRaw string
ProxyBackendHostURLMapParsed map[string]url.URL
EnableHeightBasedRouting bool
ProxyPruningBackendHostURLMapRaw string
ProxyPruningBackendHostURLMap map[string]url.URL
EvmQueryServiceURL string
DatabaseName string
DatabaseEndpointURL string
DatabaseUserName string
DatabasePassword string
DatabaseReadTimeoutSeconds int64
DatabaseWriteTimeoutSeconds int64
DatabaseSSLEnabled bool
DatabaseQueryLoggingEnabled bool
DatabaseMaxIdleConnections int64
DatabaseConnectionMaxIdleSeconds int64
DatabaseMaxOpenConnections int64
RunDatabaseMigrations bool
HTTPReadTimeoutSeconds int64
HTTPWriteTimeoutSeconds int64
MetricCompactionRoutineInterval time.Duration
MetricCollectionEnabled bool
MetricPartitioningRoutineInterval time.Duration
MetricPartitioningRoutineDelayFirstRun time.Duration
MetricPartitioningPrefillPeriodDays int
CacheEnabled bool
RedisEndpointURL string
RedisPassword string
CacheTTL time.Duration
CacheIndefinitely bool
CachePrefix string
ProxyServicePort string
LogLevel string
ProxyBackendHostURLMapRaw string
ProxyBackendHostURLMapParsed map[string]url.URL
EnableHeightBasedRouting bool
ProxyPruningBackendHostURLMapRaw string
ProxyPruningBackendHostURLMap map[string]url.URL
EvmQueryServiceURL string
DatabaseName string
DatabaseEndpointURL string
DatabaseUserName string
DatabasePassword string
DatabaseReadTimeoutSeconds int64
DatabaseWriteTimeoutSeconds int64
DatabaseSSLEnabled bool
DatabaseQueryLoggingEnabled bool
DatabaseMaxIdleConnections int64
DatabaseConnectionMaxIdleSeconds int64
DatabaseMaxOpenConnections int64
RunDatabaseMigrations bool
HTTPReadTimeoutSeconds int64
HTTPWriteTimeoutSeconds int64
MetricCompactionRoutineInterval time.Duration
MetricCollectionEnabled bool
MetricPartitioningRoutineInterval time.Duration
MetricPartitioningRoutineDelayFirstRun time.Duration
MetricPartitioningPrefillPeriodDays int
MetricPruningEnabled bool
MetricPruningRoutineInterval time.Duration
MetricPruningRoutineDelayFirstRun time.Duration
MetricPruningMaxRequestMetricsHistoryDays int
CacheEnabled bool
RedisEndpointURL string
RedisPassword string
CacheTTL time.Duration
CachePrefix string
}

const (
Expand Down Expand Up @@ -79,23 +82,31 @@ const (
DEFAULT_METRIC_PARTITIONING_ROUTINE_DELAY_FIRST_RUN_SECONDS = 10
METRIC_PARTITIONING_PREFILL_PERIOD_DAYS_ENVIRONMENT_KEY = "METRIC_PARTITIONING_PREFILL_PERIOD_DAYS"
DEFAULT_METRIC_PARTITIONING_PREFILL_PERIOD_DAYS = 7
EVM_QUERY_SERVICE_ENVIRONMENT_KEY = "EVM_QUERY_SERVICE_URL"
DATABASE_MAX_IDLE_CONNECTIONS_ENVIRONMENT_KEY = "DATABASE_MAX_IDLE_CONNECTIONS"
DEFAULT_DATABASE_MAX_IDLE_CONNECTIONS = 20
DATABASE_CONNECTION_MAX_IDLE_SECONDS_ENVIRONMENT_KEY = "DATABASE_CONNECTION_MAX_IDLE_SECONDS"
DEFAULT_DATABASE_CONNECTION_MAX_IDLE_SECONDS = 5
DATABASE_MAX_OPEN_CONNECTIONS_ENVIRONMENT_KEY = "DATABASE_MAX_OPEN_CONNECTIONS"
DEFAULT_DATABASE_MAX_OPEN_CONNECTIONS = 100
DATABASE_READ_TIMEOUT_SECONDS_ENVIRONMENT_KEY = "DATABASE_READ_TIMEOUT_SECONDS"
DEFAULT_DATABASE_READ_TIMEOUT_SECONDS = 60
DATABASE_WRITE_TIMEOUT_SECONDS_ENVIRONMENT_KEY = "DATABASE_WRITE_TIMEOUT_SECONDS"
DEFAULT_DATABASE_WRITE_TIMEOUT_SECONDS = 10
CACHE_ENABLED_ENVIRONMENT_KEY = "CACHE_ENABLED"
REDIS_ENDPOINT_URL_ENVIRONMENT_KEY = "REDIS_ENDPOINT_URL"
REDIS_PASSWORD_ENVIRONMENT_KEY = "REDIS_PASSWORD"
CACHE_TTL_ENVIRONMENT_KEY = "CACHE_TTL_SECONDS"
CACHE_INDEFINITELY_KEY = "CACHE_INDEFINITELY"
CACHE_PREFIX_ENVIRONMENT_KEY = "CACHE_PREFIX"
METRIC_PRUNING_ENABLED_ENVIRONMENT_KEY = "METRIC_PRUNING_ENABLED"
DEFAULT_METRIC_PRUNING_ENABLED = true
METRIC_PRUNING_ROUTINE_INTERVAL_SECONDS_ENVIRONMENT_KEY = "METRIC_PRUNING_ROUTINE_INTERVAL_SECONDS"
// 60 seconds * 60 minutes * 24 hours = 1 day
DEFAULT_METRIC_PRUNING_ROUTINE_INTERVAL_SECONDS = 86400
METRIC_PRUNING_ROUTINE_DELAY_FIRST_RUN_SECONDS_ENVIRONMENT_KEY = "METRIC_PRUNING_ROUTINE_DELAY_FIRST_RUN_SECONDS"
DEFAULT_METRIC_PRUNING_ROUTINE_DELAY_FIRST_RUN_SECONDS = 10
METRIC_PRUNING_MAX_REQUEST_METRICS_HISTORY_DAYS_ENVIRONMENT_KEY = "METRIC_PRUNING_MAX_REQUEST_METRICS_HISTORY_DAYS"
DEFAULT_METRIC_PRUNING_MAX_REQUEST_METRICS_HISTORY_DAYS = 1
EVM_QUERY_SERVICE_ENVIRONMENT_KEY = "EVM_QUERY_SERVICE_URL"
DATABASE_MAX_IDLE_CONNECTIONS_ENVIRONMENT_KEY = "DATABASE_MAX_IDLE_CONNECTIONS"
DEFAULT_DATABASE_MAX_IDLE_CONNECTIONS = 20
DATABASE_CONNECTION_MAX_IDLE_SECONDS_ENVIRONMENT_KEY = "DATABASE_CONNECTION_MAX_IDLE_SECONDS"
DEFAULT_DATABASE_CONNECTION_MAX_IDLE_SECONDS = 5
DATABASE_MAX_OPEN_CONNECTIONS_ENVIRONMENT_KEY = "DATABASE_MAX_OPEN_CONNECTIONS"
DEFAULT_DATABASE_MAX_OPEN_CONNECTIONS = 100
DATABASE_READ_TIMEOUT_SECONDS_ENVIRONMENT_KEY = "DATABASE_READ_TIMEOUT_SECONDS"
DEFAULT_DATABASE_READ_TIMEOUT_SECONDS = 60
DATABASE_WRITE_TIMEOUT_SECONDS_ENVIRONMENT_KEY = "DATABASE_WRITE_TIMEOUT_SECONDS"
DEFAULT_DATABASE_WRITE_TIMEOUT_SECONDS = 10
CACHE_ENABLED_ENVIRONMENT_KEY = "CACHE_ENABLED"
REDIS_ENDPOINT_URL_ENVIRONMENT_KEY = "REDIS_ENDPOINT_URL"
REDIS_PASSWORD_ENVIRONMENT_KEY = "REDIS_PASSWORD"
CACHE_TTL_ENVIRONMENT_KEY = "CACHE_TTL_SECONDS"
CACHE_PREFIX_ENVIRONMENT_KEY = "CACHE_PREFIX"
)

var ErrEmptyHostMap = errors.New("backend host url map is empty")
Expand Down Expand Up @@ -202,38 +213,41 @@ func ReadConfig() Config {
parsedProxyPruningBackendHostURLMap, _ := ParseRawProxyBackendHostURLMap(rawProxyPruningBackendHostURLMap)

return Config{
ProxyServicePort: os.Getenv(PROXY_SERVICE_PORT_ENVIRONMENT_KEY),
LogLevel: EnvOrDefault(LOG_LEVEL_ENVIRONMENT_KEY, DEFAULT_LOG_LEVEL),
ProxyBackendHostURLMapRaw: rawProxyBackendHostURLMap,
ProxyBackendHostURLMapParsed: parsedProxyBackendHostURLMap,
EnableHeightBasedRouting: EnvOrDefaultBool(PROXY_HEIGHT_BASED_ROUTING_ENABLED_KEY, false),
ProxyPruningBackendHostURLMapRaw: rawProxyPruningBackendHostURLMap,
ProxyPruningBackendHostURLMap: parsedProxyPruningBackendHostURLMap,
DatabaseName: os.Getenv(DATABASE_NAME_ENVIRONMENT_KEY),
DatabaseEndpointURL: os.Getenv(DATABASE_ENDPOINT_URL_ENVIRONMENT_KEY),
DatabaseUserName: os.Getenv(DATABASE_USERNAME_ENVIRONMENT_KEY),
DatabasePassword: os.Getenv(DATABASE_PASSWORD_ENVIRONMENT_KEY),
DatabaseSSLEnabled: EnvOrDefaultBool(DATABASE_SSL_ENABLED_ENVIRONMENT_KEY, false),
DatabaseReadTimeoutSeconds: EnvOrDefaultInt64(DATABASE_READ_TIMEOUT_SECONDS_ENVIRONMENT_KEY, DEFAULT_DATABASE_READ_TIMEOUT_SECONDS),
DatabaseWriteTimeoutSeconds: EnvOrDefaultInt64(DATABASE_WRITE_TIMEOUT_SECONDS_ENVIRONMENT_KEY, DEFAULT_DATABASE_WRITE_TIMEOUT_SECONDS),
DatabaseQueryLoggingEnabled: EnvOrDefaultBool(DATABASE_QUERY_LOGGING_ENABLED_ENVIRONMENT_KEY, true),
RunDatabaseMigrations: EnvOrDefaultBool(RUN_DATABASE_MIGRATIONS_ENVIRONMENT_KEY, false),
DatabaseMaxIdleConnections: EnvOrDefaultInt64(DATABASE_MAX_IDLE_CONNECTIONS_ENVIRONMENT_KEY, DEFAULT_DATABASE_MAX_IDLE_CONNECTIONS),
DatabaseConnectionMaxIdleSeconds: EnvOrDefaultInt64(DATABASE_CONNECTION_MAX_IDLE_SECONDS_ENVIRONMENT_KEY, DEFAULT_DATABASE_CONNECTION_MAX_IDLE_SECONDS),
DatabaseMaxOpenConnections: EnvOrDefaultInt64(DATABASE_MAX_OPEN_CONNECTIONS_ENVIRONMENT_KEY, DEFAULT_DATABASE_MAX_OPEN_CONNECTIONS),
HTTPReadTimeoutSeconds: EnvOrDefaultInt64(HTTP_READ_TIMEOUT_ENVIRONMENT_KEY, DEFAULT_HTTP_READ_TIMEOUT),
HTTPWriteTimeoutSeconds: EnvOrDefaultInt64(HTTP_WRITE_TIMEOUT_ENVIRONMENT_KEY, DEFAULT_HTTP_WRITE_TIMEOUT),
MetricCompactionRoutineInterval: time.Duration(time.Duration(EnvOrDefaultInt(METRIC_COMPACTION_ROUTINE_INTERVAL_ENVIRONMENT_KEY, DEFAULT_METRIC_COMPACTION_ROUTINE_INTERVAL_SECONDS)) * time.Second),
EvmQueryServiceURL: os.Getenv(EVM_QUERY_SERVICE_ENVIRONMENT_KEY),
MetricCollectionEnabled: EnvOrDefaultBool(METRIC_COLLECTION_ENABLED_ENVIRONMENT_KEY, DEFAULT_METRIC_COLLECTION_ENABLED),
MetricPartitioningRoutineInterval: time.Duration(time.Duration(EnvOrDefaultInt(METRIC_PARTITIONING_ROUTINE_INTERVAL_SECONDS_ENVIRONMENT_KEY, DEFAULT_METRIC_PARTITIONING_ROUTINE_INTERVAL_SECONDS)) * time.Second),
MetricPartitioningRoutineDelayFirstRun: time.Duration(time.Duration(EnvOrDefaultInt(METRIC_PARTITIONING_ROUTINE_DELAY_FIRST_RUN_SECONDS_ENVIRONMENT_KEY, DEFAULT_METRIC_PARTITIONING_ROUTINE_DELAY_FIRST_RUN_SECONDS)) * time.Second),
MetricPartitioningPrefillPeriodDays: EnvOrDefaultInt(METRIC_PARTITIONING_PREFILL_PERIOD_DAYS_ENVIRONMENT_KEY, DEFAULT_METRIC_PARTITIONING_PREFILL_PERIOD_DAYS),
CacheEnabled: EnvOrDefaultBool(CACHE_ENABLED_ENVIRONMENT_KEY, false),
RedisEndpointURL: os.Getenv(REDIS_ENDPOINT_URL_ENVIRONMENT_KEY),
RedisPassword: os.Getenv(REDIS_PASSWORD_ENVIRONMENT_KEY),
CacheTTL: time.Duration(EnvOrDefaultInt(CACHE_TTL_ENVIRONMENT_KEY, 0)) * time.Second,
CacheIndefinitely: EnvOrDefaultBool(CACHE_INDEFINITELY_KEY, false),
CachePrefix: os.Getenv(CACHE_PREFIX_ENVIRONMENT_KEY),
ProxyServicePort: os.Getenv(PROXY_SERVICE_PORT_ENVIRONMENT_KEY),
LogLevel: EnvOrDefault(LOG_LEVEL_ENVIRONMENT_KEY, DEFAULT_LOG_LEVEL),
ProxyBackendHostURLMapRaw: rawProxyBackendHostURLMap,
ProxyBackendHostURLMapParsed: parsedProxyBackendHostURLMap,
EnableHeightBasedRouting: EnvOrDefaultBool(PROXY_HEIGHT_BASED_ROUTING_ENABLED_KEY, false),
ProxyPruningBackendHostURLMapRaw: rawProxyPruningBackendHostURLMap,
ProxyPruningBackendHostURLMap: parsedProxyPruningBackendHostURLMap,
DatabaseName: os.Getenv(DATABASE_NAME_ENVIRONMENT_KEY),
DatabaseEndpointURL: os.Getenv(DATABASE_ENDPOINT_URL_ENVIRONMENT_KEY),
DatabaseUserName: os.Getenv(DATABASE_USERNAME_ENVIRONMENT_KEY),
DatabasePassword: os.Getenv(DATABASE_PASSWORD_ENVIRONMENT_KEY),
DatabaseSSLEnabled: EnvOrDefaultBool(DATABASE_SSL_ENABLED_ENVIRONMENT_KEY, false),
DatabaseReadTimeoutSeconds: EnvOrDefaultInt64(DATABASE_READ_TIMEOUT_SECONDS_ENVIRONMENT_KEY, DEFAULT_DATABASE_READ_TIMEOUT_SECONDS),
DatabaseWriteTimeoutSeconds: EnvOrDefaultInt64(DATABASE_WRITE_TIMEOUT_SECONDS_ENVIRONMENT_KEY, DEFAULT_DATABASE_WRITE_TIMEOUT_SECONDS),
DatabaseQueryLoggingEnabled: EnvOrDefaultBool(DATABASE_QUERY_LOGGING_ENABLED_ENVIRONMENT_KEY, true),
RunDatabaseMigrations: EnvOrDefaultBool(RUN_DATABASE_MIGRATIONS_ENVIRONMENT_KEY, false),
DatabaseMaxIdleConnections: EnvOrDefaultInt64(DATABASE_MAX_IDLE_CONNECTIONS_ENVIRONMENT_KEY, DEFAULT_DATABASE_MAX_IDLE_CONNECTIONS),
DatabaseConnectionMaxIdleSeconds: EnvOrDefaultInt64(DATABASE_CONNECTION_MAX_IDLE_SECONDS_ENVIRONMENT_KEY, DEFAULT_DATABASE_CONNECTION_MAX_IDLE_SECONDS),
DatabaseMaxOpenConnections: EnvOrDefaultInt64(DATABASE_MAX_OPEN_CONNECTIONS_ENVIRONMENT_KEY, DEFAULT_DATABASE_MAX_OPEN_CONNECTIONS),
HTTPReadTimeoutSeconds: EnvOrDefaultInt64(HTTP_READ_TIMEOUT_ENVIRONMENT_KEY, DEFAULT_HTTP_READ_TIMEOUT),
HTTPWriteTimeoutSeconds: EnvOrDefaultInt64(HTTP_WRITE_TIMEOUT_ENVIRONMENT_KEY, DEFAULT_HTTP_WRITE_TIMEOUT),
MetricCompactionRoutineInterval: time.Duration(time.Duration(EnvOrDefaultInt(METRIC_COMPACTION_ROUTINE_INTERVAL_ENVIRONMENT_KEY, DEFAULT_METRIC_COMPACTION_ROUTINE_INTERVAL_SECONDS)) * time.Second),
EvmQueryServiceURL: os.Getenv(EVM_QUERY_SERVICE_ENVIRONMENT_KEY),
MetricCollectionEnabled: EnvOrDefaultBool(METRIC_COLLECTION_ENABLED_ENVIRONMENT_KEY, DEFAULT_METRIC_COLLECTION_ENABLED),
MetricPartitioningRoutineInterval: time.Duration(time.Duration(EnvOrDefaultInt(METRIC_PARTITIONING_ROUTINE_INTERVAL_SECONDS_ENVIRONMENT_KEY, DEFAULT_METRIC_PARTITIONING_ROUTINE_INTERVAL_SECONDS)) * time.Second),
MetricPartitioningRoutineDelayFirstRun: time.Duration(time.Duration(EnvOrDefaultInt(METRIC_PARTITIONING_ROUTINE_DELAY_FIRST_RUN_SECONDS_ENVIRONMENT_KEY, DEFAULT_METRIC_PARTITIONING_ROUTINE_DELAY_FIRST_RUN_SECONDS)) * time.Second),
MetricPartitioningPrefillPeriodDays: EnvOrDefaultInt(METRIC_PARTITIONING_PREFILL_PERIOD_DAYS_ENVIRONMENT_KEY, DEFAULT_METRIC_PARTITIONING_PREFILL_PERIOD_DAYS),
MetricPruningEnabled: EnvOrDefaultBool(METRIC_PRUNING_ENABLED_ENVIRONMENT_KEY, DEFAULT_METRIC_PRUNING_ENABLED),
MetricPruningRoutineInterval: time.Duration(time.Duration(EnvOrDefaultInt(METRIC_PRUNING_ROUTINE_INTERVAL_SECONDS_ENVIRONMENT_KEY, DEFAULT_METRIC_PRUNING_ROUTINE_INTERVAL_SECONDS)) * time.Second),
MetricPruningRoutineDelayFirstRun: time.Duration(time.Duration(EnvOrDefaultInt(METRIC_PRUNING_ROUTINE_DELAY_FIRST_RUN_SECONDS_ENVIRONMENT_KEY, DEFAULT_METRIC_PRUNING_ROUTINE_DELAY_FIRST_RUN_SECONDS)) * time.Second),
MetricPruningMaxRequestMetricsHistoryDays: EnvOrDefaultInt(METRIC_PRUNING_MAX_REQUEST_METRICS_HISTORY_DAYS_ENVIRONMENT_KEY, DEFAULT_METRIC_PRUNING_MAX_REQUEST_METRICS_HISTORY_DAYS),
CacheEnabled: EnvOrDefaultBool(CACHE_ENABLED_ENVIRONMENT_KEY, false),
RedisEndpointURL: os.Getenv(REDIS_ENDPOINT_URL_ENVIRONMENT_KEY),
RedisPassword: os.Getenv(REDIS_PASSWORD_ENVIRONMENT_KEY),
CacheTTL: time.Duration(EnvOrDefaultInt(CACHE_TTL_ENVIRONMENT_KEY, 0)) * time.Second,
CachePrefix: os.Getenv(CACHE_PREFIX_ENVIRONMENT_KEY),
}
}
38 changes: 38 additions & 0 deletions main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,35 @@ func startMetricCompactionRoutine(serviceConfig config.Config, service service.P
return errChan
}

func startMetricPruningRoutine(serviceConfig config.Config, service service.ProxyService, serviceLogger logging.ServiceLogger) <-chan error {
metricPruningRoutineConfig := routines.MetricPruningRoutineConfig{
Interval: serviceConfig.MetricPruningRoutineInterval,
StartDelay: serviceConfig.MetricPartitioningRoutineDelayFirstRun,
Database: service.Database,
Logger: serviceLogger,
}

metricPruningRoutine, err := routines.NewMetricPruningRoutine(metricPruningRoutineConfig)

if err != nil {
serviceLogger.Error().Msg(fmt.Sprintf("error %s creating metric pruning routine with config %+v", err, metricPruningRoutineConfig))

return nil
}

errChan, err := metricPruningRoutine.Run()

if err != nil {
serviceLogger.Error().Msg(fmt.Sprintf("error %s starting metric pruning routine with config %+v", err, metricPruningRoutineConfig))

return nil
}

serviceLogger.Debug().Msg(fmt.Sprintf("started metric pruning routine with config %+v", metricPruningRoutineConfig))

return errChan
}

func main() {
serviceLogger.Debug().Msg(fmt.Sprintf("initial config: %+v", serviceConfig))

Expand Down Expand Up @@ -123,6 +152,15 @@ func main() {
}
}()

// metric pruning routine
go func() {
metricPruningErrs := startMetricPruningRoutine(serviceConfig, service, serviceLogger)

for routineErr := range metricPruningErrs {
serviceLogger.Error().Msg(fmt.Sprintf("metric pruning routine encountered error %s", routineErr))
}
}()

// run the proxy service
finalErr := service.Run()

Expand Down
Loading

0 comments on commit cbb9e50

Please sign in to comment.