diff --git a/README.md b/README.md index 4fc9c3b8..14c13921 100644 --- a/README.md +++ b/README.md @@ -79,3 +79,4 @@ Usage: server - open-source for use by all under the [GPLv3](LICENSE) license - also available under a flexible commercial license from [Interline](mailto:info@interline.io) + diff --git a/go.sum b/go.sum index 5dcf02be..04d4b60b 100644 --- a/go.sum +++ b/go.sum @@ -264,14 +264,6 @@ github.com/iancoleman/orderedmap v0.2.0 h1:sq1N/TFpYH++aViPcaKjys3bDClUEU7s5B+z6 github.com/iancoleman/orderedmap v0.2.0/go.mod h1:N0Wam8K1arqPXNWjMo21EXnBPOPp36vB07FNRdD2geA= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/interline-io/transitland-lib v0.14.0-rc1.0.20231122224424-d95957e76b4e h1:y24HWm0yVrlwE9a8oM+vKLDDW5F/zco/lFB+qX9Fx3I= -github.com/interline-io/transitland-lib v0.14.0-rc1.0.20231122224424-d95957e76b4e/go.mod h1:UcfuCX6DyKt/yn5GECFn3jQ6NcZEjt5XyPjf8a3tXZ4= -github.com/interline-io/transitland-lib v0.14.0-rc1.0.20231130233906-2adb3bfd1d44 h1:kRFPzcrs2SihxsTu+hOVTCRIICw6p+A2HlR9LzvXX6w= -github.com/interline-io/transitland-lib v0.14.0-rc1.0.20231130233906-2adb3bfd1d44/go.mod h1:UcfuCX6DyKt/yn5GECFn3jQ6NcZEjt5XyPjf8a3tXZ4= -github.com/interline-io/transitland-lib v0.14.0-rc1.0.20231201022717-a7acfe4abe4b h1:2O4SNVGuxBCQDwxC1oyiJSTWmfLytQJwO9ZokE1n5Yg= -github.com/interline-io/transitland-lib v0.14.0-rc1.0.20231201022717-a7acfe4abe4b/go.mod h1:UcfuCX6DyKt/yn5GECFn3jQ6NcZEjt5XyPjf8a3tXZ4= -github.com/interline-io/transitland-lib v0.14.0-rc1.0.20231201023749-4dbd4e03793d h1:0Tfw/JoAkRvDQvuJUhhJALVTl16zTvMd3dGUPecPJUw= -github.com/interline-io/transitland-lib v0.14.0-rc1.0.20231201023749-4dbd4e03793d/go.mod h1:UcfuCX6DyKt/yn5GECFn3jQ6NcZEjt5XyPjf8a3tXZ4= github.com/interline-io/transitland-lib v0.14.0-rc1.0.20231202005632-a9ea742322f7 h1:rwkKzYzl05Q4TM++L9RIJbXPjIvURoMN5vEr8dvUV/Q= github.com/interline-io/transitland-lib v0.14.0-rc1.0.20231202005632-a9ea742322f7/go.mod h1:UcfuCX6DyKt/yn5GECFn3jQ6NcZEjt5XyPjf8a3tXZ4= github.com/interline-io/transitland-lib v0.14.0-rc1.0.20231202021452-9af1eb605dd8 h1:2TYv/zFgZgjUU5uoNlu7chu+laGVwadHG3xA7/WnOcE= diff --git a/internal/meters/amberflo.go b/meters/amberflo.go similarity index 86% rename from internal/meters/amberflo.go rename to meters/amberflo.go index c5848574..cdbf14ea 100644 --- a/internal/meters/amberflo.go +++ b/meters/amberflo.go @@ -12,7 +12,7 @@ import ( "github.com/xtgo/uuid" ) -type Amberflo struct { +type AmberfloMeterProvider struct { apikey string interval time.Duration client *metering.Metering @@ -20,7 +20,7 @@ type Amberflo struct { cfgs map[string]amberFloConfig } -func NewAmberflo(apikey string, interval time.Duration, batchSize int) *Amberflo { +func NewAmberfloMeterProvider(apikey string, interval time.Duration, batchSize int) *AmberfloMeterProvider { afLog := &amberfloLogger{logger: log.Logger} meteringClient := metering.NewMeteringClient( apikey, @@ -32,7 +32,7 @@ func NewAmberflo(apikey string, interval time.Duration, batchSize int) *Amberflo apikey, metering.WithCustomLogger(afLog), ) - return &Amberflo{ + return &AmberfloMeterProvider{ apikey: apikey, interval: interval, client: meteringClient, @@ -48,7 +48,7 @@ type amberFloConfig struct { Dimensions Dimensions `json:"dimensions,omitempty"` } -func (m *Amberflo) LoadConfig(path string) error { +func (m *AmberfloMeterProvider) LoadConfig(path string) error { cfgs := map[string]amberFloConfig{} data, err := ioutil.ReadFile(path) if err != nil { @@ -61,24 +61,24 @@ func (m *Amberflo) LoadConfig(path string) error { return nil } -func (m *Amberflo) NewMeter(user MeterUser) ApiMeter { +func (m *AmberfloMeterProvider) NewMeter(user MeterUser) ApiMeter { return &amberFloMeter{ user: user, mp: m, } } -func (m *Amberflo) Close() error { +func (m *AmberfloMeterProvider) Close() error { return m.client.Shutdown() } -func (m *Amberflo) Flush() error { +func (m *AmberfloMeterProvider) Flush() error { // metering.Flush() // in API docs but not in library time.Sleep(m.interval) return nil } -func (m *Amberflo) getValue(user MeterUser, meterName string, startTime time.Time, endTime time.Time, checkDims Dimensions) (float64, bool) { +func (m *AmberfloMeterProvider) getValue(user MeterUser, meterName string, startTime time.Time, endTime time.Time, checkDims Dimensions) (float64, bool) { cfg, ok := m.getcfg(meterName) if !ok { return 0, false @@ -138,7 +138,7 @@ func (m *Amberflo) getValue(user MeterUser, meterName string, startTime time.Tim return total, true } -func (m *Amberflo) sendMeter(user MeterUser, meterName string, value float64, extraDimensions Dimensions) error { +func (m *AmberfloMeterProvider) sendMeter(user MeterUser, meterName string, value float64, extraDimensions Dimensions) error { cfg, ok := m.getcfg(meterName) if !ok { return nil @@ -167,7 +167,7 @@ func (m *Amberflo) sendMeter(user MeterUser, meterName string, value float64, ex }) } -func (m *Amberflo) getCustomerID(cfg amberFloConfig, user MeterUser) (string, bool) { +func (m *AmberfloMeterProvider) getCustomerID(cfg amberFloConfig, user MeterUser) (string, bool) { customerId := cfg.DefaultUser if user != nil { eidKey := cfg.ExternalIDKey @@ -184,7 +184,7 @@ func (m *Amberflo) getCustomerID(cfg amberFloConfig, user MeterUser) (string, bo return customerId, customerId != "" } -func (m *Amberflo) getcfg(meterName string) (amberFloConfig, bool) { +func (m *AmberfloMeterProvider) getcfg(meterName string) (amberFloConfig, bool) { cfg, ok := m.cfgs[meterName] if !ok { cfg = amberFloConfig{ @@ -203,7 +203,7 @@ func (m *Amberflo) getcfg(meterName string) (amberFloConfig, bool) { type amberFloMeter struct { user MeterUser addDims []eventAddDim - mp *Amberflo + mp *AmberfloMeterProvider } func (m *amberFloMeter) Meter(meterName string, value float64, extraDimensions Dimensions) error { diff --git a/internal/meters/amberflo_test.go b/meters/amberflo_test.go similarity index 89% rename from internal/meters/amberflo_test.go rename to meters/amberflo_test.go index 6f69737d..1e7b4cda 100644 --- a/internal/meters/amberflo_test.go +++ b/meters/amberflo_test.go @@ -18,7 +18,7 @@ func TestAmberfloMeter(t *testing.T) { testMeter(t, mp, testConfig) } -func getTestAmberfloMeter() (*Amberflo, testMeterConfig, error) { +func getTestAmberfloMeter() (*AmberfloMeterProvider, testMeterConfig, error) { checkKeys := []string{ "TL_TEST_AMBERFLO_APIKEY", "TL_TEST_AMBERFLO_METER1", @@ -50,7 +50,7 @@ func getTestAmberfloMeter() (*Amberflo, testMeterConfig, error) { data: map[string]string{eidKey: os.Getenv("TL_TEST_AMBERFLO_USER3")}, }, } - mp := NewAmberflo(os.Getenv("TL_TEST_AMBERFLO_APIKEY"), 1*time.Second, 1) + mp := NewAmberfloMeterProvider(os.Getenv("TL_TEST_AMBERFLO_APIKEY"), 1*time.Second, 1) mp.cfgs[testConfig.testMeter1] = amberFloConfig{Name: testConfig.testMeter1, ExternalIDKey: eidKey} mp.cfgs[testConfig.testMeter2] = amberFloConfig{Name: testConfig.testMeter2, ExternalIDKey: eidKey} return mp, testConfig, nil diff --git a/internal/meters/default.go b/meters/default.go similarity index 100% rename from internal/meters/default.go rename to meters/default.go diff --git a/internal/meters/default_test.go b/meters/default_test.go similarity index 100% rename from internal/meters/default_test.go rename to meters/default_test.go diff --git a/internal/meters/limit.go b/meters/limit.go similarity index 100% rename from internal/meters/limit.go rename to meters/limit.go diff --git a/internal/meters/limit_test.go b/meters/limit_test.go similarity index 100% rename from internal/meters/limit_test.go rename to meters/limit_test.go diff --git a/internal/meters/meters.go b/meters/meters.go similarity index 61% rename from internal/meters/meters.go rename to meters/meters.go index 5b75589d..ec6d58b7 100644 --- a/internal/meters/meters.go +++ b/meters/meters.go @@ -3,6 +3,7 @@ package meters import ( "context" "net/http" + "os" "time" "github.com/interline-io/transitland-server/auth/authn" @@ -34,8 +35,11 @@ func WithMeter(apiMeter MeterProvider, meterName string, meterValue float64, dim ctx := r.Context() ctxMeter := apiMeter.NewMeter(authn.ForContext(ctx)) r = r.WithContext(context.WithValue(ctx, meterCtxKey, ctxMeter)) + if err := ctxMeter.Meter(meterName, meterValue, dims); err != nil { + http.Error(w, "429", http.StatusTooManyRequests) + return + } next.ServeHTTP(w, r) - ctxMeter.Meter(meterName, meterValue, dims) }) } } @@ -72,3 +76,33 @@ func dimsContainedIn(checkDims Dimensions, eventDims Dimensions) bool { } return true } + +////// + +type Config struct { + EnableMetering bool + EnableRateLimits bool + MeteringProvider string + MeteringAmberfloConfig string +} + +func GetProvider(cfg Config) (MeterProvider, error) { + var meterProvider MeterProvider + meterProvider = NewDefaultMeterProvider() + if cfg.MeteringProvider == "amberflo" { + a := NewAmberfloMeterProvider(os.Getenv("AMBERFLO_APIKEY"), 30*time.Second, 100) + if cfg.MeteringAmberfloConfig != "" { + if err := a.LoadConfig(cfg.MeteringAmberfloConfig); err != nil { + return nil, err + } + } + meterProvider = a + } + if cfg.EnableRateLimits { + mp := NewLimitMeterProvider(meterProvider) + mp.Enabled = true + // mp.DefaultLimits = append(mp.DefaultLimits, meters.UserMeterLimit{Limit: 10, Period: "monthly", MeterName: "rest"}) + meterProvider = mp + } + return meterProvider, nil +} diff --git a/internal/meters/meters_test.go b/meters/meters_test.go similarity index 100% rename from internal/meters/meters_test.go rename to meters/meters_test.go diff --git a/internal/metrics/default.go b/metrics/default.go similarity index 100% rename from internal/metrics/default.go rename to metrics/default.go diff --git a/internal/metrics/http.go b/metrics/http.go similarity index 100% rename from internal/metrics/http.go rename to metrics/http.go diff --git a/internal/metrics/jobs.go b/metrics/jobs.go similarity index 100% rename from internal/metrics/jobs.go rename to metrics/jobs.go diff --git a/internal/metrics/metrics.go b/metrics/metrics.go similarity index 57% rename from internal/metrics/metrics.go rename to metrics/metrics.go index c44e9df7..a8da8433 100644 --- a/internal/metrics/metrics.go +++ b/metrics/metrics.go @@ -16,3 +16,17 @@ type MetricProvider interface { NewJobMetric(queue string) JobMetric MetricsHandler() http.Handler } + +type Config struct { + EnableMetrics bool + MetricsProvider string +} + +func GetProvider(cfg Config) (MetricProvider, error) { + var metricProvider MetricProvider + metricProvider = NewDefaultMetric() + if cfg.MetricsProvider == "prometheus" { + metricProvider = NewPromMetrics() + } + return metricProvider, nil +} diff --git a/internal/metrics/prom.go b/metrics/prom.go similarity index 100% rename from internal/metrics/prom.go rename to metrics/prom.go diff --git a/server/gql/query_resolver.go b/server/gql/query_resolver.go index aed09109..1247cd26 100644 --- a/server/gql/query_resolver.go +++ b/server/gql/query_resolver.go @@ -5,7 +5,7 @@ import ( "errors" "github.com/interline-io/transitland-server/auth/authz" - "github.com/interline-io/transitland-server/internal/meters" + "github.com/interline-io/transitland-server/meters" "github.com/interline-io/transitland-server/model" ) diff --git a/server/rest/feed_version_download.go b/server/rest/feed_version_download.go index bf5fdc91..fedf8725 100644 --- a/server/rest/feed_version_download.go +++ b/server/rest/feed_version_download.go @@ -11,8 +11,8 @@ import ( "github.com/interline-io/transitland-lib/dmfr/store" "github.com/interline-io/transitland-lib/tl" "github.com/interline-io/transitland-lib/tl/request" - "github.com/interline-io/transitland-server/internal/meters" "github.com/interline-io/transitland-server/internal/util" + "github.com/interline-io/transitland-server/meters" "github.com/tidwall/gjson" ) diff --git a/server/rest/rest.go b/server/rest/rest.go index 8cbee863..d13f3e10 100644 --- a/server/rest/rest.go +++ b/server/rest/rest.go @@ -17,8 +17,8 @@ import ( "github.com/interline-io/transitland-lib/log" "github.com/interline-io/transitland-server/auth/ancheck" "github.com/interline-io/transitland-server/config" - "github.com/interline-io/transitland-server/internal/meters" "github.com/interline-io/transitland-server/internal/util" + "github.com/interline-io/transitland-server/meters" "github.com/interline-io/transitland-server/model" ) diff --git a/server/server.go b/server/server.go index 87b560d9..58c8813a 100644 --- a/server/server.go +++ b/server/server.go @@ -17,7 +17,7 @@ import ( ) // log request and duration -func loggingMiddleware(longQueryDuration int) func(http.Handler) http.Handler { +func LoggingMiddleware(longQueryDuration int) func(http.Handler) http.Handler { return func(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { ctx := r.Context() diff --git a/server/server_cmd.go b/server/server_cmd.go index fa80054c..8d46768c 100644 --- a/server/server_cmd.go +++ b/server/server_cmd.go @@ -29,9 +29,9 @@ import ( "github.com/interline-io/transitland-server/finders/rtfinder" "github.com/interline-io/transitland-server/internal/dbutil" "github.com/interline-io/transitland-server/internal/jobs" - "github.com/interline-io/transitland-server/internal/meters" - "github.com/interline-io/transitland-server/internal/metrics" "github.com/interline-io/transitland-server/internal/playground" + "github.com/interline-io/transitland-server/meters" + "github.com/interline-io/transitland-server/metrics" "github.com/interline-io/transitland-server/model" "github.com/interline-io/transitland-server/server/gql" "github.com/interline-io/transitland-server/server/rest" @@ -56,41 +56,42 @@ type Command struct { QueuePrefix string SecretsFile string AuthMiddlewares arrayFlags - metersConfig metersConfig - metricsConfig metricsConfig + metersConfig meters.Config + metricsConfig metrics.Config AuthConfig ancheck.AuthConfig CheckerConfig azcheck.CheckerConfig config.Config } -type metricsConfig struct { - EnableMetrics bool - MetricsProvider string -} - -type metersConfig struct { - EnableMetering bool - MeteringProvider string - MeteringAmberfloConfig string -} - func (cmd *Command) Parse(args []string) error { fl := flag.NewFlagSet("sync", flag.ExitOnError) fl.Usage = func() { log.Print("Usage: server") fl.PrintDefaults() } + + // Base config fl.StringVar(&cmd.DBURL, "dburl", "", "Database URL (default: $TL_DATABASE_URL)") fl.StringVar(&cmd.RedisURL, "redisurl", "", "Redis URL (default: $TL_REDIS_URL)") - fl.IntVar(&cmd.Timeout, "timeout", 60, "") - fl.IntVar(&cmd.LongQueryDuration, "long-query", 1000, "Log queries over this duration (ms)") - fl.StringVar(&cmd.Port, "port", "8080", "") fl.StringVar(&cmd.Storage, "storage", "", "Static storage backend") fl.StringVar(&cmd.RTStorage, "rt-storage", "", "RT storage backend") - fl.StringVar(&cmd.SecretsFile, "secrets", "", "DMFR file containing secrets") fl.StringVar(&cmd.RestPrefix, "rest-prefix", "", "REST prefix for generating pagination links") + fl.BoolVar(&cmd.ValidateLargeFiles, "validate-large-files", false, "Allow validation of large files") + fl.BoolVar(&cmd.DisableImage, "disable-image", false, "Disable image generation") + + // Server config + fl.StringVar(&cmd.Port, "port", "8080", "") + fl.StringVar(&cmd.SecretsFile, "secrets", "", "DMFR file containing secrets") fl.StringVar(&cmd.QueuePrefix, "queue", "", "Job name prefix") + fl.IntVar(&cmd.Timeout, "timeout", 60, "") + fl.IntVar(&cmd.LongQueryDuration, "long-query", 1000, "Log queries over this duration (ms)") + fl.BoolVar(&cmd.DisableGraphql, "disable-graphql", false, "Disable GraphQL endpoint") + fl.BoolVar(&cmd.DisableRest, "disable-rest", false, "Disable REST endpoint") + fl.BoolVar(&cmd.EnablePlayground, "enable-playground", false, "Enable GraphQL playground") + fl.BoolVar(&cmd.EnableProfiler, "enable-profile", false, "Enable profiling") + fl.BoolVar(&cmd.LoadAdmins, "load-admins", false, "Load admin polygons from database into memory") + // Auth config fl.Var(&cmd.AuthMiddlewares, "auth", "Add one or more auth middlewares") fl.StringVar(&cmd.AuthConfig.DefaultUsername, "default-username", "", "Default user name (for --auth=admin)") fl.StringVar(&cmd.AuthConfig.JwtAudience, "jwt-audience", "", "JWT Audience (use with -auth=jwt)") @@ -104,14 +105,6 @@ func (cmd *Command) Parse(args []string) error { fl.BoolVar(&cmd.AuthConfig.GatekeeperAllowError, "gatekeeper-allow-error", false, "Gatekeeper ignore errors (use with -auth=gatekeeper)") fl.StringVar(&cmd.AuthConfig.UserHeader, "user-header", "", "Header to check for username (use with -auth=header)") - fl.BoolVar(&cmd.ValidateLargeFiles, "validate-large-files", false, "Allow validation of large files") - fl.BoolVar(&cmd.DisableImage, "disable-image", false, "Disable image generation") - fl.BoolVar(&cmd.DisableGraphql, "disable-graphql", false, "Disable GraphQL endpoint") - fl.BoolVar(&cmd.DisableRest, "disable-rest", false, "Disable REST endpoint") - fl.BoolVar(&cmd.EnablePlayground, "enable-playground", false, "Enable GraphQL playground") - fl.BoolVar(&cmd.EnableProfiler, "enable-profile", false, "Enable profiling") - fl.BoolVar(&cmd.LoadAdmins, "load-admins", false, "Load admin polygons from database into memory") - // Admin api fl.StringVar(&cmd.CheckerConfig.GlobalAdmin, "global-admin", "", "Global admin user") fl.StringVar(&cmd.CheckerConfig.Auth0ClientID, "auth0-client-id", "", "Auth0 client ID") @@ -129,9 +122,9 @@ func (cmd *Command) Parse(args []string) error { // Metering // fl.BoolVar(&cmd.EnableMetering, "enable-metering", false, "Enable metering") + fl.BoolVar(&cmd.EnableRateLimits, "enable-rate-limits", false, "Enable rate limits") fl.StringVar(&cmd.metersConfig.MeteringProvider, "metering-provider", "", "Use metering provider") fl.StringVar(&cmd.metersConfig.MeteringAmberfloConfig, "metering-amberflo-config", "", "Use provided config for Amberflo metering") - fl.BoolVar(&cmd.EnableRateLimits, "enable-rate-limits", false, "Enable rate limits") // Jobs fl.BoolVar(&cmd.EnableJobsApi, "enable-jobs-api", false, "Enable job api") @@ -225,34 +218,15 @@ func (cmd *Command) Run() error { } // Setup metrics - var metricProvider metrics.MetricProvider - metricProvider = metrics.NewDefaultMetric() - if cmd.metricsConfig.EnableMetrics { - if cmd.metricsConfig.MetricsProvider == "prometheus" { - metricProvider = metrics.NewPromMetrics() - } + metricProvider, err := metrics.GetProvider(cmd.metricsConfig) + if err != nil { + return err } // Setup metering - var meterProvider meters.MeterProvider - meterProvider = meters.NewDefaultMeterProvider() - if cmd.metersConfig.EnableMetering { - if cmd.metersConfig.MeteringProvider == "amberflo" { - a := meters.NewAmberflo(os.Getenv("AMBERFLO_APIKEY"), 30*time.Second, 100) - if cmd.metersConfig.MeteringAmberfloConfig != "" { - if err := a.LoadConfig(cmd.metersConfig.MeteringAmberfloConfig); err != nil { - return err - } - } - meterProvider = a - } - if cmd.EnableRateLimits { - mp := meters.NewLimitMeterProvider(meterProvider) - mp.Enabled = true - // mp.DefaultLimits = append(mp.DefaultLimits, meters.UserMeterLimit{Limit: 10, Period: "monthly", MeterName: "rest"}) - meterProvider = mp - } - defer meterProvider.Close() + meterProvider, err := meters.GetProvider(cmd.metersConfig) + if err != nil { + return err } // Setup router @@ -261,6 +235,12 @@ func (cmd *Command) Run() error { root.Use(middleware.RealIP) root.Use(middleware.Recoverer) root.Use(middleware.StripSlashes) + root.Use(cors.Handler(cors.Options{ + AllowedOrigins: []string{"https://*", "http://*"}, + AllowedMethods: []string{"GET", "POST", "DELETE", "OPTIONS"}, + AllowedHeaders: []string{"content-type", "apikey", "authorization"}, + AllowCredentials: true, + })) // Setup user middleware for _, k := range cmd.AuthMiddlewares { @@ -272,15 +252,7 @@ func (cmd *Command) Run() error { } // Add logging middleware - must be after auth - root.Use(loggingMiddleware(cmd.LongQueryDuration)) - - // Setup CORS - root.Use(cors.Handler(cors.Options{ - AllowedOrigins: []string{"https://*", "http://*"}, - AllowedMethods: []string{"GET", "POST", "DELETE", "OPTIONS"}, - AllowedHeaders: []string{"content-type", "apikey", "authorization"}, - AllowCredentials: true, - })) + root.Use(LoggingMiddleware(cmd.LongQueryDuration)) // Profiling if cmd.EnableProfiler {