diff --git a/app/api/handlers.go b/app/api/handlers.go index 18ae76a..e5cab84 100644 --- a/app/api/handlers.go +++ b/app/api/handlers.go @@ -10,6 +10,7 @@ import ( "strings" "time" + "github.com/go-pg/pg/v9" "github.com/go-playground/validator/v10" "github.com/go-redis/redis_rate/v7" "github.com/labstack/echo/v4" @@ -46,7 +47,6 @@ func HTTPErrorHandler(err error, c echo.Context) { if errors.As(err, &e) { Render(c, e.Code, e.Data) // nolint: errcheck - return } @@ -72,10 +72,19 @@ func HTTPErrorHandler(err error, c echo.Context) { if errors.As(err, &ve) { Render(c, http.StatusBadRequest, validators.TranslateErrors(ve)) // nolint: errcheck - return } + // Process statement_timeout error. + var pgerr pg.Error + + if errors.As(err, &pgerr) { + if pgerr.Field('C') == "57014" { + Render(c, http.StatusRequestTimeout, map[string]string{"detail": "Requested query took too long. Timeout was reached."}) // nolint: errcheck + return + } + } + Render(c, http.StatusInternalServerError, map[string]string{"detail": "Internal server error."}) // nolint: errcheck } diff --git a/app/controllers/data_object_filters.go b/app/controllers/data_object_filters.go index 5763949..164abf3 100644 --- a/app/controllers/data_object_filters.go +++ b/app/controllers/data_object_filters.go @@ -24,7 +24,7 @@ import ( var filters = map[string][]*filterOp{} const ( - maxListLength = 128 + maxListLength = 16 ) func registerFilter(op *filterOp, lookups ...string) { diff --git a/cmd/app.go b/cmd/app.go index 0296414..7332729 100644 --- a/cmd/app.go +++ b/cmd/app.go @@ -2,7 +2,6 @@ package cmd import ( _ "expvar" // Register expvar default http handler. - "fmt" "math/rand" "net/http" "runtime" @@ -37,8 +36,8 @@ var ( // App is the main structure of a cli application. App = cli.NewApp() - dbOptions = database.DefaultDBOptions() - dbInstancesOptions = database.DefaultDBOptions() + dbOptions = database.DefaultOptions + dbInstancesOptions = database.DefaultOptions redisOptions = redis.Options{} amqpChannel *amqp.Channel @@ -91,11 +90,11 @@ func init() { }, &cli.StringFlag{ Name: "db-host", Usage: "database host", - EnvVars: []string{"DB_HOST", "PGHOST"}, Value: "postgresql", + EnvVars: []string{"DB_HOST", "PGHOST"}, Value: "postgresql", Destination: &dbOptions.Host, }, &cli.StringFlag{ Name: "db-port", Usage: "database port", - EnvVars: []string{"DB_PORT", "PGPORT"}, Value: "5432", + EnvVars: []string{"DB_PORT", "PGPORT"}, Value: "5432", Destination: &dbOptions.Port, }, // Database instances options. @@ -113,11 +112,15 @@ func init() { }, &cli.StringFlag{ Name: "db-instances-host", Usage: "instances database host", - EnvVars: []string{"DB_INSTANCES_HOST", "DB_HOST", "PGHOST"}, Value: "postgresql", + EnvVars: []string{"DB_INSTANCES_HOST", "DB_HOST", "PGHOST"}, Value: "postgresql", Destination: &dbInstancesOptions.Host, }, &cli.StringFlag{ Name: "db-instances-port", Usage: "instances database port", - EnvVars: []string{"DB_INSTANCES_PORT", "DB_PORT", "PGPORT"}, Value: "5432", + EnvVars: []string{"DB_INSTANCES_PORT", "DB_PORT", "PGPORT"}, Value: "5432", Destination: &dbInstancesOptions.Port, + }, + &cli.DurationFlag{ + Name: "db-instances-statement-timeout", Usage: "instances statement timeout", + EnvVars: []string{"DB_INSTANCES_STATEMENT_TIMEOUT"}, Value: 2 * time.Second, Destination: &dbInstancesOptions.StatementTimeout, }, // Tracing options. @@ -219,9 +222,7 @@ func init() { }) // Initialize database client. - dbOptions.Addr = fmt.Sprintf("%s:%s", c.String("db-host"), c.String("db-port")) - dbInstancesOptions.Addr = fmt.Sprintf("%s:%s", c.String("db-instances-host"), c.String("db-instances-port")) - db = database.NewDB(dbOptions, dbInstancesOptions, logger, c.Bool("debug")) + db = database.NewDB(&dbOptions, &dbInstancesOptions, logger, c.Bool("debug")) fs = storage.NewStorage(settings.Common.Location, settings.Buckets, settings.API.Host, settings.API.StorageURL) diff --git a/deploy/env/eu1.secrets.gpg b/deploy/env/eu1.secrets.gpg index 2d6f7d4..bc78492 100644 Binary files a/deploy/env/eu1.secrets.gpg and b/deploy/env/eu1.secrets.gpg differ diff --git a/deploy/env/staging.env b/deploy/env/staging.env index 5434c0f..e4d526c 100644 --- a/deploy/env/staging.env +++ b/deploy/env/staging.env @@ -1,5 +1,6 @@ API_HOST=api.syncano.rocks SPACE_HOST=syncano.link +GCP_LOCATIONS=stg CODEBOX_ADDR=codebox-broker.default:9000 DEBUG=1 diff --git a/deploy/env/staging.secrets.gpg b/deploy/env/staging.secrets.gpg index 50e30b5..818d33f 100644 Binary files a/deploy/env/staging.secrets.gpg and b/deploy/env/staging.secrets.gpg differ diff --git a/go.mod b/go.mod index 94a87bc..3ac5231 100644 --- a/go.mod +++ b/go.mod @@ -5,8 +5,8 @@ go 1.15 require ( contrib.go.opencensus.io/exporter/jaeger v0.2.1 contrib.go.opencensus.io/exporter/prometheus v0.2.0 - github.com/Syncano/pkg-go/v2 v2.5.1 - github.com/Syncano/syncanoapis/gen v1.1.0 + github.com/Syncano/pkg-go/v2 v2.7.4 + github.com/Syncano/syncanoapis/gen v1.2.0 github.com/alexandrevicenzi/unchained v1.3.0 github.com/blang/semver v3.5.1+incompatible github.com/caarlos0/env/v6 v6.3.0 @@ -26,7 +26,7 @@ require ( github.com/jackc/pgtype v1.4.2 github.com/jinzhu/now v1.1.1 github.com/json-iterator/go v1.1.10 - github.com/labstack/echo/v4 v4.1.16 + github.com/labstack/echo/v4 v4.1.17 github.com/labstack/gommon v0.3.0 github.com/mitchellh/hashstructure v1.0.0 github.com/mitchellh/mapstructure v1.3.3 @@ -39,7 +39,7 @@ require ( github.com/vektra/mockery v1.1.2 go.opencensus.io v0.22.4 go.uber.org/zap v1.15.0 - golang.org/x/tools v0.0.0-20200819140908-cf83efe03cf8 - google.golang.org/grpc v1.31.0 + golang.org/x/tools v0.0.0-20200827163409-021d7c6f1ec3 + google.golang.org/grpc v1.31.1 kkn.fi/base62 v0.1.0 ) diff --git a/go.sum b/go.sum index 124800f..e12a18d 100644 --- a/go.sum +++ b/go.sum @@ -63,12 +63,12 @@ github.com/Masterminds/sprig v2.22.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuN github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk= github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0= -github.com/Syncano/pkg-go/v2 v2.5.0 h1:Kaj9DQupbOilRlETDopjoQuJSu7PdKdNN+DmWGJvSBQ= -github.com/Syncano/pkg-go/v2 v2.5.0/go.mod h1:/ssrSVW3RehtRo3YgHfE4ybuCpeM/G2V9xyeH1SSVKE= -github.com/Syncano/pkg-go/v2 v2.5.1 h1:Cqb1qpUEjwBGNZakDeSX7+48/V23wJyDaCOin/rmWVc= -github.com/Syncano/pkg-go/v2 v2.5.1/go.mod h1:/ssrSVW3RehtRo3YgHfE4ybuCpeM/G2V9xyeH1SSVKE= -github.com/Syncano/syncanoapis/gen v1.1.0 h1:CIKg+EIOh/ZKp56ROtzeDkAdShuZOYODg0+emektZw8= -github.com/Syncano/syncanoapis/gen v1.1.0/go.mod h1:I1+7CW9sF/sLeKhuAN83+sPVi/iveumVkxx/+oTHsGE= +github.com/Syncano/pkg-go/v2 v2.7.3 h1:Eqa8ljfQ2UkffsgArxZVgaemQapiO3MPN7+nNL7R43M= +github.com/Syncano/pkg-go/v2 v2.7.3/go.mod h1:T3HV28yPrYtzs7yKOUOiwI2XsOBcO9e+uGa+m68lxjU= +github.com/Syncano/pkg-go/v2 v2.7.4 h1:6oXM+SzP9xyuIJ8fwZnKnRaWjp9ajF1xJzs5L+xMJMg= +github.com/Syncano/pkg-go/v2 v2.7.4/go.mod h1:T3HV28yPrYtzs7yKOUOiwI2XsOBcO9e+uGa+m68lxjU= +github.com/Syncano/syncanoapis/gen v1.2.0 h1:U+oBiE/FEOhPY/qqhfhOiMZrxQ+LM0x/CH/w4z9HzlI= +github.com/Syncano/syncanoapis/gen v1.2.0/go.mod h1:I1+7CW9sF/sLeKhuAN83+sPVi/iveumVkxx/+oTHsGE= github.com/TheZeroSlave/zapsentry v1.5.0 h1:TpZcIE2Ie9L96WU08w49TfaDKgiaq2rGAwLT8NIkPlk= github.com/TheZeroSlave/zapsentry v1.5.0/go.mod h1:DnlnJaAHnU0ocnEAa32e1SB6+ZQ7FR/Km7uHqoZMUss= github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= @@ -395,6 +395,8 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/labstack/echo/v4 v4.1.11/go.mod h1:i541M3Fj6f76NZtHSj7TXnyM8n2gaodfvfxNnFqi74g= github.com/labstack/echo/v4 v4.1.16 h1:8swiwjE5Jkai3RPfZoahp8kjVCRNq+y7Q0hPji2Kz0o= github.com/labstack/echo/v4 v4.1.16/go.mod h1:awO+5TzAjvL8XpibdsfXxPgHr+orhtXZJZIQCVjogKI= +github.com/labstack/echo/v4 v4.1.17 h1:PQIBaRplyRy3OjwILGkPg89JRtH2x5bssi59G2EL3fo= +github.com/labstack/echo/v4 v4.1.17/go.mod h1:Tn2yRQL/UclUalpb5rPdXDevbkJ+lp/2svdyFBg6CHQ= github.com/labstack/gommon v0.3.0 h1:JEeO0bvc78PKdyHxloTKiF8BD5iGrH8T6MSeGvSgob0= github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= @@ -416,6 +418,8 @@ github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcncea github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.6 h1:6Su7aK7lXmJ/U79bYtBjLNaha4Fs1Rg9plHpcH+vvnE= github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.7 h1:bQGKb3vps/j0E9GfJQ03JyhRuxsvdAanXlT9BTw3mdw= +github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= @@ -572,6 +576,8 @@ github.com/valyala/fasttemplate v1.0.1 h1:tY9CJiPnMXf1ERmG2EyK7gNUd+c6RKGD0IfU8W github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/valyala/fasttemplate v1.1.0 h1:RZqt0yGBsps8NGvLSGW804QQqCUYYLsaOjTVHy1Ocw4= github.com/valyala/fasttemplate v1.1.0/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= +github.com/valyala/fasttemplate v1.2.1 h1:TVEnxayobAdVkhQfrfes2IzOB6o+z4roRkPF52WA1u4= +github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= github.com/vektra/mockery v1.1.2 h1:uc0Yn67rJpjt8U/mAZimdCKn9AeA97BOkjpmtBSlfP4= github.com/vektra/mockery v1.1.2/go.mod h1:VcfZjKaFOPO+MpN4ZvwPjs4c48lkq1o3Ym8yHZJu0jU= @@ -640,6 +646,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnk golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de h1:ikNHVSjEfnvz6sxdSPCaPt572qowuyMDMJLLm3Db3ig= golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a h1:vclmkQCjlDX5OydZ9wv8rBCcS0QyQY66Mpf/7BZbInM= +golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -715,6 +723,8 @@ golang.org/x/net v0.0.0-20200625001655-4c5254603344 h1:vGXIOMxbNfDTk/aXCmfdLgkrS golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202 h1:VvcQYSHwXgi7W+TpUR6A9g6Up98WAHf3f/ulnJ62IyA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -783,6 +793,8 @@ golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1 h1:ogLJMz+qpzav7lGMh10LMvAkM golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642 h1:B6caxRw+hozq68X2MY7jEpZh/cr4/aHLv9xU8Kkadrw= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200826173525-f9321e4c35a6 h1:DvY3Zkh7KabQE/kfzMvYvKirSiguP9Q/veMtkYyf0o8= +golang.org/x/sys v0.0.0-20200826173525-f9321e4c35a6/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -847,6 +859,8 @@ golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d h1:szSOL78iTCl0LF1AMjhSWJj golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200819140908-cf83efe03cf8 h1:vRX0wGqBdtbS6vDv0mxYgNYw03j7YntEHBrEb/KJeeE= golang.org/x/tools v0.0.0-20200819140908-cf83efe03cf8/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200827163409-021d7c6f1ec3 h1:OjYQxZBKJFs+sJbHkvSGIKNMkZXDJQ9JsMpebGhkafI= +golang.org/x/tools v0.0.0-20200827163409-021d7c6f1ec3/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -929,6 +943,8 @@ google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3Iji google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0 h1:T7P4R73V3SSDPhH7WW7ATbfViLtmamH0DKrP3f9AuDI= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.1 h1:SfXqXS5hkufcdZ/mHtYCh53P2b+92WQq/DZcKLgsFRs= +google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=