Merge pull request #6957 from TheThingsNetwork/merge/v3.29-in-v3.30

Merge v3.29 in v3.30

CHANGELOG.md

@@ -21,6 +21,13 @@ For details about compatibility between different releases, see the **Commitment
 
 ### Security
 
+## [3.29.1] - unreleased
+
+### Added
+
+- Option to store rate limits in Redis. When used, the rate limits are applied over the entire cluster instead of per-instance.
+- Field `complement_collaborators` was added to `SearchAccounts`. Allows an user to request the accounts that are not already attached to the entity's collaborator list.
+
 ## [3.29.0] - 2024-02-06
 
 ### Added
@@ -2766,7 +2773,8 @@ For details about compatibility between different releases, see the **Commitment
 NOTE: These links should respect backports. See https://github.com/TheThingsNetwork/lorawan-stack/pull/1444/files#r333379706.
 -->
 
-[unreleased]: https://github.com/TheThingsNetwork/lorawan-stack/compare/v3.29.0...v3.29
+[unreleased]: https://github.com/TheThingsNetwork/lorawan-stack/compare/v3.29.1...v3.29
+[3.29.1]: https://github.com/TheThingsNetwork/lorawan-stack/compare/v3.29.0...v3.29.1
 [3.29.0]: https://github.com/TheThingsNetwork/lorawan-stack/compare/v3.28.2...v3.29.0
 [3.28.2]: https://github.com/TheThingsNetwork/lorawan-stack/compare/v3.28.1...v3.28.2
 [3.28.1]: https://github.com/TheThingsNetwork/lorawan-stack/compare/v3.28.0...v3.28.1

api/ttn/lorawan/v3/api.md

@@ -10157,6 +10157,7 @@ Right is the enum that defines all the different rights to do something in the n
 | `client_ids` | [`ClientIdentifiers`](#ttn.lorawan.v3.ClientIdentifiers) |  |  |
 | `gateway_ids` | [`GatewayIdentifiers`](#ttn.lorawan.v3.GatewayIdentifiers) |  |  |
 | `organization_ids` | [`OrganizationIdentifiers`](#ttn.lorawan.v3.OrganizationIdentifiers) |  |  |
+| `complement_collaborators` | [`bool`](#bool) |  |  |
 
 #### Field Rules
 

api/ttn/lorawan/v3/api.swagger.json

@@ -941,6 +941,12 @@
             "in": "query",
             "required": false,
             "type": "string"
+          },
+          {
+            "name": "complement_collaborators",
+            "in": "query",
+            "required": false,
+            "type": "boolean"
           }
         ],
         "tags": [
@@ -4242,6 +4248,12 @@
             "in": "query",
             "required": false,
             "type": "string"
+          },
+          {
+            "name": "complement_collaborators",
+            "in": "query",
+            "required": false,
+            "type": "boolean"
           }
         ],
         "tags": [
@@ -7484,6 +7496,12 @@
             "in": "query",
             "required": false,
             "type": "string"
+          },
+          {
+            "name": "complement_collaborators",
+            "in": "query",
+            "required": false,
+            "type": "boolean"
           }
         ],
         "tags": [
@@ -10582,6 +10600,12 @@
             "required": false,
             "type": "string",
             "format": "string"
+          },
+          {
+            "name": "complement_collaborators",
+            "in": "query",
+            "required": false,
+            "type": "boolean"
           }
         ],
         "tags": [
@@ -11869,6 +11893,12 @@
             "in": "query",
             "required": false,
             "type": "string"
+          },
+          {
+            "name": "complement_collaborators",
+            "in": "query",
+            "required": false,
+            "type": "boolean"
           }
         ],
         "tags": [

api/ttn/lorawan/v3/search_services.proto

@@ -341,6 +341,8 @@ message SearchAccountsRequest {
     OrganizationIdentifiers organization_ids = 6;
   }
 
+  bool complement_collaborators = 7;
+
   // NOTE: This request intentionally does not support pagination.
 }
 

cmd/internal/shared/config.go

@@ -145,6 +145,11 @@ var DefaultKeyVaultConfig = config.KeyVault{
 	Provider: "static",
 }
 
+// DefaultRateLimitingConfig is the default config for rate limiting.
+var DefaultRateLimitingConfig = config.RateLimiting{
+	Provider: "memory",
+}
+
 // DefaultTracingConfig is the default config for telemetry tracing.
 var DefaultTracingConfig = tracing.Config{
 	Enable:   false,
@@ -184,6 +189,7 @@ var DefaultServiceBase = config.ServiceBase{
 	FrequencyPlans: DefaultFrequencyPlansConfig,
 	Rights:         DefaultRightsConfig,
 	KeyVault:       DefaultKeyVaultConfig,
+	RateLimiting:   DefaultRateLimitingConfig,
 	Tracing:        DefaultTracingConfig,
 	Telemetry:      DefaultTelemetryConfig,
 }

cmd/ttn-lw-stack/commands/start.go

@@ -103,6 +103,14 @@ func NewJoinServerSessionKeyRegistryRedis(conf *Config) *redis.Client {
 	return redis.New(conf.Redis.WithNamespace("js", "keys"))
 }
 
+// NewRateLimitingRedis instantiates a new redis client with the Rate Limiting namespace.
+func NewRateLimitingRedis(conf *Config) *redis.Client {
+	if conf.RateLimiting.Provider != "redis" {
+		return nil
+	}
+	return redis.New(conf.Cache.Redis.WithNamespace("rate-limiting"))
+}
+
 var errUnknownComponent = errors.DefineInvalidArgument("unknown_component", "unknown component `{component}`")
 
 var startCommand = &cobra.Command{
@@ -218,6 +226,7 @@ var startCommand = &cobra.Command{
 			logger.Warn("No cookie block key configured, generated a random one")
 		}
 
+		config.RateLimiting.Redis.Client = NewRateLimitingRedis(config)
 		c, err := component.New(logger, &component.Config{ServiceBase: config.ServiceBase}, componentOptions...)
 		if err != nil {
 			return shared.ErrInitializeBaseComponent.WithCause(err)

config/messages.json

@@ -5903,6 +5903,15 @@
       "file": "client_access.go"
     }
   },
+  "error:pkg/identityserver:collaborator_is_contact": {
+    "translations": {
+      "en": "collaborator `{collaborator_id}` is used as a contact"
+    },
+    "description": {
+      "package": "pkg/identityserver",
+      "file": "errors.go"
+    }
+  },
   "error:pkg/identityserver:common_password": {
     "translations": {
       "en": "must not be too common"

go.mod

@@ -2,9 +2,6 @@ module go.thethings.network/lorawan-stack/v3
 
 go 1.21
 
-// Use our fork of throttled/throttled/v2.
-replace github.com/throttled/throttled/v2 => github.com/TheThingsIndustries/throttled/v2 v2.7.1-noredis
-
 // See https://github.com/mitchellh/mapstructure/pull/278
 replace github.com/mitchellh/mapstructure => github.com/TheThingsIndustries/mapstructure v0.0.0-20230413130846-941bcd1deec3
 
@@ -16,7 +13,7 @@ require (
 	github.com/TheThingsIndustries/protoc-gen-go-flags v1.2.0
 	github.com/TheThingsIndustries/protoc-gen-go-json v1.6.0
 	github.com/TheThingsNetwork/go-cayenne-lib v1.2.0
-	github.com/aws/aws-sdk-go v1.50.11
+	github.com/aws/aws-sdk-go v1.50.26
 	github.com/blang/semver v3.5.1+incompatible
 	github.com/blevesearch/bleve v1.0.14
 	github.com/bluele/gcache v0.0.2
@@ -27,7 +24,7 @@ require (
 	github.com/emersion/go-smtp v0.20.2
 	github.com/envoyproxy/protoc-gen-validate v1.0.4
 	github.com/felixge/httpsnoop v1.0.4
-	github.com/getsentry/sentry-go v0.26.0
+	github.com/getsentry/sentry-go v0.27.0
 	github.com/golang/gddo v0.0.0-20210115222349-20d68f94ee1f
 	github.com/google/go-cmp v0.6.0
 	github.com/google/uuid v1.6.0
@@ -52,42 +49,41 @@ require (
 	github.com/jaytaylor/html2text v0.0.0-20230321000545-74c2419ad056
 	github.com/json-iterator/go v1.1.12
 	github.com/jtacoma/uritemplates v1.0.0
-	github.com/klauspost/compress v1.17.6
+	github.com/klauspost/compress v1.17.7
 	github.com/kr/pretty v0.3.1
 	github.com/lib/pq v1.10.9
 	github.com/mileusna/useragent v1.3.4
 	github.com/mitchellh/mapstructure v1.5.0
 	github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826
-	github.com/nats-io/nats-server/v2 v2.10.10
-	github.com/nats-io/nats.go v1.32.0
+	github.com/nats-io/nats-server/v2 v2.10.11
+	github.com/nats-io/nats.go v1.33.0
 	github.com/oklog/ulid/v2 v2.1.0
 	github.com/openshift/osin v1.0.2-0.20220317075346-0f4d38c6e53f
 	github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.18.0
-	github.com/redis/go-redis/v9 v9.4.0
+	github.com/redis/go-redis/v9 v9.5.1
 	github.com/sendgrid/sendgrid-go v3.14.0+incompatible
 	github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e
 	github.com/smarty/assertions v1.15.1
 	github.com/spf13/cast v1.6.0
 	github.com/spf13/cobra v1.8.0
 	github.com/spf13/pflag v1.0.5
 	github.com/spf13/viper v1.18.2
-	github.com/throttled/throttled v2.2.5+incompatible
 	github.com/throttled/throttled/v2 v2.12.0
 	github.com/uptrace/bun v1.1.17
 	github.com/uptrace/bun/dialect/pgdialect v1.1.17
 	github.com/uptrace/bun/driver/pgdriver v1.1.17
 	github.com/vmihailenco/msgpack/v5 v5.4.1
 	go.opencensus.io v0.24.0
-	go.opentelemetry.io/contrib/instrumentation/github.com/gorilla/mux/otelmux v0.47.0
-	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0
-	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0
-	go.opentelemetry.io/otel v1.22.0
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.22.0
-	go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.22.0
-	go.opentelemetry.io/otel/sdk v1.22.0
-	go.opentelemetry.io/otel/trace v1.22.0
+	go.opentelemetry.io/contrib/instrumentation/github.com/gorilla/mux/otelmux v0.49.0
+	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0
+	go.opentelemetry.io/otel v1.24.0
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.24.0
+	go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.24.0
+	go.opentelemetry.io/otel/sdk v1.24.0
+	go.opentelemetry.io/otel/trace v1.24.0
 	go.packetbroker.org/api/iam v1.8.2
 	go.packetbroker.org/api/iam/v2 v2.9.1
 	go.packetbroker.org/api/mapping/v2 v2.3.1
@@ -96,18 +92,18 @@ require (
 	go.thethings.network/lorawan-application-payload v0.0.0-20220125153912-1198ff1e403e
 	go.thethings.network/lorawan-stack-legacy/v2 v2.1.0
 	go.uber.org/automaxprocs v1.5.3
-	go.uber.org/zap v1.26.0
+	go.uber.org/zap v1.27.0
 	gocloud.dev v0.36.0
 	gocloud.dev/pubsub/natspubsub v0.36.0
-	golang.org/x/crypto v0.18.0
+	golang.org/x/crypto v0.20.0
 	golang.org/x/exp v0.0.0-20240119083558-1b970713d09a
-	golang.org/x/net v0.20.0
-	golang.org/x/oauth2 v0.16.0
+	golang.org/x/net v0.21.0
+	golang.org/x/oauth2 v0.17.0
 	golang.org/x/sync v0.6.0
 	google.golang.org/genproto v0.0.0-20240125205218-1f4bbc51befe
 	google.golang.org/genproto/googleapis/api v0.0.0-20240125205218-1f4bbc51befe
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20240125205218-1f4bbc51befe
-	google.golang.org/grpc v1.61.0
+	google.golang.org/grpc v1.61.1
 	google.golang.org/protobuf v1.32.0
 	gopkg.in/mail.v2 v2.3.1
 	gopkg.in/square/go-jose.v2 v2.6.0
@@ -186,7 +182,6 @@ require (
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
-	github.com/gomodule/redigo v1.8.9 // indirect
 	github.com/google/pprof v0.0.0-20230207041349-798e818bf904 // indirect
 	github.com/google/s2a-go v0.1.7 // indirect
 	github.com/google/wire v0.5.0 // indirect
@@ -249,13 +244,13 @@ require (
 	github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
 	github.com/willf/bitset v1.1.10 // indirect
 	go.etcd.io/bbolt v1.3.5 // indirect
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0 // indirect
-	go.opentelemetry.io/otel/metric v1.22.0 // indirect
-	go.opentelemetry.io/proto/otlp v1.0.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.24.0 // indirect
+	go.opentelemetry.io/otel/metric v1.24.0 // indirect
+	go.opentelemetry.io/proto/otlp v1.1.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	golang.org/x/image v0.14.0 // indirect
-	golang.org/x/sys v0.16.0 // indirect
-	golang.org/x/term v0.16.0 // indirect
+	golang.org/x/sys v0.17.0 // indirect
+	golang.org/x/term v0.17.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/time v0.5.0 // indirect
 	golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect