Support AWS MSK IAM (#57)

* Update modules

* Fill out MSK SASL support

* Fix lint errors

* Make config parsing strict

* Update README

* Remove unnecessary import

* Remove unnecessary log message

* Update config

* Fix config

* Update tests

* Update directories

* Update README

* Bump version

README.md

@@ -281,10 +281,10 @@ spec:
  # SASL settings (optional, not supported if using ZooKeeper)
  sasl:
  enabled: true # Whether SASL is enabled
- mechanism: SCRAM-SHA-512 # Mechanism to use;
- # choices are PLAIN, SCRAM-SHA-256, and SCRAM-SHA-512
- username: my-username # SASL username
- password: my-password # SASL password
+ mechanism: SCRAM-SHA-512 # Mechanism to use; choices are AWS-MSK-IAM, PLAIN,
+ # SCRAM-SHA-256, and SCRAM-SHA-512
+ username: my-username # SASL username; ignored for AWS-MSK-IAM
+ password: my-password # SASL password; ignored for AWS-MSK-IAM
 ```
 
 Note that the `name`, `environment`, `region`, and `description` fields are used
@@ -477,17 +477,21 @@ command-line or in a cluster config. See [this config](examples/auth/cluster.yam
 ### SASL
 
 `topicctl` supports SASL authentication when running in the exclusive broker API mode. To use this,
-either set the `--sasl-mechanism`, `--sasl-username`, and `--sasl-password` flags on the command
-line or fill out the `SASL` section of the cluster config.
+either set the `--sasl-mechanism` and other appropriate `--sasl-*` flags on the command line or
+fill out the `SASL` section of the cluster config.
 
-If using the cluster config, the username and password can still be set on the command-line
-or via the `TOPICCTL_SASL_USERNAME` and `TOPICCTL_SASL_PASSWORD` environment variables.
+The following mechanisms can be used:
 
-The tool currently supports the following SASL mechanisms:
+1. `AWS-MSK-IAM`
+2. `PLAIN`
+3. `SCRAM-SHA-256`
+4. `SCRAM-SHA-512`
 
-1. `PLAIN`
-2. `SCRAM-SHA-256`
-3. `SCRAM-SHA-512`
+If using `AWS-MSK-IAM`, then `topicctl` will attempt to discover your AWS credentials in the
+locations and order described [here](https://docs.aws.amazon.com/sdk-for-go/api/aws/session/).
+The other mechanisms require a username and password to be set in either the cluster config
+or on the command-line. See the cluster configs in the [examples/auth](/examples/auth) and
+[examples/msk](/examples/msk) directories for some specific examples.
 
 Note that SASL can be run either with or without TLS, although the former is generally more
 secure.

cmd/topicctl/subcmd/shared.go

@@ -40,6 +40,25 @@ func (s sharedOptions) validate() error {
  )
  }
 
+ if s.clusterConfig != "" {
+ clusterConfig, clusterConfigErr := config.LoadClusterFile(s.clusterConfig, s.expandEnv)
+ if clusterConfigErr != nil {
+ err = multierror.Append(
+ err,
+ clusterConfigErr,
+ )
+ } else {
+ clusterConfigValidateErr := clusterConfig.Validate()
+
+ if clusterConfigValidateErr != nil {
+ err = multierror.Append(
+ err,
+ clusterConfigValidateErr,
+ )
+ }
+ }
+ }
+
  if s.zkAddr != "" && s.brokerAddr != "" {
  err = multierror.Append(
  err,
@@ -67,9 +86,15 @@ func (s sharedOptions) validate() error {
  }
 
  if useSASL {
- if saslErr := admin.ValidateSASLMechanism(s.saslMechanism); saslErr != nil {
+ saslMechanism, saslErr := admin.SASLNameToMechanism(s.saslMechanism)
+ if saslErr != nil {
  err = multierror.Append(err, saslErr)
  }
+
+ if saslMechanism == admin.SASLMechanismAWSMSKIAM &&
+ (s.saslUsername != "" || s.saslPassword != "") {
+ log.Warn("Username and password are ignored if using SASL AWS-MSK-IAM")
+ }
  }
 
  return err
@@ -100,6 +125,17 @@ func (s sharedOptions) getAdminClient(
  saslEnabled := (s.saslMechanism != "" ||
  s.saslPassword != "" ||
  s.saslUsername != "")
+
+ var saslMechanism admin.SASLMechanism
+ var err error
+
+ if s.saslMechanism != "" {
+ saslMechanism, err = admin.SASLNameToMechanism(s.saslMechanism)
+ if err != nil {
+ return nil, err
+ }
+ }
+
  return admin.NewBrokerAdminClient(
  ctx,
  admin.BrokerAdminClientConfig{
@@ -115,7 +151,7 @@ func (s sharedOptions) getAdminClient(
  },
  SASL: admin.SASLConfig{
  Enabled: saslEnabled,
- Mechanism: s.saslMechanism,
+ Mechanism: saslMechanism,
  Password: s.saslPassword,
  Username: s.saslUsername,
  },
@@ -161,7 +197,7 @@ func addSharedFlags(cmd *cobra.Command, options *sharedOptions) {
  &options.saslMechanism,
  "sasl-mechanism",
  "",
- "SASL mechanism if using SASL (choices: PLAIN, SCRAM-SHA-256, or SCRAM-SHA-512)",
+ "SASL mechanism if using SASL (choices: AWS-MSK-IAM, PLAIN, SCRAM-SHA-256, or SCRAM-SHA-512)",
  )
  cmd.Flags().StringVar(
  &options.saslPassword,

examples/auth/cluster.yaml

@@ -21,9 +21,13 @@ spec:
  enabled: true
  mechanism: SCRAM-SHA-512
 
- # As an alternative to storing these in the config (probably not super-secure), these can be
- # set by using the --sasl-username and --sasl-password flags or the
- # TOPICCTL_SASL_USERNAME and TOPICCTL_SASL_PASSWORD environment variables when running
- # topicctl.
+ # As an alternative to storing these in plain text in the config (probably not super-secure),
+ # these can also be set via:
+ #
+ # 1. The --sasl-username and --sasl-password command-line flags,
+ # 2. The TOPICCTL_SASL_USERNAME and TOPICCTL_SASL_PASSWORD environment variables, or
+ # 3. Putting placeholder strings in the config and running with the --expand-env flag as
+ # described in the README.
+ #
  username: adminscram
  password: admin-secret-512

examples/msk/cluster.yaml

@@ -0,0 +1,23 @@
+meta:
+ name: msk-cluster
+ environment: aws-env
+ region: aws-region
+ description: |
+ Example of config for AWS MSK cluster with IAM authentication enabled.
+
+spec:
+ bootstrapAddrs:
+ # These are dummy placeholders; replace them with the broker addresses for your MSK cluster.
+ - "b-1.my-cluster.kafka.aws-region.amazonaws.com:9098"
+ - "b-2.my-cluster.kafka.aws-region.amazonaws.com:9098"
+ - "b-3.my-cluster.kafka.aws-region.amazonaws.com:9098"
+
+ tls:
+ # TLS is enabled on the IAM-authenticated broker endpoints by default
+ enabled: true
+ sasl:
+ # No credentials are set here; instead, they'll be pulled from
+ # the environment, a shared credentials file, a shared configuration file, or the EC2 metadata
+ # service as described here: https://docs.aws.amazon.com/sdk-for-go/api/aws/session/.
+ enabled: true
+ mechanism: AWS-MSK-IAM

examples/msk/topics/topic-default.yaml

@@ -0,0 +1,17 @@
+meta:
+ name: topic-default
+ cluster: msk-cluster
+ environment: aws-env
+ region: aws-region
+ description: |
+ Topic that uses default (any) strategy for assigning partition brokers.
+
+spec:
+ partitions: 3
+ replicationFactor: 3
+ retentionMinutes: 100
+ placement:
+ strategy: any
+ settings:
+ cleanup.policy: delete
+ max.message.bytes: 5542880

go.mod

@@ -6,15 +6,16 @@ go 1.17
 // replace github.com/segmentio/kafka-go => /Users/benjamin.yolken/dev/src/github.com/segmentio/kafka-go
 
 require (
- github.com/aws/aws-sdk-go v1.20.6
+ github.com/aws/aws-sdk-go v1.41.3
  github.com/briandowns/spinner v1.11.1
  github.com/c-bata/go-prompt v0.2.3
  github.com/fatih/color v1.9.0
  github.com/ghodss/yaml v1.0.0
  github.com/hashicorp/go-multierror v1.1.0
  github.com/olekukonko/tablewriter v0.0.4
  github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da
- github.com/segmentio/kafka-go v0.4.21-0.20211001205616-c03923d67699
+ github.com/segmentio/kafka-go v0.4.25
+ github.com/segmentio/kafka-go/sasl/aws_msk_iam v0.0.0-20211124042555-e88d48aa0b68
  github.com/sirupsen/logrus v1.2.0
  github.com/spf13/cobra v1.0.0
  github.com/stretchr/testify v1.6.1
@@ -27,7 +28,7 @@ require (
  github.com/golang/snappy v0.0.1 // indirect
  github.com/hashicorp/errwrap v1.0.0 // indirect
  github.com/inconshreveable/mousetrap v1.0.0 // indirect
- github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af // indirect
+ github.com/jmespath/go-jmespath v0.4.0 // indirect
  github.com/klauspost/compress v1.9.8 // indirect
  github.com/konsorten/go-windows-terminal-sequences v1.0.1 // indirect
  github.com/mattn/go-colorable v0.1.6 // indirect
@@ -43,9 +44,8 @@ require (
  github.com/spf13/pflag v1.0.3 // indirect
  github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c // indirect
  github.com/xdg/stringprep v1.0.0 // indirect
- golang.org/x/net v0.0.0-20200202094626-16171245cfb2 // indirect
- golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae // indirect
- golang.org/x/text v0.3.0 // indirect
+ golang.org/x/sys v0.0.0-20210423082822-04245dca01da // indirect
+ golang.org/x/text v0.3.6 // indirect
  gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
  gopkg.in/yaml.v2 v2.2.8 // indirect
  gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c // indirect

go.sum

@@ -4,8 +4,8 @@ github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAE
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
-github.com/aws/aws-sdk-go v1.20.6 h1:kmy4Gvdlyez1fV4kw5RYxZzWKVyuHZHgPWeU/YvRsV4=
-github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
+github.com/aws/aws-sdk-go v1.41.3 h1:deglLZ1jjHdhkd6Rbad1MZM4gL+1pfnTfjuFk6CGJFM=
+github.com/aws/aws-sdk-go v1.41.3/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/briandowns/spinner v1.11.1 h1:OixPqDEcX3juo5AjQZAnFPbeUA0jvkp2qzB5gOZJ/L0=
@@ -65,8 +65,10 @@ github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
-github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM=
-github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
+github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
+github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
@@ -114,6 +116,7 @@ github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/9
 github.com/pierrec/lz4 v2.6.0+incompatible h1:Ix9yFKn1nSPBLFl/yZknTp8TU5G4Ps0JDmguYK6iH1A=
 github.com/pierrec/lz4 v2.6.0+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/term v0.0.0-20200520122047-c3ffed290a03 h1:pd4YKIqCB0U7O2I4gWHgEUA2mCEOENmco0l/bM957bU=
 github.com/pkg/term v0.0.0-20200520122047-c3ffed290a03/go.mod h1:Z9+Ul5bCbBKnbCvdOWbLqTHhJiYV414CURZJba6L8qA=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -131,8 +134,11 @@ github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6So
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da h1:p3Vo3i64TCLY7gIfzeQaUJ+kppEO5WQG3cL8iE8tGHU=
 github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E=
-github.com/segmentio/kafka-go v0.4.21-0.20211001205616-c03923d67699 h1:DM1XDA47wY0myfsik7hUz+pkmI2uhkfKsa6ogOTNLxw=
-github.com/segmentio/kafka-go v0.4.21-0.20211001205616-c03923d67699/go.mod h1:XzMcoMjSzDGHcIwpWUI7GB43iKZ2fTVmryPSGLf/MPg=
+github.com/segmentio/kafka-go v0.4.24/go.mod h1:XzMcoMjSzDGHcIwpWUI7GB43iKZ2fTVmryPSGLf/MPg=
+github.com/segmentio/kafka-go v0.4.25 h1:QVx9yz12syKBFkxR+dVDDwTO0ItHgnjjhIdBfqizj+8=
+github.com/segmentio/kafka-go v0.4.25/go.mod h1:XzMcoMjSzDGHcIwpWUI7GB43iKZ2fTVmryPSGLf/MPg=
+github.com/segmentio/kafka-go/sasl/aws_msk_iam v0.0.0-20211124042555-e88d48aa0b68 h1:pRkq2+tKNc1aICn0L2bkmI1orIpYeexyqiC5ka+dK4o=
+github.com/segmentio/kafka-go/sasl/aws_msk_iam v0.0.0-20211124042555-e88d48aa0b68/go.mod h1:ytmdJBnHdZJOGxs17aNWHO5JQVmaKQIhmpn57IuxEm0=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
@@ -179,8 +185,8 @@ golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73r
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
-golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI=
-golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20210614182718-04defd469f4e h1:XpT3nA5TvE525Ne3hInMh6+GETgn27Zfm9dxsThnX2Q=
+golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -198,12 +204,17 @@ golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae h1:/WDfKMnPU+m5M4xB+6x4kaepxRw6jWvR5iDRdvjHgy8=
 golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da h1:b3NXsE2LusjYGGjL5bxEVZZORm/YEFFrWFjR8eFrw/c=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=