diff --git a/Dockerfile.dapper b/Dockerfile.dapper index 03c33089..5b468725 100644 --- a/Dockerfile.dapper +++ b/Dockerfile.dapper @@ -36,7 +36,7 @@ RUN curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/i ENV HUGEMEM=1024 RUN echo "vm.nr_hugepages=$((HUGEMEM/2))" >> /etc/sysctl.conf ENV SPDK_DIR /usr/src/spdk -ENV SPDK_COMMIT_ID 086836c8d1ee2d622c86965dfb0245d966420309 +ENV SPDK_COMMIT_ID 880cd9cb41a47481e46743082057781e1063dd1c RUN git clone https://github.com/longhorn/spdk.git ${SPDK_DIR} --recursive && \ cd ${SPDK_DIR} && \ git checkout ${SPDK_COMMIT_ID} && \ diff --git a/go.mod b/go.mod index 82d1ad81..c1419cf4 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/google/uuid v1.6.0 github.com/longhorn/backupstore v0.0.0-20240219094812-3a87ee02df77 github.com/longhorn/go-common-libs v0.0.0-20240319112414-b75404dc7fbc - github.com/longhorn/go-spdk-helper v0.0.0-20240326185024-d05637da5978 + github.com/longhorn/go-spdk-helper v0.0.0-20240328085119-7ab2393959d9 github.com/pkg/errors v0.9.1 github.com/sirupsen/logrus v1.9.3 go.uber.org/multierr v1.11.0 diff --git a/go.sum b/go.sum index c6d394fa..ddea92e5 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,5 @@ github.com/0xPolygon/polygon-edge v1.3.2 h1:1Dkqo9fvAwmWXlKq68/2ac/oMvF47pICiXC8a3sbJrU= github.com/0xPolygon/polygon-edge v1.3.2/go.mod h1:h3xDlSxMjhsmflbV9igX4UJooA3SgkPdsEtLLxlfhLg= -github.com/RoaringBitmap/roaring v1.9.0 h1:lwKhr90/j0jVXJyh5X+vQN1VVn77rQFfYnh6RDRGCcE= -github.com/RoaringBitmap/roaring v1.9.0/go.mod h1:6AXUsoIEzDTFFQCe1RbGA6uFONMhvejWj5rqITANK90= github.com/RoaringBitmap/roaring v1.9.1 h1:LXcSqGGGMKm+KAzUyWn7ZeREqoOkoMX+KwLOK1thc4I= github.com/RoaringBitmap/roaring v1.9.1/go.mod h1:6AXUsoIEzDTFFQCe1RbGA6uFONMhvejWj5rqITANK90= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= @@ -28,12 +26,8 @@ github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiU github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= @@ -49,20 +43,10 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/longhorn/backupstore v0.0.0-20240219094812-3a87ee02df77 h1:iJRq59kA22f9HIjFtY/lz5rKCorZJrrYXju70XoWdmE= github.com/longhorn/backupstore v0.0.0-20240219094812-3a87ee02df77/go.mod h1:4cbJWtlrD2cGTQxQLtdlPTYopiJiusXH7CpOBrn/s3k= -github.com/longhorn/go-common-libs v0.0.0-20240307063052-6e77996eda29 h1:tyzIDCMjQGQzhqAtdJaeEMAaNUZJD/sHERXp+tYc+ms= -github.com/longhorn/go-common-libs v0.0.0-20240307063052-6e77996eda29/go.mod h1:ePLGb2r/PJBUIVoVhLOt4bLOeu0S72ZB+fWDWwC8H28= -github.com/longhorn/go-common-libs v0.0.0-20240315062830-5aa0ac1dac62 h1:ovotrq4HvaSRFgLozlNyUAvJp2eykdOwfpz4VIYZGIY= -github.com/longhorn/go-common-libs v0.0.0-20240315062830-5aa0ac1dac62/go.mod h1:ePLGb2r/PJBUIVoVhLOt4bLOeu0S72ZB+fWDWwC8H28= github.com/longhorn/go-common-libs v0.0.0-20240319112414-b75404dc7fbc h1:Eh9Npc5yBcVD8E4zVQIGUtC62HcfqevrHjQ2kh7fJ/E= github.com/longhorn/go-common-libs v0.0.0-20240319112414-b75404dc7fbc/go.mod h1:ESTw7LYBF+dB5VndQNKXKrD6B9s/hF94lotGKXLovlM= -github.com/longhorn/go-spdk-helper v0.0.0-20240308030201-9b252d6f7250 h1:bc9BtfvSuXvmztYiBCebvPWPxakbiCWBvydbeqat+Ek= -github.com/longhorn/go-spdk-helper v0.0.0-20240308030201-9b252d6f7250/go.mod h1:re2QHb6FUU9G/CL3AnXDbzFjLNPwmweBvnTfeVpkZ1E= -github.com/longhorn/go-spdk-helper v0.0.0-20240315133419-fd71aaab4a11 h1:qfr6EjGt5+jEsoAizQDNb9r4BGUVCBiSE8Y1EfKsfx4= -github.com/longhorn/go-spdk-helper v0.0.0-20240315133419-fd71aaab4a11/go.mod h1:ByXr0dR+YNz0H1+k8aBXjSzQ7R2Y2UI6f4+d90YTM/M= -github.com/longhorn/go-spdk-helper v0.0.0-20240320070503-b4a1af9f0800 h1:ThB03ABY//RSD8umQNLTzuU4QNG2YfWUASJP6+nTeow= -github.com/longhorn/go-spdk-helper v0.0.0-20240320070503-b4a1af9f0800/go.mod h1:WGm84AyXymx7L0CqOw8KEr9okywD+Cj5xZi+eKeOoiU= -github.com/longhorn/go-spdk-helper v0.0.0-20240326185024-d05637da5978 h1:SjpYD1tILj/go2wSpHwmHSwFmzoOpIKjQwAtZyFcWik= -github.com/longhorn/go-spdk-helper v0.0.0-20240326185024-d05637da5978/go.mod h1:WGm84AyXymx7L0CqOw8KEr9okywD+Cj5xZi+eKeOoiU= +github.com/longhorn/go-spdk-helper v0.0.0-20240328085119-7ab2393959d9 h1:spgyLW1ND0GZJkzFyfJvbvcjVtAwBU2Msg+Jy2Yy02k= +github.com/longhorn/go-spdk-helper v0.0.0-20240328085119-7ab2393959d9/go.mod h1:WGm84AyXymx7L0CqOw8KEr9okywD+Cj5xZi+eKeOoiU= github.com/longhorn/nsfilelock v0.0.0-20200723175406-fa7c83ad0003 h1:Jw9uANsGcHTxp6HcC++/vN17LfeuDmozHI2j6DoZf5E= github.com/longhorn/nsfilelock v0.0.0-20200723175406-fa7c83ad0003/go.mod h1:0CLeXlf59Lg6C0kjLSDf47ft73Dh37CwymYRKWwAn04= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= @@ -149,8 +133,6 @@ google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80 h1: google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80/go.mod h1:PAREbraiVEVGVdTZsVWjSbbTtSyGbAgIIvni8a8CD5s= google.golang.org/grpc v1.62.1 h1:B4n+nfKzOICUXMgyrNd19h/I9oH0L1pizfk1d4zSgTk= google.golang.org/grpc v1.62.1/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -159,14 +141,10 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EV gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -k8s.io/apimachinery v0.29.2 h1:EWGpfJ856oj11C52NRCHuU7rFDwxev48z+6DSlGNsV8= -k8s.io/apimachinery v0.29.2/go.mod h1:6HVkd1FwxIagpYrHSwJlQqZI3G9LfYWRPAkUvLnXTKU= k8s.io/apimachinery v0.29.3 h1:2tbx+5L7RNvqJjn7RIuIKu9XTsIZ9Z5wX2G22XAa5EU= k8s.io/apimachinery v0.29.3/go.mod h1:hx/S4V2PNW4OMg3WizRrHutyB5la0iCUbZym+W0EQIU= k8s.io/klog/v2 v2.110.1 h1:U/Af64HJf7FcwMcXyKm2RPM22WZzyR7OSpYj5tg3cL0= k8s.io/klog/v2 v2.110.1/go.mod h1:YGtd1984u+GgbuZ7e08/yBuAfKLSO0+uR1Fhi6ExXjo= -k8s.io/mount-utils v0.29.2 h1:FrUfgvOo63nqJRPXKoqN/DW1lMnR/y0pzpFErKh6p2o= -k8s.io/mount-utils v0.29.2/go.mod h1:9IWJTMe8tG0MYMLEp60xK9GYVeCdA3g4LowmnVi+t9Y= k8s.io/mount-utils v0.29.3 h1:iEcqPP7Vv8UClH8nnMfovtmy/04fIloRW9JuSXykoZ0= k8s.io/mount-utils v0.29.3/go.mod h1:9IWJTMe8tG0MYMLEp60xK9GYVeCdA3g4LowmnVi+t9Y= k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= diff --git a/pkg/api/types.go b/pkg/api/types.go index 6feabfde..877f9dab 100644 --- a/pkg/api/types.go +++ b/pkg/api/types.go @@ -162,6 +162,7 @@ func ProtoEngineToEngine(e *spdkrpc.Engine) *Engine { type DiskInfo struct { ID string + Name string UUID string Path string Type string diff --git a/pkg/client/client.go b/pkg/client/client.go index 116b8c36..cb155d57 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -685,7 +685,7 @@ func (c *SPDKClient) ReplicaRestoreStatus(replicaName string) (*spdkrpc.ReplicaR // DiskCreate creates a disk with the given name and path. // diskUUID is optional, if not provided, it indicates the disk is newly added. -func (c *SPDKClient) DiskCreate(diskName, diskUUID, diskPath string, blockSize int64) (*spdkrpc.Disk, error) { +func (c *SPDKClient) DiskCreate(diskName, diskUUID, diskPath, diskDriver string, blockSize int64) (*spdkrpc.Disk, error) { if diskName == "" || diskPath == "" { return nil, fmt.Errorf("failed to create disk: missing required parameters") } @@ -695,14 +695,15 @@ func (c *SPDKClient) DiskCreate(diskName, diskUUID, diskPath string, blockSize i defer cancel() return client.DiskCreate(ctx, &spdkrpc.DiskCreateRequest{ - DiskName: diskName, - DiskUuid: diskUUID, - DiskPath: diskPath, - BlockSize: blockSize, + DiskName: diskName, + DiskUuid: diskUUID, + DiskPath: diskPath, + BlockSize: blockSize, + DiskDriver: diskDriver, }) } -func (c *SPDKClient) DiskGet(diskName string) (*spdkrpc.Disk, error) { +func (c *SPDKClient) DiskGet(diskName, diskPath, diskDriver string) (*spdkrpc.Disk, error) { if diskName == "" { return nil, fmt.Errorf("failed to get disk info: missing required parameter") } @@ -712,11 +713,13 @@ func (c *SPDKClient) DiskGet(diskName string) (*spdkrpc.Disk, error) { defer cancel() return client.DiskGet(ctx, &spdkrpc.DiskGetRequest{ - DiskName: diskName, + DiskName: diskName, + DiskPath: diskPath, + DiskDriver: diskDriver, }) } -func (c *SPDKClient) DiskDelete(diskName, diskUUID string) error { +func (c *SPDKClient) DiskDelete(diskName, diskUUID, diskPath, diskDriver string) error { if diskName == "" || diskUUID == "" { return fmt.Errorf("failed to delete disk: missing required parameters") } @@ -726,8 +729,10 @@ func (c *SPDKClient) DiskDelete(diskName, diskUUID string) error { defer cancel() _, err := client.DiskDelete(ctx, &spdkrpc.DiskDeleteRequest{ - DiskName: diskName, - DiskUuid: diskUUID, + DiskName: diskName, + DiskUuid: diskUUID, + DiskPath: diskPath, + DiskDriver: diskDriver, }) return err } diff --git a/pkg/spdk/disk.go b/pkg/spdk/disk.go index d6028d82..8a9346cc 100644 --- a/pkg/spdk/disk.go +++ b/pkg/spdk/disk.go @@ -2,8 +2,6 @@ package spdk import ( "fmt" - "io" - "os" "path/filepath" "github.com/pkg/errors" @@ -20,6 +18,12 @@ import ( "github.com/longhorn/longhorn-spdk-engine/pkg/util" "github.com/longhorn/longhorn-spdk-engine/proto/spdkrpc" + + "github.com/longhorn/longhorn-spdk-engine/pkg/spdk/disk" + _ "github.com/longhorn/longhorn-spdk-engine/pkg/spdk/disk/aio" + _ "github.com/longhorn/longhorn-spdk-engine/pkg/spdk/disk/nvme" + _ "github.com/longhorn/longhorn-spdk-engine/pkg/spdk/disk/virtio-blk" + _ "github.com/longhorn/longhorn-spdk-engine/pkg/spdk/disk/virtio-scsi" ) const ( @@ -29,12 +33,13 @@ const ( hostPrefix = "/host" ) -func svcDiskCreate(spdkClient *spdkclient.Client, diskName, diskUUID, diskPath string, blockSize int64) (ret *spdkrpc.Disk, err error) { +func svcDiskCreate(spdkClient *spdkclient.Client, diskName, diskUUID, diskPath, diskDriver string, blockSize int64) (ret *spdkrpc.Disk, err error) { log := logrus.WithFields(logrus.Fields{ - "diskName": diskName, - "diskUUID": diskUUID, - "diskPath": diskPath, - "blockSize": blockSize, + "diskName": diskName, + "diskUUID": diskUUID, + "diskPath": diskPath, + "blockSize": blockSize, + "diskDriver": diskDriver, }) log.Info("Creating disk") @@ -50,27 +55,25 @@ func svcDiskCreate(spdkClient *spdkclient.Client, diskName, diskUUID, diskPath s return nil, grpcstatus.Error(grpccodes.InvalidArgument, "disk name and disk path are required") } - if err := validateDiskCreation(spdkClient, diskPath); err != nil { - return nil, grpcstatus.Error(grpccodes.InvalidArgument, errors.Wrap(err, "failed to validate disk create request").Error()) - } - - if blockSize <= 0 { - blockSize = int64(defaultBlockSize) - log.Infof("Using default block size %v", blockSize) + exactDiskDriver, err := disk.GetDiskDriver(commonTypes.DiskDriver(diskDriver), diskPath) + if err != nil { + return nil, errors.Wrapf(err, "failed to get disk driver for disk %s", diskName) } - uuid, err := addBlockDevice(spdkClient, diskName, diskUUID, diskPath, blockSize) + lvstoreUUID, err := addBlockDevice(spdkClient, diskName, diskUUID, diskPath, exactDiskDriver, blockSize) if err != nil { - return nil, grpcstatus.Error(grpccodes.Internal, errors.Wrap(err, "failed to add block device").Error()) + return nil, grpcstatus.Error(grpccodes.Internal, errors.Wrap(err, "failed to add disk block device").Error()) } - return lvstoreToDisk(spdkClient, diskPath, "", uuid) + return lvstoreToDisk(spdkClient, diskPath, "", lvstoreUUID, exactDiskDriver) } -func svcDiskDelete(spdkClient *spdkclient.Client, diskName, diskUUID string) (ret *emptypb.Empty, err error) { +func svcDiskDelete(spdkClient *spdkclient.Client, diskName, diskUUID, diskPath, diskDriver string) (ret *emptypb.Empty, err error) { log := logrus.WithFields(logrus.Fields{ - "diskName": diskName, - "diskUUID": diskUUID, + "diskName": diskName, + "diskUUID": diskUUID, + "diskPath": diskPath, + "diskDriver": diskDriver, }) log.Info("Deleting disk") @@ -86,32 +89,39 @@ func svcDiskDelete(spdkClient *spdkclient.Client, diskName, diskUUID string) (re return &emptypb.Empty{}, grpcstatus.Error(grpccodes.InvalidArgument, "disk name and disk UUID are required") } - aioBdevName := "" + bdevName := "" lvstores, err := spdkClient.BdevLvolGetLvstore("", diskUUID) if err != nil { if !jsonrpc.IsJSONRPCRespErrorNoSuchDevice(err) { return nil, errors.Wrapf(err, "failed to get lvstore with UUID %v", diskUUID) } log.Warnf("Cannot find lvstore with UUID %v", diskUUID) - aioBdevName = diskName + bdevName = diskName } else { lvstore := &lvstores[0] if lvstore.Name != diskName { log.Warnf("Disk name %v does not match lvstore name %v", diskName, lvstore.Name) return nil, grpcstatus.Errorf(grpccodes.NotFound, "disk name %v does not match lvstore name %v", diskName, lvstore.Name) } - aioBdevName = lvstore.BaseBdev + bdevName = lvstore.BaseBdev } - if _, err := spdkClient.BdevAioDelete(aioBdevName); err != nil { - return nil, errors.Wrapf(err, "failed to delete AIO bdev %v", aioBdevName) + if _, err := disk.DiskDelete(spdkClient, bdevName, diskPath, diskDriver); err != nil { + return nil, errors.Wrapf(err, "failed to delete disk %v", diskName) } + return &emptypb.Empty{}, nil } -func svcDiskGet(spdkClient *spdkclient.Client, diskName string) (ret *spdkrpc.Disk, err error) { +type DeviceInfo struct { + DeviceDriver string `json:"device_driver"` +} + +func svcDiskGet(spdkClient *spdkclient.Client, diskName, diskPath, diskDriver string) (ret *spdkrpc.Disk, err error) { log := logrus.WithFields(logrus.Fields{ - "diskName": diskName, + "diskName": diskName, + "diskPath": diskPath, + "diskDriver": diskDriver, }) log.Trace("Getting disk info") @@ -128,30 +138,47 @@ func svcDiskGet(spdkClient *spdkclient.Client, diskName string) (ret *spdkrpc.Di } // Check if the disk exists - bdevs, err := spdkClient.BdevAioGet(diskName, 0) + bdevs, err := disk.DiskGet(spdkClient, diskName, diskPath, diskDriver, 0) if err != nil { if !jsonrpc.IsJSONRPCRespErrorNoSuchDevice(err) { - return nil, grpcstatus.Errorf(grpccodes.Internal, errors.Wrapf(err, "failed to get AIO bdev with name %v", diskName).Error()) + return nil, grpcstatus.Errorf(grpccodes.Internal, errors.Wrapf(err, "failed to get bdev with name %v", diskName).Error()) } } if len(bdevs) == 0 { - return nil, grpcstatus.Errorf(grpccodes.NotFound, "cannot find AIO bdev with name %v", diskName) + return nil, grpcstatus.Errorf(grpccodes.NotFound, "cannot find disk bdev with name %v", diskName) } var targetBdev *spdktypes.BdevInfo + var exactDiskDriver commonTypes.DiskDriver + for i, bdev := range bdevs { - if bdev.DriverSpecific != nil { + switch bdev.ProductName { + case spdktypes.BdevProductNameAio: + if bdev.DriverSpecific != nil { + targetBdev = &bdevs[i] + diskPath = util.RemovePrefix(bdev.DriverSpecific.Aio.FileName, hostPrefix) + } + exactDiskDriver = commonTypes.DiskDriverAio + case spdktypes.BdevProductNameVirtioBlk: + exactDiskDriver = commonTypes.DiskDriverVirtioBlk + targetBdev = &bdevs[i] + case spdktypes.BdevProductNameVirtioScsi: + exactDiskDriver = commonTypes.DiskDriverVirtioScsi targetBdev = &bdevs[i] + case spdktypes.BdevProductNameNvme: + exactDiskDriver = commonTypes.DiskDriverNvme + targetBdev = &bdevs[i] + } + if targetBdev != nil { break } } + if targetBdev == nil { - return nil, grpcstatus.Errorf(grpccodes.NotFound, errors.Wrapf(err, "failed to get AIO bdev for disk %v", diskName).Error()) + return nil, grpcstatus.Errorf(grpccodes.NotFound, errors.Wrapf(err, "failed to get disk bdev for disk %v", diskName).Error()) } - diskPath := util.RemovePrefix(targetBdev.DriverSpecific.Aio.FileName, hostPrefix) - - return lvstoreToDisk(spdkClient, diskPath, diskName, "") + return lvstoreToDisk(spdkClient, diskPath, targetBdev.Name, "", exactDiskDriver) } func getDiskPath(path string) string { @@ -172,45 +199,15 @@ func getDiskID(filename string) (string, error) { return fmt.Sprintf("%d-%d", dev.Nvme.Major, dev.Nvme.Minor), nil } -func getDiskDeviceSize(path string) (int64, error) { - file, err := os.Open(path) - if err != nil { - return 0, errors.Wrapf(err, "failed to open %s", path) - } - defer file.Close() - - pos, err := file.Seek(0, io.SeekEnd) - if err != nil { - return 0, errors.Wrapf(err, "failed to seek %s", path) - } - return pos, nil -} - -func validateDiskCreation(spdkClient *spdkclient.Client, diskPath string) error { - ok, err := spdkutil.IsBlockDevice(diskPath) - if err != nil { - return errors.Wrap(err, "failed to check if disk is a block device") - } - if !ok { - return errors.Wrapf(err, "disk %v is not a block device", diskPath) - } - - size, err := getDiskDeviceSize(diskPath) - if err != nil { - return errors.Wrap(err, "failed to get disk size") - } - if size == 0 { - return fmt.Errorf("disk %v size is 0", diskPath) - } - +func validateAioDiskCreation(spdkClient *spdkclient.Client, diskPath string, diskDriver commonTypes.DiskDriver) error { diskID, err := getDiskID(getDiskPath(diskPath)) if err != nil { return errors.Wrap(err, "failed to get disk device number") } - bdevs, err := spdkClient.BdevAioGet("", 0) + bdevs, err := disk.DiskGet(spdkClient, "", "", string(diskDriver), 0) if err != nil { - return errors.Wrap(err, "failed to get AIO bdevs") + return errors.Wrap(err, "failed to get disk bdevs") } for _, bdev := range bdevs { @@ -220,33 +217,55 @@ func validateDiskCreation(spdkClient *spdkclient.Client, diskPath string) error } if id == diskID { - return fmt.Errorf("disk %v is already used by AIO bdev %v", diskPath, bdev.Name) + return fmt.Errorf("disk %v is already used by disk bdev %v", diskPath, bdev.Name) } } return nil } -func addBlockDevice(spdkClient *spdkclient.Client, diskName, diskUUID, diskPath string, blockSize int64) (string, error) { +func addBlockDevice(spdkClient *spdkclient.Client, diskName, diskUUID, originalDiskPath string, diskDriver commonTypes.DiskDriver, blockSize int64) (string, error) { log := logrus.WithFields(logrus.Fields{ - "diskName": diskName, - "diskUUID": diskUUID, - "diskPath": diskPath, - "blockSize": blockSize, + "diskName": diskName, + "diskUUID": diskUUID, + "diskPath": originalDiskPath, + "diskDriver": diskDriver, + "blockSize": blockSize, }) - log.Infof("Creating AIO bdev %v with block size %v", diskName, blockSize) - bdevName, err := spdkClient.BdevAioCreate(getDiskPath(diskPath), diskName, uint64(blockSize)) + diskPath := originalDiskPath + if diskDriver == commonTypes.DiskDriverAio { + if err := validateAioDiskCreation(spdkClient, diskPath, diskDriver); err != nil { + return "", errors.Wrap(err, "failed to validate disk creation") + } + diskPath = getDiskPath(originalDiskPath) + } + + log.Info("Creating disk bdev") + + bdevName, err := disk.DiskCreate(spdkClient, diskName, diskPath, string(diskDriver), uint64(blockSize)) if err != nil { if !jsonrpc.IsJSONRPCRespErrorFileExists(err) { - return "", errors.Wrapf(err, "failed to create AIO bdev") + return "", errors.Wrapf(err, "failed to create disk bdev") } } - log.Infof("Creating lvstore %v", diskName) + bdevs, err := disk.DiskGet(spdkClient, bdevName, diskPath, "", 0) + if err != nil { + return "", errors.Wrapf(err, "failed to get disk bdev") + } + if len(bdevs) == 0 { + return "", fmt.Errorf("cannot find disk bdev with name %v", bdevName) + } + if len(bdevs) > 1 { + return "", fmt.Errorf("found multiple disk bdevs with name %v", bdevName) + } + bdev := bdevs[0] - // Name of the lvstore is the same as the name of the aio bdev - lvstoreName := bdevName + // Name of the lvstore is the same as the name of the disk bdev + lvstoreName := bdev.Name + + log.Infof("Creating lvstore %v", lvstoreName) lvstores, err := spdkClient.BdevLvolGetLvstore("", "") if err != nil { @@ -267,28 +286,28 @@ func addBlockDevice(spdkClient *spdkclient.Client, diskName, diskUUID, diskPath return lvstore.UUID, nil } - // Rename the existing lvstore to the name of the aio bdev if the UUID matches + // Rename the existing lvstore to the name of the disk bdev if the UUID matches log.Infof("Renaming the existing lvstore %v to %v", lvstore.Name, lvstoreName) renamed, err := spdkClient.BdevLvolRenameLvstore(lvstore.Name, lvstoreName) if err != nil { - return "", errors.Wrapf(err, "failed to rename lvstore from %v to %v", lvstore.Name, lvstoreName) + return "", errors.Wrapf(err, "failed to rename lvstore %v to %v", lvstore.Name, lvstoreName) } if !renamed { - return "", fmt.Errorf("failed to rename lvstore from %v to %v", lvstore.Name, lvstoreName) + return "", fmt.Errorf("failed to rename lvstore %v to %v", lvstore.Name, lvstoreName) } return lvstore.UUID, nil } if diskUUID == "" { log.Infof("Creating a new lvstore %v", lvstoreName) - return spdkClient.BdevLvolCreateLvstore(lvstoreName, diskName, defaultClusterSize) + return spdkClient.BdevLvolCreateLvstore(bdev.Name, lvstoreName, defaultClusterSize) } // The lvstore should be created before, but it cannot be found now. return "", grpcstatus.Error(grpccodes.NotFound, fmt.Sprintf("cannot find lvstore with UUID %v", diskUUID)) } -func lvstoreToDisk(spdkClient *spdkclient.Client, diskPath, lvstoreName, lvstoreUUID string) (*spdkrpc.Disk, error) { +func lvstoreToDisk(spdkClient *spdkclient.Client, diskPath, lvstoreName, lvstoreUUID string, diskDriver commonTypes.DiskDriver) (*spdkrpc.Disk, error) { lvstores, err := spdkClient.BdevLvolGetLvstore(lvstoreName, lvstoreUUID) if err != nil { return nil, errors.Wrapf(err, "failed to get lvstore with name %v and UUID %v", lvstoreName, lvstoreUUID) @@ -296,16 +315,21 @@ func lvstoreToDisk(spdkClient *spdkclient.Client, diskPath, lvstoreName, lvstore lvstore := &lvstores[0] // A disk does not have a fsid, so we use the device number as the disk ID - diskID, err := getDiskID(getDiskPath(diskPath)) - if err != nil { - return nil, errors.Wrapf(err, "failed to get disk ID") + diskID := diskPath + if diskDriver == commonTypes.DiskDriverAio { + diskID, err = getDiskID(getDiskPath(diskPath)) + if err != nil { + return nil, errors.Wrapf(err, "failed to get disk ID") + } } return &spdkrpc.Disk{ Id: diskID, + Name: lvstore.Name, Uuid: lvstore.UUID, Path: diskPath, Type: DiskTypeBlock, + Driver: string(diskDriver), TotalSize: int64(lvstore.TotalDataClusters * lvstore.ClusterSize), FreeSize: int64(lvstore.FreeClusters * lvstore.ClusterSize), TotalBlocks: int64(lvstore.TotalDataClusters * lvstore.ClusterSize / lvstore.BlockSize), diff --git a/pkg/spdk/disk/aio/aio.go b/pkg/spdk/disk/aio/aio.go new file mode 100644 index 00000000..74c77a0a --- /dev/null +++ b/pkg/spdk/disk/aio/aio.go @@ -0,0 +1,73 @@ +package aio + +import ( + "fmt" + "io" + "os" + + "github.com/pkg/errors" + + commonTypes "github.com/longhorn/go-common-libs/types" + spdkclient "github.com/longhorn/go-spdk-helper/pkg/spdk/client" + spdktypes "github.com/longhorn/go-spdk-helper/pkg/spdk/types" + spdkutil "github.com/longhorn/go-spdk-helper/pkg/util" + + "github.com/longhorn/longhorn-spdk-engine/pkg/spdk/disk" +) + +type DiskDriverAio struct { +} + +func init() { + driver := &DiskDriverAio{} + disk.RegisterDiskDriver(string(commonTypes.DiskDriverAio), driver) +} + +func (d *DiskDriverAio) DiskCreate(spdkClient *spdkclient.Client, diskName, diskPath string, blockSize uint64) (string, error) { + if err := validateDiskCreation(spdkClient, diskPath); err != nil { + return "", errors.Wrap(err, "failed to validate disk creation") + } + + return spdkClient.BdevAioCreate(diskPath, diskName, blockSize) +} + +func (d *DiskDriverAio) DiskDelete(spdkClient *spdkclient.Client, diskName, diskPath string) (deleted bool, err error) { + return spdkClient.BdevAioDelete(diskName) +} + +func (d *DiskDriverAio) DiskGet(spdkClient *spdkclient.Client, diskName, diskPath string, timeout uint64) ([]spdktypes.BdevInfo, error) { + return spdkClient.BdevAioGet(diskName, timeout) +} + +func validateDiskCreation(spdkClient *spdkclient.Client, diskPath string) error { + ok, err := spdkutil.IsBlockDevice(diskPath) + if err != nil { + return errors.Wrap(err, "failed to check if disk is a block device") + } + if !ok { + return errors.Wrapf(err, "disk %v is not a block device", diskPath) + } + + size, err := getDiskDeviceSize(diskPath) + if err != nil { + return errors.Wrap(err, "failed to get disk size") + } + if size == 0 { + return fmt.Errorf("disk %v size is 0", diskPath) + } + return nil +} + +func getDiskDeviceSize(path string) (int64, error) { + file, err := os.Open(path) + if err != nil { + return 0, errors.Wrapf(err, "failed to open %s", path) + } + defer file.Close() + + pos, err := file.Seek(0, io.SeekEnd) + if err != nil { + return 0, errors.Wrapf(err, "failed to seek %s", path) + } + return pos, nil +} diff --git a/pkg/spdk/disk/driver.go b/pkg/spdk/disk/driver.go new file mode 100644 index 00000000..9ab47c7a --- /dev/null +++ b/pkg/spdk/disk/driver.go @@ -0,0 +1,84 @@ +package disk + +import ( + "fmt" + + "github.com/pkg/errors" + + spdkclient "github.com/longhorn/go-spdk-helper/pkg/spdk/client" + spdktypes "github.com/longhorn/go-spdk-helper/pkg/spdk/types" +) + +type DiskDriver interface { + DiskCreate(*spdkclient.Client, string, string, uint64) (string, error) + DiskDelete(*spdkclient.Client, string, string) (bool, error) + DiskGet(*spdkclient.Client, string, string, uint64) ([]spdktypes.BdevInfo, error) +} + +var ( + diskDrivers map[string]DiskDriver +) + +func init() { + diskDrivers = make(map[string]DiskDriver) +} + +func RegisterDiskDriver(diskDriver string, ops DiskDriver) { + diskDrivers[diskDriver] = ops +} + +func DiskCreate(spdkClient *spdkclient.Client, diskName, diskPath, diskDriver string, blockSize uint64) (string, error) { + driver, ok := diskDrivers[diskDriver] + if !ok { + return "", fmt.Errorf("disk driver %s is not registered", diskDriver) + } + + return driver.DiskCreate(spdkClient, diskName, diskPath, blockSize) +} + +func DiskDelete(spdkClient *spdkclient.Client, diskName, diskPath, diskDriver string) (bool, error) { + driver, ok := diskDrivers[diskDriver] + if !ok { + return false, fmt.Errorf("disk driver %s is not registered", diskDriver) + } + + return driver.DiskDelete(spdkClient, diskName, diskPath) +} + +func DiskGet(spdkClient *spdkclient.Client, diskName, diskPath, diskDriver string, timeout uint64) ([]spdktypes.BdevInfo, error) { + if diskDriver == "" { + if !isBDF(diskPath) { + return spdkClient.BdevGetBdevs(diskName, 0) + } + bdevs, err := spdkClient.BdevGetBdevs("", 0) + if err != nil { + return nil, errors.Wrapf(err, "failed to get bdevs") + } + foundBdevs := []spdktypes.BdevInfo{} + for _, bdev := range bdevs { + if bdev.DriverSpecific == nil { + continue + } + if bdev.DriverSpecific.Nvme == nil { + if diskName == bdev.Name { + foundBdevs = append(foundBdevs, bdev) + } + } else { + nvmes := *bdev.DriverSpecific.Nvme + for _, nvme := range nvmes { + if nvme.PciAddress == diskPath { + foundBdevs = append(foundBdevs, bdev) + } + } + } + } + return foundBdevs, nil + } + + driver, ok := diskDrivers[diskDriver] + if !ok { + return nil, fmt.Errorf("disk driver %s is not registered", diskDriver) + } + + return driver.DiskGet(spdkClient, diskName, diskPath, timeout) +} diff --git a/pkg/spdk/disk/nvme/nvme.go b/pkg/spdk/disk/nvme/nvme.go new file mode 100644 index 00000000..3e24f920 --- /dev/null +++ b/pkg/spdk/disk/nvme/nvme.go @@ -0,0 +1,91 @@ +package nvme + +import ( + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + + commonTypes "github.com/longhorn/go-common-libs/types" + "github.com/longhorn/go-spdk-helper/pkg/jsonrpc" + spdkclient "github.com/longhorn/go-spdk-helper/pkg/spdk/client" + spdksetup "github.com/longhorn/go-spdk-helper/pkg/spdk/setup" + spdktypes "github.com/longhorn/go-spdk-helper/pkg/spdk/types" + helpertypes "github.com/longhorn/go-spdk-helper/pkg/types" + helperutil "github.com/longhorn/go-spdk-helper/pkg/util" + + "github.com/longhorn/longhorn-spdk-engine/pkg/spdk/disk" +) + +type DiskDriverNvme struct { +} + +func init() { + driver := &DiskDriverNvme{} + disk.RegisterDiskDriver(string(commonTypes.DiskDriverNvme), driver) +} + +func (d *DiskDriverNvme) DiskCreate(spdkClient *spdkclient.Client, diskName, diskPath string, blockSize uint64) (string, error) { + // TODO: validate the diskPath + executor, err := helperutil.NewExecutor(commonTypes.ProcDirectory) + if err != nil { + return "", errors.Wrapf(err, "failed to get the executor for NVMe disk create %v", diskPath) + } + + _, err = spdksetup.Bind(diskPath, string(commonTypes.DiskDriverUioPciGeneric), executor) + if err != nil { + return "", errors.Wrapf(err, "failed to bind NVMe disk %v with %v", diskPath, string(commonTypes.DiskDriverUioPciGeneric)) + } + + bdevs, err := spdkClient.BdevNvmeAttachController(diskName, "", diskPath, "", "PCIe", "", + helpertypes.DefaultCtrlrLossTimeoutSec, helpertypes.DefaultReconnectDelaySec, helpertypes.DefaultFastIOFailTimeoutSec) + if err != nil { + return "", errors.Wrapf(err, "failed to attach NVMe disk %v", diskPath) + } + if len(bdevs) == 0 { + return "", errors.Errorf("failed to attach NVMe disk %v", diskPath) + } + return bdevs[0], nil +} + +func (d *DiskDriverNvme) DiskDelete(spdkClient *spdkclient.Client, diskName, diskPath string) (deleted bool, err error) { + executor, err := helperutil.NewExecutor(commonTypes.ProcDirectory) + if err != nil { + return false, errors.Wrapf(err, "failed to get the executor for NVMe disk %v deletion", diskName) + } + + _, err = spdkClient.BdevNvmeDetachController(diskName) + if err != nil { + if !jsonrpc.IsJSONRPCRespErrorNoSuchDevice(err) { + return false, errors.Wrapf(err, "failed to detach NVMe disk %v", diskName) + } + } + + _, err = spdksetup.Unbind(diskPath, executor) + if err != nil { + return false, errors.Wrapf(err, "failed to unbind NVMe disk %v", diskPath) + } + return true, nil +} + +func (d *DiskDriverNvme) DiskGet(spdkClient *spdkclient.Client, diskName, diskPath string, timeout uint64) ([]spdktypes.BdevInfo, error) { + bdevs, err := spdkClient.BdevGetBdevs("", 0) + if err != nil { + return nil, errors.Wrap(err, "failed to get bdevs") + } + foundBdevs := []spdktypes.BdevInfo{} + for _, bdev := range bdevs { + if bdev.DriverSpecific == nil { + continue + } + if bdev.DriverSpecific.Nvme == nil { + continue + } + nvmes := *bdev.DriverSpecific.Nvme + for _, nvme := range nvmes { + if nvme.PciAddress == diskPath { + logrus.Infof("Found bdev %v for NVMe disk %v", bdev, diskName) + foundBdevs = append(foundBdevs, bdev) + } + } + } + return foundBdevs, nil +} diff --git a/pkg/spdk/disk/types.go b/pkg/spdk/disk/types.go new file mode 100644 index 00000000..5835b3d5 --- /dev/null +++ b/pkg/spdk/disk/types.go @@ -0,0 +1,117 @@ +package disk + +import ( + "fmt" + "regexp" + "slices" + + "github.com/pkg/errors" + + commonTypes "github.com/longhorn/go-common-libs/types" + spdksetup "github.com/longhorn/go-spdk-helper/pkg/spdk/setup" + helpertypes "github.com/longhorn/go-spdk-helper/pkg/types" + helperutil "github.com/longhorn/go-spdk-helper/pkg/util" + + "github.com/longhorn/longhorn-spdk-engine/pkg/util" +) + +type BlockDiskSubsystem string + +const ( + BlockDiskSubsystemVirtio = BlockDiskSubsystem("virtio") + BlockDiskSubsystemPci = BlockDiskSubsystem("pci") + BlockDiskSubsystemNvme = BlockDiskSubsystem("nvme") + BlockDiskSubsystemScsi = BlockDiskSubsystem("scsi") +) + +type BlockDiskType string + +const ( + BlockDiskTypeDisk = BlockDiskType("disk") + BlockDiskTypeLoop = BlockDiskType("loop") +) + +func GetDiskDriver(diskDriver commonTypes.DiskDriver, diskPathOrBdf string) (commonTypes.DiskDriver, error) { + if isBDF(diskPathOrBdf) { + return getDiskDriverForBDF(diskDriver, diskPathOrBdf) + } + + return getDiskDriverForPath(diskDriver, diskPathOrBdf) +} + +func getDiskDriverForBDF(diskDriver commonTypes.DiskDriver, bdf string) (commonTypes.DiskDriver, error) { + executor, err := helperutil.NewExecutor(commonTypes.ProcDirectory) + if err != nil { + return "", errors.Wrapf(err, "failed to get the executor for disk driver detection") + } + + diskStatus, err := spdksetup.GetDiskStatus(bdf, executor) + if err != nil { + return "", errors.Wrapf(err, "failed to get disk status for BDF %s", bdf) + } + + switch diskDriver { + case commonTypes.DiskDriverAuto: + diskPath := "" + if diskStatus.Driver != string(commonTypes.DiskDriverUioPciGeneric) { + devName, err := util.GetDevNameFromBDF(bdf) + if err != nil { + return "", errors.Wrapf(err, "failed to get device name from BDF %s", bdf) + } + diskPath = fmt.Sprintf("/dev/%s", devName) + } + return getDriverForAuto(diskStatus, diskPath) + case commonTypes.DiskDriverAio, commonTypes.DiskDriverNvme, commonTypes.DiskDriverVirtioScsi, commonTypes.DiskDriverVirtioBlk: + return diskDriver, nil + default: + return commonTypes.DiskDriverNone, fmt.Errorf("unsupported disk driver %s for BDF %s", diskDriver, bdf) + } +} + +func getDriverForAuto(diskStatus *helpertypes.DiskStatus, diskPath string) (commonTypes.DiskDriver, error) { + // SPDK supports various types of disks, including NVMe, virtio-blk, and virtio-scsi. + // + // NVMe disks can be managed by either NVMe bdev or AIO bdev. + // VirtIO disks can be managed by virtio-blk, virtio-scsi, or AIO bdev. + // + // To use the correct bdev, need to identify the disk type. + // Here's how to identify the disk type: + // - If a block device uses the subsystems virtio and pci, it's a virtio-blk disk. + // - If it uses the subsystems virtio, pci, and scsi, it's a virtio-scsi disk. + switch diskStatus.Driver { + case string(commonTypes.DiskDriverNvme): + return commonTypes.DiskDriverNvme, nil + case string(commonTypes.DiskDriverVirtioPci): + blockdevice, err := util.GetBlockDevice(diskPath) + if err != nil { + return commonTypes.DiskDriverNone, errors.Wrapf(err, "failed to get blockdevice info for %s", diskPath) + } + + if slices.Contains(blockdevice.Subsystems, string(BlockDiskSubsystemVirtio)) && slices.Contains(blockdevice.Subsystems, string(BlockDiskSubsystemPci)) { + diskDriver := commonTypes.DiskDriverVirtioBlk + if slices.Contains(blockdevice.Subsystems, string(BlockDiskSubsystemScsi)) { + diskDriver = commonTypes.DiskDriverVirtioScsi + } + return diskDriver, nil + } + + return commonTypes.DiskDriverNone, fmt.Errorf("unsupported disk driver %s for disk path %s", diskStatus.Driver, diskPath) + default: + return commonTypes.DiskDriverNone, fmt.Errorf("unsupported disk driver %s for disk path %s", diskStatus.Driver, diskPath) + } +} + +func getDiskDriverForPath(diskDriver commonTypes.DiskDriver, diskPath string) (commonTypes.DiskDriver, error) { + switch diskDriver { + case commonTypes.DiskDriverAuto, commonTypes.DiskDriverAio: + return commonTypes.DiskDriverAio, nil + default: + return commonTypes.DiskDriverNone, fmt.Errorf("unsupported disk driver %s for disk path %s", diskDriver, diskPath) + } +} + +func isBDF(addr string) bool { + bdfFormat := "[a-f0-9]{4}:[a-f0-9]{2}:[a-f0-9]{2}\\.[a-f0-9]{1}" + bdfPattern := regexp.MustCompile(bdfFormat) + return bdfPattern.MatchString(addr) +} diff --git a/pkg/spdk/disk/virtio-blk/virtio-blk.go b/pkg/spdk/disk/virtio-blk/virtio-blk.go new file mode 100644 index 00000000..2d91ab84 --- /dev/null +++ b/pkg/spdk/disk/virtio-blk/virtio-blk.go @@ -0,0 +1,68 @@ +package virtioblk + +import ( + "github.com/pkg/errors" + + commonTypes "github.com/longhorn/go-common-libs/types" + "github.com/longhorn/go-spdk-helper/pkg/jsonrpc" + spdkclient "github.com/longhorn/go-spdk-helper/pkg/spdk/client" + spdksetup "github.com/longhorn/go-spdk-helper/pkg/spdk/setup" + spdktypes "github.com/longhorn/go-spdk-helper/pkg/spdk/types" + helperutil "github.com/longhorn/go-spdk-helper/pkg/util" + + "github.com/longhorn/longhorn-spdk-engine/pkg/spdk/disk" +) + +type DiskDriverVirtioBlk struct { +} + +func init() { + driver := &DiskDriverVirtioBlk{} + disk.RegisterDiskDriver(string(commonTypes.DiskDriverVirtioBlk), driver) +} + +func (d *DiskDriverVirtioBlk) DiskCreate(spdkClient *spdkclient.Client, diskName, diskPath string, blockSize uint64) (string, error) { + // TODO: validate the diskPath + executor, err := helperutil.NewExecutor(commonTypes.ProcDirectory) + if err != nil { + return "", errors.Wrapf(err, "failed to get the executor for virtio-blk disk create %v", diskPath) + } + + _, err = spdksetup.Bind(diskPath, string(commonTypes.DiskDriverUioPciGeneric), executor) + if err != nil { + return "", errors.Wrapf(err, "failed to bind virtio-blk disk %v with %v", diskPath, string(commonTypes.DiskDriverUioPciGeneric)) + } + + bdevs, err := spdkClient.BdevVirtioAttachController(diskName, "pci", diskPath, "blk") + if err != nil { + return "", errors.Wrapf(err, "failed to attach virtio-blk disk %v", diskPath) + } + if len(bdevs) == 0 { + return "", errors.Errorf("failed to attach virtio-blk disk %v", diskPath) + } + return bdevs[0], nil +} + +func (d *DiskDriverVirtioBlk) DiskDelete(spdkClient *spdkclient.Client, diskName, diskPath string) (deleted bool, err error) { + executor, err := helperutil.NewExecutor(commonTypes.ProcDirectory) + if err != nil { + return false, errors.Wrapf(err, "failed to get the executor for virtio-blk disk %v deletion", diskName) + } + + _, err = spdkClient.BdevVirtioDetachController(diskName) + if err != nil { + if !jsonrpc.IsJSONRPCRespErrorNoSuchDevice(err) { + return false, errors.Wrapf(err, "failed to detach virtio-blk disk %v", diskName) + } + } + + _, err = spdksetup.Unbind(diskPath, executor) + if err != nil { + return false, errors.Wrapf(err, "failed to unbind virtio-blk disk %v", diskPath) + } + return true, nil +} + +func (d *DiskDriverVirtioBlk) DiskGet(spdkClient *spdkclient.Client, diskName, diskPath string, timeout uint64) ([]spdktypes.BdevInfo, error) { + return spdkClient.BdevGetBdevs(diskName, timeout) +} diff --git a/pkg/spdk/disk/virtio-scsi/virtio-scsi.go b/pkg/spdk/disk/virtio-scsi/virtio-scsi.go new file mode 100644 index 00000000..e7dbb194 --- /dev/null +++ b/pkg/spdk/disk/virtio-scsi/virtio-scsi.go @@ -0,0 +1,68 @@ +package virtioscsi + +import ( + "github.com/pkg/errors" + + commonTypes "github.com/longhorn/go-common-libs/types" + "github.com/longhorn/go-spdk-helper/pkg/jsonrpc" + spdkclient "github.com/longhorn/go-spdk-helper/pkg/spdk/client" + spdksetup "github.com/longhorn/go-spdk-helper/pkg/spdk/setup" + spdktypes "github.com/longhorn/go-spdk-helper/pkg/spdk/types" + helperutil "github.com/longhorn/go-spdk-helper/pkg/util" + + "github.com/longhorn/longhorn-spdk-engine/pkg/spdk/disk" +) + +type DiskDriverVirtioScsi struct { +} + +func init() { + driver := &DiskDriverVirtioScsi{} + disk.RegisterDiskDriver(string(commonTypes.DiskDriverVirtioScsi), driver) +} + +func (d *DiskDriverVirtioScsi) DiskCreate(spdkClient *spdkclient.Client, diskName, diskPath string, blockSize uint64) (string, error) { + // TODO: validate the diskPath + executor, err := helperutil.NewExecutor(commonTypes.ProcDirectory) + if err != nil { + return "", errors.Wrapf(err, "failed to get the executor for virtio-scsi disk create %v", diskPath) + } + + _, err = spdksetup.Bind(diskPath, string(commonTypes.DiskDriverUioPciGeneric), executor) + if err != nil { + return "", errors.Wrapf(err, "failed to bind virtio-scsi disk %v with %v", diskPath, string(commonTypes.DiskDriverUioPciGeneric)) + } + + bdevs, err := spdkClient.BdevVirtioAttachController(diskName, "pci", diskPath, "scsi") + if err != nil { + return "", errors.Wrapf(err, "failed to attach virtio-scsi disk %v", diskPath) + } + if len(bdevs) == 0 { + return "", errors.Errorf("failed to attach virtio-scsi disk %v", diskPath) + } + return bdevs[0], nil +} + +func (d *DiskDriverVirtioScsi) DiskDelete(spdkClient *spdkclient.Client, diskName, diskPath string) (deleted bool, err error) { + executor, err := helperutil.NewExecutor(commonTypes.ProcDirectory) + if err != nil { + return false, errors.Wrapf(err, "failed to get the executor for virtio-scsi disk %v deletion", diskName) + } + + _, err = spdkClient.BdevVirtioDetachController(diskName) + if err != nil { + if !jsonrpc.IsJSONRPCRespErrorNoSuchDevice(err) { + return false, errors.Wrapf(err, "failed to detach virtio-scsi disk %v", diskName) + } + } + + _, err = spdksetup.Unbind(diskPath, executor) + if err != nil { + return false, errors.Wrapf(err, "failed to unbind virtio-scsi disk %v", diskPath) + } + return true, nil +} + +func (d *DiskDriverVirtioScsi) DiskGet(spdkClient *spdkclient.Client, diskName, diskPath string, timeout uint64) ([]spdktypes.BdevInfo, error) { + return spdkClient.BdevGetBdevs(diskName, timeout) +} diff --git a/pkg/spdk/server.go b/pkg/spdk/server.go index d7e5d859..c81f715a 100644 --- a/pkg/spdk/server.go +++ b/pkg/spdk/server.go @@ -1180,7 +1180,7 @@ func (s *Server) DiskCreate(ctx context.Context, req *spdkrpc.DiskCreateRequest) spdkClient := s.spdkClient s.RUnlock() - return svcDiskCreate(spdkClient, req.DiskName, req.DiskUuid, req.DiskPath, req.BlockSize) + return svcDiskCreate(spdkClient, req.DiskName, req.DiskUuid, req.DiskPath, req.DiskDriver, req.BlockSize) } func (s *Server) DiskDelete(ctx context.Context, req *spdkrpc.DiskDeleteRequest) (ret *emptypb.Empty, err error) { @@ -1188,7 +1188,7 @@ func (s *Server) DiskDelete(ctx context.Context, req *spdkrpc.DiskDeleteRequest) spdkClient := s.spdkClient s.RUnlock() - return svcDiskDelete(spdkClient, req.DiskName, req.DiskUuid) + return svcDiskDelete(spdkClient, req.DiskName, req.DiskUuid, req.DiskPath, req.DiskDriver) } func (s *Server) DiskGet(ctx context.Context, req *spdkrpc.DiskGetRequest) (ret *spdkrpc.Disk, err error) { @@ -1196,7 +1196,7 @@ func (s *Server) DiskGet(ctx context.Context, req *spdkrpc.DiskGetRequest) (ret spdkClient := s.spdkClient s.RUnlock() - return svcDiskGet(spdkClient, req.DiskName) + return svcDiskGet(spdkClient, req.DiskName, req.DiskPath, req.DiskDriver) } func (s *Server) LogSetLevel(ctx context.Context, req *spdkrpc.LogSetLevelRequest) (ret *emptypb.Empty, err error) { diff --git a/pkg/spdk_test.go b/pkg/spdk_test.go index 4b9e2ca0..f6c485e7 100644 --- a/pkg/spdk_test.go +++ b/pkg/spdk_test.go @@ -160,6 +160,8 @@ func CleanupDiskFile(c *C, loopDevicePath string) { func (s *TestSuite) TestSPDKMultipleThread(c *C) { fmt.Println("Testing SPDK basic operations with multiple threads") + diskDriverName := "aio" + ip, err := commonNet.GetAnyExternalIP() c.Assert(err, IsNil) os.Setenv(commonNet.EnvPodIP, ip) @@ -179,13 +181,13 @@ func (s *TestSuite) TestSPDKMultipleThread(c *C) { spdkCli, err := client.NewSPDKClient(net.JoinHostPort(ip, strconv.Itoa(types.SPDKServicePort))) c.Assert(err, IsNil) - disk, err := spdkCli.DiskCreate(defaultTestDiskName, "", loopDevicePath, int64(defaultTestBlockSize)) + disk, err := spdkCli.DiskCreate(defaultTestDiskName, "", loopDevicePath, diskDriverName, int64(defaultTestBlockSize)) c.Assert(err, IsNil) c.Assert(disk.Path, Equals, loopDevicePath) c.Assert(disk.Uuid, Not(Equals), "") defer func() { - err := spdkCli.DiskDelete(defaultTestDiskName, disk.Uuid) + err := spdkCli.DiskDelete(defaultTestDiskName, disk.Uuid, disk.Path, diskDriverName) c.Assert(err, IsNil) }() @@ -428,6 +430,7 @@ func (s *TestSuite) TestSPDKMultipleThread(c *C) { func (s *TestSuite) TestSPDKMultipleThreadSnapshot(c *C) { fmt.Println("Testing SPDK snapshot operations with multiple threads") + diskDriverName := "aio" ip, err := commonNet.GetAnyExternalIP() c.Assert(err, IsNil) @@ -448,13 +451,13 @@ func (s *TestSuite) TestSPDKMultipleThreadSnapshot(c *C) { spdkCli, err := client.NewSPDKClient(net.JoinHostPort(ip, strconv.Itoa(types.SPDKServicePort))) c.Assert(err, IsNil) - disk, err := spdkCli.DiskCreate(defaultTestDiskName, "", loopDevicePath, int64(defaultTestBlockSize)) + disk, err := spdkCli.DiskCreate(defaultTestDiskName, "", loopDevicePath, diskDriverName, int64(defaultTestBlockSize)) c.Assert(err, IsNil) c.Assert(disk.Path, Equals, loopDevicePath) c.Assert(disk.Uuid, Not(Equals), "") defer func() { - err := spdkCli.DiskDelete(defaultTestDiskName, disk.Uuid) + err := spdkCli.DiskDelete(defaultTestDiskName, disk.Uuid, disk.Path, diskDriverName) c.Assert(err, IsNil) }() diff --git a/pkg/util/block.go b/pkg/util/block.go new file mode 100644 index 00000000..49a3874d --- /dev/null +++ b/pkg/util/block.go @@ -0,0 +1,108 @@ +package util + +import ( + "encoding/json" + "fmt" + "os" + "strconv" + "strings" + + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + + commonTypes "github.com/longhorn/go-common-libs/types" + "github.com/longhorn/go-spdk-helper/pkg/types" + helperutil "github.com/longhorn/go-spdk-helper/pkg/util" +) + +func GetDevNameFromBDF(bdf string) (string, error) { + ne, err := helperutil.NewExecutor(commonTypes.ProcDirectory) + if err != nil { + return "", errors.Wrap(err, "failed to create executor") + } + + cmdArgs := []string{"-n", "--nodeps", "--output", "NAME"} + output, err := ne.Execute(nil, "lsblk", cmdArgs, types.ExecuteTimeout) + if err != nil { + return "", errors.Wrap(err, "failed to list block devices") + } + + devices := strings.Fields(string(output)) + for _, dev := range devices { + link, err := os.Readlink("/sys/block/" + dev) + if err != nil { + logrus.WithError(err).Warnf("Failed to read link for %s", dev) + continue + } + + if strings.Contains(link, bdf) { + return dev, nil + } + } + + return "", fmt.Errorf("failed to find device for BDF %s", bdf) +} + +type BlockDevice struct { + Name string `json:"name"` + Path string `json:"path"` + Subsystems []string `json:"subsystems"` + Maj int `json:"maj"` + Min int `json:"min"` +} + +type BlockDevices struct { + BlockDevices []struct { + Name string `json:"name"` + Path string `json:"path"` + MajMin string `json:"maj:min"` + Subsystems string `json:"subsystems"` + } `json:"blockdevices"` +} + +func GetBlockDevice(devPath string) (BlockDevice, error) { + ne, err := helperutil.NewExecutor(commonTypes.ProcDirectory) + if err != nil { + return BlockDevice{}, errors.Wrap(err, "failed to create executor") + } + + cmdArgs := []string{"-O", "-J", devPath} + output, err := ne.Execute(nil, "lsblk", cmdArgs, types.ExecuteTimeout) + if err != nil { + return BlockDevice{}, errors.Wrap(err, "failed to get disk subsystems") + } + + var blockDevices BlockDevices + err = json.Unmarshal([]byte(output), &blockDevices) + if err != nil { + return BlockDevice{}, err + } + + if len(blockDevices.BlockDevices) == 0 { + return BlockDevice{}, fmt.Errorf("no blockdevices found") + } + + bd := blockDevices.BlockDevices[0] + majMinParts := strings.Split(bd.MajMin, ":") + if len(majMinParts) != 2 { + return BlockDevice{}, fmt.Errorf("invalid maj:min format") + } + maj, err := strconv.Atoi(majMinParts[0]) + if err != nil { + return BlockDevice{}, err + } + min, err := strconv.Atoi(majMinParts[1]) + if err != nil { + return BlockDevice{}, err + } + + subsystems := strings.Split(bd.Subsystems, ":") + + return BlockDevice{ + Name: bd.Name, + Path: bd.Path, + Subsystems: subsystems, + Maj: maj, + Min: min, + }, nil +} diff --git a/proto/spdkrpc/spdk.pb.go b/proto/spdkrpc/spdk.pb.go index 43df8ff3..b2447646 100644 --- a/proto/spdkrpc/spdk.pb.go +++ b/proto/spdkrpc/spdk.pb.go @@ -2762,6 +2762,8 @@ type Disk struct { FreeBlocks int64 `protobuf:"varint,8,opt,name=free_blocks,json=freeBlocks,proto3" json:"free_blocks,omitempty"` BlockSize int64 `protobuf:"varint,9,opt,name=block_size,json=blockSize,proto3" json:"block_size,omitempty"` ClusterSize int64 `protobuf:"varint,10,opt,name=cluster_size,json=clusterSize,proto3" json:"cluster_size,omitempty"` + Driver string `protobuf:"bytes,11,opt,name=driver,proto3" json:"driver,omitempty"` + Name string `protobuf:"bytes,12,opt,name=name,proto3" json:"name,omitempty"` } func (x *Disk) Reset() { @@ -2866,15 +2868,30 @@ func (x *Disk) GetClusterSize() int64 { return 0 } +func (x *Disk) GetDriver() string { + if x != nil { + return x.Driver + } + return "" +} + +func (x *Disk) GetName() string { + if x != nil { + return x.Name + } + return "" +} + type DiskCreateRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - DiskName string `protobuf:"bytes,1,opt,name=disk_name,json=diskName,proto3" json:"disk_name,omitempty"` - DiskUuid string `protobuf:"bytes,2,opt,name=disk_uuid,json=diskUuid,proto3" json:"disk_uuid,omitempty"` - DiskPath string `protobuf:"bytes,3,opt,name=disk_path,json=diskPath,proto3" json:"disk_path,omitempty"` - BlockSize int64 `protobuf:"varint,4,opt,name=block_size,json=blockSize,proto3" json:"block_size,omitempty"` + DiskName string `protobuf:"bytes,1,opt,name=disk_name,json=diskName,proto3" json:"disk_name,omitempty"` + DiskUuid string `protobuf:"bytes,2,opt,name=disk_uuid,json=diskUuid,proto3" json:"disk_uuid,omitempty"` + DiskPath string `protobuf:"bytes,3,opt,name=disk_path,json=diskPath,proto3" json:"disk_path,omitempty"` + BlockSize int64 `protobuf:"varint,4,opt,name=block_size,json=blockSize,proto3" json:"block_size,omitempty"` + DiskDriver string `protobuf:"bytes,5,opt,name=disk_driver,json=diskDriver,proto3" json:"disk_driver,omitempty"` } func (x *DiskCreateRequest) Reset() { @@ -2937,12 +2954,21 @@ func (x *DiskCreateRequest) GetBlockSize() int64 { return 0 } +func (x *DiskCreateRequest) GetDiskDriver() string { + if x != nil { + return x.DiskDriver + } + return "" +} + type DiskGetRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - DiskName string `protobuf:"bytes,1,opt,name=disk_name,json=diskName,proto3" json:"disk_name,omitempty"` + DiskName string `protobuf:"bytes,1,opt,name=disk_name,json=diskName,proto3" json:"disk_name,omitempty"` + DiskDriver string `protobuf:"bytes,2,opt,name=disk_driver,json=diskDriver,proto3" json:"disk_driver,omitempty"` + DiskPath string `protobuf:"bytes,3,opt,name=disk_path,json=diskPath,proto3" json:"disk_path,omitempty"` } func (x *DiskGetRequest) Reset() { @@ -2984,13 +3010,29 @@ func (x *DiskGetRequest) GetDiskName() string { return "" } +func (x *DiskGetRequest) GetDiskDriver() string { + if x != nil { + return x.DiskDriver + } + return "" +} + +func (x *DiskGetRequest) GetDiskPath() string { + if x != nil { + return x.DiskPath + } + return "" +} + type DiskDeleteRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - DiskName string `protobuf:"bytes,1,opt,name=disk_name,json=diskName,proto3" json:"disk_name,omitempty"` - DiskUuid string `protobuf:"bytes,2,opt,name=disk_uuid,json=diskUuid,proto3" json:"disk_uuid,omitempty"` + DiskName string `protobuf:"bytes,1,opt,name=disk_name,json=diskName,proto3" json:"disk_name,omitempty"` + DiskUuid string `protobuf:"bytes,2,opt,name=disk_uuid,json=diskUuid,proto3" json:"disk_uuid,omitempty"` + DiskPath string `protobuf:"bytes,3,opt,name=disk_path,json=diskPath,proto3" json:"disk_path,omitempty"` + DiskDriver string `protobuf:"bytes,4,opt,name=disk_driver,json=diskDriver,proto3" json:"disk_driver,omitempty"` } func (x *DiskDeleteRequest) Reset() { @@ -3039,6 +3081,20 @@ func (x *DiskDeleteRequest) GetDiskUuid() string { return "" } +func (x *DiskDeleteRequest) GetDiskPath() string { + if x != nil { + return x.DiskPath + } + return "" +} + +func (x *DiskDeleteRequest) GetDiskDriver() string { + if x != nil { + return x.DiskDriver + } + return "" +} + type LogSetLevelRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -3698,7 +3754,7 @@ var file_github_com_longhorn_longhorn_spdk_engine_proto_spdkrpc_spdk_proto_rawDe 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x25, 0x2e, 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x52, 0x05, - 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x94, 0x02, 0x0a, 0x04, 0x44, 0x69, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xc0, 0x02, 0x0a, 0x04, 0x44, 0x69, 0x73, 0x6b, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x75, 0x75, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x75, 0x75, 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x03, @@ -3716,267 +3772,279 @@ var file_github_com_longhorn_longhorn_spdk_engine_proto_spdkrpc_spdk_proto_rawDe 0x28, 0x03, 0x52, 0x09, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0b, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x53, 0x69, 0x7a, 0x65, - 0x22, 0x89, 0x01, 0x0a, 0x11, 0x44, 0x69, 0x73, 0x6b, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x64, 0x69, 0x73, 0x6b, 0x5f, 0x6e, - 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x64, 0x69, 0x73, 0x6b, 0x4e, - 0x61, 0x6d, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x64, 0x69, 0x73, 0x6b, 0x5f, 0x75, 0x75, 0x69, 0x64, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x64, 0x69, 0x73, 0x6b, 0x55, 0x75, 0x69, 0x64, - 0x12, 0x1b, 0x0a, 0x09, 0x64, 0x69, 0x73, 0x6b, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x08, 0x64, 0x69, 0x73, 0x6b, 0x50, 0x61, 0x74, 0x68, 0x12, 0x1d, 0x0a, - 0x0a, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, - 0x03, 0x52, 0x09, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x53, 0x69, 0x7a, 0x65, 0x22, 0x2d, 0x0a, 0x0e, - 0x44, 0x69, 0x73, 0x6b, 0x47, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, - 0x0a, 0x09, 0x64, 0x69, 0x73, 0x6b, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x08, 0x64, 0x69, 0x73, 0x6b, 0x4e, 0x61, 0x6d, 0x65, 0x22, 0x4d, 0x0a, 0x11, 0x44, - 0x69, 0x73, 0x6b, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x12, 0x1b, 0x0a, 0x09, 0x64, 0x69, 0x73, 0x6b, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x08, 0x64, 0x69, 0x73, 0x6b, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x1b, 0x0a, - 0x09, 0x64, 0x69, 0x73, 0x6b, 0x5f, 0x75, 0x75, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x08, 0x64, 0x69, 0x73, 0x6b, 0x55, 0x75, 0x69, 0x64, 0x22, 0x2a, 0x0a, 0x12, 0x4c, 0x6f, - 0x67, 0x53, 0x65, 0x74, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x12, 0x14, 0x0a, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x22, 0x2a, 0x0a, 0x12, 0x4c, 0x6f, 0x67, 0x53, 0x65, 0x74, - 0x46, 0x6c, 0x61, 0x67, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x14, 0x0a, 0x05, - 0x66, 0x6c, 0x61, 0x67, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x66, 0x6c, 0x61, - 0x67, 0x73, 0x22, 0x2b, 0x0a, 0x13, 0x4c, 0x6f, 0x67, 0x47, 0x65, 0x74, 0x4c, 0x65, 0x76, 0x65, - 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x6c, 0x65, 0x76, - 0x65, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x22, - 0x2b, 0x0a, 0x13, 0x4c, 0x6f, 0x67, 0x47, 0x65, 0x74, 0x46, 0x6c, 0x61, 0x67, 0x73, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x66, 0x6c, 0x61, 0x67, 0x73, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x66, 0x6c, 0x61, 0x67, 0x73, 0x2a, 0x26, 0x0a, 0x0b, - 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x06, 0x0a, 0x02, 0x57, - 0x4f, 0x10, 0x00, 0x12, 0x06, 0x0a, 0x02, 0x52, 0x57, 0x10, 0x01, 0x12, 0x07, 0x0a, 0x03, 0x45, - 0x52, 0x52, 0x10, 0x02, 0x32, 0x99, 0x1c, 0x0a, 0x0b, 0x53, 0x50, 0x44, 0x4b, 0x53, 0x65, 0x72, - 0x76, 0x69, 0x63, 0x65, 0x12, 0x40, 0x0a, 0x0d, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x43, - 0x72, 0x65, 0x61, 0x74, 0x65, 0x12, 0x1d, 0x2e, 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, - 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x10, 0x2e, 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, 0x52, - 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x12, 0x46, 0x0a, 0x0d, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, - 0x61, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x12, 0x1d, 0x2e, 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, - 0x63, 0x2e, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x3a, - 0x0a, 0x0a, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x47, 0x65, 0x74, 0x12, 0x1a, 0x2e, 0x73, - 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x47, 0x65, - 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x10, 0x2e, 0x73, 0x70, 0x64, 0x6b, 0x72, - 0x70, 0x63, 0x2e, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x12, 0x43, 0x0a, 0x15, 0x52, 0x65, - 0x70, 0x6c, 0x69, 0x63, 0x61, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x43, 0x72, 0x65, - 0x61, 0x74, 0x65, 0x12, 0x18, 0x2e, 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, 0x53, 0x6e, - 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x10, 0x2e, - 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x12, - 0x49, 0x0a, 0x15, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, - 0x6f, 0x74, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x12, 0x18, 0x2e, 0x73, 0x70, 0x64, 0x6b, 0x72, - 0x70, 0x63, 0x2e, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x49, 0x0a, 0x15, 0x52, 0x65, - 0x70, 0x6c, 0x69, 0x63, 0x61, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x52, 0x65, 0x76, - 0x65, 0x72, 0x74, 0x12, 0x18, 0x2e, 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, 0x53, 0x6e, - 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, - 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, - 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x43, 0x0a, 0x0b, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, - 0x4c, 0x69, 0x73, 0x74, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x1c, 0x2e, 0x73, - 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x4c, 0x69, - 0x73, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x42, 0x0a, 0x0c, 0x52, 0x65, - 0x70, 0x6c, 0x69, 0x63, 0x61, 0x57, 0x61, 0x74, 0x63, 0x68, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, - 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, - 0x74, 0x79, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x30, 0x01, 0x12, 0x60, - 0x0a, 0x19, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x52, 0x65, 0x62, 0x75, 0x69, 0x6c, 0x64, - 0x69, 0x6e, 0x67, 0x53, 0x72, 0x63, 0x53, 0x74, 0x61, 0x72, 0x74, 0x12, 0x29, 0x2e, 0x73, 0x70, - 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x52, 0x65, 0x62, - 0x75, 0x69, 0x6c, 0x64, 0x69, 0x6e, 0x67, 0x53, 0x72, 0x63, 0x53, 0x74, 0x61, 0x72, 0x74, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, - 0x12, 0x62, 0x0a, 0x1a, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x52, 0x65, 0x62, 0x75, 0x69, - 0x6c, 0x64, 0x69, 0x6e, 0x67, 0x53, 0x72, 0x63, 0x46, 0x69, 0x6e, 0x69, 0x73, 0x68, 0x12, 0x2a, + 0x12, 0x16, 0x0a, 0x06, 0x64, 0x72, 0x69, 0x76, 0x65, 0x72, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x06, 0x64, 0x72, 0x69, 0x76, 0x65, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, + 0x18, 0x0c, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x22, 0xaa, 0x01, 0x0a, + 0x11, 0x44, 0x69, 0x73, 0x6b, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x64, 0x69, 0x73, 0x6b, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x64, 0x69, 0x73, 0x6b, 0x4e, 0x61, 0x6d, 0x65, 0x12, + 0x1b, 0x0a, 0x09, 0x64, 0x69, 0x73, 0x6b, 0x5f, 0x75, 0x75, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x08, 0x64, 0x69, 0x73, 0x6b, 0x55, 0x75, 0x69, 0x64, 0x12, 0x1b, 0x0a, 0x09, + 0x64, 0x69, 0x73, 0x6b, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x08, 0x64, 0x69, 0x73, 0x6b, 0x50, 0x61, 0x74, 0x68, 0x12, 0x1d, 0x0a, 0x0a, 0x62, 0x6c, 0x6f, + 0x63, 0x6b, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x62, + 0x6c, 0x6f, 0x63, 0x6b, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x64, 0x69, 0x73, 0x6b, + 0x5f, 0x64, 0x72, 0x69, 0x76, 0x65, 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x64, + 0x69, 0x73, 0x6b, 0x44, 0x72, 0x69, 0x76, 0x65, 0x72, 0x22, 0x6b, 0x0a, 0x0e, 0x44, 0x69, 0x73, + 0x6b, 0x47, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x64, + 0x69, 0x73, 0x6b, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, + 0x64, 0x69, 0x73, 0x6b, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x64, 0x69, 0x73, 0x6b, + 0x5f, 0x64, 0x72, 0x69, 0x76, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x64, + 0x69, 0x73, 0x6b, 0x44, 0x72, 0x69, 0x76, 0x65, 0x72, 0x12, 0x1b, 0x0a, 0x09, 0x64, 0x69, 0x73, + 0x6b, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x64, 0x69, + 0x73, 0x6b, 0x50, 0x61, 0x74, 0x68, 0x22, 0x8b, 0x01, 0x0a, 0x11, 0x44, 0x69, 0x73, 0x6b, 0x44, + 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, + 0x64, 0x69, 0x73, 0x6b, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x08, 0x64, 0x69, 0x73, 0x6b, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x64, 0x69, 0x73, + 0x6b, 0x5f, 0x75, 0x75, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x64, 0x69, + 0x73, 0x6b, 0x55, 0x75, 0x69, 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x64, 0x69, 0x73, 0x6b, 0x5f, 0x70, + 0x61, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x64, 0x69, 0x73, 0x6b, 0x50, + 0x61, 0x74, 0x68, 0x12, 0x1f, 0x0a, 0x0b, 0x64, 0x69, 0x73, 0x6b, 0x5f, 0x64, 0x72, 0x69, 0x76, + 0x65, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x64, 0x69, 0x73, 0x6b, 0x44, 0x72, + 0x69, 0x76, 0x65, 0x72, 0x22, 0x2a, 0x0a, 0x12, 0x4c, 0x6f, 0x67, 0x53, 0x65, 0x74, 0x4c, 0x65, + 0x76, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x6c, 0x65, + 0x76, 0x65, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, + 0x22, 0x2a, 0x0a, 0x12, 0x4c, 0x6f, 0x67, 0x53, 0x65, 0x74, 0x46, 0x6c, 0x61, 0x67, 0x73, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x66, 0x6c, 0x61, 0x67, 0x73, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x66, 0x6c, 0x61, 0x67, 0x73, 0x22, 0x2b, 0x0a, 0x13, + 0x4c, 0x6f, 0x67, 0x47, 0x65, 0x74, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x05, 0x6c, 0x65, 0x76, 0x65, 0x6c, 0x22, 0x2b, 0x0a, 0x13, 0x4c, 0x6f, 0x67, + 0x47, 0x65, 0x74, 0x46, 0x6c, 0x61, 0x67, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x14, 0x0a, 0x05, 0x66, 0x6c, 0x61, 0x67, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x05, 0x66, 0x6c, 0x61, 0x67, 0x73, 0x2a, 0x26, 0x0a, 0x0b, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, + 0x61, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x06, 0x0a, 0x02, 0x57, 0x4f, 0x10, 0x00, 0x12, 0x06, 0x0a, + 0x02, 0x52, 0x57, 0x10, 0x01, 0x12, 0x07, 0x0a, 0x03, 0x45, 0x52, 0x52, 0x10, 0x02, 0x32, 0x99, + 0x1c, 0x0a, 0x0b, 0x53, 0x50, 0x44, 0x4b, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x40, + 0x0a, 0x0d, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x12, + 0x1d, 0x2e, 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, + 0x61, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x10, 0x2e, 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, - 0x52, 0x65, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x69, 0x6e, 0x67, 0x53, 0x72, 0x63, 0x46, 0x69, 0x6e, - 0x69, 0x73, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, - 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, - 0x74, 0x79, 0x22, 0x00, 0x12, 0x62, 0x0a, 0x1a, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x52, - 0x65, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x69, 0x6e, 0x67, 0x53, 0x72, 0x63, 0x41, 0x74, 0x74, 0x61, - 0x63, 0x68, 0x12, 0x2a, 0x2e, 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x70, - 0x6c, 0x69, 0x63, 0x61, 0x52, 0x65, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x69, 0x6e, 0x67, 0x53, 0x72, - 0x63, 0x41, 0x74, 0x74, 0x61, 0x63, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, + 0x12, 0x46, 0x0a, 0x0d, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x44, 0x65, 0x6c, 0x65, 0x74, + 0x65, 0x12, 0x1d, 0x2e, 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x70, 0x6c, + 0x69, 0x63, 0x61, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, + 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x3a, 0x0a, 0x0a, 0x52, 0x65, 0x70, 0x6c, + 0x69, 0x63, 0x61, 0x47, 0x65, 0x74, 0x12, 0x1a, 0x2e, 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, + 0x2e, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x47, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x10, 0x2e, 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x70, + 0x6c, 0x69, 0x63, 0x61, 0x12, 0x43, 0x0a, 0x15, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x53, + 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x12, 0x18, 0x2e, + 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x10, 0x2e, 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, + 0x63, 0x2e, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x12, 0x49, 0x0a, 0x15, 0x52, 0x65, 0x70, + 0x6c, 0x69, 0x63, 0x61, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x44, 0x65, 0x6c, 0x65, + 0x74, 0x65, 0x12, 0x18, 0x2e, 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, 0x53, 0x6e, 0x61, + 0x70, 0x73, 0x68, 0x6f, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, + 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, + 0x6d, 0x70, 0x74, 0x79, 0x12, 0x49, 0x0a, 0x15, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x53, + 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x52, 0x65, 0x76, 0x65, 0x72, 0x74, 0x12, 0x18, 0x2e, + 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, + 0x43, 0x0a, 0x0b, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, - 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x62, 0x0a, 0x1a, 0x52, 0x65, 0x70, 0x6c, + 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x1c, 0x2e, 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, + 0x2e, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x42, 0x0a, 0x0c, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x57, + 0x61, 0x74, 0x63, 0x68, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x16, 0x2e, 0x67, + 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, + 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x30, 0x01, 0x12, 0x60, 0x0a, 0x19, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x52, 0x65, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x69, 0x6e, 0x67, 0x53, 0x72, 0x63, - 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, 0x12, 0x2a, 0x2e, 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, - 0x2e, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x52, 0x65, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x69, - 0x6e, 0x67, 0x53, 0x72, 0x63, 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x60, 0x0a, 0x1a, - 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x53, - 0x68, 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x43, 0x6f, 0x70, 0x79, 0x12, 0x2a, 0x2e, 0x73, 0x70, 0x64, - 0x6b, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x53, 0x6e, 0x61, 0x70, - 0x73, 0x68, 0x6f, 0x74, 0x53, 0x68, 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x43, 0x6f, 0x70, 0x79, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x74, - 0x0a, 0x19, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x52, 0x65, 0x62, 0x75, 0x69, 0x6c, 0x64, - 0x69, 0x6e, 0x67, 0x44, 0x73, 0x74, 0x53, 0x74, 0x61, 0x72, 0x74, 0x12, 0x29, 0x2e, 0x73, 0x70, - 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x52, 0x65, 0x62, - 0x75, 0x69, 0x6c, 0x64, 0x69, 0x6e, 0x67, 0x44, 0x73, 0x74, 0x53, 0x74, 0x61, 0x72, 0x74, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2a, 0x2e, 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, - 0x2e, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x52, 0x65, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x69, - 0x6e, 0x67, 0x44, 0x73, 0x74, 0x53, 0x74, 0x61, 0x72, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x22, 0x00, 0x12, 0x62, 0x0a, 0x1a, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x52, - 0x65, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x69, 0x6e, 0x67, 0x44, 0x73, 0x74, 0x46, 0x69, 0x6e, 0x69, - 0x73, 0x68, 0x12, 0x2a, 0x2e, 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x70, - 0x6c, 0x69, 0x63, 0x61, 0x52, 0x65, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x69, 0x6e, 0x67, 0x44, 0x73, - 0x74, 0x46, 0x69, 0x6e, 0x69, 0x73, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, - 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, - 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x56, 0x0a, 0x22, 0x52, 0x65, 0x70, 0x6c, - 0x69, 0x63, 0x61, 0x52, 0x65, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x69, 0x6e, 0x67, 0x44, 0x73, 0x74, - 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x12, 0x18, - 0x2e, 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, - 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, + 0x53, 0x74, 0x61, 0x72, 0x74, 0x12, 0x29, 0x2e, 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, + 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x52, 0x65, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x69, 0x6e, + 0x67, 0x53, 0x72, 0x63, 0x53, 0x74, 0x61, 0x72, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, + 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x62, 0x0a, 0x1a, 0x52, 0x65, + 0x70, 0x6c, 0x69, 0x63, 0x61, 0x52, 0x65, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x69, 0x6e, 0x67, 0x53, + 0x72, 0x63, 0x46, 0x69, 0x6e, 0x69, 0x73, 0x68, 0x12, 0x2a, 0x2e, 0x73, 0x70, 0x64, 0x6b, 0x72, + 0x70, 0x63, 0x2e, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x52, 0x65, 0x62, 0x75, 0x69, 0x6c, + 0x64, 0x69, 0x6e, 0x67, 0x53, 0x72, 0x63, 0x46, 0x69, 0x6e, 0x69, 0x73, 0x68, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x62, + 0x0a, 0x1a, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x52, 0x65, 0x62, 0x75, 0x69, 0x6c, 0x64, + 0x69, 0x6e, 0x67, 0x53, 0x72, 0x63, 0x41, 0x74, 0x74, 0x61, 0x63, 0x68, 0x12, 0x2a, 0x2e, 0x73, + 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x52, 0x65, + 0x62, 0x75, 0x69, 0x6c, 0x64, 0x69, 0x6e, 0x67, 0x53, 0x72, 0x63, 0x41, 0x74, 0x74, 0x61, 0x63, + 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, - 0x12, 0x56, 0x0a, 0x22, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x52, 0x65, 0x62, 0x75, 0x69, - 0x6c, 0x64, 0x69, 0x6e, 0x67, 0x44, 0x73, 0x74, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, - 0x52, 0x65, 0x76, 0x65, 0x72, 0x74, 0x12, 0x18, 0x2e, 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, - 0x2e, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x22, 0x00, 0x12, 0x62, 0x0a, 0x1a, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x52, 0x65, 0x62, + 0x75, 0x69, 0x6c, 0x64, 0x69, 0x6e, 0x67, 0x53, 0x72, 0x63, 0x44, 0x65, 0x74, 0x61, 0x63, 0x68, + 0x12, 0x2a, 0x2e, 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x70, 0x6c, 0x69, + 0x63, 0x61, 0x52, 0x65, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x69, 0x6e, 0x67, 0x53, 0x72, 0x63, 0x44, + 0x65, 0x74, 0x61, 0x63, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, + 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, + 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x60, 0x0a, 0x1a, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, + 0x61, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x53, 0x68, 0x61, 0x6c, 0x6c, 0x6f, 0x77, + 0x43, 0x6f, 0x70, 0x79, 0x12, 0x2a, 0x2e, 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, 0x52, + 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x53, 0x68, + 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x43, 0x6f, 0x70, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, - 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x54, 0x0a, 0x13, 0x52, 0x65, 0x70, 0x6c, - 0x69, 0x63, 0x61, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x12, - 0x1c, 0x2e, 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, - 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, - 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x43, 0x72, - 0x65, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x54, - 0x0a, 0x13, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x53, - 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1c, 0x2e, 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, - 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, 0x42, 0x61, - 0x63, 0x6b, 0x75, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x22, 0x00, 0x12, 0x56, 0x0a, 0x14, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x42, - 0x61, 0x63, 0x6b, 0x75, 0x70, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x12, 0x24, 0x2e, 0x73, - 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x42, 0x61, - 0x63, 0x6b, 0x75, 0x70, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x74, 0x0a, 0x19, 0x52, 0x65, 0x70, 0x6c, + 0x69, 0x63, 0x61, 0x52, 0x65, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x69, 0x6e, 0x67, 0x44, 0x73, 0x74, + 0x53, 0x74, 0x61, 0x72, 0x74, 0x12, 0x29, 0x2e, 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, + 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x52, 0x65, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x69, 0x6e, + 0x67, 0x44, 0x73, 0x74, 0x53, 0x74, 0x61, 0x72, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x2a, 0x2e, 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x70, 0x6c, 0x69, + 0x63, 0x61, 0x52, 0x65, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x69, 0x6e, 0x67, 0x44, 0x73, 0x74, 0x53, + 0x74, 0x61, 0x72, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x62, + 0x0a, 0x1a, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x52, 0x65, 0x62, 0x75, 0x69, 0x6c, 0x64, + 0x69, 0x6e, 0x67, 0x44, 0x73, 0x74, 0x46, 0x69, 0x6e, 0x69, 0x73, 0x68, 0x12, 0x2a, 0x2e, 0x73, + 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x52, 0x65, + 0x62, 0x75, 0x69, 0x6c, 0x64, 0x69, 0x6e, 0x67, 0x44, 0x73, 0x74, 0x46, 0x69, 0x6e, 0x69, 0x73, + 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, + 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, + 0x22, 0x00, 0x12, 0x56, 0x0a, 0x22, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x52, 0x65, 0x62, + 0x75, 0x69, 0x6c, 0x64, 0x69, 0x6e, 0x67, 0x44, 0x73, 0x74, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, + 0x6f, 0x74, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x12, 0x18, 0x2e, 0x73, 0x70, 0x64, 0x6b, 0x72, + 0x70, 0x63, 0x2e, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x65, 0x0a, 0x14, - 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x53, 0x74, - 0x61, 0x74, 0x75, 0x73, 0x12, 0x24, 0x2e, 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, 0x52, - 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x53, 0x74, 0x61, - 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x73, 0x70, 0x64, - 0x6b, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x52, 0x65, 0x73, 0x74, - 0x6f, 0x72, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x22, 0x00, 0x12, 0x3d, 0x0a, 0x0c, 0x45, 0x6e, 0x67, 0x69, 0x6e, 0x65, 0x43, 0x72, 0x65, - 0x61, 0x74, 0x65, 0x12, 0x1c, 0x2e, 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, 0x45, 0x6e, - 0x67, 0x69, 0x6e, 0x65, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x0f, 0x2e, 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, 0x45, 0x6e, 0x67, 0x69, - 0x6e, 0x65, 0x12, 0x44, 0x0a, 0x0c, 0x45, 0x6e, 0x67, 0x69, 0x6e, 0x65, 0x44, 0x65, 0x6c, 0x65, - 0x74, 0x65, 0x12, 0x1c, 0x2e, 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, 0x45, 0x6e, 0x67, - 0x69, 0x6e, 0x65, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, - 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x37, 0x0a, 0x09, 0x45, 0x6e, 0x67, 0x69, - 0x6e, 0x65, 0x47, 0x65, 0x74, 0x12, 0x19, 0x2e, 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, - 0x45, 0x6e, 0x67, 0x69, 0x6e, 0x65, 0x47, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x0f, 0x2e, 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, 0x45, 0x6e, 0x67, 0x69, 0x6e, - 0x65, 0x12, 0x4b, 0x0a, 0x14, 0x45, 0x6e, 0x67, 0x69, 0x6e, 0x65, 0x53, 0x6e, 0x61, 0x70, 0x73, - 0x68, 0x6f, 0x74, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x12, 0x18, 0x2e, 0x73, 0x70, 0x64, 0x6b, - 0x72, 0x70, 0x63, 0x2e, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, 0x53, 0x6e, - 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x48, - 0x0a, 0x14, 0x45, 0x6e, 0x67, 0x69, 0x6e, 0x65, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, - 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x12, 0x18, 0x2e, 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, - 0x2e, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, - 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x48, 0x0a, 0x14, 0x45, 0x6e, 0x67, 0x69, - 0x6e, 0x65, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x52, 0x65, 0x76, 0x65, 0x72, 0x74, + 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x56, 0x0a, 0x22, 0x52, 0x65, + 0x70, 0x6c, 0x69, 0x63, 0x61, 0x52, 0x65, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x69, 0x6e, 0x67, 0x44, + 0x73, 0x74, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x52, 0x65, 0x76, 0x65, 0x72, 0x74, 0x12, 0x18, 0x2e, 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, - 0x74, 0x79, 0x12, 0x41, 0x0a, 0x0a, 0x45, 0x6e, 0x67, 0x69, 0x6e, 0x65, 0x4c, 0x69, 0x73, 0x74, - 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, - 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x1b, 0x2e, 0x73, 0x70, 0x64, 0x6b, 0x72, - 0x70, 0x63, 0x2e, 0x45, 0x6e, 0x67, 0x69, 0x6e, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x41, 0x0a, 0x0b, 0x45, 0x6e, 0x67, 0x69, 0x6e, 0x65, 0x57, - 0x61, 0x74, 0x63, 0x68, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x16, 0x2e, 0x67, + 0x74, 0x79, 0x12, 0x54, 0x0a, 0x13, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x42, 0x61, 0x63, + 0x6b, 0x75, 0x70, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x12, 0x1c, 0x2e, 0x73, 0x70, 0x64, 0x6b, + 0x72, 0x70, 0x63, 0x2e, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, + 0x63, 0x2e, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x54, 0x0a, 0x13, 0x52, 0x65, 0x70, 0x6c, + 0x69, 0x63, 0x61, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, + 0x1c, 0x2e, 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, + 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, + 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x53, 0x74, + 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x56, + 0x0a, 0x14, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x52, + 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x12, 0x24, 0x2e, 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, + 0x2e, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x52, 0x65, + 0x73, 0x74, 0x6f, 0x72, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, - 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x30, 0x01, 0x12, 0x5c, 0x0a, 0x11, 0x45, 0x6e, 0x67, 0x69, - 0x6e, 0x65, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x21, 0x2e, - 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, 0x45, 0x6e, 0x67, 0x69, 0x6e, 0x65, 0x52, 0x65, - 0x70, 0x6c, 0x69, 0x63, 0x61, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x22, 0x2e, 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, 0x45, 0x6e, 0x67, 0x69, 0x6e, - 0x65, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4e, 0x0a, 0x10, 0x45, 0x6e, 0x67, 0x69, 0x6e, 0x65, - 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x41, 0x64, 0x64, 0x12, 0x20, 0x2e, 0x73, 0x70, 0x64, + 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x65, 0x0a, 0x14, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, + 0x61, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x24, + 0x2e, 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, + 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, 0x52, + 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x53, 0x74, 0x61, + 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x3d, 0x0a, + 0x0c, 0x45, 0x6e, 0x67, 0x69, 0x6e, 0x65, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x12, 0x1c, 0x2e, + 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, 0x45, 0x6e, 0x67, 0x69, 0x6e, 0x65, 0x43, 0x72, + 0x65, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0f, 0x2e, 0x73, 0x70, + 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, 0x45, 0x6e, 0x67, 0x69, 0x6e, 0x65, 0x12, 0x44, 0x0a, 0x0c, + 0x45, 0x6e, 0x67, 0x69, 0x6e, 0x65, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x12, 0x1c, 0x2e, 0x73, + 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, 0x45, 0x6e, 0x67, 0x69, 0x6e, 0x65, 0x44, 0x65, 0x6c, + 0x65, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, + 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, + 0x74, 0x79, 0x12, 0x37, 0x0a, 0x09, 0x45, 0x6e, 0x67, 0x69, 0x6e, 0x65, 0x47, 0x65, 0x74, 0x12, + 0x19, 0x2e, 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, 0x45, 0x6e, 0x67, 0x69, 0x6e, 0x65, + 0x47, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0f, 0x2e, 0x73, 0x70, 0x64, + 0x6b, 0x72, 0x70, 0x63, 0x2e, 0x45, 0x6e, 0x67, 0x69, 0x6e, 0x65, 0x12, 0x4b, 0x0a, 0x14, 0x45, + 0x6e, 0x67, 0x69, 0x6e, 0x65, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x43, 0x72, 0x65, + 0x61, 0x74, 0x65, 0x12, 0x18, 0x2e, 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, 0x53, 0x6e, + 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, + 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x48, 0x0a, 0x14, 0x45, 0x6e, 0x67, 0x69, + 0x6e, 0x65, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, + 0x12, 0x18, 0x2e, 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, 0x53, 0x6e, 0x61, 0x70, 0x73, + 0x68, 0x6f, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, + 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, + 0x74, 0x79, 0x12, 0x48, 0x0a, 0x14, 0x45, 0x6e, 0x67, 0x69, 0x6e, 0x65, 0x53, 0x6e, 0x61, 0x70, + 0x73, 0x68, 0x6f, 0x74, 0x52, 0x65, 0x76, 0x65, 0x72, 0x74, 0x12, 0x18, 0x2e, 0x73, 0x70, 0x64, + 0x6b, 0x72, 0x70, 0x63, 0x2e, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x41, 0x0a, 0x0a, + 0x45, 0x6e, 0x67, 0x69, 0x6e, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, + 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, + 0x74, 0x79, 0x1a, 0x1b, 0x2e, 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, 0x45, 0x6e, 0x67, + 0x69, 0x6e, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x41, 0x0a, 0x0b, 0x45, 0x6e, 0x67, 0x69, 0x6e, 0x65, 0x57, 0x61, 0x74, 0x63, 0x68, 0x12, 0x16, + 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, + 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, + 0x30, 0x01, 0x12, 0x5c, 0x0a, 0x11, 0x45, 0x6e, 0x67, 0x69, 0x6e, 0x65, 0x52, 0x65, 0x70, 0x6c, + 0x69, 0x63, 0x61, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x21, 0x2e, 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, + 0x63, 0x2e, 0x45, 0x6e, 0x67, 0x69, 0x6e, 0x65, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x4c, + 0x69, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x22, 0x2e, 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, 0x45, 0x6e, 0x67, 0x69, 0x6e, 0x65, 0x52, 0x65, 0x70, 0x6c, 0x69, - 0x63, 0x61, 0x41, 0x64, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, + 0x63, 0x61, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, + 0x12, 0x4e, 0x0a, 0x10, 0x45, 0x6e, 0x67, 0x69, 0x6e, 0x65, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, + 0x61, 0x41, 0x64, 0x64, 0x12, 0x20, 0x2e, 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, 0x45, + 0x6e, 0x67, 0x69, 0x6e, 0x65, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x41, 0x64, 0x64, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, + 0x12, 0x54, 0x0a, 0x13, 0x45, 0x6e, 0x67, 0x69, 0x6e, 0x65, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, + 0x61, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x12, 0x23, 0x2e, 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, + 0x63, 0x2e, 0x45, 0x6e, 0x67, 0x69, 0x6e, 0x65, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x44, + 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, - 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x54, 0x0a, 0x13, 0x45, 0x6e, 0x67, 0x69, 0x6e, 0x65, - 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x12, 0x23, 0x2e, - 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, 0x45, 0x6e, 0x67, 0x69, 0x6e, 0x65, 0x52, 0x65, - 0x70, 0x6c, 0x69, 0x63, 0x61, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x53, 0x0a, 0x12, - 0x45, 0x6e, 0x67, 0x69, 0x6e, 0x65, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x43, 0x72, 0x65, 0x61, - 0x74, 0x65, 0x12, 0x1c, 0x2e, 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, 0x42, 0x61, 0x63, - 0x6b, 0x75, 0x70, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x1d, 0x2e, 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, 0x42, 0x61, 0x63, 0x6b, 0x75, - 0x70, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, - 0x00, 0x12, 0x53, 0x0a, 0x12, 0x45, 0x6e, 0x67, 0x69, 0x6e, 0x65, 0x42, 0x61, 0x63, 0x6b, 0x75, - 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1c, 0x2e, 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, - 0x63, 0x2e, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, - 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x62, 0x0a, 0x13, 0x45, 0x6e, 0x67, 0x69, 0x6e, 0x65, - 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x12, 0x23, 0x2e, - 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, 0x45, 0x6e, 0x67, 0x69, 0x6e, 0x65, 0x42, 0x61, - 0x63, 0x6b, 0x75, 0x70, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, 0x45, 0x6e, 0x67, - 0x69, 0x6e, 0x65, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x60, 0x0a, 0x19, 0x45, 0x6e, - 0x67, 0x69, 0x6e, 0x65, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, - 0x65, 0x46, 0x69, 0x6e, 0x69, 0x73, 0x68, 0x12, 0x29, 0x2e, 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, + 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x53, 0x0a, 0x12, 0x45, 0x6e, 0x67, 0x69, 0x6e, 0x65, + 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x12, 0x1c, 0x2e, 0x73, + 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x43, 0x72, 0x65, + 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x73, 0x70, 0x64, + 0x6b, 0x72, 0x70, 0x63, 0x2e, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x43, 0x72, 0x65, 0x61, 0x74, + 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x53, 0x0a, 0x12, 0x45, + 0x6e, 0x67, 0x69, 0x6e, 0x65, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, + 0x73, 0x12, 0x1c, 0x2e, 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, 0x42, 0x61, 0x63, 0x6b, + 0x75, 0x70, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x1d, 0x2e, 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, + 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, + 0x12, 0x62, 0x0a, 0x13, 0x45, 0x6e, 0x67, 0x69, 0x6e, 0x65, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, + 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x12, 0x23, 0x2e, 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, 0x45, 0x6e, 0x67, 0x69, 0x6e, 0x65, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x52, 0x65, - 0x73, 0x74, 0x6f, 0x72, 0x65, 0x46, 0x69, 0x6e, 0x69, 0x73, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x56, 0x0a, 0x13, - 0x45, 0x6e, 0x67, 0x69, 0x6e, 0x65, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x53, 0x74, 0x61, - 0x74, 0x75, 0x73, 0x12, 0x1d, 0x2e, 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, - 0x73, 0x74, 0x6f, 0x72, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x73, - 0x74, 0x6f, 0x72, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x22, 0x00, 0x12, 0x37, 0x0a, 0x0a, 0x44, 0x69, 0x73, 0x6b, 0x43, 0x72, 0x65, 0x61, - 0x74, 0x65, 0x12, 0x1a, 0x2e, 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, 0x44, 0x69, 0x73, - 0x6b, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0d, - 0x2e, 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, 0x44, 0x69, 0x73, 0x6b, 0x12, 0x40, 0x0a, - 0x0a, 0x44, 0x69, 0x73, 0x6b, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x12, 0x1a, 0x2e, 0x73, 0x70, - 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, 0x44, 0x69, 0x73, 0x6b, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, - 0x31, 0x0a, 0x07, 0x44, 0x69, 0x73, 0x6b, 0x47, 0x65, 0x74, 0x12, 0x17, 0x2e, 0x73, 0x70, 0x64, - 0x6b, 0x72, 0x70, 0x63, 0x2e, 0x44, 0x69, 0x73, 0x6b, 0x47, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x0d, 0x2e, 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, 0x44, 0x69, - 0x73, 0x6b, 0x12, 0x42, 0x0a, 0x0b, 0x4c, 0x6f, 0x67, 0x53, 0x65, 0x74, 0x4c, 0x65, 0x76, 0x65, - 0x6c, 0x12, 0x1b, 0x2e, 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, 0x4c, 0x6f, 0x67, 0x53, - 0x65, 0x74, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, - 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, - 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x42, 0x0a, 0x0b, 0x4c, 0x6f, 0x67, 0x53, 0x65, 0x74, - 0x46, 0x6c, 0x61, 0x67, 0x73, 0x12, 0x1b, 0x2e, 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, - 0x4c, 0x6f, 0x67, 0x53, 0x65, 0x74, 0x46, 0x6c, 0x61, 0x67, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x43, 0x0a, 0x0b, 0x4c, 0x6f, - 0x67, 0x47, 0x65, 0x74, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, - 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, - 0x79, 0x1a, 0x1c, 0x2e, 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, 0x4c, 0x6f, 0x67, 0x47, - 0x65, 0x74, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x43, 0x0a, 0x0b, 0x4c, 0x6f, 0x67, 0x47, 0x65, 0x74, 0x46, 0x6c, 0x61, 0x67, 0x73, 0x12, 0x16, - 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, - 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x1c, 0x2e, 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, - 0x2e, 0x4c, 0x6f, 0x67, 0x47, 0x65, 0x74, 0x46, 0x6c, 0x61, 0x67, 0x73, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4a, 0x0a, 0x10, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x44, - 0x65, 0x74, 0x61, 0x69, 0x6c, 0x47, 0x65, 0x74, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, + 0x73, 0x74, 0x6f, 0x72, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x73, + 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, 0x45, 0x6e, 0x67, 0x69, 0x6e, 0x65, 0x42, 0x61, 0x63, + 0x6b, 0x75, 0x70, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x22, 0x00, 0x12, 0x60, 0x0a, 0x19, 0x45, 0x6e, 0x67, 0x69, 0x6e, 0x65, 0x42, 0x61, + 0x63, 0x6b, 0x75, 0x70, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x46, 0x69, 0x6e, 0x69, 0x73, + 0x68, 0x12, 0x29, 0x2e, 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, 0x45, 0x6e, 0x67, 0x69, + 0x6e, 0x65, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x46, + 0x69, 0x6e, 0x69, 0x73, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, + 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, + 0x6d, 0x70, 0x74, 0x79, 0x22, 0x00, 0x12, 0x56, 0x0a, 0x13, 0x45, 0x6e, 0x67, 0x69, 0x6e, 0x65, + 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1d, 0x2e, + 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x53, + 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x73, + 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x53, 0x74, + 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x37, + 0x0a, 0x0a, 0x44, 0x69, 0x73, 0x6b, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x12, 0x1a, 0x2e, 0x73, + 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, 0x44, 0x69, 0x73, 0x6b, 0x43, 0x72, 0x65, 0x61, 0x74, + 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0d, 0x2e, 0x73, 0x70, 0x64, 0x6b, 0x72, + 0x70, 0x63, 0x2e, 0x44, 0x69, 0x73, 0x6b, 0x12, 0x40, 0x0a, 0x0a, 0x44, 0x69, 0x73, 0x6b, 0x44, + 0x65, 0x6c, 0x65, 0x74, 0x65, 0x12, 0x1a, 0x2e, 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, + 0x44, 0x69, 0x73, 0x6b, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x31, 0x0a, 0x07, 0x44, 0x69, 0x73, + 0x6b, 0x47, 0x65, 0x74, 0x12, 0x17, 0x2e, 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, 0x44, + 0x69, 0x73, 0x6b, 0x47, 0x65, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0d, 0x2e, + 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, 0x44, 0x69, 0x73, 0x6b, 0x12, 0x42, 0x0a, 0x0b, + 0x4c, 0x6f, 0x67, 0x53, 0x65, 0x74, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x12, 0x1b, 0x2e, 0x73, 0x70, + 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, 0x4c, 0x6f, 0x67, 0x53, 0x65, 0x74, 0x4c, 0x65, 0x76, 0x65, + 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, + 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, + 0x12, 0x42, 0x0a, 0x0b, 0x4c, 0x6f, 0x67, 0x53, 0x65, 0x74, 0x46, 0x6c, 0x61, 0x67, 0x73, 0x12, + 0x1b, 0x2e, 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, 0x4c, 0x6f, 0x67, 0x53, 0x65, 0x74, + 0x46, 0x6c, 0x61, 0x67, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, + 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, + 0x6d, 0x70, 0x74, 0x79, 0x12, 0x43, 0x0a, 0x0b, 0x4c, 0x6f, 0x67, 0x47, 0x65, 0x74, 0x4c, 0x65, + 0x76, 0x65, 0x6c, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x1c, 0x2e, 0x73, 0x70, + 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, 0x4c, 0x6f, 0x67, 0x47, 0x65, 0x74, 0x4c, 0x65, 0x76, 0x65, + 0x6c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x43, 0x0a, 0x0b, 0x4c, 0x6f, 0x67, + 0x47, 0x65, 0x74, 0x46, 0x6c, 0x61, 0x67, 0x73, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, - 0x1a, 0x1e, 0x2e, 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, 0x56, 0x65, 0x72, 0x73, 0x69, - 0x6f, 0x6e, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x47, 0x65, 0x74, 0x52, 0x65, 0x70, 0x6c, 0x79, - 0x42, 0x38, 0x5a, 0x36, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6c, - 0x6f, 0x6e, 0x67, 0x68, 0x6f, 0x72, 0x6e, 0x2f, 0x6c, 0x6f, 0x6e, 0x67, 0x68, 0x6f, 0x72, 0x6e, - 0x2d, 0x73, 0x70, 0x64, 0x6b, 0x2d, 0x65, 0x6e, 0x67, 0x69, 0x6e, 0x65, 0x2f, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x2f, 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x33, + 0x1a, 0x1c, 0x2e, 0x73, 0x70, 0x64, 0x6b, 0x72, 0x70, 0x63, 0x2e, 0x4c, 0x6f, 0x67, 0x47, 0x65, + 0x74, 0x46, 0x6c, 0x61, 0x67, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4a, + 0x0a, 0x10, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x47, + 0x65, 0x74, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x1e, 0x2e, 0x73, 0x70, 0x64, + 0x6b, 0x72, 0x70, 0x63, 0x2e, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x44, 0x65, 0x74, 0x61, + 0x69, 0x6c, 0x47, 0x65, 0x74, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x42, 0x38, 0x5a, 0x36, 0x67, 0x69, + 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6c, 0x6f, 0x6e, 0x67, 0x68, 0x6f, 0x72, + 0x6e, 0x2f, 0x6c, 0x6f, 0x6e, 0x67, 0x68, 0x6f, 0x72, 0x6e, 0x2d, 0x73, 0x70, 0x64, 0x6b, 0x2d, + 0x65, 0x6e, 0x67, 0x69, 0x6e, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x73, 0x70, 0x64, + 0x6b, 0x72, 0x70, 0x63, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/proto/spdkrpc/spdk.proto b/proto/spdkrpc/spdk.proto index 730ad6b2..1a4f3b66 100644 --- a/proto/spdkrpc/spdk.proto +++ b/proto/spdkrpc/spdk.proto @@ -341,6 +341,9 @@ message Disk { int64 block_size = 9; int64 cluster_size = 10; + + string driver = 11; + string name = 12; } message DiskCreateRequest { @@ -348,15 +351,20 @@ message DiskCreateRequest { string disk_uuid = 2; string disk_path = 3; int64 block_size = 4; + string disk_driver = 5; } message DiskGetRequest { string disk_name = 1; + string disk_driver = 2; + string disk_path = 3; } message DiskDeleteRequest { string disk_name = 1; string disk_uuid = 2; + string disk_path = 3; + string disk_driver = 4; } message LogSetLevelRequest { diff --git a/scripts/test b/scripts/test index 6146142c..f5efee3a 100755 --- a/scripts/test +++ b/scripts/test @@ -13,6 +13,16 @@ then exit 1 fi +modules=("uio_pci_generic" "nvme-tcp") + +for module in "${modules[@]}" +do + if ! modprobe "$module"; then + echo "Failed to load $module module" + exit 1 + fi +done + mount --rbind /host/dev /dev mount --rbind /host/sys /sys trap "umount /dev && umount /sys" EXIT diff --git a/vendor/github.com/longhorn/go-spdk-helper/pkg/spdk/setup/setup.go b/vendor/github.com/longhorn/go-spdk-helper/pkg/spdk/setup/setup.go new file mode 100644 index 00000000..5539f708 --- /dev/null +++ b/vendor/github.com/longhorn/go-spdk-helper/pkg/spdk/setup/setup.go @@ -0,0 +1,117 @@ +package setup + +import ( + "encoding/json" + "fmt" + "strings" + + commonNs "github.com/longhorn/go-common-libs/ns" + "github.com/pkg/errors" + + "github.com/longhorn/go-spdk-helper/pkg/types" +) + +const ( + spdkSetupPath = "/usr/src/spdk/scripts/setup.sh" +) + +func Bind(deviceAddr, deviceDriver string, executor *commonNs.Executor) (string, error) { + if deviceAddr == "" { + return "", fmt.Errorf("device address is empty") + } + + envs := []string{ + fmt.Sprintf("%s=%s", "PCI_ALLOWED", deviceAddr), + fmt.Sprintf("%s=%s", "DRIVER_OVERRIDE", deviceDriver), + } + + cmdArgs := []string{ + spdkSetupPath, + "bind", + } + + outputStr, err := executor.Execute(envs, "bash", cmdArgs, types.ExecuteTimeout) + if err != nil { + return "", err + } + + return outputStr, nil +} + +func Unbind(deviceAddr string, executor *commonNs.Executor) (string, error) { + if deviceAddr == "" { + return "", fmt.Errorf("device address is empty") + } + + cmdArgs := []string{ + spdkSetupPath, + "unbind", + deviceAddr, + } + + outputStr, err := executor.Execute(nil, "bash", cmdArgs, types.ExecuteTimeout) + if err != nil { + return "", err + } + + return outputStr, nil +} + +func GetDiskDriver(deviceAddr string, executor *commonNs.Executor) (string, error) { + if deviceAddr == "" { + return "", fmt.Errorf("device address is empty") + } + + cmdArgs := []string{ + spdkSetupPath, + "disk-driver", + deviceAddr, + } + + outputStr, err := executor.Execute(nil, "bash", cmdArgs, types.ExecuteTimeout) + if err != nil { + return "", err + } + + return extractJSON(outputStr) +} + +func GetDiskStatus(deviceAddr string, executor *commonNs.Executor) (*types.DiskStatus, error) { + if deviceAddr == "" { + return nil, fmt.Errorf("device address is empty") + } + + cmdArgs := []string{ + spdkSetupPath, + "disk-status", + deviceAddr, + } + + outputStr, err := executor.Execute(nil, "bash", cmdArgs, types.ExecuteTimeout) + if err != nil { + return nil, err + } + + jsonsStr, err := extractJSON(outputStr) + if err != nil { + return nil, err + } + + var diskStatus types.DiskStatus + err = json.Unmarshal([]byte(jsonsStr), &diskStatus) + if err != nil { + return nil, errors.Wrapf(err, "failed to unmarshal disk status for %s", deviceAddr) + } + + return &diskStatus, nil +} + +func extractJSON(outputStr string) (string, error) { + // Find the first '{' and last '}' characters, assuming valid JSON format + start := strings.Index(outputStr, "{") + end := strings.LastIndex(outputStr, "}") + if start != -1 && end != -1 { + return outputStr[start : end+1], nil + } + return "", fmt.Errorf("failed to extract JSON from output: %s", outputStr) +} diff --git a/vendor/github.com/longhorn/go-spdk-helper/pkg/types/types.go b/vendor/github.com/longhorn/go-spdk-helper/pkg/types/types.go index c5c54953..1175d634 100644 --- a/vendor/github.com/longhorn/go-spdk-helper/pkg/types/types.go +++ b/vendor/github.com/longhorn/go-spdk-helper/pkg/types/types.go @@ -37,3 +37,13 @@ const ( func GetNQN(name string) string { return fmt.Sprintf("%s:%s", NQNPrefix, name) } + +type DiskStatus struct { + Bdf string + Type string + Driver string + Vendor string + Numa string + Device string + BlockDevices string +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 5d421bff..8bfd5dd0 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -60,11 +60,12 @@ github.com/longhorn/go-common-libs/sync github.com/longhorn/go-common-libs/sys github.com/longhorn/go-common-libs/types github.com/longhorn/go-common-libs/utils -# github.com/longhorn/go-spdk-helper v0.0.0-20240326185024-d05637da5978 +# github.com/longhorn/go-spdk-helper v0.0.0-20240328085119-7ab2393959d9 ## explicit; go 1.21 github.com/longhorn/go-spdk-helper/pkg/jsonrpc github.com/longhorn/go-spdk-helper/pkg/nvme github.com/longhorn/go-spdk-helper/pkg/spdk/client +github.com/longhorn/go-spdk-helper/pkg/spdk/setup github.com/longhorn/go-spdk-helper/pkg/spdk/target github.com/longhorn/go-spdk-helper/pkg/spdk/types github.com/longhorn/go-spdk-helper/pkg/types