Add interface to provide virtio-blk QoS capabilities.

Signed-off-by: Artsiom Koltun <[email protected]>

cmd/main.go

@@ -66,16 +66,22 @@ func main() {
 
  if useKvm {
  log.Println("Creating KVM server.")
- frontendServer := frontend.NewServerWithSubsystemListener(jsonRPC,
- kvm.NewVfiouserSubsystemListener(ctrlrDir))
+ frontendServer := frontend.NewServerWithSubsystemListener(
+ jsonRPC,
+ frontend.VirtioBlkQosProviderFromMiddleendQosService(middleendServer),
+ kvm.NewVfiouserSubsystemListener(ctrlrDir),
+ )
  kvmServer := kvm.NewServer(frontendServer, qmpAddress, ctrlrDir, buses)
 
  pb.RegisterFrontendNvmeServiceServer(s, kvmServer)
  pb.RegisterFrontendVirtioBlkServiceServer(s, kvmServer)
  pb.RegisterFrontendVirtioScsiServiceServer(s, kvmServer)
  } else {
- frontendServer := frontend.NewServerWithSubsystemListener(jsonRPC,
- frontend.NewTCPSubsystemListener(tcpTransportListenAddr))
+ frontendServer := frontend.NewServerWithSubsystemListener(
+ jsonRPC,
+ frontend.VirtioBlkQosProviderFromMiddleendQosService(middleendServer),
+ frontend.NewTCPSubsystemListener(tcpTransportListenAddr),
+ )
  pb.RegisterFrontendNvmeServiceServer(s, frontendServer)
  pb.RegisterFrontendVirtioBlkServiceServer(s, frontendServer)
  pb.RegisterFrontendVirtioScsiServiceServer(s, frontendServer)

pkg/frontend/blk.go

@@ -23,6 +23,16 @@ import (
  "google.golang.org/protobuf/types/known/emptypb"
 )
 
+// VirtioBlkQosProvider provides QoS capabilities for virtio-blk
+// At the moment it is just a couple of methods from middleend QoS service, but
+// it can be changed to less verbose later.
+// If client uses VirtioBlkQosProviderFromMiddleendQosService to create an instance,
+// the interface can be changed without affecting the client code.
+type VirtioBlkQosProvider interface {
+ CreateQosVolume(context.Context, *pb.CreateQosVolumeRequest) (*pb.QosVolume, error)
+ DeleteQosVolume(context.Context, *pb.DeleteQosVolumeRequest) (*emptypb.Empty, error)
+}
+
 func sortVirtioBlks(virtioBlks []*pb.VirtioBlk) {
  sort.Slice(virtioBlks, func(i int, j int) bool {
  return virtioBlks[i].Name < virtioBlks[j].Name

pkg/frontend/blk_test.go

@@ -7,6 +7,7 @@ package frontend
 
 import (
  "bytes"
+ "context"
  "fmt"
  "reflect"
  "strings"
@@ -22,6 +23,18 @@ import (
  pb "github.com/opiproject/opi-api/storage/v1alpha1/gen/go"
 )
 
+type stubQosProvider struct {
+ err error
+}
+
+func (p stubQosProvider) CreateQosVolume(context.Context, *pb.CreateQosVolumeRequest) (*pb.QosVolume, error) {
+ return &pb.QosVolume{}, p.err
+}
+
+func (p stubQosProvider) DeleteQosVolume(context.Context, *pb.DeleteQosVolumeRequest) (*emptypb.Empty, error) {
+ return &emptypb.Empty{}, p.err
+}
+
 var (
  testVirtioCtrlID = "virtio-blk-42"
  testVirtioCtrl = pb.VirtioBlk{

pkg/frontend/frontend.go

@@ -31,6 +31,8 @@ type VirtioParameters struct {
  BlkCtrls map[string]*pb.VirtioBlk
  ScsiCtrls map[string]*pb.VirtioScsiController
  ScsiLuns map[string]*pb.VirtioScsiLun
+
+ qosProvider VirtioBlkQosProvider
 }
 
 // Server contains frontend related OPI services
@@ -47,7 +49,7 @@ type Server struct {
 
 // NewServer creates initialized instance of FrontEnd server communicating
 // with provided jsonRPC
-func NewServer(jsonRPC spdk.JSONRPC) *Server {
+func NewServer(jsonRPC spdk.JSONRPC, qosProvider VirtioBlkQosProvider) *Server {
  return &Server{
  rpc: jsonRPC,
  Nvme: NvmeParameters{
@@ -60,18 +62,32 @@ func NewServer(jsonRPC spdk.JSONRPC) *Server {
  BlkCtrls: make(map[string]*pb.VirtioBlk),
  ScsiCtrls: make(map[string]*pb.VirtioScsiController),
  ScsiLuns: make(map[string]*pb.VirtioScsiLun),
+
+ qosProvider: qosProvider,
  },
  Pagination: make(map[string]int),
  }
 }
 
 // NewServerWithSubsystemListener creates initialized instance of FrontEnd server communicating
 // with provided jsonRPC and externally created SubsystemListener instead default one.
-func NewServerWithSubsystemListener(jsonRPC spdk.JSONRPC, sysListener SubsystemListener) *Server {
+func NewServerWithSubsystemListener(
+ jsonRPC spdk.JSONRPC, qosProvider VirtioBlkQosProvider, sysListener SubsystemListener,
+) *Server {
  if sysListener == nil {
  log.Panic("nil for SubsystemListener is not allowed")
  }
- server := NewServer(jsonRPC)
+ server := NewServer(jsonRPC, qosProvider)
  server.Nvme.subsysListener = sysListener
  return server
 }
+
+// VirtioBlkQosProviderFromMiddleendQosService provides QoS provider based on middleend QoS service
+func VirtioBlkQosProviderFromMiddleendQosService(
+ s pb.MiddleendQosVolumeServiceServer,
+) VirtioBlkQosProvider {
+ if s == nil {
+ log.Panic("nil middleend QoS service is not allowed")
+ }
+ return s
+}

pkg/frontend/frontend_test.go

@@ -47,10 +47,17 @@ func (e *testEnv) Close() {
 }
 
 func createTestEnvironment(startSpdkServer bool, spdkResponses []string) *testEnv {
+ return createTestEnvironmentWithVirtioBlkQosProvider(
+ startSpdkServer, spdkResponses, stubQosProvider{})
+}
+
+func createTestEnvironmentWithVirtioBlkQosProvider(
+ startSpdkServer bool, spdkResponses []string, qosProvider VirtioBlkQosProvider,
+) *testEnv {
  env := &testEnv{}
  env.testSocket = server.GenerateSocketName("frontend")
  env.ln, env.jsonRPC = server.CreateTestSpdkServer(env.testSocket, startSpdkServer, spdkResponses)
- env.opiSpdkServer = NewServer(env.jsonRPC)
+ env.opiSpdkServer = NewServer(env.jsonRPC, qosProvider)
 
  ctx := context.Background()
  conn, err := grpc.DialContext(ctx,

pkg/kvm/blk_test.go

@@ -141,7 +141,7 @@ func TestCreateVirtioBlk(t *testing.T) {
 
  for testName, test := range tests {
  t.Run(testName, func(t *testing.T) {
- opiSpdkServer := frontend.NewServer(test.jsonRPC)
+ opiSpdkServer := frontend.NewServer(test.jsonRPC, stubQosProvider)
  qmpServer := startMockQmpServer(t, test.mockQmpCalls)
  defer qmpServer.Stop()
  qmpAddress := qmpServer.socketPath
@@ -226,7 +226,7 @@ func TestDeleteVirtioBlk(t *testing.T) {
 
  for testName, test := range tests {
  t.Run(testName, func(t *testing.T) {
- opiSpdkServer := frontend.NewServer(test.jsonRPC)
+ opiSpdkServer := frontend.NewServer(test.jsonRPC, stubQosProvider)
  opiSpdkServer.Virt.BlkCtrls[testVirtioBlkID] =
  proto.Clone(testCreateVirtioBlkRequest.VirtioBlk).(*pb.VirtioBlk)
  opiSpdkServer.Virt.BlkCtrls[testVirtioBlkID].Name = testVirtioBlkID

pkg/kvm/kvm_test.go

@@ -18,6 +18,8 @@ import (
  "time"
 
  "github.com/opiproject/gospdk/spdk"
+ pb "github.com/opiproject/opi-api/storage/v1alpha1/gen/go"
+ "github.com/opiproject/opi-spdk-bridge/pkg/frontend"
 )
 
 var (
@@ -32,6 +34,10 @@ var (
  qmplibTimeout = 250 * time.Millisecond
 
  pathRegexpStr = `\/[a-zA-Z\/\-\_0-9]*\/`
+
+ stubQosProvider = frontend.VirtioBlkQosProviderFromMiddleendQosService(
+ &pb.UnimplementedMiddleendQosVolumeServiceServer{},
+ )
 )
 
 type stubJSONRRPC struct {

pkg/kvm/nvme_test.go

@@ -274,7 +274,7 @@ func TestCreateNvmeController(t *testing.T) {
 
  for testName, test := range tests {
  t.Run(testName, func(t *testing.T) {
- opiSpdkServer := frontend.NewServer(test.jsonRPC)
+ opiSpdkServer := frontend.NewServer(test.jsonRPC, stubQosProvider)
  opiSpdkServer.Nvme.Subsystems[testSubsystemID] = &testSubsystem
  qmpServer := startMockQmpServer(t, test.mockQmpCalls)
  defer qmpServer.Stop()
@@ -402,7 +402,7 @@ func TestDeleteNvmeController(t *testing.T) {
 
  for testName, test := range tests {
  t.Run(testName, func(t *testing.T) {
- opiSpdkServer := frontend.NewServer(test.jsonRPC)
+ opiSpdkServer := frontend.NewServer(test.jsonRPC, stubQosProvider)
  opiSpdkServer.Nvme.Subsystems[testSubsystemID] = &testSubsystem
  if !test.noController {
  opiSpdkServer.Nvme.Controllers[testNvmeControllerID] =