From 0b608cface28b14329740b16b7c90aa0b5f36499 Mon Sep 17 00:00:00 2001 From: Artsiom Koltun Date: Thu, 1 Jun 2023 09:40:40 +0200 Subject: [PATCH] Add interface to provide virtio-blk QoS capabilities. Signed-off-by: Artsiom Koltun --- cmd/main.go | 14 ++++++++++---- pkg/frontend/blk.go | 10 ++++++++++ pkg/frontend/blk_test.go | 13 +++++++++++++ pkg/frontend/frontend.go | 22 +++++++++++++++++++--- pkg/frontend/frontend_test.go | 9 ++++++++- pkg/kvm/blk_test.go | 4 ++-- pkg/kvm/kvm_test.go | 6 ++++++ pkg/kvm/nvme_test.go | 4 ++-- 8 files changed, 70 insertions(+), 12 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index 7409fc49..421ae007 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -66,16 +66,22 @@ func main() { if useKvm { log.Println("Creating KVM server.") - frontendServer := frontend.NewServerWithSubsystemListener(jsonRPC, - kvm.NewVfiouserSubsystemListener(ctrlrDir)) + frontendServer := frontend.NewServerWithSubsystemListener( + jsonRPC, + frontend.VirtioBlkQosProviderFromMiddleendQosService(middleendServer), + kvm.NewVfiouserSubsystemListener(ctrlrDir), + ) kvmServer := kvm.NewServer(frontendServer, qmpAddress, ctrlrDir, buses) pb.RegisterFrontendNvmeServiceServer(s, kvmServer) pb.RegisterFrontendVirtioBlkServiceServer(s, kvmServer) pb.RegisterFrontendVirtioScsiServiceServer(s, kvmServer) } else { - frontendServer := frontend.NewServerWithSubsystemListener(jsonRPC, - frontend.NewTCPSubsystemListener(tcpTransportListenAddr)) + frontendServer := frontend.NewServerWithSubsystemListener( + jsonRPC, + frontend.VirtioBlkQosProviderFromMiddleendQosService(middleendServer), + frontend.NewTCPSubsystemListener(tcpTransportListenAddr), + ) pb.RegisterFrontendNvmeServiceServer(s, frontendServer) pb.RegisterFrontendVirtioBlkServiceServer(s, frontendServer) pb.RegisterFrontendVirtioScsiServiceServer(s, frontendServer) diff --git a/pkg/frontend/blk.go b/pkg/frontend/blk.go index 6e4d7856..9e71556e 100644 --- a/pkg/frontend/blk.go +++ b/pkg/frontend/blk.go @@ -23,6 +23,16 @@ import ( "google.golang.org/protobuf/types/known/emptypb" ) +// VirtioBlkQosProvider provides QoS capabilities for virtio-blk +// At the moment it is just a couple of methods from middleend QoS service, but +// it can be changed to less verbose later. +// If client uses VirtioBlkQosProviderFromMiddleendQosService to create an instance, +// the interface can be changed without affecting the client code. +type VirtioBlkQosProvider interface { + CreateQosVolume(context.Context, *pb.CreateQosVolumeRequest) (*pb.QosVolume, error) + DeleteQosVolume(context.Context, *pb.DeleteQosVolumeRequest) (*emptypb.Empty, error) +} + func sortVirtioBlks(virtioBlks []*pb.VirtioBlk) { sort.Slice(virtioBlks, func(i int, j int) bool { return virtioBlks[i].Name < virtioBlks[j].Name diff --git a/pkg/frontend/blk_test.go b/pkg/frontend/blk_test.go index e00a3528..e7d57857 100644 --- a/pkg/frontend/blk_test.go +++ b/pkg/frontend/blk_test.go @@ -7,6 +7,7 @@ package frontend import ( "bytes" + "context" "fmt" "reflect" "strings" @@ -22,6 +23,18 @@ import ( pb "github.com/opiproject/opi-api/storage/v1alpha1/gen/go" ) +type stubQosProvider struct { + err error +} + +func (p stubQosProvider) CreateQosVolume(context.Context, *pb.CreateQosVolumeRequest) (*pb.QosVolume, error) { + return &pb.QosVolume{}, p.err +} + +func (p stubQosProvider) DeleteQosVolume(context.Context, *pb.DeleteQosVolumeRequest) (*emptypb.Empty, error) { + return &emptypb.Empty{}, p.err +} + var ( testVirtioCtrlID = "virtio-blk-42" testVirtioCtrl = pb.VirtioBlk{ diff --git a/pkg/frontend/frontend.go b/pkg/frontend/frontend.go index 8ec852dc..64e560e7 100644 --- a/pkg/frontend/frontend.go +++ b/pkg/frontend/frontend.go @@ -31,6 +31,8 @@ type VirtioParameters struct { BlkCtrls map[string]*pb.VirtioBlk ScsiCtrls map[string]*pb.VirtioScsiController ScsiLuns map[string]*pb.VirtioScsiLun + + qosProvider VirtioBlkQosProvider } // Server contains frontend related OPI services @@ -47,7 +49,7 @@ type Server struct { // NewServer creates initialized instance of FrontEnd server communicating // with provided jsonRPC -func NewServer(jsonRPC spdk.JSONRPC) *Server { +func NewServer(jsonRPC spdk.JSONRPC, qosProvider VirtioBlkQosProvider) *Server { return &Server{ rpc: jsonRPC, Nvme: NvmeParameters{ @@ -60,6 +62,8 @@ func NewServer(jsonRPC spdk.JSONRPC) *Server { BlkCtrls: make(map[string]*pb.VirtioBlk), ScsiCtrls: make(map[string]*pb.VirtioScsiController), ScsiLuns: make(map[string]*pb.VirtioScsiLun), + + qosProvider: qosProvider, }, Pagination: make(map[string]int), } @@ -67,11 +71,23 @@ func NewServer(jsonRPC spdk.JSONRPC) *Server { // NewServerWithSubsystemListener creates initialized instance of FrontEnd server communicating // with provided jsonRPC and externally created SubsystemListener instead default one. -func NewServerWithSubsystemListener(jsonRPC spdk.JSONRPC, sysListener SubsystemListener) *Server { +func NewServerWithSubsystemListener( + jsonRPC spdk.JSONRPC, qosProvider VirtioBlkQosProvider, sysListener SubsystemListener, +) *Server { if sysListener == nil { log.Panic("nil for SubsystemListener is not allowed") } - server := NewServer(jsonRPC) + server := NewServer(jsonRPC, qosProvider) server.Nvme.subsysListener = sysListener return server } + +// VirtioBlkQosProviderFromMiddleendQosService provides QoS provider based on middleend QoS service +func VirtioBlkQosProviderFromMiddleendQosService( + s pb.MiddleendQosVolumeServiceServer, +) VirtioBlkQosProvider { + if s == nil { + log.Panic("nil middleend QoS service is not allowed") + } + return s +} diff --git a/pkg/frontend/frontend_test.go b/pkg/frontend/frontend_test.go index 2ed80845..2defdc2c 100644 --- a/pkg/frontend/frontend_test.go +++ b/pkg/frontend/frontend_test.go @@ -47,10 +47,17 @@ func (e *testEnv) Close() { } func createTestEnvironment(startSpdkServer bool, spdkResponses []string) *testEnv { + return createTestEnvironmentWithVirtioBlkQosProvider( + startSpdkServer, spdkResponses, stubQosProvider{}) +} + +func createTestEnvironmentWithVirtioBlkQosProvider( + startSpdkServer bool, spdkResponses []string, qosProvider VirtioBlkQosProvider, +) *testEnv { env := &testEnv{} env.testSocket = server.GenerateSocketName("frontend") env.ln, env.jsonRPC = server.CreateTestSpdkServer(env.testSocket, startSpdkServer, spdkResponses) - env.opiSpdkServer = NewServer(env.jsonRPC) + env.opiSpdkServer = NewServer(env.jsonRPC, qosProvider) ctx := context.Background() conn, err := grpc.DialContext(ctx, diff --git a/pkg/kvm/blk_test.go b/pkg/kvm/blk_test.go index 3ec6abe1..722fac8a 100644 --- a/pkg/kvm/blk_test.go +++ b/pkg/kvm/blk_test.go @@ -141,7 +141,7 @@ func TestCreateVirtioBlk(t *testing.T) { for testName, test := range tests { t.Run(testName, func(t *testing.T) { - opiSpdkServer := frontend.NewServer(test.jsonRPC) + opiSpdkServer := frontend.NewServer(test.jsonRPC, stubQosProvider) qmpServer := startMockQmpServer(t, test.mockQmpCalls) defer qmpServer.Stop() qmpAddress := qmpServer.socketPath @@ -226,7 +226,7 @@ func TestDeleteVirtioBlk(t *testing.T) { for testName, test := range tests { t.Run(testName, func(t *testing.T) { - opiSpdkServer := frontend.NewServer(test.jsonRPC) + opiSpdkServer := frontend.NewServer(test.jsonRPC, stubQosProvider) opiSpdkServer.Virt.BlkCtrls[testVirtioBlkID] = proto.Clone(testCreateVirtioBlkRequest.VirtioBlk).(*pb.VirtioBlk) opiSpdkServer.Virt.BlkCtrls[testVirtioBlkID].Name = testVirtioBlkID diff --git a/pkg/kvm/kvm_test.go b/pkg/kvm/kvm_test.go index 039ea97d..72846c84 100644 --- a/pkg/kvm/kvm_test.go +++ b/pkg/kvm/kvm_test.go @@ -18,6 +18,8 @@ import ( "time" "github.com/opiproject/gospdk/spdk" + pb "github.com/opiproject/opi-api/storage/v1alpha1/gen/go" + "github.com/opiproject/opi-spdk-bridge/pkg/frontend" ) var ( @@ -32,6 +34,10 @@ var ( qmplibTimeout = 250 * time.Millisecond pathRegexpStr = `\/[a-zA-Z\/\-\_0-9]*\/` + + stubQosProvider = frontend.VirtioBlkQosProviderFromMiddleendQosService( + &pb.UnimplementedMiddleendQosVolumeServiceServer{}, + ) ) type stubJSONRRPC struct { diff --git a/pkg/kvm/nvme_test.go b/pkg/kvm/nvme_test.go index 520a6c6b..d50804d1 100644 --- a/pkg/kvm/nvme_test.go +++ b/pkg/kvm/nvme_test.go @@ -274,7 +274,7 @@ func TestCreateNvmeController(t *testing.T) { for testName, test := range tests { t.Run(testName, func(t *testing.T) { - opiSpdkServer := frontend.NewServer(test.jsonRPC) + opiSpdkServer := frontend.NewServer(test.jsonRPC, stubQosProvider) opiSpdkServer.Nvme.Subsystems[testSubsystemID] = &testSubsystem qmpServer := startMockQmpServer(t, test.mockQmpCalls) defer qmpServer.Stop() @@ -402,7 +402,7 @@ func TestDeleteNvmeController(t *testing.T) { for testName, test := range tests { t.Run(testName, func(t *testing.T) { - opiSpdkServer := frontend.NewServer(test.jsonRPC) + opiSpdkServer := frontend.NewServer(test.jsonRPC, stubQosProvider) opiSpdkServer.Nvme.Subsystems[testSubsystemID] = &testSubsystem if !test.noController { opiSpdkServer.Nvme.Controllers[testNvmeControllerID] =