diff --git a/README.md b/README.md index 30444447..90eb08e9 100644 --- a/README.md +++ b/README.md @@ -286,6 +286,7 @@ opi_api.storage.v1.AioControllerService opi_api.storage.v1.FrontendNvmeService opi_api.storage.v1.FrontendVirtioBlkService opi_api.storage.v1.FrontendVirtioScsiService +opi_api.storage.v1.MiddleendService opi_api.storage.v1.NVMfRemoteControllerService opi_api.storage.v1.NullDebugService ``` @@ -328,25 +329,24 @@ $ grpc_cli ls opi-spdk-server:50051 opi_api.storage.v1.FrontendNvmeService.Creat See messages ```bash -$ grpc_cli type opi-spdk-server:50051 opi_api.storage.v1.NVMeController -message NVMeController { +$ grpc_cli type opi-spdk-server:50051 opi_api.storage.v1.NVMeControllerSpec +message NVMeControllerSpec { .opi_api.common.v1.ObjectKey id = 1 [json_name = "id"]; - uint32 nvme_controller_id = 2 [json_name = "nvmeControllerId"]; + int32 nvme_controller_id = 2 [json_name = "nvmeControllerId"]; .opi_api.common.v1.ObjectKey subsystem_id = 3 [json_name = "subsystemId"]; - .opi_api.storage.v1.NvmeControllerPciId pcie_id = 4 [json_name = "pcieId"]; - uint32 max_nsq = 5 [json_name = "maxNsq"]; - uint32 max_ncq = 6 [json_name = "maxNcq"]; - uint32 sqes = 7 [json_name = "sqes"]; - uint32 cqes = 8 [json_name = "cqes"]; - uint32 max_ns = 9 [json_name = "maxNs"]; + .opi_api.storage.v1.PciEndpoint pcie_id = 4 [json_name = "pcieId"]; + int32 max_nsq = 5 [json_name = "maxNsq"]; + int32 max_ncq = 6 [json_name = "maxNcq"]; + int32 sqes = 7 [json_name = "sqes"]; + int32 cqes = 8 [json_name = "cqes"]; + int32 max_namespaces = 9 [json_name = "maxNamespaces"]; } -$ grpc_cli type opi-spdk-server:50051 opi_api.storage.v1.NvmeControllerPciId -message NvmeControllerPciId { - uint32 bus = 1 [json_name = "bus"]; - uint32 device = 2 [json_name = "device"]; - uint32 function = 3 [json_name = "function"]; - uint32 virtual_function = 4 [json_name = "virtualFunction"]; +$ grpc_cli type opi-spdk-server:50051 opi_api.storage.v1.PciEndpoint +message PciEndpoint { + int32 port_id = 1 [json_name = "portId"]; + int32 physical_function = 2 [json_name = "physicalFunction"]; + int32 virtual_function = 3 [json_name = "virtualFunction"]; } ``` diff --git a/client/middleend.go b/client/middleend.go index 3066753d..11475346 100644 --- a/client/middleend.go +++ b/client/middleend.go @@ -4,9 +4,57 @@ import ( "context" "log" + pc "github.com/opiproject/opi-api/common/v1/gen/go" + pb "github.com/opiproject/opi-api/storage/v1alpha1/gen/go" "google.golang.org/grpc" ) func doMiddleend(ctx context.Context, conn grpc.ClientConnInterface) { log.Printf("Test middleend") + + // Crypto + c1 := pb.NewMiddleendServiceClient(conn) + log.Printf("Testing NewCryptoServiceClient") + rs1, err := c1.CreateCrypto(ctx, &pb.CreateCryptoRequest{ + Volume: &pb.Crypto{ + CryptoId: &pc.ObjectKey{Value: "OpiCrypto3"}, + VolumeId: &pc.ObjectKey{Value: "Malloc1"}, + Key: []byte("0123456789abcdef0123456789abcdef"), + }, + }) + if err != nil { + log.Fatalf("could not create CRYPTO device: %v", err) + } + log.Printf("Added: %v", rs1) + rs3, err := c1.UpdateCrypto(ctx, &pb.UpdateCryptoRequest{ + Volume: &pb.Crypto{ + CryptoId: &pc.ObjectKey{Value: "OpiCrypto3"}, + VolumeId: &pc.ObjectKey{Value: "Malloc1"}, + Key: []byte("0123456789abcdef0123456789abcdef"), + }, + }) + if err != nil { + log.Fatalf("could not update CRYPTO device: %v", err) + } + log.Printf("Updated: %v", rs3) + rs4, err := c1.ListCrypto(ctx, &pb.ListCryptoRequest{}) + if err != nil { + log.Fatalf("could not list CRYPTO device: %v", err) + } + log.Printf("Listed: %v", rs4) + rs5, err := c1.GetCrypto(ctx, &pb.GetCryptoRequest{CryptoId: &pc.ObjectKey{Value: "OpiCrypto3"}}) + if err != nil { + log.Fatalf("could not get CRYPTO device: %v", err) + } + log.Printf("Got: %s", rs5.CryptoId.Value) + rs6, err := c1.CryptoStats(ctx, &pb.CryptoStatsRequest{CryptoId: &pc.ObjectKey{Value: "OpiCrypto3"}}) + if err != nil { + log.Fatalf("could not stats CRYPTO device: %v", err) + } + log.Printf("Stats: %s", rs6.Stats) + rs2, err := c1.DeleteCrypto(ctx, &pb.DeleteCryptoRequest{CryptoId: &pc.ObjectKey{Value: "OpiCrypto3"}}) + if err != nil { + log.Fatalf("could not delete CRYPTO device: %v", err) + } + log.Printf("Deleted: %v", rs2) } diff --git a/server/middleend.go b/server/middleend.go index 242ee59a..b35021d9 100644 --- a/server/middleend.go +++ b/server/middleend.go @@ -1,4 +1,163 @@ // SPDX-License-Identifier: Apache-2.0 // Copyright (c) 2022 Dell Inc, or its subsidiaries. +// The main package of the storage server package main + +import ( + "context" + "fmt" + "log" + + pc "github.com/opiproject/opi-api/common/v1/gen/go" + pb "github.com/opiproject/opi-api/storage/v1alpha1/gen/go" + "github.com/ulule/deepcopier" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/emptypb" +) + +////////////////////////////////////////////////////////// + +func (s *server) CreateCrypto(ctx context.Context, in *pb.CreateCryptoRequest) (*pb.Crypto, error) { + log.Printf("CreateCrypto: Received from client: %v", in) + params := BdevCryptoCreateParams{ + Name: in.Volume.CryptoId.Value, + BaseBdevName: in.Volume.VolumeId.Value, + CryptoPmd: "crypto_aesni_mb", + Key: string(in.Volume.Key), + Cipher: "AES_CBC", + } + // TODO: use in.Volume.Cipher.String() + var result BdevCryptoCreateResult + err := call("bdev_crypto_create", ¶ms, &result) + if err != nil { + log.Printf("error: %v", err) + return nil, err + } + log.Printf("Received from SPDK: %v", result) + response := &pb.Crypto{} + err = deepcopier.Copy(in.Volume).To(response) + if err != nil { + log.Printf("error: %v", err) + return nil, err + } + return response, nil +} + +func (s *server) DeleteCrypto(ctx context.Context, in *pb.DeleteCryptoRequest) (*emptypb.Empty, error) { + log.Printf("DeleteCrypto: Received from client: %v", in) + params := BdevCryptoDeleteParams{ + Name: in.CryptoId.Value, + } + var result BdevCryptoDeleteResult + err := call("bdev_crypto_delete", ¶ms, &result) + if err != nil { + log.Printf("error: %v", err) + return nil, err + } + log.Printf("Received from SPDK: %v", result) + if !result { + log.Printf("Could not delete: %v", in) + } + return &emptypb.Empty{}, nil +} + +func (s *server) UpdateCrypto(ctx context.Context, in *pb.UpdateCryptoRequest) (*pb.Crypto, error) { + log.Printf("UpdateCrypto: Received from client: %v", in) + params1 := BdevCryptoDeleteParams{ + Name: in.Volume.CryptoId.Value, + } + var result1 BdevCryptoDeleteResult + err1 := call("bdev_crypto_delete", ¶ms1, &result1) + if err1 != nil { + log.Printf("error: %v", err1) + return nil, err1 + } + log.Printf("Received from SPDK: %v", result1) + if !result1 { + log.Printf("Could not delete: %v", in) + } + params2 := BdevCryptoCreateParams{ + Name: in.Volume.CryptoId.Value, + BaseBdevName: in.Volume.VolumeId.Value, + CryptoPmd: "crypto_aesni_mb", + Key: string(in.Volume.Key), + Cipher: "AES_CBC", + } + // TODO: use in.Volume.Cipher.String() + var result2 BdevCryptoCreateResult + err2 := call("bdev_crypto_create", ¶ms2, &result2) + if err2 != nil { + log.Printf("error: %v", err2) + return nil, err2 + } + log.Printf("Received from SPDK: %v", result2) + response := &pb.Crypto{} + err3 := deepcopier.Copy(in.Volume).To(response) + if err3 != nil { + log.Printf("error: %v", err3) + return nil, err3 + } + return response, nil +} + +func (s *server) ListCrypto(ctx context.Context, in *pb.ListCryptoRequest) (*pb.ListCryptoResponse, error) { + log.Printf("ListCrypto: Received from client: %v", in) + var result []BdevGetBdevsResult + err := call("bdev_get_bdevs", nil, &result) + if err != nil { + log.Printf("error: %v", err) + return nil, err + } + log.Printf("Received from SPDK: %v", result) + Blobarray := make([]*pb.Crypto, len(result)) + for i := range result { + r := &result[i] + Blobarray[i] = &pb.Crypto{CryptoId: &pc.ObjectKey{Value: r.Name}} + } + return &pb.ListCryptoResponse{Volumes: Blobarray}, nil +} + +func (s *server) GetCrypto(ctx context.Context, in *pb.GetCryptoRequest) (*pb.Crypto, error) { + log.Printf("GetCrypto: Received from client: %v", in) + params := BdevGetBdevsParams{ + Name: in.CryptoId.Value, + } + var result []BdevGetBdevsResult + err := call("bdev_get_bdevs", ¶ms, &result) + if err != nil { + log.Printf("error: %v", err) + return nil, err + } + log.Printf("Received from SPDK: %v", result) + if len(result) != 1 { + msg := fmt.Sprintf("expecting exactly 1 result, got %d", len(result)) + log.Print(msg) + return nil, status.Errorf(codes.InvalidArgument, msg) + } + return &pb.Crypto{CryptoId: &pc.ObjectKey{Value: result[0].Name}}, nil +} + +func (s *server) CryptoStats(ctx context.Context, in *pb.CryptoStatsRequest) (*pb.CryptoStatsResponse, error) { + log.Printf("CryptoStats: Received from client: %v", in) + params := BdevGetIostatParams{ + Name: in.CryptoId.Value, + } + // See https://mholt.github.io/json-to-go/ + var result BdevGetIostatResult + err := call("bdev_get_iostat", ¶ms, &result) + if err != nil { + log.Printf("error: %v", err) + return nil, err + } + log.Printf("Received from SPDK: %v", result) + if len(result.Bdevs) != 1 { + msg := fmt.Sprintf("expecting exactly 1 result, got %d", len(result.Bdevs)) + log.Print(msg) + return nil, status.Errorf(codes.InvalidArgument, msg) + } + return &pb.CryptoStatsResponse{Stats: fmt.Sprint(result.Bdevs[0])}, nil +} + +////////////////////////////////////////////////////////// diff --git a/server/server.go b/server/server.go index e6ee7a7f..b00a655a 100644 --- a/server/server.go +++ b/server/server.go @@ -25,6 +25,7 @@ type server struct { pb.UnimplementedFrontendVirtioScsiServiceServer pb.UnimplementedNullDebugServiceServer pb.UnimplementedAioControllerServiceServer + pb.UnimplementedMiddleendServiceServer } func main() { @@ -41,6 +42,7 @@ func main() { pb.RegisterFrontendVirtioScsiServiceServer(s, &server{}) pb.RegisterNullDebugServiceServer(s, &server{}) pb.RegisterAioControllerServiceServer(s, &server{}) + pb.RegisterMiddleendServiceServer(s, &server{}) reflection.Register(s) diff --git a/server/spdk.go b/server/spdk.go index 07419ca5..17945e0b 100644 --- a/server/spdk.go +++ b/server/spdk.go @@ -10,6 +10,8 @@ package main // bdev_malloc_delete // bdev_null_create // bdev_null_delete +// bdev_crypto_create +// bdev_crypto_delete // bdev_aio_create // bdev_aio_delete // bdev_nvme_attach_controller @@ -80,6 +82,26 @@ type BdevNullDeleteParams struct { // BdevNullDeleteResult is the result of deleting a Null Block Device type BdevNullDeleteResult bool +// BdevCryptoCreateParams holds the parameters required to create a Crypto Block Device +type BdevCryptoCreateParams struct { + BaseBdevName string `json:"base_bdev_name"` + Name string `json:"name"` + CryptoPmd string `json:"crypto_pmd"` + Key string `json:"key"` + Cipher string `json:"cipher"` +} + +// BdevCryptoCreateResult is the result of creating a Crypto Block Device +type BdevCryptoCreateResult string + +// BdevCryptoDeleteParams holds the parameters required to delete a Crypto Block Device +type BdevCryptoDeleteParams struct { + Name string `json:"name"` +} + +// BdevCryptoDeleteResult is the result of deleting a Crypto Block Device +type BdevCryptoDeleteResult bool + // BdevNvmeAttachControllerParams is the parameters required to create a block device based on an NVMe device type BdevNvmeAttachControllerParams struct { Name string `json:"name"` diff --git a/spdk/Dockerfile b/spdk/Dockerfile index c1ac78c6..5c943db5 100644 --- a/spdk/Dockerfile +++ b/spdk/Dockerfile @@ -16,7 +16,7 @@ RUN git clone https://github.com/spdk/spdk --branch ${TAG} --depth 1 && \ cd spdk && git submodule update --init --depth 1 && scripts/pkgdep.sh --rdma # hadolint ignore=DL3003 -RUN cd spdk && ./rpmbuild/rpm.sh --target-arch=${ARCH} --without-uring --without-crypto \ +RUN cd spdk && ./rpmbuild/rpm.sh --target-arch=${ARCH} --without-uring --with-crypto \ --without-fio --with-raid5f --with-vhost --without-pmdk --without-rbd \ --with-rdma --with-shared --with-iscsi-initiator --without-vtune