Skip to content

Commit

Permalink
Add crypto device example
Browse files Browse the repository at this point in the history
Signed-off-by: Boris Glimcher <[email protected]>
  • Loading branch information
glimchb committed Nov 16, 2022
1 parent 0c4706e commit c8567b6
Show file tree
Hide file tree
Showing 6 changed files with 247 additions and 16 deletions.
30 changes: 15 additions & 15 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -286,6 +286,7 @@ opi_api.storage.v1.AioControllerService
opi_api.storage.v1.FrontendNvmeService
opi_api.storage.v1.FrontendVirtioBlkService
opi_api.storage.v1.FrontendVirtioScsiService
opi_api.storage.v1.MiddleendService
opi_api.storage.v1.NVMfRemoteControllerService
opi_api.storage.v1.NullDebugService
```
Expand Down Expand Up @@ -328,25 +329,24 @@ $ grpc_cli ls opi-spdk-server:50051 opi_api.storage.v1.FrontendNvmeService.Creat
See messages

```bash
$ grpc_cli type opi-spdk-server:50051 opi_api.storage.v1.NVMeController
message NVMeController {
$ grpc_cli type opi-spdk-server:50051 opi_api.storage.v1.NVMeControllerSpec
message NVMeControllerSpec {
.opi_api.common.v1.ObjectKey id = 1 [json_name = "id"];
uint32 nvme_controller_id = 2 [json_name = "nvmeControllerId"];
int32 nvme_controller_id = 2 [json_name = "nvmeControllerId"];
.opi_api.common.v1.ObjectKey subsystem_id = 3 [json_name = "subsystemId"];
.opi_api.storage.v1.NvmeControllerPciId pcie_id = 4 [json_name = "pcieId"];
uint32 max_nsq = 5 [json_name = "maxNsq"];
uint32 max_ncq = 6 [json_name = "maxNcq"];
uint32 sqes = 7 [json_name = "sqes"];
uint32 cqes = 8 [json_name = "cqes"];
uint32 max_ns = 9 [json_name = "maxNs"];
.opi_api.storage.v1.PciEndpoint pcie_id = 4 [json_name = "pcieId"];
int32 max_nsq = 5 [json_name = "maxNsq"];
int32 max_ncq = 6 [json_name = "maxNcq"];
int32 sqes = 7 [json_name = "sqes"];
int32 cqes = 8 [json_name = "cqes"];
int32 max_namespaces = 9 [json_name = "maxNamespaces"];
}

$ grpc_cli type opi-spdk-server:50051 opi_api.storage.v1.NvmeControllerPciId
message NvmeControllerPciId {
uint32 bus = 1 [json_name = "bus"];
uint32 device = 2 [json_name = "device"];
uint32 function = 3 [json_name = "function"];
uint32 virtual_function = 4 [json_name = "virtualFunction"];
$ grpc_cli type opi-spdk-server:50051 opi_api.storage.v1.PciEndpoint
message PciEndpoint {
int32 port_id = 1 [json_name = "portId"];
int32 physical_function = 2 [json_name = "physicalFunction"];
int32 virtual_function = 3 [json_name = "virtualFunction"];
}
```

Expand Down
48 changes: 48 additions & 0 deletions client/middleend.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,57 @@ import (
"context"
"log"

pc "github.com/opiproject/opi-api/common/v1/gen/go"
pb "github.com/opiproject/opi-api/storage/v1alpha1/gen/go"
"google.golang.org/grpc"
)

func doMiddleend(ctx context.Context, conn grpc.ClientConnInterface) {
log.Printf("Test middleend")

// Crypto
c1 := pb.NewMiddleendServiceClient(conn)
log.Printf("Testing NewCryptoServiceClient")
rs1, err := c1.CreateCrypto(ctx, &pb.CreateCryptoRequest{
Volume: &pb.Crypto{
CryptoId: &pc.ObjectKey{Value: "OpiCrypto3"},
VolumeId: &pc.ObjectKey{Value: "Malloc1"},
Key: []byte("0123456789abcdef0123456789abcdef"),
},
})
if err != nil {
log.Fatalf("could not create CRYPTO device: %v", err)
}
log.Printf("Added: %v", rs1)
rs3, err := c1.UpdateCrypto(ctx, &pb.UpdateCryptoRequest{
Volume: &pb.Crypto{
CryptoId: &pc.ObjectKey{Value: "OpiCrypto3"},
VolumeId: &pc.ObjectKey{Value: "Malloc1"},
Key: []byte("0123456789abcdef0123456789abcdef"),
},
})
if err != nil {
log.Fatalf("could not update CRYPTO device: %v", err)
}
log.Printf("Updated: %v", rs3)
rs4, err := c1.ListCrypto(ctx, &pb.ListCryptoRequest{})
if err != nil {
log.Fatalf("could not list CRYPTO device: %v", err)
}
log.Printf("Listed: %v", rs4)
rs5, err := c1.GetCrypto(ctx, &pb.GetCryptoRequest{CryptoId: &pc.ObjectKey{Value: "OpiCrypto3"}})
if err != nil {
log.Fatalf("could not get CRYPTO device: %v", err)
}
log.Printf("Got: %s", rs5.CryptoId.Value)
rs6, err := c1.CryptoStats(ctx, &pb.CryptoStatsRequest{CryptoId: &pc.ObjectKey{Value: "OpiCrypto3"}})
if err != nil {
log.Fatalf("could not stats CRYPTO device: %v", err)
}
log.Printf("Stats: %s", rs6.Stats)
rs2, err := c1.DeleteCrypto(ctx, &pb.DeleteCryptoRequest{CryptoId: &pc.ObjectKey{Value: "OpiCrypto3"}})
if err != nil {
log.Fatalf("could not delete CRYPTO device: %v", err)
}
log.Printf("Deleted: %v", rs2)
}
159 changes: 159 additions & 0 deletions server/middleend.go
Original file line number Diff line number Diff line change
@@ -1,4 +1,163 @@
// SPDX-License-Identifier: Apache-2.0
// Copyright (c) 2022 Dell Inc, or its subsidiaries.

// The main package of the storage server
package main

import (
"context"
"fmt"
"log"

pc "github.com/opiproject/opi-api/common/v1/gen/go"
pb "github.com/opiproject/opi-api/storage/v1alpha1/gen/go"
"github.com/ulule/deepcopier"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
"google.golang.org/protobuf/types/known/emptypb"
)

//////////////////////////////////////////////////////////

func (s *server) CreateCrypto(ctx context.Context, in *pb.CreateCryptoRequest) (*pb.Crypto, error) {
log.Printf("CreateCrypto: Received from client: %v", in)
params := BdevCryptoCreateParams{
Name: in.Volume.CryptoId.Value,
BaseBdevName: in.Volume.VolumeId.Value,
CryptoPmd: "crypto_aesni_mb",
Key: string(in.Volume.Key),
Cipher: "AES_CBC",
}
// TODO: use in.Volume.Cipher.String()
var result BdevCryptoCreateResult
err := call("bdev_crypto_create", &params, &result)
if err != nil {
log.Printf("error: %v", err)
return nil, err
}
log.Printf("Received from SPDK: %v", result)
response := &pb.Crypto{}
err = deepcopier.Copy(in.Volume).To(response)
if err != nil {
log.Printf("error: %v", err)
return nil, err
}
return response, nil
}

func (s *server) DeleteCrypto(ctx context.Context, in *pb.DeleteCryptoRequest) (*emptypb.Empty, error) {
log.Printf("DeleteCrypto: Received from client: %v", in)
params := BdevCryptoDeleteParams{
Name: in.CryptoId.Value,
}
var result BdevCryptoDeleteResult
err := call("bdev_crypto_delete", &params, &result)
if err != nil {
log.Printf("error: %v", err)
return nil, err
}
log.Printf("Received from SPDK: %v", result)
if !result {
log.Printf("Could not delete: %v", in)
}
return &emptypb.Empty{}, nil
}

func (s *server) UpdateCrypto(ctx context.Context, in *pb.UpdateCryptoRequest) (*pb.Crypto, error) {
log.Printf("UpdateCrypto: Received from client: %v", in)
params1 := BdevCryptoDeleteParams{
Name: in.Volume.CryptoId.Value,
}
var result1 BdevCryptoDeleteResult
err1 := call("bdev_crypto_delete", &params1, &result1)
if err1 != nil {
log.Printf("error: %v", err1)
return nil, err1
}
log.Printf("Received from SPDK: %v", result1)
if !result1 {
log.Printf("Could not delete: %v", in)
}
params2 := BdevCryptoCreateParams{
Name: in.Volume.CryptoId.Value,
BaseBdevName: in.Volume.VolumeId.Value,
CryptoPmd: "crypto_aesni_mb",
Key: string(in.Volume.Key),
Cipher: "AES_CBC",
}
// TODO: use in.Volume.Cipher.String()
var result2 BdevCryptoCreateResult
err2 := call("bdev_crypto_create", &params2, &result2)
if err2 != nil {
log.Printf("error: %v", err2)
return nil, err2
}
log.Printf("Received from SPDK: %v", result2)
response := &pb.Crypto{}
err3 := deepcopier.Copy(in.Volume).To(response)
if err3 != nil {
log.Printf("error: %v", err3)
return nil, err3
}
return response, nil
}

func (s *server) ListCrypto(ctx context.Context, in *pb.ListCryptoRequest) (*pb.ListCryptoResponse, error) {
log.Printf("ListCrypto: Received from client: %v", in)
var result []BdevGetBdevsResult
err := call("bdev_get_bdevs", nil, &result)
if err != nil {
log.Printf("error: %v", err)
return nil, err
}
log.Printf("Received from SPDK: %v", result)
Blobarray := make([]*pb.Crypto, len(result))
for i := range result {
r := &result[i]
Blobarray[i] = &pb.Crypto{CryptoId: &pc.ObjectKey{Value: r.Name}}
}
return &pb.ListCryptoResponse{Volumes: Blobarray}, nil
}

func (s *server) GetCrypto(ctx context.Context, in *pb.GetCryptoRequest) (*pb.Crypto, error) {
log.Printf("GetCrypto: Received from client: %v", in)
params := BdevGetBdevsParams{
Name: in.CryptoId.Value,
}
var result []BdevGetBdevsResult
err := call("bdev_get_bdevs", &params, &result)
if err != nil {
log.Printf("error: %v", err)
return nil, err
}
log.Printf("Received from SPDK: %v", result)
if len(result) != 1 {
msg := fmt.Sprintf("expecting exactly 1 result, got %d", len(result))
log.Print(msg)
return nil, status.Errorf(codes.InvalidArgument, msg)
}
return &pb.Crypto{CryptoId: &pc.ObjectKey{Value: result[0].Name}}, nil
}

func (s *server) CryptoStats(ctx context.Context, in *pb.CryptoStatsRequest) (*pb.CryptoStatsResponse, error) {
log.Printf("CryptoStats: Received from client: %v", in)
params := BdevGetIostatParams{
Name: in.CryptoId.Value,
}
// See https://mholt.github.io/json-to-go/
var result BdevGetIostatResult
err := call("bdev_get_iostat", &params, &result)
if err != nil {
log.Printf("error: %v", err)
return nil, err
}
log.Printf("Received from SPDK: %v", result)
if len(result.Bdevs) != 1 {
msg := fmt.Sprintf("expecting exactly 1 result, got %d", len(result.Bdevs))
log.Print(msg)
return nil, status.Errorf(codes.InvalidArgument, msg)
}
return &pb.CryptoStatsResponse{Stats: fmt.Sprint(result.Bdevs[0])}, nil
}

//////////////////////////////////////////////////////////
2 changes: 2 additions & 0 deletions server/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ type server struct {
pb.UnimplementedFrontendVirtioScsiServiceServer
pb.UnimplementedNullDebugServiceServer
pb.UnimplementedAioControllerServiceServer
pb.UnimplementedMiddleendServiceServer
}

func main() {
Expand All @@ -41,6 +42,7 @@ func main() {
pb.RegisterFrontendVirtioScsiServiceServer(s, &server{})
pb.RegisterNullDebugServiceServer(s, &server{})
pb.RegisterAioControllerServiceServer(s, &server{})
pb.RegisterMiddleendServiceServer(s, &server{})

reflection.Register(s)

Expand Down
22 changes: 22 additions & 0 deletions server/spdk.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@ package main
// bdev_malloc_delete
// bdev_null_create
// bdev_null_delete
// bdev_crypto_create
// bdev_crypto_delete
// bdev_aio_create
// bdev_aio_delete
// bdev_nvme_attach_controller
Expand Down Expand Up @@ -80,6 +82,26 @@ type BdevNullDeleteParams struct {
// BdevNullDeleteResult is the result of deleting a Null Block Device
type BdevNullDeleteResult bool

// BdevCryptoCreateParams holds the parameters required to create a Crypto Block Device
type BdevCryptoCreateParams struct {
BaseBdevName string `json:"base_bdev_name"`
Name string `json:"name"`
CryptoPmd string `json:"crypto_pmd"`
Key string `json:"key"`
Cipher string `json:"cipher"`
}

// BdevCryptoCreateResult is the result of creating a Crypto Block Device
type BdevCryptoCreateResult string

// BdevCryptoDeleteParams holds the parameters required to delete a Crypto Block Device
type BdevCryptoDeleteParams struct {
Name string `json:"name"`
}

// BdevCryptoDeleteResult is the result of deleting a Crypto Block Device
type BdevCryptoDeleteResult bool

// BdevNvmeAttachControllerParams is the parameters required to create a block device based on an NVMe device
type BdevNvmeAttachControllerParams struct {
Name string `json:"name"`
Expand Down
2 changes: 1 addition & 1 deletion spdk/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ RUN git clone https://github.com/spdk/spdk --branch ${TAG} --depth 1 && \
cd spdk && git submodule update --init --depth 1 && scripts/pkgdep.sh --rdma

# hadolint ignore=DL3003
RUN cd spdk && ./rpmbuild/rpm.sh --target-arch=${ARCH} --without-uring --without-crypto \
RUN cd spdk && ./rpmbuild/rpm.sh --target-arch=${ARCH} --without-uring --with-crypto \
--without-fio --with-raid5f --with-vhost --without-pmdk --without-rbd \
--with-rdma --with-shared --with-iscsi-initiator --without-vtune

Expand Down

0 comments on commit c8567b6

Please sign in to comment.