List: add pagination support

go.mod

@@ -4,6 +4,7 @@ go 1.19
 
 require (
 	github.com/digitalocean/go-qemu v0.0.0-20221209210016-f035778c97f7
+	github.com/google/uuid v1.3.0
 	github.com/opiproject/opi-api v0.0.0-20230404182329-b6f178ec8cfa
 	github.com/ulule/deepcopier v0.0.0-20200430083143-45decc6639b6
 	google.golang.org/grpc v1.54.0

go.sum

@@ -8,6 +8,8 @@ github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw
 github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
+github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
+github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/opiproject/opi-api v0.0.0-20230111150933-e4b3480e8ee9 h1:Y/0Ku5yIfnLEnLa9jzYCK/7l85CHQVefs0PnaXFX+v0=
 github.com/opiproject/opi-api v0.0.0-20230111150933-e4b3480e8ee9/go.mod h1:92pv4ulvvPMuxCJ9ND3aYbmBfEMLx0VCjpkiR7ZTqPY=
 github.com/opiproject/opi-api v0.0.0-20230123165122-10e47bafd42b h1:Ho6qkBoU1vfqERZbih+Hmn7Y1lsx14YtOiqMl4lzdF4=

pkg/backend/aio.go

@@ -13,7 +13,9 @@ import (
 	pc "github.com/opiproject/opi-api/common/v1/gen/go"
 	pb "github.com/opiproject/opi-api/storage/v1alpha1/gen/go"
 	"github.com/opiproject/opi-spdk-bridge/pkg/models"
+	"github.com/opiproject/opi-spdk-bridge/pkg/server"
 
+	"github.com/google/uuid"
 	"github.com/ulule/deepcopier"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
@@ -136,10 +138,10 @@ func (s *Server) UpdateAioController(_ context.Context, in *pb.UpdateAioControll
 // ListAioControllers lists Aio controllers
 func (s *Server) ListAioControllers(_ context.Context, in *pb.ListAioControllersRequest) (*pb.ListAioControllersResponse, error) {
 	log.Printf("ListAioControllers: Received from client: %v", in)
-	if in.PageSize < 0 {
-		err := status.Error(codes.InvalidArgument, "negative PageSize is not allowed")
-		log.Printf("error: %v", err)
-		return nil, err
+	size, offset, perr := server.ExtractPagination(in.PageSize, in.PageToken, s.Pagination)
+	if perr != nil {
+		log.Printf("error: %v", perr)
+		return nil, perr
 	}
 	var result []models.BdevGetBdevsResult
 	err := s.rpc.Call("bdev_get_bdevs", nil, &result)
@@ -148,16 +150,19 @@ func (s *Server) ListAioControllers(_ context.Context, in *pb.ListAioControllers
 		return nil, err
 	}
 	log.Printf("Received from SPDK: %v", result)
-	if in.PageSize > 0 && int(in.PageSize) < len(result) {
-		log.Printf("Limiting result to: %d", in.PageSize)
-		result = result[:in.PageSize]
+	token := ""
+	log.Printf("Limiting result len(%d) to [%d:%d]", len(result), offset, size)
+	result, hasMoreElements := server.LimitPagination(result, offset, size)
+	if hasMoreElements {
+		token = uuid.New().String()
+		s.Pagination[token] = offset + size
 	}
 	Blobarray := make([]*pb.AioController, len(result))
 	for i := range result {
 		r := &result[i]
 		Blobarray[i] = &pb.AioController{Handle: &pc.ObjectKey{Value: r.Name}, BlockSize: r.BlockSize, BlocksCount: r.NumBlocks}
 	}
-	return &pb.ListAioControllersResponse{AioControllers: Blobarray}, nil
+	return &pb.ListAioControllersResponse{AioControllers: Blobarray, NextPageToken: token}, nil
 }
 
 // GetAioController gets an Aio controller

pkg/backend/aio_test.go

@@ -238,6 +238,7 @@ func TestBackEnd_ListAioControllers(t *testing.T) {
 		errMsg  string
 		start   bool
 		size    int32
+		token   string
 	}{
 		"valid request with invalid SPDK response": {
 			"volume-test",
@@ -247,6 +248,7 @@ func TestBackEnd_ListAioControllers(t *testing.T) {
 			fmt.Sprintf("Could not find any namespaces for NQN: %v", "nqn.2022-09.io.spdk:opi3"),
 			true,
 			0,
+			"",
 		},
 		"valid request with invalid marshal SPDK response": {
 			"volume-test",
@@ -256,6 +258,7 @@ func TestBackEnd_ListAioControllers(t *testing.T) {
 			fmt.Sprintf("bdev_get_bdevs: %v", "json: cannot unmarshal bool into Go value of type []models.BdevGetBdevsResult"),
 			true,
 			0,
+			"",
 		},
 		"valid request with empty SPDK response": {
 			"volume-test",
@@ -265,6 +268,7 @@ func TestBackEnd_ListAioControllers(t *testing.T) {
 			fmt.Sprintf("bdev_get_bdevs: %v", "EOF"),
 			true,
 			0,
+			"",
 		},
 		"valid request with ID mismatch SPDK response": {
 			"volume-test",
@@ -274,6 +278,7 @@ func TestBackEnd_ListAioControllers(t *testing.T) {
 			fmt.Sprintf("bdev_get_bdevs: %v", "json response ID mismatch"),
 			true,
 			0,
+			"",
 		},
 		"valid request with error code from SPDK response": {
 			"volume-test",
@@ -283,6 +288,7 @@ func TestBackEnd_ListAioControllers(t *testing.T) {
 			fmt.Sprintf("bdev_get_bdevs: %v", "json response error: myopierr"),
 			true,
 			0,
+			"",
 		},
 		"valid request with valid SPDK response": {
 			"volume-test",
@@ -303,6 +309,7 @@ func TestBackEnd_ListAioControllers(t *testing.T) {
 			"",
 			true,
 			0,
+			"",
 		},
 		"pagination overflow": {
 			"volume-test",
@@ -323,6 +330,7 @@ func TestBackEnd_ListAioControllers(t *testing.T) {
 			"",
 			true,
 			1000,
+			"",
 		},
 		"pagination negative": {
 			"volume-test",
@@ -332,6 +340,17 @@ func TestBackEnd_ListAioControllers(t *testing.T) {
 			"negative PageSize is not allowed",
 			false,
 			-10,
+			"",
+		},
+		"pagination error": {
+			"volume-test",
+			nil,
+			[]string{},
+			codes.NotFound,
+			fmt.Sprintf("unable to find pagination token %s", "unknown-pagination-token"),
+			false,
+			0,
+			"unknown-pagination-token",
 		},
 		"pagination": {
 			"volume-test",
@@ -347,6 +366,7 @@ func TestBackEnd_ListAioControllers(t *testing.T) {
 			"",
 			true,
 			1,
+			"",
 		},
 	}
 
@@ -356,12 +376,18 @@ func TestBackEnd_ListAioControllers(t *testing.T) {
 			testEnv := createTestEnvironment(tt.start, tt.spdk)
 			defer testEnv.Close()
 
-			request := &pb.ListAioControllersRequest{Parent: tt.in, PageSize: tt.size}
+			testEnv.opiSpdkServer.Pagination["existing-pagination-token"] = 1
+
+			request := &pb.ListAioControllersRequest{Parent: tt.in, PageSize: tt.size, PageToken: tt.token}
 			response, err := testEnv.client.ListAioControllers(testEnv.ctx, request)
 			if response != nil {
 				if !reflect.DeepEqual(response.AioControllers, tt.out) {
 					t.Error("response: expected", tt.out, "received", response.AioControllers)
 				}
+				// Empty NextPageToken indicates end of results list
+				if tt.size != 1 && response.NextPageToken != "" {
+					t.Error("Expected end of results, receieved non-empty next page token", response.NextPageToken)
+				}
 			}
 
 			if err != nil {

pkg/backend/backend.go

@@ -26,8 +26,9 @@ type Server struct {
 	pb.UnimplementedNullDebugServiceServer
 	pb.UnimplementedAioControllerServiceServer
 
-	rpc     server.JSONRPC
-	Volumes VolumeParameters
+	rpc        server.JSONRPC
+	Volumes    VolumeParameters
+	Pagination map[string]int
 }
 
 // NewServer creates initialized instance of BackEnd server communicating
@@ -40,5 +41,6 @@ func NewServer(jsonRPC server.JSONRPC) *Server {
 			NullVolumes: make(map[string]*pb.NullDebug),
 			NvmeVolumes: make(map[string]*pb.NVMfRemoteController),
 		},
+		Pagination: make(map[string]int),
 	}
 }

pkg/backend/null.go

@@ -13,7 +13,9 @@ import (
 	pc "github.com/opiproject/opi-api/common/v1/gen/go"
 	pb "github.com/opiproject/opi-api/storage/v1alpha1/gen/go"
 	"github.com/opiproject/opi-spdk-bridge/pkg/models"
+	"github.com/opiproject/opi-spdk-bridge/pkg/server"
 
+	"github.com/google/uuid"
 	"github.com/ulule/deepcopier"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
@@ -136,10 +138,10 @@ func (s *Server) UpdateNullDebug(_ context.Context, in *pb.UpdateNullDebugReques
 // ListNullDebugs lists Null Debug instances
 func (s *Server) ListNullDebugs(_ context.Context, in *pb.ListNullDebugsRequest) (*pb.ListNullDebugsResponse, error) {
 	log.Printf("ListNullDebugs: Received from client: %v", in)
-	if in.PageSize < 0 {
-		err := status.Error(codes.InvalidArgument, "negative PageSize is not allowed")
-		log.Printf("error: %v", err)
-		return nil, err
+	size, offset, perr := server.ExtractPagination(in.PageSize, in.PageToken, s.Pagination)
+	if perr != nil {
+		log.Printf("error: %v", perr)
+		return nil, perr
 	}
 	var result []models.BdevGetBdevsResult
 	err := s.rpc.Call("bdev_get_bdevs", nil, &result)
@@ -148,16 +150,19 @@ func (s *Server) ListNullDebugs(_ context.Context, in *pb.ListNullDebugsRequest)
 		return nil, err
 	}
 	log.Printf("Received from SPDK: %v", result)
-	if in.PageSize > 0 && int(in.PageSize) < len(result) {
-		log.Printf("Limiting result to: %d", in.PageSize)
-		result = result[:in.PageSize]
+	token := ""
+	log.Printf("Limiting result len(%d) to [%d:%d]", len(result), offset, size)
+	result, hasMoreElements := server.LimitPagination(result, offset, size)
+	if hasMoreElements {
+		token = uuid.New().String()
+		s.Pagination[token] = offset + size
 	}
 	Blobarray := make([]*pb.NullDebug, len(result))
 	for i := range result {
 		r := &result[i]
 		Blobarray[i] = &pb.NullDebug{Handle: &pc.ObjectKey{Value: r.Name}, Uuid: &pc.Uuid{Value: r.UUID}, BlockSize: r.BlockSize, BlocksCount: r.NumBlocks}
 	}
-	return &pb.ListNullDebugsResponse{NullDebugs: Blobarray}, nil
+	return &pb.ListNullDebugsResponse{NullDebugs: Blobarray, NextPageToken: token}, nil
 }
 
 // GetNullDebug gets a a Null Debug instance

pkg/backend/null_test.go

@@ -237,6 +237,7 @@ func TestBackEnd_ListNullDebugs(t *testing.T) {
 		errMsg  string
 		start   bool
 		size    int32
+		token   string
 	}{
 		"valid request with invalid SPDK response": {
 			"volume-test",
@@ -246,6 +247,7 @@ func TestBackEnd_ListNullDebugs(t *testing.T) {
 			fmt.Sprintf("Could not find any namespaces for NQN: %v", "nqn.2022-09.io.spdk:opi3"),
 			true,
 			0,
+			"",
 		},
 		"valid request with invalid marshal SPDK response": {
 			"volume-test",
@@ -255,6 +257,7 @@ func TestBackEnd_ListNullDebugs(t *testing.T) {
 			fmt.Sprintf("bdev_get_bdevs: %v", "json: cannot unmarshal bool into Go value of type []models.BdevGetBdevsResult"),
 			true,
 			0,
+			"",
 		},
 		"valid request with empty SPDK response": {
 			"volume-test",
@@ -264,6 +267,7 @@ func TestBackEnd_ListNullDebugs(t *testing.T) {
 			fmt.Sprintf("bdev_get_bdevs: %v", "EOF"),
 			true,
 			0,
+			"",
 		},
 		"valid request with ID mismatch SPDK response": {
 			"volume-test",
@@ -273,6 +277,7 @@ func TestBackEnd_ListNullDebugs(t *testing.T) {
 			fmt.Sprintf("bdev_get_bdevs: %v", "json response ID mismatch"),
 			true,
 			0,
+			"",
 		},
 		"valid request with error code from SPDK response": {
 			"volume-test",
@@ -282,6 +287,7 @@ func TestBackEnd_ListNullDebugs(t *testing.T) {
 			fmt.Sprintf("bdev_get_bdevs: %v", "json response error: myopierr"),
 			true,
 			0,
+			"",
 		},
 		"valid request with valid SPDK response": {
 			"volume-test",
@@ -304,6 +310,7 @@ func TestBackEnd_ListNullDebugs(t *testing.T) {
 			"",
 			true,
 			0,
+			"",
 		},
 		"pagination overflow": {
 			"volume-test",
@@ -326,6 +333,7 @@ func TestBackEnd_ListNullDebugs(t *testing.T) {
 			"",
 			true,
 			1000,
+			"",
 		},
 		"pagination negative": {
 			"volume-test",
@@ -335,6 +343,17 @@ func TestBackEnd_ListNullDebugs(t *testing.T) {
 			"negative PageSize is not allowed",
 			false,
 			-10,
+			"",
+		},
+		"pagination error": {
+			"volume-test",
+			nil,
+			[]string{},
+			codes.NotFound,
+			fmt.Sprintf("unable to find pagination token %s", "unknown-pagination-token"),
+			false,
+			0,
+			"unknown-pagination-token",
 		},
 		"pagination": {
 			"volume-test",
@@ -351,6 +370,7 @@ func TestBackEnd_ListNullDebugs(t *testing.T) {
 			"",
 			true,
 			1,
+			"",
 		},
 	}
 
@@ -360,12 +380,18 @@ func TestBackEnd_ListNullDebugs(t *testing.T) {
 			testEnv := createTestEnvironment(tt.start, tt.spdk)
 			defer testEnv.Close()
 
-			request := &pb.ListNullDebugsRequest{Parent: tt.in, PageSize: tt.size}
+			testEnv.opiSpdkServer.Pagination["existing-pagination-token"] = 1
+
+			request := &pb.ListNullDebugsRequest{Parent: tt.in, PageSize: tt.size, PageToken: tt.token}
 			response, err := testEnv.client.ListNullDebugs(testEnv.ctx, request)
 			if response != nil {
 				if !reflect.DeepEqual(response.NullDebugs, tt.out) {
 					t.Error("response: expected", tt.out, "received", response.NullDebugs)
 				}
+				// Empty NextPageToken indicates end of results list
+				if tt.size != 1 && response.NextPageToken != "" {
+					t.Error("Expected end of results, receieved non-empty next page token", response.NextPageToken)
+				}
 			}
 
 			if err != nil {