Skip to content

Commit

Permalink
Update feast-sdk (#281)
Browse files Browse the repository at this point in the history 
* Update feast client SDK version (#282)

* Update mock feast

* Fix tests wrt to new feast client change

Signed-off-by: khorshuheng <[email protected]>
Co-authored-by: khorshuheng <[email protected]>
Co-authored-by: ariefrahmansyah <[email protected]>
  • Loading branch information
@ariefrahmansyah @khorshuheng
ariefrahmansyah and khorshuheng authored Sep 15, 2022
1 parent 21457c6 commit bca9c42
Show file tree
Hide file tree
Showing 14 changed files with 1,022 additions and 730 deletions.
73 changes: 66 additions & 7 deletions api/cmd/mock-feast/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"context"
"fmt"
"net"
"sort"

feast "github.com/feast-dev/feast/sdk/go"
"github.com/feast-dev/feast/sdk/go/protos/feast/serving"
Expand Down Expand Up @@ -42,33 +43,35 @@ func main() {
}
}

type mockFeastOnlineServing struct{}
type mockFeastOnlineServing struct {
serving.UnimplementedServingServiceServer
}

func (m mockFeastOnlineServing) GetFeastServingInfo(ctx context.Context, req *serving.GetFeastServingInfoRequest) (*serving.GetFeastServingInfoResponse, error) {
func (m *mockFeastOnlineServing) GetFeastServingInfo(ctx context.Context, req *serving.GetFeastServingInfoRequest) (*serving.GetFeastServingInfoResponse, error) {
return &serving.GetFeastServingInfoResponse{}, nil
}

// GetOnlineFeaturesV2 return a valid GetOnlineFeaturesResponse but with all features having GetOnlineFeaturesResponse_OUTSIDE_MAX_AGE
func (m mockFeastOnlineServing) GetOnlineFeaturesV2(ctx context.Context, req *serving.GetOnlineFeaturesRequestV2) (*serving.GetOnlineFeaturesResponse, error) {
func (m *mockFeastOnlineServing) GetOnlineFeaturesV2(ctx context.Context, req *serving.GetOnlineFeaturesRequestV2) (*serving.GetOnlineFeaturesResponse, error) {
log.Infof("GetOnlineFeaturesV2.Request: %+v", req)

var fieldValues []*serving.GetOnlineFeaturesResponse_FieldValues
for _, entity := range req.EntityRows {
fields := entity.Fields
status := make(map[string]serving.GetOnlineFeaturesResponse_FieldStatus)
status := make(map[string]serving.FieldStatus)

for k := range entity.Fields {
status[k] = serving.GetOnlineFeaturesResponse_PRESENT
status[k] = serving.FieldStatus_PRESENT
}

for _, feature := range req.Features {
featureName := feature.FeatureTable + ":" + feature.Name

status[featureName] = serving.GetOnlineFeaturesResponse_OUTSIDE_MAX_AGE
status[featureName] = serving.FieldStatus_OUTSIDE_MAX_AGE
fields[featureName] = nil

if val, ok := mockOutput[feature.Name]; ok {
status[featureName] = serving.GetOnlineFeaturesResponse_PRESENT
status[featureName] = serving.FieldStatus_PRESENT
fields[featureName] = val
}
}
Expand All @@ -88,6 +91,62 @@ func (m mockFeastOnlineServing) GetOnlineFeaturesV2(ctx context.Context, req *se
return resp, nil
}

func (m *mockFeastOnlineServing) GetOnlineFeatures(ctx context.Context, req *serving.GetOnlineFeaturesRequestV2) (*serving.GetOnlineFeaturesResponseV2, error) {
log.Infof("GetOnlineFeatures.Request: %+v", req)
if len(req.EntityRows) == 0 {
log.Panicf("There should be at least one row in the entity rows")
}
sortedEntityFieldNames := make([]string, len(req.EntityRows[0].Fields))
cnt := 0
for fieldName := range req.EntityRows[0].Fields {
sortedEntityFieldNames[cnt] = fieldName
cnt++
}
sort.Strings(sortedEntityFieldNames)
featureReference := make([]string, len(req.Features))
for index, feature := range req.Features {
featureReference[index] = feature.Name
}

var fieldVectors []*serving.GetOnlineFeaturesResponseV2_FieldVector
for entityIndex, entity := range req.EntityRows {
status := make([]serving.FieldStatus, len(sortedEntityFieldNames)+len(req.Features))
entityFeatureValues := make([]*types.Value, len(status))

for index, field := range sortedEntityFieldNames {
entityFeatureValues[index] = entity.Fields[field]
status[index] = serving.FieldStatus_PRESENT
}

for index, feature := range req.Features {
status[index+len(sortedEntityFieldNames)] = serving.FieldStatus_OUTSIDE_MAX_AGE
entityFeatureValues[index+len(sortedEntityFieldNames)] = nil

if val, ok := mockOutput[feature.Name]; ok {
status[index+len(sortedEntityFieldNames)] = serving.FieldStatus_PRESENT
entityFeatureValues[index+len(sortedEntityFieldNames)] = val
}
}
fieldVector := &serving.GetOnlineFeaturesResponseV2_FieldVector{
Values: entityFeatureValues,
Statuses: status,
}
fieldVectors[entityIndex] = fieldVector
}

resp := &serving.GetOnlineFeaturesResponseV2{
Metadata: &serving.GetOnlineFeaturesResponseMetadata{
FieldNames: &serving.FieldList{
Val: append(sortedEntityFieldNames, featureReference...),
},
},
Results: fieldVectors,
}
log.Infof("GetOnlineFeaturesV2.Response: %+v", resp)

return resp, nil
}

func newMockFeastOnlineServing() serving.ServingServiceServer {
return &mockFeastOnlineServing{}
}
2 changes: 1 addition & 1 deletion api/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ require (
github.com/cespare/xxhash v1.1.0
github.com/coocood/freecache v1.1.1
github.com/fatih/color v1.9.0
github.com/feast-dev/feast/sdk/go v0.9.2
github.com/feast-dev/feast/sdk/go v0.9.4
github.com/fraugster/parquet-go v0.10.0
github.com/ghodss/yaml v1.0.0
github.com/go-gota/gota v0.0.0-00010101000000-000000000000
Expand Down
5 changes: 3 additions & 2 deletions api/go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -520,8 +520,8 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv
github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s=
github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
github.com/feast-dev/feast/sdk/go v0.9.2 h1:V+BKYKgoH9RjIIN416+OqDPpbgRZOf1CqGOVR9S2M2s=
github.com/feast-dev/feast/sdk/go v0.9.2/go.mod h1:S4pG8DusW0Tj2HK12DqV2MNlgBFfQKLVELVr15NSIVE=
github.com/feast-dev/feast/sdk/go v0.9.4 h1:ChGdqbNiaBbcm/X1TRBkikAnbhJFAp0ofzs1TmBA/M4=
github.com/feast-dev/feast/sdk/go v0.9.4/go.mod h1:RWG8U+ri5d9CEE6jGPwfaIr5TxNGisZvlCMwGYfmGn4=
github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k=
github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k=
Expand Down Expand Up @@ -2268,6 +2268,7 @@ google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3Iji
google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
google.golang.org/grpc v1.32.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0=
google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8=
Expand Down
65 changes: 33 additions & 32 deletions api/pkg/transformer/executor/transformer_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -129,46 +129,47 @@ func TestStandardTransformer_Execute(t *testing.T) {
Project: "default", // used as identifier for mocking. must match config
},
response: &feastSdk.OnlineFeaturesResponse{
RawResponse: &serving.GetOnlineFeaturesResponse{
FieldValues: []*serving.GetOnlineFeaturesResponse_FieldValues{
RawResponse: &serving.GetOnlineFeaturesResponseV2{
Metadata: &serving.GetOnlineFeaturesResponseMetadata{
FieldNames: &serving.FieldList{
Val: []string{"driver_id", "driver_feature_1", "driver_feature_2", "driver_feature_3"},
},
},
Results: []*serving.GetOnlineFeaturesResponseV2_FieldVector{
{
Fields: map[string]*feastTypes.Value{
"driver_id": feastSdk.Int64Val(1),
"driver_feature_1": feastSdk.DoubleVal(1111),
"driver_feature_2": feastSdk.DoubleVal(2222),
"driver_feature_3": {
Val: &feastTypes.Value_StringListVal{
StringListVal: &feastTypes.StringList{
Val: []string{"A", "B", "C"},
},
Values: []*feastTypes.Value{
feastSdk.Int64Val(1),
feastSdk.DoubleVal(1111),
feastSdk.DoubleVal(2222),
{Val: &feastTypes.Value_StringListVal{
StringListVal: &feastTypes.StringList{
Val: []string{"A", "B", "C"},
},
},
}},
},
Statuses: map[string]serving.GetOnlineFeaturesResponse_FieldStatus{
"driver_id": serving.GetOnlineFeaturesResponse_PRESENT,
"driver_feature_1": serving.GetOnlineFeaturesResponse_PRESENT,
"driver_feature_2": serving.GetOnlineFeaturesResponse_PRESENT,
"driver_feature_3": serving.GetOnlineFeaturesResponse_PRESENT,
Statuses: []serving.FieldStatus{
serving.FieldStatus_PRESENT,
serving.FieldStatus_PRESENT,
serving.FieldStatus_PRESENT,
serving.FieldStatus_PRESENT,
},
},
{
Fields: map[string]*feastTypes.Value{
"driver_id": feastSdk.Int64Val(2),
"driver_feature_1": feastSdk.DoubleVal(3333),
"driver_feature_2": feastSdk.DoubleVal(4444),
"driver_feature_3": {
Val: &feastTypes.Value_StringListVal{
StringListVal: &feastTypes.StringList{
Val: []string{"X", "Y", "Z"},
},
Values: []*feastTypes.Value{
feastSdk.Int64Val(2),
feastSdk.DoubleVal(3333),
feastSdk.DoubleVal(4444),
{Val: &feastTypes.Value_StringListVal{
StringListVal: &feastTypes.StringList{
Val: []string{"X", "Y", "Z"},
},
},
}},
},
Statuses: map[string]serving.GetOnlineFeaturesResponse_FieldStatus{
"driver_id": serving.GetOnlineFeaturesResponse_PRESENT,
"driver_feature_1": serving.GetOnlineFeaturesResponse_PRESENT,
"driver_feature_2": serving.GetOnlineFeaturesResponse_PRESENT,
"driver_feature_3": serving.GetOnlineFeaturesResponse_PRESENT,
Statuses: []serving.FieldStatus{
serving.FieldStatus_PRESENT,
serving.FieldStatus_PRESENT,
serving.FieldStatus_PRESENT,
serving.FieldStatus_PRESENT,
},
},
},
Expand Down
23 changes: 10 additions & 13 deletions api/pkg/transformer/feast/bigtablestore/client_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,12 @@ package bigtablestore
import (
"context"
"fmt"
"github.com/feast-dev/feast/sdk/go/protos/feast/serving"
"testing"
"time"

"cloud.google.com/go/bigtable"
feast "github.com/feast-dev/feast/sdk/go"
"github.com/feast-dev/feast/sdk/go/protos/feast/serving"
"github.com/feast-dev/feast/sdk/go/protos/feast/types"
"github.com/gojek/merlin/pkg/transformer/spec"
"github.com/golang/protobuf/proto"
Expand Down Expand Up @@ -115,19 +115,16 @@ func TestGetOnlineFeatures(t *testing.T) {
Features: []string{"driver_trips:trips_today", "driver_trips:total_distance"},
},
expected: &feast.OnlineFeaturesResponse{
RawResponse: &serving.GetOnlineFeaturesResponse{
FieldValues: []*serving.GetOnlineFeaturesResponse_FieldValues{
RawResponse: &serving.GetOnlineFeaturesResponseV2{
Metadata: &serving.GetOnlineFeaturesResponseMetadata{
FieldNames: &serving.FieldList{
Val: []string{"driver_id", "driver_trips:trips_today", "driver_trips:total_distance"},
},
},
Results: []*serving.GetOnlineFeaturesResponseV2_FieldVector{
{
Fields: map[string]*types.Value{
"driver_id": feast.Int64Val(2),
"driver_trips:trips_today": feast.Int32Val(1),
"driver_trips:total_distance": feast.DoubleVal(2.2),
},
Statuses: map[string]serving.GetOnlineFeaturesResponse_FieldStatus{
"driver_id": serving.GetOnlineFeaturesResponse_PRESENT,
"driver_trips:trips_today": serving.GetOnlineFeaturesResponse_PRESENT,
"driver_trips:total_distance": serving.GetOnlineFeaturesResponse_PRESENT,
},
Values: []*types.Value{feast.Int64Val(2), feast.Int32Val(1), feast.DoubleVal(2.2)},
Statuses: []serving.FieldStatus{serving.FieldStatus_PRESENT, serving.FieldStatus_PRESENT, serving.FieldStatus_PRESENT},
},
},
},
Expand Down
Loading

0 comments on commit bca9c42

Please sign in to comment.