From 235506490f586386e5814c3253ea9cf4aaaa60d4 Mon Sep 17 00:00:00 2001 From: guscarreon Date: Wed, 10 Jan 2024 12:24:29 -0500 Subject: [PATCH 01/21] Use Json compacter in the bidders/params endpoint --- go.mod | 1 + go.sum | 2 ++ router/router.go | 6 ++++-- router/router_test.go | 17 +++++++++++++++++ 4 files changed, 24 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 9a55cddb00a..957a3258e71 100644 --- a/go.mod +++ b/go.mod @@ -66,6 +66,7 @@ require ( github.com/spf13/pflag v1.0.5 // indirect github.com/stretchr/objx v0.5.0 // indirect github.com/subosito/gotenv v1.3.0 // indirect + github.com/tidwall/pretty v1.2.1 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82 // indirect diff --git a/go.sum b/go.sum index ad2d5ba94b8..502eb057b97 100644 --- a/go.sum +++ b/go.sum @@ -482,6 +482,8 @@ github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/subosito/gotenv v1.3.0 h1:mjC+YW8QpAdXibNi+vNWgzmgBH4+5l5dCXv8cNysBLI= github.com/subosito/gotenv v1.3.0/go.mod h1:YzJjq/33h7nrwdY+iHMhEOEEbW0ovIz0tB6t6PwAXzs= +github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= +github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/vrischmann/go-metrics-influxdb v0.1.1 h1:xneKFRjsS4BiVYvAKaM/rOlXYd1pGHksnES0ECCJLgo= github.com/vrischmann/go-metrics-influxdb v0.1.1/go.mod h1:q7YC8bFETCYopXRMtUvQQdLaoVhpsEwvQS2zZEYCqg8= diff --git a/router/router.go b/router/router.go index d89d1f59ca2..4e1a79846ec 100644 --- a/router/router.go +++ b/router/router.go @@ -44,6 +44,7 @@ import ( "github.com/julienschmidt/httprouter" _ "github.com/lib/pq" "github.com/rs/cors" + "github.com/tidwall/pretty" ) // NewJsonDirectoryServer is used to serve .json files from a directory as a single blob. For example, @@ -76,7 +77,8 @@ func newJsonDirectoryServer(schemaDirectory string, validator openrtb_ext.Bidder if !isValid { glog.Fatalf("Schema exists for an unknown bidder: %s", bidder) } - data[bidder] = json.RawMessage(validator.Schema(bidderName)) + compactedJson := pretty.Ugly([]byte(validator.Schema(bidderName))) + data[bidder] = json.RawMessage(compactedJson) } // Add in any aliases @@ -93,7 +95,7 @@ func newJsonDirectoryServer(schemaDirectory string, validator openrtb_ext.Bidder data[aliasName] = bidderData } - response, err := json.Marshal(data) + response, err := jsonutil.Marshal(data) if err != nil { glog.Fatalf("Failed to marshal bidder param JSON-schema: %v", err) } diff --git a/router/router_test.go b/router/router_test.go index cc2f077e5e6..15df2b44c99 100644 --- a/router/router_test.go +++ b/router/router_test.go @@ -5,6 +5,7 @@ import ( "net/http" "net/http/httptest" "os" + "strings" "testing" "github.com/prebid/prebid-server/v2/config" @@ -275,3 +276,19 @@ func TestValidateDefaultAliases(t *testing.T) { } } } + +func TestBidderParamsCompactedOutput(t *testing.T) { + inSchemaDirectory := "../static/bidder-params" + expected := `{"33across":{"$schema":"http://json-schema.org/draft-04/schema#","title":"33Across Adapter Params","description":"A schema which validates params accepted by the 33Across adapter","type":"object","properties":{"productId":{"type":"string","description":"Product type"},"siteId":{"type":"string","description":"Site Id"},"zoneId":{"type":"string","description":"Zone Id"}},"required":["productId","siteId"]}` + paramsValidator, err := openrtb_ext.NewBidderParamsValidator(inSchemaDirectory) + assert.NoError(t, err, "Error initialing validator") + handler := newJsonDirectoryServer(inSchemaDirectory, paramsValidator, nil, nil) + recorder := httptest.NewRecorder() + + request, err := http.NewRequest("GET", "/whatever", nil) + assert.NoError(t, err, "Error creating request") + + handler(recorder, request, nil) + + assert.True(t, strings.HasPrefix(recorder.Body.String(), expected)) +} From cd0a663c491ac0e6821ae3bccc43e27693115c40 Mon Sep 17 00:00:00 2001 From: guscarreon Date: Wed, 10 Jan 2024 12:37:03 -0500 Subject: [PATCH 02/21] Fix aliases --- router/router.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/router/router.go b/router/router.go index 4e1a79846ec..b048e1454be 100644 --- a/router/router.go +++ b/router/router.go @@ -77,13 +77,12 @@ func newJsonDirectoryServer(schemaDirectory string, validator openrtb_ext.Bidder if !isValid { glog.Fatalf("Schema exists for an unknown bidder: %s", bidder) } - compactedJson := pretty.Ugly([]byte(validator.Schema(bidderName))) - data[bidder] = json.RawMessage(compactedJson) + data[bidder] = pretty.Ugly([]byte(validator.Schema(bidderName))) } // Add in any aliases for aliasName, parentBidder := range yamlAliases { - data[string(aliasName)] = json.RawMessage(validator.Schema(parentBidder)) + data[string(aliasName)] = pretty.Ugly([]byte(validator.Schema(parentBidder))) } // Add in any default aliases From 144ec174d098251f455782ed0dbf9d7a52f71094 Mon Sep 17 00:00:00 2001 From: guscarreon Date: Thu, 11 Jan 2024 01:02:45 -0500 Subject: [PATCH 03/21] escape HTML --- router/router.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/router/router.go b/router/router.go index b048e1454be..21c9ca6a04e 100644 --- a/router/router.go +++ b/router/router.go @@ -100,6 +100,9 @@ func newJsonDirectoryServer(schemaDirectory string, validator openrtb_ext.Bidder } return func(w http.ResponseWriter, _ *http.Request, _ httprouter.Params) { + enc := json.NewEncoder(w) + enc.SetEscapeHTML(false) + w.Header().Add("Content-Type", "application/json") w.Write(response) } From 8cb7a3b09773952d171c0ad9e319bb9c126be0c6 Mon Sep 17 00:00:00 2001 From: guscarreon Date: Thu, 18 Jan 2024 22:02:44 -0500 Subject: [PATCH 04/21] Create Encoder and Decoder --- util/jsonutil/jsonutil.go | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/util/jsonutil/jsonutil.go b/util/jsonutil/jsonutil.go index b5bb47cca9a..961e168e024 100644 --- a/util/jsonutil/jsonutil.go +++ b/util/jsonutil/jsonutil.go @@ -5,8 +5,10 @@ import ( "encoding/json" "io" "strings" + "unsafe" jsoniter "github.com/json-iterator/go" + "github.com/modern-go/reflect2" "github.com/prebid/prebid-server/v2/errortypes" ) @@ -211,3 +213,34 @@ func tryExtractErrorMessage(err error) string { func isLikelyDetailedErrorMessage(msg string) bool { return !strings.HasPrefix(msg, "request.") } + +type decorateExtension struct { + jsoniter.DummyExtension +} + +type decorateCodec struct { + originEncoder jsoniter.ValEncoder +} + +func (codec *decorateCodec) IsEmpty(ptr unsafe.Pointer) bool { + return codec.originEncoder.IsEmpty(unsafe.Pointer(&ptr)) +} + +func (codec *decorateCodec) Encode(ptr unsafe.Pointer, stream *jsoniter.Stream) { + jsonRawMsg := *(*[]byte)(ptr) + + var dst *bytes.Buffer + json.Compact(dst, jsonRawMsg) + json.HTMLEscape(dst, jsonRawMsg) + + codec.originEncoder.Encode(unsafe.Pointer(&jsonRawMsg), stream) +} + +func (e *decorateExtension) DecorateEncoder(typ reflect2.Type, encoder jsoniter.ValEncoder) jsoniter.ValEncoder { + switch typ.String() { + case "jsonRawMessage", "[]uint8": + return &decorateCodec{encoder} + } + + return nil +} From d69938bd86a296c9648e12c502e53fc82ca8e865 Mon Sep 17 00:00:00 2001 From: guscarreon Date: Mon, 22 Jan 2024 14:06:25 -0500 Subject: [PATCH 05/21] Recovered code. Extension call in auction_test.go still missing --- main.go | 3 ++ router/router.go | 8 ++--- util/jsonutil/jsonutil.go | 65 +++++++++++++++++++++++++++++---------- 3 files changed, 53 insertions(+), 23 deletions(-) diff --git a/main.go b/main.go index e72f02f1f0e..d49e6e65f07 100644 --- a/main.go +++ b/main.go @@ -8,11 +8,13 @@ import ( "runtime" "time" + jsoniter "github.com/json-iterator/go" "github.com/prebid/prebid-server/v2/config" "github.com/prebid/prebid-server/v2/currency" "github.com/prebid/prebid-server/v2/openrtb_ext" "github.com/prebid/prebid-server/v2/router" "github.com/prebid/prebid-server/v2/server" + "github.com/prebid/prebid-server/v2/util/jsonutil" "github.com/prebid/prebid-server/v2/util/task" "github.com/golang/glog" @@ -48,6 +50,7 @@ func main() { garbageCollectionThreshold := make([]byte, cfg.GarbageCollectorThreshold) defer runtime.KeepAlive(garbageCollectionThreshold) + jsoniter.RegisterExtension(&jsonutil.SampleExtension{}) err = serve(cfg) if err != nil { glog.Exitf("prebid-server failed: %v", err) diff --git a/router/router.go b/router/router.go index 21c9ca6a04e..c285bbdab1b 100644 --- a/router/router.go +++ b/router/router.go @@ -44,7 +44,6 @@ import ( "github.com/julienschmidt/httprouter" _ "github.com/lib/pq" "github.com/rs/cors" - "github.com/tidwall/pretty" ) // NewJsonDirectoryServer is used to serve .json files from a directory as a single blob. For example, @@ -77,12 +76,12 @@ func newJsonDirectoryServer(schemaDirectory string, validator openrtb_ext.Bidder if !isValid { glog.Fatalf("Schema exists for an unknown bidder: %s", bidder) } - data[bidder] = pretty.Ugly([]byte(validator.Schema(bidderName))) + data[bidder] = json.RawMessage(validator.Schema(bidderName)) } // Add in any aliases for aliasName, parentBidder := range yamlAliases { - data[string(aliasName)] = pretty.Ugly([]byte(validator.Schema(parentBidder))) + data[string(aliasName)] = json.RawMessage(validator.Schema(parentBidder)) } // Add in any default aliases @@ -100,9 +99,6 @@ func newJsonDirectoryServer(schemaDirectory string, validator openrtb_ext.Bidder } return func(w http.ResponseWriter, _ *http.Request, _ httprouter.Params) { - enc := json.NewEncoder(w) - enc.SetEscapeHTML(false) - w.Header().Add("Content-Type", "application/json") w.Write(response) } diff --git a/util/jsonutil/jsonutil.go b/util/jsonutil/jsonutil.go index 961e168e024..48e0534cadd 100644 --- a/util/jsonutil/jsonutil.go +++ b/util/jsonutil/jsonutil.go @@ -214,33 +214,64 @@ func isLikelyDetailedErrorMessage(msg string) bool { return !strings.HasPrefix(msg, "request.") } -type decorateExtension struct { - jsoniter.DummyExtension +type wrapCodec struct { + encodeFunc func(ptr unsafe.Pointer, stream *jsoniter.Stream) + isEmptyFunc func(ptr unsafe.Pointer) bool + decodeFunc func(ptr unsafe.Pointer, iter *jsoniter.Iterator) } -type decorateCodec struct { - originEncoder jsoniter.ValEncoder +func (codec *wrapCodec) Encode(ptr unsafe.Pointer, stream *jsoniter.Stream) { + codec.encodeFunc(ptr, stream) } -func (codec *decorateCodec) IsEmpty(ptr unsafe.Pointer) bool { - return codec.originEncoder.IsEmpty(unsafe.Pointer(&ptr)) +func (codec *wrapCodec) IsEmpty(ptr unsafe.Pointer) bool { + if codec.isEmptyFunc == nil { + return false + } + + return codec.isEmptyFunc(ptr) } -func (codec *decorateCodec) Encode(ptr unsafe.Pointer, stream *jsoniter.Stream) { - jsonRawMsg := *(*[]byte)(ptr) +func (codec *wrapCodec) Decode(ptr unsafe.Pointer, iter *jsoniter.Iterator) { + codec.decodeFunc(ptr, iter) +} + +type SampleExtension struct { + jsoniter.DummyExtension +} - var dst *bytes.Buffer - json.Compact(dst, jsonRawMsg) - json.HTMLEscape(dst, jsonRawMsg) +func (e *SampleExtension) CreateDecoder(typ reflect2.Type) jsoniter.ValDecoder { + //if typ.String() == "json.RawMessage" { + // return &wrapCodec{ + // decodeFunc: func(ptr unsafe.Pointer, iter *jsoniter.Iterator) { + // jsonRawMsg := *(*[]byte)(ptr) - codec.originEncoder.Encode(unsafe.Pointer(&jsonRawMsg), stream) + // var dst *bytes.Buffer + // json.Compact(dst, jsonRawMsg) + + // //i := iter.ReadInt() + // //*(*int)(ptr) = i - 1000 + // }, + // } + //} + return nil } -func (e *decorateExtension) DecorateEncoder(typ reflect2.Type, encoder jsoniter.ValEncoder) jsoniter.ValEncoder { - switch typ.String() { - case "jsonRawMessage", "[]uint8": - return &decorateCodec{encoder} - } +func (e *SampleExtension) CreateEncoder(typ reflect2.Type) jsoniter.ValEncoder { + if typ.String() == "json.RawMessage" { + return &wrapCodec{ + encodeFunc: func(ptr unsafe.Pointer, stream *jsoniter.Stream) { + jsonRawMsg := *(*[]byte)(ptr) + var dst *bytes.Buffer + json.Compact(dst, jsonRawMsg) + stream.WriteStringWithHTMLEscaped(dst.String()) + + }, + isEmptyFunc: func(ptr unsafe.Pointer) bool { + return *((*string)(ptr)) == "" + }, + } + } return nil } From 56867550785f797d34864a25c399bd1b668c899f Mon Sep 17 00:00:00 2001 From: guscarreon Date: Mon, 22 Jan 2024 19:26:52 -0500 Subject: [PATCH 06/21] CreateDecoder func --- endpoints/openrtb2/auction_test.go | 7 +++++ router/router_test.go | 4 +++ util/jsonutil/jsonutil.go | 46 ++++++++++++++++++------------ 3 files changed, 39 insertions(+), 18 deletions(-) diff --git a/endpoints/openrtb2/auction_test.go b/endpoints/openrtb2/auction_test.go index c521d653cac..744181a0eec 100644 --- a/endpoints/openrtb2/auction_test.go +++ b/endpoints/openrtb2/auction_test.go @@ -18,6 +18,7 @@ import ( "time" "github.com/buger/jsonparser" + jsoniter "github.com/json-iterator/go" "github.com/julienschmidt/httprouter" "github.com/prebid/openrtb/v19/native1" nativeRequests "github.com/prebid/openrtb/v19/native1/request" @@ -122,6 +123,7 @@ func TestJsonSampleRequests(t *testing.T) { }, } + jsoniter.RegisterExtension(&jsonutil.SampleExtension{}) for _, tc := range testSuites { err := filepath.WalkDir(filepath.Join("sample-requests", tc.sampleRequestsSubDir), func(path string, info fs.DirEntry, err error) error { // According to documentation, needed to avoid panics @@ -142,6 +144,11 @@ func TestJsonSampleRequests(t *testing.T) { } } +func TestSingleJSONTest(t *testing.T) { + jsoniter.RegisterExtension(&jsonutil.SampleExtension{}) + runJsonBasedTest(t, "sample-requests/valid-whole/exemplary/simple.json", "") +} + func runJsonBasedTest(t *testing.T, filename, desc string) { t.Helper() diff --git a/router/router_test.go b/router/router_test.go index 15df2b44c99..5bc73be81f2 100644 --- a/router/router_test.go +++ b/router/router_test.go @@ -8,6 +8,7 @@ import ( "strings" "testing" + jsoniter "github.com/json-iterator/go" "github.com/prebid/prebid-server/v2/config" "github.com/prebid/prebid-server/v2/openrtb_ext" "github.com/prebid/prebid-server/v2/util/jsonutil" @@ -278,6 +279,8 @@ func TestValidateDefaultAliases(t *testing.T) { } func TestBidderParamsCompactedOutput(t *testing.T) { + jsoniter.RegisterExtension(&jsonutil.SampleExtension{}) + inSchemaDirectory := "../static/bidder-params" expected := `{"33across":{"$schema":"http://json-schema.org/draft-04/schema#","title":"33Across Adapter Params","description":"A schema which validates params accepted by the 33Across adapter","type":"object","properties":{"productId":{"type":"string","description":"Product type"},"siteId":{"type":"string","description":"Site Id"},"zoneId":{"type":"string","description":"Zone Id"}},"required":["productId","siteId"]}` paramsValidator, err := openrtb_ext.NewBidderParamsValidator(inSchemaDirectory) @@ -291,4 +294,5 @@ func TestBidderParamsCompactedOutput(t *testing.T) { handler(recorder, request, nil) assert.True(t, strings.HasPrefix(recorder.Body.String(), expected)) + assert.Equal(t, expected, recorder.Body.String()) } diff --git a/util/jsonutil/jsonutil.go b/util/jsonutil/jsonutil.go index 48e0534cadd..a255a487234 100644 --- a/util/jsonutil/jsonutil.go +++ b/util/jsonutil/jsonutil.go @@ -241,19 +241,28 @@ type SampleExtension struct { } func (e *SampleExtension) CreateDecoder(typ reflect2.Type) jsoniter.ValDecoder { - //if typ.String() == "json.RawMessage" { - // return &wrapCodec{ - // decodeFunc: func(ptr unsafe.Pointer, iter *jsoniter.Iterator) { - // jsonRawMsg := *(*[]byte)(ptr) - - // var dst *bytes.Buffer - // json.Compact(dst, jsonRawMsg) - - // //i := iter.ReadInt() - // //*(*int)(ptr) = i - 1000 - // }, - // } - //} + if t, ok := typ.(*reflect2.UnsafePtrType); ok { + decoder := jsonConfigValidationOn.DecoderOf(t) + return &wrapCodec{ + decodeFunc: decoder.Decode, + } + } + if _, ok := typ.(*reflect2.UnsafeStructType); ok { + //return jsonConfigValidationOn.DecoderOf(typ) + //jsonConfigValidationOn + return &wrapCodec{ + decodeFunc: func(ptr unsafe.Pointer, iter *jsoniter.Iterator) { + //pObj := (*interface{})(ptr) + //obj := *pObj + //iter.ReadVal(obj) + + str := *((*string)(ptr)) + r := strings.NewReader(str) + decoder := jsonConfigValidationOn.NewDecoder(r) + decoder.Decode(ptr) + }, + } + } return nil } @@ -261,12 +270,13 @@ func (e *SampleExtension) CreateEncoder(typ reflect2.Type) jsoniter.ValEncoder { if typ.String() == "json.RawMessage" { return &wrapCodec{ encodeFunc: func(ptr unsafe.Pointer, stream *jsoniter.Stream) { - jsonRawMsg := *(*[]byte)(ptr) - - var dst *bytes.Buffer - json.Compact(dst, jsonRawMsg) - stream.WriteStringWithHTMLEscaped(dst.String()) + if ptr != nil { + jsonRawMsg := *(*[]byte)(ptr) + dst := &bytes.Buffer{} + json.Compact(dst, jsonRawMsg) + stream.WriteStringWithHTMLEscaped(dst.String()) + } }, isEmptyFunc: func(ptr unsafe.Pointer) bool { return *((*string)(ptr)) == "" From 1be65a6b10056165f735d0af34809b27692b28ba Mon Sep 17 00:00:00 2001 From: guscarreon Date: Mon, 22 Jan 2024 19:34:44 -0500 Subject: [PATCH 07/21] removed go.mod and go.sum entries --- go.mod | 1 - go.sum | 2 -- 2 files changed, 3 deletions(-) diff --git a/go.mod b/go.mod index 957a3258e71..9a55cddb00a 100644 --- a/go.mod +++ b/go.mod @@ -66,7 +66,6 @@ require ( github.com/spf13/pflag v1.0.5 // indirect github.com/stretchr/objx v0.5.0 // indirect github.com/subosito/gotenv v1.3.0 // indirect - github.com/tidwall/pretty v1.2.1 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82 // indirect diff --git a/go.sum b/go.sum index 502eb057b97..ad2d5ba94b8 100644 --- a/go.sum +++ b/go.sum @@ -482,8 +482,6 @@ github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/subosito/gotenv v1.3.0 h1:mjC+YW8QpAdXibNi+vNWgzmgBH4+5l5dCXv8cNysBLI= github.com/subosito/gotenv v1.3.0/go.mod h1:YzJjq/33h7nrwdY+iHMhEOEEbW0ovIz0tB6t6PwAXzs= -github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= -github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/vrischmann/go-metrics-influxdb v0.1.1 h1:xneKFRjsS4BiVYvAKaM/rOlXYd1pGHksnES0ECCJLgo= github.com/vrischmann/go-metrics-influxdb v0.1.1/go.mod h1:q7YC8bFETCYopXRMtUvQQdLaoVhpsEwvQS2zZEYCqg8= From a93e96b18d5fd9ce87e3af7089df1360dbc35628 Mon Sep 17 00:00:00 2001 From: guscarreon Date: Mon, 22 Jan 2024 19:38:17 -0500 Subject: [PATCH 08/21] cleaned up code a little --- util/jsonutil/jsonutil.go | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/util/jsonutil/jsonutil.go b/util/jsonutil/jsonutil.go index a255a487234..ef578b91e44 100644 --- a/util/jsonutil/jsonutil.go +++ b/util/jsonutil/jsonutil.go @@ -248,18 +248,16 @@ func (e *SampleExtension) CreateDecoder(typ reflect2.Type) jsoniter.ValDecoder { } } if _, ok := typ.(*reflect2.UnsafeStructType); ok { - //return jsonConfigValidationOn.DecoderOf(typ) - //jsonConfigValidationOn return &wrapCodec{ decodeFunc: func(ptr unsafe.Pointer, iter *jsoniter.Iterator) { - //pObj := (*interface{})(ptr) - //obj := *pObj - //iter.ReadVal(obj) - - str := *((*string)(ptr)) - r := strings.NewReader(str) - decoder := jsonConfigValidationOn.NewDecoder(r) - decoder.Decode(ptr) + pObj := (*interface{})(ptr) + obj := *pObj + iter.ReadVal(&obj) + + //str := *((*string)(ptr)) + //r := strings.NewReader(str) + //decoder := jsonConfigValidationOn.NewDecoder(r) + //decoder.Decode(ptr) }, } } From 6edfb64758ac5ba9ac5e8444cdb6f4ac9497f8c6 Mon Sep 17 00:00:00 2001 From: guscarreon Date: Mon, 5 Feb 2024 21:36:52 -0500 Subject: [PATCH 09/21] Commented out CreateEncode --- util/jsonutil/jsonutil.go | 44 ++++++++++++++++++++++++++------------- 1 file changed, 30 insertions(+), 14 deletions(-) diff --git a/util/jsonutil/jsonutil.go b/util/jsonutil/jsonutil.go index ef578b91e44..062eb9c8584 100644 --- a/util/jsonutil/jsonutil.go +++ b/util/jsonutil/jsonutil.go @@ -247,21 +247,37 @@ func (e *SampleExtension) CreateDecoder(typ reflect2.Type) jsoniter.ValDecoder { decodeFunc: decoder.Decode, } } - if _, ok := typ.(*reflect2.UnsafeStructType); ok { - return &wrapCodec{ - decodeFunc: func(ptr unsafe.Pointer, iter *jsoniter.Iterator) { - pObj := (*interface{})(ptr) - obj := *pObj - iter.ReadVal(&obj) - - //str := *((*string)(ptr)) - //r := strings.NewReader(str) - //decoder := jsonConfigValidationOn.NewDecoder(r) - //decoder.Decode(ptr) - }, - } - } + //if typ.Kind() == reflect.Ptr { + // decoder := jsonConfigValidationOn.DecoderOf(typ) + + // return &wrapCodec{ + // decodeFunc: decoder.Decode, + // } + //} + + //if unsafeStruct, ok := typ.(*reflect2.UnsafeStructType); ok { + // return &wrapCodec{ + // decodeFunc: func(ptr unsafe.Pointer, iter *jsoniter.Iterator) { + // //var unsafePtr *reflect2.UnsafePtrType = *reflect2.UnsafePtrType(unsafeStruct) + // var unsafePtr *reflect2.UnsafePtrType + + // unsafePtr = unsafeStruct.Indirect(unsafePtr) + + // decoder := jsonConfigValidationOn.DecoderOf(unsafePtr) + + // return &wrapCodec{ + // decodeFunc: decoder.Decode, + // } + + // //str := *((*string)(ptr)) + // //r := strings.NewReader(str) + // //decoder := jsonConfigValidationOn.NewDecoder(r) + // //decoder.Decode(ptr) + // }, + // } + //} return nil + //return jsonConfigValidationOn.DecoderOf(typ) } func (e *SampleExtension) CreateEncoder(typ reflect2.Type) jsoniter.ValEncoder { From 0ffb7ed8bb8cf0c885337c5498306bff865d1b58 Mon Sep 17 00:00:00 2001 From: guscarreon Date: Tue, 6 Feb 2024 18:54:16 -0500 Subject: [PATCH 10/21] Tried registering the encoder --- endpoints/openrtb2/auction_test.go | 4 +- main.go | 3 +- util/jsonutil/jsonutil.go | 90 +++--------------------------- 3 files changed, 13 insertions(+), 84 deletions(-) diff --git a/endpoints/openrtb2/auction_test.go b/endpoints/openrtb2/auction_test.go index 79fc1938db6..dffa6b55587 100644 --- a/endpoints/openrtb2/auction_test.go +++ b/endpoints/openrtb2/auction_test.go @@ -123,7 +123,7 @@ func TestJsonSampleRequests(t *testing.T) { }, } - jsoniter.RegisterExtension(&jsonutil.SampleExtension{}) + jsoniter.RegisterTypeEncoder("json.RawMessage", &jsonutil.JsonCompactEncoder{}) for _, tc := range testSuites { err := filepath.WalkDir(filepath.Join("sample-requests", tc.sampleRequestsSubDir), func(path string, info fs.DirEntry, err error) error { // According to documentation, needed to avoid panics @@ -145,7 +145,7 @@ func TestJsonSampleRequests(t *testing.T) { } func TestSingleJSONTest(t *testing.T) { - jsoniter.RegisterExtension(&jsonutil.SampleExtension{}) + jsoniter.RegisterTypeEncoder("json.RawMessage", &jsonutil.JsonCompactEncoder{}) runJsonBasedTest(t, "sample-requests/valid-whole/exemplary/simple.json", "") } diff --git a/main.go b/main.go index d49e6e65f07..513c453cc90 100644 --- a/main.go +++ b/main.go @@ -50,7 +50,8 @@ func main() { garbageCollectionThreshold := make([]byte, cfg.GarbageCollectorThreshold) defer runtime.KeepAlive(garbageCollectionThreshold) - jsoniter.RegisterExtension(&jsonutil.SampleExtension{}) + //jsoniter.RegisterExtension(&jsonutil.SampleExtension{}) + jsoniter.RegisterTypeEncoder("json.RawMessage", &jsonutil.JsonCompactEncoder{}) err = serve(cfg) if err != nil { glog.Exitf("prebid-server failed: %v", err) diff --git a/util/jsonutil/jsonutil.go b/util/jsonutil/jsonutil.go index 062eb9c8584..c5fb56c991d 100644 --- a/util/jsonutil/jsonutil.go +++ b/util/jsonutil/jsonutil.go @@ -8,7 +8,6 @@ import ( "unsafe" jsoniter "github.com/json-iterator/go" - "github.com/modern-go/reflect2" "github.com/prebid/prebid-server/v2/errortypes" ) @@ -214,88 +213,17 @@ func isLikelyDetailedErrorMessage(msg string) bool { return !strings.HasPrefix(msg, "request.") } -type wrapCodec struct { - encodeFunc func(ptr unsafe.Pointer, stream *jsoniter.Stream) - isEmptyFunc func(ptr unsafe.Pointer) bool - decodeFunc func(ptr unsafe.Pointer, iter *jsoniter.Iterator) -} - -func (codec *wrapCodec) Encode(ptr unsafe.Pointer, stream *jsoniter.Stream) { - codec.encodeFunc(ptr, stream) -} - -func (codec *wrapCodec) IsEmpty(ptr unsafe.Pointer) bool { - if codec.isEmptyFunc == nil { - return false - } - - return codec.isEmptyFunc(ptr) -} - -func (codec *wrapCodec) Decode(ptr unsafe.Pointer, iter *jsoniter.Iterator) { - codec.decodeFunc(ptr, iter) -} - -type SampleExtension struct { - jsoniter.DummyExtension -} - -func (e *SampleExtension) CreateDecoder(typ reflect2.Type) jsoniter.ValDecoder { - if t, ok := typ.(*reflect2.UnsafePtrType); ok { - decoder := jsonConfigValidationOn.DecoderOf(t) - return &wrapCodec{ - decodeFunc: decoder.Decode, - } - } - //if typ.Kind() == reflect.Ptr { - // decoder := jsonConfigValidationOn.DecoderOf(typ) - - // return &wrapCodec{ - // decodeFunc: decoder.Decode, - // } - //} - - //if unsafeStruct, ok := typ.(*reflect2.UnsafeStructType); ok { - // return &wrapCodec{ - // decodeFunc: func(ptr unsafe.Pointer, iter *jsoniter.Iterator) { - // //var unsafePtr *reflect2.UnsafePtrType = *reflect2.UnsafePtrType(unsafeStruct) - // var unsafePtr *reflect2.UnsafePtrType - - // unsafePtr = unsafeStruct.Indirect(unsafePtr) - - // decoder := jsonConfigValidationOn.DecoderOf(unsafePtr) +type JsonCompactEncoder struct{} - // return &wrapCodec{ - // decodeFunc: decoder.Decode, - // } +func (encoder *JsonCompactEncoder) Encode(ptr unsafe.Pointer, stream *jsoniter.Stream) { + jsonRawMsg := *(*[]byte)(ptr) - // //str := *((*string)(ptr)) - // //r := strings.NewReader(str) - // //decoder := jsonConfigValidationOn.NewDecoder(r) - // //decoder.Decode(ptr) - // }, - // } - //} - return nil - //return jsonConfigValidationOn.DecoderOf(typ) + dst := &bytes.Buffer{} + json.Compact(dst, jsonRawMsg) + stream.WriteStringWithHTMLEscaped(dst.String()) } -func (e *SampleExtension) CreateEncoder(typ reflect2.Type) jsoniter.ValEncoder { - if typ.String() == "json.RawMessage" { - return &wrapCodec{ - encodeFunc: func(ptr unsafe.Pointer, stream *jsoniter.Stream) { - if ptr != nil { - jsonRawMsg := *(*[]byte)(ptr) - - dst := &bytes.Buffer{} - json.Compact(dst, jsonRawMsg) - stream.WriteStringWithHTMLEscaped(dst.String()) - } - }, - isEmptyFunc: func(ptr unsafe.Pointer) bool { - return *((*string)(ptr)) == "" - }, - } - } - return nil +func (encoder *JsonCompactEncoder) IsEmpty(ptr unsafe.Pointer) bool { + jsonRawMsg := *(*[]byte)(ptr) + return len(jsonRawMsg) == 0 } From f1eb5c3c5b1a08ae5f6be68293ebf9f3b44e5694 Mon Sep 17 00:00:00 2001 From: guscarreon Date: Mon, 12 Feb 2024 10:11:29 -0500 Subject: [PATCH 11/21] Scott's fix --- endpoints/openrtb2/auction_test.go | 4 ++-- main.go | 3 +-- router/router_test.go | 21 ------------------- util/jsonutil/jsonutil.go | 33 ++++++++++++++++++++++-------- 4 files changed, 27 insertions(+), 34 deletions(-) diff --git a/endpoints/openrtb2/auction_test.go b/endpoints/openrtb2/auction_test.go index dffa6b55587..e1b4d35d1fc 100644 --- a/endpoints/openrtb2/auction_test.go +++ b/endpoints/openrtb2/auction_test.go @@ -123,7 +123,7 @@ func TestJsonSampleRequests(t *testing.T) { }, } - jsoniter.RegisterTypeEncoder("json.RawMessage", &jsonutil.JsonCompactEncoder{}) + jsoniter.RegisterExtension(&jsonutil.RawMessageExtension{}) for _, tc := range testSuites { err := filepath.WalkDir(filepath.Join("sample-requests", tc.sampleRequestsSubDir), func(path string, info fs.DirEntry, err error) error { // According to documentation, needed to avoid panics @@ -145,7 +145,7 @@ func TestJsonSampleRequests(t *testing.T) { } func TestSingleJSONTest(t *testing.T) { - jsoniter.RegisterTypeEncoder("json.RawMessage", &jsonutil.JsonCompactEncoder{}) + jsoniter.RegisterExtension(&jsonutil.RawMessageExtension{}) runJsonBasedTest(t, "sample-requests/valid-whole/exemplary/simple.json", "") } diff --git a/main.go b/main.go index 513c453cc90..cc86c80814c 100644 --- a/main.go +++ b/main.go @@ -50,8 +50,7 @@ func main() { garbageCollectionThreshold := make([]byte, cfg.GarbageCollectorThreshold) defer runtime.KeepAlive(garbageCollectionThreshold) - //jsoniter.RegisterExtension(&jsonutil.SampleExtension{}) - jsoniter.RegisterTypeEncoder("json.RawMessage", &jsonutil.JsonCompactEncoder{}) + jsoniter.RegisterExtension(&jsonutil.RawMessageExtension{}) err = serve(cfg) if err != nil { glog.Exitf("prebid-server failed: %v", err) diff --git a/router/router_test.go b/router/router_test.go index 5bc73be81f2..cc2f077e5e6 100644 --- a/router/router_test.go +++ b/router/router_test.go @@ -5,10 +5,8 @@ import ( "net/http" "net/http/httptest" "os" - "strings" "testing" - jsoniter "github.com/json-iterator/go" "github.com/prebid/prebid-server/v2/config" "github.com/prebid/prebid-server/v2/openrtb_ext" "github.com/prebid/prebid-server/v2/util/jsonutil" @@ -277,22 +275,3 @@ func TestValidateDefaultAliases(t *testing.T) { } } } - -func TestBidderParamsCompactedOutput(t *testing.T) { - jsoniter.RegisterExtension(&jsonutil.SampleExtension{}) - - inSchemaDirectory := "../static/bidder-params" - expected := `{"33across":{"$schema":"http://json-schema.org/draft-04/schema#","title":"33Across Adapter Params","description":"A schema which validates params accepted by the 33Across adapter","type":"object","properties":{"productId":{"type":"string","description":"Product type"},"siteId":{"type":"string","description":"Site Id"},"zoneId":{"type":"string","description":"Zone Id"}},"required":["productId","siteId"]}` - paramsValidator, err := openrtb_ext.NewBidderParamsValidator(inSchemaDirectory) - assert.NoError(t, err, "Error initialing validator") - handler := newJsonDirectoryServer(inSchemaDirectory, paramsValidator, nil, nil) - recorder := httptest.NewRecorder() - - request, err := http.NewRequest("GET", "/whatever", nil) - assert.NoError(t, err, "Error creating request") - - handler(recorder, request, nil) - - assert.True(t, strings.HasPrefix(recorder.Body.String(), expected)) - assert.Equal(t, expected, recorder.Body.String()) -} diff --git a/util/jsonutil/jsonutil.go b/util/jsonutil/jsonutil.go index c5fb56c991d..4a06f94b2e3 100644 --- a/util/jsonutil/jsonutil.go +++ b/util/jsonutil/jsonutil.go @@ -8,6 +8,7 @@ import ( "unsafe" jsoniter "github.com/json-iterator/go" + "github.com/modern-go/reflect2" "github.com/prebid/prebid-server/v2/errortypes" ) @@ -213,17 +214,31 @@ func isLikelyDetailedErrorMessage(msg string) bool { return !strings.HasPrefix(msg, "request.") } -type JsonCompactEncoder struct{} +var jsonRawMessageType = reflect2.TypeOfPtr(&json.RawMessage{}).Elem() -func (encoder *JsonCompactEncoder) Encode(ptr unsafe.Pointer, stream *jsoniter.Stream) { - jsonRawMsg := *(*[]byte)(ptr) +type RawMessageExtension struct { + jsoniter.DummyExtension +} + +func (e *RawMessageExtension) CreateEncoder(typ reflect2.Type) jsoniter.ValEncoder { + if typ == jsonRawMessageType { + return &rawMessageCodec{} + } + return nil +} + +type rawMessageCodec struct{} - dst := &bytes.Buffer{} - json.Compact(dst, jsonRawMsg) - stream.WriteStringWithHTMLEscaped(dst.String()) +func (codec *rawMessageCodec) Encode(ptr unsafe.Pointer, stream *jsoniter.Stream) { + if ptr != nil { + jsonRawMsg := *(*[]byte)(ptr) + + dst := bytes.NewBuffer(make([]byte, 0, len(jsonRawMsg))) + json.Compact(dst, jsonRawMsg) + stream.Write(dst.Bytes()) + } } -func (encoder *JsonCompactEncoder) IsEmpty(ptr unsafe.Pointer) bool { - jsonRawMsg := *(*[]byte)(ptr) - return len(jsonRawMsg) == 0 +func (codec *rawMessageCodec) IsEmpty(ptr unsafe.Pointer) bool { + return *((*string)(ptr)) == "" } From 7747b2e67cd69ce2f579fb8087d3d4044521cdc1 Mon Sep 17 00:00:00 2001 From: guscarreon Date: Mon, 12 Feb 2024 12:09:47 -0500 Subject: [PATCH 12/21] init() --- adapters/amx/amx_test.go | 8 +++++++- endpoints/openrtb2/auction_test.go | 7 ++++++- main.go | 2 +- util/jsonutil/jsonutil.go | 2 ++ 4 files changed, 16 insertions(+), 3 deletions(-) diff --git a/adapters/amx/amx_test.go b/adapters/amx/amx_test.go index d8c73a6141f..9a6c3a5faa0 100644 --- a/adapters/amx/amx_test.go +++ b/adapters/amx/amx_test.go @@ -5,10 +5,12 @@ import ( "fmt" "testing" + jsoniter "github.com/json-iterator/go" "github.com/prebid/openrtb/v20/openrtb2" "github.com/prebid/prebid-server/v2/adapters" "github.com/prebid/prebid-server/v2/config" "github.com/prebid/prebid-server/v2/openrtb_ext" + "github.com/prebid/prebid-server/v2/util/jsonutil" "github.com/stretchr/testify/assert" "github.com/prebid/prebid-server/v2/adapters/adapterstest" @@ -45,6 +47,10 @@ func TestEndpointQueryStringMalformed(t *testing.T) { assert.Error(t, buildErr) } +func TestMain(m *testing.M) { + jsoniter.RegisterExtension(&jsonutil.RawMessageExtension{}) + m.Run() +} func TestMakeRequestsTagID(t *testing.T) { var w, h int = 300, 250 var width, height int64 = int64(w), int64(h) @@ -103,7 +109,7 @@ func TestMakeRequestsTagID(t *testing.T) { assert.Len(t, actualAdapterRequests, 1) assert.Empty(t, err) var body openrtb2.BidRequest - assert.Nil(t, json.Unmarshal(actualAdapterRequests[0].Body, &body)) + assert.Nil(t, jsonutil.Unmarshal(actualAdapterRequests[0].Body, &body)) assert.Equal(t, tc.expectedTagID, body.Imp[0].TagID) } } diff --git a/endpoints/openrtb2/auction_test.go b/endpoints/openrtb2/auction_test.go index e1b4d35d1fc..fefa7a3ac7f 100644 --- a/endpoints/openrtb2/auction_test.go +++ b/endpoints/openrtb2/auction_test.go @@ -44,6 +44,11 @@ import ( const jsonFileExtension string = ".json" +//func TestMain(m *testing.M) { +// jsoniter.RegisterExtension(&jsonutil.RawMessageExtension{}) +// m.Run() +//} + func TestJsonSampleRequests(t *testing.T) { testSuites := []struct { description string @@ -123,7 +128,7 @@ func TestJsonSampleRequests(t *testing.T) { }, } - jsoniter.RegisterExtension(&jsonutil.RawMessageExtension{}) + //jsoniter.RegisterExtension(&jsonutil.RawMessageExtension{}) for _, tc := range testSuites { err := filepath.WalkDir(filepath.Join("sample-requests", tc.sampleRequestsSubDir), func(path string, info fs.DirEntry, err error) error { // According to documentation, needed to avoid panics diff --git a/main.go b/main.go index cc86c80814c..1909199d1e9 100644 --- a/main.go +++ b/main.go @@ -23,6 +23,7 @@ import ( func init() { rand.Seed(time.Now().UnixNano()) + jsoniter.RegisterExtension(&jsonutil.RawMessageExtension{}) } func main() { @@ -50,7 +51,6 @@ func main() { garbageCollectionThreshold := make([]byte, cfg.GarbageCollectorThreshold) defer runtime.KeepAlive(garbageCollectionThreshold) - jsoniter.RegisterExtension(&jsonutil.RawMessageExtension{}) err = serve(cfg) if err != nil { glog.Exitf("prebid-server failed: %v", err) diff --git a/util/jsonutil/jsonutil.go b/util/jsonutil/jsonutil.go index 4a06f94b2e3..9e64a2bd36f 100644 --- a/util/jsonutil/jsonutil.go +++ b/util/jsonutil/jsonutil.go @@ -216,6 +216,8 @@ func isLikelyDetailedErrorMessage(msg string) bool { var jsonRawMessageType = reflect2.TypeOfPtr(&json.RawMessage{}).Elem() +// RawMessageExtension was created to call json.Compact() on every json.RawMessage field when getting marshalled. +// All other types will be marshalled as usual type RawMessageExtension struct { jsoniter.DummyExtension } From 86ee89c0fa72659c2953de587fcd367e8a04a0f3 Mon Sep 17 00:00:00 2001 From: guscarreon Date: Mon, 12 Feb 2024 12:21:47 -0500 Subject: [PATCH 13/21] TestMain(m *testing.M) --- adapters/amx/amx_test.go | 8 +------- endpoints/openrtb2/auction_test.go | 10 ++++------ router/router_test.go | 27 +++++++++++++++++++++++++++ 3 files changed, 32 insertions(+), 13 deletions(-) diff --git a/adapters/amx/amx_test.go b/adapters/amx/amx_test.go index 9a6c3a5faa0..d8c73a6141f 100644 --- a/adapters/amx/amx_test.go +++ b/adapters/amx/amx_test.go @@ -5,12 +5,10 @@ import ( "fmt" "testing" - jsoniter "github.com/json-iterator/go" "github.com/prebid/openrtb/v20/openrtb2" "github.com/prebid/prebid-server/v2/adapters" "github.com/prebid/prebid-server/v2/config" "github.com/prebid/prebid-server/v2/openrtb_ext" - "github.com/prebid/prebid-server/v2/util/jsonutil" "github.com/stretchr/testify/assert" "github.com/prebid/prebid-server/v2/adapters/adapterstest" @@ -47,10 +45,6 @@ func TestEndpointQueryStringMalformed(t *testing.T) { assert.Error(t, buildErr) } -func TestMain(m *testing.M) { - jsoniter.RegisterExtension(&jsonutil.RawMessageExtension{}) - m.Run() -} func TestMakeRequestsTagID(t *testing.T) { var w, h int = 300, 250 var width, height int64 = int64(w), int64(h) @@ -109,7 +103,7 @@ func TestMakeRequestsTagID(t *testing.T) { assert.Len(t, actualAdapterRequests, 1) assert.Empty(t, err) var body openrtb2.BidRequest - assert.Nil(t, jsonutil.Unmarshal(actualAdapterRequests[0].Body, &body)) + assert.Nil(t, json.Unmarshal(actualAdapterRequests[0].Body, &body)) assert.Equal(t, tc.expectedTagID, body.Imp[0].TagID) } } diff --git a/endpoints/openrtb2/auction_test.go b/endpoints/openrtb2/auction_test.go index fefa7a3ac7f..a1453faa7aa 100644 --- a/endpoints/openrtb2/auction_test.go +++ b/endpoints/openrtb2/auction_test.go @@ -44,10 +44,10 @@ import ( const jsonFileExtension string = ".json" -//func TestMain(m *testing.M) { -// jsoniter.RegisterExtension(&jsonutil.RawMessageExtension{}) -// m.Run() -//} +func TestMain(m *testing.M) { + jsoniter.RegisterExtension(&jsonutil.RawMessageExtension{}) + m.Run() +} func TestJsonSampleRequests(t *testing.T) { testSuites := []struct { @@ -128,7 +128,6 @@ func TestJsonSampleRequests(t *testing.T) { }, } - //jsoniter.RegisterExtension(&jsonutil.RawMessageExtension{}) for _, tc := range testSuites { err := filepath.WalkDir(filepath.Join("sample-requests", tc.sampleRequestsSubDir), func(path string, info fs.DirEntry, err error) error { // According to documentation, needed to avoid panics @@ -150,7 +149,6 @@ func TestJsonSampleRequests(t *testing.T) { } func TestSingleJSONTest(t *testing.T) { - jsoniter.RegisterExtension(&jsonutil.RawMessageExtension{}) runJsonBasedTest(t, "sample-requests/valid-whole/exemplary/simple.json", "") } diff --git a/router/router_test.go b/router/router_test.go index cc2f077e5e6..a16aba1ac5c 100644 --- a/router/router_test.go +++ b/router/router_test.go @@ -5,8 +5,10 @@ import ( "net/http" "net/http/httptest" "os" + "strings" "testing" + jsoniter "github.com/json-iterator/go" "github.com/prebid/prebid-server/v2/config" "github.com/prebid/prebid-server/v2/openrtb_ext" "github.com/prebid/prebid-server/v2/util/jsonutil" @@ -18,6 +20,11 @@ const adapterDirectory = "../adapters" type testValidator struct{} +func TestMain(m *testing.M) { + jsoniter.RegisterExtension(&jsonutil.RawMessageExtension{}) + m.Run() +} + func (validator *testValidator) Validate(name openrtb_ext.BidderName, ext json.RawMessage) error { return nil } @@ -275,3 +282,23 @@ func TestValidateDefaultAliases(t *testing.T) { } } } + +func TestBidderParamsCompactedOutput(t *testing.T) { + expectedPrefix := `{"33across":{"$schema":"http://json-schema.org/draft-04/schema#","title":"33Across Adapter Params","description":"A schema which validates params accepted by the 33Across adapter","type":"object","properties":{"productId":{"type":"string","description":"Product type"},"siteId":{"type":"string","description":"Site Id"},"zoneId":{"type":"string","description":"Zone Id"}},"required":["productId","siteId"]}` + + // Setup + inSchemaDirectory := "../static/bidder-params" + paramsValidator, err := openrtb_ext.NewBidderParamsValidator(inSchemaDirectory) + assert.NoError(t, err, "Error initialing validator") + handler := newJsonDirectoryServer(inSchemaDirectory, paramsValidator, nil, nil) + recorder := httptest.NewRecorder() + request, err := http.NewRequest("GET", "/whatever", nil) + assert.NoError(t, err, "Error creating request") + + // Run + handler(recorder, request, nil) + + // Assertions + assert.True(t, strings.HasPrefix(recorder.Body.String(), expected)) + //assert.Equal(t, expected, recorder.Body.String()) +} From 84f852bfc7b9cba930d27511f973257605c7c36a Mon Sep 17 00:00:00 2001 From: guscarreon Date: Mon, 12 Feb 2024 18:38:39 -0500 Subject: [PATCH 14/21] jsonutils test cases --- util/jsonutil/jsonutil.go | 16 ++++- util/jsonutil/jsonutil_test.go | 120 +++++++++++++++++++++++++++++++++ 2 files changed, 133 insertions(+), 3 deletions(-) diff --git a/util/jsonutil/jsonutil.go b/util/jsonutil/jsonutil.go index 9e64a2bd36f..738958e946e 100644 --- a/util/jsonutil/jsonutil.go +++ b/util/jsonutil/jsonutil.go @@ -214,14 +214,14 @@ func isLikelyDetailedErrorMessage(msg string) bool { return !strings.HasPrefix(msg, "request.") } -var jsonRawMessageType = reflect2.TypeOfPtr(&json.RawMessage{}).Elem() - -// RawMessageExtension was created to call json.Compact() on every json.RawMessage field when getting marshalled. +// RawMessageExtension will call json.Compact() on every json.RawMessage field when getting marshalled. // All other types will be marshalled as usual type RawMessageExtension struct { jsoniter.DummyExtension } +// CreateEncoder substitutes the default jsoniter encoder of the json.RawMessage type with ours, that +// calls json.Compact() before writting to the stream func (e *RawMessageExtension) CreateEncoder(typ reflect2.Type) jsoniter.ValEncoder { if typ == jsonRawMessageType { return &rawMessageCodec{} @@ -229,8 +229,14 @@ func (e *RawMessageExtension) CreateEncoder(typ reflect2.Type) jsoniter.ValEncod return nil } +// jsonRawMessageType is declared here so we don't have to call TypeOfPtr(&json.RawMessage{}).Elem() everytime we encode +var jsonRawMessageType = reflect2.TypeOfPtr(&json.RawMessage{}).Elem() + +// rawMessageCodec implements jsoniter.ValEncoder interface so we can overshadow the default json.RawMessage Encode() +// function with our implementation type rawMessageCodec struct{} +// Encode is intended to bahave as de default json.RawMessage Encode() with the addition of the json.Compact() call func (codec *rawMessageCodec) Encode(ptr unsafe.Pointer, stream *jsoniter.Stream) { if ptr != nil { jsonRawMsg := *(*[]byte)(ptr) @@ -241,6 +247,10 @@ func (codec *rawMessageCodec) Encode(ptr unsafe.Pointer, stream *jsoniter.Stream } } +// IsEmpty is the second method of the jsoniter.ValEncoder interface func (codec *rawMessageCodec) IsEmpty(ptr unsafe.Pointer) bool { + if ptr == nil { + return true + } return *((*string)(ptr)) == "" } diff --git a/util/jsonutil/jsonutil_test.go b/util/jsonutil/jsonutil_test.go index 09fb6727309..3c7922ade15 100644 --- a/util/jsonutil/jsonutil_test.go +++ b/util/jsonutil/jsonutil_test.go @@ -1,10 +1,15 @@ package jsonutil import ( + "bytes" + "encoding/json" "errors" + "fmt" "strings" "testing" + "unsafe" + jsoniter "github.com/json-iterator/go" "github.com/stretchr/testify/assert" ) @@ -240,3 +245,118 @@ func TestTryExtractErrorMessage(t *testing.T) { }) } } + +func TestCreateEncoder(t *testing.T) { + formatted := `{ + "properties": { + "string": "Blanks spaces in between words to not be removed if compacted", + "integer": 5, + "string_array": [ + "string array elem one", + "string array elem two" + ] + } +}` + compacted := `{"properties":{"string":"Blanks spaces in between words to not be removed if compacted","integer":5,"string_array":["string array elem one","string array elem two"]}}` + + type testCase struct { + desc string + test func(t *testing.T) + } + testGroup := []struct { + desc string + tests []testCase + }{ + { + desc: "Extension registered", + tests: []testCase{ + { + desc: "JSON inside a string field is passed to Marshal(), don't expect the output to be compacted", + test: func(t *testing.T) { + jsoniter.RegisterExtension(&RawMessageExtension{}) + out, err := Marshal(formatted) + assert.NoError(t, err) + assert.NotEqual(t, compacted, string(out)) + }, + }, + { + desc: "json.RawMessage is passed to Marshal(), expect inner JSON blob to be line-break-free, tab-free, spaces only found inside strings, and compacted into one line", + test: func(t *testing.T) { + jsoniter.RegisterExtension(&RawMessageExtension{}) + out, err := Marshal(json.RawMessage(formatted)) + assert.NoError(t, err) + assert.Equal(t, compacted, string(out)) + }, + }, + }, + }, + { + desc: "Extension not registered, json.RawMessage won't get compacted", + tests: []testCase{ + { + desc: "json.RawMessage not cleared of line breaks, tabs, nor compacted into one line", + test: func(t *testing.T) { + jsoniter.RegisterExtension(&RawMessageExtension{}) + out, err := Marshal(json.RawMessage(formatted)) + assert.NoError(t, err) + assert.Equal(t, compacted, string(out)) + }, + }, + }, + }, + } + + for _, group := range testGroup { + for _, tc := range group.tests { + t.Run(fmt.Sprintf("%s - %s", group.desc, tc.desc), tc.test) + } + } +} + +func TestEncode(t *testing.T) { + jsonBlob := json.RawMessage(`{ + "properties": { + "string": "Blanks spaces in between words to not be removed if compacted", + "integer": 5, + "string_array": [ + "string array elem one", + "string array elem two" + ] + } +}`) + + testCases := []struct { + desc string + inPtr unsafe.Pointer + expectedBuffer string + expectedIsEmpty bool + }{ + { + desc: "Nil pointer, expect encoder to not write anything to buffer", + inPtr: nil, + expectedIsEmpty: true, + expectedBuffer: "", + }, + { + desc: "json.RawMessage passed, expect encoder to write the corresponding compacted json data", + inPtr: unsafe.Pointer(&jsonBlob), + expectedIsEmpty: false, + expectedBuffer: `{"properties":{"string":"Blanks spaces in between words to not be removed if compacted","integer":5,"string_array":["string array elem one","string array elem two"]}}`, + }, + } + for _, tc := range testCases { + t.Run(tc.desc, func(t *testing.T) { + // set test + encoder := &rawMessageCodec{} + output := bytes.NewBuffer([]byte{}) + stream := jsoniter.NewStream(jsonConfigValidationOn, output, len(jsonBlob)) + + // run + encoder.Encode(tc.inPtr, stream) + + // assertions + assert.Equal(t, tc.expectedBuffer, output.String()) + assert.Equal(t, tc.expectedIsEmpty, encoder.IsEmpty(tc.inPtr)) + }) + } +} From b6b8ba5a1fe09f9ebd3b5fe7289800f8596393f6 Mon Sep 17 00:00:00 2001 From: guscarreon Date: Tue, 13 Feb 2024 11:07:20 -0500 Subject: [PATCH 15/21] Cleaned up code --- endpoints/openrtb2/auction_test.go | 4 ---- router/router_test.go | 3 +-- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/endpoints/openrtb2/auction_test.go b/endpoints/openrtb2/auction_test.go index a1453faa7aa..cd7b2ddc5be 100644 --- a/endpoints/openrtb2/auction_test.go +++ b/endpoints/openrtb2/auction_test.go @@ -148,10 +148,6 @@ func TestJsonSampleRequests(t *testing.T) { } } -func TestSingleJSONTest(t *testing.T) { - runJsonBasedTest(t, "sample-requests/valid-whole/exemplary/simple.json", "") -} - func runJsonBasedTest(t *testing.T, filename, desc string) { t.Helper() diff --git a/router/router_test.go b/router/router_test.go index a16aba1ac5c..bd56c1f56f2 100644 --- a/router/router_test.go +++ b/router/router_test.go @@ -299,6 +299,5 @@ func TestBidderParamsCompactedOutput(t *testing.T) { handler(recorder, request, nil) // Assertions - assert.True(t, strings.HasPrefix(recorder.Body.String(), expected)) - //assert.Equal(t, expected, recorder.Body.String()) + assert.True(t, strings.HasPrefix(recorder.Body.String(), expectedPrefix)) } From 6ed5813dbbfbdc5f9acf876cfe1f49e21558ee59 Mon Sep 17 00:00:00 2001 From: guscarreon Date: Wed, 21 Feb 2024 10:59:31 -0500 Subject: [PATCH 16/21] Scott's 2nd review part 1 --- endpoints/openrtb2/auction_test.go | 2 +- router/router_test.go | 28 ++-- router/unit_test_files/sample_schema.json | 27 ++++ util/jsonutil/jsonutil.go | 6 +- util/jsonutil/jsonutil_test.go | 162 ++++++++++++++++------ 5 files changed, 170 insertions(+), 55 deletions(-) create mode 100644 router/unit_test_files/sample_schema.json diff --git a/endpoints/openrtb2/auction_test.go b/endpoints/openrtb2/auction_test.go index cd7b2ddc5be..9df8b540ab5 100644 --- a/endpoints/openrtb2/auction_test.go +++ b/endpoints/openrtb2/auction_test.go @@ -46,7 +46,7 @@ const jsonFileExtension string = ".json" func TestMain(m *testing.M) { jsoniter.RegisterExtension(&jsonutil.RawMessageExtension{}) - m.Run() + os.Exit(m.Run()) } func TestJsonSampleRequests(t *testing.T) { diff --git a/router/router_test.go b/router/router_test.go index bd56c1f56f2..25b77b119a2 100644 --- a/router/router_test.go +++ b/router/router_test.go @@ -5,10 +5,8 @@ import ( "net/http" "net/http/httptest" "os" - "strings" "testing" - jsoniter "github.com/json-iterator/go" "github.com/prebid/prebid-server/v2/config" "github.com/prebid/prebid-server/v2/openrtb_ext" "github.com/prebid/prebid-server/v2/util/jsonutil" @@ -20,10 +18,10 @@ const adapterDirectory = "../adapters" type testValidator struct{} -func TestMain(m *testing.M) { - jsoniter.RegisterExtension(&jsonutil.RawMessageExtension{}) - m.Run() -} +//func TestMain(m *testing.M) { +// jsoniter.RegisterExtension(&jsonutil.RawMessageExtension{}) +// os.Exit(m.Run()) +//} func (validator *testValidator) Validate(name openrtb_ext.BidderName, ext json.RawMessage) error { return nil @@ -284,20 +282,30 @@ func TestValidateDefaultAliases(t *testing.T) { } func TestBidderParamsCompactedOutput(t *testing.T) { - expectedPrefix := `{"33across":{"$schema":"http://json-schema.org/draft-04/schema#","title":"33Across Adapter Params","description":"A schema which validates params accepted by the 33Across adapter","type":"object","properties":{"productId":{"type":"string","description":"Product type"},"siteId":{"type":"string","description":"Site Id"},"zoneId":{"type":"string","description":"Zone Id"}},"required":["productId","siteId"]}` + //expectedPrefix := `{"33across":{"$schema":"http://json-schema.org/draft-04/schema#","title":"33Across Adapter Params","description":"A schema which validates params accepted by the 33Across adapter","type":"object","properties":{"productId":{"type":"string","description":"Product type"},"siteId":{"type":"string","description":"Site Id"},"zoneId":{"type":"string","description":"Zone Id"}},"required":["productId","siteId"]}` + // formattedJSON := `{ + // "integer": 5, + // "string_array": [ + // "blanks in between", + // "no_blanks_in_between" + // ] + //}` + expectedResponse := `{"$schema":"http://json-schema.org/draft-04/schema#","title":"Sample schema","description":"A sample schema to test the bidder/params endpoint","type":"object","properties":{"integer_param":{"type":"integer","minimum":1,"description":"The customer id provided by AAX."},"string_param_1":{"type":"string","minLength":1,"description":"Description blanks in between"},"string_param_2":{"type":"string","minLength":1,"description":"Description_with_no_blanks_in_between"}},"required":["integer_param","string_param_2"]}` // Setup - inSchemaDirectory := "../static/bidder-params" + inSchemaDirectory := "unit_test_files" paramsValidator, err := openrtb_ext.NewBidderParamsValidator(inSchemaDirectory) + assert.NoError(t, err, "Error initialing validator") handler := newJsonDirectoryServer(inSchemaDirectory, paramsValidator, nil, nil) recorder := httptest.NewRecorder() - request, err := http.NewRequest("GET", "/whatever", nil) + request, err := http.NewRequest("GET", "/bidder/params", nil) assert.NoError(t, err, "Error creating request") // Run handler(recorder, request, nil) // Assertions - assert.True(t, strings.HasPrefix(recorder.Body.String(), expectedPrefix)) + //assert.True(t, strings.HasPrefix(recorder.Body.String(), expectedResponse)) + assert.Equal(t, expectedResponse, recorder.Body.String()) } diff --git a/router/unit_test_files/sample_schema.json b/router/unit_test_files/sample_schema.json new file mode 100644 index 00000000000..f0c4630d98c --- /dev/null +++ b/router/unit_test_files/sample_schema.json @@ -0,0 +1,27 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "title": "Sample schema", + "description": "A sample schema to test the bidder/params endpoint", + "type": "object", + "properties": { + "integer_param": { + "type": "integer", + "minimum": 1, + "description": "The customer id provided by AAX." + }, + "string_param_1": { + "type": "string", + "minLength": 1, + "description": "Description blanks in between" + }, + "string_param_2": { + "type": "string", + "minLength": 1, + "description": "Description_with_no_blanks_in_between" + } + }, + "required": [ + "integer_param", + "string_param_2" + ] +} diff --git a/util/jsonutil/jsonutil.go b/util/jsonutil/jsonutil.go index 738958e946e..df25ee00d2f 100644 --- a/util/jsonutil/jsonutil.go +++ b/util/jsonutil/jsonutil.go @@ -215,7 +215,6 @@ func isLikelyDetailedErrorMessage(msg string) bool { } // RawMessageExtension will call json.Compact() on every json.RawMessage field when getting marshalled. -// All other types will be marshalled as usual type RawMessageExtension struct { jsoniter.DummyExtension } @@ -229,14 +228,12 @@ func (e *RawMessageExtension) CreateEncoder(typ reflect2.Type) jsoniter.ValEncod return nil } -// jsonRawMessageType is declared here so we don't have to call TypeOfPtr(&json.RawMessage{}).Elem() everytime we encode var jsonRawMessageType = reflect2.TypeOfPtr(&json.RawMessage{}).Elem() -// rawMessageCodec implements jsoniter.ValEncoder interface so we can overshadow the default json.RawMessage Encode() +// rawMessageCodec implements jsoniter.ValEncoder interface so we can override the default json.RawMessage Encode() // function with our implementation type rawMessageCodec struct{} -// Encode is intended to bahave as de default json.RawMessage Encode() with the addition of the json.Compact() call func (codec *rawMessageCodec) Encode(ptr unsafe.Pointer, stream *jsoniter.Stream) { if ptr != nil { jsonRawMsg := *(*[]byte)(ptr) @@ -247,7 +244,6 @@ func (codec *rawMessageCodec) Encode(ptr unsafe.Pointer, stream *jsoniter.Stream } } -// IsEmpty is the second method of the jsoniter.ValEncoder interface func (codec *rawMessageCodec) IsEmpty(ptr unsafe.Pointer) bool { if ptr == nil { return true diff --git a/util/jsonutil/jsonutil_test.go b/util/jsonutil/jsonutil_test.go index 3c7922ade15..7c54f8ec21d 100644 --- a/util/jsonutil/jsonutil_test.go +++ b/util/jsonutil/jsonutil_test.go @@ -246,69 +246,153 @@ func TestTryExtractErrorMessage(t *testing.T) { } } +//func TestCreateEncoder(t *testing.T) { +// formatted := `{ +// "properties": { +// "string": "Blanks spaces in between words to not be removed if compacted", +// "integer": 5, +// "string_array": [ +// "string array elem one", +// "string array elem two" +// ] +// } +//}` +// compacted := `{"properties":{"string":"Blanks spaces in between words to not be removed if compacted","integer":5,"string_array":["string array elem one","string array elem two"]}}` +// +// type testCase struct { +// desc string +// test func(t *testing.T) +// } +// testGroup := []struct { +// desc string +// tests []testCase +// }{ +// { +// desc: "Extension registered", +// tests: []testCase{ +// { +// desc: "JSON inside a string field is passed to Marshal(), don't expect the output to be compacted", +// test: func(t *testing.T) { +// jsoniter.RegisterExtension(&RawMessageExtension{}) +// out, err := Marshal(formatted) +// assert.NoError(t, err) +// assert.NotEqual(t, compacted, string(out)) +// }, +// }, +// { +// desc: "json.RawMessage is passed to Marshal(), expect inner JSON blob to be line-break-free, tab-free, spaces only found inside strings, and compacted into one line", +// test: func(t *testing.T) { +// jsoniter.RegisterExtension(&RawMessageExtension{}) +// out, err := Marshal(json.RawMessage(formatted)) +// assert.NoError(t, err) +// assert.Equal(t, compacted, string(out)) +// }, +// }, +// }, +// }, +// { +// desc: "Extension not registered, json.RawMessage won't get compacted", +// tests: []testCase{ +// { +// desc: "json.RawMessage not cleared of line breaks, tabs, nor compacted into one line", +// test: func(t *testing.T) { +// jsoniter.RegisterExtension(&RawMessageExtension{}) +// out, err := Marshal(json.RawMessage(formatted)) +// assert.NoError(t, err) +// assert.Equal(t, compacted, string(out)) +// }, +// }, +// }, +// }, +// } +// +// for _, group := range testGroup { +// for _, tc := range group.tests { +// t.Run(fmt.Sprintf("%s - %s", group.desc, tc.desc), tc.test) +// } +// } +//} + func TestCreateEncoder(t *testing.T) { - formatted := `{ - "properties": { - "string": "Blanks spaces in between words to not be removed if compacted", - "integer": 5, - "string_array": [ - "string array elem one", - "string array elem two" - ] - } -}` - compacted := `{"properties":{"string":"Blanks spaces in between words to not be removed if compacted","integer":5,"string_array":["string array elem one","string array elem two"]}}` + formattedJSON := json.RawMessage(`{ + "integer": 5, + "string_array": [ + "blanks in between", + "no_blanks_in_between" + ] +}`) + compacted := []byte(`{"integer":5,"string_array":["blanks in between","no_blanks_in_between"]}`) type testCase struct { - desc string - test func(t *testing.T) + desc string + dataIsStringType bool + expectCompacted bool + tests []testCase } + testGroup := []struct { - desc string - tests []testCase + desc string + useRawMessageExtension bool + tests []testCase }{ { - desc: "Extension registered", + desc: "Not using Extension", + useRawMessageExtension: false, tests: []testCase{ { - desc: "JSON inside a string field is passed to Marshal(), don't expect the output to be compacted", - test: func(t *testing.T) { - jsoniter.RegisterExtension(&RawMessageExtension{}) - out, err := Marshal(formatted) - assert.NoError(t, err) - assert.NotEqual(t, compacted, string(out)) - }, + desc: "string type", + dataIsStringType: true, + expectCompacted: false, }, { - desc: "json.RawMessage is passed to Marshal(), expect inner JSON blob to be line-break-free, tab-free, spaces only found inside strings, and compacted into one line", - test: func(t *testing.T) { - jsoniter.RegisterExtension(&RawMessageExtension{}) - out, err := Marshal(json.RawMessage(formatted)) - assert.NoError(t, err) - assert.Equal(t, compacted, string(out)) - }, + desc: "json.RawMessage type", + dataIsStringType: false, + expectCompacted: false, }, }, }, { - desc: "Extension not registered, json.RawMessage won't get compacted", + desc: "Using Extension", + useRawMessageExtension: true, tests: []testCase{ { - desc: "json.RawMessage not cleared of line breaks, tabs, nor compacted into one line", - test: func(t *testing.T) { - jsoniter.RegisterExtension(&RawMessageExtension{}) - out, err := Marshal(json.RawMessage(formatted)) - assert.NoError(t, err) - assert.Equal(t, compacted, string(out)) - }, + desc: "string type", + dataIsStringType: true, + expectCompacted: false, + }, + { + desc: "json.RawMessage type", + dataIsStringType: false, + expectCompacted: true, }, }, }, } for _, group := range testGroup { + if group.useRawMessageExtension { + jsoniter.RegisterExtension(&RawMessageExtension{}) + } for _, tc := range group.tests { - t.Run(fmt.Sprintf("%s - %s", group.desc, tc.desc), tc.test) + t.Run(fmt.Sprintf("%s - %s", group.desc, tc.desc), func(t *testing.T) { + var out []byte + var err error + //w := http.ResponseWriter{} + + if tc.dataIsStringType { + out, err = Marshal(string(formattedJSON)) + assert.NoError(t, err) + } else { + out, err = Marshal(formattedJSON) + } + + if tc.expectCompacted { + //w.Write(out) + assert.Equal(t, compacted, out) + } else { + assert.Equal(t, []byte(formattedJSON), out) + } + }) } } } From 4d08de6cdbbd96bee73c66e834ded21bb5a8b427 Mon Sep 17 00:00:00 2001 From: guscarreon Date: Wed, 21 Feb 2024 22:37:38 -0500 Subject: [PATCH 17/21] Scott's 2nd review part 2 --- .../appnexus.json} | 0 router/router_test.go | 26 +-- util/jsonutil/jsonutil.go | 5 +- util/jsonutil/jsonutil_test.go | 158 ++---------------- 4 files changed, 27 insertions(+), 162 deletions(-) rename router/{unit_test_files/sample_schema.json => bidder_params_tests/appnexus.json} (100%) diff --git a/router/unit_test_files/sample_schema.json b/router/bidder_params_tests/appnexus.json similarity index 100% rename from router/unit_test_files/sample_schema.json rename to router/bidder_params_tests/appnexus.json diff --git a/router/router_test.go b/router/router_test.go index 25b77b119a2..b6345c572f8 100644 --- a/router/router_test.go +++ b/router/router_test.go @@ -7,6 +7,7 @@ import ( "os" "testing" + jsoniter "github.com/json-iterator/go" "github.com/prebid/prebid-server/v2/config" "github.com/prebid/prebid-server/v2/openrtb_ext" "github.com/prebid/prebid-server/v2/util/jsonutil" @@ -18,10 +19,10 @@ const adapterDirectory = "../adapters" type testValidator struct{} -//func TestMain(m *testing.M) { -// jsoniter.RegisterExtension(&jsonutil.RawMessageExtension{}) -// os.Exit(m.Run()) -//} +func TestMain(m *testing.M) { + jsoniter.RegisterExtension(&jsonutil.RawMessageExtension{}) + os.Exit(m.Run()) +} func (validator *testValidator) Validate(name openrtb_ext.BidderName, ext json.RawMessage) error { return nil @@ -282,21 +283,13 @@ func TestValidateDefaultAliases(t *testing.T) { } func TestBidderParamsCompactedOutput(t *testing.T) { - //expectedPrefix := `{"33across":{"$schema":"http://json-schema.org/draft-04/schema#","title":"33Across Adapter Params","description":"A schema which validates params accepted by the 33Across adapter","type":"object","properties":{"productId":{"type":"string","description":"Product type"},"siteId":{"type":"string","description":"Site Id"},"zoneId":{"type":"string","description":"Zone Id"}},"required":["productId","siteId"]}` - // formattedJSON := `{ - // "integer": 5, - // "string_array": [ - // "blanks in between", - // "no_blanks_in_between" - // ] - //}` - expectedResponse := `{"$schema":"http://json-schema.org/draft-04/schema#","title":"Sample schema","description":"A sample schema to test the bidder/params endpoint","type":"object","properties":{"integer_param":{"type":"integer","minimum":1,"description":"The customer id provided by AAX."},"string_param_1":{"type":"string","minLength":1,"description":"Description blanks in between"},"string_param_2":{"type":"string","minLength":1,"description":"Description_with_no_blanks_in_between"}},"required":["integer_param","string_param_2"]}` + expectedFormattedResponse := `{"appnexus":{"$schema":"http://json-schema.org/draft-04/schema#","title":"Sample schema","description":"A sample schema to test the bidder/params endpoint","type":"object","properties":{"integer_param":{"type":"integer","minimum":1,"description":"The customer id provided by AAX."},"string_param_1":{"type":"string","minLength":1,"description":"Description blanks in between"},"string_param_2":{"type":"string","minLength":1,"description":"Description_with_no_blanks_in_between"}},"required":["integer_param","string_param_2"]}}` // Setup - inSchemaDirectory := "unit_test_files" + inSchemaDirectory := "bidder_params_tests" paramsValidator, err := openrtb_ext.NewBidderParamsValidator(inSchemaDirectory) - assert.NoError(t, err, "Error initialing validator") + handler := newJsonDirectoryServer(inSchemaDirectory, paramsValidator, nil, nil) recorder := httptest.NewRecorder() request, err := http.NewRequest("GET", "/bidder/params", nil) @@ -306,6 +299,5 @@ func TestBidderParamsCompactedOutput(t *testing.T) { handler(recorder, request, nil) // Assertions - //assert.True(t, strings.HasPrefix(recorder.Body.String(), expectedResponse)) - assert.Equal(t, expectedResponse, recorder.Body.String()) + assert.Equal(t, expectedFormattedResponse, recorder.Body.String()) } diff --git a/util/jsonutil/jsonutil.go b/util/jsonutil/jsonutil.go index df25ee00d2f..5432bb9e44d 100644 --- a/util/jsonutil/jsonutil.go +++ b/util/jsonutil/jsonutil.go @@ -245,8 +245,5 @@ func (codec *rawMessageCodec) Encode(ptr unsafe.Pointer, stream *jsoniter.Stream } func (codec *rawMessageCodec) IsEmpty(ptr unsafe.Pointer) bool { - if ptr == nil { - return true - } - return *((*string)(ptr)) == "" + return ptr == nil || len(*((*json.RawMessage)(ptr))) == 0 } diff --git a/util/jsonutil/jsonutil_test.go b/util/jsonutil/jsonutil_test.go index 7c54f8ec21d..a34a4179cf0 100644 --- a/util/jsonutil/jsonutil_test.go +++ b/util/jsonutil/jsonutil_test.go @@ -4,12 +4,12 @@ import ( "bytes" "encoding/json" "errors" - "fmt" "strings" "testing" "unsafe" jsoniter "github.com/json-iterator/go" + "github.com/modern-go/reflect2" "github.com/stretchr/testify/assert" ) @@ -246,154 +246,30 @@ func TestTryExtractErrorMessage(t *testing.T) { } } -//func TestCreateEncoder(t *testing.T) { -// formatted := `{ -// "properties": { -// "string": "Blanks spaces in between words to not be removed if compacted", -// "integer": 5, -// "string_array": [ -// "string array elem one", -// "string array elem two" -// ] -// } -//}` -// compacted := `{"properties":{"string":"Blanks spaces in between words to not be removed if compacted","integer":5,"string_array":["string array elem one","string array elem two"]}}` -// -// type testCase struct { -// desc string -// test func(t *testing.T) -// } -// testGroup := []struct { -// desc string -// tests []testCase -// }{ -// { -// desc: "Extension registered", -// tests: []testCase{ -// { -// desc: "JSON inside a string field is passed to Marshal(), don't expect the output to be compacted", -// test: func(t *testing.T) { -// jsoniter.RegisterExtension(&RawMessageExtension{}) -// out, err := Marshal(formatted) -// assert.NoError(t, err) -// assert.NotEqual(t, compacted, string(out)) -// }, -// }, -// { -// desc: "json.RawMessage is passed to Marshal(), expect inner JSON blob to be line-break-free, tab-free, spaces only found inside strings, and compacted into one line", -// test: func(t *testing.T) { -// jsoniter.RegisterExtension(&RawMessageExtension{}) -// out, err := Marshal(json.RawMessage(formatted)) -// assert.NoError(t, err) -// assert.Equal(t, compacted, string(out)) -// }, -// }, -// }, -// }, -// { -// desc: "Extension not registered, json.RawMessage won't get compacted", -// tests: []testCase{ -// { -// desc: "json.RawMessage not cleared of line breaks, tabs, nor compacted into one line", -// test: func(t *testing.T) { -// jsoniter.RegisterExtension(&RawMessageExtension{}) -// out, err := Marshal(json.RawMessage(formatted)) -// assert.NoError(t, err) -// assert.Equal(t, compacted, string(out)) -// }, -// }, -// }, -// }, -// } -// -// for _, group := range testGroup { -// for _, tc := range group.tests { -// t.Run(fmt.Sprintf("%s - %s", group.desc, tc.desc), tc.test) -// } -// } -//} - func TestCreateEncoder(t *testing.T) { - formattedJSON := json.RawMessage(`{ - "integer": 5, - "string_array": [ - "blanks in between", - "no_blanks_in_between" - ] -}`) - compacted := []byte(`{"integer":5,"string_array":["blanks in between","no_blanks_in_between"]}`) - - type testCase struct { - desc string - dataIsStringType bool - expectCompacted bool - tests []testCase - } - - testGroup := []struct { - desc string - useRawMessageExtension bool - tests []testCase + testCases := []struct { + desc string + inType reflect2.Type + expectedValEncoder jsoniter.ValEncoder }{ { - desc: "Not using Extension", - useRawMessageExtension: false, - tests: []testCase{ - { - desc: "string type", - dataIsStringType: true, - expectCompacted: false, - }, - { - desc: "json.RawMessage type", - dataIsStringType: false, - expectCompacted: false, - }, - }, + desc: "Extension registered", + inType: reflect2.TypeOfPtr((*jsoniter.Any)(nil)).Elem(), + expectedValEncoder: nil, }, { - desc: "Using Extension", - useRawMessageExtension: true, - tests: []testCase{ - { - desc: "string type", - dataIsStringType: true, - expectCompacted: false, - }, - { - desc: "json.RawMessage type", - dataIsStringType: false, - expectCompacted: true, - }, - }, + desc: "Extension not registered, json.RawMessage won't get compacted", + inType: reflect2.TypeOfPtr(&json.RawMessage{}).Elem(), + expectedValEncoder: &rawMessageCodec{}, }, } - for _, group := range testGroup { - if group.useRawMessageExtension { - jsoniter.RegisterExtension(&RawMessageExtension{}) - } - for _, tc := range group.tests { - t.Run(fmt.Sprintf("%s - %s", group.desc, tc.desc), func(t *testing.T) { - var out []byte - var err error - //w := http.ResponseWriter{} - - if tc.dataIsStringType { - out, err = Marshal(string(formattedJSON)) - assert.NoError(t, err) - } else { - out, err = Marshal(formattedJSON) - } - - if tc.expectCompacted { - //w.Write(out) - assert.Equal(t, compacted, out) - } else { - assert.Equal(t, []byte(formattedJSON), out) - } - }) - } + for _, tc := range testCases { + t.Run(tc.desc, func(t *testing.T) { + extension := &RawMessageExtension{} + encoder := extension.CreateEncoder(tc.inType) + assert.IsType(t, encoder, tc.expectedValEncoder) + }) } } From 4978277b35f1c32b61244a80a101e364f03d287d Mon Sep 17 00:00:00 2001 From: guscarreon Date: Thu, 22 Feb 2024 10:33:06 -0500 Subject: [PATCH 18/21] Scott's 2nd review part 3 --- util/jsonutil/jsonutil.go | 5 +-- util/jsonutil/jsonutil_test.go | 56 ++++++++++++++++++---------------- 2 files changed, 32 insertions(+), 29 deletions(-) diff --git a/util/jsonutil/jsonutil.go b/util/jsonutil/jsonutil.go index 5432bb9e44d..695ccd8a5c1 100644 --- a/util/jsonutil/jsonutil.go +++ b/util/jsonutil/jsonutil.go @@ -239,8 +239,9 @@ func (codec *rawMessageCodec) Encode(ptr unsafe.Pointer, stream *jsoniter.Stream jsonRawMsg := *(*[]byte)(ptr) dst := bytes.NewBuffer(make([]byte, 0, len(jsonRawMsg))) - json.Compact(dst, jsonRawMsg) - stream.Write(dst.Bytes()) + if err := json.Compact(dst, jsonRawMsg); err == nil { + stream.Write(dst.Bytes()) + } } } diff --git a/util/jsonutil/jsonutil_test.go b/util/jsonutil/jsonutil_test.go index a34a4179cf0..96632d62548 100644 --- a/util/jsonutil/jsonutil_test.go +++ b/util/jsonutil/jsonutil_test.go @@ -253,12 +253,12 @@ func TestCreateEncoder(t *testing.T) { expectedValEncoder jsoniter.ValEncoder }{ { - desc: "Extension registered", + desc: "With_extension", inType: reflect2.TypeOfPtr((*jsoniter.Any)(nil)).Elem(), expectedValEncoder: nil, }, { - desc: "Extension not registered, json.RawMessage won't get compacted", + desc: "No_extension", inType: reflect2.TypeOfPtr(&json.RawMessage{}).Elem(), expectedValEncoder: &rawMessageCodec{}, }, @@ -285,38 +285,40 @@ func TestEncode(t *testing.T) { } }`) - testCases := []struct { - desc string - inPtr unsafe.Pointer - expectedBuffer string - expectedIsEmpty bool - }{ - { - desc: "Nil pointer, expect encoder to not write anything to buffer", - inPtr: nil, - expectedIsEmpty: true, - expectedBuffer: "", - }, - { - desc: "json.RawMessage passed, expect encoder to write the corresponding compacted json data", - inPtr: unsafe.Pointer(&jsonBlob), - expectedIsEmpty: false, - expectedBuffer: `{"properties":{"string":"Blanks spaces in between words to not be removed if compacted","integer":5,"string_array":["string array elem one","string array elem two"]}}`, + t.Run( + "Nil_pointer", + func(t *testing.T) { + // set test + encoder := &rawMessageCodec{} + output := bytes.NewBuffer([]byte{}) + stream := jsoniter.NewStream(jsonConfigValidationOn, output, len(jsonBlob)) + + // run + encoder.Encode(nil, stream) + + // assertions + assert.Equal(t, "", output.String()) + assert.Equal(t, true, encoder.IsEmpty(nil)) }, - } - for _, tc := range testCases { - t.Run(tc.desc, func(t *testing.T) { + ) + t.Run( + "json.RawMessage_compact_JSON", + func(t *testing.T) { // set test encoder := &rawMessageCodec{} output := bytes.NewBuffer([]byte{}) stream := jsoniter.NewStream(jsonConfigValidationOn, output, len(jsonBlob)) // run - encoder.Encode(tc.inPtr, stream) + encoder.Encode(unsafe.Pointer(&jsonBlob), stream) // assertions - assert.Equal(t, tc.expectedBuffer, output.String()) - assert.Equal(t, tc.expectedIsEmpty, encoder.IsEmpty(tc.inPtr)) - }) - } + assert.Equal( + t, + `{"properties":{"string":"Blanks spaces in between words to not be removed if compacted","integer":5,"string_array":["string array elem one","string array elem two"]}}`, + output.String(), + ) + assert.Equal(t, false, encoder.IsEmpty(unsafe.Pointer(&jsonBlob))) + }, + ) } From 0fc393391b80e69db85889986e412fe7b41fcba1 Mon Sep 17 00:00:00 2001 From: guscarreon Date: Tue, 27 Feb 2024 16:01:02 -0500 Subject: [PATCH 19/21] LoadSchema function --- openrtb_ext/bidders.go | 23 +++++++++++----- .../{appnexus.json => foo.json} | 6 ++--- router/router.go | 7 +++-- router/router_test.go | 27 ++++++++++++++----- 4 files changed, 43 insertions(+), 20 deletions(-) rename router/bidder_params_tests/{appnexus.json => foo.json} (75%) diff --git a/openrtb_ext/bidders.go b/openrtb_ext/bidders.go index 173120f7301..d7df2bb2ddf 100644 --- a/openrtb_ext/bidders.go +++ b/openrtb_ext/bidders.go @@ -612,14 +612,10 @@ func NewBidderParamsValidator(schemaDirectory string) (BidderParamValidator, err if _, ok := bidderMap[bidderName]; !ok { return nil, fmt.Errorf("File %s/%s does not match a valid BidderName.", schemaDirectory, fileInfo.Name()) } - toOpen, err := paramsValidator.abs(filepath.Join(schemaDirectory, fileInfo.Name())) - if err != nil { - return nil, fmt.Errorf("Failed to get an absolute representation of the path: %s, %v", toOpen, err) - } - schemaLoader := paramsValidator.newReferenceLoader("file:///" + filepath.ToSlash(toOpen)) - loadedSchema, err := paramsValidator.newSchema(schemaLoader) + + loadedSchema, err := LoadSchema(schemaDirectory, fileInfo.Name()) if err != nil { - return nil, fmt.Errorf("Failed to load json schema at %s: %v", toOpen, err) + return nil, err } fileBytes, err := paramsValidator.readFile(fmt.Sprintf("%s/%s", schemaDirectory, fileInfo.Name())) @@ -646,6 +642,19 @@ func NewBidderParamsValidator(schemaDirectory string) (BidderParamValidator, err }, nil } +func LoadSchema(schemaDirectory, filename string) (*gojsonschema.Schema, error) { + toOpen, err := paramsValidator.abs(filepath.Join(schemaDirectory, filename)) + if err != nil { + return nil, fmt.Errorf("Failed to get an absolute representation of the path: %s, %v", toOpen, err) + } + schemaLoader := paramsValidator.newReferenceLoader("file:///" + filepath.ToSlash(toOpen)) + loadedSchema, err := paramsValidator.newSchema(schemaLoader) + if err != nil { + return nil, fmt.Errorf("Failed to load json schema at %s: %v", toOpen, err) + } + return loadedSchema, nil +} + type bidderParamValidator struct { schemaContents map[BidderName]string parsedSchemas map[BidderName]*gojsonschema.Schema diff --git a/router/bidder_params_tests/appnexus.json b/router/bidder_params_tests/foo.json similarity index 75% rename from router/bidder_params_tests/appnexus.json rename to router/bidder_params_tests/foo.json index f0c4630d98c..11dedd41e49 100644 --- a/router/bidder_params_tests/appnexus.json +++ b/router/bidder_params_tests/foo.json @@ -7,17 +7,17 @@ "integer_param": { "type": "integer", "minimum": 1, - "description": "The customer id provided by AAX." + "description": "A customer id" }, "string_param_1": { "type": "string", "minLength": 1, - "description": "Description blanks in between" + "description": "Text with blanks in between" }, "string_param_2": { "type": "string", "minLength": 1, - "description": "Description_with_no_blanks_in_between" + "description": "Text_with_no_blanks_in_between" } }, "required": [ diff --git a/router/router.go b/router/router.go index c285bbdab1b..6729daa6838 100644 --- a/router/router.go +++ b/router/router.go @@ -57,18 +57,17 @@ import ( // This function stores the file contents in memory, and should not be used on large directories. // If the root directory, or any of the files in it, cannot be read, then the program will exit. func NewJsonDirectoryServer(schemaDirectory string, validator openrtb_ext.BidderParamValidator, aliases map[string]string) httprouter.Handle { - return newJsonDirectoryServer(schemaDirectory, validator, aliases, openrtb_ext.GetAliasBidderToParent()) + bidderMap := openrtb_ext.BuildBidderMap() + return newJsonDirectoryServer(schemaDirectory, validator, aliases, openrtb_ext.GetAliasBidderToParent(), bidderMap) } -func newJsonDirectoryServer(schemaDirectory string, validator openrtb_ext.BidderParamValidator, aliases map[string]string, yamlAliases map[openrtb_ext.BidderName]openrtb_ext.BidderName) httprouter.Handle { +func newJsonDirectoryServer(schemaDirectory string, validator openrtb_ext.BidderParamValidator, aliases map[string]string, yamlAliases map[openrtb_ext.BidderName]openrtb_ext.BidderName, bidderMap map[string]openrtb_ext.BidderName) httprouter.Handle { // Slurp the files into memory first, since they're small and it minimizes request latency. files, err := os.ReadDir(schemaDirectory) if err != nil { glog.Fatalf("Failed to read directory %s: %v", schemaDirectory, err) } - bidderMap := openrtb_ext.BuildBidderMap() - data := make(map[string]json.RawMessage, len(files)) for _, file := range files { bidder := strings.TrimSuffix(file.Name(), ".json") diff --git a/router/router_test.go b/router/router_test.go index b6345c572f8..54b25087846 100644 --- a/router/router_test.go +++ b/router/router_test.go @@ -11,6 +11,7 @@ import ( "github.com/prebid/prebid-server/v2/config" "github.com/prebid/prebid-server/v2/openrtb_ext" "github.com/prebid/prebid-server/v2/util/jsonutil" + "github.com/xeipuuv/gojsonschema" "github.com/stretchr/testify/assert" ) @@ -46,7 +47,8 @@ func ensureHasKey(t *testing.T, data map[string]json.RawMessage, key string) { func TestNewJsonDirectoryServer(t *testing.T) { defaultAlias := map[string]string{"aliastest": "appnexus"} yamlAlias := map[openrtb_ext.BidderName]openrtb_ext.BidderName{openrtb_ext.BidderName("alias"): openrtb_ext.BidderName("parentAlias")} - handler := newJsonDirectoryServer("../static/bidder-params", &testValidator{}, defaultAlias, yamlAlias) + bidderMap := openrtb_ext.BuildBidderMap() + handler := newJsonDirectoryServer("../static/bidder-params", &testValidator{}, defaultAlias, yamlAlias, bidderMap) recorder := httptest.NewRecorder() request, _ := http.NewRequest("GET", "/whatever", nil) handler(recorder, request, nil) @@ -283,14 +285,27 @@ func TestValidateDefaultAliases(t *testing.T) { } func TestBidderParamsCompactedOutput(t *testing.T) { - expectedFormattedResponse := `{"appnexus":{"$schema":"http://json-schema.org/draft-04/schema#","title":"Sample schema","description":"A sample schema to test the bidder/params endpoint","type":"object","properties":{"integer_param":{"type":"integer","minimum":1,"description":"The customer id provided by AAX."},"string_param_1":{"type":"string","minLength":1,"description":"Description blanks in between"},"string_param_2":{"type":"string","minLength":1,"description":"Description_with_no_blanks_in_between"}},"required":["integer_param","string_param_2"]}}` + expectedFormattedResponse := `{"foo":{"$schema":"http://json-schema.org/draft-04/schema#","title":"Sample schema","description":"A sample schema to test the bidder/params endpoint","type":"object","properties":{"integer_param":{"type":"integer","minimum":1,"description":"The customer id provided by AAX."},"string_param_1":{"type":"string","minLength":1,"description":"Description blanks in between"},"string_param_2":{"type":"string","minLength":1,"description":"Description_with_no_blanks_in_between"}},"required":["integer_param","string_param_2"]}}` + inSchemaDirectory := "bidder_params_tests" + inSchemaFile := "foo.json" // Setup - inSchemaDirectory := "bidder_params_tests" - paramsValidator, err := openrtb_ext.NewBidderParamsValidator(inSchemaDirectory) - assert.NoError(t, err, "Error initialing validator") + biddermap := map[string]openrtb_ext.BidderName{ + "foo": openrtb_ext.BidderName("foo"), + } + + fooSchema, err := openrtb_ext.LoadSchema(inSchemaDirectory, inSchemaFile) + assert.NoError(t, err, "Error loading schema for %s/%s: %v", inSchemaDirectory, inSchemaFile) + + fileBytes, err := os.ReadFile("bidder_params_tests/foo.json") + assert.NoError(t, err, "Error reading test schema file %s", inSchemaDirectory, inSchemaFile) + + testParamsValidator := &bidderParamValidator{ + parsedSchemas: map[BidderName]*gojsonschema.Schema{openrtb_ext.BidderName("foo"): fooSchema}, + schemaContents: map[openrtb_ext.BidderName]string{openrtb_ext.BidderName("foo"): string(fileBytes)}, + } - handler := newJsonDirectoryServer(inSchemaDirectory, paramsValidator, nil, nil) + handler := newJsonDirectoryServer(inSchemaDirectory, testParamsValidator, nil, nil, biddermap) recorder := httptest.NewRecorder() request, err := http.NewRequest("GET", "/bidder/params", nil) assert.NoError(t, err, "Error creating request") From 9f13d7567a6a6b81249cd2e2d2b72ec62d086f06 Mon Sep 17 00:00:00 2001 From: guscarreon Date: Tue, 27 Feb 2024 16:30:35 -0500 Subject: [PATCH 20/21] InitBidderParamsValidator --- openrtb_ext/bidders.go | 6 +++++- router/router_test.go | 10 +++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/openrtb_ext/bidders.go b/openrtb_ext/bidders.go index d7df2bb2ddf..220c216ef2b 100644 --- a/openrtb_ext/bidders.go +++ b/openrtb_ext/bidders.go @@ -636,10 +636,14 @@ func NewBidderParamsValidator(schemaDirectory string) (BidderParamValidator, err schemaContents[alias] = parentSchemaContents } + return InitBidderParamsValidator(schemaContents, schemas), nil +} + +func InitBidderParamsValidator(schemaContents map[BidderName]string, schemas map[BidderName]*gojsonschema.Schema) *bidderParamValidator { return &bidderParamValidator{ schemaContents: schemaContents, parsedSchemas: schemas, - }, nil + } } func LoadSchema(schemaDirectory, filename string) (*gojsonschema.Schema, error) { diff --git a/router/router_test.go b/router/router_test.go index 54b25087846..1541cb634cc 100644 --- a/router/router_test.go +++ b/router/router_test.go @@ -285,7 +285,7 @@ func TestValidateDefaultAliases(t *testing.T) { } func TestBidderParamsCompactedOutput(t *testing.T) { - expectedFormattedResponse := `{"foo":{"$schema":"http://json-schema.org/draft-04/schema#","title":"Sample schema","description":"A sample schema to test the bidder/params endpoint","type":"object","properties":{"integer_param":{"type":"integer","minimum":1,"description":"The customer id provided by AAX."},"string_param_1":{"type":"string","minLength":1,"description":"Description blanks in between"},"string_param_2":{"type":"string","minLength":1,"description":"Description_with_no_blanks_in_between"}},"required":["integer_param","string_param_2"]}}` + expectedFormattedResponse := `{"foo":{"$schema":"http://json-schema.org/draft-04/schema#","title":"Sample schema","description":"A sample schema to test the bidder/params endpoint","type":"object","properties":{"integer_param":{"type":"integer","minimum":1,"description":"A customer id"},"string_param_1":{"type":"string","minLength":1,"description":"Text with blanks in between"},"string_param_2":{"type":"string","minLength":1,"description":"Text_with_no_blanks_in_between"}},"required":["integer_param","string_param_2"]}}` inSchemaDirectory := "bidder_params_tests" inSchemaFile := "foo.json" @@ -300,10 +300,10 @@ func TestBidderParamsCompactedOutput(t *testing.T) { fileBytes, err := os.ReadFile("bidder_params_tests/foo.json") assert.NoError(t, err, "Error reading test schema file %s", inSchemaDirectory, inSchemaFile) - testParamsValidator := &bidderParamValidator{ - parsedSchemas: map[BidderName]*gojsonschema.Schema{openrtb_ext.BidderName("foo"): fooSchema}, - schemaContents: map[openrtb_ext.BidderName]string{openrtb_ext.BidderName("foo"): string(fileBytes)}, - } + testParamsValidator := openrtb_ext.InitBidderParamsValidator( + map[openrtb_ext.BidderName]string{openrtb_ext.BidderName("foo"): string(fileBytes)}, + map[openrtb_ext.BidderName]*gojsonschema.Schema{openrtb_ext.BidderName("foo"): fooSchema}, + ) handler := newJsonDirectoryServer(inSchemaDirectory, testParamsValidator, nil, nil, biddermap) recorder := httptest.NewRecorder() From dbebd64e2a25e2a7e8624f19cd7ed5ef3a96ea9c Mon Sep 17 00:00:00 2001 From: guscarreon Date: Wed, 28 Feb 2024 13:05:24 -0500 Subject: [PATCH 21/21] Rolledback functions --- openrtb_ext/bidders.go | 28 ++++++------------- .../{foo.json => appnexus.json} | 0 router/router.go | 7 +++-- router/router_test.go | 27 ++++-------------- 4 files changed, 18 insertions(+), 44 deletions(-) rename router/bidder_params_tests/{foo.json => appnexus.json} (100%) diff --git a/openrtb_ext/bidders.go b/openrtb_ext/bidders.go index 220c216ef2b..fcd2b15ce22 100644 --- a/openrtb_ext/bidders.go +++ b/openrtb_ext/bidders.go @@ -613,9 +613,14 @@ func NewBidderParamsValidator(schemaDirectory string) (BidderParamValidator, err return nil, fmt.Errorf("File %s/%s does not match a valid BidderName.", schemaDirectory, fileInfo.Name()) } - loadedSchema, err := LoadSchema(schemaDirectory, fileInfo.Name()) + toOpen, err := paramsValidator.abs(filepath.Join(schemaDirectory, fileInfo.Name())) if err != nil { - return nil, err + return nil, fmt.Errorf("Failed to get an absolute representation of the path: %s, %v", toOpen, err) + } + schemaLoader := paramsValidator.newReferenceLoader("file:///" + filepath.ToSlash(toOpen)) + loadedSchema, err := paramsValidator.newSchema(schemaLoader) + if err != nil { + return nil, fmt.Errorf("Failed to load json schema at %s: %v", toOpen, err) } fileBytes, err := paramsValidator.readFile(fmt.Sprintf("%s/%s", schemaDirectory, fileInfo.Name())) @@ -636,27 +641,10 @@ func NewBidderParamsValidator(schemaDirectory string) (BidderParamValidator, err schemaContents[alias] = parentSchemaContents } - return InitBidderParamsValidator(schemaContents, schemas), nil -} - -func InitBidderParamsValidator(schemaContents map[BidderName]string, schemas map[BidderName]*gojsonschema.Schema) *bidderParamValidator { return &bidderParamValidator{ schemaContents: schemaContents, parsedSchemas: schemas, - } -} - -func LoadSchema(schemaDirectory, filename string) (*gojsonschema.Schema, error) { - toOpen, err := paramsValidator.abs(filepath.Join(schemaDirectory, filename)) - if err != nil { - return nil, fmt.Errorf("Failed to get an absolute representation of the path: %s, %v", toOpen, err) - } - schemaLoader := paramsValidator.newReferenceLoader("file:///" + filepath.ToSlash(toOpen)) - loadedSchema, err := paramsValidator.newSchema(schemaLoader) - if err != nil { - return nil, fmt.Errorf("Failed to load json schema at %s: %v", toOpen, err) - } - return loadedSchema, nil + }, nil } type bidderParamValidator struct { diff --git a/router/bidder_params_tests/foo.json b/router/bidder_params_tests/appnexus.json similarity index 100% rename from router/bidder_params_tests/foo.json rename to router/bidder_params_tests/appnexus.json diff --git a/router/router.go b/router/router.go index 6729daa6838..c285bbdab1b 100644 --- a/router/router.go +++ b/router/router.go @@ -57,17 +57,18 @@ import ( // This function stores the file contents in memory, and should not be used on large directories. // If the root directory, or any of the files in it, cannot be read, then the program will exit. func NewJsonDirectoryServer(schemaDirectory string, validator openrtb_ext.BidderParamValidator, aliases map[string]string) httprouter.Handle { - bidderMap := openrtb_ext.BuildBidderMap() - return newJsonDirectoryServer(schemaDirectory, validator, aliases, openrtb_ext.GetAliasBidderToParent(), bidderMap) + return newJsonDirectoryServer(schemaDirectory, validator, aliases, openrtb_ext.GetAliasBidderToParent()) } -func newJsonDirectoryServer(schemaDirectory string, validator openrtb_ext.BidderParamValidator, aliases map[string]string, yamlAliases map[openrtb_ext.BidderName]openrtb_ext.BidderName, bidderMap map[string]openrtb_ext.BidderName) httprouter.Handle { +func newJsonDirectoryServer(schemaDirectory string, validator openrtb_ext.BidderParamValidator, aliases map[string]string, yamlAliases map[openrtb_ext.BidderName]openrtb_ext.BidderName) httprouter.Handle { // Slurp the files into memory first, since they're small and it minimizes request latency. files, err := os.ReadDir(schemaDirectory) if err != nil { glog.Fatalf("Failed to read directory %s: %v", schemaDirectory, err) } + bidderMap := openrtb_ext.BuildBidderMap() + data := make(map[string]json.RawMessage, len(files)) for _, file := range files { bidder := strings.TrimSuffix(file.Name(), ".json") diff --git a/router/router_test.go b/router/router_test.go index 1541cb634cc..866a8440f3f 100644 --- a/router/router_test.go +++ b/router/router_test.go @@ -11,7 +11,6 @@ import ( "github.com/prebid/prebid-server/v2/config" "github.com/prebid/prebid-server/v2/openrtb_ext" "github.com/prebid/prebid-server/v2/util/jsonutil" - "github.com/xeipuuv/gojsonschema" "github.com/stretchr/testify/assert" ) @@ -47,8 +46,7 @@ func ensureHasKey(t *testing.T, data map[string]json.RawMessage, key string) { func TestNewJsonDirectoryServer(t *testing.T) { defaultAlias := map[string]string{"aliastest": "appnexus"} yamlAlias := map[openrtb_ext.BidderName]openrtb_ext.BidderName{openrtb_ext.BidderName("alias"): openrtb_ext.BidderName("parentAlias")} - bidderMap := openrtb_ext.BuildBidderMap() - handler := newJsonDirectoryServer("../static/bidder-params", &testValidator{}, defaultAlias, yamlAlias, bidderMap) + handler := newJsonDirectoryServer("../static/bidder-params", &testValidator{}, defaultAlias, yamlAlias) recorder := httptest.NewRecorder() request, _ := http.NewRequest("GET", "/whatever", nil) handler(recorder, request, nil) @@ -285,27 +283,14 @@ func TestValidateDefaultAliases(t *testing.T) { } func TestBidderParamsCompactedOutput(t *testing.T) { - expectedFormattedResponse := `{"foo":{"$schema":"http://json-schema.org/draft-04/schema#","title":"Sample schema","description":"A sample schema to test the bidder/params endpoint","type":"object","properties":{"integer_param":{"type":"integer","minimum":1,"description":"A customer id"},"string_param_1":{"type":"string","minLength":1,"description":"Text with blanks in between"},"string_param_2":{"type":"string","minLength":1,"description":"Text_with_no_blanks_in_between"}},"required":["integer_param","string_param_2"]}}` - inSchemaDirectory := "bidder_params_tests" - inSchemaFile := "foo.json" + expectedFormattedResponse := `{"appnexus":{"$schema":"http://json-schema.org/draft-04/schema#","title":"Sample schema","description":"A sample schema to test the bidder/params endpoint","type":"object","properties":{"integer_param":{"type":"integer","minimum":1,"description":"A customer id"},"string_param_1":{"type":"string","minLength":1,"description":"Text with blanks in between"},"string_param_2":{"type":"string","minLength":1,"description":"Text_with_no_blanks_in_between"}},"required":["integer_param","string_param_2"]}}` // Setup - biddermap := map[string]openrtb_ext.BidderName{ - "foo": openrtb_ext.BidderName("foo"), - } - - fooSchema, err := openrtb_ext.LoadSchema(inSchemaDirectory, inSchemaFile) - assert.NoError(t, err, "Error loading schema for %s/%s: %v", inSchemaDirectory, inSchemaFile) - - fileBytes, err := os.ReadFile("bidder_params_tests/foo.json") - assert.NoError(t, err, "Error reading test schema file %s", inSchemaDirectory, inSchemaFile) - - testParamsValidator := openrtb_ext.InitBidderParamsValidator( - map[openrtb_ext.BidderName]string{openrtb_ext.BidderName("foo"): string(fileBytes)}, - map[openrtb_ext.BidderName]*gojsonschema.Schema{openrtb_ext.BidderName("foo"): fooSchema}, - ) + inSchemaDirectory := "bidder_params_tests" + paramsValidator, err := openrtb_ext.NewBidderParamsValidator(inSchemaDirectory) + assert.NoError(t, err, "Error initialing validator") - handler := newJsonDirectoryServer(inSchemaDirectory, testParamsValidator, nil, nil, biddermap) + handler := newJsonDirectoryServer(inSchemaDirectory, paramsValidator, nil, nil) recorder := httptest.NewRecorder() request, err := http.NewRequest("GET", "/bidder/params", nil) assert.NoError(t, err, "Error creating request")