diff --git a/core/templating/templating_test.go b/core/templating/templating_test.go index e2bef290d..be468ca1f 100644 --- a/core/templating/templating_test.go +++ b/core/templating/templating_test.go @@ -495,6 +495,18 @@ func Test_ApplyTemplate_Request_Body_Jsonpath_LargeInt(t *testing.T) { Expect(template).To(Equal("5553686208582")) } +func Test_ApplyTemplate_Request_Body_Jsonpath_From_Xml(t *testing.T) { + RegisterTestingT(t) + + template, err := ApplyTemplate(&models.RequestDetails{ + Body: `Johnny`, + }, make(map[string]string), `{{ Request.Body 'jsonpathfromxml' '$.name' }}`) + + Expect(err).To(BeNil()) + + Expect(template).To(Equal("Johnny")) +} + func Test_ApplyTemplate_ReplaceStringInQueryParams(t *testing.T) { RegisterTestingT(t) diff --git a/core/util/util.go b/core/util/util.go index fe96e65f3..c4ba2f9d6 100644 --- a/core/util/util.go +++ b/core/util/util.go @@ -25,6 +25,7 @@ import ( "strconv" "time" + xj "github.com/basgys/goxml2json" "github.com/tdewolff/minify/v2" mjson "github.com/tdewolff/minify/v2/json" "github.com/tdewolff/minify/v2/xml" @@ -374,24 +375,22 @@ func RandStringFromTimestamp(length int) string { func FetchFromRequestBody(queryType, query, toMatch string) interface{} { if queryType == "jsonpath" { - result := jsonPath(query, toMatch) - var data interface{} - err := json.Unmarshal([]byte(result), &data) - - arrayData, ok := data.([]interface{}) - - if err != nil || !ok { - return result - } - return arrayData + return jsonPath(query, toMatch) } else if queryType == "xpath" { return xPath(query, toMatch) + } else if queryType == "jsonpathfromxml" { + xmlReader := strings.NewReader(toMatch) + jsonBytes, err := xj.Convert(xmlReader) + if err != nil { + return "" + } + return jsonPath(query, jsonBytes.String()) } log.Errorf("Unknown query type \"%s\" for templating Request.Body", queryType) return "" } -func jsonPath(query, toMatch string) string { +func jsonPath(query, toMatch string) interface{} { query = PrepareJsonPathQuery(query) result, err := JsonPathExecution(query, toMatch) @@ -408,7 +407,16 @@ func jsonPath(query, toMatch string) string { result = strconv.Itoa(intResult) } - return result + // convert to array data if applicable + var data interface{} + err = json.Unmarshal([]byte(result), &data) + + arrayData, ok := data.([]interface{}) + + if err != nil || !ok { + return result + } + return arrayData } func xPath(query, toMatch string) string { diff --git a/go.mod b/go.mod index e29815001..2ea7afdbf 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/SpectoLabs/goproxy/ext v0.0.0-20220724221645-71c396c297b7 github.com/SpectoLabs/raymond v2.0.3-0.20240313210732-e0e216cf0920+incompatible github.com/antonholmquist/jason v1.0.1-0.20160829104012-962e09b85496 + github.com/basgys/goxml2json v1.1.0 github.com/beevik/etree v1.1.0 github.com/boltdb/bolt v1.2.1-0.20160424201119-d97499360d1e github.com/brianvoe/gofakeit/v6 v6.19.0 @@ -46,6 +47,7 @@ require ( require ( github.com/BurntSushi/toml v1.2.1 // indirect github.com/aymerick/raymond v2.0.2+incompatible // indirect + github.com/bitly/go-simplejson v0.5.1 // indirect github.com/corpix/uarand v0.0.0-20170903190822-2b8494104d86 // indirect github.com/fsnotify/fsnotify v1.5.4 // indirect github.com/google/go-cmp v0.5.9 // indirect diff --git a/go.sum b/go.sum index 175d00a37..07f2d7989 100644 --- a/go.sum +++ b/go.sum @@ -14,8 +14,12 @@ github.com/antonholmquist/jason v1.0.1-0.20160829104012-962e09b85496 h1:dESITduf github.com/antonholmquist/jason v1.0.1-0.20160829104012-962e09b85496/go.mod h1:+GxMEKI0Va2U8h3os6oiUAetHAlGMvxjdpAH/9uvUMA= github.com/aymerick/raymond v2.0.2+incompatible h1:VEp3GpgdAnv9B2GFyTvqgcKvY+mfKMjPOA3SbKLtnU0= github.com/aymerick/raymond v2.0.2+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g= +github.com/basgys/goxml2json v1.1.0 h1:4ln5i4rseYfXNd86lGEB+Vi652IsIXIvggKM/BhUKVw= +github.com/basgys/goxml2json v1.1.0/go.mod h1:wH7a5Np/Q4QoECFIU8zTQlZwZkrilY0itPfecMw41Dw= github.com/beevik/etree v1.1.0 h1:T0xke/WvNtMoCqgzPhkX2r4rjY3GDZFi+FjpRZY2Jbs= github.com/beevik/etree v1.1.0/go.mod h1:r8Aw8JqVegEf0w2fDnATrX9VpkMcyFeM0FhwO62wh+A= +github.com/bitly/go-simplejson v0.5.1 h1:xgwPbetQScXt1gh9BmoJ6j9JMr3TElvuIyjR8pgdoow= +github.com/bitly/go-simplejson v0.5.1/go.mod h1:YOPVLzCfwK14b4Sff3oP1AmGhI9T9Vsg84etUnlyp+Q= github.com/boltdb/bolt v1.2.1-0.20160424201119-d97499360d1e h1:ZjpTXDvUplNMT6aktSoMffTkAGIQJGNGPQW/xL4kPwA= github.com/boltdb/bolt v1.2.1-0.20160424201119-d97499360d1e/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= github.com/brianvoe/gofakeit/v6 v6.19.0 h1:g+yJ+meWVEsAmR+bV4mNM/eXI0N+0pZ3D+Mi+G5+YQo= diff --git a/vendor/github.com/basgys/goxml2json/.gitignore b/vendor/github.com/basgys/goxml2json/.gitignore new file mode 100644 index 000000000..6bfad5422 --- /dev/null +++ b/vendor/github.com/basgys/goxml2json/.gitignore @@ -0,0 +1,25 @@ +# Compiled Object files, Static and Dynamic libs (Shared Objects) +*.o +*.a +*.so + +# Folders +_obj +_test + +# Architecture specific extensions/prefixes +*.[568vq] +[568vq].out + +*.cgo1.go +*.cgo2.c +_cgo_defun.c +_cgo_gotypes.go +_cgo_export.* + +_testmain.go + +*.exe +*.test +*.prof +/.tags diff --git a/vendor/github.com/basgys/goxml2json/LICENSE b/vendor/github.com/basgys/goxml2json/LICENSE new file mode 100644 index 000000000..dc5a2e3eb --- /dev/null +++ b/vendor/github.com/basgys/goxml2json/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2016 Bastien Gysler + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/vendor/github.com/basgys/goxml2json/README.md b/vendor/github.com/basgys/goxml2json/README.md new file mode 100644 index 000000000..04e0b7faf --- /dev/null +++ b/vendor/github.com/basgys/goxml2json/README.md @@ -0,0 +1,82 @@ +# goxml2json [![CircleCI](https://circleci.com/gh/basgys/goxml2json.svg?style=svg)](https://circleci.com/gh/basgys/goxml2json) + +Go package that converts XML to JSON + +### Install + + go get -u github.com/basgys/goxml2json + +### Importing + + import github.com/basgys/goxml2json + +### Usage + +**Code example** + +```go + package main + + import ( + "fmt" + "strings" + + xj "github.com/basgys/goxml2json" + ) + + func main() { + // xml is an io.Reader + xml := strings.NewReader(`world`) + json, err := xj.Convert(xml) + if err != nil { + panic("That's embarrassing...") + } + + fmt.Println(json.String()) + // {"hello": "world"} + } + +``` + +**Input** + +```xml + + + + bar + +``` + +**Output** + +```json + { + "osm": { + "-version": "0.6", + "-generator": "CGImap 0.0.2", + "bounds": { + "-minlat": "54.0889580", + "-minlon": "12.2487570", + "-maxlat": "54.0913900", + "-maxlon": "12.2524800" + }, + "foo": "bar" + } + } +``` + +### Contributing +Feel free to contribute to this project if you want to fix/extend/improve it. + +### Contributors + + - [DirectX](https://github.com/directx) + - [samuelhug](https://github.com/samuelhug) + +### TODO + + * Extract data types in JSON (numbers, boolean, ...) + * Categorise errors + * Option to prettify the JSON output + * Benchmark diff --git a/vendor/github.com/basgys/goxml2json/converter.go b/vendor/github.com/basgys/goxml2json/converter.go new file mode 100644 index 000000000..3e4cbce2a --- /dev/null +++ b/vendor/github.com/basgys/goxml2json/converter.go @@ -0,0 +1,25 @@ +package xml2json + +import ( + "bytes" + "io" +) + +// Convert converts the given XML document to JSON +func Convert(r io.Reader) (*bytes.Buffer, error) { + // Decode XML document + root := &Node{} + err := NewDecoder(r).Decode(root) + if err != nil { + return nil, err + } + + // Then encode it in JSON + buf := new(bytes.Buffer) + err = NewEncoder(buf).Encode(root) + if err != nil { + return nil, err + } + + return buf, nil +} diff --git a/vendor/github.com/basgys/goxml2json/decoder.go b/vendor/github.com/basgys/goxml2json/decoder.go new file mode 100644 index 000000000..af03356c9 --- /dev/null +++ b/vendor/github.com/basgys/goxml2json/decoder.go @@ -0,0 +1,140 @@ +package xml2json + +import ( + "encoding/xml" + "io" + "unicode" + + "golang.org/x/net/html/charset" +) + +const ( + attrPrefix = "-" + contentPrefix = "#" +) + +// A Decoder reads and decodes XML objects from an input stream. +type Decoder struct { + r io.Reader + err error + attributePrefix string + contentPrefix string +} + +type element struct { + parent *element + n *Node + label string +} + +func (dec *Decoder) SetAttributePrefix(prefix string) { + dec.attributePrefix = prefix +} + +func (dec *Decoder) SetContentPrefix(prefix string) { + dec.contentPrefix = prefix +} + +func (dec *Decoder) DecodeWithCustomPrefixes(root *Node, contentPrefix string, attributePrefix string) error { + dec.contentPrefix = contentPrefix + dec.attributePrefix = attributePrefix + return dec.Decode(root) +} + +// NewDecoder returns a new decoder that reads from r. +func NewDecoder(r io.Reader) *Decoder { + return &Decoder{r: r} +} + +// Decode reads the next JSON-encoded value from its +// input and stores it in the value pointed to by v. +func (dec *Decoder) Decode(root *Node) error { + + if dec.contentPrefix == "" { + dec.contentPrefix = contentPrefix + } + if dec.attributePrefix == "" { + dec.attributePrefix = attrPrefix + } + + xmlDec := xml.NewDecoder(dec.r) + + // That will convert the charset if the provided XML is non-UTF-8 + xmlDec.CharsetReader = charset.NewReaderLabel + + // Create first element from the root node + elem := &element{ + parent: nil, + n: root, + } + + for { + t, _ := xmlDec.Token() + if t == nil { + break + } + + switch se := t.(type) { + case xml.StartElement: + // Build new a new current element and link it to its parent + elem = &element{ + parent: elem, + n: &Node{}, + label: se.Name.Local, + } + + // Extract attributes as children + for _, a := range se.Attr { + elem.n.AddChild(dec.attributePrefix+a.Name.Local, &Node{Data: a.Value}) + } + case xml.CharData: + // Extract XML data (if any) + elem.n.Data = trimNonGraphic(string(xml.CharData(se))) + case xml.EndElement: + // And add it to its parent list + if elem.parent != nil { + elem.parent.n.AddChild(elem.label, elem.n) + } + + // Then change the current element to its parent + elem = elem.parent + } + } + + return nil +} + +// trimNonGraphic returns a slice of the string s, with all leading and trailing +// non graphic characters and spaces removed. +// +// Graphic characters include letters, marks, numbers, punctuation, symbols, +// and spaces, from categories L, M, N, P, S, Zs. +// Spacing characters are set by category Z and property Pattern_White_Space. +func trimNonGraphic(s string) string { + if s == "" { + return s + } + + var first *int + var last int + for i, r := range []rune(s) { + if !unicode.IsGraphic(r) || unicode.IsSpace(r) { + continue + } + + if first == nil { + f := i // copy i + first = &f + last = i + } else { + last = i + } + } + + // If first is nil, it means there are no graphic characters + if first == nil { + return "" + } + + return string([]rune(s)[*first : last+1]) +} diff --git a/vendor/github.com/basgys/goxml2json/doc.go b/vendor/github.com/basgys/goxml2json/doc.go new file mode 100644 index 000000000..8a68bd30f --- /dev/null +++ b/vendor/github.com/basgys/goxml2json/doc.go @@ -0,0 +1,2 @@ +// Package xml2json is an XML to JSON converter +package xml2json diff --git a/vendor/github.com/basgys/goxml2json/encoder.go b/vendor/github.com/basgys/goxml2json/encoder.go new file mode 100644 index 000000000..0a542b8a8 --- /dev/null +++ b/vendor/github.com/basgys/goxml2json/encoder.go @@ -0,0 +1,197 @@ +package xml2json + +import ( + "bytes" + "io" + "unicode/utf8" +) + +// An Encoder writes JSON objects to an output stream. +type Encoder struct { + w io.Writer + err error + contentPrefix string + attributePrefix string +} + +// NewEncoder returns a new encoder that writes to w. +func NewEncoder(w io.Writer) *Encoder { + return &Encoder{w: w} +} + +func (enc *Encoder) SetAttributePrefix(prefix string) { + enc.attributePrefix = prefix +} + +func (enc *Encoder) SetContentPrefix(prefix string) { + enc.contentPrefix = prefix +} + +func (enc *Encoder) EncodeWithCustomPrefixes(root *Node, contentPrefix string, attributePrefix string) error { + enc.contentPrefix = contentPrefix + enc.attributePrefix = attributePrefix + return enc.Encode(root) +} + +// Encode writes the JSON encoding of v to the stream +func (enc *Encoder) Encode(root *Node) error { + if enc.err != nil { + return enc.err + } + if root == nil { + return nil + } + if enc.contentPrefix == "" { + enc.contentPrefix = contentPrefix + } + if enc.attributePrefix == "" { + enc.attributePrefix = attrPrefix + } + + enc.err = enc.format(root, 0) + + // Terminate each value with a newline. + // This makes the output look a little nicer + // when debugging, and some kind of space + // is required if the encoded value was a number, + // so that the reader knows there aren't more + // digits coming. + enc.write("\n") + + return enc.err +} + +func (enc *Encoder) format(n *Node, lvl int) error { + if n.IsComplex() { + enc.write("{") + + // Add data as an additional attibute (if any) + if len(n.Data) > 0 { + enc.write("\"") + enc.write(enc.contentPrefix) + enc.write("content") + enc.write("\": ") + enc.write(sanitiseString(n.Data)) + enc.write(", ") + } + + i := 0 + tot := len(n.Children) + for label, children := range n.Children { + enc.write("\"") + enc.write(label) + enc.write("\": ") + + if len(children) > 1 { + // Array + enc.write("[") + for j, c := range children { + enc.format(c, lvl+1) + + if j < len(children)-1 { + enc.write(", ") + } + } + enc.write("]") + } else { + // Map + enc.format(children[0], lvl+1) + } + + if i < tot-1 { + enc.write(", ") + } + i++ + } + + enc.write("}") + } else { + // TODO : Extract data type + enc.write(sanitiseString(n.Data)) + } + + return nil +} + +func (enc *Encoder) write(s string) { + enc.w.Write([]byte(s)) +} + +// https://golang.org/src/encoding/json/encode.go?s=5584:5627#L788 +var hex = "0123456789abcdef" + +func sanitiseString(s string) string { + var buf bytes.Buffer + + buf.WriteByte('"') + start := 0 + for i := 0; i < len(s); { + if b := s[i]; b < utf8.RuneSelf { + if 0x20 <= b && b != '\\' && b != '"' && b != '<' && b != '>' && b != '&' { + i++ + continue + } + if start < i { + buf.WriteString(s[start:i]) + } + switch b { + case '\\', '"': + buf.WriteByte('\\') + buf.WriteByte(b) + case '\n': + buf.WriteByte('\\') + buf.WriteByte('n') + case '\r': + buf.WriteByte('\\') + buf.WriteByte('r') + case '\t': + buf.WriteByte('\\') + buf.WriteByte('t') + default: + // This encodes bytes < 0x20 except for \n and \r, + // as well as <, > and &. The latter are escaped because they + // can lead to security holes when user-controlled strings + // are rendered into JSON and served to some browsers. + buf.WriteString(`\u00`) + buf.WriteByte(hex[b>>4]) + buf.WriteByte(hex[b&0xF]) + } + i++ + start = i + continue + } + c, size := utf8.DecodeRuneInString(s[i:]) + if c == utf8.RuneError && size == 1 { + if start < i { + buf.WriteString(s[start:i]) + } + buf.WriteString(`\ufffd`) + i += size + start = i + continue + } + // U+2028 is LINE SEPARATOR. + // U+2029 is PARAGRAPH SEPARATOR. + // They are both technically valid characters in JSON strings, + // but don't work in JSONP, which has to be evaluated as JavaScript, + // and can lead to security holes there. It is valid JSON to + // escape them, so we do so unconditionally. + // See http://timelessrepo.com/json-isnt-a-javascript-subset for discussion. + if c == '\u2028' || c == '\u2029' { + if start < i { + buf.WriteString(s[start:i]) + } + buf.WriteString(`\u202`) + buf.WriteByte(hex[c&0xF]) + i += size + start = i + continue + } + i += size + } + if start < len(s) { + buf.WriteString(s[start:]) + } + buf.WriteByte('"') + return buf.String() +} diff --git a/vendor/github.com/basgys/goxml2json/struct.go b/vendor/github.com/basgys/goxml2json/struct.go new file mode 100644 index 000000000..1f423b1ce --- /dev/null +++ b/vendor/github.com/basgys/goxml2json/struct.go @@ -0,0 +1,25 @@ +package xml2json + +// Node is a data element on a tree +type Node struct { + Children map[string]Nodes + Data string +} + +// Nodes is a list of nodes +type Nodes []*Node + +// AddChild appends a node to the list of children +func (n *Node) AddChild(s string, c *Node) { + // Lazy lazy + if n.Children == nil { + n.Children = map[string]Nodes{} + } + + n.Children[s] = append(n.Children[s], c) +} + +// IsComplex returns whether it is a complex type (has children) +func (n *Node) IsComplex() bool { + return len(n.Children) > 0 +} diff --git a/vendor/modules.txt b/vendor/modules.txt index a5e20e396..abd258148 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -32,9 +32,14 @@ github.com/antonholmquist/jason github.com/aymerick/raymond/ast github.com/aymerick/raymond/lexer github.com/aymerick/raymond/parser +# github.com/basgys/goxml2json v1.1.0 +## explicit +github.com/basgys/goxml2json # github.com/beevik/etree v1.1.0 ## explicit github.com/beevik/etree +# github.com/bitly/go-simplejson v0.5.1 +## explicit; go 1.17 # github.com/boltdb/bolt v1.2.1-0.20160424201119-d97499360d1e ## explicit github.com/boltdb/bolt