Merge pull request #210 from containeroo/CO-68

CO-68 Rework dynamic IP response filtering and processing

api/v1beta1/ip_types.go

@@ -39,12 +39,12 @@ type IPSpecIPSources struct {
 	// +kubebuilder:validation:Enum=GET;POST;PUT;DELETE
 	// +kubebuilder:default=GET
 	RequestMethod string `json:"requestMethod,omitempty"`
-	// ResponseJSONPath defines the JSON path to the value to be used as IP
+	// ResponseJQFilter applies a JQ filter to the response to extract the IP
 	// +optional
-	ResponseJSONPath string `json:"responseJSONPath,omitempty"`
-	// ResponseRegex defines the regular expression to be used to extract the IP from the response or a JSON path result
+	ResponseJQFilter string `json:"responseJQFilter,omitempty"`
+	// PostProcessingRegex defines the regular expression to be used to extract the IP from the response or a JQ filter result
 	// +optional
-	ResponseRegex string `json:"responseRegex,omitempty"`
+	PostProcessingRegex string `json:"postProcessingRegex,omitempty"`
 }
 
 // IPSpec defines the desired state of IP

config/crd/bases/cf.containeroo.ch_ips.yaml

@@ -56,6 +56,11 @@ spec:
                   source (e.g. an API or public IP echo service)
                 items:
                   properties:
+                    postProcessingRegex:
+                      description: PostProcessingRegex defines the regular expression
+                        to be used to extract the IP from the response or a JQ filter
+                        result
+                      type: string
                     requestBody:
                       description: RequestBody to be sent to the URL
                       type: string
@@ -89,14 +94,9 @@ spec:
                       - PUT
                       - DELETE
                       type: string
-                    responseJSONPath:
-                      description: ResponseJSONPath defines the JSON path to the value
-                        to be used as IP
-                      type: string
-                    responseRegex:
-                      description: ResponseRegex defines the regular expression to
-                        be used to extract the IP from the response or a JSON path
-                        result
+                    responseJQFilter:
+                      description: ResponseJQFilter applies a JQ filter to the response
+                        to extract the IP
                       type: string
                     url:
                       description: URL of the IP source (e.g. https://checkip.amazonaws.com)

controllers/ip_controller.go

@@ -32,12 +32,12 @@ import (
 
 	cfv1beta1 "github.com/containeroo/cloudflare-operator/api/v1beta1"
 	"github.com/go-logr/logr"
+	"github.com/itchyny/gojq"
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	apimeta "k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
-	"k8s.io/client-go/util/jsonpath"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
@@ -278,35 +278,37 @@ func (r *IPReconciler) getIPSource(ctx context.Context, source cfv1beta1.IPSpecI
 	}
 
 	extractedIP := string(response)
-	if source.ResponseJSONPath != "" {
-		var jsonResponse map[string]interface{}
+	if source.ResponseJQFilter != "" {
+		var jsonResponse interface{}
 		err := json.Unmarshal(response, &jsonResponse)
 		if err != nil {
 			return "", fmt.Errorf("failed to get IP from %s: %s", source.URL, err)
 		}
-		j := jsonpath.New("jsonpath")
-		buf := new(bytes.Buffer)
-		if err := j.Parse(source.ResponseJSONPath); err != nil {
-			return "", fmt.Errorf("failed to parse jsonpath %s: %s", source.ResponseJSONPath, err)
+		jq, err := gojq.Parse(source.ResponseJQFilter)
+		if err != nil {
+			return "", fmt.Errorf("failed to parse jq filter %s: %s", source.ResponseJQFilter, err)
 		}
-		if err := j.Execute(buf, jsonResponse); err != nil {
-			return "", fmt.Errorf("failed to extract IP from %s: %s", source.URL, err)
+		iter := jq.Run(jsonResponse)
+		result, ok := iter.Next()
+		if !ok {
+			return "", fmt.Errorf("failed to extract IP from %s. jq returned no results", source.URL)
 		}
-
-		extractedIP = buf.String()
+		extractedIP = fmt.Sprintf("%v", result)
 	}
 
-	if source.ResponseRegex != "" {
-		re, err := regexp.Compile(source.ResponseRegex)
+	if source.PostProcessingRegex != "" {
+		re, err := regexp.Compile(source.PostProcessingRegex)
 		if err != nil {
-			return "", fmt.Errorf("failed to compile regex %s: %s", source.ResponseRegex, err)
+			return "", fmt.Errorf("failed to compile regex %s: %s", source.PostProcessingRegex, err)
 		}
-		extractedIPBytes := []byte(extractedIP)
-		match := re.Find(extractedIPBytes)
+		match := re.FindStringSubmatch(extractedIP)
 		if match == nil {
 			return "", fmt.Errorf("failed to extract IP from %s. regex returned no matches", source.URL)
 		}
-		extractedIP = string(match)
+		if len(match) < 2 {
+			return "", fmt.Errorf("failed to extract IP from %s. regex returned no matches", source.URL)
+		}
+		extractedIP = match[1]
 	}
 
 	if net.ParseIP(extractedIP) == nil {

go.mod

@@ -5,6 +5,7 @@ go 1.19
 require (
 	github.com/cloudflare/cloudflare-go v0.62.0
 	github.com/go-logr/logr v1.2.3
+	github.com/itchyny/gojq v0.12.12
 	github.com/prometheus/client_golang v1.14.0
 	k8s.io/api v0.26.2
 	k8s.io/apimachinery v0.26.2
@@ -34,6 +35,7 @@ require (
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
 	github.com/hashicorp/go-retryablehttp v0.7.2 // indirect
 	github.com/imdario/mergo v0.3.12 // indirect
+	github.com/itchyny/timefmt-go v0.1.5 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/mailru/easyjson v0.7.6 // indirect

go.sum

@@ -178,6 +178,10 @@ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU=
 github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
+github.com/itchyny/gojq v0.12.12 h1:x+xGI9BXqKoJQZkr95ibpe3cdrTbY8D9lonrK433rcA=
+github.com/itchyny/gojq v0.12.12/go.mod h1:j+3sVkjxwd7A7Z5jrbKibgOLn0ZfLWkV+Awxr/pyzJE=
+github.com/itchyny/timefmt-go v0.1.5 h1:G0INE2la8S6ru/ZI5JecgyzbbJNs5lG1RcBqa7Jm6GE=
+github.com/itchyny/timefmt-go v0.1.5/go.mod h1:nEP7L+2YmAbT2kZ2HfSs1d8Xtw9LY8D2stDBckWakZ8=
 github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
@@ -208,7 +212,7 @@ github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN
 github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA=
 github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
 github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40=
-github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
+github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/matttproud/golang_protobuf_extensions v1.0.2 h1:hAHbPm5IJGijwng3PWk09JkG9WeqChjprR5s9bBZ+OM=
 github.com/matttproud/golang_protobuf_extensions v1.0.2/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=