test: Adding hubble metrics E2E tests.

test/e2e/common/validate-metric.go

@@ -0,0 +1,52 @@
+package common
+
+import (
+	"errors"
+	"fmt"
+	"log"
+
+	prom "github.com/microsoft/retina/test/e2e/framework/prometheus"
+)
+
+var (
+	ErrMetricFound = errors.New("unexpected metric found")
+)
+
+type ValidateMetric struct {
+	ForwardedPort string
+	MetricName    string
+	ValidMetrics  []map[string]string
+	ExpectMetric  bool
+}
+
+func (v *ValidateMetric) Run() error {
+	promAddress := fmt.Sprintf("http://localhost:%s/metrics", v.ForwardedPort)
+
+	for _, validMetric := range v.ValidMetrics {
+		err := prom.CheckMetric(promAddress, v.MetricName, validMetric)
+		if err != nil {
+			// If we expect the metric not to be found, return nil if it's not found.
+			if !v.ExpectMetric && errors.Is(err, prom.ErrNoMetricFound) {
+				log.Printf("metric %s not found, as expected\n", v.MetricName)
+				return nil
+			}
+			return fmt.Errorf("failed to verify prometheus metrics: %w", err)
+		}
+
+		// if we expect the metric not to be found, return an error if it is found
+		if !v.ExpectMetric {
+			return fmt.Errorf("did not expect to find metric %s matching %+v: %w", v.MetricName, validMetric, ErrMetricFound)
+		}
+
+		log.Printf("found metric %s matching %+v\n", v.MetricName, validMetric)
+	}
+	return nil
+}
+
+func (v *ValidateMetric) Prevalidate() error {
+	return nil
+}
+
+func (v *ValidateMetric) Stop() error {
+	return nil
+}

test/e2e/framework/constants/hubble.go

@@ -0,0 +1,31 @@
+package constants
+
+const (
+	// Metrics Port
+	HubbleMetricsPort = "9965"
+
+	// MetricsName
+	HubbleDNSQueryMetricName    = "hubble_dns_queries_total"
+	HubbleDNSResponseMetricName = "hubble_dns_responses_total"
+	HubbleFlowMetricName        = "hubble_flows_processed_total"
+	HubbleDropMetricName        = "hubble_drop_total"
+	HubbleTCPFlagsMetricName    = "hubble_tcp_flags_total"
+
+	// Labels
+	HubbleDestinationLabel = "destination"
+	HubbleSourceLabel      = "source"
+	HubbleIPsRetunedLabel  = "ips_returned"
+	HubbleQTypesLabel      = "qtypes"
+	HubbleRCodeLabel       = "rcode"
+	HubbleQueryLabel       = "query"
+
+	HubbleProtocolLabel = "protocol"
+	HubbleReasonLabel   = "reason"
+
+	HubbleSubtypeLabel = "subtype"
+	HubbleTypeLabel    = "type"
+	HubbleVerdictLabel = "verdict"
+
+	HubbleFamilyLabel = "family"
+	HubbleFlagLabel   = "flag"
+)

test/e2e/framework/constants/networking.go

@@ -0,0 +1,13 @@
+package constants
+
+const (
+	TCP             = "TCP"
+	UDP             = "UDP"
+	IPV4            = "IPv4"
+	IPTableRuleDrop = "IPTABLE_RULE_DROP"
+	SYN             = "SYN"
+	SYNACK          = "SYN-ACK"
+	ACK             = "ACK"
+	FIN             = "FIN"
+	RST             = "RST"
+)

test/e2e/framework/kubernetes/install-hubble-helm.go

@@ -23,15 +23,15 @@ const (
 	HubbleRelayApp  = "hubble-relay"
 )
 
-type ValidateHubbleStep struct {
+type InstallHubbleHelmChart struct {
 	Namespace          string
 	ReleaseName        string
 	KubeConfigFilePath string
 	ChartPath          string
 	TagEnv             string
 }
 
-func (v *ValidateHubbleStep) Run() error {
+func (v *InstallHubbleHelmChart) Run() error {
 	ctx, cancel := context.WithTimeout(context.Background(), defaultTimeoutSeconds*time.Second)
 	defer cancel()
 
@@ -146,10 +146,10 @@ func (v *ValidateHubbleStep) Run() error {
 	return nil
 }
 
-func (v *ValidateHubbleStep) Prevalidate() error {
+func (v *InstallHubbleHelmChart) Prevalidate() error {
 	return nil
 }
 
-func (v *ValidateHubbleStep) Stop() error {
+func (v *InstallHubbleHelmChart) Stop() error {
 	return nil
 }

test/e2e/framework/scaletest/validate-options.go

@@ -37,7 +37,6 @@ func (po *ValidateAndPrintOptions) Prevalidate() error {
 
 // Returning an error will cause the test to fail
 func (po *ValidateAndPrintOptions) Run() error {
-
 	log.Printf("Starting to scale with folowing options: %+v", po.Options)
 
 	return nil

test/e2e/jobs/jobs.go

@@ -9,9 +9,12 @@
 	"github.com/microsoft/retina/test/e2e/framework/generic"
 	"github.com/microsoft/retina/test/e2e/framework/kubernetes"
 	"github.com/microsoft/retina/test/e2e/framework/types"
-	"github.com/microsoft/retina/test/e2e/hubble"
 	"github.com/microsoft/retina/test/e2e/scenarios/dns"
 	"github.com/microsoft/retina/test/e2e/scenarios/drop"
+	hubble_dns "github.com/microsoft/retina/test/e2e/scenarios/hubble/dns"
+	hubble_flow "github.com/microsoft/retina/test/e2e/scenarios/hubble/flow"
+	hubble_service "github.com/microsoft/retina/test/e2e/scenarios/hubble/service"
+	hubble_tcp "github.com/microsoft/retina/test/e2e/scenarios/hubble/tcp"
 	"github.com/microsoft/retina/test/e2e/scenarios/latency"
 	"github.com/microsoft/retina/test/e2e/scenarios/perf"
 	tcp "github.com/microsoft/retina/test/e2e/scenarios/tcp"
@@ -245,68 +248,28 @@
 	return job
 }
 
-func ValidateHubble(kubeConfigFilePath, chartPath string, testPodNamespace string) *types.Job {
+func InstallAndTestHubbleMetrics(kubeConfigFilePath, chartPath string, testPodNamespace string) *types.Job {
 	job := types.NewJob("Validate Hubble")
 
-	job.AddStep(&kubernetes.ValidateHubbleStep{
+	job.AddStep(&kubernetes.InstallHubbleHelmChart{
 		Namespace:          common.KubeSystemNamespace,
 		ReleaseName:        "retina",
 		KubeConfigFilePath: kubeConfigFilePath,
 		ChartPath:          chartPath,
 		TagEnv:             generic.DefaultTagEnv,
 	}, nil)
 
-	job.AddScenario(hubble.ValidateHubbleRelayService())
-
-	job.AddScenario(hubble.ValidateHubbleUIService(kubeConfigFilePath))
-
-	job.AddScenario(drop.ValidateDropMetric(testPodNamespace))
-
-	job.AddScenario(tcp.ValidateTCPMetrics(testPodNamespace))
-
-	dnsScenarios := []struct {
-		name string
-		req  *dns.RequestValidationParams
-		resp *dns.ResponseValidationParams
-	}{
-		{
-			name: "Validate basic DNS request and response metrics for a valid domain",
-			req: &dns.RequestValidationParams{
-				NumResponse: "0",
-				Query:       "kubernetes.default.svc.cluster.local.",
-				QueryType:   "A",
-				Command:     "nslookup kubernetes.default",
-				ExpectError: false,
-			},
-			resp: &dns.ResponseValidationParams{
-				NumResponse: "1",
-				Query:       "kubernetes.default.svc.cluster.local.",
-				QueryType:   "A",
-				ReturnCode:  "No Error",
-				Response:    "10.0.0.1",
-			},
-		},
-		{
-			name: "Validate basic DNS request and response metrics for a non-existent domain",
-			req: &dns.RequestValidationParams{
-				NumResponse: "0",
-				Query:       "some.non.existent.domain.",
-				QueryType:   "A",
-				Command:     "nslookup some.non.existent.domain",
-				ExpectError: true,
-			},
-			resp: &dns.ResponseValidationParams{
-				NumResponse: "0",
-				Query:       "some.non.existent.domain.",
-				QueryType:   "A",
-				Response:    dns.EmptyResponse, // hacky way to bypass the framework for now
-				ReturnCode:  "Non-Existent Domain",
-			},
-		},
+	hubbleScrenarios := []*types.Scenario{
+		hubble_dns.ValidateDNSMetric(),
+		hubble_flow.ValidateFlowMetric(),
+		// hubble_drop.ValidateDropMetric(), TODO Needs to investigate why drop metrics are not present.
+		hubble_tcp.ValidateTCPMetric(),
+		hubble_service.ValidateHubbleRelayService(),
+		hubble_service.ValidateHubbleUIService(kubeConfigFilePath),
 	}
 
-	for _, scenario := range dnsScenarios {
-		job.AddScenario(dns.ValidateBasicDNSMetrics(scenario.name, scenario.req, scenario.resp, testPodNamespace))
+	for _, scenario := range hubbleScrenarios {
+		job.AddScenario(scenario)
 	}
 
 	job.AddStep(&kubernetes.EnsureStableComponent{
@@ -365,4 +328,3 @@
 
 	return job
 }
-

test/e2e/retina_e2e_test.go

@@ -71,7 +71,7 @@ func TestE2ERetina(t *testing.T) {
 	advanceMetricsE2E := types.NewRunner(t, jobs.UpgradeAndTestRetinaAdvancedMetrics(kubeConfigFilePath, chartPath, profilePath, common.TestPodNamespace))
 	advanceMetricsE2E.Run(ctx)
 
-	// Install and test Hubble basic metrics
-	validatehubble := types.NewRunner(t, jobs.ValidateHubble(kubeConfigFilePath, hubblechartPath, common.TestPodNamespace))
-	validatehubble.Run(ctx)
+	// Install and test Hubble  metrics
+	hubbleMetricsE2E := types.NewRunner(t, jobs.InstallAndTestHubbleMetrics(kubeConfigFilePath, hubblechartPath, common.TestPodNamespace))
+	hubbleMetricsE2E.Run(ctx)
 }

test/e2e/scenarios/hubble/dns/labels.go

@@ -0,0 +1,26 @@
+package dns
+
+import (
+	"github.com/microsoft/retina/test/e2e/common"
+	"github.com/microsoft/retina/test/e2e/framework/constants"
+)
+
+var (
+	podName                   = "agnhost-dns-0"
+	validDNSQueryMetricLabels = map[string]string{
+		constants.HubbleDestinationLabel: "",
+		constants.HubbleSourceLabel:      common.TestPodNamespace + "/" + podName,
+		constants.HubbleIPsRetunedLabel:  "0",
+		constants.HubbleQTypesLabel:      "A",
+		constants.HubbleRCodeLabel:       "",
+		constants.HubbleQueryLabel:       "one.one.one.one.",
+	}
+	validDNSResponseMetricLabels = map[string]string{
+		constants.HubbleDestinationLabel: common.TestPodNamespace + "/" + podName,
+		constants.HubbleSourceLabel:      "",
+		constants.HubbleIPsRetunedLabel:  "2",
+		constants.HubbleQTypesLabel:      "A",
+		constants.HubbleRCodeLabel:       "No Error",
+		constants.HubbleQueryLabel:       "one.one.one.one.",
+	}
+)

test/e2e/scenarios/hubble/dns/scenario.go

@@ -0,0 +1,85 @@
+package dns
+
+import (
+	"time"
+
+	"github.com/microsoft/retina/test/e2e/common"
+	"github.com/microsoft/retina/test/e2e/framework/constants"
+	"github.com/microsoft/retina/test/e2e/framework/kubernetes"
+	"github.com/microsoft/retina/test/e2e/framework/types"
+)
+
+const (
+	sleepDelay = 5 * time.Second
+)
+
+func ValidateDNSMetric() *types.Scenario {
+	name := "DNS Metrics"
+	agnhostName := "agnhost-dns"
+	podName := agnhostName + "-0"
+	steps := []*types.StepWrapper{
+		{
+			Step: &kubernetes.CreateAgnhostStatefulSet{
+				AgnhostName:      agnhostName,
+				AgnhostNamespace: common.TestPodNamespace,
+			},
+		},
+		{
+			Step: &kubernetes.PortForward{
+				LabelSelector:         "k8s-app=retina",
+				LocalPort:             constants.HubbleMetricsPort,
+				RemotePort:            constants.HubbleMetricsPort,
+				Namespace:             common.KubeSystemNamespace,
+				Endpoint:              "metrics",
+				OptionalLabelAffinity: "app=" + agnhostName, // port forward hubble metrics to a pod on a node that also has this pod with this label, assuming same namespace
+			},
+			Opts: &types.StepOptions{
+				RunInBackgroundWithID: "hubble-dns-port-forward",
+			},
+		},
+		{
+			Step: &kubernetes.ExecInPod{
+				PodName:      podName,
+				PodNamespace: common.TestPodNamespace,
+				Command:      "nslookup -type=a one.one.one.one",
+			},
+			Opts: &types.StepOptions{
+				ExpectError:               false,
+				SkipSavingParametersToJob: true,
+			},
+		},
+		{
+			Step: &types.Sleep{
+				Duration: sleepDelay,
+			},
+		},
+		{
+			Step: &common.ValidateMetric{
+				ForwardedPort: constants.HubbleMetricsPort,
+				MetricName:    constants.HubbleDNSQueryMetricName,
+				ValidMetrics:  []map[string]string{validDNSQueryMetricLabels},
+				ExpectMetric:  true,
+			},
+			Opts: &types.StepOptions{
+				SkipSavingParametersToJob: true,
+			},
+		},
+		{
+			Step: &common.ValidateMetric{
+				ForwardedPort: constants.HubbleMetricsPort,
+				MetricName:    constants.HubbleDNSResponseMetricName,
+				ValidMetrics:  []map[string]string{validDNSResponseMetricLabels},
+				ExpectMetric:  true,
+			},
+			Opts: &types.StepOptions{
+				SkipSavingParametersToJob: true,
+			},
+		},
+		{
+			Step: &types.Stop{
+				BackgroundID: "hubble-dns-port-forward",
+			},
+		},
+	}
+	return types.NewScenario(name, steps...)
+}

test/e2e/scenarios/hubble/drop/labels.go

@@ -0,0 +1,16 @@
+package drop
+
+import (
+	"github.com/microsoft/retina/test/e2e/common"
+	"github.com/microsoft/retina/test/e2e/framework/constants"
+)
+
+var (
+	podName                     = "agnhost-drop-0"
+	validHubbleDropMetricLabels = map[string]string{
+		constants.HubbleSourceLabel:      common.TestPodNamespace + "/" + podName,
+		constants.HubbleDestinationLabel: "",
+		constants.HubbleProtocolLabel:    constants.UDP,
+		constants.HubbleReasonLabel:      "POLICY_DENIED",
+	}
+)