diff --git a/README.md b/README.md index a0bfa54a..a61231f6 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ For example, consider the following ss entry: LISTEN 0 4096 127.0.0.1:10248 0.0.0.0:* users:(("kubelet",pid=6187,fd=20)) ``` -The `ss` package provides the `CreateComDetailsFromNode` function that runs +The `ss` package provides the `CreateSSOutputFromNode` function that runs the `ss` command on each node, and converts the output into a corresponding ComDetails list. ### Communication Matrix Creation Guide @@ -31,7 +31,7 @@ Examples are available in the `example-custom-entries` files. The following environment variables are used to configure: ``` -FORMAT (csv/json/yaml) +FORMAT (csv/json/yaml/nft) CLUSTER_ENV (baremetal/cloud) DEST_DIR (path to the directory containing the artifacts) DEPLOYMENT (mno/sno) diff --git a/cmd/main.go b/cmd/main.go index c9526d42..9f8bbe6a 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -1,56 +1,34 @@ package main import ( - "context" "flag" "fmt" "os" - "path" "path/filepath" - "sync" - "golang.org/x/sync/errgroup" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - clientutil "github.com/openshift-kni/commatrix/client" "github.com/openshift-kni/commatrix/commatrix" - "github.com/openshift-kni/commatrix/consts" - "github.com/openshift-kni/commatrix/debug" - "github.com/openshift-kni/commatrix/ss" - "github.com/openshift-kni/commatrix/types" ) -func main() { - var ( - destDir string - format string - envStr string - deploymentStr string - customEntriesPath string - customEntriesFormat string - printFn func(m types.ComMatrix) ([]byte, error) - ) +var ( + destDir string + format string + envStr string + deploymentStr string + customEntriesPath string + customEntriesFormat string +) +func init() { flag.StringVar(&destDir, "destDir", "communication-matrix", "Output files dir") - flag.StringVar(&format, "format", "csv", "Desired format (json,yaml,csv)") + flag.StringVar(&format, "format", "csv", "Desired format (json,yaml,csv,nft)") flag.StringVar(&envStr, "env", "baremetal", "Cluster environment (baremetal/cloud)") flag.StringVar(&deploymentStr, "deployment", "mno", "Deployment type (mno/sno)") flag.StringVar(&customEntriesPath, "customEntriesPath", "", "Add custom entries from a file to the matrix") flag.StringVar(&customEntriesFormat, "customEntriesFormat", "", "Set the format of the custom entries file (json,yaml,csv)") - flag.Parse() +} - switch format { - case "json": - printFn = types.ToJSON - case "csv": - printFn = types.ToCSV - case "yaml": - printFn = types.ToYAML - default: - panic(fmt.Sprintf("invalid format: %s. Please specify json, csv, or yaml.", format)) - } - +func main() { kubeconfig, ok := os.LookupEnv("KUBECONFIG") if !ok { panic("must set the KUBECONFIG environment variable") @@ -79,136 +57,37 @@ func main() { if customEntriesPath != "" && customEntriesFormat == "" { panic("error, variable customEntriesFormat is not set") } - + // generate the endpointslice matrix mat, err := commatrix.New(kubeconfig, customEntriesPath, customEntriesFormat, env, deployment) if err != nil { - panic(fmt.Sprintf("failed to create the communication matrix: %s", err)) - } - - res, err := printFn(*mat) - if err != nil { - panic(err) - } - - comMatrixFileName := filepath.Join(destDir, fmt.Sprintf("communication-matrix.%s", format)) - err = os.WriteFile(comMatrixFileName, res, 0644) - if err != nil { - panic(err) - } - - cs, err := clientutil.New(kubeconfig) - if err != nil { - panic(err) - } - - tcpFile, err := os.OpenFile(path.Join(destDir, "raw-ss-tcp"), os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0644) - if err != nil { - panic(err) - } - defer tcpFile.Close() - - udpFile, err := os.OpenFile(path.Join(destDir, "raw-ss-udp"), os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0644) - if err != nil { - panic(err) + panic(fmt.Errorf("failed to create the communication matrix: %s", err)) } - defer udpFile.Close() - - nodesList, err := cs.CoreV1Interface.Nodes().List(context.TODO(), metav1.ListOptions{}) + // write the endpoint slice matrix to file + err = commatrix.WriteMatrixToFileByType(*mat, "communication-matrix", format, deployment, destDir) if err != nil { - panic(err) + panic(fmt.Sprintf("Error while writing the endpoint slice matrix to file :%v", err)) } - - nodesComDetails := []types.ComDetails{} - - err = debug.CreateNamespace(cs, consts.DefaultDebugNamespace) + // generate the ss matrix and ss raws + ssMat, ssOutTCP, ssOutUDP, err := commatrix.GenerateSS(kubeconfig, customEntriesPath, customEntriesFormat, format, env, deployment, destDir) if err != nil { - panic(err) - } - defer func() { - err := debug.DeleteNamespace(cs, consts.DefaultDebugNamespace) - if err != nil { - panic(err) - } - }() - - nLock := &sync.Mutex{} - g := new(errgroup.Group) - for _, n := range nodesList.Items { - node := n - g.Go(func() error { - debugPod, err := debug.New(cs, node.Name, consts.DefaultDebugNamespace, consts.DefaultDebugPodImage) - if err != nil { - return err - } - defer func() { - err := debugPod.Clean() - if err != nil { - fmt.Printf("failed cleaning debug pod %s: %v", debugPod, err) - } - }() - - cds, err := ss.CreateComDetailsFromNode(debugPod, &node, tcpFile, udpFile) - if err != nil { - return err - } - nLock.Lock() - nodesComDetails = append(nodesComDetails, cds...) - nLock.Unlock() - return nil - }) + panic(fmt.Sprintf("Error while generating the ss matrix and ss raws :%v", err)) } - - err = g.Wait() + // write ss raw files + err = commatrix.WriteSSRawFiles(destDir, ssOutTCP, ssOutUDP) if err != nil { - panic(err) + panic(fmt.Sprintf("Error while writing the ss raw files :%v", err)) } - - cleanedComDetails := types.CleanComDetails(nodesComDetails) - ssComMat := types.ComMatrix{Matrix: cleanedComDetails} - - res, err = printFn(ssComMat) + // write the ss matrix to file + err = commatrix.WriteMatrixToFileByType(*ssMat, "ss-generated-matrix", format, deployment, destDir) if err != nil { - panic(err) + panic(fmt.Sprintf("Error while writing ss matrix to file :%v", err)) } + // generate the diff matrix between the enpointslice and the ss matrix + diff := commatrix.GenerateMatrixDiff(*mat, *ssMat) - ssMatrixFileName := filepath.Join(destDir, fmt.Sprintf("ss-generated-matrix.%s", format)) - err = os.WriteFile(ssMatrixFileName, res, 0644) + // write the diff matrix between the enpointslice and the ss matrix to file + err = os.WriteFile(filepath.Join(destDir, "matrix-diff-ss"), []byte(diff), 0644) if err != nil { - panic(err) + panic(fmt.Sprintf("Error writing the diff matrix :%v", err)) } - - diff := buildMatrixDiff(*mat, ssComMat) - - err = os.WriteFile(filepath.Join(destDir, "matrix-diff-ss"), - []byte(diff), - 0644) - if err != nil { - panic(err) - } -} - -func buildMatrixDiff(mat1 types.ComMatrix, mat2 types.ComMatrix) string { - diff := consts.CSVHeaders + "\n" - for _, cd := range mat1.Matrix { - if mat2.Contains(cd) { - diff += fmt.Sprintf("%s\n", cd) - continue - } - - diff += fmt.Sprintf("+ %s\n", cd) - } - - for _, cd := range mat2.Matrix { - // Skip "rpc.statd" ports, these are randomly open ports on the node, - // no need to mention them in the matrix diff - if cd.Service == "rpc.statd" { - continue - } - - if !mat1.Contains(cd) { - diff += fmt.Sprintf("- %s\n", cd) - } - } - - return diff } diff --git a/commatrix/commatrix_test.go b/commatrix/commatrix_test.go new file mode 100644 index 00000000..b720e3b5 --- /dev/null +++ b/commatrix/commatrix_test.go @@ -0,0 +1,52 @@ +package commatrix + +import ( + "fmt" + "path/filepath" + "testing" + + "github.com/openshift-kni/commatrix/types" + "github.com/stretchr/testify/assert" +) + +func TestGetPrintFunction(t *testing.T) { + tests := []struct { + format string + expectedFnType string + expectedErr bool + }{ + {"json", "func(types.ComMatrix) ([]uint8, error)", false}, + {"csv", "func(types.ComMatrix) ([]uint8, error)", false}, + {"yaml", "func(types.ComMatrix) ([]uint8, error)", false}, + {"nft", "func(types.ComMatrix) ([]uint8, error)", false}, + {"invalid", "", true}, + } + + for _, tt := range tests { + t.Run(tt.format, func(t *testing.T) { + fn, err := getPrintFunction(tt.format) + if tt.expectedErr { + assert.Error(t, err) + } else { + assert.NoError(t, err) + assert.Equal(t, fmt.Sprintf("%T", fn), tt.expectedFnType) + } + }) + } +} + +func TestWriteMatrixToFile(t *testing.T) { + destDir := t.TempDir() + matrix := types.ComMatrix{ + Matrix: []types.ComDetails{ + {NodeRole: "master", Service: "testService"}, + }, + } + printFn := types.ToJSON + fileName := "test-matrix" + format := "json" + + err := writeMatrixToFile(matrix, fileName, format, printFn, destDir) + assert.NoError(t, err) + assert.FileExists(t, filepath.Join(destDir, "test-matrix.json")) +} diff --git a/commatrix/generate.go b/commatrix/generate.go new file mode 100644 index 00000000..a866f8a0 --- /dev/null +++ b/commatrix/generate.go @@ -0,0 +1,204 @@ +package commatrix + +import ( + "context" + "fmt" + "os" + "path" + "path/filepath" + "sync" + + "golang.org/x/sync/errgroup" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + clientutil "github.com/openshift-kni/commatrix/client" + "github.com/openshift-kni/commatrix/consts" + "github.com/openshift-kni/commatrix/debug" + "github.com/openshift-kni/commatrix/ss" + "github.com/openshift-kni/commatrix/types" +) + +func GenerateSS(kubeconfig, customEntriesPath, customEntriesFormat, format string, env Env, deployment Deployment, destDir string) (ssMat *types.ComMatrix, ssOutTCP, ssOutUDP []byte, err error) { + cs, err := clientutil.New(kubeconfig) + if err != nil { + return nil, nil, nil, err + } + + nodesList, err := cs.CoreV1Interface.Nodes().List(context.TODO(), metav1.ListOptions{}) + if err != nil { + return nil, nil, nil, err + } + + nodesComDetails := []types.ComDetails{} + err = debug.CreateNamespace(cs, consts.DefaultDebugNamespace) + if err != nil { + return nil, nil, nil, err + } + defer func() { + err := debug.DeleteNamespace(cs, consts.DefaultDebugNamespace) + if err != nil { + fmt.Printf("failed to delete debug namespace: %v", err) + } + }() + + nLock := &sync.Mutex{} + g := new(errgroup.Group) + for _, n := range nodesList.Items { + node := n + g.Go(func() error { + debugPod, err := debug.New(cs, node.Name, consts.DefaultDebugNamespace, consts.DefaultDebugPodImage) + if err != nil { + return err + } + defer func() { + err := debugPod.Clean() + if err != nil { + fmt.Printf("failed cleaning debug pod %s: %v", debugPod, err) + } + }() + + cds, ssTCP, ssUDP, err := ss.CreateSSOutputFromNode(debugPod, &node) + if err != nil { + return err + } + nLock.Lock() + ssTCPLine := fmt.Sprintf("node: %s\n%s\n", node.Name, string(ssTCP)) + ssUDPLine := fmt.Sprintf("node: %s\n%s\n", node.Name, string(ssUDP)) + + nodesComDetails = append(nodesComDetails, cds...) + ssOutTCP = append(ssOutTCP, []byte(ssTCPLine)...) + ssOutUDP = append(ssOutUDP, []byte(ssUDPLine)...) + nLock.Unlock() + return nil + }) + } + + err = g.Wait() + if err != nil { + return nil, nil, nil, err + } + + cleanedComDetails := types.CleanComDetails(nodesComDetails) + ssComMat := types.ComMatrix{Matrix: cleanedComDetails} + return &ssComMat, ssOutTCP, ssOutUDP, nil +} + +func getPrintFunction(format string) (func(m types.ComMatrix) ([]byte, error), error) { + switch format { + case "json": + return types.ToJSON, nil + case "csv": + return types.ToCSV, nil + case "yaml": + return types.ToYAML, nil + case "nft": + return types.ToNFTables, nil + default: + return nil, fmt.Errorf("invalid format: %s. Please specify json, csv, yaml, or nft", format) + } +} + +func WriteSSRawFiles(destDir string, ssOutTCP, ssOutUDP []byte) error { + tcpFile, err := os.OpenFile(path.Join(destDir, "raw-ss-tcp"), os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + return err + } + + udpFile, err := os.OpenFile(path.Join(destDir, "raw-ss-udp"), os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + tcpFile.Close() + return err + } + + defer tcpFile.Close() + defer udpFile.Close() + + _, err = tcpFile.Write([]byte(string(ssOutTCP))) + if err != nil { + return fmt.Errorf("failed writing to file: %s", err) + } + + _, err = udpFile.Write([]byte(string(ssOutUDP))) + if err != nil { + return fmt.Errorf("failed writing to file: %s", err) + } + + return nil +} + +func WriteMatrixToFileByType(mat types.ComMatrix, fileNamePrefix, format string, deployment Deployment, destDir string) error { + printFn, err := getPrintFunction(format) + if err != nil { + return err + } + + if format == types.FormatNFT { + masterMatrix, workerMatrix := separateMatrixByRole(mat) + err := writeMatrixToFile(masterMatrix, fileNamePrefix+"-master", format, printFn, destDir) + if err != nil { + return err + } + if deployment == MNO { + err := writeMatrixToFile(workerMatrix, fileNamePrefix+"-worker", format, printFn, destDir) + if err != nil { + return err + } + } + } else { + err := writeMatrixToFile(mat, fileNamePrefix, format, printFn, destDir) + if err != nil { + return err + } + } + + return nil +} + +func writeMatrixToFile(matrix types.ComMatrix, fileName, format string, printFn func(m types.ComMatrix) ([]byte, error), destDir string) error { + res, err := printFn(matrix) + if err != nil { + return err + } + + comMatrixFileName := filepath.Join(destDir, fmt.Sprintf("%s.%s", fileName, format)) + return os.WriteFile(comMatrixFileName, res, 0644) +} + +func GenerateMatrixDiff(mat1 types.ComMatrix, mat2 types.ComMatrix) string { + diff := consts.CSVHeaders + "\n" + for _, cd := range mat1.Matrix { + if mat2.Contains(cd) { + diff += fmt.Sprintf("%s\n", cd) + continue + } + + diff += fmt.Sprintf("+ %s\n", cd) + } + + for _, cd := range mat2.Matrix { + // Skip "rpc.statd" ports, these are randomly open ports on the node, + // no need to mention them in the matrix diff + if cd.Service == "rpc.statd" { + continue + } + + if !mat1.Contains(cd) { + diff += fmt.Sprintf("- %s\n", cd) + } + } + + return diff +} + +func separateMatrixByRole(matrix types.ComMatrix) (types.ComMatrix, types.ComMatrix) { + var masterMatrix, workerMatrix types.ComMatrix + for _, entry := range matrix.Matrix { + if entry.NodeRole == "master" { + masterMatrix.Matrix = append(masterMatrix.Matrix, entry) + } else if entry.NodeRole == "worker" { + workerMatrix.Matrix = append(workerMatrix.Matrix, entry) + } + } + + return masterMatrix, workerMatrix +} diff --git a/endpointslices/endpointslices.go b/endpointslices/endpointslices.go index f30d2cf8..2db184b7 100644 --- a/endpointslices/endpointslices.go +++ b/endpointslices/endpointslices.go @@ -5,6 +5,7 @@ import ( "fmt" "strings" + "github.com/openshift-kni/commatrix/types" log "github.com/sirupsen/logrus" corev1 "k8s.io/api/core/v1" discoveryv1 "k8s.io/api/discovery/v1" @@ -15,7 +16,6 @@ import ( "github.com/openshift-kni/commatrix/client" "github.com/openshift-kni/commatrix/consts" nodesutil "github.com/openshift-kni/commatrix/nodes" - "github.com/openshift-kni/commatrix/types" ) type EndpointSlicesInfo struct { diff --git a/go.mod b/go.mod index b9ba8e49..fc9782aa 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/gocarina/gocsv v0.0.0-20231116093920-b87c2d0e983a github.com/openshift/client-go v0.0.0-20240415214935-be70f772f157 github.com/sirupsen/logrus v1.9.3 + github.com/stretchr/testify v1.8.4 k8s.io/api v0.29.3 k8s.io/apimachinery v0.29.3 k8s.io/client-go v0.29.3 @@ -14,7 +15,10 @@ require ( sigs.k8s.io/yaml v1.4.0 ) -require github.com/openshift/api v0.0.0-20240415161129-d7aff303fa1a // indirect +require ( + github.com/openshift/api v0.0.0-20240415161129-d7aff303fa1a // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect +) require ( github.com/davecgh/go-spew v1.1.1 // indirect diff --git a/ss/ss.go b/ss/ss.go index a881c1ac..11f9cd1b 100644 --- a/ss/ss.go +++ b/ss/ss.go @@ -3,7 +3,6 @@ package ss import ( "encoding/json" "fmt" - "os" "regexp" "strconv" "strings" @@ -24,23 +23,14 @@ const ( duration = time.Second * 5 ) -func CreateComDetailsFromNode(debugPod *debug.DebugPod, node *corev1.Node, tcpFile, udpFile *os.File) ([]types.ComDetails, error) { - ssOutTCP, err := debugPod.ExecWithRetry("ss -anpltH", interval, duration) +func CreateSSOutputFromNode(debugPod *debug.DebugPod, node *corev1.Node) (res []types.ComDetails, ssOutTCP, ssOutUDP []byte, err error) { + ssOutTCP, err = debugPod.ExecWithRetry("ss -anpltH", interval, duration) if err != nil { - return nil, err + return nil, nil, nil, err } - ssOutUDP, err := debugPod.ExecWithRetry("ss -anpluH", interval, duration) + ssOutUDP, err = debugPod.ExecWithRetry("ss -anpluH", interval, duration) if err != nil { - return nil, err - } - - _, err = tcpFile.Write([]byte(fmt.Sprintf("node: %s\n%s\n", node.Name, string(ssOutTCP)))) - if err != nil { - return nil, fmt.Errorf("failed writing to file: %s", err) - } - _, err = udpFile.Write([]byte(fmt.Sprintf("node: %s\n%s\n", node.Name, string(ssOutUDP)))) - if err != nil { - return nil, fmt.Errorf("failed writing to file: %s", err) + return nil, nil, nil, err } ssOutFilteredTCP := filterEntries(splitByLines(ssOutTCP)) @@ -49,11 +39,11 @@ func CreateComDetailsFromNode(debugPod *debug.DebugPod, node *corev1.Node, tcpFi tcpComDetails := toComDetails(debugPod, ssOutFilteredTCP, "TCP", node) udpComDetails := toComDetails(debugPod, ssOutFilteredUDP, "UDP", node) - res := []types.ComDetails{} + res = []types.ComDetails{} res = append(res, udpComDetails...) res = append(res, tcpComDetails...) - return res, nil + return res, ssOutTCP, ssOutUDP, nil } func splitByLines(bytes []byte) []string { diff --git a/types/types.go b/types/types.go index 344a0559..9acbaea3 100644 --- a/types/types.go +++ b/types/types.go @@ -26,6 +26,7 @@ const ( FormatJSON = "json" FormatYAML = "yaml" FormatCSV = "csv" + FormatNFT = "nft" ) type ComMatrix struct { @@ -160,3 +161,46 @@ func (m ComMatrix) Contains(cd ComDetails) bool { return false } + +func ToNFTables(m ComMatrix) ([]byte, error) { + var tcpPorts []string + var udpPorts []string + for _, line := range m.Matrix { + if line.Protocol == "TCP" { + tcpPorts = append(tcpPorts, fmt.Sprint(line.Port)) + } else if line.Protocol == "UDP" { + udpPorts = append(udpPorts, fmt.Sprint(line.Port)) + } + } + + tcpPortsStr := strings.Join(tcpPorts, ", ") + udpPortsStr := strings.Join(udpPorts, ", ") + + result := fmt.Sprintf(`#!/usr/sbin/nft -f + + table inet openshift_filter { + chain OPENSHIFT { + type filter hook input priority 1; policy accept; + + # Allow loopback traffic + iif lo accept + + # Allow established and related traffic + ct state established,related accept + + # Allow ICMP on ipv4 + ip protocol icmp accept + # Allow ICMP on ipv6 + ip6 nexthdr ipv6-icmp accept + + # Allow specific TCP and UDP ports + tcp dport { %s } accept + udp dport { %s } accept + + # Logging and default drop + log prefix "firewall " drop + } + }`, tcpPortsStr, udpPortsStr) + + return []byte(result), nil +}