cmd/govulncheck: restructure testdata tests

Each subdir in testdata has the same structure that testdata used to
have: modules, testfiles, vulndb-v1. The are two test cases for now:
common and strip.

For now, the tests are executed sequentially, due to racing on moddir
environment variable. Parallelism willl be restored in a follow-up CL.

This CL is part of a series of CLs trying to improve govulncheck unit
testing.

Change-Id: Ia5b734c2ddcf5830a415c1ed1e46ffe2c4f29924
Reviewed-on: https://go-review.googlesource.com/c/vuln/+/571115
LUCI-TryBot-Result: Go LUCI <[email protected]>
Run-TryBot: Zvonimir Pavlinovic <[email protected]>
Reviewed-by: Maceo Thompson <[email protected]>
TryBot-Result: Gopher Robot <[email protected]>

cmd/govulncheck/main_test.go

@@ -76,7 +76,7 @@ var fixups = []fixup{
 		pattern: `"scanner_version": "[^"]*"`,
 		replace: `"scanner_version": "v0.0.0-00000000000-20000101010101"`,
 	}, {
-		pattern: `file:///(.*)/testdata/vulndb`,
+		pattern: `file:///(.*)/testdata/(.*)/vulndb`,
 		replace: `testdata/vulndb`,
 	}, {
 		pattern: `package (.*) is not in (GOROOT|std) (.*)`,
@@ -110,6 +110,31 @@ func init() {
 	}
 }
 
+// testCase models a group of tests in cmd/govulncheck testdata.
+type testCase struct {
+	dir      string   // subdirectory in testdata containing tests
+	skipGOOS []string // GOOS to skip, if any
+	strip    bool     // indicates if binaries should be stripped
+}
+
+func (tc testCase) skip() bool {
+	for _, sg := range tc.skipGOOS {
+		if runtime.GOOS == sg {
+			return true
+		}
+	}
+	return false
+}
+
+// testCases contains the list of major groups of tests in
+// cmd/govulncheck/testdata folder with information on how
+// to run them.
+var testCases = []testCase{
+	{dir: "common"},
+	// Binaries are not stripped on darwin with go1.21 and earlier. See #61051.
+	{dir: "strip", skipGOOS: []string{"darwin"}, strip: true},
+}
+
 func TestCommand(t *testing.T) {
 	if testing.Short() {
 		t.Skip("skipping test that uses internet in short mode")
@@ -119,55 +144,59 @@ func TestCommand(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	vulndbDir, err := filepath.Abs(filepath.Join(testDir, "testdata", "vulndb-v1"))
-	if err != nil {
-		t.Fatal(err)
-	}
-	govulndbURI, err := web.URLFromFilePath(vulndbDir)
-	if err != nil {
-		t.Fatalf("failed to create make vulndb url: %v", err)
-	}
-
-	moduleDirs, err := filepath.Glob("testdata/modules/*")
-	if err != nil {
-		t.Fatal(err)
-	}
 
-	os.Setenv("moddir", filepath.Join(testDir, "testdata", "modules"))
-	for _, md := range moduleDirs {
-		// Skip nogomod module. It has intended build issues.
-		if filepath.Base(md) == "nogomod" {
-			noModDir, err := filepath.Abs(t.TempDir())
-			if err != nil {
-				t.Fatal(err)
-			}
-			os.Setenv("nomoddir", noModDir)
-			b, err := os.ReadFile(filepath.Join(md, "vuln.go"))
-			if err != nil {
-				t.Fatal(err)
-			}
-			err = os.WriteFile(filepath.Join(noModDir, "vuln.go"), b, 0644)
-			if err != nil {
-				t.Fatal(err)
-			}
+	for _, tc := range testCases {
+		if tc.skip() {
 			continue
 		}
+		base := tc.dir
 
-		// Build test module binary.
-		binary, cleanup := test.GoBuild(t, md, "", filepath.Base(md) == "strip")
-		t.Cleanup(cleanup)
-		// Set an environment variable to the path to the binary, so tests
-		// can refer to it.
-		varName := filepath.Base(md) + "_binary"
-		os.Setenv(varName, binary)
-	}
-	testFilesDir := filepath.Join(testDir, "testdata", "testfiles")
-	os.Setenv("testdir", testFilesDir)
+		vulndbDir, err := filepath.Abs(filepath.Join(testDir, "testdata", base, "vulndb-v1"))
+		if err != nil {
+			t.Fatal(err)
+		}
+		govulndbURI, err := web.URLFromFilePath(vulndbDir)
+		if err != nil {
+			t.Fatalf("failed to create make vulndb url: %v", err)
+		}
 
-	runTestSuite(t, testFilesDir, govulndbURI.String(), *update)
-	if runtime.GOOS != "darwin" {
-		// Binaries are not stripped on darwin with go1.21 and earlier. See #61051.
-		runTestSuite(t, filepath.Join(testDir, "testdata", "strip"), govulndbURI.String(), *update)
+		moduleDirs, err := filepath.Glob(fmt.Sprintf("testdata/%s/modules/*", base))
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		os.Setenv("moddir", filepath.Join(testDir, "testdata", base, "modules"))
+		for _, md := range moduleDirs {
+			// Skip nogomod module. It has intended build issues.
+			if filepath.Base(md) == "nogomod" {
+				noModDir, err := filepath.Abs(t.TempDir())
+				if err != nil {
+					t.Fatal(err)
+				}
+				os.Setenv("nomoddir", noModDir)
+				b, err := os.ReadFile(filepath.Join(md, "vuln.go"))
+				if err != nil {
+					t.Fatal(err)
+				}
+				err = os.WriteFile(filepath.Join(noModDir, "vuln.go"), b, 0644)
+				if err != nil {
+					t.Fatal(err)
+				}
+				continue
+			}
+
+			// Build test module binary.
+			binary, cleanup := test.GoBuild(t, md, "", tc.strip)
+			t.Cleanup(cleanup)
+			// Set an environment variable to the path to the binary, so tests
+			// can refer to it. The binary name is unique across all test cases.
+			varName := tc.dir + "_" + filepath.Base(md) + "_binary"
+			os.Setenv(varName, binary)
+		}
+		testFilesDir := filepath.Join(testDir, "testdata", base, "testfiles")
+		os.Setenv("testdir", testFilesDir)
+
+		runTestSuite(t, testFilesDir, govulndbURI.String(), *update)
 	}
 }
 
@@ -189,22 +218,18 @@ var (
 	parallelLimiterInit sync.Once
 )
 
-// testSuite creates a cmdtest suite from dir. It also defines
+// testSuite creates a cmdtest suite from tsReadDir. It also defines
 // a govulncheck command on the suite that runs govulncheck
 // against vulnerability database available at vulndbDir.
-func runTestSuite(t *testing.T, dir string, govulndb string, update bool) {
+func runTestSuite(t *testing.T, tsReadDir string, govulndb string, update bool) {
 	parallelLimiterInit.Do(func() {
 		limit := (runtime.GOMAXPROCS(0) + 3) / 4
 		if limit > 2 && unsafe.Sizeof(uintptr(0)) < 8 {
 			limit = 2
 		}
 		parallelLimiter = make(chan struct{}, limit)
 	})
-	tsReadDir := dir
-	if filepath.Base(dir) != "strip" {
-		tsReadDir = filepath.Join(tsReadDir, "*")
-	}
-	ts, err := cmdtest.Read(tsReadDir)
+	ts, err := cmdtest.Read(filepath.Join(tsReadDir, "*"))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -221,7 +246,7 @@ func runTestSuite(t *testing.T, dir string, govulndb string, update bool) {
 		cmd.Stdout = buf
 		cmd.Stderr = buf
 		if inputFile != "" {
-			input, err := os.Open(filepath.Join(dir, inputFile))
+			input, err := os.Open(filepath.Join(tsReadDir, inputFile))
 			if err != nil {
 				return nil, err
 			}
@@ -299,7 +324,7 @@ func runTestSuite(t *testing.T, dir string, govulndb string, update bool) {
 		ts.Run(t, true)
 		return
 	}
-	ts.RunParallel(t, false)
+	ts.Run(t, false) // TODO: make parallel
 }
 
 func isJSONMode(args []string) bool {

cmd/govulncheck/testdata/testfiles/binary-call/binary_call_json.ct → cmd/govulncheck/testdata/common/testfiles/binary-call/binary_call_json.ct

@@ -1,6 +1,6 @@
 #####
 # Test basic binary scanning with json output
-$ govulncheck -format json -mode binary ${vuln_binary}
+$ govulncheck -format json -mode binary ${common_vuln_binary}
 {
   "config": {
     "protocol_version": "v1.0.0",

cmd/govulncheck/testdata/testfiles/binary-call/binary_call_text.ct → cmd/govulncheck/testdata/common/testfiles/binary-call/binary_call_text.ct

@@ -1,6 +1,6 @@
 #####
 # Test basic binary scanning with text output
-$ govulncheck -mode=binary ${vuln_binary} --> FAIL 3
+$ govulncheck -mode=binary ${common_vuln_binary} --> FAIL 3
 Scanning your binary for known vulnerabilities...
 
 === Symbol Results ===

cmd/govulncheck/testdata/testfiles/binary-call/binary_sarif.ct → cmd/govulncheck/testdata/common/testfiles/binary-call/binary_sarif.ct

@@ -1,6 +1,6 @@
 #####
 # Test basic binary scanning with sarif output
-$ govulncheck -format sarif -mode binary ${vuln_binary}
+$ govulncheck -format sarif -mode binary ${common_vuln_binary}
 {
   "version": "2.1.0",
   "$schema": "https://json.schemastore.org/sarif-2.1.0.json",

cmd/govulncheck/testdata/testfiles/binary-call/binary_vendored_json.ct → cmd/govulncheck/testdata/common/testfiles/binary-call/binary_vendored_json.ct

@@ -1,6 +1,6 @@
 #####
 # Test basic binary scanning with json output
-$ govulncheck -format json -mode binary ${vendored_binary}
+$ govulncheck -format json -mode binary ${common_vendored_binary}
 {
   "config": {
     "protocol_version": "v1.0.0",

cmd/govulncheck/testdata/testfiles/binary-module/binary_module_json.ct → cmd/govulncheck/testdata/common/testfiles/binary-module/binary_module_json.ct

@@ -1,6 +1,6 @@
 #####
 # Test binary scanning at the module level with json output
-$ govulncheck -format json -mode binary -scan module ${vuln_binary}
+$ govulncheck -format json -mode binary -scan module ${common_vuln_binary}
 {
   "config": {
     "protocol_version": "v1.0.0",

cmd/govulncheck/testdata/testfiles/binary-module/binary_module_text.ct → cmd/govulncheck/testdata/common/testfiles/binary-module/binary_module_text.ct

@@ -1,6 +1,6 @@
 #####
 # Test binary scanning at the module level
-$ govulncheck -mode=binary -scan module ${vuln_binary} --> FAIL 3
+$ govulncheck -mode=binary -scan module ${common_vuln_binary} --> FAIL 3
 Scanning your binary for known vulnerabilities...
 
 === Module Results ===

cmd/govulncheck/testdata/testfiles/binary-package/binary_package_json.ct → cmd/govulncheck/testdata/common/testfiles/binary-package/binary_package_json.ct

@@ -1,6 +1,6 @@
 #####
 # Test binary scanning at the package level with json output
-$ govulncheck -format json -mode binary -scan package ${vuln_binary}
+$ govulncheck -format json -mode binary -scan package ${common_vuln_binary}
 {
   "config": {
     "protocol_version": "v1.0.0",

cmd/govulncheck/testdata/testfiles/binary-package/binary_package_text.ct → cmd/govulncheck/testdata/common/testfiles/binary-package/binary_package_text.ct

@@ -1,5 +1,5 @@
 # Test binary scanning at the package level.
-$ govulncheck -mode=binary -scan package ${vuln_binary} --> FAIL 3
+$ govulncheck -mode=binary -scan package ${common_vuln_binary} --> FAIL 3
 Scanning your binary for known vulnerabilities...
 
 === Package Results ===

cmd/govulncheck/testdata/testfiles/failures/binary_fail.ct → cmd/govulncheck/testdata/common/testfiles/failures/binary_fail.ct

@@ -55,15 +55,15 @@ govulncheck: unrecognized binary format
 
 #####
 # Test of trying to analyze multiple binaries
-$ govulncheck -mode=binary ${vuln_binary} ${vuln_binary} --> FAIL 2
+$ govulncheck -mode=binary ${common_vuln_binary} ${common_vuln_binary} --> FAIL 2
 only 1 binary can be analyzed at a time
 
 #####
 # Test of trying to run -mode=binary with -tags flag
-$ govulncheck -tags=foo -mode=binary ${vuln_binary} --> FAIL 2
+$ govulncheck -tags=foo -mode=binary ${common_vuln_binary} --> FAIL 2
 the -tags flag is not supported in binary mode
 
 #####
 # Test of trying to run -mode=binary with the -test flag
-$ govulncheck -test -mode=binary ${vuln_binary} --> FAIL 2
+$ govulncheck -test -mode=binary ${common_vuln_binary} --> FAIL 2
 the -test flag is not supported in binary mode

cmd/govulncheck/testdata/testfiles/failures/source_fail.ct → cmd/govulncheck/testdata/common/testfiles/failures/source_fail.ct

@@ -10,7 +10,7 @@ See https://go.dev/doc/modules/managing-dependencies for more information.
 
 #####
 # Test of handing a binary to source mode
-$ govulncheck ${vuln_binary} --> FAIL 2
+$ govulncheck ${common_vuln_binary} --> FAIL 2
 govulncheck: myfile is a file.
 
 By default, govulncheck runs source analysis on Go modules.

cmd/govulncheck/testdata/strip/strip.ct → cmd/govulncheck/testdata/strip/testfiles/binary/strip.ct

@@ -1,6 +1,6 @@
 #####
 # Test for stripped binaries (see #57764)
-$ govulncheck -mode=binary ${strip_binary} --> FAIL 3
+$ govulncheck -mode=binary ${strip_vuln_binary} --> FAIL 3
 Scanning your binary for known vulnerabilities...
 
 === Symbol Results ===

cmd/govulncheck/testdata/strip/vulndb-v1/ID/GO-2020-0015.json

@@ -0,0 +1 @@
+{"schema_version":"1.3.1","id":"GO-2020-0015","modified":"2023-04-03T15:57:51Z","published":"2021-04-14T20:04:52Z","aliases":["CVE-2020-14040","GHSA-5rcv-m4m3-hfh7"],"summary":"Infinite loop when decoding some inputs in golang.org/x/text","details":"An attacker could provide a single byte to a UTF16 decoder instantiated with UseBOM or ExpectBOM to trigger an infinite loop if the String function on the Decoder is called, or the Decoder is passed to transform.String. If used to parse user supplied input, this may be used as a denial of service vector.","affected":[{"package":{"name":"golang.org/x/text","ecosystem":"Go"},"ranges":[{"type":"SEMVER","events":[{"introduced":"0"},{"fixed":"0.3.3"}]}],"ecosystem_specific":{"imports":[{"path":"golang.org/x/text/encoding/unicode","symbols":["bomOverride.Transform","utf16Decoder.Transform"]},{"path":"golang.org/x/text/transform","symbols":["String"]}]}}],"references":[{"type":"FIX","url":"https://go.dev/cl/238238"},{"type":"FIX","url":"https://go.googlesource.com/text/+/23ae387dee1f90d29a23c0e87ee0b46038fbed0e"},{"type":"REPORT","url":"https://go.dev/issue/39491"},{"type":"WEB","url":"https://groups.google.com/g/golang-announce/c/bXVeAmGOqz0"}],"credits":[{"name":"@abacabadabacaba and Anton Gyllenberg"}],"database_specific":{"url":"https://pkg.go.dev/vuln/GO-2020-0015"}}

cmd/govulncheck/testdata/strip/vulndb-v1/ID/GO-2021-0113.json

@@ -0,0 +1 @@
+{"schema_version":"1.3.1","id":"GO-2021-0113","modified":"2023-04-03T15:57:51Z","published":"2021-10-06T17:51:21Z","aliases":["CVE-2021-38561","GHSA-ppp9-7jff-5vj2"],"details":"Due to improper index calculation, an incorrectly formatted language tag can cause Parse to panic via an out of bounds read. If Parse is used to process untrusted user inputs, this may be used as a vector for a denial of service attack.","affected":[{"package":{"name":"golang.org/x/text","ecosystem":"Go"},"ranges":[{"type":"SEMVER","events":[{"introduced":"0"},{"fixed":"0.3.7"}]}],"ecosystem_specific":{"imports":[{"path":"golang.org/x/text/language","symbols":["MatchStrings","MustParse","Parse","ParseAcceptLanguage"]}]}}],"references":[{"type":"FIX","url":"https://go.dev/cl/340830"},{"type":"FIX","url":"https://go.googlesource.com/text/+/383b2e75a7a4198c42f8f87833eefb772868a56f"}],"credits":[{"name":"Guido Vranken"}],"database_specific":{"url":"https://pkg.go.dev/vuln/GO-2021-0113"}}

cmd/govulncheck/testdata/strip/vulndb-v1/index/db.json

@@ -0,0 +1 @@
+{"modified":"2023-04-03T15:57:51Z"}

cmd/govulncheck/testdata/strip/vulndb-v1/index/modules.json

@@ -0,0 +1 @@
+[{"path":"golang.org/x/text","vulns":[{"id":"GO-2020-0015","modified":"2023-04-03T15:57:51Z","fixed":"0.3.3"},{"id":"GO-2021-0113","modified":"2023-04-03T15:57:51Z","fixed":"0.3.7"}]}]

cmd/govulncheck/testdata/strip/vulndb-v1/index/vulns.json

@@ -0,0 +1 @@
+[{"id":"GO-2020-0015","modified":"2023-04-03T15:57:51Z","aliases":["CVE-2020-14040","GHSA-5rcv-m4m3-hfh7"]},{"id":"GO-2021-0113","modified":"2023-04-03T15:57:51Z","aliases":["CVE-2021-38561","GHSA-ppp9-7jff-5vj2"]}]