feat: correctly index go stdlib (#184)

lsif-go now correctly indexes the go standard library.

It will redirect any references to the standard library in the following way:

"fmt" -> "github.com/golang/go/std/fmt"

This allows cross repo jump-to-def and references.

cmd/lsif-go/index.go

@@ -12,7 +12,7 @@ import (
 	"github.com/sourcegraph/sourcegraph/lib/codeintel/lsif/protocol/writer"
 )
 
-func writeIndex(repositoryRoot, repositoryRemote, projectRoot, moduleName, moduleVersion string, dependencies map[string]gomod.Module, outFile string, outputOptions output.Options) error {
+func writeIndex(repositoryRoot, repositoryRemote, projectRoot, moduleName, moduleVersion string, dependencies map[string]gomod.GoModule, outFile string, outputOptions output.Options) error {
 	start := time.Now()
 
 	out, err := os.Create(outFile)

go.mod

@@ -12,6 +12,7 @@ require (
 	github.com/slimsag/godocmd v0.0.0-20161025000126-a1005ad29fe3
 	github.com/sourcegraph/lsif-static-doc v0.0.0-20210728160750-2383b286c98b
 	github.com/sourcegraph/sourcegraph/lib v0.0.0-20210728181912-41b2ce5ea74c
+	golang.org/x/mod v0.4.2 // indirect
 	golang.org/x/tools v0.1.3
 	mvdan.cc/gofumpt v0.1.1 // indirect
 )

internal/gomod/dependencies.go

@@ -2,6 +2,7 @@ package gomod
 
 import (
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"log"
@@ -17,7 +18,7 @@ import (
 	"golang.org/x/tools/go/vcs"
 )
 
-type Module struct {
+type GoModule struct {
 	Name    string
 	Version string
 }
@@ -26,7 +27,7 @@ type Module struct {
 // and version as declared by the go.mod file in the current directory. The given root module
 // and version are used to resolve replace directives with local file paths. The root module
 // is expected to be a resolved import path (a valid URL, including a scheme).
-func ListDependencies(dir, rootModule, rootVersion string, outputOptions output.Options) (dependencies map[string]Module, err error) {
+func ListDependencies(dir, rootModule, rootVersion string, outputOptions output.Options) (dependencies map[string]GoModule, err error) {
 	if !isModule(dir) {
 		log.Println("WARNING: No go.mod file found in current directory.")
 		return nil, nil
@@ -39,7 +40,21 @@ func ListDependencies(dir, rootModule, rootVersion string, outputOptions output.
 			return
 		}
 
-		dependencies, err = parseGoListOutput(output, rootVersion)
+		// The reason we run this command separate is because we want the
+		// information about this package specifically. Currently, it seems
+		// that "go list all" will place the current modules information first
+		// in the list, but we don't know that that is guaranteed.
+		//
+		// Because of that, we do a separate execution to guarantee we get only
+		// this package information to use to determine the corresponding
+		// goVersion.
+		modOutput, err := command.Run(dir, "go", "list", "-mod=readonly", "-m", "-json")
+		if err != nil {
+			err = fmt.Errorf("failed to list module info: %v\n%s", err, output)
+			return
+		}
+
+		dependencies, err = parseGoListOutput(output, modOutput, rootVersion)
 		if err != nil {
 			return
 		}
@@ -61,15 +76,18 @@ type jsonModule struct {
 	Name    string      `json:"Path"`
 	Version string      `json:"Version"`
 	Replace *jsonModule `json:"Replace"`
+
+	// The Golang version required for this module
+	GoVersion string `json:"GoVersion"`
 }
 
 // parseGoListOutput parse the JSON output of `go list -m`. This method returns a map from
 // import paths to pairs of declared (unresolved) module names and version pairs that respect
 // replacement directives specified in go.mod. Replace directives indicating a local file path
 // will create a module with the given root version, which is expected to be the same version
 // as the module being indexed.
-func parseGoListOutput(output, rootVersion string) (map[string]Module, error) {
-	dependencies := map[string]Module{}
+func parseGoListOutput(output, modOutput, rootVersion string) (map[string]GoModule, error) {
+	dependencies := map[string]GoModule{}
 	decoder := json.NewDecoder(strings.NewReader(output))
 
 	for {
@@ -95,15 +113,60 @@ func parseGoListOutput(output, rootVersion string) (map[string]Module, error) {
 			module.Version = rootVersion
 		}
 
-		dependencies[importPath] = Module{
+		dependencies[importPath] = GoModule{
 			Name:    module.Name,
 			Version: cleanVersion(module.Version),
 		}
 	}
 
+	var thisModule jsonModule
+	if err := json.NewDecoder(strings.NewReader(modOutput)).Decode(&thisModule); err != nil {
+		return nil, err
+	}
+
+	if thisModule.GoVersion == "" {
+		return nil, errors.New("could not find GoVersion for current module")
+	}
+
+	setGolangDependency(dependencies, thisModule.GoVersion)
+
 	return dependencies, nil
 }
 
+// The repository to find the source code for golang.
+var golangRepository = "github.com/golang/go"
+
+func setGolangDependency(dependencies map[string]GoModule, goVersion string) {
+	dependencies[golangRepository] = GoModule{
+		Name: golangRepository,
+
+		// The reason we prefix version with "go" is because in golang/go, all the release
+		// tags are prefixed with "go". So turn "1.15" -> "go1.15"
+		Version: fmt.Sprintf("go%s", goVersion),
+	}
+}
+
+func GetGolangDependency(dependencies map[string]GoModule) GoModule {
+	return dependencies[golangRepository]
+}
+
+// NormalizeMonikerPackage returns a normalized path to ensure that all
+// standard library paths are handled the same. Primarily to make sure
+// that both the golangRepository and "std/" paths are normalized.
+func NormalizeMonikerPackage(path string) string {
+	// When indexing _within_ the golang/go repository, `std/` is prefixed
+	// to packages. So we trim that here just to be sure that we keep
+	// consistent names.
+	normalizedPath := strings.TrimPrefix(path, "std/")
+
+	if !isStandardlibPackge(normalizedPath) {
+		return path
+	}
+
+	// Make sure we don't see double "std/" in the package for the moniker
+	return fmt.Sprintf("%s/std/%s", golangRepository, normalizedPath)
+}
+
 // versionPattern matches a versioning ending in a 12-digit sha, e.g., vX.Y.Z.-yyyymmddhhmmss-abcdefabcdef
 var versionPattern = regexp.MustCompile(`^.*-([a-f0-9]{12})$`)
 
@@ -147,20 +210,18 @@ func resolveImportPaths(rootModule string, modules []string) map[string]string {
 				// Try to resolve the import path if it looks like a local path
 				name, err := resolveLocalPath(name, rootModule)
 				if err != nil {
-					log.Println(fmt.Sprintf("WARNING: Failed to resolve %s (%s).", name, err))
+					log.Println(fmt.Sprintf("WARNING: Failed to resolve local %s (%s).", name, err))
 					continue
 				}
 
 				// Determine path suffix relative to the import path
-				repoRoot, err := vcs.RepoRootForImportPath(name, false)
-				if err != nil {
-					log.Println(fmt.Sprintf("WARNING: Failed to resolve %s (%s).", name, err))
+				resolved, ok := resolveRepoRootForImportPath(name)
+				if !ok {
 					continue
 				}
-				suffix := strings.TrimPrefix(name, repoRoot.Root)
 
 				m.Lock()
-				namesToResolve[originalName] = repoRoot.Repo + suffix
+				namesToResolve[originalName] = resolved
 				m.Unlock()
 			}
 		}()
@@ -170,6 +231,28 @@ func resolveImportPaths(rootModule string, modules []string) map[string]string {
 	return namesToResolve
 }
 
+// resolveRepoRootForImportPath will get the resolved name after handling vsc RepoRoots and any
+// necessary handling of the standard library
+func resolveRepoRootForImportPath(name string) (string, bool) {
+	// When indexing golang/go, there are some references to the package "std" itself.
+	//    Generally, "std/" is not referenced directly (it is just assumed when you have "fmt" or similar
+	//    in your imports), but inside of golang/go, it is directly referenced.
+	//
+	//    In that case, we just return it directly, there is no other resolving to do.
+	if name == "std" {
+		return name, true
+	}
+
+	repoRoot, err := vcs.RepoRootForImportPath(name, false)
+	if err != nil {
+		log.Println(fmt.Sprintf("WARNING: Failed to resolve repo %s (%s) %s.", name, err, repoRoot))
+		return "", false
+	}
+
+	suffix := strings.TrimPrefix(name, repoRoot.Root)
+	return repoRoot.Repo + suffix, true
+}
+
 // resolveLocalPath converts the given name to an import path if it looks like a local path based on
 // the given root module. The root module, if non-empty, is expected to be a resolved import path
 // (a valid URL, including a scheme). If the name does not look like a local path, it will be returned
@@ -193,10 +276,10 @@ func resolveLocalPath(name, rootModule string) (string, error) {
 
 // mapImportPaths replace each module name with the value in the given resolved import paths
 // map. If the module name is not present in the map, no change is made to the module value.
-func mapImportPaths(dependencies map[string]Module, resolvedImportPaths map[string]string) {
+func mapImportPaths(dependencies map[string]GoModule, resolvedImportPaths map[string]string) {
 	for importPath, module := range dependencies {
 		if name, ok := resolvedImportPaths[module.Name]; ok {
-			dependencies[importPath] = Module{
+			dependencies[importPath] = GoModule{
 				Name:    name,
 				Version: module.Version,
 			}

internal/gomod/dependencies_test.go

@@ -59,12 +59,23 @@ func TestParseGoListOutput(t *testing.T) {
 		}
 	`
 
-	modules, err := parseGoListOutput(output, "v1.2.3")
+	modOutput := `
+		{
+				"Path": "github.com/sourcegraph/lsif-go",
+				"Main": true,
+				"Dir": "/home/tjdevries/sourcegraph/lsif-go.git/asdf",
+				"GoMod": "/home/tjdevries/sourcegraph/lsif-go.git/asdf/go.mod",
+				"GoVersion": "1.15"
+		}
+	`
+
+	modules, err := parseGoListOutput(output, modOutput, "v1.2.3")
 	if err != nil {
 		t.Fatalf("unexpected error: %s", err)
 	}
 
-	expected := map[string]Module{
+	expected := map[string]GoModule{
+		"github.com/golang/go":                              {Name: "github.com/golang/go", Version: "go1.15"},
 		"github.com/gavv/httpexpect":                        {Name: "github.com/gavv/httpexpect", Version: "v2.0.0"},
 		"github.com/getsentry/raven-go":                     {Name: "github.com/getsentry/raven-go", Version: "v0.2.0"},
 		"github.com/gfleury/go-bitbucket-v1":                {Name: "github.com/gfleury/go-bitbucket-v1", Version: "e5170e3280fb"},
@@ -125,3 +136,24 @@ func TestResolveImportPaths(t *testing.T) {
 		t.Errorf("unexpected import paths (-want +got): %s", diff)
 	}
 }
+
+func TestNormalizeMonikerPackage(t *testing.T) {
+	testCases := map[string]string{
+		"fmt": "github.com/golang/go/std/fmt",
+
+		// This happens sometimes in the standard library, that we have "std/" prefixed.
+		"std/hash": "github.com/golang/go/std/hash",
+
+		// User libs should be unchanged.
+		"github.com/sourcegraph/sourcegraph/lib": "github.com/sourcegraph/sourcegraph/lib",
+
+		// Unknown libs should not be changed (for example, custom proxy)
+		"myCustomPackage": "myCustomPackage",
+	}
+
+	for path, expected := range testCases {
+		if diff := cmp.Diff(expected, NormalizeMonikerPackage(path)); diff != "" {
+			t.Errorf("unexpected normalized moniker package (-want +got): %s", diff)
+		}
+	}
+}