API docs: update to latest protocol spec (#163)

This is a few changes (sent as one because they all depend on each-other, sorry): * Updating `sourcegraph/enterprise/lib` and dealing with the fact it moved to `sourcegraph/lib` * Updating the API docs indexing code to emit according to the revised spec: https://github.com/sourcegraph/sourcegraph/pull/22080 Signed-off-by: Stephen Gutekanst <[email protected]>
sourcegraph · Jun 16, 2021 · 55b2a79 · 55b2a79
1 parent 095a862
commit 55b2a79
Show file tree

Hide file tree

Showing 15 changed files with 449 additions and 190 deletions.
diff --git a/cmd/lsif-go/args.go b/cmd/lsif-go/args.go
@@ -8,7 +8,7 @@ import (
 
 	"github.com/alecthomas/kingpin"
 	"github.com/sourcegraph/lsif-go/internal/git"
-	protocol "github.com/sourcegraph/sourcegraph/enterprise/lib/codeintel/lsif/protocol"
+	protocol "github.com/sourcegraph/sourcegraph/lib/codeintel/lsif/protocol"
 )
 
 var app = kingpin.New(

diff --git a/cmd/lsif-go/index.go b/cmd/lsif-go/index.go
@@ -7,8 +7,8 @@ import (
 
 	"github.com/sourcegraph/lsif-go/internal/gomod"
 	"github.com/sourcegraph/lsif-go/internal/indexer"
-	protocol "github.com/sourcegraph/sourcegraph/enterprise/lib/codeintel/lsif/protocol"
-	"github.com/sourcegraph/sourcegraph/enterprise/lib/codeintel/lsif/protocol/writer"
+	protocol "github.com/sourcegraph/sourcegraph/lib/codeintel/lsif/protocol"
+	"github.com/sourcegraph/sourcegraph/lib/codeintel/lsif/protocol/writer"
 )
 
 func writeIndex(repositoryRoot, repositoryRemote, projectRoot, moduleName, moduleVersion string, dependencies map[string]gomod.Module, outFile string) error {

diff --git a/go.mod b/go.mod
@@ -8,13 +8,10 @@ require (
 	github.com/google/go-cmp v0.5.5
 	github.com/hashicorp/go-multierror v1.1.1
 	github.com/hexops/autogold v1.3.0
-	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
-	github.com/modern-go/reflect2 v1.0.1 // indirect
 	github.com/pkg/errors v0.9.1
 	github.com/slimsag/godocmd v0.0.0-20161025000126-a1005ad29fe3
-	github.com/sourcegraph/sourcegraph/enterprise/lib v0.0.0-20210510223937-fa85dfb33caa
+	github.com/sourcegraph/sourcegraph/lib v0.0.0-20210615182129-22e00c9fc8cd
 	golang.org/x/mod v0.4.2 // indirect
-	golang.org/x/sys v0.0.0-20210331175145-43e1dd70ce54 // indirect
 	golang.org/x/tools v0.1.0
 	mvdan.cc/gofumpt v0.1.1 // indirect
 )
diff --git a/go.sum b/go.sum
diff --git a/internal/gomod/dependencies_test.go b/internal/gomod/dependencies_test.go
@@ -50,11 +50,11 @@ func TestParseGoListOutput(t *testing.T) {
 			"Replace": {
 				"Path": "./enterprise/lib",
 				"Dir": "/Users/efritz/dev/sourcegraph/sourcegraph/enterprise/lib",
-				"GoMod": "/Users/efritz/dev/sourcegraph/sourcegraph/enterprise/lib/go.mod",
+				"GoMod": "/Users/efritz/dev/sourcegraph/sourcegraph/lib/go.mod",
 				"GoVersion": "1.16"
 			},
 			"Dir": "/Users/efritz/dev/sourcegraph/sourcegraph/enterprise/lib",
-			"GoMod": "/Users/efritz/dev/sourcegraph/sourcegraph/enterprise/lib/go.mod",
+			"GoMod": "/Users/efritz/dev/sourcegraph/sourcegraph/lib/go.mod",
 			"GoVersion": "1.16"
 		}
 	`

diff --git a/internal/indexer/doctomarkdown_test.go b/internal/indexer/doctomarkdown_test.go
@@ -9,8 +9,8 @@ import (
 	"sort"
 	"strings"
 
-	"github.com/sourcegraph/sourcegraph/enterprise/lib/codeintel/lsif/protocol"
-	"github.com/sourcegraph/sourcegraph/enterprise/lib/codeintel/lsif/protocol/reader"
+	"github.com/sourcegraph/sourcegraph/lib/codeintel/lsif/protocol"
+	"github.com/sourcegraph/sourcegraph/lib/codeintel/lsif/protocol/reader"
 )
 
 // doctomarkdown converts the LSIF data in r to Markdown by scanning for data in the Sourcegraph LSIF
@@ -368,7 +368,7 @@ func (c *converter) findDocumentationChildrenFor(result int) (children []*reader
 	return
 }
 
-func (c *converter) recurse(this *reader.Element, depth int, slug string) error {
+func (c *converter) recurse(this *reader.Element, depth int, pathID string) error {
 	labelVertex, detailVertex, err := c.findDocumentationStringsFor(this.ID)
 	if err != nil {
 		return err
@@ -380,14 +380,16 @@ func (c *converter) recurse(this *reader.Element, depth int, slug string) error
 	if !tagsMatch(c.matchingTags, doc.Tags) {
 		return nil
 	}
-	emptyRootDocumentation := depth == 0 && detail.Value == ""
-	if !emptyRootDocumentation {
-		slug = joinSlugs(slug, doc.Slug)
-	}
 	depthStr := strings.Repeat("#", depth+1)
+
 	var infos []string
+	var thisPathID string
 	if doc.NewPage {
+		thisPathID = joinIdentifiers(pathID, cleanPathElement(doc.Identifier))
+		pathID = thisPathID
 		infos = append(infos, "new page")
+	} else {
+		thisPathID = pathID + "---" + cleanPathElement(doc.Identifier)
 	}
 	for _, tag := range doc.Tags {
 		infos = append(infos, string(tag))
@@ -396,7 +398,7 @@ func (c *converter) recurse(this *reader.Element, depth int, slug string) error
 	if annotations != "" {
 		annotations = fmt.Sprintf(" <small>(%s)</small>", annotations)
 	}
-	if _, err := fmt.Fprintf(c.out, "%s <a name=\"%s\">%s%s</a>\n\n", depthStr, slug, label.Value, annotations); err != nil {
+	if _, err := fmt.Fprintf(c.out, "%s <a name=\"%s\">%s%s</a>\n\n", depthStr, thisPathID, label.Value, annotations); err != nil {
 		return err
 	}
 	writeDetails := func(summary string, contents string) error {
@@ -441,14 +443,14 @@ func (c *converter) recurse(this *reader.Element, depth int, slug string) error
 		return err
 	}
 	for _, child := range children {
-		if err := c.recurse(child, depth+1, slug); err != nil {
+		if err := c.recurse(child, depth+1, pathID); err != nil {
 			return err
 		}
 	}
 	return nil
 }
 
-func (c *converter) recurseIndex(this *reader.Element, depth, parentDepth int, end bool, slug string) error {
+func (c *converter) recurseIndex(this *reader.Element, depth, parentDepth int, end bool, pathID string) error {
 	labelVertex, _, err := c.findDocumentationStringsFor(this.ID)
 	if err != nil {
 		return err
@@ -460,16 +462,20 @@ func (c *converter) recurseIndex(this *reader.Element, depth, parentDepth int, e
 		return nil
 	}
 
-	if parentDepth != 0 || depth != 0 {
-		slug = joinSlugs(slug, doc.Slug)
+	var thisPathID string
+	if doc.NewPage {
+		thisPathID = joinIdentifiers(pathID, cleanPathElement(doc.Identifier))
+		pathID = thisPathID
+	} else {
+		thisPathID = pathID + "---" + cleanPathElement(doc.Identifier)
 	}
 	if depth == 0 {
 		if _, err := fmt.Fprintf(c.out, "%s Index\n\n", strings.Repeat("#", parentDepth+1)); err != nil {
 			return err
 		}
 	} else {
 		depthStr := strings.Repeat("  ", depth-1)
-		if _, err := fmt.Fprintf(c.out, "%s- [%s](#%s)\n", depthStr, label.Value, slug); err != nil {
+		if _, err := fmt.Fprintf(c.out, "%s- [%s](#%s)\n", depthStr, label.Value, thisPathID); err != nil {
 			return err
 		}
 	}
@@ -483,7 +489,7 @@ func (c *converter) recurseIndex(this *reader.Element, depth, parentDepth int, e
 	}
 	for _, child := range children {
 		childDoc := child.Payload.(protocol.Documentation)
-		if err := c.recurseIndex(child, depth+1, parentDepth, childDoc.NewPage, slug); err != nil {
+		if err := c.recurseIndex(child, depth+1, parentDepth, childDoc.NewPage, pathID); err != nil {
 			return err
 		}
 	}
@@ -505,13 +511,19 @@ func tagsMatch(want, have []protocol.DocumentationTag) bool {
 	return true
 }
 
-func joinSlugs(a, b string) string {
+func joinIdentifiers(a, b string) string {
 	s := []string{}
 	if a != "" {
 		s = append(s, a)
 	}
 	if b != "" {
 		s = append(s, b)
 	}
-	return strings.Join(s, "-")
+	return strings.Join(s, "/")
+}
+
+func cleanPathElement(s string) string {
+	s = strings.Replace(s, "/", "-", -1)
+	s = strings.Replace(s, "#", "-", -1)
+	return s
 }
diff --git a/internal/indexer/documentation.go b/internal/indexer/documentation.go
@@ -14,8 +14,8 @@ import (
 	"github.com/hashicorp/go-multierror"
 	"github.com/pkg/errors"
 	doc "github.com/slimsag/godocmd"
-	"github.com/sourcegraph/sourcegraph/enterprise/lib/codeintel/lsif/protocol"
-	"github.com/sourcegraph/sourcegraph/enterprise/lib/codeintel/lsif/protocol/writer"
+	"github.com/sourcegraph/sourcegraph/lib/codeintel/lsif/protocol"
+	"github.com/sourcegraph/sourcegraph/lib/codeintel/lsif/protocol/writer"
 	"golang.org/x/tools/go/packages"
 )
 
@@ -75,11 +75,11 @@ func (i *Indexer) indexDocumentation() error {
 	// project itself.
 	rootDocumentationID := (&documentationResult{
 		Documentation: protocol.Documentation{
-			Slug:    "index",
-			NewPage: true,
-			Tags:    []protocol.DocumentationTag{protocol.DocumentationExported},
+			Identifier: "",
+			NewPage:    true,
+			Tags:       []protocol.DocumentationTag{protocol.DocumentationExported},
 		},
-		Label:  protocol.NewMarkupContent("Index", protocol.PlainText),
+		Label:  protocol.NewMarkupContent("", protocol.PlainText),
 		Detail: protocol.NewMarkupContent("", protocol.PlainText),
 	}).emit(i.emitter)
 	_ = i.emitter.EmitDocumentationResultEdge(rootDocumentationID, i.projectID)
@@ -171,20 +171,20 @@ func (d *docsIndexer) indexPackage(p *packages.Package) (docsPackage, error) {
 	}
 	packageDocsID := (&documentationResult{
 		Documentation: protocol.Documentation{
-			Slug:    slugify(shortestUniquePkgPath),
-			NewPage: true,
-			Tags:    pkgTags,
+			Identifier: shortestUniquePkgPath,
+			NewPage:    true,
+			Tags:       pkgTags,
 		},
 		Label:  protocol.NewMarkupContent("Package "+p.Name, protocol.PlainText),
 		Detail: protocol.NewMarkupContent(pkgDocsMarkdown, protocol.Markdown),
 	}).emit(d.i.emitter)
 
-	newSection := func(label, slug string, children []uint64) uint64 {
+	newSection := func(label, identifier string, children []uint64) uint64 {
 		sectionID := (&documentationResult{
 			Documentation: protocol.Documentation{
-				Slug:    slugify(slug),
-				NewPage: false,
-				Tags:    pkgTags,
+				Identifier: identifier,
+				NewPage:    false,
+				Tags:       pkgTags,
 			},
 			Label:  protocol.NewMarkupContent(label, protocol.PlainText),
 			Detail: protocol.NewMarkupContent("", protocol.PlainText),
@@ -432,9 +432,9 @@ func (t constVarDocs) result() *documentationResult {
 
 	return &documentationResult{
 		Documentation: protocol.Documentation{
-			Slug:    slugify(t.name),
-			NewPage: false,
-			Tags:    tags,
+			Identifier: t.name,
+			NewPage:    false,
+			Tags:       tags,
 		},
 		Label:  protocol.NewMarkupContent(t.label, protocol.PlainText),
 		Detail: protocol.NewMarkupContent(detail.String(), protocol.Markdown),
@@ -513,9 +513,9 @@ func (t typeDocs) result() *documentationResult {
 
 	return &documentationResult{
 		Documentation: protocol.Documentation{
-			Slug:    slugify(t.name),
-			NewPage: false,
-			Tags:    tags,
+			Identifier: t.name,
+			NewPage:    false,
+			Tags:       tags,
 		},
 		Label:  protocol.NewMarkupContent(t.label, protocol.PlainText),
 		Detail: protocol.NewMarkupContent(detail.String(), protocol.Markdown),
@@ -564,6 +564,9 @@ type funcDocs struct {
 	// The type of the receiver, or nil.
 	recvType ast.Expr
 
+	// The name of the receiver type, or an empty string.
+	recvTypeName string
+
 	// The type of return values, or nil.
 	resultTypes []ast.Expr
 
@@ -589,11 +592,15 @@ func (f funcDocs) result() *documentationResult {
 	detail.WriteString("```\n\n")
 	detail.WriteString(f.docsMarkdown)
 
+	identifier := f.name
+	if f.recvTypeName != "" {
+		identifier = f.recvTypeName + "." + f.name
+	}
 	return &documentationResult{
 		Documentation: protocol.Documentation{
-			Slug:    slugify(f.name),
-			NewPage: false,
-			Tags:    tags,
+			Identifier: identifier,
+			NewPage:    false,
+			Tags:       tags,
 		},
 		Label:  protocol.NewMarkupContent(f.label, protocol.PlainText),
 		Detail: protocol.NewMarkupContent(detail.String(), protocol.Markdown),
@@ -630,6 +637,7 @@ func (d *docsIndexer) indexFuncDecl(fset *token.FileSet, p *packages.Package, in
 			// Mark functions as unexported if they are an exported method of a type that is
 			// unexported.
 			if named, ok := dereference(p.TypesInfo.TypeOf(result.recvType)).(*types.Named); ok {
+				result.recvTypeName = named.Obj().Name()
 				if !named.Obj().Exported() {
 					result.exported = false
 				}
@@ -752,12 +760,6 @@ func dereference(t types.Type) types.Type {
 	return t
 }
 
-func slugify(s string) string {
-	s = strings.Replace(s, " ", "-", -1)
-	s = strings.Replace(s, "/", "-", -1)
-	return s
-}
-
 func godocToMarkdown(godoc string) string {
 	var buf bytes.Buffer
 	doc.ToMarkdown(&buf, godoc, nil)

diff --git a/internal/indexer/helpers_test.go b/internal/indexer/helpers_test.go
@@ -10,7 +10,7 @@ import (
 	"sync"
 	"testing"
 
-	protocol "github.com/sourcegraph/sourcegraph/enterprise/lib/codeintel/lsif/protocol"
+	protocol "github.com/sourcegraph/sourcegraph/lib/codeintel/lsif/protocol"
 	"golang.org/x/tools/go/packages"
 )
 

diff --git a/internal/indexer/hover.go b/internal/indexer/hover.go
@@ -4,7 +4,7 @@ import (
 	"fmt"
 	"go/types"
 
-	protocol "github.com/sourcegraph/sourcegraph/enterprise/lib/codeintel/lsif/protocol"
+	protocol "github.com/sourcegraph/sourcegraph/lib/codeintel/lsif/protocol"
 	"golang.org/x/tools/go/packages"
 )
 

diff --git a/internal/indexer/indexer.go b/internal/indexer/indexer.go
@@ -13,8 +13,8 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/sourcegraph/lsif-go/internal/gomod"
-	"github.com/sourcegraph/sourcegraph/enterprise/lib/codeintel/lsif/protocol"
-	"github.com/sourcegraph/sourcegraph/enterprise/lib/codeintel/lsif/protocol/writer"
+	"github.com/sourcegraph/sourcegraph/lib/codeintel/lsif/protocol"
+	"github.com/sourcegraph/sourcegraph/lib/codeintel/lsif/protocol/writer"
 	"golang.org/x/tools/go/packages"
 )
 

diff --git a/internal/indexer/indexer_test.go b/internal/indexer/indexer_test.go
@@ -9,8 +9,8 @@ import (
 	"testing"
 
 	"github.com/hexops/autogold"
-	"github.com/sourcegraph/sourcegraph/enterprise/lib/codeintel/lsif/protocol"
-	"github.com/sourcegraph/sourcegraph/enterprise/lib/codeintel/lsif/protocol/writer"
+	"github.com/sourcegraph/sourcegraph/lib/codeintel/lsif/protocol"
+	"github.com/sourcegraph/sourcegraph/lib/codeintel/lsif/protocol/writer"
 )
 
 func TestIndexer(t *testing.T) {

diff --git a/internal/indexer/moniker_test.go b/internal/indexer/moniker_test.go
@@ -8,7 +8,7 @@ import (
 
 	"github.com/google/go-cmp/cmp"
 	"github.com/sourcegraph/lsif-go/internal/gomod"
-	"github.com/sourcegraph/sourcegraph/enterprise/lib/codeintel/lsif/protocol/writer"
+	"github.com/sourcegraph/sourcegraph/lib/codeintel/lsif/protocol/writer"
 )
 
 func TestEmitExportMoniker(t *testing.T) {

diff --git a/internal/indexer/protocol.go b/internal/indexer/protocol.go
@@ -7,7 +7,7 @@ import (
 	"strings"
 
 	doc "github.com/slimsag/godocmd"
-	protocol "github.com/sourcegraph/sourcegraph/enterprise/lib/codeintel/lsif/protocol"
+	protocol "github.com/sourcegraph/sourcegraph/lib/codeintel/lsif/protocol"
 )
 
 const languageGo = "go"

diff --git a/internal/indexer/protocol_test.go b/internal/indexer/protocol_test.go
@@ -7,7 +7,7 @@ import (
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
-	protocol "github.com/sourcegraph/sourcegraph/enterprise/lib/codeintel/lsif/protocol"
+	protocol "github.com/sourcegraph/sourcegraph/lib/codeintel/lsif/protocol"
 )
 
 func TestRangeForObject(t *testing.T) {