From 9f9fac2e64d205d79488372e3871a26ac3784121 Mon Sep 17 00:00:00 2001 From: Joe Chen Date: Sun, 21 Jul 2019 13:10:09 -0700 Subject: [PATCH] Initial version --- .gitignore | 18 ++ LICENSE | 21 ++ README.md | 44 ++++ cmd.go | 110 +++++++++ export.go | 65 ++++++ export/exporter.go | 508 ++++++++++++++++++++++++++++++++++++++++++ export/helper.go | 144 ++++++++++++ export/types.go | 39 ++++ go.mod | 8 + go.sum | 9 + main.go | 37 ++++ protocol/protocol.go | 518 +++++++++++++++++++++++++++++++++++++++++++ version.go | 45 ++++ 13 files changed, 1566 insertions(+) create mode 100644 .gitignore create mode 100644 LICENSE create mode 100644 README.md create mode 100644 cmd.go create mode 100644 export.go create mode 100644 export/exporter.go create mode 100644 export/helper.go create mode 100644 export/types.go create mode 100644 go.mod create mode 100644 go.sum create mode 100644 main.go create mode 100644 protocol/protocol.go create mode 100644 version.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..ac1557e5 --- /dev/null +++ b/.gitignore @@ -0,0 +1,18 @@ +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib + +# Test binary, build with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +.DS_Store +/.vscode +/.idea +/lsif-go +/data.lsif diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..c0e8dfd0 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2019 ᴊ. ᴄʜᴇɴ + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 00000000..59cfd6e9 --- /dev/null +++ b/README.md @@ -0,0 +1,44 @@ +# Language Server Indexing Format Implementation for Go + +🚨 This implementation is still in very early stage and follows the latest LSIF specification closely. + +## Language Server Index Format + +The purpose of the Language Server Index Format (LSIF) is to define a standard format for language servers or other programming tools to dump their knowledge about a workspace. This dump can later be used to answer language server [LSP](https://microsoft.github.io/language-server-protocol/) requests for the same workspace without running the language server itself. Since much of the information would be invalidated by a change to the workspace, the dumped information typically excludes requests used when mutating a document. So, for example, the result of a code complete request is typically not part of such a dump. + +A first draft specification can be found [here](https://github.com/Microsoft/language-server-protocol/blob/master/indexFormat/specification.md). + +## Quickstart + +1. Download and build this program via `go get github.com/sourcegraph/lsif-go`. +2. The binary `lsif-go` should be installed into your `$GOPATH/bin` directory. +3. Make sure you have added `$GOPATH/bin` to your `$PATH` envrionment variable. +4. Go to a root directory of a Go project, then execute `lsif-go export`: + +``` +➜ lsif-go export +Package: protocol + File: /Users/unknwon/Work/Sourcegraph/lsif-go/protocol/protocol.go + +Package: export + File: /Users/unknwon/Work/Sourcegraph/lsif-go/export/exporter.go + File: /Users/unknwon/Work/Sourcegraph/lsif-go/export/helper.go + File: /Users/unknwon/Work/Sourcegraph/lsif-go/export/types.go + +Package: main + File: /Users/unknwon/Work/Sourcegraph/lsif-go/cmd.go + File: /Users/unknwon/Work/Sourcegraph/lsif-go/export.go + File: /Users/unknwon/Work/Sourcegraph/lsif-go/main.go + File: /Users/unknwon/Work/Sourcegraph/lsif-go/version.go + +Processed in 950.942253ms +``` + +By default, the exporter dumps LSIF data to the file `data.lsif` in the working directory. + +Use `lsif-go -h` for more information + +## Testing Commands + +- Validate: `lsif-util validate data.lsif` +- Visualize: `lsif-util visualize data.lsif --distance 2 | dot -Tpng -o image.png` diff --git a/cmd.go b/cmd.go new file mode 100644 index 00000000..5ff6bb2b --- /dev/null +++ b/cmd.go @@ -0,0 +1,110 @@ +// This file is a modified version of sourcegraph/src-cli/cmd/src/cmd.go. + +package main + +import ( + "flag" + "fmt" + "log" + "os" +) + +// command is a subcommand handler and its flag set. +type command struct { + // flagSet is the flag set for the command. + flagSet *flag.FlagSet + + // handler is the function that is invoked to handle this command. + handler func(args []string) error + + // flagSet.Usage function to invoke on e.g. -h flag. If nil, a default one + // one is used. + usageFunc func() +} + +// matches tells if the given name matches this command. +func (c *command) matches(name string) bool { + if name == c.flagSet.Name() { + return true + } + return false +} + +// commander represents a top-level command with subcommands. +type commander []*command + +// run runs the command. +func (c commander) run(flagSet *flag.FlagSet, cmdName, usageText string, args []string) { + // Parse flags + flagSet.Usage = func() { + fmt.Fprintf(flag.CommandLine.Output(), usageText) + } + if !flagSet.Parsed() { + flagSet.Parse(args) + } + + // Print usage if the command is "help" + if flagSet.Arg(0) == "help" || flagSet.NArg() == 0 { + flagSet.Usage() + os.Exit(0) + } + + // Configure default usage funcs for commands + for _, cmd := range c { + cmd := cmd + if cmd.usageFunc != nil { + cmd.flagSet.Usage = cmd.usageFunc + continue + } + cmd.flagSet.Usage = func() { + fmt.Fprintf(flag.CommandLine.Output(), "Usage of '%s %s':\n", cmdName, cmd.flagSet.Name()) + cmd.flagSet.PrintDefaults() + } + } + + // Find the subcommand to execute + name := flagSet.Arg(0) + for _, cmd := range c { + if !cmd.matches(name) { + continue + } + + // Parse subcommand flags + args := flagSet.Args()[1:] + if err := cmd.flagSet.Parse(args); err != nil { + panic(fmt.Sprintf("all registered commands should use flag.ExitOnError: error: %s", err)) + } + + // Execute the subcommand. + if err := cmd.handler(flagSet.Args()[1:]); err != nil { + if _, ok := err.(*usageError); ok { + log.Println(err) + cmd.flagSet.Usage() + os.Exit(2) + } + if e, ok := err.(*exitCodeError); ok { + if e.error != nil { + log.Println(e.error) + } + os.Exit(e.exitCode) + } + log.Fatal(err) + } + os.Exit(0) + } + log.Printf("%s: unknown subcommand %q", cmdName, name) + log.Fatalf("Run '%s help' for usage.", cmdName) +} + +// usageError is an error type that subcommands can return in order to signal +// that a usage error has occurred. +type usageError struct { + error +} + +// exitCodeError is an error type that subcommands can return in order to +// specify the exact exit code. +type exitCodeError struct { + error + exitCode int +} diff --git a/export.go b/export.go new file mode 100644 index 00000000..1789fa0a --- /dev/null +++ b/export.go @@ -0,0 +1,65 @@ +package main + +import ( + "flag" + "fmt" + "log" + "os" + "time" + + "github.com/sourcegraph/lsif-go/export" + "github.com/sourcegraph/lsif-go/protocol" +) + +func init() { + usage := ` +Examples: + + Generate an LSIF dump for a workspace: + + $ lsif-go export -workspace=myrepo -output=myrepo.lsif + +` + + flagSet := flag.NewFlagSet("export", flag.ExitOnError) + usageFunc := func() { + fmt.Fprintf(flag.CommandLine.Output(), "Usage of 'lsif-go %s':\n", flagSet.Name()) + flagSet.PrintDefaults() + fmt.Println(usage) + } + var ( + workspaceFlag = flagSet.String("workspace", "", `The path to the workspace. (required)`) + outputFlag = flagSet.String("output", "data.lsif", `The output location of the dump.`) + ) + + handler := func(args []string) error { + flagSet.Parse(args) + + start := time.Now() + + out, err := os.Create(*outputFlag) + if err != nil { + return fmt.Errorf("create dump file: %v", err) + } + defer out.Close() + + err = export.Export(*workspaceFlag, out, protocol.ToolInfo{ + Name: "lsif-go", + Version: version, + Args: args, + }) + if err != nil { + return fmt.Errorf("export: %v", err) + } + + log.Println("Processed in", time.Since(start)) + return nil + } + + // Register the command + commands = append(commands, &command{ + flagSet: flagSet, + handler: handler, + usageFunc: usageFunc, + }) +} diff --git a/export/exporter.go b/export/exporter.go new file mode 100644 index 00000000..bb4b5632 --- /dev/null +++ b/export/exporter.go @@ -0,0 +1,508 @@ +// Package export is used to generate an LSIF dump for a workspace. +package export + +import ( + "bytes" + "encoding/json" + "fmt" + "go/token" + "go/types" + "io" + "io/ioutil" + "log" + "path/filepath" + "strconv" + + doc "github.com/slimsag/godocmd" + "github.com/sourcegraph/lsif-go/protocol" + "golang.org/x/tools/go/packages" +) + +// Export generates an LSIF dump for a workspace by traversing through source files +// and storing LSP responses to output source that implements io.Writer. It is +// caller's responsibility to close the output source if applicable. +func Export(workspace string, w io.Writer, toolInfo protocol.ToolInfo) error { + projectRoot, err := filepath.Abs(workspace) + if err != nil { + return fmt.Errorf("get abspath of project root: %v", err) + } + + pkgs, err := packages.Load(&packages.Config{ + Mode: packages.NeedName | packages.NeedFiles | + packages.NeedImports | packages.NeedDeps | + packages.NeedTypes | packages.NeedSyntax | packages.NeedTypesInfo, + Dir: projectRoot, + }, "./...") + if err != nil { + return fmt.Errorf("load packages: %v", err) + } + + return (&exporter{ + projectRoot: projectRoot, + w: w, + + pkgs: pkgs, + files: make(map[string]*fileInfo), + funcs: make(map[string]*defInfo), + vars: make(map[token.Pos]*defInfo), + types: make(map[string]*defInfo), + refs: make(map[string]*refResultInfo), + }).export(toolInfo) +} + +// exporter keeps track of all information needed to generate a LSIF dump. +type exporter struct { + projectRoot string + w io.Writer + + id int // The ID counter of the last element emitted + pkgs []*packages.Package + files map[string]*fileInfo // Keys: filename + funcs map[string]*defInfo // Keys: full name (with receiver for methods) + vars map[token.Pos]*defInfo // Keys: definition position + types map[string]*defInfo // Keys: type name + refs map[string]*refResultInfo // Keys: definition range ID +} + +func (e *exporter) export(info protocol.ToolInfo) error { + _, err := e.emitMetaData("file://"+e.projectRoot, info) + if err != nil { + return fmt.Errorf(`emit "metadata": %v`, err) + } + proID, err := e.emitProject() + if err != nil { + return fmt.Errorf(`emit "project": %v`, err) + } + + for _, p := range e.pkgs { + if err = e.exportPkg(p, proID); err != nil { + return fmt.Errorf("export package %q: %v", p.Name, err) + } + } + + return nil +} + +func (e *exporter) exportPkg(p *packages.Package, proID string) error { + // TODO(jchen): support "-verbose" flag + log.Println("Package:", p.Name) + defer log.Println() + + err := e.exportDefs(p, proID) + if err != nil { + return fmt.Errorf("export defs: %v", err) + } + + err = e.exportUses(p, proID) + if err != nil { + return fmt.Errorf("export uses: %v", err) + } + + for name, f := range e.files { + for _, rangeID := range e.files[name].defRangeIDs { + refResultID, err := e.emitReferenceResult() + if err != nil { + return fmt.Errorf(`emit "referenceResult": %v`, err) + } + + _, err = e.emitTextDocumentReferences(e.refs[rangeID].resultSetID, refResultID) + if err != nil { + return fmt.Errorf(`emit "textDocument/references": %v`, err) + } + + _, err = e.emitItemOfDefinitions(refResultID, e.refs[rangeID].defRangeIDs, f.docID) + if err != nil { + return fmt.Errorf(`emit "item": %v`, err) + } + + if len(e.refs[rangeID].refRangeIDs) > 0 { + _, err = e.emitItemOfReferences(refResultID, e.refs[rangeID].refRangeIDs, f.docID) + if err != nil { + return fmt.Errorf(`emit "item": %v`, err) + } + } + } + + _, err = e.emitContains(f.docID, append(f.defRangeIDs, f.useRangeIDs...)) + if err != nil { + return fmt.Errorf(`emit "contains": %v`, err) + } + } + + return nil +} + +func (e *exporter) exportDefs(p *packages.Package, proID string) error { + for _, f := range p.Syntax { + fpos := p.Fset.Position(f.Package) + // TODO(jchen): support "-verbose" flag + log.Println("\tFile:", fpos.Filename) + + docID, err := e.emitDocument(fpos.Filename) + if err != nil { + return fmt.Errorf(`emit "document": %v`, err) + } + + _, err = e.emitContains(proID, []string{docID}) + if err != nil { + return fmt.Errorf(`emit "contains": %v`, err) + } + + var rangeIDs []string + for ident, obj := range p.TypesInfo.Defs { + // Object is nil when not denote an object + if obj == nil { + continue + } + + // Only emit if the object belongs to current file + // TODO(jchen): mayeb emit other documents on the fly + ipos := p.Fset.Position(ident.Pos()) + if ipos.Filename != fpos.Filename { + continue + } + + resultSetID, err := e.emitResultSet() + if err != nil { + return fmt.Errorf(`emit "resultSet": %v`, err) + } + + rangeID, err := e.emitRange(lspRange(ipos, ident.Name)) + if err != nil { + return fmt.Errorf(`emit "range": %v`, err) + } + + _, err = e.emitNext(rangeID, resultSetID) + if err != nil { + return fmt.Errorf(`emit "next": %v`, err) + } + + qf := func(*types.Package) string { return "" } + var s string + var extra string + if f, ok := obj.(*types.Var); ok && f.IsField() { + // TODO(jchen): make this be like (T).F not "struct field F string". + s = "struct " + obj.String() + } else { + if obj, ok := obj.(*types.TypeName); ok { + typ := obj.Type().Underlying() + if _, ok := typ.(*types.Struct); ok { + s = "type " + obj.Name() + " struct" + extra = prettyPrintTypesString(types.TypeString(typ, qf)) + } + if _, ok := typ.(*types.Interface); ok { + s = "type " + obj.Name() + " interface" + extra = prettyPrintTypesString(types.TypeString(typ, qf)) + } + } + if s == "" { + s = types.ObjectString(obj, qf) + } + } + + contents := []protocol.MarkedString{ + protocol.NewMarkedString(s), + } + comments, err := findComments(f, obj) + if err != nil { + return fmt.Errorf("find comments: %v", err) + } + if comments != "" { + var b bytes.Buffer + doc.ToMarkdown(&b, comments, nil) + contents = append(contents, protocol.RawMarkedString(b.String())) + } + if extra != "" { + contents = append(contents, protocol.NewMarkedString(extra)) + } + + switch v := obj.(type) { + case *types.Func: + // TODO(jchen): support "-verbose" flag + //fmt.Printf("---> %T\n", obj) + //fmt.Println("Def:", ident.Name) + //fmt.Println("FullName:", v.FullName()) + //fmt.Println("iPos:", ipos) + //fmt.Println("vPos:", p.Fset.Position(v.Pos())) + e.funcs[v.FullName()] = &defInfo{ + rangeID: rangeID, + resultSetID: resultSetID, + contents: contents, + } + + // TODO(jchen): case *types.Const: + + case *types.Var: + // TODO(jchen): support "-verbose" flag + //fmt.Printf("---> %T\n", obj) + //fmt.Println("Def:", ident.Name) + //fmt.Println("iPos:", ipos) + e.vars[ident.Pos()] = &defInfo{ + rangeID: rangeID, + resultSetID: resultSetID, + contents: contents, + } + + case *types.TypeName: + // TODO(jchen): support "-verbose" flag + //fmt.Println("Def:", ident.Name) + //fmt.Println("Type:", obj.Type()) + //fmt.Println("Pos:", ipos) + e.types[obj.Type().String()] = &defInfo{ + rangeID: rangeID, + resultSetID: resultSetID, + contents: contents, + } + + // TODO(jchen): case *types.Label: + + // TODO(jchen): case *types.PkgName: + + // TODO(jchen): case *types.Builtin: + + // TODO(jchen): case *types.Nil: + + default: + // TODO(jchen): remove this case-branch + //fmt.Printf("---> %T\n", obj) + //fmt.Println("(default)") + //fmt.Println("Def:", ident) + //fmt.Println("Pos:", ipos) + //spew.Dump(obj) + } + + rangeIDs = append(rangeIDs, rangeID) + + if e.refs[rangeID] == nil { + e.refs[rangeID] = &refResultInfo{} + } + refResult := e.refs[rangeID] + refResult.resultSetID = resultSetID + refResult.defRangeIDs = append(refResult.defRangeIDs, rangeID) + } + + if e.files[fpos.Filename] == nil { + e.files[fpos.Filename] = &fileInfo{ + docID: docID, + } + } + e.files[fpos.Filename].defRangeIDs = append(e.files[fpos.Filename].defRangeIDs, rangeIDs...) + } + return nil +} + +func (e *exporter) exportUses(p *packages.Package, docID string) error { + for _, f := range p.Syntax { + fpos := p.Fset.Position(f.Package) + var rangeIDs []string + for ident, obj := range p.TypesInfo.Uses { + // Only emit if the object belongs to current file + ipos := p.Fset.Position(ident.Pos()) + if ipos.Filename != fpos.Filename { + continue + } + + var def *defInfo + switch v := obj.(type) { + case *types.Func: + // TODO(jchen): support "-verbose" flag + //fmt.Printf("---> %T\n", obj) + //fmt.Println("Use:", ident.Name) + //fmt.Println("FullName:", v.FullName()) + //fmt.Println("Pos:", ipos) + //fmt.Println("Scope.Parent.Pos:", p.Fset.Position(v.Scope().Parent().Pos())) + //fmt.Println("Scope.Pos:", p.Fset.Position(v.Scope().Pos())) + def = e.funcs[v.FullName()] + + // TODO(jchen): case *types.Const: + + case *types.Var: + // TODO(jchen): support "-verbose" flag + //fmt.Printf("---> %T\n", obj) + //fmt.Println("Use:", ident) + //fmt.Println("iPos:", ipos) + //fmt.Println("vPos:", p.Fset.Position(v.Pos())) + def = e.vars[v.Pos()] + + // TODO(jchen): case *types.PkgName: + //fmt.Println("Use:", ident) + //fmt.Println("Pos:", ipos) + //def = e.imports[ident.Name] + + case *types.TypeName: + // TODO(jchen): support "-verbose" flag + //fmt.Printf("---> %T\n", obj) + //fmt.Println("Use:", ident.Name) + //fmt.Println("Type:", obj.Type()) + //fmt.Println("Pos:", ipos) + def = e.types[obj.Type().String()] + + // TODO(jchen): case *types.Label: + + // TODO(jchen): case *types.PkgName: + + // TODO(jchen): case *types.Builtin: + + // TODO(jchen): case *types.Nil: + + default: + // TODO(jchen): remove this case-branch + //fmt.Printf("---> %T\n", obj) + //fmt.Println("(default)") + //fmt.Println("Use:", ident) + //fmt.Println("iPos:", ipos) + //fmt.Println("vPos:", p.Fset.Position(v.Pos())) + //spew.Dump(obj) + } + + if def == nil { + continue + } + + rangeID, err := e.emitRange(lspRange(ipos, ident.Name)) + if err != nil { + return fmt.Errorf(`emit "range": %v`, err) + } + rangeIDs = append(rangeIDs, rangeID) + + _, err = e.emitNext(rangeID, def.resultSetID) + if err != nil { + return fmt.Errorf(`emit "next": %v`, err) + } + + defResultID, err := e.emitDefinitionResult() + if err != nil { + return fmt.Errorf(`emit "definitionResult": %v`, err) + } + + _, err = e.emitTextDocumentDefinition(def.resultSetID, defResultID) + if err != nil { + return fmt.Errorf(`emit "textDocument/definition": %v`, err) + } + + _, err = e.emitItem(defResultID, []string{def.rangeID}, docID) + if err != nil { + return fmt.Errorf(`emit "item": %v`, err) + } + + hoverResultID, err := e.emitHoverResult(def.contents) + if err != nil { + return fmt.Errorf(`emit "hoverResult": %v`, err) + } + + _, err = e.emitTextDocumentHover(def.resultSetID, hoverResultID) + if err != nil { + return fmt.Errorf(`emit "textDocument/hover": %v`, err) + } + + rangeIDs = append(rangeIDs, rangeID) + + refResult := e.refs[def.rangeID] + if refResult != nil { + refResult.refRangeIDs = append(refResult.refRangeIDs, rangeID) + } + } + + e.files[fpos.Filename].useRangeIDs = append(e.files[fpos.Filename].useRangeIDs, rangeIDs...) + } + return nil +} + +func (e *exporter) writeNewLine() error { + _, err := e.w.Write([]byte("\n")) + return err +} + +func (e *exporter) nextID() string { + e.id++ + return strconv.Itoa(e.id) +} + +func (e *exporter) emit(v interface{}) error { + return json.NewEncoder(e.w).Encode(v) +} + +func (e *exporter) emitMetaData(root string, info protocol.ToolInfo) (string, error) { + id := e.nextID() + return id, e.emit(protocol.NewMetaData(id, root, info)) +} + +func (e *exporter) emitProject() (string, error) { + id := e.nextID() + return id, e.emit(protocol.NewProject(id)) +} + +func (e *exporter) emitDocument(path string) (string, error) { + contents, err := ioutil.ReadFile(path) + if err != nil { + return "", fmt.Errorf("read file: %v", err) + } + + id := e.nextID() + return id, e.emit(protocol.NewDocument(id, "file://"+path, contents)) +} + +func (e *exporter) emitContains(outV string, inVs []string) (string, error) { + id := e.nextID() + return id, e.emit(protocol.NewContains(id, outV, inVs)) +} + +func (e *exporter) emitResultSet() (string, error) { + id := e.nextID() + return id, e.emit(protocol.NewResultSet(id)) +} + +func (e *exporter) emitRange(start, end protocol.Pos) (string, error) { + id := e.nextID() + return id, e.emit(protocol.NewRange(id, start, end)) +} + +func (e *exporter) emitNext(outV, inV string) (string, error) { + id := e.nextID() + return id, e.emit(protocol.NewNext(id, outV, inV)) +} + +func (e *exporter) emitDefinitionResult() (string, error) { + id := e.nextID() + return id, e.emit(protocol.NewDefinitionResult(id)) +} + +func (e *exporter) emitTextDocumentDefinition(outV, inV string) (string, error) { + id := e.nextID() + return id, e.emit(protocol.NewTextDocumentDefinition(id, outV, inV)) +} + +func (e *exporter) emitHoverResult(contents []protocol.MarkedString) (string, error) { + id := e.nextID() + return id, e.emit(protocol.NewHoverResult(id, contents)) +} + +func (e *exporter) emitTextDocumentHover(outV, inV string) (string, error) { + id := e.nextID() + return id, e.emit(protocol.NewTextDocumentHover(id, outV, inV)) +} + +func (e *exporter) emitReferenceResult() (string, error) { + id := e.nextID() + return id, e.emit(protocol.NewReferenceResult(id)) +} + +func (e *exporter) emitTextDocumentReferences(outV, inV string) (string, error) { + id := e.nextID() + return id, e.emit(protocol.NewTextDocumentReferences(id, outV, inV)) +} + +func (e *exporter) emitItem(outV string, inVs []string, docID string) (string, error) { + id := e.nextID() + return id, e.emit(protocol.NewItem(id, outV, inVs, docID)) +} + +func (e *exporter) emitItemOfDefinitions(outV string, inVs []string, docID string) (string, error) { + id := e.nextID() + return id, e.emit(protocol.NewItemOfDefinitions(id, outV, inVs, docID)) +} + +func (e *exporter) emitItemOfReferences(outV string, inVs []string, docID string) (string, error) { + id := e.nextID() + return id, e.emit(protocol.NewItemOfReferences(id, outV, inVs, docID)) +} diff --git a/export/helper.go b/export/helper.go new file mode 100644 index 00000000..9e81503b --- /dev/null +++ b/export/helper.go @@ -0,0 +1,144 @@ +package export + +import ( + "bytes" + "go/ast" + "go/token" + "go/types" + "strings" + + "github.com/sourcegraph/lsif-go/protocol" + "golang.org/x/tools/go/ast/astutil" +) + +// lspRange transforms go/token.Position (1-based) to LSP start end end ranges (0-based) +// which takes in consideration of identifier's name length. +func lspRange(pos token.Position, name string) (start protocol.Pos, end protocol.Pos) { + return protocol.Pos{ + Line: pos.Line - 1, + Character: pos.Column - 1, + }, protocol.Pos{ + Line: pos.Line - 1, + Character: pos.Column - 1 + len(name), + } +} + +// prettyPrintTypesString is pretty printing specific to the output of +// types.*String. Instead of re-implementing the printer, we can just +// transform its output. +// +// This function is copied from +// https://sourcegraph.com/github.com/sourcegraph/go-langserver@02f4198/-/blob/langserver/hover.go#L332 +func prettyPrintTypesString(s string) string { + // Don't bother including the fields if it is empty + if strings.HasSuffix(s, "{}") { + return "" + } + var b bytes.Buffer + b.Grow(len(s)) + depth := 0 + for i := 0; i < len(s); i++ { + c := s[i] + switch c { + case ';': + b.WriteByte('\n') + for j := 0; j < depth; j++ { + b.WriteString(" ") + } + // Skip following space + i++ + + case '{': + if i == len(s)-1 { + // This should never happen, but in case it + // does give up + return s + } + + n := s[i+1] + if n == '}' { + // Do not modify {} + b.WriteString("{}") + // We have already written }, so skip + i++ + } else { + // We expect fields to follow, insert a newline and space + depth++ + b.WriteString(" {\n") + for j := 0; j < depth; j++ { + b.WriteString(" ") + } + } + + case '}': + depth-- + if depth < 0 { + return s + } + b.WriteString("\n}") + + default: + b.WriteByte(c) + } + } + return b.String() +} + +// findComments traverses the paths found within enclosing interval of the object +// to collect comments. +// +// This function is modified from +// https://sourcegraph.com/github.com/sourcegraph/go-langserver@02f4198/-/blob/langserver/hover.go#L106 +func findComments(f *ast.File, o types.Object) (string, error) { + if o == nil { + return "", nil + } + + if _, ok := o.(*types.PkgName); ok { + // TODO(jchen): add helper to find package doc + return "", nil + } + + // Resolve the object o into its respective ast.Node + paths, _ := astutil.PathEnclosingInterval(f, o.Pos(), o.Pos()) + if paths == nil { + return "", nil + } + + // Pull the comment out of the comment map for the file. Do + // not search too far away from the current path. + var comments string + for i := 0; i < 3 && i < len(paths) && comments == ""; i++ { + switch v := paths[i].(type) { + case *ast.Field: + // Concat associated documentation with any inline comments + comments = joinCommentGroups(v.Doc, v.Comment) + case *ast.ValueSpec: + comments = v.Doc.Text() + case *ast.TypeSpec: + comments = v.Doc.Text() + case *ast.GenDecl: + comments = v.Doc.Text() + case *ast.FuncDecl: + comments = v.Doc.Text() + } + } + return comments, nil +} + +// joinCommentGroups joins the resultant non-empty comment text from two +// CommentGroups with a newline. +// +// This function is copied from +// https://sourcegraph.com/github.com/sourcegraph/go-langserver@02f4198/-/blob/langserver/hover.go#L190 +func joinCommentGroups(a, b *ast.CommentGroup) string { + aText := a.Text() + bText := b.Text() + if aText == "" { + return bText + } else if bText == "" { + return aText + } else { + return aText + "\n" + bText + } +} diff --git a/export/types.go b/export/types.go new file mode 100644 index 00000000..ff3df728 --- /dev/null +++ b/export/types.go @@ -0,0 +1,39 @@ +package export + +import ( + "github.com/sourcegraph/lsif-go/protocol" +) + +// fileInfo contains LSIF information of a file. +type fileInfo struct { + // The vertex ID of the document that represents the file. + docID string + // The vertices ID of ranges that represents the definition. + // This information is collected to emit "contains" edge. + defRangeIDs []string + // The vertices ID of ranges that represents the definition use cases. + // This information is collected to emit "contains" edge. + useRangeIDs []string +} + +// defInfo contains LSIF information of a definition. +type defInfo struct { + // The vertex ID of the range that represents the definition. + rangeID string + // The vertex ID of the resultSet that represents the definition. + resultSetID string + // The contents will be used as the hover information. + contents []protocol.MarkedString +} + +// refResultInfo contains LSIF information of a reference result. +type refResultInfo struct { + // The vertex ID of the resultSet that represents the referenceResult. + resultSetID string + // The vertices ID of definition ranges that are referenced by the referenceResult. + // This information is collected to emit `{"label":"item", "property":"definitions"}` edge. + defRangeIDs []string + // The vertices ID of reference ranges that are represented by the referenceResult. + // This information is collected to emit `{"label":"item", "property":"references"}` edge. + refRangeIDs []string +} diff --git a/go.mod b/go.mod new file mode 100644 index 00000000..c40ada8b --- /dev/null +++ b/go.mod @@ -0,0 +1,8 @@ +module github.com/sourcegraph/lsif-go + +go 1.12 + +require ( + github.com/slimsag/godocmd v0.0.0-20161025000126-a1005ad29fe3 + golang.org/x/tools v0.0.0-20190717194535-128ec6dfca09 +) diff --git a/go.sum b/go.sum new file mode 100644 index 00000000..e8c12edf --- /dev/null +++ b/go.sum @@ -0,0 +1,9 @@ +github.com/slimsag/godocmd v0.0.0-20161025000126-a1005ad29fe3 h1:sAARUcYbwxnebBeWHzKX2MeyXtzy25TEglCTz9BhueY= +github.com/slimsag/godocmd v0.0.0-20161025000126-a1005ad29fe3/go.mod h1:AIBPxLCkKUFc2ZkjCXzs/Kk9OUhQLw/Zicdd0Rhqz2U= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/tools v0.0.0-20190717194535-128ec6dfca09 h1:4E8vPHnP5LziSZdudGZpi697p5agtemBV2l/jWYmG+M= +golang.org/x/tools v0.0.0-20190717194535-128ec6dfca09/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI= diff --git a/main.go b/main.go new file mode 100644 index 00000000..0c83910b --- /dev/null +++ b/main.go @@ -0,0 +1,37 @@ +// The program lsif-go is an LSIF exporter for Go. +package main + +import ( + "flag" + "log" + "os" +) + +const usageText = `lsif-go is an LSIF exporter for Go. +For more information, see https://github.com/sourcegraph/lsif-go + +Usage: + + lsif-go [options] command [command options] + +The options are: + +The commands are: + + export generates an LSIF dump for a workspace + version display version information + +Use "lsif-go [command] -h" for more information about a command. + +` + +// commands contains all registered subcommands. +var commands commander + +func main() { + // Configure logging + log.SetFlags(0) + log.SetPrefix("") + + commands.run(flag.CommandLine, "lsif-go", usageText, os.Args[1:]) +} diff --git a/protocol/protocol.go b/protocol/protocol.go new file mode 100644 index 00000000..2f128d56 --- /dev/null +++ b/protocol/protocol.go @@ -0,0 +1,518 @@ +package protocol + +import ( + "encoding/base64" + "encoding/json" + "strings" +) + +/* + Reference: https://github.com/microsoft/lsif-node/blob/master/protocol/src/protocol.ts +*/ + +const ( + // Version represnets the current LSIF version of implementation. + Version = "0.4.0" + // LanguageID is the language ID in LSP, For Go it's "go". + LanguageID = "go" + // PositionEncoding is the encoding used to compute line and character values in positions and ranges. + PositionEncoding = "utf-16" +) + +// Element contains basic information of an element in the graph. +type Element struct { + // The unique identifier of this element within the scope of project. + ID string `json:"id"` + // The kind of element in the graph. + Type ElementType `json:"type"` +} + +// ElementType represents the kind of element. +type ElementType string + +const ( + ElementVertex ElementType = "vertex" + ElementEdge ElementType = "edge" +) + +// Vertex contains information of a vertex in the graph. +type Vertex struct { + Element + // The kind of vertex in the graph. + Label VertexLabel `json:"label"` +} + +// VertexLabel represents the purpose of vertex. +type VertexLabel string + +const ( + VertexMetaData VertexLabel = "metaData" + VertexEvent VertexLabel = "$event" + VertexProject VertexLabel = "project" + VertexRange VertexLabel = "range" + VertexLocation VertexLabel = "location" + VertexDocument VertexLabel = "document" + VertexMoniker VertexLabel = "moniker" + VertexPackageInformation VertexLabel = "packageInformation" + VertexResultSet VertexLabel = "resultSet" + VertexDocumentSymbolResult VertexLabel = "documentSymbolResult" + VertexFoldingRangeResult VertexLabel = "foldingRangeResult" + VertexDocumentLinkResult VertexLabel = "documentLinkResult" + VertexDianosticResult VertexLabel = "diagnosticResult" + VertexDeclarationResult VertexLabel = "declarationResult" + VertexDefinitionResult VertexLabel = "definitionResult" + VertexTypeDefinitionResult VertexLabel = "typeDefinitionResult" + VertexHoverResult VertexLabel = "hoverResult" + VertexReferenceResult VertexLabel = "referenceResult" + VertexImplementationResult VertexLabel = "implementationResult" +) + +// ToolInfo contains information about the tool that created the dump. +type ToolInfo struct { + // The name of the tool. + Name string `json:"name"` + // The version of the tool. + Version string `json:"version,omitempty"` + // The arguments passed to the tool. + Args []string `json:"args,omitempty"` +} + +// MetaData contains basic information about the dump. +type MetaData struct { + Vertex + // The version of the LSIF format using semver notation. + Version string `json:"version"` + // The project root (in form of an URI) used to compute this dump. + ProjectRoot string `json:"projectRoot"` + // The string encoding used to compute line and character values in + // positions and ranges. Currently only 'utf-16' is support due to the + // limitations in LSP. + PositionEncoding string `json:"positionEncoding"` + // The information about the tool that created the dump. + ToolInfo ToolInfo `json:"toolInfo"` +} + +// NewMetaData returns a new MetaData object with given ID, project root +// and tool information. +func NewMetaData(id, root string, info ToolInfo) *MetaData { + return &MetaData{ + Vertex: Vertex{ + Element: Element{ + ID: id, + Type: ElementVertex, + }, + Label: VertexMetaData, + }, + Version: Version, + ProjectRoot: root, + PositionEncoding: PositionEncoding, + ToolInfo: info, + } +} + +// Project declares the language of the dump. +type Project struct { + Vertex + // The kind of language of the dump. + Kind string `json:"kind"` +} + +// NewProject returns a new Project object with given ID. +func NewProject(id string) *Project { + return &Project{ + Vertex: Vertex{ + Element: Element{ + ID: id, + Type: ElementVertex, + }, + Label: VertexProject, + }, + Kind: LanguageID, + } +} + +// Document is a vertex of document in the project. +type Document struct { + Vertex + // The URI indicates the location of the document. + URI string `json:"uri"` + // The language identifier of the document. + LanguageID string `json:"languageId"` + // The contents of the the document. + Contents string `json:"contents,omitempty"` +} + +// NewDocument returns a new Document object with given ID, URI and contents. +func NewDocument(id, uri string, contents []byte) *Document { + d := &Document{ + Vertex: Vertex{ + Element: Element{ + ID: id, + Type: ElementVertex, + }, + Label: VertexDocument, + }, + URI: uri, + LanguageID: LanguageID, + } + + if len(contents) > 0 { + d.Contents = base64.StdEncoding.EncodeToString(contents) + } + + return d +} + +// ResultSet acts as a hub to be able to store information common to a set of ranges. +type ResultSet struct { + Vertex +} + +// NewResultSet returns a new ResultSet object with given ID. +func NewResultSet(id string) *ResultSet { + return &ResultSet{ + Vertex: Vertex{ + Element: Element{ + ID: id, + Type: ElementVertex, + }, + Label: VertexResultSet, + }, + } +} + +// ReferenceResult acts as a hub to be able to store reference information common to a set of ranges. +type ReferenceResult struct { + Vertex +} + +// NewReferenceResult returns a new ReferenceResult object with given ID. +func NewReferenceResult(id string) *ResultSet { + return &ResultSet{ + Vertex: Vertex{ + Element: Element{ + ID: id, + Type: ElementVertex, + }, + Label: VertexReferenceResult, + }, + } +} + +// Pos contains the precise position information. +type Pos struct { + // The line number (0-based index) + Line int `json:"line"` + // The column of the character (0-based index) + Character int `json:"character"` +} + +// Range contains range information of a vertex object. +type Range struct { + Vertex + // The start position of the range. + Start Pos `json:"start"` + // The end position of the range. + End Pos `json:"end"` +} + +// NewRange returns a new Range object with given ID and position information. +func NewRange(id string, start, end Pos) *Range { + return &Range{ + Vertex: Vertex{ + Element: Element{ + ID: id, + Type: ElementVertex, + }, + Label: VertexRange, + }, + Start: start, + End: end, + } +} + +// DefinitionResult connects a definition that is spread over multiple ranges or multiple documents. +type DefinitionResult struct { + Vertex +} + +// NewDefinitionResult returns a new DefinitionResult object with given ID. +func NewDefinitionResult(id string) *DefinitionResult { + return &DefinitionResult{ + Vertex: Vertex{ + Element: Element{ + ID: id, + Type: ElementVertex, + }, + Label: VertexDefinitionResult, + }, + } +} + +// MarkedString is the object to describe marked string. +type MarkedString markedString + +type markedString struct { + // The language of the marked string. + Language string `json:"language"` + // The value of the marked string. + Value string `json:"value"` + // Indicates whether to marshal JSON as raw string. + isRawString bool +} + +func (m *MarkedString) UnmarshalJSON(data []byte) error { + if d := strings.TrimSpace(string(data)); len(d) > 0 && d[0] == '"' { + // Raw string + var s string + if err := json.Unmarshal(data, &s); err != nil { + return err + } + m.Value = s + m.isRawString = true + return nil + } + // Language string + ms := (*markedString)(m) + return json.Unmarshal(data, ms) +} + +func (m MarkedString) MarshalJSON() ([]byte, error) { + if m.isRawString { + return json.Marshal(m.Value) + } + return json.Marshal((markedString)(m)) +} + +// NewMarkedString returns a MarkedString with given string in language "go". +func NewMarkedString(s string) MarkedString { + return MarkedString{ + Language: LanguageID, + Value: s, + } +} + +// RawMarkedString returns a MarkedString consisting of only a raw string +// (i.e., "foo" instead of {"value":"foo", "language":"bar"}). +func RawMarkedString(s string) MarkedString { + return MarkedString{ + Value: s, + isRawString: true, + } +} + +type hoverResult struct { + Contents []MarkedString `json:"contents"` +} + +// HoverResult connects a hover that is spread over multiple ranges or multiple documents. +type HoverResult struct { + Vertex + // The result contents as the hover information. + Result hoverResult `json:"result"` +} + +// NewHoverResult returns a new HoverResult object with given ID, signature and extra contents. +func NewHoverResult(id string, contents []MarkedString) *HoverResult { + return &HoverResult{ + Vertex: Vertex{ + Element: Element{ + ID: id, + Type: ElementVertex, + }, + Label: VertexHoverResult, + }, + Result: hoverResult{ + Contents: contents, + }, + } +} + +// Edge contains information of an edge in the graph. +type Edge struct { + Element + // The kind of edge in the graph. + Label EdgeLabel `json:"label"` +} + +// EdgeLabel represents the purpose of an edge. +type EdgeLabel string + +const ( + EdgeContains EdgeLabel = "contains" + EdgeItem EdgeLabel = "item" + EdgeNext EdgeLabel = "next" + EdgeMoniker EdgeLabel = "moniker" + EdgeNextMoniker EdgeLabel = "nextMoniker" + EdgePackageInformation EdgeLabel = "packageInformation" + EdgeTextDocumentDocumentSymbol EdgeLabel = "textDocument/documentSymbol" + EdgeTextDocumentFoldingRange EdgeLabel = "textDocument/foldingRange" + EdgeTextDocumentDocumentLink EdgeLabel = "textDocument/documentLink" + EdgeTextDocumentDiagnostic EdgeLabel = "textDocument/diagnostic" + EdgeTextDocumentDefinition EdgeLabel = "textDocument/definition" + EdgeTextDocumentDeclaration EdgeLabel = "textDocument/declaration" + EdgeTextDocumentTypeDefinition EdgeLabel = "textDocument/typeDefinition" + EdgeTextDocumentHover EdgeLabel = "textDocument/hover" + EdgeTextDocumentReferences EdgeLabel = "textDocument/references" + EdgeTextDocumentImplementation EdgeLabel = "textDocument/implementation" +) + +// Next is an edge object that represents "next" relation. +type Next struct { + Edge + OutV string `json:"outV"` + InV string `json:"inV"` +} + +// NewNext returns a new Next object with given ID and vertices information. +func NewNext(id, outV, inV string) *Next { + return &Next{ + Edge: Edge{ + Element: Element{ + ID: id, + Type: ElementEdge, + }, + Label: EdgeNext, + }, + OutV: outV, + InV: inV, + } +} + +// Contains is an edge object that represents 1:n "contains" relation. +type Contains struct { + Edge + OutV string `json:"outV"` + InVs []string `json:"inVs"` +} + +// NewContains returns a new Contains object with given ID and vertices information. +func NewContains(id, outV string, inVs []string) *Contains { + return &Contains{ + Edge: Edge{ + Element: Element{ + ID: id, + Type: ElementEdge, + }, + Label: EdgeContains, + }, + OutV: outV, + InVs: inVs, + } +} + +// TextDocumentDefinition is an edge object that represents "textDocument/definition" relation. +type TextDocumentDefinition struct { + Edge + OutV string `json:"outV"` + InV string `json:"inV"` +} + +// NewTextDocumentDefinition returns a new TextDocumentDefinition object with given ID and +// vertices information. +func NewTextDocumentDefinition(id, outV, inV string) *TextDocumentDefinition { + return &TextDocumentDefinition{ + Edge: Edge{ + Element: Element{ + ID: id, + Type: ElementEdge, + }, + Label: EdgeTextDocumentDefinition, + }, + OutV: outV, + InV: inV, + } +} + +// TextDocumentHover is an edge object that represents "textDocument/hover" relation. +type TextDocumentHover struct { + Edge + OutV string `json:"outV"` + InV string `json:"inV"` +} + +// NewTextDocumentHover returns a new TextDocumentHover object with given ID and +// vertices information. +func NewTextDocumentHover(id, outV, inV string) *TextDocumentHover { + return &TextDocumentHover{ + Edge: Edge{ + Element: Element{ + ID: id, + Type: ElementEdge, + }, + Label: EdgeTextDocumentHover, + }, + OutV: outV, + InV: inV, + } +} + +// TextDocumentReferences is an edge object that represents "textDocument/references" relation. +type TextDocumentReferences struct { + Edge + OutV string `json:"outV"` + InV string `json:"inV"` +} + +// NewTextDocumentReferences returns a new TextDocumentReferences object with given ID and +// vertices information. +func NewTextDocumentReferences(id, outV, inV string) *TextDocumentReferences { + return &TextDocumentReferences{ + Edge: Edge{ + Element: Element{ + ID: id, + Type: ElementEdge, + }, + Label: EdgeTextDocumentReferences, + }, + OutV: outV, + InV: inV, + } +} + +// Item is an edge object that represents "item" relation. +type Item struct { + Edge + OutV string `json:"outV"` + InVs []string `json:"inVs"` + // The document the item belongs to. + Document string `json:"document"` + // The relationship property of the item. + Property string `json:"property,omitempty"` +} + +// NewItem returns a new Item object with given ID and vertices information. +func NewItem(id, outV string, inVs []string, document string) *Item { + return &Item{ + Edge: Edge{ + Element: Element{ + ID: id, + Type: ElementEdge, + }, + Label: EdgeItem, + }, + OutV: outV, + InVs: inVs, + Document: document, + } +} + +// NewItemWithProperty returns a new Item object with given ID, vertices, document and +// property information. +func NewItemWithProperty(id, outV string, inVs []string, document, property string) *Item { + i := NewItem(id, outV, inVs, document) + i.Property = property + return i +} + +// NewItemOfDefinitions returns a new Item object with given ID, vertices and document +// informationand in "definitions" relationship. +func NewItemOfDefinitions(id, outV string, inVs []string, document string) *Item { + return NewItemWithProperty(id, outV, inVs, document, "definitions") +} + +// NewItemOfReferences returns a new Item object with given ID, vertices and document +// informationand in "references" relationship. +func NewItemOfReferences(id, outV string, inVs []string, document string) *Item { + return NewItemWithProperty(id, outV, inVs, document, "references") +} diff --git a/version.go b/version.go new file mode 100644 index 00000000..7ef9a1b4 --- /dev/null +++ b/version.go @@ -0,0 +1,45 @@ +package main + +import ( + "flag" + "fmt" + "log" + + "github.com/sourcegraph/lsif-go/protocol" +) + +const version = "0.1.0" + +func init() { + usage := ` +Examples: + + Display the tool and protocol version: + + $ lsif-go version + +` + + flagSet := flag.NewFlagSet("version", flag.ExitOnError) + usageFunc := func() { + fmt.Fprintf(flag.CommandLine.Output(), "Usage of 'lsif-go %s':\n", flagSet.Name()) + flagSet.PrintDefaults() + fmt.Println(usage) + } + + handler := func(args []string) error { + flagSet.Parse(args) + + log.Println("Go LSIF exporter:", version) + log.Println("Protocol version:", protocol.Version) + + return nil + } + + // Register the command + commands = append(commands, &command{ + flagSet: flagSet, + handler: handler, + usageFunc: usageFunc, + }) +}