UAST diff

uast/diff/apply.go

@@ -0,0 +1,70 @@
+package diff
+
+import (
+	"errors"
+	"fmt"
+
+	"gopkg.in/bblfsh/sdk.v2/uast/nodes"
+)
+
+// Apply is a method that takes a tree (nodes.Node) and applies the current changelist to
+// that tree.
+func (cl Changelist) Apply(root nodes.Node) (nodes.Node, error) {
+	nodeDict := make(map[ID]nodes.Node)
+	nodes.WalkPreOrder(root, func(node nodes.Node) bool {
+		nodeDict[nodes.UniqueKey(node)] = node
+		return true
+	})
+
+	for _, change := range cl {
+		switch ch := change.(type) {
+		case Create:
+			// create a node and add to the dictionary
+			nodeDict[nodes.UniqueKey(ch.Node)] = ch.Node
+
+		case Attach:
+			// get src and chld from the dictionary, attach (modify src)
+			parent, ok := nodeDict[ch.Parent]
+			if !ok {
+				return nil, errors.New("diff: invalid attachment point")
+			}
+			child, ok := nodeDict[ch.Child]
+			if !ok {
+				child, ok = ch.Child.(nodes.Value)
+				if !ok {
+					return nil, fmt.Errorf("diff: unknown type of a child: %T", ch.Child)
+				}
+			}
+
+			switch key := ch.Key.(type) {
+			case String:
+				parent := parent.(nodes.Object)
+				parent[string(key)] = child
+			case Int:
+				parent := parent.(nodes.Array)
+				parent[int(key)] = child
+			default:
+				return nil, fmt.Errorf("diff: unknown type of a key: %T", ch.Key)
+			}
+		case Detach:
+			// get the src from the dictionary, deatach (modify src)
+			parent := nodeDict[ch.Parent]
+
+			switch key := ch.Key.(type) {
+			case String:
+				parent := parent.(nodes.Object)
+				delete(parent, string(key))
+			case Int:
+				return nil, errors.New("diff: cannot detach from an Array")
+			default:
+				return nil, fmt.Errorf("diff: unknown type of a key: %T", ch.Key)
+			}
+
+		case Delete:
+			return nil, errors.New("diff: delete is not supported in a Changelist")
+		default:
+			return nil, fmt.Errorf("diff: unknown change of type %T", change)
+		}
+	}
+	return root, nil
+}

uast/diff/changelist.go

@@ -0,0 +1,88 @@
+package diff
+
+import (
+	"gopkg.in/bblfsh/sdk.v2/uast/nodes"
+)
+
+// Changelist is a list of changes, a result of tree difference. Applying all changes from a
+// changelist on a source tree will result in it being transformed into the destination tree.
+type Changelist []Change
+
+// Change is a single operation performed
+type Change interface {
+	isChange()
+	// GroupID is the same for multiple changes that are a part of one high-level operation.
+	GroupID() uint64
+}
+
+// ID is a type representing node unique ID that can be compared in O(1)
+type ID nodes.Comparable
+
+// Key in a node, string for nodes.Object and int for nodes.Array
+type Key interface{ isKey() }
+
+// String is a wrapped string type for the Key interface.
+type String string
+
+// Int is a wrapped int type for the Key interface.
+type Int int
+
+func (Int) isKey()    {}
+func (String) isKey() {}
+
+// four change types
+
+// Create a node. Each array and object is created separately.
+type Create struct {
+	group uint64
+
+	// Node to create.
+	Node nodes.Node
+}
+
+func (Create) isChange() {}
+
+func (ch Create) GroupID() uint64 { return ch.group }
+
+// Delete a node by ID
+type Delete struct {
+	group uint64
+
+	// Node to delete.
+	Node ID
+}
+
+func (Delete) isChange() {}
+
+func (ch Delete) GroupID() uint64 { return ch.group }
+
+// Attach a node as a child of another node with a given key
+type Attach struct {
+	group uint64
+
+	// Parent node ID to attach the key to.
+	Parent ID
+	// Key to attach.
+	Key Key
+	// Child to attach on a given key.
+	Child ID
+}
+
+func (Attach) isChange() {}
+
+func (ch Attach) GroupID() uint64 { return ch.group }
+
+// Detach a child from a node
+type Detach struct {
+	group uint64
+
+	// Parent node ID to detach the key from.
+	Parent ID
+	// Key to detach. Currently detach semantics are only defined for nodes.Object so the
+	// Key is practically always a String.
+	Key Key
+}
+
+func (Detach) isChange() {}
+
+func (ch Detach) GroupID() uint64 { return ch.group }

uast/diff/changelist_test.go

@@ -0,0 +1,49 @@
+package diff
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"gopkg.in/bblfsh/sdk.v2/uast/nodes"
+	uastyml "gopkg.in/bblfsh/sdk.v2/uast/yaml"
+)
+
+const dataDir = "./testdata"
+
+func readUAST(t testing.TB, path string) nodes.Node {
+	data, err := ioutil.ReadFile(path)
+	require.NoError(t, err)
+	nd, err := uastyml.Unmarshal(data)
+	require.NoError(t, err)
+	return nd
+}
+
+func TestChangelist(t *testing.T) {
+	dir, err := os.Open(dataDir)
+	require.NoError(t, err)
+	defer dir.Close()
+	names, err := dir.Readdirnames(-1)
+	require.NoError(t, err)
+
+	for _, fname := range names {
+		if strings.HasSuffix(fname, "_src.uast") {
+			name := fname[:len(fname)-len("_src.uast")]
+
+			t.Run(name, func(t *testing.T) {
+				srcName := filepath.Join(dataDir, name+"_src.uast")
+				dstName := filepath.Join(dataDir, name+"_dst.uast")
+				src := readUAST(t, srcName)
+				dst := readUAST(t, dstName)
+
+				changes := Changes(src, dst)
+				newsrc, err := changes.Apply(src)
+				require.NoError(t, err)
+				require.True(t, nodes.Equal(newsrc, dst))
+			})
+		}
+	}
+}

uast/diff/create-testdata/main.go

@@ -0,0 +1,65 @@
+package main
+
+import (
+	"bufio"
+	"flag"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strconv"
+)
+
+// This is a script used to generate test data for the diff library. It uses pairs of files
+// to get diffed acquired from
+// https://github.com/vmarkovtsev/treediff/blob/49356e7f85c261ed88cf46326791765c58c22b5b/dataset/flask.tar.xz
+// It uses https://github.com/bblfsh/client-go#Installation to convert python sources into
+// uast yaml files.
+// This program outputs commands on the stdout - they are to be piped to sh/bash. It's a good
+// idea to inspect them before running by just reading the textual output of the program.
+// The program needs to be ran with proper commandline arguments. Example below. The cli arguments
+// are also documented if you run `go run ./uast-diff-create-testdata --help`.
+// $ go run ./create-testdata/ -d ~/data/sourced/treediff/python-dataset -f smalltest.txt -o . | sh -
+
+var (
+	datasetPath   = flag.String("d", "./", "Path to the python-dataset (unpacked flask.tar.gz)")
+	testnamesPath = flag.String("f", "./smalltest.txt", "File with testnames")
+	outPath       = flag.String("o", "./", "Output directory")
+)
+
+func firstFile(dirname string, fnamePattern string) string {
+	fns, err := filepath.Glob(filepath.Join(dirname, fnamePattern))
+	if err != nil {
+		panic(err)
+	}
+	return fns[0]
+}
+
+func main() {
+	flag.Parse()
+	_, err := os.Stat(*datasetPath)
+	if err != nil {
+		panic(err)
+	}
+
+	testnames, err := os.Open(*testnamesPath)
+	if err != nil {
+		panic(err)
+	}
+	defer testnames.Close()
+
+	scanner := bufio.NewScanner(testnames)
+
+	for i := 0; scanner.Scan(); i++ {
+		name := scanner.Text()
+		if name == "" {
+			continue
+		}
+		src := firstFile(*datasetPath, name+"_before*.src")
+		dst := firstFile(*datasetPath, name+"_after*.src")
+		iStr := strconv.Itoa(i)
+		fmt.Println("bblfsh-cli -l python " + src + " -o yaml > " +
+			filepath.Join(*outPath, iStr+"_src.uast"))
+		fmt.Println("bblfsh-cli -l python " + dst + " -o yaml > " +
+			filepath.Join(*outPath, iStr+"_dst.uast"))
+	}
+}