feat: faster parser (#3)

* fix(load): support nested objects * feat: initial golang parser Moves most of load.libsonnet to a Golang app for hopefully much faster parsing (fingers crossed) * feat(fast): nested objects, subpackages * feat: incorporate fast mode into main binary Moves the work done in the fast directory into the main executable, removing the Jsonnet based parser in favor of the much faster Golang one.
jsonnet-libs · May 10, 2020 · 4e4d4a7 · 4e4d4a7
1 parent 75f8e73
commit 4e4d4a7
Show file tree

Hide file tree

Showing 7 changed files with 441 additions and 56 deletions.
diff --git a/fast/main.go b/fast/main.go
@@ -0,0 +1,234 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/google/go-jsonnet"
+	"github.com/sh0rez/docsonnet/pkg/docsonnet"
+	"github.com/sh0rez/docsonnet/pkg/render"
+)
+
+func main() {
+	data, err := eval()
+	if err != nil {
+		log.Fatalln(err)
+	}
+	// fmt.Println(string(data))
+
+	var d DS
+	if err := json.Unmarshal(data, &d); err != nil {
+		log.Fatalln(err)
+	}
+
+	pkg := load(d)
+
+	fmt.Println("render")
+	res := render.Render(pkg)
+	for k, v := range res {
+		fmt.Println(k)
+		if err := ioutil.WriteFile(filepath.Join("docs", k), []byte(v), 0644); err != nil {
+			log.Fatalln(err)
+		}
+	}
+}
+
+// load docsonnet
+//
+// Data assumptions:
+// - only map[string]interface{} and docsonnet fields
+// - docsonnet fields (#...) coming first
+func load(d DS) docsonnet.Package {
+	start := time.Now()
+
+	pkg := d.Package()
+	fmt.Println("load", pkg.Name)
+
+	pkg.API = make(docsonnet.Fields)
+	pkg.Sub = make(map[string]docsonnet.Package)
+
+	for k, v := range d {
+		if k == "#" {
+			continue
+		}
+
+		f := v.(map[string]interface{})
+
+		// docsonnet field
+		name := strings.TrimPrefix(k, "#")
+		if strings.HasPrefix(k, "#") {
+			pkg.API[name] = loadField(name, f, d)
+			continue
+		}
+
+		// non-docsonnet
+		// subpackage?
+		if _, ok := f["#"]; ok {
+			p := load(DS(f))
+			pkg.Sub[p.Name] = p
+			continue
+		}
+
+		// non-annotated nested?
+		// try to load, but skip when already loaded as annotated above
+		if nested, ok := loadNested(name, f); ok && !fieldsHas(pkg.API, name) {
+			pkg.API[name] = *nested
+			continue
+		}
+	}
+
+	fmt.Println("done load", pkg.Name, time.Since(start))
+	return pkg
+}
+
+func fieldsHas(f docsonnet.Fields, key string) bool {
+	_, b := f[key]
+	return b
+}
+
+func loadNested(name string, msi map[string]interface{}) (*docsonnet.Field, bool) {
+	out := docsonnet.Object{
+		Name:   name,
+		Fields: make(docsonnet.Fields),
+	}
+
+	ok := false
+	for k, v := range msi {
+		f := v.(map[string]interface{})
+		n := strings.TrimPrefix(k, "#")
+
+		if !strings.HasPrefix(k, "#") {
+			if l, ok := loadNested(k, f); ok {
+				out.Fields[n] = *l
+			}
+			continue
+		}
+
+		ok = true
+		l := loadField(n, f, msi)
+		out.Fields[n] = l
+	}
+
+	if !ok {
+		return nil, false
+	}
+
+	return &docsonnet.Field{Object: &out}, true
+}
+
+func loadField(name string, field map[string]interface{}, parent map[string]interface{}) docsonnet.Field {
+	if ifn, ok := field["function"]; ok {
+		return loadFn(name, ifn.(map[string]interface{}))
+	}
+
+	if iobj, ok := field["object"]; ok {
+		return loadObj(name, iobj.(map[string]interface{}), parent)
+	}
+
+	panic("docsonnet field lacking {function | object}")
+}
+
+func loadFn(name string, msi map[string]interface{}) docsonnet.Field {
+	fn := docsonnet.Function{
+		Name: name,
+		Help: msi["help"].(string),
+	}
+	if args, ok := msi["args"]; ok {
+		fn.Args = loadArgs(args.([]interface{}))
+	}
+	return docsonnet.Field{Function: &fn}
+}
+
+func loadArgs(is []interface{}) []docsonnet.Argument {
+	args := make([]docsonnet.Argument, len(is))
+	for i := range is {
+		arg := is[i].(map[string]interface{})
+		args[i] = docsonnet.Argument{
+			Name:    arg["name"].(string),
+			Type:    docsonnet.Type(arg["type"].(string)),
+			Default: arg["default"],
+		}
+	}
+	return args
+}
+
+func fieldNames(msi map[string]interface{}) []string {
+	out := make([]string, 0, len(msi))
+	for k := range msi {
+		out = append(out, k)
+	}
+	return out
+}
+
+func loadObj(name string, msi map[string]interface{}, parent map[string]interface{}) docsonnet.Field {
+	obj := docsonnet.Object{
+		Name:   name,
+		Help:   msi["help"].(string),
+		Fields: make(docsonnet.Fields),
+	}
+
+	// look for children in same key without #
+	var iChilds interface{}
+	var ok bool
+	if iChilds, ok = parent[name]; !ok {
+		fmt.Println("aborting, no", name, strings.Join(fieldNames(parent), ", "))
+		return docsonnet.Field{Object: &obj}
+	}
+
+	childs := iChilds.(map[string]interface{})
+	for k, v := range childs {
+		name := strings.TrimPrefix(k, "#")
+		f := v.(map[string]interface{})
+		if !strings.HasPrefix(k, "#") {
+			if l, ok := loadNested(k, f); ok {
+				obj.Fields[name] = *l
+			}
+			continue
+		}
+
+		obj.Fields[name] = loadField(name, f, childs)
+	}
+
+	return docsonnet.Field{Object: &obj}
+}
+
+type DS map[string]interface{}
+
+func (d DS) Package() docsonnet.Package {
+	hash, ok := d["#"]
+	if !ok {
+		log.Fatalln("Package declaration missing")
+	}
+
+	pkg := hash.(map[string]interface{})
+	return docsonnet.Package{
+		Help:   pkg["help"].(string),
+		Name:   pkg["name"].(string),
+		Import: pkg["import"].(string),
+	}
+}
+
+func eval() ([]byte, error) {
+	fmt.Println("eval start")
+	start := time.Now()
+
+	vm := jsonnet.MakeVM()
+	vm.Importer(&jsonnet.FileImporter{JPaths: []string{".."}})
+	data, err := ioutil.ReadFile("fast.libsonnet")
+	if err != nil {
+		return nil, err
+	}
+
+	out, err := vm.EvaluateSnippet("fast.libsonnet", string(data))
+	if err != nil {
+		return nil, err
+	}
+
+	fmt.Println("eval:", time.Since(start))
+	return []byte(out), nil
+}
diff --git a/go.sum b/go.sum
@@ -2,8 +2,6 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
-github.com/go-clix/cli v0.1.1 h1:T9N0AdMbmpFM9cLw42TcLL5sQ3YgyxTyHUhBK0GW1LI=
-github.com/go-clix/cli v0.1.1/go.mod h1:dYJevXraB9mXZFhz5clyQestG0qGcmT5rRC/P9etoRQ=
 github.com/go-clix/cli v0.1.2-0.20200502172020-b8f4629e879a h1:nh+UOawbjKgiUAJAgi8JHctNebEu6mjwDXsv8Xdln8w=
 github.com/go-clix/cli v0.1.2-0.20200502172020-b8f4629e879a/go.mod h1:dYJevXraB9mXZFhz5clyQestG0qGcmT5rRC/P9etoRQ=
 github.com/gobuffalo/here v0.6.0 h1:hYrd0a6gDmWxBM4TnrGw8mQg24iSVoIkHEk7FodQcBI=

diff --git a/load.libsonnet b/load.libsonnet
@@ -1,59 +1,24 @@
 local lib = {
-  // reshape converts the Jsonnet structure to the one used by docsonnet:
-  // - put fields into an `api` key
-  // - put subpackages into `sub` key
-  reshape(pkg)::
-    local aux(old, key) =
-      if key == '#' then
-        old
-      else if std.objectHas(pkg[key], '#') then
-        old { sub+: { [key]: $.package(pkg[key]) } }
-      else
-        old { api+: { [key]: pkg[key] } };
-
-    std.foldl(aux, std.objectFields(pkg), {})
-    + pkg['#'],
-
-  // fillObjects creates docsonnet objects from Jsonnet ones,
-  // also filling those that have been specified explicitely
-  fillObjects(api)::
+  scan(obj)::
     local aux(old, key) =
       if std.startsWith(key, '#') then
-        old { [key]: api[key] }
-      else if std.isObject(api[key]) && std.length(std.objectFields(api[key])) > 0 then
-        old { ['#' + key]+: { object+: {
-          fields: api[key],
-        } } }
+        true
+      else if std.isObject(obj[key]) then
+        old || $.scan(obj[key])
       else old;
+    std.foldl(aux, std.objectFieldsAll(obj), false),
 
-    std.foldl(aux, std.objectFields(api), {}),
-
-  // clean removes all hashes from field names
-  clean(api):: {
-    [std.lstripChars(key, '#')]:
-      if std.isObject(api[key]) then $.clean(api[key])
-      else api[key]
-    for key in std.objectFields(api)
-  },
-
-  cleanNonObj(api):: {
-    [key]:
-      if std.startsWith(key, "#") then api[key]
-      else if std.isObject(api[key]) then $.cleanNonObj(api[key])
-      else api[key]
-    for key in std.objectFieldsAll(api)
-    if std.isObject(api[key])
-  },
+  load(pkg)::
+    local aux(old, key) =
+      if !std.isObject(pkg[key]) then
+        old
+      else if std.startsWith(key, '#') then
+        old { [key]: pkg[key] }
+      else if self.scan(pkg[key]) then
+        old { [key]: $.load(pkg[key]) }
+      else old;
 
-  // package loads docsonnet from a Jsonnet package
-  package(pkg)::
-    local cleaned = self.cleanNonObj(pkg);
-    local reshaped = self.reshape(cleaned);
-    local filled =
-      if std.objectHas(reshaped, 'api')
-      then reshaped { api: $.fillObjects(reshaped.api) }
-      else reshaped;
-    self.clean(filled),
+    std.foldl(aux, std.objectFieldsAll(pkg), {}),
 };
 
-lib.package(std.extVar("main"))
+lib.load(std.extVar('main'))