docs: Add list of Runtime transactions

.github/workflows/ci-lint.yml

@@ -198,3 +198,51 @@ jobs:
         uses: nosborn/[email protected]
         with:
           files: docs
+
+  lint-go-tools-extract-runtime-txs:
+    name: lint-go-tools-extract-runtime-txs
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Setup Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: '1.17.x'
+
+      - name: Lint Go cli
+        uses: golangci/[email protected]
+        with:
+          # NOTE: The version must be specified without the patch version.
+          version: v1.40
+          working-directory: tools/extract-runtime-txs
+
+  lint-generated-docs:
+    name: lint-generated-docs
+    runs-on: ubuntu-latest
+    env:
+      TXS_FILE: ../../docs/runtime/transactions.md
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Set up Go
+        uses: actions/setup-go@v3
+
+      - name: Compile extract-runtime-txs
+        working-directory: tools/extract-runtime-txs
+        run: go build
+
+      - name: Check runtime transactions sync
+        working-directory: tools/extract-runtime-txs
+        run: |
+          cp ${TXS_FILE} ${TXS_FILE}.orig
+          ./extract-runtime-txs \
+            --codebase.path ../.. \
+            --markdown \
+            --markdown.template.file ${TXS_FILE}.tpl \
+            --codebase.url https://github.com/oasisprotocol/oasis-sdk/tree/master/ \
+            > ${TXS_FILE}
+          diff -u ${TXS_FILE}.orig ${TXS_FILE} || \
+            (echo "Runtime transactions docs should be updated. Check tools/extract-runtime-txs/README.md on how to do it." && exit 1)

docs/runtime/transactions.md.tpl

@@ -0,0 +1,12 @@
+# Runtime Transactions
+
+This section describes the format of all supported runtime methods and queries
+with references to Go, Rust and TypeScript bindings in Oasis SDK.
+
+<!-- DO NOT EDIT LIST OF TRANSACTIONS BELOW MANUALLY! The list was generated by
+     tools/extract-runtime-txs. -->
+
+<!-- markdownlint-disable line-length -->
+
+<!--- OASIS_RUNTIME_TRANSACTIONS -->
+<!-- markdownlint-enable line-length -->

runtime-sdk-macros/src/lib.rs

@@ -59,8 +59,8 @@ pub fn sdk_derive(args: TokenStream, input: TokenStream) -> TokenStream {
     }
 }
 
-/// A helper attribute for `#[sdk_derive(...)]`. It doesn't do anyting on its own;
-/// it only mark functions that represent a paratime method handler.
+/// A helper attribute for `#[sdk_derive(...)]`. It doesn't do anything on its own;
+/// it only marks functions that represent a paratime method handler.
 /// The permitted forms are:
 ///  - `#[handler(call = "my_module.MyCall")]`: Marks a function that handles
 ///        the "my_module.MyCall" call and can be passed to

tools/extract-runtime-txs/README.md

@@ -0,0 +1,33 @@
+# extract-runtime-txs
+
+This tool parses Rust, Go, and TypeScript code and generates markdown docs
+describing all runtime transactions with their call types and results.
+
+## Compilation and Testing
+
+
+```sh
+go build
+go test ./...
+```
+
+## Execution
+
+This should output all runtime transactions in JSON format:
+
+```sh
+./extract-runtime-txs \
+	--codebase.path ../..
+```
+
+For oasis-sdk documentation an existing Markdown template should be used and
+any source files should be referenced relative to github.com URL:
+
+```sh
+./extract-runtime-txs \
+	--codebase.path ../.. \
+	--markdown \
+	--markdown.template.file ../../docs/runtime/transactions.md.tpl \
+	--codebase.url https://github.com/oasisprotocol/oasis-sdk/tree/master/ \
+	> ../../docs/runtime/transactions.md
+```

tools/extract-runtime-txs/go.mod

@@ -0,0 +1,30 @@
+module github.com/oasisprotocol/oasis-sdk/tools/extract-runtime-txs
+
+go 1.17
+
+require (
+	github.com/spf13/cobra v1.3.0
+	github.com/spf13/viper v1.10.1
+	github.com/stretchr/testify v1.7.0
+)
+
+require (
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/fsnotify/fsnotify v1.5.1 // indirect
+	github.com/hashicorp/hcl v1.0.0 // indirect
+	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/magiconair/properties v1.8.5 // indirect
+	github.com/mitchellh/mapstructure v1.4.3 // indirect
+	github.com/pelletier/go-toml v1.9.4 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/spf13/afero v1.6.0 // indirect
+	github.com/spf13/cast v1.4.1 // indirect
+	github.com/spf13/jwalterweatherman v1.1.0 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/subosito/gotenv v1.2.0 // indirect
+	golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9 // indirect
+	golang.org/x/text v0.3.7 // indirect
+	gopkg.in/ini.v1 v1.66.2 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
+)

tools/extract-runtime-txs/main.go

@@ -0,0 +1,220 @@
+// extract-runtime-txs extracts runtime transactions from Rust, Go, and TypeScript sources
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+
+	"github.com/oasisprotocol/oasis-sdk/tools/extract-runtime-txs/parsers"
+	"github.com/oasisprotocol/oasis-sdk/tools/extract-runtime-txs/types"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+)
+
+const (
+	CfgMarkdown               = "markdown"
+	CfgMarkdownTplFile        = "markdown.template.file"
+	CfgMarkdownTplPlaceholder = "markdown.template.placeholder"
+	CfgCodebasePath           = "codebase.path"
+	CfgCodebaseURL            = "codebase.url"
+)
+
+var (
+	scriptName = filepath.Base(os.Args[0])
+
+	rootCmd = &cobra.Command{
+		Use:   scriptName,
+		Short: "Extracts Runtime transactions from formatted Rust, Go and TypeScript code.",
+		Example: `./extract-runtime-txs \
+        --codebase.path ../.. \
+        --markdown \
+        --markdown.template.file ../../docs/runtime/transactions.md.tpl \
+        --codebase.url https://github.com/oasisprotocol/oasis-sdk/tree/master/ \
+        > ../../docs/runtime/transactions.md`,
+		Run: doExtractRuntimeTxs,
+	}
+)
+
+// refAnchor returns the reference name.
+func refAnchor(l types.Lang, fullName string, t types.RefType) string {
+	refTypeStr := ""
+	if t != types.Base {
+		refTypeStr = fmt.Sprintf("-%s", t)
+	}
+
+	return fmt.Sprintf("%s-%s%s", l, fullName, refTypeStr)
+}
+
+// markdownRefSrcs generates a sorted list by language of URL sources for the given references.
+func markdownRefSrcs(fullName string, refs map[types.Lang]types.Snippet, refType types.RefType) string {
+	markdown := ""
+	for _, lang := range []types.Lang{types.Rust, types.Go, types.TypeScript} {
+		if _, valid := refs[lang]; !valid {
+			continue
+		}
+		markdown += fmt.Sprintf("[%s]: %s\n", refAnchor(lang, fullName, refType), snippetPath(refs[lang]))
+	}
+
+	return markdown
+}
+
+// markdownRef generates [ Go | Rust | TypeScript ] for the provided snippet.
+func markdownRef(fullName string, snippets map[types.Lang]types.Snippet, t types.RefType) string {
+	langMarkdown := []string{}
+	for _, lang := range []types.Lang{types.Rust, types.Go, types.TypeScript} {
+		if _, valid := snippets[lang]; !valid {
+			continue
+		}
+		ref := fmt.Sprintf("[%s][%s]", lang.ToString(), refAnchor(lang, fullName, t))
+		langMarkdown = append(langMarkdown, ref)
+	}
+
+	return fmt.Sprintf("[%s]", strings.Join(langMarkdown, " | "))
+}
+
+// markdownParams generates a markdown list of parameter or results fields of the transaction.
+func markdownParams(params []types.Parameter) string {
+	paramsStr := ""
+	for _, p := range params {
+		paramsStr += fmt.Sprintf("- `%s: %s`\n", p.Name, p.Type)
+		if p.Description != "" {
+			paramsStr += fmt.Sprintf("\n  %s\n", p.Description)
+		}
+	}
+	return paramsStr
+}
+
+// snippetPath populates the file name with line numbers and optionally replaces the local filename
+// with the github's or other git repository base URL.
+func snippetPath(s types.Snippet) string {
+	baseDir := viper.GetString(CfgCodebasePath)
+	if viper.IsSet(CfgMarkdownTplFile) && !viper.IsSet(CfgCodebaseURL) {
+		baseDir = filepath.Dir(viper.GetString(CfgMarkdownTplFile))
+	}
+	fileURL, _ := filepath.Rel(baseDir, s.Path)
+	if viper.IsSet(CfgCodebaseURL) {
+		fileURL = viper.GetString(CfgCodebaseURL) + fileURL
+	}
+	linesStr := ""
+	if s.LineFrom != 0 {
+		linesStr = fmt.Sprintf("#L%d", s.LineFrom)
+		if s.LineTo != s.LineFrom {
+			linesStr += fmt.Sprintf("-L%d", s.LineTo)
+		}
+	}
+	return fmt.Sprintf("%s%s", fileURL, linesStr)
+}
+
+// sortTxs sorts the given map of transactions by their key and returns an
+// ordered list of transactions.
+func sortTxs(txs map[string]types.Tx) []types.Tx {
+	sortedTxs := make([]types.Tx, 0, len(txs))
+	for _, tx := range txs {
+		sortedTxs = append(sortedTxs, tx)
+	}
+	// Sort by module name, then by tx type and then by name.
+	sort.Slice(sortedTxs, func(i, j int) bool {
+		return sortedTxs[i].Module+string(sortedTxs[i].Type)+sortedTxs[i].Name < sortedTxs[j].Module+string(sortedTxs[j].Type)+sortedTxs[j].Name
+	})
+	return sortedTxs
+}
+
+func printMarkdown(transactions map[string]types.Tx) {
+	markdown := ""
+	lastModule := ""
+	for _, tx := range sortTxs(transactions) {
+		if tx.Module != lastModule {
+			markdown += fmt.Sprintf("## %s\n\n", tx.Module)
+			lastModule = tx.Module
+		}
+		markdown += fmt.Sprintf("### %s (%s) {#%s}\n\n", tx.FullName(), tx.Type, tx.Module+"-"+strings.ToLower(tx.Name))
+		markdown += fmt.Sprintf("%s\n\n", markdownRef(tx.FullName(), tx.Ref, types.Base))
+		markdown += fmt.Sprintf("#### Parameters %s\n\n%s\n", markdownRef(tx.FullName(), tx.ParametersRef, types.Params), markdownParams(tx.Parameters))
+
+		if len(tx.ResultRef) > 0 {
+			markdown += fmt.Sprintf("#### Result %s\n\n", markdownRef(tx.FullName(), tx.ResultRef, types.Result))
+			if tx.Result != nil {
+				markdown += fmt.Sprintf("%s\n", markdownParams(tx.Result))
+			}
+		}
+
+		markdown += markdownRefSrcs(tx.FullName(), tx.Ref, types.Base)
+		markdown += markdownRefSrcs(tx.FullName(), tx.ParametersRef, types.Params)
+		markdown += markdownRefSrcs(tx.FullName(), tx.ResultRef, types.Result)
+
+		markdown += "\n"
+	}
+
+	if !viper.IsSet(CfgMarkdownTplFile) {
+		// Print Markdown only.
+		fmt.Print(markdown)
+		return
+	}
+
+	md, err := ioutil.ReadFile(viper.GetString(CfgMarkdownTplFile))
+	if err != nil {
+		panic(err)
+	}
+
+	mdStr := strings.Replace(string(md), viper.GetString(CfgMarkdownTplPlaceholder)+"\n", markdown, 1)
+	fmt.Print(mdStr)
+}
+
+func printJSON(txs map[string]types.Tx) {
+	data, err := json.Marshal(sortTxs(txs))
+	if err != nil {
+		panic(err)
+	}
+	fmt.Printf("%s", data)
+}
+
+func printWarnings(parser parsers.Parser) {
+	for _, w := range parser.GetWarnings() {
+		fmt.Fprintln(os.Stderr, w)
+	}
+}
+
+func doExtractRuntimeTxs(cmd *cobra.Command, args []string) {
+	rustParser := parsers.NewRustParser(viper.GetString(CfgCodebasePath) + "/runtime-sdk")
+	transactions, err := rustParser.GenerateInitialTransactions()
+	if err != nil {
+		log.Fatal(err)
+	}
+	printWarnings(rustParser)
+
+	prsrs := []parsers.Parser{
+		parsers.NewGolangParser(viper.GetString(CfgCodebasePath) + "/client-sdk/go"),
+		parsers.NewTypeScriptParser(viper.GetString(CfgCodebasePath) + "/client-sdk/ts-web"),
+	}
+	for _, p := range prsrs {
+		p.PopulateRefs(transactions)
+		for _, w := range p.GetWarnings() {
+			fmt.Fprintln(os.Stderr, w)
+		}
+		printWarnings(p)
+	}
+
+	if viper.GetBool(CfgMarkdown) {
+		printMarkdown(transactions)
+	} else {
+		printJSON(transactions)
+	}
+}
+
+func main() {
+	rootCmd.Flags().Bool(CfgMarkdown, false, "print metrics in markdown format")
+	rootCmd.Flags().String(CfgCodebasePath, "", "path to Go codebase")
+	rootCmd.Flags().String(CfgCodebaseURL, "", "show URL to Go files with this base instead of relative path (optional) (e.g. https://github.com/oasisprotocol/oasis-sdk/tree/master/)")
+	rootCmd.Flags().String(CfgMarkdownTplFile, "", "path to Markdown template file")
+	rootCmd.Flags().String(CfgMarkdownTplPlaceholder, "<!--- OASIS_RUNTIME_TRANSACTIONS -->", "placeholder for Markdown table in the template")
+	_ = cobra.MarkFlagRequired(rootCmd.Flags(), CfgCodebasePath)
+	_ = viper.BindPFlags(rootCmd.Flags())
+
+	_ = rootCmd.Execute()
+}