Tolk v0.8 grammar and updates

CHANGELOG.md

@@ -3,6 +3,11 @@
 Historically, tolk-vscode has been cloned from [vscode-func](https://github.com/tonwhales/vscode-func)
 and completely refactored.
 
+## [0.8.0] - 2025-01-27
+
+* Support Tolk v0.8 (syntax `tensorVar.0` / `tupleVar.0`)
+* Fix a bug that variables in `catch` were considered unknown symbols
+
 ## [0.7.0] - 2025-01-13
 
 * Support Tolk v0.7 (`bool` type, `as` operator, generics instantiations `f<int>`)

package.json

@@ -10,7 +10,7 @@
         "Smart contract"
     ],
     "icon": "logo.png",
-    "version": "0.7.0",
+    "version": "0.8.0",
     "engines": {
         "vscode": "^1.63.0"
     },
@@ -94,7 +94,7 @@
                         }
                     },
                     "default": {
-                        "tolkCompilerVersion": "0.7",
+                        "tolkCompilerVersion": "0.8",
                         "stdlibFolder": "/path/to/folder/stdlib-tolk"
                     }
                 },

server/src/config-scheme.ts

@@ -26,7 +26,7 @@ export const defaultConfig: TolkPluginConfigScheme = {
   experimentalDiagnostics: false,
   autoDetectSDK: true,
   manualSDKSettings: {
-    tolkCompilerVersion: "0.7",
+    tolkCompilerVersion: "0.8",
     stdlibFolder: "/path/to/folder/stdlib-tolk"
   }
 }

server/src/lsp-features/diagnostics.ts

@@ -112,7 +112,7 @@ class TreeVisitor {
         let locals = findLocalVariables(this.rootNode, node.endPosition)
         found = !!locals.find(a => a.name === name)
       }
-      if (!found) {
+      if (!found && !name.startsWith('__')) {   // don't warn on `__expect_type` and other special names
         this.diagnostics.error(`Cannot find symbol '${name}'`, node)
       }
     }

server/src/lsp-features/find-locals.ts

@@ -33,6 +33,20 @@ export function findLocalVariables(rootNode: Parser.SyntaxNode, cursorPosition:
       }
     }
 
+    if (descendant.type === 'try_catch_statement') {
+      for (let childFieldName of ['catch_var1', 'catch_var2']) {
+        let catchVarNode = descendant.childForFieldName(childFieldName)
+        if (catchVarNode) {
+          result.push({
+            kind: 'variable',
+            node: catchVarNode,
+            name: extractNameFromNode(catchVarNode),
+            type: childFieldName === 'catch_var1' ? { kind: 'primitive', name: 'int' } : { kind: 'unknown' }
+          })
+        }
+      }
+    }
+
     if (descendant.type === 'function_declaration' || descendant.type === 'get_method_declaration') {
       let parameterNodes = descendant.childForFieldName('parameters')?.descendantsOfType('parameter_declaration') || []
       for (let paramNode of parameterNodes) {

server/tests/parsing.spec.ts

@@ -164,3 +164,23 @@ it('should parse type hints', () => {
   expect(fType.kind === 'tensor' && fType.items.length === 3 && fType.items[1].kind === 'primitive' && fType.items[2].kind === 'tuple' && fType.items[2].items[0].kind === 'primitive').toBeTruthy()
   expect(vType.kind === 'type_identifier' && vType.name === 'T').toBeTruthy()
 })
+
+it('should parse indexed access', () => {
+  let rootNode = parseTolkSource('fun main() { t.0; t.id; }');
+  let f_body = rootNode.firstChild!.childForFieldName('body')!
+  expect(rootNode.hasError()).toBeFalsy()
+  expect(f_body.namedChild(0)!.namedChild(0)!.type).toBe('dot_access')
+  expect(f_body.namedChild(0)!.namedChild(0)!.childForFieldName('field')!.text).toBe('0')
+  expect(f_body.namedChild(1)!.namedChild(0)!.type).toBe('dot_access')
+  expect(f_body.namedChild(1)!.namedChild(0)!.childForFieldName('field')!.text).toBe('id')
+})
+
+it('should parse indexed access', () => {
+  let rootNode = parseTolkSource('fun main() { t.0; t.id; }');
+  let f_body = rootNode.firstChild!.childForFieldName('body')!
+  expect(rootNode.hasError()).toBeFalsy()
+  expect(f_body.namedChild(0)!.namedChild(0)!.type).toBe('dot_access')
+  expect(f_body.namedChild(0)!.namedChild(0)!.childForFieldName('field')!.text).toBe('0')
+  expect(f_body.namedChild(1)!.namedChild(0)!.type).toBe('dot_access')
+  expect(f_body.namedChild(1)!.namedChild(0)!.childForFieldName('field')!.text).toBe('id')
+})

server/tests/server-side.spec.ts

@@ -69,3 +69,59 @@ fun main(cs: slice) {
     expect(locals2[1].name).toBe('cs')
   })
 })
+
+it('should find catch variables', () => {
+  let tolkSource = `
+fun main(m: int) {
+  try { }
+  catch (excNo, arg) { 
+    // cursor1
+    return (excNo, arg); 
+  }
+  // cursor2
+}
+`
+  let tree = createParser().parse(tolkSource)
+
+  let cursor1 = tree.rootNode.descendantsOfType('comment').find(c => c.text.includes('cursor1'))!
+  let locals1 = findLocalVariables(tree.rootNode, cursor1.startPosition)
+  expect(locals1.length).toBe(3)
+  expect(locals1[0].name).toBe('excNo')
+  expect(locals1[0].type).toEqual({ kind: 'primitive', name: 'int' })
+  expect(locals1[1].name).toBe('arg')
+  expect(locals1[1].type).toEqual({ kind: 'unknown' })
+  expect(locals1[2].name).toBe('m')
+
+  let cursor2 = tree.rootNode.descendantsOfType('comment').find(c => c.text.includes('cursor2'))!
+  let locals2 = findLocalVariables(tree.rootNode, cursor2.startPosition)
+  expect(locals2.length).toBe(1)
+  expect(locals2[0].name).toBe('m')
+})
+
+it('should find catch variables', () => {
+  let tolkSource = `
+fun main(m: int) {
+  try { }
+  catch (excNo, arg) { 
+    // cursor1
+    return (excNo, arg); 
+  }
+  // cursor2
+}
+`
+  let tree = createParser().parse(tolkSource)
+
+  let cursor1 = tree.rootNode.descendantsOfType('comment').find(c => c.text.includes('cursor1'))!
+  let locals1 = findLocalVariables(tree.rootNode, cursor1.startPosition)
+  expect(locals1.length).toBe(3)
+  expect(locals1[0].name).toBe('excNo')
+  expect(locals1[0].type).toEqual({ kind: 'primitive', name: 'int' })
+  expect(locals1[1].name).toBe('arg')
+  expect(locals1[1].type).toEqual({ kind: 'unknown' })
+  expect(locals1[2].name).toBe('m')
+
+  let cursor2 = tree.rootNode.descendantsOfType('comment').find(c => c.text.includes('cursor2'))!
+  let locals2 = findLocalVariables(tree.rootNode, cursor2.startPosition)
+  expect(locals2.length).toBe(1)
+  expect(locals2[0].name).toBe('m')
+})

tree-sitter-tolk/grammar.js

@@ -338,7 +338,7 @@ const TOLK_GRAMMAR = {
   dot_access: $ => prec(80, seq(
     field('obj', $._expression),
     '.',
-    field('field', $.identifier)    // for method call, dot_access is wrapped into function_call, "field" actually means method name
+    field('field', choice($.identifier, $.numeric_index))    // for method call, dot_access is wrapped into function_call, "field" actually means method name
   )),
 
   function_call: $ => prec.left(90, seq(
@@ -405,6 +405,7 @@ const TOLK_GRAMMAR = {
   null_literal: $ => 'null',
   underscore: $ => '_',
   identifier: $ => /`[^`]+`|[a-zA-Z$_][a-zA-Z0-9$_]*/,
+  numeric_index: $ => /[0-9]+/,
 
   // http://stackoverflow.com/questions/13014947/regex-to-match-a-c-style-multiline-comment/36328890#36328890
   comment: $ => token(choice(

tree-sitter-tolk/src/grammar.json

@@ -1918,8 +1918,17 @@
             "type": "FIELD",
             "name": "field",
             "content": {
-              "type": "SYMBOL",
-              "name": "identifier"
+              "type": "CHOICE",
+              "members": [
+                {
+                  "type": "SYMBOL",
+                  "name": "identifier"
+                },
+                {
+                  "type": "SYMBOL",
+                  "name": "numeric_index"
+                }
+              ]
             }
           }
         ]
@@ -2516,6 +2525,10 @@
       "type": "PATTERN",
       "value": "`[^`]+`|[a-zA-Z$_][a-zA-Z0-9$_]*"
     },
+    "numeric_index": {
+      "type": "PATTERN",
+      "value": "[0-9]+"
+    },
     "comment": {
       "type": "TOKEN",
       "content": {

tree-sitter-tolk/src/node-types.json

@@ -1098,6 +1098,10 @@
           {
             "type": "identifier",
             "named": true
+          },
+          {
+            "type": "numeric_index",
+            "named": true
           }
         ]
       },
@@ -3644,6 +3648,10 @@
     "type": "number_literal",
     "named": true
   },
+  {
+    "type": "numeric_index",
+    "named": true
+  },
   {
     "type": "redef",
     "named": false

tree-sitter-tolk/test/corpus/simple.test.txt

@@ -80,6 +80,8 @@ fun main() {
     obj.f1;
     obj.f1.f2;
     obj.f2.f2.f3;
+    tensorVar.0;
+    tupleVar.0.1;
 }
 ---
 (source_file (function_declaration (identifier) (parameter_list) (block_statement
@@ -92,6 +94,12 @@ fun main() {
   (expression_statement
     (dot_access (dot_access (dot_access (identifier) (identifier)) (identifier)) (identifier))
   )
+  (expression_statement
+    (dot_access (identifier) (numeric_index))
+  )
+  (expression_statement
+    (dot_access (dot_access (identifier) (numeric_index)) (numeric_index))
+  )
 )))
 
 ===
@@ -102,6 +110,7 @@ fun main() {
     obj.method().method();
     obj.field.method();
     obj.method().field;
+    tensorVar.0();
 }
 ---
 (source_file (function_declaration (identifier) (parameter_list) (block_statement
@@ -125,4 +134,9 @@ fun main() {
       (function_call (dot_access (identifier) (identifier)) (argument_list))
       (identifier))
   )
+  (expression_statement
+    (function_call
+      (dot_access (identifier) (numeric_index))
+    (argument_list))
+  )
 )))