Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Tolk v0.8 grammar and updates #5

Merged
merged 1 commit into from
Feb 10, 2025
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -3,6 +3,11 @@
Historically, tolk-vscode has been cloned from [vscode-func](https://github.com/tonwhales/vscode-func)
and completely refactored.

## [0.8.0] - 2025-01-27

* Support Tolk v0.8 (syntax `tensorVar.0` / `tupleVar.0`)
* Fix a bug that variables in `catch` were considered unknown symbols

## [0.7.0] - 2025-01-13

* Support Tolk v0.7 (`bool` type, `as` operator, generics instantiations `f<int>`)
4 changes: 2 additions & 2 deletions package.json
Original file line number Diff line number Diff line change
@@ -10,7 +10,7 @@
"Smart contract"
],
"icon": "logo.png",
"version": "0.7.0",
"version": "0.8.0",
"engines": {
"vscode": "^1.63.0"
},
@@ -94,7 +94,7 @@
}
},
"default": {
"tolkCompilerVersion": "0.7",
"tolkCompilerVersion": "0.8",
"stdlibFolder": "/path/to/folder/stdlib-tolk"
}
},
2 changes: 1 addition & 1 deletion server/src/config-scheme.ts
Original file line number Diff line number Diff line change
@@ -26,7 +26,7 @@ export const defaultConfig: TolkPluginConfigScheme = {
experimentalDiagnostics: false,
autoDetectSDK: true,
manualSDKSettings: {
tolkCompilerVersion: "0.7",
tolkCompilerVersion: "0.8",
stdlibFolder: "/path/to/folder/stdlib-tolk"
}
}
2 changes: 1 addition & 1 deletion server/src/lsp-features/diagnostics.ts
Original file line number Diff line number Diff line change
@@ -112,7 +112,7 @@ class TreeVisitor {
let locals = findLocalVariables(this.rootNode, node.endPosition)
found = !!locals.find(a => a.name === name)
}
if (!found) {
if (!found && !name.startsWith('__')) { // don't warn on `__expect_type` and other special names
this.diagnostics.error(`Cannot find symbol '${name}'`, node)
}
}
14 changes: 14 additions & 0 deletions server/src/lsp-features/find-locals.ts
Original file line number Diff line number Diff line change
@@ -33,6 +33,20 @@ export function findLocalVariables(rootNode: Parser.SyntaxNode, cursorPosition:
}
}

if (descendant.type === 'try_catch_statement') {
for (let childFieldName of ['catch_var1', 'catch_var2']) {
let catchVarNode = descendant.childForFieldName(childFieldName)
if (catchVarNode) {
result.push({
kind: 'variable',
node: catchVarNode,
name: extractNameFromNode(catchVarNode),
type: childFieldName === 'catch_var1' ? { kind: 'primitive', name: 'int' } : { kind: 'unknown' }
})
}
}
}

if (descendant.type === 'function_declaration' || descendant.type === 'get_method_declaration') {
let parameterNodes = descendant.childForFieldName('parameters')?.descendantsOfType('parameter_declaration') || []
for (let paramNode of parameterNodes) {
20 changes: 20 additions & 0 deletions server/tests/parsing.spec.ts
Original file line number Diff line number Diff line change
@@ -164,3 +164,23 @@ it('should parse type hints', () => {
expect(fType.kind === 'tensor' && fType.items.length === 3 && fType.items[1].kind === 'primitive' && fType.items[2].kind === 'tuple' && fType.items[2].items[0].kind === 'primitive').toBeTruthy()
expect(vType.kind === 'type_identifier' && vType.name === 'T').toBeTruthy()
})

it('should parse indexed access', () => {
let rootNode = parseTolkSource('fun main() { t.0; t.id; }');
let f_body = rootNode.firstChild!.childForFieldName('body')!
expect(rootNode.hasError()).toBeFalsy()
expect(f_body.namedChild(0)!.namedChild(0)!.type).toBe('dot_access')
expect(f_body.namedChild(0)!.namedChild(0)!.childForFieldName('field')!.text).toBe('0')
expect(f_body.namedChild(1)!.namedChild(0)!.type).toBe('dot_access')
expect(f_body.namedChild(1)!.namedChild(0)!.childForFieldName('field')!.text).toBe('id')
})

it('should parse indexed access', () => {
let rootNode = parseTolkSource('fun main() { t.0; t.id; }');
let f_body = rootNode.firstChild!.childForFieldName('body')!
expect(rootNode.hasError()).toBeFalsy()
expect(f_body.namedChild(0)!.namedChild(0)!.type).toBe('dot_access')
expect(f_body.namedChild(0)!.namedChild(0)!.childForFieldName('field')!.text).toBe('0')
expect(f_body.namedChild(1)!.namedChild(0)!.type).toBe('dot_access')
expect(f_body.namedChild(1)!.namedChild(0)!.childForFieldName('field')!.text).toBe('id')
})
56 changes: 56 additions & 0 deletions server/tests/server-side.spec.ts
Original file line number Diff line number Diff line change
@@ -69,3 +69,59 @@ fun main(cs: slice) {
expect(locals2[1].name).toBe('cs')
})
})

it('should find catch variables', () => {
let tolkSource = `
fun main(m: int) {
try { }
catch (excNo, arg) {
// cursor1
return (excNo, arg);
}
// cursor2
}
`
let tree = createParser().parse(tolkSource)

let cursor1 = tree.rootNode.descendantsOfType('comment').find(c => c.text.includes('cursor1'))!
let locals1 = findLocalVariables(tree.rootNode, cursor1.startPosition)
expect(locals1.length).toBe(3)
expect(locals1[0].name).toBe('excNo')
expect(locals1[0].type).toEqual({ kind: 'primitive', name: 'int' })
expect(locals1[1].name).toBe('arg')
expect(locals1[1].type).toEqual({ kind: 'unknown' })
expect(locals1[2].name).toBe('m')

let cursor2 = tree.rootNode.descendantsOfType('comment').find(c => c.text.includes('cursor2'))!
let locals2 = findLocalVariables(tree.rootNode, cursor2.startPosition)
expect(locals2.length).toBe(1)
expect(locals2[0].name).toBe('m')
})

it('should find catch variables', () => {
let tolkSource = `
fun main(m: int) {
try { }
catch (excNo, arg) {
// cursor1
return (excNo, arg);
}
// cursor2
}
`
let tree = createParser().parse(tolkSource)

let cursor1 = tree.rootNode.descendantsOfType('comment').find(c => c.text.includes('cursor1'))!
let locals1 = findLocalVariables(tree.rootNode, cursor1.startPosition)
expect(locals1.length).toBe(3)
expect(locals1[0].name).toBe('excNo')
expect(locals1[0].type).toEqual({ kind: 'primitive', name: 'int' })
expect(locals1[1].name).toBe('arg')
expect(locals1[1].type).toEqual({ kind: 'unknown' })
expect(locals1[2].name).toBe('m')

let cursor2 = tree.rootNode.descendantsOfType('comment').find(c => c.text.includes('cursor2'))!
let locals2 = findLocalVariables(tree.rootNode, cursor2.startPosition)
expect(locals2.length).toBe(1)
expect(locals2[0].name).toBe('m')
})
Binary file modified server/tree-sitter-tolk.wasm
Binary file not shown.
3 changes: 2 additions & 1 deletion tree-sitter-tolk/grammar.js
Original file line number Diff line number Diff line change
@@ -338,7 +338,7 @@ const TOLK_GRAMMAR = {
dot_access: $ => prec(80, seq(
field('obj', $._expression),
'.',
field('field', $.identifier) // for method call, dot_access is wrapped into function_call, "field" actually means method name
field('field', choice($.identifier, $.numeric_index)) // for method call, dot_access is wrapped into function_call, "field" actually means method name
)),

function_call: $ => prec.left(90, seq(
@@ -405,6 +405,7 @@ const TOLK_GRAMMAR = {
null_literal: $ => 'null',
underscore: $ => '_',
identifier: $ => /`[^`]+`|[a-zA-Z$_][a-zA-Z0-9$_]*/,
numeric_index: $ => /[0-9]+/,

// http://stackoverflow.com/questions/13014947/regex-to-match-a-c-style-multiline-comment/36328890#36328890
comment: $ => token(choice(
17 changes: 15 additions & 2 deletions tree-sitter-tolk/src/grammar.json
Original file line number Diff line number Diff line change
@@ -1918,8 +1918,17 @@
"type": "FIELD",
"name": "field",
"content": {
"type": "SYMBOL",
"name": "identifier"
"type": "CHOICE",
"members": [
{
"type": "SYMBOL",
"name": "identifier"
},
{
"type": "SYMBOL",
"name": "numeric_index"
}
]
}
}
]
@@ -2516,6 +2525,10 @@
"type": "PATTERN",
"value": "`[^`]+`|[a-zA-Z$_][a-zA-Z0-9$_]*"
},
"numeric_index": {
"type": "PATTERN",
"value": "[0-9]+"
},
"comment": {
"type": "TOKEN",
"content": {
8 changes: 8 additions & 0 deletions tree-sitter-tolk/src/node-types.json
Original file line number Diff line number Diff line change
@@ -1098,6 +1098,10 @@
{
"type": "identifier",
"named": true
},
{
"type": "numeric_index",
"named": true
}
]
},
@@ -3644,6 +3648,10 @@
"type": "number_literal",
"named": true
},
{
"type": "numeric_index",
"named": true
},
{
"type": "redef",
"named": false
1,011 changes: 522 additions & 489 deletions tree-sitter-tolk/src/parser.c

Large diffs are not rendered by default.

14 changes: 14 additions & 0 deletions tree-sitter-tolk/test/corpus/simple.test.txt
Original file line number Diff line number Diff line change
@@ -80,6 +80,8 @@ fun main() {
obj.f1;
obj.f1.f2;
obj.f2.f2.f3;
tensorVar.0;
tupleVar.0.1;
}
---
(source_file (function_declaration (identifier) (parameter_list) (block_statement
@@ -92,6 +94,12 @@ fun main() {
(expression_statement
(dot_access (dot_access (dot_access (identifier) (identifier)) (identifier)) (identifier))
)
(expression_statement
(dot_access (identifier) (numeric_index))
)
(expression_statement
(dot_access (dot_access (identifier) (numeric_index)) (numeric_index))
)
)))

===
@@ -102,6 +110,7 @@ fun main() {
obj.method().method();
obj.field.method();
obj.method().field;
tensorVar.0();
}
---
(source_file (function_declaration (identifier) (parameter_list) (block_statement
@@ -125,4 +134,9 @@ fun main() {
(function_call (dot_access (identifier) (identifier)) (argument_list))
(identifier))
)
(expression_statement
(function_call
(dot_access (identifier) (numeric_index))
(argument_list))
)
)))