Add AST support of JSONC (for theme check, language features, etc.)

.changeset/witty-chairs-kneel.md

@@ -0,0 +1,5 @@
+---
+'@shopify/theme-check-common': minor
+---
+
+[Internal] Add JSONC support to the AST parser

ThirdPartyNotices.txt

@@ -7,6 +7,7 @@ The theme-language-server project incorporates third party material from the pro
 3. @lezer/json (https://github.com/lezer-parser/json)
 4. @codemirror/lang-json (https://github.com/codemirror/lang-json)
 5. monaco-editor (https://github.com/microsoft/monaco-editor)
+6. json-to-ast (https://github.com/vtrushin/json-to-ast)
 
 %% vscode-languageserver-node NOTICES, INFORMATION, AND LICENSE BEGIN HERE
 =========================================
@@ -133,3 +134,27 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE.
 =========================================
 END OF monaco-editor NOTICES, INFORMATION, AND LICENSE
+
+%% json-to-ast NOTICES, INFORMATION, AND LICENSE BEGIN HERE
+=========================================
+Copyright (C) 2016 by Vlad Trushin
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+=========================================
+END OF json-to-ast NOTICES, INFORMATION, AND LICENSE

packages/theme-check-common/package.json

@@ -28,7 +28,6 @@
   "dependencies": {
     "@shopify/liquid-html-parser": "2.1.2",
     "cross-fetch": "^4.0.0",
-    "json-to-ast": "^2.1.0",
     "jsonc-parser": "^3.2.0",
     "line-column": "^1.0.2",
     "lodash-es": "^4.17.21",
@@ -37,7 +36,6 @@
     "vscode-uri": "^3.0.7"
   },
   "devDependencies": {
-    "@types/json-to-ast": "^2.1.2",
     "@types/line-column": "^1.0.0",
     "@types/lodash-es": "^4.17.12"
   }

packages/theme-check-common/src/checks/json-syntax-error/index.spec.ts

@@ -10,12 +10,14 @@ describe('Module: JSONSyntaxError', () => {
     }`;
 
     const offenses = await runJSONCheck(JSONSyntaxError, invalidJson, 'file.json');
-    expect(offenses).to.have.length(1);
-    expect(offenses[0].message).to.equal('Unexpected token <,>');
+    expect(offenses).to.have.length(2);
+    expect(offenses[0].message).to.equal('Property name expected');
+    expect(offenses[1].message).to.equal('Expecting a value');
 
     const highlights = highlightedOffenses({ 'file.json': invalidJson }, offenses);
-    expect(highlights).to.have.length(1);
+    expect(highlights).to.have.length(2);
     expect(highlights[0]).to.equal(',');
+    expect(highlights[1]).to.equal(',');
   });
 
   it('should report an error for invalid JSON (1)', async () => {
@@ -26,11 +28,11 @@ describe('Module: JSONSyntaxError', () => {
 
     const offenses = await runJSONCheck(JSONSyntaxError, invalidJson, 'file.json');
     expect(offenses).to.have.length(1);
-    expect(offenses[0].message).to.equal('Unexpected end of input');
+    expect(offenses[0].message).to.equal('Expecting a closing brace (})');
 
     const highlights = highlightedOffenses({ 'file.json': invalidJson }, offenses);
     expect(highlights).to.have.length(1);
-    expect(highlights[0]).to.equal('\n');
+    expect(highlights[0]).to.equal('');
   });
 
   it('should report an error for invalid JSON (2)', async () => {
@@ -40,12 +42,14 @@ describe('Module: JSONSyntaxError', () => {
     }`;
 
     const offenses = await runJSONCheck(JSONSyntaxError, invalidJson, 'file.json');
-    expect(offenses).to.have.length(1);
-    expect(offenses[0].message).to.equal("Unexpected symbol <'>");
+    expect(offenses).to.have.length(3);
+    expect(offenses[0].message).to.equal('Invalid symbol');
+    expect(offenses[1].message).to.equal('Property name expected');
+    expect(offenses[2].message).to.equal('Expecting a value');
 
     const highlights = highlightedOffenses({ 'file.json': invalidJson }, offenses);
-    expect(highlights).to.have.length(1);
-    expect(highlights[0]).to.equal("'");
+    expect(highlights).to.have.length(3);
+    expect(highlights[0]).to.equal("'key1'");
   });
 
   it('should not report any errors for valid JSON', async () => {

packages/theme-check-common/src/checks/json-syntax-error/index.ts

@@ -1,5 +1,7 @@
-import { Severity, SourceCodeType, JSONCheckDefinition } from '../../types';
-import { getOffset, isError } from '../../utils';
+import { ParseErrorCode, printParseErrorCode } from 'jsonc-parser';
+import { JSONCParseErrors } from '../../jsonc/parse';
+import { JSONCheckDefinition, Severity, SourceCodeType } from '../../types';
+import { isError } from '../../utils';
 
 function cleanErrorMessage(error: Error) {
   const message = 'rawMessage' in error ? (error.rawMessage as string) : error.message;
@@ -30,20 +32,14 @@ export const JSONSyntaxError: JSONCheckDefinition = {
 
     return {
       async onCodePathStart(file) {
-        if (
-          'line' in error &&
-          typeof error.line === 'number' &&
-          'column' in error &&
-          typeof error.column === 'number'
-        ) {
-          const { line, column } = error;
-          const startIndex = getOffset(file.source, line, column);
-          const endIndex = getOffset(file.source, line, column) + 1;
-          context.report({
-            message: cleanErrorMessage(error),
-            startIndex,
-            endIndex: endIndex,
-          });
+        if (file.ast instanceof JSONCParseErrors) {
+          for (const error of file.ast.errors) {
+            context.report({
+              message: jsoncParseErrorMessage(error.error),
+              startIndex: error.offset,
+              endIndex: error.offset + error.length,
+            });
+          }
         } else {
           context.report({
             message: cleanErrorMessage(error),
@@ -55,3 +51,42 @@ export const JSONSyntaxError: JSONCheckDefinition = {
     };
   },
 };
+
+function jsoncParseErrorMessage(errorType: ParseErrorCode) {
+  switch (errorType) {
+    case ParseErrorCode.InvalidSymbol:
+      return 'Invalid symbol';
+    case ParseErrorCode.InvalidNumberFormat:
+      return 'Invalid number format';
+    case ParseErrorCode.PropertyNameExpected:
+      return 'Property name expected';
+    case ParseErrorCode.ValueExpected:
+      return 'Expecting a value';
+    case ParseErrorCode.ColonExpected:
+      return 'Expecting a colon after a property name (:)';
+    case ParseErrorCode.CommaExpected:
+      return 'Expecting a comma';
+    case ParseErrorCode.CloseBraceExpected:
+      return 'Expecting a closing brace (})';
+    case ParseErrorCode.CloseBracketExpected:
+      return 'Expecting a closing bracket (])';
+    case ParseErrorCode.EndOfFileExpected:
+      return 'Expecting end of file';
+    case ParseErrorCode.InvalidCommentToken:
+      return 'Invalid comment token';
+    case ParseErrorCode.UnexpectedEndOfComment:
+      return 'Unexpected end of comment';
+    case ParseErrorCode.UnexpectedEndOfString:
+      return 'Unexpected end of string';
+    case ParseErrorCode.UnexpectedEndOfNumber:
+      return 'Unexpected end of number';
+    case ParseErrorCode.InvalidUnicode:
+      return 'Invalid unicode';
+    case ParseErrorCode.InvalidEscapeCharacter:
+      return 'Invalid escape character';
+    case ParseErrorCode.InvalidCharacter:
+      return 'Invalid character';
+    default:
+      return 'Something went wrong with this JSON';
+  }
+}

packages/theme-check-common/src/checks/liquid-free-settings/index.ts

@@ -1,7 +1,12 @@
-import { JSONNode, LiquidCheckDefinition, Severity, SourceCodeType } from '../../types';
 import { toJSONAST } from '../../to-source-code';
+import {
+  JSONNode,
+  LiquidCheckDefinition,
+  LiteralNode,
+  Severity,
+  SourceCodeType,
+} from '../../types';
 import { visit } from '../../visitor';
-import { LiteralNode } from 'json-to-ast';
 
 export const LiquidFreeSettings: LiquidCheckDefinition = {
   meta: {

packages/theme-check-common/src/checks/matching-translations/index.ts

@@ -1,10 +1,10 @@
-import { PropertyNode } from 'json-to-ast';
 import {
   JSONCheckDefinition,
   JSONNode,
   JSONSourceCode,
   Severity,
   SourceCodeType,
+  PropertyNode,
 } from '../../types';
 
 const PLURALIZATION_KEYS = new Set(['zero', 'one', 'two', 'few', 'many', 'other']);

packages/theme-check-common/src/checks/schema-presets-block-order/index.ts

@@ -1,11 +1,15 @@
-import { ArrayNode } from 'json-to-ast';
 import { getLocEnd, getLocStart, nodeAtPath } from '../../json';
 import { basename } from '../../path';
 import { isBlock, isSection } from '../../to-schema';
-import { JSONNode, LiquidCheckDefinition, Severity, SourceCodeType } from '../../types';
+import {
+  ArrayNode,
+  Context,
+  JSONNode,
+  LiquidCheckDefinition,
+  Severity,
+  SourceCodeType,
+} from '../../types';
 import { Preset } from '../../types/schemas/preset';
-import { ThemeBlock } from '../../types/schemas/theme-block';
-import { Context } from '../../types';
 
 export const SchemaPresetsBlockOrder: LiquidCheckDefinition = {
   meta: {

packages/theme-check-common/src/checks/valid-block-target/block-utils.ts

@@ -1,5 +1,12 @@
-import { JSONNode, Preset, Section, SourceCodeType, Theme, ThemeBlock } from '../../types';
-import { LiteralNode } from 'json-to-ast';
+import {
+  JSONNode,
+  LiteralNode,
+  Preset,
+  Section,
+  SourceCodeType,
+  Theme,
+  ThemeBlock,
+} from '../../types';
 import { getLocEnd, getLocStart, nodeAtPath } from '../../json';
 import { Context } from '../../types';
 import { doesFileExist } from '../../utils/file-utils';

packages/theme-check-common/src/checks/valid-block-target/index.ts

@@ -5,8 +5,8 @@ import {
   Preset,
   Section,
   ThemeBlock,
+  LiteralNode,
 } from '../../types';
-import { LiteralNode } from 'json-to-ast';
 import { nodeAtPath } from '../../json';
 import { getSchema } from '../../to-schema';
 import {

packages/theme-check-common/src/checks/valid-html-translation/index.ts

@@ -1,6 +1,5 @@
-import { SourceCodeType, JSONCheckDefinition, Severity, Problem, JSONNode } from '../../types';
+import { SourceCodeType, JSONCheckDefinition, Severity, Problem, LiteralNode } from '../../types';
 import { toLiquidHtmlAST } from '@shopify/liquid-html-parser';
-import { Location, LiteralNode } from 'json-to-ast';
 
 export const ValidHTMLTranslation: JSONCheckDefinition = {
   meta: {
@@ -31,7 +30,7 @@ export const ValidHTMLTranslation: JSONCheckDefinition = {
         try {
           toLiquidHtmlAST(node.value);
         } catch (error) {
-          const loc = node.loc as Location;
+          const loc = node.loc;
 
           const problem: Problem<SourceCodeType.JSON> = {
             message: `${error}.`,

packages/theme-check-common/src/checks/valid-json/index.ts

@@ -1,10 +1,4 @@
-import { Severity, SourceCodeType, JSONCheckDefinition } from '../../types';
-import { getOffset, isError } from '../../utils';
-
-function cleanErrorMessage(error: Error) {
-  const message = 'rawMessage' in error ? (error.rawMessage as string) : error.message;
-  return message.replace(/\s+at \d+:\d+/, '');
-}
+import { JSONCheckDefinition, Severity, SourceCodeType } from '../../types';
 
 export const ValidJSON: JSONCheckDefinition = {
   meta: {

packages/theme-check-common/src/checks/valid-local-blocks/index.ts

@@ -1,5 +1,11 @@
-import { LiquidCheckDefinition, Preset, Severity, SourceCodeType, Section } from '../../types';
-import { LiteralNode } from 'json-to-ast';
+import {
+  LiquidCheckDefinition,
+  Preset,
+  Severity,
+  SourceCodeType,
+  Section,
+  LiteralNode,
+} from '../../types';
 import { nodeAtPath } from '../../json';
 import { getSchema } from '../../to-schema';
 import { isBlock, isSection } from '../../to-schema';

packages/theme-check-common/src/checks/valid-local-blocks/valid-block-utils.ts

@@ -1,6 +1,5 @@
-import { LiteralNode } from 'json-to-ast';
 import { getLocEnd, getLocStart } from '../../json';
-import { Preset, ThemeBlock, Section, Context, SourceCodeType } from '../../types';
+import { Preset, ThemeBlock, Section, Context, SourceCodeType, LiteralNode } from '../../types';
 
 type BlockNodeWithPath = {
   node: Section.Block | ThemeBlock.Block | Preset.Block;

packages/theme-check-common/src/checks/valid-schema-name/index.ts

@@ -1,7 +1,6 @@
-import { LiteralNode } from 'json-to-ast';
 import { getLocEnd, getLocStart, nodeAtPath } from '../../json';
 import { getSchema } from '../../to-schema';
-import { LiquidCheckDefinition, Severity, SourceCodeType } from '../../types';
+import { LiquidCheckDefinition, LiteralNode, Severity, SourceCodeType } from '../../types';
 import { deepGet } from '../../utils';
 
 const MAX_SCHEMA_NAME_LENGTH = 25;