From 4d3c2636b9722c4f936ed99362370b41feb98f40 Mon Sep 17 00:00:00 2001 From: vityaman Date: Mon, 12 Aug 2024 12:36:45 +0300 Subject: [PATCH 1/4] #136 Remove FollowSet cache dependency on ignoredTokens Signed-off-by: vityaman --- ports/cpp/ci/test.sh | 2 -- ports/cpp/source/antlr4-c3/CodeCompletionCore.cpp | 5 +++-- ports/cpp/source/antlr4-c3/CodeCompletionCore.hpp | 2 +- ports/cpp/test/expr/ExprTest.cpp | 5 ++--- 4 files changed, 6 insertions(+), 8 deletions(-) diff --git a/ports/cpp/ci/test.sh b/ports/cpp/ci/test.sh index f871ded..8812865 100644 --- a/ports/cpp/ci/test.sh +++ b/ports/cpp/ci/test.sh @@ -8,8 +8,6 @@ if [ $? -ne 0 ]; then exit 1 fi -exit 0 # https://github.com/mike-lischke/antlr4-c3/issues/136 - set -e TESTS=" diff --git a/ports/cpp/source/antlr4-c3/CodeCompletionCore.cpp b/ports/cpp/source/antlr4-c3/CodeCompletionCore.cpp index 07fda7b..c2fdc6d 100644 --- a/ports/cpp/source/antlr4-c3/CodeCompletionCore.cpp +++ b/ports/cpp/source/antlr4-c3/CodeCompletionCore.cpp @@ -286,7 +286,7 @@ bool CodeCompletionCore::translateToRuleIndex( * @returns A list of toke types. */ std::vector CodeCompletionCore::getFollowingTokens(const antlr4::atn::Transition* transition -) const { +) { std::vector result; std::vector pipeline = {transition->target}; @@ -302,7 +302,7 @@ std::vector CodeCompletionCore::getFollowingTokens(const antlr4::atn::Tr if (outgoing->getTransitionType() == antlr4::atn::TransitionType::ATOM) { if (!outgoing->isEpsilon()) { const auto list = outgoing->label(); - if (list.size() == 1 && !ignoredTokens.contains(list.get(0))) { + if (list.size() == 1) { result.push_back(list.get(0)); pipeline.push_back(outgoing->target); } @@ -501,6 +501,7 @@ CodeCompletionCore::RuleEndStatus CodeCompletionCore::processRule( // NOLINT antlr4::atn::RuleStopState* stop = atn->ruleToStopState[startState->ruleIndex]; setsPerState[startState->stateNumber] = determineFollowSets(startState, stop); } + const FollowSetsHolder& followSets = setsPerState[startState->stateNumber]; // Get the token index where our rule starts from our (possibly filtered) diff --git a/ports/cpp/source/antlr4-c3/CodeCompletionCore.hpp b/ports/cpp/source/antlr4-c3/CodeCompletionCore.hpp index e852146..5ee770f 100644 --- a/ports/cpp/source/antlr4-c3/CodeCompletionCore.hpp +++ b/ports/cpp/source/antlr4-c3/CodeCompletionCore.hpp @@ -225,7 +225,7 @@ class CodeCompletionCore { bool translateToRuleIndex(size_t index, RuleWithStartTokenList const& ruleWithStartTokenList); - std::vector getFollowingTokens(const antlr4::atn::Transition* transition) const; + static std::vector getFollowingTokens(const antlr4::atn::Transition* transition); FollowSetsHolder determineFollowSets(antlr4::atn::ATNState* start, antlr4::atn::ATNState* stop); diff --git a/ports/cpp/test/expr/ExprTest.cpp b/ports/cpp/test/expr/ExprTest.cpp index 65d287f..db9c022 100644 --- a/ports/cpp/test/expr/ExprTest.cpp +++ b/ports/cpp/test/expr/ExprTest.cpp @@ -98,9 +98,8 @@ TEST(SimpleExpressionParser, TypicalSetup) { auto candidates = completion.collectCandidates(0); EXPECT_THAT(Keys(candidates.tokens), UnorderedElementsAre(ExprLexer::VAR, ExprLexer::LET)); - // NOTE: Behaviour differs from TypeScript version - EXPECT_THAT(candidates.tokens[ExprLexer::VAR], UnorderedElementsAre()); - EXPECT_THAT(candidates.tokens[ExprLexer::LET], UnorderedElementsAre()); + EXPECT_THAT(candidates.tokens[ExprLexer::VAR], ElementsAre(ExprLexer::ID, ExprLexer::EQUAL)); + EXPECT_THAT(candidates.tokens[ExprLexer::LET], ElementsAre(ExprLexer::ID, ExprLexer::EQUAL)); } { // 2) On the variable name ('c'). From 81a1329ed9ac5f85f090b79ba85505d89b1e8ecd Mon Sep 17 00:00:00 2001 From: vityaman Date: Mon, 12 Aug 2024 12:40:21 +0300 Subject: [PATCH 2/4] #136 Fix test assertion for unordered follow tokens Signed-off-by: vityaman --- ports/cpp/test/whitebox/WhiteboxTest.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ports/cpp/test/whitebox/WhiteboxTest.cpp b/ports/cpp/test/whitebox/WhiteboxTest.cpp index cbc2944..a07451d 100644 --- a/ports/cpp/test/whitebox/WhiteboxTest.cpp +++ b/ports/cpp/test/whitebox/WhiteboxTest.cpp @@ -82,7 +82,7 @@ TEST(WhiteboxGrammarTests, CaretAtOneOfMultiplePossibleStates) { auto candidates = completion.collectCandidates(2, {ctx}); EXPECT_THAT(Keys(candidates.tokens), UnorderedElementsAre(WhiteboxLexer::DOLOR)); - EXPECT_THAT(candidates.tokens[WhiteboxLexer::DOLOR], UnorderedElementsAre()); + EXPECT_THAT(candidates.tokens[WhiteboxLexer::DOLOR], ElementsAre()); } } @@ -95,7 +95,7 @@ TEST(WhiteboxGrammarTests, CaretAtOneOfMultiplePossibleStatesWithCommonFollowLis auto candidates = completion.collectCandidates(2, {ctx}); EXPECT_THAT(Keys(candidates.tokens), UnorderedElementsAre(WhiteboxLexer::DOLOR)); - EXPECT_THAT(candidates.tokens[WhiteboxLexer::DOLOR], UnorderedElementsAre(WhiteboxLexer::SIT)); + EXPECT_THAT(candidates.tokens[WhiteboxLexer::DOLOR], ElementsAre(WhiteboxLexer::SIT)); } } // namespace c3::test From 7bbae9506648ad96fd99372ccd151ea28e74f514 Mon Sep 17 00:00:00 2001 From: vityaman Date: Mon, 26 Aug 2024 08:48:52 +0300 Subject: [PATCH 3/4] #136 Remove ignored following tokens Signed-off-by: vityaman --- ports/cpp/source/antlr4-c3/CodeCompletionCore.cpp | 7 +++++++ ports/cpp/test/expr/ExprTest.cpp | 4 ++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/ports/cpp/source/antlr4-c3/CodeCompletionCore.cpp b/ports/cpp/source/antlr4-c3/CodeCompletionCore.cpp index c2fdc6d..49995c4 100644 --- a/ports/cpp/source/antlr4-c3/CodeCompletionCore.cpp +++ b/ports/cpp/source/antlr4-c3/CodeCompletionCore.cpp @@ -131,6 +131,13 @@ CandidatesCollection CodeCompletionCore::collectCandidates( processRule(atn->ruleToStartState[startRule], 0, callStack, 0, 0, candidates.isCancelled); + for (auto& [_, following] : candidates.tokens) { + auto removed = std::ranges::remove_if(following, [&](size_t token) { + return ignoredTokens.contains(token); + }); + following.erase(std::begin(removed), std::end(removed)); + } + if (debugOptions.showResult) { if (candidates.isCancelled) { std::cout << "*** TIMED OUT ***\n"; diff --git a/ports/cpp/test/expr/ExprTest.cpp b/ports/cpp/test/expr/ExprTest.cpp index db9c022..f0b945d 100644 --- a/ports/cpp/test/expr/ExprTest.cpp +++ b/ports/cpp/test/expr/ExprTest.cpp @@ -98,8 +98,8 @@ TEST(SimpleExpressionParser, TypicalSetup) { auto candidates = completion.collectCandidates(0); EXPECT_THAT(Keys(candidates.tokens), UnorderedElementsAre(ExprLexer::VAR, ExprLexer::LET)); - EXPECT_THAT(candidates.tokens[ExprLexer::VAR], ElementsAre(ExprLexer::ID, ExprLexer::EQUAL)); - EXPECT_THAT(candidates.tokens[ExprLexer::LET], ElementsAre(ExprLexer::ID, ExprLexer::EQUAL)); + EXPECT_THAT(candidates.tokens[ExprLexer::VAR], ElementsAre()); + EXPECT_THAT(candidates.tokens[ExprLexer::LET], ElementsAre()); } { // 2) On the variable name ('c'). From c1ad6509d11c4aae8e233b17215696b5fab033ec Mon Sep 17 00:00:00 2001 From: vityaman Date: Mon, 26 Aug 2024 08:55:01 +0300 Subject: [PATCH 4/4] #136 Extract overall results output to procedure Signed-off-by: vityaman --- .../source/antlr4-c3/CodeCompletionCore.cpp | 70 ++++++++++--------- .../source/antlr4-c3/CodeCompletionCore.hpp | 2 + 2 files changed, 39 insertions(+), 33 deletions(-) diff --git a/ports/cpp/source/antlr4-c3/CodeCompletionCore.cpp b/ports/cpp/source/antlr4-c3/CodeCompletionCore.cpp index 49995c4..30fc3d1 100644 --- a/ports/cpp/source/antlr4-c3/CodeCompletionCore.cpp +++ b/ports/cpp/source/antlr4-c3/CodeCompletionCore.cpp @@ -138,39 +138,7 @@ CandidatesCollection CodeCompletionCore::collectCandidates( following.erase(std::begin(removed), std::end(removed)); } - if (debugOptions.showResult) { - if (candidates.isCancelled) { - std::cout << "*** TIMED OUT ***\n"; - } - - std::cout << "States processed: " << statesProcessed << "\n\n"; - - std::cout << "Collected rules:\n"; - for (const auto& [tokenIndex, rule] : candidates.rules) { - std::cout << ruleNames->at(tokenIndex); - std::cout << ", path: "; - - for (const size_t token : rule.ruleList) { - std::cout << ruleNames->at(token) << " "; - } - } - std::cout << "\n\n"; - - std::set sortedTokens; - for (const auto& [token, tokenList] : candidates.tokens) { - std::string value = vocabulary->getDisplayName(token); - for (const size_t following : tokenList) { - value += " " + vocabulary->getDisplayName(following); - } - sortedTokens.emplace(value); - } - - std::cout << "Collected tokens:\n"; - for (const std::string& symbol : sortedTokens) { - std::cout << symbol; - } - std::cout << "\n\n"; - } + printOverallResults(); return candidates; } @@ -857,4 +825,40 @@ void CodeCompletionCore::printRuleState(RuleWithStartTokenList const& stack) { std::cout << "\n"; } +void CodeCompletionCore::printOverallResults() { + if (debugOptions.showResult) { + if (candidates.isCancelled) { + std::cout << "*** TIMED OUT ***\n"; + } + + std::cout << "States processed: " << statesProcessed << "\n\n"; + + std::cout << "Collected rules:\n"; + for (const auto& [tokenIndex, rule] : candidates.rules) { + std::cout << ruleNames->at(tokenIndex); + std::cout << ", path: "; + + for (const size_t token : rule.ruleList) { + std::cout << ruleNames->at(token) << " "; + } + } + std::cout << "\n\n"; + + std::set sortedTokens; + for (const auto& [token, tokenList] : candidates.tokens) { + std::string value = vocabulary->getDisplayName(token); + for (const size_t following : tokenList) { + value += " " + vocabulary->getDisplayName(following); + } + sortedTokens.emplace(value); + } + + std::cout << "Collected tokens:\n"; + for (const std::string& symbol : sortedTokens) { + std::cout << symbol; + } + std::cout << "\n\n"; + } +} + } // namespace c3 diff --git a/ports/cpp/source/antlr4-c3/CodeCompletionCore.hpp b/ports/cpp/source/antlr4-c3/CodeCompletionCore.hpp index 5ee770f..de90341 100644 --- a/ports/cpp/source/antlr4-c3/CodeCompletionCore.hpp +++ b/ports/cpp/source/antlr4-c3/CodeCompletionCore.hpp @@ -258,6 +258,8 @@ class CodeCompletionCore { ); void printRuleState(RuleWithStartTokenList const& stack); + + void printOverallResults(); }; } // namespace c3