#136 Remove FollowSet cache dependency on ignoredTokens

ports/cpp/ci/test.sh

@@ -8,8 +8,6 @@ if [ $? -ne 0 ]; then
     exit 1
 fi
 
-exit 0 # https://github.com/mike-lischke/antlr4-c3/issues/136
-
 set -e
 
 TESTS="

ports/cpp/source/antlr4-c3/CodeCompletionCore.cpp

@@ -131,40 +131,15 @@ CandidatesCollection CodeCompletionCore::collectCandidates(
 
   processRule(atn->ruleToStartState[startRule], 0, callStack, 0, 0, candidates.isCancelled);
 
-  if (debugOptions.showResult) {
-    if (candidates.isCancelled) {
-      std::cout << "*** TIMED OUT ***\n";
-    }
-
-    std::cout << "States processed: " << statesProcessed << "\n\n";
-
-    std::cout << "Collected rules:\n";
-    for (const auto& [tokenIndex, rule] : candidates.rules) {
-      std::cout << ruleNames->at(tokenIndex);
-      std::cout << ", path: ";
-
-      for (const size_t token : rule.ruleList) {
-        std::cout << ruleNames->at(token) << " ";
-      }
-    }
-    std::cout << "\n\n";
-
-    std::set<std::string> sortedTokens;
-    for (const auto& [token, tokenList] : candidates.tokens) {
-      std::string value = vocabulary->getDisplayName(token);
-      for (const size_t following : tokenList) {
-        value += " " + vocabulary->getDisplayName(following);
-      }
-      sortedTokens.emplace(value);
-    }
-
-    std::cout << "Collected tokens:\n";
-    for (const std::string& symbol : sortedTokens) {
-      std::cout << symbol;
-    }
-    std::cout << "\n\n";
+  for (auto& [_, following] : candidates.tokens) {
+    auto removed = std::ranges::remove_if(following, [&](size_t token) {
+      return ignoredTokens.contains(token);
+    });
+    following.erase(std::begin(removed), std::end(removed));
   }
 
+  printOverallResults();
+
   return candidates;
 }
 
@@ -286,7 +261,7 @@ bool CodeCompletionCore::translateToRuleIndex(
  * @returns A list of toke types.
  */
 std::vector<size_t> CodeCompletionCore::getFollowingTokens(const antlr4::atn::Transition* transition
-) const {
+) {
   std::vector<size_t> result;
   std::vector<antlr4::atn::ATNState*> pipeline = {transition->target};
 
@@ -302,7 +277,7 @@ std::vector<size_t> CodeCompletionCore::getFollowingTokens(const antlr4::atn::Tr
       if (outgoing->getTransitionType() == antlr4::atn::TransitionType::ATOM) {
         if (!outgoing->isEpsilon()) {
           const auto list = outgoing->label();
-          if (list.size() == 1 && !ignoredTokens.contains(list.get(0))) {
+          if (list.size() == 1) {
             result.push_back(list.get(0));
             pipeline.push_back(outgoing->target);
           }
@@ -501,6 +476,7 @@ CodeCompletionCore::RuleEndStatus CodeCompletionCore::processRule(  // NOLINT
     antlr4::atn::RuleStopState* stop = atn->ruleToStopState[startState->ruleIndex];
     setsPerState[startState->stateNumber] = determineFollowSets(startState, stop);
   }
+
   const FollowSetsHolder& followSets = setsPerState[startState->stateNumber];
 
   // Get the token index where our rule starts from our (possibly filtered)
@@ -849,4 +825,40 @@ void CodeCompletionCore::printRuleState(RuleWithStartTokenList const& stack) {
   std::cout << "\n";
 }
 
+void CodeCompletionCore::printOverallResults() {
+  if (debugOptions.showResult) {
+    if (candidates.isCancelled) {
+      std::cout << "*** TIMED OUT ***\n";
+    }
+
+    std::cout << "States processed: " << statesProcessed << "\n\n";
+
+    std::cout << "Collected rules:\n";
+    for (const auto& [tokenIndex, rule] : candidates.rules) {
+      std::cout << ruleNames->at(tokenIndex);
+      std::cout << ", path: ";
+
+      for (const size_t token : rule.ruleList) {
+        std::cout << ruleNames->at(token) << " ";
+      }
+    }
+    std::cout << "\n\n";
+
+    std::set<std::string> sortedTokens;
+    for (const auto& [token, tokenList] : candidates.tokens) {
+      std::string value = vocabulary->getDisplayName(token);
+      for (const size_t following : tokenList) {
+        value += " " + vocabulary->getDisplayName(following);
+      }
+      sortedTokens.emplace(value);
+    }
+
+    std::cout << "Collected tokens:\n";
+    for (const std::string& symbol : sortedTokens) {
+      std::cout << symbol;
+    }
+    std::cout << "\n\n";
+  }
+}
+
 }  // namespace c3

ports/cpp/source/antlr4-c3/CodeCompletionCore.hpp

@@ -225,7 +225,7 @@ class CodeCompletionCore {
 
   bool translateToRuleIndex(size_t index, RuleWithStartTokenList const& ruleWithStartTokenList);
 
-  std::vector<size_t> getFollowingTokens(const antlr4::atn::Transition* transition) const;
+  static std::vector<size_t> getFollowingTokens(const antlr4::atn::Transition* transition);
 
   FollowSetsHolder determineFollowSets(antlr4::atn::ATNState* start, antlr4::atn::ATNState* stop);
 
@@ -258,6 +258,8 @@ class CodeCompletionCore {
   );
 
   void printRuleState(RuleWithStartTokenList const& stack);
+
+  void printOverallResults();
 };
 
 }  // namespace c3

ports/cpp/test/expr/ExprTest.cpp

@@ -98,9 +98,8 @@ TEST(SimpleExpressionParser, TypicalSetup) {
     auto candidates = completion.collectCandidates(0);
     EXPECT_THAT(Keys(candidates.tokens), UnorderedElementsAre(ExprLexer::VAR, ExprLexer::LET));
 
-    // NOTE: Behaviour differs from TypeScript version
-    EXPECT_THAT(candidates.tokens[ExprLexer::VAR], UnorderedElementsAre());
-    EXPECT_THAT(candidates.tokens[ExprLexer::LET], UnorderedElementsAre());
+    EXPECT_THAT(candidates.tokens[ExprLexer::VAR], ElementsAre());
+    EXPECT_THAT(candidates.tokens[ExprLexer::LET], ElementsAre());
   }
   {
     // 2) On the variable name ('c').

ports/cpp/test/whitebox/WhiteboxTest.cpp

@@ -82,7 +82,7 @@ TEST(WhiteboxGrammarTests, CaretAtOneOfMultiplePossibleStates) {
     auto candidates = completion.collectCandidates(2, {ctx});
 
     EXPECT_THAT(Keys(candidates.tokens), UnorderedElementsAre(WhiteboxLexer::DOLOR));
-    EXPECT_THAT(candidates.tokens[WhiteboxLexer::DOLOR], UnorderedElementsAre());
+    EXPECT_THAT(candidates.tokens[WhiteboxLexer::DOLOR], ElementsAre());
   }
 }
 
@@ -95,7 +95,7 @@ TEST(WhiteboxGrammarTests, CaretAtOneOfMultiplePossibleStatesWithCommonFollowLis
   auto candidates = completion.collectCandidates(2, {ctx});
 
   EXPECT_THAT(Keys(candidates.tokens), UnorderedElementsAre(WhiteboxLexer::DOLOR));
-  EXPECT_THAT(candidates.tokens[WhiteboxLexer::DOLOR], UnorderedElementsAre(WhiteboxLexer::SIT));
+  EXPECT_THAT(candidates.tokens[WhiteboxLexer::DOLOR], ElementsAre(WhiteboxLexer::SIT));
 }
 
 }  // namespace c3::test