From abc70da7bbf9fcc3c30d7499185118b9cac0bc1d Mon Sep 17 00:00:00 2001 From: Christopher Gagner Date: Tue, 15 Oct 2024 17:33:40 -0400 Subject: [PATCH] fix: Fixed all lexers `tokenize_macro` methods Fixed all lexers `tokenize_macro` methods to return a boolean `true` if a token is parsed or `false` if a token is not parsed. This fixes an issue where the `#` symbol appears in a comment or in another string field. This was an issue with `pMarineViewer` button fields. --- moos-parser/src/behavior/lexer.rs | 17 ++++++++++++----- moos-parser/src/moos/lexer.rs | 17 ++++++++++++----- moos-parser/src/nsplug/lexer.rs | 26 +++++++++++++++----------- 3 files changed, 39 insertions(+), 21 deletions(-) diff --git a/moos-parser/src/behavior/lexer.rs b/moos-parser/src/behavior/lexer.rs index d42b4b1..65df89e 100644 --- a/moos-parser/src/behavior/lexer.rs +++ b/moos-parser/src/behavior/lexer.rs @@ -295,15 +295,17 @@ impl<'input> Lexer<'input> { ); } - fn tokenize_macro(&mut self, _i: usize) { + /// Tokenize macros starting at position `_i`. + /// Returns true if a token is parsed; false if no tokens are parsed. + fn tokenize_macro(&mut self, _i: usize) -> bool { // If its not the start of the line, it can't be a macro. if !self.start_of_line { - return; + return false; } if let Some((_prev_i, unhandled)) = self.get_unhandled_string(self.input.len(), true) { if !unhandled.trim_start().starts_with("#") { - return; + return false; } } // Skip lines that start with # @@ -316,7 +318,7 @@ impl<'input> Lexer<'input> { // Setting the previous index to drop previous tokens self.previous_index = self.get_safe_index(i); self.tokenize_new_line(i, false); - return; + return true; } _ => {} } @@ -324,6 +326,7 @@ impl<'input> Lexer<'input> { // Should only get in here if we have reached the end of the input. self.previous_index = self.get_safe_index(self.input.len()); + return true; } fn tokenize_new_line(&mut self, i: usize, drop_unhandled: bool) { @@ -590,7 +593,11 @@ impl<'input> Lexer<'input> { break; } } - '#' => self.tokenize_macro(i), + '#' => { + if self.tokenize_macro(i) { + return; + } + } '{' => self.tokenize_curly_brace(i, Token::CurlyOpen), '}' => self.tokenize_curly_brace(i, Token::CurlyClose), ',' => self.tokenize_comma(i), diff --git a/moos-parser/src/moos/lexer.rs b/moos-parser/src/moos/lexer.rs index efb9940..7280fe5 100644 --- a/moos-parser/src/moos/lexer.rs +++ b/moos-parser/src/moos/lexer.rs @@ -279,15 +279,17 @@ impl<'input> Lexer<'input> { ); } - fn tokenize_macro(&mut self, _i: usize) { + /// Tokenize macros starting at position `_i`. + /// Returns true if a token is parsed; false if no tokens are parsed. + fn tokenize_macro(&mut self, _i: usize) -> bool { // If its not the start of the line, it can't be a macro. if !self.start_of_line { - return; + return false; } if let Some((_prev_i, unhandled)) = self.get_unhandled_string(self.input.len(), true) { if !unhandled.trim_start().starts_with("#") { - return; + return false; } } // Skip lines that start with # @@ -300,7 +302,7 @@ impl<'input> Lexer<'input> { // Setting the previous index to drop previous tokens self.previous_index = self.get_safe_index(i); self.tokenize_new_line(i, false); - return; + return true; } _ => {} } @@ -308,6 +310,7 @@ impl<'input> Lexer<'input> { // Should only get in here if we have reached the end of the input. self.previous_index = self.get_safe_index(self.input.len()); + return true; } fn tokenize_new_line(&mut self, i: usize, drop_unhandled: bool) { @@ -553,7 +556,11 @@ impl<'input> Lexer<'input> { break; } } - '#' => self.tokenize_macro(i), + '#' => { + if self.tokenize_macro(i) { + return; + } + } '{' => self.tokenize_curly_brace(i, Token::CurlyOpen), '}' => self.tokenize_curly_brace(i, Token::CurlyClose), _ => {} diff --git a/moos-parser/src/nsplug/lexer.rs b/moos-parser/src/nsplug/lexer.rs index feeb5c8..a73cbc2 100644 --- a/moos-parser/src/nsplug/lexer.rs +++ b/moos-parser/src/nsplug/lexer.rs @@ -270,17 +270,19 @@ impl<'input> Lexer<'input> { } } - fn tokenize_macro(&mut self, i: usize) { + /// Tokenize macros starting at position `_i`. + /// Returns true if a token is parsed; false if no tokens are parsed. + fn tokenize_macro(&mut self, i: usize) -> bool { // If its not the start of the line, it can't be a macro. if !self.start_of_line { - return; + return false; } // Make sure the current line starts with nothing but whitespace before // the '#' if let Some((prev_i, unhandled)) = self.get_unhandled_string(i) { if !unhandled.trim().is_empty() { - return; + return false; } // Push the indent as a whitespace token. self.push_token(prev_i, Token::WhiteSpace(unhandled), i); @@ -307,7 +309,7 @@ impl<'input> Lexer<'input> { match ccc { '\n' => { // Handle this back in the main tokenize method - return; + return false; } _ => {} } @@ -318,7 +320,7 @@ impl<'input> Lexer<'input> { let token = Self::get_macro_token(line); self.push_token(i, token, self.input.len()); self.previous_index = None; - return; + return true; }; let is_include = match token { @@ -358,12 +360,12 @@ impl<'input> Lexer<'input> { } '\n' => { self.tokenize_new_line(i, false); - return; + return true; } '"' => { let found_quote = self.tokenize_quote(i); if !found_quote { - return; + return false; } } c if (c == '$' && cc == '(') => { @@ -376,7 +378,7 @@ impl<'input> Lexer<'input> { } }); if !found_variable { - return; + return false; } } c if (c == '%' && cc == '(') => { @@ -389,7 +391,7 @@ impl<'input> Lexer<'input> { } }); if !found_variable { - return; + return false; } } '|' => { @@ -413,6 +415,7 @@ impl<'input> Lexer<'input> { } self.previous_index = self.get_safe_index(self.input.len()); } + return true; } fn tokenize_new_line(&mut self, i: usize, drop_unhandled: bool) { @@ -641,8 +644,9 @@ impl<'input> Lexer<'input> { }); } '#' => { - self.tokenize_macro(i); - return; + if self.tokenize_macro(i) { + return; + } } _ => {} }