Updated nsplug to handle include tags.

moos-ivp-language-server/src/cache.rs

@@ -11,6 +11,32 @@ pub struct SemanticTokenInfo {
     pub token_modifiers: u32,
 }
 
+/*
+ * namespace	For identifiers that declare or reference a namespace, module, or package.
+ * class	For identifiers that declare or reference a class type.
+ * enum	For identifiers that declare or reference an enumeration type.
+ * interface	For identifiers that declare or reference an interface type.
+ * struct	For identifiers that declare or reference a struct type.
+ * typeParameter	For identifiers that declare or reference a type parameter.
+ * type	For identifiers that declare or reference a type that is not covered above.
+ * parameter	For identifiers that declare or reference a function or method parameters.
+ * variable	For identifiers that declare or reference a local or global variable.
+ * property	For identifiers that declare or reference a member property, member field, or member variable.
+ * enumMember	For identifiers that declare or reference an enumeration property, constant, or member.
+ * decorator	For identifiers that declare or reference decorators and annotations.
+ * event	For identifiers that declare an event property.
+ * function	For identifiers that declare a function.
+ * method	For identifiers that declare a member function or method.
+ * macro	For identifiers that declare a macro.
+ * label	For identifiers that declare a label.
+ * comment	For tokens that represent a comment.
+ * string	For tokens that represent a string literal.
+ * keyword	For tokens that represent a language keyword.
+ * number	For tokens that represent a number literal.
+ * regexp	For tokens that represent a regular expression literal.
+ * operator	For tokens that represent an operator.
+ */
+
 #[derive(Debug, Copy, Clone)]
 pub enum TokenTypes {
     /// For tokens that represent a comment.
@@ -27,6 +53,10 @@ pub enum TokenTypes {
     Macro,
     /// For tokens that represent an operator
     Operator,
+    /// For types
+    Type,
+    /// Namespace
+    Namespace,
 }
 
 impl Into<u32> for TokenTypes {

moos-ivp-language-server/src/main.rs

@@ -138,6 +138,8 @@ fn main() -> Result<(), Box<dyn Error + Sync + Send>> {
         semantic_tokens_provider: Some(
             SemanticTokensOptions {
                 legend: SemanticTokensLegend {
+                    /// TODO: These should get moved to where we define the
+                    /// SemanticToken enum
                     token_types: vec![
                         SemanticTokenType::COMMENT,
                         SemanticTokenType::KEYWORD,
@@ -146,6 +148,8 @@ fn main() -> Result<(), Box<dyn Error + Sync + Send>> {
                         SemanticTokenType::NUMBER,
                         SemanticTokenType::MACRO,
                         SemanticTokenType::OPERATOR,
+                        SemanticTokenType::TYPE,
+                        SemanticTokenType::NAMESPACE,
                     ],
                     token_modifiers: vec![
                         SemanticTokenModifier::DECLARATION,

moos-ivp-language-server/src/parsers/nsplug.rs

@@ -7,8 +7,8 @@ use moos_parser::{
         error::{PlugParseError, PlugParseErrorKind},
         lexer::{State, Token},
         tree::{
-            IfDefBranch, IfNotDefBranch, IncludePath, Line, Lines, MacroCondition, MacroDefinition,
-            MacroType, Quote, Values, Variable, VariableStrings,
+            IfDefBranch, IfNotDefBranch, IncludePath, IncludeTag, Line, Lines, MacroCondition,
+            MacroDefinition, MacroType, Quote, Values, Variable, VariableStrings,
         },
     },
     ParseError, PlugParser,
@@ -159,8 +159,8 @@ fn handle_lines(document: &mut Document, lines: &Lines) {
                         );
                         handle_values(document, line, &definition.value);
                     }
-                    MacroType::Include { path, range } => {
-                        handle_include(document, line, &path, &range);
+                    MacroType::Include { path, tag, range } => {
+                        handle_include(document, line, &path, &tag, &range);
                     }
 
                     MacroType::IfDef {
@@ -217,7 +217,13 @@ fn handle_macro_token(document: &mut Document, line: u32, range: &TokenRange) {
     );
 }
 
-fn handle_include(document: &mut Document, line: u32, path: &IncludePath, range: &TokenRange) {
+fn handle_include(
+    document: &mut Document,
+    line: u32,
+    path: &IncludePath,
+    tag: &Option<IncludeTag>,
+    range: &TokenRange,
+) {
     document.semantic_tokens.insert(
         line,
         range.clone(),
@@ -232,6 +238,17 @@ fn handle_include(document: &mut Document, line: u32, path: &IncludePath, range:
         }
         IncludePath::Quote(quote) => handle_quote(document, line, &quote),
     }
+
+    if let Some(tag) = tag {
+        document.semantic_tokens.insert(
+            line,
+            tag.range.clone(),
+            SemanticTokenInfo {
+                token_type: TokenTypes::Namespace as u32,
+                token_modifiers: 0,
+            },
+        );
+    }
 }
 
 fn handle_quote(document: &mut Document, line: u32, quote: &Quote) {

moos-parser/src/nsplug/lexer.rs

@@ -54,6 +54,8 @@ pub enum Token<'input> {
     UnknownMacro(&'input str),
     OrOperator,
     AndOperator,
+    LeftAngleBracket,
+    RightAngleBracket,
     Space,
     /// End of Line
     EOL,
@@ -372,17 +374,18 @@ impl<'input> Lexer<'input> {
             return;
         };
 
-        let mut is_ifndef = false;
+        let (is_ifndef, is_include) = match token {
+            Token::MacroInclude => (false, true),
+            Token::MacroIfNotDef => (true, false),
+            _ => (false, false),
+        };
 
         let has_conditions = match token {
             Token::MacroIfDef | Token::MacroElseIfDef => true,
             // #ifndef doesn't really support conditions, but we will handle
             // that in the parser. For now, enable the tokenization of the
             // && and || operators so we can throw an in the parser.
-            Token::MacroIfNotDef => {
-                is_ifndef = true;
-                true
-            }
+            Token::MacroIfNotDef => true,
             _ => false,
         };
 
@@ -400,15 +403,23 @@ impl<'input> Lexer<'input> {
         while let Some(((i, c), (_ii, cc))) = self.iter.find(|&((_i, c), (_ii, cc))| {
             c == '\n'
                 || c == '"'
-                || (c == '=')
                 || (has_whitespace && (c == ' ' || c == '\t')) // Whitespace
                 || (has_comments && (c == '/' && cc == '/')) // Comment
                 || (c == '$' && cc == '(') // Plug variable
                 || (c == '%' && cc == '(') // Plug Upper Variable
                 || (has_conditions && c == '|' && cc == '|') // Or operator
                 || (has_conditions && c == '&' && cc == '&' ) // And operator
+                || (is_include && (c == ' ' || c == '\t') && cc == '<') // Handle include tags
         }) {
             match c {
+                c if is_include && (c == ' ' || c == '\t') && cc == '<' => {
+                    // Handle include - This needs to happen before handling
+                    // the spaces below.
+                    let found_include_tag = self.tokenize_include_tag(i);
+                    if found_include_tag {
+                        return;
+                    }
+                }
                 '\n' => {
                     self.tokenize_new_line(i, false);
                     return;
@@ -463,7 +474,7 @@ impl<'input> Lexer<'input> {
                     self.trim_start = true;
                     self.found_assign_op = false;
                 }
-                ' ' | '\t' => {
+                c if has_whitespace && (c == ' ' || c == '\t') => {
                     if !is_ifndef {
                         self.found_assign_op = true; // Enables parsing primitives
                     }
@@ -497,6 +508,102 @@ impl<'input> Lexer<'input> {
         }
     }
 
+    fn tokenize_include_tag(&mut self, index: usize) -> bool {
+        // The include tag is a space followed by a '<', then a tag, then
+        // a '>' and the end of the line. If it does not follow that format,
+        // the characters should be treated as part of the include path.
+
+        // NOTE: We do NOT handle nested tags (E.G. <<my_tag>>) because
+        // nsplug doesn't care. It finds the first '<' then the last '>'.
+
+        // Make a clone of the iterator so we can look forward to the end of
+        // the line to see if this is a valid tag
+        let mut local_iter = self.iter.clone();
+
+        let mut right_bracket_location: Option<usize> = None;
+        let mut new_line_index: Option<usize> = None;
+
+        // Search until we find the end of the line. Mark the location of the
+        // last '>'.
+        while let Some(((i, c), (_ii, _cc))) = local_iter.find(|&((_i, c), (_ii, _cc))| {
+            c == '\n' // New line
+            || c == '>' // Right angle bracket
+        }) {
+            match c {
+                '>' => right_bracket_location = Some(i),
+                '\n' => {
+                    new_line_index = Some(i);
+                    break;
+                }
+                _ => {}
+            }
+        }
+
+        if let Some(right_bracket_location) = right_bracket_location {
+            let remaining = if let Some(i) = new_line_index {
+                // Up to, but not including the new line
+                &self.input[index + 1..i]
+            } else {
+                // Until the end of the file
+                &self.input[index + 1..]
+            }
+            .trim();
+
+            // Check that the right bracket is the last character in the trimmed
+            // string.
+            if remaining.len() < 2 && !remaining.ends_with(">") {
+                return false;
+            }
+
+            // Check that there isn't any whitespace in the remaining
+            if let Some(_i) = remaining.find(char::is_whitespace) {
+                return false;
+            }
+
+            // We have found a tag.
+            // Push unhandled before the tag
+            self.trim_end = true;
+            if let Some((prev_i, unhandled)) = self.get_unhandled_string(index, true) {
+                if !unhandled.is_empty() {
+                    self.scan_value(unhandled, prev_i);
+                    self.start_of_line = false;
+                    self.trim_start = false;
+                }
+            }
+
+            // Push the left angle bracket
+            self.push_token(index + 1, Token::LeftAngleBracket, index + 2);
+
+            // Push the tag as a value string
+            self.push_token(
+                index + 2,
+                Token::ValueString(&remaining[1..remaining.len() - 1]),
+                right_bracket_location,
+            );
+
+            // Push the right angle bracket
+            self.push_token(
+                right_bracket_location,
+                Token::RightAngleBracket,
+                right_bracket_location + 1,
+            );
+
+            // If we found a new line, EOL
+            if let Some(i) = new_line_index {
+                self._handle_new_line(i);
+            } else {
+                self.previous_index = None;
+            }
+
+            // Update our iterator to our local copy
+            self.iter = local_iter;
+
+            return true;
+        } else {
+            return false;
+        }
+    }
+
     fn tokenize_new_line(&mut self, i: usize, drop_unhandled: bool) {
         // Trim up to the new line
         self.trim_end = true;

moos-parser/src/nsplug/nsplug.lalrpop

@@ -46,12 +46,6 @@ pub Line: Line<'input> = {
     },
 }
 
-// TODO: 
-//   1. Add MacroIfDef
-//   2. Add MacroElseIfDef
-//   3. Add MacroIfNotDef
-
-
 MacroDefine: Line<'input> = {
     <ml:@L> <d:"#define"> <mr:@R> <definition:MacroDefinition> <comment:"Comment"?> "EOL" => 
         Line::Macro{
@@ -71,7 +65,7 @@ MacroDefinition: MacroDefinition<'input> = {
 }
 
 MacroInclude: Line<'input> = {
-    <ml:@L> <d:"#include"> <mr:@R> <path:IncludePath> <l:@L> <comment:"Comment"?> <r:@R> "EOL" => {
+    <ml:@L> <d:"#include"> <mr:@R> <path:IncludePath> <tag: IncludeTag?> <l:@L> <comment:"Comment"?> <r:@R> "EOL" => {
         if let Some(comment) = comment {
             let e = lalrpop_util::ErrorRecovery { 
                 error: lalrpop_util::ParseError::User {
@@ -82,7 +76,7 @@ MacroInclude: Line<'input> = {
             state.errors.push(e);
         }
         Line::Macro{
-            macro_type: MacroType::Include{path, range: TokenRange::new_line(ml,mr).unwrap()}, 
+            macro_type: MacroType::Include{path, tag, range: TokenRange::new_line(ml,mr).unwrap()}, 
             comment,
             line: ml.line,
         }
@@ -147,6 +141,10 @@ IncludePath: tree::IncludePath<'input> = {
     Quote => tree::IncludePath::Quote(<>),
 }
 
+IncludeTag: tree::IncludeTag<'input> = {
+    <l:@L> "<" <tag: "ValueString"> ">" <r:@R> => tree::IncludeTag{tag, range: TokenRange::new_line(l,r).expect("Invalid token range while parsing `IncludeTag`")},
+}
+
 IfDef: Line<'input> = {
     <ml:@L> "#ifdef" <mr:@R> <condition:MacroCondition> "EOL" <body:Lines> <branch: IfDefBranch> => 
         Line::Macro{
@@ -201,8 +199,6 @@ MacroCondition: tree::MacroCondition<'input> = {
         },
 }
 
-// TODO: Need to change IfNotDef to use clauses not conditions
-
 IfNotDef: Line<'input> = {
     <ml:@L> "#ifndef" <mr:@R> <clauses:IfNotDefClauses> " "* "EOL" <body:Lines> <branch: IfNotDefBranch> => 
         Line::Macro{
@@ -273,6 +269,8 @@ extern {
         "#unknown" => Token::UnknownMacro(<&'input str>),
         "||" => Token::OrOperator,
         "&&" => Token::AndOperator,
+        "<" => Token::LeftAngleBracket,
+        ">" => Token::RightAngleBracket,
         " " => Token::Space,
     }
 }

moos-parser/src/nsplug/tree.rs

@@ -178,6 +178,25 @@ impl<'input> From<Quote<'input>> for IncludePath<'input> {
     }
 }
 
+#[derive(Debug, Clone, Copy)]
+pub struct IncludeTag<'input> {
+    pub tag: &'input str,
+    /// Range of the Include tag. THis includes the start and ending brackets
+    pub range: TokenRange,
+}
+
+impl<'input> IncludeTag<'input> {
+    pub fn new(tag: &'input str, range: TokenRange) -> Self {
+        Self { tag, range }
+    }
+}
+
+impl<'input> ToString for IncludeTag<'input> {
+    fn to_string(&self) -> String {
+        format!("<{}>", self.tag)
+    }
+}
+
 #[derive(Debug)]
 pub enum MacroType<'input> {
     Define {
@@ -187,6 +206,8 @@ pub enum MacroType<'input> {
     },
     Include {
         path: IncludePath<'input>,
+        /// Optional include tag. Added in 2020.
+        tag: Option<IncludeTag<'input>>,
         /// Range of the "#include"
         range: TokenRange,
     },
@@ -212,8 +233,12 @@ impl<'input> ToString for MacroType<'input> {
             MacroType::Define { definition, range } => {
                 format!("#define {}", definition.to_string())
             }
-            MacroType::Include { path, range } => {
-                format!("#include {}", path.to_string())
+            MacroType::Include { path, tag, range } => {
+                if let Some(tag) = tag {
+                    format!("#include {} {}", path.to_string(), tag.to_string())
+                } else {
+                    format!("#include {}", path.to_string())
+                }
             }
             MacroType::IfDef {
                 condition,