diff --git a/moos-ivp-language-server/src/cache.rs b/moos-ivp-language-server/src/cache.rs index 92d2287..75ab6ed 100644 --- a/moos-ivp-language-server/src/cache.rs +++ b/moos-ivp-language-server/src/cache.rs @@ -11,6 +11,32 @@ pub struct SemanticTokenInfo { pub token_modifiers: u32, } +/* + * namespace For identifiers that declare or reference a namespace, module, or package. + * class For identifiers that declare or reference a class type. + * enum For identifiers that declare or reference an enumeration type. + * interface For identifiers that declare or reference an interface type. + * struct For identifiers that declare or reference a struct type. + * typeParameter For identifiers that declare or reference a type parameter. + * type For identifiers that declare or reference a type that is not covered above. + * parameter For identifiers that declare or reference a function or method parameters. + * variable For identifiers that declare or reference a local or global variable. + * property For identifiers that declare or reference a member property, member field, or member variable. + * enumMember For identifiers that declare or reference an enumeration property, constant, or member. + * decorator For identifiers that declare or reference decorators and annotations. + * event For identifiers that declare an event property. + * function For identifiers that declare a function. + * method For identifiers that declare a member function or method. + * macro For identifiers that declare a macro. + * label For identifiers that declare a label. + * comment For tokens that represent a comment. + * string For tokens that represent a string literal. + * keyword For tokens that represent a language keyword. + * number For tokens that represent a number literal. + * regexp For tokens that represent a regular expression literal. + * operator For tokens that represent an operator. + */ + #[derive(Debug, Copy, Clone)] pub enum TokenTypes { /// For tokens that represent a comment. @@ -27,6 +53,10 @@ pub enum TokenTypes { Macro, /// For tokens that represent an operator Operator, + /// For types + Type, + /// Namespace + Namespace, } impl Into for TokenTypes { diff --git a/moos-ivp-language-server/src/main.rs b/moos-ivp-language-server/src/main.rs index 88e438f..446e25c 100644 --- a/moos-ivp-language-server/src/main.rs +++ b/moos-ivp-language-server/src/main.rs @@ -138,6 +138,8 @@ fn main() -> Result<(), Box> { semantic_tokens_provider: Some( SemanticTokensOptions { legend: SemanticTokensLegend { + /// TODO: These should get moved to where we define the + /// SemanticToken enum token_types: vec![ SemanticTokenType::COMMENT, SemanticTokenType::KEYWORD, @@ -146,6 +148,8 @@ fn main() -> Result<(), Box> { SemanticTokenType::NUMBER, SemanticTokenType::MACRO, SemanticTokenType::OPERATOR, + SemanticTokenType::TYPE, + SemanticTokenType::NAMESPACE, ], token_modifiers: vec![ SemanticTokenModifier::DECLARATION, diff --git a/moos-ivp-language-server/src/parsers/nsplug.rs b/moos-ivp-language-server/src/parsers/nsplug.rs index e2e7a61..f9bf1c0 100644 --- a/moos-ivp-language-server/src/parsers/nsplug.rs +++ b/moos-ivp-language-server/src/parsers/nsplug.rs @@ -7,8 +7,8 @@ use moos_parser::{ error::{PlugParseError, PlugParseErrorKind}, lexer::{State, Token}, tree::{ - IfDefBranch, IfNotDefBranch, IncludePath, Line, Lines, MacroCondition, MacroDefinition, - MacroType, Quote, Values, Variable, VariableStrings, + IfDefBranch, IfNotDefBranch, IncludePath, IncludeTag, Line, Lines, MacroCondition, + MacroDefinition, MacroType, Quote, Values, Variable, VariableStrings, }, }, ParseError, PlugParser, @@ -159,8 +159,8 @@ fn handle_lines(document: &mut Document, lines: &Lines) { ); handle_values(document, line, &definition.value); } - MacroType::Include { path, range } => { - handle_include(document, line, &path, &range); + MacroType::Include { path, tag, range } => { + handle_include(document, line, &path, &tag, &range); } MacroType::IfDef { @@ -217,7 +217,13 @@ fn handle_macro_token(document: &mut Document, line: u32, range: &TokenRange) { ); } -fn handle_include(document: &mut Document, line: u32, path: &IncludePath, range: &TokenRange) { +fn handle_include( + document: &mut Document, + line: u32, + path: &IncludePath, + tag: &Option, + range: &TokenRange, +) { document.semantic_tokens.insert( line, range.clone(), @@ -232,6 +238,17 @@ fn handle_include(document: &mut Document, line: u32, path: &IncludePath, range: } IncludePath::Quote(quote) => handle_quote(document, line, "e), } + + if let Some(tag) = tag { + document.semantic_tokens.insert( + line, + tag.range.clone(), + SemanticTokenInfo { + token_type: TokenTypes::Namespace as u32, + token_modifiers: 0, + }, + ); + } } fn handle_quote(document: &mut Document, line: u32, quote: &Quote) { diff --git a/moos-parser/src/nsplug/lexer.rs b/moos-parser/src/nsplug/lexer.rs index dddde64..db20c0c 100644 --- a/moos-parser/src/nsplug/lexer.rs +++ b/moos-parser/src/nsplug/lexer.rs @@ -54,6 +54,8 @@ pub enum Token<'input> { UnknownMacro(&'input str), OrOperator, AndOperator, + LeftAngleBracket, + RightAngleBracket, Space, /// End of Line EOL, @@ -372,17 +374,18 @@ impl<'input> Lexer<'input> { return; }; - let mut is_ifndef = false; + let (is_ifndef, is_include) = match token { + Token::MacroInclude => (false, true), + Token::MacroIfNotDef => (true, false), + _ => (false, false), + }; let has_conditions = match token { Token::MacroIfDef | Token::MacroElseIfDef => true, // #ifndef doesn't really support conditions, but we will handle // that in the parser. For now, enable the tokenization of the // && and || operators so we can throw an in the parser. - Token::MacroIfNotDef => { - is_ifndef = true; - true - } + Token::MacroIfNotDef => true, _ => false, }; @@ -400,15 +403,23 @@ impl<'input> Lexer<'input> { while let Some(((i, c), (_ii, cc))) = self.iter.find(|&((_i, c), (_ii, cc))| { c == '\n' || c == '"' - || (c == '=') || (has_whitespace && (c == ' ' || c == '\t')) // Whitespace || (has_comments && (c == '/' && cc == '/')) // Comment || (c == '$' && cc == '(') // Plug variable || (c == '%' && cc == '(') // Plug Upper Variable || (has_conditions && c == '|' && cc == '|') // Or operator || (has_conditions && c == '&' && cc == '&' ) // And operator + || (is_include && (c == ' ' || c == '\t') && cc == '<') // Handle include tags }) { match c { + c if is_include && (c == ' ' || c == '\t') && cc == '<' => { + // Handle include - This needs to happen before handling + // the spaces below. + let found_include_tag = self.tokenize_include_tag(i); + if found_include_tag { + return; + } + } '\n' => { self.tokenize_new_line(i, false); return; @@ -463,7 +474,7 @@ impl<'input> Lexer<'input> { self.trim_start = true; self.found_assign_op = false; } - ' ' | '\t' => { + c if has_whitespace && (c == ' ' || c == '\t') => { if !is_ifndef { self.found_assign_op = true; // Enables parsing primitives } @@ -497,6 +508,102 @@ impl<'input> Lexer<'input> { } } + fn tokenize_include_tag(&mut self, index: usize) -> bool { + // The include tag is a space followed by a '<', then a tag, then + // a '>' and the end of the line. If it does not follow that format, + // the characters should be treated as part of the include path. + + // NOTE: We do NOT handle nested tags (E.G. <>) because + // nsplug doesn't care. It finds the first '<' then the last '>'. + + // Make a clone of the iterator so we can look forward to the end of + // the line to see if this is a valid tag + let mut local_iter = self.iter.clone(); + + let mut right_bracket_location: Option = None; + let mut new_line_index: Option = None; + + // Search until we find the end of the line. Mark the location of the + // last '>'. + while let Some(((i, c), (_ii, _cc))) = local_iter.find(|&((_i, c), (_ii, _cc))| { + c == '\n' // New line + || c == '>' // Right angle bracket + }) { + match c { + '>' => right_bracket_location = Some(i), + '\n' => { + new_line_index = Some(i); + break; + } + _ => {} + } + } + + if let Some(right_bracket_location) = right_bracket_location { + let remaining = if let Some(i) = new_line_index { + // Up to, but not including the new line + &self.input[index + 1..i] + } else { + // Until the end of the file + &self.input[index + 1..] + } + .trim(); + + // Check that the right bracket is the last character in the trimmed + // string. + if remaining.len() < 2 && !remaining.ends_with(">") { + return false; + } + + // Check that there isn't any whitespace in the remaining + if let Some(_i) = remaining.find(char::is_whitespace) { + return false; + } + + // We have found a tag. + // Push unhandled before the tag + self.trim_end = true; + if let Some((prev_i, unhandled)) = self.get_unhandled_string(index, true) { + if !unhandled.is_empty() { + self.scan_value(unhandled, prev_i); + self.start_of_line = false; + self.trim_start = false; + } + } + + // Push the left angle bracket + self.push_token(index + 1, Token::LeftAngleBracket, index + 2); + + // Push the tag as a value string + self.push_token( + index + 2, + Token::ValueString(&remaining[1..remaining.len() - 1]), + right_bracket_location, + ); + + // Push the right angle bracket + self.push_token( + right_bracket_location, + Token::RightAngleBracket, + right_bracket_location + 1, + ); + + // If we found a new line, EOL + if let Some(i) = new_line_index { + self._handle_new_line(i); + } else { + self.previous_index = None; + } + + // Update our iterator to our local copy + self.iter = local_iter; + + return true; + } else { + return false; + } + } + fn tokenize_new_line(&mut self, i: usize, drop_unhandled: bool) { // Trim up to the new line self.trim_end = true; diff --git a/moos-parser/src/nsplug/nsplug.lalrpop b/moos-parser/src/nsplug/nsplug.lalrpop index f7a7a25..d52aaac 100644 --- a/moos-parser/src/nsplug/nsplug.lalrpop +++ b/moos-parser/src/nsplug/nsplug.lalrpop @@ -46,12 +46,6 @@ pub Line: Line<'input> = { }, } -// TODO: -// 1. Add MacroIfDef -// 2. Add MacroElseIfDef -// 3. Add MacroIfNotDef - - MacroDefine: Line<'input> = { "EOL" => Line::Macro{ @@ -71,7 +65,7 @@ MacroDefinition: MacroDefinition<'input> = { } MacroInclude: Line<'input> = { - "EOL" => { + "EOL" => { if let Some(comment) = comment { let e = lalrpop_util::ErrorRecovery { error: lalrpop_util::ParseError::User { @@ -82,7 +76,7 @@ MacroInclude: Line<'input> = { state.errors.push(e); } Line::Macro{ - macro_type: MacroType::Include{path, range: TokenRange::new_line(ml,mr).unwrap()}, + macro_type: MacroType::Include{path, tag, range: TokenRange::new_line(ml,mr).unwrap()}, comment, line: ml.line, } @@ -147,6 +141,10 @@ IncludePath: tree::IncludePath<'input> = { Quote => tree::IncludePath::Quote(<>), } +IncludeTag: tree::IncludeTag<'input> = { + "<" ">" => tree::IncludeTag{tag, range: TokenRange::new_line(l,r).expect("Invalid token range while parsing `IncludeTag`")}, +} + IfDef: Line<'input> = { "#ifdef" "EOL" => Line::Macro{ @@ -201,8 +199,6 @@ MacroCondition: tree::MacroCondition<'input> = { }, } -// TODO: Need to change IfNotDef to use clauses not conditions - IfNotDef: Line<'input> = { "#ifndef" " "* "EOL" => Line::Macro{ @@ -273,6 +269,8 @@ extern { "#unknown" => Token::UnknownMacro(<&'input str>), "||" => Token::OrOperator, "&&" => Token::AndOperator, + "<" => Token::LeftAngleBracket, + ">" => Token::RightAngleBracket, " " => Token::Space, } } diff --git a/moos-parser/src/nsplug/tree.rs b/moos-parser/src/nsplug/tree.rs index 4f5af01..838b8cb 100644 --- a/moos-parser/src/nsplug/tree.rs +++ b/moos-parser/src/nsplug/tree.rs @@ -178,6 +178,25 @@ impl<'input> From> for IncludePath<'input> { } } +#[derive(Debug, Clone, Copy)] +pub struct IncludeTag<'input> { + pub tag: &'input str, + /// Range of the Include tag. THis includes the start and ending brackets + pub range: TokenRange, +} + +impl<'input> IncludeTag<'input> { + pub fn new(tag: &'input str, range: TokenRange) -> Self { + Self { tag, range } + } +} + +impl<'input> ToString for IncludeTag<'input> { + fn to_string(&self) -> String { + format!("<{}>", self.tag) + } +} + #[derive(Debug)] pub enum MacroType<'input> { Define { @@ -187,6 +206,8 @@ pub enum MacroType<'input> { }, Include { path: IncludePath<'input>, + /// Optional include tag. Added in 2020. + tag: Option>, /// Range of the "#include" range: TokenRange, }, @@ -212,8 +233,12 @@ impl<'input> ToString for MacroType<'input> { MacroType::Define { definition, range } => { format!("#define {}", definition.to_string()) } - MacroType::Include { path, range } => { - format!("#include {}", path.to_string()) + MacroType::Include { path, tag, range } => { + if let Some(tag) = tag { + format!("#include {} {}", path.to_string(), tag.to_string()) + } else { + format!("#include {}", path.to_string()) + } } MacroType::IfDef { condition,