From df2d872bfa126c7f2a6e95c5b2c86515d33b0704 Mon Sep 17 00:00:00 2001 From: Christopher Gagner Date: Sat, 30 Mar 2024 13:24:11 -0400 Subject: [PATCH] Implemented #ifndef for nsplug parser. --- .../src/parsers/nsplug.rs | 69 ++++++++++----- moos-parser/src/helpers.rs | 7 ++ moos-parser/src/nsplug/error.rs | 10 ++- moos-parser/src/nsplug/lexer.rs | 47 ++++++++-- moos-parser/src/nsplug/nsplug.lalrpop | 85 +++++++++++++++++-- moos-parser/src/nsplug/tree.rs | 65 ++++++++++---- 6 files changed, 233 insertions(+), 50 deletions(-) diff --git a/moos-ivp-language-server/src/parsers/nsplug.rs b/moos-ivp-language-server/src/parsers/nsplug.rs index d8feb94..e2e7a61 100644 --- a/moos-ivp-language-server/src/parsers/nsplug.rs +++ b/moos-ivp-language-server/src/parsers/nsplug.rs @@ -7,8 +7,8 @@ use moos_parser::{ error::{PlugParseError, PlugParseErrorKind}, lexer::{State, Token}, tree::{ - IfDefBranch, IncludePath, Line, Lines, MacroCondition, MacroDefinition, MacroType, - Quote, Values, Variable, VariableStrings, + IfDefBranch, IfNotDefBranch, IncludePath, Line, Lines, MacroCondition, MacroDefinition, + MacroType, Quote, Values, Variable, VariableStrings, }, }, ParseError, PlugParser, @@ -59,7 +59,7 @@ pub fn parse(document: &mut Document) { } let iter = document.diagnostics.iter(); - iter.for_each(|e| { + state.errors.iter().for_each(|e| { error!("Parse Error: {e:?}"); }); // TODO: Add new method to handle converting errors into diagnostics @@ -101,6 +101,14 @@ pub fn parse(document: &mut Document) { ); document.diagnostics.push(d); } + PlugParseErrorKind::MissingEndIf => { + let d = new_error_diagnostic( + &error.loc_start, + &error.loc_end, + format!("Missing #endif"), + ); + document.diagnostics.push(d); + } }, ParseError::UnrecognizedToken { token, expected } => { let (loc_start, token, loc_end) = token; @@ -167,21 +175,24 @@ fn handle_lines(document: &mut Document, lines: &Lines) { handle_lines(document, body); handle_ifdef_branch(document, line, branch); } - MacroType::IfNotDef { range } => { - // - handle_macro_token(document, line, &range); - } - MacroType::ElseIfDef { range } => { - // - handle_macro_token(document, line, &range); - } - MacroType::Else { range } => { - // - handle_macro_token(document, line, &range); - } - MacroType::EndIf { range } => { - // + MacroType::IfNotDef { + clauses, + branch, + body, + range, + } => { handle_macro_token(document, line, &range); + for clause in clauses { + handle_variable_strings( + document, + line, + clause, + TokenTypes::Variable, + 0, + ); + } + handle_lines(document, body); + handle_ifndef_branch(document, line, branch); } } } @@ -200,7 +211,7 @@ fn handle_macro_token(document: &mut Document, line: u32, range: &TokenRange) { line, range.clone(), SemanticTokenInfo { - token_type: TokenTypes::Macro as u32, + token_type: TokenTypes::Keyword as u32, token_modifiers: 0, }, ); @@ -306,7 +317,7 @@ fn handle_values(document: &mut Document, line: u32, values: &Values) { line, range.clone(), SemanticTokenInfo { - token_type: TokenTypes::Keyword as u32, + token_type: TokenTypes::Macro as u32, // TODO: Should this be a type? token_modifiers: 0, }, ); @@ -405,6 +416,26 @@ fn handle_ifdef_branch(document: &mut Document, _parent_line: u32, input_branch: } } +fn handle_ifndef_branch(document: &mut Document, _parent_line: u32, input_branch: &IfNotDefBranch) { + // TODO: Add folding ranges + match input_branch { + IfNotDefBranch::Else { + line, + macro_range, + body, + endif_line, + endif_macro_range, + } => { + handle_macro_token(document, *line, ¯o_range); + handle_lines(document, body); + handle_macro_token(document, *endif_line, &endif_macro_range); + } + IfNotDefBranch::EndIf { line, macro_range } => { + handle_macro_token(document, *line, ¯o_range); + } + } +} + /* TODO: These are reminders of tokens that we should be handling. diff --git a/moos-parser/src/helpers.rs b/moos-parser/src/helpers.rs index 3ef797c..1c8b25a 100644 --- a/moos-parser/src/helpers.rs +++ b/moos-parser/src/helpers.rs @@ -120,6 +120,13 @@ macro_rules! vec_wrapper { } } + impl<'lt> From<$type<'lt>> for $name<'lt> { + fn from(value: $type<'lt>) -> Self { + let values: Vec<$type<'lt>> = vec![value]; + Self(values.into()) + } + } + impl<'lt> ToString for $name<'lt> { fn to_string(&self) -> String { let rtn = "".to_owned(); diff --git a/moos-parser/src/nsplug/error.rs b/moos-parser/src/nsplug/error.rs index 12a6adf..306d2c0 100644 --- a/moos-parser/src/nsplug/error.rs +++ b/moos-parser/src/nsplug/error.rs @@ -46,10 +46,17 @@ impl<'input> PlugParseError<'input> { loc_end, } } + pub fn new_missing_endif(loc_start: Location, loc_end: Location) -> PlugParseError<'input> { + PlugParseError { + kind: PlugParseErrorKind::MissingEndIf, + loc_start, + loc_end, + } + } pub fn new_missing_new_line(loc_start: Location, loc_end: Location) -> PlugParseError<'input> { PlugParseError { kind: PlugParseErrorKind::MissingNewLine, - loc_start: loc_start, + loc_start, loc_end, } } @@ -70,6 +77,7 @@ impl<'input> PlugParseError<'input> { #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub enum PlugParseErrorKind<'input> { + MissingEndIf, MissingTrailing(char), MissingNewLine, UnexpectedComment(&'input str), diff --git a/moos-parser/src/nsplug/lexer.rs b/moos-parser/src/nsplug/lexer.rs index 4364a98..dddde64 100644 --- a/moos-parser/src/nsplug/lexer.rs +++ b/moos-parser/src/nsplug/lexer.rs @@ -265,7 +265,7 @@ impl<'input> Lexer<'input> { } fn tokenize_or_operator(&mut self, i: usize) { - self.iter.next(); + let mut tokens = self.iter.next(); if let Some((prev_i, unhandled)) = self.get_unhandled_string(i, true) { if !unhandled.is_empty() { self.scan_value(unhandled, prev_i); @@ -274,10 +274,19 @@ impl<'input> Lexer<'input> { } self.push_token(i, Token::OrOperator, i + 2); + + // Consume tokens until the next token is a non-white space or we reach + // the end of the file + while let Some(((_current_i, _current_c), (_next_i, next_c))) = tokens { + match next_c { + ' ' | '\t' => tokens = self.iter.next(), + _ => break, + } + } } fn tokenize_and_operator(&mut self, i: usize) { - self.iter.next(); + let mut tokens = self.iter.next(); if let Some((prev_i, unhandled)) = self.get_unhandled_string(i, true) { if !unhandled.is_empty() { self.scan_value(unhandled, prev_i); @@ -286,6 +295,15 @@ impl<'input> Lexer<'input> { } self.push_token(i, Token::AndOperator, i + 2); + + // Consume tokens until the next token is a non-white space or we reach + // the end of the file + while let Some(((_current_i, _current_c), (_next_i, next_c))) = tokens { + match next_c { + ' ' | '\t' => tokens = self.iter.next(), + _ => break, + } + } } fn get_macro_token(line: &'input str) -> Token<'input> { @@ -354,13 +372,28 @@ impl<'input> Lexer<'input> { return; }; + let mut is_ifndef = false; + let has_conditions = match token { Token::MacroIfDef | Token::MacroElseIfDef => true, + // #ifndef doesn't really support conditions, but we will handle + // that in the parser. For now, enable the tokenization of the + // && and || operators so we can throw an in the parser. + Token::MacroIfNotDef => { + is_ifndef = true; + true + } + _ => false, + }; + + let has_comments = match token { + Token::MacroElse | Token::MacroEndIf => true, _ => false, }; let mut has_whitespace = match token { Token::MacroDefine | Token::MacroIfDef | Token::MacroElseIfDef => true, + Token::MacroIfNotDef => true, _ => false, }; @@ -369,7 +402,7 @@ impl<'input> Lexer<'input> { || c == '"' || (c == '=') || (has_whitespace && (c == ' ' || c == '\t')) // Whitespace - || (c == '/' && cc == '/') // Comment + || (has_comments && (c == '/' && cc == '/')) // Comment || (c == '$' && cc == '(') // Plug variable || (c == '%' && cc == '(') // Plug Upper Variable || (has_conditions && c == '|' && cc == '|') // Or operator @@ -431,12 +464,16 @@ impl<'input> Lexer<'input> { self.found_assign_op = false; } ' ' | '\t' => { - self.found_assign_op = true; // Enables parsing primitives + if !is_ifndef { + self.found_assign_op = true; // Enables parsing primitives + } self.trim_end = true; if let Some((prev_i, unhandled)) = self.get_unhandled_string(i, true) { if !unhandled.is_empty() { self.scan_value(unhandled, prev_i); - has_whitespace = false; + if !is_ifndef { + has_whitespace = false; + } self.push_token(i, Token::Space, i + 1); } self.previous_index = self.get_safe_index(i + 1); diff --git a/moos-parser/src/nsplug/nsplug.lalrpop b/moos-parser/src/nsplug/nsplug.lalrpop index d1fc13c..f7a7a25 100644 --- a/moos-parser/src/nsplug/nsplug.lalrpop +++ b/moos-parser/src/nsplug/nsplug.lalrpop @@ -14,13 +14,36 @@ pub Lines: tree::Lines<'input> = { pub Line: Line<'input> = { IfDef => <>, + IfNotDef => <>, MacroDefine => <>, MacroInclude => <>, "EOL" => Line::EndOfLine, // => Line::Comment(comment), UnknownMacro => <>, => Line::Variable{variable, line: l.line}, - ! => { state.errors.push(<>); Line::Error }, + => { + match &error.error { + lalrpop_util::ParseError::UnrecognizedEOF { location, expected } => { + if expected.contains(&"\"#endif\"".to_string()) { + let endl = if l.line < r.line { + Location::new(l.line + 1, 0) + } else { + r + }; + let new_error = lalrpop_util::ErrorRecovery { + error: lalrpop_util::ParseError::User { + error: PlugParseError::new_missing_endif(l, endl) + }, + dropped_tokens: vec![] + }; + state.errors.push(new_error); + } + } + _ => {} + } + state.errors.push(error); + Line::Error(l.line, r.line) + }, } // TODO: @@ -39,11 +62,11 @@ MacroDefine: Line<'input> = { } MacroDefinition: MacroDefinition<'input> = { - " "+ => { - MacroDefinition::new(name.into(), value.into()) + " "+ => { + MacroDefinition::new(name, value.into()) }, - " "* => { - MacroDefinition::new(name.into(), Vec::new().into()) + " "* => { + MacroDefinition::new(name, Vec::new().into()) } } @@ -69,7 +92,7 @@ MacroInclude: Line<'input> = { } UnknownMacro: Line<'input> = { - => { + => { let e = lalrpop_util::ErrorRecovery { error: lalrpop_util::ParseError::User { error: PlugParseError::new_unknown_macro(l, m) @@ -77,7 +100,7 @@ UnknownMacro: Line<'input> = { dropped_tokens: vec![] }; state.errors.push(e); - Line::Error + Line::Error(l.line, r.line) } } @@ -111,12 +134,16 @@ VariableString: tree::VariableString<'input> = { Variable => tree::VariableString::Variable(<>), } +VariableStrings: tree::VariableStrings<'input> = { + => variables.into() +} + Quote: tree::Quote<'input> = { QuoteBegin QuoteEnd => tree::Quote{content: content.into(), range: TokenRange::new_line(l,r).expect("Invalid token range while parsing `Quote`")} } IncludePath: tree::IncludePath<'input> = { - => tree::IncludePath::VariableStrings(content.into(), TokenRange::new_line(l,r).expect("Invalid token range while parsing `IncludePath`")), + => tree::IncludePath::VariableStrings(content, TokenRange::new_line(l,r).expect("Invalid token range while parsing `IncludePath`")), Quote => tree::IncludePath::Quote(<>), } @@ -125,7 +152,7 @@ IfDef: Line<'input> = { Line::Macro{ macro_type: MacroType::IfDef{ condition, - branch: branch, + branch, body, range: TokenRange::new_line(ml,mr).unwrap(), }, @@ -174,6 +201,46 @@ MacroCondition: tree::MacroCondition<'input> = { }, } +// TODO: Need to change IfNotDef to use clauses not conditions + +IfNotDef: Line<'input> = { + "#ifndef" " "* "EOL" => + Line::Macro{ + macro_type: MacroType::IfNotDef{ + clauses, + branch, + body, + range: TokenRange::new_line(ml,mr).unwrap(), + }, + comment: None, + line: ml.line, + }, +} + +IfNotDefBranch: tree::IfNotDefBranch<'input> = { + "#else" "EOL" "#endif" => + tree::IfNotDefBranch::Else { + line: ml.line, + macro_range: TokenRange::new_line(ml,mr).unwrap(), + body, + endif_line: endl.line, + endif_macro_range: TokenRange::new_line(endl,endr).unwrap(), + }, + "#endif" "EOL" => + tree::IfNotDefBranch::EndIf { + line: ml.line, + macro_range: TokenRange::new_line(ml,mr).unwrap(), + }, +} + +IfNotDefClauses: tree::IfNotDefClauses<'input> = { + => clause.into(), + " "+ => { + clauses.push(clause.into()); + clauses + }, +} + // --------------------------------------------------------------------------- // Token Definitions diff --git a/moos-parser/src/nsplug/tree.rs b/moos-parser/src/nsplug/tree.rs index 146121a..4f5af01 100644 --- a/moos-parser/src/nsplug/tree.rs +++ b/moos-parser/src/nsplug/tree.rs @@ -198,21 +198,12 @@ pub enum MacroType<'input> { range: TokenRange, }, IfNotDef { + clauses: IfNotDefClauses<'input>, + branch: IfNotDefBranch<'input>, + body: Lines<'input>, /// Range of the "#ifndef" range: TokenRange, }, - ElseIfDef { - /// Range of the "#elseifdef" - range: TokenRange, - }, - Else { - /// Range of the "#else" - range: TokenRange, - }, - EndIf { - /// Range of the "#endif" - range: TokenRange, - }, } impl<'input> ToString for MacroType<'input> { @@ -233,7 +224,17 @@ impl<'input> ToString for MacroType<'input> { // TODO: Need to recursively print the branch and lines format!("#ifdef {}", condition.to_string()) } - _ => "".to_string(), + MacroType::IfNotDef { + clauses, + branch, + body, + range, + } => { + let mut rtn = "#ifndef ".to_string(); + clauses + .iter() + .fold(rtn, |acc, v| acc + " " + v.to_string().as_str()) + } } } } @@ -339,6 +340,38 @@ impl<'input> ToString for IfDefBranch<'input> { } } +vec_wrapper!(IfNotDefClauses, VariableStrings); + +#[derive(Debug)] +pub enum IfNotDefBranch<'input> { + Else { + line: u32, + macro_range: TokenRange, + body: Lines<'input>, + endif_line: u32, + endif_macro_range: TokenRange, + }, + EndIf { + line: u32, + macro_range: TokenRange, + }, +} + +impl<'input> ToString for IfNotDefBranch<'input> { + fn to_string(&self) -> String { + match self { + IfNotDefBranch::Else { + line, + macro_range, + body, + endif_line, + endif_macro_range, + } => "#else".to_string(), + IfNotDefBranch::EndIf { line, macro_range } => "#endif".to_string(), + } + } +} + #[derive(Debug)] pub enum Line<'input> { /// NOTE: Comments are not really supported by NSPlug. We have them here @@ -356,7 +389,7 @@ pub enum Line<'input> { variable: Variable<'input>, line: u32, }, - Error, + Error(u32, u32), EndOfLine, } @@ -377,8 +410,8 @@ impl<'input> ToString for Line<'input> { macro_type.to_string() } } - Line::Variable { variable, line } => todo!(), - Line::Error => "".to_string(), + Line::Variable { variable, line } => variable.to_string(), + Line::Error(_, _) => "".to_string(), Line::EndOfLine => "".to_string(), } }