Skip to content

Commit

Permalink
Fixed parsers handled EOF.
Browse files Browse the repository at this point in the history
  • Loading branch information
@cgagner
cgagner committed Sep 23, 2024
1 parent 6112cbb commit 9854c6c
Show file tree
Hide file tree
Showing 6 changed files with 80 additions and 54 deletions.
21 changes: 13 additions & 8 deletions moos-parser/src/behavior/behavior.lalrpop
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,9 @@ pub Lines: tree::Lines = {
}

pub Line: Line = {
<l:@L> "EOL" => Line::EndOfLine{line: l.line, index: l.index},
<l:@L> <comment:Comment> <r:@R> "EOL" => Line::Comment{comment, line: l.line},
<l:@L> <assignment:Assignment> <r:@R> "EOL" => {
<l:@L> END => Line::EndOfLine{line: l.line, index: l.index},
<l:@L> <comment:Comment> <r:@R> END => Line::Comment{comment, line: l.line},
<l:@L> <assignment:Assignment> <r:@R> END => {
let e = lalrpop_util::ErrorRecovery {
error: lalrpop_util::ParseError::User {
error: BehaviorParseError::new_unexpected_assignment(l, r)
Expand All @@ -35,7 +35,7 @@ pub Line: Line = {
}
},
SetBlock => <>,
<l:@L> <variable:Variable> "EOL" => Line::Variable{variable, line: l.line},
<l:@L> <variable:Variable> END => Line::Variable{variable, line: l.line},
<l:@L> <error:!> <r:@R> => {
state.errors.push(error);
Line::Error{start_line: l.line, end_line: r.line}
Expand Down Expand Up @@ -108,8 +108,8 @@ Assignments: tree::Assignments = {
}

Initialize: Line = {
<l:@L> "initialize" <r:@R> <assignments: Assignments> "EOL" => Line::Initialize{assignments, deferred: false, line: l.line, range: TokenRange::new_line(l,r).expect("Invalid token range while parsing `initialize`")},
<l:@L> "initialize_" <r:@R> <assignments: Assignments> "EOL" => Line::Initialize{assignments, deferred: true, line: l.line, range: TokenRange::new_line(l,r).expect("Invalid token range while parsing `initialize_`")},
<l:@L> "initialize" <r:@R> <assignments: Assignments> END => Line::Initialize{assignments, deferred: false, line: l.line, range: TokenRange::new_line(l,r).expect("Invalid token range while parsing `initialize`")},
<l:@L> "initialize_" <r:@R> <assignments: Assignments> END => Line::Initialize{assignments, deferred: true, line: l.line, range: TokenRange::new_line(l,r).expect("Invalid token range while parsing `initialize_`")},
}

CommentOrEmptyLine: Line = {
Expand Down Expand Up @@ -154,7 +154,7 @@ BehaviorBlock: Line = {
Block: (tree::UnknownBlock, Location, Location) = {
<ocl:@L> "{" <ocr:@R> <open_curly_comment: Comment?> "EOL"
<body: BehaviorBlockLine*>
<ccl:@L> "}" <close_curly_comment: Comment?> "EOL"
<ccl:@L> "}" <close_curly_comment: Comment?> END
=> (
tree::UnknownBlock {
open_curly_line: ocl.line,
Expand Down Expand Up @@ -185,7 +185,7 @@ SetBlock: Line = {
<set_block_comment: Comment?> "EOL"?
<ocl:@L> "{" <open_curly_comment: Comment?> "EOL"
<body: SetBlockLine*>
<ccl:@L> "}" <else_value: VariableStrings?><close_curly_comment: Comment?> "EOL"
<ccl:@L> "}" <else_value: VariableStrings?><close_curly_comment: Comment?> END
=> Line::SetBlock {
set_block: tree::SetBlock{
set_block_comment: None,
Expand All @@ -206,6 +206,10 @@ SetBlock: Line = {
},
}

END: () = {
"EOL" => {},
"EOF" => {},
}

// ---------------------------------------------------------------------------
// Token Definitions
Expand All @@ -221,6 +225,7 @@ extern {
QuoteEnd => Token::QuoteEnd,
"ValueString" => Token::ValueString(<&'input str>),
"EOL" => Token::EOL,
"EOF" => Token::EOF,
"EnvVariable" => Token::EnvVariable(<&'input str>),
"PartialEnvVariable" => Token::PartialEnvVariable(<&'input str>),
"int" => Token::Integer(<i64>, <&'input str>),
Expand Down
29 changes: 15 additions & 14 deletions moos-parser/src/behavior/lexer.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@ pub struct Lexer<'input> {
trim_start: bool,
trim_end: bool,
token_queue: TokenQueue<'input>,
reached_eof: bool,
}

impl<'input> Lexer<'input> {
Expand Down Expand Up @@ -92,6 +93,7 @@ impl<'input> Lexer<'input> {
trim_start: true,
trim_end: false,
token_queue: TokenQueue::new(),
reached_eof: false,
}
}

Expand Down Expand Up @@ -556,8 +558,6 @@ impl<'input> Lexer<'input> {
// 9. # Macro => Skip entire line
//
// Ignore other tokens

let mut found_new_line = false;
while let Some(((i, c), (_ii, cc))) = self.iter.find(|&((_i, c), (_ii, cc))| {
c == '\n'
|| (c == '/' && cc == '/') // Comment
Expand All @@ -572,9 +572,7 @@ impl<'input> Lexer<'input> {
match c {
'\n' => {
self.tokenize_new_line(i, false);
found_new_line = true;
// Break out of the tokenize for-loop after each line
break;
return;
}
'/' => self.tokenize_comment(i),
c if (c == '$' && cc == '{') => {
Expand All @@ -600,16 +598,19 @@ impl<'input> Lexer<'input> {
}
}

if !found_new_line {
// Should only get in here if we have reached the end of the input.
// If so, check that there isn't some straggling unhandled string.
self.trim_end = true;
if let Some((prev_i, unhandled)) = self.get_unhandled_string(self.input.len(), true) {
if !unhandled.is_empty() {
self.scan_keywords_and_values(unhandled, prev_i);
}
self.previous_index = self.get_safe_index(self.input.len());
// Should only get in here if we have reached the end of the input.
// If so, check that there isn't some straggling unhandled string.
self.trim_end = true;
if let Some((prev_i, unhandled)) = self.get_unhandled_string(self.input.len(), true) {
if !unhandled.is_empty() {
self.scan_keywords_and_values(unhandled, prev_i);
}
self.previous_index = self.get_safe_index(self.input.len());
}

if !self.reached_eof {
self.push_token(self.input.len(), Token::EOF, self.input.len());
self.reached_eof = true;
}
}
}
Expand Down
28 changes: 15 additions & 13 deletions moos-parser/src/moos/lexer.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ pub struct Lexer<'input> {
trim_start: bool,
trim_end: bool,
token_queue: TokenQueue<'input>,
reached_eof: bool,
}

impl<'input> Lexer<'input> {
Expand All @@ -83,6 +84,7 @@ impl<'input> Lexer<'input> {
trim_start: true,
trim_end: false,
token_queue: TokenQueue::new(),
reached_eof: false,
}
}

Expand Down Expand Up @@ -521,7 +523,6 @@ impl<'input> Lexer<'input> {
//
// Ignore other tokens

let mut found_new_line = false;
while let Some(((i, c), (_ii, cc))) = self.iter.find(|&((_i, c), (_ii, cc))| {
c == '\n'
|| (c == '/' && cc == '/') // Comment
Expand All @@ -534,9 +535,7 @@ impl<'input> Lexer<'input> {
match c {
'\n' => {
self.tokenize_new_line(i, false);
found_new_line = true;
// Break out of the tokenize for-loop after each line
break;
return;
}
'/' => self.tokenize_comment(i),
c if (c == '$' && cc == '{') => {
Expand All @@ -561,16 +560,19 @@ impl<'input> Lexer<'input> {
}
}

if !found_new_line {
// Should only get in here if we have reached the end of the input.
// If so, check that there isn't some straggling unhandled string.
self.trim_end = true;
if let Some((prev_i, unhandled)) = self.get_unhandled_string(self.input.len(), true) {
if !unhandled.is_empty() {
self.scan_keywords_and_values(unhandled, prev_i);
}
self.previous_index = self.get_safe_index(self.input.len());
// Should only get in here if we have reached the end of the input.
// If so, check that there isn't some straggling unhandled string.
self.trim_end = true;
if let Some((prev_i, unhandled)) = self.get_unhandled_string(self.input.len(), true) {
if !unhandled.is_empty() {
self.scan_keywords_and_values(unhandled, prev_i);
}
self.previous_index = self.get_safe_index(self.input.len());
}

if !self.reached_eof {
self.push_token(self.input.len(), Token::EOF, self.input.len());
self.reached_eof = true;
}
}
}
Expand Down
15 changes: 10 additions & 5 deletions moos-parser/src/moos/moos.lalrpop
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,12 @@ pub Lines: tree::Lines = {
}

pub Line: Line = {
<l:@L> "EOL" => Line::EndOfLine{line: l.line, index: l.index},
<l:@L> <comment:Comment> <r:@R> "EOL" => Line::Comment{comment, line: l.line},
<l:@L> END => Line::EndOfLine{line: l.line, index: l.index},
<l:@L> <comment:Comment> <r:@R> END => Line::Comment{comment, line: l.line},
<l:@L> <assignment:Assignment> => Line::Assignment{assignment, line: l.line},
Define => <>,
ProcessConfig => <>,
<l:@L> <variable:Variable> "EOL" => Line::Variable{variable, line: l.line},
<l:@L> <variable:Variable> END => Line::Variable{variable, line: l.line},
<l:@L> <error:!> <r:@R> => {
state.errors.push(error);
Line::Error{start_line: l.line, end_line: r.line}
Expand Down Expand Up @@ -66,7 +66,7 @@ Comment: tree::Comment = {
}

Assignment: tree::Assignment = {
<name:VariableStrings> "=" <value:Value*> <comment: Comment?> "EOL" => tree::Assignment{name, value: value.into(), comment},
<name:VariableStrings> "=" <value:Value*> <comment: Comment?> END => tree::Assignment{name, value: value.into(), comment},
}

Define: Line = {
Expand Down Expand Up @@ -95,7 +95,7 @@ ProcessConfig: Line = {
<prelude_comments: CommentOrEmptyLine*>
<ocl:@L> "{" <open_curly_comment: Comment?> "EOL"
<body: ProcessConfigLine*>
<ccl:@L> "}" <close_curly_comment: Comment?> "EOL"
<ccl:@L> "}" <close_curly_comment: Comment?> END
=> Line::ProcessConfig {
process_config: tree::ProcessConfig{
process_config_comment,
Expand All @@ -114,6 +114,10 @@ ProcessConfig: Line = {
},
}

END: () = {
"EOL" => {},
"EOF" => {},
}

// ---------------------------------------------------------------------------
// Token Definitions
Expand All @@ -129,6 +133,7 @@ extern {
QuoteEnd => Token::QuoteEnd,
"ValueString" => Token::ValueString(<&'input str>),
"EOL" => Token::EOL,
"EOF" => Token::EOF,
"EnvVariable" => Token::EnvVariable(<&'input str>),
"PartialEnvVariable" => Token::PartialEnvVariable(<&'input str>),
"int" => Token::Integer(<i64>, <&'input str>),
Expand Down
14 changes: 11 additions & 3 deletions moos-parser/src/nsplug/lexer.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ pub struct Lexer<'input> {
char_count: usize,
start_of_line: bool,
token_queue: TokenQueue<'input>,
reached_eof: bool,
}

impl<'input> Lexer<'input> {
Expand All @@ -82,6 +83,7 @@ impl<'input> Lexer<'input> {
char_count: 0,
start_of_line: true,
token_queue: TokenQueue::new(),
reached_eof: false,
}
}

Expand Down Expand Up @@ -612,8 +614,7 @@ impl<'input> Lexer<'input> {
match c {
'\n' => {
self.tokenize_new_line(i, true);
// Break out of the tokenize for-loop after each line
break;
return;
}
c if (c == '$' && cc == '(') => {
// drop the unhandled tokens before this because we are not
Expand All @@ -639,13 +640,20 @@ impl<'input> Lexer<'input> {
}
});
}
'#' => self.tokenize_macro(i),
'#' => {
self.tokenize_macro(i);
return;
}
_ => {}
}
}

// NOTE: There could still be tokens to be parse, but we don't care
// about them.
if !self.reached_eof {
self.push_token(self.input.len(), Token::EOF, self.input.len());
self.reached_eof = true;
}
}
}

Expand Down
Loading

0 comments on commit 9854c6c

Please sign in to comment.