feat: add bullet list element parsing and rendering (#111)

* arch: add bullet-list element structure

* feat: add bullet list parsing and html rendering

* fix: prevent list entries from being taken as body

* fix: correct list entry body parsing

* fix: parse directly nested bullet lists

* fix: add test for different list entry keywords

* feat: indent blocks for snapshot tests

* fix: adapt SymbolIterator peek to work like next

peek now also considers matching functions.

* fix: remove checks needed with old peek behavior

* fix: add documentation for the bullet list element

* fix: add deeply-nested bullet list test

* fix: apply cargo fmt after rustup update

* fix: improve doc for symbol iterator state fields

* arch: rename kind to parent in symbol iterator

* arch: switch condition order for better readability

Co-authored-by: Nadir Fejzić <[email protected]>

* fix: fix clippy warnings

---------

Co-authored-by: Nadir Fejzić <[email protected]>

commons/src/scanner/mod.rs

@@ -21,7 +21,7 @@ pub fn scan_str(input: &str) -> Vec<Symbol<'_>> {
     // skip(1) to ignore break at start of input
     for offset in segmenter.segment_str(input).skip(1) {
         if let Some(grapheme) = input.get(prev_offset..offset) {
-            let mut kind = SymbolKind::from(grapheme);
+            let kind = SymbolKind::from(grapheme);
 
             let end_pos = if kind == SymbolKind::Newline {
                 SymPos {
@@ -37,11 +37,6 @@ pub fn scan_str(input: &str) -> Vec<Symbol<'_>> {
                 }
             };
 
-            if curr_pos.col_utf8 == 1 && kind == SymbolKind::Newline {
-                // newline at the start of line -> Blankline
-                kind = SymbolKind::Blankline;
-            }
-
             symbols.push(Symbol {
                 input,
                 kind,

commons/src/scanner/symbol/iterator/matcher.rs

@@ -26,7 +26,8 @@ pub trait EndMatcher {
     fn is_empty_line(&mut self) -> bool;
 
     /// Wrapper around [`Self::is_empty_line()`] that additionally consumes the matched empty line.
-    /// Consuming means the related iterator advances over the matched empty line.
+    /// Consuming means the related iterator advances over the matched empty line, but not the end newline.
+    /// Not consuming the end newline allows to consume contiguous empty lines.
     ///
     /// **Note:** The iterator is only advanced if an empty line is matched.
     ///
@@ -65,69 +66,86 @@ pub trait PrefixMatcher {
     ///
     /// [`Symbol`]: super::Symbol
     fn consumed_prefix(&mut self, sequence: &[SymbolKind]) -> bool;
+
+    /// Returns `true` if the upcoming [`Symbol`] sequence is an empty line.
+    /// Meaning that a line contains no [`Symbol`] or only [`SymbolKind::Whitespace`].
+    ///
+    /// **Note:** This is also `true` if a parent iterator stripped non-whitespace symbols, and the nested iterator only has whitespace symbols.
+    ///
+    /// [`Symbol`]: super::Symbol
+    fn empty_line(&mut self) -> bool;
 }
 
 impl<'input> EndMatcher for SymbolIterator<'input> {
     fn is_empty_line(&mut self) -> bool {
-        // Note: Multiple matches may be set in the match closure, so we need to ensure that all start at the same index
-        self.reset_peek();
+        let peek_index = self.peek_index();
 
         let next = self
-            .peeking_next(|s| {
-                matches!(
-                    s.kind,
-                    SymbolKind::Newline | SymbolKind::Blankline | SymbolKind::EOI
-                )
-            })
+            .peeking_next(|s| matches!(s.kind, SymbolKind::Newline | SymbolKind::EOI))
             .map(|s| s.kind);
 
         let is_empty_line = if Some(SymbolKind::Newline) == next {
             let _whitespaces = self
                 .peeking_take_while(|s| s.kind == SymbolKind::Whitespace)
                 .count();
 
-            let new_line = self.peeking_next(|s| {
-                matches!(
-                    s.kind,
-                    SymbolKind::Newline | SymbolKind::Blankline | SymbolKind::EOI
-                )
-            });
+            let new_line =
+                self.peeking_next(|s| matches!(s.kind, SymbolKind::Newline | SymbolKind::EOI));
+
+            if Some(SymbolKind::Newline) == new_line.map(|s| s.kind) {
+                self.set_peek_index(self.peek_index() - 1); // Do not consume next newline to enable consumption of contiguous empty lines
+            }
+
             new_line.is_some()
         } else {
             next.is_some()
         };
 
+        self.set_match_index(self.peek_index());
+        self.set_peek_index(peek_index);
+
         is_empty_line
     }
 
     fn consumed_is_empty_line(&mut self) -> bool {
         let is_empty_line = self.is_empty_line();
 
         if is_empty_line {
-            self.set_index(self.peek_index()); // To consume peeked symbols
+            self.set_peek_index(self.match_index()); // To consume matched symbols for `peeking_next()`
+
+            if !self.peek_matching {
+                self.set_index(self.match_index()); // To consume matched symbols for `next()`
+            }
         }
 
         is_empty_line
     }
 
     fn matches(&mut self, sequence: &[SymbolKind]) -> bool {
         // Note: Multiple matches may be set in the match closure, so we need to ensure that all start at the same index
-        self.reset_peek();
+        let peek_index = self.peek_index();
 
         for kind in sequence {
             if self.peeking_next(|s| s.kind == *kind).is_none() {
+                self.set_peek_index(peek_index);
                 return false;
             }
         }
 
+        self.set_match_index(self.peek_index());
+        self.set_peek_index(peek_index);
         true
     }
 
     fn consumed_matches(&mut self, sequence: &[SymbolKind]) -> bool {
         let matched = self.matches(sequence);
 
         if matched {
-            self.set_index(self.peek_index()); // To consume peeked symbols
+            self.set_peek_index(self.match_index()); // To consume matched symbols for `peeking_next()`
+
+            if !self.peek_matching {
+                self.set_index(self.match_index()); // To consume matched symbols for `next()`
+            }
         }
 
         matched
@@ -147,4 +165,19 @@ impl<'input> PrefixMatcher for SymbolIterator<'input> {
 
         self.consumed_matches(sequence)
     }
+
+    fn empty_line(&mut self) -> bool {
+        let peek_index = self.peek_index();
+
+        // NOTE: `Newline` at start is already ensured for prefix matches.
+        let _whitespaces = self
+            .peeking_take_while(|s| s.kind == SymbolKind::Whitespace)
+            .count();
+
+        let new_line =
+            self.peeking_next(|s| matches!(s.kind, SymbolKind::Newline | SymbolKind::EOI));
+
+        self.set_peek_index(peek_index);
+        new_line.is_some()
+    }
 }