Merge pull request #113 from swisstopo/LGD-564/improve-OCR-mistake-ha…

…ndling

Lgd 564/improve ocr mistake handling

src/stratigraphy/sidebar/a_above_b_sidebar.py

@@ -4,6 +4,7 @@
 
 import statistics
 from dataclasses import dataclass
+from itertools import product
 
 import fitz
 import numpy as np
@@ -116,19 +117,31 @@ def remove_entry_by_correlation_gradient(self) -> AAboveBSidebar | None:
 
     def make_ascending(self):
         median_value = np.median(np.array([entry.value for entry in self.entries]))
-        for i, entry in enumerate(self.entries):
-            if entry.value.is_integer() and entry.value > median_value:
-                factor100_value = entry.value / 100
-                previous_ok = i == 0 or all(entry.value < factor100_value for entry in self.entries[:i])
-                next_ok = i + 1 == len(self.entries) or factor100_value < self.entries[i + 1].value
 
-                if previous_ok and next_ok:
-                    # Create a new entry instead of modifying the value of the current one, as this entry might be
-                    # used in different sidebars as well.
-                    self.entries[i] = DepthColumnEntry(rect=entry.rect, value=factor100_value)
+        for i, entry in enumerate(self.entries):
+            new_values = []
 
+            if entry.value.is_integer() and entry.value > median_value:
+                new_values.extend([entry.value / 100, entry.value / 10])
+
+            # Correct common OCR mistakes where "4" is recognized instead of "1"
+            # We don't control for OCR mistakes recognizing "9" as "3" (example zurich/680244005-bp.pdf)
+            if "4" in str(entry.value) and not self._valid_value(i, entry.value):
+                new_values.extend(generate_alternatives(entry.value))
+
+            # Assign the first valid correction
+            for new_value in new_values:
+                if self._valid_value(i, new_value):
+                    self.entries[i] = DepthColumnEntry(rect=entry.rect, value=new_value)
+                    break
         return self
 
+    def _valid_value(self, index: int, new_value: float) -> bool:
+        """Check if new value at given index is maintaining ascending order."""
+        previous_ok = index == 0 or all(other_entry.value < new_value for other_entry in self.entries[:index])
+        next_ok = index + 1 == len(self.entries) or new_value < self.entries[index + 1].value
+        return previous_ok and next_ok
+
     def break_on_double_descending(self) -> list[AAboveBSidebar]:
         segments = []
         segment_start = 0
@@ -227,3 +240,15 @@ def identify_groups(
             groups.append(IntervalBlockGroup(depth_intervals=current_intervals, blocks=current_blocks))
 
         return groups
+
+
+def generate_alternatives(value: float) -> list[float]:
+    """Generate a list of all possible alternatives by replacing each '4' with '1'."""
+    value_str = str(value)
+    alternatives = []
+    options = [(char if char != "4" else ["4", "1"]) for char in value_str]
+
+    for combo in product(*options):
+        alternatives.append(float("".join(combo)))
+
+    return alternatives

src/stratigraphy/sidebar/a_above_b_sidebar_extractor.py

@@ -42,7 +42,6 @@ def find_in_words(
         clusters = Cluster[DepthColumnEntry].create_clusters(entries)
 
         numeric_columns = [AAboveBSidebar(cluster.entries) for cluster in clusters if len(cluster.entries) >= 3]
-        sidebar_validator = AAboveBSidebarValidator(**sidebar_params)
 
         filtered_columns = [
             column

src/stratigraphy/sidebar/a_above_b_sidebar_validator.py

@@ -6,7 +6,6 @@
 
 from .a_above_b_sidebar import AAboveBSidebar
 from .sidebar import SidebarNoise, noise_count
-from .sidebarentry import DepthColumnEntry
 
 
 @dataclasses.dataclass
@@ -82,10 +81,6 @@ def reduce_until_valid(
             if self.is_valid(sidebar_noise):
                 return sidebar_noise
 
-            corrected_sidebar_noise = self.correct_OCR_mistakes(sidebar_noise, word_rtree)
-            if corrected_sidebar_noise:
-                return corrected_sidebar_noise
-
             new_sidebar = sidebar_noise.sidebar.remove_entry_by_correlation_gradient()
             if not new_sidebar:
                 return None
@@ -94,68 +89,3 @@ def reduce_until_valid(
             sidebar_noise = SidebarNoise(sidebar=new_sidebar, noise_count=new_noise_count)
 
         return None
-
-    def correct_OCR_mistakes(self, sidebar_noise: SidebarNoise, word_rtree: rtree.index.Index) -> SidebarNoise | None:
-        """Corrects OCR mistakes in the Sidebar entries.
-
-        Loops through all values and corrects common OCR mistakes for the given entry. Then, the column with the
-        highest pearson correlation coefficient is selected and checked for validity.
-
-        This is useful if one or more entries have an OCR mistake, and the column is not valid because of it.
-
-        Currently, there is no limit on the number of corrections per depth column. Indeed, there are examples of depth
-        columns with multiple OCR errors on different depth values. On the other hand, allowing an unlimited number of
-        corrections increases the risk, that a random column of different values is incorrectly accepted as a depth
-        column after making the corrections, especially if the column has a low number of entries. A more robust
-        solution might be to allow corrections on less than 50% of all entries, or something similar. However, we
-        currently don't have enough examples to properly tune this parameter.
-
-        Note: Common mistakes should be extended as needed.
-
-        Args:
-            sidebar_noise (SidebarNoise): The SidebarNoise wrapping the sidebar to validate.
-            word_rtree (index.Index): R-tree of all words on page for efficient spatial queries.
-
-        Returns:
-            SidebarNoise | None: The corrected SidebarNoise, or None if no correction was possible.
-        """
-        sidebar = sidebar_noise.sidebar
-        new_columns = [AAboveBSidebar(entries=[])]
-
-        for entry in sidebar.entries:
-            new_columns = [
-                AAboveBSidebar([*column.entries, DepthColumnEntry(entry.rect, new_value)])
-                for column in new_columns
-                for new_value in _value_alternatives(entry.value)
-            ]
-            # Immediately require strictly increasing values, to avoid exponential complexity when many implausible
-            # alternative values are suggested
-            new_columns = [column for column in new_columns if column.is_strictly_increasing()]
-
-        if new_columns:
-            best_column = max(new_columns, key=lambda column: column.pearson_correlation_coef())
-            new_noise_count = noise_count(best_column, word_rtree)
-
-            # We require a higher correlation coefficient when corrections are made
-            if self.is_valid(
-                SidebarNoise(sidebar=best_column, noise_count=new_noise_count), corr_coef_threshold=0.999
-            ):
-                return SidebarNoise(sidebar=best_column, noise_count=new_noise_count)
-
-        return None
-
-
-def _value_alternatives(value: float) -> set[float]:
-    """Corrects frequent OCR errors in depth column entries.
-
-    Args:
-        value (float): The depth values to find plausible alternatives for
-
-    Returns:
-        set(float): all plausible values (including the original one)
-    """
-    alternatives = {value}
-    # In older documents, OCR sometimes mistakes 1 for 4
-    alternatives.add(float(str(value).replace("4", "1")))
-
-    return alternatives

tests/test_aabovebsidebar.py

@@ -2,6 +2,7 @@
 
 import fitz
 from stratigraphy.sidebar import AAboveBSidebar
+from stratigraphy.sidebar.a_above_b_sidebar import generate_alternatives
 from stratigraphy.sidebar.sidebarentry import DepthColumnEntry
 
 
@@ -47,20 +48,48 @@ def test_aabovebsidebar_makeascending():  # noqa: D103
 
     def test(in_values, out_values):
         sidebar = AAboveBSidebar([DepthColumnEntry(fitz.Rect(), value=value) for value in in_values])
-        assert [
-            entry.value for entry in sidebar.make_ascending().entries
-        ] == out_values, "The depth values from the sidebar are not converted correctly"
+        result = [entry.value for entry in sidebar.make_ascending().entries]
+        assert result == out_values, f"Expected {out_values}, but got {result}"
 
+    # Basic transformation for values greater than the median, correct by factor 100
     test([1.0, 200.0, 3.0], [1.0, 2.0, 3.0])
     test([100.0, 2.0, 3.0], [1.0, 2.0, 3.0])
     test([1.0, 2.0, 300.0], [1.0, 2.0, 3.0])
-    test([1.0, 200.0, 300.0, 4.0, 5.0, 6.0, 100.0], [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 100.0])
+
+    # Basic transformation for values greater than the median, correct by factor 10
+    test([1.0, 20.0, 300.0], [1.0, 20.0, 30.0])
+    test([1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 100.0], [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 10.0])
     test([100.0, 200.0, 300.0], [100.0, 200.0, 300.0])
 
+    ## Transforming OCR mistakes
+    test([0.5, 4.0, 2.0, 5.0], [0.5, 1.0, 2.0, 5.0])
+    test([4.0, 4.4, 4.4, 5.0], [4.0, 4.1, 4.4, 5.0])
+
     # ensure a "noise" value "0.0" does not influence the result
     test([1.0, 2.0, 3.0, 0.0, 4.0], [1.0, 2.0, 3.0, 0.0, 4.0])
 
 
+def test_generate_alternatives():
+    """Test generate_alternatives function for alternative options to OCR mistakes."""
+    assert generate_alternatives(4) == [4, 1]
+    assert generate_alternatives(14) == [14, 11]
+    assert generate_alternatives(441) == [441, 411, 141, 111]
+    assert generate_alternatives(123) == [123]
+    assert generate_alternatives(4.4) == [4.4, 4.1, 1.4, 1.1]
+
+
+def test_valid_value():
+    """Test _valid_value helper function for make_ascending method of the AAboveBSidebar class."""
+    entries = [DepthColumnEntry(None, 1), DepthColumnEntry(None, 2), DepthColumnEntry(None, 3)]
+    sidebar = AAboveBSidebar(entries)
+
+    assert sidebar._valid_value(1, 2) is True
+    assert sidebar._valid_value(1, 3) is False
+    assert sidebar._valid_value(1, 1.5) is True
+    assert sidebar._valid_value(0, 2) is False
+    assert sidebar._valid_value(2, 3.5) is True
+
+
 def test_aabovebsidebar_isstrictlyincreasing():  # noqa: D103
     """Test the is_strictly_increasing method of the AAboveBSidebar class."""
     # Case 1: Strictly increasing values