Ensure that damage is non-overlapping

The previous logic was blindly merging and doing only a forward
pass, however this won't work when after merging 2 rectangles
already processed one could end up in overlap.

This is an example of old renderer doing so:

```
[ 697101.942] [email protected]_buffer(66, 350, 36, 48)
[ 697101.956] [email protected]_buffer(126, 446, 828, 168)
[ 697101.965] [email protected]_buffer(90, 638, 744, 528)
[ 697101.974] [email protected]_buffer(162, 1262, 420, 72)
[ 697101.984] [email protected]_buffer(114, 1358, 312, 48)
[ 697101.994] [email protected]()

damage to be rendered: [
    Rectangle<smithay::utils::geometry::Physical> {
        x: 158,
        y: 478,
        width: 828,
        height: 168,
    },
    Rectangle<smithay::utils::geometry::Physical> {
        x: 194,
        y: 1294,
        width: 420,
        height: 72,
    },
    Rectangle<smithay::utils::geometry::Physical> {
        x: 146,
        y: 1390,
        width: 312,
        height: 48,
    },
    Rectangle<smithay::utils::geometry::Physical> {
        x: 32,
        y: 382,
        width: 834,
        height: 1152,
    },
]
```

The new algorithm trying splits damage bounding box into non-overlapping
regions containing damage, and then running tile-based damage shaping,
where for each tile we try to find total intersection + union of
damage rectangles.

src/backend/renderer/damage.rs → src/backend/renderer/damage/mod.rs

@@ -214,6 +214,10 @@ use super::{
 
 use super::{Renderer, Texture};
 
+mod shaper;
+
+use shaper::DamageShaper;
+
 const MAX_AGE: usize = 4;
 
 #[derive(Debug, Clone, Copy)]
@@ -256,6 +260,7 @@ struct RendererState {
 pub struct OutputDamageTracker {
     mode: OutputModeSource,
     last_state: RendererState,
+    damage_shaper: DamageShaper,
     span: tracing::Span,
 }
 
@@ -314,6 +319,7 @@ impl OutputDamageTracker {
                 transform,
             },
             last_state: Default::default(),
+            damage_shaper: Default::default(),
             span: info_span!("renderer_damage"),
         }
     }
@@ -326,6 +332,7 @@ impl OutputDamageTracker {
     pub fn from_output(output: &Output) -> Self {
         Self {
             mode: OutputModeSource::Auto(output.clone()),
+            damage_shaper: Default::default(),
             last_state: Default::default(),
             span: info_span!("renderer_damage", output = output.name()),
         }
@@ -339,8 +346,9 @@ impl OutputDamageTracker {
     pub fn from_mode_source(output_mode_source: impl Into<OutputModeSource>) -> Self {
         Self {
             mode: output_mode_source.into(),
-            last_state: Default::default(),
             span: info_span!("render_damage"),
+            damage_shaper: Default::default(),
+            last_state: Default::default(),
         }
     }
 
@@ -621,25 +629,18 @@ impl OutputDamageTracker {
         };
 
         // Optimize the damage for rendering
-        damage.dedup();
-        damage.retain(|rect| rect.overlaps_or_touches(output_geo));
-        damage.retain(|rect| !rect.is_empty());
-        // filter damage outside of the output gep and merge overlapping rectangles
-        *damage = damage
-            .drain(..)
-            .filter_map(|rect| rect.intersection(output_geo))
-            .fold(Vec::new(), |new_damage, mut rect| {
-                // replace with drain_filter, when that becomes stable to reuse the original Vec's memory
-                let (overlapping, mut new_damage): (Vec<_>, Vec<_>) = new_damage
-                    .into_iter()
-                    .partition(|other| other.overlaps_or_touches(rect));
-
-                for overlap in overlapping {
-                    rect = rect.merge(overlap);
-                }
-                new_damage.push(rect);
-                new_damage
-            });
+
+        // Clamp all rectangles to the bounds removing the ones without intersection.
+        damage.retain_mut(|rect| {
+            if let Some(intersected) = rect.intersection(output_geo) {
+                *rect = intersected;
+                true
+            } else {
+                false
+            }
+        });
+
+        self.damage_shaper.shape_damage(damage);
 
         if damage.is_empty() {
             trace!("nothing damaged, exiting early");

src/backend/renderer/damage/shaper.rs

@@ -0,0 +1,261 @@
+use std::mem;
+
+use crate::utils::{Physical, Rectangle, Size};
+
+/// The recommended minimum tile side.
+const DEFAULT_MIN_TILE_SIDE: i32 = 16;
+
+/// The maximum ratio of the largest damage rectangle to the current damage bbox.
+const MAX_DAMAGE_TO_DAMAGE_BBOX_RATIO: f32 = 0.9;
+
+/// State of the damage shaping.
+#[derive(Debug, Default)]
+pub struct DamageShaper<const MIN_TILE_SIDE: i32 = DEFAULT_MIN_TILE_SIDE> {
+    /// Cache for tiles to avoid re-allocation.
+    tiles_cache: Vec<Tile>,
+    /// The damage accumulated during shaping.
+    out_damage: Vec<Rectangle<i32, Physical>>,
+}
+
+impl<const MIN_TILE_SIDE: i32> DamageShaper<MIN_TILE_SIDE> {
+    /// Shape damage rectangles.
+    #[profiling::function]
+    pub fn shape_damage(&mut self, in_damage: &mut Vec<Rectangle<i32, Physical>>) {
+        self.out_damage.clear();
+
+        // Call the implementation without direction.
+        self.shape_damage_impl(in_damage, None, false);
+
+        // The shaped damage is inside of `out_damage`, so swap it with `in_damage` since
+        // it's irrelevant.
+        mem::swap(&mut self.out_damage, in_damage);
+    }
+
+    // A divide and conquer hybrid damage shaping algorithm.
+    //
+    // The key idea is to split rectangles by non-overlapping segments on dominating axis (largest
+    // side of the damage bounding box).
+    //
+    // When damage is small enough (less than 3) manual damage shaping is done by checking
+    // intersection or just forwarding damage.
+    //
+    // When damage is overlapping through the entire `in_damage` span we shape damage using the
+    // _tile_ based shaper, where the damage bounding box is split into tiles and damage is being
+    // computed for each tile individually by walking all the current damage rectangles.
+    #[inline(never)]
+    fn shape_damage_impl(
+        &mut self,
+        in_damage: &mut [Rectangle<i32, Physical>],
+        last_direction: Option<DamageSplitAxis>,
+        invert_direction: bool,
+    ) {
+        // Optimize small damage input.
+        if in_damage.len() < 3 {
+            self.shape_damage_less_than_3(in_damage);
+            return;
+        }
+
+        // Compute the effective damage bounding box and the maximum damaged area rectangle.
+
+        let (x_min, y_min, x_max, y_max, max_damage_area) =
+            in_damage.iter().fold((i32::MAX, i32::MAX, 0, 0, 0), |acc, rect| {
+                let area = rect.size.w * rect.size.h;
+                (
+                    acc.0.min(rect.loc.x),
+                    acc.1.min(rect.loc.y),
+                    acc.2.max(rect.loc.x + rect.size.w),
+                    acc.3.max(rect.loc.y + rect.size.h),
+                    acc.4.max(area),
+                )
+            });
+
+        let bbox_w = x_max - x_min;
+        let bbox_h = y_max - y_min;
+
+        let damage_bbox = Rectangle::<i32, Physical>::from_loc_and_size((x_min, y_min), (bbox_w, bbox_h));
+
+        // Damage the current bounding box when there's a damage rect covering near all the area.
+        if max_damage_area as f32 / (damage_bbox.size.w * damage_bbox.size.h) as f32
+            > MAX_DAMAGE_TO_DAMAGE_BBOX_RATIO
+        {
+            self.out_damage.push(damage_bbox);
+            return;
+        }
+
+        // Now we try to split bounding box to process non-overlapping damage rects separately.
+        //
+        // The whole approach is recursive and splits viewport if and only if we have a gap
+        // in the current segment or rectangles touch each other, since the renderer excludes
+        // borders.
+        //
+        // Examples:
+        //      [0, 3], [1, 2] [2, 3] [3, 4]
+        //      will have a split at ^
+        //      [0, 3], [1, 2] [2, 3] [6, 10]
+        //      will have a split at ^
+        //
+        //  Resulting in recursively trying to shape damage before and after
+        //  split point.
+
+        let mut direction = if bbox_w >= bbox_h {
+            DamageSplitAxis::X
+        } else {
+            DamageSplitAxis::Y
+        };
+
+        if invert_direction {
+            direction = direction.invert();
+        }
+
+        // The coordinate where the first rectangle ends and where the potential overlap
+        // will end.
+        let mut overlap_end = match direction {
+            DamageSplitAxis::X => {
+                if Some(direction) != last_direction {
+                    in_damage.sort_unstable_by(|lhs, rhs| {
+                        // Sort ascending by X and then descending by width, so the first
+                        // rectangle will overlap the most.
+                        lhs.loc.x.cmp(&rhs.loc.x).then(rhs.size.w.cmp(&lhs.size.w))
+                    });
+                }
+                in_damage[0].loc.x + in_damage[0].size.w
+            }
+            DamageSplitAxis::Y => {
+                if Some(direction) != last_direction {
+                    in_damage.sort_unstable_by(|lhs, rhs| {
+                        // Sort ascending by Y and then descending by height, so the first
+                        // rectangle will overlap the most.
+                        lhs.loc.y.cmp(&rhs.loc.y).then(rhs.size.h.cmp(&lhs.size.h))
+                    });
+                }
+                in_damage[0].loc.y + in_damage[0].size.h
+            }
+        };
+
+        // The start of overlap.
+        let mut overlap_start_idx = 0;
+        for idx in overlap_start_idx + 1..in_damage.len() {
+            let rect = in_damage[idx];
+            let (rect_start, rect_end) = match direction {
+                DamageSplitAxis::X => (rect.loc.x, rect.loc.x + rect.size.w),
+                DamageSplitAxis::Y => (rect.loc.y, rect.loc.y + rect.size.h),
+            };
+
+            // NOTE the renderer excludes the boundary, otherwise we need `>`.
+            if rect_start >= overlap_end {
+                self.shape_damage_impl(&mut in_damage[overlap_start_idx..idx], Some(direction), false);
+
+                // Advance the overlap.
+                overlap_start_idx = idx;
+                overlap_end = rect_end;
+            } else {
+                overlap_end = overlap_end.max(rect_end);
+            }
+        }
+
+        // When rectangle covers the entire bounding box and we've tried different direction of
+        // splitting perform the .
+        if overlap_start_idx == 0 && invert_direction {
+            // We pick more steps for edges which don't have full overlap.
+            const NUM_TILES: i32 = 4;
+            // NOTE we need to revert direction back to use larger side preferences.
+            let (tile_w, tile_h) = match direction.invert() {
+                DamageSplitAxis::X => (bbox_w / NUM_TILES, bbox_h / (NUM_TILES * 2)),
+                DamageSplitAxis::Y => (bbox_w / (NUM_TILES * 2), bbox_h / NUM_TILES),
+            };
+            let tile_size = (tile_w.max(MIN_TILE_SIDE), tile_h.max(MIN_TILE_SIDE)).into();
+
+            self.shape_damage_tiled(in_damage, damage_bbox, tile_size);
+        } else {
+            self.shape_damage_impl(
+                &mut in_damage[overlap_start_idx..],
+                Some(direction),
+                overlap_start_idx == 0,
+            );
+        }
+    }
+
+    #[inline]
+    fn shape_damage_tiled(
+        &mut self,
+        in_damage: &[Rectangle<i32, Physical>],
+        bbox: Rectangle<i32, Physical>,
+        tile_size: Size<i32, Physical>,
+    ) {
+        self.tiles_cache.clear();
+
+        for x in (bbox.loc.x..bbox.loc.x + bbox.size.w).step_by(tile_size.w as usize) {
+            for y in (bbox.loc.y..bbox.loc.y + bbox.size.h).step_by(tile_size.h as usize) {
+                // NOTE the in_damage is constrained to the `bbox`, so it can't go outside
+                // the tile, even though some tiles could go outside the `bbox`.
+                let bbox = Rectangle::<i32, Physical>::from_loc_and_size((x, y), tile_size);
+                let mut tile = Tile { bbox, damage: None };
+
+                // Intersect damage regions with the given tile bounding box.
+                for damage_rect in in_damage.iter() {
+                    if let Some(intersection) = tile.bbox.intersection(*damage_rect) {
+                        tile.damage = if let Some(tile_damage) = tile.damage {
+                            Some(intersection.merge(tile_damage))
+                        } else {
+                            Some(intersection)
+                        };
+                    }
+                }
+
+                self.tiles_cache.push(tile);
+            }
+        }
+
+        self.out_damage
+            .extend(self.tiles_cache.iter().filter_map(|tile| tile.damage));
+    }
+
+    /// Shape damage when there're at max 2 rectangles in it.
+    #[inline]
+    fn shape_damage_less_than_3(&mut self, in_damage: &[Rectangle<i32, Physical>]) {
+        if in_damage.is_empty() {
+            return;
+        }
+
+        let first = in_damage[0];
+        if in_damage.len() == 1 {
+            self.out_damage.push(first);
+        } else {
+            let second = in_damage[1];
+            if first.overlaps(second) {
+                self.out_damage.push(first.merge(second));
+            } else {
+                self.out_damage.push(first);
+                self.out_damage.push(second);
+            }
+        }
+    }
+}
+
+/// Tile with the damage tracking information.
+#[derive(Debug)]
+pub struct Tile {
+    /// Bounding box for the given tile.
+    bbox: Rectangle<i32, Physical>,
+    /// The accumulated damage in the tile.
+    damage: Option<Rectangle<i32, Physical>>,
+}
+
+/// Direction to split damage bounding box.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum DamageSplitAxis {
+    /// Split damage bounding box on x axis.
+    X,
+    /// Split damage bounding box on y axis.
+    Y,
+}
+
+impl DamageSplitAxis {
+    /// Invert the split direction.
+    fn invert(self) -> Self {
+        match self {
+            DamageSplitAxis::X => DamageSplitAxis::Y,
+            DamageSplitAxis::Y => DamageSplitAxis::X,
+        }
+    }
+}