fix(recovery): Clear recovery directory instead of deleting it (#2823)

We've recently set up the GuestOS so that the we always have a recovery
directory in `var/lib/ic/data/ic_state`, which `ic-recovery` uses by
default (see #2131).

During clean-up, ic-recovery attempts to completely delete the
directory, which results in a permission error. Instead, we should
simply clear the directory's contents.

rs/recovery/src/file_sync_helper.rs

@@ -183,6 +183,26 @@ pub fn remove_dir(path: &Path) -> RecoveryResult<()> {
     }
 }
 
+pub fn clear_dir(path: &Path) -> RecoveryResult<()> {
+    if path_exists(path)? {
+        for entry in fs::read_dir(path).map_err(|e| RecoveryError::dir_error(path, e))? {
+            let entry = entry.map_err(|e| RecoveryError::dir_error(path, e))?;
+            let file_type = entry
+                .file_type()
+                .map_err(|e| RecoveryError::dir_error(path, e))?;
+            if file_type.is_dir() {
+                fs::remove_dir_all(entry.path())
+            } else {
+                fs::remove_file(entry.path())
+            }
+            .map_err(|e| RecoveryError::dir_error(entry.path().as_path(), e))?
+        }
+        Ok(())
+    } else {
+        Ok(())
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use tempfile::tempdir;

rs/recovery/src/steps.rs

@@ -2,7 +2,7 @@ use crate::{
     admin_helper::IcAdmin,
     command_helper::exec_cmd,
     error::{RecoveryError, RecoveryResult},
-    file_sync_helper::{create_dir, read_dir, remove_dir, rsync, rsync_with_retries},
+    file_sync_helper::{clear_dir, create_dir, read_dir, rsync, rsync_with_retries},
     get_member_ips, get_node_heights_from_metrics,
     registry_helper::RegistryHelper,
     replay_helper,
@@ -662,11 +662,11 @@ pub struct CleanupStep {
 
 impl Step for CleanupStep {
     fn descr(&self) -> String {
-        format!("Deleting directory {}.", self.recovery_dir.display())
+        format!("Clearing directory {}.", self.recovery_dir.display())
     }
 
     fn exec(&self) -> RecoveryResult<()> {
-        remove_dir(&self.recovery_dir)
+        clear_dir(&self.recovery_dir)
     }
 }