Merge pull request #147 from pacholoamit/perf/improve-disk-scanning-p…

…erformance-2

refactor: Improve sorting logic for sub items in DiskItem

CHANGELOG.md

@@ -2,6 +2,16 @@
 
 All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](https://semver.org/).
 
+## 0.9.2
+- Performance improvements for disk analysis 
+- Added System information metrics to dashboard
+- UI performance improvements
+- Improve responsiveness of Disks analytics page
+
+
+## 0.9.1
+- Perf improvements
+
 ## 0.9.0
 - Added Disk Analysis feature
 - Added File explorer feature

package.json

@@ -22,6 +22,7 @@
     "classnames": "^2.5.1",
     "highcharts": "^11.4.1",
     "highcharts-react-official": "^3.2.1",
+    "immer": "^10.1.1",
     "lodash.sortby": "^4.7.0",
     "mantine-datatable": "^2.9.14",
     "non.geist": "^1.0.3",
@@ -30,6 +31,7 @@
     "react": "^18.3.1",
     "react-accessible-treeview": "^2.9.0",
     "react-apexcharts": "^1.4.1",
+    "react-arborist": "^3.4.0",
     "react-dom": "^18.3.1",
     "react-geiger": "^1.2.0",
     "react-github-btn": "^1.4.0",
@@ -38,7 +40,8 @@
     "react-text-gradients": "^1.0.2",
     "tauri-plugin-autostart-api": "https://github.com/tauri-apps/tauri-plugin-autostart",
     "tauri-plugin-log-api": "https://github.com/tauri-apps/tauri-plugin-log",
-    "tauri-plugin-store": "https://github.com/tauri-apps/tauri-plugin-store"
+    "tauri-plugin-store": "https://github.com/tauri-apps/tauri-plugin-store",
+    "zustand": "^4.5.2"
   },
   "devDependencies": {
     "@tauri-apps/cli": "^1.5.14",
@@ -51,4 +54,4 @@
     "vite": "^5.2.11"
   },
   "packageManager": "[email protected]"
-}
+}

src-tauri/src/app.rs

@@ -1,15 +1,13 @@
 use log::info;
-use rayon::iter::{IntoParallelRefIterator, ParallelIterator};
-use std::ffi::OsStr;
-use std::fs;
+
 use std::path::PathBuf;
 use std::sync::{Arc, Mutex};
+use std::thread;
 use tauri::{State, Window};
 
-use crate::dirstat::{DiskItem, DiskItemMetadata, FileInfo};
+use crate::dirstat::{DiskItem, FileInfo};
 use crate::metrics::Metrics;
 use crate::models::*;
-use rayon::prelude::*;
 
 pub struct AppState(Arc<Mutex<App>>);
 
@@ -132,12 +130,52 @@ pub fn delete_folder(path: String) {
     }
 }
 
-// Result<Vec<FileEntry>, String>
+#[tauri::command]
+// multi threaded fast version but emits data
+pub fn deep_scan_emit(window: tauri::Window, path: String) -> Result<(), String> {
+    let time = std::time::Instant::now();
+    dbg!("Scanning folder:", &path);
+    let path_buf = PathBuf::from(&path);
+
+    let file_info = FileInfo::from_path(&path_buf, true).map_err(|e| e.to_string())?;
+
+    let callback: Arc<dyn Fn(&DiskItem) + Send + Sync> = Arc::new(move |item: &DiskItem| {
+        if let Err(e) = window.emit("deep_scan_analysis", item) {
+            eprintln!("Failed to send signal: {}", e);
+        }
+    });
+
+    thread::spawn(move || {
+        let analysed = match file_info {
+            FileInfo::Directory { volume_id } => {
+                DiskItem::from_analyze_callback(&path_buf, true, volume_id, Arc::clone(&callback))
+                    .map_err(|e| e.to_string())
+            }
+            _ => Err("Not a directory".into()),
+        };
+
+        match analysed {
+            Ok(_) => {
+                dbg!("Scanning complete");
+                dbg!("Time taken:", time.elapsed().as_secs_f32());
+            }
+            Err(err) => {
+                eprintln!("Error during analysis: {}", err);
+            }
+        }
+    });
+
+    Ok(())
+}
 
 #[tauri::command]
+// Multithreaded fast version, uses high cpu/memory
 pub async fn deep_scan(path: String) -> Result<DiskItem, String> {
     let time = std::time::Instant::now();
     dbg!("Scanning folder:", &path);
+    dbg!("Triggering Deep scan");
+
+
     let path_buf = PathBuf::from(&path);
     let file_info = FileInfo::from_path(&path_buf, true).map_err(|e| e.to_string())?;
 

src-tauri/src/dirstat/mod.rs

@@ -6,10 +6,28 @@ use std::error::Error;
 use std::ffi::OsStr;
 use std::fs;
 use std::path::Path;
+use std::sync::atomic::{AtomicU64, Ordering};
+use std::sync::{Arc, Once};
 use ts_rs::TS;
 #[cfg(target_os = "windows")]
 use winapi_util::{file, Handle};
 
+static INIT: Once = Once::new();
+static mut COUNTER: Option<AtomicU64> = None;
+
+fn get_next_id() -> String {
+    unsafe {
+        INIT.call_once(|| {
+            COUNTER = Some(AtomicU64::new(1));
+        });
+        COUNTER
+            .as_ref()
+            .expect("COUNTER is not initialized")
+            .fetch_add(1, Ordering::SeqCst)
+            .to_string()
+    }
+}
+
 #[derive(Serialize, Deserialize, Debug, TS)]
 #[serde(rename_all = "camelCase")]
 #[ts(export, export_to = "../../src/lib/bindings/")]
@@ -18,14 +36,13 @@ pub struct DiskItemMetadata {
     #[ts(type = "number")]
     pub size: u64,
 }
-// TODO Reduce dupication after confirmed is working
+
 #[derive(Serialize, Deserialize, Debug, TS)]
 #[serde(rename_all = "camelCase")]
 #[ts(export, export_to = "../../src/lib/bindings/")]
 pub struct DiskItem {
+    pub id: String,
     pub name: String,
-    // #[ts(type = "number")]
-    // pub size: u64,
     pub children: Option<Vec<DiskItem>>,
     pub metadata: DiskItemMetadata,
 }
@@ -36,6 +53,7 @@ impl DiskItem {
         apparent: bool,
         root_dev: u64,
     ) -> Result<Self, Box<dyn Error>> {
+        let id = get_next_id();
         let name = path
             .file_name()
             .unwrap_or(OsStr::new("."))
@@ -54,30 +72,109 @@ impl DiskItem {
                     .filter_map(Result::ok)
                     .collect::<Vec<_>>();
 
-                let mut sub_items = sub_entries
-                    .par_iter()
+                let sub_items: Vec<DiskItem> = sub_entries
+                    .into_par_iter()
                     .filter_map(|entry| {
                         DiskItem::from_analyze(&entry.path(), apparent, root_dev).ok()
                     })
-                    .collect::<Vec<_>>();
+                    .collect();
 
-                sub_items.sort_unstable_by(|a, b| a.metadata.size.cmp(&b.metadata.size).reverse());
+                let mut sorted_sub_items = sub_items;
+                sorted_sub_items
+                    .sort_unstable_by(|a, b| a.metadata.size.cmp(&b.metadata.size).reverse());
 
                 Ok(DiskItem {
+                    id,
                     name,
                     metadata: DiskItemMetadata {
-                        size: sub_items.iter().map(|di| di.metadata.size).sum(),
+                        size: sorted_sub_items.iter().map(|di| di.metadata.size).sum(),
                     },
-                    children: Some(sub_items),
+                    children: Some(sorted_sub_items),
                 })
             }
             FileInfo::File { size, .. } => Ok(DiskItem {
+                id,
                 name,
                 metadata: DiskItemMetadata { size },
                 children: None,
             }),
         }
     }
+
+    pub fn from_analyze_callback(
+        path: &Path,
+        apparent: bool,
+        root_dev: u64,
+        callback: Arc<dyn Fn(&DiskItem) + Send + Sync>,
+    ) -> Result<Self, Box<dyn Error>> {
+        let id = get_next_id();
+        let name = path
+            .file_name()
+            .unwrap_or(OsStr::new("."))
+            .to_string_lossy()
+            .to_string();
+
+        let file_info = FileInfo::from_path(path, apparent)?;
+
+        match file_info {
+            FileInfo::Directory { volume_id } => {
+                if volume_id != root_dev {
+                    return Err("Filesystem boundary crossed".into());
+                }
+
+                let sub_entries = fs::read_dir(path)?
+                    .filter_map(Result::ok)
+                    .collect::<Vec<_>>();
+
+                let sub_items: Vec<DiskItem> = sub_entries
+                    .into_par_iter()
+                    .filter_map(|entry| {
+                        match DiskItem::from_analyze_callback(
+                            &entry.path(),
+                            apparent,
+                            root_dev,
+                            Arc::clone(&callback),
+                        ) {
+                            Ok(item) => {
+                                callback(&item);
+                                Some(item)
+                            }
+                            Err(_) => None,
+                        }
+                    })
+                    .collect();
+
+                let mut sorted_sub_items = sub_items;
+                sorted_sub_items
+                    .sort_unstable_by(|a, b| a.metadata.size.cmp(&b.metadata.size).reverse());
+
+                let item = DiskItem {
+                    id,
+                    name,
+                    metadata: DiskItemMetadata {
+                        size: sorted_sub_items.iter().map(|di| di.metadata.size).sum(),
+                    },
+                    children: Some(sorted_sub_items),
+                };
+
+                callback(&item);
+
+                Ok(item)
+            }
+            FileInfo::File { size, .. } => {
+                let item = DiskItem {
+                    id,
+                    name,
+                    metadata: DiskItemMetadata { size },
+                    children: None,
+                };
+
+                callback(&item);
+
+                Ok(item)
+            }
+        }
+    }
 }
 
 pub enum FileInfo {

src-tauri/src/main.rs

@@ -109,7 +109,8 @@ fn build_and_run_app(app: AppState) {
             app::kill_process,
             app::show_folder,
             app::deep_scan,
-            app::delete_folder
+            app::delete_folder,
+            app::deep_scan_emit,
         ])
         .run(tauri::generate_context!())
         .expect("error while running tauri application");