[casr-cluster] Add cluster updating support

casr/src/util.rs

@@ -1,16 +1,21 @@
 //! Common utility functions.
 extern crate libcasr;
 
+use libcasr::cluster::{Cluster, ReportInfo};
 use libcasr::report::CrashReport;
 use libcasr::stacktrace::{
     STACK_FRAME_FILEPATH_IGNORE_REGEXES, STACK_FRAME_FUNCTION_IGNORE_REGEXES,
 };
 
 use anyhow::{bail, Context, Result};
 use clap::ArgMatches;
+use is_executable::IsExecutable;
 use log::{info, warn};
+use rayon::iter::{IntoParallelIterator, ParallelIterator};
 use simplelog::*;
-use std::collections::HashSet;
+use wait_timeout::ChildExt;
+
+use std::collections::{HashMap, HashSet};
 use std::fs::{self, OpenOptions};
 use std::io::Write;
 use std::io::{BufRead, BufReader};
@@ -19,9 +24,6 @@
 use std::sync::RwLock;
 use std::time::Duration;
 
-use is_executable::IsExecutable;
-use wait_timeout::ChildExt;
-
 /// Call casr-san with the provided options
 ///
 /// # Arguments
@@ -411,3 +413,147 @@
         );
     }
 }
+
+/// Get CASR reports from specified directory
+///
+/// # Arguments
+///
+/// * `dir` - directory path
+///
+/// # Return value
+///
+/// A vector of reports paths
+pub fn get_reports(dir: &Path) -> Result<Vec<PathBuf>> {
+    let dir = fs::read_dir(dir).with_context(|| format!("File: {}", dir.display()))?;
+    let casreps: Vec<PathBuf> = dir
+        .map(|path| path.unwrap().path())
+        .filter(|s| s.extension().is_some() && s.extension().unwrap() == "casrep")
+        .collect();
+    Ok(casreps)
+}
+
+/// Parse CASR reports from specified paths.
+///
+/// # Arguments
+///
+/// * `casreps` - a vector of report paths
+///
+/// * `jobs` - number of jobs for parsing process
+///
+/// # Return value
+///
+/// * A vector of correctly parsed report info: paths, stacktraces and crashlines
+/// * A vector of bad reports
+pub fn reports_from_paths(casreps: Vec<PathBuf>, jobs: usize) -> (Vec<ReportInfo>, Vec<PathBuf>) {
+    // Get len
+    let len = casreps.len();
+    // Start thread pool.
+    let custom_pool = rayon::ThreadPoolBuilder::new()
+        .num_threads(jobs.min(len))
+        .build()
+        .unwrap();
+    // Report info from casreps: (casrep, (trace, crashline))
+    let mut casrep_info: RwLock<Vec<ReportInfo>> = RwLock::new(Vec::new());
+    // Casreps with stacktraces, that we cannot parse
+    let mut badreports: RwLock<Vec<PathBuf>> = RwLock::new(Vec::new());
+    custom_pool.install(|| {
+        (0..len).into_par_iter().for_each(|i| {
+            if let Ok(report) = report_from_file(casreps[i].as_path()) {
+                if let Ok(trace) = report.filtered_stacktrace() {
+                    casrep_info
+                        .write()
+                        .unwrap()
+                        .push((casreps[i].clone(), (trace, report.crashline)));
+                } else {
+                    badreports.write().unwrap().push(casreps[i].clone());
+                }
+            } else {
+                badreports.write().unwrap().push(casreps[i].clone());
+            }
+        })
+    });
+    let casrep_info = casrep_info.get_mut().unwrap();
+    let badreports = badreports.get_mut().unwrap().to_vec();
+    // Sort by casrep filename
+    casrep_info.sort_by(|a, b| {
+        a.0.file_name()
+            .unwrap()
+            .to_str()
+            .unwrap()
+            .cmp(b.0.file_name().unwrap().to_str().unwrap())
+    });
+
+    (casrep_info.to_vec(), badreports)
+}
+
+/// Get `Cluster` structure from specified directory path.
+///
+/// # Arguments
+///
+/// * `dir` - valid cluster dir path
+///
+/// * `jobs` - number of jobs for parsing process
+///
+/// # Return value
+///
+/// `Cluster` structure
+pub fn cluster_from_dir(dir: &Path, jobs: usize) -> Result<Cluster> {
+    // Get cluster number
+    let i = dir.file_name().unwrap().to_str().unwrap()[2..]
+        .to_string()
+        .parse::<usize>()
+        .unwrap();
+    // Get casreps from cluster
+    let casreps = get_reports(dir)?;
+    let (casreps, _) = reports_from_paths(casreps, jobs);
+    let (_, (stacktraces, crashlines)): (Vec<_>, (Vec<_>, Vec<_>)) =
+        casreps.iter().cloned().unzip();
+    // Create cluster
+    // NOTE: We don't care about paths of casreps from existing clusters
+    Ok(Cluster::new(i, Vec::new(), stacktraces, crashlines))
+}
+
+/// Save clusters to given directory
+///
+/// # Arguments
+///
+/// * `clusters` - given `Cluster` structures for saving
+///
+/// * `dir` - out directory
+pub fn save_clusters(clusters: &HashMap<usize, Cluster>, dir: &Path) -> Result<()> {
+    for cluster in clusters.values() {
+        fs::create_dir_all(format!("{}/cl{}", &dir.display(), cluster.number))?;
+        for casrep in cluster.paths() {
+            fs::copy(
+                casrep,
+                format!(
+                    "{}/cl{}/{}",
+                    &dir.display(),
+                    cluster.number,
+                    &casrep.file_name().unwrap().to_str().unwrap()
+                ),
+            )?;
+        }
+    }
+    Ok(())
+}
+
+/// Save CASR reports to given directory
+///
+/// # Arguments
+///
+/// * `reports` - A vector of CASR reports
+///
+/// * `dir` - out directory
+pub fn save_reports(reports: Vec<PathBuf>, dir: String) -> Result<()> {
+    if !Path::new(&dir).exists() {
+        fs::create_dir_all(&dir)?;
+    }
+    for report in reports {
+        fs::copy(
+            &report,
+            format!("{}/{}", dir, &report.file_name().unwrap().to_str().unwrap()),
+        )?;
+    }
+    Ok(())
+}

casr/tests/casr_tests/casrep/test_clustering_small/40.casrep

@@ -0,0 +1,87 @@
+{
+    "Date": "2021-07-14T19:56:09.276635+03:00",
+    "Uname": "Linux titanfall 5.8.0-59-generic #66~20.04.1-Ubuntu SMP Thu Jun 17 11:14:10 UTC 2021 x86_64 x86_64 x86_64 GNU/Linux",
+    "OS": "Ubuntu",
+    "OSRelease": "20.04",
+    "Architecture": "amd64",
+    "ExecutablePath": "/usr/local/bin/tiff2pdf",
+    "ProcCmdline": "tiff2pdf ./fuz3tiff2pdf/main/crashes/id:000009,sig:06,src:000040+000049,time:43718,op:splice,rep:4",
+    "ProcMaps": [
+        "      0x555555554000     0x555555556000     0x2000        0x0 /usr/local/bin/tiff2pdf",
+        "      0x555555556000     0x555555561000     0xb000     0x2000 /usr/local/bin/tiff2pdf",
+        "      0x555555561000     0x555555565000     0x4000     0xd000 /usr/local/bin/tiff2pdf",
+        "      0x555555565000     0x555555566000     0x1000    0x10000 /usr/local/bin/tiff2pdf",
+        "      0x555555566000     0x555555567000     0x1000    0x11000 /usr/local/bin/tiff2pdf",
+        "      0x555555567000     0x555555588000    0x21000        0x0 [heap]",
+        "      0x7ffff7945000     0x7ffff7949000     0x4000        0x0 ",
+        "      0x7ffff7949000     0x7ffff7958000     0xf000        0x0 /usr/lib/x86_64-linux-gnu/libm-2.31.so",
+        "      0x7ffff7958000     0x7ffff79ff000    0xa7000     0xf000 /usr/lib/x86_64-linux-gnu/libm-2.31.so",
+        "      0x7ffff79ff000     0x7ffff7a96000    0x97000    0xb6000 /usr/lib/x86_64-linux-gnu/libm-2.31.so",
+        "      0x7ffff7a96000     0x7ffff7a97000     0x1000   0x14c000 /usr/lib/x86_64-linux-gnu/libm-2.31.so",
+        "      0x7ffff7a97000     0x7ffff7a98000     0x1000   0x14d000 /usr/lib/x86_64-linux-gnu/libm-2.31.so",
+        "      0x7ffff7a98000     0x7ffff7a9a000     0x2000        0x0 /usr/lib/x86_64-linux-gnu/libz.so.1.2.11",
+        "      0x7ffff7a9a000     0x7ffff7aab000    0x11000     0x2000 /usr/lib/x86_64-linux-gnu/libz.so.1.2.11",
+        "      0x7ffff7aab000     0x7ffff7ab1000     0x6000    0x13000 /usr/lib/x86_64-linux-gnu/libz.so.1.2.11",
+        "      0x7ffff7ab1000     0x7ffff7ab2000     0x1000    0x19000 /usr/lib/x86_64-linux-gnu/libz.so.1.2.11",
+        "      0x7ffff7ab2000     0x7ffff7ab3000     0x1000    0x19000 /usr/lib/x86_64-linux-gnu/libz.so.1.2.11",
+        "      0x7ffff7ab3000     0x7ffff7ab4000     0x1000    0x1a000 /usr/lib/x86_64-linux-gnu/libz.so.1.2.11",
+        "      0x7ffff7ab4000     0x7ffff7ab8000     0x4000        0x0 /usr/lib/x86_64-linux-gnu/libjpeg.so.8.2.2",
+        "      0x7ffff7ab8000     0x7ffff7afc000    0x44000     0x4000 /usr/lib/x86_64-linux-gnu/libjpeg.so.8.2.2",
+        "      0x7ffff7afc000     0x7ffff7b36000    0x3a000    0x48000 /usr/lib/x86_64-linux-gnu/libjpeg.so.8.2.2",
+        "      0x7ffff7b36000     0x7ffff7b37000     0x1000    0x82000 /usr/lib/x86_64-linux-gnu/libjpeg.so.8.2.2",
+        "      0x7ffff7b37000     0x7ffff7b38000     0x1000    0x82000 /usr/lib/x86_64-linux-gnu/libjpeg.so.8.2.2",
+        "      0x7ffff7b38000     0x7ffff7b39000     0x1000    0x83000 /usr/lib/x86_64-linux-gnu/libjpeg.so.8.2.2",
+        "      0x7ffff7b39000     0x7ffff7b44000     0xb000        0x0 /usr/lib/x86_64-linux-gnu/libjbig.so.0",
+        "      0x7ffff7b44000     0x7ffff7d43000   0x1ff000     0xb000 /usr/lib/x86_64-linux-gnu/libjbig.so.0",
+        "      0x7ffff7d43000     0x7ffff7d44000     0x1000     0xa000 /usr/lib/x86_64-linux-gnu/libjbig.so.0",
+        "      0x7ffff7d44000     0x7ffff7d47000     0x3000     0xb000 /usr/lib/x86_64-linux-gnu/libjbig.so.0",
+        "      0x7ffff7d47000     0x7ffff7d6c000    0x25000        0x0 /usr/lib/x86_64-linux-gnu/libc-2.31.so",
+        "      0x7ffff7d6c000     0x7ffff7ee4000   0x178000    0x25000 /usr/lib/x86_64-linux-gnu/libc-2.31.so",
+        "      0x7ffff7ee4000     0x7ffff7f2e000    0x4a000   0x19d000 /usr/lib/x86_64-linux-gnu/libc-2.31.so",
+        "      0x7ffff7f2e000     0x7ffff7f2f000     0x1000   0x1e7000 /usr/lib/x86_64-linux-gnu/libc-2.31.so",
+        "      0x7ffff7f2f000     0x7ffff7f32000     0x3000   0x1e7000 /usr/lib/x86_64-linux-gnu/libc-2.31.so",
+        "      0x7ffff7f32000     0x7ffff7f35000     0x3000   0x1ea000 /usr/lib/x86_64-linux-gnu/libc-2.31.so",
+        "      0x7ffff7f35000     0x7ffff7f39000     0x4000        0x0 ",
+        "      0x7ffff7f39000     0x7ffff7f41000     0x8000        0x0 /usr/local/lib/libtiff.so.3.9.6",
+        "      0x7ffff7f41000     0x7ffff7f76000    0x35000     0x8000 /usr/local/lib/libtiff.so.3.9.6",
+        "      0x7ffff7f76000     0x7ffff7f9f000    0x29000    0x3d000 /usr/local/lib/libtiff.so.3.9.6",
+        "      0x7ffff7f9f000     0x7ffff7fa0000     0x1000    0x66000 /usr/local/lib/libtiff.so.3.9.6",
+        "      0x7ffff7fa0000     0x7ffff7fa2000     0x2000    0x66000 /usr/local/lib/libtiff.so.3.9.6",
+        "      0x7ffff7fa2000     0x7ffff7fa3000     0x1000    0x68000 /usr/local/lib/libtiff.so.3.9.6",
+        "      0x7ffff7fa3000     0x7ffff7fa5000     0x2000        0x0 ",
+        "      0x7ffff7fc8000     0x7ffff7fc9000     0x1000        0x0 ",
+        "      0x7ffff7fc9000     0x7ffff7fcd000     0x4000        0x0 [vvar]",
+        "      0x7ffff7fcd000     0x7ffff7fcf000     0x2000        0x0 [vdso]",
+        "      0x7ffff7fcf000     0x7ffff7fd0000     0x1000        0x0 /usr/lib/x86_64-linux-gnu/ld-2.31.so",
+        "      0x7ffff7fd0000     0x7ffff7ff3000    0x23000     0x1000 /usr/lib/x86_64-linux-gnu/ld-2.31.so",
+        "      0x7ffff7ff3000     0x7ffff7ffb000     0x8000    0x24000 /usr/lib/x86_64-linux-gnu/ld-2.31.so",
+        "      0x7ffff7ffb000     0x7ffff7ffc000     0x1000        0x0 /home/avgor46/testdoc/fuz3tiff2pdf/main/crashes/id:000009,sig:06,src:000040+000049,time:43718,op:splice,rep:4",
+        "      0x7ffff7ffc000     0x7ffff7ffd000     0x1000    0x2c000 /usr/lib/x86_64-linux-gnu/ld-2.31.so",
+        "      0x7ffff7ffd000     0x7ffff7ffe000     0x1000    0x2d000 /usr/lib/x86_64-linux-gnu/ld-2.31.so",
+        "      0x7ffff7ffe000     0x7ffff7fff000     0x1000        0x0 ",
+        "      0x7ffffffde000     0x7ffffffff000    0x21000        0x0 [stack]",
+        "  0xffffffffff600000 0xffffffffff601000     0x1000        0x0 [vsyscall]"
+    ],
+    "CrashSeverity": {
+        "Type": "NOT_CRITICAL",
+        "ShortDescription": "SafeFunctionCheck",
+        "Description": "Buffer overflow in safe function",
+        "Explanation": "The target stopped while handling a signal that was generated by libc due to detection of buffer overflow in safe copy function."
+    },
+    "Stacktrace": [
+        "#0  __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:50",
+        "#1  0x00007ffff7d6c859 in __GI_abort () at abort.c:79",
+        "#2  0x00007ffff7dd73ee in __libc_message (action=action@entry=do_abort, fmt=fmt@entry=0x7ffff7f0107c \"*** %s ***: terminated\\n\") at ../sysdeps/posix/libc_fatal.c:155",
+        "#3  0x00007ffff7e79b4a in __GI___fortify_fail (msg=msg@entry=0x7ffff7f01012 \"buffer overflow detected\") at fortify_fail.c:26",
+        "#4  0x00007ffff7e783e6 in __GI___chk_fail () at chk_fail.c:28",
+        "#5  0x00007ffff7dcf1cf in _IO_str_chk_overflow (fp=<optimized out>, c=<optimized out>) at iovsprintf.c:35",
+        "#6  0x00007ffff7da7db0 in __GI___printf_fp_l (fp=<optimized out>, loc=<optimized out>, info=<optimized out>, args=<optimized out>) at printf_fp.c:1246",
+        "#7  0x00007ffff7dc163a in __vfprintf_internal (s=s@entry=0x7fffffffe070, format=format@entry=0x5555555613df \"%.4f\", ap=ap@entry=0x7fffffffe1b0, mode_flags=mode_flags@entry=6) at vfprintf-internal.c:1687",
+        "#8  0x00007ffff7dcf279 in __vsprintf_internal (string=0x7fffffffe2a0 \"79725330432.000\", maxlen=<optimized out>, format=0x5555555613df \"%.4f\", args=args@entry=0x7fffffffe1b0, mode_flags=6) at iovsprintf.c:95",
+        "#9  0x00007ffff7e77edb in ___sprintf_chk (s=<optimized out>, flag=<optimized out>, slen=<optimized out>, format=<optimized out>) at sprintf_chk.c:40",
+        "#10 0x000055555555c7a1 in sprintf (__fmt=0x5555555613df \"%.4f\", __s=0x7fffffffe2a0 \"79725330432.000\") at /usr/include/x86_64-linux-gnu/bits/stdio2.h:36",
+        "#12 0x00005555555601b8 in t2p_write_pdf (output=0x555555568f80, input=0x555555567ea0, t2p=0x5555555672a0) at tiff2pdf.c:5175",
+        "#13 t2p_write_pdf (t2p=0x5555555672a0, input=0x555555567ea0, output=0x555555568f80) at tiff2pdf.c:5133",
+        "#14 0x00005555555568d4 in main (argc=<optimized out>, argv=<optimized out>) at tiff2pdf.c:763"
+    ]
+}