Skip to content

Commit

Permalink
[casr-cluster] Add cluster updating support (#183)
Browse files Browse the repository at this point in the history
  • Loading branch information
@hkctkuy
hkctkuy authored Feb 2, 2024
1 parent b64a7f4 commit df9eecf
Show file tree
Hide file tree
Showing 8 changed files with 1,096 additions and 110 deletions.
428 changes: 331 additions & 97 deletions casr/src/bin/casr-cluster.rs
View file

Large diffs are not rendered by default.

155 changes: 151 additions & 4 deletions casr/src/util.rs
View file
Original file line number Diff line number Diff line change
@@ -1,16 +1,21 @@
//! Common utility functions.
extern crate libcasr;

use libcasr::cluster::{Cluster, ReportInfo};
use libcasr::report::CrashReport;
use libcasr::stacktrace::{
STACK_FRAME_FILEPATH_IGNORE_REGEXES, STACK_FRAME_FUNCTION_IGNORE_REGEXES,
};

use anyhow::{bail, Context, Result};
use clap::ArgMatches;
use is_executable::IsExecutable;
use log::{info, warn};
use rayon::iter::{IntoParallelIterator, ParallelIterator};
use simplelog::*;
use std::collections::HashSet;
use wait_timeout::ChildExt;

use std::collections::{HashMap, HashSet};
use std::fs::{self, OpenOptions};
use std::io::Write;
use std::io::{BufRead, BufReader};
Expand All @@ -19,9 +24,6 @@ use std::process::{Command, Output, Stdio};
use std::sync::RwLock;
use std::time::Duration;

use is_executable::IsExecutable;
use wait_timeout::ChildExt;

/// Call casr-san with the provided options
///
/// # Arguments
Expand Down Expand Up @@ -411,3 +413,148 @@ pub fn get_path(tool: &str) -> Result<PathBuf> {
);
}
}

/// Get CASR reports from specified directory
///
/// # Arguments
///
/// * `dir` - directory path
///
/// # Return value
///
/// A vector of reports paths
pub fn get_reports(dir: &Path) -> Result<Vec<PathBuf>> {
let dir = fs::read_dir(dir).with_context(|| format!("File: {}", dir.display()))?;
let casreps: Vec<PathBuf> = dir
.map(|path| path.unwrap().path())
.filter(|s| s.extension().is_some() && s.extension().unwrap() == "casrep")
.collect();
Ok(casreps)
}

/// Parse CASR reports from specified paths.
///
/// # Arguments
///
/// * `casreps` - a vector of report paths
///
/// * `jobs` - number of jobs for parsing process
///
/// # Return value
///
/// * A vector of correctly parsed report info: paths, stacktraces and crashlines
/// * A vector of bad reports
pub fn reports_from_paths(casreps: &Vec<PathBuf>, jobs: usize) -> (Vec<ReportInfo>, Vec<PathBuf>) {
// Get len
let len = casreps.len();
// Start thread pool.
let custom_pool = rayon::ThreadPoolBuilder::new()
.num_threads(jobs.min(len))
.build()
.unwrap();
// Report info from casreps: (casrep, (trace, crashline))
let mut casrep_info: RwLock<Vec<ReportInfo>> = RwLock::new(Vec::new());
// Casreps with stacktraces, that we cannot parse
let mut badreports: RwLock<Vec<PathBuf>> = RwLock::new(Vec::new());
custom_pool.install(|| {
(0..len).into_par_iter().for_each(|i| {
if let Ok(report) = report_from_file(casreps[i].as_path()) {
if let Ok(trace) = report.filtered_stacktrace() {
casrep_info
.write()
.unwrap()
.push((casreps[i].clone(), (trace, report.crashline)));
} else {
badreports.write().unwrap().push(casreps[i].clone());
}
} else {
badreports.write().unwrap().push(casreps[i].clone());
}
})
});
let casrep_info = casrep_info.get_mut().unwrap();
let badreports = badreports.get_mut().unwrap().to_vec();
// Sort by casrep filename
casrep_info.sort_by(|a, b| {
a.0.file_name()
.unwrap()
.to_str()
.unwrap()
.cmp(b.0.file_name().unwrap().to_str().unwrap())
});

(casrep_info.to_vec(), badreports)
}

/// Get `Cluster` structure from specified directory path.
///
/// # Arguments
///
/// * `dir` - valid cluster dir path
///
/// * `jobs` - number of jobs for parsing process
///
/// # Return value
///
/// `Cluster` structure
/// NOTE: Resulting cluster does not contains path info
pub fn load_cluster(dir: &Path, jobs: usize) -> Result<Cluster> {
// Get cluster number
let i = dir.file_name().unwrap().to_str().unwrap();
if i.len() < 3 {
bail!("Invalid cluster path: {}", &dir.display());
}
let i = i[2..].to_string().parse::<usize>()?;
// Get casreps from cluster
let casreps = get_reports(dir)?;
let (casreps, _) = reports_from_paths(&casreps, jobs);
let (_, (stacktraces, crashlines)): (Vec<_>, (Vec<_>, Vec<_>)) =
casreps.iter().cloned().unzip();
// Create cluster
Ok(Cluster::new(i, Vec::new(), stacktraces, crashlines))
}

/// Save clusters to given directory
///
/// # Arguments
///
/// * `clusters` - given `Cluster` structures for saving
///
/// * `dir` - out directory
pub fn save_clusters(clusters: &HashMap<usize, Cluster>, dir: &Path) -> Result<()> {
for cluster in clusters.values() {
fs::create_dir_all(format!("{}/cl{}", &dir.display(), cluster.number))?;
for casrep in cluster.paths() {
fs::copy(
casrep,
format!(
"{}/cl{}/{}",
&dir.display(),
cluster.number,
&casrep.file_name().unwrap().to_str().unwrap()
),
)?;
}
}
Ok(())
}

/// Save CASR reports to given directory
///
/// # Arguments
///
/// * `reports` - A vector of CASR reports
///
/// * `dir` - out directory
pub fn save_reports(reports: &Vec<PathBuf>, dir: &str) -> Result<()> {
if !Path::new(&dir).exists() {
fs::create_dir_all(dir)?;
}
for report in reports {
fs::copy(
report,
format!("{}/{}", dir, &report.file_name().unwrap().to_str().unwrap()),
)?;
}
Ok(())
}
87 changes: 87 additions & 0 deletions casr/tests/casr_tests/casrep/test_clustering_small/40.casrep
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
{
"Date": "2021-07-14T19:56:09.276635+03:00",
"Uname": "Linux titanfall 5.8.0-59-generic #66~20.04.1-Ubuntu SMP Thu Jun 17 11:14:10 UTC 2021 x86_64 x86_64 x86_64 GNU/Linux",
"OS": "Ubuntu",
"OSRelease": "20.04",
"Architecture": "amd64",
"ExecutablePath": "/usr/local/bin/tiff2pdf",
"ProcCmdline": "tiff2pdf ./fuz3tiff2pdf/main/crashes/id:000009,sig:06,src:000040+000049,time:43718,op:splice,rep:4",
"ProcMaps": [
" 0x555555554000 0x555555556000 0x2000 0x0 /usr/local/bin/tiff2pdf",
" 0x555555556000 0x555555561000 0xb000 0x2000 /usr/local/bin/tiff2pdf",
" 0x555555561000 0x555555565000 0x4000 0xd000 /usr/local/bin/tiff2pdf",
" 0x555555565000 0x555555566000 0x1000 0x10000 /usr/local/bin/tiff2pdf",
" 0x555555566000 0x555555567000 0x1000 0x11000 /usr/local/bin/tiff2pdf",
" 0x555555567000 0x555555588000 0x21000 0x0 [heap]",
" 0x7ffff7945000 0x7ffff7949000 0x4000 0x0 ",
" 0x7ffff7949000 0x7ffff7958000 0xf000 0x0 /usr/lib/x86_64-linux-gnu/libm-2.31.so",
" 0x7ffff7958000 0x7ffff79ff000 0xa7000 0xf000 /usr/lib/x86_64-linux-gnu/libm-2.31.so",
" 0x7ffff79ff000 0x7ffff7a96000 0x97000 0xb6000 /usr/lib/x86_64-linux-gnu/libm-2.31.so",
" 0x7ffff7a96000 0x7ffff7a97000 0x1000 0x14c000 /usr/lib/x86_64-linux-gnu/libm-2.31.so",
" 0x7ffff7a97000 0x7ffff7a98000 0x1000 0x14d000 /usr/lib/x86_64-linux-gnu/libm-2.31.so",
" 0x7ffff7a98000 0x7ffff7a9a000 0x2000 0x0 /usr/lib/x86_64-linux-gnu/libz.so.1.2.11",
" 0x7ffff7a9a000 0x7ffff7aab000 0x11000 0x2000 /usr/lib/x86_64-linux-gnu/libz.so.1.2.11",
" 0x7ffff7aab000 0x7ffff7ab1000 0x6000 0x13000 /usr/lib/x86_64-linux-gnu/libz.so.1.2.11",
" 0x7ffff7ab1000 0x7ffff7ab2000 0x1000 0x19000 /usr/lib/x86_64-linux-gnu/libz.so.1.2.11",
" 0x7ffff7ab2000 0x7ffff7ab3000 0x1000 0x19000 /usr/lib/x86_64-linux-gnu/libz.so.1.2.11",
" 0x7ffff7ab3000 0x7ffff7ab4000 0x1000 0x1a000 /usr/lib/x86_64-linux-gnu/libz.so.1.2.11",
" 0x7ffff7ab4000 0x7ffff7ab8000 0x4000 0x0 /usr/lib/x86_64-linux-gnu/libjpeg.so.8.2.2",
" 0x7ffff7ab8000 0x7ffff7afc000 0x44000 0x4000 /usr/lib/x86_64-linux-gnu/libjpeg.so.8.2.2",
" 0x7ffff7afc000 0x7ffff7b36000 0x3a000 0x48000 /usr/lib/x86_64-linux-gnu/libjpeg.so.8.2.2",
" 0x7ffff7b36000 0x7ffff7b37000 0x1000 0x82000 /usr/lib/x86_64-linux-gnu/libjpeg.so.8.2.2",
" 0x7ffff7b37000 0x7ffff7b38000 0x1000 0x82000 /usr/lib/x86_64-linux-gnu/libjpeg.so.8.2.2",
" 0x7ffff7b38000 0x7ffff7b39000 0x1000 0x83000 /usr/lib/x86_64-linux-gnu/libjpeg.so.8.2.2",
" 0x7ffff7b39000 0x7ffff7b44000 0xb000 0x0 /usr/lib/x86_64-linux-gnu/libjbig.so.0",
" 0x7ffff7b44000 0x7ffff7d43000 0x1ff000 0xb000 /usr/lib/x86_64-linux-gnu/libjbig.so.0",
" 0x7ffff7d43000 0x7ffff7d44000 0x1000 0xa000 /usr/lib/x86_64-linux-gnu/libjbig.so.0",
" 0x7ffff7d44000 0x7ffff7d47000 0x3000 0xb000 /usr/lib/x86_64-linux-gnu/libjbig.so.0",
" 0x7ffff7d47000 0x7ffff7d6c000 0x25000 0x0 /usr/lib/x86_64-linux-gnu/libc-2.31.so",
" 0x7ffff7d6c000 0x7ffff7ee4000 0x178000 0x25000 /usr/lib/x86_64-linux-gnu/libc-2.31.so",
" 0x7ffff7ee4000 0x7ffff7f2e000 0x4a000 0x19d000 /usr/lib/x86_64-linux-gnu/libc-2.31.so",
" 0x7ffff7f2e000 0x7ffff7f2f000 0x1000 0x1e7000 /usr/lib/x86_64-linux-gnu/libc-2.31.so",
" 0x7ffff7f2f000 0x7ffff7f32000 0x3000 0x1e7000 /usr/lib/x86_64-linux-gnu/libc-2.31.so",
" 0x7ffff7f32000 0x7ffff7f35000 0x3000 0x1ea000 /usr/lib/x86_64-linux-gnu/libc-2.31.so",
" 0x7ffff7f35000 0x7ffff7f39000 0x4000 0x0 ",
" 0x7ffff7f39000 0x7ffff7f41000 0x8000 0x0 /usr/local/lib/libtiff.so.3.9.6",
" 0x7ffff7f41000 0x7ffff7f76000 0x35000 0x8000 /usr/local/lib/libtiff.so.3.9.6",
" 0x7ffff7f76000 0x7ffff7f9f000 0x29000 0x3d000 /usr/local/lib/libtiff.so.3.9.6",
" 0x7ffff7f9f000 0x7ffff7fa0000 0x1000 0x66000 /usr/local/lib/libtiff.so.3.9.6",
" 0x7ffff7fa0000 0x7ffff7fa2000 0x2000 0x66000 /usr/local/lib/libtiff.so.3.9.6",
" 0x7ffff7fa2000 0x7ffff7fa3000 0x1000 0x68000 /usr/local/lib/libtiff.so.3.9.6",
" 0x7ffff7fa3000 0x7ffff7fa5000 0x2000 0x0 ",
" 0x7ffff7fc8000 0x7ffff7fc9000 0x1000 0x0 ",
" 0x7ffff7fc9000 0x7ffff7fcd000 0x4000 0x0 [vvar]",
" 0x7ffff7fcd000 0x7ffff7fcf000 0x2000 0x0 [vdso]",
" 0x7ffff7fcf000 0x7ffff7fd0000 0x1000 0x0 /usr/lib/x86_64-linux-gnu/ld-2.31.so",
" 0x7ffff7fd0000 0x7ffff7ff3000 0x23000 0x1000 /usr/lib/x86_64-linux-gnu/ld-2.31.so",
" 0x7ffff7ff3000 0x7ffff7ffb000 0x8000 0x24000 /usr/lib/x86_64-linux-gnu/ld-2.31.so",
" 0x7ffff7ffb000 0x7ffff7ffc000 0x1000 0x0 /home/avgor46/testdoc/fuz3tiff2pdf/main/crashes/id:000009,sig:06,src:000040+000049,time:43718,op:splice,rep:4",
" 0x7ffff7ffc000 0x7ffff7ffd000 0x1000 0x2c000 /usr/lib/x86_64-linux-gnu/ld-2.31.so",
" 0x7ffff7ffd000 0x7ffff7ffe000 0x1000 0x2d000 /usr/lib/x86_64-linux-gnu/ld-2.31.so",
" 0x7ffff7ffe000 0x7ffff7fff000 0x1000 0x0 ",
" 0x7ffffffde000 0x7ffffffff000 0x21000 0x0 [stack]",
" 0xffffffffff600000 0xffffffffff601000 0x1000 0x0 [vsyscall]"
],
"CrashSeverity": {
"Type": "NOT_CRITICAL",
"ShortDescription": "SafeFunctionCheck",
"Description": "Buffer overflow in safe function",
"Explanation": "The target stopped while handling a signal that was generated by libc due to detection of buffer overflow in safe copy function."
},
"Stacktrace": [
"#0 __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:50",
"#1 0x00007ffff7d6c859 in __GI_abort () at abort.c:79",
"#2 0x00007ffff7dd73ee in __libc_message (action=action@entry=do_abort, fmt=fmt@entry=0x7ffff7f0107c \"*** %s ***: terminated\\n\") at ../sysdeps/posix/libc_fatal.c:155",
"#3 0x00007ffff7e79b4a in __GI___fortify_fail (msg=msg@entry=0x7ffff7f01012 \"buffer overflow detected\") at fortify_fail.c:26",
"#4 0x00007ffff7e783e6 in __GI___chk_fail () at chk_fail.c:28",
"#5 0x00007ffff7dcf1cf in _IO_str_chk_overflow (fp=<optimized out>, c=<optimized out>) at iovsprintf.c:35",
"#6 0x00007ffff7da7db0 in __GI___printf_fp_l (fp=<optimized out>, loc=<optimized out>, info=<optimized out>, args=<optimized out>) at printf_fp.c:1246",
"#7 0x00007ffff7dc163a in __vfprintf_internal (s=s@entry=0x7fffffffe070, format=format@entry=0x5555555613df \"%.4f\", ap=ap@entry=0x7fffffffe1b0, mode_flags=mode_flags@entry=6) at vfprintf-internal.c:1687",
"#8 0x00007ffff7dcf279 in __vsprintf_internal (string=0x7fffffffe2a0 \"79725330432.000\", maxlen=<optimized out>, format=0x5555555613df \"%.4f\", args=args@entry=0x7fffffffe1b0, mode_flags=6) at iovsprintf.c:95",
"#9 0x00007ffff7e77edb in ___sprintf_chk (s=<optimized out>, flag=<optimized out>, slen=<optimized out>, format=<optimized out>) at sprintf_chk.c:40",
"#10 0x000055555555c7a1 in sprintf (__fmt=0x5555555613df \"%.4f\", __s=0x7fffffffe2a0 \"79725330432.000\") at /usr/include/x86_64-linux-gnu/bits/stdio2.h:36",
"#12 0x00005555555601b8 in t2p_write_pdf (output=0x555555568f80, input=0x555555567ea0, t2p=0x5555555672a0) at tiff2pdf.c:5175",
"#13 t2p_write_pdf (t2p=0x5555555672a0, input=0x555555567ea0, output=0x555555568f80) at tiff2pdf.c:5133",
"#14 0x00005555555568d4 in main (argc=<optimized out>, argv=<optimized out>) at tiff2pdf.c:763"
]
}
Loading

0 comments on commit df9eecf

Please sign in to comment.