Skip to content

Commit

Permalink
Refactor test utils to a separate dev crate and enable version 1 test…
Browse files Browse the repository at this point in the history 
…s without validation.
  • Loading branch information
@daemontus
daemontus committed Mar 6, 2024
1 parent a3969f4 commit c8b7ce4
Show file tree
Hide file tree
Showing 26 changed files with 338 additions and 395 deletions.
13 changes: 8 additions & 5 deletions Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,15 +6,18 @@ edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

[features]
# When enabled, rust the SBML syntactic test suite as part of unit tests.
# When enabled, run the SBML syntactic test suite as part of unit tests.
# This is mainly used for the purpose of code coverage computation.
sbml_test_suite = []

[dependencies]
const_format = "0.2.31"
macros = { path = "macros" }
phf = { version = "0.11.2", features = ["macros"] }
strum = "0.25.0"
strum_macros = "0.25"
xml-doc = { git = "https://github.com/daemontus/xml-doc" }
strum = "0.26"
strum_macros = "0.26"
regex = "1.10.3"
xml-doc = { git = "https://github.com/daemontus/xml-doc" }
sbml-macros = { path = "sbml-macros" }

[dev-dependencies]
sbml-test-suite = { path = "sbml-test-suite" }
6 changes: 5 additions & 1 deletion examples/basic_example.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,11 @@ use biodivine_lib_sbml::Sbml;
fn main() {
// let doc = Sbml::read_path("test-inputs/COVID19_immunotherapy_Mathematical_Model.xml").unwrap();
// let doc = Sbml::read_path("test-inputs/cholesterol_metabolism_and_atherosclerosis.xml").unwrap();
let doc = Sbml::read_path("test-inputs/Mukandavire2020.xml").unwrap();
//let doc = Sbml::read_path("test-inputs/Mukandavire2020.xml").unwrap();
let doc = Sbml::read_path(
"/Users/daemontus/Code/biodivine-lib-sbml/syntactic/21226/21226-fail-02-08-sev2-l3v2.xml",
)
.unwrap();

// let model = doc.model().get().unwrap();
// Print the whole document:
Expand Down
207 changes: 24 additions & 183 deletions examples/test-suite-syntactic.rs
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
use biodivine_lib_sbml::{Sbml, SbmlIssue, SbmlIssueSeverity};
use std::collections::{HashMap, HashSet};
use std::path::Path;
use biodivine_lib_sbml::{Sbml, SbmlIssueSeverity};
use sbml_test_suite::test_helper;
use std::collections::HashSet;

/// This is an integration test that uses the examples from the SBML test suite
/// to validate the functionality of the library.
Expand All @@ -23,7 +23,7 @@ fn main() {
None
};

let result = test_inner(filter);
let result = test_helper("./syntactic", filter, true, test_document);

let error_problems = result.error.clone();
let warning_problems = result.warning.clone();
Expand All @@ -50,185 +50,26 @@ fn main() {
assert!(info_problems.is_empty());
}

struct TestResults {
error: Vec<String>,
warning: Vec<String>,
info: Vec<String>,
}

/// A helper functions that actually runs the test.
fn test_inner(filter: Option<HashSet<String>>) -> TestResults {
let dir_path = "./syntactic";

if !Path::new(dir_path).is_dir() {
panic!("Test data is missing.")
}

if let Some(filter) = filter.as_ref() {
println!(
"Test suite restricted to {} rules: {:?}",
filter.len(),
filter
);
}

let test_issue = |id: &str| {
if let Some(filter) = filter.as_ref() {
filter.contains(id)
} else {
true
}
};

let mut tested = HashSet::new();

let mut error_problems = Vec::new();
let mut warning_problems = Vec::new();
let mut info_problems = Vec::new();

for rule_dir in std::fs::read_dir(dir_path).unwrap() {
let rule_dir = rule_dir.unwrap();
let name = rule_dir.file_name();
let name = name.to_str().unwrap();
if !rule_dir.path().is_dir() {
println!("Skipping file {} (not a directory).", name);
continue;
}
tested.insert(name.to_string());

let mut test_cases = Vec::new();
for test_file in std::fs::read_dir(rule_dir.path()).unwrap() {
let test_file = test_file.unwrap();
let test_name = test_file.file_name();
let test_name = test_name.to_str().unwrap();
if !test_name.ends_with(".xml") {
continue;
}
if !test_name.contains("l3v2") {
// Skip any tests that are not for SBML level 3 version 1.
continue;
}

test_cases.push(test_name.to_string());
}

println!("Found {} test cases for rule {}.", test_cases.len(), name);

for test_case in test_cases {
let mut test_file = rule_dir.path();
test_file.push(test_case.clone());
let mut result_file = rule_dir.path();
result_file.push(test_case.replace(".xml", ".txt"));

println!(" > Testing {:?}", test_file);
let mut expected = read_expected_issues(result_file.to_str().unwrap());

let Ok(doc) = Sbml::read_path(test_file.to_str().unwrap()) else {
// This process *can* fail if the document is not encoded correctly.
// In that case, rule 10101 should appear in the expected results. If it does
// not appear in this list, it is an error.
if !expected.contains_key(&"10101".to_string()) {
let report = format!(
"Test {}/{}: Found unexpected issue 10101 (severity Error).",
name, test_case
);
error_problems.push(report);
}
continue;
};

let issues: Vec<SbmlIssue> = doc.validate();

for issue in issues {
if test_issue(issue.rule.as_str()) {
if let Some(entry) = expected.get_mut(&issue.rule) {
entry.1 -= 1;
} else {
println!(
" >> Found issue {} that is not in the expected list: {}",
issue.rule, issue.message,
);
let report = format!(
"Test {}/{}: Found unexpected issue {} (severity {:?}).",
name, test_case, issue.rule, issue.severity
);
match issue.severity {
SbmlIssueSeverity::Error => error_problems.push(report),
SbmlIssueSeverity::Warning => warning_problems.push(report),
SbmlIssueSeverity::Info => info_problems.push(report),
};
}
}
}

for (id, (sev, count)) in expected {
if count == 0 {
// All issues of this type have been discovered.
continue;
}
if test_issue(id.as_str()) {
println!(" >> Missed expected issue {}.", id);
let report = format!(
"Test {}/{}: Missed issue {} (severity {:?}).",
name, test_case, id, sev,
);
match sev {
SbmlIssueSeverity::Error => error_problems.push(report),
SbmlIssueSeverity::Warning => warning_problems.push(report),
SbmlIssueSeverity::Info => info_problems.push(report),
};
}
}
/// A method that can be passed to `test_helper` to validate a document.
pub fn test_document(document: &str) -> Vec<(String, String)> {
match Sbml::read_str(document) {
Ok(doc) => doc
.validate()
.into_iter()
.map(|issue| {
let severity = match issue.severity {
SbmlIssueSeverity::Error => "Error",
SbmlIssueSeverity::Warning => "Warning",
SbmlIssueSeverity::Info => "Informational",
};
(issue.rule, severity.to_string())
})
.collect(),
Err(_e) => {
// This process *can* fail if the document is not encoded correctly.
// In that case, rule 10101 should appear in the expected results. If it does
// not appear in this list, it is an error.
vec![("10101".to_string(), "Error".to_string())]
}
}

if let Some(filter) = filter {
let missing = Vec::from_iter(filter.difference(&tested));
println!(
"WARNING: {} rules were requested but not found in the test suite: {:?}",
missing.len(),
missing
);
}

TestResults {
error: error_problems,
warning: warning_problems,
info: info_problems,
}
}

fn read_expected_issues(result_file: &str) -> HashMap<String, (SbmlIssueSeverity, usize)> {
let content = std::fs::read_to_string(result_file).unwrap();
let mut last_rule = None;
let mut result = HashMap::new();
for line in content.lines() {
let split = Vec::from_iter(line.split(':'));
if split.len() != 2 {
continue;
}
if split[0].trim() == "Validation id" {
assert!(last_rule.is_none());
last_rule = Some(split[1].trim().to_string());
}
if split[0].trim() == "Severity" {
assert!(last_rule.is_some());
let s = match split[1].trim() {
"Error" => SbmlIssueSeverity::Error,
"Warning" => SbmlIssueSeverity::Warning,
"Informational" => SbmlIssueSeverity::Info,
_ => {
panic!("Unknown severity {}", split[1].trim());
}
};
let key = last_rule.as_ref().unwrap().clone();
let entry = result.entry(key);
let value = entry.or_insert((s, 0));
assert_eq!(value.0, s);
value.1 += 1;
last_rule = None;
}
}

result
}
2 changes: 1 addition & 1 deletion macros/Cargo.toml → sbml-macros/Cargo.toml
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
[package]
name = "macros"
name = "sbml-macros"
version = "0.1.0"
edition = "2021"

Expand Down
2 changes: 1 addition & 1 deletion macros/README.md → sbml-macros/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,4 +2,4 @@

This crate provides useful macros for generating boilerplate code
for the main `lib-sbml` project. It is not intended to be published
or used independently on the main project.
or used independently of the main project.
0 macros/src/lib.rs → sbml-macros/src/lib.rs
View file
File renamed without changes.
8 changes: 8 additions & 0 deletions sbml-test-suite/Cargo.toml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
[package]
name = "sbml-test-suite"
version = "0.1.0"
edition = "2021"

# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

[dependencies]
9 changes: 9 additions & 0 deletions sbml-test-suite/README.md
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
# Test suite utils

This internal project contains utility code to run the official
SBML syntactic test suite. The code is then imported as a dev dependency
and used in the `test-suite-syntactic` example binary as well as
the `test-suite` validation module.

It's not super optimized or generalized, because it basically just
needs to work in this project alone.
Loading

0 comments on commit c8b7ce4

Please sign in to comment.