diff --git a/lib/bolero-engine/src/target_location.rs b/lib/bolero-engine/src/target_location.rs index 54af956f..533fb0df 100644 --- a/lib/bolero-engine/src/target_location.rs +++ b/lib/bolero-engine/src/target_location.rs @@ -175,7 +175,7 @@ impl TargetLocation { components.join("__") } - fn item_path(&self) -> String { + pub fn item_path(&self) -> String { Self::format_symbol_name(self.item_path) } diff --git a/lib/bolero/Cargo.toml b/lib/bolero/Cargo.toml index e933fe6c..14322054 100644 --- a/lib/bolero/Cargo.toml +++ b/lib/bolero/Cargo.toml @@ -21,7 +21,6 @@ arbitrary = ["bolero-generator/arbitrary"] bolero-engine = { version = "0.13", path = "../bolero-engine" } bolero-generator = { version = "0.13", path = "../bolero-generator", default-features = false } cfg-if = "1" - [target.'cfg(fuzzing_afl)'.dependencies] bolero-afl = { version = "0.13", path = "../bolero-afl" } diff --git a/lib/bolero/src/lib.rs b/lib/bolero/src/lib.rs index 3ccf4745..2c3867f8 100644 --- a/lib/bolero/src/lib.rs +++ b/lib/bolero/src/lib.rs @@ -17,7 +17,7 @@ cfg_if::cfg_if! { } else if #[cfg(kani)] { pub use bolero_kani::KaniEngine as DefaultEngine; } else { - mod test; + pub mod test; /// The default engine used when defining a test target pub use crate::test::TestEngine as DefaultEngine; diff --git a/lib/bolero/src/test/mod.rs b/lib/bolero/src/test/mod.rs index 892367da..35ecec7d 100644 --- a/lib/bolero/src/test/mod.rs +++ b/lib/bolero/src/test/mod.rs @@ -6,12 +6,28 @@ use bolero_engine::{ }; use core::{fmt, mem::size_of, time::Duration}; use std::path::PathBuf; - +use std::env; type ExhastiveDriver = Box>; +use std::collections::HashMap; +use std::sync::Mutex; + + + +mod outcome; mod input; mod report; + +/*pub fn event_with_payload(key: &str, value: T) { + let mut context = GLOBAL_CONTEXT.lock().unwrap(); + if let Some(map) = context.as_mut() { + map.insert(key.to_string(), value.to_string()); + } +} + +*/ + /// Engine implementation which mimics Rust's default test /// harness. By default, the test inputs will include any present /// `corpus` and `crashes` files, as well as generating @@ -154,19 +170,36 @@ impl TestEngine { driver: &mut driver, buffer: &mut buffer, }; + let mut representation = String::from(""); + let tyche_on = match env::var("BOLERO_TYCHE") { + Ok(v) => v, + Err(_) => "".to_string(), + }; + let result = match test.test(&mut input) { - Ok(is_valid) => Ok(is_valid), + Ok(is_valid) => { + // restart the driver to replay what was selected + if tyche_on != "" { + input.driver.replay(); + + + let value = test.generate_value(&mut input); + representation = format!("{:?}", value); + } + Ok((is_valid, representation)) + } Err(error) => { // restart the driver to replay what was selected input.driver.replay(); let input = test.generate_value(&mut input); + let representation = format!("{:?}", input); let error = Failure { seed: None, error, input, }; - Err(error.to_string()) + Err((error.to_string(), representation)) } }; @@ -191,52 +224,71 @@ impl TestEngine { file.read_into(&mut buffer); let mut input = input::Bytes::new(&buffer, file_options); - test.test(&mut input).map_err(|error| { - let shrunken = test.shrink(buffer.clone(), data.seed(), file_options); - - if let Some(shrunken) = shrunken { - format!("{:#}", shrunken) - } else { - format!( - "{:#}", - Failure { - seed: data.seed(), - error, - input: buffer.clone() - } - ) - } - }) + + + let result = test.test(&mut input); + // Generate a value for representation after the test + let mut repr_input = input::Bytes::new(&buffer, file_options); + let value = test.generate_value(&mut repr_input); + let representation = format!("{:?}", value); + + + + result.map(|is_valid| (is_valid, representation.clone())) + .map_err(|error| { + let shrunken = test.shrink(buffer.clone(), data.seed(), file_options); + + if let Some(shrunken) = shrunken { + (format!("{:#}", shrunken), representation) + } else { + (format!( + "{:#}", + (Failure { + seed: data.seed(), + error, + input: buffer.clone() + }) + ), representation) + } + }) } input::Test::Rng(conf) => { let mut input = conf.input(&mut buffer, &mut cache, rng_options); - test.test(&mut input).map_err(|error| { - let shrunken = if rng_options.shrink_time_or_default().is_zero() { - None - } else { - // reseed the input and buffer the rng for shrinking - let mut input = conf.buffered_input(&mut buffer, rng_options); - let _ = test.generate_value(&mut input); - - test.shrink(buffer.clone(), data.seed(), rng_options) - }; - - if let Some(shrunken) = shrunken { - format!("{:#}", shrunken) - } else { - buffer.clear(); - let mut input = conf.input(&mut buffer, &mut cache, rng_options); - let input = test.generate_value(&mut input); - format!( - "{:#}", - Failure { - seed: data.seed(), - error, - input - } - ) - } - }) + let result = test.test(&mut input); + + buffer.clear(); + let mut repr_input = conf.input(&mut buffer, &mut cache, rng_options); + let value = test.generate_value(&mut repr_input); + let representation = format!("{:?}", value); + + result.map(|is_valid| (is_valid, representation.clone())) + .map_err(|error| { + let shrunken = if rng_options.shrink_time_or_default().is_zero() { + None + } else { + // reseed the input and buffer the rng for shrinking + let mut input = conf.buffered_input(&mut buffer, rng_options); + let _ = test.generate_value(&mut input); + + test.shrink(buffer.clone(), data.seed(), rng_options) + }; + + if let Some(shrunken) = shrunken { + (format!("{:#}", shrunken), representation) + } else { + buffer.clear(); + let mut input = conf.input(&mut buffer, &mut cache, rng_options); + let input = test.generate_value(&mut input); + (format!( + "{:#}", + Failure { + seed: data.seed(), + error, + input + } + ),representation) + } + }) } } }; @@ -253,13 +305,17 @@ impl TestEngine { if options.exhaustive() { let testfn = |driver: ExhastiveDriver, test: &mut T| { let (driver, result) = bolero_engine::any::run(driver, test); - let result = result.map_err(|error| { - Failure { + let result = result.map(|r| { + // For scope tests, we don't have a good way to get a representation + // so we'll use a placeholder + (r, "".to_string()) + }).map_err(|error| { + (Failure { seed: None, error, input: (), } - .to_string() + .to_string(), "".to_string()) }); (driver, result) }; @@ -274,8 +330,6 @@ impl TestEngine { let rng_options = &rng_options; let mut buffer = vec![]; - // TODO - // let mut cache = driver::cache::Cache::default(); let file_driver = bolero_engine::driver::bytes::Driver::new(vec![], file_options); let file_driver = bolero_engine::driver::object::Object(file_driver); let file_driver = Box::new(file_driver); @@ -294,16 +348,16 @@ impl TestEngine { buffer = driver.reset(vec![], file_options); file_driver = Some(driver); - // TODO shrinking - - result.map_err(|error| { - Failure { - seed: None, - error, - input: (), // TODO figure out a better input to show - } - .to_string() - }) + // For scope tests, use a placeholder representation + result.map(|r| (r, "".to_string())) + .map_err(|error| { + (Failure { + seed: None, + error, + input: (), // TODO figure out a better input to show + } + .to_string(), "".to_string()) + }) } input::Test::Rng(conf) => { let seed = conf.seed; @@ -311,16 +365,16 @@ impl TestEngine { let driver = Box::new(Object(driver)); let (_driver, result) = bolero_engine::any::run(driver, test); - // TODO shrinking - - result.map_err(|error| { - Failure { - seed: Some(seed), - error, - input: (), // TODO figure out a better input to show - } - .to_string() - }) + // For scope tests, use a placeholder representation + result.map(|r| (r, "".to_string())) + .map_err(|error| { + (Failure { + seed: Some(seed), + error, + input: (), // TODO figure out a better input to show + } + .to_string(), "".to_string()) + }) } } }; @@ -330,7 +384,7 @@ impl TestEngine { fn run_tests(mut self, mut state: S, mut testfn: T) where - T: FnMut(&mut S, &input::Test) -> Result, + T: FnMut(&mut S, &input::Test) -> Result<(bool, String), (String, String)>, { // if we're fuzzing with cargo-bolero and the iteration count isn't specified // then go forever @@ -351,25 +405,58 @@ impl TestEngine { if cfg!(fuzzing_random) { report.spawn_timer(); } + let mut outcome = outcome::Outcome::new(&self.location, start_time); + let tyche_on = match env::var("BOLERO_TYCHE") { + Ok(v) => v, + Err(_) => "".to_string(), + }; + outcome.set_jsonpath(tyche_on.clone()); bolero_engine::panic::set_hook(); bolero_engine::panic::forward_panic(false); for input in tests { + if let Some(test_time) = test_time { if start_time.elapsed() > test_time { + outcome.on_exit(outcome::ExitReason::MaxDurationExceeded { + limit: test_time, + default: self.rng_cfg.test_time.is_none(), + }); break; } + } - progress(); + outcome.on_named_test(&input.data); + + match testfn(&mut state, &input.data){ + Ok((is_valid, representation)) => { + + + - match testfn(&mut state, &input.data) { - Ok(is_valid) => { report.on_result(is_valid); + outcome.set_representation(representation); + if tyche_on != "" { + let _ = outcome.output_json(); + } + + + } - Err(err) => { + Err((err, rep)) => { + + outcome.set_representation(rep); + + + + bolero_engine::panic::forward_panic(true); + outcome.on_exit(outcome::ExitReason::TestFailure); + if tyche_on != "" { + let _ = outcome.output_json(); + } eprintln!("{}", err); panic!("test failed"); } @@ -379,7 +466,7 @@ impl TestEngine { fn run_exhaustive(self, mut state: S, mut testfn: F, options: driver::Options) where - F: FnMut(ExhastiveDriver, &mut S) -> (ExhastiveDriver, Result), + F: FnMut(ExhastiveDriver, &mut S) -> (ExhastiveDriver, Result<(bool, String), (String, String)>), { bolero_engine::panic::set_hook(); bolero_engine::panic::forward_panic(false); @@ -392,6 +479,8 @@ impl TestEngine { let mut report = report::Report::default(); // when running exhaustive tests, it's nice to have the progress displayed report.spawn_timer(); + let _outcome = outcome::Outcome::new(&self.location, start_time); + while driver.step().is_continue() { if let Some(test_time) = test_time { @@ -404,11 +493,11 @@ impl TestEngine { driver = drvr; match result { - Ok(is_valid) => { + Ok((is_valid, _representation)) => { report.on_estimate(driver.estimate()); report.on_result(is_valid); } - Err(error) => { + Err((error, rep)) => { bolero_engine::panic::forward_panic(true); eprintln!("{error}"); panic!("test failed"); @@ -453,4 +542,4 @@ fn progress() { #[allow(clippy::explicit_write)] let _ = write!(stderr(), "."); } -} +} \ No newline at end of file diff --git a/lib/bolero/src/test/outcome.rs b/lib/bolero/src/test/outcome.rs new file mode 100644 index 00000000..008f6f7a --- /dev/null +++ b/lib/bolero/src/test/outcome.rs @@ -0,0 +1,201 @@ +use bolero_engine::TargetLocation; +use core::{fmt, time::Duration}; +use std::time::Instant; +use std::time::SystemTime; +use std::time::UNIX_EPOCH; +use std::collections::HashMap; +use std::io::Write; + +pub enum ExitReason { + MaxDurationExceeded { limit: Duration, default: bool }, + TestFailure, +} + +impl fmt::Display for ExitReason { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + ExitReason::MaxDurationExceeded { limit, default } => { + write!( + f, + "max duration ({:?}{}) exceeded", + limit, + if *default { " - default" } else { "" } + ) + } + ExitReason::TestFailure => write!(f, "test failure"), + } + } +} + +pub struct Outcome<'a> { + location: &'a TargetLocation, + start_time: Instant, + corpus_input: u64, + rng_input: u64, + exhaustive_input: u64, + total: u64, + exit_reason: Option, + features: String, + arguments: String, + coverage: String, + representation: String, + json_path: String, + json_time: std::time::Duration, +} + +impl fmt::Display for Outcome<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let runtime = self.start_time.elapsed(); + + if let Some(name) = self.location.test_name.as_ref() { + write!(f, "test {name} ...\t")?; + } else { + write!(f, "test {} ...\t", self.location.item_path())?; + } + + write!(f, "run time: {runtime:?} | ")?; + + let mut ips = self.total as f64 / runtime.as_secs_f64(); + if ips > 10.0 { + ips = ips.round(); + write!(f, "iterations/s: {ips}")?; + } else { + write!(f, "iterations/s: {ips:0.2}")?; + } + + for (label, count) in [ + ("corpus inputs", self.corpus_input), + ("rng inputs", self.rng_input), + ("exhaustive inputs", self.exhaustive_input), + ] { + if count > 0 { + write!(f, " | {label}: {count}")?; + } + } + + if let Some(reason) = &self.exit_reason { + write!(f, " | exit reason: {}", reason)?; + } + + Ok(()) + } +} + +impl<'a> Outcome<'a> { + pub fn new(location: &'a TargetLocation, start_time: Instant) -> Self { + Self { + location, + start_time, + corpus_input: 0, + rng_input: 0, + exhaustive_input: 0, + total: 0, + representation: String::from("{}"), + exit_reason: None, + features: String::from("{}"), + arguments: String::from("{}"), + coverage: String::from("{}"), + json_path: String::new(), + json_time: SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("Time went backwards"), + + } + } + + + pub fn on_named_test(&mut self, test: &super::input::Test) { + match test { + super::input::Test::Rng(_) => self.on_rng_input(), + super::input::Test::File(_) => self.on_corpus_input(), + } + } + + pub fn on_corpus_input(&mut self) { + progress(); + self.corpus_input += 1; + self.total += 1; + } + + pub fn on_rng_input(&mut self) { + progress(); + self.rng_input += 1; + self.total += 1; + } + + pub fn on_exhaustive_input(&mut self) { + self.exhaustive_input += 1; + self.total += 1; + } + + pub fn on_exit(&mut self, reason: ExitReason) { + self.exit_reason = Some(reason); + } + pub fn set_representation(&mut self, representation: String) { + self.representation = representation; + } + pub fn set_jsonpath(&mut self, json_path: String) { + self.json_path = json_path; + } + pub fn output_json(&self) -> std::io::Result<()>{ + let status = match &self.exit_reason { + Some(ExitReason::TestFailure) => "failed", + _ => "passed", + }; + + let status_reason = match &self.exit_reason { + Some(reason) => reason.to_string(), + None => String::new(), + }; + + let property = self.location.test_name.as_ref() + .map(|s| s.to_string()) + .unwrap_or_else(|| self.location.item_path.to_string()); + + let how_generated = "generated during unknown phase"; + + let metadata = String::from("{\"traceback\":null}"); + let file_name = self.json_path.clone(); + + let output_string = format!("{{\"type\":\"{typ}\",\ + \"run_start\":{run_start},\ + \"property\":\"{prop}\",\ + \"status\":\"{status}\",\ + \"status_reason\":\"{sr}\",\ + \"representation\":\"{rep}\",\ + \"arguments\":{arg},\ + \"how_generated\":\"{hg}\",\ + \"features\":{feat},\ + \"metadata\":{meta},\ + \"coverage\":{cov}}}", typ="test_case", run_start=self.json_time.as_secs().to_string(),prop=property, + sr= status_reason, rep=self.representation, arg=self.arguments, hg=how_generated, feat=self.features, + meta=metadata, cov=self.coverage); + let file = std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(file_name)?; + + let mut buffered_writer = std::io::BufWriter::new(file); + writeln!(buffered_writer, "{}", output_string); + let _ = buffered_writer.flush(); + Ok(()) + + } +} + + +impl Drop for Outcome<'_> { + fn drop(&mut self) { + eprintln!("{}", self.to_string()); + } +} + +fn progress() { + if cfg!(miri) { + use std::io::{stderr, Write}; + + // miri doesn't capture explicit writes to stderr + #[allow(clippy::explicit_write)] + let _ = write!(stderr(), "."); + } +}