From 13d450251575c40d81e155c29985cb481ae94a6d Mon Sep 17 00:00:00 2001 From: Hugo C <911307+hugocaillard@users.noreply.github.com> Date: Tue, 6 Jun 2023 09:17:17 +0200 Subject: [PATCH] feat: improve clarity test code coverage and handle code branches (#1030) * feat: improve clarity test code coverate and handle code branches * refactor: improve coverage unit tests * chore: ugprade dprint-plugin-json dependency --- .gitignore | 1 + Cargo.lock | 40 +- README.md | 4 +- components/clarinet-cli/Cargo.toml | 2 +- .../clarity-repl/src/analysis/coverage.rs | 313 ++++++---- .../src/analysis/coverage_tests.rs | 550 ++++++++++++++++++ components/clarity-repl/src/analysis/mod.rs | 2 + docs/how-to-guides/how-to-test-contract.md | 4 +- 8 files changed, 788 insertions(+), 128 deletions(-) create mode 100644 components/clarity-repl/src/analysis/coverage_tests.rs diff --git a/.gitignore b/.gitignore index 175c1dd27..0de6610ca 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,4 @@ components/stacks-devnet-js/build *.tar.gz *.zip *.rdb +*.lcov diff --git a/Cargo.lock b/Cargo.lock index f6048b8df..a9120d367 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -606,9 +606,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.12.0" +version = "3.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535" +checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1" [[package]] name = "byte-tools" @@ -861,7 +861,7 @@ dependencies = [ "hyper", "import_map", "indexmap", - "jsonc-parser", + "jsonc-parser 0.20.0", "lazy_static", "libc", "libsecp256k1 0.7.1", @@ -2046,17 +2046,31 @@ dependencies = [ "serde", ] +[[package]] +name = "dprint-core" +version = "0.62.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d966e6047321db5f011567c1819b89972a457ab49c2f4b56f074e67a59214112" +dependencies = [ + "anyhow", + "bumpalo", + "indexmap", + "rustc-hash", + "serde", + "unicode-width", +] + [[package]] name = "dprint-plugin-json" -version = "0.15.4" +version = "0.17.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db127f7ccb9b497b5b32e5e8eca4b19a7f191e38a3505195f029d5fbb728e51a" +checksum = "00905c12671f1be023a8e12915b97a701a6561bacf39221ad314884c99f55c74" dependencies = [ "anyhow", - "dprint-core", - "jsonc-parser", + "dprint-core 0.62.0", + "jsonc-parser 0.21.1", "serde", - "text_lines 0.4.1", + "text_lines 0.6.0", ] [[package]] @@ -2066,7 +2080,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "749753ef284b5eea8ab11e8baf01a735351c83747cdd72c5913e2351e6b8a309" dependencies = [ "anyhow", - "dprint-core", + "dprint-core 0.58.3", "pulldown-cmark", "regex", "serde", @@ -2080,7 +2094,7 @@ checksum = "e3a4ce966b327d5eba1df51bd9b0e373a741317f35a00abe7a71b01cfd582261" dependencies = [ "anyhow", "deno_ast", - "dprint-core", + "dprint-core 0.58.3", "rustc-hash", "serde", ] @@ -3166,6 +3180,12 @@ dependencies = [ "serde_json", ] +[[package]] +name = "jsonc-parser" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b56a20e76235284255a09fcd1f45cf55d3c524ea657ebd3854735925c57743d" + [[package]] name = "jsonrpc" version = "0.12.1" diff --git a/README.md b/README.md index b04f76c68..0670fa87a 100644 --- a/README.md +++ b/README.md @@ -385,8 +385,8 @@ From there, you can use the `lcov` tooling suite to produce HTML reports: ```bash $ brew install lcov -$ genhtml coverage.lcov -$ open index.html +$ genhtml --branch-coverage -o coverage coverage.lcov +$ open coverage/index.html ``` ![lcov](docs/images/lcov.png) diff --git a/components/clarinet-cli/Cargo.toml b/components/clarinet-cli/Cargo.toml index 7a76168ee..be6f0cde3 100644 --- a/components/clarinet-cli/Cargo.toml +++ b/components/clarinet-cli/Cargo.toml @@ -88,7 +88,7 @@ sys-info = "0.9.1" zstd = '=0.11.1' semver-parser = "=0.10.2" netif = "0.1.3" -dprint-plugin-json = "=0.15.4" +dprint-plugin-json = "=0.17.3" dprint-plugin-markdown = "=0.13.3" dprint-plugin-typescript = "=0.71.1" signal-hook-registry = "1.4.0" diff --git a/components/clarity-repl/src/analysis/coverage.rs b/components/clarity-repl/src/analysis/coverage.rs index 070f413af..da97997c6 100644 --- a/components/clarity-repl/src/analysis/coverage.rs +++ b/components/clarity-repl/src/analysis/coverage.rs @@ -1,16 +1,18 @@ use std::{ collections::{BTreeMap, BTreeSet, HashMap, HashSet}, + convert::{TryFrom, TryInto}, fs::{create_dir_all, File}, io::{Error, ErrorKind, Write}, mem, path::{Path, PathBuf}, }; -use clarity::vm::ast::ContractAST; -use clarity::vm::functions::define::DefineFunctionsParsed; -use clarity::vm::representations::SymbolicExpression; -use clarity::vm::types::QualifiedContractIdentifier; -use clarity::vm::EvalHook; +use clarity::vm::{ + ast::ContractAST, + functions::{define::DefineFunctionsParsed, NativeFunctions}, + types::QualifiedContractIdentifier, + EvalHook, SymbolicExpression, +}; use serde_json::Value as JsonValue; #[derive(Serialize, Deserialize, Debug, Default, Clone)] @@ -20,17 +22,15 @@ pub struct CoverageReporter { pub contract_paths: BTreeMap, } +type ExprCoverage = HashMap; +type ExecutableLines = HashMap>; +type ExecutableBranches = HashMap>; + +/// One `TestCoverageReport` per test file. #[derive(Serialize, Deserialize, Debug, Default, Clone)] pub struct TestCoverageReport { pub test_name: String, - pub contracts_coverage: HashMap, -} - -#[derive(Serialize, Deserialize, Debug, Default, Clone)] -pub struct ContractCoverageReport { - functions_coverage: HashMap, - execution_counts: HashMap, - executed_statements: BTreeSet, + pub contracts_coverage: HashMap, } pub fn parse_coverage_str(path: &str) -> Result { @@ -42,6 +42,17 @@ pub fn parse_coverage_str(path: &str) -> Result { } } +// LCOV format: +// TN: test name +// SF: source file path +// FN: line number,function name +// FNF: number functions found +// FNH: number functions hit +// DA: line data: line number, hit count +// BRF: number branches found +// BRH: number branches hit +// BRDA: branch data: line number, expr_id, branch_nb, hit count + impl CoverageReporter { pub fn new() -> CoverageReporter { CoverageReporter { @@ -51,22 +62,27 @@ impl CoverageReporter { } } - pub fn register_contract(&mut self, contract_name: String, contract_path: String) { - self.contract_paths.insert(contract_name, contract_path); - } - - pub fn add_asts(&mut self, asts: &BTreeMap) { - self.asts.append(&mut asts.clone()); - } - - pub fn add_reports(&mut self, reports: &Vec) { - self.reports.append(&mut reports.clone()); - } - pub fn write_lcov_file + Copy>( &self, filename: P, ) -> std::io::Result<()> { + let filepath = filename.as_ref().to_path_buf(); + let filepath = filepath.parent().ok_or(Error::new( + ErrorKind::NotFound, + "could not get directory to create coverage file", + ))?; + create_dir_all(filepath)?; + let mut out = File::create(filename)?; + let content = self.build_lcov_file(); + + write!(out, "{}", content)?; + + Ok(()) + } + + pub fn build_lcov_file(&self) -> String { + let mut file_content = String::new(); + let mut filtered_asts = HashMap::new(); for (contract_id, ast) in self.asts.iter() { let contract_name = contract_id.name.to_string(); @@ -76,7 +92,7 @@ impl CoverageReporter { ( contract_id, self.retrieve_functions(&ast.expressions), - self.filter_executable_lines(&ast.expressions), + self.retrieve_executable_lines_and_branches(&ast.expressions), ), ); } @@ -87,43 +103,44 @@ impl CoverageReporter { test_names.insert(report.test_name.to_string()); } - let filepath = filename.as_ref().to_path_buf(); - let filepath = filepath.parent().ok_or(Error::new( - ErrorKind::NotFound, - "could not get directory to create coverage file", - ))?; - create_dir_all(filepath)?; - let mut out = File::create(filename)?; - for (index, test_name) in test_names.iter().enumerate() { for (contract_name, contract_path) in self.contract_paths.iter() { - writeln!(out, "TN:{}", test_name)?; - writeln!(out, "SF:{}", contract_path)?; + file_content.push_str(&format!("TN:{}\n", test_name)); + file_content.push_str(&format!("SF:{}\n", contract_path)); - if let Some((contract_id, functions, executable_lines)) = - filtered_asts.get(contract_name) + if let Some((contract_id, functions, executable)) = filtered_asts.get(contract_name) { for (function, line_start, line_end) in functions.iter() { - writeln!(out, "FN:{},{}", line_start, function)?; + file_content.push_str(&format!("FN:{},{}\n", line_start, function)); } + let (executable_lines, executables_branches) = executable; let mut function_hits = BTreeMap::new(); - let mut consolidated_execution_counts = BTreeMap::new(); + let mut line_execution_counts = BTreeMap::new(); + + let mut branches = HashSet::new(); + let mut branches_hits = HashSet::new(); + let mut branch_execution_counts = BTreeMap::new(); + for report in self.reports.iter() { if &report.test_name == test_name { - if let Some(contract) = report.contracts_coverage.get(contract_id) { + if let Some(coverage) = report.contracts_coverage.get(contract_id) { let mut local_function_hits = BTreeSet::new(); - for line in executable_lines.iter() { - let count = contract.execution_counts.get(line).unwrap_or(&0); - - if let Some(line_count) = - consolidated_execution_counts.get_mut(line) - { - *line_count += *count; - } else { - consolidated_execution_counts.insert(*line, *count); + for (line, expr_ids) in executable_lines.iter() { + // in case of code branches on the line + // retrieve the expression with the most hits + let mut counts = vec![]; + for id in expr_ids { + if let Some(c) = coverage.get(id) { + counts.push(c.clone()); + } } + let count = counts.iter().max().unwrap_or(&0); + + let total_count = + line_execution_counts.entry(line).or_insert(0); + *total_count += count; if count == &0 { continue; @@ -136,32 +153,55 @@ impl CoverageReporter { } } - for function in local_function_hits.into_iter() { - if let Some(total_hit) = function_hits.get_mut(function) { - *total_hit += 1; - } else { - function_hits.insert(function, 1); + for (expr_id, args) in executables_branches.iter() { + for (i, (line, arg_expr_id)) in args.iter().enumerate() { + let count = coverage.get(arg_expr_id).unwrap_or(&0); + + branches.insert(arg_expr_id); + if count > &0 { + branches_hits.insert(arg_expr_id); + } + + let total_count = branch_execution_counts + .entry((line, expr_id, i)) + .or_insert(0); + *total_count += count; } } + + for function in local_function_hits.into_iter() { + let hits = function_hits.entry(function).or_insert(0); + *hits += 1 + } } } } for (function, hits) in function_hits.iter() { - writeln!(out, "FNDA:{},{}", hits, function)?; + file_content.push_str(&format!("FNDA:{},{}\n", hits, function)); + } + file_content.push_str(&format!("FNF:{}\n", functions.len())); + file_content.push_str(&format!("FNH:{}\n", function_hits.len())); + + for (line_number, count) in line_execution_counts.iter() { + file_content.push_str(&format!("DA:{},{}\n", line_number, count)); } - writeln!(out, "FNF:{}", functions.len())?; - writeln!(out, "FNH:{}", function_hits.len())?; - for (line_number, count) in consolidated_execution_counts.iter() { - writeln!(out, "DA:{},{}", line_number, count)?; + file_content.push_str(&format!("BRF:{}\n", branches.len())); + file_content.push_str(&format!("BRH:{}\n", branches_hits.len())); + + for ((line, block_id, branch_nb), count) in branch_execution_counts.iter() { + file_content.push_str(&format!( + "BRDA:{},{},{},{}\n", + line, block_id, branch_nb, count + )); } } - writeln!(out, "end_of_record")?; + file_content.push_str("end_of_record\n"); } } - Ok(()) + file_content } fn retrieve_functions(&self, exprs: &Vec) -> Vec<(String, u32, u32)> { @@ -189,11 +229,16 @@ impl CoverageReporter { functions } - fn filter_executable_lines(&self, exprs: &Vec) -> Vec { - let mut lines = vec![]; - let mut lines_seen = HashSet::new(); + fn retrieve_executable_lines_and_branches( + &self, + exprs: &Vec, + ) -> (ExecutableLines, ExecutableBranches) { + let mut lines: ExecutableLines = HashMap::new(); + let mut branches: ExecutableBranches = HashMap::new(); + for expression in exprs.iter() { let mut frontier = vec![expression]; + while let Some(cur_expr) = frontier.pop() { // Only consider body functions if let Some(define_expr) = DefineFunctionsParsed::try_parse(cur_expr).ok().flatten() @@ -204,35 +249,81 @@ impl CoverageReporter { | DefineFunctionsParsed::ReadOnlyFunction { signature: _, body } => { frontier.push(body); } - DefineFunctionsParsed::BoundedFungibleToken { .. } => {} - DefineFunctionsParsed::Constant { .. } => {} - DefineFunctionsParsed::PersistedVariable { .. } => {} - DefineFunctionsParsed::NonFungibleToken { .. } => {} - DefineFunctionsParsed::UnboundedFungibleToken { .. } => {} - DefineFunctionsParsed::Map { .. } => {} - DefineFunctionsParsed::Trait { .. } => {} - DefineFunctionsParsed::UseTrait { .. } => {} - DefineFunctionsParsed::ImplTrait { .. } => {} + _ => {} } continue; } if let Some(children) = cur_expr.match_list() { + if let Some((func, args)) = try_parse_native_func(children) { + // handle codes branches + // (if, asserts!, and, or, match) + match func { + NativeFunctions::If | NativeFunctions::Asserts => { + let (_cond, args) = args.split_first().unwrap(); + branches.insert( + cur_expr.id, + args.iter() + .map(|a| { + let expr = extract_expr_from_list(a); + (expr.span.start_line, expr.id) + }) + .collect(), + ); + } + NativeFunctions::And | NativeFunctions::Or => { + branches.insert( + cur_expr.id, + args.iter() + .map(|a| { + let expr = extract_expr_from_list(a); + (expr.span.start_line, expr.id) + }) + .collect(), + ); + } + NativeFunctions::Match => { + // for match ignore bindings children - some, ok, err + if args.len() == 4 || args.len() == 5 { + let input = args.first().unwrap(); + let left_branch = args.get(2).unwrap(); + let right_branch = args.last().unwrap(); + + let match_branches = [left_branch, right_branch]; + branches.insert( + cur_expr.id, + match_branches + .iter() + .map(|a| { + let expr = extract_expr_from_list(a); + (expr.span.start_line, expr.id) + }) + .collect(), + ); + + frontier.extend([input]); + frontier.extend(match_branches); + } + continue; + } + _ => {} + }; + }; + // don't count list expressions as a whole, just their children frontier.extend(children); } else { let line = cur_expr.span.start_line; - if !lines_seen.contains(&line) { - lines_seen.insert(line); - lines.push(line); + if let Some(line) = lines.get_mut(&line) { + line.push(cur_expr.id); + } else { + lines.insert(line, vec![cur_expr.id]); } } } } - - lines.sort(); - lines + (lines, branches) } } @@ -248,16 +339,16 @@ impl TestCoverageReport { impl EvalHook for TestCoverageReport { fn will_begin_eval( &mut self, - env: &mut clarity::vm::contexts::Environment, - context: &clarity::vm::contexts::LocalContext, + env: &mut clarity::vm::Environment, + context: &clarity::vm::LocalContext, expr: &SymbolicExpression, ) { let contract = &env.contract_context.contract_identifier; let mut contract_report = match self.contracts_coverage.remove(contract) { Some(e) => e, - _ => ContractCoverageReport::new(), + _ => HashMap::new(), }; - contract_report.report_eval(expr); + report_eval(&mut contract_report, expr); self.contracts_coverage .insert(contract.clone(), contract_report); } @@ -267,43 +358,39 @@ impl EvalHook for TestCoverageReport { _env: &mut clarity::vm::Environment, _context: &clarity::vm::LocalContext, _expr: &SymbolicExpression, - _res: &core::result::Result, + _res: &Result, ) { } - fn did_complete( - &mut self, - _result: core::result::Result<&mut clarity::vm::ExecutionResult, String>, - ) { - } + fn did_complete(&mut self, _result: Result<&mut clarity::vm::ExecutionResult, String>) {} +} + +fn try_parse_native_func( + expr: &[SymbolicExpression], +) -> Option<(NativeFunctions, &[SymbolicExpression])> { + let (name, args) = expr.split_first()?; + let atom = name.match_atom()?; + let func = NativeFunctions::lookup_by_name(atom)?; + Some((func, args)) } -impl ContractCoverageReport { - pub fn new() -> ContractCoverageReport { - ContractCoverageReport { - functions_coverage: HashMap::new(), - execution_counts: HashMap::new(), - executed_statements: BTreeSet::new(), +fn report_eval(expr_coverage: &mut ExprCoverage, expr: &SymbolicExpression) { + if let Some(children) = expr.match_list() { + if let Some((function_variable, rest)) = children.split_first() { + report_eval(expr_coverage, function_variable); } + return; } - pub fn report_eval(&mut self, expr: &SymbolicExpression) { - if let Some(children) = expr.match_list() { - // Handle the function variable, then the rest of the list will be - // eval'ed later. - if let Some((function_variable, rest)) = children.split_first() { - self.report_eval(function_variable); - } - return; - } + let count = expr_coverage.entry(expr.id).or_insert(0); + *count += 1; +} - // other sexps can only span 1 line - let line_executed = expr.span.start_line; - if let Some(execution_count) = self.execution_counts.get_mut(&line_executed) { - *execution_count += 1; - } else { - self.execution_counts.insert(line_executed, 1); - } - self.executed_statements.insert(expr.id); +// because list expressions are not considered as evaluated +// this helpers returns evaluatable expr from list +fn extract_expr_from_list(expr: &SymbolicExpression) -> SymbolicExpression { + if let Some(first) = expr.match_list().and_then(|l| l.first()) { + return extract_expr_from_list(first); } + return expr.to_owned(); } diff --git a/components/clarity-repl/src/analysis/coverage_tests.rs b/components/clarity-repl/src/analysis/coverage_tests.rs new file mode 100644 index 000000000..66a71cd8f --- /dev/null +++ b/components/clarity-repl/src/analysis/coverage_tests.rs @@ -0,0 +1,550 @@ +use std::convert::TryInto; +use std::fmt::format; + +use super::coverage::{CoverageReporter, TestCoverageReport}; +use crate::repl::session::{self, Session}; +use crate::repl::{ + ClarityCodeSource, ClarityContract, ContractDeployer, SessionSettings, DEFAULT_CLARITY_VERSION, + DEFAULT_EPOCH, +}; + +fn get_coverage_report(contract: &str, snippets: Vec) -> (TestCoverageReport, String) { + let mut session = Session::new(SessionSettings::default()); + + let mut report = TestCoverageReport::new("test_scenario".into()); + let _ = session.eval(contract.into(), Some(vec![&mut report]), false); + for snippet in snippets { + let _ = session.eval(snippet.into(), Some(vec![&mut report]), false); + } + + let (contract_id, ast) = session.asts.pop_first().unwrap(); + let coverage_reporter = CoverageReporter::new(); + + let mut coverage_reporter = CoverageReporter::new(); + coverage_reporter + .asts + .insert(contract_id.clone(), ast.clone()); + coverage_reporter + .contract_paths + .insert(contract_id.name.to_string(), "/contract-0.clar".into()); + coverage_reporter.reports.append(&mut vec![report.clone()]); + + let lcov_content = coverage_reporter.build_lcov_file(); + + (report, lcov_content) +} + +fn get_expected_report(body: String) -> String { + return format!("TN:test_scenario\nSF:/contract-0.clar\n{body}\nend_of_record\n"); +} + +#[test] +fn line_is_executed() { + let contract = "(define-read-only (add) (+ 1 2))"; + let snippet = "(contract-call? .contract-0 add)"; + let (report, cov) = get_coverage_report(contract, vec![snippet.into()]); + + let expect = get_expected_report( + vec![ + "FN:1,add", + "FNDA:1,add", + "FNF:1", + "FNH:1", + "DA:1,1", + "BRF:0", + "BRH:0", + ] + .join("\n"), + ); + assert_eq!(cov, expect); +} + +#[test] +fn line_is_executed_twice() { + let contract = "(define-read-only (add) (+ 1 2))"; + // call it twice + let snippet = "(contract-call? .contract-0 add) (contract-call? .contract-0 add)"; + let (_, cov) = get_coverage_report(contract.into(), vec![snippet.into()]); + + let expect = get_expected_report( + vec![ + "FN:1,add", + "FNDA:1,add", + "FNF:1", + "FNH:1", + "DA:1,2", + "BRF:0", + "BRH:0", + ] + .join("\n"), + ); + assert_eq!(cov, expect); +} + +#[test] +fn line_count_in_iterator() { + let contract = vec![ + "(define-private (add-1 (n uint)) (+ n u1))", + "(define-public (map-add-1)", + " (ok (map add-1 (list u2 u3)))", + ")", + ] + .join("\n"); + let snippet = "(contract-call? .contract-0 map-add-1)"; + let (_, cov) = get_coverage_report(contract.as_str(), vec![snippet.into()]); + + let expect = get_expected_report( + vec![ + "FN:1,add-1", + "FN:2,map-add-1", + "FNDA:1,add-1", + "FNDA:1,map-add-1", + "FNF:2", + "FNH:2", + "DA:1,2", + "DA:3,1", + "BRF:0", + "BRH:0", + ] + .join("\n"), + ); + assert_eq!(cov, expect); +} + +#[test] +fn multiple_line_execution() { + let contract = vec![ + "(define-read-only (add)", + " (begin", + " (+ (+ 1 1) (+ 1 2))", + " (+ 1 2 3)", + " )", + ")", + ] + .join("\n"); + + let snippet = "(contract-call? .contract-0 add)"; + let (_, cov) = get_coverage_report(contract.as_str(), vec![snippet.into()]); + + let expect = get_expected_report( + vec![ + "FN:1,add", + "FNDA:1,add", + "FNF:1", + "FNH:1", + "DA:2,1", + "DA:3,1", + "DA:4,1", + "BRF:0", + "BRH:0", + ] + .join("\n"), + ); + assert_eq!(cov, expect); +} + +#[test] +fn let_binding() { + let contract = vec![ + "(define-public (add-print)", + " (let (", + " (c (+ 1 1))", + " )", + " (ok c)", + " )", + ")", + ] + .join("\n"); + + let snippet = "(contract-call? .contract-0 add-print)"; + let (_, cov) = get_coverage_report(contract.as_str(), vec![snippet.into()]); + + let expect = get_expected_report( + vec![ + "FN:1,add-print", + "FNDA:1,add-print", + "FNF:1", + "FNH:1", + "DA:2,1", + "DA:3,1", + "DA:5,1", + "BRF:0", + "BRH:0", + ] + .join("\n"), + ); + assert_eq!(cov, expect); +} + +#[test] +fn simple_if_branching() { + let contract = vec![ + "(define-read-only (one-or-two (one bool))", + " (if one 1 2)", + ")", + ] + .join("\n"); + + let expect_base = vec![ + "FN:1,one-or-two", + "FNDA:1,one-or-two", + "FNF:1", + "FNH:1", + "DA:2,1", + "BRF:2", + "BRH:1", + ]; + + // left path + let snippet = "(contract-call? .contract-0 one-or-two true)"; + let (_, cov) = get_coverage_report(contract.as_str(), vec![snippet.into()]); + + let expect = get_expected_report( + [&expect_base[..], &["BRDA:2,8,0,1", "BRDA:2,8,1,0"]] + .concat() + .join("\n"), + ); + assert_eq!(cov, expect); + + // right path + let snippet = "(contract-call? .contract-0 one-or-two false)"; + let (_, cov) = get_coverage_report(contract.as_str(), vec![snippet.into()]); + + let expect = get_expected_report( + [&expect_base[..], &["BRDA:2,8,0,0", "BRDA:2,8,1,1"]] + .concat() + .join("\n"), + ); + assert_eq!(cov, expect); +} + +#[test] +fn simple_if_branches_with_exprs() { + let contract = vec![ + "(define-read-only (add-or-sub (add bool))", + " (if add (+ 1 1) (- 1 1))", + ")", + ] + .join("\n"); + let snippet = "(contract-call? .contract-0 add-or-sub true)"; + let (report, cov) = get_coverage_report(contract.as_str(), vec![snippet.into()]); + + let expect = get_expected_report( + vec![ + "FN:1,add-or-sub", + "FNDA:1,add-or-sub", + "FNF:1", + "FNH:1", + "DA:2,1", + "BRF:2", + "BRH:1", + "BRDA:2,8,0,1", + "BRDA:2,8,1,0", + ] + .join("\n"), + ); + assert_eq!(cov, expect); +} + +#[test] +fn hit_all_if_branches() { + let contract = vec![ + "(define-read-only (add-or-sub (add bool))", + " (if add (+ 1 1) (- 1 1))", + ")", + ] + .join("\n"); + + // hit left branch 3 times and right branch 2 + let snippets: Vec = vec![ + "(contract-call? .contract-0 add-or-sub true)".into(), + "(contract-call? .contract-0 add-or-sub true)".into(), + "(contract-call? .contract-0 add-or-sub true)".into(), + "(contract-call? .contract-0 add-or-sub false)".into(), + "(contract-call? .contract-0 add-or-sub false)".into(), + ]; + let (report, cov) = get_coverage_report(&contract, snippets); + + let expect = get_expected_report( + vec![ + "FN:1,add-or-sub", + "FNDA:1,add-or-sub", + "FNF:1", + "FNH:1", + "DA:2,5", // 3 + 2 + "BRF:2", + "BRH:2", + "BRDA:2,8,0,3", + "BRDA:2,8,1,2", + ] + .join("\n"), + ); + assert_eq!(cov, expect); +} + +#[test] +fn simple_asserts_branching() { + let contract = vec![ + "(define-read-only (is-one (v int))", + " (ok (asserts! (is-eq v 1) (err u1)))", + ")", + ] + .join("\n"); + + // no hit on (err u1) + let snippets: Vec = vec!["(contract-call? .contract-0 is-one 1)".into()]; + let (report, cov) = get_coverage_report(&contract, snippets); + + let expect = get_expected_report( + vec![ + "FN:1,is-one", + "FNDA:1,is-one", + "FNF:1", + "FNH:1", + "DA:2,1", + "BRF:1", + "BRH:0", + "BRDA:2,10,0,0", + ] + .join("\n"), + ); + assert_eq!(cov, expect); + + // hit on (err u1) + let snippets: Vec = vec!["(contract-call? .contract-0 is-one 2)".into()]; + let (report, cov) = get_coverage_report(&contract, snippets); + + let expect = get_expected_report( + vec![ + "FN:1,is-one", + "FNDA:1,is-one", + "FNF:1", + "FNH:1", + "DA:2,1", + "BRF:1", + "BRH:1", + "BRDA:2,10,0,1", + ] + .join("\n"), + ); + assert_eq!(cov, expect); +} + +#[test] +fn branch_if_plus_and() { + let contract = vec![ + "(define-read-only (unecessary-ifs (v int))", + " (if (and (> v 0) (> v 1) (> v 2) (> v 3))", + " (ok \"greater\")", + " (ok \"lower\")", + " )", + ")", + ] + .join("\n"); + // calling with `2`, so that evualuation should stop at (> v 2) (which is false) + let snippet = "(contract-call? .contract-0 unecessary-ifs 2)"; + let (report, cov) = get_coverage_report(contract.as_str(), vec![snippet.into()]); + + let expect = get_expected_report( + vec![ + "FN:1,unecessary-ifs", + "FNDA:1,unecessary-ifs", + "FNF:1", + "FNH:1", + "DA:2,1", + "DA:3,0", // left if path + "DA:4,1", // right if path + "BRF:6", + "BRH:4", + "BRDA:2,10,0,1", + "BRDA:2,10,1,1", + "BRDA:2,10,2,1", + "BRDA:2,10,3,0", // (> v 3) not hit + "BRDA:3,8,0,0", // left if path not hit + "BRDA:4,8,1,1", + ] + .join("\n"), + ); + assert_eq!(cov, expect); +} + +#[test] +fn branch_if_plus_or() { + let contract = vec![ + "(define-read-only (unecessary-ors (v int))", + " (if (or (is-eq v 0) (is-eq v 1) (is-eq v 2) (is-eq v 3))", + " (ok \"match\")", + " (ok \"no match\")", + " )", + ")", + ] + .join("\n"); + // calling with 1, so that evualuation should stop at (is-eq v 1) + let snippet = "(contract-call? .contract-0 unecessary-ors 1)"; + let (report, cov) = get_coverage_report(contract.as_str(), vec![snippet.into()]); + + let expect = get_expected_report( + vec![ + "FN:1,unecessary-ors", + "FNDA:1,unecessary-ors", + "FNF:1", + "FNH:1", + "DA:2,1", + "DA:3,1", // left if path + "DA:4,0", // right if path + "BRF:6", + "BRH:3", + "BRDA:2,10,0,1", + "BRDA:2,10,1,1", // stop + "BRDA:2,10,2,0", + "BRDA:2,10,3,0", + "BRDA:3,8,0,1", + "BRDA:4,8,1,0", + ] + .join("\n"), + ); + assert_eq!(cov, expect); +} + +#[test] +fn match_opt_oneline() { + let contract = vec![ + "(define-public (match-opt (opt? (optional int)))", + " (match opt? opt (ok opt) (err u1))", + ")", + ] + .join("\n"); + + let expect_base = [ + "FN:1,match-opt", + "FNDA:1,match-opt", + "FNF:1", + "FNH:1", + "DA:2,1", + "BRF:2", + "BRH:1", + ]; + + // left path + let snippets: Vec = vec!["(contract-call? .contract-0 match-opt (some 1))".into()]; + let (report, cov) = get_coverage_report(&contract, snippets); + + let expect = get_expected_report( + vec![&expect_base[..], &["BRDA:2,10,0,1", "BRDA:2,10,1,0"]] + .concat() + .join("\n"), + ); + assert_eq!(cov, expect); + + // right path + let snippets: Vec = vec!["(contract-call? .contract-0 match-opt none)".into()]; + let (report, cov) = get_coverage_report(&contract, snippets); + + let expect = get_expected_report( + vec![&expect_base[..], &["BRDA:2,10,0,0", "BRDA:2,10,1,1"]] + .concat() + .join("\n"), + ); + assert_eq!(cov, expect); +} + +#[test] +fn match_opt_multiline() { + let contract = vec![ + "(define-public (match-opt (opt? (optional int)))", + " (match opt?", + " opt", + " (ok opt)", + " (err u1)", + " )", + ")", + ] + .join("\n"); + + let expect_base = [ + "FN:1,match-opt", + "FNDA:1,match-opt", + "FNF:1", + "FNH:1", + "DA:2,1", + ]; + + // left path + let snippets: Vec = vec!["(contract-call? .contract-0 match-opt (some 1))".into()]; + let (report, cov) = get_coverage_report(&contract, snippets); + + let expect = get_expected_report( + vec![ + &expect_base[..], + &[ + "DA:4,1", + "DA:5,0", + "BRF:2", + "BRH:1", + "BRDA:4,10,0,1", + "BRDA:5,10,1,0", + ], + ] + .concat() + .join("\n"), + ); + assert_eq!(cov, expect); + + // right path + let snippets: Vec = vec!["(contract-call? .contract-0 match-opt none)".into()]; + let (report, cov) = get_coverage_report(&contract, snippets); + + let expect = get_expected_report( + vec![ + &expect_base[..], + &[ + "DA:4,0", + "DA:5,1", + "BRF:2", + "BRH:1", + "BRDA:4,10,0,0", + "BRDA:5,10,1,1", + ], + ] + .concat() + .join("\n"), + ); + assert_eq!(cov, expect); +} + +#[test] +fn match_res_oneline() { + // very similar to match opt + // lighter test strategy, only test one liner and call both paths in same session + let contract = vec![ + "(define-public (match-res (res (response int uint)))", + " (match res o (ok o) e (err e))", + ")", + ] + .join("\n"); + + // call left path twice and right path once + let snippets: Vec = vec![ + "(contract-call? .contract-0 match-res (ok 1))".into(), + "(contract-call? .contract-0 match-res (ok 2))".into(), + "(contract-call? .contract-0 match-res (err u1))".into(), + ]; + let (report, cov) = get_coverage_report(&contract, snippets); + + let expect = get_expected_report( + vec![ + "FN:1,match-res", + "FNDA:1,match-res", + "FNF:1", + "FNH:1", + "DA:2,3", + "BRF:2", + "BRH:2", + "BRDA:2,11,0,2", + "BRDA:2,11,1,1", + ] + .join("\n"), + ); + + assert_eq!(cov, expect); +} diff --git a/components/clarity-repl/src/analysis/mod.rs b/components/clarity-repl/src/analysis/mod.rs index b21d76317..2a1aab5fe 100644 --- a/components/clarity-repl/src/analysis/mod.rs +++ b/components/clarity-repl/src/analysis/mod.rs @@ -4,6 +4,8 @@ pub mod ast_visitor; pub mod call_checker; pub mod check_checker; pub mod coverage; +#[cfg(test)] +mod coverage_tests; use serde::de::Deserialize; use serde::Serialize; diff --git a/docs/how-to-guides/how-to-test-contract.md b/docs/how-to-guides/how-to-test-contract.md index 8a1699872..b255790c6 100644 --- a/docs/how-to-guides/how-to-test-contract.md +++ b/docs/how-to-guides/how-to-test-contract.md @@ -175,8 +175,8 @@ From there, you can use the `lcov` tooling suite to produce HTML reports. ```bash $ brew install lcov -$ genhtml coverage.lcov -$ open index.html +$ genhtml --branch-coverage -o coverage coverage.lcov +$ open coverage/index.html ``` ![lcov](../images/lcov.png)