Filter out generated samples that are empty & ignore samples that have any invalid parts (for decomposed setting).

Author: ewynx

test_suite/src/tester.rs

@@ -159,7 +159,8 @@ pub fn test_regex(regex_input: &DbEntry, code: &mut Code) -> anyhow::Result<Test
     let test_result = match &regex_input.samples_pass {
         SamplesPass::WithSubstrs(samples) => {
             // Random sample testing for substrings is only done for decomposed setting
-            let (random_samples_correct, incorrect_substring_random_test) = match &regex_input.regex {
+            let (random_samples_correct, incorrect_substring_random_test) = match &regex_input.regex
+            {
                 RegexInput::Decomposed(parts) => {
                     test_random_samples_gen_substrs(parts, regex_input.input_size as u32, code)?
                 }
@@ -306,38 +307,49 @@ fn test_random_samples_gen_substrs(
     for _ in 0..constants::DEFAULT_SAMPLE_NUMBER {
         let mut substrings = Vec::<String>::new();
         let mut total_string = String::new();
+        let mut valid_sample = true;
 
         for part in regex_parts {
             // Create the regex for this part
             let regex_part = Regex::compile(&part.regex_def, max_inputsize);
             let sample = match regex_part {
                 Ok(regex_part) => regex_part.sample(&mut rng),
                 Err(err) => {
-                    log::error!(
+                    log::info!(
                         "ignoring the random testing - 
                 the random samples were not generated due to the following error: {:?}",
                         err
                     );
+                    valid_sample = false;
                     String::new()
                 }
             };
 
-            if part.is_public {
-                substrings.push(sample.clone());
+            // If any of the parts wasn't generated correctly we consider it an invalid sample and
+            // it will be ignored
+            if valid_sample {
+                if part.is_public {
+                    substrings.push(sample.clone());
+                }
+                // Concatenate this sample to the total_string
+                total_string.push_str(&sample);
             }
-            // Concatenate this sample to the total_string
-            total_string.push_str(&sample);
         }
 
         let input_with_substring = InputWithSubstrs {
             input: total_string.clone(),
-            expected_substrings: substrings,
+            // Don't pass empty substrings
+            expected_substrings: substrings.into_iter().filter(|s| !s.is_empty()).collect(),
         };
-        let test_passed = run_single_gen_substr_test(code, &input_with_substring)?;
-        if test_passed {
-            random_samples_correct.push(total_string);
-        } else {
-            incorrect_substring_tests.push(total_string);
+
+        // Ignore empty samples
+        if !input_with_substring.input.is_empty() {
+            let test_passed = run_single_gen_substr_test(code, &input_with_substring)?;
+            if test_passed {
+                random_samples_correct.push(total_string);
+            } else {
+                incorrect_substring_tests.push(total_string);
+            }
         }
     }
 
@@ -361,14 +373,18 @@ fn test_for_random_samples(
                 .collect::<Vec<String>>()
         }
         Err(err) => {
-            log::error!(
+            log::info!(
                 "ignoring the random testing - 
                 the random samples were not generated due to the following error: {:?}",
                 err
             );
             Vec::new()
         }
-    };
+    }
+    .into_iter()
+    .filter(|s| !s.is_empty()) // Filter out empty strings
+    .collect();
+
     let (random_samples_correct, random_samples_wrong) =
         evaluate_test_set(code, &regex_input.regex.complete_regex(), &random_samples)?;
     Ok((random_samples_correct, random_samples_wrong))