Fail CI check if 'record_soft_failure' without valid reference

poo#115343

CONTRIBUTING.md

@@ -126,6 +126,16 @@ if (match_has_tag('yast2_missing_package')) {
     assert_screen 'yast2_console-finished';
 }
 ```
+* Please add a bug/ticket reference for `record_soft_failure`, otherwise
+  CI checks may fail. you can use formats like below:
+  bsc#12345 -> Bugzilla bug
+  poo#12345 -> Progress ticket
+  jsc#SLE-19640 -> Jira ticket
+  Maniphest#T5531
+  fate.suse.com/123
+  If you don't have a reference ticket, and still  want to mark a specific
+  step as soft_fail, please use `record_info` with softfail tag:
+  record_info($title [, $output] [, result => softfail] [, resultname => $resultname]);
 
 ### Preparing a new Pull Request
 * All code needs to be tidy, for this use `make prepare` the first time you

Makefile

@@ -152,7 +152,7 @@ test-deleted-renamed-referenced-files:
 
 .PHONY: test-soft_failure-no-reference
 test-soft_failure-no-reference:
-	@! git --no-pager grep -E -e 'soft_failure\>.*\;' --and --not -e '([$$0-9a-z]+#[$$0-9a-zA-Z]+|fate.suse.com/[0-9]|\$$[a-z]+)' lib/ tests/
+	@! git --no-pager grep -E -e 'record_soft_failure\>.*\;' --and --not -e '([a-zA-Z]+#[a-zA-Z-]*[0-9]+|fate.suse.com/[0-9]+)' lib/ tests/
 
 .PHONY: test-invalid-syntax
 test-invalid-syntax:

lib/audit_test.pm

@@ -174,14 +174,14 @@ sub _parse_results_with_diff_baseline {
     my ($name, $result, $msg, $flag) = @_;
     my $softfail_tests = {};
     if ($result eq 'PASS') {
-        record_soft_failure($msg);
+        record_info($msg, result => 'softfail');
         $flag = 'softfail' if ($flag ne 'fail');
     }
     else {
         my $arch = get_var('ARCH');
         if ($softfail_tests->{$arch}) {
             if (my $reason = $softfail_tests->{$arch}->{$name}) {
-                record_soft_failure($msg . "\n" . $reason);
+                record_info($msg . "\n" . $reason, result => 'softfail');
                 return 'softfail';
             }
         }

lib/containers/utils.pm

@@ -217,8 +217,8 @@ sub basic_container_tests {
     die("error: $runtime image rmi -a $leap") if ($output_containers =~ m/Untagged:.*opensuse\/leap/);
     die("error: $runtime image rmi -a $tumbleweed") if ($output_containers =~ m/Untagged:.*opensuse\/tumbleweed/);
     die("error: $runtime image rmi -a tw:saved") if ($output_containers =~ m/Untagged:.*tw:saved/);
-    record_soft_failure("error: $runtime image rmi -a $alpine") if ($output_containers =~ m/Untagged:.*alpine/);
-    record_soft_failure("error: $runtime image rmi -a $hello_world:latest") if ($output_containers =~ m/Untagged:.*hello-world:latest/);
+    record_info("error: $runtime image rmi -a $alpine", result => 'softfail') if ($output_containers =~ m/Untagged:.*alpine/);
+    record_info("error: $runtime image rmi -a $hello_world:latest", result => 'softfail') if ($output_containers =~ m/Untagged:.*hello-world:latest/);
 }
 
 =head2 can_build_sle_base

lib/known_bugs.pm

@@ -143,7 +143,7 @@ sub upload_journal {
                 $regexp_matched{$regexp} = 1;
                 my $fail_type = 'softfail';
                 if ($type eq 'hard') {
-                    record_soft_failure $message. "\n\n" . "$line";
+                    record_info $message. "\n\n" . "$line", result => 'softfail';
                 }
                 elsif ($type eq 'soft') {
                     force_soft_failure $message. "\n\n" . "$line";

lib/opensusebasetest.pm

@@ -386,7 +386,7 @@ sub investigate_yast2_failure {
             # Save detected error to indetify if have new regressions
             push @detected_errors, $y2log_error;
             if (my $bug = $y2log_errors{$y2log_error}) {
-                record_soft_failure("$bug\n\nDetails:\n$y2log_error_result");
+                record_info("$bug\n\nDetails:\n$y2log_error_result", result => 'softfail');
                 next;
             }
             $detected_errors_detailed .= "$y2log_error_result\n\n$delimiter\n\n";

lib/power_action_utils.pm

@@ -433,7 +433,7 @@ sub assert_shutdown_with_soft_timeout {
         my $ret = check_shutdown $args->{soft_timeout};
         return if $ret;
         $args->{soft_failure_reason} //= "$args->{bugref}: Machine didn't shut down within $args->{soft_timeout} sec";
-        record_soft_failure "$args->{soft_failure_reason}";
+        record_info("$args->{soft_failure_reason}", result => 'softfail');
     }
     assert_shutdown($args->{timeout} - $args->{soft_timeout});
 }

lib/trento.pm

@@ -602,7 +602,7 @@ sub cypress_exec {
         $self->result("fail");
     }
     if ($failok) {
-        record_soft_failure("Cypress exit code:$ret at $log_prefix") if ($ret);
+        record_info("Cypress exit code:$ret at $log_prefix", result => 'softfail') if ($ret);
         $ret = 0;
     }
     die "Cypress exec error at '$cmd'" unless ($ret == 0);

lib/utils.pm

@@ -1063,7 +1063,7 @@ sub assert_screen_with_soft_timeout {
         die "soft timeout has to be smaller than timeout" unless ($args{soft_timeout} < $args{timeout});
         my $ret = check_screen $mustmatch, $args{soft_timeout};
         return $ret if $ret;
-        record_soft_failure "$args{soft_failure_reason}";
+        record_info("$args{soft_failure_reason}", result => 'softfail');
     }
     return assert_screen $mustmatch, $args{timeout} - $args{soft_timeout};
 }
@@ -2350,7 +2350,7 @@ sub package_upgrade_check {
             die "Error: package $pkg_name is not upgraded yet, please check with developer";
         }
         else {
-            record_soft_failure "Warning: package $pkg_name is not upgraded yet";
+            record_info("Warning: package $pkg_name is not upgraded yet", result => 'softfail');
         }
     }
 }

lib/virt_autotest/kernel.pm

@@ -51,7 +51,7 @@ sub check_virt_kernel {
 
     my $dmesg = "dmesg | grep -i 'fail\\|error\\|segmentation\\|stack\\|buffer' | grep -vi 'acpi\\|ERST\\|bar\\|mouse\\|vesafb\\|firmware\\|calibration\\|thermal\\|Correctable Errors\\|calibration failed\\|PM-Timer\\|dmi\\|irqstacks\\|auto-init\\|TSC ADJUST\\|xapic not enabled\\|Firmware\\|missing monitors config\\|perfctr\\|mitigation\\|vesa\\|ram buffer\\|microcode\\|frame\\|nmi\\|pci-dma\\|pm-timer\\|tsc\\|drm\\|hv_vmbus\\|floppy\\|fd0\\|nmi\\|x2apic\\|show_stack\\|dump_stack\\|pstore\\|pagetables\\|page allocation failure\\|amd64_edac_mod\\|FW version\\|Failed to check link status\\|task [0-9a-zA-Z]{4,16} task.stack: [0-9a-zA-Z]{4,16}\\|segfault at .* in libwicked.*.so'";
     if (script_run("$go_to_target $dmesg") != 1) {
-        record_soft_failure "The $target needs to be checked manually!";
+        record_info "The $target needs to be checked manually!", result => 'softfail';
         assert_script_run("$go_to_target $dmesg | tee -a $log_file");
     }
     save_screenshot;

lib/virt_autotest/utils.pm

@@ -157,7 +157,7 @@ sub download_script {
     $cmd = "ssh root\@$args{machine} " . "\"$cmd\"" if ($args{machine} ne 'localhost');
     my $ret = script_run($cmd);
     unless ($ret == 0) {
-        record_soft_failure("Failed to download $args{script_name} from $args{script_url}!");
+        record_info("Failed to download $args{script_name} from $args{script_url}!", result => 'softfail');
         return $ret;
     }
     $cmd = "chmod +x ~/$args{script_name}";

tests/console/cleanup_qam_testrepos.pm

@@ -40,9 +40,9 @@ sub run {
             if ($behav eq "" || $behav eq "fail") {
                 die "Repository $repo is invalid";
             } elsif ($behav eq "softfail") {
-                record_soft_failure("repository $repo is invalid");
+                record_info("repository $repo is invalid", result => 'softfail');
             } elsif ($behav eq "clear") {
-                record_soft_failure("removing invalid repository: $repo");
+                record_info("removing invalid repository: $repo", result => 'softfail');
                 zypper_call("rr '$repo'");
             } elsif ($behav eq "purge") {
                 record_info("removing invalid repository: $repo");

tests/console/journal_check.pm

@@ -70,7 +70,7 @@ sub run {
             } elsif ($bug_pattern->{$bug}->{type} eq 'ignore') {
                 bmwqemu::diag("Ignoring log message:\n$buffer\n");
             } else {
-                record_soft_failure("$bug:\n$buffer");
+                record_info("$bug:\n$buffer", result => 'softfail');
             }
         }
     }
@@ -98,7 +98,7 @@ sub run {
             my $failed_service_output = script_output("systemctl status $service -l || true");
             foreach my $bsc (@matched_bugs) {
                 if ($failed_service_output =~ /$bug_pattern->{$bsc}->{description}/) {
-                    record_soft_failure("Service: $service failed due to $bsc\n$failed_service_output");
+                    record_info("Service: $service failed due to $bsc\n$failed_service_output", result => 'softfail');
                     next SRV;
                 }
             }

tests/console/ping.pm

@@ -58,7 +58,7 @@ sub run {
         $bug = "bsc#1199927" if is_sle('=15');
 
         if (defined($bug)) {
-            record_soft_failure $bug;
+            record_info $bug, result => 'softfail';
         } else {
             $self->result("fail");
             record_info("Unknown failure on $cmd, maybe related to: bsc#1200617, bsc#1195826, bsc#1196840, bsc#1199918, bsc#1199926, bsc#1199927");

tests/console/python_scientific.pm

@@ -28,7 +28,7 @@ sub run_python_script {
             record_info("scipy-fft", "scipy-fft module not available", result => 'softfail');
         } else {
             my $failmsg = script_output("grep 'Softfail' '$logfile'");
-            record_soft_failure("$failmsg");
+            record_info("$failmsg", result => 'softfail');
         }
     }
 }

tests/console/snapper_cleanup.pm

@@ -60,7 +60,7 @@ sub snapper_cleanup {
         my $msg = "bsc#998360: qgroup 1/0: Exclusive space is above user-defined limit:\n"
           . "$exp_excl_space (expected exclusive disk space) < $qgroup_excl_space (consumed exclusive disk space)";
         if (check_var('VERSION', '12-SP2')) {
-            record_soft_failure $msg;
+            record_info $msg, result => 'softfail';
         }
         else {
             die $msg;

tests/console/yast2_lan_device_settings.pm

@@ -56,7 +56,7 @@ sub run {
             send_key "alt-n";
             send_key "alt-c";
             close_yast2_lan();
-            record_soft_failure("Duplicate IP, $static_ip is currently unavailable. Skipping static IP assignment");
+            record_info("Duplicate IP, $static_ip is currently unavailable. Skipping static IP assignment", result => 'softfail');
         } elsif (match_has_tag 'static-ip-address-set') {
             close_yast2_lan();
 

tests/console/zfs.pm

@@ -59,7 +59,7 @@ sub install_zfs {
         $repo = get_repository();
         if (!check_available($repo)) {
             my $msg = "Sorry, zfs repository is not (yet) available for this distribution";
-            record_soft_failure($msg);
+            record_info($msg, result => 'softfail');
             return 0;
         }
     }

tests/containers/bci_test.pm

@@ -38,18 +38,18 @@ sub run_tox_cmd {
     my $ret = script_run("timeout $bci_timeout $cmd", timeout => ($bci_timeout + 3));
     if ($ret == 124) {
         # man timeout: If  the command times out, and --preserve-status is not set, then exit with status 124.
-        record_soft_failure("The command <tox -e $env> timed out.");
+        record_info("The command <tox -e $env> timed out.", result => 'softfail');
         $error_count += 1;
     } elsif ($ret != 0) {
-        record_soft_failure("The command <tox -e $env> failed $ret.");
+        record_info("The command <tox -e $env> failed.", result => 'softfail');
         $error_count += 1;
     } else {
         record_info('PASSED');
     }
     # Cut the tox log from the header onward and filter the text
     my $cmd_xf = "awk '/short test summary info/{f=1}f' $tox_out | grep XFAIL";
     my $ret_xf = script_run("$cmd_xf", timeout => ($bci_timeout + 3));
-    record_soft_failure("The command <tox -e $env> has softfailures(XFAIL)") if ($ret_xf == 0);
+    record_info("The command <tox -e $env> has softfailures(XFAIL)", result => 'softfail') if ($ret_xf == 0);
     # Rename resulting junit file because it will be overwritten if we run
     # the same tox command later with another container engine. This way,
     # we will be able to parse the results for both container engines tox runs.

tests/kernel/tuned.pm

@@ -50,7 +50,7 @@ sub run {
         for (keys %known_errors) {
             if ($error =~ /${known_errors{$_}}/) {
                 my $bugref = $_ =~ s/_/\#/r;
-                record_soft_failure "$bugref - ${known_errors{$_}}";
+                record_info "$bugref - ${known_errors{$_}}", result => 'softfail';
                 last;
             }
             record_info 'unknown error', $error, result => 'fail';

tests/publiccloud/download_repos.pm

@@ -61,7 +61,7 @@ sub run {
             $ret = script_run "wget --no-clobber -r -R 'robots.txt,*.ico,*.png,*.gif,*.css,*.js,*.htm*' --reject-regex='s390x\\/|ppc64le\\/|kernel*debuginfo*.rpm|src\\/' --domains $domain --no-parent $maintrepo/", timeout => 600;
             if ($ret !~ /0|8/) {
                 # softfailure, if repo doesn't exist (anymore). This is required for cloning jobs, because the original test repos could be empty already
-                record_soft_failure("Download failed (rc=$ret):\n$maintrepo");
+                record_info("Download failed (rc=$ret):\n$maintrepo", result => 'softfail');
                 script_run("echo 'Download failed for $maintrepo ...' >> ~/repos/qem_download_status.txt");
             } else {
                 assert_script_run("echo -en '\\n" . ('#' x 80) . "\\n# $maintrepo:\\n' >> /tmp/repos.list.txt");
@@ -70,7 +70,7 @@ sub run {
                     assert_script_run(sprintf(q(sed -i '1 s/]/_%s]/' %s/*.repo), random_string(4), $parent));
                     assert_script_run("find $parent >> /tmp/repos.list.txt");
                 } else {
-                    record_soft_failure("No .repo file found in $parent. This directory will be removed.");
+                    record_info("No .repo file found in $parent. This directory will be removed.", result => 'softfail');
                     assert_script_run("echo 'No .repo found for $maintrepo' >> ~/repos/qem_download_status.txt");
                     assert_script_run("rm -rf $parent");
                 }

tests/publiccloud/storage_perf.pm

@@ -111,7 +111,7 @@ sub analyze_previous_series {
             record_info('ANALYZE', $analyze);
             # This detects if mean values differs more than 10%
             if ($diff_percents > 10) {
-                record_soft_failure("Deviation occurred. $generic_message");
+                record_info("Deviation occurred. $generic_message", result => 'softfail');
                 $result = 1;
             } else {
                 record_info('PASS', "The data looks good. $generic_message");

tests/sles4sap/sapconf.pm

@@ -58,7 +58,7 @@ sub run_developers_tests {
         # Skip results of fate#325548 on SLES4SAP versions before 15
         next if ($summary =~ /fate325548/ and is_sle('<15'));
         if ($summary =~ /Test #(bsc|fate)([0-9]+)/) {
-            record_soft_failure "$1#$2";
+            record_info "$1#$2", result => 'softfail';
         }
         else {
             record_info $summary, "Test summary: $summary", result => 'fail';

tests/sles4sap/sys_param_check.pm

@@ -36,7 +36,7 @@ sub add_softfail {
     my ($module, $os_version, $bsc_number, @parameters) = @_;
     foreach my $parameter (@parameters) {
         if (check_var("VERSION", $os_version) && check_failure($module, $parameter)) {
-            record_soft_failure "$bsc_number - Wrong value for $parameter";
+            record_info "$bsc_number - Wrong value for $parameter", result => 'softfail';
             remove_value($module, $parameter);
         }
     }

tests/virt_autotest/hotplugging_utils.pm

@@ -39,7 +39,7 @@ sub set_guest_memory {
     my $guestmemory = get_guest_memory($guest);
     # Memory reposts are not precise, we allow for a +/-10% acceptance range
     my $within_tolerance = ($min_memory <= $guestmemory) && ($guestmemory <= $max_memory);
-    record_soft_failure("Set live memory failed - expected $memory but got $guestmemory") unless ($within_tolerance);
+    record_info("Set live memory failed - expected $memory but got $guestmemory", result => 'softfail') unless ($within_tolerance);
 }
 
 # Returns guest memory in MB

tests/virt_autotest/sriov_network_card_pci_passthrough.pm

@@ -422,7 +422,7 @@ sub restore_original_guests {
             record_info "Guest $guest is restored.";
         }
         else {
-            record_soft_failure "Fail to restore $guest!";
+            record_info("Fail to restore $guest!", result => 'softfail');
         }
     }
 }

tests/virt_autotest/virt_utils.pm

@@ -394,7 +394,7 @@ sub get_guest_list {
     my $qa_guest_config_file = "/usr/share/qa/virtautolib/data/vm_guest_config_in_vh_update";
     my $hypervisor_type = get_var('SYSTEM_ROLE', '');
     my $guest_list = script_output "source /usr/share/qa/virtautolib/lib/virtlib; get_vms_from_config_file $qa_guest_config_file $guest_pattern $hypervisor_type";
-    record_soft_failure("Not found guest pattern $guest_pattern in $qa_guest_config_file") if ($guest_list eq '');
+    record_info("Not found guest pattern $guest_pattern in $qa_guest_config_file", result => 'softfail') if ($guest_list eq '');
     return $guest_list;
 }
 
@@ -437,7 +437,7 @@ sub download_guest_assets {
             push @available_guests, $guest;
         }
         else {
-            record_soft_failure("$vm_disk_url not found!");
+            record_info("$vm_disk_url not found!", result => 'softfail');
         }
     }
     return 0 unless @available_guests;
@@ -467,7 +467,7 @@ sub download_guest_assets {
         # download vm xml file
         my $rc = script_run("cp $mount_point/$remote_guest_xml_file $vm_xml_dir/$guest.xml", 60);
         if ($rc) {
-            record_soft_failure("Failed copying: $mount_point/$remote_guest_xml_file");
+            record_info("Failed copying: $mount_point/$remote_guest_xml_file", result => 'softfail');
             next;
         }
         script_run("ls -l $vm_xml_dir", 10);
@@ -490,7 +490,7 @@ sub download_guest_assets {
         $rc = script_run("cp $mount_point/$remote_guest_disk $local_guest_image", 300);    #it took 75 seconds copy from vh016 to vh001
         script_run "ls -l $local_guest_image";
         if ($rc) {
-            record_soft_failure("Failed to download: $remote_guest_disk");
+            record_info("Failed to download: $remote_guest_disk", result => 'softfail');
             next;
         }
         $guest_count++;

tests/virt_autotest/xen_guest_irqbalance.pm

@@ -71,7 +71,7 @@ sub run_test {
         else {
             my $default_affinity = script_output("ssh root\@$guest \"cat /proc/irq/default_smp_affinity\"");
             foreach (@affinities_with_irqbalance) {
-                record_soft_failure("The value of one NIC IRQ smp_affinity, '$_', did not follow the default_smp_affinity, '$default_affinity', with irqbalance enabled.") if $_ ne $default_affinity;
+                record_info("The value of one NIC IRQ smp_affinity, '$_', did not follow the default_smp_affinity, '$default_affinity', with irqbalance enabled.", result => 'softfail') if $_ ne $default_affinity;
             }
         }
 
@@ -90,7 +90,7 @@ sub run_test {
             #at least a few interrupts on each cpu core
             if ($increased_irqs_on_cpu[$cpu_id] < 10) {
                 #Please look into the soft failure to identify if it is a product bug or temporary lack of network load coverage
-                record_soft_failure("IRQ are not balanced as the vif interrupts for CPU" . $cpu_id . " is " . $increased_irqs_on_cpu[$cpu_id]);
+                record_info("IRQ are not balanced as the vif interrupts for CPU" . $cpu_id . " is " . $increased_irqs_on_cpu[$cpu_id], result => 'softfail');
             }
         }
         record_info("NIC IRQs distribution on $nproc cpu cores", "@increased_irqs_on_cpu");
@@ -121,7 +121,7 @@ sub restore_original_guests {
             restore_downloaded_guests($guest, $vm_xml_save_dir);
         }
         else {
-            record_soft_failure "Fail to restore $guest!";
+            record_info "Fail to restore $guest!", result => 'softfail';
         }
     }
 }