fix

openfoodfacts · Oct 18, 2024 · 15e5436 · 15e5436
1 parent bca62d3
commit 15e5436
Show file tree

Hide file tree

Showing 2 changed files with 36 additions and 22 deletions.
diff --git a/lib/ProductOpener/Store.pm b/lib/ProductOpener/Store.pm
@@ -307,6 +307,8 @@ sub sto_iter ($initial_path, $pattern = qr/\.sto$/i) {
 				foreach my $file (@candidates) {
 					# avoid ..
 					next if $file =~ /^\.\.?$/;
+					# avoid conflicting-codes and invalid-codes
+					next if $file =~ /^(conflicting|invalid)-codes$/;
 					my $path = "$current_dir/$file";
 					if (-d $path) {
 						# explore sub dirs

diff --git a/scripts/check_products_in_sto_files.pl b/scripts/check_products_in_sto_files.pl
@@ -52,24 +52,36 @@ ($product_path)
 	# return a list with path, product_id and normalized_id
 	my $iter = sto_iter($BASE_DIRS{PRODUCTS}, qr/product\.sto$/i);
 	my @anomalous = ();
+	my $i = 0;
 	while (my $product_path = $iter->()) {
-
+		print STDERR "Processing $product_path\n";
 		my $product_ref = retrieve($product_path);
 		if (defined $product_ref) {
 			my $code = $product_ref->{code};
-        my $product_id = $product_ref->{_id};
-		my $normalized_code = normalize_code($code);
-        my $normalized_product_id = product_id_for_owner(undef, $normalized_code);
-        my $normalized_product_path = product_path_from_id($normalized_product_id);
-
-        $product_path =~ s/.*\/products\///;
-        $product_path =~ s/\/product\.sto$//;
-        #print STDERR "code: $code - normalized_code: $normalized_code - product_id: $product_id - normalized_product_id: $normalized_product_id - product_path: $product_path - normalized_product_path: $normalized_product_path\n";
-
-		if (($code ne $normalized_code)  or ($product_id ne $normalized_product_id) or ($product_path ne $normalized_product_path)) {
-			push(@anomalous, [$product_path, $normalized_product_path, $code, $normalized_code, $product_id, $normalized_product_id]);
+			my $product_id = $product_ref->{_id};
+			my $normalized_code = normalize_code($code);
+			my $normalized_product_id = product_id_for_owner(undef, $normalized_code);
+			my $normalized_product_path = product_path_from_id($normalized_product_id);
+
+			$product_path =~ s/.*\/products\///;
+			$product_path =~ s/\/product\.sto$//;
+			#print STDERR "code: $code - normalized_code: $normalized_code - product_id: $product_id - normalized_product_id: $normalized_product_id - product_path: $product_path - normalized_product_path: $normalized_product_path\n";
+
+			if (   ($code ne $normalized_code)
+				or ($product_id ne $normalized_product_id)
+				or ($product_path ne $normalized_product_path))
+			{
+				push(
+					@anomalous,
+					[
+						$product_path, $normalized_product_path, $code,
+						$normalized_code, $product_id, $normalized_product_id
+					]
+				);
+			}
 		}
-        }
+		$i++;
+		($i % 1000 == 0) && print STDERR "Processed $i products - current path: $product_path\n";
 	}
 	return @anomalous;
 }
@@ -78,18 +90,20 @@ ($product_path, $dry_run, $out)
 	my @items = find_non_normalized_sto($product_path);
 
 	foreach my $item (@items) {
-        my ($product_path, $normalized_product_path, $code, $normalized_code, $product_id, $normalized_id) = @$item;
-		
+		my ($product_path, $normalized_product_path, $code, $normalized_code, $product_id, $normalized_id) = @$item;
+
 		my $is_duplicate = (-e "$BASE_DIRS{PRODUCTS}/$normalized_product_path") || 0;
-    
+
 		my $is_invalid = ($normalized_product_path eq "invalid") || 0;
 
-        print STDERR "product_path: $product_path - normalized_product_path: $normalized_product_path - code: $code - normalized_code: $normalized_code - product_id: $product_id - normalized_id: $normalized_id - is_duplicate: $is_duplicate - is_invalid: $is_invalid\n";
-
+		print STDERR
+			"product_path: $product_path - normalized_product_path: $normalized_product_path - code: $code - normalized_code: $normalized_code - product_id: $product_id - normalized_id: $normalized_id - is_duplicate: $is_duplicate - is_invalid: $is_invalid\n";
+
 	}
 
-    print STDERR "Found " . scalar(@items) . " non normalized codes / ids / paths\n";
-return;}
+	print STDERR "Found " . scalar(@items) . " non normalized codes / ids / paths\n";
+	return;
+}
 
 my $int_codes_query_ref = {'code' => {'$not' => {'$type' => 'string'}}};
 
@@ -114,7 +128,6 @@ ()
 
 }
 
-
 ### script
 my $usage = <<TXT
 fix_non_normalized_codes.pl is a script that updates checks and fix for products with non normalized codes
@@ -133,4 +146,3 @@ ()
 my $product_path = $BASE_DIRS{PRODUCTS};
 fix_non_normalized_sto($product_path, $dry_run, \*STDOUT);
 
-