uk_freeview: add retries to fetching web pages (#241)

XMLTV · Oct 15, 2024 · 25d8408 · 25d8408
1 parent e0a0d49
commit 25d8408
Showing 1 changed file with 44 additions and 2 deletions.
diff --git a/grab/uk_freeview/tv_grab_uk_freeview b/grab/uk_freeview/tv_grab_uk_freeview
@@ -563,7 +563,10 @@ sub get_programmes {
 		t $url;
 
 		# fetch json content (will be decoded from utf8)
-		my $data = get_nice_json( $url, \&fudgeprogs );
+		# (#244) my $data = get_nice_json( $url, \&fudgeprogs );
+		# fetch a url with up to 5 retries
+		my $data = fetch_url_json( $url, 'get', undef, \&fudgeprogs );
+
 		if ($data->{status} ne 'success') { print STDERR " PROGRAMME fetch failed : ".$data->{status}."\n" if $opt_debug; }
 
 		#print STDERR Dumper($data);die();
@@ -675,7 +678,10 @@ sub get_programmes {
 					t $url;
 
 					# fetch json content (will be decoded from utf8)
-					my $data = get_nice_json( $url, \&fudgeprogs );
+					# (#244) my $data = get_nice_json( $url, \&fudgeprogs );
+					# fetch a url with up to 5 retries
+					my $data = fetch_url_json( $url, 'get', undef, \&fudgeprogs );
+
 					if ($data->{status} ne 'success') { print STDERR " DETAILS fetch failed : ".$data->{status}."\n" if $opt_debug; }
 
 					#print STDERR Dumper($data);die();
@@ -974,3 +980,39 @@ sub get_region {
 
 	return ( $region_id );
 }
+
+
+# get data from url (with retries on fetch fail)
+sub fetch_url_json ($;$$$$) {
+	# fetch a url with up to 5 retries
+	my ($url, $method, $varhash, $filter, $utf8) = @_;
+	$XMLTV::Get_nice::FailOnError = 0;
+	my $content;
+	my $maxretry = 5;
+	my $retry = 0;
+	if (defined $method && lc($method) eq 'post') {
+		# NOT TESTED
+		while ( (not defined($content = XMLTV::Get_nice::post_nice_json($url, $varhash))) || (length($content) == 0) ) {
+			my $r = $XMLTV::Get_nice::Response;
+			print STDERR "HTTP error: ".$r->status_line."\n";
+			$retry++;
+			#return undef if $retry > $maxretry;
+			die "could not fetch $url, error: " . $r->status_line . ", aborting\n" if $retry > $maxretry;
+			print STDERR "Retrying URL: $url (attempt $retry of $maxretry) \n";
+		}
+
+	} else {
+
+		while ( (not defined($content = XMLTV::Get_nice::get_nice_json($url, $filter, $utf8))) || (length($content) == 0) ) {
+			my $r = $XMLTV::Get_nice::Response;
+			print STDERR "HTTP error: ".$r->status_line."\n";
+			$retry++;
+			#return undef if $retry > $maxretry;
+			die "could not fetch $url, error: " . $r->status_line . ", aborting\n" if $retry > $maxretry;
+			print STDERR "Retrying URL: $url (attempt $retry of $maxretry) \n";
+		}
+
+	}
+
+	return $content;
+}