Skip to content

Commit

Permalink
added time limit
Browse files Browse the repository at this point in the history
  • Loading branch information
AlexanderPico committed Mar 27, 2023
1 parent 0b7e251 commit f200de1
Show file tree
Hide file tree
Showing 2 changed files with 23 additions and 6 deletions.
File renamed without changes.
29 changes: 23 additions & 6 deletions scripts/fetch_figures.R
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,11 @@ library(dplyr)
library(magrittr)
library(stringr)

# Keep track of time to stay under 6 hour limit per GH Action job.
# Exit at 5 hours and let subsequent runs complete a given query
job_start_time <- Sys.time()
exit_flag <- FALSE

# Multiple attempts of a given query or until successful
for (j in 1:5) {
reload.flag <- FALSE
Expand Down Expand Up @@ -107,7 +112,7 @@ for (j in 1:5) {
exfigids <- read.table(config$exclude_figids, sep = "\t", stringsAsFactors = F)[,1]

res.fig.count <- 0

for (i in 1:page.count){
cat(sprintf("\nPage %i of %i", i, page.count))
cat(sprintf("\nPage %i of %i", i, page.count), file="figures/fetch.log", append = T)
Expand Down Expand Up @@ -311,6 +316,14 @@ for (j in 1:5) {

} # end if results on page

# Check time elapsed
check_time <- difftime(Sys.time(), job_start_time, units = "hours")
if (check_time >= 5) {
message("Total runtime nearing 6 hour limit. Time to wrap things up!")
exit_flag <- TRUE
break
}

# Turn the page
if (i < page.count-1){
result <- NULL
Expand All @@ -329,7 +342,7 @@ for (j in 1:5) {
if (!is.null(result)) {
next.page.button <- remDr$findElement(using = "xpath", "//*[@class='active page_link next']")
next.page.button$clickElement()
remDr$screenshot(display = TRUE)
#remDr$screenshot(display = TRUE)
} else {
message("Page failed to load after 15 seconds. Reloading query results and trying again...")
reload.flag <- TRUE
Expand All @@ -341,14 +354,18 @@ for (j in 1:5) {
break
} #end for each page

## Log totals for this round
cat(sprintf("\n\n%i new figures total\n",res.fig.count))
cat(sprintf("\n\n%i new figures total\n",res.fig.count), file="figures/fetch.log", append = T)

## Exit if time is running out
if (exit_flag)
break

## Reload query if prior result aborted
if (reload.flag)
next

## Log final results
cat(sprintf("\n\n%i new figures total\n",res.fig.count))
cat(sprintf("\n\n%i new figures total\n",res.fig.count), file="figures/fetch.log", append = T)

## log last_run
config$last_run <- to.date
yaml::write_yaml(config, "query_config.yml")
Expand Down

0 comments on commit f200de1

Please sign in to comment.