run all output scripts on one slurm, improve logging

modules/11_aerosols/exoGAINS/postsolve.gms

@@ -19,7 +19,7 @@ if((o_modelstat le 2),
 );
 
 *** Calculate AP emissions
-Execute "Rscript exoGAINSAirpollutants.R";
+Execute "Rscript --vanilla exoGAINSAirpollutants.R";
 
 *** Read input ref results for tall with following dimensions: pm_emiAPexsolve(tall,all_regi,all_sectorEmi,emiRCP)
 if((cm_startyear gt 2005),

modules/15_climate/magicc/postsolve.gms

@@ -18,11 +18,11 @@
 *** Generate MAGICC scenario file
 $include "./core/magicc.gms";
 *** execute MAGICC (this is cheap enough, ~2s)
-Execute "Rscript run_magicc.R";
+Execute "Rscript --vanilla run_magicc.R";
 *** read in results
-Execute "Rscript read_DAT_TOTAL_ANTHRO_RF.R";
+Execute "Rscript --vanilla read_DAT_TOTAL_ANTHRO_RF.R";
 Execute_Loadpoint 'p15_forc_magicc'  p15_forc_magicc;
-Execute "Rscript read_DAT_SURFACE_TEMP.R";
+Execute "Rscript --vanilla read_DAT_SURFACE_TEMP.R";
 Execute_Loadpoint 'p15_magicc_temp' pm_globalMeanTemperature = pm_globalMeanTemperature;
 *** MAGICC only reports unitl 2300:
 pm_globalMeanTemperature(tall)$(tall.val gt 2300) = 0;
@@ -62,7 +62,7 @@ $ifthen.cm_magicc_tirf "%cm_magicc_temperatureImpulseResponse%" == "on"
 * thus only compute TIRF after each of the first 10 iterations, then only every fifth iteration. 
 * runtime is ca 30s, so switching on TIRF adds ca 10min to runtime
 if( ((iteration.val le 10) or ( mod(iteration.val,5 ) eq 0)) ,
-    execute "Rscript run_magicc_temperatureImpulseResponse.R";
+    execute "Rscript --vanilla run_magicc_temperatureImpulseResponse.R";
     execute_loadpoint 'pm_magicc_temperatureImpulseResponse'  pm_temperatureImpulseResponseCO2 = pm_temperatureImpulseResponse;
 );
 *NOTE the MAGICC results (*.OUT files) are from  the last pulse experiment now, so take care if reading them in after this point.

output.R

@@ -199,7 +199,7 @@ if (comp %in% c("comparison", "export")) {
   if (! exists("source_include")) {
     # for selected output scripts, only slurm configurations matching these regex are available
     slurmExceptions <- if ("reporting" %in% output) "--mem=[0-9]*[0-9]{3}" else NULL
-    if (all(output %in% outputUsingDirect)) slurmConfig <- "direct"
+    if (any(output %in% outputUsingDirect)) slurmConfig <- "direct"
     # if this script is not being sourced by another script but called from the command line via Rscript let the user
     # choose the slurm options
     if (!exists("slurmConfig")) {
@@ -257,38 +257,38 @@ if (comp %in% c("comparison", "export")) {
       message("\nNo output generation, as output was set to NA, as for example for --testOneRegi or --quick.")
     } else {
       message("\nStarting output generation for ", outputdir, "\n")
-      for (rout in output) {
-        name <- paste(rout, ".R", sep = "")
-        if ("--test" %in% flags) {
-          message("Test mode, not executing ", paste0("scripts/output/single/", name))
-        } else {
-          if (file.exists(paste0("scripts/output/single/", name))) {
-            if (slurmConfig == "direct" | rout %in% outputUsingDirect) {
-              # execute output script directly (without sending it to slurm)
-              message("Executing ", name)
-              tmp.env <- new.env()
-              tmp.error <- try(sys.source(paste0("scripts/output/single/", name), envir = tmp.env))
-              #        rm(list=ls(tmp.env),envir=tmp.env)
-              rm(tmp.env)
-              gc()
-              if (!is.null(tmp.error)) {
-                warning("Script ", name, " was stopped by an error and not executed properly!")
-              }
-            } else {
-              # send the output script to slurm
-              logfile <- paste0(outputdir, "/log_", rout, ".txt")
-              slurmcmd <- paste0("sbatch ", slurmConfig, " --job-name=", logfile, " --output=", logfile,
-                                 " --mail-type=END --comment=", rout, " --wrap=\"Rscript scripts/output/single/", rout,
-                                 ".R  outputdir=", outputdir, "\"")
-              message("Sending to slurm: ", name, ". Find log in ", logfile)
-              system(slurmcmd)
-              Sys.sleep(1)
+      name <- paste0(output, ".R")
+      if ("--test" %in% flags) {
+        message("Test mode, not executing scripts/output/single/", paste(name, collapse = ", "))
+      } else if (all(file.exists(paste0("scripts/output/single/", name)))) {
+        if (slurmConfig == "direct") {
+          # execute output script directly (without sending it to slurm)
+          for (n in name) {
+            message("Executing ", n)
+            tmp.env <- new.env()
+            tmp.error <- try(sys.source(paste0("scripts/output/single/", n), envir = tmp.env))
+            #        rm(list=ls(tmp.env),envir=tmp.env)
+            rm(tmp.env)
+            gc()
+            if (!is.null(tmp.error)) {
+              warning("Script ", n, " was stopped by an error and not executed properly!")
             }
           }
+        } else {
+          # send the output script to slurm
+          logfile <- file.path(outputdir, "log_output.txt")
+          Rscripts <- paste0("Rscript scripts/output/single/", name, " outputdir=", outputdir, collapse = "; ")
+          slurmcmd <- paste0("sbatch ", slurmConfig, " --job-name=", logfile, " --output=", logfile,
+                       " --mail-type=END --comment=output.R --wrap='", Rscripts, "'")
+          message("Sending to slurm: ", paste(name, collapse = ", "), ". Find log in ", logfile)
+          system(slurmcmd)
         }
+        # finished
+        message("\nFinished ", ifelse(slurmConfig == "direct", "", "starting job for "), "output generation for ", outputdir, "!\n")
+      } else {
+        warning("Skipping ", outputdir, " because some output script selected could not be found ",
+                "in scripts/output/single: ", name[! name %in% dir("scripts/output/single")])
       }
-      # finished
-      message("\nFinished ", ifelse(slurmConfig == "direct", "", "starting jobs for "), "output generation for ", outputdir, "!\n")
     }
 
     rm(source_include)

scripts/input/exoGAINSAirpollutants.R

@@ -30,12 +30,17 @@ load("config.Rdata")
 ssp_scenario <- cfg$gms$cm_APscen
 
 # read in REMIND avtivities, use input.gdx if the fulldata_exoGAINS.gdx is not available
-gdx <- if (file.exists("fulldata_exoGAINS.gdx")) "fulldata_exoGAINS.gdx" else "input.gdx"
+if (file.exists("fulldata_exoGAINS.gdx")) {
+  gdx <- "fulldata_exoGAINS.gdx"
+  iterationInfo <- paste("iteration", as.numeric(readGDX(gdx = gdx, "o_iterationNumber", format = "simplest")))
+} else {
+  gdx <- "input.gdx"
+  iterationInfo <- gdx
+}
 
-iterationNumber <- as.numeric(readGDX(gdx = gdx, "o_iterationNumber", format = "simplest"))
 t <- c(seq(2005,2060,5),seq(2070,2110,10),2130,2150)
 rem_in_mo <- NULL
-message("Iteration ", iterationNumber, ": exoGAINSAirpollutants.R calls remind2::reportMacroEconomy ", appendLF = FALSE)
+message("With data from ", iterationInfo, ": exoGAINSAirpollutants.R calls remind2::reportMacroEconomy ", appendLF = FALSE)
 rem_in_mo <- mbind(rem_in_mo,reportMacroEconomy(gdx)[,t,])
 message("- reportPE ", appendLF = FALSE)
 rem_in_mo <- mbind(rem_in_mo,reportPE(gdx)[,t,])

scripts/output/single/reporting.R

@@ -123,18 +123,18 @@ message("### end generation of mif files at ", Sys.time())
 
 ## produce REMIND LCOE reporting *.csv based on gdx information
 if (! isTRUE(envir$cfg$gms$c_empty_model == "on") || ! grepl("^C_TESTTHAT", scenario)) {
-  message("start generation of LCOE reporting")
+  message("### start generation of LCOE reporting at ", Sys.time())
   tmp <- try(convGDX2CSV_LCOE(gdx,file=LCOE_reporting_file,scen=scenario)) # execute convGDX2MIF_LCOE
-  message("end generation of LCOE reporting")
+  message("### end generation of LCOE reporting at ", Sys.time())
 }
 
 ## generate DIETER reporting if it is needed
 ## the reporting is appended to REMIND_generic_<scenario>.MIF in "DIETER" Sub Directory
 DIETERGDX <- "report_DIETER.gdx"
 if(file.exists(file.path(outputdir, DIETERGDX))){
-  message("start generation of DIETER reporting")
+  message("start generation of DIETER reporting at ", Sys.time())
   remind2::reportDIETER(DIETERGDX,outputdir)
-  message("end generation of DIETER reporting")
+  message("end generation of DIETER reporting at ", Sys.time())
 }
 
 message("### reporting finished.")

scripts/start/modelSummary.R

@@ -63,7 +63,7 @@ modelSummary <- function(folder = ".", gams_runtime = NULL) {
         message("  Modelstat after ", as.numeric(names(modelstat)), " iterations: ", modelstat,
                 if (modelstat %in% names(explain_modelstat)) paste0(" (", explain_modelstat[modelstat], ")"))
       }
-      logStatus <- grep("*** Status", readLines(file.path(folder, "full.log")), fixed = TRUE, value = TRUE)
+      logStatus <- grep("*** Status", readLines(file.path(folder, "full.log"), warn = FALSE), fixed = TRUE, value = TRUE)
       message("  full.log states: ", paste(logStatus, collapse = ", "))
       if (! all("*** Status: Normal completion" == logStatus)) stoprun <- TRUE
     }

scripts/start/prepare.R

@@ -263,8 +263,8 @@ prepare <- function() {
                       paste0("rev",cfg$inputRevision,"_", madrat::regionscode(cfg$regionmapping),ifelse(cfg$extramappings_historic == "","",paste0("-", madrat::regionscode(cfg$extramappings_historic))),"_", tolower(cfg$validationmodel_name),".tgz"),
                       paste0("CESparametersAndGDX_",cfg$CESandGDXversion,".tgz"))
   # download and distribute needed data
-  if(!setequal(input_new, input_old) | cfg$force_download) {
-      message(if (cfg$force_download) "You set 'cfg$force_download = TRUE'"
+  if (! setequal(input_new, input_old) || isTRUE(cfg$force_download)) {
+      message(if (isTRUE(cfg$force_download)) "You set 'cfg$force_download = TRUE'"
               else "Your input data are outdated or in a different regional resolution",
               ". New input data are downloaded and distributed.")
       download_distribute(files        = input_new,
@@ -277,7 +277,7 @@ prepare <- function() {
   }
 
   # extract BAU emissions for NDC runs to set up emission goals for region where only some countries have a target
-  if ((!is.null(cfg$gms$carbonprice) && (cfg$gms$carbonprice == "NDC")) | (!is.null(cfg$gms$carbonpriceRegi) && (cfg$gms$carbonpriceRegi == "NDC")) ){
+  if (isTRUE(cfg$gms$carbonprice == "NDC") || isTRUE(cfg$gms$carbonpriceRegi == "NDC")) {
     cat("\nRun scripts/input/prepare_NDC.R.\n")
     source("scripts/input/prepare_NDC.R")
     prepare_NDC(as.character(cfg$files2export$start["input_bau.gdx"]), cfg)

start.R

@@ -73,12 +73,16 @@ config.file <- NULL
 if(!exists("argv")) argv <- commandArgs(trailingOnly = TRUE)
 argv <- argv[! grepl("^-", argv) & ! grepl("=", argv)]
 # check if user provided any unknown arguments or config files that do not exist
-if (length(argv) > 0) {
-  file_exists <- file.exists(argv)
-  if (sum(file_exists) > 1) stop("You provided more than one file, start.R can only handle one.")
-  if (!all(file_exists)) stop("Unknown parameter provided: ", paste(argv[!file_exists], collapse = ", "))
-  # set config file to not known parameter where the file actually exists
-  config.file <- argv[[1]]
+if (length(argv) == 1) {
+  if (file.exists(argv)) {
+    config.file <- argv
+  } else if (file.exists(file.path("config", argv))) {
+    config.file <- file.path("config", argv)
+  } else {
+    stop("Unknown parameter provided: ", paste(argv, collapse = ", "))
+  }
+} else if (length(argv) > 1) {
+  stop("You provided more than one file or other command line argument, start.R can only handle one.")
 }
 
 if ("--help" %in% flags) {