auto-commit

Author: yvanrichard

makefile

@@ -1,40 +1,48 @@
 all: all-results/all-results.rdata
 
-#  1: park previous results with "make park"
-#  2: specify computers and threads to use in parallel-pars.r
-#  3: split job among computers with "make preparesims"
-#  4: --- !!! COMMIT AND PUSH CHANGES !!! ---
-#  5: synchronise computers with "make gitsync"
-#  6: "make runallsims" to start all the simulations
-#  7: harvest results on different computers with "make harvest"
+#  1: specify computers and threads to use in parallel-pars.r
+#  2: split job among computers with "make preparesims"
+#  3: park previous results with "make park"
+#  4: "make runallsims" to start all the simulations
+#  5: harvest results on different computers with "make harvest"
 # Check runs with "make checkruns"
-# If something wrong: "make killr", to kill all the R processes on all computers, before starting again
+# If something goes wrong: "make killr", to kill all the R processes on all computers, before starting again
 
 
 ## collect results from all computers, and append them
-all-results/all-results.rdata: parallelisation/gather-results.r
+all-results/all-results.rdata: parallelisation/gather-results.r  .runallsims
 	cd parallelisation; Rscript gather-results.r
 
 ## Run simulations
-runallsims: .compile   parallelisation/run-all-sims.r 
+.runallsims: parallelisation/run-all-sims.r   .compile   .preparesims
 	cd parallelisation; Rscript run-all-sims.r
+	touch .runallsims
 
 ## Prepare simulations; split between cores; 
 ## run "make preparsesims a=test" just for testing what's about to be run
-preparesims: 
+.preparesims: 
 	cd parallelisation; Rscript prepare-sims.r $(a)
+	touch .preparesims
+
+## Byte compile R functions to go a bit faster
+.compile: functions.r  functions-model.r  parallelisation/compile-functions.r 
+	cd parallelisation; Rscript compile-functions.r
+	touch .compile
 
 ## Park previous results (results, all-results)
 park:
 	cd parallelisation; Rscript park-prev.r
 
-## Do a git pull on dfly computers (as in pars_parallel.r)
-gitsync:
-	cd parallelisation; Rscript gitsync.r
 
 
 
 
+###  Various utilities
+
+## Do a git pull on dfly computers (as in pars_parallel.r)
+gitsync:
+	cd parallelisation; Rscript gitsync.r
+
 ## Kill screens on all dfly computers specified in pars_parallel.r
 killscreens:
 	cd parallelisation; Rscript kill-all-screens.r
@@ -43,12 +51,11 @@ killscreens:
 killr:
 	cd parallelisation; Rscript kill-all-r.r
 
-
+## Check log of screen on each computer and check R processes running on all computers
 checkruns:
 	cd parallelisation; Rscript check-screens.r
 	Rscript -e "proc_in_dfly()"
 
-
 ## make tags for emacs
 tags:
 	Rscript -e "rtags(ofile='TAGS', recursive=T)"
@@ -59,6 +66,3 @@ clean:
 cleantmp: 
 	rm *~
 
-.compile: functions.r  functions-model.r  parallelisation/compile-functions.r 
-	cd parallelisation; Rscript compile-functions.r
-	touch .compile

parallelisation/gather-results.r

@@ -1,25 +1,23 @@
 ###  Fetch results from all computers  ###
 ##########################################
 
-rm(list=ls())
 library(compiler)
 
-basefold <- '~/dragonfly/sra-foundations/modelling/bh-dd-k50/'
-setwd(basefold)
-
-source('parallel-pars.r')
+source('../pars_parallel.r')
 ## source('functions.r')
 loadcmp('functions_cmp.r')
 
-load(datafile)
+load(sprintf('../%s', datafile))
+data <- eval(parse(text=dataname))
+if (dataname != 'data') {rm(dataname); gc()}
 
 load('cores_alloc.rdata')
 othercomps <- unique(cores_alloc$comp[!(cores_alloc$comp %in% thiscomp)])
 
 c=othercomps[1]
 for (c in othercomps)
   {
-    fold <- sprintf('%s-%s', outputfold, c)
+    fold <- sprintf('../%s-%s', outputfold, c)
     dir.create(fold, showWarnings=F)
     ## Mount results from other computers
     if (!length(dir(fold)))
@@ -35,26 +33,26 @@ folds <- c(outputfold, sprintf('%s-%s', outputfold, othercomps))
 for (d in folds)  # d='results-taiko'    # d='results'
   {
     cat('\nMerging files in', d, '...\n')
-    setwd(sprintf('%s%s', basefold, d))
+    setwd(sprintf('%s/%s/%s/%s', basefold, projectname, runfold, d))
 
-    inter <- dir(pattern='.*pars1-steps.*..rdata')
-    fin <- dir(patter='.*pars-steps.*..rdata')
+    inter <- dir(pattern='.*incomplete.*..rdata')
+    fin <- dir(patter='.*finished.*..rdata')
 
     for (f in inter)                    # f=inter[1]
       {
         load(f)
-        resi <- rbind2(resi, pars1)
+        resi <- rbind2(resi, data1)
       }
 
     for (f in fin)                      # f=fin[2]
       {
         load(f)
-        resf <- rbind2(resf, pars)
+        resf <- rbind2(resf, data)
       }
   }
 
 
-setwd(basefold)
+setwd(sprintf('%s/%s/%s', basefold, projectname, runfold))
 
 ## Unmount other computers
 for (c in othercomps)  # c='frank'
@@ -64,21 +62,20 @@ for (c in othercomps)  # c='frank'
     unlink(fold, recursive=T)
   }
 
-allresdir <- sprintf('%sall-results', basefold)
-dir.create(allresdir, showWarnings=F)
+dir.create(allresfold, showWarnings=F)
 setwd(allresdir)
 
 
 ## Merge intermediate and final results
-resi2 <- resi[!(resi$row %in% resf$row),]  # remove finished rows from intermediate results
+resi2 <- resi[!(resi[[rowidcol]] %in% resf[[rowidcol]]),]  # remove finished rows from intermediate results
 res <- rbind(resi2, resf)                  
 
-res <- res[order(res$row),]
+res <- res[order(res[[rowidcol]]),]
 
 
 
 ## Check missing rows and summarise
-missingrows <- dem$row[!(dem$row %in% res$row)]
+missingrows <- data[[rowidcol]][!(data[[rowidcol]] %in% res[[rowidcol]])]
 if (length(missingrows))
   {
     mr <- as.numeric(missingrows)
@@ -90,17 +87,11 @@ if (length(missingrows))
   }
 
 cat(sprintf('\n%i rows done (%i duplicates) after merging intermediates and final results.
-   Details:', nrow(res), sum(duplicated(res$row))),'\n')
-print(table(res$species))
-
-if ('pbr0' %in% names(res))
-  cat(sprintf('\n%i PBR values (%0.1f%%) already calculated.\n', sum(!is.na(res$pbr0)),
-      100*sum(!is.na(res$pbr0))/nrow(res))) else
-         cat('\nNo PBR values already calculated.\n')
+   Details:', nrow(res), sum(duplicated(res[[rowidcol]]))),'\n')
 
 
 ### Save appended results
-res <- res[order(res$row),]
+res <- res[order(res[[rowidcol]]),]
 
 setwd(allresdir)
 save(res, file='results_res.rdata')

parallelisation/park-prev.r

@@ -14,8 +14,9 @@ for (cp in comps)
 
     ## results
     f <- sprintf('%s/%s/%s/%s', basefold, projectname, runfold, outputfold)
-    a <- suppressWarnings(system(sprintf('ssh -A %s@%s "ls -d %s" 2>&1', user, cp, f), intern=T))
-    if (!grepl('^ls:',a))
+    suppressWarnings(ex <- system(sprintf("ssh %s@%s test -d '%s' && echo TRUE", user, cp, f),
+                                  intern=T))
+    if (length(ex))
       {
         cmd <- sprintf('ssh -A %s@%s "cd %s/%s/%s; mv %s %s_%s"', user, cp,
                        basefold, projectname, runfold, outputfold, outputfold, timeflag); cat(cmd,'\n')
@@ -25,8 +26,9 @@ for (cp in comps)
 
     ## all-results
     f <- sprintf('%s/%s/%s/%s', basefold, projectname, runfold, allresfold)
-    a <- suppressWarnings(system(sprintf('ssh -A %s@%s "ls -d %s" 2>&1', user, cp, f), intern=T))
-    if (!grepl('^ls:',a))
+    suppressWarnings(ex <- system(sprintf("ssh %s@%s test -d '%s' && echo TRUE", user, cp, f),
+                                  intern=T))
+    if (length(ex))
       {
         cmd <- sprintf('ssh -A %s@%s "cd %s/%s/%s; mv %s %s_%s"', user, cp,
                        basefold, projectname, runfold, allresfold, allresfold, timeflag); cat(cmd,'\n')

run1.r

@@ -8,7 +8,7 @@ source('pars_parallel.r')
 loadcmp('parallelisation/functions_cmp.r')
 loadcmp('parallelisation/functions-model_cmp.r')
 
-load('data.rdata')
+load(datafile)
 data <- eval(parse(text=dataname))
 if (dataname != 'data') {rm(dataname); gc()}