results

Author: bottler

README.md

@@ -1,5 +1,5 @@
 # PythonMisc
-Some standalone python scripts
+Some fairly standalone python scripts
 
 * readSketchnet: tool to read data from the sketch-a-net dataset http://cybertron.cg.tu-berlin.de/eitz/projects/classifysketch/
 
@@ -10,3 +10,5 @@ Some standalone python scripts
 * tinisMultiRun: script for parallelizing another script across the 4 GPUs you get on tinis
 
 * tinisDummyProgram: a demo program for tinisMultiRun
+
+* results: code for managing results of long neural network training runs

results/Readme.md

@@ -0,0 +1,39 @@
+##Results database for longish-running nnet training
+
+This system, all in results.py, helps look after results from trainings of models.
+I usually find that I present data to the model in blocks, which usually contain a few minibatches of training data and a small sample of test data.
+
+#Using from training code
+Two functions are provided which are called by the training routine:
+*results.startRun
+This is called before any training, and stores information about the run for future reference. It also creates the database file (by default `results.sqlite`) if it doesn't exist.
+
+The supplied information I store for each run are pretty much the arguments of this function and the columns of the table RUNS. Exception: `callerFilePath` is a list of filenames whose contents we store - e.g. the code. I use the nonemptiness of `continuation` to indicate that the model was not trained from scratch, but from the saved weights of the previous run. `architecture` and `solver` are basically assumed in diffRuns and describeRun to be long JSON strings. You can change any of this.
+
+*results.step
+This stores the reported training and test objective and accuracy from each step of training.
+These are stored in the STEPS table.
+The time for the first 10 steps is remembered in the steps table.
+
+These functions are also available in lua after something like this
+```
+results = require 'results'
+```
+##warning
+Only one process should modify the database at any one time; you cannot train two models to the same database. But you can have as many processes as you like connected to the database and using the query functions, even during training.
+
+#Using the results
+I usually have an alias like 'ipython -i -c "import numpy as np; import results as r"'.
+The following functions are easy to use analysis functions.
+* `r.runList()` and `r.runList2()` provide summaries of all the runs.
+* `r.plotRun(x)` draws a graph of the training of the xth run
+* `r.plotRuns(x,y)` draws a graph comparing the training of two runs
+* `r.rrs(x,y)` is a text mode comparison of how two runs' training compare.
+* `r.describeRun(x)` prints the information which `startRun` supplied
+* `r.diffRuns(x,y)` makes a comparison of the startRun info of two runs
+* `r.steps(x)` prints a table of all the steps
+* `r.lastSteps()` prints a table of all the steps of the latest (e.g. current) run
+* `r.l()` is a very small summary of whether the latest (e.g. current) run is making progress.
+It can be desirable to take moving averages of the accuracy and error. Only `runList2` does this by default.
+
+At the moment whether to draw graphs on screen or to file is determined by the `oncampus` module in this repo. This will want modifying for other users, as well as the file location used (in `pushplot`).

results/results.lua

@@ -0,0 +1,66 @@
+require "lsqlite3"
+require 'sys'
+
+local results = {}
+
+filename = os.getenv("JR_RESULTSFILE") or "results.sqlite"
+con = sqlite3.open(filename)
+filecontents = ""
+
+function append_file_to_filecontents_string(f)
+   local ff = io.open(f,"r")
+   local content = ff:read("*all")
+   ff:close()
+   filecontents = filecontents .. f .."\n"..content
+end
+
+function check(x)
+   if x~=sqlite3.OK then
+      error(con:errmsg())
+   end
+end
+
+--if sql is a statement which doesn't return results,
+--resultless(sql, param1, param2) executes it with the given parameters
+function resultless(st, ...)
+   local s = con:prepare(st)
+   assert(s)
+   s:bind_values(...)
+   local res = s:step()
+   if res~=sqlite3.DONE then
+      error(con:errmsg())
+   end
+   s:finalize()
+end
+
+function results.startRun(repnname, repnlength, continuation, batchTr, batchTe, layertype, layers, width, architecture, solver, callerFilePath)
+   if "table" == type(callerFilePath) then
+      for i,name in ipairs(callerFilePath) do
+         append_file_to_filecontents_string(name)
+      end
+   else
+      append_file_to_filecontents_string(callerFilePath)
+   end
+   local setup=[[create table if not exists RUNS(COUNT INTEGER PRIMARY KEY, TIME TEXT DEFAULT CURRENT_TIMESTAMP NOT NULL, REPN TEXT, REPNLENGTH INT, CONTINUATION TEXT, BATCHTR INT, BATCHTE INT, LAYERTYPE TEXT, LAYERS INT, WIDTH INT, ARCHITECTURE TEXT, SOLVER TEXT, CODE TEXT)   ;
+   create table if not exists STEPS(STEP INTEGER PRIMARY KEY, RUN int, OBJECTIVE real, TRAINACC real, TESTOBJECTIVE real, TESTACC REAL );
+   create table if not exists TIMES(RUN INT, TIME real)
+   ]]
+   check(con:exec(setup))
+   local infoquery = "insert into RUNS (REPN, REPNLENGTH, CONTINUATION, BATCHTR, BATCHTE, LAYERTYPE, LAYERS, WIDTH, ARCHITECTURE, SOLVER, CODE) VALUES (?,?,?,?,?,?,?,?,?,?,?)"
+   local info = {repnname, repnlength, continuation, batchTr, batchTe, layertype, layers, width, architecture, solver, filecontents}
+   resultless(infoquery,unpack(info))
+   nrun = con:last_insert_rowid()
+   sys.tic()
+   nsteps = 0
+   filecontents = nil
+end
+
+function results.step(obj,train,objte,test)
+   nsteps = 1+nsteps
+   resultless("insert into steps values (NULL, ?, ?, ?, ?, ?)", nrun,obj,train,objte, test)
+   if nsteps == 10 then
+      resultless("insert into TIMES VALUES (?,?)", nrun, sys.toc())
+   end
+end
+
+return results