Fits dependencies compiler into compile-config and -code

It turns out compile-dependencies for generating dataflow.dot and
Makefile codegen was a special case, so refactored it into the other two
main compilation steps.

Author: netj

compiler/app-has-been-compiled

@@ -13,7 +13,7 @@ export DEEPDIVE_APP
 
 [[ $# -gt 0 ]] ||
     # default set of expected compilation results
-    set -- "$DEEPDIVE_APP"/run/{compiled/{{schema,config}.json,Makefile},Makefile,process/}
+    set -- "$DEEPDIVE_APP"/run/{compiled/{schema,config}.json,Makefile,process/}
 
 for path; do
     [[ -e "$path" ]] ||

compiler/compile-dependencies/compile-dependencies2make renamed to compiler/compile-code/compile-code-Makefile

@@ -1,12 +1,15 @@
 #!/usr/bin/env jq-f
-# compile-dependencies2make -- Generates a GNU Make recipes corresponding to the input dependency graph in JSON
+# compile-code-Makefile -- Generates a GNU Makefile with recipes for the compiled processes considering their dependencies
 ##
 
 def mktarget(suffix): "\(.).\(suffix)";
 def mktargets(suffix): map(mktarget(suffix)) | join(" ");
 def mktarget: mktarget("done");
 def mktargets: mktargets("done");
 
+.deepdive_.execution.dependencies |
+{ path: "Makefile", content: [
+
 # make variables
 "
 # default commands
@@ -64,3 +67,5 @@ reset: \(keys | mktargets("reset"))
 " else empty end
 
 )
+
+] | join("") }

compiler/compile-dependencies/compile-dependencies2dot renamed to compiler/compile-code/compile-code-dataflow_dot

@@ -1,5 +1,5 @@
 #!/usr/bin/env jq-f
-# compile-dependencies2dot -- Generates a DOT corresponding to the input dependency graph in JSON
+# compile-code-dataflow_dot -- Generates dependency graph of processes in Graphviz DOT language
 ##
 
 def nodeId: sub("/"; "/\n");
@@ -13,6 +13,8 @@ def nodeType: sub("/.*$"; "");
 , model    : "shape=septagon color=\"#ffffff\" style=filled fillcolor=\"#cc22cc\""
 } as $nodeAttr |
 
+.deepdive_.execution.dependencies |
+{ path: "dataflow.dot", content: [
 "
 digraph \"\(env.DEEPDIVE_APP | sub(".*/"; "")) data flow\" {
     graph[rankdir=BT  # bottom-to-top drawing (or RL/right-to-left may also make sense)
@@ -51,3 +53,4 @@ end
 
 }
 "
+] | join("") }

compiler/compile-codegen

@@ -4,10 +4,25 @@
 # Author: Jaeho Shin <netj@cs.stanford.edu>
 # Created: 2015-12-05
 set -euo pipefail
-eval "$(jq -r '"
-x=\(.path | @sh)
+eval "$(jq -r '
+@sh "
+x=\(.path)
 echo >&2 \" run/$x\"
-mkdir -p \"$(dirname \"$x\")\"
-echo \(.content | @sh) >\"$x\"
-\(.mode | if . then "chmod \(@sh) \"$x\"" else "" end)
-"')"
+# ensure parent dir exists
+xd=\"$(dirname \"$x\")\"
+mkdir -p \"$xd\"
+# move existing files/dirs out of the way
+if [[ -e \"$x\" ]]; then
+    chmod +w \"$x\" \"$xd\" || true
+    mv -f --no-target-directory \"$x\" \"$x\"~
+fi
+# write contents
+echo \(.content) >\"$x\"
+"
+
++ (.mode | if . then @sh "
+# adjust mode
+chmod \(.) \"$x\""
+else ""
+end)
+')"

compiler/compile-dependencies/compile-config2dependencies renamed to compiler/compile-config/compile-config-9.99-dependencies

@@ -1,5 +1,5 @@
 #!/usr/bin/env jq-f
-# compile-config2dependencies -- Derives a dependency graph from deepdive.conf in JSON
+# compile-config-9.99-dependencies -- Derives a dependency graph for easier Makefile and .dot codegen
 ##
 .deepdive_ as $deepdive
 
@@ -8,7 +8,8 @@
   ) as $output_relation
 
 # form a dependency graph
-| [ # every process/* and the processes or input data it depends on
+| .deepdive_.execution.dependencies =
+( [ # every process/* and the processes or input data it depends on
     ( $deepdive.execution.processes | to_entries[]
     | .value |= ((.dependencies_ // [] | map($output_relation[.] // .)) + .input_ | unique)
     )
@@ -25,3 +26,4 @@
 #    )
   ]
 | from_entries
+)

compiler/deepdive-compile

@@ -40,7 +40,6 @@ ln -sfnv "$compileDir" run/RUNNING.COMPILE >&2
 # augment PATH to access compiler components
 PATH="$DEEPDIVE_HOME"/util/compile-config:"$PATH"
 PATH="$DEEPDIVE_HOME"/util/compile-code:"$PATH"
-PATH="$DEEPDIVE_HOME"/util/compile-dependencies:"$PATH"
 export DEEPDIVE_ERROR_PREFIX="[ERROR] "
 
 ## compile a full deepdive.conf from source code
@@ -128,20 +127,6 @@ ln -sfn "$config" config.json
 STEP "Validating run/compiled/config.json:"
 deepdive-check -a -c "$PWD"/config.json 'compiled_*' 2>&1 | sed 's/^/ /' >&2
 
-if [[ -z $DEEPDIVE_COMPILE_SKIP_DATAFLOW ]]; then
-STEP "Analyzing data flow dependencies to compile:"
-# derive a dependency graph from the extractors, factors, pipelines defined in deepdive.conf
-compile-config2dependencies <config.json >dependencies.json
-# compile a Makefile from the dependency graph
-STEP " run/Makefile"
-compile-dependencies2make <dependencies.json -r >Makefile
-# compile an SVG from the dependency graph
-STEP " run/compiled/dataflow.svg"
-compile-dependencies2dot <dependencies.json -r >dataflow.dot
-dot -Tsvg <dataflow.dot >dataflow.svg
-STEP "  (file://$(pwd -P)/dataflow.svg)"  # XXX a handy file: URL for opening in a browser
-fi
-
 STEP "Compiling executable code into:"
 # compile extractors and factors under process/ and factor/
 for cc in "$DEEPDIVE_HOME"/util/compile-code/compile-code-*; do
@@ -151,17 +136,24 @@ for cc in "$DEEPDIVE_HOME"/util/compile-code/compile-code-*; do
     STEP " run/compiled/code-$name.json"
     "$cc" config.json >code-$name.json
 done
+
+#STEP "Validating run/compiled/code-*.json:"
+#deepdive-check -a -c "$PWD"/code-\*.json 'codegen_*' 2>&1 | sed 's/^/ /' >&2
 # TODO check path name collisions in code-*.json
+
 # mark everything compiled as read-only
 chmod a-w *
 
 if [[ -z $DEEPDIVE_COMPILE_SKIP_CODEGEN ]]; then
 # generate actual code under run/
 cd "$DEEPDIVE_APP"/run
-STEP "Generating executable files:"
+STEP "Generating files:"
 cat "$compileDir"/code-*.json |
 compile-codegen
-ln -sfn compiled/Makefile .
+# some postprocessing over generated files
+dot -Tsvg <dataflow.dot >dataflow.svg
+STEP " run/dataflow.svg"
+STEP "  (file://$(pwd -P)/dataflow.svg)"  # XXX a handy file: URL for opening in a browser
 fi
 
 # point to the successfully compiled one, leaving the previous one as backup

stage.sh

@@ -57,7 +57,6 @@ stage .build/submodule/compiler/hocon2json/target/scala-2.10/hocon2json-assembly
 stage src/main/resources/application.conf                         etc/deepdive-default.conf
 stage compiler/compile-config                                     util/
 stage compiler/compile-check                                      util/
-stage compiler/compile-dependencies                               util/
 stage compiler/compile-code                                       util/
 stage compiler/compile-codegen                                    util/