Issue #419 - Provide more details from Travis on PRs.

This uses a node.js module to post `status` updates to Github, and uses a Travis
secret to authenticate.

- Post comments from static analysis tools
- Change to posting from LetsEncryptBot
- For integration testing, only fail if the compile fails, or
  the NodeJS-client fails. Log if the Python client fails.

.gitignore

@@ -11,6 +11,7 @@ bin
 # Test files
 test/js/node_modules
 test/js/*.pem
+test/github-secret.json
 
 # Architecture specific extensions/prefixes
 *.[568vq]

.travis.yml

@@ -19,11 +19,16 @@ branches:
 sudo: required
 
 before_install:
+  # Github-PR-Status secret
+  - openssl aes-256-cbc -K $encrypted_f63987375ff8_key -iv $encrypted_f63987375ff8_iv -in test/github-secret.json.enc -out test/github-secret.json -d
+  - travis_retry npm install github-pr-status
+
   - go get golang.org/x/tools/cmd/vet
   - go get golang.org/x/tools/cmd/cover
   - go get github.com/golang/lint/golint
   - go get github.com/mattn/goveralls
   - go get github.com/modocache/gover
+
   # Boulder consists of multiple Go packages, which
   # refer to each other by their absolute GitHub path,
   # e.g. github.com/letsencrypt/boulder/analysis. That means, by default, if

test.sh

@@ -21,25 +21,108 @@ TESTDIRS="analysis \
           # cmd
           # Godeps
 
+# We need to know, for github-pr-status, what the triggering commit is.
+# Assume first it's the travis commit (for builds of master), unless we're
+# a PR, when it's actually the first parent.
+TRIGGER_COMMIT=${TRAVIS_COMMIT}
+if [ "${TRAVIS_PULL_REQUEST}" != "false" ] ; then
+  revs=$(git rev-list --parents -n 1 HEAD)
+  # The trigger commit is the last ID in the space-delimited rev-list
+  TRIGGER_COMMIT=${revs##* }
+fi
+
+start_context() {
+  CONTEXT="$1"
+  echo "Starting [${CONTEXT}]"
+}
+
+end_context() {
+  CONTEXT=""
+}
+
+update_status() {
+  if [ "${TRAVIS}" == "true" ] && [ "x${CONTEXT}" != "x" ] ; then
+    node node_modules/github-pr-status/github-pr-status.js \
+      --authfile "$(pwd)/test/github-secret.json" --sha "${TRIGGER_COMMIT}" \
+      --user "letsencrypt" --repo "boulder" --context "${CONTEXT}" $*
+  fi
+}
+
 run() {
   echo "$*"
   if $*; then
+    update_status --state success
     echo "success: $*"
   else
     FAILURE=1
+    update_status --state failure
     echo "failure: $*"
   fi
 }
 
+run_and_comment() {
+  if [ "${TRAVIS_PULL_REQUEST}" == "false" ] ; then
+    run $*
+  else
+    run $* | node node_modules/github-pr-status/github-pr-comment.js \
+      --authfile "$(pwd)/test/github-secret.json" --pr ${TRAVIS_PULL_REQUEST} \
+      --user "letsencrypt" --repo "boulder"
+  fi
+}
 
 # Path for installed go package binaries. If yours is different, override with
 # GOBIN=/my/path/to/bin ./test.sh
 GOBIN=${GOBIN:-$HOME/gopath/bin}
 
-# Ask vet to check in on things
-run go vet -x ./...
+#
+# Run Go Vet, a static analysis tool
+#
+
+start_context "test/vet"
+run_and_comment go vet ./...
 
-[ -e $GOBIN/golint ] && run $GOBIN/golint ./...
+#
+# Run Go Lint, another static analysis tool
+#
+
+start_context "test/golint"
+[ -e $GOBIN/golint ] && run_and_comment $GOBIN/golint ./...
+
+#
+# Ensure all files are formatted per the `go fmt` tool
+#
+start_context "test/gofmt"
+unformatted=$(find . -name "*.go" -not -path "./Godeps/*" -print | xargs -n1  gofmt -l)
+if [ "x${unformatted}" == "x" ] ; then
+  update_status --state success
+else
+
+  V="Unformatted files found.
+  Please run 'go fmt' on each of these files and amend your commit to continue."
+
+  for f in ${unformatted}; do
+    V=$(printf "%s\n - %s" "${V}" "${f}")
+  done
+
+  # Print to stdout
+  printf "%s\n\n" "${V}"
+  update_status --state failure --description "${V}"
+
+  # Post a comment with the unformatted list
+  if [ "${TRAVIS_PULL_REQUEST}" != "false" ] ; then
+    run_and_comment printf "%s\n\n" "${V}"
+  fi
+
+  [ "${TRAVIS}" == "true" ] || exit 1 # Stop here if running locally
+fi
+
+#
+# Unit Tests. These do not receive a context or status updates,
+# as they are reflected in our eventual exit code.
+#
+
+# Clear Status context
+end_context
 
 # Ensure SQLite is installed so we don't recompile it each time
 go install ./Godeps/_workspace/src/github.com/mattn/go-sqlite3
@@ -70,9 +153,21 @@ fi
 
 # If the unittests failed, exit before trying to run the integration test.
 if [ ${FAILURE} != 0 ]; then
+  echo "--------------------------------------------------"
+  echo "---          A unit test failed.               ---"
+  echo "--- Stopping before running integration tests. ---"
+  echo "--------------------------------------------------"
   exit ${FAILURE}
 fi
 
+#
+# Integration tests
+#
+
+# Set context to integration, and force a pending state
+start_context "test/integration"
+update_status --state pending --description "Integration Tests in progress"
+
 if [ -z "$LETSENCRYPT_PATH" ]; then
   LETSENCRYPT_PATH=$(mktemp -d -t leXXXX)
 
@@ -94,16 +189,23 @@ fi
 source $LETSENCRYPT_PATH/venv/bin/activate
 export LETSENCRYPT_PATH
 
-run python test/amqp-integration-test.py
-
-unformatted=$(find . -name "*.go" -not -path "./Godeps/*" -print | xargs -n1  gofmt -l)
-if [ "x${unformatted}" != "x" ] ; then
-  echo "Unformatted files found; setting failure state."
-  echo "Please run 'go fmt' on each of these files and amend your commit to continue."
-  FAILURE=1
-  for f in ${unformatted}; do
-    echo "- ${f}"
-  done
-fi
+python test/amqp-integration-test.py
+case $? in
+  0) # Success
+    update_status --state success
+    ;;
+  1) # Python client failed, but Node client didn't, which does
+     # not constitute failure
+    update_status --state success --description "Python integration failed."
+    ;;
+  2) # Node client failed
+    update_status --state failure --description "NodeJS integration failed."
+    FAILURE=1
+    ;;
+  *) # Error occurred
+    update_status --state error --description "Error occurred."
+    FAILURE=1
+    ;;
+esac
 
 exit ${FAILURE}

test/amqp-integration-test.py

@@ -9,11 +9,14 @@
 
 tempdir = tempfile.mkdtemp()
 
+class ExitStatus:
+    OK, PythonFailure, NodeFailure, Error = range(4)
+
 exit_status = 0
 
-def die():
+def die(status):
     global exit_status
-    exit_status = 1
+    exit_status = status
     sys.exit(1)
 
 processes = []
@@ -24,7 +27,7 @@ def run(path):
     cmd = 'go build -tags pkcs11 -o %s %s' % (binary, path)
     print(cmd)
     if subprocess.Popen(cmd, shell=True).wait() != 0:
-        die()
+        die(ExitStatus.Error)
     processes.append(subprocess.Popen('''
         exec %s --config test/boulder-test-config.json
         ''' % binary, shell=True))
@@ -42,25 +45,25 @@ def run_node_test():
         s.connect(('localhost', 4300))
     except socket.error, e:
         print("Cannot connect to WFE")
-        die()
+        die(ExitStatus.Error)
 
     os.chdir('test/js')
 
     if subprocess.Popen('npm install', shell=True).wait() != 0:
         print("\n Installing NPM modules failed")
-        die()
+        die(ExitStatus.Error)
     if subprocess.Popen('''
         node test.js --email foo@letsencrypt.org --agree true \
           --domains foo.com --new-reg http://localhost:4300/acme/new-reg \
           --certKey %s/key.pem --cert %s/cert.der
         ''' % (tempdir, tempdir), shell=True).wait() != 0:
         print("\nIssuing failed")
-        die()
+        die(ExitStatus.NodeFailure)
     if subprocess.Popen('''
         node revoke.js %s/cert.der %s/key.pem http://localhost:4300/acme/revoke-cert
         ''' % (tempdir, tempdir), shell=True).wait() != 0:
         print("\nRevoking failed")
-        die()
+        die(ExitStatus.NodeFailure)
 
     return 0
 
@@ -72,14 +75,14 @@ def run_client_tests():
     os.environ['SERVER'] = 'http://localhost:4300/acme/new-reg'
     test_script_path = os.path.join(root, 'tests', 'boulder-integration.sh')
     if subprocess.Popen(test_script_path, shell=True, cwd=root).wait() != 0:
-        die()
+        die(ExitStatus.PythonFailure)
 
 try:
     start()
     run_node_test()
     run_client_tests()
 except Exception as e:
-    exit_status = 1
+    exit_status = ExitStatus.Error
     print e
 finally:
     for p in processes:

test/github-secret.json.enc

@@ -0,0 +1,2 @@
+P��gxO
+֫�i�������eq�U�E�J����gD}�Oc�G�t�N��x�����=�L��_ �3s�a���4�*)�yE�hZ� �y���U��ԭ(�ƀ�����x(Kv�:�fuE��.޿�0�