Merge pull request intelsdi-x#25 from intelsdi-x/build

Large tests for psutil

.gitignore

@@ -29,6 +29,7 @@ build/
 *.swp
 profile.cov
 gin-bin
+*.pyc
 
 # we don't vendor godep _workspace
 **/Godeps/_workspace/**

.travis.yml

@@ -1,23 +1,48 @@
 sudo: false
 language: go
 go:
-- 1.5.3
-- 1.6
-before_install:
-- go get github.com/tools/godep
-- if [ ! -d $SNAP_PLUGIN_SOURCE ]; then mkdir -p $HOME/gopath/src/github.com/intelsdi-x; ln -s $TRAVIS_BUILD_DIR $SNAP_PLUGIN_SOURCE; fi # CI for forks not from intelsdi-x
+- 1.6.3
+- 1.7
 env:
   global:
-    - SNAP_PLUGIN_SOURCE=/home/travis/gopath/src/github.com/intelsdi-x/snap-plugin-collector-psutil
+    - ORG_PATH=/home/travis/gopath/src/github.com/intelsdi-x
+    - SNAP_PLUGIN_SOURCE=/home/travis/gopath/src/github.com/${TRAVIS_REPO_SLUG}
+    - GO15VENDOREXPERIMENT=1
   matrix:
-    - TEST=unit
+    - TEST=legacy
+    - TEST=build
+matrix:
+  exclude:
+    - go: 1.6.3
+      env: TEST_TYPE=build
+before_install:
+  - go get github.com/tools/godep
+  - if [ ! -d $SNAP_PLUGIN_SOURCE ]; then mkdir -p $HOME/gopath/src/github.com/intelsdi-x; ln -s $TRAVIS_BUILD_DIR $SNAP_PLUGIN_SOURCE; fi # CI for forks not from intelsdi-x
 install:
-- export TMPDIR=$HOME/tmp
-- mkdir -p $TMPDIR
-- cd $SNAP_PLUGIN_SOURCE # change dir into source
-- make deps
+  - export TMPDIR=$HOME/tmp
+  - mkdir -p $TMPDIR
+  - cd $SNAP_PLUGIN_SOURCE # change dir into source
+  - make deps
 script:
-- make check TEST=$TEST 2>&1 # Run test suite
+  - make check TEST=$TEST 2>&1 # Run test suite
+before_deploy:
+  - make all
+  - "./scripts/pre_deploy.sh"
+deploy:
+  provider: s3
+  access_key_id: AKIAINMB43VSSPFZISAA
+  secret_access_key:
+    secure: YvbymtPIqf/VvjBGlZNNTr1bQ5BWvBETEmTq9xprCzfYz5WvNKS7VYDrLmN8ZKF9SJAjpad0a93TCSkR/78Z/awMIKscCWODsYyYfSDkzN+uIqFhFKse6YnaU3795qmt8gZAf4guNQthOYGtlLEwsl3O0vlmesCXAR6Dwi1IGN+t9BUnCgM7OmcTcNpRHoedwIgY0M7c+JbFJtx1vP1+Tc1JL5w8q7PaMdsTPlbhW7w+i+zrjYpuskxcc+A6VoYB+lKf+wrX/uHM8pHSPYJOS5Kp0m8khbLED820geoZ79knUTcxMxrqPsXXk9/QqSK0dR0M5Jey8j2H9jUfAEgN/4nAftSVTJitVnDlIU2blx6Fly/nNkiZHqXw9T87Sdl9ZF/sTOwDJaPnsDbh9gwf9QXeWwi7k+qFBu1IhYtoV6K/ynRVHWBl1SaHOBD3GcaA4rLYQM1Bk2vC4Cs75IA5o1wFsg9OFkPAjwawasaXKypQpP/Zk0hfDS3HsX3fDObHOelH1i5ajqxltZEg4OJCMkKyRfjAHtulxtRTLzxMm+Cj/flBRvr8UvhUEeUuCMDXszatbwKMLLFGY0hgKDqpjCliSnRqnHBfbDhc1t5So02x2ngx4Uw2tzySvWjDCBGRDRNeI+rE78VCD4EQWZCJRH10+Eqdxe209aNLMI/dli0=
+  bucket: snap.ci.snap-telemetry.io
+  region: us-west-2
+  skip_cleanup: true
+  local-dir: build
+  upload-dir: plugin
+  acl: public_read
+  on:
+    repo: intelsdi-x/snap-plugin-collector-psutil
+    branch: build
+    condition: $TEST_TYPE=build && $TRAVIS_GO_VERSION="1.7"
 notifications:
   slack:
     secure: VkbZLIc2RH8yf3PtIAxUNPdAu3rQQ7yQx0GcK124JhbEnZGaHyK615V0rbG7HcVmYKGPdB0cXqZiLBDKGqGKb2zR1NepOe1nF03jxGSpPq8jIFeEXSJGEYGL34ScDzZZGuG6qwbjFcXiW5lqn6t8igzp7v2+URYBaZo5ktCS2xY=

psutil/psutil_test.go

@@ -1,5 +1,5 @@
 //
-// +build unit
+// +build small
 
 /*
 http://www.apache.org/licenses/LICENSE-2.0.txt

scripts/build.sh

@@ -1,5 +1,22 @@
 #!/bin/bash -e
 
+#http://www.apache.org/licenses/LICENSE-2.0.txt
+#
+#
+#Copyright 2016 Intel Corporation
+#
+#Licensed under the Apache License, Version 2.0 (the "License");
+#you may not use this file except in compliance with the License.
+#You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+#Unless required by applicable law or agreed to in writing, software
+#distributed under the License is distributed on an "AS IS" BASIS,
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#See the License for the specific language governing permissions and
+#limitations under the License.
+
 GITVERSION=`git describe --always`
 SOURCEDIR=$1
 BUILDDIR=$SOURCEDIR/build

scripts/common.sh

@@ -0,0 +1,112 @@
+#!/bin/bash
+
+#http://www.apache.org/licenses/LICENSE-2.0.txt
+#
+#
+#Copyright 2016 Intel Corporation
+#
+#Licensed under the Apache License, Version 2.0 (the "License");
+#you may not use this file except in compliance with the License.
+#You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+#Unless required by applicable law or agreed to in writing, software
+#distributed under the License is distributed on an "AS IS" BASIS,
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#See the License for the specific language governing permissions and
+#limitations under the License.
+
+set -e
+set -u
+set -o pipefail
+
+LOG_LEVEL="${LOG_LEVEL:-6}"
+NO_COLOR="${NO_COLOR:-}"
+
+trap_exitcode() {
+  exit $?
+}
+
+trap trap_exitcode SIGINT
+
+_fmt () {
+  local color_debug="\x1b[35m"
+  local color_info="\x1b[32m"
+  local color_notice="\x1b[34m"
+  local color_warning="\x1b[33m"
+  local color_error="\x1b[31m"
+  local colorvar=color_$1
+
+  local color="${!colorvar:-$color_error}"
+  local color_reset="\x1b[0m"
+  if [ "${NO_COLOR}" = "true" ] || [[ "${TERM:-}" != "xterm"* ]] || [ -t 1 ]; then
+    # Don't use colors on pipes or non-recognized terminals
+    color=""; color_reset=""
+  fi
+  echo -e "$(date -u +"%Y-%m-%d %H:%M:%S UTC") ${color}$(printf "[%9s]" "${1}")${color_reset}";
+}
+
+_debug ()   { [ "${LOG_LEVEL}" -ge 7 ] && echo "$(_fmt debug) ${*}" 1>&2 || true; }
+_info ()    { [ "${LOG_LEVEL}" -ge 6 ] && echo "$(_fmt info) ${*}" 1>&2 || true; }
+_notice ()  { [ "${LOG_LEVEL}" -ge 5 ] && echo "$(_fmt notice) ${*}" 1>&2 || true; }
+_warning () { [ "${LOG_LEVEL}" -ge 4 ] && echo "$(_fmt warning) ${*}" 1>&2 || true; }
+_error ()   { [ "${LOG_LEVEL}" -ge 3 ] && echo "$(_fmt error) ${*}" 1>&2 || true; exit 1; }
+
+_test_dirs() {
+  local test_dirs=$(find . -type f -name '*.go' -not -path "./.*" -not -path "*/_*" -not -path "./Godeps/*" -not -path "./vendor/*" -print0 | xargs -0 -n1 dirname| sort -u)
+  echo "$test_dirs"
+}
+
+#_debug "go code directories:
+#${test_dirs}"
+
+_go_get() {
+  local _url=$1
+  local _util
+
+  _util=$(basename "${_url}")
+
+  type -p "${_util}" > /dev/null || go get "${_url}" && _debug "go get ${_util} ${_url}"
+}
+
+_goimports() {
+  _go_get golang.org/x/tools/cmd/goimports
+  test -z "$(goimports -l -d $(find . -type f -name '*.go' -not -path "./vendor/*") | tee /dev/stderr)"
+}
+
+_golint() {
+  _go_get github.com/golang/lint/golint
+  golint ./...
+}
+
+_go_vet() {
+  go vet $(_test_dirs)
+}
+
+_go_race() {
+  go test -race ./...
+}
+
+_go_test() {
+  _info "running test type: ${TEST_TYPE}"
+  # Standard go tooling behavior is to ignore dirs with leading underscors
+  for dir in $(_test_dirs);
+  do
+    if [[ -z ${go_cover+x} ]]; then
+      _debug "running go test with cover in ${dir}"
+      go test -v --tags="${TEST_TYPE}" -covermode=count -coverprofile="${dir}/profile.tmp" "${dir}"
+      if [ -f "${dir}/profile.tmp" ]; then
+        tail -n +2 "${dir}/profile.tmp" >> profile.cov
+        rm "${dir}/profile.tmp"
+      fi
+    else
+      _debug "running go test without cover in ${dir}"
+      go test -v --tags="${TEST_TYPE}" "${dir}"
+    fi
+  done
+}
+
+_go_cover() {
+  go tool cover -func profile.cov
+}

scripts/docker/large/docker-compose.yml

@@ -0,0 +1,9 @@
+version: '2'
+services:
+  main:
+    container_name: psutil_large_test
+    image: mkrolik/alpine-python:latest
+    network_mode: "host"
+    volumes:
+      - ${PLUGIN_SRC}:/snap-plugin-collector-psutil
+    entrypoint: sh -c 'python /snap-plugin-collector-psutil/scripts/test/large.py'

scripts/docker/large/psutil-task.yml

@@ -0,0 +1,10 @@
+---
+  version: 1
+  schedule: 
+    type: "simple"
+    interval: "1s"
+  max-failures: 10
+  workflow: 
+    collect: 
+      metrics: 
+        /intel/psutil/vm/total: {}

scripts/large.sh

@@ -0,0 +1,53 @@
+#!/bin/bash 
+
+#http://www.apache.org/licenses/LICENSE-2.0.txt
+#
+#
+#Copyright 2016 Intel Corporation
+#
+#Licensed under the Apache License, Version 2.0 (the "License");
+#you may not use this file except in compliance with the License.
+#You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+#Unless required by applicable law or agreed to in writing, software
+#distributed under the License is distributed on an "AS IS" BASIS,
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#See the License for the specific language governing permissions and
+#limitations under the License.
+
+set -e
+set -u
+set -o pipefail
+
+__dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+__proj_dir="$(dirname "$__dir")"
+__proj_name="$(basename $__proj_dir)"
+
+. "${__dir}/common.sh"
+
+# NOTE: these variables control the docker-compose image.
+export PLUGIN_SRC="${__proj_dir}"
+export LOG_LEVEL="${LOG_LEVEL:-"7"}"
+export PROJECT_NAME="${__proj_name}"
+
+TEST_TYPE="${TEST_TYPE:-"large"}"
+
+docker_folder="${__proj_dir}/scripts/docker/${TEST_TYPE}"
+
+_docker_project () {
+  echo ${docker_folder}
+  cd "${docker_folder}" && "$@"
+}
+
+_debug "building docker compose images"
+_docker_project docker-compose build
+_debug "running test: ${TEST_TYPE}"
+_docker_project docker-compose up
+test_res=`docker-compose ps -q | xargs docker inspect -f '{{ .Name }} exited with status {{ .State.ExitCode }}' | awk '{print $5}'`
+echo "exit code from large_jenkins $test_res"
+_debug "stopping docker compose images"
+_docker_project docker-compose stop
+_docker_project docker-compose rm -f
+exit $test_res

scripts/pre_deploy.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+
+#http://www.apache.org/licenses/LICENSE-2.0.txt
+#
+#
+#Copyright 2016 Intel Corporation
+#
+#Licensed under the Apache License, Version 2.0 (the "License");
+#you may not use this file except in compliance with the License.
+#You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+#Unless required by applicable law or agreed to in writing, software
+#distributed under the License is distributed on an "AS IS" BASIS,
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#See the License for the specific language governing permissions and
+#limitations under the License.
+
+set -e
+set -u
+set -o pipefail
+
+__dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+__proj_dir="$(dirname "$__dir")"
+
+# shellcheck source=scripts/common.sh
+. "${__dir}/common.sh"
+
+build_path="${__proj_dir}/build"
+git_sha=$(git log --pretty=format:"%H" -1)
+git_path="${build_path}/${TRAVIS_BRANCH}/${git_sha}"
+latest_path="${build_path}/${TRAVIS_BRANCH}/latest"
+
+mkdir -p "${git_path}"
+mkdir -p "${latest_path}"
+
+_info "copying binary to ${git_path}"
+cp "${build_path}/rootfs/"* "${git_path}"
+_info "copying snap binaries to ${latest_path}"
+mv "${build_path}/rootfs/"* "${latest_path}"