standard test invocation: take 2

Makefile

@@ -20,13 +20,13 @@ default: run
 root/:
 	mkdir -p ./root
 
-$(RENDERED_DOCKERFILE_MD): $(SOURCE_DOCKERFILE_MD)
+$(RENDERED_DOCKERFILE_MD): $(SOURCE_DOCKERFILE_MD) specs/*
 	$(DG_EXEC) --template $(SOURCE_DOCKERFILE_MD) --output $(RENDERED_DOCKERFILE_MD)
 
-$(RENDERED_README_MD): $(SOURCE_README_MD)
+$(RENDERED_README_MD): $(SOURCE_README_MD) specs/*
 	$(DG_EXEC) --template $(SOURCE_README_MD) --output $(RENDERED_README_MD)
 
-$(RENDERED_HELP_MD): $(SOURCE_HELP_MD) specs/multispec.yaml
+$(RENDERED_HELP_MD): $(SOURCE_HELP_MD) specs/*
 	@# FIXME: current go-md2man can't convert tables :<
 	@# go-md2man -in=${SOURCE_HELP_MD} -out=./root/help.1
 	$(shell TOOLS_CONTAINER_SKIP_ENUMERATION=false $(DG_EXEC) --template $(SOURCE_HELP_MD) --output $(RENDERED_HELP_MD))
@@ -53,6 +53,9 @@ check: test
 test: build
 	make -C tests/ check-local IMAGE_NAME=$(REPOSITORY)
 
+check-in-vm: build
+	make -C tests/ check-in-vm IMAGE_NAME=$(REPOSITORY)
+
 clean:
 	rm Dockerfile || :
 	rm root/README.md || :

tests/Makefile

@@ -1,14 +1,16 @@
 .PHONY: check-local check-in-vm
 
 IMAGE_NAME := ""
-VM_IMAGE := ../Fedora-Cloud-Base-27-1.6.x86_64.qcow2
+VM_IMAGE_NAME := Fedora-Cloud-Base-27-1.6.x86_64.qcow2
+VM_IMAGE_PATH = ../$(VM_IMAGE_NAME)
+INVENTORY := /usr/share/ansible/inventory/standard-inventory-qcow2
 
 check-local:
-	ansible-playbook -e subject=$(IMAGE_NAME) -v ./tests.yml
+	ansible-playbook -e subject=$(IMAGE_NAME) ./local.yml
 
-check-in-vm: $(VM_IMAGE)
-	ansible-playbook -e setup=true -e vm_image=$(VM_IMAGE) ./tests.yml
+check-in-vm: $(VM_IMAGE_PATH)
+	TEST_SUBJECTS=$(VM_IMAGE_PATH) ansible-playbook -e ansible_python_interpreter=/usr/bin/python3 -e subject=$(IMAGE_NAME) -i $(INVENTORY) -e setup=true -e vm_image=$(VM_IMAGE) ./in-vm.yml
 
-$(VM_IMAGE):
-	curl -o $(VM_IMAGE) -s https://download.fedoraproject.org/fedora/releases/27/CloudImages/x86_64/images/$(VM_IMAGE)
+$(VM_IMAGE_PATH):
+	curl -o $(VM_IMAGE_PATH) -s https://download.fedoraproject.org/fedora/releases/27/CloudImages/x86_64/images/$(VM_IMAGE_NAME)
 

tests/ansible.cfg

@@ -1,3 +1,4 @@
 [defaults]
 inventory = ./inventory
 retry_files_enabled = false
+# roles_path = </path/to/the/repo>/roles

tests/in-vm.yml

@@ -0,0 +1,80 @@
+# support use cases:
+#  * testing an image built locally in current environment (set pull variable to false)
+#  * testing an image built locally inside a VM (supply vm_image variable)
+#  * testing an image present in registry in current environment (set pull to true)
+#  * testing an image present in registry inside a VM (set pull and vm_image variables)
+---
+- name: Integration tests for tools container image executed in current environment
+  hosts: localhost
+  vars:
+    # don't pull the test subject by default
+    pull: false
+
+    # don't set up the environment by default (instal and start container runtime)
+    setup: false
+
+    required_packages:
+     - python3-conu
+     - python3-pytest
+
+    tests:
+    - integration
+
+    # our test subject
+    subject: ""
+
+    # path where the test artifacts will be stored - logs
+    artifacts: "{{ playbook_dir }}/artifacts/"
+
+  tasks:
+  - name: prepare the environment to run tests
+    block:
+    - name: Install the container engine
+      package:
+        name: docker
+        state: present
+      become: true
+    - name: Start the container engine
+      systemd:
+        name: docker
+        state: started
+      become: true
+    when: setup
+
+  - name: Pull the test subject (=container image)
+    command: docker pull {{ subject }}
+    when: pull
+
+  - name: Copy test subject from host inside the VM
+    block:
+    # FIXME: make this configurable
+    - name: Create temporary directory for the image
+      tempfile:
+        state: directory
+      register: tmp
+    - name: Save the image to a file
+      command: 'docker save -o {{ tmp.path + "/image.tar.gz" }} {{ subject }}'
+    - name: Copy the image from host to the target
+      # synchronize is so unreliable
+      synchronize:
+        src: '{{ tmp.path + "/image.tar.gz" }}'
+        dest: '/tmp/'
+        mode: push
+        ssh_args: "-o UserKnownHostsFile=/dev/null -i {{ ansible_ssh_private_key_file }}"
+    - file:
+        state: absent
+        path: "{{ tmp.path }}"
+    when: not pull
+    delegate_to: localhost
+
+  - block:
+    - name: Load the image on the target into dockerd
+      command: 'docker load -i /tmp/image.tar.gz'
+    - file:
+        state: absent
+        path: "/tmp/image.tar.gz"
+    when: not pull
+
+  - name: Execute the role which performs testing
+    import_role:
+      name: standard-test-basic

tests/local.yml

@@ -0,0 +1,61 @@
+# support use cases:
+#  * testing an image built locally in current environment (set pull variable to false)
+#  * testing an image present in registry in current environment (set pull to true)
+---
+- name: Integration tests for tools container image executed in current environment
+  hosts: localhost
+  vars:
+    # don't pull the test subject by default
+    pull: false
+
+    # don't set up the environment by default (install and start container runtime)
+    setup: false
+
+    required_packages:
+     - python3-conu
+     - python3-pytest
+
+    tests:
+    - integration
+
+    # our test subject
+    subject: ""
+
+    # path where the test artifacts will be stored - logs
+    artifacts: "{{ playbook_dir }}/artifacts/"
+
+  tasks:
+  - name: prepare the environment to run tests
+    block:
+    - name: Install the container engine
+      package:
+        name: docker
+        state: present
+      become: true
+    - name: Start the container engine
+      systemd:
+        name: docker
+        state: started
+      become: true
+    when: setup
+
+  - name: Pull the test subject (=container image)
+    command: docker pull {{ subject }}
+    when: pull
+
+  - block:
+    # should this be configurable?
+    - name: Create temp dir to store tests
+      tempfile:
+        state: directory
+      register: tmp_tests
+    - name: Execute the role which performs testing
+      import_role:
+        name: standard-test-basic
+      vars:
+        tenv_workdir: "{{ tmp_tests.path }}"
+    always:
+    - name: delete the temp dir
+      file:
+        path: "{{ tmp_tests.path }}"
+        state: absent