docker-build-httpd: continue on failure

tests/docker-build-httpd/cleanup.yml

@@ -0,0 +1,25 @@
+---
+# set ft=ansible
+#
+
+- name: Set MTU on docker daemon
+  lineinfile:
+    dest: /etc/sysconfig/docker-network
+    regexp: "^DOCKER_NETWORK_OPTIONS"
+    line: "DOCKER_NETWORK_OPTIONS="
+
+- name: Restart docker daemon
+  service:
+    name: docker
+    state: restarted
+
+- import_role:
+    name: docker_remove_all
+  tags:
+    - docker_remove_all
+
+- when: ansible_distribution == "RedHat"
+  import_role:
+    name: redhat_unsubscribe
+  tags:
+    - redhat_unsubscribe

tests/docker-build-httpd/docker-build-httpd.yml

@@ -0,0 +1,49 @@
+---
+# set ft=ansible
+#
+
+- name: Fail if variables are not defined
+  when: base_images is undefined or image_names is undefined
+  fail:
+    msg: |
+      Required variables are not defined.
+      Please check tests/docker-build-httpd/vars.yml.
+
+- name: Fail if RHEL variables are not defined
+  when:
+    - (rhel_base_images is undefined or rhel_image_names is undefined)
+    - ansible_distribution == "RedHat"
+  fail:
+    msg: |
+      Required RHEL variables are not defined.
+      Please check tests/docker-build-httpd/vars.yml
+
+- name: Fail if RHEL variables are not defined
+  when: (rhel_base_images is undefined or rhel_image_names is undefined) and
+        ansible_distribution == "RedHat"
+  fail:
+    msg: "Required RHEL variables are not defined.  Please check tests/docker-build-httpd/vars.yml"
+
+# I would have used a 'block' of 'roles' here, but that is not
+# compatible with the 'with_items' loop control, so I stuck all
+# the tasks into a single role file and used an 'include_tasks:'
+# statement on that file.
+#
+# Added the ability to skip broken images with the 'cli_skipped_images'
+# variable.  This can be set via the CLI like so:
+# ansible-playbook -e "cli_skipped_images=busybox_httpd,debian_httpd"
+#
+- include_tasks: tasks/build_run_remove.yml
+  vars:
+    base_dir: "{{ working_dir }}"
+    image_name: "{{ item }}"
+    skipped_images: "{{ cli_skipped_images | default('') }}"
+  with_items: "{{ image_names }}"
+
+- when: ansible_distribution == "RedHat"
+  include_tasks: tasks/build_run_remove.yml
+  vars:
+    base_dir: "{{ working_dir }}"
+    image_name: "{{ item }}"
+    skipped_images: "{{ cli_skipped_images | default('') }}"
+  with_items: "{{ rhel_image_names }}"

tests/docker-build-httpd/main.yml

@@ -1,189 +1,59 @@
----
-# vim: set ft=ansible:
-#
-# !!!NOTE!!! This playbook was tested using Ansible 2.2; any other versions
-# are not supported.
-#
-# This playbook aims to verify that the changes to the Atomic Host platform
-# do not disrupt the ability for users to build/run containers built on top
-# of various base images.  The base images from Docker Hub were selected
-# based on 'popularity' (aka number of pulls).
-#
-# Each base image is used to build an httpd container, which is then run,
-# and the content returned by the container is verified to be correct.
-# Afterwards, the container and build image are removed from the system.
-#
-# In order to use less 'hacks', the playbook is actually multiple playbooks
-# that are divided into setup, tests, and cleanup.  As such, this playbook
-# is meant to be run in its entirety and not separately.
-#
-- name: Docker build httpd - setup
+- name: Docker Build HTTPD - Test Suite
   hosts: all
   become: true
 
-  tags:
-    - docker_build_setup
-
   vars_files:
     - vars.yml
 
-  pre_tasks:
-    - name: Fail if variables are not defined
-      when: base_images is undefined or image_names is undefined
-      fail:
-        msg: |
-          Required variables are not defined.
-          Please check tests/docker-build-httpd/vars.yml.
-
-    - name: Fail if RHEL variables are not defined
-      when: (rhel_base_images is undefined or rhel_image_names is undefined) and
-            ansible_distribution == "RedHat"
-      fail:
-        msg: |
-          Required RHEL variables are not defined.
-          Please check tests/docker-build-httpd/vars.yml
-
-    # Determining the right MTU for Docker interfaces running on OpenStack
-    # can be tricky:
-    #  - https://bugzilla.redhat.com/show_bug.cgi?id=1475460
-    #  - https://bugzilla.redhat.com/show_bug.cgi?id=1357116
-    - name: Set MTU on docker daemon
-      lineinfile:
-        dest: /etc/sysconfig/docker-network
-        regexp: "^DOCKER_NETWORK_OPTIONS"
-        line: "DOCKER_NETWORK_OPTIONS='--mtu={{ ansible_default_ipv4['mtu'] }}'"
+  vars:
+    tests: []
 
-    - name: Restart docker daemon
-      service:
-        name: docker
-        state: restarted
-
-  roles:
-    - role: ansible_version_check
-      tags:
-        - ansible_version_check
-
-    - when: ansible_distribution == "RedHat"
-      role: redhat_subscription
-      tags:
-        - redhat_subscription
-
-  post_tasks:
-    - name: Create working directory
-      command: mktemp -d
-      register: mktemp
-
-    - name: Set the working_dir fact
+  tasks:
+    - name: Set logging
       set_fact:
-        working_dir: "{{ mktemp.stdout }}"
-
-    - name: Copy Dockerfiles
-      synchronize:
-        src: files/
-        dest: "{{ working_dir }}/"
-        recursive: true
-
-    - name: Pull all the upstream base images
-      command: "docker pull {{ item }}"
-      with_items: "{{ base_images }}"
-      register: dp_upstream
-      retries: 5
-      delay: 60
-      until: dp_upstream|success
-
-    - name: Pull the Red Hat base images
-      when: ansible_distribution == "RedHat"
-      command: "docker pull {{ item }}"
-      with_items: "{{ rhel_base_images }}"
-      register: dp_rhel
-      retries: 5
-      delay: 60
-      until: dp_rhel|success
-
-
-- name: Docker build httpd - test
-  hosts: all
-  become: true
-
-  tags:
-    - docker_build_test
-
-  vars_files:
-    - vars.yml
-
-  pre_tasks:
-    - name: Fail if variables are not defined
-      when: base_images is undefined or image_names is undefined
-      fail:
-        msg: |
-          Required variables are not defined.
-          Please check tests/docker-build-httpd/vars.yml.
-
-    - name: Fail if RHEL variables are not defined
-      when:
-        - (rhel_base_images is undefined or rhel_image_names is undefined)
-        - ansible_distribution == "RedHat"
+        log_results: true
+        result_file: "{{ playbook_dir }}/docker-build-httpd-result.log"
+      tags: setup
+
+    - include_tasks: 'setup.yml'
+      tags: setup
+
+    # TEST
+    # Verify user can build httpd with Docker
+    - block:
+        - include_tasks: 'docker-build-httpd.yml'
+        - set_fact:
+            tests: "{{ tests + [ { 'name':'Docker Build HTTPD Test', 'result':'Passed', 'result_details': '' } ] }}"
+      rescue:
+        - set_fact:
+            tests: "{{ tests + [ { 'name':'Docker Build HTTPD Test', 'result':'Failed', 'result_details': ansible_failed_result } ] }}"
+      tags: docker_build_httpd
+
+    # CLEANUP
+    - block:
+        - include_tasks: 'cleanup.yml'
+        - set_fact:
+            tests: "{{ tests + [ { 'name': 'Cleanup', 'result':'Passed', 'result_details': '' } ] }}"
+      rescue:
+        - set_fact:
+            tests: "{{ tests + [ { 'name':'Cleanup', 'result':'Failed', 'result_details': ansible_failed_result } ] }}"
+      always:
+        # WRITE RESULTS TO FILE
+        - name: Remove existing log files
+          local_action: file path={{ result_file }} state=absent
+          become: false
+
+        - name: Save result to file
+          when: log_results
+          local_action: copy content={{ tests | to_nice_yaml(indent=2) }} dest={{ result_file }}
+          become: false
+      tags: cleanup
+
+    # Handled exceptions show up as failures in Ansible but the playbook
+    # itself does not return 0, so explicitly fail the test by checking
+    # the test results
+    - name: Explicitly fail based on test results
+      when: item['result']|lower == "failed"
       fail:
-        msg: |
-          Required RHEL variables are not defined.
-          Please check tests/docker-build-httpd/vars.yml
-
-    - name: Fail if RHEL variables are not defined
-      when: (rhel_base_images is undefined or rhel_image_names is undefined) and
-            ansible_distribution == "RedHat"
-      fail:
-        msg: "Required RHEL variables are not defined.  Please check tests/docker-build-httpd/vars.yml"
-  tasks:
-    # I would have used a 'block' of 'roles' here, but that is not
-    # compatible with the 'with_items' loop control, so I stuck all
-    # the tasks into a single role file and used an 'include_tasks:'
-    # statement on that file.
-    #
-    # Added the ability to skip broken images with the 'cli_skipped_images'
-    # variable.  This can be set via the CLI like so:
-    # ansible-playbook -e "cli_skipped_images=busybox_httpd,debian_httpd"
-    #
-    - include_tasks: tasks/build_run_remove.yml
-      vars:
-        base_dir: "{{ working_dir }}"
-        image_name: "{{ item }}"
-        skipped_images: "{{ cli_skipped_images | default('') }}"
-      with_items: "{{ image_names }}"
-
-    - when: ansible_distribution == "RedHat"
-      include_tasks: tasks/build_run_remove.yml
-      vars:
-        base_dir: "{{ working_dir }}"
-        image_name: "{{ item }}"
-        skipped_images: "{{ cli_skipped_images | default('') }}"
-      with_items: "{{ rhel_image_names }}"
-
-
-- name: Docker build httpd - Cleanup
-  hosts: all
-  become: true
-
-  tags:
-    - docker_build_cleanup
-
-  pre_tasks:
-    - name: Set MTU on docker daemon
-      lineinfile:
-        dest: /etc/sysconfig/docker-network
-        regexp: "^DOCKER_NETWORK_OPTIONS"
-        line: "DOCKER_NETWORK_OPTIONS="
-
-    - name: Restart docker daemon
-      service:
-        name: docker
-        state: restarted
-
-  roles:
-    - role: docker_remove_all
-      tags:
-        - docker_remove_all
-
-    - role: redhat_unsubscribe
-      when: ansible_distribution == "RedHat"
-      tags:
-        - redhat_unsubscribe
+        msg: "Failure found in test"
+      with_items: "{{ tests }}"

tests/docker-build-httpd/setup.yml

@@ -0,0 +1,75 @@
+---
+# set ft=ansible
+#
+
+- import_role:
+    name: ansible_version_check
+  tags:
+    - ansible_version_check
+
+- when: ansible_distribution == "RedHat"
+  import_role:
+    name: redhat_subscription
+  tags:
+    - redhat_subscription
+
+- name: Fail if variables are not defined
+  when: base_images is undefined or image_names is undefined
+  fail:
+    msg: |
+      Required variables are not defined.
+      Please check tests/docker-build-httpd/vars.yml.
+
+- name: Fail if RHEL variables are not defined
+  when: (rhel_base_images is undefined or rhel_image_names is undefined) and
+        ansible_distribution == "RedHat"
+  fail:
+    msg: |
+      Required RHEL variables are not defined.
+      Please check tests/docker-build-httpd/vars.yml
+
+# Determining the right MTU for Docker interfaces running on OpenStack
+# can be tricky:
+#  - https://bugzilla.redhat.com/show_bug.cgi?id=1475460
+#  - https://bugzilla.redhat.com/show_bug.cgi?id=1357116
+- name: Set MTU on docker daemon
+  lineinfile:
+    dest: /etc/sysconfig/docker-network
+    regexp: "^DOCKER_NETWORK_OPTIONS"
+    line: "DOCKER_NETWORK_OPTIONS='--mtu={{ ansible_default_ipv4['mtu'] }}'"
+
+- name: Restart docker daemon
+  service:
+    name: docker
+    state: restarted
+
+- name: Create working directory
+  command: mktemp -d
+  register: mktemp
+
+- name: Set the working_dir fact
+  set_fact:
+    working_dir: "{{ mktemp.stdout }}"
+
+- name: Copy Dockerfiles
+  synchronize:
+    src: files/
+    dest: "{{ working_dir }}/"
+    recursive: true
+
+- name: Pull all the upstream base images
+  command: "docker pull {{ item }}"
+  with_items: "{{ base_images }}"
+  register: dp_upstream
+  retries: 5
+  delay: 60
+  until: dp_upstream|success
+
+- name: Pull the Red Hat base images
+  when: ansible_distribution == "RedHat"
+  command: "docker pull {{ item }}"
+  with_items: "{{ rhel_base_images }}"
+  register: dp_rhel
+  retries: 5
+  delay: 60
+  until: dp_rhel|success