tests: Merge installed/ and fedora-str/ directories

Let's be opinionated now, and our installed/ test story *is*
Ansible/STR.  Merge `tests/fedora-str` into `tests/installed/`.

Rework the nondestructive tests into a separate playbook run, and parallelize
them for more efficiency.

The destructive tests are also changed to use Ansible more.

Add a higher level `run.sh` entrypoint and update the `README.md`
with some useful tips.

.papr.yml

@@ -15,11 +15,10 @@ tests:
   - ./tests/fedora-str/provision.sh
   # TODO: enhance papr to have caching, a bit like https://docs.travis-ci.com/user/caching/
   - curl -Lo fedora-atomic-host.qcow2 https://getfedora.org/atomic_qcow2_latest
-  - env "TEST_SUBJECTS=$(pwd)/fedora-atomic-host.qcow2" ./tests/fedora-str/playbook-run.sh tests/fedora-str/sysinstall-tests.yml
+  - env "TEST_SUBJECTS=$(pwd)/fedora-atomic-host.qcow2" ./tests/installed/run.sh
 
 artifacts:
-  - tests/fedora-str/artifacts/fedora-atomic-host.qcow2.log
-  - tests/fedora-str/artifacts/installed-tests.log
+  - tests/installed/artifacts/
 
 ---
 

tests/installed/README.md

@@ -1,2 +1,12 @@
-This suite of tests is currently run from redhat-ci;
-they're intended to run as root.
+This directory holds tests that use the
+[Fedora Standard Test Interface](https://fedoraproject.org/wiki/CI/Standard_Test_Interface).
+
+The high level structure is that we take a qcow2 file, inject
+built RPMs into it, and then use Ansible to run tests.
+
+See `papr.yml` for canonical usage.
+
+For local development, you should cache the qcow2 somewhere
+stable (outside of this git repo).  Also note that `../ci/build-rpms.sh`
+does *not* pick up uncommitted changes!  Stated more strongly, you
+currently need to run `build-rpms.sh` after every change.

tests/installed/destructive.yml

@@ -0,0 +1,35 @@
+# This entrypoint right now just runs the sysinstalled-tests.
+---
+- hosts: localhost
+  tags:
+  - atomic
+  remote_user: root
+  vars:
+    use_git_build: True
+    tests: "."
+  tasks:
+    - import_tasks: overlay-git.yml
+      when: use_git_build
+    # Next copy all of the tests/ directory
+    - name: Copy test data
+      synchronize: src=../../ dest=/root/tests/ archive=yes
+    - find:
+        paths: /root/tests/installed/destructive
+        patterns: "itest-*.sh"
+      register: all_tests
+    - set_fact:
+        selected_tests: "{{ all_tests.files|map(attribute='path') | select('match', tests) | list }}"
+    - assert:
+        that:
+          - "{{ selected_tests|length }} != 0"
+    - file: path=/root/logs state=directory
+    - block:
+        - name: Run destructive tests
+          shell: "{{ item }} &> /root/logs/$(basename {{ item }}).log"
+          with_items:
+            - "{{ selected_tests }}"
+      always:
+        - synchronize:
+            src: /root/logs/
+            dest: artifacts/installed-destructive
+            mode: pull

tests/installed/destructive/README.md

@@ -0,0 +1,5 @@
+This suite of tests is run from PAPR. Everything in here is destructive; it's
+recommended to only run them in disposable virtual machines.  This is done
+in `tests/fedora-str/sysinstalled-tests.yml`, which currently uses a single VM
+and runs the tests serially.  It's likely in the future this will be changed
+to do one VM per test.

tests/installed/itest-bare-root.sh → tests/installed/destructive/itest-bare-root.sh

@@ -5,7 +5,7 @@
 set -xeuo pipefail
 
 dn=$(dirname $0)
-. ${dn}/libinsttest.sh
+. ${dn}/../libinsttest.sh
 
 echo "1..2"
 

tests/installed/itest-deploy-selinux.sh → tests/installed/destructive/itest-deploy-selinux.sh

@@ -5,7 +5,7 @@
 set -xeuo pipefail
 
 dn=$(dirname $0)
-. ${dn}/libinsttest.sh
+. ${dn}/../libinsttest.sh
 
 # Create a new deployment
 ostree admin deploy --karg-proc-cmdline ${host_refspec}

tests/installed/itest-label-selinux.sh → tests/installed/destructive/itest-label-selinux.sh

@@ -5,7 +5,7 @@
 set -xeuo pipefail
 
 dn=$(dirname $0)
-. ${dn}/libinsttest.sh
+. ${dn}/../libinsttest.sh
 
 cd /ostree/repo/tmp
 rm co -rf

tests/installed/execute_batch.yml

@@ -0,0 +1,22 @@
+#####################
+# execute_batch.yml
+#####################
+- name: Begin async command execution
+  shell: "{{ async_item }} &> {{ logdir }}/{{ async_item|basename }}.log"
+  # 10 minutes; the PAPR tester generally times out before that
+  async: 600
+  poll: 0
+  with_items: "{{ async_commands }}"
+  loop_control:
+    loop_var: "async_item"
+  register: async_results
+
+- name: Check async command status
+  async_status:
+    jid: "{{ async_result_item.ansible_job_id }}"
+  with_items: "{{ async_results.results }}"
+  loop_control:
+    loop_var: "async_result_item"
+  register: async_poll_results
+  until: async_poll_results.finished
+  retries: 240

tests/installed/libinsttest.sh

@@ -20,7 +20,7 @@
 # Boston, MA 02111-1307, USA.
 
 dn=$(dirname $0)
-. ${dn}/libtest-core.sh
+. ${dn}/../libtest-core.sh
 
 # Copy of bits from tap-test
 test_tmpdir=

tests/installed/nondestructive.yml

@@ -0,0 +1,40 @@
+# Nondestructive sysinstalled tests, run in parallel.
+---
+- hosts: localhost
+  tags:
+  - atomic
+  remote_user: root
+  vars:
+    use_git_build: True
+    tests: "."
+    # Arbitrary...we want some parallelism
+    batching_factor: 4
+  tasks:
+    - import_tasks: overlay-git.yml
+      when: use_git_build
+    # Next copy all of the tests/ directory
+    - name: Copy test data
+      synchronize: src=../../ dest=/root/tests/ archive=yes
+    - find:
+        paths: /root/tests/installed/nondestructive
+        patterns: "itest-*.sh"
+      register: all_tests
+    - set_fact:
+        selected_tests: "{{ all_tests.files|map(attribute='path') | select('match', tests) | list }}"
+    - assert:
+        that:
+          - "{{ selected_tests|length }} != 0"
+    - file: path=/root/logs state=directory
+    - block:
+        - name: Run nondestructive tests
+          vars:
+            logdir: /root/logs
+          async_commands: "{{ item }}"
+          include_tasks: execute_batch.yml
+          with_items:
+            - "{{ selected_tests | batch('{{ batching_factor }}') | list }}"
+      always:
+        - synchronize:
+            src: /root/logs
+            dest: artifacts/installed-nondestructive
+            mode: pull

tests/installed/itest-bare-unit.sh → tests/installed/nondestructive/itest-bare-unit.sh

@@ -6,13 +6,13 @@
 set -xeuo pipefail
 
 dn=$(dirname $0)
-. ${dn}/libinsttest.sh
+. ${dn}/../libinsttest.sh
 
 # These tests sort of bypass the installed-tests spec;
 # fixing that would require installing g-d-t-r, though
 # more ideally we architect things with a "control" container
 # distinct from the host.
-export G_TEST_SRCDIR=$(realpath $dn/../..)
+export G_TEST_SRCDIR=$(realpath $dn/../../..)
 
 # Use /var/tmp to hopefully use XFS + O_TMPFILE etc.
 prepare_tmpdir /var/tmp

tests/installed/itest-bare-user-root.sh → tests/installed/nondestructive/itest-bare-user-root.sh

@@ -5,7 +5,7 @@
 set -xeuo pipefail
 
 dn=$(dirname $0)
-. ${dn}/libinsttest.sh
+. ${dn}/../libinsttest.sh
 
 echo "1..1"
 

tests/installed/itest-bareuser-nouserxattrs.sh → tests/installed/nondestructive/itest-bareuser-nouserxattrs.sh

@@ -8,7 +8,7 @@
 set -xeuo pipefail
 
 dn=$(dirname $0)
-. ${dn}/libinsttest.sh
+. ${dn}/../libinsttest.sh
 
 prepare_tmpdir
 trap _tmpdir_cleanup EXIT

tests/installed/itest-payload-link.sh → tests/installed/nondestructive/itest-payload-link.sh

@@ -22,7 +22,7 @@
 set -xeuo pipefail
 
 dn=$(dirname $0)
-. ${dn}/libinsttest.sh
+. ${dn}/../libinsttest.sh
 
 echo "1..1"
 

tests/installed/itest-pull-space.sh → tests/installed/nondestructive/itest-pull-space.sh

@@ -4,18 +4,19 @@
 set -xeuo pipefail
 
 dn=$(dirname $0)
-. ${dn}/libinsttest.sh
+. ${dn}/../libinsttest.sh
 
 prepare_tmpdir
 trap _tmpdir_cleanup EXIT
 
 cd ${test_tmpdir}
-truncate -s 100MB testblk.img
+truncate -s 20MB testblk.img
 blkdev=$(losetup --find --show $(pwd)/testblk.img)
 mkfs.xfs ${blkdev}
 mkdir mnt
 mount ${blkdev} mnt
 ostree --repo=mnt/repo init --mode=bare-user
+echo 'fsync=false' >> mnt/repo/config
 if ostree --repo=mnt/repo pull-local /ostree/repo ${host_commit} 2>err.txt; then
     fatal "succeeded in doing a pull with no free space"
 fi

tests/installed/itest-pull.sh → tests/installed/nondestructive/itest-pull.sh

@@ -5,7 +5,7 @@
 set -xeuo pipefail
 
 dn=$(dirname $0)
-. ${dn}/libinsttest.sh
+. ${dn}/../libinsttest.sh
 
 prepare_tmpdir /var/tmp
 trap _tmpdir_cleanup EXIT
@@ -37,3 +37,6 @@ ostree --repo=repo pull-local /ostree/repo ${host_commit}
 ostree --repo=repo fsck
 cd ..
 umount mnt
+
+kill -TERM $(cat ${test_tmpdir}/httpd-pid)
+echo "ok"

tests/installed/itest-remotes.sh → tests/installed/nondestructive/itest-remotes.sh

@@ -5,7 +5,7 @@
 set -xeuo pipefail
 
 dn=$(dirname $0)
-. ${dn}/libinsttest.sh
+. ${dn}/../libinsttest.sh
 
 prepare_tmpdir
 trap _tmpdir_cleanup EXIT

tests/installed/nondestructive/libtest-core.sh

@@ -0,0 +1 @@
+../libtest-core.sh

tests/fedora-str/overlay-git.yml → tests/installed/overlay-git.yml

@@ -6,7 +6,7 @@
 - set_fact:
     ostree_orig_version_yaml: "{{ ostree_orig_version.stdout | from_yaml }}"
 - name: Copy locally built RPMs
-  synchronize: src=x86_64/ dest=/root/x86_64/ archive=yes
+  synchronize: src=build/x86_64/ dest=/root/x86_64/ archive=yes
 - shell: ostree admin unlock || true
 # Install the RPMs we already have.  For the test suite we use rpm2cpio
 # since it depends on libsoup, but we're not using that yet for the sysinstalled tests

tests/installed/run.sh

@@ -1,17 +1,17 @@
-#!/bin/bash
-
+#!/usr/bin/bash
+# Run all installed tests; see README.md in this directory for more
+# information.
 set -xeuo pipefail
 
-dn=$(dirname $0)
-for tn in ${dn}/itest-*.sh; do
-    if [ -n "${TESTS:-}" ]; then
-      tbn=$(basename "$tn" .sh)
-      tbn=" ${tbn#itest-} "
-      if [[ " $TESTS " != *$tbn* ]]; then
-        echo "Skipping: ${tn}"
-        continue
-      fi
-    fi
-    echo Executing: ${tn}
-    ${tn}
+dn=$(cd $(dirname $0) && pwd)
+
+if ! test -d build; then
+    mkdir -p build
+    (cd build && ${dn}/../../ci/build-rpm.sh)
+fi
+
+# TODO: parallelize this
+PLAYBOOKS=${PLAYBOOKS:-destructive.yml nondestructive.yml}
+for playbook in $PLAYBOOKS; do
+    ${dn}/playbook-run.sh ${playbook}
 done