tests: Merge installed/ and fedora-str/ directories

.papr.yml

@@ -7,24 +7,29 @@ branches:
 context: FAH27-insttests
 required: true
 
-container:
-  image: registry.fedoraproject.org/fedora:27
+# FIXME; temporary workaround
+# https://github.com/ostreedev/ostree/pull/1513#issuecomment-378784162
+host:
+  distro: fedora/27/atomic
+  specs:
+    ram: 4096
+#container:
+#  image: registry.fedoraproject.org/fedora:27
 
 tests:
-  - cd tests/fedora-str && ../../ci/build-rpm.sh
-  - ./tests/fedora-str/provision.sh
-  # TODO: enhance papr to have caching, a bit like https://docs.travis-ci.com/user/caching/
-  - curl -Lo fedora-atomic-host.qcow2 https://getfedora.org/atomic_qcow2_latest
-  - env "TEST_SUBJECTS=$(pwd)/fedora-atomic-host.qcow2" ./tests/fedora-str/playbook-run.sh tests/fedora-str/sysinstall-tests.yml
+  - cd /etc/yum.repos.d/ && curl -L -O https://copr.fedorainfracloud.org/coprs/walters/oci-kvm-hook/repo/fedora-27/walters-oci-kvm-hook-fedora-27.repo
+  - rpm-ostree install oci-kvm-hook && rpm-ostree ex livefs
+  - docker run --device /dev/kvm --rm -v $(pwd):/srv/code:z registry.fedoraproject.org/fedora:27 /bin/sh -c "cd /srv/code && ./ci/fah27-insttests.sh"
 
 artifacts:
-  - tests/fedora-str/artifacts/fedora-atomic-host.qcow2.log
-  - tests/fedora-str/artifacts/installed-tests.log
+  - tests/installed/artifacts/
 
 ---
 
 # This suite skips the RPMs and does the build+unit tests in a container
-inherit: true
+inherit: false
+container:
+  image: registry.fedoraproject.org/fedora:27
 context: f27-primary
 env:
   # We only use -Werror=maybe-uninitialized here with a "fixed" toolchain

ci/fah27-insttests.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/bash
+set -xeuo pipefail
+
+./tests/installed/provision.sh
+# TODO: enhance papr to have caching, a bit like https://docs.travis-ci.com/user/caching/
+cd tests/installed
+curl -Lo fedora-atomic-host.qcow2 https://getfedora.org/atomic_qcow2_latest
+exec env "TEST_SUBJECTS=$(pwd)/fedora-atomic-host.qcow2" ./run.sh

tests/installed/README.md

@@ -1,2 +1,15 @@
-This suite of tests is currently run from redhat-ci;
-they're intended to run as root.
+This directory holds tests that use the
+[Fedora Standard Test Interface](https://fedoraproject.org/wiki/CI/Standard_Test_Interface).
+
+The high level structure is that we take a qcow2 file, inject
+built RPMs into it, and then use Ansible to run tests.
+
+See `.papr.yml` for canonical usage.
+
+For local development, you should cache the qcow2 somewhere
+stable (outside of this git repo).  Also note that `../ci/build-rpms.sh`
+does *not* pick up uncommitted changes!  Stated more strongly, you
+currently need to run `build-rpms.sh` after every change.
+
+To run just a specific test, use e.g.:
+`env TEST_SUBJECTS=/path/to/qcow2 ./playbook-run.sh -e tests=.*pull nondestructive.yml`

tests/installed/destructive.yml

@@ -0,0 +1,35 @@
+# This entrypoint right now just runs the sysinstalled-tests.
+---
+- hosts: localhost
+  tags:
+  - atomic
+  remote_user: root
+  vars:
+    use_git_build: True
+    tests: "."
+  tasks:
+    - import_tasks: overlay-git.yml
+      when: use_git_build
+    # Next copy all of the tests/ directory
+    - name: Copy test data
+      synchronize: src=../../ dest=/root/tests/ archive=yes
+    - find:
+        paths: /root/tests/installed/destructive
+        patterns: "itest-*.sh"
+      register: all_tests
+    - set_fact:
+        selected_tests: "{{ all_tests.files|map(attribute='path') | select('match', tests) | list }}"
+    - assert:
+        that:
+          - "{{ selected_tests|length }} != 0"
+    - file: path=/root/logs state=directory
+    - block:
+        - name: Run destructive tests
+          shell: "{{ item }} &> /root/logs/$(basename {{ item }}).log"
+          with_items:
+            - "{{ selected_tests }}"
+      always:
+        - synchronize:
+            src: /root/logs/
+            dest: artifacts/installed-destructive
+            mode: pull

tests/installed/destructive/README.md

@@ -0,0 +1,5 @@
+This suite of tests is run from PAPR. Everything in here is destructive; it's
+recommended to only run them in disposable virtual machines.  This is done
+in `tests/fedora-str/sysinstalled-tests.yml`, which currently uses a single VM
+and runs the tests serially.  It's likely in the future this will be changed
+to do one VM per test.

tests/installed/itest-bare-root.sh → tests/installed/destructive/itest-bare-root.sh

@@ -5,9 +5,10 @@
 set -xeuo pipefail
 
 dn=$(dirname $0)
-. ${dn}/libinsttest.sh
+. ${dn}/../libinsttest.sh
 
 echo "1..2"
+date
 
 cd /ostree/repo/tmp
 rm co -rf
@@ -40,3 +41,4 @@ rm co -rf
 rm co-testref -rf
 
 echo "ok xattrs"
+date

tests/installed/itest-deploy-selinux.sh → tests/installed/destructive/itest-deploy-selinux.sh

@@ -5,8 +5,9 @@
 set -xeuo pipefail
 
 dn=$(dirname $0)
-. ${dn}/libinsttest.sh
+. ${dn}/../libinsttest.sh
 
+date
 # Create a new deployment
 ostree admin deploy --karg-proc-cmdline ${host_refspec}
 new_deployment_path=/ostree/deploy/${host_osname}/deploy/${host_commit}.1
@@ -54,3 +55,4 @@ assert_file_has_content_literal bootlsz.txt 'system_u:object_r:boot_t:s0 initram
 
 ostree admin undeploy 0
 ostree refs --delete test-label
+date

tests/installed/itest-label-selinux.sh → tests/installed/destructive/itest-label-selinux.sh

@@ -5,8 +5,9 @@
 set -xeuo pipefail
 
 dn=$(dirname $0)
-. ${dn}/libinsttest.sh
+. ${dn}/../libinsttest.sh
 
+date
 cd /ostree/repo/tmp
 rm co -rf
 ostree checkout -H ${host_refspec} co
@@ -85,3 +86,4 @@ assert_streq "${oldcon}" "${newcon}"
 rm co -rf
 ostree refs --delete testbranch
 echo "ok checkout selinux and skip-list"
+date

tests/installed/execute_batch.yml

@@ -0,0 +1,22 @@
+#####################
+# execute_batch.yml
+#####################
+- name: Begin async command execution
+  shell: "{{ async_item }} &> {{ logdir }}/{{ async_item|basename }}.log"
+  # 10 minutes; the PAPR tester generally times out before that
+  async: 600
+  poll: 0
+  with_items: "{{ async_commands }}"
+  loop_control:
+    loop_var: "async_item"
+  register: async_results
+
+- name: Check async command status
+  async_status:
+    jid: "{{ async_result_item.ansible_job_id }}"
+  with_items: "{{ async_results.results }}"
+  loop_control:
+    loop_var: "async_result_item"
+  register: async_poll_results
+  until: async_poll_results.finished
+  retries: 240

tests/installed/libinsttest.sh

@@ -20,7 +20,7 @@
 # Boston, MA 02111-1307, USA.
 
 dn=$(dirname $0)
-. ${dn}/libtest-core.sh
+. ${dn}/../libtest-core.sh
 
 # Copy of bits from tap-test
 test_tmpdir=

tests/installed/nondestructive.yml

@@ -0,0 +1,40 @@
+# Nondestructive sysinstalled tests, run in parallel.
+---
+- hosts: localhost
+  tags:
+  - atomic
+  remote_user: root
+  vars:
+    use_git_build: True
+    tests: "."
+    # Arbitrary...we want some parallelism
+    batching_factor: 4
+  tasks:
+    - import_tasks: overlay-git.yml
+      when: use_git_build
+    # Next copy all of the tests/ directory
+    - name: Copy test data
+      synchronize: src=../../ dest=/root/tests/ archive=yes
+    - find:
+        paths: /root/tests/installed/nondestructive
+        patterns: "itest-*.sh"
+      register: all_tests
+    - set_fact:
+        selected_tests: "{{ all_tests.files|map(attribute='path') | select('match', tests) | list }}"
+    - assert:
+        that:
+          - "{{ selected_tests|length }} != 0"
+    - file: path=/root/logs state=directory
+    - block:
+        - name: Run nondestructive tests
+          vars:
+            logdir: /root/logs
+          async_commands: "{{ item }}"
+          include_tasks: execute_batch.yml
+          with_items:
+            - "{{ selected_tests | batch('{{ batching_factor }}') | list }}"
+      always:
+        - synchronize:
+            src: /root/logs
+            dest: artifacts/installed-nondestructive
+            mode: pull

tests/installed/itest-bare-unit.sh → tests/installed/nondestructive/itest-bare-unit.sh

@@ -6,16 +6,18 @@
 set -xeuo pipefail
 
 dn=$(dirname $0)
-. ${dn}/libinsttest.sh
+. ${dn}/../libinsttest.sh
 
+date
 # These tests sort of bypass the installed-tests spec;
 # fixing that would require installing g-d-t-r, though
 # more ideally we architect things with a "control" container
 # distinct from the host.
-export G_TEST_SRCDIR=$(realpath $dn/../..)
+export G_TEST_SRCDIR=$(realpath $dn/../../..)
 
 # Use /var/tmp to hopefully use XFS + O_TMPFILE etc.
 prepare_tmpdir /var/tmp
 trap _tmpdir_cleanup EXIT
 /usr/libexec/installed-tests/libostree/test-basic.sh
 /usr/libexec/installed-tests/libostree/test-basic-c
+date

tests/installed/itest-bare-user-root.sh → tests/installed/nondestructive/itest-bare-user-root.sh

@@ -5,9 +5,10 @@
 set -xeuo pipefail
 
 dn=$(dirname $0)
-. ${dn}/libinsttest.sh
+. ${dn}/../libinsttest.sh
 
 echo "1..1"
+date
 
 prepare_tmpdir
 ostree --repo=repo init --mode=bare-user
@@ -37,3 +38,4 @@ ostree --repo=repo ls -X rootfs /usr/lib/dbus-daemon-helper >ls.txt
 assert_file_has_content ls.txt '^-007.. 0 81 .*security.selinux.*/usr/lib/dbus-daemon-helper'
 assert_not_file_has_content ls.txt 'user.ostreemeta'
 echo "ok bare-user link-checkout-speedup with modified xattrs maintains uids"
+date

tests/installed/itest-bareuser-nouserxattrs.sh → tests/installed/nondestructive/itest-bareuser-nouserxattrs.sh

@@ -8,10 +8,11 @@
 set -xeuo pipefail
 
 dn=$(dirname $0)
-. ${dn}/libinsttest.sh
+. ${dn}/../libinsttest.sh
 
 prepare_tmpdir
 trap _tmpdir_cleanup EXIT
+date
 
 mkdir mnt
 mount -t tmpfs tmpfs mnt
@@ -21,3 +22,4 @@ if ostree --repo=mnt/repo init --mode=bare-user 2>err.txt; then
 fi
 umount mnt
 assert_file_has_content err.txt "Operation not supported"
+date

tests/installed/itest-payload-link.sh → tests/installed/nondestructive/itest-payload-link.sh

@@ -22,9 +22,10 @@
 set -xeuo pipefail
 
 dn=$(dirname $0)
-. ${dn}/libinsttest.sh
+. ${dn}/../libinsttest.sh
 
 echo "1..1"
+date
 
 # Use /var/tmp so we have O_TMPFILE etc.
 prepare_tmpdir /var/tmp
@@ -86,3 +87,4 @@ cat payload-links.txt | while read i; do
 done
 set -x
 echo "ok pull creates .payload-link"
+date

tests/installed/itest-pull-space.sh → tests/installed/nondestructive/itest-pull-space.sh

@@ -4,21 +4,24 @@
 set -xeuo pipefail
 
 dn=$(dirname $0)
-. ${dn}/libinsttest.sh
+. ${dn}/../libinsttest.sh
+date
 
 prepare_tmpdir
 trap _tmpdir_cleanup EXIT
 
 cd ${test_tmpdir}
-truncate -s 100MB testblk.img
+truncate -s 20MB testblk.img
 blkdev=$(losetup --find --show $(pwd)/testblk.img)
 mkfs.xfs ${blkdev}
 mkdir mnt
 mount ${blkdev} mnt
 ostree --repo=mnt/repo init --mode=bare-user
+echo 'fsync=false' >> mnt/repo/config
 if ostree --repo=mnt/repo pull-local /ostree/repo ${host_commit} 2>err.txt; then
     fatal "succeeded in doing a pull with no free space"
 fi
 assert_file_has_content err.txt "min-free-space-percent"
 umount mnt
 losetup -d ${blkdev}
+date