Native zuulv3 tests

This finalises the ports of the legacy jobs to zuul native jobs.

The dib-setup-gate-mirrors role preconfigures the repo templates,
etc. for the openstack-ci-mirrors element.

The dib-functests role runs the tests as specified by dib_functests,
and can run under python2 or 3.

Change-Id: Ied67a31f0d31503d13eccad8662c29740c93f33e

.zuul.d/jobs.yaml

@@ -3,130 +3,226 @@
       - tripleo-buildimage-jobs
     check:
       jobs:
-        - dib-dsvm-functests-python2-ubuntu-trusty:
+        - dib-functests-centos7-python2
-            branches: (master|feature/v2)
+        - dib-functests-centos7-python2-image
-            irrelevant-files:
+        - dib-functests-centos7-python2-extras
-              - ^.*\.rst$
+        - dib-functests-trusty-python2
-              - ^doc/.*$
+        - dib-functests-trusty-python2-image
-        - dib-dsvm-functests-python2-ubuntu-trusty-extras:
+        - dib-functests-trusty-python2-extras
-            voting: false
+        - dib-functests-xenial-python3
-            branches: (master|feature/v2)
+        - dib-functests-xenial-python3-image
-            irrelevant-files:
+        - dib-functests-xenial-python3-extras
-              - ^.*\.rst$
+        - dib-functests-bionic-python3
-              - ^doc/.*$
+        - dib-functests-bionic-python3-image
-        - dib-dsvm-functests-python2-ubuntu-trusty-image:
+        - dib-functests-bionic-python3-extras
-            voting: false
+        - tripleo-buildimage-overcloud-full-centos-7
-            branches: (master|feature/v2)
-            irrelevant-files:
-              - ^.*\.rst$
-              - ^doc/.*$
-        - dib-dsvm-functests-python3:
-            irrelevant-files:
-              - ^.*\.rst$
-              - ^doc/.*$
-        - dib-dsvm-functests-python3-extras:
-            voting: false
-            irrelevant-files:
-              - ^.*\.rst$
-              - ^doc/.*$
-        - dib-dsvm-functests-python3-image:
-            voting: false
-            irrelevant-files:
-              - ^.*\.rst$
-              - ^doc/.*$
-        - dib-dsvm-functests-python2-centos-7:
-            irrelevant-files:
-              - ^.*\.rst$
-              - ^doc/.*$
-        - dib-dsvm-functests-python2-centos-7-extras:
-            voting: false
-            irrelevant-files:
-              - ^.*\.rst$
-              - ^doc/.*$
-        - dib-dsvm-functests-python2-centos-7-image:
-            voting: false
-            irrelevant-files:
-              - ^.*\.rst$
-              - ^doc/.*$
         - openstack-tox-lower-constraints
 
     gate:
       jobs:
-        - dib-dsvm-functests-python2-ubuntu-trusty:
+        - dib-functests-trusty-python2
-            branches: (master|feature/v2)
+        - dib-functests-xenial-python3
-            irrelevant-files:
+        - dib-functests-centos7-python2
-              - ^.*\.rst$
-              - ^doc/.*$
-        - dib-dsvm-functests-python3:
-            irrelevant-files:
-              - ^.*\.rst$
-              - ^doc/.*$
-        - dib-dsvm-functests-python2-centos-7:
-            irrelevant-files:
-              - ^.*\.rst$
-              - ^doc/.*$
         - tripleo-buildimage-overcloud-full-centos-7
         - openstack-tox-lower-constraints
 
 - job:
-    name: dib-dsvm-functests-python2-centos-7
+    name: dib-functests-base
-    parent: legacy-dsvm-base
+    abstract: true
-    run: playbooks/legacy/dib-dsvm-functests-python2-centos-7/run.yaml
+    parent: base
-    post-run: playbooks/legacy/dib-dsvm-functests-python2-centos-7/post.yaml
+    run: playbooks/dib-functests/run.yaml
+    post-run: playbooks/dib-functests/post.yaml
     timeout: 3600
-    nodeset: legacy-centos-7
     required-projects:
       - openstack/diskimage-builder
+      - openstack/requirements
+    irrelevant-files:
+      - ^.*\.rst$
+      - ^doc/.*$
 
 - job:
-    name: dib-dsvm-functests-python2-centos-7-extras
+    name: dib-functests-default
-    parent: legacy-dsvm-base
+    description: |
-    run: playbooks/legacy/dib-dsvm-functests-python2-centos-7-extras/run.yaml
+      This is the base set of stable tests
-    post-run: playbooks/legacy/dib-dsvm-functests-python2-centos-7-extras/post.yaml
+    abstract: true
-    timeout: 3600
+    parent: dib-functests-base
-    nodeset: legacy-centos-7
+    vars:
-    required-projects:
+      dib_functests:
-      - openstack/diskimage-builder
+        - apt-sources/test-sources
+        - centos-minimal/build-succeeds
+        - fedora-minimal/build-succeeds
+        - ironic-agent/build-succeeds-fedora
+        - pip-and-virtualenv/source-install-fedora
+        - pip-and-virtualenv/source-install-ubuntu
+        - ubuntu-minimal/trusty-build-succeeds
+        - ubuntu-minimal/xenial-build-succeeds
+        - ubuntu-minimal/bionic-build-succeeds
 
 - job:
-    name: dib-dsvm-functests-python2-centos-7-image
+    name: dib-functests-image
-    parent: legacy-dsvm-base
+    description: |
-    run: playbooks/legacy/dib-dsvm-functests-python2-centos-7-image/run.yaml
+      This is tests that rely on downloading an upstream
-    post-run: playbooks/legacy/dib-dsvm-functests-python2-centos-7-image/post.yaml
+      image.  We generally do not have mirrors for these images
-    timeout: 3600
+      so they can be slighlty unreliable.
-    nodeset: legacy-centos-7
+    abstract: true
-    required-projects:
+    voting: false
-      - openstack/diskimage-builder
+    parent: dib-functests-base
+    vars:
+      dib_functests:
+        - centos7/build-succeeds
+        - opensuse/build-succeeds
+        - opensuse/opensuse423-build-succeeds
+        - fedora/build-succeeds
+        - ubuntu/trusty-build-succeeds
+        - ubuntu/xenial-build-succeeds
+        - ubuntu/bionic-build-succeeds
 
 - job:
-    name: dib-dsvm-functests-python2-ubuntu-trusty
+    name: dib-functests-extras
-    parent: legacy-dsvm-base
+    description: |
-    run: playbooks/legacy/dib-dsvm-functests-python2-ubuntu-trusty/run.yaml
+      These are tests which are not as widely supported as the
-    post-run: playbooks/legacy/dib-dsvm-functests-python2-ubuntu-trusty/post.yaml
+      core platforms
-    timeout: 3600
+    abstract: true
-    nodeset: legacy-ubuntu-trusty
+    voting: false
-    required-projects:
+    parent: dib-functests-base
-      - openstack/diskimage-builder
+    vars:
+      dib_functests:
+        - gentoo/build-succeeds
+        - debian-minimal/stable-vm
+
+# Bionic / Python3 tests
+- job:
+    name: dib-functests-trusty-python2
+    description: |
+      Run the default tests on a Trusty build host with Python 2
+    parent: dib-functests-default
+    nodeset: ubuntu-trusty
+    vars:
+      dib_python: python2
 
 - job:
-    name: dib-dsvm-functests-python2-ubuntu-trusty-extras
+    name: dib-functests-trusty-python2-image
-    parent: legacy-dsvm-base
+    description: |
-    run: playbooks/legacy/dib-dsvm-functests-python2-ubuntu-trusty-extras/run.yaml
+      The image tests on a Trusty build host with Python 2
-    post-run: playbooks/legacy/dib-dsvm-functests-python2-ubuntu-trusty-extras/post.yaml
+    parent: dib-functests-image
-    timeout: 3600
+    nodeset: ubuntu-trusty
-    nodeset: legacy-ubuntu-trusty
+    vars:
-    required-projects:
+      dib_python: python2
-      - openstack/diskimage-builder
 
 - job:
-    name: dib-dsvm-functests-python2-ubuntu-trusty-image
+    name: dib-functests-trusty-python2-extras
-    parent: legacy-dsvm-base
+    description: |
-    run: playbooks/legacy/dib-dsvm-functests-python2-ubuntu-trusty-image/run.yaml
+      The extras tests on a Trusty build host with Python 2
-    post-run: playbooks/legacy/dib-dsvm-functests-python2-ubuntu-trusty-image/post.yaml
+    parent: dib-functests-extras
-    timeout: 3600
+    nodeset: ubuntu-trusty
-    nodeset: legacy-ubuntu-trusty
+    vars:
-    required-projects:
+      dib_python: python2
-      - openstack/diskimage-builder
+
+# Bionic / Python3 tests
+- job:
+    name: dib-functests-bionic-python3
+    description: |
+      Run the default tests on a Xenial build host with Python 3
+    parent: dib-functests-default
+    nodeset: ubuntu-bionic
+    voting: false
+
+- job:
+    name: dib-functests-bionic-python3-image
+    description: |
+      The image tests on a Xenial build host with Python 3
+    parent: dib-functests-image
+    nodeset: ubuntu-bionic
+    voting: false
+
+- job:
+    name: dib-functests-bionic-python3-extras
+    description: |
+      The extras tests on a Xenial build host with Python 3
+    parent: dib-functests-extras
+    nodeset: ubuntu-bionic
+    voting: false
+
+# Xenial / Python3 tests
+- job:
+    name: dib-functests-xenial-python3
+    description: |
+      Run the default tests on a Xenial build host with Python 3
+    parent: dib-functests-default
+    nodeset: ubuntu-xenial
+
+- job:
+    name: dib-functests-xenial-python3-image
+    description: |
+      The image tests on a Xenial build host with Python 3
+    parent: dib-functests-image
+    nodeset: ubuntu-xenial
+
+- job:
+    name: dib-functests-xenial-python3-extras
+    description: |
+      The extras tests on a Xenial build host with Python 3
+    parent: dib-functests-extras
+    nodeset: ubuntu-xenial
+    vars:
+      # opensuse-minimal can only build on xenial ATM, as other
+      # platforms don't have zypper
+      dib_functests_extra:
+        - opensuse-minimal/opensuse423-build-succeeds
+
+# Bionic / Python3 tests
+- job:
+    name: dib-functests-bionic-python3
+    description: |
+      Run the default tests on a Xenial build host with Python 3
+    parent: dib-functests-default
+    nodeset: ubuntu-bionic
+    voting: false
+
+- job:
+    name: dib-functests-bionic-python3-image
+    description: |
+      The image tests on a Xenial build host with Python 3
+    parent: dib-functests-image
+    nodeset: ubuntu-bionic
+    voting: false
+
+- job:
+    name: dib-functests-bionic-python3-extras
+    description: |
+      The extras tests on a Xenial build host with Python 3
+    parent: dib-functests-extras
+    nodeset: ubuntu-bionic
+    voting: false
+
+# Centos 7 / Python2 tests
+- job:
+    name: dib-functests-centos7-python2
+    description: |
+      Run the default tests on a Centos 7 build host with Python 2
+    parent: dib-functests-default
+    nodeset: centos-7
+    vars:
+      dib_python: python2
+
+- job:
+    name: dib-functests-centos7-python2-image
+    description: |
+      The image tests on a Centos 7 build host with Python 2
+    parent: dib-functests-image
+    nodeset: centos-7
+    vars:
+      dib_python: python2
+
+- job:
+    name: dib-functests-centos7-python2-extras
+    description: |
+      The extras tests on a Centos 7 build host with Python 2
+    parent: dib-functests-extras
+    nodeset: centos-7
+    vars:
+      dib_python: python2
+
 
 - job:
     name: dib-dsvm-functests-python3

contrib/setup-gate-mirrors.sh

@@ -12,7 +12,7 @@ source /etc/ci/mirror_info.sh
 
 # Tests should probe for this directory and then use the repos/sources
 # files inside it for the gate tests.
-BASE_DIR=$WORKSPACE/dib-mirror
+BASE_DIR=${WORKSPACE:-$(pwd)}/dib-mirror
 
 mkdir -p $BASE_DIR
 

diskimage_builder/elements/openstack-ci-mirrors/environment.d/11-dib-distribution-mirror.bash

@@ -44,12 +44,16 @@ export DIB_DISTRIBUTION_MIRROR_UBUNTU_INSECURE=1
 
 # These repo files are pre-created for the fedora/centos-minimal jobs
 # in the gate.  Not relevant inside the chroot.
-if [[ -d ${WORKSPACE:-/not/a/path/}/dib-mirror ]]; then
+if [[ -d ${DIB_OS_CI_YUM_REPOS:-/not/a/path/} ]]; then
 
     if [[ "${DISTRO_NAME}" == "fedora" ]]; then
-        export DIB_YUM_MINIMAL_BOOTSTRAP_REPOS=${WORKSPACE}/dib-mirror/fedora-minimal/yum.repos.d
+        if [[ -d ${DIB_OS_CI_YUM_REPOS}/fedora-minimal/${DIB_RELEASE} ]]; then
+            export DIB_YUM_MINIMAL_BOOTSTRAP_REPOS=${DIB_OS_CI_YUM_REPOS}/fedora-minimal/${DIB_RELEASE}/yum.repos.d
+        else
+            export DIB_YUM_MINIMAL_BOOTSTRAP_REPOS=${DIB_OS_CI_YUM_REPOS}/fedora-minimal/default/yum.repos.d
+        fi
     elif [[ "${DISTRO_NAME}" == "centos" ]]; then
-        export DIB_YUM_MINIMAL_BOOTSTRAP_REPOS=${WORKSPACE}/dib-mirror/centos-minimal/yum.repos.d
+        export DIB_YUM_MINIMAL_BOOTSTRAP_REPOS=${DIB_OS_CI_YUM_REPOS}/centos-minimal/yum.repos.d
     fi
 
 fi

playbooks/dib-functests/post.yaml

@@ -0,0 +1,13 @@
+- hosts: all
+  tasks:
+
+    - name: Check logs directory exists
+      stat: path='{{ ansible_user_dir }}/logs/'
+      register: check_path
+
+    - name: Copy log files from {{ ansible_user_dir }} on node
+      synchronize:
+        src: '{{ ansible_user_dir }}/logs/'
+        dest: '{{ zuul.executor.log_root }}/logs'
+        mode: pull
+      when: check_path.stat.exists == true

playbooks/dib-functests/run.yaml

@@ -0,0 +1,7 @@
+- hosts: all
+  name: Run dib functional tests
+  roles:
+    - bindep
+    - dib-setup-gate-mirrors
+    - dib-functests
+

roles/dib-functests/README.rst

@@ -0,0 +1,18 @@
+dib-functests
+-------------
+
+Run diskimage-builder functional tests
+
+Installs dib and dependencies, and runs functional tests
+
+**Role Variables**
+
+.. zuul:rolevar:: python_version
+    :default: 2
+
+    The python version to run the test under
+
+.. zuul:rolevar:: dib_functests
+    :default: []
+
+    The list of functional tests to run

roles/dib-functests/defaults/main.yaml

@@ -0,0 +1,8 @@
+---
+dib_python: python3
+dib_virtualenv: "{{ ansible_user_dir }}/dib-venv"
+dib_functests: []
+# This is a bit of hack; it allows us to template out jobs where the
+# base tests are defined in dib_functests, but then add a few extra
+# tests to run in this variable for a specific test.
+dib_functests_extra: []

roles/dib-functests/tasks/main.yaml

@@ -0,0 +1,49 @@
+- name: Install dib extra dependencies
+  command: ./install_test_deps.sh
+  args:
+    chdir: "{{ zuul.project.src_dir }}/tests"
+  become: yes
+
+- name: Setup mounted space
+  file:
+    path: '/opt/dib_cache'
+    state: directory
+    mode: ugo+rw
+  become: yes
+
+- name: Define upper_constraints
+  set_fact:
+    upper_constraints: "{{ ansible_user_dir }}/{{ zuul.projects['git.openstack.org/openstack/requirements'].src_dir }}/upper-constraints.txt"
+
+- name: Take ourselves out of uppper constraints
+  lineinfile:
+    path: "{{ upper_constraints }}"
+    state: absent
+    regexp: '^diskimage-builder'
+
+- name: Install diskimage-builder
+  pip:
+    name: "file://{{ ansible_user_dir }}/{{ zuul.project.src_dir }}"
+    virtualenv: "{{ dib_virtualenv }}"
+    virtualenv_python: "{{ dib_python }}"
+    extra_args: "-c {{ upper_constraints }}"
+
+- name: Setup log output
+  file:
+    path: "{{ ansible_user_dir }}/logs"
+    state: directory
+    mode: ugo+rw
+
+- name: Run dib functional tests
+  shell:
+    cmd: |
+      source {{ dib_virtualenv }}/bin/activate
+      {{ zuul.project.src_dir }}/tests/run_functests.sh \
+        -L {{ ansible_user_dir }}/logs {{ dib_functests | join(' ') }} \
+        {{ dib_functests_extra | join(' ') }}
+  args:
+    executable: /bin/bash
+  environment:
+    DIB_NO_TMPFS: 1
+    TMPDIR: /opt/dib_cache
+    DIB_OS_CI_YUM_REPOS: "{{ dib_gate_mirror_repos|default(omit) }}"

roles/dib-setup-gate-mirrors/README.rst

@@ -0,0 +1,6 @@
+dib-setup-gate-mirrors
+======================
+
+Create customised repository files as required by the
+openstack-ci-mirrors element.  Path to repo files is set into
+``dib_gate_mirror_repos``

roles/dib-setup-gate-mirrors/defaults/main.yaml

@@ -0,0 +1 @@
+mirror_fqdn: "{{ zuul_site_mirror_fqdn|default(omit) }}"

roles/dib-setup-gate-mirrors/tasks/main.yaml

@@ -0,0 +1,42 @@
+- name: Set repo path
+  set_fact:
+    dib_gate_mirror_repos: "{{ ansible_user_dir }}/dib-mirror"
+
+- debug: msg="zuul site mirror {{ zuul_site_mirror_fqdn }}"
+- debug: msg="mirror_fqdn {{ mirror_fqdn }}"
+
+- name: Create centos-minimal directory
+  file:
+    path: "{{ dib_gate_mirror_repos }}/centos-minimal/yum.repos.d"
+    state: directory
+    mode: 0775
+    recurse: yes
+
+- name: Install centos-minimal repo files
+  template:
+    dest: "{{ dib_gate_mirror_repos }}/centos-minimal/yum.repos.d/dib-mirror-{{ item }}"
+    mode: 0644
+    src: "centos-minimal/{{ item }}.j2"
+  with_items:
+    - base.repo
+    - updates.repo
+    - extras.repo
+
+- name: Create fedora-minimal directories
+  file:
+    path: "{{ dib_gate_mirror_repos }}/fedora-minimal/{{ item }}/yum.repos.d"
+    state: directory
+    mode: 0775
+    recurse: yes
+  with_items:
+    - default
+    - '28'
+
+- name: Install fedora-minimal repo files
+  template:
+    dest: "{{ dib_gate_mirror_repos }}/fedora-minimal/{{ item[0] }}/yum.repos.d/dib-mirror-{{ item[1] }}"
+    mode: 0644
+    src: "fedora-minimal/{{ item[0] }}/{{ item[1] }}.j2"
+  with_nested:
+    - [ 'default', '28' ]
+    - [ 'fedora.repo', 'fedora-updates.repo' ]

roles/dib-setup-gate-mirrors/templates/centos-minimal/base.repo.j2

@@ -0,0 +1,4 @@
+[base]
+name=CentOS-$releasever - Base
+baseurl=http://{{ mirror_fqdn }}/centos/$releasever/os/$basearch/
+gpgcheck=0

roles/dib-setup-gate-mirrors/templates/centos-minimal/extras.repo.j2

@@ -0,0 +1,5 @@
+#additional packages that may be useful
+[extras]
+name=CentOS-$releasever - Extras
+baseurl=http://{{ mirror_fqdn }}/centos/$releasever/extras/$basearch/
+gpgcheck=0

roles/dib-setup-gate-mirrors/templates/centos-minimal/updates.repo.j2

@@ -0,0 +1,5 @@
+#released updates
+[updates]
+name=CentOS-$releasever - Updates
+baseurl=http://{{ mirror_fqdn }}/centos/$releasever/updates/$basearch/
+gpgcheck=0

roles/dib-setup-gate-mirrors/templates/fedora-minimal/28/fedora-updates.repo.j2

@@ -0,0 +1,10 @@
+[updates]
+name=Fedora $releasever - $basearch - Updates
+failovermethod=priority
+baseurl=http://{{ mirror_fqdn }}/fedora/updates/$releasever/Everything/$basearch/
+enabled=1
+gpgcheck=0
+metadata_expire=6h
+skip_if_unavailable=False
+deltarpm=False
+deltarpm_percentage=0

roles/dib-setup-gate-mirrors/templates/fedora-minimal/28/fedora.repo.j2

@@ -0,0 +1,10 @@
+[fedora]
+name=Fedora $releasever - $basearch
+failovermethod=priority
+baseurl=http://{{ mirror_fqdn }}/fedora/releases/$releasever/Everything/$basearch/os/
+enabled=1
+metadata_expire=7d
+gpgcheck=0
+skip_if_unavailable=False
+deltarpm=False
+deltarpm_percentage=0

roles/dib-setup-gate-mirrors/templates/fedora-minimal/default/fedora-updates.repo.j2

@@ -0,0 +1,10 @@
+[updates]
+name=Fedora $releasever - $basearch - Updates
+failovermethod=priority
+baseurl=http://{{ mirror_fqdn }}/fedora/updates/$releasever/$basearch/
+enabled=1
+gpgcheck=0
+metadata_expire=6h
+skip_if_unavailable=False
+deltarpm=False
+deltarpm_percentage=0

roles/dib-setup-gate-mirrors/templates/fedora-minimal/default/fedora.repo.j2

@@ -0,0 +1,10 @@
+[fedora]
+name=Fedora $releasever - $basearch
+failovermethod=priority
+baseurl=http://{{ mirror_fqdn }}/fedora/releases/$releasever/Everything/$basearch/os/
+enabled=1
+metadata_expire=7d
+gpgcheck=0
+skip_if_unavailable=False
+deltarpm=False
+deltarpm_percentage=0