sig_cloud/diskimage-builder
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Native zuulv3 tests

Browse Source 
This finalises the ports of the legacy jobs to zuul native jobs.

The dib-setup-gate-mirrors role preconfigures the repo templates,
etc. for the openstack-ci-mirrors element.

The dib-functests role runs the tests as specified by dib_functests,
and can run under python2 or 3.

Change-Id: Ied67a31f0d31503d13eccad8662c29740c93f33e
This commit is contained in:
Ian Wienand 2018-05-23 12:11:13 +10:00
parent c48a40d479
commit 36d642a6a0
18 changed files with 405 additions and 107 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

302
.zuul.d/jobs.yaml
View File

@ -3,130 +3,226 @@
- tripleo-buildimage-jobs
check:
jobs:
- dib-dsvm-functests-python2-ubuntu-trusty:
branches: (master|feature/v2)
irrelevant-files:
- ^.*\.rst$
- ^doc/.*$
- dib-dsvm-functests-python2-ubuntu-trusty-extras:
voting: false
branches: (master|feature/v2)
irrelevant-files:
- ^.*\.rst$
- ^doc/.*$
- dib-dsvm-functests-python2-ubuntu-trusty-image:
voting: false
branches: (master|feature/v2)
irrelevant-files:
- ^.*\.rst$
- ^doc/.*$
- dib-dsvm-functests-python3:
irrelevant-files:
- ^.*\.rst$
- ^doc/.*$
- dib-dsvm-functests-python3-extras:
voting: false
irrelevant-files:
- ^.*\.rst$
- ^doc/.*$
- dib-dsvm-functests-python3-image:
voting: false
irrelevant-files:
- ^.*\.rst$
- ^doc/.*$
- dib-dsvm-functests-python2-centos-7:
irrelevant-files:
- ^.*\.rst$
- ^doc/.*$
- dib-dsvm-functests-python2-centos-7-extras:
voting: false
irrelevant-files:
- ^.*\.rst$
- ^doc/.*$
- dib-dsvm-functests-python2-centos-7-image:
voting: false
irrelevant-files:
- ^.*\.rst$
- ^doc/.*$
- dib-functests-centos7-python2
- dib-functests-centos7-python2-image
- dib-functests-centos7-python2-extras
- dib-functests-trusty-python2
- dib-functests-trusty-python2-image
- dib-functests-trusty-python2-extras
- dib-functests-xenial-python3
- dib-functests-xenial-python3-image
- dib-functests-xenial-python3-extras
- dib-functests-bionic-python3
- dib-functests-bionic-python3-image
- dib-functests-bionic-python3-extras
- tripleo-buildimage-overcloud-full-centos-7
- openstack-tox-lower-constraints
gate:
jobs:
- dib-dsvm-functests-python2-ubuntu-trusty:
branches: (master|feature/v2)
irrelevant-files:
- ^.*\.rst$
- ^doc/.*$
- dib-dsvm-functests-python3:
irrelevant-files:
- ^.*\.rst$
- ^doc/.*$
- dib-dsvm-functests-python2-centos-7:
irrelevant-files:
- ^.*\.rst$
- ^doc/.*$
- dib-functests-trusty-python2
- dib-functests-xenial-python3
- dib-functests-centos7-python2
- tripleo-buildimage-overcloud-full-centos-7
- openstack-tox-lower-constraints
- job:
name: dib-dsvm-functests-python2-centos-7
parent: legacy-dsvm-base
run: playbooks/legacy/dib-dsvm-functests-python2-centos-7/run.yaml
post-run: playbooks/legacy/dib-dsvm-functests-python2-centos-7/post.yaml
name: dib-functests-base
abstract: true
parent: base
run: playbooks/dib-functests/run.yaml
post-run: playbooks/dib-functests/post.yaml
timeout: 3600
nodeset: legacy-centos-7
required-projects:
- openstack/diskimage-builder
- openstack/requirements
irrelevant-files:
- ^.*\.rst$
- ^doc/.*$
- job:
name: dib-dsvm-functests-python2-centos-7-extras
parent: legacy-dsvm-base
run: playbooks/legacy/dib-dsvm-functests-python2-centos-7-extras/run.yaml
post-run: playbooks/legacy/dib-dsvm-functests-python2-centos-7-extras/post.yaml
timeout: 3600
nodeset: legacy-centos-7
required-projects:
- openstack/diskimage-builder
name: dib-functests-default
description: |
This is the base set of stable tests
abstract: true
parent: dib-functests-base
vars:
dib_functests:
- apt-sources/test-sources
- centos-minimal/build-succeeds
- fedora-minimal/build-succeeds
- ironic-agent/build-succeeds-fedora
- pip-and-virtualenv/source-install-fedora
- pip-and-virtualenv/source-install-ubuntu
- ubuntu-minimal/trusty-build-succeeds
- ubuntu-minimal/xenial-build-succeeds
- ubuntu-minimal/bionic-build-succeeds
- job:
name: dib-dsvm-functests-python2-centos-7-image
parent: legacy-dsvm-base
run: playbooks/legacy/dib-dsvm-functests-python2-centos-7-image/run.yaml
post-run: playbooks/legacy/dib-dsvm-functests-python2-centos-7-image/post.yaml
timeout: 3600
nodeset: legacy-centos-7
required-projects:
- openstack/diskimage-builder
name: dib-functests-image
description: |
This is tests that rely on downloading an upstream
image. We generally do not have mirrors for these images
so they can be slighlty unreliable.
abstract: true
voting: false
parent: dib-functests-base
vars:
dib_functests:
- centos7/build-succeeds
- opensuse/build-succeeds
- opensuse/opensuse423-build-succeeds
- fedora/build-succeeds
- ubuntu/trusty-build-succeeds
- ubuntu/xenial-build-succeeds
- ubuntu/bionic-build-succeeds
- job:
name: dib-dsvm-functests-python2-ubuntu-trusty
parent: legacy-dsvm-base
run: playbooks/legacy/dib-dsvm-functests-python2-ubuntu-trusty/run.yaml
post-run: playbooks/legacy/dib-dsvm-functests-python2-ubuntu-trusty/post.yaml
timeout: 3600
nodeset: legacy-ubuntu-trusty
required-projects:
- openstack/diskimage-builder
name: dib-functests-extras
description: |
These are tests which are not as widely supported as the
core platforms
abstract: true
voting: false
parent: dib-functests-base
vars:
dib_functests:
- gentoo/build-succeeds
- debian-minimal/stable-vm
# Bionic / Python3 tests
- job:
name: dib-functests-trusty-python2
description: |
Run the default tests on a Trusty build host with Python 2
parent: dib-functests-default
nodeset: ubuntu-trusty
vars:
dib_python: python2
- job:
name: dib-dsvm-functests-python2-ubuntu-trusty-extras
parent: legacy-dsvm-base
run: playbooks/legacy/dib-dsvm-functests-python2-ubuntu-trusty-extras/run.yaml
post-run: playbooks/legacy/dib-dsvm-functests-python2-ubuntu-trusty-extras/post.yaml
timeout: 3600
nodeset: legacy-ubuntu-trusty
required-projects:
- openstack/diskimage-builder
name: dib-functests-trusty-python2-image
description: |
The image tests on a Trusty build host with Python 2
parent: dib-functests-image
nodeset: ubuntu-trusty
vars:
dib_python: python2
- job:
name: dib-dsvm-functests-python2-ubuntu-trusty-image
parent: legacy-dsvm-base
run: playbooks/legacy/dib-dsvm-functests-python2-ubuntu-trusty-image/run.yaml
post-run: playbooks/legacy/dib-dsvm-functests-python2-ubuntu-trusty-image/post.yaml
timeout: 3600
nodeset: legacy-ubuntu-trusty
required-projects:
- openstack/diskimage-builder
name: dib-functests-trusty-python2-extras
description: |
The extras tests on a Trusty build host with Python 2
parent: dib-functests-extras
nodeset: ubuntu-trusty
vars:
dib_python: python2
# Bionic / Python3 tests
- job:
name: dib-functests-bionic-python3
description: |
Run the default tests on a Xenial build host with Python 3
parent: dib-functests-default
nodeset: ubuntu-bionic
voting: false
- job:
name: dib-functests-bionic-python3-image
description: |
The image tests on a Xenial build host with Python 3
parent: dib-functests-image
nodeset: ubuntu-bionic
voting: false
- job:
name: dib-functests-bionic-python3-extras
description: |
The extras tests on a Xenial build host with Python 3
parent: dib-functests-extras
nodeset: ubuntu-bionic
voting: false
# Xenial / Python3 tests
- job:
name: dib-functests-xenial-python3
description: |
Run the default tests on a Xenial build host with Python 3
parent: dib-functests-default
nodeset: ubuntu-xenial
- job:
name: dib-functests-xenial-python3-image
description: |
The image tests on a Xenial build host with Python 3
parent: dib-functests-image
nodeset: ubuntu-xenial
- job:
name: dib-functests-xenial-python3-extras
description: |
The extras tests on a Xenial build host with Python 3
parent: dib-functests-extras
nodeset: ubuntu-xenial
vars:
# opensuse-minimal can only build on xenial ATM, as other
# platforms don't have zypper
dib_functests_extra:
- opensuse-minimal/opensuse423-build-succeeds
# Bionic / Python3 tests
- job:
name: dib-functests-bionic-python3
description: |
Run the default tests on a Xenial build host with Python 3
parent: dib-functests-default
nodeset: ubuntu-bionic
voting: false
- job:
name: dib-functests-bionic-python3-image
description: |
The image tests on a Xenial build host with Python 3
parent: dib-functests-image
nodeset: ubuntu-bionic
voting: false
- job:
name: dib-functests-bionic-python3-extras
description: |
The extras tests on a Xenial build host with Python 3
parent: dib-functests-extras
nodeset: ubuntu-bionic
voting: false
# Centos 7 / Python2 tests
- job:
name: dib-functests-centos7-python2
description: |
Run the default tests on a Centos 7 build host with Python 2
parent: dib-functests-default
nodeset: centos-7
vars:
dib_python: python2
- job:
name: dib-functests-centos7-python2-image
description: |
The image tests on a Centos 7 build host with Python 2
parent: dib-functests-image
nodeset: centos-7
vars:
dib_python: python2
- job:
name: dib-functests-centos7-python2-extras
description: |
The extras tests on a Centos 7 build host with Python 2
parent: dib-functests-extras
nodeset: centos-7
vars:
dib_python: python2
- job:
name: dib-dsvm-functests-python3

2
contrib/setup-gate-mirrors.sh
View File

@ -12,7 +12,7 @@ source /etc/ci/mirror_info.sh
# Tests should probe for this directory and then use the repos/sources
# files inside it for the gate tests.
BASE_DIR=$WORKSPACE/dib-mirror
BASE_DIR=${WORKSPACE:-$(pwd)}/dib-mirror
mkdir -p $BASE_DIR

10
diskimage_builder/elements/openstack-ci-mirrors/environment.d/11-dib-distribution-mirror.bash
View File

@ -44,12 +44,16 @@ export DIB_DISTRIBUTION_MIRROR_UBUNTU_INSECURE=1
# These repo files are pre-created for the fedora/centos-minimal jobs
# in the gate. Not relevant inside the chroot.
if [[ -d ${WORKSPACE:-/not/a/path/}/dib-mirror ]]; then
if [[ -d ${DIB_OS_CI_YUM_REPOS:-/not/a/path/} ]]; then
if [[ "${DISTRO_NAME}" == "fedora" ]]; then
export DIB_YUM_MINIMAL_BOOTSTRAP_REPOS=${WORKSPACE}/dib-mirror/fedora-minimal/yum.repos.d
if [[ -d ${DIB_OS_CI_YUM_REPOS}/fedora-minimal/${DIB_RELEASE} ]]; then
export DIB_YUM_MINIMAL_BOOTSTRAP_REPOS=${DIB_OS_CI_YUM_REPOS}/fedora-minimal/${DIB_RELEASE}/yum.repos.d
else
export DIB_YUM_MINIMAL_BOOTSTRAP_REPOS=${DIB_OS_CI_YUM_REPOS}/fedora-minimal/default/yum.repos.d
fi
elif [[ "${DISTRO_NAME}" == "centos" ]]; then
export DIB_YUM_MINIMAL_BOOTSTRAP_REPOS=${WORKSPACE}/dib-mirror/centos-minimal/yum.repos.d
export DIB_YUM_MINIMAL_BOOTSTRAP_REPOS=${DIB_OS_CI_YUM_REPOS}/centos-minimal/yum.repos.d
fi
fi

13
playbooks/dib-functests/post.yaml Normal file
View File

@ -0,0 +1,13 @@
- hosts: all
tasks:
- name: Check logs directory exists
stat: path='{{ ansible_user_dir }}/logs/'
register: check_path
- name: Copy log files from {{ ansible_user_dir }} on node
synchronize:
src: '{{ ansible_user_dir }}/logs/'
dest: '{{ zuul.executor.log_root }}/logs'
mode: pull
when: check_path.stat.exists == true

7
playbooks/dib-functests/run.yaml Normal file
View File

@ -0,0 +1,7 @@
- hosts: all
name: Run dib functional tests
roles:
- bindep
- dib-setup-gate-mirrors
- dib-functests

18
roles/dib-functests/README.rst Normal file
View File

@ -0,0 +1,18 @@
dib-functests
-------------
Run diskimage-builder functional tests
Installs dib and dependencies, and runs functional tests
**Role Variables**
.. zuul:rolevar:: python_version
:default: 2
The python version to run the test under
.. zuul:rolevar:: dib_functests
:default: []
The list of functional tests to run

8
roles/dib-functests/defaults/main.yaml Normal file
View File

@ -0,0 +1,8 @@
---
dib_python: python3
dib_virtualenv: "{{ ansible_user_dir }}/dib-venv"
dib_functests: []
# This is a bit of hack; it allows us to template out jobs where the
# base tests are defined in dib_functests, but then add a few extra
# tests to run in this variable for a specific test.
dib_functests_extra: []

49
roles/dib-functests/tasks/main.yaml Normal file
View File

@ -0,0 +1,49 @@
- name: Install dib extra dependencies
command: ./install_test_deps.sh
args:
chdir: "{{ zuul.project.src_dir }}/tests"
become: yes
- name: Setup mounted space
file:
path: '/opt/dib_cache'
state: directory
mode: ugo+rw
become: yes
- name: Define upper_constraints
set_fact:
upper_constraints: "{{ ansible_user_dir }}/{{ zuul.projects['git.openstack.org/openstack/requirements'].src_dir }}/upper-constraints.txt"
- name: Take ourselves out of uppper constraints
lineinfile:
path: "{{ upper_constraints }}"
state: absent
regexp: '^diskimage-builder'
- name: Install diskimage-builder
pip:
name: "file://{{ ansible_user_dir }}/{{ zuul.project.src_dir }}"
virtualenv: "{{ dib_virtualenv }}"
virtualenv_python: "{{ dib_python }}"
extra_args: "-c {{ upper_constraints }}"
- name: Setup log output
file:
path: "{{ ansible_user_dir }}/logs"
state: directory
mode: ugo+rw
- name: Run dib functional tests
shell:
cmd: |
source {{ dib_virtualenv }}/bin/activate
{{ zuul.project.src_dir }}/tests/run_functests.sh \
-L {{ ansible_user_dir }}/logs {{ dib_functests | join(' ') }} \
{{ dib_functests_extra | join(' ') }}
args:
executable: /bin/bash
environment:
DIB_NO_TMPFS: 1
TMPDIR: /opt/dib_cache
DIB_OS_CI_YUM_REPOS: "{{ dib_gate_mirror_repos|default(omit) }}"

6
roles/dib-setup-gate-mirrors/README.rst Normal file
View File

@ -0,0 +1,6 @@
dib-setup-gate-mirrors
======================
Create customised repository files as required by the
openstack-ci-mirrors element. Path to repo files is set into
``dib_gate_mirror_repos``

1
roles/dib-setup-gate-mirrors/defaults/main.yaml Normal file
View File

@ -0,0 +1 @@
mirror_fqdn: "{{ zuul_site_mirror_fqdn|default(omit) }}"

42
roles/dib-setup-gate-mirrors/tasks/main.yaml Normal file
View File

@ -0,0 +1,42 @@
- name: Set repo path
set_fact:
dib_gate_mirror_repos: "{{ ansible_user_dir }}/dib-mirror"
- debug: msg="zuul site mirror {{ zuul_site_mirror_fqdn }}"
- debug: msg="mirror_fqdn {{ mirror_fqdn }}"
- name: Create centos-minimal directory
file:
path: "{{ dib_gate_mirror_repos }}/centos-minimal/yum.repos.d"
state: directory
mode: 0775
recurse: yes
- name: Install centos-minimal repo files
template:
dest: "{{ dib_gate_mirror_repos }}/centos-minimal/yum.repos.d/dib-mirror-{{ item }}"
mode: 0644
src: "centos-minimal/{{ item }}.j2"
with_items:
- base.repo
- updates.repo
- extras.repo
- name: Create fedora-minimal directories
file:
path: "{{ dib_gate_mirror_repos }}/fedora-minimal/{{ item }}/yum.repos.d"
state: directory
mode: 0775
recurse: yes
with_items:
- default
- '28'
- name: Install fedora-minimal repo files
template:
dest: "{{ dib_gate_mirror_repos }}/fedora-minimal/{{ item[0] }}/yum.repos.d/dib-mirror-{{ item[1] }}"
mode: 0644
src: "fedora-minimal/{{ item[0] }}/{{ item[1] }}.j2"
with_nested:
- [ 'default', '28' ]
- [ 'fedora.repo', 'fedora-updates.repo' ]

4
roles/dib-setup-gate-mirrors/templates/centos-minimal/base.repo.j2 Normal file
View File

@ -0,0 +1,4 @@
[base]
name=CentOS-$releasever - Base
baseurl=http://{{ mirror_fqdn }}/centos/$releasever/os/$basearch/
gpgcheck=0

5
roles/dib-setup-gate-mirrors/templates/centos-minimal/extras.repo.j2 Normal file
View File

@ -0,0 +1,5 @@
#additional packages that may be useful
[extras]
name=CentOS-$releasever - Extras
baseurl=http://{{ mirror_fqdn }}/centos/$releasever/extras/$basearch/
gpgcheck=0

5
roles/dib-setup-gate-mirrors/templates/centos-minimal/updates.repo.j2 Normal file
View File

@ -0,0 +1,5 @@
#released updates
[updates]
name=CentOS-$releasever - Updates
baseurl=http://{{ mirror_fqdn }}/centos/$releasever/updates/$basearch/
gpgcheck=0

10
roles/dib-setup-gate-mirrors/templates/fedora-minimal/28/fedora-updates.repo.j2 Normal file
View File

@ -0,0 +1,10 @@
[updates]
name=Fedora $releasever - $basearch - Updates
failovermethod=priority
baseurl=http://{{ mirror_fqdn }}/fedora/updates/$releasever/Everything/$basearch/
enabled=1
gpgcheck=0
metadata_expire=6h
skip_if_unavailable=False
deltarpm=False
deltarpm_percentage=0

10
roles/dib-setup-gate-mirrors/templates/fedora-minimal/28/fedora.repo.j2 Normal file
View File

@ -0,0 +1,10 @@
[fedora]
name=Fedora $releasever - $basearch
failovermethod=priority
baseurl=http://{{ mirror_fqdn }}/fedora/releases/$releasever/Everything/$basearch/os/
enabled=1
metadata_expire=7d
gpgcheck=0
skip_if_unavailable=False
deltarpm=False
deltarpm_percentage=0

10
roles/dib-setup-gate-mirrors/templates/fedora-minimal/default/fedora-updates.repo.j2 Normal file
View File

@ -0,0 +1,10 @@
[updates]
name=Fedora $releasever - $basearch - Updates
failovermethod=priority
baseurl=http://{{ mirror_fqdn }}/fedora/updates/$releasever/$basearch/
enabled=1
gpgcheck=0
metadata_expire=6h
skip_if_unavailable=False
deltarpm=False
deltarpm_percentage=0

10
roles/dib-setup-gate-mirrors/templates/fedora-minimal/default/fedora.repo.j2 Normal file
View File

@ -0,0 +1,10 @@
[fedora]
name=Fedora $releasever - $basearch
failovermethod=priority
baseurl=http://{{ mirror_fqdn }}/fedora/releases/$releasever/Everything/$basearch/os/
enabled=1
metadata_expire=7d
gpgcheck=0
skip_if_unavailable=False
deltarpm=False
deltarpm_percentage=0