diff --git a/.zuul.d/jobs.yaml b/.zuul.d/jobs.yaml index 57e837c2..9beff8af 100644 --- a/.zuul.d/jobs.yaml +++ b/.zuul.d/jobs.yaml @@ -3,130 +3,226 @@ - tripleo-buildimage-jobs check: jobs: - - dib-dsvm-functests-python2-ubuntu-trusty: - branches: (master|feature/v2) - irrelevant-files: - - ^.*\.rst$ - - ^doc/.*$ - - dib-dsvm-functests-python2-ubuntu-trusty-extras: - voting: false - branches: (master|feature/v2) - irrelevant-files: - - ^.*\.rst$ - - ^doc/.*$ - - dib-dsvm-functests-python2-ubuntu-trusty-image: - voting: false - branches: (master|feature/v2) - irrelevant-files: - - ^.*\.rst$ - - ^doc/.*$ - - dib-dsvm-functests-python3: - irrelevant-files: - - ^.*\.rst$ - - ^doc/.*$ - - dib-dsvm-functests-python3-extras: - voting: false - irrelevant-files: - - ^.*\.rst$ - - ^doc/.*$ - - dib-dsvm-functests-python3-image: - voting: false - irrelevant-files: - - ^.*\.rst$ - - ^doc/.*$ - - dib-dsvm-functests-python2-centos-7: - irrelevant-files: - - ^.*\.rst$ - - ^doc/.*$ - - dib-dsvm-functests-python2-centos-7-extras: - voting: false - irrelevant-files: - - ^.*\.rst$ - - ^doc/.*$ - - dib-dsvm-functests-python2-centos-7-image: - voting: false - irrelevant-files: - - ^.*\.rst$ - - ^doc/.*$ + - dib-functests-centos7-python2 + - dib-functests-centos7-python2-image + - dib-functests-centos7-python2-extras + - dib-functests-trusty-python2 + - dib-functests-trusty-python2-image + - dib-functests-trusty-python2-extras + - dib-functests-xenial-python3 + - dib-functests-xenial-python3-image + - dib-functests-xenial-python3-extras + - dib-functests-bionic-python3 + - dib-functests-bionic-python3-image + - dib-functests-bionic-python3-extras + - tripleo-buildimage-overcloud-full-centos-7 - openstack-tox-lower-constraints gate: jobs: - - dib-dsvm-functests-python2-ubuntu-trusty: - branches: (master|feature/v2) - irrelevant-files: - - ^.*\.rst$ - - ^doc/.*$ - - dib-dsvm-functests-python3: - irrelevant-files: - - ^.*\.rst$ - - ^doc/.*$ - - dib-dsvm-functests-python2-centos-7: - irrelevant-files: - - ^.*\.rst$ - - ^doc/.*$ + - dib-functests-trusty-python2 + - dib-functests-xenial-python3 + - dib-functests-centos7-python2 - tripleo-buildimage-overcloud-full-centos-7 - openstack-tox-lower-constraints - job: - name: dib-dsvm-functests-python2-centos-7 - parent: legacy-dsvm-base - run: playbooks/legacy/dib-dsvm-functests-python2-centos-7/run.yaml - post-run: playbooks/legacy/dib-dsvm-functests-python2-centos-7/post.yaml + name: dib-functests-base + abstract: true + parent: base + run: playbooks/dib-functests/run.yaml + post-run: playbooks/dib-functests/post.yaml timeout: 3600 - nodeset: legacy-centos-7 required-projects: - openstack/diskimage-builder + - openstack/requirements + irrelevant-files: + - ^.*\.rst$ + - ^doc/.*$ - job: - name: dib-dsvm-functests-python2-centos-7-extras - parent: legacy-dsvm-base - run: playbooks/legacy/dib-dsvm-functests-python2-centos-7-extras/run.yaml - post-run: playbooks/legacy/dib-dsvm-functests-python2-centos-7-extras/post.yaml - timeout: 3600 - nodeset: legacy-centos-7 - required-projects: - - openstack/diskimage-builder + name: dib-functests-default + description: | + This is the base set of stable tests + abstract: true + parent: dib-functests-base + vars: + dib_functests: + - apt-sources/test-sources + - centos-minimal/build-succeeds + - fedora-minimal/build-succeeds + - ironic-agent/build-succeeds-fedora + - pip-and-virtualenv/source-install-fedora + - pip-and-virtualenv/source-install-ubuntu + - ubuntu-minimal/trusty-build-succeeds + - ubuntu-minimal/xenial-build-succeeds + - ubuntu-minimal/bionic-build-succeeds - job: - name: dib-dsvm-functests-python2-centos-7-image - parent: legacy-dsvm-base - run: playbooks/legacy/dib-dsvm-functests-python2-centos-7-image/run.yaml - post-run: playbooks/legacy/dib-dsvm-functests-python2-centos-7-image/post.yaml - timeout: 3600 - nodeset: legacy-centos-7 - required-projects: - - openstack/diskimage-builder + name: dib-functests-image + description: | + This is tests that rely on downloading an upstream + image. We generally do not have mirrors for these images + so they can be slighlty unreliable. + abstract: true + voting: false + parent: dib-functests-base + vars: + dib_functests: + - centos7/build-succeeds + - opensuse/build-succeeds + - opensuse/opensuse423-build-succeeds + - fedora/build-succeeds + - ubuntu/trusty-build-succeeds + - ubuntu/xenial-build-succeeds + - ubuntu/bionic-build-succeeds - job: - name: dib-dsvm-functests-python2-ubuntu-trusty - parent: legacy-dsvm-base - run: playbooks/legacy/dib-dsvm-functests-python2-ubuntu-trusty/run.yaml - post-run: playbooks/legacy/dib-dsvm-functests-python2-ubuntu-trusty/post.yaml - timeout: 3600 - nodeset: legacy-ubuntu-trusty - required-projects: - - openstack/diskimage-builder + name: dib-functests-extras + description: | + These are tests which are not as widely supported as the + core platforms + abstract: true + voting: false + parent: dib-functests-base + vars: + dib_functests: + - gentoo/build-succeeds + - debian-minimal/stable-vm + +# Bionic / Python3 tests +- job: + name: dib-functests-trusty-python2 + description: | + Run the default tests on a Trusty build host with Python 2 + parent: dib-functests-default + nodeset: ubuntu-trusty + vars: + dib_python: python2 - job: - name: dib-dsvm-functests-python2-ubuntu-trusty-extras - parent: legacy-dsvm-base - run: playbooks/legacy/dib-dsvm-functests-python2-ubuntu-trusty-extras/run.yaml - post-run: playbooks/legacy/dib-dsvm-functests-python2-ubuntu-trusty-extras/post.yaml - timeout: 3600 - nodeset: legacy-ubuntu-trusty - required-projects: - - openstack/diskimage-builder + name: dib-functests-trusty-python2-image + description: | + The image tests on a Trusty build host with Python 2 + parent: dib-functests-image + nodeset: ubuntu-trusty + vars: + dib_python: python2 - job: - name: dib-dsvm-functests-python2-ubuntu-trusty-image - parent: legacy-dsvm-base - run: playbooks/legacy/dib-dsvm-functests-python2-ubuntu-trusty-image/run.yaml - post-run: playbooks/legacy/dib-dsvm-functests-python2-ubuntu-trusty-image/post.yaml - timeout: 3600 - nodeset: legacy-ubuntu-trusty - required-projects: - - openstack/diskimage-builder + name: dib-functests-trusty-python2-extras + description: | + The extras tests on a Trusty build host with Python 2 + parent: dib-functests-extras + nodeset: ubuntu-trusty + vars: + dib_python: python2 + +# Bionic / Python3 tests +- job: + name: dib-functests-bionic-python3 + description: | + Run the default tests on a Xenial build host with Python 3 + parent: dib-functests-default + nodeset: ubuntu-bionic + voting: false + +- job: + name: dib-functests-bionic-python3-image + description: | + The image tests on a Xenial build host with Python 3 + parent: dib-functests-image + nodeset: ubuntu-bionic + voting: false + +- job: + name: dib-functests-bionic-python3-extras + description: | + The extras tests on a Xenial build host with Python 3 + parent: dib-functests-extras + nodeset: ubuntu-bionic + voting: false + +# Xenial / Python3 tests +- job: + name: dib-functests-xenial-python3 + description: | + Run the default tests on a Xenial build host with Python 3 + parent: dib-functests-default + nodeset: ubuntu-xenial + +- job: + name: dib-functests-xenial-python3-image + description: | + The image tests on a Xenial build host with Python 3 + parent: dib-functests-image + nodeset: ubuntu-xenial + +- job: + name: dib-functests-xenial-python3-extras + description: | + The extras tests on a Xenial build host with Python 3 + parent: dib-functests-extras + nodeset: ubuntu-xenial + vars: + # opensuse-minimal can only build on xenial ATM, as other + # platforms don't have zypper + dib_functests_extra: + - opensuse-minimal/opensuse423-build-succeeds + +# Bionic / Python3 tests +- job: + name: dib-functests-bionic-python3 + description: | + Run the default tests on a Xenial build host with Python 3 + parent: dib-functests-default + nodeset: ubuntu-bionic + voting: false + +- job: + name: dib-functests-bionic-python3-image + description: | + The image tests on a Xenial build host with Python 3 + parent: dib-functests-image + nodeset: ubuntu-bionic + voting: false + +- job: + name: dib-functests-bionic-python3-extras + description: | + The extras tests on a Xenial build host with Python 3 + parent: dib-functests-extras + nodeset: ubuntu-bionic + voting: false + +# Centos 7 / Python2 tests +- job: + name: dib-functests-centos7-python2 + description: | + Run the default tests on a Centos 7 build host with Python 2 + parent: dib-functests-default + nodeset: centos-7 + vars: + dib_python: python2 + +- job: + name: dib-functests-centos7-python2-image + description: | + The image tests on a Centos 7 build host with Python 2 + parent: dib-functests-image + nodeset: centos-7 + vars: + dib_python: python2 + +- job: + name: dib-functests-centos7-python2-extras + description: | + The extras tests on a Centos 7 build host with Python 2 + parent: dib-functests-extras + nodeset: centos-7 + vars: + dib_python: python2 + - job: name: dib-dsvm-functests-python3 diff --git a/contrib/setup-gate-mirrors.sh b/contrib/setup-gate-mirrors.sh index bf50f867..78d024ae 100755 --- a/contrib/setup-gate-mirrors.sh +++ b/contrib/setup-gate-mirrors.sh @@ -12,7 +12,7 @@ source /etc/ci/mirror_info.sh # Tests should probe for this directory and then use the repos/sources # files inside it for the gate tests. -BASE_DIR=$WORKSPACE/dib-mirror +BASE_DIR=${WORKSPACE:-$(pwd)}/dib-mirror mkdir -p $BASE_DIR diff --git a/diskimage_builder/elements/openstack-ci-mirrors/environment.d/11-dib-distribution-mirror.bash b/diskimage_builder/elements/openstack-ci-mirrors/environment.d/11-dib-distribution-mirror.bash index 61d79ca7..83119084 100644 --- a/diskimage_builder/elements/openstack-ci-mirrors/environment.d/11-dib-distribution-mirror.bash +++ b/diskimage_builder/elements/openstack-ci-mirrors/environment.d/11-dib-distribution-mirror.bash @@ -44,12 +44,16 @@ export DIB_DISTRIBUTION_MIRROR_UBUNTU_INSECURE=1 # These repo files are pre-created for the fedora/centos-minimal jobs # in the gate. Not relevant inside the chroot. -if [[ -d ${WORKSPACE:-/not/a/path/}/dib-mirror ]]; then +if [[ -d ${DIB_OS_CI_YUM_REPOS:-/not/a/path/} ]]; then if [[ "${DISTRO_NAME}" == "fedora" ]]; then - export DIB_YUM_MINIMAL_BOOTSTRAP_REPOS=${WORKSPACE}/dib-mirror/fedora-minimal/yum.repos.d + if [[ -d ${DIB_OS_CI_YUM_REPOS}/fedora-minimal/${DIB_RELEASE} ]]; then + export DIB_YUM_MINIMAL_BOOTSTRAP_REPOS=${DIB_OS_CI_YUM_REPOS}/fedora-minimal/${DIB_RELEASE}/yum.repos.d + else + export DIB_YUM_MINIMAL_BOOTSTRAP_REPOS=${DIB_OS_CI_YUM_REPOS}/fedora-minimal/default/yum.repos.d + fi elif [[ "${DISTRO_NAME}" == "centos" ]]; then - export DIB_YUM_MINIMAL_BOOTSTRAP_REPOS=${WORKSPACE}/dib-mirror/centos-minimal/yum.repos.d + export DIB_YUM_MINIMAL_BOOTSTRAP_REPOS=${DIB_OS_CI_YUM_REPOS}/centos-minimal/yum.repos.d fi fi diff --git a/playbooks/dib-functests/post.yaml b/playbooks/dib-functests/post.yaml new file mode 100644 index 00000000..2d7a3d73 --- /dev/null +++ b/playbooks/dib-functests/post.yaml @@ -0,0 +1,13 @@ +- hosts: all + tasks: + + - name: Check logs directory exists + stat: path='{{ ansible_user_dir }}/logs/' + register: check_path + + - name: Copy log files from {{ ansible_user_dir }} on node + synchronize: + src: '{{ ansible_user_dir }}/logs/' + dest: '{{ zuul.executor.log_root }}/logs' + mode: pull + when: check_path.stat.exists == true diff --git a/playbooks/dib-functests/run.yaml b/playbooks/dib-functests/run.yaml new file mode 100644 index 00000000..2cecfa44 --- /dev/null +++ b/playbooks/dib-functests/run.yaml @@ -0,0 +1,7 @@ +- hosts: all + name: Run dib functional tests + roles: + - bindep + - dib-setup-gate-mirrors + - dib-functests + diff --git a/roles/dib-functests/README.rst b/roles/dib-functests/README.rst new file mode 100644 index 00000000..a55ce479 --- /dev/null +++ b/roles/dib-functests/README.rst @@ -0,0 +1,18 @@ +dib-functests +------------- + +Run diskimage-builder functional tests + +Installs dib and dependencies, and runs functional tests + +**Role Variables** + +.. zuul:rolevar:: python_version + :default: 2 + + The python version to run the test under + +.. zuul:rolevar:: dib_functests + :default: [] + + The list of functional tests to run diff --git a/roles/dib-functests/defaults/main.yaml b/roles/dib-functests/defaults/main.yaml new file mode 100644 index 00000000..56cb5b11 --- /dev/null +++ b/roles/dib-functests/defaults/main.yaml @@ -0,0 +1,8 @@ +--- +dib_python: python3 +dib_virtualenv: "{{ ansible_user_dir }}/dib-venv" +dib_functests: [] +# This is a bit of hack; it allows us to template out jobs where the +# base tests are defined in dib_functests, but then add a few extra +# tests to run in this variable for a specific test. +dib_functests_extra: [] \ No newline at end of file diff --git a/roles/dib-functests/tasks/main.yaml b/roles/dib-functests/tasks/main.yaml new file mode 100644 index 00000000..51e2363d --- /dev/null +++ b/roles/dib-functests/tasks/main.yaml @@ -0,0 +1,49 @@ +- name: Install dib extra dependencies + command: ./install_test_deps.sh + args: + chdir: "{{ zuul.project.src_dir }}/tests" + become: yes + +- name: Setup mounted space + file: + path: '/opt/dib_cache' + state: directory + mode: ugo+rw + become: yes + +- name: Define upper_constraints + set_fact: + upper_constraints: "{{ ansible_user_dir }}/{{ zuul.projects['git.openstack.org/openstack/requirements'].src_dir }}/upper-constraints.txt" + +- name: Take ourselves out of uppper constraints + lineinfile: + path: "{{ upper_constraints }}" + state: absent + regexp: '^diskimage-builder' + +- name: Install diskimage-builder + pip: + name: "file://{{ ansible_user_dir }}/{{ zuul.project.src_dir }}" + virtualenv: "{{ dib_virtualenv }}" + virtualenv_python: "{{ dib_python }}" + extra_args: "-c {{ upper_constraints }}" + +- name: Setup log output + file: + path: "{{ ansible_user_dir }}/logs" + state: directory + mode: ugo+rw + +- name: Run dib functional tests + shell: + cmd: | + source {{ dib_virtualenv }}/bin/activate + {{ zuul.project.src_dir }}/tests/run_functests.sh \ + -L {{ ansible_user_dir }}/logs {{ dib_functests | join(' ') }} \ + {{ dib_functests_extra | join(' ') }} + args: + executable: /bin/bash + environment: + DIB_NO_TMPFS: 1 + TMPDIR: /opt/dib_cache + DIB_OS_CI_YUM_REPOS: "{{ dib_gate_mirror_repos|default(omit) }}" diff --git a/roles/dib-setup-gate-mirrors/README.rst b/roles/dib-setup-gate-mirrors/README.rst new file mode 100644 index 00000000..ccacee88 --- /dev/null +++ b/roles/dib-setup-gate-mirrors/README.rst @@ -0,0 +1,6 @@ +dib-setup-gate-mirrors +====================== + +Create customised repository files as required by the +openstack-ci-mirrors element. Path to repo files is set into +``dib_gate_mirror_repos`` diff --git a/roles/dib-setup-gate-mirrors/defaults/main.yaml b/roles/dib-setup-gate-mirrors/defaults/main.yaml new file mode 100644 index 00000000..ac832bea --- /dev/null +++ b/roles/dib-setup-gate-mirrors/defaults/main.yaml @@ -0,0 +1 @@ +mirror_fqdn: "{{ zuul_site_mirror_fqdn|default(omit) }}" \ No newline at end of file diff --git a/roles/dib-setup-gate-mirrors/tasks/main.yaml b/roles/dib-setup-gate-mirrors/tasks/main.yaml new file mode 100644 index 00000000..da7521d0 --- /dev/null +++ b/roles/dib-setup-gate-mirrors/tasks/main.yaml @@ -0,0 +1,42 @@ +- name: Set repo path + set_fact: + dib_gate_mirror_repos: "{{ ansible_user_dir }}/dib-mirror" + +- debug: msg="zuul site mirror {{ zuul_site_mirror_fqdn }}" +- debug: msg="mirror_fqdn {{ mirror_fqdn }}" + +- name: Create centos-minimal directory + file: + path: "{{ dib_gate_mirror_repos }}/centos-minimal/yum.repos.d" + state: directory + mode: 0775 + recurse: yes + +- name: Install centos-minimal repo files + template: + dest: "{{ dib_gate_mirror_repos }}/centos-minimal/yum.repos.d/dib-mirror-{{ item }}" + mode: 0644 + src: "centos-minimal/{{ item }}.j2" + with_items: + - base.repo + - updates.repo + - extras.repo + +- name: Create fedora-minimal directories + file: + path: "{{ dib_gate_mirror_repos }}/fedora-minimal/{{ item }}/yum.repos.d" + state: directory + mode: 0775 + recurse: yes + with_items: + - default + - '28' + +- name: Install fedora-minimal repo files + template: + dest: "{{ dib_gate_mirror_repos }}/fedora-minimal/{{ item[0] }}/yum.repos.d/dib-mirror-{{ item[1] }}" + mode: 0644 + src: "fedora-minimal/{{ item[0] }}/{{ item[1] }}.j2" + with_nested: + - [ 'default', '28' ] + - [ 'fedora.repo', 'fedora-updates.repo' ] diff --git a/roles/dib-setup-gate-mirrors/templates/centos-minimal/base.repo.j2 b/roles/dib-setup-gate-mirrors/templates/centos-minimal/base.repo.j2 new file mode 100644 index 00000000..1883758a --- /dev/null +++ b/roles/dib-setup-gate-mirrors/templates/centos-minimal/base.repo.j2 @@ -0,0 +1,4 @@ +[base] +name=CentOS-$releasever - Base +baseurl=http://{{ mirror_fqdn }}/centos/$releasever/os/$basearch/ +gpgcheck=0 diff --git a/roles/dib-setup-gate-mirrors/templates/centos-minimal/extras.repo.j2 b/roles/dib-setup-gate-mirrors/templates/centos-minimal/extras.repo.j2 new file mode 100644 index 00000000..d0ff4601 --- /dev/null +++ b/roles/dib-setup-gate-mirrors/templates/centos-minimal/extras.repo.j2 @@ -0,0 +1,5 @@ +#additional packages that may be useful +[extras] +name=CentOS-$releasever - Extras +baseurl=http://{{ mirror_fqdn }}/centos/$releasever/extras/$basearch/ +gpgcheck=0 diff --git a/roles/dib-setup-gate-mirrors/templates/centos-minimal/updates.repo.j2 b/roles/dib-setup-gate-mirrors/templates/centos-minimal/updates.repo.j2 new file mode 100644 index 00000000..ce0b0779 --- /dev/null +++ b/roles/dib-setup-gate-mirrors/templates/centos-minimal/updates.repo.j2 @@ -0,0 +1,5 @@ +#released updates +[updates] +name=CentOS-$releasever - Updates +baseurl=http://{{ mirror_fqdn }}/centos/$releasever/updates/$basearch/ +gpgcheck=0 diff --git a/roles/dib-setup-gate-mirrors/templates/fedora-minimal/28/fedora-updates.repo.j2 b/roles/dib-setup-gate-mirrors/templates/fedora-minimal/28/fedora-updates.repo.j2 new file mode 100644 index 00000000..5725c665 --- /dev/null +++ b/roles/dib-setup-gate-mirrors/templates/fedora-minimal/28/fedora-updates.repo.j2 @@ -0,0 +1,10 @@ +[updates] +name=Fedora $releasever - $basearch - Updates +failovermethod=priority +baseurl=http://{{ mirror_fqdn }}/fedora/updates/$releasever/Everything/$basearch/ +enabled=1 +gpgcheck=0 +metadata_expire=6h +skip_if_unavailable=False +deltarpm=False +deltarpm_percentage=0 diff --git a/roles/dib-setup-gate-mirrors/templates/fedora-minimal/28/fedora.repo.j2 b/roles/dib-setup-gate-mirrors/templates/fedora-minimal/28/fedora.repo.j2 new file mode 100644 index 00000000..74853f8d --- /dev/null +++ b/roles/dib-setup-gate-mirrors/templates/fedora-minimal/28/fedora.repo.j2 @@ -0,0 +1,10 @@ +[fedora] +name=Fedora $releasever - $basearch +failovermethod=priority +baseurl=http://{{ mirror_fqdn }}/fedora/releases/$releasever/Everything/$basearch/os/ +enabled=1 +metadata_expire=7d +gpgcheck=0 +skip_if_unavailable=False +deltarpm=False +deltarpm_percentage=0 diff --git a/roles/dib-setup-gate-mirrors/templates/fedora-minimal/default/fedora-updates.repo.j2 b/roles/dib-setup-gate-mirrors/templates/fedora-minimal/default/fedora-updates.repo.j2 new file mode 100644 index 00000000..79ecb21e --- /dev/null +++ b/roles/dib-setup-gate-mirrors/templates/fedora-minimal/default/fedora-updates.repo.j2 @@ -0,0 +1,10 @@ +[updates] +name=Fedora $releasever - $basearch - Updates +failovermethod=priority +baseurl=http://{{ mirror_fqdn }}/fedora/updates/$releasever/$basearch/ +enabled=1 +gpgcheck=0 +metadata_expire=6h +skip_if_unavailable=False +deltarpm=False +deltarpm_percentage=0 diff --git a/roles/dib-setup-gate-mirrors/templates/fedora-minimal/default/fedora.repo.j2 b/roles/dib-setup-gate-mirrors/templates/fedora-minimal/default/fedora.repo.j2 new file mode 100644 index 00000000..74853f8d --- /dev/null +++ b/roles/dib-setup-gate-mirrors/templates/fedora-minimal/default/fedora.repo.j2 @@ -0,0 +1,10 @@ +[fedora] +name=Fedora $releasever - $basearch +failovermethod=priority +baseurl=http://{{ mirror_fqdn }}/fedora/releases/$releasever/Everything/$basearch/os/ +enabled=1 +metadata_expire=7d +gpgcheck=0 +skip_if_unavailable=False +deltarpm=False +deltarpm_percentage=0