From 649f0b66d9011bc041f33281c1b25949b8d8628d Mon Sep 17 00:00:00 2001 From: Ian Wienand Date: Tue, 20 Jun 2017 10:42:10 +1000 Subject: [PATCH] Start at using CI mirrors for fedora/centos fedora/centos-minimal don't obey DIB_DISTRIBUTION_MIRROR currently. I don't really want them too -- we want to be able to separate the mirrors used during the build process from those embedded into the final image. Add DIB_YUM_MINIMAL_BOOTSTRAP_REPOS which is a directory with repo files to use during the install. This introduces setup-gate-mirrors.sh which is intended to setup repo/sources/whatever files in the openstack gate that point to the local region mirror. It pulls the info from the mirror_info.sh script on each CI node. The openstack-ci-mirrors element is updated to export these variables. elements are updated to depend on it. Tests are restored Change-Id: I7604fc4d41cb1483be16b8d628a24e8fc764f515 --- contrib/setup-gate-mirrors.sh | 81 +++++++++++++++++++ .../test-elements/build-succeeds/element-deps | 1 + .../test-elements/build-succeeds/element-deps | 1 + .../test-elements/build-succeeds/element-deps | 1 + .../environment.d/10-fedora-pin-mirror.bash | 2 - .../test-elements/build-succeeds/element-deps | 1 + .../environment.d/10-fedora-pin-mirror.bash | 2 - .../environment.d/10-fedora-pin-mirror.bash | 9 ++- .../11-dib-distribution-mirror.bash | 12 +++ .../source-install-fedora/element-deps | 1 + .../elements/yum-minimal/README.rst | 6 ++ .../cleanup.d/95-remove-yum-mirror | 35 ++++++++ .../elements/yum-minimal/root.d/08-yum-chroot | 24 +++++- tests/run_functests.sh | 8 +- 14 files changed, 172 insertions(+), 12 deletions(-) create mode 100755 contrib/setup-gate-mirrors.sh create mode 100644 diskimage_builder/elements/centos-minimal/test-elements/build-succeeds/element-deps create mode 100644 diskimage_builder/elements/fedora-minimal/test-elements/build-succeeds/element-deps delete mode 100644 diskimage_builder/elements/fedora-minimal/test-elements/build-succeeds/environment.d/10-fedora-pin-mirror.bash delete mode 100644 diskimage_builder/elements/fedora/test-elements/build-succeeds/environment.d/10-fedora-pin-mirror.bash create mode 100755 diskimage_builder/elements/yum-minimal/cleanup.d/95-remove-yum-mirror diff --git a/contrib/setup-gate-mirrors.sh b/contrib/setup-gate-mirrors.sh new file mode 100755 index 00000000..7cd7010c --- /dev/null +++ b/contrib/setup-gate-mirrors.sh @@ -0,0 +1,81 @@ +#!/bin/bash + +set -x + +# +# This tool creates repo/sources files that point to the mirrors for +# the host region in the OpenStack CI gate. +# + +# This pre-created on CI nodes by slave scripts +source /etc/ci/mirror_info.sh + +# Tests should probe for this directory and then use the repos/sources +# files inside it for the gate tests. +BASE_DIR=$WORKSPACE/dib-mirror + +mkdir -p $BASE_DIR + +## REPOS + +# all should start with "dib-mirror-" +# gpg check turned off, because we don't have the keys outside the chroot + +# fedora-minimal +FEDORA_MIN_DIR=$BASE_DIR/fedora-minimal/yum.repos.d +mkdir -p $FEDORA_MIN_DIR + +cat < $FEDORA_MIN_DIR/dib-mirror-fedora.repo +[fedora] +name=Fedora \$releasever - \$basearch +failovermethod=priority +baseurl=$NODEPOOL_FEDORA_MIRROR/releases/\$releasever/Everything/\$basearch/os/ +enabled=1 +metadata_expire=7d +gpgcheck=0 +skip_if_unavailable=False +deltarpm=False +deltarpm_percentage=0 +EOF + +cat < $FEDORA_MIN_DIR/dib-mirror-fedora-updates.repo +[updates] +name=Fedora \$releasever - \$basearch - Updates +failovermethod=priority +baseurl=$NODEPOOL_FEDORA_MIRROR/updates/\$releasever/\$basearch/ +enabled=1 +gpgcheck=0 +metadata_expire=6h +skip_if_unavailable=False +deltarpm=False +deltarpm_percentage=0 +EOF + +# Centos Minimal +CENTOS_MIN_DIR=$BASE_DIR/centos-minimal/yum.repos.d +mkdir -p $CENTOS_MIN_DIR + +cat < $CENTOS_MIN_DIR/dib-mirror-base.repo +[base] +name=CentOS-\$releasever - Base +baseurl=$NODEPOOL_CENTOS_MIRROR/\$releasever/os/\$basearch/ +gpgcheck=0 +EOF + +cat < $CENTOS_MIN_DIR/dib-mirror-updates.repo +#released updates +[updates] +name=CentOS-\$releasever - Updates +baseurl=$NODEPOOL_CENTOS_MIRROR/\$releasever/updates/\$basearch/ +gpgcheck=0 +EOF + +cat < $CENTOS_MIN_DIR/dib-mirror-extras.repo +#additional packages that may be useful +[extras] +name=CentOS-\$releasever - Extras +baseurl=$NODEPOOL_CENTOS_MIRROR/\$releasever/extras/\$basearch/ +gpgcheck=0 +EOF + +## apt sources (todo) diff --git a/diskimage_builder/elements/centos-minimal/test-elements/build-succeeds/element-deps b/diskimage_builder/elements/centos-minimal/test-elements/build-succeeds/element-deps new file mode 100644 index 00000000..7791c84f --- /dev/null +++ b/diskimage_builder/elements/centos-minimal/test-elements/build-succeeds/element-deps @@ -0,0 +1 @@ +openstack-ci-mirrors \ No newline at end of file diff --git a/diskimage_builder/elements/centos7/test-elements/build-succeeds/element-deps b/diskimage_builder/elements/centos7/test-elements/build-succeeds/element-deps index df967b96..7b69c4f1 100644 --- a/diskimage_builder/elements/centos7/test-elements/build-succeeds/element-deps +++ b/diskimage_builder/elements/centos7/test-elements/build-succeeds/element-deps @@ -1 +1,2 @@ base +openstack-ci-mirrors diff --git a/diskimage_builder/elements/fedora-minimal/test-elements/build-succeeds/element-deps b/diskimage_builder/elements/fedora-minimal/test-elements/build-succeeds/element-deps new file mode 100644 index 00000000..7791c84f --- /dev/null +++ b/diskimage_builder/elements/fedora-minimal/test-elements/build-succeeds/element-deps @@ -0,0 +1 @@ +openstack-ci-mirrors \ No newline at end of file diff --git a/diskimage_builder/elements/fedora-minimal/test-elements/build-succeeds/environment.d/10-fedora-pin-mirror.bash b/diskimage_builder/elements/fedora-minimal/test-elements/build-succeeds/environment.d/10-fedora-pin-mirror.bash deleted file mode 100644 index 13224cfa..00000000 --- a/diskimage_builder/elements/fedora-minimal/test-elements/build-succeeds/environment.d/10-fedora-pin-mirror.bash +++ /dev/null @@ -1,2 +0,0 @@ -# Pin to this mirror because the roundrobin is fairly unreliable -export DIB_DISTRIBUTION_MIRROR=http://dl.fedoraproject.org/pub/fedora/linux diff --git a/diskimage_builder/elements/fedora/test-elements/build-succeeds/element-deps b/diskimage_builder/elements/fedora/test-elements/build-succeeds/element-deps index df967b96..7b69c4f1 100644 --- a/diskimage_builder/elements/fedora/test-elements/build-succeeds/element-deps +++ b/diskimage_builder/elements/fedora/test-elements/build-succeeds/element-deps @@ -1 +1,2 @@ base +openstack-ci-mirrors diff --git a/diskimage_builder/elements/fedora/test-elements/build-succeeds/environment.d/10-fedora-pin-mirror.bash b/diskimage_builder/elements/fedora/test-elements/build-succeeds/environment.d/10-fedora-pin-mirror.bash deleted file mode 100644 index 13224cfa..00000000 --- a/diskimage_builder/elements/fedora/test-elements/build-succeeds/environment.d/10-fedora-pin-mirror.bash +++ /dev/null @@ -1,2 +0,0 @@ -# Pin to this mirror because the roundrobin is fairly unreliable -export DIB_DISTRIBUTION_MIRROR=http://dl.fedoraproject.org/pub/fedora/linux diff --git a/diskimage_builder/elements/ironic-agent/test-elements/build-succeeds-fedora/environment.d/10-fedora-pin-mirror.bash b/diskimage_builder/elements/ironic-agent/test-elements/build-succeeds-fedora/environment.d/10-fedora-pin-mirror.bash index 13224cfa..3740c09d 100644 --- a/diskimage_builder/elements/ironic-agent/test-elements/build-succeeds-fedora/environment.d/10-fedora-pin-mirror.bash +++ b/diskimage_builder/elements/ironic-agent/test-elements/build-succeeds-fedora/environment.d/10-fedora-pin-mirror.bash @@ -1,2 +1,7 @@ -# Pin to this mirror because the roundrobin is fairly unreliable -export DIB_DISTRIBUTION_MIRROR=http://dl.fedoraproject.org/pub/fedora/linux +# Set DIB_DISTRIBUTION_MIRROR if running in openstack gate +if [ -f /etc/ci/mirror_info.sh ]; then + source /etc/ci/mirror_info.sh + export DIB_DISTRIBUTION_MIRROR=$NODEPOOL_FEDORA_MIRROR +else + export DIB_DISTRIBUTION_MIRROR=http://dl.fedoraproject.org/pub/fedora/linux +fi diff --git a/diskimage_builder/elements/openstack-ci-mirrors/environment.d/11-dib-distribution-mirror.bash b/diskimage_builder/elements/openstack-ci-mirrors/environment.d/11-dib-distribution-mirror.bash index 03af8220..33ae67c3 100644 --- a/diskimage_builder/elements/openstack-ci-mirrors/environment.d/11-dib-distribution-mirror.bash +++ b/diskimage_builder/elements/openstack-ci-mirrors/environment.d/11-dib-distribution-mirror.bash @@ -24,3 +24,15 @@ if [ -f /etc/ci/mirror_info.sh ]; then fi fi + +# This is repo files pre-created for the fedora/centos-minimal jobs in +# the gate +if [[ -d ${WORKSPACE:-/not/a/path/}/dib-mirror ]]; then + + if [[ "${DISTRO_NAME}" == "fedora" ]]; then + export DIB_YUM_MINIMAL_BOOTSTRAP_REPOS=${WORKSPACE}/dib-mirror/fedora-minimal/yum.repos.d + elif [[ "${DISTRO_NAME}" == "centos" ]]; then + export DIB_YUM_MINIMAL_BOOTSTRAP_REPOS=${WORKSPACE}/dib-mirror/centos-minimal/yum.repos.d + fi + +fi diff --git a/diskimage_builder/elements/pip-and-virtualenv/test-elements/source-install-fedora/element-deps b/diskimage_builder/elements/pip-and-virtualenv/test-elements/source-install-fedora/element-deps index b767efbb..33b08441 100644 --- a/diskimage_builder/elements/pip-and-virtualenv/test-elements/source-install-fedora/element-deps +++ b/diskimage_builder/elements/pip-and-virtualenv/test-elements/source-install-fedora/element-deps @@ -1 +1,2 @@ fedora-minimal +openstack-ci-mirrors diff --git a/diskimage_builder/elements/yum-minimal/README.rst b/diskimage_builder/elements/yum-minimal/README.rst index e68d704d..7d1889a2 100644 --- a/diskimage_builder/elements/yum-minimal/README.rst +++ b/diskimage_builder/elements/yum-minimal/README.rst @@ -12,3 +12,9 @@ Ubuntu and Debian. Nothing additional is needed on Fedora or CentOS. If you wish to have DHCP networking setup for eth0 & eth1 via /etc/sysconfig/network-config scripts/ifcfg-eth[0|1], set the environment variable `DIB_YUM_MINIMAL_CREATE_INTERFACES` to `1`. + +If you wish to build from specific mirrors, set +``DIB_YUM_MINIMAL_BOOTSTRAP_REPOS`` to a directory with the ``.repo`` +files to use during bootstrap and build. The repo files should be +named with a prefix ``dib-mirror-`` and will be removed from the final +image. diff --git a/diskimage_builder/elements/yum-minimal/cleanup.d/95-remove-yum-mirror b/diskimage_builder/elements/yum-minimal/cleanup.d/95-remove-yum-mirror new file mode 100755 index 00000000..343c16d8 --- /dev/null +++ b/diskimage_builder/elements/yum-minimal/cleanup.d/95-remove-yum-mirror @@ -0,0 +1,35 @@ +#!/bin/bash +# +# Copyright 2015 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + +# dib-lint: disable=safe_sudo + +if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then + set -x +fi +set -eu +set -o pipefail + +# This removes any build-time repos that might be set in 08-yum-chroot +if [[ -n ${DIB_YUM_MINIMAL_BOOTSTRAP_REPOS:-} ]]; then + + sudo rm -f $TMP_MOUNT_PATH/etc/yum.repos.d/dib-mirror*.repo + + for repo in $TMP_MOUNT_PATH/etc/yum.repos.d/*.USING_MIRROR; do + sudo mv $repo ${repo/.USING_MIRROR/} + done + +fi diff --git a/diskimage_builder/elements/yum-minimal/root.d/08-yum-chroot b/diskimage_builder/elements/yum-minimal/root.d/08-yum-chroot index ee96e552..5289eae3 100755 --- a/diskimage_builder/elements/yum-minimal/root.d/08-yum-chroot +++ b/diskimage_builder/elements/yum-minimal/root.d/08-yum-chroot @@ -78,15 +78,23 @@ function _install_repos { packages+="${DISTRO_NAME}-repos " fi + # By default, parent elements (fedora-minimal, centos-minimal) + # have a yum.repos.d directory in the element with a default repo; + # this is copied to TMP_HOOK_PATH by the usual hook-copying + # routines. In the gate, environment.d files for the funtional + # tests will set DIB_YUM_MINIMAL_BOOTSTRAP_REPOS -- this contains + # mirrors correct for the region setup by contrib/setup-gate-mirrors.sh + local repo=${DIB_YUM_MINIMAL_BOOTSTRAP_REPOS:-${TMP_HOOKS_PATH}/yum.repos.d} + # yumdownloader puts repo xml files and such into a directory # ${TMPDIR}/yum-$USER-random. Since we don't need this once the # initial download happens, redirect TMPDIR for this call so we # can clean it up nicely local temp_tmp temp_tmp=$(mktemp -d) - TMPDIR=${temp_tmp} yumdownloader \ + TMPDIR=${temp_tmp} yumdownloader --verbose \ --releasever=$DIB_RELEASE \ - --setopt=reposdir=$TMP_HOOKS_PATH/yum.repos.d \ + --setopt=reposdir=$repo \ --destdir=$WORKING \ ${packages} && rc=$? || rc=$? rm -rf ${temp_tmp} @@ -97,6 +105,18 @@ function _install_repos { # --nodeps works around these wanting /bin/sh in some fedora # releases, see rhbz#1265873 sudo $_RPM --root $TARGET_ROOT --nodeps -ivh $WORKING/*rpm + + # install the bootstrap mirror repos over the default ones, if + # set. we will remove this at the end so the final image has + # regular mirrors + if [[ -n ${DIB_YUM_MINIMAL_BOOTSTRAP_REPOS:-} ]]; then + for repo in $TARGET_ROOT/etc/yum.repos.d/*.repo; do + sudo mv $repo $repo.USING_MIRROR + done + sudo cp ${DIB_YUM_MINIMAL_BOOTSTRAP_REPOS}/* \ + $TARGET_ROOT/etc/yum.repos.d/ + fi + } # _install_pkg_manager packages... diff --git a/tests/run_functests.sh b/tests/run_functests.sh index 8a4fd1af..5af62c21 100755 --- a/tests/run_functests.sh +++ b/tests/run_functests.sh @@ -11,6 +11,10 @@ export DIB_ELEMENTS=$(python -c ' import diskimage_builder.paths diskimage_builder.paths.show_path("elements")') +# XXX: This will move into the gate scripts, and happen before +# we run this script ... just here for initial bringup +./openstack/diskimage-builder/contrib/setup-gate-mirrors.sh + # Setup sane locale defaults, because this information is leaked into DIB. export LANG=en_US.utf8 export LC_ALL= @@ -42,10 +46,6 @@ DEFAULT_SKIP_TESTS=( debian-minimal/stable-build-succeeds debian-minimal/stable-vm debian/build-succeeds - # temporary disable until change to use mirrors - pip-and-virtualenv/source-install-fedora - ironic-agent/build-succeeds-fedora - fedora-minimal/build-succeeds ) # The default output formats (specified to disk-image-create's "-t"