Start at using CI mirrors for fedora/centos
fedora/centos-minimal don't obey DIB_DISTRIBUTION_MIRROR currently. I don't really want them too -- we want to be able to separate the mirrors used during the build process from those embedded into the final image. Add DIB_YUM_MINIMAL_BOOTSTRAP_REPOS which is a directory with repo files to use during the install. This introduces setup-gate-mirrors.sh which is intended to setup repo/sources/whatever files in the openstack gate that point to the local region mirror. It pulls the info from the mirror_info.sh script on each CI node. The openstack-ci-mirrors element is updated to export these variables. elements are updated to depend on it. Tests are restored Change-Id: I7604fc4d41cb1483be16b8d628a24e8fc764f515
This commit is contained in:
Ian Wienand
parent
f0b70211c6
commit
649f0b66d9
81
contrib/setup-gate-mirrors.sh
Executable file
81
contrib/setup-gate-mirrors.sh
Executable file
@ -0,0 +1,81 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -x
|
||||
|
||||
#
|
||||
# This tool creates repo/sources files that point to the mirrors for
|
||||
# the host region in the OpenStack CI gate.
|
||||
#
|
||||
|
||||
# This pre-created on CI nodes by slave scripts
|
||||
source /etc/ci/mirror_info.sh
|
||||
|
||||
# Tests should probe for this directory and then use the repos/sources
|
||||
# files inside it for the gate tests.
|
||||
BASE_DIR=$WORKSPACE/dib-mirror
|
||||
|
||||
mkdir -p $BASE_DIR
|
||||
|
||||
## REPOS
|
||||
|
||||
# all should start with "dib-mirror-"
|
||||
# gpg check turned off, because we don't have the keys outside the chroot
|
||||
|
||||
# fedora-minimal
|
||||
FEDORA_MIN_DIR=$BASE_DIR/fedora-minimal/yum.repos.d
|
||||
mkdir -p $FEDORA_MIN_DIR
|
||||
|
||||
cat <<EOF > $FEDORA_MIN_DIR/dib-mirror-fedora.repo
|
||||
[fedora]
|
||||
name=Fedora \$releasever - \$basearch
|
||||
failovermethod=priority
|
||||
baseurl=$NODEPOOL_FEDORA_MIRROR/releases/\$releasever/Everything/\$basearch/os/
|
||||
enabled=1
|
||||
metadata_expire=7d
|
||||
gpgcheck=0
|
||||
skip_if_unavailable=False
|
||||
deltarpm=False
|
||||
deltarpm_percentage=0
|
||||
EOF
|
||||
|
||||
cat <<EOF > $FEDORA_MIN_DIR/dib-mirror-fedora-updates.repo
|
||||
[updates]
|
||||
name=Fedora \$releasever - \$basearch - Updates
|
||||
failovermethod=priority
|
||||
baseurl=$NODEPOOL_FEDORA_MIRROR/updates/\$releasever/\$basearch/
|
||||
enabled=1
|
||||
gpgcheck=0
|
||||
metadata_expire=6h
|
||||
skip_if_unavailable=False
|
||||
deltarpm=False
|
||||
deltarpm_percentage=0
|
||||
EOF
|
||||
|
||||
# Centos Minimal
|
||||
CENTOS_MIN_DIR=$BASE_DIR/centos-minimal/yum.repos.d
|
||||
mkdir -p $CENTOS_MIN_DIR
|
||||
|
||||
cat <<EOF > $CENTOS_MIN_DIR/dib-mirror-base.repo
|
||||
[base]
|
||||
name=CentOS-\$releasever - Base
|
||||
baseurl=$NODEPOOL_CENTOS_MIRROR/\$releasever/os/\$basearch/
|
||||
gpgcheck=0
|
||||
EOF
|
||||
|
||||
cat <<EOF > $CENTOS_MIN_DIR/dib-mirror-updates.repo
|
||||
#released updates
|
||||
[updates]
|
||||
name=CentOS-\$releasever - Updates
|
||||
baseurl=$NODEPOOL_CENTOS_MIRROR/\$releasever/updates/\$basearch/
|
||||
gpgcheck=0
|
||||
EOF
|
||||
|
||||
cat <<EOF > $CENTOS_MIN_DIR/dib-mirror-extras.repo
|
||||
#additional packages that may be useful
|
||||
[extras]
|
||||
name=CentOS-\$releasever - Extras
|
||||
baseurl=$NODEPOOL_CENTOS_MIRROR/\$releasever/extras/\$basearch/
|
||||
gpgcheck=0
|
||||
EOF
|
||||
|
||||
## apt sources (todo)
|
||||
@ -0,0 +1 @@
|
||||
openstack-ci-mirrors
|
||||
@ -1 +1,2 @@
|
||||
base
|
||||
openstack-ci-mirrors
|
||||
|
||||
@ -0,0 +1 @@
|
||||
openstack-ci-mirrors
|
||||
@ -1,2 +0,0 @@
|
||||
# Pin to this mirror because the roundrobin is fairly unreliable
|
||||
export DIB_DISTRIBUTION_MIRROR=http://dl.fedoraproject.org/pub/fedora/linux
|
||||
@ -1 +1,2 @@
|
||||
base
|
||||
openstack-ci-mirrors
|
||||
|
||||
@ -1,2 +0,0 @@
|
||||
# Pin to this mirror because the roundrobin is fairly unreliable
|
||||
export DIB_DISTRIBUTION_MIRROR=http://dl.fedoraproject.org/pub/fedora/linux
|
||||
@ -1,2 +1,7 @@
|
||||
# Pin to this mirror because the roundrobin is fairly unreliable
|
||||
# Set DIB_DISTRIBUTION_MIRROR if running in openstack gate
|
||||
if [ -f /etc/ci/mirror_info.sh ]; then
|
||||
source /etc/ci/mirror_info.sh
|
||||
export DIB_DISTRIBUTION_MIRROR=$NODEPOOL_FEDORA_MIRROR
|
||||
else
|
||||
export DIB_DISTRIBUTION_MIRROR=http://dl.fedoraproject.org/pub/fedora/linux
|
||||
fi
|
||||
|
||||
@ -24,3 +24,15 @@ if [ -f /etc/ci/mirror_info.sh ]; then
|
||||
fi
|
||||
|
||||
fi
|
||||
|
||||
# This is repo files pre-created for the fedora/centos-minimal jobs in
|
||||
# the gate
|
||||
if [[ -d ${WORKSPACE:-/not/a/path/}/dib-mirror ]]; then
|
||||
|
||||
if [[ "${DISTRO_NAME}" == "fedora" ]]; then
|
||||
export DIB_YUM_MINIMAL_BOOTSTRAP_REPOS=${WORKSPACE}/dib-mirror/fedora-minimal/yum.repos.d
|
||||
elif [[ "${DISTRO_NAME}" == "centos" ]]; then
|
||||
export DIB_YUM_MINIMAL_BOOTSTRAP_REPOS=${WORKSPACE}/dib-mirror/centos-minimal/yum.repos.d
|
||||
fi
|
||||
|
||||
fi
|
||||
|
||||
@ -1 +1,2 @@
|
||||
fedora-minimal
|
||||
openstack-ci-mirrors
|
||||
|
||||
@ -12,3 +12,9 @@ Ubuntu and Debian. Nothing additional is needed on Fedora or CentOS.
|
||||
If you wish to have DHCP networking setup for eth0 & eth1 via
|
||||
/etc/sysconfig/network-config scripts/ifcfg-eth[0|1], set the
|
||||
environment variable `DIB_YUM_MINIMAL_CREATE_INTERFACES` to `1`.
|
||||
|
||||
If you wish to build from specific mirrors, set
|
||||
``DIB_YUM_MINIMAL_BOOTSTRAP_REPOS`` to a directory with the ``.repo``
|
||||
files to use during bootstrap and build. The repo files should be
|
||||
named with a prefix ``dib-mirror-`` and will be removed from the final
|
||||
image.
|
||||
|
||||
35
diskimage_builder/elements/yum-minimal/cleanup.d/95-remove-yum-mirror
Executable file
35
diskimage_builder/elements/yum-minimal/cleanup.d/95-remove-yum-mirror
Executable file
@ -0,0 +1,35 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Copyright 2015 Hewlett-Packard Development Company, L.P.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
#
|
||||
|
||||
# dib-lint: disable=safe_sudo
|
||||
|
||||
if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then
|
||||
set -x
|
||||
fi
|
||||
set -eu
|
||||
set -o pipefail
|
||||
|
||||
# This removes any build-time repos that might be set in 08-yum-chroot
|
||||
if [[ -n ${DIB_YUM_MINIMAL_BOOTSTRAP_REPOS:-} ]]; then
|
||||
|
||||
sudo rm -f $TMP_MOUNT_PATH/etc/yum.repos.d/dib-mirror*.repo
|
||||
|
||||
for repo in $TMP_MOUNT_PATH/etc/yum.repos.d/*.USING_MIRROR; do
|
||||
sudo mv $repo ${repo/.USING_MIRROR/}
|
||||
done
|
||||
|
||||
fi
|
||||
@ -78,15 +78,23 @@ function _install_repos {
|
||||
packages+="${DISTRO_NAME}-repos "
|
||||
fi
|
||||
|
||||
# By default, parent elements (fedora-minimal, centos-minimal)
|
||||
# have a yum.repos.d directory in the element with a default repo;
|
||||
# this is copied to TMP_HOOK_PATH by the usual hook-copying
|
||||
# routines. In the gate, environment.d files for the funtional
|
||||
# tests will set DIB_YUM_MINIMAL_BOOTSTRAP_REPOS -- this contains
|
||||
# mirrors correct for the region setup by contrib/setup-gate-mirrors.sh
|
||||
local repo=${DIB_YUM_MINIMAL_BOOTSTRAP_REPOS:-${TMP_HOOKS_PATH}/yum.repos.d}
|
||||
|
||||
# yumdownloader puts repo xml files and such into a directory
|
||||
# ${TMPDIR}/yum-$USER-random. Since we don't need this once the
|
||||
# initial download happens, redirect TMPDIR for this call so we
|
||||
# can clean it up nicely
|
||||
local temp_tmp
|
||||
temp_tmp=$(mktemp -d)
|
||||
TMPDIR=${temp_tmp} yumdownloader \
|
||||
TMPDIR=${temp_tmp} yumdownloader --verbose \
|
||||
--releasever=$DIB_RELEASE \
|
||||
--setopt=reposdir=$TMP_HOOKS_PATH/yum.repos.d \
|
||||
--setopt=reposdir=$repo \
|
||||
--destdir=$WORKING \
|
||||
${packages} && rc=$? || rc=$?
|
||||
rm -rf ${temp_tmp}
|
||||
@ -97,6 +105,18 @@ function _install_repos {
|
||||
# --nodeps works around these wanting /bin/sh in some fedora
|
||||
# releases, see rhbz#1265873
|
||||
sudo $_RPM --root $TARGET_ROOT --nodeps -ivh $WORKING/*rpm
|
||||
|
||||
# install the bootstrap mirror repos over the default ones, if
|
||||
# set. we will remove this at the end so the final image has
|
||||
# regular mirrors
|
||||
if [[ -n ${DIB_YUM_MINIMAL_BOOTSTRAP_REPOS:-} ]]; then
|
||||
for repo in $TARGET_ROOT/etc/yum.repos.d/*.repo; do
|
||||
sudo mv $repo $repo.USING_MIRROR
|
||||
done
|
||||
sudo cp ${DIB_YUM_MINIMAL_BOOTSTRAP_REPOS}/* \
|
||||
$TARGET_ROOT/etc/yum.repos.d/
|
||||
fi
|
||||
|
||||
}
|
||||
|
||||
# _install_pkg_manager packages...
|
||||
|
||||
@ -11,6 +11,10 @@ export DIB_ELEMENTS=$(python -c '
|
||||
import diskimage_builder.paths
|
||||
diskimage_builder.paths.show_path("elements")')
|
||||
|
||||
# XXX: This will move into the gate scripts, and happen before
|
||||
# we run this script ... just here for initial bringup
|
||||
./openstack/diskimage-builder/contrib/setup-gate-mirrors.sh
|
||||
|
||||
# Setup sane locale defaults, because this information is leaked into DIB.
|
||||
export LANG=en_US.utf8
|
||||
export LC_ALL=
|
||||
@ -42,10 +46,6 @@ DEFAULT_SKIP_TESTS=(
|
||||
debian-minimal/stable-build-succeeds
|
||||
debian-minimal/stable-vm
|
||||
debian/build-succeeds
|
||||
# temporary disable until change to use mirrors
|
||||
pip-and-virtualenv/source-install-fedora
|
||||
ironic-agent/build-succeeds-fedora
|
||||
fedora-minimal/build-succeeds
|
||||
)
|
||||
|
||||
# The default output formats (specified to disk-image-create's "-t"
|
||||
|
||||
Loading…
Reference in New Issue
Block a user