Start at using CI mirrors for fedora/centos

fedora/centos-minimal don't obey DIB_DISTRIBUTION_MIRROR currently.  I
don't really want them too -- we want to be able to separate the
mirrors used during the build process from those embedded into the
final image.  Add DIB_YUM_MINIMAL_BOOTSTRAP_REPOS which is a directory
with repo files to use during the install.

This introduces setup-gate-mirrors.sh which is intended to setup
repo/sources/whatever files in the openstack gate that point to the
local region mirror.  It pulls the info from the mirror_info.sh script
on each CI node.

The openstack-ci-mirrors element is updated to export these variables.
elements are updated to depend on it.  Tests are restored

Change-Id: I7604fc4d41cb1483be16b8d628a24e8fc764f515

contrib/setup-gate-mirrors.sh

@@ -0,0 +1,81 @@
+#!/bin/bash
+
+set -x
+
+#
+# This tool creates repo/sources files that point to the mirrors for
+# the host region in the OpenStack CI gate.
+#
+
+# This pre-created on CI nodes by slave scripts
+source /etc/ci/mirror_info.sh
+
+# Tests should probe for this directory and then use the repos/sources
+# files inside it for the gate tests.
+BASE_DIR=$WORKSPACE/dib-mirror
+
+mkdir -p $BASE_DIR
+
+## REPOS
+
+# all should start with "dib-mirror-"
+# gpg check turned off, because we don't have the keys outside the chroot
+
+# fedora-minimal
+FEDORA_MIN_DIR=$BASE_DIR/fedora-minimal/yum.repos.d
+mkdir -p $FEDORA_MIN_DIR
+
+cat <<EOF > $FEDORA_MIN_DIR/dib-mirror-fedora.repo
+[fedora]
+name=Fedora \$releasever - \$basearch
+failovermethod=priority
+baseurl=$NODEPOOL_FEDORA_MIRROR/releases/\$releasever/Everything/\$basearch/os/
+enabled=1
+metadata_expire=7d
+gpgcheck=0
+skip_if_unavailable=False
+deltarpm=False
+deltarpm_percentage=0
+EOF
+
+cat <<EOF > $FEDORA_MIN_DIR/dib-mirror-fedora-updates.repo
+[updates]
+name=Fedora \$releasever - \$basearch - Updates
+failovermethod=priority
+baseurl=$NODEPOOL_FEDORA_MIRROR/updates/\$releasever/\$basearch/
+enabled=1
+gpgcheck=0
+metadata_expire=6h
+skip_if_unavailable=False
+deltarpm=False
+deltarpm_percentage=0
+EOF
+
+# Centos Minimal
+CENTOS_MIN_DIR=$BASE_DIR/centos-minimal/yum.repos.d
+mkdir -p $CENTOS_MIN_DIR
+
+cat <<EOF > $CENTOS_MIN_DIR/dib-mirror-base.repo
+[base]
+name=CentOS-\$releasever - Base
+baseurl=$NODEPOOL_CENTOS_MIRROR/\$releasever/os/\$basearch/
+gpgcheck=0
+EOF
+
+cat <<EOF > $CENTOS_MIN_DIR/dib-mirror-updates.repo
+#released updates
+[updates]
+name=CentOS-\$releasever - Updates
+baseurl=$NODEPOOL_CENTOS_MIRROR/\$releasever/updates/\$basearch/
+gpgcheck=0
+EOF
+
+cat <<EOF > $CENTOS_MIN_DIR/dib-mirror-extras.repo
+#additional packages that may be useful
+[extras]
+name=CentOS-\$releasever - Extras
+baseurl=$NODEPOOL_CENTOS_MIRROR/\$releasever/extras/\$basearch/
+gpgcheck=0
+EOF
+
+## apt sources (todo)

diskimage_builder/elements/centos-minimal/test-elements/build-succeeds/element-deps

@@ -0,0 +1 @@
+openstack-ci-mirrors

diskimage_builder/elements/centos7/test-elements/build-succeeds/element-deps

@@ -1 +1,2 @@
 base
+openstack-ci-mirrors

diskimage_builder/elements/fedora-minimal/test-elements/build-succeeds/element-deps

@@ -0,0 +1 @@
+openstack-ci-mirrors

diskimage_builder/elements/fedora-minimal/test-elements/build-succeeds/environment.d/10-fedora-pin-mirror.bash

@@ -1,2 +0,0 @@
-# Pin to this mirror because the roundrobin is fairly unreliable
-export DIB_DISTRIBUTION_MIRROR=http://dl.fedoraproject.org/pub/fedora/linux

diskimage_builder/elements/fedora/test-elements/build-succeeds/element-deps

@@ -1 +1,2 @@
 base
+openstack-ci-mirrors

diskimage_builder/elements/fedora/test-elements/build-succeeds/environment.d/10-fedora-pin-mirror.bash

@@ -1,2 +0,0 @@
-# Pin to this mirror because the roundrobin is fairly unreliable
-export DIB_DISTRIBUTION_MIRROR=http://dl.fedoraproject.org/pub/fedora/linux

diskimage_builder/elements/ironic-agent/test-elements/build-succeeds-fedora/environment.d/10-fedora-pin-mirror.bash

@@ -1,2 +1,7 @@
-# Pin to this mirror because the roundrobin is fairly unreliable
+# Set DIB_DISTRIBUTION_MIRROR if running in openstack gate
+if [ -f /etc/ci/mirror_info.sh ]; then
+    source /etc/ci/mirror_info.sh
+    export DIB_DISTRIBUTION_MIRROR=$NODEPOOL_FEDORA_MIRROR
+else
     export DIB_DISTRIBUTION_MIRROR=http://dl.fedoraproject.org/pub/fedora/linux
+fi

diskimage_builder/elements/openstack-ci-mirrors/environment.d/11-dib-distribution-mirror.bash

@@ -24,3 +24,15 @@ if [ -f /etc/ci/mirror_info.sh ]; then
     fi
 
 fi
+
+# This is repo files pre-created for the fedora/centos-minimal jobs in
+# the gate
+if [[ -d ${WORKSPACE:-/not/a/path/}/dib-mirror ]]; then
+
+    if [[ "${DISTRO_NAME}" == "fedora" ]]; then
+        export DIB_YUM_MINIMAL_BOOTSTRAP_REPOS=${WORKSPACE}/dib-mirror/fedora-minimal/yum.repos.d
+    elif [[ "${DISTRO_NAME}" == "centos" ]]; then
+        export DIB_YUM_MINIMAL_BOOTSTRAP_REPOS=${WORKSPACE}/dib-mirror/centos-minimal/yum.repos.d
+    fi
+
+fi

diskimage_builder/elements/pip-and-virtualenv/test-elements/source-install-fedora/element-deps

@@ -1 +1,2 @@
 fedora-minimal
+openstack-ci-mirrors

diskimage_builder/elements/yum-minimal/README.rst

@@ -12,3 +12,9 @@ Ubuntu and Debian. Nothing additional is needed on Fedora or CentOS.
 If you wish to have DHCP networking setup for eth0 & eth1 via
 /etc/sysconfig/network-config scripts/ifcfg-eth[0|1], set the
 environment variable `DIB_YUM_MINIMAL_CREATE_INTERFACES` to `1`.
+
+If you wish to build from specific mirrors, set
+``DIB_YUM_MINIMAL_BOOTSTRAP_REPOS`` to a directory with the ``.repo``
+files to use during bootstrap and build.  The repo files should be
+named with a prefix ``dib-mirror-`` and will be removed from the final
+image.

diskimage_builder/elements/yum-minimal/cleanup.d/95-remove-yum-mirror

@@ -0,0 +1,35 @@
+#!/bin/bash
+#
+# Copyright 2015 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+
+# dib-lint: disable=safe_sudo
+
+if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then
+    set -x
+fi
+set -eu
+set -o pipefail
+
+# This removes any build-time repos that might be set in 08-yum-chroot
+if [[ -n ${DIB_YUM_MINIMAL_BOOTSTRAP_REPOS:-} ]]; then
+
+    sudo rm -f $TMP_MOUNT_PATH/etc/yum.repos.d/dib-mirror*.repo
+
+    for repo in $TMP_MOUNT_PATH/etc/yum.repos.d/*.USING_MIRROR; do
+        sudo mv $repo ${repo/.USING_MIRROR/}
+    done
+
+fi

diskimage_builder/elements/yum-minimal/root.d/08-yum-chroot

@@ -78,15 +78,23 @@ function _install_repos {
         packages+="${DISTRO_NAME}-repos "
     fi
 
+    # By default, parent elements (fedora-minimal, centos-minimal)
+    # have a yum.repos.d directory in the element with a default repo;
+    # this is copied to TMP_HOOK_PATH by the usual hook-copying
+    # routines.  In the gate, environment.d files for the funtional
+    # tests will set DIB_YUM_MINIMAL_BOOTSTRAP_REPOS -- this contains
+    # mirrors correct for the region setup by contrib/setup-gate-mirrors.sh
+    local repo=${DIB_YUM_MINIMAL_BOOTSTRAP_REPOS:-${TMP_HOOKS_PATH}/yum.repos.d}
+
     # yumdownloader puts repo xml files and such into a directory
     # ${TMPDIR}/yum-$USER-random.  Since we don't need this once the
     # initial download happens, redirect TMPDIR for this call so we
     # can clean it up nicely
     local temp_tmp
     temp_tmp=$(mktemp -d)
-    TMPDIR=${temp_tmp} yumdownloader \
+    TMPDIR=${temp_tmp} yumdownloader --verbose \
         --releasever=$DIB_RELEASE \
-        --setopt=reposdir=$TMP_HOOKS_PATH/yum.repos.d \
+        --setopt=reposdir=$repo \
         --destdir=$WORKING \
         ${packages} && rc=$? || rc=$?
     rm -rf ${temp_tmp}
@@ -97,6 +105,18 @@ function _install_repos {
     # --nodeps works around these wanting /bin/sh in some fedora
     # releases, see rhbz#1265873
     sudo $_RPM --root $TARGET_ROOT --nodeps -ivh $WORKING/*rpm
+
+    # install the bootstrap mirror repos over the default ones, if
+    # set.  we will remove this at the end so the final image has
+    # regular mirrors
+    if [[ -n ${DIB_YUM_MINIMAL_BOOTSTRAP_REPOS:-} ]]; then
+        for repo in $TARGET_ROOT/etc/yum.repos.d/*.repo; do
+            sudo mv $repo $repo.USING_MIRROR
+        done
+        sudo cp ${DIB_YUM_MINIMAL_BOOTSTRAP_REPOS}/* \
+            $TARGET_ROOT/etc/yum.repos.d/
+    fi
+
 }
 
 # _install_pkg_manager packages...

tests/run_functests.sh

@@ -11,6 +11,10 @@ export DIB_ELEMENTS=$(python -c '
 import diskimage_builder.paths
 diskimage_builder.paths.show_path("elements")')
 
+# XXX: This will move into the gate scripts, and happen before
+# we run this script ... just here for initial bringup
+./openstack/diskimage-builder/contrib/setup-gate-mirrors.sh
+
 # Setup sane locale defaults, because this information is leaked into DIB.
 export LANG=en_US.utf8
 export LC_ALL=
@@ -42,10 +46,6 @@ DEFAULT_SKIP_TESTS=(
     debian-minimal/stable-build-succeeds
     debian-minimal/stable-vm
     debian/build-succeeds
-    # temporary disable until change to use mirrors
-    pip-and-virtualenv/source-install-fedora
-    ironic-agent/build-succeeds-fedora
-    fedora-minimal/build-succeeds
 )
 
 # The default output formats (specified to disk-image-create's "-t"