From f0b70211c68451d386cbed3597c5a9854e1fbd80 Mon Sep 17 00:00:00 2001 From: Ian Wienand Date: Wed, 21 Jun 2017 05:37:06 +1000 Subject: [PATCH] Use local mirror for ubuntu-minimal jobs This adds "openstack-ci-mirrors" element which performs various settings to get builds using local mirrors. As a first step, we convert ubuntu-minimal jobs The main trick is that since infra mirrors are created with rerepo they are not signed (they are recreated, not cloned, and not signing is seen as a feature in that it deters external use). So we need to instruct debootstrap to ignore signing and also turn it off for in-chroot apt. Other than that, the existing DIB_DISTRIBUTION_MIRROR works to redirect installs. Remove "restricted" as it's not mirrored, and I don't think we want it in here by default. (I think DIB_DISTRIBUTION_MIRROR is a bit of an anti-pattern, because it leaves the mirrors in the final image -- just because you use them to build, doesn't mean you want them at runtime). But we don't need to fix that now, and we don't use any created images.) This pauses fedora testing until the next change, which moves to using local mirrors for testing on fedora/centos Change-Id: I778bd05a1e615c27edf1c9f0a1409119a6b3a850 --- .../test-elements/test-sources/element-deps | 1 + .../elements/openstack-ci-mirrors/README.rst | 8 ++++ .../11-dib-distribution-mirror.bash | 26 +++++++++++++ .../root.d/10-apt-disable-gpg | 37 +++++++++++++++++++ .../source-install-ubuntu/element-deps | 1 + .../environment.d/10-ubuntu-distro-name.bash | 2 +- .../trusty-build-succeeds/element-deps | 1 + .../xenial-build-succeeds/element-deps | 3 +- tests/run_functests.sh | 4 ++ 9 files changed, 81 insertions(+), 2 deletions(-) create mode 100644 diskimage_builder/elements/openstack-ci-mirrors/README.rst create mode 100644 diskimage_builder/elements/openstack-ci-mirrors/environment.d/11-dib-distribution-mirror.bash create mode 100755 diskimage_builder/elements/openstack-ci-mirrors/root.d/10-apt-disable-gpg create mode 100644 diskimage_builder/elements/ubuntu-minimal/test-elements/trusty-build-succeeds/element-deps diff --git a/diskimage_builder/elements/apt-sources/test-elements/test-sources/element-deps b/diskimage_builder/elements/apt-sources/test-elements/test-sources/element-deps index dbb1c56b..81da378c 100644 --- a/diskimage_builder/elements/apt-sources/test-elements/test-sources/element-deps +++ b/diskimage_builder/elements/apt-sources/test-elements/test-sources/element-deps @@ -1,2 +1,3 @@ base +openstack-ci-mirrors ubuntu-minimal diff --git a/diskimage_builder/elements/openstack-ci-mirrors/README.rst b/diskimage_builder/elements/openstack-ci-mirrors/README.rst new file mode 100644 index 00000000..339e86cc --- /dev/null +++ b/diskimage_builder/elements/openstack-ci-mirrors/README.rst @@ -0,0 +1,8 @@ +==================== +openstack-ci-mirrors +==================== + +This element contains various settings to setup mirrors for openstack +ci gate testing in a generic fashion. It is intended to be used as a +dependency of testing elements that run in the gate. It should do +nothing outside that environment. diff --git a/diskimage_builder/elements/openstack-ci-mirrors/environment.d/11-dib-distribution-mirror.bash b/diskimage_builder/elements/openstack-ci-mirrors/environment.d/11-dib-distribution-mirror.bash new file mode 100644 index 00000000..03af8220 --- /dev/null +++ b/diskimage_builder/elements/openstack-ci-mirrors/environment.d/11-dib-distribution-mirror.bash @@ -0,0 +1,26 @@ +# Set DIB_DISTRIBUTION_MIRROR if running in openstack gate +if [ -f /etc/ci/mirror_info.sh ]; then + + # don't spam logs with this source + _xtrace=$(set +o | grep xtrace) + set +o xtrace + source /etc/ci/mirror_info.sh + $_xtrace + + # note 11- is after 10- which is where DISTRO_NAME is set usually + + if [[ "${DISTRO_NAME}" == "ubuntu" ]]; then + export DIB_DISTRIBUTION_MIRROR=$NODEPOOL_UBUNTU_MIRROR + export DIB_DEBOOTSTRAP_EXTRA_ARGS+=" --no-check-gpg" + elif [[ "${DISTRO_NAME}" == "debian" ]]; then + export DIB_DISTRIBUTION_MIRROR=$NODEPOOL_DEBIAN_MIRROR + export DIB_DEBOOTSTRAP_EXTRA_ARGS+=" --no-check-gpg" + elif [[ "${DISTRO_NAME}" == "fedora" ]]; then + export DIB_DISTRIBUTION_MIRROR=$NODEPOOL_FEDORA_MIRROR + elif [[ "${DISTRO_NAME}" == "centos" ]]; then + export DIB_DISTRIBUTION_MIRROR=$NODEPOOL_CENTOS_MIRROR + elif [[ "${DISTRO_NAME}" == "centos7" ]]; then + export DIB_DISTRIBUTION_MIRROR=$NODEPOOL_CENTOS_MIRROR + fi + +fi diff --git a/diskimage_builder/elements/openstack-ci-mirrors/root.d/10-apt-disable-gpg b/diskimage_builder/elements/openstack-ci-mirrors/root.d/10-apt-disable-gpg new file mode 100755 index 00000000..95832c67 --- /dev/null +++ b/diskimage_builder/elements/openstack-ci-mirrors/root.d/10-apt-disable-gpg @@ -0,0 +1,37 @@ +#!/bin/bash +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# dib-lint: disable=safe_sudo + +if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then + set -x +fi +set -eu +set -o pipefail + +# Disable gpg signing for apt openstack ci repos are created by reprepro +# and not signed. + +# This has to be after debootstrap, but before anything else tries to +# use the mirror. + +APT_DIR=${TARGET_ROOT}/etc/apt/apt.conf.d + +if [ -d ${APT_DIR} ]; then + FILE=${APT_DIR}/99unauthenticated + sudo dd of=${FILE} <