From 9b4d2a22e4d1ee3d85bf582035d966cda256d4a4 Mon Sep 17 00:00:00 2001 From: "d.marlin" Date: Wed, 16 Nov 2016 21:47:26 -0500 Subject: [PATCH 1/6] Fedora AArch64 (64-bit ARM) support in diskimage-builder Add some checks for AArch64 to avoid the "Unknown architecture" or "architecture not supported" messages, and allow builds to complete. Change-Id: I89ba609abaeeb7019eb317cf13473929b2065230 --- .../extra-data.d/01-copy-binary | 2 +- elements/rpm-distro/pre-install.d/01-override-yum-arch | 3 +++ lib/common-defaults | 2 ++ 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/elements/architecture-emulation-binaries/extra-data.d/01-copy-binary b/elements/architecture-emulation-binaries/extra-data.d/01-copy-binary index 01409906..ee970b01 100755 --- a/elements/architecture-emulation-binaries/extra-data.d/01-copy-binary +++ b/elements/architecture-emulation-binaries/extra-data.d/01-copy-binary @@ -53,7 +53,7 @@ case "$ARCH" in qemu_binary_file="/usr/bin/qemu-arm-static" copy_binary $qemu_binary_file $ARCH ;; - "arm64") + "arm64" | "aarch64") qemu_binary_file="/usr/bin/qemu-aarch64-static" copy_binary $qemu_binary_file $ARCH ;; diff --git a/elements/rpm-distro/pre-install.d/01-override-yum-arch b/elements/rpm-distro/pre-install.d/01-override-yum-arch index 138bf313..1e0aaf50 100755 --- a/elements/rpm-distro/pre-install.d/01-override-yum-arch +++ b/elements/rpm-distro/pre-install.d/01-override-yum-arch @@ -18,6 +18,9 @@ elif [[ "$ARCH" = "ppc64" ]]; then elif [[ "$ARCH" = "ppc64el" ]]; then basearch=ppc64el arch=ppc64el +elif [[ "$ARCH" = "aarch64" ]]; then + basearch=aarch64 + arch=aarch64 else echo "********************" echo "Unknown arch '$ARCH'" diff --git a/lib/common-defaults b/lib/common-defaults index e0471855..428f58fb 100644 --- a/lib/common-defaults +++ b/lib/common-defaults @@ -26,6 +26,8 @@ else "armv"*) _ARCH="armhf" ;; + "aarch64") + ;; *) echo "WARNING: Unknown architecture: $_ARCH" ;; From e0c346d479f86a4543c349fef22e3be9e6207b02 Mon Sep 17 00:00:00 2001 From: Ian Wienand Date: Fri, 18 Nov 2016 14:49:21 +1100 Subject: [PATCH 2/6] Turn off tracing around pid/chroot check In the error case, we get a spew of output as this check goes though every pid checking if its in the chroot. Disable tracing around the call. Change-Id: Ie84f12974755c0c2c51d7e7697337ed9b32a4a1c --- lib/common-functions | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/common-functions b/lib/common-functions index 92c42f88..6762139f 100644 --- a/lib/common-functions +++ b/lib/common-functions @@ -135,6 +135,10 @@ function eval_run_d () { } function kill_chroot_processes () { + local xtrace + xtrace=$(set +o | grep xtrace) + set +o xtrace + if [ -z "${1}" ]; then echo "ERROR: no chroot directory specified" exit 1 @@ -149,6 +153,8 @@ function kill_chroot_processes () { sudo kill $pid fi done + + $xtrace } function cleanup_build_dir () { From c5ec1348c39ffdf520d36616a4354d4e13b3f4e0 Mon Sep 17 00:00:00 2001 From: Clark Boylan Date: Fri, 18 Nov 2016 16:37:59 -0800 Subject: [PATCH 3/6] Fix runtime ssh host keys script The script is set -e and set -o pipefail, unfortauntely this intersects with `yes n`'s non zero exit code behavior when it receives an interrupt like sigpipe. As a result stop setting pipefail so that we treat those errors as "normal" and only fail if ssh-keygen fails. Change-Id: I5447df97c9888cae3007e235e2fea44df61af28e --- .../static/usr/local/sbin/runtime-ssh-host-keys.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/elements/runtime-ssh-host-keys/static/usr/local/sbin/runtime-ssh-host-keys.sh b/elements/runtime-ssh-host-keys/static/usr/local/sbin/runtime-ssh-host-keys.sh index 4fa2374d..60757d3e 100755 --- a/elements/runtime-ssh-host-keys/static/usr/local/sbin/runtime-ssh-host-keys.sh +++ b/elements/runtime-ssh-host-keys/static/usr/local/sbin/runtime-ssh-host-keys.sh @@ -12,10 +12,9 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. -# dib-lint: disable=dibdebugtrace +# dib-lint: disable=dibdebugtrace setpipefail set -exu -set -o pipefail # We are running into race conditions with glean, which ssh-keygen -A is # not handling properly. So, create a new script to first check if the @@ -23,7 +22,7 @@ set -o pipefail for key in dsa ecdsa ed25519 rsa; do FILE=/etc/ssh/ssh_host_${key}_key - if ! [ -e $FILE ]; then + if ! [ -f $FILE ]; then /usr/bin/yes n | /usr/bin/ssh-keygen -f $FILE -N '' -t $key fi done From bbcc22751f689fb1002a85e641a854006280ad66 Mon Sep 17 00:00:00 2001 From: Markos Chandras Date: Tue, 18 Oct 2016 23:21:07 +0100 Subject: [PATCH 4/6] elements: Add new openssh-server element Add new 'openssh-server' element to ensure that openssh server is installed and enabled during boot. This is mostly useful for *-minimal images which do not come with openssh installed and/or enabled in order to keep a small dependency footprint. Change-Id: Ide15ee04f5de123dbc8ce4bb56d638d8a167c341 --- elements/debian/element-deps | 1 + elements/debian/package-installs.yaml | 1 - elements/dynamic-login/element-deps | 2 +- elements/local-config/element-deps | 2 +- elements/local-config/package-installs.yaml | 1 - elements/openssh-server/README.rst | 14 +++++++++ elements/openssh-server/element-deps | 3 ++ .../package-installs.yaml | 0 elements/openssh-server/pkg-map | 10 +++++++ .../post-install.d/80-enable-sshd-service | 29 +++++++++++++++++++ .../openssh-server-0f6d065748a2fc18.yaml | 4 +++ 11 files changed, 63 insertions(+), 4 deletions(-) delete mode 100644 elements/local-config/package-installs.yaml create mode 100644 elements/openssh-server/README.rst create mode 100644 elements/openssh-server/element-deps rename elements/{dynamic-login => openssh-server}/package-installs.yaml (100%) create mode 100644 elements/openssh-server/pkg-map create mode 100755 elements/openssh-server/post-install.d/80-enable-sshd-service create mode 100644 releasenotes/notes/openssh-server-0f6d065748a2fc18.yaml diff --git a/elements/debian/element-deps b/elements/debian/element-deps index 0f508629..1c0ebe24 100644 --- a/elements/debian/element-deps +++ b/elements/debian/element-deps @@ -1 +1,2 @@ debian-minimal +openssh-server diff --git a/elements/debian/package-installs.yaml b/elements/debian/package-installs.yaml index 862c274f..58880aca 100644 --- a/elements/debian/package-installs.yaml +++ b/elements/debian/package-installs.yaml @@ -1,4 +1,3 @@ -openssh-server: file: less: kbd: diff --git a/elements/dynamic-login/element-deps b/elements/dynamic-login/element-deps index 74451ffb..34c57c76 100644 --- a/elements/dynamic-login/element-deps +++ b/elements/dynamic-login/element-deps @@ -1,3 +1,3 @@ dib-init-system install-static -package-installs +openssh-server diff --git a/elements/local-config/element-deps b/elements/local-config/element-deps index 7076aba9..dd21b5aa 100644 --- a/elements/local-config/element-deps +++ b/elements/local-config/element-deps @@ -1 +1 @@ -package-installs +openssh-server diff --git a/elements/local-config/package-installs.yaml b/elements/local-config/package-installs.yaml deleted file mode 100644 index c342a0be..00000000 --- a/elements/local-config/package-installs.yaml +++ /dev/null @@ -1 +0,0 @@ -openssh-server: diff --git a/elements/openssh-server/README.rst b/elements/openssh-server/README.rst new file mode 100644 index 00000000..7190deab --- /dev/null +++ b/elements/openssh-server/README.rst @@ -0,0 +1,14 @@ +============== +openssh-server +============== +This element ensures that openssh server is installed and enabled during boot. + + +Note +---- +Most cloud images come with the openssh server service installed and enabled +during boot. However, certain cloud images, especially those created by the +\*-minimal elements may not have it installed or enabled. In these cases, +using this element may be helpful to ensure your image will accessible via SSH. +It's usually helpful to combine this element with others such as the +`runtime-ssh-host-keys`. diff --git a/elements/openssh-server/element-deps b/elements/openssh-server/element-deps new file mode 100644 index 00000000..fe8fa3f8 --- /dev/null +++ b/elements/openssh-server/element-deps @@ -0,0 +1,3 @@ +dib-init-system +package-installs +runtime-ssh-host-keys diff --git a/elements/dynamic-login/package-installs.yaml b/elements/openssh-server/package-installs.yaml similarity index 100% rename from elements/dynamic-login/package-installs.yaml rename to elements/openssh-server/package-installs.yaml diff --git a/elements/openssh-server/pkg-map b/elements/openssh-server/pkg-map new file mode 100644 index 00000000..9b039341 --- /dev/null +++ b/elements/openssh-server/pkg-map @@ -0,0 +1,10 @@ +{ + "family": { + "suse": { + "openssh-server": "openssh" + }, + "gentoo": { + "openssh-server": "openssh" + } + } +} diff --git a/elements/openssh-server/post-install.d/80-enable-sshd-service b/elements/openssh-server/post-install.d/80-enable-sshd-service new file mode 100755 index 00000000..50238bbc --- /dev/null +++ b/elements/openssh-server/post-install.d/80-enable-sshd-service @@ -0,0 +1,29 @@ +#!/bin/bash + +if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then + set -x +fi +set -eu +set -o pipefail + +case "$DIB_INIT_SYSTEM" in + upstart) + # nothing to do + exit 0 + ;; + systemd) + if [[ $DISTRO_NAME = "ubuntu" || $DISTRO_NAME = "debian" ]]; then + systemctl enable ssh.service + else + systemctl enable sshd.service + fi + ;; + openrc) + # let dib-init-system's postinstall handle enabling init scripts + exit 0 + ;; + *) + echo "Unsupported init system" + exit 1 + ;; +esac diff --git a/releasenotes/notes/openssh-server-0f6d065748a2fc18.yaml b/releasenotes/notes/openssh-server-0f6d065748a2fc18.yaml new file mode 100644 index 00000000..985475b8 --- /dev/null +++ b/releasenotes/notes/openssh-server-0f6d065748a2fc18.yaml @@ -0,0 +1,4 @@ +--- +features: + - New openssh-server element to ensure that the openssh + server is installed and enabled during boot. From f15550f9fe2db8a9fe86fb89f28039aa5c46323b Mon Sep 17 00:00:00 2001 From: Ian Wienand Date: Wed, 23 Nov 2016 19:44:50 +1100 Subject: [PATCH 5/6] Special case dib-python in dib-lint It seems that on Xenial, it does not take much to confuse "file" and it's mime guessing such that it thinks some files are not python. "package-installs-v2" is a good example, since it has an interpreter "dib-python" that "file" doesn't know about, and no extension. While looking at this, I've added emacs vars here so it opens in python mode. Change-Id: I01994b08c5ad8987925f1eec4062f5b6ee72eb8f --- bin/dib-lint | 9 +++++++-- elements/package-installs/bin/package-installs-v2 | 5 +++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/bin/dib-lint b/bin/dib-lint index c58865e5..a8c10d4d 100755 --- a/bin/dib-lint +++ b/bin/dib-lint @@ -83,10 +83,15 @@ for i in $(find elements -type f \ error "$i is not executable" fi - # Ensure 4 spaces indent are used - if [[ "$(file -b -k --mime-type $i)" =~ "text/x-python" ]]; then + # run flake8 over python files. note our "dib-python" + # interpreter can confuse the magic matching being done in + # "file" and make it think the file is not python; + # special-case it. + if [[ "$(file -b -k --mime-type $i)" =~ "text/x-python" ]] || \ + [[ $firstline =~ "dib-python" ]]; then flake8 $i || error "$i failed flake8" else + # Ensure 4 spaces indent are used if ! excluded indent ; then indent_regex='^\( \{4\}\)* \{1,3\}[^ ]' if grep -q "$indent_regex" ${i}; then diff --git a/elements/package-installs/bin/package-installs-v2 b/elements/package-installs/bin/package-installs-v2 index f1814dd7..b4a4d0d8 100755 --- a/elements/package-installs/bin/package-installs-v2 +++ b/elements/package-installs/bin/package-installs-v2 @@ -101,3 +101,8 @@ def main(): if __name__ == '__main__': main() + +# Tell emacs to use python-mode +# Local variables: +# mode: python +# End: From 08d6a9f93dcfeb25086f15255a1b4661ddac208d Mon Sep 17 00:00:00 2001 From: Ian Wienand Date: Thu, 24 Nov 2016 09:52:28 +1100 Subject: [PATCH 6/6] yum-minimal: add systemd to initial install It seems in the grub cleanup in Iafe3611f4eec3c6357587a6cae6a30a261686ead I managed to unintentionally drop systemd from the yum-minimal builds. By not pre-installing grub we dropped some dependencies; the path is tortured ... grub2 -> os-prober -> udev -> systemd-udev -> systemd (we don't even want os-prober! So this whole thing was working by accident). This manifests in *very* confusing ways. Currently centos-minimal builds are failing late in the build with services unable to enabled. dib-init-system was actually trying to tell us that it didn't know what init was installed (because systemd wasn't actually installed), but unfortunately it was not really failing. This meant the service files were not copied correctly from other elements, and thus fail to be enabled. I have corrected this with I076c08190d40c315ad6a6d96a3823e9fc52630be which would at least alert us earlier. For Fedora 24, due to a bug in dracut dependencies [1], missing the systemd-udev package fails the build of the initrd during the kernel install. This then results in an initrd-less, unbootable system (see also Ibaaa81124098f3c6febe48e455d3e1cd0a5f1761). Add these dependencies explicitly. [1] https://bugzilla.redhat.com/show_bug.cgi?id=1398505 Change-Id: I24ce648485c3d6f3c27ab8f87a638516b3727017 --- elements/yum-minimal/root.d/08-yum-chroot | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/elements/yum-minimal/root.d/08-yum-chroot b/elements/yum-minimal/root.d/08-yum-chroot index 10fe1d25..7760897b 100755 --- a/elements/yum-minimal/root.d/08-yum-chroot +++ b/elements/yum-minimal/root.d/08-yum-chroot @@ -242,7 +242,17 @@ else sudo -E chroot $TARGET_ROOT ${YUM} -y update sudo -E chroot $TARGET_ROOT ${YUM} -y \ --setopt=cachedir=/tmp/yum/$ARCH/$DIB_RELEASE \ - install passwd findutils sudo util-linux-ng + install systemd passwd findutils sudo util-linux-ng + + # This package is split out from systemd on >F24, dracut is + # missing the dependency and will fail to make an initrd without + # it; see + # https://bugzilla.redhat.com/show_bug.cgi?id=1398505 + if [ $DISTRO_NAME = "fedora" -a $DIB_RELEASE -ge 24 ]; then + sudo -E chroot $TARGET_ROOT ${YUM} -y \ + --setopt=cachedir=/tmp/yum/$ARCH/$DIB_RELEASE \ + install systemd-udev + fi # Put in a dummy /etc/resolv.conf over the temporary one we used # to bootstrap. systemd has a bug/feature [1] that it will assume