From f5dff9c52a1d1960fd29ef2d18dfe7344d96f3b9 Mon Sep 17 00:00:00 2001 From: Ian Wienand Date: Tue, 31 May 2016 04:53:05 +0000 Subject: [PATCH] Rework yum-minimal locale cleanup It turns out our manual locale cleanup is causing issues (see I54490b17a7f8b2f977369044fcc6bb49cc13768e). Upon further investigation, I think this is a better approach than manually deleting repos. glibc on Fedora obeys the %_install_langs macro for reducing the installed locales (as mentioned in the comments, F24 has moved to having different packages, but worry about that later). So our existing clear-out is really only required for CentOS, whose glibc does not have any way to indicate to build less locales. However, %_install_langs is still correct there, as it restricts some of the translation files and other things installed with the %lang macro in spec files. This is complicated by us having to set this at glibc-common install time, which happens with the "yum" from outside the chroot (i.e. on trusty). Since this is too old to have flags to pass this, we need to fiddle with rpmmacros. I've tested this with fedora-minimal builds and the locales file is about 2MiB, which is what it was after the cleanups, and the listed locales are only those we expect (i.e. it appears to be working). Change-Id: I528a68beeb7b2ceec25ccbec1900670501608158 --- .../yum-minimal/pre-install.d/03-yum-cleanup | 98 +++++++++---------- elements/yum-minimal/root.d/08-yum-chroot | 30 +++++- 2 files changed, 77 insertions(+), 51 deletions(-) diff --git a/elements/yum-minimal/pre-install.d/03-yum-cleanup b/elements/yum-minimal/pre-install.d/03-yum-cleanup index fb5da262..29154c72 100755 --- a/elements/yum-minimal/pre-install.d/03-yum-cleanup +++ b/elements/yum-minimal/pre-install.d/03-yum-cleanup @@ -23,58 +23,56 @@ set -o pipefail # effectively: febootstrap-minimize --keep-zoneinfo --keep-rpmdb --keep-services "$target" -# Fedora 24 has a much better way to handle just installing some -# languages; see bug. We should support that at the right time. -if [[ $DISTRO_NAME == "fedora" && $DIB_RELEASE -gt 23 ]]; then - echo "Locale support for Fedora 24 is incomplete" - echo " see: https://bugs.launchpad.net/diskimage-builder/+bug/1571488" - die "Cannot cleanup locales on > Fedora 23" +# This is only required on CentOS ... see notes in +# root.d/08-yum-chroot about %_install_langs +if [[ $DISTRO_NAME != "fedora" ]]; then + + # Stripping *all* locales is a bad idea. For now, we take the + # suggestion in [1] for reducing this + # [1] https://bugzilla.redhat.com/show_bug.cgi?id=156477 + + if [ ! -f /usr/lib/locale/locale-archive ]; then + die "locale-archive not found? Can not do cleanup." + fi + + # now the archive has everything in it, and is about 100MiB. Strip it + # to just en_US (basically, this is the locale we support if you ssh + # in, other than POSIX) + localedef --delete-from-archive \ + $(localedef --list-archive | grep -v '^en_US' | xargs) + + # This removes the locales from the archive index but doesn't rebuild + # the file, so it is still the same size (maybe it is sparse? + # presumably as it's mmapped you don't want to fiddle with the offsets + # of locales in the archive on a live system. We are not live). + + # build-locale-archive is a tool provided by the RH packaging of + # glibc. Documentation is scarce, but it takes the pre-built locales + # in the tmpl file and creates an archive. It seems originally the + # installer would set some flags to rpm to tell the package what + # languages to pick out of the template, but along the way, this was + # reverted to install them (size considered less important than locale + # support). + + # We hack this by moving the locale-archive we've removed the extra + # locales from to the template, then re-run the build. + mv /usr/lib/locale/locale-archive /usr/lib/locale/locale-archive.tmpl + + # rebuild archive from template + /usr/sbin/build-locale-archive + + # leave empty template behind as package does. I think this stops + # upgrades redoing things + echo > /usr/lib/locale/locale-archive.tmpl + + # remove the unnecessary source locale files and iconv files + pushd /usr/share/locale + find . ! -name 'locale.alias' -delete + popd + rm -rf {lib.lib64}/gconv + fi -# Stripping *all* locales is a bad idea. For now, we take the -# suggestion in [1] for reducing this -# [1] https://bugzilla.redhat.com/show_bug.cgi?id=156477 - -if [ ! -f /usr/lib/locale/locale-archive ]; then - die "locale-archive not found? Can not do cleanup." -fi - -# now the archive has everything in it, and is about 100MiB. Strip it -# to just en_US (basically, this is the locale we support if you ssh -# in, other than POSIX) -localedef --delete-from-archive \ - $(localedef --list-archive | grep -v '^en_US' | xargs) - -# This removes the locales from the archive index but doesn't rebuild -# the file, so it is still the same size (maybe it is sparse? -# presumably as it's mmapped you don't want to fiddle with the offsets -# of locales in the archive on a live system. We are not live). - -# build-locale-archive is a tool provided by the RH packaging of -# glibc. Documentation is scarce, but it takes the pre-built locales -# in the tmpl file and creates an archive. It seems originally the -# installer would set some flags to rpm to tell the package what -# languages to pick out of the template, but along the way, this was -# reverted to install them (size considered less important than locale -# support). - -# We hack this by moving the locale-archive we've removed the extra -# locales from to the template, then re-run the build. -mv /usr/lib/locale/locale-archive /usr/lib/locale/locale-archive.tmpl - -# rebuild archive from template -/usr/sbin/build-locale-archive - -# leave empty template behind as package does. I think this stops -# upgrades redoing things -echo > /usr/lib/locale/locale-archive.tmpl - -# remove the unnecessary source locale files and iconv files -pushd /usr/share/locale -find . ! -name 'locale.alias' -delete -popd -rm -rf {lib.lib64}/gconv - # docs rm -rf /usr/share/{doc,info,gnome/help} # cracklib diff --git a/elements/yum-minimal/root.d/08-yum-chroot b/elements/yum-minimal/root.d/08-yum-chroot index 104f8cdb..c3ef5183 100755 --- a/elements/yum-minimal/root.d/08-yum-chroot +++ b/elements/yum-minimal/root.d/08-yum-chroot @@ -128,6 +128,29 @@ function _install_pkg_manager { flock -w 1200 9 || die "Can not lock .rpmmacros" echo "%_dbpath /var/lib/rpm" >> $HOME/.rpmmacros + # Fedora 24 has a much better way to handle just installing some + # languages; see bug. We should support that at the right time. + if [[ $DISTRO_NAME == "fedora" && $DIB_RELEASE -gt 23 ]]; then + echo "Locale support for Fedora 24 is incomplete" + echo " see: https://bugs.launchpad.net/diskimage-builder/+bug/1571488" + die "Cannot cleanup locales on > Fedora 23" + fi + + # _install_langs is a rpm macro that limits the translation + # files, etc installed by packages. For Fedora 23 [1], the + # glibc-common package will obey this to only install the + # listed locales, keeping things much smaller (we still have + # to clean up locales manually on centos7). We install just + # en_US because people often ssh in with that locale, but + # leave out everything else. Note that yum has an option to + # set this from the command-line [2], but the yum in trusty we + # are using is too old to have it. So we set it directly in + # the macros file + # + # [1] http://pkgs.fedoraproject.org/cgit/rpms/glibc.git/commit/glibc.spec?h=f23&id=91764bd9ec690d4b8a886c0a3a104aac12d340d2 + # [2] http://yum.baseurl.org/gitweb?p=yum.git;a=commit;h=26128173b362474456e8f0642073ecb0322ed031 + echo "%_install_langs C:en_US:en_US.UTF-8" >> $HOME/.rpmmacros + sudo -E yum -y \ --setopt=cachedir=$YUM_CACHE/$ARCH/$DIB_RELEASE \ --setopt=reposdir=$TARGET_ROOT/etc/yum.repos.d \ @@ -137,7 +160,8 @@ function _install_pkg_manager { # We modified the base system - make sure we clean up always! rm $HOME/.rpmmacros.dib.lock - sed -i '$ d' $HOME/.rpmmacros + # sed makes it easy to remove last line, but not last n lines... + sed -i '$ d' $HOME/.rpmmacros; sed -i '$ d' $HOME/.rpmmacros; if [ $rc != 0 ]; then die "Initial yum install to chroot failed! Can not continue." fi @@ -193,6 +217,10 @@ else sudo -E chroot $TARGET_ROOT rpm --rebuilddb sudo -E chroot $TARGET_ROOT ${YUM} clean all + # populate the lang reduction macro in the chroot + echo "%_install_langs C:en_US:en_US.UTF-8" | \ + sudo tee -a $TARGET_ROOT/etc/rpm/macros.langs > /dev/null + # bootstrap the environment within the chroot; bring in new # metadata with an update and install some base packages we need. sudo -E chroot $TARGET_ROOT ${YUM} -y update