add -I option to guard/allow insane base-packages<->db sync

3 files changed, 92 insertions, 41 deletions

diff --git a/src/chroot-tools/chcleanup.in b/src/chroot-tools/chcleanup.in
index a76dd67..89a25c4 100644
--- a/src/chroot-tools/chcleanup.in
+++ b/src/chroot-tools/chcleanup.in
@@ -3,6 +3,7 @@ set -eE
 
 # Performs chroot cleanup smartly.
 # Removes all and only non-essential packages, leaving a clean base-devel system.
+# This script is only intended to be executed implicitly by `librechroot`.
 #
 # Copyright (C) 2011-2012 Nicolás Reynolds <fauno@parabola.nu>
 # Copyright (C) 2012-2013, 2015, 2017-2018 Luke Shumaker <lukeshu@parabola.nu>
@@ -85,11 +86,13 @@ fi
 
 msg "Cleaning chroot..."
 
-# Sync the local repo with pacman (a limited form of `pacman -Sy`)
+# Sync the local repo with pacman (a crude form of `pacman -Sy`)
 cp /repo/repo.db /var/lib/pacman/sync/repo.db
 
 # Setup the temporary directory
 TEMPDIR="$(mktemp --tmpdir -d "${0##*/}.XXXXXXXXXX")"
+ERROR_PKGS_FILE="$TEMPDIR/err_pkgs.txt"
+WHITELIST_PKGS_FILE="$TEMPDIR/pkglist.txt"
 trap "rm -rf -- ${TEMPDIR@Q}" EXIT
 
 # Set up a scratch pacman DB
@@ -100,45 +103,79 @@ cp -a -t "${TEMPDIR}/db" -- /var/lib/pacman/sync
 		ln -sfT -- /dev/null "$TEMPDIR/hooks/${hook##*/}"
 	done
 done
-pkglist_cmd=(pacman --dbpath="$TEMPDIR/db" --hookdir="$TEMPDIR/hooks")
+pacman_cmd=(pacman --dbpath="$TEMPDIR/db" --hookdir="$TEMPDIR/hooks")
 
 # Do our best to preload the scratch DB with CHROOTPKG and
-# CHROOTEXTRAPKG packages.  This is purely an optimization step.  The
-# safety of this optimization assumes that none of CHROOTPKG,
-# CHROOTEXTRAPKG, *or their dependancies* are virtual packages.  We
-# don't include DEPENDS in this optimization, because this assumption
-# doesn't hold for them.
-while read -r pkg; do
-	if [[ -d /var/lib/pacman/local/$pkg ]]; then
-		cp -a -T -- "/var/lib/pacman/local/$pkg" "$TEMPDIR/db/local/$pkg"
-	fi
-done < <("${pkglist_cmd[@]}" -Sp --print-format='%n-%v' -- "${CHROOTPKG[@]}" "${CHROOTEXTRAPKG[@]}")
-
-# Get the full list of packages needed by dependencies, including the base system
-msg2 "Creating a full list of packages..."
-for pkglist in CHROOTPKG CHROOTEXTRAPKG DEPENDS; do
-	declare -n pkgsref="$pkglist"
-	if [[ $pkglist = DEPENDS ]]; then
-		mapfile -t pkgs < <("${pkglist_cmd[@]}" -T -- "${pkgsref[@]}")
+# CHROOTEXTRAPKG packages.  This is purely an optimization step.
+# The safety of this optimization assumes that none of CHROOTPKG,
+# CHROOTEXTRAPKG, *or their dependancies* are virtual packages.
+# DEPENDS are not included in this optimization,
+# because this assumption doesn't hold for them.
+mapfile -t fresh_pkgs < <("${pacman_cmd[@]}" -Sp --print-format='%n %v'                 \
+                                             -- "${CHROOTPKG[@]}" "${CHROOTEXTRAPKG[@]}")
+stale_pkgs=()
+for pkg in "${fresh_pkgs[@]}"; do
+	pkg_name_ver=${pkg/ /-}
+	pkg_name=${pkg% *}
+	pkg_dir=/var/lib/pacman/local/$pkg_name_ver
+	if [[ -d "$pkg_dir" ]]; then
+		cp -a -T -- "$pkg_dir" "$TEMPDIR/db/local/$pkg_name_ver"
 	else
-		pkgs=("${pkgsref[@]}")
+		stale_pkgs+=($pkg_name)
 	fi
-	(( ${#pkgs[@]} > 0 )) || continue
+done
 
+# Collect the minimal set of packages, which are necessary,
+# in order to build the PKGBUILD recipe
+# (base+base-devel, user-packages, PKGBUILD dependencies,
+# and the entire resolved dependency chain).
+# This is done by installing those into a temporary pacman DB;
+# then querying the DB for it's complete package list.
+msg2 "Collecting the minimal set of packages needed ..."
+"${pacman_cmd[@]}" -S --dbonly --noscriptlet --needed --noconfirm        \
+                   -- ${CHROOTPKG[*]} ${CHROOTEXTRAPKG[*]} ${DEPENDS[*]} \
+                   <&- >& "$ERROR_PKGS_FILE"
+if (( $? != 0 )); then
+	error "Could not create a full list of packages, exiting."
+	plain "This is likely caused by a dependency that could not be found."
+	sed 's/^/ > /' < "$ERROR_PKGS_FILE" >&2
+	exit 1
+fi
+
+# Generate the list of packages which should be installed
+# Any installed package with an out-dated pkgver; will not be reported as installed
+# in the temp DB; because the 'pkgver' will not match in the initial pre-load stage.
+# This can be the case, if the chroot package database was upadted manually,
+# while enabling the [testing] repo, for example.
+# These package would not be marked as needed in the whitelist collection stage;
+# because they are installed, and the '--needed' option ignores upgrades.
+# Thus, these would not be present on the whitelist, signalling their removal;
+# which for essential dependencies, would result in a fatal conflict.
+# This could be avoided by omitting the '--needed' option;
+# but that could very likely result in a "partially uograded" system.
+# Another option would be to begin this script with forceful complete system upgrade,
+# This situation is as likely to be unintentional as it may be intentional;
+# so it is best to require explicit initiation of a build in this insane configuration
+# (via the libremakepkg '-I' option), and to exit otherwise, warning the user to upgrade.
+# If the '-I' option was detected (!SANE), the stale packages are added to the whitelist.
+"${pacman_cmd[@]}" -Qq > "$WHITELIST_PKGS_FILE"
+if (( ${#stale_pkgs[*]} > 0 )); then
+	insane_msg_1="Some (%d) essential packages are out-of-sync with the database."
+	insane_msg_2="Consider upgrading the chroot system before building this package."
+	if ! $SANE; then
+		warning "$insane_msg_1" "${#stale_pkgs[*]}" ; plain "$insane_msg_2" ;
+		plain "(ignoring, per the '-I' option)" ;
+		printf "%s\n" "${stale_pkgs[@]}" >> "$WHITELIST_PKGS_FILE"
+	else
+		error "$insane_msg_1" "${#stale_pkgs[*]}" ; plain "$insane_msg_2" ;
+		plain "If absolutely necessary, this guard may be bypassed via the '-I' option."
+		exit 1
 	fi
-	"${pkglist_cmd[@]}" -S --dbonly --noscriptlet --needed --noconfirm -- "${pkgs[@]}" <&- >& "$TEMPDIR/pacman.txt" || ret=$?
-	if (( ret != 0 )); then
-		error "Could not create a full list of packages, exiting."
-		plain "This is likely caused by a dependency that could not be found."
-		sed 's/^/ > /' <  "$TEMPDIR/pacman.txt" >&2
-		exit $ret
-	fi
-done
-"${pkglist_cmd[@]}" -Qq > "$TEMPDIR/pkglist.txt"
+fi
 
 # Diff installed packages against a clean chroot then remove leftovers
 packages=($(comm -23 <(pacman -Qq | sort -u) \
-                     <(sort -u "$TEMPDIR/pkglist.txt")))
+                     <(sort -u "$WHITELIST_PKGS_FILE")))
 if [[ ${#packages[@]} = 0 ]]; then
 	msg2 "No packages to remove"
 else
@@ -153,7 +190,7 @@ else
 fi
 
 packages=($(comm -13 <(pacman -Qq | sort -u) \
-                     <(sort -u "$TEMPDIR/pkglist.txt")))
+                     <(sort -u "$WHITELIST_PKGS_FILE")))
 if [[ ${#packages[@]} = 0 ]]; then
 	msg2 "No packages to add"
 else
@@ -165,3 +202,6 @@ else
 		pacman --noconfirm -S "${packages[@]}"
 	fi
 fi
+
+
+${DRYRUN} && warning "exiting per \$DRYRUN" && exit 1 || :
diff --git a/src/chroot-tools/hooks-chcleanup.sh b/src/chroot-tools/hooks-chcleanup.sh
index 154c58f..8ee32c7 100644
--- a/src/chroot-tools/hooks-chcleanup.sh
+++ b/src/chroot-tools/hooks-chcleanup.sh
@@ -23,8 +23,8 @@ clean_chroot() (
 
 	if $INCHROOT; then
 		cd /startdir
-		"$(librelib chroot/chcleanup)"
+		SANE=$SANE "$(librelib chroot/chcleanup)"
 	else
-		librechroot "${librechroot_flags[@]}" clean-pkgs
+		librechroot "${librechroot_flags[@]}" $($SANE || echo '-I') clean-pkgs
 	fi
 )
diff --git a/src/chroot-tools/libremakepkg b/src/chroot-tools/libremakepkg
index 14ec0ff..3718166 100755
--- a/src/chroot-tools/libremakepkg
+++ b/src/chroot-tools/libremakepkg
@@ -36,6 +36,7 @@ umask 0022
 # Global variables:
 readonly _indent="$(librelib chroot/indent)"
 readonly INCHROOT=$([[ -f /.arch-chroot ]] && echo true || echo false)
+SANE=true  # can be changed with the -I flag
 NONET=true # can be changed with the -N flag
 # {PKG,SRC,SRCPKG,LOG}DEST set at runtime by makepkg.conf
 # MAKEFLAGS, PACKAGER set at runtime by makepkg.conf
@@ -134,6 +135,7 @@ build() (
   $NONET || run_nnet=("${run_ynet[@]}")
 
   prepare_chroot "$copydir" "$LIBREHOME" "$repack" false
+
   run_hook pre_build "$copydir"
   trap "run_hook post_build ${copydir@Q}; rm -rf -- ${startdir@Q}" EXIT
   "${run_nnet[@]}" /chrootbuild "${makepkg_args[@]}" </dev/null |& indent
@@ -170,13 +172,21 @@ usage() {
          environment variables for {SRC,SRCPKG,PKG,LOG}DEST, MAKEFLAGS and
          PACKAGER override the settings in makepkg.conf(5).'
   echo
-  prose 'The `-n` and `-l` options behave identically to librechroot (see
-         the librechroot documentation).'
+  prose 'The `-n` and `-l` options are passed to `librechroot`
+        (see the librechroot documentation).'
   echo
-  flag 'Options (librechroot):'                                                   \
-       "-n <$(_ NAME)>"    'Name of the chroot to use'                            \
-       "-l <$(_ COPY)>"    'Name of, or absolute path to, the chroot copy to use' \
-       "-w <${mount_msg}>" 'Bind mount a file or directory, read/write'           \
+  flag 'Options (librechroot):'                                                        \
+       "-n <$(_ NAME)>"    'Name of the chroot to use'                                 \
+       "-l <$(_ COPY)>"    'Name of, or absolute path to, the chroot copy to use'      \
+       '-I'                "Allow packages older than their database entries to remain
+                            installed, after updating the database manually.
+                            This option can lead to an insane system configuaration.
+                            Replacement packages will still be applied, for example,
+                            unless they are helpd back explcitly; and there may be
+                            dependency conflicts or shared library mis-matches.
+                            Use this only in extreme circumstances; such as if you
+                            need to enable [testing] without synchronizing afterward." \
+       "-w <${mount_msg}>" 'Bind mount a file or directory, read/write'                \
        "-r <${mount_msg}>" 'Bind mount a file or directory, read-only'
   flag 'Options (libremakepkg):'                                               \
        '-N'             "Don't disable networking during prepare(),
@@ -204,11 +214,12 @@ main() {
   local srcpkg=''
 
   # Parse command line options ###########################################
-  while getopts 'n:l:w:r:NRS:h' flag ; do
+  while getopts 'n:l:w:r:INRS:h' flag ; do
     case "${flag}" in
       n  ) chroot=$OPTARG                        ; ! $INCHROOT || err_chflag "$flag";;
       l  ) copy=$OPTARG                          ; ! $INCHROOT || err_chflag "$flag";;
       w|r) librechroot_flags+=(-$flag "$OPTARG") ; ! $INCHROOT || err_chflag "$flag";;
+      I  ) SANE=false;;
       N  ) NONET=false;;
       R  ) repack=true; makepkg_args+=(-R);;
       S  ) srcpkg='';; # srcpkg=$OPTARG;; TODO: not yet implemented