db-import-pkg: Use db-{move,update,remove}

1 files changed, 76 insertions, 221 deletions

diff --git a/db-import-pkg b/db-import-pkg
index c1dffe0..6dff6bf 100755
--- a/db-import-pkg
+++ b/db-import-pkg
@@ -83,29 +83,6 @@ filter_blacklisted() {
 		<(libreblacklist cat | libreblacklist get-pkg | sort -u)
 }
 
-# usage: filter_duplicates <FULL_LIST >FILTERED_LIST
-#
-# Given a list of packages in the format:
-#
-#     pkgname [epoch:]pkgver-pkgrel
-#
-# filter out arch=(any) packages present elsewhere, as it confuses
-# parabolaweb, librechroot, and who-knows-what-else.  This only
-# filters exact pkgname/epoch/pkgver/pkgrel matches.
-filter_duplicates() {
-	sort -u | comm -23 - <(
-		for pool in "${INHERIT[@]}"; do
-			for f in "${FTP_BASE}/${pool}"/*-any${PKGEXTS}; do
-				f=${f##*/}
-				f=${f%-any$PKGEXTS}
-				pkgname=${f%-*-*}
-				fullpkgver=${f#"${pkgname}-"}
-				printf '%s %s\n' "$pkgname" "$fullpkgver"
-			done
-		done | sort -u
-	)
-}
-
 # usage: sync_pool <from> <path-to-whitelist> <into>
 #
 # Sync excluding everything but whitelist
@@ -118,7 +95,7 @@ sync_pool() {
 	fi
 
 	mkdir -p -- "$_into"
-	msg2 "Retrieving %d packages from %s pool" \
+	msg "Retrieving up to %d files from %s pool" \
 		"$(wc -l < "$_whitelist")" \
 		"$(basename "$_into")"
 
@@ -133,97 +110,6 @@ sync_pool() {
 		"$_into"
 }
 
-# usage: poolify <arch>
-#
-# Given a list of packages in the format:
-#
-#     pkgname [epoch:]pkgver-pkgrel
-#
-# Resolve each to a file in `${FTP_BASE}/pool/`.  The output is
-# relative to `${FTP_BASE}/pool/`.  That is, something along the lines
-# of:
-#
-#     poolname/pkgname-[epoch:]pkgver-pkgrel-arch.pkg.tar.xz
-#     archlinux32/zip-3.0-7-i686.pkg.tar.xz
-#     packages/rhino-1.7.7.1-1-any.pkg.tar.xz
-poolify() {
-	local -r arch=$1
-
-	local pkgname fullpkgver
-	local restore paths path
-	while read -r pkgname fullpkgver; do
-		paths=()
-		for pool in "${INHERIT[@]}" "$PKGPOOL"; do
-			paths+=(
-				"${FTP_BASE}/${pool}/${pkgname}-${fullpkgver}-any"${PKGEXTS}
-				"${FTP_BASE}/${pool}/${pkgname}-${fullpkgver}-${arch}"${PKGEXTS}
-			)
-		done
-		path="${paths[0]:-}"
-		if ! [[ -f "$path" && -f "${path}.sig" ]]; then
-			error "No file was found for %q=%q, aborting" "$pkgname" "$fullpkgver"
-			exit 1
-		fi
-		printf '%s\n' "${path#"${FTP_BASE}/pool/"}"
-	done
-}
-
-# usage: make_repo_symlinks TAG <POOLFILELIST
-#
-# Uses the ${WORKDIR} global
-make_repo_symlinks() {
-	local -r tag=$1
-
-	local -r repo=${tag%-*}
-	local -r arch=${tag##*-}
-	local -r repodir=${WORKDIR}/staging-rsync/${repo}/os/${arch}
-
-	msg2 "Putting symlinks in %s" "${repo}/os/${arch}"
-	mkdir -p -- "${repodir}"
-
-	local poolfile
-	while read -r poolfile; do
-		ln -sfvT "../../../pool/${poolfile##*/pool/}"     "${repodir}/${poolfile##*/}"
-		ln -sfvT "../../../pool/${poolfile##*/pool/}.sig" "${repodir}/${poolfile##*/}.sig"
-	done
-}
-
-# usage: make_repo_dbs <repo> <arch>
-#
-# Uses the ${WORKDIR} global
-make_repo_dbs() {
-	local -r from=${WORKDIR}/staging-rsync/${1}/os/${2}
-	local -r into=${FTP_BASE}/${1}/os/${2}/
-	local -r db_file=${from}/${1}${DBEXT}
-	local -r files_file=${from}/${1}${FILESEXT}
-
-	# create fresh databases to reflect actual `any.pkg.tar.xz` packages.
-	# this also avoids corrupt upstream metadata (ALARM)
-	msg2 "Adding whitelisted packages to clean DBs ..."
-
-	pushd "${from}"
-	local -r UMASK=$(umask)
-	umask 002
-	repo-add "${db_file##*/}" *${PKGEXTS}
-	umask "$UMASK" >/dev/null
-	popd >/dev/null
-
-	mkdir -p -- "$into"
-	# This bit is based on db-functions:set_repo_permission()
-	local -r group=$(/usr/bin/stat --printf='%G' "${into}")
-	chgrp "$group" "${db_file}"
-	chgrp "$group" "${files_file}"
-	chmod g+w "${db_file}"
-	chmod g+w "${files_file}"
-
-	msg2 "Updating %s-%s databases" "$2" "$1"
-	rsync "${extra[@]}" --no-motd -rtlpH --no-t \
-		--delay-updates \
-		--delete-after \
-		--links \
-		"$from/" "$into"
-}
-
 # Main function. Process the databases and get the libre packages
 # Outline:
 #  1. Fetch package info
@@ -234,32 +120,30 @@ make_repo_dbs() {
 #       blacklist.txt) the desired repo state, and how to get from one
 #       to the other.
 #  3. Fetch the packages we want
-#     * Create sync whitelist (based on package blacklist)
-#     * Call sync_pool to fetch packages and signatures
-#  4. Put the packages in the repos
-#     * Create new repo.db with them (repo-add)
-#     * rsync scratch directory => repos
+#     * Symlink to files with the same name in INHERIT pools
+#     * sync_pool to download the others
+#  4. Modify the repos
+#     * db-move
+#     * db-update
+#     * db-remove
 #
 # Files:
 #   (misc)
 #     - ${WORKDIR}/expac/                 : Scratch directory for expac_file()
 #   (download)
 #     - ${WORKDIR}/rsync/                 : Where we download '.db' files to
-#     - ${FTP_BASE}/${PKGPOOL}            : Where we download packages to
-#     - ${FTP_BASE}/${SRCPOOL}            : Where we download sources to
+#     - ${WORKDIR}/staging/${repo}        : Where we download packages to
+#     - ${WORKDIR}/staging/${SRCPOOL}     : Where we download sources to
 #   (analysis)
 #     - ${FTP_BASE}/${INHERIT}            : Where we look for duplicate files
 #     - ${FTP_BASE}/.../${repo}.db        : Where we generate ${WORKDIR}/old/ from
 #     - ${WORKDIR}/old/                   : .txt files describing the way the repos are
 #     - ${WORKDIR}/new/                   : .txt files describing the way we want them to be
 #     - ${WORKDIR}/dif/                   : .txt files describing how to make it happen
-#     - ${WORKDIR}/all.whitelist          : Glob list of (source-)package filenames to download
-#     - ${WORKDIR}/${tag}.whitelist       : Glob list of (source-)package filenames to download
-#     - ${WORKDIR}/${tag}.pool            : See poolify()
+#     - ${WORKDIR}/${repo}.pkg.whitelist  : List of package filenames to download
+#     - ${WORKDIR}/all.src.whitelist      : Glob list of source-package filenames to download
 #   (release)
-#     - ${WORKDIR}/staging-rsync/         : .db files and symlinks we'll rsync in to ${FTP_BASE}
-#     - ${FTP_BASE}/                      : Where we rsync ${WORKDIR}/staging-rsync/ to
-#     - ${FTP_BASE}/lastupdate            : Timestamp file of last repo change
+#     - ${WORKDIR}/staging/               : STAGING= directory for db-update
 main() {
 	##############################################################
 	# 0. Initialization                                          #
@@ -279,7 +163,7 @@ main() {
 	source "$config_file"
 
 	local ret=0 varname varref
-	for varname in PKGEXTS DBEXT FILESEXT FTP_BASE PKGPOOL SRCPOOL; do
+	for varname in PKGEXTS FTP_BASE PKGPOOL SRCPOOL; do
 		if [[ -z ${!varname:-} ]] || is_array "$varname"; then
 			print "Configure '%s' as a non-empty string in %q (or %q):" "$varname" "$config_file" "$LOCAL_CONFIG"
 			ret=$EXIT_NOTCONFIGURED
@@ -299,15 +183,6 @@ main() {
 		fi
 	done
 
-	if [[ -n ${ARCHSRCPOOL:-} && -z ${ARCHPKGPOOL:-} ]]; then
-		print '%s requires that %s is also set' ARCH{SRC,PKG}POOL
-		ret=$EXIT_NOTCONFIGURED
-	fi
-
-	if (( ret != 0 )); then
-		exit $ret
-	fi
-
 	WORKDIR=$(mktemp -dt "${0##*/}.XXXXXXXXXX")
 	readonly WORKDIR
 	trap "rm -rf -- ${WORKDIR@Q}" EXIT
@@ -388,98 +263,78 @@ main() {
 	# 3. Fetch the packages we want                              #
 	##############################################################
 
-	# OK, now we have ${WORKDIR}/old/ describing the way the repos
-	# are, ${WORKDIR}/new/ describing the way we want them to be,
-	# and ${WORKDIR}/dif/ describing how to get from `old` to
-	# `new`.  We should (TODO) now use db-move, db-update, and
-	# db-remove to apply that diff.
-	#
-	# But,
-	#  - db-move is broken
-	#  - The code that populates /dif/ isn't finished
-	# So, just nuke the current repos and entirely re-create
-	# everything from /new/.
-
-	local whitelists=()
-	for _tag in "${ARCHTAGS[@]}"; do
-		msg "Processing %s" "$_tag"
-		_repo=${_tag%-*}
-		_arch=${_tag##*-}
-		# Create a whitelist, add * wildcard to end.
-		#
-		# FIXME: due to lack of -arch suffix, the pool sync
-		# retrieves every arch even if we aren't syncing them.
-		#
-		# IMPORTANT: the . in the sed command is needed
-		# because an empty whitelist would consist of a single
-		# * allowing any package to pass through.
-		filter_duplicates \
-			<"${WORKDIR}/new/${_tag}.txt" \
-			| sed -e 's/ /-/' -e 's|.$|&*|g' \
-			> "${WORKDIR}/${_tag}.whitelist"
-		if [[ -n ${ARCHPKGPOOL:-} ]]; then
-			# Append to whitelists array so that we can
-			# later sync_pool() all packages
-			whitelists+=("${WORKDIR}/${_tag}.whitelist")
-		else
-			# Upstream doesn't use an $ARCHPKGPOOL
-			sync_pool \
-				"${ARCHMIRROR}/$(get_repo_dir "${_repo}" "${_arch}")/" \
-				"${WORKDIR}/${_tag}.whitelist" \
-				"${FTP_BASE}/${PKGPOOL}/"
-			poolify "${_arch}" \
-				<"${WORKDIR}/new/${_tag}.txt" \
-				>"${WORKDIR}/${_tag}.pool"
-			make_repo_symlinks "$_tag" \
-				<"${WORKDIR}/${_tag}.pool"
-		fi
+	# For some packages, "fetch" means to create a symlink to a
+	# pool we INHERIT from.  For others, it means to actually
+	# download it from arg_upstream with rsync.
+	for tag in "${ARCHTAGS[@]}";do
+		msg "Processing %s" "$tag"
+		repo=${tag%-*}
+		arch=${tag##*-}
+		mkdir -p -- "${WORKDIR}/staging/${repo}"
+
+		local pkgname filename pgpsig
+		while read -r pkgname filename pgpsig; do
+			local pool staged=false
+			for pool in "${INHERIT[@]}"; do
+				filepath=("${FTP_BASE}/${pool}/${filename}")
+				if [[ -f $filepath && ! -h $filepath ]]; then
+					ln -srT -- "$filepath"     "${WORKDIR}/staging/${repo}/${filename}"
+					ln -srT -- "$filepath".sig "${WORKDIR}/staging/${repo}/${filename}".sig
+					staged=true
+				fi
+			done
+			if ! $staged; then
+				printf '%s\n' "$filename"{,.sig} >> "${WORKDIR}/${repo}.pkg.whitelist"
+				printf '%s\n' "${filename%$PKGEXTS}*.src.tar*" >> "${WORKDIR}/all.src.whitelist"
+			fi
+		done < <(
+			mapfile -t pkgnames < <(cut -d' ' -f1 <"${WORKDIR}/dif/update:${tag}.txt")
+			db_file="${WORKDIR}/rsync/$(get_repo_dir "${repo}" "${arch}")/${repo}.db"
+			expac_file "$db_file" '%n %f %g' "${pkgnames[@]}"
+		)
 	done
-
-	if (( ${#whitelists[@]} > 0 )); then
-		# Concatenate all whitelists, check for single *s just in case
-		cat "${whitelists[@]}" | grep -v "^\*$" |
-			sort -u > "${WORKDIR}/all.whitelist"
-		# FIXME: make_whitelist() wildcards should be narrowed
-		#        down to respect the architecture of the tag
-
-		msg "Syncing package pool"
+	for whitelist in "${WORKDIR}"/*.pkg.whitelist; do
 		sync_pool \
-			"${ARCHMIRROR}/${ARCHPKGPOOL}/" \
-			"${WORKDIR}/all.whitelist" \
-			"${FTP_BASE}/${PKGPOOL}/"
-		for _tag in "${ARCHTAGS[@]}"; do
-			_repo=${_tag%-*}
-			_arch=${_tag##*-}
-			poolify "${_arch}" "${PKGPOOL}" \
-				<"${WORKDIR}/new/${_tag}.txt" \
-				>"${WORKDIR}/${_tag}.pool"
-			make_repo_symlinks "$_tag" \
-				<"${WORKDIR}/${_tag}.pool"
-		done
-
-		if [[ -n ${ARCHSRCPOOL:-} ]]; then
-			msg "Syncing source pool"
-			sync_pool \
-				"${ARCHMIRROR}/${ARCHSRCPOOL}/" \
-				"${WORKDIR}/all.whitelist" \
-				"${FTP_BASE}/${SRCPOOL}/"
-		fi
+			"${ARCHMIRROR}/${ARCHPKGPOOL:-$(get_repo_dir "${_repo}" "${_arch}")}/" \
+			"$whitelist" \
+			"${WORKDIR}/staging/${repo}/"
+	done
+	if [[ -n ${ARCHSRCPOOL:-} ]]; then
+		sync_pool \
+			"${ARCHMIRROR}/${ARCHSRCPOOL}/" \
+			"${WORKDIR}/all.src.whitelist" \
+			"${WORKDIR}/staging/${SRCPOOL}/"
 	fi
 
 	##############################################################
-	# 4. Put the packages in the repos                           #
+	# 4. Modify the repos                                        #
 	##############################################################
 
-	msg "Putting databases back in place"
+	msg "Modifying the actual repos"
 
-	# FIXME: all repo DBs should be replaced at once (per architecture)
-	ln -srT "$FTP_BASE/pool" "${WORKDIR}/staging-rsync/pool"
-	for _tag in "${ARCHTAGS[@]}"; do
-		_repo=${_tag%-*}
-		_arch=${_tag##*-}
-		make_repo_dbs "$_repo" "$_arch"
+	# db-move
+	for tag_from in "${ARCHTAGS[@]}"; do
+		repo_from=${tag_from%-*}
+		arch_from=${tag_from##*-}
+		for tag_to in "${ARCHTAGS[@]}"; do
+			repo_to=${tag_to%-*}
+			arch_to=${tag_to##*-}
+			[[ $tag_from != $tag_to ]] || continue
+			[[ $arch_from == $arch_to ]] || continue
+
+			< "${WORKDIR}/diff/move:${tag_from}:${tag_to}.txt" \
+				cut -d' ' -f1 | \
+				xargs -r0 -d $'\n' db-move "$repo_from" "$repo_to"
+		done
+	done
+	# db-update
+	STAGING=${WORKDIR}/staging db-update
+	# db-remove
+	for tag in "${ARCHTAGS[@]}"; do
+		< "${WORKDIR}/diff/remove:${tag}.txt" \
+			cut -d' ' -f1 | \
+			xargs -r0 -d $'\n' db-move "$repo_from" "$repo_to"
 	done
-	date +%s > "${FTP_BASE}/lastupdate"
 }
 
 main "$@"