merge db-import-archlinux-pkg into db-import-pkg

This is a follow up from the previous commit. All three upstreams now
share the same import code.

db-import-pkg has been refactored to be somewhat more extensible and
maintainable. In particular:

 - Both script's main procedures have been broken up in reusable
   functional units.
 - Functions now contain some documentation on their interface and
   operation.
 - We took a sizeable wack against duplication.

The gates are more open than ever to continue with cleanups.

3 files changed, 235 insertions, 323 deletions

diff --git a/db-import-archlinux-pkg b/db-import-archlinux-pkg
deleted file mode 100755
index ceb874f..0000000
--- a/db-import-archlinux-pkg
+++ /dev/null
@@ -1,209 +0,0 @@
-#!/bin/bash
-# Syncs Arch repos based on info contained in repo.db files
-# License: GPLv3
-
-# Principles
-# * Get repo.db from an Arch-like repo
-# * Generate a list of available packages
-# * Create sync whitelist (based on package blacklist)
-# * Get packages and signatures
-# * Sync repo.db => repo.db
-
-# TODO
-# * make a tarball of files used for forensics
-
-set -e
-
-# Run as `V=true db-import-pkg-archlinux` to get verbose output
-VERBOSE=${V}
-extra=()
-${VERBOSE} && extra+=(-v)
-
-WORKDIR=$(mktemp -dt "${0##*/}.XXXXXXXXXX")
-trap "rm -rf -- $(printf '%q' "${WORKDIR}")" EXIT
-
-# Returns contents of a repo
-get_repos() {
-	# Exclude everything but db files
-	rsync "${extra[@]}" --no-motd -mrtlH --no-p --include="*/" \
-		--include="*.db" \
-		--include="*${DBEXT}" \
-		--include="*.files" \
-		--include="*${FILESEXT}" \
-		--exclude="*" \
-		--delete-after \
-		"rsync://${mirror}/${mirrorpath}/" "$WORKDIR"
-}
-
-get_repo_content() {
-	# Return all contents
-	bsdtar tf "${1}" | \
-		cut -d "/" -f 1 | \
-		sort -u
-}
-
-# repo
-# arch
-get_repo_file() {
-	echo "${WORKDIR}/${1}/os/${2}/${1}"
-}
-
-# Process the databases and get the libre packages
-init() {
-
-	# Get the blacklisted packages
-	libreblacklist update
-	blacklist=($(libreblacklist cat | libreblacklist get-pkg))
-	# Store all the whitelist files
-	whitelists=()
-
-	msg "%d packages in blacklist" ${#blacklist[@]}
-
-	test ${#blacklist[@]} -eq 0 && fatal_error "Empty blacklist"
-
-	# Sync the repos databases
-	get_repos
-
-	# Traverse all repo-arch pairs
-	for _repo in "${ARCHREPOS[@]}"; do
-		for _arch in "${ARCHARCHES[@]}"; do
-			msg "Processing %s-%s" "${_repo}" "${_arch}"
-
-			db_file=$(get_repo_file "${_repo}" "${_arch}")${DBEXT}
-			files_file=$(get_repo_file "${_repo}" "${_arch}")${FILESEXT}
-
-			if [ ! -f "${db_file}" ]; then
-				warning "%s doesn't exist, skipping this repo-arch" "${db_file}"
-				continue
-			fi
-			if [ ! -f "${files_file}" ]; then
-				warning "%s doesn't exist, skipping this repo-arch" "${files_file}"
-				continue
-			fi
-
-			# Remove blacklisted packages and count them
-			# TODO capture all removed packages for printing on debug mode
-			msg2 "Removing blacklisted packages from %s and %s databases..." .db .files
-			LC_ALL=C repo-remove "${db_file}" "${blacklist[@]}" \
-			|& sed -n 's/-> Removing/	&/p'
-			# Get db contents
-			db=($(get_repo_content "${db_file}"))
-
-			msg2 "Process clean db for syncing..."
-
-			# Create a whitelist, add * wildcard to end
-			# TODO due to lack of -arch suffix, the pool sync retrieves every arch even if
-			# we aren't syncing them
-			# IMPORTANT: the . in the sed command is needed because an empty
-			# whitelist would consist of a single * allowing any package to
-			# pass through
-			printf '%s\n' "${db[@]}" | sed "s|.$|&*|g" > "/tmp/${_repo}-${_arch}.whitelist"
-
-			msg2 "%d packages in whitelist" "$(wc -l /tmp/${_repo}-${_arch}.whitelist | cut -d' ' -f1)"
-
-			mkdir -p -- "${FTP_BASE}/${_repo}/os/${_arch}/"
-			# Sync excluding everything but whitelist
-			# We delete here for cleanup
-			rsync "${extra[@]}" --no-motd -rtlH \
-				--delete-after \
-				--delete-excluded \
-				--delay-updates \
-				--include-from="/tmp/${_repo}-${_arch}.whitelist" \
-				--exclude="*" \
-				"rsync://${mirror}/${mirrorpath}/${_repo}/os/${_arch}/" \
-				"${FTP_BASE}/${_repo}/os/${_arch}/"
-
-			# Add a new whitelist
-			whitelists+=(/tmp/${_repo}-${_arch}.whitelist)
-
-			msg "Putting databases back in place"
-			mkdir -p -- "${FTP_BASE}/${_repo}/os/${_arch}/"
-			rsync "${extra[@]}" --no-motd -rtlH \
-				--delay-updates \
-				--safe-links \
-				"${WORKDIR}/${_repo}/os/${_arch}/" \
-				"${FTP_BASE}/${_repo}/os/${_arch}/"
-
-			# Cleanup
-			unset db
-		done
-	done
-
-
-	msg "Syncing package pool"
-	# Concatenate all whitelists, check for single *s just in case
-	cat "${whitelists[@]}" | grep -v "^\*$" | sort -u > /tmp/any.whitelist
-
-	msg2 "Retrieving %d packages from pool" "$(wc -l /tmp/any.whitelist | cut -d' ' -f1)"
-
-	# Sync
-	# *Don't delete-after*, this is the job of cleanup scripts. It will remove our
-	# packages too
-	local pkgpool
-	for pkgpool in "${ARCHPKGPOOLS[@]}"; do
-		# FIXME:  whitelist wildcards should be narrowed down to respect ${ARCHARCHES[@]}
-		rsync "${extra[@]}" --no-motd -rtlH \
-			--delay-updates \
-			--safe-links \
-			--include-from=/tmp/any.whitelist \
-			--exclude="*" \
-			"rsync://${mirror}/${mirrorpath}/${pkgpool}/" \
-			"${FTP_BASE}/${pkgpool}/"
-	done
-
-	# Sync sources
-	msg "Syncing source pool"
-	#sed "s|\.pkg\.tar\.|.src.tar.|" /tmp/any.whitelist > /tmp/any-src.whitelist
-	#msg2 "Retrieving %d sources from pool" $(wc -l < /tmp/any-src.whitelist)
-
-	# Sync
-	# *Don't delete-after*, this is the job of cleanup scripts. It will remove our
-	# packages too
-	local srcpool
-	for srcpool in "${ARCHSRCPOOLS[@]}"; do
-		rsync "${extra[@]}" --no-motd -rtlH \
-			--delay-updates \
-			--safe-links \
-			--include-from=/tmp/any.whitelist \
-			--exclude="*" \
-			"rsync://${mirror}/${mirrorpath}/${srcpool}/" \
-			"${FTP_BASE}/${srcpool}/"
-	done
-
-	date -u +%s > "${FTP_BASE}/lastsync"
-
-	# Cleanup
-	unset blacklist whitelists _arch _repo repo_file
-}
-
-trap_exit() {
-	local signal=$1; shift
-	echo
-	error "$@"
-	trap -- "$signal"
-	kill "-$signal" "$$"
-}
-
-fatal_error() {
-	error "$@"
-	exit 1
-}
-
-source "$(dirname "$(readlink -e "$0")")/config"
-source "$(dirname "$(readlink -e "$0")")/db-import-archlinux.conf"
-source "$(librelib messages)"
-
-# Check variables presence
-for var in DBEXT FILESEXT mirror mirrorpath WORKDIR FTP_BASE ARCHSRCPOOLS ARCHPKGPOOLS; do
-	test -z "${!var}" && fatal_error "Empty %s" "${var}"
-done
-
-# From makepkg
-set -E
-for signal in TERM HUP QUIT; do
-	trap "trap_exit $signal '%s signal caught. Exiting...' $signal" "$signal"
-done
-trap 'trap_exit INT "Aborted by user! Exiting..."' INT
-trap 'trap_exit USR1 "An unknown error has occurred. Exiting..."' ERR
-
-init
diff --git a/db-import-archlinux32.conf b/db-import-archlinux32.conf
index a6f17c9..d8bedb4 100644
--- a/db-import-archlinux32.conf
+++ b/db-import-archlinux32.conf
@@ -9,7 +9,7 @@ ARCHARCHES=(i686)
 
 ## mirrors without sources folder
 mirror="mirror.archlinux32.org"
-mirror="32.arlm.tyzoid.com"
+# mirror="32.arlm.tyzoid.com"
 
 mirrorpath="archlinux32"
 
diff --git a/db-import-pkg b/db-import-pkg
index c4c6781..78aeda1 100755
--- a/db-import-pkg
+++ b/db-import-pkg
@@ -23,9 +23,12 @@ ${VERBOSE} && extra+=(-v)
 WORKDIR=$(mktemp -dt "${0##*/}.XXXXXXXXXX")
 trap "rm -rf -- $(printf '%q' "${WORKDIR}")" EXIT
 
-# Returns contents of a repo
-get_repos() {
-	# Exclude everything but db files
+# usage: sync_dbs <from> <into>
+#
+# Sync excluding everything but db files
+# TODO: we could be doing without things other than what is in
+#       ${ARCHARCHES[@]} and ${ARCHREPOS[@]}
+sync_dbs() {
 	rsync "${extra[@]}" --no-motd -mrtlH --no-p --include="*/" \
 		--include="*.db" \
 		--include="*${DBEXT}" \
@@ -33,144 +36,259 @@ get_repos() {
 		--include="*${FILESEXT}" \
 		--exclude="*" \
 		--delete-after \
-		"rsync://${mirror}/${mirrorpath}/" "$WORKDIR"
+		"$1" "$2"
 }
 
+# usage: get_repo_workdir <repo> <arch>
+#
+# Prints workdir path for given <repo> <arch> combination
+get_repo_workdir() {
+    case "$UPSTREAM" in
+	archlinux)
+	    printf -- '%s' "${WORKDIR}/${1}/os/${2}/" ;;
+	archlinux32|archlinuxarm)
+	    printf -- '%s' "${WORKDIR}/${2}/${1}/" ;;
+    esac
+}
+
+# usage: get_repo_content <path-to-db>
+#
+# Prints a list of packages within a given <path-to-db>
 get_repo_content() {
-	# Return all contents
 	bsdtar tf "${1}" | \
 		cut -d "/" -f 1 | \
 		sort -u
 }
 
-# repo
-# arch
-get_repo_file() {
-	echo "${WORKDIR}/${2}/${1}/${1}"
+# usage: make_whitelist <output-file> <path-to-db> <blacklisted-pkg1> [...]
+#
+# Has 2 side effects:
+#   1. Notably, overwrites <output-file> with the whitelist created from...
+#   2. Cleaning <path-to-db> from <blacklisted-pkg1> [...] in the process.
+#   2.1. repo-remove will also clean the corresponding .files db during 2.
+make_whitelist() {
+	local -r output_file=$1 db_file=$2 blacklist=(${@:3})
+	# Remove blacklisted packages and count them
+	# TODO: capture all removed packages for printing on debug mode
+	msg2 "Removing blacklisted packages from %s ..." "${db_file##*/}"
+	LC_ALL=C repo-remove "$db_file" "${blacklist[@]}" |&
+		sed -n 's/-> Removing/	&/p'
+
+	# Get db contents
+	local -r db=($(get_repo_content "${db_file}"))
+	msg2 "%d packages in whitelist" ${#db[@]}
+
+	# Create a whitelist, add * wildcard to end.
+	# FIXME: due to lack of -arch suffix, the pool sync retrieves every arch even if
+	# we aren't syncing them.
+	# IMPORTANT: the . in the sed command is needed because an empty
+	# whitelist would consist of a single * allowing any package to
+	# pass through.
+	printf '%s\n' "${db[@]}" | sed "s|.$|&*|g" > "$output_file"
+}
+
+# usage: < <whitelist> filter_duplicates
+#
+# Don't import arch=(any) packages present elsewhere, it confuses parabolaweb.
+# This reads a whitelist from stdin and prints it without said duplicates.
+filter_duplicates() {
+	grep -vf <(find "${FTP_BASE}/pool/" \
+	                -name "*-any${PKGEXT}" \
+	                -printf "%f\n" | sed  's/-any\.pkg.*/*/') --
+}
+
+# usage: sync_pool <from> <path-to-whitelist> <into>
+#
+# Sync excluding everything but whitelist
+sync_pool() {
+	local -r _from=$1 _whitelist=$2 _into=$3
+
+	mkdir -p -- "$_into"
+	msg2 "Retrieving %d packages from %s pool" "$(wc -l "$_whitelist" | cut -d' ' -f1)" "$(basename "$_from")"
+
+	# *Don't delete-after*, this is the job of
+	# cleanup scripts. It will remove our packages too
+	rsync "${extra[@]}" --no-motd -rtlH \
+	      --delay-updates \
+	      --safe-links \
+	      --include-from="$_whitelist" \
+	      --exclude="*" \
+	      "$_from" \
+	      "$_into"
+}
+
+# usage: sync_repo <from> <path-to-whitelist> <into>
+#
+# Sync excluding everything but whitelist.
+# TODO: this is too similar to sync_pool(). Merge?
+sync_repo() {
+	local -r _from=$1 _whitelist=$2 _into=$3
+	mkdir -p -- "$_into"
+	msg2 "Retrieving %d files from repo" "$(wc -l "$_whitelist" | cut -d' ' -f1)"
+
+	# We delete here for cleanup
+	rsync "${extra[@]}" --no-motd -rtlH \
+	      --delete-after \
+	      --delete-excluded \
+	      --delay-updates \
+	      --include-from="$_whitelist" \
+	      --exclude="*" \
+	      "$_from" \
+	      "$_into"
+}
+
+# usage: make_repo_symlinks <pool> <path-to-whitelist> <repo> <arch>
+#
+# Generate symbolic links to target packages <repo-whitelist> lying in
+# some of our <pool>s, and put them in $FTP_BASE/<repo>/os/<arch>.
+#
+# Use this after `sync_pool`ing from an upstream with no pool(s) and
+# therefore no symlinks inside <repo>/os/<arch>.
+make_repo_symlinks() {
+	local -r _pool=$1 _whitelist=$2 _repo=$3 _arch=$4
+
+	msg2 "Putting symlinks in ${_repo}/os/${_arch}"
+	mkdir -p -- "${FTP_BASE}/${_repo}/os/${_arch}"
+
+	local _pkgfile
+	while read _pkgfile; do
+		local _path="${FTP_BASE}/${_pool}/${_pkgfile}"
+		if [[ ! -f "$_path" ]]; then
+			# pkg was an `any.pkg.tar.?z`, find which pool it's in.
+			local _any_pkgs=(${FTP_BASE}/pool/*/${_pkgfile/${_arch}/any})
+			_path="${_any_pkgs[0]}"
+		fi
+		# give up
+		[[ ! (-f "$_path" && -f "${_path}.sig") ]] && continue
+
+		ln -sfv "../../../pool/${_path##*/pool/}" \
+			"${FTP_BASE}/${_repo}/os/${_arch}/${_path##*/}"
+		ln -sfv "../../../pool/${_path##*/pool/}.sig" \
+			"${FTP_BASE}/${_repo}/os/${_arch}/${_path##*/}.sig"
+	done < <(sed "s/*/-${_arch}.pkg.tar.xz/" "$_whitelist")
+}
+
+# usage: make_repo_dbs <repo> <arch>
+make_repo_dbs() {
+	local -r from=$(get_repo_workdir "$1" "$2")/
+	local -r into=${FTP_BASE}/${1}/os/${2}/
+	local -r db_file=${from}/${1}${DBEXT}
+	local -r files_file=${from}/${1}${FILESEXT}
+	local -r whitelist=/tmp/${1}-${2}.whitelist
+
+	# create fresh databases to reflect actual `any.pkg.tar.xz` packages.
+	# this also avoids corrupt upstream metadata (ALARM)
+	msg2 "Adding whitelisted packages to clean %s and %s ..." \
+	     "${db_file##*/}" "${files_file##*/}"
+	rm "$db_file" "$files_file"
+	LC_ALL=C repo-add "$db_file" \
+	      $(sed "s|^|${into}|; s|$|${PKGEXT}|" "$whitelist") |&
+		sed -n 's/==> Adding/	-> Adding/p'
+
+	msg2 "Updating %s-%s databases" "$2" "$1"
+	mkdir -p -- "$into"
+	rsync "${extra[@]}" --no-motd -rtlH \
+	      --delay-updates \
+	      --safe-links \
+	      "$from" "$into"
 }
 
 # Process the databases and get the libre packages
 init() {
-
 	# Get the blacklisted packages
 	libreblacklist update
-	blacklist=($(libreblacklist cat | libreblacklist get-pkg))
-
-	msg "%d packages in blacklist" ${#blacklist[@]}
-
+	local -a blacklist=($(libreblacklist cat | libreblacklist get-pkg))
 	test ${#blacklist[@]} -eq 0 && fatal_error "Empty blacklist"
+	msg2 "%d packages in blacklist" ${#blacklist[@]}
 
 	# Sync the repos databases
-	get_repos
+	msg 'Retrieving .db and .files files'
+	sync_dbs "rsync://${mirror}/${mirrorpath}/" "$WORKDIR"
 
 	# Traverse all repo-arch pairs
+	local _arch _repo
 	for _arch in "${ARCHARCHES[@]}"; do
 		for _repo in "${ARCHREPOS[@]}"; do
-			msg "Processing %s-%s" "${_repo}" "${_arch}"
-
-			db_file=$(get_repo_file "${_repo}" "${_arch}")${DBEXT}
-			files_file=$(get_repo_file "${_repo}" "${_arch}")${FILESEXT}
-
-			if [ ! -f "${db_file}" ]; then
-				warning "%s doesn't exist, skipping this arch-repo" "${db_file}"
-				continue
-			fi
-			if [ ! -f "${files_file}" ]; then
-				warning "%s doesn't exist, skipping this arch-repo" "${files_file}"
-				continue
-			fi
-
-			# Remove blacklisted packages and count them
-			# TODO capture all removed packages for printing on debug mode
-			msg2 "Removing blacklisted packages from %s and %s databases..." .db .files
-			LC_ALL=C repo-remove "${db_file}" "${blacklist[@]}" \
-			|& sed -n 's/-> Removing/	&/p'
-			# Get db contents
-			db=($(get_repo_content "${db_file}"))
-
-			msg2 "Process clean db for syncing..."
-
-			# Create a whitelist, add * wildcard to end
-			# TODO due to lack of -arch suffix, the pool sync retrieves every arch even if
-			# we aren't syncing them
-			# IMPORTANT: the . in the sed command is needed because an empty
-			# whitelist would consist of a single * allowing any package to
-			# pass through
-			printf '%s\n' "${db[@]}" | sed "s|.$|&*|g" > "/tmp/${_repo}-${_arch}.whitelist"
-
-			msg2 "%d packages in whitelist" "$(wc -l /tmp/${_repo}-${_arch}.whitelist | cut -d' ' -f1)"
-
-			# FIXME: number might be lower, because of next line
-			msg2 "Retrieving %d packages to pool" "$(wc -l /tmp/${_repo}-${_arch}.whitelist | cut -d' ' -f1)"
-
-			mkdir -p -- "${FTP_BASE}/${ARCHPKGPOOLS}"
-			# Don't import arch=(any) packages present elsewhere
-			grep -vf <(find "${FTP_BASE}/pool/" -name "*-any${PKGEXT}" -printf "%f\n" | sed  's/-any\.pkg.*/*/') \
-				-- "/tmp/${_repo}-${_arch}.whitelist" |
-				# Sync excluding everything but stdin
-				rsync "${extra[@]}" --no-motd -rtlH \
-					--delay-updates \
-					--safe-links \
-					--include-from=- \
-					--exclude="*" \
-					"rsync://${mirror}/${mirrorpath}/${_arch}/${_repo}/" \
-					"${FTP_BASE}/${ARCHPKGPOOLS}/"
-
-			unset db
+			msg "Processing %s-%s" "${_arch}" "${_repo}"
+
+			local db_file=$(get_repo_workdir "${_repo}" "${_arch}")/${_repo}${DBEXT}
+			local files_file=$(get_repo_workdir "${_repo}" "${_arch}")/${_repo}${FILESEXT}
+			local _file
+			for _file in db_file files_file; do
+				if [ ! -f "${!_file}" ]; then
+					warning "%s doesn't exist, skipping this arch-repo" "${!_file}"
+					continue
+				fi
+			done
+
+			make_whitelist "/tmp/${_repo}-${_arch}.whitelist" "$db_file" "${blacklist[@]}"
+			case "$UPSTREAM" in
+			    archlinux)
+				# Append to whitelists array so that we can
+                                # later sync_pool() all packages
+				local -a whitelists+=(/tmp/${_repo}-${_arch}.whitelist)
+				# Get repo packages (symlinks)
+				sync_repo "rsync://${mirror}/${mirrorpath}/${_repo}/os/${_arch}/" \
+				          "/tmp/${_repo}-${_arch}.whitelist" \
+				          "${FTP_BASE}/${_repo}/os/${_arch}/"
+				;;
+			    archlinux32|archlinuxarm)
+				# Upstream doesn't use an $ARCHPKGPOOL
+				filter_duplicates < "/tmp/${_repo}-${_arch}.whitelist" > "/tmp/${_repo}-${_arch}-nodups.whitelist"
+				sync_pool "rsync://${mirror}/${mirrorpath}/${_arch}/${_repo}/" \
+				          "/tmp/${_repo}-${_arch}-nodups.whitelist" \
+				          "${FTP_BASE}/${ARCHPKGPOOLS}/"
+				;;
+			esac
 		done
 	done
 
-	msg "Generating symbolic links to pool"
+	case "$UPSTREAM" in
+	    archlinux)
+		# Concatenate all whitelists, check for single *s just in case
+		cat "${whitelists[@]}" | grep -v "^\*$" | sort -u > "/tmp/${UPSTREAM}-all.whitelist"
+		# FIXME:  make_whitelist() wildcards should be narrowed down to respect ${ARCHARCHES[@]}
 
-	for _arch in "${ARCHARCHES[@]}"; do
-		for _repo in "${ARCHREPOS[@]}"; do
-			msg "Putting symlinks in ${_repo}/os/${_arch}"
-			mkdir -p -- "${FTP_BASE}/${_repo}/os/${_arch}"
-
-			local _pkgfile
-			while read _pkgfile; do
-				local _path="${FTP_BASE}/${ARCHPKGPOOLS}/${_pkgfile}"
-				if [[ ! -f "$_path" ]]; then
-					# pkg was an `any.pkg.tar.?z`, find which pool it's in.
-					local _any_pkgs=(${FTP_BASE}/pool/*/${_pkgfile/${_arch}/any})
-					_path="${_any_pkgs[0]}"
-				fi
-				# give up
-				[[ ! (-f "$_path" && -f "${_path}.sig") ]] && continue
-
-				ln -sfv "../../../pool/${_path##*/pool/}" \
-					"${FTP_BASE}/${_repo}/os/${_arch}/${_path##*/}"
-				ln -sfv "../../../pool/${_path##*/pool/}.sig" \
-					"${FTP_BASE}/${_repo}/os/${_arch}/${_path##*/}.sig"
-			done < <(sed "s/*/-${_arch}.pkg.tar.xz/g" "/tmp/${_repo}-${_arch}.whitelist")
+		msg "Syncing package pools"
+		local pkgpool
+		for pkgpool in "${ARCHPKGPOOLS[@]}"; do
+		    sync_pool "rsync://${mirror}/${mirrorpath}/${pkgpool}/" "/tmp/${UPSTREAM}-all.whitelist" "${FTP_BASE}/${pkgpool}/"
 		done
-	done
 
-	# create fresh databases to reflect actual `any.pkg.tar.xz` packages.
-	# this also avoids corrupt upstream metadata (ALARM)
+		msg "Syncing source pool"
+		local srcpool
+		for srcpool in "${ARCHSRCPOOLS[@]}"; do
+		    sync_pool "rsync://${mirror}/${mirrorpath}/${srcpool}/" "/tmp/${UPSTREAM}-all.whitelist" "${FTP_BASE}/${srcpool}/"
+		done
+		;;
+	    archlinux32|archlinuxarm)
+		msg "Generating symbolic links to pool"
+
+		local _arch _repo _pkgpool
+		for _arch in "${ARCHARCHES[@]}"; do
+			for _repo in "${ARCHREPOS[@]}"; do
+				for _pkgpool in "${ARCHPKGPOOLS[@]}"; do
+					make_repo_symlinks \
+                                                "$_pkgpool" \
+					         "/tmp/${_repo}-${_arch}.whitelist" \
+					         "$_repo" \
+					         "$_arch"
+				done
+			done
+		done
+		;;
+	esac
+
+	msg "Putting databases back in place"
+
+	# FIXME: all repo databases should be replaced at once (per architecture)
 	for _arch in "${ARCHARCHES[@]}"; do
 		for _repo in "${ARCHREPOS[@]}"; do
-			msg2 "Adding whitelisted packages to clean %s and %s databases..." .db .files
-			rm "$db_file" "$files_file"
-			LC_ALL=C repo-add "$db_file" \
-				$(sed "s|^|${FTP_BASE}/${_repo}/os/${_arch}/|; s|$|${PKGEXT}|" \
-				      "/tmp/${_repo}-${_arch}.whitelist") |&
-				sed -n 's/==> Adding/	-> Adding/p'
-
-			msg "Putting databases back in place"
-			# TODO: all repos should be updated at once (per architecture)
-			rsync "${extra[@]}" --no-motd -rtlH \
-			      --delay-updates \
-			      --safe-links \
-			      "${WORKDIR}/${_arch}/${_repo}/" \
-			      "${FTP_BASE}/${_repo}/os/${_arch}/"
+                        make_repo_dbs "$_repo" "$_arch"
 		done
 	done
 
 	date -u +%s > "${FTP_BASE}/lastsync"
-
-	# Cleanup
-	unset blacklist _arch _repo repo_file _pkgfile
 }
 
 trap_exit() {
@@ -189,17 +307,20 @@ fatal_error() {
 source "$(librelib messages)"
 source "$(dirname "$(readlink -e "$0")")/config"
 
-readonly -a remotes=(archlinux{32,arm})
+readonly -a UPSTREAMS=(archlinux{,32,arm})
 
-if [[ $# -ne 1 ]] || ! in_array "$1" "${remotes[@]}" ; then
-	msg 'usage: %s {%s\b}' "${0##*/}" "$(printf -- '%s |' "${remotes[@]}")"
+if [[ $# -ne 1 ]] || ! in_array "$1" "${UPSTREAMS[@]}" ; then
+	msg 'usage: %s {%s\b}' "${0##*/}" "$(printf -- ' %s |' "${UPSTREAMS[@]}")"
 	exit 1
 fi
 
-source "$(dirname "$(readlink -e "$0")")/db-import-${1}.conf"
+readonly UPSTREAM=$1
+source "$(dirname "$(readlink -e "$0")")/db-import-${UPSTREAM}.conf"
 
 # Check variables presence
-for var in DBEXT FILESEXT mirror mirrorpath WORKDIR FTP_BASE; do
+vars=(DBEXT FILESEXT mirror mirrorpath WORKDIR FTP_BASE ARCHREPOS ARCHPKGPOOLS)
+[[ $UPSTREAM == archlinux ]] && vars+=(ARCHSRCPOOLS)
+for var in "${vars[@]}"; do
 	test -z "${!var}" && fatal_error "Empty %s" "${var}"
 done