db-import-pkg: Don't do any work directly in $FTP_BASE/$repo or $WORKDIR/rsyncparabola/20180505

Currently, it
 - rsync's/creates (depending on $UPSTREAM) symlinks in
   "${FTP_BASE}/${repo}/os/${arch}" directly
 - runs repo-add on DBs in ${WORKDIR}/rsync/$(get_repo_dir "$repo" "$arch")
Instead, do both of these things in $WORKDIR/staging-rsync/$tag.  There's
no reason to re-use the initial /rsync/ dir (we don't re-use any files in
it--we rm them), and the sync_repo's --delete-excluded means that we
temporarily delete the .db files on the live repo.

Also, to do this efficiently, we need to create hardlinks between package
files in $FTP_BASE and $WORKDIR, so fiddle with tmp directories in the
config.

This also has the effect that imported repos no longer have '.db.old'
files, which is fine because they were never Parabola's old version of the
file, they were the verbatim .old file from where we're importing from.

3 files changed, 18 insertions, 12 deletions

diff --git a/config b/config
index e632a1e..5b94388 100644
--- a/config
+++ b/config
@@ -22,7 +22,9 @@ SOURCE_CLEANUP_KEEP=30
 LOCK_DELAY=10
 
 [ -n "${STAGING:-}" ] || STAGING="$HOME/staging/unknown/staging"
-TMPDIR="/tmp"
+#TMPDIR="/tmp"
+TMPDIR=/srv/repo/tmp
+mkdir -p "$TMPDIR"
 ARCHES=(x86_64 i686 armv7h)
 DBEXT=".db.tar.gz"
 FILESEXT=".files.tar.gz"
diff --git a/db-import-pkg b/db-import-pkg
index 5b702e7..3591419 100755
--- a/db-import-pkg
+++ b/db-import-pkg
@@ -125,8 +125,10 @@ sync_repo() {
 make_repo_symlinks() {
 	local -r pool=$1 whitelist=$2 repo=$3 arch=$4
 
+	local repodir="${WORKDIR}/staging-rsync/${repo}-${arch}"
+
 	msg2 "Putting symlinks in ${repo}/os/${arch}"
-	mkdir -p -- "${FTP_BASE}/${repo}/os/${arch}"
+	mkdir -p -- "${repodir}"
 
 	local pkgfile
 	while read pkgfile; do
@@ -148,7 +150,7 @@ make_repo_symlinks() {
 				"${pkgfile%-*}"
 			continue
 		fi
-		local symlink="${FTP_BASE}/${repo}/os/${arch}/${path##*/}"
+		local symlink="${repodir}/${path##*/}"
 		ln -sfv "../../../pool/${path##*/pool/}" "$symlink"
 		ln -sfv "../../../pool/${path##*/pool/}.sig" "${symlink}.sig"
 		local -a new_whitelist+=($symlink)
@@ -158,7 +160,7 @@ make_repo_symlinks() {
 
 # usage: make_repo_dbs <repo> <arch>
 make_repo_dbs() {
-	local -r from=${WORKDIR}/rsync/$(get_repo_dir "$1" "$2")
+	local -r from=${WORKDIR}/staging-rsync/${1}-${2}
 	local -r into=${FTP_BASE}/${1}/os/${2}/
 	local -r db_file=${from}/${1}${DBEXT}
 	local -r files_file=${from}/${1}${FILESEXT}
@@ -166,9 +168,7 @@ make_repo_dbs() {
 
 	# create fresh databases to reflect actual `any.pkg.tar.xz` packages.
 	# this also avoids corrupt upstream metadata (ALARM)
-	msg2 "Adding whitelisted packages to clean %s and %s ..." \
-		"${db_file##*/}" "${files_file##*/}"
-	rm "$db_file" "$files_file"
+	msg2 "Adding whitelisted packages to clean DBs ..."
 	local pkgfiles=()
 	case "$UPSTREAM" in
 		packages|community)
@@ -194,7 +194,8 @@ make_repo_dbs() {
 	msg2 "Updating %s-%s databases" "$2" "$1"
 	rsync "${extra[@]}" --no-motd -rtlpH \
 		--delay-updates \
-		--safe-links \
+		--delete-after \
+		--links \
 		"$from/" "$into"
 }
 
@@ -310,6 +311,8 @@ main() {
 	# So, just nuke the current repos and entirely re-create
 	# everything from /new/.
 
+	mkdir "${WORKDIR}/staging-rsync"
+
 	local whitelists=()
 	local reponame
 	for _tag in "${ARCHTAGS[@]}"; do
@@ -331,10 +334,13 @@ main() {
 				# later sync_pool() all packages
 				whitelists+=("${WORKDIR}/${reponame}-${_arch}.whitelist")
 				# Get repo packages (symlinks)
+				mkdir "${WORKDIR}/staging-rsync/${_tag}"
+				ln -t "${WORKDIR}/staging-rsync/${_tag}" \
+					"${FTP_BASE}/${reponame}/os/${_arch}"/*
 				sync_repo \
-					"${ARCHMIRROR_fullmodule}/${reponame}/os/${_arch}/" \
+					"${ARCHMIRROR_fullmodule}/$(get_repo_dir "${reponame}" "${_arch}")/" \
 					"${WORKDIR}/${reponame}-${_arch}.whitelist" \
-					"${FTP_BASE}/${reponame}/os/${_arch}/"
+					"${WORKDIR}/staging-rsync/${_tag}/"
 				;;
 			archlinux32|archlinuxarm)
 				# Upstream doesn't use an $ARCHPKGPOOL
diff --git a/systemd/db-import@.service b/systemd/db-import@.service
index 884d4af..f8d0679 100644
--- a/systemd/db-import@.service
+++ b/systemd/db-import@.service
@@ -8,5 +8,3 @@ Type=oneshot
 User=repo
 Environment=DBSCRIPTS_CONFIG=/etc/dbscripts/config.local.%I
 ExecStart=/usr/bin/db-import-pkg %I
-
-PrivateTmp=true