path: root/db-import-pkg

#!/bin/bash
# Imports packages from upstream Arch repos.
#
# License: GPLv3

set -eE
shopt -s extglob globstar nullglob
source "$(librelib messages)"
setup_traps

# usage: expac_file <file.db> <expac_args>
#
# Uses the ${WORKDIR} global
expac_file() {
	local dbfile=$1
	local args=("${@:2}")

	local reponame=${dbfile##*/}
	reponame=${reponame%%.*}

	mkdir -p -- "${WORKDIR}/expac/root"
	cat >"${WORKDIR}/expac/pacman.conf" <<-EOT
	[options]
	RootDir = ${WORKDIR}/expac/root
	DBPath = ${WORKDIR}/expac/root

	[${reponame}]
	Server = file://$(realpath --no-symlinks -- "${dbfile%/*}")
	EOT

	fakeroot pacman --config="${WORKDIR}/expac/pacman.conf" -Syy >/dev/null
	# expac exits with non-zero on emtpy databases, so ignore errors
	expac --config="${WORKDIR}/expac/pacman.conf" --sync "${args[@]}" || true
}

# usage: fetch_dbs <from> <into>
#
# Fetch excluding everything but db files
# TODO: we could be doing without things other than what is in
#       ${ARCHTAGS[@]}
fetch_dbs() {
	local extra=()
	if [[ $arg_verbose = true ]]; then
		extra+=(-v)
	fi

	rsync "${extra[@]}" --no-motd -mrtLH --no-p \
		--include="*/" \
		--include="*.db" \
		--exclude="*" \
		--delete-after \
		"$1" "$2"
}

# usage: get_repo_dir <repo> <arch>
#
# Prints repo directory path for the given <repo> <arch> combination,
# relative to the rsync root.
get_repo_dir() {
	repo=$1 arch=$2 envsubst '$repo $arch' <<<"$ARCHPATH"
}

# usage: db_list_pkgs <path-to-db>
#
# Prints a list of packages within a given <path-to-db>, one-per-line,
# in the format:
#
#     pkgname [epoch:]pkgver-pkgrel
db_list_pkgs() {
	expac_file "$1" '%n %v' | sort -u
}

# usage: filter_blacklisted <FULL_LIST >FILTERED_LIST
#
# Given a list of packages in the format:
#
#     pkgname [epoch:]pkgver-pkgrel
#
# filter out all of the packages named in blacklist.txt.
filter_blacklisted() {
	sort -u | join -v1 \
		- \
		<(libreblacklist cat | libreblacklist get-pkg | sort -u)
}

# usage: filter_duplicates <FULL_LIST >FILTERED_LIST
#
# Given a list of packages in the format:
#
#     pkgname [epoch:]pkgver-pkgrel
#
# filter out arch=(any) packages present elsewhere, as it confuses
# parabolaweb, librechroot, and who-knows-what-else.  This only
# filters exact pkgname/epoch/pkgver/pkgrel matches.
filter_duplicates() {
	sort -u | comm -23 - <(
		for pool in "${INHERIT[@]}"; do
			for f in "${FTP_BASE}/${pool}"/*-any${PKGEXTS}; do
				f=${f##*/}
				f=${f%-any$PKGEXTS}
				pkgname=${f%-*-*}
				fullpkgver=${f#"${pkgname}-"}
				printf '%s %s\n' "$pkgname" "$fullpkgver"
			done
		done | sort -u
	)
}

# usage: sync_pool <from> <path-to-whitelist> <into>
#
# Sync excluding everything but whitelist
sync_pool() {
	local -r _from=$1 _whitelist=$2 _into=$3

	local extra=()
	if [[ $arg_verbose = true ]]; then
		extra+=(-v)
	fi

	mkdir -p -- "$_into"
	msg2 "Retrieving %d packages from %s pool" \
		"$(wc -l < "$_whitelist")" \
		"$(basename "$_into")"

	# *Don't delete-after*, this is the job of
	# cleanup scripts. It will remove our packages too
	rsync "${extra[@]}" --no-motd -rtlH --no-t \
		--delay-updates \
		--safe-links \
		--include-from="$_whitelist" \
		--exclude="*" \
		"$_from" \
		"$_into"
}

# usage: poolify <arch>
#
# Given a list of packages in the format:
#
#     pkgname [epoch:]pkgver-pkgrel
#
# Resolve each to a file in `${FTP_BASE}/pool/`.  The output is
# relative to `${FTP_BASE}/pool/`.  That is, something along the lines
# of:
#
#     poolname/pkgname-[epoch:]pkgver-pkgrel-arch.pkg.tar.xz
#     archlinux32/zip-3.0-7-i686.pkg.tar.xz
#     packages/rhino-1.7.7.1-1-any.pkg.tar.xz
poolify() {
	local -r arch=$1

	local pkgname fullpkgver
	local restore paths path
	while read -r pkgname fullpkgver; do
		paths=()
		for pool in "${INHERIT[@]}" "$PKGPOOL"; do
			paths+=(
				"${FTP_BASE}/${pool}/${pkgname}-${fullpkgver}-any"${PKGEXTS}
				"${FTP_BASE}/${pool}/${pkgname}-${fullpkgver}-${arch}"${PKGEXTS}
			)
		done
		path="${paths[0]:-}"
		if ! [[ -f "$path" && -f "${path}.sig" ]]; then
			error "No file was found for %q=%q, aborting" "$pkgname" "$fullpkgver"
			exit 1
		fi
		printf '%s\n' "${path#"${FTP_BASE}/pool/"}"
	done
}

# usage: make_repo_symlinks TAG <POOLFILELIST
#
# Uses the ${WORKDIR} global
make_repo_symlinks() {
	local -r tag=$1

	local -r repo=${tag%-*}
	local -r arch=${tag##*-}
	local -r repodir=${WORKDIR}/staging-rsync/${repo}/os/${arch}

	msg2 "Putting symlinks in %s" "${repo}/os/${arch}"
	mkdir -p -- "${repodir}"

	local poolfile
	while read -r poolfile; do
		ln -sfvT "../../../pool/${poolfile##*/pool/}"     "${repodir}/${poolfile##*/}"
		ln -sfvT "../../../pool/${poolfile##*/pool/}.sig" "${repodir}/${poolfile##*/}.sig"
	done
}

# usage: make_repo_dbs <repo> <arch>
#
# Uses the ${WORKDIR} global
make_repo_dbs() {
	local -r from=${WORKDIR}/staging-rsync/${1}/os/${2}
	local -r into=${FTP_BASE}/${1}/os/${2}/
	local -r db_file=${from}/${1}${DBEXT}
	local -r files_file=${from}/${1}${FILESEXT}

	# create fresh databases to reflect actual `any.pkg.tar.xz` packages.
	# this also avoids corrupt upstream metadata (ALARM)
	msg2 "Adding whitelisted packages to clean DBs ..."

	pushd "${from}"
	local -r UMASK=$(umask)
	umask 002
	repo-add "${db_file##*/}" *${PKGEXTS}
	umask "$UMASK" >/dev/null
	popd >/dev/null

	mkdir -p -- "$into"
	# This bit is based on db-functions:set_repo_permission()
	local -r group=$(/usr/bin/stat --printf='%G' "${into}")
	chgrp "$group" "${db_file}"
	chgrp "$group" "${files_file}"
	chmod g+w "${db_file}"
	chmod g+w "${files_file}"

	msg2 "Updating %s-%s databases" "$2" "$1"
	rsync "${extra[@]}" --no-motd -rtlpH --no-t \
		--delay-updates \
		--delete-after \
		--links \
		"$from/" "$into"
}

# Main function. Process the databases and get the libre packages
# Outline:
#  1. Fetch package info
#     * Get blacklist.txt
#     * Get repo.db from an Arch-like repo
#  2. Figure out what we want
#     * Generate textfiles describing the current repo state, (using
#       blacklist.txt) the desired repo state, and how to get from one
#       to the other.
#  3. Fetch the packages we want
#     * Create sync whitelist (based on package blacklist)
#     * Call sync_pool to fetch packages and signatures
#  4. Put the packages in the repos
#     * Create new repo.db with them (repo-add)
#     * rsync scratch directory => repos
#
# Files:
#   (misc)
#     - ${WORKDIR}/expac/                 : Scratch directory for expac_file()
#   (download)
#     - ${WORKDIR}/rsync/                 : Where we download '.db' files to
#     - ${FTP_BASE}/${PKGPOOL}            : Where we download packages to
#     - ${FTP_BASE}/${SRCPOOL}            : Where we download sources to
#   (analysis)
#     - ${FTP_BASE}/${INHERIT}            : Where we look for duplicate files
#     - ${FTP_BASE}/.../${repo}.db        : Where we generate ${WORKDIR}/old/ from
#     - ${WORKDIR}/old/                   : .txt files describing the way the repos are
#     - ${WORKDIR}/new/                   : .txt files describing the way we want them to be
#     - ${WORKDIR}/dif/                   : .txt files describing how to make it happen
#     - ${WORKDIR}/all.whitelist          : Glob list of (source-)package filenames to download
#     - ${WORKDIR}/${tag}.whitelist       : Glob list of (source-)package filenames to download
#     - ${WORKDIR}/${tag}.pool            : See poolify()
#   (release)
#     - ${WORKDIR}/staging-rsync/         : .db files and symlinks we'll rsync in to ${FTP_BASE}
#     - ${FTP_BASE}/                      : Where we rsync ${WORKDIR}/staging-rsync/ to
#     - ${FTP_BASE}/lastupdate            : Timestamp file of last repo change
main() {
	##############################################################
	# 0. Initialization                                          #
	##############################################################

	# Run as `V=true db-import-pkg` to get verbose output
	declare -r arg_verbose="$V"

	# Print usage message
	if [[ $# -ne 0 ]] || [[ -z "$DBSCRIPTS_CONFIG" ]] || ! grep -q ARCHMIRROR -- "$DBSCRIPTS_CONFIG"; then
		msg 'usage: [V=true] DBSCRIPTS_CONFIG=/path/to/file %s' "${0##*/}"
		exit $EXIT_INVALIDARGUMENT
	fi

	local config_file
	config_file="$(dirname "$(readlink -e "$0")")/config"
	source "$config_file"

	local ret=0 varname varref
	for varname in PKGEXTS DBEXT FILESEXT FTP_BASE PKGPOOL SRCPOOL; do
		if [[ -z ${!varname:-} ]] || is_array "$varname"; then
			print "Configure '%s' as a non-empty string in %q (or %q):" "$varname" "$config_file" "$LOCAL_CONFIG"
			ret=$EXIT_NOTCONFIGURED
		fi
	done
	for varname in ARCHMIRROR ARCHPATH; do # optional: ARCHPKGPOOL ARCHSRCPOOL
		if [[ -z ${!varname:-} ]] || is_array "$varname"; then
			print "Configure '%s' as a non-empty string in DBSCRIPTS_CONFIG=%q (did you set DBSCRIPTS_CONFIG correctly?):" "$varname" "$LOCAL_CONFIG"
			ret=$EXIT_NOTCONFIGURED
		fi
	done
	for varname in ARCHTAGS; do # optional: INHERIT
		declare -n varref="$varname"
		if (( ${#varref[*]} == 0 )) || ! is_array "$varname"; then
			print "Configure '%s' as a non-empty array in DBSCRIPTS_CONFIG=%q (did you set DBSCRIPTS_CONFIG correctly?):" "$varname" "$LOCAL_CONFIG"
			ret=$EXIT_NOTCONFIGURED
		fi
	done

	if [[ -n ${ARCHSRCPOOL:-} && -z ${ARCHPKGPOOL:-} ]]; then
		print '%s requires that %s is also set' ARCH{SRC,PKG}POOL
		ret=$EXIT_NOTCONFIGURED
	fi

	if (( ret != 0 )); then
		exit $ret
	fi

	WORKDIR=$(mktemp -dt "${0##*/}.XXXXXXXXXX")
	readonly WORKDIR
	trap "rm -rf -- ${WORKDIR@Q}" EXIT

	##############################################################
	# 1. Fetch package info                                      #
	##############################################################

	# Get the blacklisted packages
	libreblacklist update

	# Sync the repos databases
	msg 'Downloading .db and .files files to import'
	mkdir "${WORKDIR}/rsync"
	fetch_dbs "${ARCHMIRROR}/" "${WORKDIR}/rsync"

	##############################################################
	# 2. Figure out what we want                                 #
	##############################################################

	mkdir "${WORKDIR}"/{old,new,dif}
	local _tag _repo _arch db_file
	for _tag in "${ARCHTAGS[@]}"; do
		_repo=${_tag%-*}
		_arch=${_tag##*-}
		# FIXME: this should use db-functions to lock the
		# repos while we read them.
		db_file="${FTP_BASE}/${_repo}/os/${_arch}/${_repo}.db"
		db_list_pkgs "$db_file" > "${WORKDIR}/old/${_tag}.txt"

		db_file="${WORKDIR}/rsync/$(get_repo_dir "${_repo}" "${_arch}")/${_repo}.db"
		db_list_pkgs "$db_file" | filter_blacklisted > "${WORKDIR}/new/${_tag}.txt"
	done

	# We now have ${WORKDIR}/old/ describing the way the repos
	# are, and ${WORKDIR}/new/ describing the way we want them to
	# be.  We now create ${WORKDIR}/dif/ describing how to get
	# from point A to point B.
	cat "${WORKDIR}"/old/*-*.txt | sort -u > "${WORKDIR}/old/all.txt"
	cat "${WORKDIR}"/new/*-*.txt | sort -u > "${WORKDIR}/new/all.txt"
	# db-move <repo-from> <repo-to> <pkgname|pkgbase> ...
	#
	# db-move doesn't allow us to limit the operation to a
	# specific arch, but the DBSCRIPTS_CONFIG will limit what
	# arches it applies to, and we currently only import 1 arch
	# from each upstream.
	local tag_from tag_to
	for tag_from in "${ARCHTAGS[@]}"; do
		arch_from=${tag_from##*-}
		for tag_to in "${ARCHTAGS[@]}"; do
			arch_to=${tag_to##*-}
			[[ $tag_from != $tag_to ]] || continue
			[[ $arch_from == $arch_to ]] || continue

			comm -12 \
			     "${WORKDIR}/old/${tag_from}.txt" \
			     "${WORKDIR}/new/${tag_to}.txt" \
			     >> "${WORKDIR}/dif/move:${tag_from}:${tag_to}.txt"
		done
	done
	# db-update
	local tag
	for tag in "${ARCHTAGS[@]}"; do
		comm -13 \
			"${WORKDIR}/old/all.txt" \
			"${WORKDIR}/new/${tag}.txt" \
			> "${WORKDIR}/dif/update:${tag}.txt"
	done
	# db-remove <repo> <arch> <pkgname|pkgbase> ...
	for tag in "${ARCHTAGS[@]}"; do
		comm -23 \
		     "${WORKDIR}/old/${tag}.txt" \
		     "${WORKDIR}/new/all.txt" \
		     > "${WORKDIR}/dif/remove:${tag}.txt"
	done

	##############################################################
	# 3. Fetch the packages we want                              #
	##############################################################

	# OK, now we have ${WORKDIR}/old/ describing the way the repos
	# are, ${WORKDIR}/new/ describing the way we want them to be,
	# and ${WORKDIR}/dif/ describing how to get from `old` to
	# `new`.  We should (TODO) now use db-move, db-update, and
	# db-remove to apply that diff.
	#
	# But,
	#  - db-move is broken
	#  - The code that populates /dif/ isn't finished
	# So, just nuke the current repos and entirely re-create
	# everything from /new/.

	local whitelists=()
	for _tag in "${ARCHTAGS[@]}"; do
		msg "Processing %s" "$_tag"
		_repo=${_tag%-*}
		_arch=${_tag##*-}
		# Create a whitelist, add * wildcard to end.
		#
		# FIXME: due to lack of -arch suffix, the pool sync
		# retrieves every arch even if we aren't syncing them.
		#
		# IMPORTANT: the . in the sed command is needed
		# because an empty whitelist would consist of a single
		# * allowing any package to pass through.
		filter_duplicates \
			<"${WORKDIR}/new/${_tag}.txt" \
			| sed -e 's/ /-/' -e 's|.$|&*|g' \
			> "${WORKDIR}/${_tag}.whitelist"
		if [[ -n ${ARCHPKGPOOL:-} ]]; then
			# Append to whitelists array so that we can
			# later sync_pool() all packages
			whitelists+=("${WORKDIR}/${_tag}.whitelist")
		else
			# Upstream doesn't use an $ARCHPKGPOOL
			sync_pool \
				"${ARCHMIRROR}/$(get_repo_dir "${_repo}" "${_arch}")/" \
				"${WORKDIR}/${_tag}.whitelist" \
				"${FTP_BASE}/${PKGPOOL}/"
			poolify "${_arch}" \
				<"${WORKDIR}/new/${_tag}.txt" \
				>"${WORKDIR}/${_tag}.pool"
			make_repo_symlinks "$_tag" \
				<"${WORKDIR}/${_tag}.pool"
		fi
	done

	if (( ${#whitelists[@]} > 0 )); then
		# Concatenate all whitelists, check for single *s just in case
		cat "${whitelists[@]}" | grep -v "^\*$" |
			sort -u > "${WORKDIR}/all.whitelist"
		# FIXME: make_whitelist() wildcards should be narrowed
		#        down to respect the architecture of the tag

		msg "Syncing package pool"
		sync_pool \
			"${ARCHMIRROR}/${ARCHPKGPOOL}/" \
			"${WORKDIR}/all.whitelist" \
			"${FTP_BASE}/${PKGPOOL}/"
		for _tag in "${ARCHTAGS[@]}"; do
			_repo=${_tag%-*}
			_arch=${_tag##*-}
			poolify "${_arch}" "${PKGPOOL}" \
				<"${WORKDIR}/new/${_tag}.txt" \
				>"${WORKDIR}/${_tag}.pool"
			make_repo_symlinks "$_tag" \
				<"${WORKDIR}/${_tag}.pool"
		done

		if [[ -n ${ARCHSRCPOOL:-} ]]; then
			msg "Syncing source pool"
			sync_pool \
				"${ARCHMIRROR}/${ARCHSRCPOOL}/" \
				"${WORKDIR}/all.whitelist" \
				"${FTP_BASE}/${SRCPOOL}/"
		fi
	fi

	##############################################################
	# 4. Put the packages in the repos                           #
	##############################################################

	msg "Putting databases back in place"

	# FIXME: all repo DBs should be replaced at once (per architecture)
	ln -srT "$FTP_BASE/pool" "${WORKDIR}/staging-rsync/pool"
	for _tag in "${ARCHTAGS[@]}"; do
		_repo=${_tag%-*}
		_arch=${_tag##*-}
		make_repo_dbs "$_repo" "$_arch"
	done
	date +%s > "${FTP_BASE}/lastupdate"
}

main "$@"