add db-import-archlinux32-pkg

These are the bare minimum changes to start importing i686 packages
from archlinux32.org

9 files changed, 262 insertions, 41 deletions

diff --git a/config b/config
index 36c0f99..b4fab5a 100644..100755
--- a/config
+++ b/config
@@ -13,7 +13,7 @@ PKGREPOS=(
 	'nonprism' 'nonprism-testing' 'nonprism-multilib'
 	'nonprism-multilib-testing' 'pcr' 'pcr-testing' 'pcr-multilib'
 	'pcr-multilib-testing' 'pcr-testing' 'kernels' 'kernels-testing'
-	'cross' 'java' 'unmaintained')
+	'cross' 'java' 'unmaintained' 'build-support')
 PKGPOOL='pool/parabola'
 SRCPOOL='sources/parabola'
 
@@ -33,7 +33,7 @@ LOCK_DELAY=10
 
 [ -n "${STAGING:-}" ] || STAGING="$HOME/staging/unknown/staging"
 TMPDIR="/tmp"
-ARCHES=(i686 x86_64 armv7h)
+ARCHES=(x86_64 i686 armv7h)
 DBEXT=".db.tar.gz"
 FILESEXT=".files.tar.gz"
 PKGEXT=".pkg.tar.?z"
diff --git a/db-cleanup.conf b/db-cleanup.conf
index b8d5b86..12d65f9 100644..100755
--- a/db-cleanup.conf
+++ b/db-cleanup.conf
@@ -4,16 +4,18 @@
 
 # Directories where packages are shared between repos
 PKGPOOLS=(
-	pool/parabola  # Parabola GNU/Linux-libre
-	pool/alarm     # Arch Linux ARM
-	pool/packages  # Arch Linux (project)
-	pool/community # Arch Linux (community)
+	pool/parabola    # Parabola GNU/Linux-libre
+	pool/alarm       # Arch Linux ARM
+	pool/archlinux32 # Arch Linux 32-bits
+	pool/packages    # Arch Linux (project)
+	pool/community   # Arch Linux (community)
 )
 
 # Directories where sources are stored
 SRCPOOLS=(
-	sources/parabola  # Parabola GNU/Linux-libre
-	sources/alarm     # Arch Linux ARM
-	sources/packages  # Arch Linux (project)
-	sources/community # Arch Linux (community)
+	sources/parabola    # Parabola GNU/Linux-libre
+	sources/alarm       # Arch Linux ARM
+	sources/archlinux32 # Arch Linux 32-bits
+	sources/packages    # Arch Linux (project)
+	sources/community   # Arch Linux (community)
 )
diff --git a/db-import-archlinux-pkg b/db-import-archlinux-pkg
index d43a245..2e0b1d1 100755
--- a/db-import-archlinux-pkg
+++ b/db-import-archlinux-pkg
@@ -6,10 +6,8 @@
 # * Get repo.db from an Arch-like repo
 # * Generate a list of available packages
 # * Create sync whitelist (based on package blacklist)
-# * Get packages
-# * Check package signatures
-# * Check database signatures
-# * Sync repo => repo
+# * Get packages and signatures
+# * Sync repo.db => repo.db
 
 # TODO
 # * make a tarball of files used for forensics
@@ -118,6 +116,7 @@ init() {
 			whitelists+=(/tmp/${_repo}-${_arch}.whitelist)
 
 			msg "Putting databases back in place"
+			mkdir -p -- "${FTP_BASE}/${_repo}/os/${_arch}/"
 			rsync "${extra[@]}" --no-motd -rtlH \
 				--delay-updates \
 				--safe-links \
@@ -125,7 +124,7 @@ init() {
 				"${FTP_BASE}/${_repo}/os/${_arch}/"
 
 			# Cleanup
-			unset db 
+			unset db
 		done
 	done
 
diff --git a/db-import-archlinux.conf b/db-import-archlinux.conf
index 891cb98..3c4af63 100644
--- a/db-import-archlinux.conf
+++ b/db-import-archlinux.conf
@@ -1,8 +1,8 @@
 ARCHREPOS=('core' 'testing' 'extra' 'community' 'multilib' 'multilib-testing')
 ARCHPKGPOOLS=(pool/{packages,community})
 ARCHSRCPOOLS=(sources/{packages,community})
-ARCHARCHES=(i686 x86_64)
-OURARCHES=(armv7h)
+ARCHARCHES=(x86_64)
+OURARCHES=(armv7h i686)
 
 mirror="archlinux.mirror.pkern.at"
 #mirror="mirrors.kernel.org"
diff --git a/db-import-archlinux32-pkg b/db-import-archlinux32-pkg
new file mode 100755
index 0000000..233f153
--- /dev/null
+++ b/db-import-archlinux32-pkg
@@ -0,0 +1,195 @@
+#!/bin/bash
+# Syncs Arch repos based on info contained in repo.db files
+# License: GPLv3
+
+# Principles
+# * Get repo.db from an Arch-like repo
+# * Generate a list of available packages
+# * Create sync whitelist (based on package blacklist)
+# * Get packages and signatures
+# * Create new repo.db with them
+# * Sync repo.db => repo.db
+
+# TODO
+# * make a tarball of files used for forensics
+
+set -e
+
+# Run as `V=true db-import-pkg-archlinux` to get verbose output
+VERBOSE=${V}
+extra=()
+${VERBOSE} && extra+=(-v)
+
+WORKDIR=$(mktemp -dt "${0##*/}.XXXXXXXXXX")
+trap "rm -rf -- $(printf '%q' "${WORKDIR}")" EXIT
+
+# Returns contents of a repo
+get_repos() {
+	# Exclude everything but db files
+	rsync "${extra[@]}" --no-motd -mrtlH --no-p --include="*/" \
+		--include="*.db" \
+		--include="*${DBEXT}" \
+		--include="*.files" \
+		--include="*${FILESEXT}" \
+		--exclude="*" \
+		--delete-after \
+		"rsync://${mirror}/${mirrorpath}/" "$WORKDIR"
+}
+
+get_repo_content() {
+	# Return all contents
+	bsdtar tf "${1}" | \
+		cut -d "/" -f 1 | \
+		sort -u
+}
+
+# repo
+# arch
+get_repo_file() {
+	echo "${WORKDIR}/${2}/${1}/${1}"
+}
+
+# Process the databases and get the libre packages
+init() {
+
+	# Get the blacklisted packages
+	libreblacklist update
+	blacklist=($(libreblacklist cat | libreblacklist get-pkg))
+
+	msg "%d packages in blacklist" ${#blacklist[@]}
+
+	test ${#blacklist[@]} -eq 0 && fatal_error "Empty blacklist"
+
+	# Sync the repos databases
+	get_repos
+
+	# Traverse all repo-arch pairs
+	for _arch in "${ARCHARCHES[@]}"; do
+		for _repo in "${ARCHREPOS[@]}"; do
+			msg "Processing %s-%s" "${_repo}" "${_arch}"
+
+			db_file=$(get_repo_file "${_repo}" "${_arch}")${DBEXT}
+			files_file=$(get_repo_file "${_repo}" "${_arch}")${FILESEXT}
+
+			if [ ! -f "${db_file}" ]; then
+				warning "%s doesn't exist, skipping this arch-repo" "${db_file}"
+				continue
+			fi
+			if [ ! -f "${files_file}" ]; then
+				warning "%s doesn't exist, skipping this arch-repo" "${files_file}"
+				continue
+			fi
+
+			# Remove blacklisted packages and count them
+			# TODO capture all removed packages for printing on debug mode
+			msg2 "Removing blacklisted packages from %s and %s databases..." .db .files
+			LC_ALL=C repo-remove "${db_file}" "${blacklist[@]}" \
+			|& sed -n 's/-> Removing/	&/p'
+			# Get db contents
+			db=($(get_repo_content "${db_file}"))
+
+			msg2 "Process clean db for syncing..."
+
+			# Create a whitelist, add * wildcard to end
+			# TODO due to lack of -arch suffix, the pool sync retrieves every arch even if
+			# we aren't syncing them
+			# IMPORTANT: the . in the sed command is needed because an empty
+			# whitelist would consist of a single * allowing any package to
+			# pass through
+			printf '%s\n' "${db[@]}" | sed "s|.$|&*|g" > "/tmp/${_repo}-${_arch}.whitelist"
+
+			msg2 "%d packages in whitelist" "$(wc -l /tmp/${_repo}-${_arch}.whitelist | cut -d' ' -f1)"
+
+			msg2 "Retrieving %d packages to pool" "$(wc -l /tmp/${_repo}-${_arch}.whitelist | cut -d' ' -f1)"
+
+			# Sync excluding everything but whitelist
+			rsync "${extra[@]}" --no-motd -rtlH \
+				--delay-updates \
+				--safe-links \
+				--include-from="/tmp/${_repo}-${_arch}.whitelist" \
+				--exclude="*" \
+				"rsync://${mirror}/${mirrorpath}/${_arch}/${_repo}/" \
+				"${FTP_BASE}/${ARCHPKGPOOLS}/"
+
+
+			msg "Putting databases back in place"
+			mkdir -p -- "${FTP_BASE}/${_repo}/os/${_arch}/"
+			rsync "${extra[@]}" --no-motd -rtlH \
+				--delay-updates \
+				--safe-links \
+				"${WORKDIR}/${_arch}/${_repo}/" \
+				"${FTP_BASE}/${_repo}/os/${_arch}/"
+
+			# Cleanup
+			unset db
+		done
+	done
+
+	msg "Generating symbolic links to pool"
+
+	for _arch in "${ARCHARCHES[@]}"; do
+		for _repo in "${ARCHREPOS[@]}"; do
+			# Modify whitelist to search packages and create symlinks
+			sed -i "s/*/-${_arch}.pkg.tar.xz/g" "/tmp/${_repo}-${_arch}.whitelist"
+
+			msg "Putting symlinks in ${_repo}/os/${_arch}"
+
+			while read _pkgfile; do
+				# Symlink to package
+				if [ -f "${FTP_BASE}/${ARCHPKGPOOLS}/${_pkgfile}" ]; then
+					ln -sfv "../../../${ARCHPKGPOOLS}/${_pkgfile}" \
+					"${FTP_BASE}/${_repo}/os/${_arch}/${_pkgfile}"
+				elif [ -f "${FTP_BASE}/${ARCHPKGPOOLS}/${_pkgfile/${_arch}/any}" ]; then
+					ln -sfv "../../../${ARCHPKGPOOLS}/${_pkgfile/${_arch}/any}" \
+					"${FTP_BASE}/${_repo}/os/${_arch}/${_pkgfile/${_arch}/any}"
+				fi
+
+				# Symlink to signature
+				if [ -f "${FTP_BASE}/${ARCHPKGPOOLS}/${_pkgfile}.sig" ]; then
+					ln -sfv "../../../${ARCHPKGPOOLS}/${_pkgfile}.sig" \
+					"${FTP_BASE}/${_repo}/os/${_arch}/${_pkgfile}.sig"
+				elif [ -f "${FTP_BASE}/${ARCHPKGPOOLS}/${_pkgfile/${_arch}/any}.sig" ]; then
+					ln -sfv "../../../${ARCHPKGPOOLS}/${_pkgfile/${_arch}/any}.sig" \
+					"${FTP_BASE}/${_repo}/os/${_arch}/${_pkgfile/${_arch}/any}.sig"
+				fi
+			done < "/tmp/${_repo}-${_arch}.whitelist"
+		done
+	done
+
+	date -u +%s > "${FTP_BASE}/lastsync"
+
+	# Cleanup
+	unset blacklist _arch _repo repo_file _pkgfile
+}
+
+trap_exit() {
+	local signal=$1; shift
+	echo
+	error "$@"
+	trap -- "$signal"
+	kill "-$signal" "$$"
+}
+
+fatal_error() {
+	error "$@"
+	exit 1
+}
+
+source "$(dirname "$(readlink -e "$0")")/config"
+source "$(dirname "$(readlink -e "$0")")/db-import-archlinux32.conf"
+source "$(librelib messages)"
+
+# Check variables presence
+for var in DBEXT FILESEXT mirror mirrorpath WORKDIR FTP_BASE; do
+	test -z "${!var}" && fatal_error "Empty %s" "${var}"
+done
+
+# From makepkg
+set -E
+for signal in TERM HUP QUIT; do
+	trap "trap_exit $signal '%s signal caught. Exiting...' $signal" "$signal"
+done
+trap 'trap_exit INT "Aborted by user! Exiting..."' INT
+trap 'trap_exit USR1 "An unknown error has occurred. Exiting..."' ERR
+
+init
diff --git a/db-import-archlinux32.conf b/db-import-archlinux32.conf
new file mode 100644
index 0000000..a6f17c9
--- /dev/null
+++ b/db-import-archlinux32.conf
@@ -0,0 +1,26 @@
+ARCHREPOS=('build-support' 'core' 'testing' 'extra' 'community')
+ARCHPKGPOOLS='pool/archlinux32'
+# ARCHSRCPOOLS='sources/archlinux32' # archlinux32 doesn't provide .src tarballs
+ARCHARCHES=(i686)
+# OURARCHES=(i686) # useless?
+
+## mirrors WITH sources folder
+# (none)
+
+## mirrors without sources folder
+mirror="mirror.archlinux32.org"
+mirror="32.arlm.tyzoid.com"
+
+mirrorpath="archlinux32"
+
+# The followng is for db-import-archlinux32-src
+
+# NB: I am unsure if the presence or absence of a trailing slash in
+#     the _ROOT variables is significant. -- lukeshu
+SYNCARGS='-mrtv --no-motd --delete-after --no-p --no-o --no-g --quiet'
+ABSLIBRE_ROOT='/srv/repo/db-import-archlinux32-src/abslibre'
+ABSLIBRE_GIT='https://git.parabola.nu/abslibre.git'
+ABS_ROOT='/srv/repo/db-import-archlinux32-src/abs/'
+ABS_SERVER="${mirror}/${mirrorpath}/"
+ALARM_GIT='https://github.com/archlinux32/packages'
+ALARM_ROOT='/srv/repo/db-import-archlinux32-src/archlinux32/'
diff --git a/db-import-archlinuxarm-pkg b/db-import-archlinuxarm-pkg
index 96dfc41..d6465e6 100755
--- a/db-import-archlinuxarm-pkg
+++ b/db-import-archlinuxarm-pkg
@@ -43,7 +43,6 @@ get_repo_content() {
 		sort -u
 }
 
-
 # repo
 # arch
 get_repo_file() {
@@ -56,8 +55,6 @@ init() {
 	# Get the blacklisted packages
 	libreblacklist update
 	blacklist=($(libreblacklist cat | libreblacklist get-pkg))
-	# Store all the whitelist files
-	whitelists=()
 
 	msg "%d packages in blacklist" ${#blacklist[@]}
 
@@ -67,8 +64,8 @@ init() {
 	get_repos
 
 	# Traverse all repo-arch pairs
-	for _arch in "${OURARCHES[@]}"; do
-		for _repo in "${ARMREPOS[@]}"; do
+	for _arch in "${ARCHARCHES[@]}"; do
+		for _repo in "${ARCHREPOS[@]}"; do
 			msg "Processing %s-%s" "${_repo}" "${_arch}"
 
 			db_file=$(get_repo_file "${_repo}" "${_arch}")${DBEXT}
@@ -112,7 +109,7 @@ init() {
 				--include-from="/tmp/${_repo}-${_arch}.whitelist" \
 				--exclude="*" \
 				"rsync://${mirror}/${mirrorpath}/${_arch}/${_repo}/" \
-				"${FTP_BASE}/${PKGPOOLARM}/"
+				"${FTP_BASE}/${ARCHPKGPOOLS}/"
 
 			# Avoid sloppy upstream databases with corrupt package descriptions,
 			# exclude .sig files from consideration.
@@ -120,11 +117,12 @@ init() {
 			# Optional, repo-add overwrites entries by default
 			rm "$db_file" "$files_file"
 			LC_ALL=C repo-add "$db_file" \
-				$(sed "s|^|${FTP_BASE}/${PKGPOOLARM}/|; s|$|${PKGEXT}|" \
+				$(sed "s|^|${FTP_BASE}/${ARCHPKGPOOLS}/|; s|$|${PKGEXT}|" \
 					"/tmp/${_repo}-${_arch}.whitelist") \
 			|& sed -n 's/==> Adding/	-> Adding/p'
 
 			msg "Putting databases back in place"
+			mkdir -p -- "${FTP_BASE}/${_repo}/os/${_arch}/"
 			rsync "${extra[@]}" --no-motd -rtlH \
 				--delay-updates \
 				--safe-links \
@@ -136,11 +134,10 @@ init() {
 		done
 	done
 
-
 	msg "Generating symbolic links to pool"
 
-	for _arch in "${OURARCHES[@]}"; do
-		for _repo in "${ARMREPOS[@]}"; do
+	for _arch in "${ARCHARCHES[@]}"; do
+		for _repo in "${ARCHREPOS[@]}"; do
 			# Modify whitelist to search packages and create symlinks
 			sed -i "s/*/-${_arch}.pkg.tar.xz/g" "/tmp/${_repo}-${_arch}.whitelist"
 
@@ -148,20 +145,20 @@ init() {
 
 			while read _pkgfile; do
 				# Symlink to package
-				if [ -f "${FTP_BASE}/${PKGPOOLARM}/${_pkgfile}" ]; then
-					ln -sfv "../../../${PKGPOOLARM}/${_pkgfile}" \
+				if [ -f "${FTP_BASE}/${ARCHPKGPOOLS}/${_pkgfile}" ]; then
+					ln -sfv "../../../${ARCHPKGPOOLS}/${_pkgfile}" \
 					"${FTP_BASE}/${_repo}/os/${_arch}/${_pkgfile}"
-				elif [ -f "${FTP_BASE}/${PKGPOOLARM}/${_pkgfile/${_arch}/any}" ]; then
-					ln -sfv "../../../${PKGPOOLARM}/${_pkgfile/${_arch}/any}" \
+				elif [ -f "${FTP_BASE}/${ARCHPKGPOOLS}/${_pkgfile/${_arch}/any}" ]; then
+					ln -sfv "../../../${ARCHPKGPOOLS}/${_pkgfile/${_arch}/any}" \
 					"${FTP_BASE}/${_repo}/os/${_arch}/${_pkgfile/${_arch}/any}"
 				fi
 
 				# Symlink to signature
-				if [ -f "${FTP_BASE}/${PKGPOOLARM}/${_pkgfile}.sig" ]; then
-					ln -sfv "../../../${PKGPOOLARM}/${_pkgfile}.sig" \
+				if [ -f "${FTP_BASE}/${ARCHPKGPOOLS}/${_pkgfile}.sig" ]; then
+					ln -sfv "../../../${ARCHPKGPOOLS}/${_pkgfile}.sig" \
 					"${FTP_BASE}/${_repo}/os/${_arch}/${_pkgfile}.sig"
-				elif [ -f "${FTP_BASE}/${PKGPOOLARM}/${_pkgfile/${_arch}/any}.sig" ]; then
-					ln -sfv "../../../${PKGPOOLARM}/${_pkgfile/${_arch}/any}.sig" \
+				elif [ -f "${FTP_BASE}/${ARCHPKGPOOLS}/${_pkgfile/${_arch}/any}.sig" ]; then
+					ln -sfv "../../../${ARCHPKGPOOLS}/${_pkgfile/${_arch}/any}.sig" \
 					"${FTP_BASE}/${_repo}/os/${_arch}/${_pkgfile/${_arch}/any}.sig"
 				fi
 			done < "/tmp/${_repo}-${_arch}.whitelist"
@@ -171,7 +168,7 @@ init() {
 	date -u +%s > "${FTP_BASE}/lastsync"
 
 	# Cleanup
-	unset blacklist whitelists _arch _repo repo_file _pkgfile
+	unset blacklist _arch _repo repo_file _pkgfile
 }
 
 trap_exit() {
diff --git a/db-import-archlinuxarm-src b/db-import-archlinuxarm-src
index c4ed86d..10e00a4 100755
--- a/db-import-archlinuxarm-src
+++ b/db-import-archlinuxarm-src
@@ -17,7 +17,7 @@ function sync_abs() {
 	rm -rf -- "$TMP_SUBDIR" "$ABS_ROOT"
 	mkdir -- "$TMP_SUBDIR"
 	for ARCH in armv7h; do
-		for REPO in "${ARMREPOS[@]}"; do
+		for REPO in "${ARCHREPOS[@]}"; do
 			rsync ${SYNCARGS} \
 			      rsync://${ABS_SERVER}/${ARCH}/${REPO}/${REPO}.abs.tar.gz \
 			      "$TMP_SUBDIR" || return $?
@@ -37,7 +37,7 @@ function sync_abs() {
 	fi
 	# FIXME: $ALARM_ROOT doesn't separate packages by architecture
 	for ARCH in armv7h; do
-		for REPO in "${ARMREPOS[@]}"; do
+		for REPO in "${ARCHREPOS[@]}"; do
 			rsync -mrtq --no-p --no-o --no-g --exclude='README' \
 			      "${ALARM_ROOT}/${REPO}/" \
 			      "${ABS_ROOT}/${ARCH}/${REPO}"
diff --git a/db-import-archlinuxarm.conf b/db-import-archlinuxarm.conf
index fd3a8bf..392acf0 100644
--- a/db-import-archlinuxarm.conf
+++ b/db-import-archlinuxarm.conf
@@ -1,6 +1,8 @@
-ARMREPOS=('core' 'extra' 'community')
-PKGPOOLARM='pool/alarm'
-OURARCHES=(armv7h)
+ARCHREPOS=('core' 'extra' 'community')
+ARCHPKGPOOLS='pool/alarm'
+# ARCHSRCPOOLS='sources/alarm'    # ALARM doesn't provide .src tarballs
+ARCHARCHES=(armv7h)
+# OURARCHES=(armv7h)             # useless?
 
 ## mirrors WITH sources folder
 # (none)