From e3973262cca737d8c174b6292f64de6b1baeb6e6 Mon Sep 17 00:00:00 2001 From: Isaac David Date: Sat, 11 Nov 2017 11:44:16 -0600 Subject: db-import-pkg: move all global setup code to a single place --- db-import-pkg | 137 ++++++++++++++++++++++++++++------------------------------ 1 file changed, 67 insertions(+), 70 deletions(-) diff --git a/db-import-pkg b/db-import-pkg index fd37e37..d5ea8b4 100755 --- a/db-import-pkg +++ b/db-import-pkg @@ -1,19 +1,14 @@ #!/bin/bash -# Syncs Arch repos based on info contained in repo.db files +# Syncs Arch, ALARM or Arch32 repos based on info contained in the +# accompanying .conf files. # License: GPLv3 -# Principles -# * Get repo.db from an Arch-like repo -# * Generate a list of available packages -# * Create sync whitelist (based on package blacklist) -# * Get packages and signatures -# * Create new repo.db with them -# * Sync repo.db => repo.db +set -eE -# TODO -# * make a tarball of files used for forensics +source "$(librelib messages)" +source "$(dirname "$(readlink -e "$0")")/config" -set -e +readonly -a UPSTREAMS=(archlinux{,32,arm}) # Run as `V=true db-import-pkg-archlinux` to get verbose output VERBOSE=${V} @@ -23,6 +18,44 @@ ${VERBOSE} && extra+=(-v) WORKDIR=$(mktemp -dt "${0##*/}.XXXXXXXXXX") trap "rm -rf -- $(printf '%q' "${WORKDIR}")" EXIT +trap_exit() { + local signal=$1; shift + echo + error "$@" + trap -- "$signal" + kill "-$signal" "$$" +} + +# From makepkg +for signal in TERM HUP QUIT; do + trap "trap_exit $signal '%s signal caught. Exiting...' $signal" \ + "$signal" +done +trap 'trap_exit INT "Aborted by user! Exiting..."' INT +trap 'trap_exit USR1 "An unknown error has occurred. Exiting..."' ERR + +fatal_error() { + error "$@" + exit 1 +} + +# Print usage message +if [[ $# -ne 1 ]] || ! in_array "$1" "${UPSTREAMS[@]}" ; then + fatal_error 'usage: %s {%s\b}' \ + "${0##*/}" \ + "$(printf -- ' %s |' "${UPSTREAMS[@]}")" +fi + +readonly UPSTREAM=$1 +source "$(dirname "$(readlink -e "$0")")/db-import-${UPSTREAM}.conf" + +# Check variables presence +vars=(DBEXT FILESEXT mirror mirrorpath WORKDIR FTP_BASE ARCHREPOS ARCHPKGPOOLS) +[[ $UPSTREAM == archlinux ]] && vars+=(ARCHSRCPOOLS) +for var in "${vars[@]}"; do + test -z "${!var}" && fatal_error "Empty %s" "${var}" +done + # usage: sync_dbs # # Sync excluding everything but db files @@ -196,7 +229,14 @@ make_repo_dbs() { "$from" "$into" } -# Process the databases and get the libre packages +# Main function. Process the databases and get the libre packages +# Outline: +# * Get repo.db from an Arch-like repo +# * Generate a list of available packages +# * Create sync whitelist (based on package blacklist) +# * Get packages and signatures +# * Create new repo.db with them +# * Sync repo.db => repo.db init() { # Get the blacklisted packages libreblacklist update @@ -211,41 +251,42 @@ init() { # Traverse all repo-arch pairs local _arch _repo for _arch in "${ARCHARCHES[@]}"; do - for _repo in "${ARCHREPOS[@]}"; do - msg "Processing %s-%s" "${_arch}" "${_repo}" - - local db_file=$(get_repo_workdir "${_repo}" "${_arch}")/${_repo}${DBEXT} - local files_file=$(get_repo_workdir "${_repo}" "${_arch}")/${_repo}${FILESEXT} + for (( _repo = 0; _repo < ${#ARCHREPOS[@]}; ++_repo )); do + local reponame=${ARCHREPOS[$_repo]} + msg "Processing %s-%s" "${_arch}" "${reponame}" + local db_file=$(get_repo_workdir "${reponame}" "${_arch}")/${reponame}${DBEXT} + local files_file=$(get_repo_workdir "${reponame}" "${_arch}")/${reponame}${FILESEXT} local _file for _file in db_file files_file; do if [ ! -f "${!_file}" ]; then warning "%s doesn't exist, skipping this arch-repo" \ "${!_file}" + unset ARCHREPOS[$_repo] continue fi done - make_whitelist "/tmp/${_repo}-${_arch}.whitelist" \ + make_whitelist "/tmp/${reponame}-${_arch}.whitelist" \ "$db_file" \ "${blacklist[@]}" case "$UPSTREAM" in archlinux) # Append to whitelists array so that we can # later sync_pool() all packages - local -a whitelists+=(/tmp/${_repo}-${_arch}.whitelist) + local -a whitelists+=(/tmp/${reponame}-${_arch}.whitelist) # Get repo packages (symlinks) sync_repo \ - "rsync://${mirror}/${mirrorpath}/${_repo}/os/${_arch}/" \ - "/tmp/${_repo}-${_arch}.whitelist" \ - "${FTP_BASE}/${_repo}/os/${_arch}/" + "rsync://${mirror}/${mirrorpath}/${reponame}/os/${_arch}/" \ + "/tmp/${reponame}-${_arch}.whitelist" \ + "${FTP_BASE}/${reponame}/os/${_arch}/" ;; archlinux32|archlinuxarm) # Upstream doesn't use an $ARCHPKGPOOL filter_duplicates \ - < "/tmp/${_repo}-${_arch}.whitelist" \ - > "/tmp/${_repo}-${_arch}-nodups.whitelist" - sync_pool "rsync://${mirror}/${mirrorpath}/${_arch}/${_repo}/" \ - "/tmp/${_repo}-${_arch}-nodups.whitelist" \ + < "/tmp/${reponame}-${_arch}.whitelist" \ + > "/tmp/${reponame}-${_arch}-nodups.whitelist" + sync_pool "rsync://${mirror}/${mirrorpath}/${_arch}/${reponame}/" \ + "/tmp/${reponame}-${_arch}-nodups.whitelist" \ "${FTP_BASE}/${ARCHPKGPOOLS}/" ;; esac @@ -308,48 +349,4 @@ init() { date -u +%s > "${FTP_BASE}/lastsync" } -trap_exit() { - local signal=$1; shift - echo - error "$@" - trap -- "$signal" - kill "-$signal" "$$" -} - -fatal_error() { - error "$@" - exit 1 -} - -source "$(librelib messages)" -source "$(dirname "$(readlink -e "$0")")/config" - -readonly -a UPSTREAMS=(archlinux{,32,arm}) - -if [[ $# -ne 1 ]] || ! in_array "$1" "${UPSTREAMS[@]}" ; then - fatal_error 'usage: %s {%s\b}' \ - "${0##*/}" \ - "$(printf -- ' %s |' "${UPSTREAMS[@]}")" -fi - -readonly UPSTREAM=$1 - -source "$(dirname "$(readlink -e "$0")")/db-import-${UPSTREAM}.conf" - -# Check variables presence -vars=(DBEXT FILESEXT mirror mirrorpath WORKDIR FTP_BASE ARCHREPOS ARCHPKGPOOLS) -[[ $UPSTREAM == archlinux ]] && vars+=(ARCHSRCPOOLS) -for var in "${vars[@]}"; do - test -z "${!var}" && fatal_error "Empty %s" "${var}" -done - -# From makepkg -set -E -for signal in TERM HUP QUIT; do - trap "trap_exit $signal '%s signal caught. Exiting...' $signal" \ - "$signal" -done -trap 'trap_exit INT "Aborted by user! Exiting..."' INT -trap 'trap_exit USR1 "An unknown error has occurred. Exiting..."' ERR - init -- cgit v1.2.2