diff options
Diffstat (limited to 'src/abslibre-tools')
| -rwxr-xr-x | src/abslibre-tools/libredbdiff | 1 | ||||
| -rw-r--r-- | src/abslibre-tools/libredbdiff.conf | 2 | ||||
| -rwxr-xr-x | src/abslibre-tools/librerelease | 383 | ||||
| -rwxr-xr-x | src/abslibre-tools/librestage | 18 |
4 files changed, 241 insertions, 163 deletions
diff --git a/src/abslibre-tools/libredbdiff b/src/abslibre-tools/libredbdiff index 28031a2..b5e6331 100755 --- a/src/abslibre-tools/libredbdiff +++ b/src/abslibre-tools/libredbdiff @@ -3,6 +3,7 @@ # # Copyright (C) 2014 Esteban Carnevale <alfplayer@mailoo.org> # Copyright (C) 2014, 2017-2018 Luke Shumaker <lukeshu@parabola.nu> +# Copyright (C) 2018 Omar Vega Ramos <ovruni@gnu.org.pe> # # License: GNU GPLv3+ # diff --git a/src/abslibre-tools/libredbdiff.conf b/src/abslibre-tools/libredbdiff.conf index d92a832..23c8c2d 100644 --- a/src/abslibre-tools/libredbdiff.conf +++ b/src/abslibre-tools/libredbdiff.conf @@ -3,7 +3,7 @@ statedir='/var/lib/libredbdiff' -mirror_prbl='https://repomirror.parabola.nu/$repo/os/$arch' +mirror_prbl='https://repo.parabola.nu/$repo/os/$arch' mirror_arch='https://mirrors.kernel.org/archlinux/$repo/os/$arch' repos=(libre{,-multilib} pcr{,-multilib} nonprism{,-multilib}) diff --git a/src/abslibre-tools/librerelease b/src/abslibre-tools/librerelease index 8ebdc54..cb07229 100755 --- a/src/abslibre-tools/librerelease +++ b/src/abslibre-tools/librerelease @@ -1,24 +1,25 @@ #!/usr/bin/env bash -# Librerelease -# Uploads packages and releases them -# Copyright (C) 2010-2012 Joshua Ismael Haase Hernández (xihh) <hahj87@gmail.com> -# Copyright (C) 2010-2013 Nicolás Reynolds <fauno@parabola.nu> -# Copyright (C) 2013 Michał Masłowski <mtjm@mtjm.eu> -# Copyright (C) 2013-2014, 2017-2018 Luke Shumaker <lukeshu@parabola.nu> +# librerelease - uploads packages to the repo server and publishes them +# +# Copyright (C) 2010-2012 Joshua Ismael Haase Hernández (xihh) <hahj87@gmail.com> +# Copyright (C) 2010-2013 Nicolás Reynolds <fauno@parabola.nu> +# Copyright (C) 2013 Michał Masłowski <mtjm@mtjm.eu> +# Copyright (C) 2013-2014,2017-2018,2024 Luke Shumaker <lukeshu@parabola.nu> +# Copyright (C) 2019-2020,2022-2024 Bill Auger <mr.j.spam.me@gmail.com> # # For just the create_signature() function: # Copyright (C) 2006-2013 Pacman Development Team <pacman-dev@archlinux.org> # Copyright (C) 2002-2006 Judd Vinet <jvinet@zeroflux.org> -# Copyright (C) 2005 Aurelien Foret <orelien@chez.com> -# Copyright (C) 2006 Miklos Vajna <vmiklos@frugalware.org> -# Copyright (C) 2005 Christian Hamar <krics@linuxforum.hu> -# Copyright (C) 2006 Alex Smith <alex@alex-smith.me.uk> -# Copyright (C) 2006 Andras Voroskoi <voroskoi@frugalware.org> +# Copyright (C) 2005 Aurelien Foret <orelien@chez.com> +# Copyright (C) 2006 Miklos Vajna <vmiklos@frugalware.org> +# Copyright (C) 2005 Christian Hamar <krics@linuxforum.hu> +# Copyright (C) 2006 Alex Smith <alex@alex-smith.me.uk> +# Copyright (C) 2006 Andras Voroskoi <voroskoi@frugalware.org> # -# License: GNU GPLv3+ +# SPDX-License-Identifier: GPL-3.0-or-later # -# This file is part of Parabola. +# This file is part of Parabola Libretools. # # Parabola is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -36,14 +37,16 @@ # create_signature() is taken from pacman:makepkg, which is GPLv2+, # so we take the '+' to combine it with our GPLv3+. + set -euE -. "$(librelib messages)" -. "$(librelib conf)" -setup_traps -dryrun="" -upload_only=false -readonly rsync_flags=( +source "$(librelib conf )" # LIBREUSER, load_conf() +source "$(librelib messages )" # setup_traps(), msg(), msg2() error(), print(), prose(), flag() +source "$(librelib notifications)" # notify_release() + + +declare -ri STAGING_LOCK=8 +declare -ra RSYNC_FLAGS=( --no-group --no-perms --copy-links @@ -52,17 +55,34 @@ readonly rsync_flags=( --human-readable --progress ) +DRY_RUN='' # main() +UPLOAD_ONLY='' # main() +TIER0_HOST='' # main() +TIER0_STAGING='' # main() +TIER0_LOGIN='' # main() +TIER0_PORT='' # main() +SSH_CMD='' # main() +RSYNC_DEST='' # main() + -# Functions #################################################################### +## helpers ## + +lock_staging() { + lock $STAGING_LOCK "${WORKDIR}/staging.lock" \ + "Waiting for an exclusive lock on the staging directory ...." +} + +unlock_staging() { + lock_close $STAGING_LOCK +} list0_files() { find -L "${WORKDIR}/staging" -type f -not -name '*.lock' \ - -exec realpath -z --relative-to="${WORKDIR}/staging" {} + + -exec realpath -z --relative-to="${WORKDIR}/staging" {} + | sort -z } # This function is taken almost verbatim from makepkg create_signature() { - local ret=$EXIT_SUCCESS local filename="$1" msg "Signing package..." @@ -71,14 +91,13 @@ create_signature() { SIGNWITHKEY=(-u "${GPGKEY}") fi - gpg --detach-sign --use-agent "${SIGNWITHKEY[@]}" --no-armor "$filename" &>/dev/null || ret=$EXIT_FAILURE - - - if (( ! ret )); then + if gpg --detach-sign --use-agent "${SIGNWITHKEY[@]}" \ + --no-armor "$filename" &>/dev/null ; then msg2 "Created signature file %s." "$filename.sig" + return $EXIT_SUCCESS else error "Failed to sign package file." - return $ret + return $EXIT_FAILURE fi } @@ -106,118 +125,52 @@ sign_packages() { # Clean everything if not in dry-run mode clean_files() ( local file_list=$1 + local rmcmd - local rmcmd=(rm -fv) - if [[ -n "${dryrun}" ]]; then - rmcmd=(printf "$(_ "removed '%s' (dry-run)")\n") + if [[ -z "$DRY_RUN" ]]; then + rmcmd=( rm -fv ) + else + rmcmd=( printf "$(_ "removed '%s' (dry-run)")\n" ) fi msg "Removing files from local staging directory" cd "${WORKDIR}/staging" xargs -0r -a "$file_list" "${rmcmd[@]}" find . -depth -mindepth 1 -type d \ - -exec rmdir --ignore-fail-on-non-empty -- '{}' + + -exec rmdir --ignore-fail-on-non-empty -- '{}' + ) -################################################################################ + +## The different modes ## usage() { print "Usage: %s [OPTIONS]" "${0##*/}" - print 'Upload packages in $WORKDIR/staging to the Parabola server' + prose 'Upload packages staged in %s (per `librestage`) to the configured server, + and publish them to their respective repositories.' \$WORKDIR/staging + echo + prose 'This requires the `gpg` program, configured with your GPG key, + and a staging directory (%s), writable by %s.' \$WORKDIR/staging '$LIBREUSER' + prose '%s is determined at runtime, by %s. + %s is normally the local login of the invoking user, + but may be over-ridden by setting %s in the environment.' \ + '$LIBREUSER' /usr/lib/libretools/conf.sh '$LIBREUSER' '$SUDO_USER' + prose 'By default, %s is assumed to match the hackers.git login + for which %s has login credentials for the repo server. + If %s does not match the hackers.git login, + you must specify %s as the remote login, (eg: in %s)' \ + '$LIBREUSER' '$LIBREUSER' '$LIBREUSER' '$TIER0_LOGIN' \ + \$XDG_CONFIG_HOME/libretools/libretools.conf echo print "Options:" - flag '-c' 'Clean; delete packages in $WORKDIR/staging' - flag '-l' "List; list packages but not upload them" - flag '-u' "Upload-only; do not run db-update on the server" - - flag '-n' "Dry-run; don't actually do anything" - flag '-h' "Show this message" -} - -main() { - if [[ -w / ]]; then - error "This program should be run as regular user" - return $EXIT_NOPERMISSION - fi - - # Parse options - local mode="release_packages" - while getopts 'clunh' arg; do - case $arg in - c) mode=clean ;; - l) mode=pretty_print_packages ;; - u) upload_only=true ;; - n) dryrun="--dry-run" ;; - h) mode=usage ;; - *) usage >&2; return $EXIT_INVALIDARGUMENT ;; - esac - done - shift $((OPTIND - 1)) - if [[ $# != 0 ]]; then - usage >&2 - return $EXIT_INVALIDARGUMENT - fi - - if [[ $mode == usage ]]; then - usage - return $EXIT_SUCCESS - fi - - declare -i ret=0 - load_conf makepkg.conf GPGKEY || ret=$? - load_conf libretools.conf WORKDIR REPODEST DBSCRIPTS_CONFIG || ret=$? # and HOOK{PRE,POST}RELEASE, which are optional - [[ $ret = 0 ]] || exit $ret - - local re_url='^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?$' - local re_authority='^(([^@]*)@)?([^][@:]*|\[[^]]*\])(:([0-9]*))?$' - local REPODEST_ok=false - if [[ "$REPODEST" =~ $re_url ]]; then - REPODEST_ok=true - - REPODEST_scheme=${BASH_REMATCH[2]} - REPODEST_authority=${BASH_REMATCH[4]} - REPODEST_path=${BASH_REMATCH[5]} - REPODEST_query=${BASH_REMATCH[7]} - REPODEST_fragment=${BASH_REMATCH[9]} - - if [[ "$REPODEST_authority" =~ $re_authority ]]; then - REPODEST_userinfo=${BASH_REMATCH[2]} - REPODEST_host=${BASH_REMATCH[3]} - REPODEST_port=${BASH_REMATCH[5]} - - if [[ "$REPODEST_host" = '['*']' ]]; then - REPODEST_host=${REPODEST_HOST#'['} - REPODEST_host=${REPODEST_HOST#']'} - fi - else - REPODEST_ok=false - fi - - [[ $REPODEST_scheme == ssh ]] || REPODEST_ok=false - [[ -n $REPODEST_host ]] || REPODEST_ok=false - [[ -n $REPODEST_path ]] || REPODEST_ok=false - fi - if ! $REPODEST_ok; then - error 'The format of libretools.conf:REPODEST has changed.' - plain 'Merge the /etc/libretools.conf.pacnew file!' - return $EXIT_NOTCONFIGURED - fi - if [[ "$REPODEST_path" = '/~'* ]]; then - if [[ "$REPODEST_path" = '/~/'* ]]; then - REPODEST_path=${REPODEST_path#'/~/'} - else - error 'Unfortunately, `~user` home-directory expansion is not (yet?) supported in libretools.conf:REPODEST' - return $EXIT_NOTCONFIGURED - fi - fi - REPODEST_userhost="${REPODEST_userinfo:+${REPODEST_userinfo%%:*}@}${REPODEST_host}" - - "$mode" + flag '-c' "Clean Local: delete packages in local staging directory" + flag '-C' "Clean Remote: delete packages in remote staging directory" + flag '-h' "Help: Show this message" + flag '-l' "List: list packages but not upload them" + flag '-n' "Dry-run: don't actually do anything" + flag '-u' "Upload-only: do not run db-update on the server" } -# The different modes (sans 'usage') ########################################### - -pretty_print_packages() { +list() { find "$WORKDIR/staging/" -mindepth 1 -maxdepth 1 -type d -not -empty | sort | while read -r path; do msg2 "${path##*/}" @@ -226,21 +179,44 @@ pretty_print_packages() { done } -clean() { - lock 8 "${WORKDIR}/staging.lock" \ - 'Waiting for an exclusive lock on the staging directory' +clean_local() { + lock_staging local file_list file_list="$(mktemp -t "${0##*/}.XXXXXXXXXX")" trap "rm -f -- ${file_list@Q}" EXIT list0_files > "$file_list" - lock_close 8 + unlock_staging clean_files "$file_list" } -release_packages() { +clean_remote() { + msg "Removing files from remote staging directory" + ${SSH_CMD[*]} "rm -rfv ${TIER0_STAGING@Q}/*" +} + +release() { + local file_list="$( mktemp -t ${0##*/}_lst.XXXXXXXXXX)" + local dbupdate_log="$(mktemp -t ${0##*/}_log.XXXXXXXXXX)" + local mkdir_cmd="mkdir -p -- ${TIER0_STAGING@Q} && cd ${TIER0_STAGING@Q} && xargs -0r mkdir -pv --" + local dbupdate_cmd="STAGING=${TIER0_STAGING@Q} DBSCRIPTS_CONFIG=${DBSCRIPTS_CONFIG@Q} db-update" + local upload_size pbotsay_msg pbotsay_cmd + + trap "rm -f -- ${file_list@Q} ${dbupdate_log@Q}" INT RETURN TERM + + lock_staging + + # verify connection and login + if ! ${SSH_CMD[0]} -fN ${SSH_CMD[*]:1}; then + error "Connection or login failed." + return $EXIT_FAILURE + fi + + + ## prepare ## + if [[ -n $HOOKPRERELEASE ]]; then msg "Running HOOKPRERELEASE..." ( @@ -249,31 +225,29 @@ release_packages() { ) fi - lock 8 "${WORKDIR}/staging.lock" \ - 'Waiting for an exclusive lock on the staging directory' - - sign_packages || return + sign_packages || return $EXIT_FAILURE - # Make the permissions of the packages 644 otherwise the user will get access - # denied error when they try to download (rsync --no-perms doesn't seem to - # work). + # collect staged files and set permissions for repository-bound files + list0_files > "$file_list" + upload_size="$(cd "${WORKDIR}/staging" && du -hc --files0-from="$file_list" | sed -n '$s/\t.*//p')" find "${WORKDIR}/staging" -type f -exec chmod 644 {} + find "${WORKDIR}/staging" -type d -exec chmod 755 {} + - local file_list="$(mktemp -t ${0##*/}.XXXXXXXXXX)" - trap "rm -f -- ${file_list@Q}" EXIT - list0_files > "$file_list" + unlock_staging + + # prepare remote staging directory tree + msg "%s to upload" "$upload_size" + xargs -0r -a "$file_list" dirname -z | ${SSH_CMD[*]} "$mkdir_cmd" - lock_close 8 - msg "%s to upload" "$(cd "${WORKDIR}/staging" && du -hc --files0-from="$file_list" | sed -n '$s/\t.*//p')" + ## upload ## + msg "Uploading packages..." - xargs -0r -a "$file_list" dirname -z | ssh ${REPODEST_port:+-p "$REPODEST_port"} "${REPODEST_userhost}" "mkdir -p -- ${REPODEST_path@Q} && cd ${REPODEST_path@Q} && xargs -0r mkdir -pv --" - if ! rsync ${dryrun} "${rsync_flags[@]}" \ - -e "ssh ${REPODEST_port:+-p $REPODEST_port}" \ - -0 --files-from="$file_list" \ - "${WORKDIR}/staging" \ - "$REPODEST_userhost:$REPODEST_path/" + if ! rsync ${DRY_RUN} "${RSYNC_FLAGS[@]}" \ + -e "ssh $SSH_PORT" \ + -0 --files-from="$file_list" \ + "${WORKDIR}/staging" \ + "$RSYNC_DEST" then error "Sync failed, try again" return $EXIT_FAILURE @@ -281,28 +255,117 @@ release_packages() { clean_files "$file_list" - if $upload_only; then + if $UPLOAD_ONLY; then return $EXIT_SUCCESS fi + + ## publish ## + msg "Running db-update on repos" - ssh ${REPODEST_port:+-p "$REPODEST_port"} "${REPODEST_userhost}" "STAGING=${REPODEST_path@Q} DBSCRIPTS_CONFIG=${DBSCRIPTS_CONFIG@Q} db-update" + ( + # this contraption allows detecting `db-update` exit failure, + # while logging output both to file and to the local shell in real-time, + # while preserving (restoring) colors lost in the pipeline + set -o pipefail ; ${SSH_CMD[*]} "$dbupdate_cmd" | tee "$dbupdate_log" | + while read line ; do ( [[ "$line" =~ ^==\> ]] && msg "${line#==\> }" ) || + ( [[ "$line" =~ ^\ *-\> ]] && msg2 "${line#*\> }" ) || + echo "$line" ; done + ) + + if grep -Eq "^==> Updating \[" "$dbupdate_log"; then + pbotsay_msg=$(release_notification "${TIER0_LOGIN:-${LIBREUSER}}" < "$dbupdate_log") + pbotsay_cmd="if type pbot-say &>/dev/null ; then pbot-say ${pbotsay_msg@Q} ; fi" + + if [[ -n $HOOKPOSTRELEASE ]]; then + msg "Running HOOKPOSTRELEASE..." + ( + PS4=" \\[$BOLD\\]\$\\[$ALL_OFF\\] " + eval -- "set -x; $HOOKPOSTRELEASE" + ) + fi - if [[ -n $HOOKPOSTRELEASE ]]; then - msg "Running HOOKPOSTRELEASE..." - ( - PS4=" \\[$BOLD\\]\$\\[$ALL_OFF\\] " - eval -- "set -x; $HOOKPOSTRELEASE" - ) + # notify pbot of the excellent work that we have done today + msg2 "Notifying pbot:" ; print " $pbotsay_msg" ; + ${SSH_CMD[*]} "$pbotsay_cmd" &> /dev/null || : + else + msg2 "Nothing was published" fi +} + - # notify pbot of the excellent work we have done - local packages=${file_list} # TODO: parse package names? - this is gonna be messy - local login=${REPODEST_userinfo:-somebody} ; login=${login%%:*} ; - ssh ${REPODEST_port:+-p "$REPODEST_port"} "${REPODEST_userhost}" \ - "which pbot-say && pbot-say ${login} just published: ${packages}" +## main entry ## - return $EXIT_SUCCESS +main() { + # Parse CLI options + local mode=release # publish packages to public repo (default) + UPLOAD_ONLY=false # upload and publish + while getopts 'cChlnu' arg; do + case $arg in + c) mode=clean_local ;; # empties local staging area + C) mode=clean_remote ;; # empties remote staging area + h) mode=usage ;; # print 'Usage' message + l) mode=list ;; # pretty-print locally-staged packages + n) DRY_RUN='--dry-run' ;; # only show what would be done + u) UPLOAD_ONLY=true ;; # upload, but do not publish + *) usage >&2 ; return $EXIT_INVALIDARGUMENT ;; + esac + done + shift $(( OPTIND - 1 )) + if [[ -w / ]]; then + error "This program should be run as unprivileged user" + return $EXIT_NOPERMISSION + elif (( $# )); then + usage >&2 + return $EXIT_INVALIDARGUMENT + elif [[ $mode == usage ]]; then + usage + return $EXIT_SUCCESS + fi + + # source makepkg and libretools configuration files + # the specified config vars will be used in this script, and so are mandatory + # optional config vars used in this script, if specified in libretools.conf: + # TIER0_LOGIN, TIER0_PORT, TIER0_STAGING, HOOKPRERELEASE, HOOKPOSTRELEASE + if ! load_conf makepkg.conf GPGKEY; then + error "The %s config variable is misconfigured." \$GPGKEY + return $EXIT_NOTCONFIGURED + elif ! load_conf libretools.conf WORKDIR TIER0_HOST DBSCRIPTS_CONFIG; then + for var in WORKDIR TIER0_HOST DBSCRIPTS_CONFIG; do + [[ -n ${!var} ]] || error "The $%s config variable is misconfigured." \$$var + done + prose "The format of %s variables may have changed. + Merge the %s file, if one is present; and adapt any custom %s to it." \ + libretools.conf /etc/libretools.conf.pacnew ~/.config/libretools/libretools.conf + return $EXIT_NOTCONFIGURED + fi + + # validate/sanitize tier-0 repo URL components + if [[ "$TIER0_STAGING" == '/~/'* ]]; then + TIER0_STAGING=${TIER0_STAGING#'/~/'} + elif [[ "$TIER0_STAGING" == '/~'* ]]; then + error "Unfortunately, tilde expansion ('~' home directory) is not supported in libretools.conf::TIER0_STAGING" + return $EXIT_NOTCONFIGURED + fi + + # finalize state + readonly DRY_RUN + readonly UPLOAD_ONLY + + # construct the SSH and rsync destination parameters + readonly TIER0_LOGIN + readonly TIER0_HOST + readonly TIER0_PORT + readonly TIER0_STAGING=${TIER0_STAGING:-/home/${TIER0_LOGIN:-$LIBREUSER}/staging} + readonly SSH_URL=${TIER0_LOGIN:+${TIER0_LOGIN}@}${TIER0_HOST} + readonly SSH_PORT=${TIER0_PORT:+-p $TIER0_PORT} + readonly SSH_CMD=( ssh ${SSH_PORT} ${SSH_URL} ) + readonly RSYNC_DEST=${SSH_URL}:${TIER0_STAGING%/}/ + + # do the requested business + $mode } + +setup_traps main "$@" diff --git a/src/abslibre-tools/librestage b/src/abslibre-tools/librestage index bec536c..7098aac 100755 --- a/src/abslibre-tools/librestage +++ b/src/abslibre-tools/librestage @@ -5,6 +5,7 @@ # Copyright (C) 2010-2012 Nicolás Reynolds <fauno@parabola.nu> # Copyright (C) 2011 Joshua Ismael Haase Hernández (xihh) <hahj87@gmail.com> # Copyright (C) 2013-2014, 2017-2018 Luke Shumaker <lukeshu@parabola.nu> +# Copyright (C) 2020 Andreas Grapentin <andreas@grapentin.org> # # License: GNU GPLv3+ # @@ -26,12 +27,19 @@ . "$(librelib messages)" . "$(librelib conf)" +LimitArch="*" + usage() { - print "Usage: %s [REPO]" "${0##*/}" + print "Usage: %s [-A <arch>] [REPO]" "${0##*/}" print "Stages the package(s) build by ./PKGBUILD for upload." echo prose "The package(s) are staged for the named repository, or the name of the parent directory if a repository is not named." + echo + echo "Supported options:" + echo " -h Display this help and exit" + echo " -A <arch> Stage only packages built for arches matching the given" + echo " wildcard string. Default: *" } main() { @@ -41,12 +49,16 @@ main() { fi # Parse options, set up - while getopts 'h' arg; do + while getopts 'hA:' arg; do case $arg in h) usage; return $EXIT_SUCCESS;; + A) LimitArch="$OPTARG";; *) usage >&2; return $EXIT_INVALIDARGUMENT;; esac done + local shiftlen=$(( OPTIND - 1 )) + shift $shiftlen + local repo= case $# in 0) repo="$(basename "$(dirname "$PWD")")";; @@ -79,6 +91,8 @@ main() { # Look for (libre)makepkg output local CARCH _pkgname pkgnames pkgfile for CARCH in "${arch[@]}"; do + # Skip this arch if necessary + [[ "$CARCH" == $LimitArch ]] || continue # This supports both pacman < 5.1 pkgname-debug # packages and pacman >= 5.1 pkgbase-debug packages. pkgnames=("${pkgname[@]}" "${pkgname[@]/%/-debug}") |