From 34fc8f48c893714be74e90d65b8e3132ae4ef6f0 Mon Sep 17 00:00:00 2001 From: Nicolas Reynolds Date: Thu, 9 Jun 2011 19:20:09 -0300 Subject: Implemented offline mode (aka don't use shared build queue) --- fullpkg | 39 ++++++++++++++++++++++++++++----------- 1 file changed, 28 insertions(+), 11 deletions(-) (limited to 'fullpkg') diff --git a/fullpkg b/fullpkg index f281df3..2ca5226 100755 --- a/fullpkg +++ b/fullpkg @@ -7,6 +7,12 @@ source /etc/abs.conf source /etc/libretools.conf source /usr/bin/libremessages +# Avoid /libretools dir doesn't exist errors +[ -z $XDG_CONFIG_HOME ] && { + error "There's no XDG_CONFIG_HOME var set" + exit 1 +} + [ -r $XDG_CONFIG_HOME/libretools/libretools.conf ] && \ source $XDG_CONFIG_HOME/libretools/libretools.conf @@ -27,6 +33,7 @@ function usage { echo " -n : don't update pacman db." echo " -m max_level : check deps until this level" echo " -r \"command\" : use this instead of \"$FULLBUILDCMD\"" + echo " -o : work offline (avoid queue)" # printf " -f pkgname : build even when a package has been built. " # printf " Use it as many times as needed\n" echo @@ -42,6 +49,8 @@ function usage { # Get the queue list from the server get_queue() { + [[ "$OFFLINE" = "true" ]] && return 0 + rsync -e ssh -aq $PARABOLAHOST:mips64el/queue $queue_file >/dev/null 2>&1 || { error "Failed to retrieve queue list" return 1 @@ -50,6 +59,8 @@ get_queue() { # Put the queue list on the server put_queue() { + [[ "$OFFLINE" = "true" ]] && return 0 + rsync -e ssh -aq $queue_file $PARABOLAHOST:mips64el/queue >/dev/null 2>&1 || { error "Failed to put queue list" return 1 @@ -65,6 +76,8 @@ update_queue() { # Remove a package from the queue remove_queue() { + [[ "$OFFLINE" = "true" ]] && return 0 + get_queue grep -Evw "^$(basename $PWD)" $queue_file > $queue_file.2 @@ -75,6 +88,8 @@ remove_queue() { # Checks if a package is listed check_queue() { + [[ "$OFFLINE" = "true" ]] && return 0 + get_queue local packager=$(grep -w "$(basename $PWD)" ${queue_file} | cut -d ':' -f2) @@ -161,17 +176,17 @@ function find_deps { # if search pkgname in repo doesn't work # this should find pkgsplits - elif _dir=($(find "$ABSROOT/${_repo}" -type f -name PKGBUILD -print0 2>/dev/null | \ - "xargs" -0 -e grep -HEw "pkgname=|pkgbase=|provides=" | grep -w "$_dep" 2>&1)); - then - _dir=$(dirname $(echo $_dir | cut -d: -f1)) - plain "guess for $_dep -> $_dir" - pushd $_dir > /dev/null - $0 -c -d ${build_dir} -l ${next_level} +# elif _dir=($(find "$ABSROOT/${_repo}" -type f -name PKGBUILD -print0 2>/dev/null | \ +# "xargs" -0 -e grep -HEw "pkgname=|pkgbase=|provides=" | grep -w "$_dep" 2>&1)); +# then +# _dir=$(dirname $(echo $_dir | cut -d: -f1)) +# plain "guess for $_dep -> $_dir" +# pushd $_dir > /dev/null +# $0 -c -d ${build_dir} -l ${next_level} # Circular deps must fail - [ $? -eq 20 ] && return 20 - popd > /dev/null - break 1 # found, go to next dep +# [ $? -eq 20 ] && return 20 +# popd > /dev/null +# break 1 # found, go to next dep else echo "dep_not_found:$_dep:$_repo" >> $build_dir/log @@ -307,7 +322,8 @@ build_only='n' check_deps_only='n' do_cleanup='n' max_level=21 -while getopts 'ha:bcCd:l:nm:r:' arg; do +OFFLINE=false +while getopts 'ha:bcCd:l:nm:r:o' arg; do case $arg in h) usage; exit 0 ;; a) ABSROOT="$OPTARG" ;; @@ -322,6 +338,7 @@ while getopts 'ha:bcCd:l:nm:r:' arg; do n) noupdate='y';; m) max_level=$OPTARG ;; r) FULLBUILDCMD="$OPTARG" ;; + o) OFFLINE=true ;; esac done -- cgit v1.2.2 From 4d4439b1b9b5ab83981407e06e3a85bf1144d6cd Mon Sep 17 00:00:00 2001 From: Nicolas Reynolds Date: Thu, 9 Jun 2011 20:00:26 -0300 Subject: Changed mipsrelease into a hook that's meant to run for local releases. For mipsrelease, it adds packages to a local repo and copies all needed build files to a dir on WORKDIR. --- fullpkg | 39 +++++++++++++++++++++++++++++++-------- 1 file changed, 31 insertions(+), 8 deletions(-) (limited to 'fullpkg') diff --git a/fullpkg b/fullpkg index 2ca5226..b4475a8 100755 --- a/fullpkg +++ b/fullpkg @@ -257,8 +257,10 @@ function _pkg_build () { 0) plain "The build was succesful." source .INFO && [ -n $repo ] && { -# Release locally - mipsrelease *.pkg.tar.?z + +# Calls a local release script if it's used + [ -z $HOOKLOCALRELEASE ] || \ + $HOOKLOCALRELEASE $repo *.pkg.tar.?z # Stage for releasing librestage $repo || { @@ -268,6 +270,7 @@ function _pkg_build () { msg "Updating pacman db and packages" sudo pacman -Syu --noconfirm } + echo "built:$(basename $PWD)" >> $build_dir/log cleanup "$(basename $PWD)" ;; @@ -313,8 +316,27 @@ function _pkg_build () { popd > /dev/null } + +# End inmediately but print a useful message +trap_exit() { + remove_queue + + error "$@" + warning "Leftover files left on $build_dir" + + exit 1 +} + ## END FUNCTIONS ## +## Trap signals +# From makepkg +set -E +trap 'clean_up' 0 +trap 'trap_exit "TERM signal caught. Exiting..."' TERM HUP QUIT +trap 'trap_exit "Aborted by user! Exiting..."' INT +trap 'trap_exit "An unknown error has occurred. Exiting..."' ERR + force_build="" level=0 noupdate='n' @@ -386,7 +408,7 @@ done } ## if $level = 20 there is highly likely there are circular deps -[ $level -eq $max_level -o $level -gt $max_level ] && exit 20 +[ $level -ge $max_level ] && exit 20 find_deps || { @@ -425,11 +447,12 @@ find_deps || { ## START Building msg "Building packages:" -[ ! -w $queue_file ] && error "can't write queue file" -# Exit loop on Ctrl+C -trap "break" INT -# Remove from queue package being built on error -trap "remove_queue" EXIT INT QUIT TERM KILL +# If the queue file isn't writable go into offline mode +[ ! -w $queue_file ] && { + error "Can't write queue file" + OFFLINE=true +} + _pkg_build echo -- cgit v1.2.2 From d96b9e839c3eca1f6015e967f90e044d5f7ab09e Mon Sep 17 00:00:00 2001 From: Nicolas Reynolds Date: Thu, 9 Jun 2011 21:20:15 -0300 Subject: Did some cleanup plus moving things around to make them nicer. No new features. --- fullpkg | 95 ++++++++++++++++++++++++++++++++++++----------------------------- 1 file changed, 52 insertions(+), 43 deletions(-) (limited to 'fullpkg') diff --git a/fullpkg b/fullpkg index b4475a8..2e25ba2 100755 --- a/fullpkg +++ b/fullpkg @@ -44,9 +44,6 @@ function usage { # * Add/Remove from queue # * Check if a package is listed -# TODO -# * Check for concurrence # @ fauno: What do you mean? - # Get the queue list from the server get_queue() { [[ "$OFFLINE" = "true" ]] && return 0 @@ -81,7 +78,7 @@ remove_queue() { get_queue grep -Evw "^$(basename $PWD)" $queue_file > $queue_file.2 - mv $queue_file.2 $queue_file + cat $queue_file.2 > $queue_file; rm -f $queue_file.2 put_queue && return 0 || return $? } @@ -104,6 +101,18 @@ check_queue() { # END Queue Management # +## Build order management ## + +# Removes a package from the buildorder +# $1 package name +# $2 buildorder file +remove_buildorder() { + grep -Evw "$1" $2 > $2.2 + cat $2.2 > $2; rm -f $2.2 +} + +# Guesses the repo name according to the parent dir of the current package +# assuming the form repo/package/PKGBUILD guess_repo() { basename $(dirname $(pwd)) } @@ -130,27 +139,30 @@ function find_deps { ## Check this level. source PKGBUILD local repo=${repo:-$(guess_repo)} + local pkgbase=${pkgbase:-${pkgname[0]}} # If package is built exit # TODO?: If this package is in force_build: skip this step - if is_built "${pkgbase:-${pkgname[0]}}>=${pkgver}-${pkgrel}"; then + if is_built "${pkgbase}>=${pkgver}-${pkgrel}"; then exit 0 fi # Tell which packages are deeper in deps (even if they are on build_dir) # so we can build them first. - echo "${level}:$(basename $PWD)" >> "${build_dir}/BUILDORDER" + echo "${level}:${pkgbase}" >> "${build_dir}/BUILDORDER" # if pkgbuild directory is on build_dir, do not copy and exit - if [ -d "${build_dir}/$(basename $PWD)" ]; then + if [ -d "${build_dir}/${pkgbase}" ]; then exit 0 else - cp -r ../$(basename $PWD) ${build_dir}/ +# Copy dir to build_dir + cp -r ../${pkgbase}/ ${build_dir}/ # Info to eval later - echo "repo=$repo" > "${build_dir}/$(basename $PWD)/.INFO" + echo "repo=$repo" > "${build_dir}/${pkgbase}/.INFO" fi - msg2 "${pkgbase:-${pkgname[0]}}>=${pkgver}-${pkgrel}" +# Inform the current package + msg2 "${pkgbase}-${pkgver}-${pkgrel}" ## Check next levels # Clean version checking @@ -165,8 +177,8 @@ function find_deps { for _dep in ${deps[@]}; do for _repo in ${REPOS[@]}; do # try to find $_dep on each repo from dirname - if [ -e "$ABSROOT/${_repo}/$_dep/PKGBUILD" ]; then - pushd "$ABSROOT/${_repo}/$_dep" > /dev/null + if [ -e "${ABSROOT}/${_repo}/${_dep}/PKGBUILD" ]; then + pushd "${ABSROOT}/${_repo}/${_dep}" > /dev/null $0 -c -d ${build_dir} -l ${next_level} # Circular deps must fail @@ -190,7 +202,6 @@ function find_deps { else echo "dep_not_found:$_dep:$_repo" >> $build_dir/log - fi done done @@ -199,7 +210,7 @@ function find_deps { # unset PKGBUILD variables unset pkgname pkgver pkgrel epoch pkgdesc arch url license groups depends \ makedepens checkdepends optdepends provides conflicts replaces backup \ - options install changelog source noextract md5sums build check package + options install changelog source noextract md5sums build check package } function _pkg_build () { @@ -213,18 +224,13 @@ function _pkg_build () { source PKGBUILD msg2 "${pkgbase:-${pkgname[0]}} $pkgver-$pkgrel" -# Add mips64el - if ! grep mips64el PKGBUILD >/dev/null; then - plain "Adding mips64el arch" - sed -i "s/^\(arch=([^)anym]\+\))/\1 'mips64el')/" "PKGBUILD" - fi # Check if pkg is being built msg2 "Checking build queue" check_queue || { echo "someone_is_building:$(basename $PWD)" >> $build_dir/log - grep -vwh "$(basename $PWD)" $buildorder > $buildorder.2 - mv $buildorder.2 $buildorder + + remove_buildorder "$(basename $PWD)" $buildorder continue } @@ -240,12 +246,11 @@ function _pkg_build () { # fails with something else if blacklist can't be retrieved if [ $? -eq 15 ]; then # log they have nonfree deps and so didn't build - echo "nonfree:$(basename $PWD)" >> $build_dir/log + echo "nonfree:$(basename $PWD)" >> $build_dir/log # take out package from $buildorder - grep -vwh "$(basename $PWD)" $buildorder > $buildorder.2 - mv $buildorder.2 $buildorder + remove_buildorder "$(basename $PWD)" $buildorder # continue building next package - continue + continue fi } @@ -282,8 +287,7 @@ function _pkg_build () { esac # Package was built or failed: take it out of $buildorder - grep -vwh "${build_packages[0]}" $buildorder > $buildorder.2 - mv $buildorder.2 $buildorder + remove_buildorder "${build_packages[0]}" $buildorder # Take package out from queue remove_queue @@ -293,7 +297,7 @@ function _pkg_build () { popd > /dev/null done - pkgs=$(cat $build_dir/log | grep "nonfree:") && { + pkgs=($(grep "nonfree:" $build_dir/log)) && { error "Those packages contain nonfree deps:" echo ${pkgs[@]} | tr " " "\n" | cut -d: -f2 } @@ -332,7 +336,7 @@ trap_exit() { ## Trap signals # From makepkg set -E -trap 'clean_up' 0 +trap 'cleanup' 0 trap 'trap_exit "TERM signal caught. Exiting..."' TERM HUP QUIT trap 'trap_exit "Aborted by user! Exiting..."' INT trap 'trap_exit "An unknown error has occurred. Exiting..."' ERR @@ -365,7 +369,7 @@ while getopts 'ha:bcCd:l:nm:r:o' arg; do done # Only on level 0 -[ $level -eq 0 ] && { +if [ $level -eq 0 ]; then # if build_dir exist use it, else make a build_dir build_dir=${build_dir:-$(mktemp -d /tmp/fullpkg.XXXXXX)} @@ -391,23 +395,19 @@ done # Build only [ $build_only == 'y' ] && { -# Exit loop on Ctrl+C - trap "break" INT -# Remove from queue package being built on error - trap "remove_queue" EXIT INT QUIT TERM KILL _pkg_build exit 0 } msg "Checking dependencies" -} +fi [ ! -r PKGBUILD ] && { error "This isn't a build directory" usage && exit 1 } -## if $level = 20 there is highly likely there are circular deps +## if $level = 20 it's highly likely there are circular deps [ $level -ge $max_level ] && exit 20 @@ -426,19 +426,21 @@ find_deps || { # check BUILDORDER to not include banned deps and [ $level -eq 0 -a -d $build_dir ] && { - # Check for banned deps +# Check for banned deps if [ -w $ban_file -a -r $ban_file ]; then chmod o+rw $ban_file || error "Ban file is not readable/writable ($ban_file)" else rsync -e ssh -aq $PARABOLAHOST:mips64el/ban >/dev/null 2>&1 || { warning "Failed to get ban list" && [ -r $ban_file ] && { - # continue if download failed but local copy - search=$(cat $ban_file | tr "\n" "|") - echo ${@} | tr " " "\n" | egrep -w "$search" $buildorder >> $build_dir/banned - echo ${@} | tr " " "\n" | egrep -vw "$search" $buildorder > $buildorder.2 - mv $buildorder.2 $buildorder - unset search + +# Continue if download failed but local copy +# @xihh i don't understand this -- fauno + search=$(cat $ban_file | tr "\n" "|") + echo ${@} | tr " " "\n" | egrep -w "$search" $buildorder >> $build_dir/banned + echo ${@} | tr " " "\n" | egrep -vw "$search" $buildorder > $buildorder.2 + cat $buildorder.2 > $buildorder + unset search } } fi @@ -453,6 +455,13 @@ msg "Building packages:" OFFLINE=true } +# Add mips64el if missing from arch=() and it isn't an 'any' package +if ! grep mips64el PKGBUILD >/dev/null; then + plain "Adding mips64el arch" + sed -i "s/^\(arch=([^)anym]\+\))/\1 'mips64el')/" "PKGBUILD" +fi + +# Build the package _pkg_build echo -- cgit v1.2.2 From e8724e43c334dfe57882dbe2b5d8a83f37c25af0 Mon Sep 17 00:00:00 2001 From: Nicolas Reynolds Date: Mon, 13 Jun 2011 21:29:57 -0300 Subject: Moved stuff around --- fullpkg | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) (limited to 'fullpkg') diff --git a/fullpkg b/fullpkg index 2e25ba2..15d6cc1 100755 --- a/fullpkg +++ b/fullpkg @@ -16,6 +16,11 @@ source /usr/bin/libremessages [ -r $XDG_CONFIG_HOME/libretools/libretools.conf ] && \ source $XDG_CONFIG_HOME/libretools/libretools.conf +# set queue_file and ban_file +mkdir -p $XDG_CONFIG_HOME/libretools +queue_file=$XDG_CONFIG_HOME/libretools/queue +ban_file=$XDG_CONFIG_HOME/libretools/ban + ##### START FUNCTIONS ##### function usage { @@ -33,7 +38,7 @@ function usage { echo " -n : don't update pacman db." echo " -m max_level : check deps until this level" echo " -r \"command\" : use this instead of \"$FULLBUILDCMD\"" - echo " -o : work offline (avoid queue)" + echo " -o : work offline (avoid queuing)" # printf " -f pkgname : build even when a package has been built. " # printf " Use it as many times as needed\n" echo @@ -46,7 +51,7 @@ function usage { # Get the queue list from the server get_queue() { - [[ "$OFFLINE" = "true" ]] && return 0 + [[ "$OFFLINE" -eq true ]] && return 0 rsync -e ssh -aq $PARABOLAHOST:mips64el/queue $queue_file >/dev/null 2>&1 || { error "Failed to retrieve queue list" @@ -56,7 +61,7 @@ get_queue() { # Put the queue list on the server put_queue() { - [[ "$OFFLINE" = "true" ]] && return 0 + [[ "$OFFLINE" = true ]] && return 0 rsync -e ssh -aq $queue_file $PARABOLAHOST:mips64el/queue >/dev/null 2>&1 || { error "Failed to put queue list" @@ -73,7 +78,7 @@ update_queue() { # Remove a package from the queue remove_queue() { - [[ "$OFFLINE" = "true" ]] && return 0 + [[ "$OFFLINE" -eq true ]] && return 0 get_queue @@ -120,8 +125,9 @@ guess_repo() { # Usage: cleanup [ $(basename $PWD) ] from PKGBUILD dir # cleans the build_dir function cleanup { +# Already clean if [ ! -d $build_dir ]; then - return 1 + return 0 elif [ -d $build_dir -a ${#@} -gt 0 ]; then for _dir in $@; do @@ -129,7 +135,8 @@ function cleanup { done else - rm -rf $build_dir/* +# Only do full cleanup on level 0 + [ $level -eq 0 ] && rm -rf $build_dir/* fi } @@ -211,6 +218,7 @@ function find_deps { unset pkgname pkgver pkgrel epoch pkgdesc arch url license groups depends \ makedepens checkdepends optdepends provides conflicts replaces backup \ options install changelog source noextract md5sums build check package + } function _pkg_build () { @@ -379,11 +387,6 @@ if [ $level -eq 0 ]; then [ $build_only == 'y' ] && exit 0 } -# set queue_file and ban_file - mkdir -p $XDG_CONFIG_HOME/libretools - queue_file=$XDG_CONFIG_HOME/libretools/queue - ban_file=$XDG_CONFIG_HOME/libretools/ban - # make files for log and buildorder touch $build_dir/{log,BUILDORDER} buildorder=$build_dir/BUILDORDER @@ -439,7 +442,7 @@ find_deps || { search=$(cat $ban_file | tr "\n" "|") echo ${@} | tr " " "\n" | egrep -w "$search" $buildorder >> $build_dir/banned echo ${@} | tr " " "\n" | egrep -vw "$search" $buildorder > $buildorder.2 - cat $buildorder.2 > $buildorder + cat $buildorder.2 > $buildorder; rm $buildorder.2 unset search } } -- cgit v1.2.2