From 43340dd5019bce02af94dc1717b6ad60b84a1ead Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Wed, 1 Jan 2014 22:22:33 -0300 Subject: cleanup after release --- mips64el/mipsrelease | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mips64el/mipsrelease b/mips64el/mipsrelease index 4d7a7de..586adfc 100755 --- a/mips64el/mipsrelease +++ b/mips64el/mipsrelease @@ -43,7 +43,7 @@ fi source PKGBUILD fullver=$(get_full_version ${epoch:-0} ${pkgver} ${pkgrel}) pkgs=() -makepkg --source -f --skippgpcheck +makepkg --source -f --skippgpcheck -c msg "Adding packages to [stage3]..." for name in ${pkgname[@]}; do -- cgit v1.2.2 From 7f3926f2aab774eacc5fc5963725d44c70f50bdc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Wed, 1 Jan 2014 22:22:59 -0300 Subject: minor checks --- treepkg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/treepkg b/treepkg index 1350428..dc2575d 100755 --- a/treepkg +++ b/treepkg @@ -127,7 +127,7 @@ BUILD=${B:-true} CLEANUP=${C:-true} # Skip BUILDORDER creation and build anything on BUILDDIR BUILDNOW=${N:-false} -IGNORE=${I:-} +IGNORE=${I} if [ ! -z "${1}" -a ${DEPTH} -eq 0 ]; then BUILDNOW=true @@ -233,7 +233,7 @@ if [ ${DEPTH} -eq 0 ]; then ${VERBOSE} || echo "${BUILDORDER}" || true fi -if ${CLEANUP} ; then +if (( CLEANUP )) ; then msg2 "Removing ${BUILDDIR}" rm -rf "${BUILDDIR}" fi -- cgit v1.2.2 From 0d1aabe28efaabc910dcbca4508bb7bbc97b23df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Tue, 7 Jan 2014 10:13:49 -0300 Subject: First test version of dagpkg, a topologically sorted recursive abs builder --- dagpkg | 154 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 154 insertions(+) create mode 100755 dagpkg diff --git a/dagpkg b/dagpkg new file mode 100755 index 0000000..d3fb723 --- /dev/null +++ b/dagpkg @@ -0,0 +1,154 @@ +#!/usr/bin/env bash +set -e + +# Source variables from makepkg +source /etc/makepkg.conf +source $XDG_CONFIG_HOME/.makepkg.conf &>/dev/null || true + +# Source variables from libretools +source /etc/libretools.conf +source $XDG_CONFIG_HOME/libretools/libretools.conf &>/dev/null || true + +# End inmediately but print an useful message +trap_exit() { + term_title "error!" + error "($(basename $0)) $@ (leftovers on ${temp_dir})" + exit 1 +} + +# Trap signals from makepkg +trap 'trap_exit "TERM signal caught. Exiting..."' TERM HUP QUIT +trap 'trap_exit "Aborted by user! Exiting..."' INT +trap 'trap_exit "An unknown error has occurred. Exiting..."' ERR + +# Source this PKGBUILD, if it doesn't exist, exit +if ! source PKGBUILD &>/dev/null ; then + error "No PKGBUILD in %s" "$PWD" + exit 1 +fi + +# This is the name of the package +name="${pkgbase:-${pkgname[0]}}" + +# The name of the previous package +prev="${3}" +depth="${2:-0}" +let next=${depth}+1 || true + +# A temporary work dir and log file +temp_dir="${1:-$(mktemp -d /tmp/${name}-testpkg-XXXX)}" +log="${temp_dir}/buildorder" + +# Generate the full version with epoch +get_fullver() { + if [ $1 -eq 0 ]; then +# zero epoch case, don't include it in version + echo $2-$3 + else + echo $1:$2-$3 + fi + +} + +# If it's already built we don't bother +is_built ${pkgname[0]} $(get_fullver ${epoch:-0} ${pkgver} ${pkgrel}) && +exit + +# If the envvar I contains this package, ignore it and exit +echo "$I" | grep -q "$name" && exit + +msg "%s(%d)" $name $depth + +build=false +if [ ! -z "${1}" -a ${depth} -eq 0 ]; then + build=true +fi + + # If we specified a work dir on the cli it means we want to skip + # dependency graph creation and jump to build whatever is there + if ! ${build}; then + + # Export a pair of current and previous package to get a list of graph + # edges + echo -e "${name}\t${prev:--}" | tee -a ${log} + + # Recurse into dependencies + for d in ${depends[@]} ${makedepends[@]}; do + # Cleanup dependency versions + d=$(echo $d | sed "s/[<>=].*//") + # Where's the pkgbuild? + w=$(toru-where $d) + + # Skip if not available + test -z "$w" && continue + + # Go to this dir + pushd $w &>/dev/null + + # Infinite loop detection, if the inverted pair current+prev was + # already seen, skip + if grep -q "${prev}[[:space:]]${d}" ${log} ; then + msg2 "infinite loop %s<->%s" $d $prev + continue + fi + + # Edge detection + if grep -q "^${d}[[:space:]]" ${log} ; then + msg2 "edge %s already visited" ${d} + continue + fi + + # Run this same command giving work dir, depth and previous package + $0 ${temp_dir} ${next} ${name} + + popd &>/dev/null + done +else + msg "Resuming build..." +fi + +# end here if we're not the first package +test ${depth} -ne 0 && exit + + +# enter work dir +pushd "${temp_dir}" &>/dev/null +tsort ${log} | head -n-1 | tac | nl | tac | while read order pkg; do + # skip if already built + test -f "${pkg}/built_ok" && continue + + # where's this package? + w="$(toru-where "$pkg")" + test -z "$w" && continue + + # copy to work dir if not already + test -d "$pkg" || cp -r "$w" "$pkg" + pushd "$pkg" &>/dev/null + + term_title "$pkg($order)" + + msg "Building %s" ${pkg} + + # upgrade the system + sudo -E pacman -Syu --noconfirm + + # run the pre build command from libretools.conf + ${HOOKPREBUILD} + + # run the build command + ${FULLBUILDCMD} + + # Run local release hook with $1 = $repo + ${HOOKLOCALRELEASE} "$(basename "$(dirname "$w")")" + + # it's built! + touch built_ok + + popd &>/dev/null +done + +popd &>/dev/null +# cleanup +rm -rf ${log} "${temp_dir}" + +term_title "done" -- cgit v1.2.2 From 2d127aeaab734b41ba6387a9cf352f488d1ecd1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Wed, 8 Jan 2014 13:07:19 -0300 Subject: dagpkg: working version * check if package is ported * discover edges for same pkgbase but different pkgnames * general commentary and tips --- dagpkg | 92 +++++++++++++++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 68 insertions(+), 24 deletions(-) diff --git a/dagpkg b/dagpkg index d3fb723..578203e 100755 --- a/dagpkg +++ b/dagpkg @@ -1,18 +1,38 @@ #!/usr/bin/env bash +# +# dagpkg - create a directed graph of package dependencies and build +# them in topological order +# +# (c) 2014 Nicolás Reynolds +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . set -e -# Source variables from makepkg -source /etc/makepkg.conf -source $XDG_CONFIG_HOME/.makepkg.conf &>/dev/null || true - # Source variables from libretools source /etc/libretools.conf source $XDG_CONFIG_HOME/libretools/libretools.conf &>/dev/null || true +# Source variables from makepkg +source /etc/makepkg.conf +source $XDG_CONFIG_HOME/.makepkg.conf &>/dev/null || true + # End inmediately but print an useful message trap_exit() { term_title "error!" - error "($(basename $0)) $@ (leftovers on ${temp_dir})" + test ${depth} -eq 0 && + error "(%s) %s (leftovers on %s)" \ + "${0##*/}" "$@" "${temp_dir}" exit 1 } @@ -27,6 +47,16 @@ if ! source PKGBUILD &>/dev/null ; then exit 1 fi +# Save resources +unset pkgdesc arch license groups backup install md5sums sha1sums \ + sha256sums source options &>/dev/null + +unset build package &>/dev/null + +for _pkg in ${pkgname[@]}; do + unset package_${_pkg} &>/dev/null || true +done + # This is the name of the package name="${pkgbase:-${pkgname[0]}}" @@ -54,50 +84,59 @@ get_fullver() { is_built ${pkgname[0]} $(get_fullver ${epoch:-0} ${pkgver} ${pkgrel}) && exit +echo "${arch[@]}" | grep -qw "$CARCH" || warning "%s isn't ported to %s yet" ${name} ${CARCH} + # If the envvar I contains this package, ignore it and exit -echo "$I" | grep -q "$name" && exit +echo "$I" | grep -qw "$name" && +msg2 "%s ignored" ${name} && +exit -msg "%s(%d)" $name $depth +msg "%s (%s)" ${name} ${prev} build=false if [ ! -z "${1}" -a ${depth} -eq 0 ]; then build=true fi - # If we specified a work dir on the cli it means we want to skip - # dependency graph creation and jump to build whatever is there - if ! ${build}; then +# If we specified a work dir on the cli it means we want to skip +# dependency graph creation and jump to build whatever is there +if ! ${build}; then # Export a pair of current and previous package to get a list of graph # edges - echo -e "${name}\t${prev:--}" | tee -a ${log} + echo -e "${name}\t${prev:--}" >>${log} + + # Infinite loop detection, if the inverted pair current+prev was + # already seen, skip + if grep -q "${prev}[[:space:]]${name}" ${log} ; then + msg2 "infinite loop %s<->%s" $name $prev + exit + fi # Recurse into dependencies for d in ${depends[@]} ${makedepends[@]}; do # Cleanup dependency versions d=$(echo $d | sed "s/[<>=].*//") + # Where's the pkgbuild? w=$(toru-where $d) # Skip if not available test -z "$w" && continue - # Go to this dir - pushd $w &>/dev/null - - # Infinite loop detection, if the inverted pair current+prev was - # already seen, skip - if grep -q "${prev}[[:space:]]${d}" ${log} ; then - msg2 "infinite loop %s<->%s" $d $prev - continue - fi - - # Edge detection - if grep -q "^${d}[[:space:]]" ${log} ; then + # revisited edge detection + # we use the basename of the package dir as pkgbase to avoid + # recalling an edge using one of the other pkgname's + if grep -q "^${w##*/}[[:space:]]" ${log} ; then msg2 "edge %s already visited" ${d} + # add edge anyway but avoid reprocessing + echo -e "${w##*/}\t${name}" >>${log} continue fi + # Go to this dir + pushd $w &>/dev/null + # Run this same command giving work dir, depth and previous package $0 ${temp_dir} ${next} ${name} @@ -110,9 +149,9 @@ fi # end here if we're not the first package test ${depth} -ne 0 && exit - # enter work dir pushd "${temp_dir}" &>/dev/null +# TODO do something when loops are discovered (fail? skip?) tsort ${log} | head -n-1 | tac | nl | tac | while read order pkg; do # skip if already built test -f "${pkg}/built_ok" && continue @@ -122,6 +161,9 @@ tsort ${log} | head -n-1 | tac | nl | tac | while read order pkg; do test -z "$w" && continue # copy to work dir if not already + # this means you can make modifications to the pkgbuild during the + # graph build or remove the dir after a build failure and let dagpkg + # copy a new version test -d "$pkg" || cp -r "$w" "$pkg" pushd "$pkg" &>/dev/null @@ -130,6 +172,8 @@ tsort ${log} | head -n-1 | tac | nl | tac | while read order pkg; do msg "Building %s" ${pkg} # upgrade the system + # this would probably have to go on HOOKPREBUILD if you're working + # outside chroots sudo -E pacman -Syu --noconfirm # run the pre build command from libretools.conf -- cgit v1.2.2 From e574f9574a984545d76ed05c361ba92deabe873b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Wed, 8 Jan 2014 13:26:52 -0300 Subject: dagpkg: lukeshu's suggestions --- dagpkg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dagpkg b/dagpkg index 578203e..b4ef9e1 100755 --- a/dagpkg +++ b/dagpkg @@ -42,7 +42,7 @@ trap 'trap_exit "Aborted by user! Exiting..."' INT trap 'trap_exit "An unknown error has occurred. Exiting..."' ERR # Source this PKGBUILD, if it doesn't exist, exit -if ! source PKGBUILD &>/dev/null ; then +if ! source ./PKGBUILD &>/dev/null ; then error "No PKGBUILD in %s" "$PWD" exit 1 fi @@ -66,7 +66,7 @@ depth="${2:-0}" let next=${depth}+1 || true # A temporary work dir and log file -temp_dir="${1:-$(mktemp -d /tmp/${name}-testpkg-XXXX)}" +temp_dir="${1:-$(mktemp -dt ${name}-testpkg-XXXX)}" log="${temp_dir}/buildorder" # Generate the full version with epoch -- cgit v1.2.2 From fb5ee96054490596e4febdf459e5f5bd1b70e807 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Mas=C5=82owski?= Date: Thu, 9 Jan 2014 00:49:21 -0300 Subject: dagpkg: implement a better sorting algorithm --- dagpkg | 127 +++++++++++++++++++++++++++++++++-------------------------------- 1 file changed, 65 insertions(+), 62 deletions(-) diff --git a/dagpkg b/dagpkg index b4ef9e1..290c775 100755 --- a/dagpkg +++ b/dagpkg @@ -4,6 +4,7 @@ # them in topological order # # (c) 2014 Nicolás Reynolds +# Michał Masłowski # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -30,7 +31,6 @@ source $XDG_CONFIG_HOME/.makepkg.conf &>/dev/null || true # End inmediately but print an useful message trap_exit() { term_title "error!" - test ${depth} -eq 0 && error "(%s) %s (leftovers on %s)" \ "${0##*/}" "$@" "${temp_dir}" exit 1 @@ -41,29 +41,28 @@ trap 'trap_exit "TERM signal caught. Exiting..."' TERM HUP QUIT trap 'trap_exit "Aborted by user! Exiting..."' INT trap 'trap_exit "An unknown error has occurred. Exiting..."' ERR -# Source this PKGBUILD, if it doesn't exist, exit -if ! source ./PKGBUILD &>/dev/null ; then - error "No PKGBUILD in %s" "$PWD" - exit 1 -fi +source_pkgbuild() { + # Source this PKGBUILD, if it doesn't exist, exit + if ! source ./PKGBUILD &>/dev/null ; then + error "No PKGBUILD in %s" "$PWD" + exit 1 + fi -# Save resources -unset pkgdesc arch license groups backup install md5sums sha1sums \ - sha256sums source options &>/dev/null + # Save resources + unset pkgdesc arch license groups backup install md5sums sha1sums \ + sha256sums source options &>/dev/null -unset build package &>/dev/null + unset build package &>/dev/null -for _pkg in ${pkgname[@]}; do - unset package_${_pkg} &>/dev/null || true -done + for _pkg in ${pkgname[@]}; do + unset package_${_pkg} &>/dev/null || true + done -# This is the name of the package -name="${pkgbase:-${pkgname[0]}}" + # This is the name of the package + name="${pkgbase:-${pkgname[0]}}" +} -# The name of the previous package -prev="${3}" -depth="${2:-0}" -let next=${depth}+1 || true +source_pkgbuild # A temporary work dir and log file temp_dir="${1:-$(mktemp -dt ${name}-testpkg-XXXX)}" @@ -80,38 +79,42 @@ get_fullver() { } -# If it's already built we don't bother -is_built ${pkgname[0]} $(get_fullver ${epoch:-0} ${pkgver} ${pkgrel}) && -exit +# Mark array for DFS-based topological sort. See +# https://en.wikipedia.org/wiki/Topological_sort for an explanation of +# the algorithm. Key: package name, value: 0 for unvisited package, 1 +# during visit, 2 after visit. +declare -A marks -echo "${arch[@]}" | grep -qw "$CARCH" || warning "%s isn't ported to %s yet" ${name} ${CARCH} +# Visit a PKGBUILD for graph building. +visit_pkgbuild() { + # The name of the previous package + prev="${1}" -# If the envvar I contains this package, ignore it and exit -echo "$I" | grep -qw "$name" && -msg2 "%s ignored" ${name} && -exit + local name + source_pkgbuild -msg "%s (%s)" ${name} ${prev} + # Detect cycle or already visited package + case "${marks[$name]:-0}" in + 1) msg2 "cycle found with %s depending on %s" $prev $name + exit 1;; + 2) return;; + esac -build=false -if [ ! -z "${1}" -a ${depth} -eq 0 ]; then - build=true -fi + # If it's already built we don't bother + is_built ${pkgname[0]} $(get_fullver ${epoch:-0} ${pkgver} ${pkgrel}) && + return -# If we specified a work dir on the cli it means we want to skip -# dependency graph creation and jump to build whatever is there -if ! ${build}; then + echo "${arch[@]}" | grep -qw "$CARCH" || warning "%s isn't ported to %s yet" ${name} ${CARCH} - # Export a pair of current and previous package to get a list of graph - # edges - echo -e "${name}\t${prev:--}" >>${log} + # If the envvar I contains this package, ignore it and exit + echo "$I" | grep -qw "$name" && + msg2 "%s ignored" ${name} && + return - # Infinite loop detection, if the inverted pair current+prev was - # already seen, skip - if grep -q "${prev}[[:space:]]${name}" ${log} ; then - msg2 "infinite loop %s<->%s" $name $prev - exit - fi + msg "%s (%s)" ${name} ${prev} + + # Mark the package as being visited + marks[$name]=1 # Recurse into dependencies for d in ${depends[@]} ${makedepends[@]}; do @@ -124,37 +127,37 @@ if ! ${build}; then # Skip if not available test -z "$w" && continue - # revisited edge detection - # we use the basename of the package dir as pkgbase to avoid - # recalling an edge using one of the other pkgname's - if grep -q "^${w##*/}[[:space:]]" ${log} ; then - msg2 "edge %s already visited" ${d} - # add edge anyway but avoid reprocessing - echo -e "${w##*/}\t${name}" >>${log} - continue - fi - # Go to this dir pushd $w &>/dev/null - # Run this same command giving work dir, depth and previous package - $0 ${temp_dir} ${next} ${name} + visit_pkgbuild "$name" popd &>/dev/null done + + # Mark the package as finished + marks[$name]=2 + # Append it to the reversed list of packages to build. + echo "$name" >> "${log}" +} + +# If we specified a work dir on the cli it means we want to skip +# dependency graph creation and jump to build whatever is there +if [ -z "${1}" ]; then + # Visit the root PKGBUILD to make the graph. + visit_pkgbuild "" else msg "Resuming build..." fi -# end here if we're not the first package -test ${depth} -ne 0 && exit - # enter work dir pushd "${temp_dir}" &>/dev/null -# TODO do something when loops are discovered (fail? skip?) -tsort ${log} | head -n-1 | tac | nl | tac | while read order pkg; do +nl ${log} | while read order pkg; do # skip if already built - test -f "${pkg}/built_ok" && continue + if test -f "${pkg}/built_ok"; then + warning "tried to build %s twice" "%{pkg}" + continue + fi # where's this package? w="$(toru-where "$pkg")" -- cgit v1.2.2 From 3fababaa704ea9cd0ed06c62a9af1b83301b9302 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Thu, 9 Jan 2014 00:50:19 -0300 Subject: dagpkg: correctly handle variables --- dagpkg | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/dagpkg b/dagpkg index 290c775..49d6ef1 100755 --- a/dagpkg +++ b/dagpkg @@ -43,13 +43,15 @@ trap 'trap_exit "An unknown error has occurred. Exiting..."' ERR source_pkgbuild() { # Source this PKGBUILD, if it doesn't exist, exit + unset pkgbase pkgname depends makedepends + unset pkgrel pkgver epoch if ! source ./PKGBUILD &>/dev/null ; then error "No PKGBUILD in %s" "$PWD" exit 1 fi # Save resources - unset pkgdesc arch license groups backup install md5sums sha1sums \ + unset pkgdesc license groups backup install md5sums sha1sums \ sha256sums source options &>/dev/null unset build package &>/dev/null @@ -93,6 +95,10 @@ visit_pkgbuild() { local name source_pkgbuild + # If it's already built we don't bother + is_built ${pkgname[0]} $(get_fullver ${epoch:-0} ${pkgver} ${pkgrel}) && + return + # Detect cycle or already visited package case "${marks[$name]:-0}" in 1) msg2 "cycle found with %s depending on %s" $prev $name @@ -100,19 +106,16 @@ visit_pkgbuild() { 2) return;; esac - # If it's already built we don't bother - is_built ${pkgname[0]} $(get_fullver ${epoch:-0} ${pkgver} ${pkgrel}) && - return + msg "%s (%s)" ${name} ${prev} - echo "${arch[@]}" | grep -qw "$CARCH" || warning "%s isn't ported to %s yet" ${name} ${CARCH} + echo "${arch[@]}" | grep -qw "$CARCH" || + warning "%s isn't ported to %s yet" ${name} ${CARCH} # If the envvar I contains this package, ignore it and exit echo "$I" | grep -qw "$name" && msg2 "%s ignored" ${name} && return - msg "%s (%s)" ${name} ${prev} - # Mark the package as being visited marks[$name]=1 -- cgit v1.2.2