From 2b7bb0c6b7aa4f7a43c82db1cf9a18d27600c62e Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 18 Dec 2011 14:16:30 +0100 Subject: Validate package signatures on db-update --- db-functions | 15 +++++++++++++++ db-update | 3 +++ test/lib/common.inc | 29 +++++++++++++++++++++++++---- test/test.d/signed-packages.sh | 20 +++++++++++++++++++- 4 files changed, 62 insertions(+), 5 deletions(-) diff --git a/db-functions b/db-functions index a3e2168..7c4a7fe 100644 --- a/db-functions +++ b/db-functions @@ -466,6 +466,21 @@ check_repo_permission() { return 0 } +check_pkgsig() { + local signature=$1 + local ret=1 + local fd="$(mktemp --tmpdir="${WORKDIR}")" + + exec 4>"${fd}" + gpg --homedir /etc/pacman.d/gnupg/ --status-fd 4 --verify "${signature}" >/dev/null 2>&1 + exec 4>&- + if grep -q TRUST_FULLY "${fd}"; then + ret=0 + fi + + return $ret +} + set_repo_permission() { local repo=$1 local arch=$2 diff --git a/db-update b/db-update index 4b9c78f..60af79f 100755 --- a/db-update +++ b/db-update @@ -35,6 +35,9 @@ for repo in ${repos[@]}; do if ! check_pkgfile "${pkg}"; then die "Package ${repo}/$(basename ${pkg}) is not consistent with its meta data" fi + if ${REQUIRE_SIGNATURE} && ! check_pkgsig "${pkg}.sig"; then + die "Package ${repo}/$(basename ${pkg}) does not have a valid signature" + fi if ! check_pkgsvn "${pkg}" "${repo}"; then die "Package ${repo}/$(basename ${pkg}) is not consistent with svn repository" fi diff --git a/test/lib/common.inc b/test/lib/common.inc index a2dee10..e0e7048 100644 --- a/test/lib/common.inc +++ b/test/lib/common.inc @@ -3,6 +3,21 @@ set -E . "$(dirname ${BASH_SOURCE[0]})/../../config" . "$(dirname ${BASH_SOURCE[0]})/../../db-functions" +signpkg() { + if [[ -r '/etc/makepkg.conf' ]]; then + source '/etc/makepkg.conf' + else + die '/etc/makepkg.conf not found!' + fi + if [[ -r ~/.makepkg.conf ]]; then + . ~/.makepkg.conf + fi + if [[ -n $GPGKEY ]]; then + SIGNWITHKEY="-u ${GPGKEY}" + fi + gpg --detach-sign --use-agent ${SIGNWITHKEY} ${@} || die +} + oneTimeSetUp() { local p local d @@ -110,18 +125,24 @@ releasePackage() { local repo=$1 local pkgbase=$2 local arch=$3 + local a + local p + local pkgver + local pkgname pushd "${TMP}/svn-packages-copy"/${pkgbase}/trunk/ >/dev/null archrelease ${repo}-${arch} >/dev/null 2&>1 pkgver=$(. PKGBUILD; echo $(get_full_version ${epoch:-0} ${pkgver} ${pkgrel})) + pkgname=($(. PKGBUILD; echo ${pkgname[@]})) popd >/dev/null cp "${pkgdir}/${pkgbase}"/*-${pkgver}-${arch}${PKGEXT} "${STAGING}"/${repo}/ if ${REQUIRE_SIGNATURE}; then - # TODO: really sign the packages with a valid key - find "${STAGING}"/${repo}/ -type f \ - -name "*-${pkgver}-${arch}${PKGEXT}" \ - -exec touch {}.sig \; + for a in ${arch[@]}; do + for p in ${pkgname[@]}; do + signpkg "${STAGING}"/${repo}/${p}-${pkgver}-${a}${PKGEXT} + done + done fi } diff --git a/test/test.d/signed-packages.sh b/test/test.d/signed-packages.sh index 5d6f4ff..20ad844 100755 --- a/test/test.d/signed-packages.sh +++ b/test/test.d/signed-packages.sh @@ -5,9 +5,27 @@ curdir=$(readlink -e $(dirname $0)) testAddUnsignedPackage() { releasePackage extra 'pkg-simple-a' 'i686' - # remove any signature rm "${STAGING}"/extra/*.sig ../db-update >/dev/null 2>&1 && fail "db-update should fail when a signature is missing!" } +testAddInvalidSignedPackage() { + local p + releasePackage extra 'pkg-simple-a' 'i686' + for p in "${STAGING}"/extra/*${PKGEXT}; do + unxz $p + xz -0 ${p%%.xz} + done + ../db-update >/dev/null 2>&1 && fail "db-update should fail when a signature is invalid!" +} + +testAddBrokenSignature() { + local s + releasePackage extra 'pkg-simple-a' 'i686' + for s in "${STAGING}"/extra/*.sig; do + echo 0 > $s + done + ../db-update >/dev/null 2>&1 && fail "db-update should fail when a signature is broken!" +} + . "${curdir}/../lib/shunit2" -- cgit v1.2.2 From 5c7220a4aa370d1f726eb27d99d8f9294646433d Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Mon, 19 Dec 2011 14:40:07 +0100 Subject: Avoid calls to basename --- cron-jobs/ftpdir-cleanup | 2 +- cron-jobs/integrity-check | 2 +- cron-jobs/update-web-db | 2 +- db-functions | 24 ++++++++++++------------ db-move | 6 +++--- db-remove | 4 ++-- db-repo-add | 2 +- db-repo-remove | 2 +- db-update | 14 +++++++------- test/lib/common.inc | 8 ++++---- test/test.d/db-move.sh | 4 ++-- test/test.d/db-update.sh | 2 +- test/test.d/ftpdir-cleanup.sh | 2 +- test/test.d/pool-transition.sh | 12 ++++++------ testing2x | 2 +- 15 files changed, 44 insertions(+), 44 deletions(-) diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index bb1661a..7839d50 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -16,7 +16,7 @@ clean_pkg() { if [ -e "$pkg.sig" ]; then mv -f "$pkg.sig" "$CLEANUP_DESTDIR" fi - touch "${CLEANUP_DESTDIR}/$(basename ${pkg})" + touch "${CLEANUP_DESTDIR}/${pkg##*/}" fi done fi diff --git a/cron-jobs/integrity-check b/cron-jobs/integrity-check index d4f9694..044f093 100755 --- a/cron-jobs/integrity-check +++ b/cron-jobs/integrity-check @@ -8,7 +8,7 @@ dirname="$(dirname $0)" script_lock if [ $# -ne 1 ]; then - die "usage: $(basename $0) " + die "usage: ${0##*/} " fi mailto=$1 diff --git a/cron-jobs/update-web-db b/cron-jobs/update-web-db index 6ced4c1..b1767ca 100755 --- a/cron-jobs/update-web-db +++ b/cron-jobs/update-web-db @@ -14,7 +14,7 @@ REPOS=('community-testing' 'multilib-testing' 'multilib' 'community' 'extra' 'te LOGOUT="/tmp/archweb_update.log" # figure out what operation to perform -cmd="$(basename $0)" +cmd="${0##*/}" if [[ $cmd != "update-web-db" && $cmd != "update-web-files-db" ]]; then die "Invalid command name '$cmd' specified!" fi diff --git a/db-functions b/db-functions index 7c4a7fe..3100e31 100644 --- a/db-functions +++ b/db-functions @@ -16,7 +16,7 @@ restore_umask () { } # set up general environment -WORKDIR=$(mktemp -d /tmp/$(basename $0).XXXXXXXXXX) +WORKDIR=$(mktemp -d /tmp/${0##*/}.XXXXXXXXXX) LOCKS=() # check if messages are to be printed using color @@ -85,10 +85,10 @@ get_full_version() { } script_lock() { - local LOCKDIR="$TMPDIR/.scriptlock.$(basename $0)" + local LOCKDIR="$TMPDIR/.scriptlock.${0##*/}" if ! mkdir "$LOCKDIR" >/dev/null 2>&1 ; then local _owner="$(/usr/bin/stat -c %U $LOCKDIR)" - error "Script $(basename $0) is already locked by $_owner." + error "Script ${0##*/} is already locked by $_owner." exit 1 else set_umask @@ -97,9 +97,9 @@ script_lock() { } script_unlock() { - local LOCKDIR="$TMPDIR/.scriptlock.$(basename $0)" + local LOCKDIR="$TMPDIR/.scriptlock.${0##*/}" if [ ! -d "$LOCKDIR" ]; then - warning "Script $(basename $0) was not locked!" + warning "Script ${0##*/} was not locked!" restore_umask return 1 else @@ -123,8 +123,8 @@ cleanup() { repo_unlock $repo $arch fi done - if [ -d "$TMPDIR/.scriptlock.$(basename $0)" ]; then - msg "Removing left over lock from $(basename $0)" + if [ -d "$TMPDIR/.scriptlock.${0##*/}" ]; then + msg "Removing left over lock from ${0##*/}" script_unlock fi rm -rf "$WORKDIR" @@ -324,7 +324,7 @@ check_pkgfile() { in_array "${pkgarch}" ${ARCHES[@]} 'any' || return 1 - if echo "$(basename ${pkgfile})" | grep -q "${pkgname}-${pkgver}-${pkgarch}"; then + if echo "${pkgfile##*/}" | grep -q "${pkgname}-${pkgver}-${pkgarch}"; then return 0 else return 1 @@ -418,8 +418,8 @@ check_pkgrepos() { [ -f "${FTP_BASE}/${PKGPOOL}/${pkgname}-${pkgver}-${pkgarch}"${PKGEXT} ] && return 1 [ -f "${FTP_BASE}/${PKGPOOL}/${pkgname}-${pkgver}-${pkgarch}"${PKGEXT}.sig ] && return 1 - [ -f "${FTP_BASE}/${PKGPOOL}/$(basename ${pkgfile})" ] && return 1 - [ -f "${FTP_BASE}/${PKGPOOL}/$(basename ${pkgfile}).sig" ] && return 1 + [ -f "${FTP_BASE}/${PKGPOOL}/${pkgfile##*/}" ] && return 1 + [ -f "${FTP_BASE}/${PKGPOOL}/${pkgfile##*/}.sig" ] && return 1 local repo local arch @@ -427,8 +427,8 @@ check_pkgrepos() { for arch in ${ARCHES[@]}; do [ -f "${FTP_BASE}/${repo}/os/${arch}/${pkgname}-${pkgver}-${pkgarch}"${PKGEXT} ] && return 1 [ -f "${FTP_BASE}/${repo}/os/${arch}/${pkgname}-${pkgver}-${pkgarch}"${PKGEXT}.sig ] && return 1 - [ -f "${FTP_BASE}/${repo}/os/${arch}/$(basename ${pkgfile})" ] && return 1 - [ -f "${FTP_BASE}/${repo}/os/${arch}/$(basename ${pkgfile}).sig" ] && return 1 + [ -f "${FTP_BASE}/${repo}/os/${arch}/${pkgfile##*/}" ] && return 1 + [ -f "${FTP_BASE}/${repo}/os/${arch}/${pkgfile##*/}.sig" ] && return 1 done done diff --git a/db-move b/db-move index 019faa2..9166a99 100755 --- a/db-move +++ b/db-move @@ -4,7 +4,7 @@ . "$(dirname $0)/config" if [ $# -lt 3 ]; then - msg "usage: $(basename $0) ..." + msg "usage: ${0##*/} ..." exit 1 fi @@ -90,12 +90,12 @@ for pkgbase in ${args[@]:2}; do /usr/bin/svn mv -q -r HEAD "${svnrepo_from}/$file" "${svnrepo_to}/" done /usr/bin/svn rm --force -q "${svnrepo_from}" - /usr/bin/svn commit -q "${WORKDIR}/svn/${pkgbase}" -m "$(basename $0): moved ${pkgbase} from [${repo_from}] to [${repo_to}] (${pkgarch})" + /usr/bin/svn commit -q "${WORKDIR}/svn/${pkgbase}" -m "${0##*/}: moved ${pkgbase} from [${repo_from}] to [${repo_to}] (${pkgarch})" for pkgname in ${pkgnames[@]}; do for tarch in ${tarches[@]}; do pkgpath=$(getpkgfile "${ftppath_from}/${tarch}/"${pkgname}-${pkgver}-${pkgarch}${PKGEXT}) - pkgfile=$(basename "${pkgpath}") + pkgfile="${pkgpath##*/}" # copy package to pool if needed # TODO: can be removed once every package has been moved to the package pool diff --git a/db-remove b/db-remove index a3e03e6..512ce51 100755 --- a/db-remove +++ b/db-remove @@ -4,7 +4,7 @@ . "$(dirname $0)/config" if [ $# -lt 3 ]; then - msg "usage: $(basename $0) ..." + msg "usage: ${0##*/} ..." exit 1 fi @@ -37,7 +37,7 @@ for pkgbase in ${pkgbases[@]}; do if [ -d "${WORKDIR}/svn/$pkgbase/repos/$svnrepo" ]; then remove_pkgs=(${remove_pkgs[@]} $(. "${WORKDIR}/svn/$pkgbase/repos/$svnrepo/PKGBUILD"; echo ${pkgname[@]})) /usr/bin/svn rm --force -q "${WORKDIR}/svn/$pkgbase/repos/$svnrepo" - /usr/bin/svn commit -q "${WORKDIR}/svn/$pkgbase" -m "$(basename $0): $pkgbase removed by $(id -un)" + /usr/bin/svn commit -q "${WORKDIR}/svn/$pkgbase" -m "${0##*/}: $pkgbase removed by $(id -un)" else warning "$pkgbase not found in $svnrepo" warning "Removing only $pkgbase from the repo" diff --git a/db-repo-add b/db-repo-add index b83fb77..99bb4b5 100755 --- a/db-repo-add +++ b/db-repo-add @@ -4,7 +4,7 @@ . "$(dirname $0)/config" if [ $# -lt 3 ]; then - msg "usage: $(basename $0) ..." + msg "usage: ${0##*/} ..." exit 1 fi diff --git a/db-repo-remove b/db-repo-remove index 4f04ed1..43be772 100755 --- a/db-repo-remove +++ b/db-repo-remove @@ -4,7 +4,7 @@ . "$(dirname $0)/config" if [ $# -lt 3 ]; then - msg "usage: $(basename $0) ..." + msg "usage: ${0##*/} ..." exit 1 fi diff --git a/db-update b/db-update index 60af79f..a9d307f 100755 --- a/db-update +++ b/db-update @@ -4,7 +4,7 @@ . "$(dirname $0)/config" if [ $# -ge 1 ]; then - warning "Calling $(basename $0) with a specific repository is no longer supported" + warning "Calling ${0##*/} with a specific repository is no longer supported" exit 1 fi @@ -30,19 +30,19 @@ for repo in ${repos[@]}; do if [ $? -eq 0 ]; then for pkg in ${pkgs[@]}; do if [ -h "${pkg}" ]; then - die "Package ${repo}/$(basename ${pkg}) is a symbolic link" + die "Package ${repo}/${pkg##*/} is a symbolic link" fi if ! check_pkgfile "${pkg}"; then - die "Package ${repo}/$(basename ${pkg}) is not consistent with its meta data" + die "Package ${repo}/${pkg##*/} is not consistent with its meta data" fi if ${REQUIRE_SIGNATURE} && ! check_pkgsig "${pkg}.sig"; then - die "Package ${repo}/$(basename ${pkg}) does not have a valid signature" + die "Package ${repo}/${pkg##*/} does not have a valid signature" fi if ! check_pkgsvn "${pkg}" "${repo}"; then - die "Package ${repo}/$(basename ${pkg}) is not consistent with svn repository" + die "Package ${repo}/${pkg##*/} is not consistent with svn repository" fi if ! check_pkgrepos "${pkg}"; then - die "Package ${repo}/$(basename ${pkg}) already exists in another repository" + die "Package ${repo}/${pkg##*/} already exists in another repository" fi done if ! check_splitpkgs ${repo} ${pkgs[@]}; then @@ -60,7 +60,7 @@ for repo in ${repos[@]}; do add_pkgs=() arch_pkgs=($(getpkgfiles "${STAGING}/${repo}/"*-${pkgarch}${PKGEXT} 2>/dev/null)) for pkg in ${arch_pkgs[@]} ${any_pkgs[@]}; do - pkgfile="$(basename ${pkg})" + pkgfile="${pkg##*/}" msg2 "${pkgfile} (${pkgarch})" # any packages might have been moved by the previous run if [ -f "${pkg}" ]; then diff --git a/test/lib/common.inc b/test/lib/common.inc index e0e7048..3eab68e 100644 --- a/test/lib/common.inc +++ b/test/lib/common.inc @@ -26,7 +26,7 @@ oneTimeSetUp() { local pkgarch local pkgversion local build - pkgdir="$(mktemp -d /tmp/$(basename $0).XXXXXXXXXX)" + pkgdir="$(mktemp -d /tmp/${0##*/}.XXXXXXXXXX)" cp -Lr $(dirname ${BASH_SOURCE[0]})/../packages/* "${pkgdir}" msg 'Building packages...' for d in "${pkgdir}"/*; do @@ -52,7 +52,7 @@ oneTimeSetUp() { fi for a in ${pkgarch[@]}; do for p in ${pkgname[@]}; do - cp ${p}-${pkgversion}-${a}${PKGEXT} $(dirname ${BASH_SOURCE[0]})/../packages/$(basename ${d}) + cp ${p}-${pkgversion}-${a}${PKGEXT} $(dirname ${BASH_SOURCE[0]})/../packages/${d##*/} done done fi @@ -71,7 +71,7 @@ setUp() { local a [ -f "$(dirname ${BASH_SOURCE[0]})/../../config.local" ] && die "$(dirname ${BASH_SOURCE[0]})/../../config.local exists" - TMP="$(mktemp -d /dev/shm/$(basename $0).XXXXXXXXXX)" + TMP="$(mktemp -d /dev/shm/${0##*/}.XXXXXXXXXX)" #msg "Using ${TMP}" PKGREPOS=('core' 'extra' 'testing') @@ -92,7 +92,7 @@ setUp() { svn checkout -q "file://${TMP}/svn-packages-repo" "${TMP}/svn-packages-copy" for p in "${pkgdir}"/*; do - pkg=$(basename $p) + pkg=${p##*/} mkdir -p "${TMP}/svn-packages-copy/${pkg}"/{trunk,repos} cp "${p}"/* "${TMP}/svn-packages-copy"/${pkg}/trunk/ svn add -q "${TMP}/svn-packages-copy"/${pkg} diff --git a/test/test.d/db-move.sh b/test/test.d/db-move.sh index 9d7c1f6..890ffc4 100755 --- a/test/test.d/db-move.sh +++ b/test/test.d/db-move.sh @@ -107,12 +107,12 @@ testMoveSplitPackages() { for arch in ${arches[@]}; do for pkg in "${pkgdir}/pkg-split-a"/*-${arch}${PKGEXT}; do - checkPackage extra $(basename ${pkg}) ${arch} + checkPackage extra ${pkg##*/} ${arch} done done for arch in ${arches[@]}; do for pkg in "${pkgdir}/pkg-split-b"/*-${arch}${PKGEXT}; do - checkPackage testing $(basename ${pkg}) ${arch} + checkPackage testing ${pkg##*/} ${arch} done done diff --git a/test/test.d/db-update.sh b/test/test.d/db-update.sh index e38c328..2021993 100755 --- a/test/test.d/db-update.sh +++ b/test/test.d/db-update.sh @@ -69,7 +69,7 @@ testAddSplitPackages() { for pkgbase in ${pkgs[@]}; do for arch in ${arches[@]}; do for pkg in "${pkgdir}/${pkgbase}"/*-${arch}${PKGEXT}; do - checkPackage extra $(basename ${pkg}) ${arch} + checkPackage extra ${pkg##*/} ${arch} done done done diff --git a/test/test.d/ftpdir-cleanup.sh b/test/test.d/ftpdir-cleanup.sh index 20026b4..bfea7ea 100755 --- a/test/test.d/ftpdir-cleanup.sh +++ b/test/test.d/ftpdir-cleanup.sh @@ -113,7 +113,7 @@ testCleanupSplitPackages() { done for pkg in "${pkgdir}/${pkgs[1]}"/*-${arch}${PKGEXT}; do - checkPackage extra $(basename ${pkg}) ${arch} + checkPackage extra ${pkg##*/} ${arch} done done } diff --git a/test/test.d/pool-transition.sh b/test/test.d/pool-transition.sh index 5873f00..4912968 100755 --- a/test/test.d/pool-transition.sh +++ b/test/test.d/pool-transition.sh @@ -23,7 +23,7 @@ testMovePackagesWithoutPool() { for arch in ${arches[@]}; do for old in 0 2; do for pkg in "${pkgdir}/${pkgs[${old}]}"/*-${arch}${PKGEXT}; do - pkg=$(basename $pkg) + pkg=${pkg##*/} mv -f "${FTP_BASE}/${PKGPOOL}/${pkg}" "${FTP_BASE}/testing/os/${arch}/${pkg}" done done @@ -38,7 +38,7 @@ testMovePackagesWithoutPool() { for pkgbase in ${pkgs[@]}; do for arch in ${arches[@]}; do for pkg in "${pkgdir}/${pkgbase}"/*-${arch}${PKGEXT}; do - checkPackage extra $(basename ${pkg}) ${arch} + checkPackage extra ${pkg##*/} ${arch} done checkRemovedPackage testing ${pkgbase} ${arch} done @@ -95,7 +95,7 @@ testMoveAnyPackagesWithoutPool() { # transform a package to old style layout for pkg in "${pkgdir}/${pkgs[0]}"/*-any${PKGEXT}; do - pkg=$(basename $pkg) + pkg=${pkg##*/} mv -f "${FTP_BASE}/${PKGPOOL}/${pkg}" "${FTP_BASE}/testing/os/any/${pkg}" for arch in i686 x86_64; do ln -sf "../any/${pkg}" "${FTP_BASE}/testing/os/${arch}/${pkg}" @@ -110,13 +110,13 @@ testMoveAnyPackagesWithoutPool() { for pkgbase in ${pkgs[@]}; do for pkg in "${pkgdir}/${pkgbase}"/*-any${PKGEXT}; do - checkAnyPackage extra $(basename ${pkg}) + checkAnyPackage extra ${pkg##*/} done checkRemovedAnyPackage testing ${pkgbase} done for pkg in "${pkgdir}/${pkgs[0]}"/*-any${PKGEXT}; do - pkg=$(basename $pkg) + pkg=${pkg##*/} for arch in any i686 x86_64; do [ -f "${FTP_BASE}/testing/os/${arch}/${pkg}" ] && fail "testing/os/${arch}/${pkg} found" done @@ -132,7 +132,7 @@ testUpdateSameAnyPackageToDifferentRepositoriesWithoutPool() { # transform a package to old style layout for pkg in "${pkgdir}/pkg-any-a"/*-any${PKGEXT}; do - pkg=$(basename $pkg) + pkg=${pkg##*/} mv -f "${FTP_BASE}/${PKGPOOL}/${pkg}" "${FTP_BASE}/extra/os/any/${pkg}" for arch in i686 x86_64; do ln -sf "../any/${pkg}" "${FTP_BASE}/extra/os/${arch}/${pkg}" diff --git a/testing2x b/testing2x index 54cae11..caf6df1 100755 --- a/testing2x +++ b/testing2x @@ -4,7 +4,7 @@ . "$(dirname $0)/config" if [ $# -lt 1 ]; then - msg "usage: $(basename $0) ..." + msg "usage: ${0##*/} ..." exit 1 fi -- cgit v1.2.2 From fb35f77bbbf729cdf5cf054e6e82706af76fce9a Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sat, 14 Jan 2012 20:12:33 +0100 Subject: Add [multilib-staging] to sigurd config --- config.local.sigurd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.local.sigurd b/config.local.sigurd index d28aa37..468ec64 100644 --- a/config.local.sigurd +++ b/config.local.sigurd @@ -1,4 +1,4 @@ -PKGREPOS=('community' 'community-testing' 'community-staging' 'multilib' 'multilib-testing') +PKGREPOS=('community' 'community-testing' 'community-staging' 'multilib' 'multilib-testing' 'multilib-staging') PKGPOOL='pool/community' SRCPOOL='sources/community' SVNREPO='file:///srv/svn-packages' -- cgit v1.2.2 From 8d85e0d1617fa319701e1c18b2dfb63ee7012fe7 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 19 Feb 2012 02:21:11 +0100 Subject: Add at least one positive signing test --- test/test.d/signed-packages.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/test.d/signed-packages.sh b/test/test.d/signed-packages.sh index 20ad844..03566ef 100755 --- a/test/test.d/signed-packages.sh +++ b/test/test.d/signed-packages.sh @@ -3,6 +3,11 @@ curdir=$(readlink -e $(dirname $0)) . "${curdir}/../lib/common.inc" +testAddSignedPackage() { + releasePackage extra 'pkg-simple-a' 'i686' + ../db-update || fail "db-update failed!" +} + testAddUnsignedPackage() { releasePackage extra 'pkg-simple-a' 'i686' rm "${STAGING}"/extra/*.sig -- cgit v1.2.2 From f6235619f379b20fdb5fb11e849bdad60c8503bc Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 19 Feb 2012 02:21:59 +0100 Subject: Fix typo and leftover shm usage --- test/lib/common.inc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/lib/common.inc b/test/lib/common.inc index 3eab68e..a703152 100644 --- a/test/lib/common.inc +++ b/test/lib/common.inc @@ -71,7 +71,7 @@ setUp() { local a [ -f "$(dirname ${BASH_SOURCE[0]})/../../config.local" ] && die "$(dirname ${BASH_SOURCE[0]})/../../config.local exists" - TMP="$(mktemp -d /dev/shm/${0##*/}.XXXXXXXXXX)" + TMP="$(mktemp -d /tmp/${0##*/}.XXXXXXXXXX)" #msg "Using ${TMP}" PKGREPOS=('core' 'extra' 'testing') @@ -131,7 +131,7 @@ releasePackage() { local pkgname pushd "${TMP}/svn-packages-copy"/${pkgbase}/trunk/ >/dev/null - archrelease ${repo}-${arch} >/dev/null 2&>1 + archrelease ${repo}-${arch} >/dev/null 2>&1 pkgver=$(. PKGBUILD; echo $(get_full_version ${epoch:-0} ${pkgver} ${pkgrel})) pkgname=($(. PKGBUILD; echo ${pkgname[@]})) popd >/dev/null -- cgit v1.2.2 From 3b63496ceb3cc063588a5ffefbe4a21c9c938b98 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 19 Feb 2012 02:23:16 +0100 Subject: Sourceballs: Output details if fetching the sources failed --- cron-jobs/sourceballs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index b55de05..87897f3 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -88,7 +88,7 @@ for repo in ${PKGREPOS[@]}; do # Build the actual source package pushd "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" >/dev/null - makepkg --nocolor --allsource --ignorearch >/dev/null 2>&1 + makepkg --nocolor --allsource --ignorearch >"${WORKDIR}/${pkgbase}.log" 2>&1 if [ $? -eq 0 ] && [ -f "${pkgbase}-${pkgver}${SRCEXT}" ]; then mv "${pkgbase}-${pkgver}${SRCEXT}" "${FTP_BASE}/${SRCPOOL}" # Avoid creating the same source package for every arch @@ -96,6 +96,7 @@ for repo in ${PKGREPOS[@]}; do newpkgs[${#newpkgs[*]}]="${pkgbase}-${pkgver}${SRCEXT}" else failedpkgs[${#failedpkgs[*]}]="${pkgbase}-${pkgver}${SRCEXT}" + cat "${WORKDIR}/${pkgbase}.log" >> "${WORKDIR}/makepkg-fail.log" fi popd >/dev/null fi @@ -141,4 +142,9 @@ if [ ${#old_pkgs[@]} -ge 1 ]; then done fi +if [ -f "${WORKDIR}/makepkg-fail.log" ]; then + msg "Log of failed packages" + cat "${WORKDIR}/makepkg-fail.log" +fi + script_unlock -- cgit v1.2.2 From a14cb544ed9edf6cc07f8cdd5bf4d6c5f27a0492 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 19 Feb 2012 11:26:57 +0100 Subject: Do not check source gpg signatures --- cron-jobs/sourceballs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 87897f3..4c4cbfd 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -88,7 +88,7 @@ for repo in ${PKGREPOS[@]}; do # Build the actual source package pushd "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" >/dev/null - makepkg --nocolor --allsource --ignorearch >"${WORKDIR}/${pkgbase}.log" 2>&1 + makepkg --nocolor --allsource --ignorearch --skippgpcheck >"${WORKDIR}/${pkgbase}.log" 2>&1 if [ $? -eq 0 ] && [ -f "${pkgbase}-${pkgver}${SRCEXT}" ]; then mv "${pkgbase}-${pkgver}${SRCEXT}" "${FTP_BASE}/${SRCPOOL}" # Avoid creating the same source package for every arch -- cgit v1.2.2 From 6c29c8e05486934cb52bfba9baef02496d26199a Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Wed, 7 Mar 2012 14:56:58 +0100 Subject: sourceballs: If ALLOWED_LICENSES is empty create source packages for every package --- cron-jobs/sourceballs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 4c4cbfd..a24676d 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -64,7 +64,7 @@ for repo in ${PKGREPOS[@]}; do continue fi # Check if the license or .force file does not enforce creating a source package - if ! (chk_license ${pkglicense[@]} || grep -Fqx "${pkgbase}" "${dirname}/sourceballs.force"); then + if ! ([[ -z ${ALLOWED_LICENSES[@]} ]] || chk_license ${pkglicense[@]} || grep -Fqx "${pkgbase}" "${dirname}/sourceballs.force"); then continue fi # Store the expected file name of the source package -- cgit v1.2.2 From 08439d5fd18c5b64670aed4f3025216ce52a8bb7 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Wed, 7 Mar 2012 15:07:34 +0100 Subject: Update sourceballs.skip --- cron-jobs/sourceballs.skip | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/cron-jobs/sourceballs.skip b/cron-jobs/sourceballs.skip index 14d6f4b..0e1731c 100644 --- a/cron-jobs/sourceballs.skip +++ b/cron-jobs/sourceballs.skip @@ -1,14 +1,28 @@ -nexuiz-data +0ad-data +alienarena-data +blobwars-data +btanks-data +dangerdeep-data +egoboo-data +fillets-ng-data +flightgear-data +frogatto-data +gcompris-data +naev-data +openarena-data +rocksndiamonds-data +smc-data +speed-dreams-data torcs-data tremulous-data ufoai-data -frogatto-data vdrift-data -naev-data -btanks-data +warmux-data wesnoth-data -texlive-bin +widelands-data +xonotic-data texlive-bibtexextra +texlive-bin texlive-core texlive-fontsextra texlive-formatsextra -- cgit v1.2.2 From 31c1ebb69681e9b9b9d17795210f6bbd24ee2381 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Mon, 26 Mar 2012 11:26:41 +0200 Subject: Use gpg status-file instead of status-fd --- db-functions | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/db-functions b/db-functions index 3100e31..1b77035 100644 --- a/db-functions +++ b/db-functions @@ -471,9 +471,7 @@ check_pkgsig() { local ret=1 local fd="$(mktemp --tmpdir="${WORKDIR}")" - exec 4>"${fd}" - gpg --homedir /etc/pacman.d/gnupg/ --status-fd 4 --verify "${signature}" >/dev/null 2>&1 - exec 4>&- + gpg --homedir /etc/pacman.d/gnupg/ --status-file "${fd}" --verify "${signature}" >/dev/null 2>&1 if grep -q TRUST_FULLY "${fd}"; then ret=0 fi -- cgit v1.2.2 From 180e655748dc6fd8c62ca7fa269e4f614ac1e8ca Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 29 Apr 2012 09:18:52 +0200 Subject: Respect TMPDIR config when creating the working directory --- db-functions | 2 +- test/lib/common.inc | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/db-functions b/db-functions index 1b77035..c992fe5 100644 --- a/db-functions +++ b/db-functions @@ -16,7 +16,7 @@ restore_umask () { } # set up general environment -WORKDIR=$(mktemp -d /tmp/${0##*/}.XXXXXXXXXX) +WORKDIR=$(mktemp -d "${TMPDIR}/${0##*/}.XXXXXXXXXX") LOCKS=() # check if messages are to be printed using color diff --git a/test/lib/common.inc b/test/lib/common.inc index a703152..3001ad8 100644 --- a/test/lib/common.inc +++ b/test/lib/common.inc @@ -1,6 +1,8 @@ set -E . "$(dirname ${BASH_SOURCE[0]})/../../config" +# override the default TMPDIR +export TMPDIR="$(mktemp -d /tmp/${0##*/}.XXXXXXXXXX)" . "$(dirname ${BASH_SOURCE[0]})/../../db-functions" signpkg() { -- cgit v1.2.2 From 33cfe12c34747bd52e2730e45566e5b21cd76153 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 29 Apr 2012 10:10:43 +0200 Subject: Source the config before the functions as the latter references the former --- cron-jobs/ftpdir-cleanup | 2 +- cron-jobs/integrity-check | 2 +- cron-jobs/sourceballs | 2 +- cron-jobs/update-web-db | 2 +- db-move | 2 +- db-remove | 2 +- db-repo-add | 2 +- db-repo-remove | 2 +- db-update | 2 +- test/lib/common.inc | 2 +- testing2x | 2 +- 11 files changed, 11 insertions(+), 11 deletions(-) diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index 7839d50..9bc88d7 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -1,7 +1,7 @@ #!/bin/bash -. "$(dirname $0)/../db-functions" . "$(dirname $0)/../config" +. "$(dirname $0)/../db-functions" clean_pkg() { local pkg diff --git a/cron-jobs/integrity-check b/cron-jobs/integrity-check index 044f093..f6c26cf 100755 --- a/cron-jobs/integrity-check +++ b/cron-jobs/integrity-check @@ -2,8 +2,8 @@ dirname="$(dirname $0)" -. "${dirname}/../db-functions" . "${dirname}/../config" +. "${dirname}/../db-functions" script_lock diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index a24676d..d494905 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -1,8 +1,8 @@ #!/bin/bash dirname="$(dirname $(readlink -e $0))" -. "${dirname}/../db-functions" . "${dirname}/../config" +. "${dirname}/../db-functions" pushd "${WORKDIR}" >/dev/null script_lock diff --git a/cron-jobs/update-web-db b/cron-jobs/update-web-db index b1767ca..195d1fc 100755 --- a/cron-jobs/update-web-db +++ b/cron-jobs/update-web-db @@ -1,7 +1,7 @@ #!/bin/bash -. "$(dirname $0)/../db-functions" . "$(dirname $0)/../config" +. "$(dirname $0)/../db-functions" # setup paths SPATH="/srv/http/archweb" diff --git a/db-move b/db-move index 9166a99..a23aebe 100755 --- a/db-move +++ b/db-move @@ -1,7 +1,7 @@ #!/bin/bash -. "$(dirname $0)/db-functions" . "$(dirname $0)/config" +. "$(dirname $0)/db-functions" if [ $# -lt 3 ]; then msg "usage: ${0##*/} ..." diff --git a/db-remove b/db-remove index 512ce51..da32c78 100755 --- a/db-remove +++ b/db-remove @@ -1,7 +1,7 @@ #!/bin/bash -. "$(dirname $0)/db-functions" . "$(dirname $0)/config" +. "$(dirname $0)/db-functions" if [ $# -lt 3 ]; then msg "usage: ${0##*/} ..." diff --git a/db-repo-add b/db-repo-add index 99bb4b5..5d5b653 100755 --- a/db-repo-add +++ b/db-repo-add @@ -1,7 +1,7 @@ #!/bin/bash -. "$(dirname $0)/db-functions" . "$(dirname $0)/config" +. "$(dirname $0)/db-functions" if [ $# -lt 3 ]; then msg "usage: ${0##*/} ..." diff --git a/db-repo-remove b/db-repo-remove index 43be772..2a693f4 100755 --- a/db-repo-remove +++ b/db-repo-remove @@ -1,7 +1,7 @@ #!/bin/bash -. "$(dirname $0)/db-functions" . "$(dirname $0)/config" +. "$(dirname $0)/db-functions" if [ $# -lt 3 ]; then msg "usage: ${0##*/} ..." diff --git a/db-update b/db-update index a9d307f..8be0d54 100755 --- a/db-update +++ b/db-update @@ -1,7 +1,7 @@ #!/bin/bash -. "$(dirname $0)/db-functions" . "$(dirname $0)/config" +. "$(dirname $0)/db-functions" if [ $# -ge 1 ]; then warning "Calling ${0##*/} with a specific repository is no longer supported" diff --git a/test/lib/common.inc b/test/lib/common.inc index 3001ad8..6efd517 100644 --- a/test/lib/common.inc +++ b/test/lib/common.inc @@ -2,7 +2,7 @@ set -E . "$(dirname ${BASH_SOURCE[0]})/../../config" # override the default TMPDIR -export TMPDIR="$(mktemp -d /tmp/${0##*/}.XXXXXXXXXX)" +TMPDIR="$(mktemp -d /tmp/${0##*/}.XXXXXXXXXX)" . "$(dirname ${BASH_SOURCE[0]})/../../db-functions" signpkg() { diff --git a/testing2x b/testing2x index caf6df1..f209e83 100755 --- a/testing2x +++ b/testing2x @@ -1,7 +1,7 @@ #!/bin/bash -. "$(dirname $0)/db-functions" . "$(dirname $0)/config" +. "$(dirname $0)/db-functions" if [ $# -lt 1 ]; then msg "usage: ${0##*/} ..." -- cgit v1.2.2 From 75a16bafa3d79499af710414c57ff50451b158a3 Mon Sep 17 00:00:00 2001 From: Florian Pritz Date: Thu, 12 Apr 2012 11:41:04 +0200 Subject: db-move: commit all arches at once Signed-off-by: Florian Pritz Signed-off-by: Pierre Schmitz --- db-move | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/db-move b/db-move index a23aebe..2bb2ea8 100755 --- a/db-move +++ b/db-move @@ -63,6 +63,7 @@ msg "Moving packages from [${repo_from}] to [${repo_to}]..." declare -A add_pkgs declare -A remove_pkgs for pkgbase in ${args[@]:2}; do + tag_list="" for pkgarch in ${ARCHES[@]} 'any'; do svnrepo_from="${WORKDIR}/svn/${pkgbase}/repos/${repo_from}-${pkgarch}" svnrepo_to="${WORKDIR}/svn/${pkgbase}/repos/${repo_to}-${pkgarch}" @@ -90,7 +91,7 @@ for pkgbase in ${args[@]:2}; do /usr/bin/svn mv -q -r HEAD "${svnrepo_from}/$file" "${svnrepo_to}/" done /usr/bin/svn rm --force -q "${svnrepo_from}" - /usr/bin/svn commit -q "${WORKDIR}/svn/${pkgbase}" -m "${0##*/}: moved ${pkgbase} from [${repo_from}] to [${repo_to}] (${pkgarch})" + tag_list="$tag_list, $pkgarch" for pkgname in ${pkgnames[@]}; do for tarch in ${tarches[@]}; do @@ -112,6 +113,8 @@ for pkgbase in ${args[@]:2}; do done fi done + tag_list="${tag_list#, }" + /usr/bin/svn commit -q "${WORKDIR}/svn/${pkgbase}" -m "${0##*/}: moved ${pkgbase} from [${repo_from}] to [${repo_to}] (${tag_list})" done for tarch in ${ARCHES[@]}; do -- cgit v1.2.2 From c639ae7d7a9fa5544d495e0b4d4f3b49611a7af9 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 10 Jun 2012 11:45:59 +0200 Subject: Remove support for packages in legacy $repo/os/any directories --- cron-jobs/ftpdir-cleanup | 23 --------- test/lib/common.inc | 17 +++---- test/test.d/pool-transition.sh | 104 ----------------------------------------- 3 files changed, 7 insertions(+), 137 deletions(-) diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index 9bc88d7..15eb46c 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -75,29 +75,6 @@ if [ ${#old_pkgs[@]} -ge 1 ]; then done fi -# cleanup of legacy $repo/os/any directories -for repo in ${PKGREPOS[@]}; do - if [ ! -d "${FTP_BASE}/${repo}/os/any" ]; then - continue - fi - if [ -n "$(find "${FTP_BASE}/${repo}/os/any" -type d -empty)" ]; then - msg "Removing empty legacy directory ${repo}/os/any" - ${CLEANUP_DRYRUN} || rmdir "${FTP_BASE}/${repo}/os/any" - continue - fi - find "${FTP_BASE}/${repo}/os/any" -name "*${PKGEXT}" -printf '%f\n' | sort > "${WORKDIR}/any-${repo}" - cat "${WORKDIR}/db-${repo}-"* | sort -u > "${WORKDIR}/all-${repo}" - - old_pkgs=($(comm -23 "${WORKDIR}/any-${repo}" "${WORKDIR}/all-${repo}")) - if [ ${#old_pkgs[@]} -ge 1 ]; then - msg "Removing old packages from [${repo}] (any)..." - for old_pkg in ${old_pkgs[@]}; do - msg2 "${old_pkg}" - clean_pkg "${FTP_BASE}/${repo}/os/any/${old_pkg}" - done - fi -done - old_pkgs=($(find ${CLEANUP_DESTDIR} -type f -name "*${PKGEXT}" -mtime +${CLEANUP_KEEP} -printf '%f\n')) if [ ${#old_pkgs[@]} -ge 1 ]; then msg "Removing old packages from the cleanup directory..." diff --git a/test/lib/common.inc b/test/lib/common.inc index 6efd517..827c207 100644 --- a/test/lib/common.inc +++ b/test/lib/common.inc @@ -82,7 +82,7 @@ setUp() { for r in ${PKGREPOS[@]}; do mkdir -p "${TMP}/staging/${r}" - for a in ${ARCHES[@]} any; do + for a in ${ARCHES[@]}; do mkdir -p "${TMP}/ftp/${r}/os/${a}" done done @@ -169,18 +169,15 @@ checkAnyPackageDB() { [ "$(readlink -e "${FTP_BASE}/${repo}/os/${arch}/${pkg}.sig")" == "${FTP_BASE}/${PKGPOOL}/${pkg}.sig" ] \ || fail "${repo}/os/${arch}/${pkg}.sig does not link to ${PKGPOOL}/${pkg}.sig" fi + + for db in ${DBEXT} ${FILESEXT}; do + ( [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" ] \ + && bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" -O | grep -q ${pkg}) \ + || fail "${pkg} not in ${repo}/os/${arch}/${repo}${db%.tar.*}" + done done [ -r "${STAGING}"/${repo}/${pkg} ] && fail "${repo}/${pkg} found in staging dir" [ -r "${STAGING}"/${repo}/${pkg}.sig ] && fail "${repo}/${pkg}.sig found in staging dir" - - for db in ${DBEXT} ${FILESEXT}; do - ( [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" ] \ - && bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" -O | grep -q ${pkg}) \ - || fail "${pkg} not in ${repo}/os/${arch}/${repo}${db%.tar.*}" - done - - [ -r "${FTP_BASE}/${repo}/os/any/${pkg}" ] && fail "${repo}/os/any/${pkg} should not exist" - [ -r "${FTP_BASE}/${repo}/os/any/${pkg}.sig" ] && fail "${repo}/os/any/${pkg}.sig should not exist" } checkAnyPackage() { diff --git a/test/test.d/pool-transition.sh b/test/test.d/pool-transition.sh index 4912968..7880b0a 100755 --- a/test/test.d/pool-transition.sh +++ b/test/test.d/pool-transition.sh @@ -45,108 +45,4 @@ testMovePackagesWithoutPool() { done } -testUpdateAnyPackageWithoutPool() { - local pkgname='pkg-any-a' - local pkg1='pkg-any-a-1-1-any.pkg.tar.xz' - local pkg2='pkg-any-a-1-2-any.pkg.tar.xz' - local arch - - - releasePackage extra pkg-any-a any - ../db-update - # transform two packages to old style layout - mv -f "${FTP_BASE}/${PKGPOOL}/${pkg1}" "${FTP_BASE}/extra/os/any" - for arch in i686 x86_64; do - ln -sf "../any/${pkg1}" "${FTP_BASE}/extra/os/${arch}" - done - - pushd "${TMP}/svn-packages-copy/${pkgname}/trunk/" >/dev/null - sed 's/pkgrel=1/pkgrel=2/g' -i PKGBUILD - svn commit -q -m"update pkg to pkgrel=2" >/dev/null - sudo extra-i686-build >/dev/null 2>&1 - mv "${pkg2}" "${pkgdir}/${pkgname}/" - popd >/dev/null - - releasePackage extra ${pkgname} any - ../db-update - rm -f "${pkgdir}/${pkgname}/${pkg2}" - - ../cron-jobs/ftpdir-cleanup >/dev/null - - checkAnyPackage extra "${pkg2}" - - [ -f "${FTP_BASE}/${PKGPOOL}/${pkg1}" ] && fail "${PKGPOOL}/${pkg1} found" - for arch in any i686 x86_64; do - [ -f "${FTP_BASE}/extra/os/${arch}/${pkg1}" ] && fail "extra/os/${arch}/${pkg1} found" - done -} - -testMoveAnyPackagesWithoutPool() { - local pkgs=('pkg-any-a' 'pkg-any-b') - local pkgbase - local arch - local pkg - - for pkgbase in ${pkgs[@]}; do - releasePackage testing ${pkgbase} any - done - - ../db-update - - # transform a package to old style layout - for pkg in "${pkgdir}/${pkgs[0]}"/*-any${PKGEXT}; do - pkg=${pkg##*/} - mv -f "${FTP_BASE}/${PKGPOOL}/${pkg}" "${FTP_BASE}/testing/os/any/${pkg}" - for arch in i686 x86_64; do - ln -sf "../any/${pkg}" "${FTP_BASE}/testing/os/${arch}/${pkg}" - done - done - - ../cron-jobs/ftpdir-cleanup >/dev/null - - ../db-move testing extra ${pkgs[@]} - - ../cron-jobs/ftpdir-cleanup >/dev/null - - for pkgbase in ${pkgs[@]}; do - for pkg in "${pkgdir}/${pkgbase}"/*-any${PKGEXT}; do - checkAnyPackage extra ${pkg##*/} - done - checkRemovedAnyPackage testing ${pkgbase} - done - - for pkg in "${pkgdir}/${pkgs[0]}"/*-any${PKGEXT}; do - pkg=${pkg##*/} - for arch in any i686 x86_64; do - [ -f "${FTP_BASE}/testing/os/${arch}/${pkg}" ] && fail "testing/os/${arch}/${pkg} found" - done - done -} - -testUpdateSameAnyPackageToDifferentRepositoriesWithoutPool() { - local pkg - local arch - - releasePackage extra pkg-any-a any - ../db-update - - # transform a package to old style layout - for pkg in "${pkgdir}/pkg-any-a"/*-any${PKGEXT}; do - pkg=${pkg##*/} - mv -f "${FTP_BASE}/${PKGPOOL}/${pkg}" "${FTP_BASE}/extra/os/any/${pkg}" - for arch in i686 x86_64; do - ln -sf "../any/${pkg}" "${FTP_BASE}/extra/os/${arch}/${pkg}" - done - done - - releasePackage testing pkg-any-a any - ../db-update >/dev/null 2>&1 && (fail 'Adding an existing package to another repository should fail'; return 1) - - for arch in i686 x86_64; do - ( [ -r "${FTP_BASE}/testing/os/${arch}/testing${DBEXT%.tar.*}" ] \ - && bsdtar -xf "${FTP_BASE}/testing/os/${arch}/testing${DBEXT%.tar.*}" -O | grep -q pkg-any-a) \ - && fail "pkg-any-a should not be in testing/os/${arch}/testing${DBEXT%.tar.*}" - done -} - . "${curdir}/../lib/shunit2" -- cgit v1.2.2 From fe19d9cff1cb41ed8456e4cce7be474599cbf503 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 10 Jun 2012 12:02:50 +0200 Subject: Remove support for packages that are not in the package pool --- db-functions | 11 ---------- db-move | 5 ----- test/test.d/pool-transition.sh | 48 ------------------------------------------ 3 files changed, 64 deletions(-) delete mode 100755 test/test.d/pool-transition.sh diff --git a/db-functions b/db-functions index c992fe5..de5e8b8 100644 --- a/db-functions +++ b/db-functions @@ -421,17 +421,6 @@ check_pkgrepos() { [ -f "${FTP_BASE}/${PKGPOOL}/${pkgfile##*/}" ] && return 1 [ -f "${FTP_BASE}/${PKGPOOL}/${pkgfile##*/}.sig" ] && return 1 - local repo - local arch - for repo in ${PKGREPOS[@]}; do - for arch in ${ARCHES[@]}; do - [ -f "${FTP_BASE}/${repo}/os/${arch}/${pkgname}-${pkgver}-${pkgarch}"${PKGEXT} ] && return 1 - [ -f "${FTP_BASE}/${repo}/os/${arch}/${pkgname}-${pkgver}-${pkgarch}"${PKGEXT}.sig ] && return 1 - [ -f "${FTP_BASE}/${repo}/os/${arch}/${pkgfile##*/}" ] && return 1 - [ -f "${FTP_BASE}/${repo}/os/${arch}/${pkgfile##*/}.sig" ] && return 1 - done - done - return 0 } diff --git a/db-move b/db-move index 2bb2ea8..8ea7431 100755 --- a/db-move +++ b/db-move @@ -98,11 +98,6 @@ for pkgbase in ${args[@]:2}; do pkgpath=$(getpkgfile "${ftppath_from}/${tarch}/"${pkgname}-${pkgver}-${pkgarch}${PKGEXT}) pkgfile="${pkgpath##*/}" - # copy package to pool if needed - # TODO: can be removed once every package has been moved to the package pool - if [ ! -f ${FTP_BASE}/${PKGPOOL}/${pkgfile} ]; then - cp ${pkgpath} ${FTP_BASE}/${PKGPOOL} - fi ln -s "../../../${PKGPOOL}/${pkgfile}" ${ftppath_to}/${tarch}/ if [ -f ${FTP_BASE}/${PKGPOOL}/${pkgfile}.sig ]; then ln -s "../../../${PKGPOOL}/${pkgfile}.sig" ${ftppath_to}/${tarch}/ diff --git a/test/test.d/pool-transition.sh b/test/test.d/pool-transition.sh deleted file mode 100755 index 7880b0a..0000000 --- a/test/test.d/pool-transition.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/bin/bash - -curdir=$(readlink -e $(dirname $0)) -. "${curdir}/../lib/common.inc" - -testMovePackagesWithoutPool() { - local arches=('i686' 'x86_64') - local pkgs=('pkg-simple-a' 'pkg-simple-b' 'pkg-split-a' 'pkg-split-b') - local pkgbase - local arch - local pkg - local old - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage testing ${pkgbase} ${arch} - done - done - - ../db-update - - # transform two packages to old style layout - for arch in ${arches[@]}; do - for old in 0 2; do - for pkg in "${pkgdir}/${pkgs[${old}]}"/*-${arch}${PKGEXT}; do - pkg=${pkg##*/} - mv -f "${FTP_BASE}/${PKGPOOL}/${pkg}" "${FTP_BASE}/testing/os/${arch}/${pkg}" - done - done - done - - ../cron-jobs/ftpdir-cleanup >/dev/null - - ../db-move testing extra ${pkgs[@]} - - ../cron-jobs/ftpdir-cleanup >/dev/null - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - for pkg in "${pkgdir}/${pkgbase}"/*-${arch}${PKGEXT}; do - checkPackage extra ${pkg##*/} ${arch} - done - checkRemovedPackage testing ${pkgbase} ${arch} - done - done -} - -. "${curdir}/../lib/shunit2" -- cgit v1.2.2 From bc7cc36f4741c7a800afd94952dfccfdb4da2707 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 10 Jun 2012 12:19:47 +0200 Subject: Use our own copy of makepkg.conf instead of depending on the host one --- cron-jobs/makepkg.conf | 121 +++++++++++++++++++++++++++++++++++++++++++++++++ cron-jobs/sourceballs | 2 +- 2 files changed, 122 insertions(+), 1 deletion(-) create mode 100644 cron-jobs/makepkg.conf diff --git a/cron-jobs/makepkg.conf b/cron-jobs/makepkg.conf new file mode 100644 index 0000000..ff460ec --- /dev/null +++ b/cron-jobs/makepkg.conf @@ -0,0 +1,121 @@ +# +# /etc/makepkg.conf +# + +######################################################################### +# SOURCE ACQUISITION +######################################################################### +# +#-- The download utilities that makepkg should use to acquire sources +# Format: 'protocol::agent' +DLAGENTS=('ftp::/usr/bin/curl -fC - --ftp-pasv --retry 3 --retry-delay 3 -o %o %u' + 'http::/usr/bin/curl -fLC - --retry 3 --retry-delay 3 -o %o %u' + 'https::/usr/bin/curl -fLC - --retry 3 --retry-delay 3 -o %o %u' + 'rsync::/usr/bin/rsync -z %u %o' + 'scp::/usr/bin/scp -C %u %o') + +# Other common tools: +# /usr/bin/snarf +# /usr/bin/lftpget -c +# /usr/bin/wget + +######################################################################### +# ARCHITECTURE, COMPILE FLAGS +######################################################################### +# +CARCH="x86_64" +CHOST="x86_64-unknown-linux-gnu" + +#-- Compiler and Linker Flags +# -march (or -mcpu) builds exclusively for an architecture +# -mtune optimizes for an architecture, but builds for whole processor family +CFLAGS="-march=x86-64 -mtune=generic -O2 -pipe -fstack-protector --param=ssp-buffer-size=4 -D_FORTIFY_SOURCE=2" +CXXFLAGS="-march=x86-64 -mtune=generic -O2 -pipe -fstack-protector --param=ssp-buffer-size=4 -D_FORTIFY_SOURCE=2" +LDFLAGS="-Wl,-O1,--sort-common,--as-needed,-z,relro,--hash-style=gnu" +#-- Make Flags: change this for DistCC/SMP systems +#MAKEFLAGS="-j2" + +######################################################################### +# BUILD ENVIRONMENT +######################################################################### +# +# Defaults: BUILDENV=(fakeroot !distcc color !ccache check !sign) +# A negated environment option will do the opposite of the comments below. +# +#-- fakeroot: Allow building packages as a non-root user +#-- distcc: Use the Distributed C/C++/ObjC compiler +#-- color: Colorize output messages +#-- ccache: Use ccache to cache compilation +#-- check: Run the check() function if present in the PKGBUILD +#-- sign: Generate PGP signature file +# +BUILDENV=(fakeroot !distcc color !ccache check !sign) +# +#-- If using DistCC, your MAKEFLAGS will also need modification. In addition, +#-- specify a space-delimited list of hosts running in the DistCC cluster. +#DISTCC_HOSTS="" +# +#-- Specify a directory for package building. +#BUILDDIR=/tmp/makepkg + +######################################################################### +# GLOBAL PACKAGE OPTIONS +# These are default values for the options=() settings +######################################################################### +# +# Default: OPTIONS=(strip docs libtool emptydirs zipman purge !upx) +# A negated option will do the opposite of the comments below. +# +#-- strip: Strip symbols from binaries/libraries +#-- docs: Save doc directories specified by DOC_DIRS +#-- libtool: Leave libtool (.la) files in packages +#-- emptydirs: Leave empty directories in packages +#-- zipman: Compress manual (man and info) pages in MAN_DIRS with gzip +#-- purge: Remove files specified by PURGE_TARGETS +#-- upx: Compress binary executable files using UPX +# +OPTIONS=(strip docs libtool emptydirs zipman purge !upx) + +#-- File integrity checks to use. Valid: md5, sha1, sha256, sha384, sha512 +INTEGRITY_CHECK=(md5) +#-- Options to be used when stripping binaries. See `man strip' for details. +STRIP_BINARIES="--strip-all" +#-- Options to be used when stripping shared libraries. See `man strip' for details. +STRIP_SHARED="--strip-unneeded" +#-- Options to be used when stripping static libraries. See `man strip' for details. +STRIP_STATIC="--strip-debug" +#-- Manual (man and info) directories to compress (if zipman is specified) +MAN_DIRS=({usr{,/local}{,/share},opt/*}/{man,info}) +#-- Doc directories to remove (if !docs is specified) +DOC_DIRS=(usr/{,local/}{,share/}{doc,gtk-doc} opt/*/{doc,gtk-doc}) +#-- Files to be removed from all packages (if purge is specified) +PURGE_TARGETS=(usr/{,share}/info/dir .packlist *.pod) + +######################################################################### +# PACKAGE OUTPUT +######################################################################### +# +# Default: put built package and cached source in build directory +# +#-- Destination: specify a fixed directory where all packages will be placed +#PKGDEST=/home/packages +#-- Source cache: specify a fixed directory where source files will be cached +#SRCDEST=/home/sources +#-- Source packages: specify a fixed directory where all src packages will be placed +#SRCPKGDEST=/home/srcpackages +#-- Packager: name/email of the person or organization building packages +#PACKAGER="John Doe " +#-- Specify a key to use for package signing +#GPGKEY="" + +######################################################################### +# EXTENSION DEFAULTS +######################################################################### +# +# WARNING: Do NOT modify these variables unless you know what you are +# doing. +# +PKGEXT='.pkg.tar.xz' +SRCEXT='.src.tar.gz' + +# vim: set ft=sh ts=2 sw=2 et: diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index d494905..aee3e02 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -88,7 +88,7 @@ for repo in ${PKGREPOS[@]}; do # Build the actual source package pushd "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" >/dev/null - makepkg --nocolor --allsource --ignorearch --skippgpcheck >"${WORKDIR}/${pkgbase}.log" 2>&1 + makepkg --nocolor --allsource --ignorearch --skippgpcheck --config "${dirname}/makepkg.conf" >"${WORKDIR}/${pkgbase}.log" 2>&1 if [ $? -eq 0 ] && [ -f "${pkgbase}-${pkgver}${SRCEXT}" ]; then mv "${pkgbase}-${pkgver}${SRCEXT}" "${FTP_BASE}/${SRCPOOL}" # Avoid creating the same source package for every arch -- cgit v1.2.2 From 75deac81c2ca88219a4bc31d4a9a3da4f8274863 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 10 Jun 2012 12:28:38 +0200 Subject: Make repositories for testing2x configurable --- config | 2 ++ config.local.gerolde | 2 ++ config.local.sigurd | 2 ++ test/lib/common.inc | 2 ++ testing2x | 16 ++++++++-------- 5 files changed, 16 insertions(+), 8 deletions(-) diff --git a/config b/config index 4abddd5..44bfccc 100644 --- a/config +++ b/config @@ -3,6 +3,8 @@ SVNREPO='' PKGREPOS=() PKGPOOL='' SRCPOOL='' +TESTING_REPO='' +STABLE_REPOS=() CLEANUP_DESTDIR="/srv/package-cleanup" CLEANUP_DRYRUN=false diff --git a/config.local.gerolde b/config.local.gerolde index 4501a93..5e16b76 100644 --- a/config.local.gerolde +++ b/config.local.gerolde @@ -2,3 +2,5 @@ PKGREPOS=('core' 'extra' 'testing' 'staging' 'kde-unstable' 'gnome-unstable') PKGPOOL='pool/packages' SRCPOOL='sources/packages' SVNREPO='file:///srv/svn-packages' +TESTING_REPO='testing' +STABLE_REPOS=('core' 'extra') diff --git a/config.local.sigurd b/config.local.sigurd index 468ec64..4f95bd2 100644 --- a/config.local.sigurd +++ b/config.local.sigurd @@ -2,3 +2,5 @@ PKGREPOS=('community' 'community-testing' 'community-staging' 'multilib' 'multil PKGPOOL='pool/community' SRCPOOL='sources/community' SVNREPO='file:///srv/svn-packages' +TESTING_REPO='community-testing' +STABLE_REPOS=('community') diff --git a/test/lib/common.inc b/test/lib/common.inc index 827c207..35cdaee 100644 --- a/test/lib/common.inc +++ b/test/lib/common.inc @@ -106,6 +106,8 @@ setUp() { SVNREPO="file://${TMP}/svn-packages-repo" PKGREPOS=(${PKGREPOS[@]}) PKGPOOL="${PKGPOOL}" + TESTING_REPO='testing' + STABLE_REPOS=('core' 'extra') CLEANUP_DESTDIR="${TMP}/package-cleanup" SOURCE_CLEANUP_DESTDIR="${TMP}/source-cleanup" STAGING="${TMP}/staging" diff --git a/testing2x b/testing2x index f209e83..a05cb53 100755 --- a/testing2x +++ b/testing2x @@ -10,7 +10,7 @@ fi # Lock everything to reduce possibility of interfering task between the different repo-updates script_lock -for repo in 'core' 'extra' 'testing'; do +for repo in ${TESTING_REPO} ${STABLE_REPOS[@]}; do for pkgarch in ${ARCHES[@]}; do repo_lock ${repo} ${pkgarch} || exit 1 done @@ -24,16 +24,16 @@ for pkgbase in $*; do found_source=false for pkgarch in ${ARCHES[@]} 'any'; do - svnrepo_from="${WORKDIR}/${pkgbase}/testing-${pkgarch}" + svnrepo_from="${WORKDIR}/${pkgbase}/${TESTING_REPO}-${pkgarch}" if [ -r "${svnrepo_from}/PKGBUILD" ]; then found_source=true break fi done - ${found_source} || die "${pkgbase} not found in [testing]" + ${found_source} || die "${pkgbase} not found in [${TESTING_REPO}]" found_target=false for pkgarch in ${ARCHES[@]} 'any'; do - for repo in 'core' 'extra'; do + for repo in ${STABLE_REPOS[@]}; do svnrepo_to="${WORKDIR}/${pkgbase}/${repo}-${pkgarch}" if [ -r "${svnrepo_to}/PKGBUILD" ]; then found_target=true @@ -42,19 +42,19 @@ for pkgbase in $*; do fi done done - ${found_target} || die "${pkgbase} neither found in [core] nor [extra]" + ${found_target} || die "${pkgbase} not found in any of these repos: ${STABLE_REPOS[@]}" fi done for pkgarch in ${ARCHES[@]}; do - repo_unlock 'testing' ${pkgarch} + repo_unlock ${TESTING_REPO} ${pkgarch} done -for repo in 'core' 'extra'; do +for repo in ${STABLE_REPOS[@]}; do for pkgarch in ${ARCHES[@]}; do repo_unlock ${repo} ${pkgarch} done if [ -n "${pkgs[${repo}]}" ]; then - "$(dirname $0)/db-move" 'testing' "${repo}" ${pkgs[${repo}]} + "$(dirname $0)/db-move" ${TESTING_REPO} "${repo}" ${pkgs[${repo}]} fi done -- cgit v1.2.2 From 8bd650da942c2a112de8f3d33ba727ae7be7ffd3 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Mon, 11 Jun 2012 11:21:42 +0200 Subject: Reduce output when downloading sources --- cron-jobs/makepkg.conf | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cron-jobs/makepkg.conf b/cron-jobs/makepkg.conf index ff460ec..603edba 100644 --- a/cron-jobs/makepkg.conf +++ b/cron-jobs/makepkg.conf @@ -8,11 +8,11 @@ # #-- The download utilities that makepkg should use to acquire sources # Format: 'protocol::agent' -DLAGENTS=('ftp::/usr/bin/curl -fC - --ftp-pasv --retry 3 --retry-delay 3 -o %o %u' - 'http::/usr/bin/curl -fLC - --retry 3 --retry-delay 3 -o %o %u' - 'https::/usr/bin/curl -fLC - --retry 3 --retry-delay 3 -o %o %u' - 'rsync::/usr/bin/rsync -z %u %o' - 'scp::/usr/bin/scp -C %u %o') +DLAGENTS=('ftp::/usr/bin/curl -sS -fC - --ftp-pasv --retry 3 --retry-delay 3 -o %o %u' + 'http::/usr/bin/curl -sS -fLC - --retry 3 --retry-delay 3 -o %o %u' + 'https::/usr/bin/curl -sS -fLC - --retry 3 --retry-delay 3 -o %o %u' + 'rsync::/usr/bin/rsync -q --no-motd -z %u %o' + 'scp::/usr/bin/scp -q -C %u %o') # Other common tools: # /usr/bin/snarf -- cgit v1.2.2 From 7fdc74786a59c69d41524d5f311f856cfa9a99c6 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sat, 16 Jun 2012 19:48:25 +0200 Subject: Fix handling of package sources containg files with an @ character --- db-move | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/db-move b/db-move index 8ea7431..3ff1843 100755 --- a/db-move +++ b/db-move @@ -80,7 +80,7 @@ for pkgbase in ${args[@]:2}; do if [ -d "${svnrepo_to}" ]; then for file in $(/usr/bin/svn ls "${svnrepo_to}"); do - /usr/bin/svn rm -q "${svnrepo_to}/$file" + /usr/bin/svn rm -q "${svnrepo_to}/$file@" done else mkdir "${svnrepo_to}" @@ -88,7 +88,7 @@ for pkgbase in ${args[@]:2}; do fi for file in $(svn ls "${svnrepo_from}"); do - /usr/bin/svn mv -q -r HEAD "${svnrepo_from}/$file" "${svnrepo_to}/" + /usr/bin/svn mv -q -r HEAD "${svnrepo_from}/$file@" "${svnrepo_to}/" done /usr/bin/svn rm --force -q "${svnrepo_from}" tag_list="$tag_list, $pkgarch" -- cgit v1.2.2 From ce6e51f159fb1b0d292ac0f3d5edfaa0f0f6be2e Mon Sep 17 00:00:00 2001 From: Florian Pritz Date: Mon, 23 Jul 2012 20:42:00 +0200 Subject: db-move: Remove double space in tag_list This fixes this message which shouldn't have one space after the opening brace and two space before x86_64. "db-move: moved cups-filters from [testing] to [extra] ( i686, x86_64)" Signed-off-by: Florian Pritz Signed-off-by: Pierre Schmitz --- db-move | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db-move b/db-move index 3ff1843..1a89ded 100755 --- a/db-move +++ b/db-move @@ -91,7 +91,7 @@ for pkgbase in ${args[@]:2}; do /usr/bin/svn mv -q -r HEAD "${svnrepo_from}/$file@" "${svnrepo_to}/" done /usr/bin/svn rm --force -q "${svnrepo_from}" - tag_list="$tag_list, $pkgarch" + tag_list="$tag_list, $pkgarch" for pkgname in ${pkgnames[@]}; do for tarch in ${tarches[@]}; do -- cgit v1.2.2 From 332346d06375c1272bee6dce2c91d31a027fe212 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 5 Aug 2012 12:19:28 +0200 Subject: grep has been moved to /usr --- db-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db-functions b/db-functions index de5e8b8..e0d693b 100644 --- a/db-functions +++ b/db-functions @@ -211,7 +211,7 @@ repo_unlock () { #repo_unlock _grep_pkginfo() { local _ret - _ret="$(/usr/bin/bsdtar -xOqf "$1" .PKGINFO | /bin/grep -m 1 "^${2} = ")" + _ret="$(/usr/bin/bsdtar -xOqf "$1" .PKGINFO | grep -m 1 "^${2} = ")" echo "${_ret#${2} = }" } -- cgit v1.2.2 From b0047186ad33c5ef8bb6560dd9ba4d8a724dabce Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Wed, 3 Oct 2012 14:49:02 +0200 Subject: test: Do not suppress devtools output --- test/test.d/db-update.sh | 4 ++-- test/test.d/testing2x.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/test.d/db-update.sh b/test/test.d/db-update.sh index 2021993..67bca9f 100755 --- a/test/test.d/db-update.sh +++ b/test/test.d/db-update.sh @@ -82,7 +82,7 @@ testUpdateAnyPackage() { pushd "${TMP}/svn-packages-copy/pkg-any-a/trunk/" >/dev/null sed 's/pkgrel=1/pkgrel=2/g' -i PKGBUILD svn commit -q -m"update pkg to pkgrel=2" >/dev/null - sudo extra-i686-build >/dev/null 2>&1 + sudo extra-i686-build mv pkg-any-a-1-2-any.pkg.tar.xz "${pkgdir}/pkg-any-a/" popd >/dev/null @@ -100,7 +100,7 @@ testUpdateAnyPackageToDifferentRepositoriesAtOnce() { pushd "${TMP}/svn-packages-copy/pkg-any-a/trunk/" >/dev/null sed 's/pkgrel=1/pkgrel=2/g' -i PKGBUILD svn commit -q -m"update pkg to pkgrel=2" >/dev/null - sudo extra-i686-build >/dev/null 2>&1 + sudo extra-i686-build mv pkg-any-a-1-2-any.pkg.tar.xz "${pkgdir}/pkg-any-a/" popd >/dev/null diff --git a/test/test.d/testing2x.sh b/test/test.d/testing2x.sh index eda6cd6..357f3c2 100755 --- a/test/test.d/testing2x.sh +++ b/test/test.d/testing2x.sh @@ -10,7 +10,7 @@ testTesting2xAnyPackage() { pushd "${TMP}/svn-packages-copy/pkg-any-a/trunk/" >/dev/null sed 's/pkgrel=1/pkgrel=2/g' -i PKGBUILD svn commit -q -m"update pkg to pkgrel=2" >/dev/null - sudo extra-i686-build >/dev/null 2>&1 + sudo extra-i686-build mv pkg-any-a-1-2-any.pkg.tar.xz "${pkgdir}/pkg-any-a/" popd >/dev/null -- cgit v1.2.2 From a539598c640caa92e2b1967949836d73c5264036 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Wed, 3 Oct 2012 14:50:03 +0200 Subject: Use pacman-key to check signatures as it now uses a correct exit value --- db-functions | 13 ------------- db-update | 2 +- 2 files changed, 1 insertion(+), 14 deletions(-) diff --git a/db-functions b/db-functions index e0d693b..e3485d3 100644 --- a/db-functions +++ b/db-functions @@ -455,19 +455,6 @@ check_repo_permission() { return 0 } -check_pkgsig() { - local signature=$1 - local ret=1 - local fd="$(mktemp --tmpdir="${WORKDIR}")" - - gpg --homedir /etc/pacman.d/gnupg/ --status-file "${fd}" --verify "${signature}" >/dev/null 2>&1 - if grep -q TRUST_FULLY "${fd}"; then - ret=0 - fi - - return $ret -} - set_repo_permission() { local repo=$1 local arch=$2 diff --git a/db-update b/db-update index 8be0d54..caddbe6 100755 --- a/db-update +++ b/db-update @@ -35,7 +35,7 @@ for repo in ${repos[@]}; do if ! check_pkgfile "${pkg}"; then die "Package ${repo}/${pkg##*/} is not consistent with its meta data" fi - if ${REQUIRE_SIGNATURE} && ! check_pkgsig "${pkg}.sig"; then + if ${REQUIRE_SIGNATURE} && ! pacman-key -v "${pkg}.sig" >/dev/null 2>&1; then die "Package ${repo}/${pkg##*/} does not have a valid signature" fi if ! check_pkgsvn "${pkg}" "${repo}"; then -- cgit v1.2.2 From b2952784db9995ecc21d3a770530a19394234d3b Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sat, 27 Oct 2012 14:32:39 +0200 Subject: Add the possibility to run all svn commands with a different user * A SVNUSER can be configured in the config file * This user needs to be able to call svn without a password --- config | 1 + config.local.gerolde | 1 + config.local.sigurd | 1 + cron-jobs/sourceballs | 2 +- db-functions | 12 ++++++++++-- db-move | 18 +++++++++--------- db-remove | 6 +++--- test/lib/common.inc | 14 +++++++------- test/test.d/db-update.sh | 4 ++-- test/test.d/testing2x.sh | 2 +- testing2x | 2 +- 11 files changed, 37 insertions(+), 26 deletions(-) diff --git a/config b/config index 44bfccc..9e3041c 100644 --- a/config +++ b/config @@ -1,5 +1,6 @@ FTP_BASE="/srv/ftp" SVNREPO='' +SVNUSER='' PKGREPOS=() PKGPOOL='' SRCPOOL='' diff --git a/config.local.gerolde b/config.local.gerolde index 5e16b76..f19eeec 100644 --- a/config.local.gerolde +++ b/config.local.gerolde @@ -2,5 +2,6 @@ PKGREPOS=('core' 'extra' 'testing' 'staging' 'kde-unstable' 'gnome-unstable') PKGPOOL='pool/packages' SRCPOOL='sources/packages' SVNREPO='file:///srv/svn-packages' +SVNUSER='' TESTING_REPO='testing' STABLE_REPOS=('core' 'extra') diff --git a/config.local.sigurd b/config.local.sigurd index 4f95bd2..afd5cba 100644 --- a/config.local.sigurd +++ b/config.local.sigurd @@ -2,5 +2,6 @@ PKGREPOS=('community' 'community-testing' 'community-staging' 'multilib' 'multil PKGPOOL='pool/community' SRCPOOL='sources/community' SVNREPO='file:///srv/svn-packages' +SVNUSER='' TESTING_REPO='community-testing' STABLE_REPOS=('community') diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index aee3e02..5d2b086 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -79,7 +79,7 @@ for repo in ${PKGREPOS[@]}; do # Get the sources from svn mkdir -p "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}" - svn export -q "${SVNREPO}/${pkgbase}/repos/${repo}-${pkgarch}" \ + arch_svn export -q "${SVNREPO}/${pkgbase}/repos/${repo}-${pkgarch}" \ "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" >/dev/null 2>&1 if [ $? -ge 1 ]; then failedpkgs[${#failedpkgs[*]}]="${pkgbase}-${pkgver}${SRCEXT}" diff --git a/db-functions b/db-functions index e3485d3..b9d0d2f 100644 --- a/db-functions +++ b/db-functions @@ -347,7 +347,7 @@ check_pkgsvn() { if [ ! -f "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}" ]; then mkdir -p "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}" - svn export -q "${SVNREPO}/${_pkgbase}/repos/${repo}-${_pkgarch}/PKGBUILD" \ + arch_svn export -q "${SVNREPO}/${_pkgbase}/repos/${repo}-${_pkgarch}/PKGBUILD" \ "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}" >/dev/null [ $? -ge 1 ] && return 1 fi @@ -382,7 +382,7 @@ check_splitpkgs() { if [ ! -f "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}" ]; then mkdir -p "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}" - svn export -q "${SVNREPO}/${_pkgbase}/repos/${repo}-${_pkgarch}/PKGBUILD" \ + arch_svn export -q "${SVNREPO}/${_pkgbase}/repos/${repo}-${_pkgarch}/PKGBUILD" \ "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}" >/dev/null [ $? -ge 1 ] && return 1 fi @@ -504,3 +504,11 @@ arch_repo_remove() { || error "repo-remove ${filesfile} ${pkgs[@]}" set_repo_permission "${repo}" "${arch}" } + +arch_svn() { + if [ -z "${SVNUSER}" ]; then + /usr/bin/svn "${@}" + else + sudo -u "${SVNUSER}" -- /usr/bin/svn "${@}" + fi +} diff --git a/db-move b/db-move index 1a89ded..1fa44d4 100755 --- a/db-move +++ b/db-move @@ -25,9 +25,9 @@ for pkgarch in ${ARCHES[@]}; do done # check if packages to be moved exist in svn and ftp dir -/usr/bin/svn checkout -q -N "${SVNREPO}" "${WORKDIR}/svn" >/dev/null +arch_svn checkout -q -N "${SVNREPO}" "${WORKDIR}/svn" >/dev/null for pkgbase in ${args[@]:2}; do - /usr/bin/svn up -q "${WORKDIR}/svn/${pkgbase}" >/dev/null + arch_svn up -q "${WORKDIR}/svn/${pkgbase}" >/dev/null for pkgarch in ${ARCHES[@]} 'any'; do svnrepo_from="${WORKDIR}/svn/${pkgbase}/repos/${repo_from}-${pkgarch}" if [ -r "${svnrepo_from}/PKGBUILD" ]; then @@ -79,18 +79,18 @@ for pkgbase in ${args[@]:2}; do pkgver=$(. "${svnrepo_from}/PKGBUILD"; echo $(get_full_version ${epoch:-0} ${pkgver} ${pkgrel})) if [ -d "${svnrepo_to}" ]; then - for file in $(/usr/bin/svn ls "${svnrepo_to}"); do - /usr/bin/svn rm -q "${svnrepo_to}/$file@" + for file in $(arch_svn ls "${svnrepo_to}"); do + arch_svn rm -q "${svnrepo_to}/$file@" done else mkdir "${svnrepo_to}" - /usr/bin/svn add -q "${svnrepo_to}" + arch_svn add -q "${svnrepo_to}" fi - for file in $(svn ls "${svnrepo_from}"); do - /usr/bin/svn mv -q -r HEAD "${svnrepo_from}/$file@" "${svnrepo_to}/" + for file in $(arch_svn ls "${svnrepo_from}"); do + arch_svn mv -q -r HEAD "${svnrepo_from}/$file@" "${svnrepo_to}/" done - /usr/bin/svn rm --force -q "${svnrepo_from}" + arch_svn rm --force -q "${svnrepo_from}" tag_list="$tag_list, $pkgarch" for pkgname in ${pkgnames[@]}; do @@ -109,7 +109,7 @@ for pkgbase in ${args[@]:2}; do fi done tag_list="${tag_list#, }" - /usr/bin/svn commit -q "${WORKDIR}/svn/${pkgbase}" -m "${0##*/}: moved ${pkgbase} from [${repo_from}] to [${repo_to}] (${tag_list})" + arch_svn commit -q "${WORKDIR}/svn/${pkgbase}" -m "${0##*/}: moved ${pkgbase} from [${repo_from}] to [${repo_to}] (${tag_list})" done for tarch in ${ARCHES[@]}; do diff --git a/db-remove b/db-remove index da32c78..25cb9a7 100755 --- a/db-remove +++ b/db-remove @@ -32,12 +32,12 @@ done remove_pkgs=() for pkgbase in ${pkgbases[@]}; do msg "Removing $pkgbase from [$repo]..." - /usr/bin/svn checkout -q "${SVNREPO}/${pkgbase}" "${WORKDIR}/svn/${pkgbase}" >/dev/null + arch_svn checkout -q "${SVNREPO}/${pkgbase}" "${WORKDIR}/svn/${pkgbase}" >/dev/null if [ -d "${WORKDIR}/svn/$pkgbase/repos/$svnrepo" ]; then remove_pkgs=(${remove_pkgs[@]} $(. "${WORKDIR}/svn/$pkgbase/repos/$svnrepo/PKGBUILD"; echo ${pkgname[@]})) - /usr/bin/svn rm --force -q "${WORKDIR}/svn/$pkgbase/repos/$svnrepo" - /usr/bin/svn commit -q "${WORKDIR}/svn/$pkgbase" -m "${0##*/}: $pkgbase removed by $(id -un)" + arch_svn rm --force -q "${WORKDIR}/svn/$pkgbase/repos/$svnrepo" + arch_svn commit -q "${WORKDIR}/svn/$pkgbase" -m "${0##*/}: $pkgbase removed by $(id -un)" else warning "$pkgbase not found in $svnrepo" warning "Removing only $pkgbase from the repo" diff --git a/test/lib/common.inc b/test/lib/common.inc index 35cdaee..aeceece 100644 --- a/test/lib/common.inc +++ b/test/lib/common.inc @@ -91,14 +91,14 @@ setUp() { msg 'Creating svn repository...' svnadmin create "${TMP}/svn-packages-repo" - svn checkout -q "file://${TMP}/svn-packages-repo" "${TMP}/svn-packages-copy" + arch_svn checkout -q "file://${TMP}/svn-packages-repo" "${TMP}/svn-packages-copy" for p in "${pkgdir}"/*; do pkg=${p##*/} mkdir -p "${TMP}/svn-packages-copy/${pkg}"/{trunk,repos} cp "${p}"/* "${TMP}/svn-packages-copy"/${pkg}/trunk/ - svn add -q "${TMP}/svn-packages-copy"/${pkg} - svn commit -q -m"initial commit of ${pkg}" "${TMP}/svn-packages-copy" + arch_svn add -q "${TMP}/svn-packages-copy"/${pkg} + arch_svn commit -q -m"initial commit of ${pkg}" "${TMP}/svn-packages-copy" done cat < "$(dirname ${BASH_SOURCE[0]})/../../config.local" @@ -189,7 +189,7 @@ checkAnyPackage() { checkAnyPackageDB $repo $pkg local pkgbase=$(getpkgbase "${FTP_BASE}/${PKGPOOL}/${pkg}") - svn up -q "${TMP}/svn-packages-copy/${pkgbase}" + arch_svn up -q "${TMP}/svn-packages-copy/${pkgbase}" [ -d "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo}-any" ] \ || fail "svn-packages-copy/${pkgbase}/repos/${repo}-any does not exist" } @@ -231,7 +231,7 @@ checkPackage() { checkPackageDB $repo $pkg $arch local pkgbase=$(getpkgbase "${FTP_BASE}/${PKGPOOL}/${pkg}") - svn up -q "${TMP}/svn-packages-copy/${pkgbase}" + arch_svn up -q "${TMP}/svn-packages-copy/${pkgbase}" [ -d "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo}-${arch}" ] \ || fail "svn-packages-copy/${pkgbase}/repos/${repo}-${arch} does not exist" } @@ -256,7 +256,7 @@ checkRemovedPackage() { checkRemovedPackageDB $repo $pkgbase $arch - svn up -q "${TMP}/svn-packages-copy/${pkgbase}" + arch_svn up -q "${TMP}/svn-packages-copy/${pkgbase}" [ -d "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo}-${arch}" ] \ && fail "svn-packages-copy/${pkgbase}/repos/${repo}-${arch} should not exist" } @@ -282,7 +282,7 @@ checkRemovedAnyPackage() { checkRemovedAnyPackageDB $repo $pkgbase - svn up -q "${TMP}/svn-packages-copy/${pkgbase}" + arch_svn up -q "${TMP}/svn-packages-copy/${pkgbase}" [ -d "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo}-any" ] \ && fail "svn-packages-copy/${pkgbase}/repos/${repo}-any should not exist" } diff --git a/test/test.d/db-update.sh b/test/test.d/db-update.sh index 67bca9f..ea1e130 100755 --- a/test/test.d/db-update.sh +++ b/test/test.d/db-update.sh @@ -81,7 +81,7 @@ testUpdateAnyPackage() { pushd "${TMP}/svn-packages-copy/pkg-any-a/trunk/" >/dev/null sed 's/pkgrel=1/pkgrel=2/g' -i PKGBUILD - svn commit -q -m"update pkg to pkgrel=2" >/dev/null + arch_svn commit -q -m"update pkg to pkgrel=2" >/dev/null sudo extra-i686-build mv pkg-any-a-1-2-any.pkg.tar.xz "${pkgdir}/pkg-any-a/" popd >/dev/null @@ -99,7 +99,7 @@ testUpdateAnyPackageToDifferentRepositoriesAtOnce() { pushd "${TMP}/svn-packages-copy/pkg-any-a/trunk/" >/dev/null sed 's/pkgrel=1/pkgrel=2/g' -i PKGBUILD - svn commit -q -m"update pkg to pkgrel=2" >/dev/null + arch_svn commit -q -m"update pkg to pkgrel=2" >/dev/null sudo extra-i686-build mv pkg-any-a-1-2-any.pkg.tar.xz "${pkgdir}/pkg-any-a/" popd >/dev/null diff --git a/test/test.d/testing2x.sh b/test/test.d/testing2x.sh index 357f3c2..31d85b4 100755 --- a/test/test.d/testing2x.sh +++ b/test/test.d/testing2x.sh @@ -9,7 +9,7 @@ testTesting2xAnyPackage() { pushd "${TMP}/svn-packages-copy/pkg-any-a/trunk/" >/dev/null sed 's/pkgrel=1/pkgrel=2/g' -i PKGBUILD - svn commit -q -m"update pkg to pkgrel=2" >/dev/null + arch_svn commit -q -m"update pkg to pkgrel=2" >/dev/null sudo extra-i686-build mv pkg-any-a-1-2-any.pkg.tar.xz "${pkgdir}/pkg-any-a/" popd >/dev/null diff --git a/testing2x b/testing2x index a05cb53..369857f 100755 --- a/testing2x +++ b/testing2x @@ -20,7 +20,7 @@ declare -A pkgs for pkgbase in $*; do if [ ! -d "${WORKDIR}/${pkgbase}" ]; then - /usr/bin/svn export -q "${SVNREPO}/${pkgbase}/repos" "${WORKDIR}/${pkgbase}" >/dev/null + arch_svn export -q "${SVNREPO}/${pkgbase}/repos" "${WORKDIR}/${pkgbase}" >/dev/null found_source=false for pkgarch in ${ARCHES[@]} 'any'; do -- cgit v1.2.2 From bef62b16c5e79bc81c8e37465477755690dc54cf Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sat, 3 Nov 2012 17:52:06 +0100 Subject: Adjust configuration for use on nymeria.archlinux.org --- config | 6 +++--- config.local.gerolde | 7 ------- config.local.sigurd | 7 ------- config.local.svn-community | 11 +++++++++++ config.local.svn-packages | 11 +++++++++++ 5 files changed, 25 insertions(+), 17 deletions(-) delete mode 100644 config.local.gerolde delete mode 100644 config.local.sigurd create mode 100644 config.local.svn-community create mode 100644 config.local.svn-packages diff --git a/config b/config index 9e3041c..3df6c95 100644 --- a/config +++ b/config @@ -7,12 +7,12 @@ SRCPOOL='' TESTING_REPO='' STABLE_REPOS=() -CLEANUP_DESTDIR="/srv/package-cleanup" +CLEANUP_DESTDIR="/var/tmp" CLEANUP_DRYRUN=false # Time in days to keep moved packages CLEANUP_KEEP=30 -SOURCE_CLEANUP_DESTDIR="/srv/source-cleanup" +SOURCE_CLEANUP_DESTDIR="/var/tmp" SOURCE_CLEANUP_DRYRUN=false # Time in days to keep moved sourcepackages SOURCE_CLEANUP_KEEP=14 @@ -23,7 +23,7 @@ LOCK_DELAY=10 LOCK_TIMEOUT=300 STAGING="$HOME/staging" -TMPDIR="/srv/tmp" +TMPDIR="/var/tmp" ARCHES=(i686 x86_64) DBEXT=".db.tar.gz" FILESEXT=".files.tar.gz" diff --git a/config.local.gerolde b/config.local.gerolde deleted file mode 100644 index f19eeec..0000000 --- a/config.local.gerolde +++ /dev/null @@ -1,7 +0,0 @@ -PKGREPOS=('core' 'extra' 'testing' 'staging' 'kde-unstable' 'gnome-unstable') -PKGPOOL='pool/packages' -SRCPOOL='sources/packages' -SVNREPO='file:///srv/svn-packages' -SVNUSER='' -TESTING_REPO='testing' -STABLE_REPOS=('core' 'extra') diff --git a/config.local.sigurd b/config.local.sigurd deleted file mode 100644 index afd5cba..0000000 --- a/config.local.sigurd +++ /dev/null @@ -1,7 +0,0 @@ -PKGREPOS=('community' 'community-testing' 'community-staging' 'multilib' 'multilib-testing' 'multilib-staging') -PKGPOOL='pool/community' -SRCPOOL='sources/community' -SVNREPO='file:///srv/svn-packages' -SVNUSER='' -TESTING_REPO='community-testing' -STABLE_REPOS=('community') diff --git a/config.local.svn-community b/config.local.svn-community new file mode 100644 index 0000000..105ea66 --- /dev/null +++ b/config.local.svn-community @@ -0,0 +1,11 @@ +PKGREPOS=('community' 'community-testing' 'community-staging' 'multilib' 'multilib-testing' 'multilib-staging') +PKGPOOL='pool/community' +SRCPOOL='sources/community' +SVNREPO='file:///srv/repos/svn-community/svn' +SVNUSER='svn-community' +TESTING_REPO='community-testing' +STABLE_REPOS=('community') + +CLEANUP_DESTDIR="/srv/repos/svn-community/package-cleanup" +SOURCE_CLEANUP_DESTDIR="/srv/repos/svn-community/source-cleanup" +TMPDIR="/srv/repos/svn-community/tmp" diff --git a/config.local.svn-packages b/config.local.svn-packages new file mode 100644 index 0000000..958a483 --- /dev/null +++ b/config.local.svn-packages @@ -0,0 +1,11 @@ +PKGREPOS=('core' 'extra' 'testing' 'staging' 'kde-unstable' 'gnome-unstable') +PKGPOOL='pool/packages' +SRCPOOL='sources/packages' +SVNREPO='file:///srv/repos/svn-packages/svn' +SVNUSER='svn-packages' +TESTING_REPO='testing' +STABLE_REPOS=('core' 'extra') + +CLEANUP_DESTDIR="/srv/repos/svn-packages/package-cleanup" +SOURCE_CLEANUP_DESTDIR="/srv/repos/svn-packages/source-cleanup" +TMPDIR="/srv/repos/svn-packages/tmp" -- cgit v1.2.2 From 868adfa4049fcb136d8b7aaaa347b9ffd918c7b9 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 4 Nov 2012 19:04:38 +0100 Subject: Grant the SVNUSER access to the working directory --- db-functions | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/db-functions b/db-functions index b9d0d2f..b3a4293 100644 --- a/db-functions +++ b/db-functions @@ -17,6 +17,11 @@ restore_umask () { # set up general environment WORKDIR=$(mktemp -d "${TMPDIR}/${0##*/}.XXXXXXXXXX") +if [ -n "${SVNUSER}" ]; then + setfacl -m u:"${SVNUSER}":rwx "${WORKDIR}" + setfacl -m d:u:"${USER}":rwx "${WORKDIR}" + setfacl -m d:u:"${SVNUSER}":rwx "${WORKDIR}" +fi LOCKS=() # check if messages are to be printed using color @@ -509,6 +514,6 @@ arch_svn() { if [ -z "${SVNUSER}" ]; then /usr/bin/svn "${@}" else - sudo -u "${SVNUSER}" -- /usr/bin/svn "${@}" + sudo -u "${SVNUSER}" -- /usr/bin/svn --username "${USER}" "${@}" fi } -- cgit v1.2.2 From 1609e48d3301dfc2c897ed22d413eb74260b6f4b Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 25 Nov 2012 11:52:33 +0100 Subject: Fix sourceballs permissions --- cron-jobs/sourceballs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 5d2b086..6f75ccc 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -78,7 +78,7 @@ for repo in ${PKGREPOS[@]}; do fi # Get the sources from svn - mkdir -p "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}" + mkdir -p -m0770 "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}" arch_svn export -q "${SVNREPO}/${pkgbase}/repos/${repo}-${pkgarch}" \ "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" >/dev/null 2>&1 if [ $? -ge 1 ]; then -- cgit v1.2.2 From c2ad0298b74877bb2cd85f8f0e899460714bb019 Mon Sep 17 00:00:00 2001 From: Florian Pritz Date: Mon, 7 Jan 2013 12:08:30 +0100 Subject: ftpdir-cleanup: fix ACL issues on nymeria Signed-off-by: Florian Pritz Signed-off-by: Pierre Schmitz --- cron-jobs/ftpdir-cleanup | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index 15eb46c..e1294bd 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -3,6 +3,15 @@ . "$(dirname $0)/../config" . "$(dirname $0)/../db-functions" +# just like mv -f, but we touch the file and then copy the content so +# default ACLs in the target dir will be applied +mv_acl() { + rm -f "$2" + touch "$2" + cat "$1" >"$2" || return 1 + rm -f "$1" +} + clean_pkg() { local pkg local target @@ -12,9 +21,9 @@ clean_pkg() { if [ -h "$pkg" ]; then rm -f "$pkg" "$pkg.sig" else - mv -f "$pkg" "$CLEANUP_DESTDIR" + mv_acl "$pkg" "$CLEANUP_DESTDIR/${pkg##*/}" if [ -e "$pkg.sig" ]; then - mv -f "$pkg.sig" "$CLEANUP_DESTDIR" + mv_acl "$pkg.sig" "$CLEANUP_DESTDIR/${pkg##*/}.sig" fi touch "${CLEANUP_DESTDIR}/${pkg##*/}" fi -- cgit v1.2.2 From 0992cc36f525898ff07032d0ee697ecb62fedc07 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Thu, 31 Jan 2013 23:24:07 +0100 Subject: db-update: Ignore forgeign staging packages --- db-update | 9 ++++++++- test/test.d/db-update.sh | 10 ++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/db-update b/db-update index caddbe6..576fe2b 100755 --- a/db-update +++ b/db-update @@ -9,11 +9,18 @@ if [ $# -ge 1 ]; then fi # Find repos with packages to release -repos=($(find "${STAGING}" -mindepth 1 -type d ! -empty -printf '%f ' 2>/dev/null)) +staging_repos=($(find "${STAGING}" -mindepth 1 -type f -name "*${PKGEXT}" -printf '%h\n' | sort -u)) if [ $? -ge 1 ]; then die "Could not read ${STAGING}" fi +repos=() +for staging_repo in ${staging_repos[@]##*/}; do + if in_array ${staging_repo} ${PKGREPOS[@]}; then + repos+=(${staging_repo}) + fi +done + # TODO: this might lock too much (architectures) for repo in ${repos[@]}; do for pkgarch in ${ARCHES[@]}; do diff --git a/test/test.d/db-update.sh b/test/test.d/db-update.sh index ea1e130..7a5ce4c 100755 --- a/test/test.d/db-update.sh +++ b/test/test.d/db-update.sh @@ -162,4 +162,14 @@ testAddIncompleteSplitPackage() { done } +testUnknownRepo() { + mkdir "${STAGING}/unknown/" + releasePackage extra 'pkg-simple-a' 'i686' + releasePackage unknown 'pkg-simple-b' 'i686' + ../db-update + checkPackage extra 'pkg-simple-a-1-1-i686.pkg.tar.xz' 'i686' + [ -e "${FTP_BASE}/unknown" ] && fail "db-update pushed a package into an unknown repository" + rm -rf "${STAGING}/unknown/" +} + . "${curdir}/../lib/shunit2" -- cgit v1.2.2 From fbce7db101feaba0d69bf185b54270c1aa6d65ab Mon Sep 17 00:00:00 2001 From: "Jan Alexander Steffens (heftig)" Date: Tue, 19 Mar 2013 19:53:38 +0100 Subject: sourceballs: fix ACL issues on nymeria ACLs from the WORKDIR were used, leading to problems as these weren't world-readable. Fix follows ftpdir-cleanup. Signed-off-by: Jan Alexander Steffens (heftig) Signed-off-by: Pierre Schmitz --- cron-jobs/ftpdir-cleanup | 9 --------- cron-jobs/sourceballs | 4 ++-- db-functions | 9 +++++++++ 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index e1294bd..8f5cb3c 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -3,15 +3,6 @@ . "$(dirname $0)/../config" . "$(dirname $0)/../db-functions" -# just like mv -f, but we touch the file and then copy the content so -# default ACLs in the target dir will be applied -mv_acl() { - rm -f "$2" - touch "$2" - cat "$1" >"$2" || return 1 - rm -f "$1" -} - clean_pkg() { local pkg local target diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 6f75ccc..be66e99 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -90,7 +90,7 @@ for repo in ${PKGREPOS[@]}; do pushd "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" >/dev/null makepkg --nocolor --allsource --ignorearch --skippgpcheck --config "${dirname}/makepkg.conf" >"${WORKDIR}/${pkgbase}.log" 2>&1 if [ $? -eq 0 ] && [ -f "${pkgbase}-${pkgver}${SRCEXT}" ]; then - mv "${pkgbase}-${pkgver}${SRCEXT}" "${FTP_BASE}/${SRCPOOL}" + mv_acl "${pkgbase}-${pkgver}${SRCEXT}" "${FTP_BASE}/${SRCPOOL}/${pkgbase}-${pkgver}${SRCEXT}" # Avoid creating the same source package for every arch echo "${pkgbase}-${pkgver}${SRCEXT}" >> "${WORKDIR}/available-src-pkgs" newpkgs[${#newpkgs[*]}]="${pkgbase}-${pkgver}${SRCEXT}" @@ -127,7 +127,7 @@ if [ ${#old_pkgs[@]} -ge 1 ]; then for old_pkg in ${old_pkgs[@]}; do msg2 "${old_pkg}" if ! ${SOURCE_CLEANUP_DRYRUN}; then - mv "$FTP_BASE/${SRCPOOL}/${old_pkg}" "${SOURCE_CLEANUP_DESTDIR}" + mv_acl "$FTP_BASE/${SRCPOOL}/${old_pkg}" "${SOURCE_CLEANUP_DESTDIR}/${old_pkg}" touch "${SOURCE_CLEANUP_DESTDIR}/${old_pkg}" fi done diff --git a/db-functions b/db-functions index b3a4293..4a9a42a 100644 --- a/db-functions +++ b/db-functions @@ -15,6 +15,15 @@ restore_umask () { umask $UMASK >/dev/null } +# just like mv -f, but we touch the file and then copy the content so +# default ACLs in the target dir will be applied +mv_acl() { + rm -f "$2" + touch "$2" + cat "$1" >"$2" || return 1 + rm -f "$1" +} + # set up general environment WORKDIR=$(mktemp -d "${TMPDIR}/${0##*/}.XXXXXXXXXX") if [ -n "${SVNUSER}" ]; then -- cgit v1.2.2 From 0b43e8cdee1ee46ea79e4d089136e76e767b4d5b Mon Sep 17 00:00:00 2001 From: Florian Pritz Date: Wed, 23 Jan 2013 23:03:35 +0100 Subject: Add lastupdate file This allows for faster checking if an update might be needed and helps to let reporead run when something changed. Signed-off-by: Florian Pritz --- db-functions | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/db-functions b/db-functions index 4a9a42a..bb49894 100644 --- a/db-functions +++ b/db-functions @@ -32,6 +32,7 @@ if [ -n "${SVNUSER}" ]; then setfacl -m d:u:"${SVNUSER}":rwx "${WORKDIR}" fi LOCKS=() +REPO_MODIFIED=0 # check if messages are to be printed using color unset ALL_OFF BOLD BLUE GREEN RED YELLOW @@ -142,6 +143,11 @@ cleanup() { script_unlock fi rm -rf "$WORKDIR" + + if (( REPO_MODIFIED )); then + date +%s > "${FTP_BASE}/lastupdate" + fi + [ "$1" ] && exit $1 } @@ -499,6 +505,8 @@ arch_repo_add() { || error "repo-add -f ${repo}${FILESEXT} ${pkgs[@]}" popd >/dev/null set_repo_permission "${repo}" "${arch}" + + REPO_MODIFIED=1 } arch_repo_remove() { @@ -517,6 +525,8 @@ arch_repo_remove() { /usr/bin/repo-remove -q "${filesfile}" ${pkgs[@]} \ || error "repo-remove ${filesfile} ${pkgs[@]}" set_repo_permission "${repo}" "${arch}" + + REPO_MODIFIED=1 } arch_svn() { -- cgit v1.2.2 From 278d07fb2b72951e1f03154df55bb833cc38d29b Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sat, 1 Nov 2014 12:11:37 +0100 Subject: cron-jobs: Fix issues with empty repositories --- cron-jobs/ftpdir-cleanup | 2 +- cron-jobs/sourceballs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index 8f5cb3c..2683676 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -64,7 +64,7 @@ done # get a list of all available packages in the pacakge pool find "$FTP_BASE/${PKGPOOL}" -name "*${PKGEXT}" -printf '%f\n' | sort > "${WORKDIR}/pool" # create a list of packages in our db -cat "${WORKDIR}/db-"* | sort -u > "${WORKDIR}/db" +find "${WORKDIR}" -maxdepth 1 -type f -name 'db-*' -exec cat {} \; | sort -u > "${WORKDIR}/db" old_pkgs=($(comm -23 "${WORKDIR}/pool" "${WORKDIR}/db")) if [ ${#old_pkgs[@]} -ge 1 ]; then diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index be66e99..4126c37 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -117,8 +117,8 @@ for repo in ${PKGREPOS[@]}; do done # Cleanup old source packages -cat "${WORKDIR}/expected-src-pkgs" | sort -u > "${WORKDIR}/expected-src-pkgs.sort" -cat "${WORKDIR}/available-src-pkgs" | sort -u > "${WORKDIR}/available-src-pkgs.sort" +find "${WORKDIR}" -maxdepth 1 -type f -name 'expected-src-pkgs' -exec cat {} \; | sort -u > "${WORKDIR}/expected-src-pkgs.sort" +find "${WORKDIR}" -maxdepth 1 -type f -name 'available-src-pkgs' -exec cat {} \; | sort -u > "${WORKDIR}/available-src-pkgs.sort" old_pkgs=($(comm -23 "${WORKDIR}/available-src-pkgs.sort" "${WORKDIR}/expected-src-pkgs.sort")) if [ ${#old_pkgs[@]} -ge 1 ]; then -- cgit v1.2.2 From 4fcedea4ae5fb8788c7ef0ddd907ed8b02d14a5b Mon Sep 17 00:00:00 2001 From: Dave Reisner Date: Tue, 9 Sep 2014 19:04:25 -0400 Subject: disallow packages without a valid PACKAGER Shame. Signed-off-by: Pierre Schmitz --- db-functions | 7 +++++++ db-update | 3 +++ 2 files changed, 10 insertions(+) diff --git a/db-functions b/db-functions index bb49894..7e1b3aa 100644 --- a/db-functions +++ b/db-functions @@ -297,6 +297,13 @@ getpkgarch() { echo "$_ver" } +check_packager() { + local _packager + + _packager=$(_grep_pkginfo "$1" "packager") + [[ $_packager && $_packager != 'Unknown Packager' ]] +} + getpkgfile() { if [[ ${#} -ne 1 ]]; then error 'No canonical package found!' diff --git a/db-update b/db-update index 576fe2b..cdc35a5 100755 --- a/db-update +++ b/db-update @@ -51,6 +51,9 @@ for repo in ${repos[@]}; do if ! check_pkgrepos "${pkg}"; then die "Package ${repo}/${pkg##*/} already exists in another repository" fi + if ! check_packager "${pkg}"; then + die "Package ${repo}/${pkg##*/} does not have a valid packager" + fi done if ! check_splitpkgs ${repo} ${pkgs[@]}; then die "Missing split packages for ${repo}" -- cgit v1.2.2 From fbd17dc8edc59e3b5c67acafebfcdda73e409e00 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 7 Feb 2016 09:47:03 +0100 Subject: Remove separate creation of file databases With pacman 5.0 repo-add and repo-remove handle file databases by default. --- db-functions | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/db-functions b/db-functions index 7e1b3aa..667ac8d 100644 --- a/db-functions +++ b/db-functions @@ -169,7 +169,6 @@ trap cleanup EXIT repo_lock () { local LOCKDIR="$TMPDIR/.repolock.$1.$2" local DBLOCKFILE="${FTP_BASE}/${1}/os/${2}/${1}${DBEXT}.lck" - local FILESLOCKFILE="${FTP_BASE}/${1}/os/${2}/${1}${FILESEXT}.lck" local _count local _trial local _timeout @@ -181,10 +180,6 @@ repo_lock () { error "Repo [${1}] (${2}) is already locked by repo-{add,remove} process $(cat $DBLOCKFILE)" return 1 fi - if [ -f "${FILESLOCKFILE}" ]; then - error "Repo [${1}] (${2}) is already locked by repo-{add,remove} process $(cat ${FILESLOCKFILE})" - return 1 - fi if [ $# -eq 2 ]; then _lockblock=true @@ -508,8 +503,6 @@ arch_repo_add() { pushd "${FTP_BASE}/${repo}/os/${arch}" >/dev/null /usr/bin/repo-add -q "${repo}${DBEXT}" ${pkgs[@]} \ || error "repo-add ${repo}${DBEXT} ${pkgs[@]}" - /usr/bin/repo-add -f -q "${repo}${FILESEXT}" ${pkgs[@]} \ - || error "repo-add -f ${repo}${FILESEXT} ${pkgs[@]}" popd >/dev/null set_repo_permission "${repo}" "${arch}" @@ -521,7 +514,6 @@ arch_repo_remove() { local arch=$2 local pkgs=(${@:3}) local dbfile="${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" - local filesfile="${FTP_BASE}/${repo}/os/${arch}/${repo}${FILESEXT}" if [ ! -f "${dbfile}" ]; then error "No database found at '${dbfile}'" @@ -529,8 +521,6 @@ arch_repo_remove() { fi /usr/bin/repo-remove -q "${dbfile}" ${pkgs[@]} \ || error "repo-remove ${dbfile} ${pkgs[@]}" - /usr/bin/repo-remove -q "${filesfile}" ${pkgs[@]} \ - || error "repo-remove ${filesfile} ${pkgs[@]}" set_repo_permission "${repo}" "${arch}" REPO_MODIFIED=1 -- cgit v1.2.2 From 7f1c367f8d8504126caf3daa07a15d259fd072fa Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 7 Feb 2016 10:01:42 +0100 Subject: Update bundled makepkg.conf --- cron-jobs/makepkg.conf | 58 ++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 42 insertions(+), 16 deletions(-) diff --git a/cron-jobs/makepkg.conf b/cron-jobs/makepkg.conf index 603edba..d8512fb 100644 --- a/cron-jobs/makepkg.conf +++ b/cron-jobs/makepkg.conf @@ -19,6 +19,13 @@ DLAGENTS=('ftp::/usr/bin/curl -sS -fC - --ftp-pasv --retry 3 --retry-delay 3 -o # /usr/bin/lftpget -c # /usr/bin/wget +#-- The package required by makepkg to download VCS sources +# Format: 'protocol::package' +VCSCLIENTS=('bzr::bzr' + 'git::git' + 'hg::mercurial' + 'svn::subversion') + ######################################################################### # ARCHITECTURE, COMPILE FLAGS ######################################################################### @@ -29,27 +36,30 @@ CHOST="x86_64-unknown-linux-gnu" #-- Compiler and Linker Flags # -march (or -mcpu) builds exclusively for an architecture # -mtune optimizes for an architecture, but builds for whole processor family -CFLAGS="-march=x86-64 -mtune=generic -O2 -pipe -fstack-protector --param=ssp-buffer-size=4 -D_FORTIFY_SOURCE=2" -CXXFLAGS="-march=x86-64 -mtune=generic -O2 -pipe -fstack-protector --param=ssp-buffer-size=4 -D_FORTIFY_SOURCE=2" -LDFLAGS="-Wl,-O1,--sort-common,--as-needed,-z,relro,--hash-style=gnu" +CPPFLAGS="-D_FORTIFY_SOURCE=2" +CFLAGS="-march=x86-64 -mtune=generic -O2 -pipe -fstack-protector-strong" +CXXFLAGS="-march=x86-64 -mtune=generic -O2 -pipe -fstack-protector-strong" +LDFLAGS="-Wl,-O1,--sort-common,--as-needed,-z,relro" #-- Make Flags: change this for DistCC/SMP systems #MAKEFLAGS="-j2" +#-- Debugging flags +DEBUG_CFLAGS="-g -fvar-tracking-assignments" +DEBUG_CXXFLAGS="-g -fvar-tracking-assignments" ######################################################################### # BUILD ENVIRONMENT ######################################################################### # -# Defaults: BUILDENV=(fakeroot !distcc color !ccache check !sign) +# Defaults: BUILDENV=(!distcc color !ccache check !sign) # A negated environment option will do the opposite of the comments below. # -#-- fakeroot: Allow building packages as a non-root user #-- distcc: Use the Distributed C/C++/ObjC compiler #-- color: Colorize output messages #-- ccache: Use ccache to cache compilation #-- check: Run the check() function if present in the PKGBUILD #-- sign: Generate PGP signature file # -BUILDENV=(fakeroot !distcc color !ccache check !sign) +BUILDENV=(!distcc color !ccache check !sign) # #-- If using DistCC, your MAKEFLAGS will also need modification. In addition, #-- specify a space-delimited list of hosts running in the DistCC cluster. @@ -63,18 +73,21 @@ BUILDENV=(fakeroot !distcc color !ccache check !sign) # These are default values for the options=() settings ######################################################################### # -# Default: OPTIONS=(strip docs libtool emptydirs zipman purge !upx) +# Default: OPTIONS=(strip docs !libtool !staticlibs emptydirs zipman purge !optipng !upx !debug) # A negated option will do the opposite of the comments below. # -#-- strip: Strip symbols from binaries/libraries -#-- docs: Save doc directories specified by DOC_DIRS -#-- libtool: Leave libtool (.la) files in packages -#-- emptydirs: Leave empty directories in packages -#-- zipman: Compress manual (man and info) pages in MAN_DIRS with gzip -#-- purge: Remove files specified by PURGE_TARGETS -#-- upx: Compress binary executable files using UPX -# -OPTIONS=(strip docs libtool emptydirs zipman purge !upx) +#-- strip: Strip symbols from binaries/libraries +#-- docs: Save doc directories specified by DOC_DIRS +#-- libtool: Leave libtool (.la) files in packages +#-- staticlibs: Leave static library (.a) files in packages +#-- emptydirs: Leave empty directories in packages +#-- zipman: Compress manual (man and info) pages in MAN_DIRS with gzip +#-- purge: Remove files specified by PURGE_TARGETS +#-- upx: Compress binary executable files using UPX +#-- optipng: Optimize PNG images with optipng +#-- debug: Add debugging flags as specified in DEBUG_* variables +# +OPTIONS=(strip docs !libtool !staticlibs emptydirs zipman purge !optipng !upx !debug) #-- File integrity checks to use. Valid: md5, sha1, sha256, sha384, sha512 INTEGRITY_CHECK=(md5) @@ -103,11 +116,24 @@ PURGE_TARGETS=(usr/{,share}/info/dir .packlist *.pod) #SRCDEST=/home/sources #-- Source packages: specify a fixed directory where all src packages will be placed #SRCPKGDEST=/home/srcpackages +#-- Log files: specify a fixed directory where all log files will be placed +#LOGDEST=/home/makepkglogs #-- Packager: name/email of the person or organization building packages #PACKAGER="John Doe " #-- Specify a key to use for package signing #GPGKEY="" +######################################################################### +# COMPRESSION DEFAULTS +######################################################################### +# +COMPRESSGZ=(gzip -c -f -n) +COMPRESSBZ2=(bzip2 -c -f) +COMPRESSXZ=(xz -c -z -) +COMPRESSLRZ=(lrzip -q) +COMPRESSLZO=(lzop -q) +COMPRESSZ=(compress -c -f) + ######################################################################### # EXTENSION DEFAULTS ######################################################################### -- cgit v1.2.2 From 282bf65c81e278b9237b4c202d325642bc0aa1a3 Mon Sep 17 00:00:00 2001 From: Levente Polyak Date: Mon, 8 Feb 2016 14:26:23 +0100 Subject: disallow packages not built in a chroot double shame! This will detect both, packages with missing .BUILDINFO (built in non updated environments) and packages that are not built in a chroot at all (indicated by non default builddir). Signed-off-by: Levente Polyak Signed-off-by: Pierre Schmitz --- db-functions | 18 ++++++++++++++++++ db-update | 6 ++++++ 2 files changed, 24 insertions(+) diff --git a/db-functions b/db-functions index 667ac8d..3101c0c 100644 --- a/db-functions +++ b/db-functions @@ -230,6 +230,13 @@ _grep_pkginfo() { echo "${_ret#${2} = }" } +# usage: _grep_buildinfo pkgfile pattern +_grep_buildinfo() { + local _ret + + _ret="$(/usr/bin/bsdtar -xOqf "$1" .BUILDINFO | grep -m 1 "^${2} = ")" + echo "${_ret#${2} = }" +} # Get the package base or name as fallback getpkgbase() { @@ -299,6 +306,17 @@ check_packager() { [[ $_packager && $_packager != 'Unknown Packager' ]] } +check_buildinfo() { + /usr/bin/bsdtar -tf "$1" .BUILDINFO >/dev/null 2>&1 +} + +check_builddir() { + local _builddir + + _builddir=$(_grep_buildinfo "$1" "builddir") + [[ $_builddir && $_builddir = '/build' ]] +} + getpkgfile() { if [[ ${#} -ne 1 ]]; then error 'No canonical package found!' diff --git a/db-update b/db-update index cdc35a5..db8cb91 100755 --- a/db-update +++ b/db-update @@ -54,6 +54,12 @@ for repo in ${repos[@]}; do if ! check_packager "${pkg}"; then die "Package ${repo}/${pkg##*/} does not have a valid packager" fi + if ! check_buildinfo "${pkg}"; then + die "Package ${repo}/${pkg##*/} does not have a .BUILDINFO file" + fi + if ! check_builddir "${pkg}"; then + die "Package ${repo}/${pkg##*/} was not built in a chroot" + fi done if ! check_splitpkgs ${repo} ${pkgs[@]}; then die "Missing split packages for ${repo}" -- cgit v1.2.2 From 446edc13a9b3e354f93f05bf46be484a5d5f10e7 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sat, 15 Apr 2017 14:37:49 +0200 Subject: Run tests using docker --- .gitignore | 3 ++- Makefile | 7 +++++++ config | 3 ++- test/Dockerfile | 22 ++++++++++++++++++++++ test/lib/common.inc | 15 ++++----------- test/packages/pkg-any-a/PKGBUILD | 4 ++-- test/packages/pkg-any-b/PKGBUILD | 4 ++-- test/packages/pkg-simple-a/Makefile | 1 - test/packages/pkg-simple-a/PKGBUILD | 13 +------------ test/packages/pkg-simple-a/test.c | 1 - test/packages/pkg-simple-b/Makefile | 1 - test/packages/pkg-simple-b/PKGBUILD | 13 +------------ test/packages/pkg-simple-b/test.c | 1 - test/packages/pkg-simple-epoch/Makefile | 1 - test/packages/pkg-simple-epoch/PKGBUILD | 13 +------------ test/packages/pkg-simple-epoch/test.c | 1 - test/packages/pkg-split-a/Makefile | 1 - test/packages/pkg-split-a/PKGBUILD | 16 ++-------------- test/packages/pkg-split-a/test.c | 1 - test/packages/pkg-split-b/Makefile | 1 - test/packages/pkg-split-b/PKGBUILD | 17 ++--------------- test/packages/pkg-split-b/test.c | 1 - test/src/Makefile | 5 ----- test/src/test.c | 7 ------- test/test.d/db-update.sh | 4 ++-- test/test.d/testing2x.sh | 2 +- 26 files changed, 51 insertions(+), 107 deletions(-) create mode 100644 Makefile create mode 100644 test/Dockerfile delete mode 120000 test/packages/pkg-simple-a/Makefile delete mode 120000 test/packages/pkg-simple-a/test.c delete mode 120000 test/packages/pkg-simple-b/Makefile delete mode 120000 test/packages/pkg-simple-b/test.c delete mode 120000 test/packages/pkg-simple-epoch/Makefile delete mode 120000 test/packages/pkg-simple-epoch/test.c delete mode 120000 test/packages/pkg-split-a/Makefile delete mode 120000 test/packages/pkg-split-a/test.c delete mode 120000 test/packages/pkg-split-b/Makefile delete mode 120000 test/packages/pkg-split-b/test.c delete mode 100644 test/src/Makefile delete mode 100644 test/src/test.c diff --git a/.gitignore b/.gitignore index f47d96a..fcc397a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ *~ /config.local -test/packages/*/*.pkg.tar.?z +/test/packages/*/*.pkg.tar.?z +/.idea diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..3a59211 --- /dev/null +++ b/Makefile @@ -0,0 +1,7 @@ +test-image: + docker build --pull -t dbscripts/test test + +test: test-image + docker run --rm --network=none -v $(PWD):/dbscripts:ro --tmpfs=/tmp:exec -w /dbscripts/test dbscripts/test ./runTest + +.PHONY: test-image test diff --git a/config b/config index 3df6c95..e5440d2 100644 --- a/config +++ b/config @@ -34,4 +34,5 @@ SRCEXT=".src.tar.gz" ALLOWED_LICENSES=('GPL' 'GPL1' 'GPL2' 'LGPL' 'LGPL1' 'LGPL2' 'LGPL2.1') # Override default config with config.local -[ -f "$(dirname ${BASH_SOURCE[0]})/config.local" ] && . "$(dirname ${BASH_SOURCE[0]})/config.local" +LOCAL_CONFIG=${DBSCRIPTS_CONFIG:-"$(dirname ${BASH_SOURCE[0]})/config.local"} +[ -f "${LOCAL_CONFIG}" ] && . "${LOCAL_CONFIG}" diff --git a/test/Dockerfile b/test/Dockerfile new file mode 100644 index 0000000..30247bf --- /dev/null +++ b/test/Dockerfile @@ -0,0 +1,22 @@ +FROM pierres/archlinux +RUN pacman -Syu --noconfirm --needed base-devel devtools +RUN pacman-key --init +RUN echo '%wheel ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/wheel +RUN useradd -N -g users -G wheel -m tester +RUN install -d -o tester -g users /build +USER tester +RUN echo -e "\ +%echo Generating signing key...\n\ +Key-Type: RSA\n\ +Key-Length: 1024\n\ +Key-Usage: sign\n\ +Name-Real: Bob Tester\n\ +Name-Email: tester@localhost\n\ +Expire-Date: 0\n\ +%no-protection\n\ +%commit\n\ +%echo Done\n"\ +| gpg --quiet --batch --no-tty --no-permission-warning --gen-key +RUN gpg --export | sudo pacman-key -a - +RUN sudo pacman-key --lsign-key tester@localhost +RUN echo -e 'BUILDDIR=/build\nPACKAGER="Bob Tester "' > /home/tester/.makepkg.conf diff --git a/test/lib/common.inc b/test/lib/common.inc index aeceece..d1ea100 100644 --- a/test/lib/common.inc +++ b/test/lib/common.inc @@ -46,17 +46,12 @@ oneTimeSetUp() { if ! ${build}; then if [ "${pkgarch[0]}" == 'any' ]; then - sudo extra-x86_64-build || die 'extra-x86_64-build failed' + makepkg -cCf || die 'makepkg failed' else for a in ${pkgarch[@]}; do - sudo extra-${a}-build || die "extra-${a}-build failed" + CARCH=${a} makepkg -cCf || die "makepkg failed" done fi - for a in ${pkgarch[@]}; do - for p in ${pkgname[@]}; do - cp ${p}-${pkgversion}-${a}${PKGEXT} $(dirname ${BASH_SOURCE[0]})/../packages/${d##*/} - done - done fi popd >/dev/null done @@ -72,9 +67,7 @@ setUp() { local r local a - [ -f "$(dirname ${BASH_SOURCE[0]})/../../config.local" ] && die "$(dirname ${BASH_SOURCE[0]})/../../config.local exists" TMP="$(mktemp -d /tmp/${0##*/}.XXXXXXXXXX)" - #msg "Using ${TMP}" PKGREPOS=('core' 'extra' 'testing') PKGPOOL='pool/packages' @@ -101,7 +94,8 @@ setUp() { arch_svn commit -q -m"initial commit of ${pkg}" "${TMP}/svn-packages-copy" done - cat < "$(dirname ${BASH_SOURCE[0]})/../../config.local" + export DBSCRIPTS_CONFIG=${TMP}/config.local + cat < "${DBSCRIPTS_CONFIG}" FTP_BASE="${TMP}/ftp" SVNREPO="file://${TMP}/svn-packages-repo" PKGREPOS=(${PKGREPOS[@]}) @@ -121,7 +115,6 @@ eot tearDown() { rm -rf "${TMP}" - rm -f "$(dirname ${BASH_SOURCE[0]})/../../config.local" echo } diff --git a/test/packages/pkg-any-a/PKGBUILD b/test/packages/pkg-any-a/PKGBUILD index 8749a35..3026919 100644 --- a/test/packages/pkg-any-a/PKGBUILD +++ b/test/packages/pkg-any-a/PKGBUILD @@ -7,6 +7,6 @@ url='http://www.archlinux.org/' license=('GPL') package() { - install -d -m755 ${pkgdir}/usr/share/${pkgname} - echo 'test' > ${pkgdir}/usr/share/${pkgname}/test + install -d ${pkgdir}/usr/share/${pkgname} + :>${pkgdir}/usr/share/${pkgname}/test } diff --git a/test/packages/pkg-any-b/PKGBUILD b/test/packages/pkg-any-b/PKGBUILD index e6a0498..cca1859 100644 --- a/test/packages/pkg-any-b/PKGBUILD +++ b/test/packages/pkg-any-b/PKGBUILD @@ -7,6 +7,6 @@ url='http://www.archlinux.org/' license=('GPL') package() { - install -d -m755 ${pkgdir}/usr/share/${pkgname} - echo 'test' > ${pkgdir}/usr/share/${pkgname}/test + install -d ${pkgdir}/usr/share/${pkgname} + :>${pkgdir}/usr/share/${pkgname}/test } diff --git a/test/packages/pkg-simple-a/Makefile b/test/packages/pkg-simple-a/Makefile deleted file mode 120000 index 50be211..0000000 --- a/test/packages/pkg-simple-a/Makefile +++ /dev/null @@ -1 +0,0 @@ -../../src/Makefile \ No newline at end of file diff --git a/test/packages/pkg-simple-a/PKGBUILD b/test/packages/pkg-simple-a/PKGBUILD index 953ecfa..25dbed3 100644 --- a/test/packages/pkg-simple-a/PKGBUILD +++ b/test/packages/pkg-simple-a/PKGBUILD @@ -5,18 +5,7 @@ pkgdesc="A package called ${pkgname}" arch=('i686' 'x86_64') url='http://www.archlinux.org/' license=('GPL') -depends=('glibc') -makedepends=('gcc') -source=('Makefile' 'test.c') -md5sums=('c6cb8dcc86253355fed559416d0c8dcf' - '3c1e4279feb678fd9cabaccdb28e40d0') - -build() { - cd ${srcdir} - make -} package() { - cd ${srcdir} - make install DESTDIR=${pkgdir} DESTBIN=${pkgname} + install -D -m755 /bin/true ${pkgdir}/usr/bin/${pkgname} } diff --git a/test/packages/pkg-simple-a/test.c b/test/packages/pkg-simple-a/test.c deleted file mode 120000 index ed5b5ac..0000000 --- a/test/packages/pkg-simple-a/test.c +++ /dev/null @@ -1 +0,0 @@ -../../src/test.c \ No newline at end of file diff --git a/test/packages/pkg-simple-b/Makefile b/test/packages/pkg-simple-b/Makefile deleted file mode 120000 index 50be211..0000000 --- a/test/packages/pkg-simple-b/Makefile +++ /dev/null @@ -1 +0,0 @@ -../../src/Makefile \ No newline at end of file diff --git a/test/packages/pkg-simple-b/PKGBUILD b/test/packages/pkg-simple-b/PKGBUILD index 95ffd09..49bffab 100644 --- a/test/packages/pkg-simple-b/PKGBUILD +++ b/test/packages/pkg-simple-b/PKGBUILD @@ -5,18 +5,7 @@ pkgdesc="A package called ${pkgname}" arch=('i686' 'x86_64') url='http://www.archlinux.org/' license=('GPL') -depends=('glibc') -makedepends=('gcc') -source=('Makefile' 'test.c') -md5sums=('c6cb8dcc86253355fed559416d0c8dcf' - '3c1e4279feb678fd9cabaccdb28e40d0') - -build() { - cd ${srcdir} - make -} package() { - cd ${srcdir} - make install DESTDIR=${pkgdir} DESTBIN=${pkgname} + install -D -m755 /bin/true ${pkgdir}/usr/bin/${pkgname} } diff --git a/test/packages/pkg-simple-b/test.c b/test/packages/pkg-simple-b/test.c deleted file mode 120000 index ed5b5ac..0000000 --- a/test/packages/pkg-simple-b/test.c +++ /dev/null @@ -1 +0,0 @@ -../../src/test.c \ No newline at end of file diff --git a/test/packages/pkg-simple-epoch/Makefile b/test/packages/pkg-simple-epoch/Makefile deleted file mode 120000 index 50be211..0000000 --- a/test/packages/pkg-simple-epoch/Makefile +++ /dev/null @@ -1 +0,0 @@ -../../src/Makefile \ No newline at end of file diff --git a/test/packages/pkg-simple-epoch/PKGBUILD b/test/packages/pkg-simple-epoch/PKGBUILD index eebe2bd..bdce203 100644 --- a/test/packages/pkg-simple-epoch/PKGBUILD +++ b/test/packages/pkg-simple-epoch/PKGBUILD @@ -6,18 +6,7 @@ pkgdesc="A package called ${pkgname}" arch=('i686' 'x86_64') url='http://www.archlinux.org/' license=('GPL') -depends=('glibc') -makedepends=('gcc') -source=('Makefile' 'test.c') -md5sums=('c6cb8dcc86253355fed559416d0c8dcf' - '3c1e4279feb678fd9cabaccdb28e40d0') - -build() { - cd ${srcdir} - make -} package() { - cd ${srcdir} - make install DESTDIR=${pkgdir} DESTBIN=${pkgname} + install -D -m755 /bin/true ${pkgdir}/usr/bin/${pkgname} } diff --git a/test/packages/pkg-simple-epoch/test.c b/test/packages/pkg-simple-epoch/test.c deleted file mode 120000 index ed5b5ac..0000000 --- a/test/packages/pkg-simple-epoch/test.c +++ /dev/null @@ -1 +0,0 @@ -../../src/test.c \ No newline at end of file diff --git a/test/packages/pkg-split-a/Makefile b/test/packages/pkg-split-a/Makefile deleted file mode 120000 index 50be211..0000000 --- a/test/packages/pkg-split-a/Makefile +++ /dev/null @@ -1 +0,0 @@ -../../src/Makefile \ No newline at end of file diff --git a/test/packages/pkg-split-a/PKGBUILD b/test/packages/pkg-split-a/PKGBUILD index e941976..9b764db 100644 --- a/test/packages/pkg-split-a/PKGBUILD +++ b/test/packages/pkg-split-a/PKGBUILD @@ -6,23 +6,11 @@ pkgdesc="A split package called ${pkgbase}" arch=('i686' 'x86_64') url='http://www.archlinux.org/' license=('GPL') -depends=('glibc') -makedepends=('gcc') -source=('Makefile' 'test.c') -md5sums=('c6cb8dcc86253355fed559416d0c8dcf' - '3c1e4279feb678fd9cabaccdb28e40d0') - -build() { - cd ${srcdir} - make -} package_pkg-split-a1() { - cd ${srcdir} - make install DESTDIR=${pkgdir} DESTBIN=${pkgname[0]} + install -D -m755 /bin/true ${pkgdir}/usr/bin/${pkgname} } package_pkg-split-a2() { - cd ${srcdir} - make install DESTDIR=${pkgdir} DESTBIN=${pkgname[1]} + install -D -m755 /bin/true ${pkgdir}/usr/bin/${pkgname} } diff --git a/test/packages/pkg-split-a/test.c b/test/packages/pkg-split-a/test.c deleted file mode 120000 index ed5b5ac..0000000 --- a/test/packages/pkg-split-a/test.c +++ /dev/null @@ -1 +0,0 @@ -../../src/test.c \ No newline at end of file diff --git a/test/packages/pkg-split-b/Makefile b/test/packages/pkg-split-b/Makefile deleted file mode 120000 index 50be211..0000000 --- a/test/packages/pkg-split-b/Makefile +++ /dev/null @@ -1 +0,0 @@ -../../src/Makefile \ No newline at end of file diff --git a/test/packages/pkg-split-b/PKGBUILD b/test/packages/pkg-split-b/PKGBUILD index 6ddbc45..3c05371 100644 --- a/test/packages/pkg-split-b/PKGBUILD +++ b/test/packages/pkg-split-b/PKGBUILD @@ -7,23 +7,10 @@ arch=('i686' 'x86_64') url='http://www.archlinux.org/' license=('GPL') -depends=('glibc') -makedepends=('gcc') -source=('Makefile' 'test.c') -md5sums=('c6cb8dcc86253355fed559416d0c8dcf' - '3c1e4279feb678fd9cabaccdb28e40d0') - -build() { - cd ${srcdir} - make -} - package_pkg-split-b1() { - cd ${srcdir} - make install DESTDIR=${pkgdir} DESTBIN=${pkgname[0]} + install -D -m755 /bin/true ${pkgdir}/usr/bin/${pkgname} } package_pkg-split-b2() { - cd ${srcdir} - make install DESTDIR=${pkgdir} DESTBIN=${pkgname[1]} + install -D -m755 /bin/true ${pkgdir}/usr/bin/${pkgname} } diff --git a/test/packages/pkg-split-b/test.c b/test/packages/pkg-split-b/test.c deleted file mode 120000 index ed5b5ac..0000000 --- a/test/packages/pkg-split-b/test.c +++ /dev/null @@ -1 +0,0 @@ -../../src/test.c \ No newline at end of file diff --git a/test/src/Makefile b/test/src/Makefile deleted file mode 100644 index 105b730..0000000 --- a/test/src/Makefile +++ /dev/null @@ -1,5 +0,0 @@ -all: - gcc $(CFLAGS) -o test test.c - -install: - install -D -m755 test $(DESTDIR)/usr/bin/$(DESTBIN) diff --git a/test/src/test.c b/test/src/test.c deleted file mode 100644 index a661689..0000000 --- a/test/src/test.c +++ /dev/null @@ -1,7 +0,0 @@ -#include -#include - -int main(void) { - printf("Arch is the best!\n"); - return EXIT_SUCCESS; -} diff --git a/test/test.d/db-update.sh b/test/test.d/db-update.sh index 7a5ce4c..7ae7a06 100755 --- a/test/test.d/db-update.sh +++ b/test/test.d/db-update.sh @@ -82,7 +82,7 @@ testUpdateAnyPackage() { pushd "${TMP}/svn-packages-copy/pkg-any-a/trunk/" >/dev/null sed 's/pkgrel=1/pkgrel=2/g' -i PKGBUILD arch_svn commit -q -m"update pkg to pkgrel=2" >/dev/null - sudo extra-i686-build + makepkg -cCf mv pkg-any-a-1-2-any.pkg.tar.xz "${pkgdir}/pkg-any-a/" popd >/dev/null @@ -100,7 +100,7 @@ testUpdateAnyPackageToDifferentRepositoriesAtOnce() { pushd "${TMP}/svn-packages-copy/pkg-any-a/trunk/" >/dev/null sed 's/pkgrel=1/pkgrel=2/g' -i PKGBUILD arch_svn commit -q -m"update pkg to pkgrel=2" >/dev/null - sudo extra-i686-build + makepkg -cCf mv pkg-any-a-1-2-any.pkg.tar.xz "${pkgdir}/pkg-any-a/" popd >/dev/null diff --git a/test/test.d/testing2x.sh b/test/test.d/testing2x.sh index 31d85b4..fdc7ca3 100755 --- a/test/test.d/testing2x.sh +++ b/test/test.d/testing2x.sh @@ -10,7 +10,7 @@ testTesting2xAnyPackage() { pushd "${TMP}/svn-packages-copy/pkg-any-a/trunk/" >/dev/null sed 's/pkgrel=1/pkgrel=2/g' -i PKGBUILD arch_svn commit -q -m"update pkg to pkgrel=2" >/dev/null - sudo extra-i686-build + makepkg -cCf mv pkg-any-a-1-2-any.pkg.tar.xz "${pkgdir}/pkg-any-a/" popd >/dev/null -- cgit v1.2.2 From aaf67a3b12239fe11cc8e4346340be67930ffa0b Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sat, 15 Apr 2017 14:39:42 +0200 Subject: Add travis configuration --- .travis.yml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..2e3480a --- /dev/null +++ b/.travis.yml @@ -0,0 +1,7 @@ +sudo: required + +services: + - docker + +script: + - make test -- cgit v1.2.2 From abe255d51bc450ee9c0ac1a4486b4d7d10c9794f Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sat, 15 Apr 2017 15:36:55 +0200 Subject: Add editor configuration --- .editorconfig | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .editorconfig diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..ffdaa55 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,7 @@ +root = true +[*] +indent_style = tab +end_of_line = lf +charset = utf-8 +[*.yml] +indent_style = space -- cgit v1.2.2 From 355b457c97a55c9ee16ef2f81fc06d49c47c129b Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sat, 15 Apr 2017 16:30:41 +0200 Subject: Compile tests into one file to reduce time spent on setup --- test/runTest | 23 ++++++++++------------- test/test.d/create-filelists.sh | 7 ------- test/test.d/db-move.sh | 7 ------- test/test.d/db-remove.sh | 7 ------- test/test.d/db-repo-add.sh | 7 ------- test/test.d/db-repo-remove.sh | 7 ------- test/test.d/db-update.sh | 7 ------- test/test.d/ftpdir-cleanup.sh | 7 ------- test/test.d/packages.sh | 7 ------- test/test.d/signed-packages.sh | 7 ------- test/test.d/sourceballs.sh | 7 ------- test/test.d/testing2x.sh | 7 ------- 12 files changed, 10 insertions(+), 90 deletions(-) diff --git a/test/runTest b/test/runTest index b8713d8..adeb84e 100755 --- a/test/runTest +++ b/test/runTest @@ -1,15 +1,12 @@ #!/bin/bash -. "$(dirname ${BASH_SOURCE[0]})/lib/common.inc" - -for t in "$(dirname ${BASH_SOURCE[0]})/test.d/"*.sh; do - l=$(basename ${t} .sh) - if [ -x ${t} ]; then - msg "Running test '${l}'" - ${t} - [ $? -ne 0 ] && die "Test '${l}' failed" - echo -e "\n\n\n" - else - warning "Skipping test ${l}" - fi -done +curdir=$(readlink -e $(dirname $0)) + +testSuite=$(mktemp) +trap "rm -rf '${testSuite}'" EXIT + +echo ". ${curdir}/lib/common.inc" > ${testSuite} +cat "${curdir}/test.d"/*.sh >> ${testSuite} +echo ". ${curdir}/lib/shunit2" >> ${testSuite} + +bash ${testSuite} diff --git a/test/test.d/create-filelists.sh b/test/test.d/create-filelists.sh index 49734c4..0010bf3 100755 --- a/test/test.d/create-filelists.sh +++ b/test/test.d/create-filelists.sh @@ -1,8 +1,3 @@ -#!/bin/bash - -curdir=$(readlink -e $(dirname $0)) -. "${curdir}/../lib/common.inc" - testCreateSimpleFileLists() { local arches=('i686' 'x86_64') local pkgs=('pkg-simple-a' 'pkg-simple-b' 'pkg-simple-epoch') @@ -101,5 +96,3 @@ testCleanupFileLists() { done } - -. "${curdir}/../lib/shunit2" diff --git a/test/test.d/db-move.sh b/test/test.d/db-move.sh index 890ffc4..a27dfbb 100755 --- a/test/test.d/db-move.sh +++ b/test/test.d/db-move.sh @@ -1,8 +1,3 @@ -#!/bin/bash - -curdir=$(readlink -e $(dirname $0)) -. "${curdir}/../lib/common.inc" - testMoveSimplePackages() { local arches=('i686' 'x86_64') local pkgs=('pkg-simple-a' 'pkg-simple-b') @@ -118,5 +113,3 @@ testMoveSplitPackages() { checkRemovedAnyPackage testing pkg-split-a } - -. "${curdir}/../lib/shunit2" diff --git a/test/test.d/db-remove.sh b/test/test.d/db-remove.sh index 416e693..52b9584 100755 --- a/test/test.d/db-remove.sh +++ b/test/test.d/db-remove.sh @@ -1,8 +1,3 @@ -#!/bin/bash - -curdir=$(readlink -e $(dirname $0)) -. "${curdir}/../lib/common.inc" - testRemovePackages() { local arches=('i686' 'x86_64') local pkgs=('pkg-simple-a' 'pkg-simple-b' 'pkg-split-a' 'pkg-split-b' 'pkg-simple-epoch') @@ -73,5 +68,3 @@ testRemoveAnyPackages() { checkRemovedAnyPackage extra ${pkgbase} done } - -. "${curdir}/../lib/shunit2" diff --git a/test/test.d/db-repo-add.sh b/test/test.d/db-repo-add.sh index 8603104..90e619b 100755 --- a/test/test.d/db-repo-add.sh +++ b/test/test.d/db-repo-add.sh @@ -1,8 +1,3 @@ -#!/bin/bash - -curdir=$(readlink -e $(dirname $0)) -. "${curdir}/../lib/common.inc" - testAddSimplePackages() { local arches=('i686' 'x86_64') local pkgs=('pkg-simple-a' 'pkg-simple-b') @@ -50,5 +45,3 @@ testAddMultiplePackages() { done done } - -. "${curdir}/../lib/shunit2" diff --git a/test/test.d/db-repo-remove.sh b/test/test.d/db-repo-remove.sh index 315d63d..0f294b5 100755 --- a/test/test.d/db-repo-remove.sh +++ b/test/test.d/db-repo-remove.sh @@ -1,8 +1,3 @@ -#!/bin/bash - -curdir=$(readlink -e $(dirname $0)) -. "${curdir}/../lib/common.inc" - testRemovePackages() { local arches=('i686' 'x86_64') local pkgs=('pkg-simple-a' 'pkg-simple-b' 'pkg-simple-epoch') @@ -54,5 +49,3 @@ testRemoveMultiplePackages() { done done } - -. "${curdir}/../lib/shunit2" diff --git a/test/test.d/db-update.sh b/test/test.d/db-update.sh index 7ae7a06..f9e7d1a 100755 --- a/test/test.d/db-update.sh +++ b/test/test.d/db-update.sh @@ -1,8 +1,3 @@ -#!/bin/bash - -curdir=$(readlink -e $(dirname $0)) -. "${curdir}/../lib/common.inc" - testAddSimplePackages() { local arches=('i686' 'x86_64') local pkgs=('pkg-simple-a' 'pkg-simple-b') @@ -171,5 +166,3 @@ testUnknownRepo() { [ -e "${FTP_BASE}/unknown" ] && fail "db-update pushed a package into an unknown repository" rm -rf "${STAGING}/unknown/" } - -. "${curdir}/../lib/shunit2" diff --git a/test/test.d/ftpdir-cleanup.sh b/test/test.d/ftpdir-cleanup.sh index bfea7ea..73779a2 100755 --- a/test/test.d/ftpdir-cleanup.sh +++ b/test/test.d/ftpdir-cleanup.sh @@ -1,8 +1,3 @@ -#!/bin/bash - -curdir=$(readlink -e $(dirname $0)) -. "${curdir}/../lib/common.inc" - testCleanupSimplePackages() { local arches=('i686' 'x86_64') local pkgs=('pkg-simple-a' 'pkg-simple-b') @@ -117,5 +112,3 @@ testCleanupSplitPackages() { done done } - -. "${curdir}/../lib/shunit2" diff --git a/test/test.d/packages.sh b/test/test.d/packages.sh index 488cb15..a5e432c 100755 --- a/test/test.d/packages.sh +++ b/test/test.d/packages.sh @@ -1,11 +1,4 @@ -#!/bin/bash - -curdir=$(readlink -e $(dirname $0)) -. "${curdir}/../lib/common.inc" - testPackages() { # TODO: namcap -r sodepends fails with i686 packages find "${pkgdir}" -name "*${PKGEXT}" -exec namcap -e sodepends,pkgnameindesc {} + || fail 'namcap failed' } - -. "${curdir}/../lib/shunit2" diff --git a/test/test.d/signed-packages.sh b/test/test.d/signed-packages.sh index 03566ef..148929f 100755 --- a/test/test.d/signed-packages.sh +++ b/test/test.d/signed-packages.sh @@ -1,8 +1,3 @@ -#!/bin/bash - -curdir=$(readlink -e $(dirname $0)) -. "${curdir}/../lib/common.inc" - testAddSignedPackage() { releasePackage extra 'pkg-simple-a' 'i686' ../db-update || fail "db-update failed!" @@ -32,5 +27,3 @@ testAddBrokenSignature() { done ../db-update >/dev/null 2>&1 && fail "db-update should fail when a signature is broken!" } - -. "${curdir}/../lib/shunit2" diff --git a/test/test.d/sourceballs.sh b/test/test.d/sourceballs.sh index fdcf08c..a004cdb 100755 --- a/test/test.d/sourceballs.sh +++ b/test/test.d/sourceballs.sh @@ -1,8 +1,3 @@ -#!/bin/bash - -curdir=$(readlink -e $(dirname $0)) -. "${curdir}/../lib/common.inc" - testSourceballs() { local arches=('i686' 'x86_64') local pkgs=('pkg-simple-a' 'pkg-simple-b' 'pkg-simple-epoch') @@ -80,5 +75,3 @@ testSourceballsCleanup() { [ -r ${FTP_BASE}/${SRCPOOL}/pkg-simple-a-*${SRCEXT} ] && fail "source package was not removed!" [ ! -r ${FTP_BASE}/${SRCPOOL}/pkg-simple-b-*${SRCEXT} ] && fail "source package not found!" } - -. "${curdir}/../lib/shunit2" diff --git a/test/test.d/testing2x.sh b/test/test.d/testing2x.sh index fdc7ca3..8fac2ad 100755 --- a/test/test.d/testing2x.sh +++ b/test/test.d/testing2x.sh @@ -1,8 +1,3 @@ -#!/bin/bash - -curdir=$(readlink -e $(dirname $0)) -. "${curdir}/../lib/common.inc" - testTesting2xAnyPackage() { releasePackage core pkg-any-a any ../db-update @@ -23,5 +18,3 @@ testTesting2xAnyPackage() { checkAnyPackage core pkg-any-a-1-2-any.pkg.tar.xz any checkRemovedAnyPackage testing pkg-any-a } - -. "${curdir}/../lib/shunit2" -- cgit v1.2.2 From d21565347c01139c03ddfccffa67a756b9436f4a Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sat, 15 Apr 2017 17:40:17 +0200 Subject: Simplify test setup --- test/lib/common.inc | 281 ---------------------------------------------------- test/lib/common.sh | 259 ++++++++++++++++++++++++++++++++++++++++++++++++ test/runTest | 2 +- 3 files changed, 260 insertions(+), 282 deletions(-) delete mode 100644 test/lib/common.inc create mode 100644 test/lib/common.sh diff --git a/test/lib/common.inc b/test/lib/common.inc deleted file mode 100644 index d1ea100..0000000 --- a/test/lib/common.inc +++ /dev/null @@ -1,281 +0,0 @@ -set -E - -. "$(dirname ${BASH_SOURCE[0]})/../../config" -# override the default TMPDIR -TMPDIR="$(mktemp -d /tmp/${0##*/}.XXXXXXXXXX)" -. "$(dirname ${BASH_SOURCE[0]})/../../db-functions" - -signpkg() { - if [[ -r '/etc/makepkg.conf' ]]; then - source '/etc/makepkg.conf' - else - die '/etc/makepkg.conf not found!' - fi - if [[ -r ~/.makepkg.conf ]]; then - . ~/.makepkg.conf - fi - if [[ -n $GPGKEY ]]; then - SIGNWITHKEY="-u ${GPGKEY}" - fi - gpg --detach-sign --use-agent ${SIGNWITHKEY} ${@} || die -} - -oneTimeSetUp() { - local p - local d - local a - local pkgname - local pkgarch - local pkgversion - local build - pkgdir="$(mktemp -d /tmp/${0##*/}.XXXXXXXXXX)" - cp -Lr $(dirname ${BASH_SOURCE[0]})/../packages/* "${pkgdir}" - msg 'Building packages...' - for d in "${pkgdir}"/*; do - pushd $d >/dev/null - pkgname=($(. PKGBUILD; echo ${pkgname[@]})) - pkgarch=($(. PKGBUILD; echo ${arch[@]})) - pkgversion=$(. PKGBUILD; echo $(get_full_version ${epoch:-0} ${pkgver} ${pkgrel})) - - build=true - for a in ${pkgarch[@]}; do - for p in ${pkgname[@]}; do - [ ! -f ${p}-${pkgversion}-${a}${PKGEXT} ] && build=false - done - done - - if ! ${build}; then - if [ "${pkgarch[0]}" == 'any' ]; then - makepkg -cCf || die 'makepkg failed' - else - for a in ${pkgarch[@]}; do - CARCH=${a} makepkg -cCf || die "makepkg failed" - done - fi - fi - popd >/dev/null - done -} - -oneTimeTearDown() { - rm -rf "${pkgdir}" -} - -setUp() { - local p - local pkg - local r - local a - - TMP="$(mktemp -d /tmp/${0##*/}.XXXXXXXXXX)" - - PKGREPOS=('core' 'extra' 'testing') - PKGPOOL='pool/packages' - mkdir -p "${TMP}/"{ftp,tmp,staging,{package,source}-cleanup,svn-packages-{copy,repo}} - - for r in ${PKGREPOS[@]}; do - mkdir -p "${TMP}/staging/${r}" - for a in ${ARCHES[@]}; do - mkdir -p "${TMP}/ftp/${r}/os/${a}" - done - done - mkdir -p "${TMP}/ftp/${PKGPOOL}" - mkdir -p "${TMP}/ftp/${SRCPOOL}" - - msg 'Creating svn repository...' - svnadmin create "${TMP}/svn-packages-repo" - arch_svn checkout -q "file://${TMP}/svn-packages-repo" "${TMP}/svn-packages-copy" - - for p in "${pkgdir}"/*; do - pkg=${p##*/} - mkdir -p "${TMP}/svn-packages-copy/${pkg}"/{trunk,repos} - cp "${p}"/* "${TMP}/svn-packages-copy"/${pkg}/trunk/ - arch_svn add -q "${TMP}/svn-packages-copy"/${pkg} - arch_svn commit -q -m"initial commit of ${pkg}" "${TMP}/svn-packages-copy" - done - - export DBSCRIPTS_CONFIG=${TMP}/config.local - cat < "${DBSCRIPTS_CONFIG}" - FTP_BASE="${TMP}/ftp" - SVNREPO="file://${TMP}/svn-packages-repo" - PKGREPOS=(${PKGREPOS[@]}) - PKGPOOL="${PKGPOOL}" - TESTING_REPO='testing' - STABLE_REPOS=('core' 'extra') - CLEANUP_DESTDIR="${TMP}/package-cleanup" - SOURCE_CLEANUP_DESTDIR="${TMP}/source-cleanup" - STAGING="${TMP}/staging" - TMPDIR="${TMP}/tmp" - CLEANUP_DRYRUN=false - SOURCE_CLEANUP_DRYRUN=false - REQUIRE_SIGNATURE=true -eot - . "$(dirname ${BASH_SOURCE[0]})/../../config" -} - -tearDown() { - rm -rf "${TMP}" - echo -} - -releasePackage() { - local repo=$1 - local pkgbase=$2 - local arch=$3 - local a - local p - local pkgver - local pkgname - - pushd "${TMP}/svn-packages-copy"/${pkgbase}/trunk/ >/dev/null - archrelease ${repo}-${arch} >/dev/null 2>&1 - pkgver=$(. PKGBUILD; echo $(get_full_version ${epoch:-0} ${pkgver} ${pkgrel})) - pkgname=($(. PKGBUILD; echo ${pkgname[@]})) - popd >/dev/null - cp "${pkgdir}/${pkgbase}"/*-${pkgver}-${arch}${PKGEXT} "${STAGING}"/${repo}/ - - if ${REQUIRE_SIGNATURE}; then - for a in ${arch[@]}; do - for p in ${pkgname[@]}; do - signpkg "${STAGING}"/${repo}/${p}-${pkgver}-${a}${PKGEXT} - done - done - fi -} - -checkAnyPackageDB() { - local repo=$1 - local pkg=$2 - local arch - local db - - [ -r "${FTP_BASE}/${PKGPOOL}/${pkg}" ] || fail "${PKGPOOL}/${pkg} not found" - if ${REQUIRE_SIGNATURE}; then - [ -r "${FTP_BASE}/${PKGPOOL}/${pkg}.sig" ] || fail "${PKGPOOL}/${pkg}.sig not found" - fi - - for arch in i686 x86_64; do - [ -L "${FTP_BASE}/${repo}/os/${arch}/${pkg}" ] || fail "${repo}/os/${arch}/${pkg} is not a symlink" - [ "$(readlink -e "${FTP_BASE}/${repo}/os/${arch}/${pkg}")" == "${FTP_BASE}/${PKGPOOL}/${pkg}" ] \ - || fail "${repo}/os/${arch}/${pkg} does not link to ${PKGPOOL}/${pkg}" - - if ${REQUIRE_SIGNATURE}; then - [ -L "${FTP_BASE}/${repo}/os/${arch}/${pkg}.sig" ] || fail "${repo}/os/${arch}/${pkg}.sig is not a symlink" - [ "$(readlink -e "${FTP_BASE}/${repo}/os/${arch}/${pkg}.sig")" == "${FTP_BASE}/${PKGPOOL}/${pkg}.sig" ] \ - || fail "${repo}/os/${arch}/${pkg}.sig does not link to ${PKGPOOL}/${pkg}.sig" - fi - - for db in ${DBEXT} ${FILESEXT}; do - ( [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" ] \ - && bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" -O | grep -q ${pkg}) \ - || fail "${pkg} not in ${repo}/os/${arch}/${repo}${db%.tar.*}" - done - done - [ -r "${STAGING}"/${repo}/${pkg} ] && fail "${repo}/${pkg} found in staging dir" - [ -r "${STAGING}"/${repo}/${pkg}.sig ] && fail "${repo}/${pkg}.sig found in staging dir" -} - -checkAnyPackage() { - local repo=$1 - local pkg=$2 - - checkAnyPackageDB $repo $pkg - - local pkgbase=$(getpkgbase "${FTP_BASE}/${PKGPOOL}/${pkg}") - arch_svn up -q "${TMP}/svn-packages-copy/${pkgbase}" - [ -d "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo}-any" ] \ - || fail "svn-packages-copy/${pkgbase}/repos/${repo}-any does not exist" -} - -checkPackageDB() { - local repo=$1 - local pkg=$2 - local arch=$3 - local db - - [ -r "${FTP_BASE}/${PKGPOOL}/${pkg}" ] || fail "${PKGPOOL}/${pkg} not found" - [ -L "${FTP_BASE}/${repo}/os/${arch}/${pkg}" ] || fail "${repo}/os/${arch}/${pkg} not a symlink" - [ -r "${STAGING}"/${repo}/${pkg} ] && fail "${repo}/${pkg} found in staging dir" - - [ "$(readlink -e "${FTP_BASE}/${repo}/os/${arch}/${pkg}")" == "${FTP_BASE}/${PKGPOOL}/${pkg}" ] \ - || fail "${repo}/os/${arch}/${pkg} does not link to ${PKGPOOL}/${pkg}" - - if ${REQUIRE_SIGNATURE}; then - [ -r "${FTP_BASE}/${PKGPOOL}/${pkg}.sig" ] || fail "${PKGPOOL}/${pkg}.sig not found" - [ -L "${FTP_BASE}/${repo}/os/${arch}/${pkg}.sig" ] || fail "${repo}/os/${arch}/${pkg}.sig is not a symlink" - [ -r "${STAGING}"/${repo}/${pkg}.sig ] && fail "${repo}/${pkg}.sig found in staging dir" - - [ "$(readlink -e "${FTP_BASE}/${repo}/os/${arch}/${pkg}.sig")" == "${FTP_BASE}/${PKGPOOL}/${pkg}.sig" ] \ - || fail "${repo}/os/${arch}/${pkg}.sig does not link to ${PKGPOOL}/${pkg}.sig" - fi - - for db in ${DBEXT} ${FILESEXT}; do - ( [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" ] \ - && bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" -O | grep -q ${pkg}) \ - || fail "${pkg} not in ${repo}/os/${arch}/${repo}${db%.tar.*}" - done -} - -checkPackage() { - local repo=$1 - local pkg=$2 - local arch=$3 - - checkPackageDB $repo $pkg $arch - - local pkgbase=$(getpkgbase "${FTP_BASE}/${PKGPOOL}/${pkg}") - arch_svn up -q "${TMP}/svn-packages-copy/${pkgbase}" - [ -d "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo}-${arch}" ] \ - || fail "svn-packages-copy/${pkgbase}/repos/${repo}-${arch} does not exist" -} - -checkRemovedPackageDB() { - local repo=$1 - local pkgbase=$2 - local arch=$3 - local db - - for db in ${DBEXT} ${FILESEXT}; do - ( [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" ] \ - && bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" -O | grep -q ${pkgbase}) \ - && fail "${pkgbase} should not be in ${repo}/os/${arch}/${repo}${db%.tar.*}" - done -} - -checkRemovedPackage() { - local repo=$1 - local pkgbase=$2 - local arch=$3 - - checkRemovedPackageDB $repo $pkgbase $arch - - arch_svn up -q "${TMP}/svn-packages-copy/${pkgbase}" - [ -d "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo}-${arch}" ] \ - && fail "svn-packages-copy/${pkgbase}/repos/${repo}-${arch} should not exist" -} - -checkRemovedAnyPackageDB() { - local repo=$1 - local pkgbase=$2 - local arch - local db - - for db in ${DBEXT} ${FILESEXT}; do - for arch in i686 x86_64; do - ( [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" ] \ - && bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" -O | grep -q ${pkgbase}) \ - && fail "${pkgbase} should not be in ${repo}/os/${arch}/${repo}${db%.tar.*}" - done - done -} - -checkRemovedAnyPackage() { - local repo=$1 - local pkgbase=$2 - - checkRemovedAnyPackageDB $repo $pkgbase - - arch_svn up -q "${TMP}/svn-packages-copy/${pkgbase}" - [ -d "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo}-any" ] \ - && fail "svn-packages-copy/${pkgbase}/repos/${repo}-any should not exist" -} diff --git a/test/lib/common.sh b/test/lib/common.sh new file mode 100644 index 0000000..07e533e --- /dev/null +++ b/test/lib/common.sh @@ -0,0 +1,259 @@ +set -E + +. "$(dirname ${BASH_SOURCE[0]})/../../config" +# override the default TMPDIR +TMPDIR="$(mktemp -d /tmp/${0##*/}.XXXXXXXXXX)" +. "$(dirname ${BASH_SOURCE[0]})/../../db-functions" + +signpkg() { + gpg --detach-sign --no-armor --use-agent ${@} || die +} + +oneTimeSetUp() { + local p + local d + local a + local pkgname + local pkgarch + local pkgversion + local build + pkgdir="$(mktemp -d /tmp/${0##*/}.XXXXXXXXXX)" + cp -Lr $(dirname ${BASH_SOURCE[0]})/../packages/* "${pkgdir}" + msg 'Building packages...' + for d in "${pkgdir}"/*; do + pushd $d >/dev/null + pkgarch=($(. PKGBUILD; echo ${arch[@]})) + + if [ "${pkgarch[0]}" == 'any' ]; then + makepkg -cCf || die 'makepkg failed' + else + for a in ${pkgarch[@]}; do + CARCH=${a} makepkg -cCf || die "makepkg failed" + done + fi + popd >/dev/null + done +} + +oneTimeTearDown() { + rm -rf "${pkgdir}" +} + +setUp() { + local p + local pkg + local r + local a + + TMP="$(mktemp -d /tmp/${0##*/}.XXXXXXXXXX)" + + PKGREPOS=('core' 'extra' 'testing') + PKGPOOL='pool/packages' + mkdir -p "${TMP}/"{ftp,tmp,staging,{package,source}-cleanup,svn-packages-{copy,repo}} + + for r in ${PKGREPOS[@]}; do + mkdir -p "${TMP}/staging/${r}" + for a in ${ARCHES[@]}; do + mkdir -p "${TMP}/ftp/${r}/os/${a}" + done + done + mkdir -p "${TMP}/ftp/${PKGPOOL}" + mkdir -p "${TMP}/ftp/${SRCPOOL}" + + msg 'Creating svn repository...' + svnadmin create "${TMP}/svn-packages-repo" + arch_svn checkout -q "file://${TMP}/svn-packages-repo" "${TMP}/svn-packages-copy" + + for p in "${pkgdir}"/*; do + pkg=${p##*/} + mkdir -p "${TMP}/svn-packages-copy/${pkg}"/{trunk,repos} + cp "${p}"/* "${TMP}/svn-packages-copy"/${pkg}/trunk/ + arch_svn add -q "${TMP}/svn-packages-copy"/${pkg} + arch_svn commit -q -m"initial commit of ${pkg}" "${TMP}/svn-packages-copy" + done + + export DBSCRIPTS_CONFIG=${TMP}/config.local + cat < "${DBSCRIPTS_CONFIG}" + FTP_BASE="${TMP}/ftp" + SVNREPO="file://${TMP}/svn-packages-repo" + PKGREPOS=(${PKGREPOS[@]}) + PKGPOOL="${PKGPOOL}" + TESTING_REPO='testing' + STABLE_REPOS=('core' 'extra') + CLEANUP_DESTDIR="${TMP}/package-cleanup" + SOURCE_CLEANUP_DESTDIR="${TMP}/source-cleanup" + STAGING="${TMP}/staging" + TMPDIR="${TMP}/tmp" + CLEANUP_DRYRUN=false + SOURCE_CLEANUP_DRYRUN=false + REQUIRE_SIGNATURE=true +eot + . "$(dirname ${BASH_SOURCE[0]})/../../config" +} + +tearDown() { + rm -rf "${TMP}" + echo +} + +releasePackage() { + local repo=$1 + local pkgbase=$2 + local arch=$3 + local a + local p + local pkgver + local pkgname + + pushd "${TMP}/svn-packages-copy"/${pkgbase}/trunk/ >/dev/null + archrelease ${repo}-${arch} >/dev/null 2>&1 + pkgver=$(. PKGBUILD; echo $(get_full_version ${epoch:-0} ${pkgver} ${pkgrel})) + pkgname=($(. PKGBUILD; echo ${pkgname[@]})) + popd >/dev/null + cp "${pkgdir}/${pkgbase}"/*-${pkgver}-${arch}${PKGEXT} "${STAGING}"/${repo}/ + + if ${REQUIRE_SIGNATURE}; then + for a in ${arch[@]}; do + for p in ${pkgname[@]}; do + signpkg "${STAGING}"/${repo}/${p}-${pkgver}-${a}${PKGEXT} + done + done + fi +} + +checkAnyPackageDB() { + local repo=$1 + local pkg=$2 + local arch + local db + + [ -r "${FTP_BASE}/${PKGPOOL}/${pkg}" ] || fail "${PKGPOOL}/${pkg} not found" + if ${REQUIRE_SIGNATURE}; then + [ -r "${FTP_BASE}/${PKGPOOL}/${pkg}.sig" ] || fail "${PKGPOOL}/${pkg}.sig not found" + fi + + for arch in i686 x86_64; do + [ -L "${FTP_BASE}/${repo}/os/${arch}/${pkg}" ] || fail "${repo}/os/${arch}/${pkg} is not a symlink" + [ "$(readlink -e "${FTP_BASE}/${repo}/os/${arch}/${pkg}")" == "${FTP_BASE}/${PKGPOOL}/${pkg}" ] \ + || fail "${repo}/os/${arch}/${pkg} does not link to ${PKGPOOL}/${pkg}" + + if ${REQUIRE_SIGNATURE}; then + [ -L "${FTP_BASE}/${repo}/os/${arch}/${pkg}.sig" ] || fail "${repo}/os/${arch}/${pkg}.sig is not a symlink" + [ "$(readlink -e "${FTP_BASE}/${repo}/os/${arch}/${pkg}.sig")" == "${FTP_BASE}/${PKGPOOL}/${pkg}.sig" ] \ + || fail "${repo}/os/${arch}/${pkg}.sig does not link to ${PKGPOOL}/${pkg}.sig" + fi + + for db in ${DBEXT} ${FILESEXT}; do + ( [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" ] \ + && bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" -O | grep -q ${pkg}) \ + || fail "${pkg} not in ${repo}/os/${arch}/${repo}${db%.tar.*}" + done + done + [ -r "${STAGING}"/${repo}/${pkg} ] && fail "${repo}/${pkg} found in staging dir" + [ -r "${STAGING}"/${repo}/${pkg}.sig ] && fail "${repo}/${pkg}.sig found in staging dir" +} + +checkAnyPackage() { + local repo=$1 + local pkg=$2 + + checkAnyPackageDB $repo $pkg + + local pkgbase=$(getpkgbase "${FTP_BASE}/${PKGPOOL}/${pkg}") + arch_svn up -q "${TMP}/svn-packages-copy/${pkgbase}" + [ -d "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo}-any" ] \ + || fail "svn-packages-copy/${pkgbase}/repos/${repo}-any does not exist" +} + +checkPackageDB() { + local repo=$1 + local pkg=$2 + local arch=$3 + local db + + [ -r "${FTP_BASE}/${PKGPOOL}/${pkg}" ] || fail "${PKGPOOL}/${pkg} not found" + [ -L "${FTP_BASE}/${repo}/os/${arch}/${pkg}" ] || fail "${repo}/os/${arch}/${pkg} not a symlink" + [ -r "${STAGING}"/${repo}/${pkg} ] && fail "${repo}/${pkg} found in staging dir" + + [ "$(readlink -e "${FTP_BASE}/${repo}/os/${arch}/${pkg}")" == "${FTP_BASE}/${PKGPOOL}/${pkg}" ] \ + || fail "${repo}/os/${arch}/${pkg} does not link to ${PKGPOOL}/${pkg}" + + if ${REQUIRE_SIGNATURE}; then + [ -r "${FTP_BASE}/${PKGPOOL}/${pkg}.sig" ] || fail "${PKGPOOL}/${pkg}.sig not found" + [ -L "${FTP_BASE}/${repo}/os/${arch}/${pkg}.sig" ] || fail "${repo}/os/${arch}/${pkg}.sig is not a symlink" + [ -r "${STAGING}"/${repo}/${pkg}.sig ] && fail "${repo}/${pkg}.sig found in staging dir" + + [ "$(readlink -e "${FTP_BASE}/${repo}/os/${arch}/${pkg}.sig")" == "${FTP_BASE}/${PKGPOOL}/${pkg}.sig" ] \ + || fail "${repo}/os/${arch}/${pkg}.sig does not link to ${PKGPOOL}/${pkg}.sig" + fi + + for db in ${DBEXT} ${FILESEXT}; do + ( [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" ] \ + && bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" -O | grep -q ${pkg}) \ + || fail "${pkg} not in ${repo}/os/${arch}/${repo}${db%.tar.*}" + done +} + +checkPackage() { + local repo=$1 + local pkg=$2 + local arch=$3 + + checkPackageDB $repo $pkg $arch + + local pkgbase=$(getpkgbase "${FTP_BASE}/${PKGPOOL}/${pkg}") + arch_svn up -q "${TMP}/svn-packages-copy/${pkgbase}" + [ -d "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo}-${arch}" ] \ + || fail "svn-packages-copy/${pkgbase}/repos/${repo}-${arch} does not exist" +} + +checkRemovedPackageDB() { + local repo=$1 + local pkgbase=$2 + local arch=$3 + local db + + for db in ${DBEXT} ${FILESEXT}; do + ( [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" ] \ + && bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" -O | grep -q ${pkgbase}) \ + && fail "${pkgbase} should not be in ${repo}/os/${arch}/${repo}${db%.tar.*}" + done +} + +checkRemovedPackage() { + local repo=$1 + local pkgbase=$2 + local arch=$3 + + checkRemovedPackageDB $repo $pkgbase $arch + + arch_svn up -q "${TMP}/svn-packages-copy/${pkgbase}" + [ -d "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo}-${arch}" ] \ + && fail "svn-packages-copy/${pkgbase}/repos/${repo}-${arch} should not exist" +} + +checkRemovedAnyPackageDB() { + local repo=$1 + local pkgbase=$2 + local arch + local db + + for db in ${DBEXT} ${FILESEXT}; do + for arch in i686 x86_64; do + ( [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" ] \ + && bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" -O | grep -q ${pkgbase}) \ + && fail "${pkgbase} should not be in ${repo}/os/${arch}/${repo}${db%.tar.*}" + done + done +} + +checkRemovedAnyPackage() { + local repo=$1 + local pkgbase=$2 + + checkRemovedAnyPackageDB $repo $pkgbase + + arch_svn up -q "${TMP}/svn-packages-copy/${pkgbase}" + [ -d "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo}-any" ] \ + && fail "svn-packages-copy/${pkgbase}/repos/${repo}-any should not exist" +} diff --git a/test/runTest b/test/runTest index adeb84e..ae1fbe5 100755 --- a/test/runTest +++ b/test/runTest @@ -5,7 +5,7 @@ curdir=$(readlink -e $(dirname $0)) testSuite=$(mktemp) trap "rm -rf '${testSuite}'" EXIT -echo ". ${curdir}/lib/common.inc" > ${testSuite} +echo ". ${curdir}/lib/common.sh" > ${testSuite} cat "${curdir}/test.d"/*.sh >> ${testSuite} echo ". ${curdir}/lib/shunit2" >> ${testSuite} -- cgit v1.2.2 From f11014797ff002eaea04c4290115965d2b78082e Mon Sep 17 00:00:00 2001 From: Jelle van der Waa Date: Mon, 20 Mar 2017 19:37:34 +0100 Subject: Disable integrity check for i686 i686 will be deprecated, so this check is no longer required. Signed-off-by: Jelle van der Waa Signed-off-by: Pierre Schmitz --- cron-jobs/integrity-check | 4 ---- 1 file changed, 4 deletions(-) diff --git a/cron-jobs/integrity-check b/cron-jobs/integrity-check index f6c26cf..211a24b 100755 --- a/cron-jobs/integrity-check +++ b/cron-jobs/integrity-check @@ -21,10 +21,6 @@ check() { 2>&1 | ${dirname}/devlist-mailer "Integrity Check ${arch}: ${repos}" "${mailto}" } -repos='core,extra,community' -arch='i686' -check - repos='core,extra,community,multilib' arch='x86_64' check -- cgit v1.2.2 From 4c83281eedf2a27d088f2b1f3a31919983417a74 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sat, 15 Apr 2017 18:24:48 +0200 Subject: Use a minimal travis environment --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 2e3480a..0b0519c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,7 @@ sudo: required +language: bash + services: - docker -- cgit v1.2.2 From 96fdd934bed3dda6feade4a9a2f4f718ce5bcfc6 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sat, 15 Apr 2017 19:40:55 +0200 Subject: testSuite is a simple file --- test/runTest | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/runTest b/test/runTest index ae1fbe5..0953eb3 100755 --- a/test/runTest +++ b/test/runTest @@ -3,7 +3,7 @@ curdir=$(readlink -e $(dirname $0)) testSuite=$(mktemp) -trap "rm -rf '${testSuite}'" EXIT +trap "rm -f '${testSuite}'" EXIT echo ". ${curdir}/lib/common.sh" > ${testSuite} cat "${curdir}/test.d"/*.sh >> ${testSuite} -- cgit v1.2.2 From 62fab76a3ec1ee9caf3abab5eb2c73b8f06fe4ce Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sat, 15 Apr 2017 20:19:15 +0200 Subject: Fix package test namcap does not return a non-zero code on error --- test/packages/pkg-simple-a/PKGBUILD | 1 + test/packages/pkg-simple-b/PKGBUILD | 1 + test/packages/pkg-simple-epoch/PKGBUILD | 1 + test/packages/pkg-split-a/PKGBUILD | 1 + test/packages/pkg-split-b/PKGBUILD | 1 + test/test.d/packages.sh | 7 +++++-- 6 files changed, 10 insertions(+), 2 deletions(-) diff --git a/test/packages/pkg-simple-a/PKGBUILD b/test/packages/pkg-simple-a/PKGBUILD index 25dbed3..870339d 100644 --- a/test/packages/pkg-simple-a/PKGBUILD +++ b/test/packages/pkg-simple-a/PKGBUILD @@ -5,6 +5,7 @@ pkgdesc="A package called ${pkgname}" arch=('i686' 'x86_64') url='http://www.archlinux.org/' license=('GPL') +depends=('glibc') package() { install -D -m755 /bin/true ${pkgdir}/usr/bin/${pkgname} diff --git a/test/packages/pkg-simple-b/PKGBUILD b/test/packages/pkg-simple-b/PKGBUILD index 49bffab..22f7ece 100644 --- a/test/packages/pkg-simple-b/PKGBUILD +++ b/test/packages/pkg-simple-b/PKGBUILD @@ -5,6 +5,7 @@ pkgdesc="A package called ${pkgname}" arch=('i686' 'x86_64') url='http://www.archlinux.org/' license=('GPL') +depends=('glibc') package() { install -D -m755 /bin/true ${pkgdir}/usr/bin/${pkgname} diff --git a/test/packages/pkg-simple-epoch/PKGBUILD b/test/packages/pkg-simple-epoch/PKGBUILD index bdce203..f81d067 100644 --- a/test/packages/pkg-simple-epoch/PKGBUILD +++ b/test/packages/pkg-simple-epoch/PKGBUILD @@ -6,6 +6,7 @@ pkgdesc="A package called ${pkgname}" arch=('i686' 'x86_64') url='http://www.archlinux.org/' license=('GPL') +depends=('glibc') package() { install -D -m755 /bin/true ${pkgdir}/usr/bin/${pkgname} diff --git a/test/packages/pkg-split-a/PKGBUILD b/test/packages/pkg-split-a/PKGBUILD index 9b764db..2acfd92 100644 --- a/test/packages/pkg-split-a/PKGBUILD +++ b/test/packages/pkg-split-a/PKGBUILD @@ -6,6 +6,7 @@ pkgdesc="A split package called ${pkgbase}" arch=('i686' 'x86_64') url='http://www.archlinux.org/' license=('GPL') +depends=('glibc') package_pkg-split-a1() { install -D -m755 /bin/true ${pkgdir}/usr/bin/${pkgname} diff --git a/test/packages/pkg-split-b/PKGBUILD b/test/packages/pkg-split-b/PKGBUILD index 3c05371..70470dd 100644 --- a/test/packages/pkg-split-b/PKGBUILD +++ b/test/packages/pkg-split-b/PKGBUILD @@ -6,6 +6,7 @@ pkgdesc="A split package called ${pkgbase}" arch=('i686' 'x86_64') url='http://www.archlinux.org/' license=('GPL') +#depends=('glibc') package_pkg-split-b1() { install -D -m755 /bin/true ${pkgdir}/usr/bin/${pkgname} diff --git a/test/test.d/packages.sh b/test/test.d/packages.sh index a5e432c..e758c72 100755 --- a/test/test.d/packages.sh +++ b/test/test.d/packages.sh @@ -1,4 +1,7 @@ testPackages() { - # TODO: namcap -r sodepends fails with i686 packages - find "${pkgdir}" -name "*${PKGEXT}" -exec namcap -e sodepends,pkgnameindesc {} + || fail 'namcap failed' + local result + for pkg in $(find "${pkgdir}" -name "*${PKGEXT}"); do + result=$(namcap -e pkgnameindesc ${pkg}) + [[ -n "${result}" ]] && fail "${result}" + done; } -- cgit v1.2.2 From 5156ce59a6fc730a92153bec83603f97939ded3a Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sat, 15 Apr 2017 20:26:27 +0200 Subject: Fix test package dependency --- test/packages/pkg-split-b/PKGBUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/packages/pkg-split-b/PKGBUILD b/test/packages/pkg-split-b/PKGBUILD index 70470dd..8b11f73 100644 --- a/test/packages/pkg-split-b/PKGBUILD +++ b/test/packages/pkg-split-b/PKGBUILD @@ -6,7 +6,7 @@ pkgdesc="A split package called ${pkgbase}" arch=('i686' 'x86_64') url='http://www.archlinux.org/' license=('GPL') -#depends=('glibc') +depends=('glibc') package_pkg-split-b1() { install -D -m755 /bin/true ${pkgdir}/usr/bin/${pkgname} -- cgit v1.2.2 From 1f2a8069ee8a352061adbb317249af2fb04e549a Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Tue, 18 Apr 2017 12:20:37 +0200 Subject: Add ability to generate test coverage --- .gitignore | 1 + Makefile | 14 +++++++++++--- test/Dockerfile | 2 +- test/runTest | 11 ++++++++++- 4 files changed, 23 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index fcc397a..4e3ce99 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ /config.local /test/packages/*/*.pkg.tar.?z /.idea +/coverage diff --git a/Makefile b/Makefile index 3a59211..67beca5 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,15 @@ +IMAGE:=dbscripts/test +RUN_OPTIONS:=--rm --network=none -v $(PWD):/dbscripts:ro --tmpfs=/tmp:exec -w /dbscripts/test + test-image: - docker build --pull -t dbscripts/test test + docker build --pull -t $(IMAGE) test test: test-image - docker run --rm --network=none -v $(PWD):/dbscripts:ro --tmpfs=/tmp:exec -w /dbscripts/test dbscripts/test ./runTest + docker run $(RUN_OPTIONS) $(IMAGE) ./runTest + +test-coverage: test-image + rm -rf ${PWD}/coverage + mkdir ${PWD}/coverage + docker run $(RUN_OPTIONS) -v ${PWD}/coverage:/coverage -e COVERAGE_DIR=/coverage $(IMAGE) ./runTest -.PHONY: test-image test +.PHONY: test-image test test-coverage diff --git a/test/Dockerfile b/test/Dockerfile index 30247bf..ef38d54 100644 --- a/test/Dockerfile +++ b/test/Dockerfile @@ -1,5 +1,5 @@ FROM pierres/archlinux -RUN pacman -Syu --noconfirm --needed base-devel devtools +RUN pacman -Syu --noconfirm --needed base-devel devtools kcov RUN pacman-key --init RUN echo '%wheel ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/wheel RUN useradd -N -g users -G wheel -m tester diff --git a/test/runTest b/test/runTest index 0953eb3..fcec2ff 100755 --- a/test/runTest +++ b/test/runTest @@ -1,6 +1,7 @@ #!/bin/bash curdir=$(readlink -e $(dirname $0)) +projectdir=$(readlink -e ${curdir}/../) testSuite=$(mktemp) trap "rm -f '${testSuite}'" EXIT @@ -9,4 +10,12 @@ echo ". ${curdir}/lib/common.sh" > ${testSuite} cat "${curdir}/test.d"/*.sh >> ${testSuite} echo ". ${curdir}/lib/shunit2" >> ${testSuite} -bash ${testSuite} +if [[ -n "${COVERAGE_DIR}" ]]; then +kcov \ + --include-path=${projectdir} \ + --exclude-path=${projectdir}/test,${projectdir}/cron-jobs/makepkg.conf,${projectdir}/config \ + "${COVERAGE_DIR}" \ + ${testSuite} +else + bash ${testSuite} +fi -- cgit v1.2.2 From 64ae83ea4004e82c0d1faa9f2e8e4d62332991ef Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Tue, 18 Apr 2017 12:56:54 +0200 Subject: Replace bash script with Makefile --- Makefile | 4 ++-- test/Makefile | 20 ++++++++++++++++++++ test/runTest | 21 --------------------- 3 files changed, 22 insertions(+), 23 deletions(-) create mode 100644 test/Makefile delete mode 100755 test/runTest diff --git a/Makefile b/Makefile index 67beca5..a109059 100644 --- a/Makefile +++ b/Makefile @@ -5,11 +5,11 @@ test-image: docker build --pull -t $(IMAGE) test test: test-image - docker run $(RUN_OPTIONS) $(IMAGE) ./runTest + docker run $(RUN_OPTIONS) $(IMAGE) make test test-coverage: test-image rm -rf ${PWD}/coverage mkdir ${PWD}/coverage - docker run $(RUN_OPTIONS) -v ${PWD}/coverage:/coverage -e COVERAGE_DIR=/coverage $(IMAGE) ./runTest + docker run $(RUN_OPTIONS) -v ${PWD}/coverage:/coverage -e COVERAGE_DIR=/coverage $(IMAGE) make test-coverage .PHONY: test-image test test-coverage diff --git a/test/Makefile b/test/Makefile new file mode 100644 index 0000000..0df7569 --- /dev/null +++ b/test/Makefile @@ -0,0 +1,20 @@ +TESTSUITE_FILE:=$(shell mktemp) + +test-suite: + echo "#!/usr/bin/bash" > $(TESTSUITE_FILE) + echo ". $(CURDIR)/lib/common.sh" >> $(TESTSUITE_FILE) + cat "$(CURDIR)/test.d"/*.sh >> $(TESTSUITE_FILE) + echo ". $(CURDIR)/lib/shunit2" >> $(TESTSUITE_FILE) + chmod +x $(TESTSUITE_FILE) + +test: test-suite + $(TESTSUITE_FILE) + +test-coverage: test-suite + kcov \ + --include-path=$(PWD) \ + --exclude-path=$(PWD)/test,$(PWD)/cron-jobs/makepkg.conf,$(PWD)/config \ + $(COVERAGE_DIR) \ + $(TESTSUITE_FILE) + +.PHONY: test-suite test test-coverage diff --git a/test/runTest b/test/runTest deleted file mode 100755 index fcec2ff..0000000 --- a/test/runTest +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash - -curdir=$(readlink -e $(dirname $0)) -projectdir=$(readlink -e ${curdir}/../) - -testSuite=$(mktemp) -trap "rm -f '${testSuite}'" EXIT - -echo ". ${curdir}/lib/common.sh" > ${testSuite} -cat "${curdir}/test.d"/*.sh >> ${testSuite} -echo ". ${curdir}/lib/shunit2" >> ${testSuite} - -if [[ -n "${COVERAGE_DIR}" ]]; then -kcov \ - --include-path=${projectdir} \ - --exclude-path=${projectdir}/test,${projectdir}/cron-jobs/makepkg.conf,${projectdir}/config \ - "${COVERAGE_DIR}" \ - ${testSuite} -else - bash ${testSuite} -fi -- cgit v1.2.2 From 46dd607da7cf84a6f1d0dfcf5104d481c8651665 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Tue, 18 Apr 2017 16:05:52 +0200 Subject: Make test suite independent from code under test --- test/Makefile | 5 ++- test/lib/common.sh | 97 +++++++++++++++++++++-------------------- test/test.d/create-filelists.sh | 1 - test/test.d/db-update.sh | 4 +- test/test.d/testing2x.sh | 2 +- 5 files changed, 56 insertions(+), 53 deletions(-) diff --git a/test/Makefile b/test/Makefile index 0df7569..ea1d157 100644 --- a/test/Makefile +++ b/test/Makefile @@ -2,6 +2,7 @@ TESTSUITE_FILE:=$(shell mktemp) test-suite: echo "#!/usr/bin/bash" > $(TESTSUITE_FILE) + echo ". /usr/share/makepkg/util/pkgbuild.sh" >> $(TESTSUITE_FILE) echo ". $(CURDIR)/lib/common.sh" >> $(TESTSUITE_FILE) cat "$(CURDIR)/test.d"/*.sh >> $(TESTSUITE_FILE) echo ". $(CURDIR)/lib/shunit2" >> $(TESTSUITE_FILE) @@ -12,8 +13,8 @@ test: test-suite test-coverage: test-suite kcov \ - --include-path=$(PWD) \ - --exclude-path=$(PWD)/test,$(PWD)/cron-jobs/makepkg.conf,$(PWD)/config \ + --include-path=$(CURDIR)/../ \ + --exclude-path=$(CURDIR)/../test,$(CURDIR)/../cron-jobs/makepkg.conf,$(CURDIR)/../config \ $(COVERAGE_DIR) \ $(TESTSUITE_FILE) diff --git a/test/lib/common.sh b/test/lib/common.sh index 07e533e..c968a1e 100644 --- a/test/lib/common.sh +++ b/test/lib/common.sh @@ -1,34 +1,19 @@ -set -E - -. "$(dirname ${BASH_SOURCE[0]})/../../config" -# override the default TMPDIR -TMPDIR="$(mktemp -d /tmp/${0##*/}.XXXXXXXXXX)" -. "$(dirname ${BASH_SOURCE[0]})/../../db-functions" - -signpkg() { - gpg --detach-sign --no-armor --use-agent ${@} || die -} - oneTimeSetUp() { - local p local d local a - local pkgname local pkgarch - local pkgversion - local build - pkgdir="$(mktemp -d /tmp/${0##*/}.XXXXXXXXXX)" + + pkgdir="$(mktemp -d)" cp -Lr $(dirname ${BASH_SOURCE[0]})/../packages/* "${pkgdir}" - msg 'Building packages...' for d in "${pkgdir}"/*; do pushd $d >/dev/null pkgarch=($(. PKGBUILD; echo ${arch[@]})) if [ "${pkgarch[0]}" == 'any' ]; then - makepkg -cCf || die 'makepkg failed' + makepkg -cCf || fail 'makepkg failed' else for a in ${pkgarch[@]}; do - CARCH=${a} makepkg -cCf || die "makepkg failed" + CARCH=${a} makepkg -cCf || fail "makepkg failed" done fi popd >/dev/null @@ -45,10 +30,27 @@ setUp() { local r local a - TMP="$(mktemp -d /tmp/${0##*/}.XXXXXXXXXX)" + TMP="$(mktemp -d)" + export DBSCRIPTS_CONFIG=${TMP}/config.local + cat < "${DBSCRIPTS_CONFIG}" + FTP_BASE="${TMP}/ftp" + SVNREPO="file://${TMP}/svn-packages-repo" PKGREPOS=('core' 'extra' 'testing') PKGPOOL='pool/packages' + SRCPOOL='sources/packages' + TESTING_REPO='testing' + STABLE_REPOS=('core' 'extra') + CLEANUP_DESTDIR="${TMP}/package-cleanup" + SOURCE_CLEANUP_DESTDIR="${TMP}/source-cleanup" + STAGING="${TMP}/staging" + TMPDIR="${TMP}/tmp" + CLEANUP_DRYRUN=false + SOURCE_CLEANUP_DRYRUN=false + REQUIRE_SIGNATURE=true +eot + . "$(dirname ${BASH_SOURCE[0]})/../../config" + mkdir -p "${TMP}/"{ftp,tmp,staging,{package,source}-cleanup,svn-packages-{copy,repo}} for r in ${PKGREPOS[@]}; do @@ -60,40 +62,37 @@ setUp() { mkdir -p "${TMP}/ftp/${PKGPOOL}" mkdir -p "${TMP}/ftp/${SRCPOOL}" - msg 'Creating svn repository...' svnadmin create "${TMP}/svn-packages-repo" - arch_svn checkout -q "file://${TMP}/svn-packages-repo" "${TMP}/svn-packages-copy" + svn checkout -q "file://${TMP}/svn-packages-repo" "${TMP}/svn-packages-copy" for p in "${pkgdir}"/*; do pkg=${p##*/} mkdir -p "${TMP}/svn-packages-copy/${pkg}"/{trunk,repos} cp "${p}"/* "${TMP}/svn-packages-copy"/${pkg}/trunk/ - arch_svn add -q "${TMP}/svn-packages-copy"/${pkg} - arch_svn commit -q -m"initial commit of ${pkg}" "${TMP}/svn-packages-copy" + svn add -q "${TMP}/svn-packages-copy"/${pkg} + svn commit -q -m"initial commit of ${pkg}" "${TMP}/svn-packages-copy" done - - export DBSCRIPTS_CONFIG=${TMP}/config.local - cat < "${DBSCRIPTS_CONFIG}" - FTP_BASE="${TMP}/ftp" - SVNREPO="file://${TMP}/svn-packages-repo" - PKGREPOS=(${PKGREPOS[@]}) - PKGPOOL="${PKGPOOL}" - TESTING_REPO='testing' - STABLE_REPOS=('core' 'extra') - CLEANUP_DESTDIR="${TMP}/package-cleanup" - SOURCE_CLEANUP_DESTDIR="${TMP}/source-cleanup" - STAGING="${TMP}/staging" - TMPDIR="${TMP}/tmp" - CLEANUP_DRYRUN=false - SOURCE_CLEANUP_DRYRUN=false - REQUIRE_SIGNATURE=true -eot - . "$(dirname ${BASH_SOURCE[0]})/../../config" } tearDown() { rm -rf "${TMP}" - echo +} + +getpkgbase() { + local _base + _grep_pkginfo() { + local _ret + + _ret="$(/usr/bin/bsdtar -xOqf "$1" .PKGINFO | grep -m 1 "^${2} = ")" + echo "${_ret#${2} = }" + } + + _base="$(_grep_pkginfo "$1" "pkgbase")" + if [ -z "$_base" ]; then + _grep_pkginfo "$1" "pkgname" + else + echo "$_base" + fi } releasePackage() { @@ -105,6 +104,10 @@ releasePackage() { local pkgver local pkgname + signpkg() { + gpg --detach-sign --no-armor --use-agent ${@} || fail 'gpg failed' + } + pushd "${TMP}/svn-packages-copy"/${pkgbase}/trunk/ >/dev/null archrelease ${repo}-${arch} >/dev/null 2>&1 pkgver=$(. PKGBUILD; echo $(get_full_version ${epoch:-0} ${pkgver} ${pkgrel})) @@ -160,7 +163,7 @@ checkAnyPackage() { checkAnyPackageDB $repo $pkg local pkgbase=$(getpkgbase "${FTP_BASE}/${PKGPOOL}/${pkg}") - arch_svn up -q "${TMP}/svn-packages-copy/${pkgbase}" + svn up -q "${TMP}/svn-packages-copy/${pkgbase}" [ -d "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo}-any" ] \ || fail "svn-packages-copy/${pkgbase}/repos/${repo}-any does not exist" } @@ -202,7 +205,7 @@ checkPackage() { checkPackageDB $repo $pkg $arch local pkgbase=$(getpkgbase "${FTP_BASE}/${PKGPOOL}/${pkg}") - arch_svn up -q "${TMP}/svn-packages-copy/${pkgbase}" + svn up -q "${TMP}/svn-packages-copy/${pkgbase}" [ -d "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo}-${arch}" ] \ || fail "svn-packages-copy/${pkgbase}/repos/${repo}-${arch} does not exist" } @@ -227,7 +230,7 @@ checkRemovedPackage() { checkRemovedPackageDB $repo $pkgbase $arch - arch_svn up -q "${TMP}/svn-packages-copy/${pkgbase}" + svn up -q "${TMP}/svn-packages-copy/${pkgbase}" [ -d "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo}-${arch}" ] \ && fail "svn-packages-copy/${pkgbase}/repos/${repo}-${arch} should not exist" } @@ -253,7 +256,7 @@ checkRemovedAnyPackage() { checkRemovedAnyPackageDB $repo $pkgbase - arch_svn up -q "${TMP}/svn-packages-copy/${pkgbase}" + svn up -q "${TMP}/svn-packages-copy/${pkgbase}" [ -d "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo}-any" ] \ && fail "svn-packages-copy/${pkgbase}/repos/${repo}-any should not exist" } diff --git a/test/test.d/create-filelists.sh b/test/test.d/create-filelists.sh index 0010bf3..27158cd 100755 --- a/test/test.d/create-filelists.sh +++ b/test/test.d/create-filelists.sh @@ -94,5 +94,4 @@ testCleanupFileLists() { fail "usr/bin/pkg-simple-a still found in ${arch}/extra${FILESEXT}" fi done - } diff --git a/test/test.d/db-update.sh b/test/test.d/db-update.sh index f9e7d1a..fb1991b 100755 --- a/test/test.d/db-update.sh +++ b/test/test.d/db-update.sh @@ -76,7 +76,7 @@ testUpdateAnyPackage() { pushd "${TMP}/svn-packages-copy/pkg-any-a/trunk/" >/dev/null sed 's/pkgrel=1/pkgrel=2/g' -i PKGBUILD - arch_svn commit -q -m"update pkg to pkgrel=2" >/dev/null + svn commit -q -m"update pkg to pkgrel=2" >/dev/null makepkg -cCf mv pkg-any-a-1-2-any.pkg.tar.xz "${pkgdir}/pkg-any-a/" popd >/dev/null @@ -94,7 +94,7 @@ testUpdateAnyPackageToDifferentRepositoriesAtOnce() { pushd "${TMP}/svn-packages-copy/pkg-any-a/trunk/" >/dev/null sed 's/pkgrel=1/pkgrel=2/g' -i PKGBUILD - arch_svn commit -q -m"update pkg to pkgrel=2" >/dev/null + svn commit -q -m"update pkg to pkgrel=2" >/dev/null makepkg -cCf mv pkg-any-a-1-2-any.pkg.tar.xz "${pkgdir}/pkg-any-a/" popd >/dev/null diff --git a/test/test.d/testing2x.sh b/test/test.d/testing2x.sh index 8fac2ad..0add9fe 100755 --- a/test/test.d/testing2x.sh +++ b/test/test.d/testing2x.sh @@ -4,7 +4,7 @@ testTesting2xAnyPackage() { pushd "${TMP}/svn-packages-copy/pkg-any-a/trunk/" >/dev/null sed 's/pkgrel=1/pkgrel=2/g' -i PKGBUILD - arch_svn commit -q -m"update pkg to pkgrel=2" >/dev/null + svn commit -q -m"update pkg to pkgrel=2" >/dev/null makepkg -cCf mv pkg-any-a-1-2-any.pkg.tar.xz "${pkgdir}/pkg-any-a/" popd >/dev/null -- cgit v1.2.2 From 10439da698c8eb723b1637b803cb37370245a2f8 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Tue, 18 Apr 2017 16:33:21 +0200 Subject: db-repo-add: Fix adding of 'any' packages --- db-repo-add | 4 ++-- test/test.d/db-repo-add.sh | 27 +++++++++++++++++++++++++-- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/db-repo-add b/db-repo-add index 5d5b653..78f387f 100755 --- a/db-repo-add +++ b/db-repo-add @@ -30,8 +30,8 @@ done for tarch in ${tarches[@]}; do for pkgfile in ${pkgfiles[@]}; do - if [[ ! -f "${FTP_BASE}/${repo}/os/${arch}/${pkgfile##*/}" ]]; then - die "Package file ${pkgfile##*/} not found in ${FTP_BASE}/${repo}/os/${arch}/" + if [[ ! -f "${FTP_BASE}/${repo}/os/${tarch}/${pkgfile##*/}" ]]; then + die "Package file ${pkgfile##*/} not found in ${FTP_BASE}/${repo}/os/${tarch}/" else msg "Adding $pkgfile to [$repo]..." fi diff --git a/test/test.d/db-repo-add.sh b/test/test.d/db-repo-add.sh index 90e619b..21ea538 100755 --- a/test/test.d/db-repo-add.sh +++ b/test/test.d/db-repo-add.sh @@ -1,4 +1,4 @@ -testAddSimplePackages() { +testRepoAddSimplePackages() { local arches=('i686' 'x86_64') local pkgs=('pkg-simple-a' 'pkg-simple-b') local pkgbase @@ -21,7 +21,7 @@ testAddSimplePackages() { done } -testAddMultiplePackages() { +testRepoAddMultiplePackages() { local arches=('i686' 'x86_64') local pkgs=('pkg-simple-a' 'pkg-simple-b') local pkgbase @@ -45,3 +45,26 @@ testAddMultiplePackages() { done done } + +testRepoAddAnyPackages() { + local arches=('i686' 'x86_64') + local pkgs=('pkg-any-a' 'pkg-any-b') + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + cp "${pkgdir}/${pkgbase}/${pkgbase}-1-1-any.pkg.tar.xz" "${FTP_BASE}/${PKGPOOL}/" + touch "${FTP_BASE}/${PKGPOOL}/${pkgbase}-1-1-any.pkg.tar.xz.sig" + for arch in ${arches[@]}; do + ln -s "${FTP_BASE}/${PKGPOOL}/${pkgbase}-1-1-any.pkg.tar.xz" "${FTP_BASE}/extra/os/${arch}/" + ln -s "${FTP_BASE}/${PKGPOOL}/${pkgbase}-1-1-any.pkg.tar.xz.sig" "${FTP_BASE}/extra/os/${arch}/" + done + ../db-repo-add extra any ${pkgbase}-1-1-any.pkg.tar.xz + done + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + checkPackageDB extra ${pkgbase}-1-1-any.pkg.tar.xz ${arch} + done + done +} -- cgit v1.2.2 From 5d8d01bef27b358c09267a66411c643bfc78503b Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Tue, 18 Apr 2017 16:58:41 +0200 Subject: db-repo-remove: Test removal of 'any' packages --- test/test.d/db-repo-remove.sh | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/test/test.d/db-repo-remove.sh b/test/test.d/db-repo-remove.sh index 0f294b5..6369251 100755 --- a/test/test.d/db-repo-remove.sh +++ b/test/test.d/db-repo-remove.sh @@ -1,4 +1,4 @@ -testRemovePackages() { +testRepoRemovePackages() { local arches=('i686' 'x86_64') local pkgs=('pkg-simple-a' 'pkg-simple-b' 'pkg-simple-epoch') local pkgbase @@ -25,7 +25,7 @@ testRemovePackages() { done } -testRemoveMultiplePackages() { +testRepoRemoveMultiplePackages() { local arches=('i686' 'x86_64') local pkgs=('pkg-simple-a' 'pkg-simple-b' 'pkg-simple-epoch') local pkgbase @@ -49,3 +49,22 @@ testRemoveMultiplePackages() { done done } + +testRepoRemoveAnyPackages() { + local pkgs=('pkg-any-a' 'pkg-any-b') + local pkgbase + + for pkgbase in ${pkgs[@]}; do + releasePackage extra ${pkgbase} any + done + + ../db-update + + for pkgbase in ${pkgs[@]}; do + ../db-repo-remove extra any ${pkgbase} + done + + for pkgbase in ${pkgs[@]}; do + checkRemovedAnyPackageDB extra ${pkgbase} + done +} -- cgit v1.2.2 From 41f4b54c56702610e23ba9130b2022c3f0fe5961 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Tue, 18 Apr 2017 17:09:21 +0200 Subject: db-update: Check that packages with invalid signatures wont get added --- test/test.d/signed-packages.sh | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/test/test.d/signed-packages.sh b/test/test.d/signed-packages.sh index 148929f..b18f5d5 100755 --- a/test/test.d/signed-packages.sh +++ b/test/test.d/signed-packages.sh @@ -3,13 +3,15 @@ testAddSignedPackage() { ../db-update || fail "db-update failed!" } -testAddUnsignedPackage() { +testAddUnsignedPackageFails() { releasePackage extra 'pkg-simple-a' 'i686' rm "${STAGING}"/extra/*.sig ../db-update >/dev/null 2>&1 && fail "db-update should fail when a signature is missing!" + + checkRemovedPackage extra pkg-simple-a-1-1-i686.pkg.tar.xz i686 } -testAddInvalidSignedPackage() { +testAddInvalidSignedPackageFails() { local p releasePackage extra 'pkg-simple-a' 'i686' for p in "${STAGING}"/extra/*${PKGEXT}; do @@ -17,13 +19,17 @@ testAddInvalidSignedPackage() { xz -0 ${p%%.xz} done ../db-update >/dev/null 2>&1 && fail "db-update should fail when a signature is invalid!" + + checkRemovedPackage extra pkg-simple-a-1-1-i686.pkg.tar.xz i686 } -testAddBrokenSignature() { +testAddBrokenSignatureFails() { local s releasePackage extra 'pkg-simple-a' 'i686' for s in "${STAGING}"/extra/*.sig; do echo 0 > $s done ../db-update >/dev/null 2>&1 && fail "db-update should fail when a signature is broken!" + + checkRemovedPackage extra pkg-simple-a-1-1-i686.pkg.tar.xz i686 } -- cgit v1.2.2 From ec7d4fdced1aac8116d68427a8606c9f27b06b41 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Tue, 18 Apr 2017 17:20:28 +0200 Subject: Signing packages is no longer optional --- config | 2 -- db-functions | 4 ++-- db-update | 2 +- test/lib/common.sh | 35 +++++++++++++---------------------- test/test.d/create-filelists.sh | 1 - test/test.d/db-update.sh | 31 +++++++++++++++++++++++++++++++ test/test.d/signed-packages.sh | 35 ----------------------------------- 7 files changed, 47 insertions(+), 63 deletions(-) delete mode 100755 test/test.d/signed-packages.sh diff --git a/config b/config index e5440d2..6d76ef3 100644 --- a/config +++ b/config @@ -17,8 +17,6 @@ SOURCE_CLEANUP_DRYRUN=false # Time in days to keep moved sourcepackages SOURCE_CLEANUP_KEEP=14 -REQUIRE_SIGNATURE=true - LOCK_DELAY=10 LOCK_TIMEOUT=300 diff --git a/db-functions b/db-functions index 3101c0c..2ce7811 100644 --- a/db-functions +++ b/db-functions @@ -324,7 +324,7 @@ getpkgfile() { elif [ ! -f "${1}" ]; then error "Package ${1} not found!" exit 1 - elif ${REQUIRE_SIGNATURE} && [ ! -f "${1}.sig" ]; then + elif [ ! -f "${1}.sig" ]; then error "Package signature ${1}.sig not found!" exit 1 fi @@ -343,7 +343,7 @@ getpkgfiles() { if [ ! -f "${f}" ]; then error "Package ${f} not found!" exit 1 - elif ${REQUIRE_SIGNATURE} && [ ! -f "${f}.sig" ]; then + elif [ ! -f "${f}.sig" ]; then error "Package signature ${f}.sig not found!" exit 1 fi diff --git a/db-update b/db-update index db8cb91..54e60cb 100755 --- a/db-update +++ b/db-update @@ -42,7 +42,7 @@ for repo in ${repos[@]}; do if ! check_pkgfile "${pkg}"; then die "Package ${repo}/${pkg##*/} is not consistent with its meta data" fi - if ${REQUIRE_SIGNATURE} && ! pacman-key -v "${pkg}.sig" >/dev/null 2>&1; then + if ! pacman-key -v "${pkg}.sig" >/dev/null 2>&1; then die "Package ${repo}/${pkg##*/} does not have a valid signature" fi if ! check_pkgsvn "${pkg}" "${repo}"; then diff --git a/test/lib/common.sh b/test/lib/common.sh index c968a1e..7ac03ec 100644 --- a/test/lib/common.sh +++ b/test/lib/common.sh @@ -47,7 +47,6 @@ setUp() { TMPDIR="${TMP}/tmp" CLEANUP_DRYRUN=false SOURCE_CLEANUP_DRYRUN=false - REQUIRE_SIGNATURE=true eot . "$(dirname ${BASH_SOURCE[0]})/../../config" @@ -115,13 +114,11 @@ releasePackage() { popd >/dev/null cp "${pkgdir}/${pkgbase}"/*-${pkgver}-${arch}${PKGEXT} "${STAGING}"/${repo}/ - if ${REQUIRE_SIGNATURE}; then - for a in ${arch[@]}; do - for p in ${pkgname[@]}; do - signpkg "${STAGING}"/${repo}/${p}-${pkgver}-${a}${PKGEXT} - done + for a in ${arch[@]}; do + for p in ${pkgname[@]}; do + signpkg "${STAGING}"/${repo}/${p}-${pkgver}-${a}${PKGEXT} done - fi + done } checkAnyPackageDB() { @@ -131,20 +128,16 @@ checkAnyPackageDB() { local db [ -r "${FTP_BASE}/${PKGPOOL}/${pkg}" ] || fail "${PKGPOOL}/${pkg} not found" - if ${REQUIRE_SIGNATURE}; then - [ -r "${FTP_BASE}/${PKGPOOL}/${pkg}.sig" ] || fail "${PKGPOOL}/${pkg}.sig not found" - fi + [ -r "${FTP_BASE}/${PKGPOOL}/${pkg}.sig" ] || fail "${PKGPOOL}/${pkg}.sig not found" for arch in i686 x86_64; do [ -L "${FTP_BASE}/${repo}/os/${arch}/${pkg}" ] || fail "${repo}/os/${arch}/${pkg} is not a symlink" [ "$(readlink -e "${FTP_BASE}/${repo}/os/${arch}/${pkg}")" == "${FTP_BASE}/${PKGPOOL}/${pkg}" ] \ || fail "${repo}/os/${arch}/${pkg} does not link to ${PKGPOOL}/${pkg}" - if ${REQUIRE_SIGNATURE}; then - [ -L "${FTP_BASE}/${repo}/os/${arch}/${pkg}.sig" ] || fail "${repo}/os/${arch}/${pkg}.sig is not a symlink" - [ "$(readlink -e "${FTP_BASE}/${repo}/os/${arch}/${pkg}.sig")" == "${FTP_BASE}/${PKGPOOL}/${pkg}.sig" ] \ - || fail "${repo}/os/${arch}/${pkg}.sig does not link to ${PKGPOOL}/${pkg}.sig" - fi + [ -L "${FTP_BASE}/${repo}/os/${arch}/${pkg}.sig" ] || fail "${repo}/os/${arch}/${pkg}.sig is not a symlink" + [ "$(readlink -e "${FTP_BASE}/${repo}/os/${arch}/${pkg}.sig")" == "${FTP_BASE}/${PKGPOOL}/${pkg}.sig" ] \ + || fail "${repo}/os/${arch}/${pkg}.sig does not link to ${PKGPOOL}/${pkg}.sig" for db in ${DBEXT} ${FILESEXT}; do ( [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" ] \ @@ -181,14 +174,12 @@ checkPackageDB() { [ "$(readlink -e "${FTP_BASE}/${repo}/os/${arch}/${pkg}")" == "${FTP_BASE}/${PKGPOOL}/${pkg}" ] \ || fail "${repo}/os/${arch}/${pkg} does not link to ${PKGPOOL}/${pkg}" - if ${REQUIRE_SIGNATURE}; then - [ -r "${FTP_BASE}/${PKGPOOL}/${pkg}.sig" ] || fail "${PKGPOOL}/${pkg}.sig not found" - [ -L "${FTP_BASE}/${repo}/os/${arch}/${pkg}.sig" ] || fail "${repo}/os/${arch}/${pkg}.sig is not a symlink" - [ -r "${STAGING}"/${repo}/${pkg}.sig ] && fail "${repo}/${pkg}.sig found in staging dir" + [ -r "${FTP_BASE}/${PKGPOOL}/${pkg}.sig" ] || fail "${PKGPOOL}/${pkg}.sig not found" + [ -L "${FTP_BASE}/${repo}/os/${arch}/${pkg}.sig" ] || fail "${repo}/os/${arch}/${pkg}.sig is not a symlink" + [ -r "${STAGING}"/${repo}/${pkg}.sig ] && fail "${repo}/${pkg}.sig found in staging dir" - [ "$(readlink -e "${FTP_BASE}/${repo}/os/${arch}/${pkg}.sig")" == "${FTP_BASE}/${PKGPOOL}/${pkg}.sig" ] \ - || fail "${repo}/os/${arch}/${pkg}.sig does not link to ${PKGPOOL}/${pkg}.sig" - fi + [ "$(readlink -e "${FTP_BASE}/${repo}/os/${arch}/${pkg}.sig")" == "${FTP_BASE}/${PKGPOOL}/${pkg}.sig" ] \ + || fail "${repo}/os/${arch}/${pkg}.sig does not link to ${PKGPOOL}/${pkg}.sig" for db in ${DBEXT} ${FILESEXT}; do ( [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" ] \ diff --git a/test/test.d/create-filelists.sh b/test/test.d/create-filelists.sh index 27158cd..6b55a0b 100755 --- a/test/test.d/create-filelists.sh +++ b/test/test.d/create-filelists.sh @@ -68,7 +68,6 @@ testCreateSplitFileLists() { done } - testCleanupFileLists() { local arches=('i686' 'x86_64') local pkgs=('pkg-simple-a' 'pkg-simple-b') diff --git a/test/test.d/db-update.sh b/test/test.d/db-update.sh index fb1991b..77a712b 100755 --- a/test/test.d/db-update.sh +++ b/test/test.d/db-update.sh @@ -166,3 +166,34 @@ testUnknownRepo() { [ -e "${FTP_BASE}/unknown" ] && fail "db-update pushed a package into an unknown repository" rm -rf "${STAGING}/unknown/" } + +testAddUnsignedPackageFails() { + releasePackage extra 'pkg-simple-a' 'i686' + rm "${STAGING}"/extra/*.sig + ../db-update >/dev/null 2>&1 && fail "db-update should fail when a signature is missing!" + + checkRemovedPackage extra pkg-simple-a-1-1-i686.pkg.tar.xz i686 +} + +testAddInvalidSignedPackageFails() { + local p + releasePackage extra 'pkg-simple-a' 'i686' + for p in "${STAGING}"/extra/*${PKGEXT}; do + unxz $p + xz -0 ${p%%.xz} + done + ../db-update >/dev/null 2>&1 && fail "db-update should fail when a signature is invalid!" + + checkRemovedPackage extra pkg-simple-a-1-1-i686.pkg.tar.xz i686 +} + +testAddBrokenSignatureFails() { + local s + releasePackage extra 'pkg-simple-a' 'i686' + for s in "${STAGING}"/extra/*.sig; do + echo 0 > $s + done + ../db-update >/dev/null 2>&1 && fail "db-update should fail when a signature is broken!" + + checkRemovedPackage extra pkg-simple-a-1-1-i686.pkg.tar.xz i686 +} diff --git a/test/test.d/signed-packages.sh b/test/test.d/signed-packages.sh deleted file mode 100755 index b18f5d5..0000000 --- a/test/test.d/signed-packages.sh +++ /dev/null @@ -1,35 +0,0 @@ -testAddSignedPackage() { - releasePackage extra 'pkg-simple-a' 'i686' - ../db-update || fail "db-update failed!" -} - -testAddUnsignedPackageFails() { - releasePackage extra 'pkg-simple-a' 'i686' - rm "${STAGING}"/extra/*.sig - ../db-update >/dev/null 2>&1 && fail "db-update should fail when a signature is missing!" - - checkRemovedPackage extra pkg-simple-a-1-1-i686.pkg.tar.xz i686 -} - -testAddInvalidSignedPackageFails() { - local p - releasePackage extra 'pkg-simple-a' 'i686' - for p in "${STAGING}"/extra/*${PKGEXT}; do - unxz $p - xz -0 ${p%%.xz} - done - ../db-update >/dev/null 2>&1 && fail "db-update should fail when a signature is invalid!" - - checkRemovedPackage extra pkg-simple-a-1-1-i686.pkg.tar.xz i686 -} - -testAddBrokenSignatureFails() { - local s - releasePackage extra 'pkg-simple-a' 'i686' - for s in "${STAGING}"/extra/*.sig; do - echo 0 > $s - done - ../db-update >/dev/null 2>&1 && fail "db-update should fail when a signature is broken!" - - checkRemovedPackage extra pkg-simple-a-1-1-i686.pkg.tar.xz i686 -} -- cgit v1.2.2 From 961c9407982315ba52f5fa4371b20e0db95e7cdb Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Tue, 18 Apr 2017 17:45:27 +0200 Subject: ftpdir-cleanup: Test removal of old packages --- test/test.d/ftpdir-cleanup.sh | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/test/test.d/ftpdir-cleanup.sh b/test/test.d/ftpdir-cleanup.sh index 73779a2..f29e2e3 100755 --- a/test/test.d/ftpdir-cleanup.sh +++ b/test/test.d/ftpdir-cleanup.sh @@ -112,3 +112,37 @@ testCleanupSplitPackages() { done done } + +testCleanupOldPackages() { + local arches=('i686' 'x86_64') + local pkgs=('pkg-simple-a' 'pkg-simple-b') + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + releasePackage extra ${pkgbase} ${arch} + done + done + + ../db-update + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + ../db-remove extra ${arch} ${pkgbase} + done + done + + ../cron-jobs/ftpdir-cleanup >/dev/null + + local pkgfilea="pkg-simple-a-1-1-${arch}.pkg.tar.xz" + local pkgfileb="pkg-simple-b-1-1-${arch}.pkg.tar.xz" + for arch in ${arches[@]}; do + touch -d "-$(expr ${CLEANUP_KEEP} + 1)days" ${CLEANUP_DESTDIR}/${pkgfilea}{,.sig} + done + + ../cron-jobs/ftpdir-cleanup >/dev/null + + [ -f ${CLEANUP_DESTDIR}/${pkgfilea} ] && fail "${pkgfilea} was not removed" + [ -f ${CLEANUP_DESTDIR}/${pkgfileb} ] || fail "${pkgfileb} was removed" +} -- cgit v1.2.2 From e84b36059f82edb6d5f9eec1c99695cc80e6fb82 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Tue, 18 Apr 2017 18:13:01 +0200 Subject: db-update: Disallow invalid package names --- db-functions | 2 +- test/test.d/db-update.sh | 25 ++++++++++++++++++++++++- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/db-functions b/db-functions index 2ce7811..521924b 100644 --- a/db-functions +++ b/db-functions @@ -364,7 +364,7 @@ check_pkgfile() { in_array "${pkgarch}" ${ARCHES[@]} 'any' || return 1 - if echo "${pkgfile##*/}" | grep -q "${pkgname}-${pkgver}-${pkgarch}"; then + if echo "${pkgfile##*/}" | grep -q "^${pkgname}-${pkgver}-${pkgarch}"; then return 0 else return 1 diff --git a/test/test.d/db-update.sh b/test/test.d/db-update.sh index 77a712b..e789e6f 100755 --- a/test/test.d/db-update.sh +++ b/test/test.d/db-update.sh @@ -134,7 +134,6 @@ testUpdateSameAnyPackageToDifferentRepositories() { done } - testAddIncompleteSplitPackage() { local arches=('i686' 'x86_64') local repo='extra' @@ -197,3 +196,27 @@ testAddBrokenSignatureFails() { checkRemovedPackage extra pkg-simple-a-1-1-i686.pkg.tar.xz i686 } + +testAddPackageWithInconsistentVersionFails() { + local p + releasePackage extra 'pkg-simple-a' 'i686' + + for p in "${STAGING}"/extra/*; do + mv "${p}" "${p/1/2}" + done + + ../db-update >/dev/null 2>&1 && fail "db-update should fail when a package is not consistent!" + checkRemovedPackage extra 'pkg-simple-a-2-1-i686.pkg.tar.xz' 'i686' +} + +testAddPackageWithInconsistentNameFails() { + local p + releasePackage extra 'pkg-simple-a' 'i686' + + for p in "${STAGING}"/extra/*; do + mv "${p}" "${p/pkg-/foo-pkg-}" + done + + ../db-update >/dev/null 2>&1 && fail "db-update should fail when a package is not consistent!" + checkRemovedPackage extra 'foo-pkg-simple-a-1-1-i686.pkg.tar.xz' 'i686' +} -- cgit v1.2.2 From 32d99fb5c287d2dc3bb35b41aa471e9f48ea7910 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Tue, 18 Apr 2017 18:23:17 +0200 Subject: db-update: Test if package cannot be added if svn is inconsistent --- test/test.d/db-update.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/test/test.d/db-update.sh b/test/test.d/db-update.sh index e789e6f..c691c40 100755 --- a/test/test.d/db-update.sh +++ b/test/test.d/db-update.sh @@ -220,3 +220,15 @@ testAddPackageWithInconsistentNameFails() { ../db-update >/dev/null 2>&1 && fail "db-update should fail when a package is not consistent!" checkRemovedPackage extra 'foo-pkg-simple-a-1-1-i686.pkg.tar.xz' 'i686' } + +testAddPackageWithInconsistentSVNFails() { + releasePackage extra 'pkg-simple-a' 'i686' + + pushd "${TMP}/svn-packages-copy/pkg-simple-a/repos/extra-i686" >/dev/null + sed 's/pkgrel=1/pkgrel=2/g' -i PKGBUILD + svn commit -q -m"update pkg to pkgrel=2" >/dev/null + popd >/dev/null + + ../db-update >/dev/null 2>&1 && fail "db-update should fail when a package is not consistent!" + checkRemovedPackage extra 'foo-pkg-simple-a-1-1-i686.pkg.tar.xz' 'i686' +} -- cgit v1.2.2 From c33e9409cbec408222875f8a71d324362b9cb043 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Tue, 18 Apr 2017 18:49:23 +0200 Subject: db-update: Test package validation --- test/test.d/db-update.sh | 36 +++++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/test/test.d/db-update.sh b/test/test.d/db-update.sh index c691c40..f0a8416 100755 --- a/test/test.d/db-update.sh +++ b/test/test.d/db-update.sh @@ -230,5 +230,39 @@ testAddPackageWithInconsistentSVNFails() { popd >/dev/null ../db-update >/dev/null 2>&1 && fail "db-update should fail when a package is not consistent!" - checkRemovedPackage extra 'foo-pkg-simple-a-1-1-i686.pkg.tar.xz' 'i686' + checkRemovedPackage extra 'pkg-simple-a-1-1-i686.pkg.tar.xz' 'i686' +} + +testAddPackageWithInsufficientPermissionsFails() +{ + releasePackage core 'pkg-simple-a' 'i686' + releasePackage extra 'pkg-simple-b' 'i686' + + chmod -xwr ${FTP_BASE}/core/os/i686 + ../db-update >/dev/null 2>&1 && fail "db-update should fail when permissions are insufficient!" + chmod +xwr ${FTP_BASE}/core/os/i686 + + checkRemovedPackage core 'pkg-simple-a-1-1-i686.pkg.tar.xz' 'i686' + checkRemovedPackage extra 'pkg-simple-b-1-1-i686.pkg.tar.xz' 'i686' +} + +testPackageHasToBeARegularFile() +{ + local p + local target=$(mktemp -d) + local arches=('i686' 'x86_64') + + for arch in ${arches[@]}; do + releasePackage extra 'pkg-simple-a' $arch + done + + for p in "${STAGING}"/extra/*i686*; do + mv "${p}" "${target}" + ln -s "${target}/${p##*/}" "${p}" + done + + ../db-update >/dev/null 2>&1 && fail "db-update should fail when a package is a symlink!" + for arch in ${arches[@]}; do + checkRemovedPackage extra "pkg-simple-a-1-1-${arch}.pkg.tar.xz" $arch + done } -- cgit v1.2.2 From 114a8817003e582555a0a51fe5742ceeb207a6ed Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Tue, 18 Apr 2017 19:01:08 +0200 Subject: Make coverage folder writeable to the test user --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index a109059..988d335 100644 --- a/Makefile +++ b/Makefile @@ -9,7 +9,7 @@ test: test-image test-coverage: test-image rm -rf ${PWD}/coverage - mkdir ${PWD}/coverage + mkdir -m 777 ${PWD}/coverage docker run $(RUN_OPTIONS) -v ${PWD}/coverage:/coverage -e COVERAGE_DIR=/coverage $(IMAGE) make test-coverage .PHONY: test-image test test-coverage -- cgit v1.2.2 From 1e8a10b3c8cef45513eb76c557f35dc8dd918334 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Tue, 18 Apr 2017 19:11:19 +0200 Subject: Avoid replacing the temporary dir by accident --- test/test.d/db-update.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test.d/db-update.sh b/test/test.d/db-update.sh index f0a8416..3c9765a 100755 --- a/test/test.d/db-update.sh +++ b/test/test.d/db-update.sh @@ -202,7 +202,7 @@ testAddPackageWithInconsistentVersionFails() { releasePackage extra 'pkg-simple-a' 'i686' for p in "${STAGING}"/extra/*; do - mv "${p}" "${p/1/2}" + mv "${p}" "${p/pkg-simple-a-1/pkg-simple-a-2}" done ../db-update >/dev/null 2>&1 && fail "db-update should fail when a package is not consistent!" -- cgit v1.2.2 From 625fa02f6ae5642be95f20e4ed482cf07fa75420 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Tue, 18 Apr 2017 20:20:49 +0200 Subject: use makepkg library instead of local function copies --- db-functions | 71 ++++-------------------------------------------------- db-move | 4 +-- test/lib/common.sh | 2 +- 3 files changed, 8 insertions(+), 69 deletions(-) diff --git a/db-functions b/db-functions index 521924b..45e8d38 100644 --- a/db-functions +++ b/db-functions @@ -1,5 +1,9 @@ #!/bin/bash +. /usr/share/makepkg/util/message.sh +. /usr/share/makepkg/util/util.sh +. /usr/share/makepkg/util/pkgbuild.sh + # Some PKGBUILDs need CARCH to be set CARCH="x86_64" @@ -34,71 +38,6 @@ fi LOCKS=() REPO_MODIFIED=0 -# check if messages are to be printed using color -unset ALL_OFF BOLD BLUE GREEN RED YELLOW -if [[ -t 2 ]]; then - ALL_OFF="$(tput sgr0)" - BOLD="$(tput bold)" - BLUE="${BOLD}$(tput setaf 4)" - GREEN="${BOLD}$(tput setaf 2)" - RED="${BOLD}$(tput setaf 1)" - YELLOW="${BOLD}$(tput setaf 3)" -fi -readonly ALL_OFF BOLD BLUE GREEN RED YELLOW - -plain() { - local mesg=$1; shift - printf "${BOLD} ${mesg}${ALL_OFF}\n" "$@" -} - -msg() { - local mesg=$1; shift - printf "${GREEN}==>${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" -} - -msg2() { - local mesg=$1; shift - printf "${BLUE} ->${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" -} - -warning() { - local mesg=$1; shift - printf "${YELLOW}==> WARNING:${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2 -} - -error() { - local mesg=$1; shift - printf "${RED}==> ERROR${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2 -} - -## -# usage : in_array( $needle, $haystack ) -# return : 0 - found -# 1 - not found -## -in_array() { - local needle=$1; shift - [[ -z $1 ]] && return 1 # Not Found - local item - for item in "$@"; do - [[ $item = $needle ]] && return 0 # Found - done - return 1 # Not Found -} - -## -# usage : get_full_version( $epoch, $pkgver, $pkgrel ) -# return : full version spec, including epoch (if necessary), pkgver, pkgrel -## -get_full_version() { - if [[ $1 -eq 0 ]]; then - # zero epoch case, don't include it in version - echo $2-$3 - else - echo $1:$2-$3 - fi -} - script_lock() { local LOCKDIR="$TMPDIR/.scriptlock.${0##*/}" if ! mkdir "$LOCKDIR" >/dev/null 2>&1 ; then @@ -392,7 +331,7 @@ check_pkgsvn() { [ $? -ge 1 ] && return 1 fi - local svnver="$(. "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}"; echo $(get_full_version ${epoch:-0} ${pkgver} ${pkgrel}) )" + local svnver="$(. "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}"; get_full_version)" [ "${svnver}" == "${_pkgver}" ] || return 1 local svnnames=($(. "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}"; echo ${pkgname[@]})) diff --git a/db-move b/db-move index 1fa44d4..ce3fcf9 100755 --- a/db-move +++ b/db-move @@ -36,7 +36,7 @@ for pkgbase in ${args[@]:2}; do die "Could not read pkgname" fi - pkgver=$(. "${svnrepo_from}/PKGBUILD"; echo $(get_full_version ${epoch:-0} ${pkgver} ${pkgrel})) + pkgver=$(. "${svnrepo_from}/PKGBUILD"; get_full_version) if [ -z "${pkgver}" ]; then die "Could not read pkgver" fi @@ -76,7 +76,7 @@ for pkgbase in ${args[@]:2}; do fi msg2 "${pkgbase} ($(echo ${tarches[@]}))" pkgnames=($(. "${svnrepo_from}/PKGBUILD"; echo ${pkgname[@]})) - pkgver=$(. "${svnrepo_from}/PKGBUILD"; echo $(get_full_version ${epoch:-0} ${pkgver} ${pkgrel})) + pkgver=$(. "${svnrepo_from}/PKGBUILD"; get_full_version) if [ -d "${svnrepo_to}" ]; then for file in $(arch_svn ls "${svnrepo_to}"); do diff --git a/test/lib/common.sh b/test/lib/common.sh index 7ac03ec..3513ab7 100644 --- a/test/lib/common.sh +++ b/test/lib/common.sh @@ -109,7 +109,7 @@ releasePackage() { pushd "${TMP}/svn-packages-copy"/${pkgbase}/trunk/ >/dev/null archrelease ${repo}-${arch} >/dev/null 2>&1 - pkgver=$(. PKGBUILD; echo $(get_full_version ${epoch:-0} ${pkgver} ${pkgrel})) + pkgver=$(. PKGBUILD; get_full_version) pkgname=($(. PKGBUILD; echo ${pkgname[@]})) popd >/dev/null cp "${pkgdir}/${pkgbase}"/*-${pkgver}-${arch}${PKGEXT} "${STAGING}"/${repo}/ -- cgit v1.2.2 From 51aa6217229ad9164122e904b184c673db190216 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Tue, 18 Apr 2017 22:25:36 +0200 Subject: Move package build and update functions to common library --- test/lib/common.sh | 75 ++++++++++++++++++++++++++++++++++++------------ test/test.d/db-update.sh | 25 +++------------- test/test.d/testing2x.sh | 8 +----- 3 files changed, 62 insertions(+), 46 deletions(-) diff --git a/test/lib/common.sh b/test/lib/common.sh index 3513ab7..052312a 100644 --- a/test/lib/common.sh +++ b/test/lib/common.sh @@ -1,21 +1,11 @@ oneTimeSetUp() { local d - local a - local pkgarch pkgdir="$(mktemp -d)" cp -Lr $(dirname ${BASH_SOURCE[0]})/../packages/* "${pkgdir}" for d in "${pkgdir}"/*; do pushd $d >/dev/null - pkgarch=($(. PKGBUILD; echo ${arch[@]})) - - if [ "${pkgarch[0]}" == 'any' ]; then - makepkg -cCf || fail 'makepkg failed' - else - for a in ${pkgarch[@]}; do - CARCH=${a} makepkg -cCf || fail "makepkg failed" - done - fi + buildPackage popd >/dev/null done } @@ -103,22 +93,71 @@ releasePackage() { local pkgver local pkgname - signpkg() { - gpg --detach-sign --no-armor --use-agent ${@} || fail 'gpg failed' - } - pushd "${TMP}/svn-packages-copy"/${pkgbase}/trunk/ >/dev/null archrelease ${repo}-${arch} >/dev/null 2>&1 pkgver=$(. PKGBUILD; get_full_version) pkgname=($(. PKGBUILD; echo ${pkgname[@]})) - popd >/dev/null - cp "${pkgdir}/${pkgbase}"/*-${pkgver}-${arch}${PKGEXT} "${STAGING}"/${repo}/ for a in ${arch[@]}; do for p in ${pkgname[@]}; do - signpkg "${STAGING}"/${repo}/${p}-${pkgver}-${a}${PKGEXT} + cp ${p}-${pkgver}-${a}${PKGEXT}{,.sig} "${STAGING}"/${repo}/ done done + + popd >/dev/null +} + +buildPackage() { + local pkgarch + local pkgver + local pkgname + local a + local p + + pkgname=($(. PKGBUILD; echo ${pkgname[@]})) + pkgver=$(. PKGBUILD; get_full_version) + pkgarch=($(. PKGBUILD; echo ${arch[@]})) + + if [ "${pkgarch[0]}" == 'any' ]; then + makepkg -cCf || fail 'makepkg failed' + else + for a in ${pkgarch[@]}; do + CARCH=${a} makepkg -cCf || fail "makepkg failed" + done + fi + + for a in ${pkgarch[@]}; do + for p in ${pkgname[@]}; do + gpg --detach-sign --no-armor --use-agent ${p}-${pkgver}-${a}* || fail 'gpg failed' + done + done +} + +__updatePKGBUILD() { + local pkgrel + + pkgrel=$(. PKGBUILD; expr ${pkgrel} + 1) + sed "s/pkgrel=.*/pkgrel=${pkgrel}/" -i PKGBUILD + svn commit -q -m"update pkg to pkgrel=${pkgrel}" >/dev/null +} + +updatePackage() { + local pkgbase=$1 + + pushd "${TMP}/svn-packages-copy/${pkgbase}/trunk/" >/dev/null + __updatePKGBUILD + buildPackage + popd >/dev/null +} + +updateRepoPKGBUILD() { + local pkgbase=$1 + local repo=$2 + local arch=$3 + + pushd "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo}-${arch}/" >/dev/null + __updatePKGBUILD + popd >/dev/null } checkAnyPackageDB() { diff --git a/test/test.d/db-update.sh b/test/test.d/db-update.sh index 3c9765a..e9e5958 100755 --- a/test/test.d/db-update.sh +++ b/test/test.d/db-update.sh @@ -74,30 +74,18 @@ testUpdateAnyPackage() { releasePackage extra pkg-any-a any ../db-update - pushd "${TMP}/svn-packages-copy/pkg-any-a/trunk/" >/dev/null - sed 's/pkgrel=1/pkgrel=2/g' -i PKGBUILD - svn commit -q -m"update pkg to pkgrel=2" >/dev/null - makepkg -cCf - mv pkg-any-a-1-2-any.pkg.tar.xz "${pkgdir}/pkg-any-a/" - popd >/dev/null + updatePackage pkg-any-a releasePackage extra pkg-any-a any ../db-update checkAnyPackage extra pkg-any-a-1-2-any.pkg.tar.xz any - - rm -f "${pkgdir}/pkg-any-a/pkg-any-a-1-2-any.pkg.tar.xz" } testUpdateAnyPackageToDifferentRepositoriesAtOnce() { releasePackage extra pkg-any-a any - pushd "${TMP}/svn-packages-copy/pkg-any-a/trunk/" >/dev/null - sed 's/pkgrel=1/pkgrel=2/g' -i PKGBUILD - svn commit -q -m"update pkg to pkgrel=2" >/dev/null - makepkg -cCf - mv pkg-any-a-1-2-any.pkg.tar.xz "${pkgdir}/pkg-any-a/" - popd >/dev/null + updatePackage pkg-any-a releasePackage testing pkg-any-a any @@ -105,8 +93,6 @@ testUpdateAnyPackageToDifferentRepositoriesAtOnce() { checkAnyPackage extra pkg-any-a-1-1-any.pkg.tar.xz any checkAnyPackage testing pkg-any-a-1-2-any.pkg.tar.xz any - - rm -f "${pkgdir}/pkg-any-a/pkg-any-a-1-2-any.pkg.tar.xz" } testUpdateSameAnyPackageToSameRepository() { @@ -221,13 +207,10 @@ testAddPackageWithInconsistentNameFails() { checkRemovedPackage extra 'foo-pkg-simple-a-1-1-i686.pkg.tar.xz' 'i686' } -testAddPackageWithInconsistentSVNFails() { +testAddPackageWithInconsistentPKGBUILDFails() { releasePackage extra 'pkg-simple-a' 'i686' - pushd "${TMP}/svn-packages-copy/pkg-simple-a/repos/extra-i686" >/dev/null - sed 's/pkgrel=1/pkgrel=2/g' -i PKGBUILD - svn commit -q -m"update pkg to pkgrel=2" >/dev/null - popd >/dev/null + updateRepoPKGBUILD 'pkg-simple-a' extra i686 ../db-update >/dev/null 2>&1 && fail "db-update should fail when a package is not consistent!" checkRemovedPackage extra 'pkg-simple-a-1-1-i686.pkg.tar.xz' 'i686' diff --git a/test/test.d/testing2x.sh b/test/test.d/testing2x.sh index 0add9fe..1168874 100755 --- a/test/test.d/testing2x.sh +++ b/test/test.d/testing2x.sh @@ -2,16 +2,10 @@ testTesting2xAnyPackage() { releasePackage core pkg-any-a any ../db-update - pushd "${TMP}/svn-packages-copy/pkg-any-a/trunk/" >/dev/null - sed 's/pkgrel=1/pkgrel=2/g' -i PKGBUILD - svn commit -q -m"update pkg to pkgrel=2" >/dev/null - makepkg -cCf - mv pkg-any-a-1-2-any.pkg.tar.xz "${pkgdir}/pkg-any-a/" - popd >/dev/null + updatePackage pkg-any-a releasePackage testing pkg-any-a any ../db-update - rm -f "${pkgdir}/pkg-any-a/pkg-any-a-1-2-any.pkg.tar.xz" ../testing2x pkg-any-a -- cgit v1.2.2 From f3cf9ff346afc960ccad843e44c1c71334cc84ee Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Tue, 18 Apr 2017 22:39:31 +0200 Subject: Remove test for file database as this feature was moved to pacman/repo-add a long time ago --- test/test.d/create-filelists.sh | 96 ----------------------------------------- 1 file changed, 96 deletions(-) delete mode 100755 test/test.d/create-filelists.sh diff --git a/test/test.d/create-filelists.sh b/test/test.d/create-filelists.sh deleted file mode 100755 index 6b55a0b..0000000 --- a/test/test.d/create-filelists.sh +++ /dev/null @@ -1,96 +0,0 @@ -testCreateSimpleFileLists() { - local arches=('i686' 'x86_64') - local pkgs=('pkg-simple-a' 'pkg-simple-b' 'pkg-simple-epoch') - local pkgbase - local arch - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage extra ${pkgbase} ${arch} - done - done - ../db-update - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - if ! bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra${FILESEXT}" | grep -q "usr/bin/${pkgbase}"; then - fail "usr/bin/${pkgbase} not found in ${arch}/extra${FILESEXT}" - fi - done - done -} - -testCreateAnyFileLists() { - local arches=('i686' 'x86_64') - local pkgs=('pkg-any-a' 'pkg-any-b') - local pkgbase - local arch - - for pkgbase in ${pkgs[@]}; do - releasePackage extra ${pkgbase} any - done - ../db-update - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - if ! bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra${FILESEXT}" | grep -q "usr/share/${pkgbase}/test"; then - fail "usr/share/${pkgbase}/test not found in ${arch}/extra${FILESEXT}" - fi - done - done -} - -testCreateSplitFileLists() { - local arches=('i686' 'x86_64') - local pkgs=('pkg-split-a' 'pkg-split-b') - local pkg - local pkgbase - local pkgname - local pkgnames - local arch - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage extra ${pkgbase} ${arch} - done - done - ../db-update - - for pkgbase in ${pkgs[@]}; do - pkgnames=($(source "${TMP}/svn-packages-copy/${pkgbase}/trunk/PKGBUILD"; echo ${pkgname[@]})) - for pkgname in ${pkgnames[@]}; do - for arch in ${arches[@]}; do - if ! bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra${FILESEXT}" | grep -q "usr/bin/${pkgname}"; then - fail "usr/bin/${pkgname} not found in ${arch}/extra${FILESEXT}" - fi - done - done - done -} - -testCleanupFileLists() { - local arches=('i686' 'x86_64') - local pkgs=('pkg-simple-a' 'pkg-simple-b') - local pkgbase - local arch - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage extra ${pkgbase} ${arch} - done - done - ../db-update - - for arch in ${arches[@]}; do - ../db-remove extra ${arch} pkg-simple-a - done - - for arch in ${arches[@]}; do - if ! bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra${FILESEXT}" | grep -q "usr/bin/pkg-simple-b"; then - fail "usr/bin/pkg-simple-b not found in ${arch}/extra${FILESEXT}" - fi - if bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra${FILESEXT}" | grep -q "usr/bin/pkg-simple-a"; then - fail "usr/bin/pkg-simple-a still found in ${arch}/extra${FILESEXT}" - fi - done -} -- cgit v1.2.2 From cc328dca993b52c2266d3c18f8b6321db037c618 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Tue, 18 Apr 2017 22:50:12 +0200 Subject: Mark private test functions --- test/lib/common.sh | 54 +++++++++++++++++++++++++++--------------------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/test/lib/common.sh b/test/lib/common.sh index 052312a..693c794 100644 --- a/test/lib/common.sh +++ b/test/lib/common.sh @@ -1,3 +1,28 @@ +__getPackageBaseFromPackage() { + local _base + _grep_pkginfo() { + local _ret + + _ret="$(/usr/bin/bsdtar -xOqf "$1" .PKGINFO | grep -m 1 "^${2} = ")" + echo "${_ret#${2} = }" + } + + _base="$(_grep_pkginfo "$1" "pkgbase")" + if [ -z "$_base" ]; then + _grep_pkginfo "$1" "pkgname" + else + echo "$_base" + fi +} + +__updatePKGBUILD() { + local pkgrel + + pkgrel=$(. PKGBUILD; expr ${pkgrel} + 1) + sed "s/pkgrel=.*/pkgrel=${pkgrel}/" -i PKGBUILD + svn commit -q -m"update pkg to pkgrel=${pkgrel}" >/dev/null +} + oneTimeSetUp() { local d @@ -67,23 +92,6 @@ tearDown() { rm -rf "${TMP}" } -getpkgbase() { - local _base - _grep_pkginfo() { - local _ret - - _ret="$(/usr/bin/bsdtar -xOqf "$1" .PKGINFO | grep -m 1 "^${2} = ")" - echo "${_ret#${2} = }" - } - - _base="$(_grep_pkginfo "$1" "pkgbase")" - if [ -z "$_base" ]; then - _grep_pkginfo "$1" "pkgname" - else - echo "$_base" - fi -} - releasePackage() { local repo=$1 local pkgbase=$2 @@ -133,14 +141,6 @@ buildPackage() { done } -__updatePKGBUILD() { - local pkgrel - - pkgrel=$(. PKGBUILD; expr ${pkgrel} + 1) - sed "s/pkgrel=.*/pkgrel=${pkgrel}/" -i PKGBUILD - svn commit -q -m"update pkg to pkgrel=${pkgrel}" >/dev/null -} - updatePackage() { local pkgbase=$1 @@ -194,7 +194,7 @@ checkAnyPackage() { checkAnyPackageDB $repo $pkg - local pkgbase=$(getpkgbase "${FTP_BASE}/${PKGPOOL}/${pkg}") + local pkgbase=$(__getPackageBaseFromPackage "${FTP_BASE}/${PKGPOOL}/${pkg}") svn up -q "${TMP}/svn-packages-copy/${pkgbase}" [ -d "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo}-any" ] \ || fail "svn-packages-copy/${pkgbase}/repos/${repo}-any does not exist" @@ -234,7 +234,7 @@ checkPackage() { checkPackageDB $repo $pkg $arch - local pkgbase=$(getpkgbase "${FTP_BASE}/${PKGPOOL}/${pkg}") + local pkgbase=$(__getPackageBaseFromPackage "${FTP_BASE}/${PKGPOOL}/${pkg}") svn up -q "${TMP}/svn-packages-copy/${pkgbase}" [ -d "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo}-${arch}" ] \ || fail "svn-packages-copy/${pkgbase}/repos/${repo}-${arch} does not exist" -- cgit v1.2.2 From f57490022949d94eccc2b7caab79e99c050918cc Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Tue, 18 Apr 2017 23:15:14 +0200 Subject: Set PATH outside the test environment --- test/Makefile | 1 + test/lib/common.sh | 4 ++-- test/test.d/db-move.sh | 20 ++++++++++---------- test/test.d/db-remove.sh | 12 ++++++------ test/test.d/db-repo-add.sh | 6 +++--- test/test.d/db-repo-remove.sh | 12 ++++++------ test/test.d/db-update.sh | 44 +++++++++++++++++++++---------------------- test/test.d/ftpdir-cleanup.sh | 32 +++++++++++++++---------------- test/test.d/sourceballs.sh | 20 ++++++++++---------- test/test.d/testing2x.sh | 6 +++--- 10 files changed, 79 insertions(+), 78 deletions(-) diff --git a/test/Makefile b/test/Makefile index ea1d157..2cab2f6 100644 --- a/test/Makefile +++ b/test/Makefile @@ -4,6 +4,7 @@ test-suite: echo "#!/usr/bin/bash" > $(TESTSUITE_FILE) echo ". /usr/share/makepkg/util/pkgbuild.sh" >> $(TESTSUITE_FILE) echo ". $(CURDIR)/lib/common.sh" >> $(TESTSUITE_FILE) + echo "PATH=$(CURDIR)/../:$(CURDIR)/../cron-jobs/:$(PATH)" >> $(TESTSUITE_FILE) cat "$(CURDIR)/test.d"/*.sh >> $(TESTSUITE_FILE) echo ". $(CURDIR)/lib/shunit2" >> $(TESTSUITE_FILE) chmod +x $(TESTSUITE_FILE) diff --git a/test/lib/common.sh b/test/lib/common.sh index 693c794..42abed8 100644 --- a/test/lib/common.sh +++ b/test/lib/common.sh @@ -27,7 +27,7 @@ oneTimeSetUp() { local d pkgdir="$(mktemp -d)" - cp -Lr $(dirname ${BASH_SOURCE[0]})/../packages/* "${pkgdir}" + cp -Lr packages/* "${pkgdir}" for d in "${pkgdir}"/*; do pushd $d >/dev/null buildPackage @@ -63,7 +63,7 @@ setUp() { CLEANUP_DRYRUN=false SOURCE_CLEANUP_DRYRUN=false eot - . "$(dirname ${BASH_SOURCE[0]})/../../config" + . config mkdir -p "${TMP}/"{ftp,tmp,staging,{package,source}-cleanup,svn-packages-{copy,repo}} diff --git a/test/test.d/db-move.sh b/test/test.d/db-move.sh index a27dfbb..4da5ae2 100755 --- a/test/test.d/db-move.sh +++ b/test/test.d/db-move.sh @@ -10,9 +10,9 @@ testMoveSimplePackages() { done done - ../db-update + db-update - ../db-move testing extra pkg-simple-a + db-move testing extra pkg-simple-a for arch in ${arches[@]}; do checkPackage extra pkg-simple-a-1-1-${arch}.pkg.tar.xz ${arch} @@ -34,9 +34,9 @@ testMoveMultiplePackages() { done done - ../db-update + db-update - ../db-move testing extra pkg-simple-a pkg-simple-b + db-move testing extra pkg-simple-a pkg-simple-b for pkgbase in ${pkgs[@]}; do for arch in ${arches[@]}; do @@ -58,9 +58,9 @@ testMoveEpochPackages() { done done - ../db-update + db-update - ../db-move testing extra pkg-simple-epoch + db-move testing extra pkg-simple-epoch for arch in ${arches[@]}; do checkPackage extra pkg-simple-epoch-1:1-1-${arch}.pkg.tar.xz ${arch} @@ -76,8 +76,8 @@ testMoveAnyPackages() { releasePackage testing ${pkgbase} any done - ../db-update - ../db-move testing extra pkg-any-a + db-update + db-move testing extra pkg-any-a checkAnyPackage extra pkg-any-a-1-1-any.pkg.tar.xz checkRemovedAnyPackage testing pkg-any-a @@ -97,8 +97,8 @@ testMoveSplitPackages() { done done - ../db-update - ../db-move testing extra pkg-split-a + db-update + db-move testing extra pkg-split-a for arch in ${arches[@]}; do for pkg in "${pkgdir}/pkg-split-a"/*-${arch}${PKGEXT}; do diff --git a/test/test.d/db-remove.sh b/test/test.d/db-remove.sh index 52b9584..eab1ef4 100755 --- a/test/test.d/db-remove.sh +++ b/test/test.d/db-remove.sh @@ -10,11 +10,11 @@ testRemovePackages() { done done - ../db-update + db-update for pkgbase in ${pkgs[@]}; do for arch in ${arches[@]}; do - ../db-remove extra ${arch} ${pkgbase} + db-remove extra ${arch} ${pkgbase} done done @@ -37,10 +37,10 @@ testRemoveMultiplePackages() { done done - ../db-update + db-update for arch in ${arches[@]}; do - ../db-remove extra ${arch} ${pkgs[@]} + db-remove extra ${arch} ${pkgs[@]} done for pkgbase in ${pkgs[@]}; do @@ -58,10 +58,10 @@ testRemoveAnyPackages() { releasePackage extra ${pkgbase} any done - ../db-update + db-update for pkgbase in ${pkgs[@]}; do - ../db-remove extra any ${pkgbase} + db-remove extra any ${pkgbase} done for pkgbase in ${pkgs[@]}; do diff --git a/test/test.d/db-repo-add.sh b/test/test.d/db-repo-add.sh index 21ea538..ed72957 100755 --- a/test/test.d/db-repo-add.sh +++ b/test/test.d/db-repo-add.sh @@ -10,7 +10,7 @@ testRepoAddSimplePackages() { touch "${FTP_BASE}/${PKGPOOL}/${pkgbase}-1-1-${arch}.pkg.tar.xz.sig" ln -s "${FTP_BASE}/${PKGPOOL}/${pkgbase}-1-1-${arch}.pkg.tar.xz" "${FTP_BASE}/extra/os/${arch}/" ln -s "${FTP_BASE}/${PKGPOOL}/${pkgbase}-1-1-${arch}.pkg.tar.xz.sig" "${FTP_BASE}/extra/os/${arch}/" - ../db-repo-add extra ${arch} ${pkgbase}-1-1-${arch}.pkg.tar.xz + db-repo-add extra ${arch} ${pkgbase}-1-1-${arch}.pkg.tar.xz done done @@ -36,7 +36,7 @@ testRepoAddMultiplePackages() { ln -s "${FTP_BASE}/${PKGPOOL}/${pkgbase}-1-1-${arch}.pkg.tar.xz.sig" "${FTP_BASE}/extra/os/${arch}/" add_pkgs[${#add_pkgs[*]}]=${pkgbase}-1-1-${arch}.pkg.tar.xz done - ../db-repo-add extra ${arch} ${add_pkgs[@]} + db-repo-add extra ${arch} ${add_pkgs[@]} done for pkgbase in ${pkgs[@]}; do @@ -59,7 +59,7 @@ testRepoAddAnyPackages() { ln -s "${FTP_BASE}/${PKGPOOL}/${pkgbase}-1-1-any.pkg.tar.xz" "${FTP_BASE}/extra/os/${arch}/" ln -s "${FTP_BASE}/${PKGPOOL}/${pkgbase}-1-1-any.pkg.tar.xz.sig" "${FTP_BASE}/extra/os/${arch}/" done - ../db-repo-add extra any ${pkgbase}-1-1-any.pkg.tar.xz + db-repo-add extra any ${pkgbase}-1-1-any.pkg.tar.xz done for pkgbase in ${pkgs[@]}; do diff --git a/test/test.d/db-repo-remove.sh b/test/test.d/db-repo-remove.sh index 6369251..1aced25 100755 --- a/test/test.d/db-repo-remove.sh +++ b/test/test.d/db-repo-remove.sh @@ -10,11 +10,11 @@ testRepoRemovePackages() { done done - ../db-update + db-update for pkgbase in ${pkgs[@]}; do for arch in ${arches[@]}; do - ../db-repo-remove extra ${arch} ${pkgbase} + db-repo-remove extra ${arch} ${pkgbase} done done @@ -37,10 +37,10 @@ testRepoRemoveMultiplePackages() { done done - ../db-update + db-update for arch in ${arches[@]}; do - ../db-repo-remove extra ${arch} ${pkgs[@]} + db-repo-remove extra ${arch} ${pkgs[@]} done for pkgbase in ${pkgs[@]}; do @@ -58,10 +58,10 @@ testRepoRemoveAnyPackages() { releasePackage extra ${pkgbase} any done - ../db-update + db-update for pkgbase in ${pkgs[@]}; do - ../db-repo-remove extra any ${pkgbase} + db-repo-remove extra any ${pkgbase} done for pkgbase in ${pkgs[@]}; do diff --git a/test/test.d/db-update.sh b/test/test.d/db-update.sh index e9e5958..db37b82 100755 --- a/test/test.d/db-update.sh +++ b/test/test.d/db-update.sh @@ -10,7 +10,7 @@ testAddSimplePackages() { done done - ../db-update + db-update for pkgbase in ${pkgs[@]}; do for arch in ${arches[@]}; do @@ -21,13 +21,13 @@ testAddSimplePackages() { testAddSingleSimplePackage() { releasePackage extra 'pkg-simple-a' 'i686' - ../db-update + db-update checkPackage extra 'pkg-simple-a-1-1-i686.pkg.tar.xz' 'i686' } testAddSingleEpochPackage() { releasePackage extra 'pkg-simple-epoch' 'i686' - ../db-update + db-update checkPackage extra 'pkg-simple-epoch-1:1-1-i686.pkg.tar.xz' 'i686' } @@ -39,7 +39,7 @@ testAddAnyPackages() { releasePackage extra ${pkgbase} any done - ../db-update + db-update for pkgbase in ${pkgs[@]}; do checkAnyPackage extra ${pkgbase}-1-1-any.pkg.tar.xz @@ -59,7 +59,7 @@ testAddSplitPackages() { done done - ../db-update + db-update for pkgbase in ${pkgs[@]}; do for arch in ${arches[@]}; do @@ -72,12 +72,12 @@ testAddSplitPackages() { testUpdateAnyPackage() { releasePackage extra pkg-any-a any - ../db-update + db-update updatePackage pkg-any-a releasePackage extra pkg-any-a any - ../db-update + db-update checkAnyPackage extra pkg-any-a-1-2-any.pkg.tar.xz any } @@ -89,7 +89,7 @@ testUpdateAnyPackageToDifferentRepositoriesAtOnce() { releasePackage testing pkg-any-a any - ../db-update + db-update checkAnyPackage extra pkg-any-a-1-1-any.pkg.tar.xz any checkAnyPackage testing pkg-any-a-1-2-any.pkg.tar.xz any @@ -97,20 +97,20 @@ testUpdateAnyPackageToDifferentRepositoriesAtOnce() { testUpdateSameAnyPackageToSameRepository() { releasePackage extra pkg-any-a any - ../db-update + db-update checkAnyPackage extra pkg-any-a-1-1-any.pkg.tar.xz any releasePackage extra pkg-any-a any - ../db-update >/dev/null 2>&1 && (fail 'Adding an existing package to the same repository should fail'; return 1) + db-update >/dev/null 2>&1 && (fail 'Adding an existing package to the same repository should fail'; return 1) } testUpdateSameAnyPackageToDifferentRepositories() { releasePackage extra pkg-any-a any - ../db-update + db-update checkAnyPackage extra pkg-any-a-1-1-any.pkg.tar.xz any releasePackage testing pkg-any-a any - ../db-update >/dev/null 2>&1 && (fail 'Adding an existing package to another repository should fail'; return 1) + db-update >/dev/null 2>&1 && (fail 'Adding an existing package to another repository should fail'; return 1) local arch for arch in i686 x86_64; do @@ -133,7 +133,7 @@ testAddIncompleteSplitPackage() { # remove a split package to make db-update fail rm "${STAGING}"/extra/${pkgbase}1-* - ../db-update >/dev/null 2>&1 && fail "db-update should fail when a split package is missing!" + db-update >/dev/null 2>&1 && fail "db-update should fail when a split package is missing!" for arch in ${arches[@]}; do ( [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" ] \ @@ -146,7 +146,7 @@ testUnknownRepo() { mkdir "${STAGING}/unknown/" releasePackage extra 'pkg-simple-a' 'i686' releasePackage unknown 'pkg-simple-b' 'i686' - ../db-update + db-update checkPackage extra 'pkg-simple-a-1-1-i686.pkg.tar.xz' 'i686' [ -e "${FTP_BASE}/unknown" ] && fail "db-update pushed a package into an unknown repository" rm -rf "${STAGING}/unknown/" @@ -155,7 +155,7 @@ testUnknownRepo() { testAddUnsignedPackageFails() { releasePackage extra 'pkg-simple-a' 'i686' rm "${STAGING}"/extra/*.sig - ../db-update >/dev/null 2>&1 && fail "db-update should fail when a signature is missing!" + db-update >/dev/null 2>&1 && fail "db-update should fail when a signature is missing!" checkRemovedPackage extra pkg-simple-a-1-1-i686.pkg.tar.xz i686 } @@ -167,7 +167,7 @@ testAddInvalidSignedPackageFails() { unxz $p xz -0 ${p%%.xz} done - ../db-update >/dev/null 2>&1 && fail "db-update should fail when a signature is invalid!" + db-update >/dev/null 2>&1 && fail "db-update should fail when a signature is invalid!" checkRemovedPackage extra pkg-simple-a-1-1-i686.pkg.tar.xz i686 } @@ -178,7 +178,7 @@ testAddBrokenSignatureFails() { for s in "${STAGING}"/extra/*.sig; do echo 0 > $s done - ../db-update >/dev/null 2>&1 && fail "db-update should fail when a signature is broken!" + db-update >/dev/null 2>&1 && fail "db-update should fail when a signature is broken!" checkRemovedPackage extra pkg-simple-a-1-1-i686.pkg.tar.xz i686 } @@ -191,7 +191,7 @@ testAddPackageWithInconsistentVersionFails() { mv "${p}" "${p/pkg-simple-a-1/pkg-simple-a-2}" done - ../db-update >/dev/null 2>&1 && fail "db-update should fail when a package is not consistent!" + db-update >/dev/null 2>&1 && fail "db-update should fail when a package is not consistent!" checkRemovedPackage extra 'pkg-simple-a-2-1-i686.pkg.tar.xz' 'i686' } @@ -203,7 +203,7 @@ testAddPackageWithInconsistentNameFails() { mv "${p}" "${p/pkg-/foo-pkg-}" done - ../db-update >/dev/null 2>&1 && fail "db-update should fail when a package is not consistent!" + db-update >/dev/null 2>&1 && fail "db-update should fail when a package is not consistent!" checkRemovedPackage extra 'foo-pkg-simple-a-1-1-i686.pkg.tar.xz' 'i686' } @@ -212,7 +212,7 @@ testAddPackageWithInconsistentPKGBUILDFails() { updateRepoPKGBUILD 'pkg-simple-a' extra i686 - ../db-update >/dev/null 2>&1 && fail "db-update should fail when a package is not consistent!" + db-update >/dev/null 2>&1 && fail "db-update should fail when a package is not consistent!" checkRemovedPackage extra 'pkg-simple-a-1-1-i686.pkg.tar.xz' 'i686' } @@ -222,7 +222,7 @@ testAddPackageWithInsufficientPermissionsFails() releasePackage extra 'pkg-simple-b' 'i686' chmod -xwr ${FTP_BASE}/core/os/i686 - ../db-update >/dev/null 2>&1 && fail "db-update should fail when permissions are insufficient!" + db-update >/dev/null 2>&1 && fail "db-update should fail when permissions are insufficient!" chmod +xwr ${FTP_BASE}/core/os/i686 checkRemovedPackage core 'pkg-simple-a-1-1-i686.pkg.tar.xz' 'i686' @@ -244,7 +244,7 @@ testPackageHasToBeARegularFile() ln -s "${target}/${p##*/}" "${p}" done - ../db-update >/dev/null 2>&1 && fail "db-update should fail when a package is a symlink!" + db-update >/dev/null 2>&1 && fail "db-update should fail when a package is a symlink!" for arch in ${arches[@]}; do checkRemovedPackage extra "pkg-simple-a-1-1-${arch}.pkg.tar.xz" $arch done diff --git a/test/test.d/ftpdir-cleanup.sh b/test/test.d/ftpdir-cleanup.sh index f29e2e3..2749a87 100755 --- a/test/test.d/ftpdir-cleanup.sh +++ b/test/test.d/ftpdir-cleanup.sh @@ -10,13 +10,13 @@ testCleanupSimplePackages() { done done - ../db-update + db-update for arch in ${arches[@]}; do - ../db-remove extra ${arch} pkg-simple-a + db-remove extra ${arch} pkg-simple-a done - ../cron-jobs/ftpdir-cleanup >/dev/null + ftpdir-cleanup >/dev/null for arch in ${arches[@]}; do local pkg1="pkg-simple-a-1-1-${arch}.pkg.tar.xz" @@ -41,13 +41,13 @@ testCleanupEpochPackages() { done done - ../db-update + db-update for arch in ${arches[@]}; do - ../db-remove extra ${arch} pkg-simple-epoch + db-remove extra ${arch} pkg-simple-epoch done - ../cron-jobs/ftpdir-cleanup >/dev/null + ftpdir-cleanup >/dev/null for arch in ${arches[@]}; do local pkg1="pkg-simple-epoch-1:1-1-${arch}.pkg.tar.xz" @@ -66,9 +66,9 @@ testCleanupAnyPackages() { releasePackage extra ${pkgbase} any done - ../db-update - ../db-remove extra any pkg-any-a - ../cron-jobs/ftpdir-cleanup >/dev/null + db-update + db-remove extra any pkg-any-a + ftpdir-cleanup >/dev/null local pkg1='pkg-any-a-1-1-any.pkg.tar.xz' checkRemovedAnyPackage extra 'pkg-any-a' @@ -92,13 +92,13 @@ testCleanupSplitPackages() { done done - ../db-update + db-update for arch in ${arches[@]}; do - ../db-remove extra ${arch} ${pkgs[0]} + db-remove extra ${arch} ${pkgs[0]} done - ../cron-jobs/ftpdir-cleanup >/dev/null + ftpdir-cleanup >/dev/null for arch in ${arches[@]}; do for pkg in "${pkgdir}/${pkgs[0]}"/*-${arch}${PKGEXT}; do @@ -125,15 +125,15 @@ testCleanupOldPackages() { done done - ../db-update + db-update for pkgbase in ${pkgs[@]}; do for arch in ${arches[@]}; do - ../db-remove extra ${arch} ${pkgbase} + db-remove extra ${arch} ${pkgbase} done done - ../cron-jobs/ftpdir-cleanup >/dev/null + ftpdir-cleanup >/dev/null local pkgfilea="pkg-simple-a-1-1-${arch}.pkg.tar.xz" local pkgfileb="pkg-simple-b-1-1-${arch}.pkg.tar.xz" @@ -141,7 +141,7 @@ testCleanupOldPackages() { touch -d "-$(expr ${CLEANUP_KEEP} + 1)days" ${CLEANUP_DESTDIR}/${pkgfilea}{,.sig} done - ../cron-jobs/ftpdir-cleanup >/dev/null + ftpdir-cleanup >/dev/null [ -f ${CLEANUP_DESTDIR}/${pkgfilea} ] && fail "${pkgfilea} was not removed" [ -f ${CLEANUP_DESTDIR}/${pkgfileb} ] || fail "${pkgfileb} was removed" diff --git a/test/test.d/sourceballs.sh b/test/test.d/sourceballs.sh index a004cdb..04d0fd3 100755 --- a/test/test.d/sourceballs.sh +++ b/test/test.d/sourceballs.sh @@ -9,9 +9,9 @@ testSourceballs() { releasePackage extra ${pkgbase} ${arch} done done - ../db-update + db-update - ../cron-jobs/sourceballs + sourceballs for pkgbase in ${pkgs[@]}; do [ ! -r ${FTP_BASE}/${SRCPOOL}/${pkgbase}-*${SRCEXT} ] && fail "source package not found!" done @@ -24,9 +24,9 @@ testAnySourceballs() { for pkgbase in ${pkgs[@]}; do releasePackage extra ${pkgbase} any done - ../db-update + db-update - ../cron-jobs/sourceballs + sourceballs for pkgbase in ${pkgs[@]}; do [ ! -r ${FTP_BASE}/${SRCPOOL}/${pkgbase}-*${SRCEXT} ] && fail "source package not found!" done @@ -45,9 +45,9 @@ testSplitSourceballs() { done done - ../db-update + db-update - ../cron-jobs/sourceballs + sourceballs for pkgbase in ${pkgs[@]}; do [ ! -r ${FTP_BASE}/${SRCPOOL}/${pkgbase}-*${SRCEXT} ] && fail "source package not found!" done @@ -64,14 +64,14 @@ testSourceballsCleanup() { releasePackage extra ${pkgbase} ${arch} done done - ../db-update - ../cron-jobs/sourceballs + db-update + sourceballs for arch in ${arches[@]}; do - ../db-remove extra ${arch} pkg-simple-a + db-remove extra ${arch} pkg-simple-a done - ../cron-jobs/sourceballs + sourceballs [ -r ${FTP_BASE}/${SRCPOOL}/pkg-simple-a-*${SRCEXT} ] && fail "source package was not removed!" [ ! -r ${FTP_BASE}/${SRCPOOL}/pkg-simple-b-*${SRCEXT} ] && fail "source package not found!" } diff --git a/test/test.d/testing2x.sh b/test/test.d/testing2x.sh index 1168874..cecf5c2 100755 --- a/test/test.d/testing2x.sh +++ b/test/test.d/testing2x.sh @@ -1,13 +1,13 @@ testTesting2xAnyPackage() { releasePackage core pkg-any-a any - ../db-update + db-update updatePackage pkg-any-a releasePackage testing pkg-any-a any - ../db-update + db-update - ../testing2x pkg-any-a + testing2x pkg-any-a checkAnyPackage core pkg-any-a-1-2-any.pkg.tar.xz any checkRemovedAnyPackage testing pkg-any-a -- cgit v1.2.2 From d0b808b5a58023899c7e76958cdae732bbdfc08e Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Wed, 19 Apr 2017 12:33:57 +0200 Subject: Switch from shuint2 to bats testing framework --- Makefile | 6 +- test/Dockerfile | 2 +- test/Makefile | 23 +- test/lib/common.bash | 276 +++++++++++ test/lib/common.sh | 292 ----------- test/lib/shunit2 | 1048 --------------------------------------- test/test.d/db-move.bats | 117 +++++ test/test.d/db-move.sh | 115 ----- test/test.d/db-remove.bats | 72 +++ test/test.d/db-remove.sh | 70 --- test/test.d/db-repo-add.bats | 72 +++ test/test.d/db-repo-add.sh | 70 --- test/test.d/db-repo-remove.bats | 72 +++ test/test.d/db-repo-remove.sh | 70 --- test/test.d/db-update.bats | 263 ++++++++++ test/test.d/db-update.sh | 251 ---------- test/test.d/ftpdir-cleanup.bats | 150 ++++++ test/test.d/ftpdir-cleanup.sh | 148 ------ test/test.d/packages.bats | 34 ++ test/test.d/packages.sh | 7 - test/test.d/sourceballs.bats | 79 +++ test/test.d/sourceballs.sh | 77 --- test/test.d/testing2x.bats | 16 + test/test.d/testing2x.sh | 14 - 24 files changed, 1161 insertions(+), 2183 deletions(-) create mode 100644 test/lib/common.bash delete mode 100644 test/lib/common.sh delete mode 100755 test/lib/shunit2 create mode 100644 test/test.d/db-move.bats delete mode 100755 test/test.d/db-move.sh create mode 100644 test/test.d/db-remove.bats delete mode 100755 test/test.d/db-remove.sh create mode 100644 test/test.d/db-repo-add.bats delete mode 100755 test/test.d/db-repo-add.sh create mode 100644 test/test.d/db-repo-remove.bats delete mode 100755 test/test.d/db-repo-remove.sh create mode 100644 test/test.d/db-update.bats delete mode 100755 test/test.d/db-update.sh create mode 100644 test/test.d/ftpdir-cleanup.bats delete mode 100755 test/test.d/ftpdir-cleanup.sh create mode 100644 test/test.d/packages.bats delete mode 100755 test/test.d/packages.sh create mode 100644 test/test.d/sourceballs.bats delete mode 100755 test/test.d/sourceballs.sh create mode 100644 test/test.d/testing2x.bats delete mode 100755 test/test.d/testing2x.sh diff --git a/Makefile b/Makefile index 988d335..92b3fec 100644 --- a/Makefile +++ b/Makefile @@ -1,13 +1,13 @@ IMAGE:=dbscripts/test -RUN_OPTIONS:=--rm --network=none -v $(PWD):/dbscripts:ro --tmpfs=/tmp:exec -w /dbscripts/test +RUN_OPTIONS:=-t --rm --network=none -v $(PWD):/dbscripts:ro --tmpfs=/tmp:exec -w /dbscripts/test test-image: docker build --pull -t $(IMAGE) test -test: test-image +test: docker run $(RUN_OPTIONS) $(IMAGE) make test -test-coverage: test-image +test-coverage: rm -rf ${PWD}/coverage mkdir -m 777 ${PWD}/coverage docker run $(RUN_OPTIONS) -v ${PWD}/coverage:/coverage -e COVERAGE_DIR=/coverage $(IMAGE) make test-coverage diff --git a/test/Dockerfile b/test/Dockerfile index ef38d54..e428f0a 100644 --- a/test/Dockerfile +++ b/test/Dockerfile @@ -1,5 +1,5 @@ FROM pierres/archlinux -RUN pacman -Syu --noconfirm --needed base-devel devtools kcov +RUN pacman -Syu --noconfirm --needed base-devel devtools kcov bash-bats RUN pacman-key --init RUN echo '%wheel ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/wheel RUN useradd -N -g users -G wheel -m tester diff --git a/test/Makefile b/test/Makefile index 2cab2f6..afc3cff 100644 --- a/test/Makefile +++ b/test/Makefile @@ -1,22 +1,11 @@ -TESTSUITE_FILE:=$(shell mktemp) +test: + PATH=$(CURDIR)/../:$(CURDIR)/../cron-jobs/:$(PATH) bats test.d -test-suite: - echo "#!/usr/bin/bash" > $(TESTSUITE_FILE) - echo ". /usr/share/makepkg/util/pkgbuild.sh" >> $(TESTSUITE_FILE) - echo ". $(CURDIR)/lib/common.sh" >> $(TESTSUITE_FILE) - echo "PATH=$(CURDIR)/../:$(CURDIR)/../cron-jobs/:$(PATH)" >> $(TESTSUITE_FILE) - cat "$(CURDIR)/test.d"/*.sh >> $(TESTSUITE_FILE) - echo ". $(CURDIR)/lib/shunit2" >> $(TESTSUITE_FILE) - chmod +x $(TESTSUITE_FILE) - -test: test-suite - $(TESTSUITE_FILE) - -test-coverage: test-suite - kcov \ +test-coverage: + PATH=$(CURDIR)/../:$(CURDIR)/../cron-jobs/:$(PATH) kcov \ --include-path=$(CURDIR)/../ \ --exclude-path=$(CURDIR)/../test,$(CURDIR)/../cron-jobs/makepkg.conf,$(CURDIR)/../config \ $(COVERAGE_DIR) \ - $(TESTSUITE_FILE) + bats test.d -.PHONY: test-suite test test-coverage +.PHONY: test test-coverage diff --git a/test/lib/common.bash b/test/lib/common.bash new file mode 100644 index 0000000..aba3336 --- /dev/null +++ b/test/lib/common.bash @@ -0,0 +1,276 @@ +. /usr/share/makepkg/util/pkgbuild.sh + +__getPackageBaseFromPackage() { + local _base + _grep_pkginfo() { + local _ret + + _ret="$(/usr/bin/bsdtar -xOqf "$1" .PKGINFO | grep -m 1 "^${2} = ")" + echo "${_ret#${2} = }" + } + + _base="$(_grep_pkginfo "$1" "pkgbase")" + if [ -z "$_base" ]; then + _grep_pkginfo "$1" "pkgname" + else + echo "$_base" + fi +} + +__updatePKGBUILD() { + local pkgrel + + pkgrel=$(. PKGBUILD; expr ${pkgrel} + 1) + sed "s/pkgrel=.*/pkgrel=${pkgrel}/" -i PKGBUILD + svn commit -q -m"update pkg to pkgrel=${pkgrel}" +} + +__buildPackage() { + local arch=$1 + local pkgver + local pkgname + local a + local p + + pkgname=($(. PKGBUILD; echo ${pkgname[@]})) + pkgver=$(. PKGBUILD; get_full_version) + + # FIXME: Avoid rebuilding of the same package + for p in ${pkgname[@]}; do + [ -f ${p}-${pkgver}-${arch}${PKGEXT} ] && return 0 + done + + if [ "${arch}" == 'any' ]; then + makepkg -c + else + CARCH=${arch} makepkg -c + fi + + for p in ${pkgname[@]}; do + gpg --detach-sign --no-armor --use-agent ${p}-${pkgver}-${arch}* + done +} + +setup() { + local p + local pkg + local r + local a + + TMP="$(mktemp -d)" + + export DBSCRIPTS_CONFIG=${TMP}/config.local + cat < "${DBSCRIPTS_CONFIG}" + FTP_BASE="${TMP}/ftp" + SVNREPO="file://${TMP}/svn-packages-repo" + PKGREPOS=('core' 'extra' 'testing') + PKGPOOL='pool/packages' + SRCPOOL='sources/packages' + TESTING_REPO='testing' + STABLE_REPOS=('core' 'extra') + CLEANUP_DESTDIR="${TMP}/package-cleanup" + SOURCE_CLEANUP_DESTDIR="${TMP}/source-cleanup" + STAGING="${TMP}/staging" + TMPDIR="${TMP}/tmp" + CLEANUP_DRYRUN=false + SOURCE_CLEANUP_DRYRUN=false +eot + . config + + mkdir -p "${TMP}/"{ftp,tmp,staging,{package,source}-cleanup,svn-packages-{copy,repo}} + + for r in ${PKGREPOS[@]}; do + mkdir -p "${TMP}/staging/${r}" + for a in ${ARCHES[@]}; do + mkdir -p "${TMP}/ftp/${r}/os/${a}" + done + done + mkdir -p "${TMP}/ftp/${PKGPOOL}" + mkdir -p "${TMP}/ftp/${SRCPOOL}" + + svnadmin create "${TMP}/svn-packages-repo" + svn checkout -q "file://${TMP}/svn-packages-repo" "${TMP}/svn-packages-copy" +} + +teardown() { + rm -rf "${TMP}" +} + +releasePackage() { + local repo=$1 + local pkgbase=$2 + local arch=$3 + local a + local p + local pkgver + local pkgname + + if [ ! -d "${TMP}/svn-packages-copy/${pkgbase}/trunk" ]; then + mkdir -p "${TMP}/svn-packages-copy/${pkgbase}"/{trunk,repos} + cp -r "packages/${pkgbase}"/* "${TMP}/svn-packages-copy"/${pkgbase}/trunk/ + svn add -q "${TMP}/svn-packages-copy"/${pkgbase} + svn commit -q -m"initial commit of ${pkgbase}" "${TMP}/svn-packages-copy" + fi + + pushd "${TMP}/svn-packages-copy"/${pkgbase}/trunk/ + __buildPackage ${arch} + archrelease -f ${repo}-${arch} + pkgver=$(. PKGBUILD; get_full_version) + pkgname=($(. PKGBUILD; echo ${pkgname[@]})) + + for a in ${arch[@]}; do + for p in ${pkgname[@]}; do + cp ${p}-${pkgver}-${a}${PKGEXT}{,.sig} "${STAGING}"/${repo}/ + done + done + + popd +} + +updatePackage() { + local pkgbase=$1 + local arch=$2 + + pushd "${TMP}/svn-packages-copy/${pkgbase}/trunk/" + __updatePKGBUILD + __buildPackage ${arch} + popd +} + +updateRepoPKGBUILD() { + local pkgbase=$1 + local repo=$2 + local arch=$3 + + pushd "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo}-${arch}/" + __updatePKGBUILD + popd +} + +getPackageNamesFromPackageBase() { + local pkgbase=$1 + + $(. "packages/${pkgbase}/PKGBUILD"; echo ${pkgname[@]}) +} + +checkAnyPackageDB() { + local repo=$1 + local pkg=$2 + local arch + local db + + [ -r "${FTP_BASE}/${PKGPOOL}/${pkg}" ] + [ -r "${FTP_BASE}/${PKGPOOL}/${pkg}.sig" ] + + for arch in i686 x86_64; do + [ -L "${FTP_BASE}/${repo}/os/${arch}/${pkg}" ] + [ "$(readlink -e "${FTP_BASE}/${repo}/os/${arch}/${pkg}")" == "${FTP_BASE}/${PKGPOOL}/${pkg}" ] + + [ -L "${FTP_BASE}/${repo}/os/${arch}/${pkg}.sig" ] + [ "$(readlink -e "${FTP_BASE}/${repo}/os/${arch}/${pkg}.sig")" == "${FTP_BASE}/${PKGPOOL}/${pkg}.sig" ] + + for db in ${DBEXT} ${FILESEXT}; do + [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" ] + bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" -O | grep -q ${pkg} + done + done + + [ ! -r "${STAGING}"/${repo}/${pkg} ] + [ ! -r "${STAGING}"/${repo}/${pkg}.sig ] +} + +checkAnyPackage() { + local repo=$1 + local pkg=$2 + + checkAnyPackageDB $repo $pkg + + local pkgbase=$(__getPackageBaseFromPackage "${FTP_BASE}/${PKGPOOL}/${pkg}") + svn up -q "${TMP}/svn-packages-copy/${pkgbase}" + [ -d "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo}-any" ] +} + +checkPackageDB() { + local repo=$1 + local pkg=$2 + local arch=$3 + local db + + [ -r "${FTP_BASE}/${PKGPOOL}/${pkg}" ] + [ -L "${FTP_BASE}/${repo}/os/${arch}/${pkg}" ] + [ ! -r "${STAGING}"/${repo}/${pkg} ] + + [ "$(readlink -e "${FTP_BASE}/${repo}/os/${arch}/${pkg}")" == "${FTP_BASE}/${PKGPOOL}/${pkg}" ] + + [ -r "${FTP_BASE}/${PKGPOOL}/${pkg}.sig" ] + [ -L "${FTP_BASE}/${repo}/os/${arch}/${pkg}.sig" ] + [ ! -r "${STAGING}"/${repo}/${pkg}.sig ] + + [ "$(readlink -e "${FTP_BASE}/${repo}/os/${arch}/${pkg}.sig")" == "${FTP_BASE}/${PKGPOOL}/${pkg}.sig" ] + + for db in ${DBEXT} ${FILESEXT}; do + [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" ] + bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" -O | grep -q ${pkg} + done +} + +checkPackage() { + local repo=$1 + local pkg=$2 + local arch=$3 + + checkPackageDB $repo $pkg $arch + + local pkgbase=$(__getPackageBaseFromPackage "${FTP_BASE}/${PKGPOOL}/${pkg}") + svn up -q "${TMP}/svn-packages-copy/${pkgbase}" + [ -d "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo}-${arch}" ] +} + +checkRemovedPackageDB() { + local repo=$1 + local pkgbase=$2 + local arch=$3 + local db + + for db in ${DBEXT} ${FILESEXT}; do + if [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" ]; then + echo "$(bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" -O)" | grep -qv ${pkgbase} + fi + done +} + +checkRemovedPackage() { + local repo=$1 + local pkgbase=$2 + local arch=$3 + + checkRemovedPackageDB $repo $pkgbase $arch + + svn up -q "${TMP}/svn-packages-copy/${pkgbase}" + [ ! -d "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo}-${arch}" ] +} + +checkRemovedAnyPackageDB() { + local repo=$1 + local pkgbase=$2 + local arch + local db + + for db in ${DBEXT} ${FILESEXT}; do + for arch in i686 x86_64; do + if [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" ]; then + echo "$(bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" -O)" | grep -qv ${pkgbase} + fi + done + done +} + +checkRemovedAnyPackage() { + local repo=$1 + local pkgbase=$2 + + checkRemovedAnyPackageDB $repo $pkgbase + + svn up -q "${TMP}/svn-packages-copy/${pkgbase}" + [ ! -d "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo}-any" ] +} diff --git a/test/lib/common.sh b/test/lib/common.sh deleted file mode 100644 index 42abed8..0000000 --- a/test/lib/common.sh +++ /dev/null @@ -1,292 +0,0 @@ -__getPackageBaseFromPackage() { - local _base - _grep_pkginfo() { - local _ret - - _ret="$(/usr/bin/bsdtar -xOqf "$1" .PKGINFO | grep -m 1 "^${2} = ")" - echo "${_ret#${2} = }" - } - - _base="$(_grep_pkginfo "$1" "pkgbase")" - if [ -z "$_base" ]; then - _grep_pkginfo "$1" "pkgname" - else - echo "$_base" - fi -} - -__updatePKGBUILD() { - local pkgrel - - pkgrel=$(. PKGBUILD; expr ${pkgrel} + 1) - sed "s/pkgrel=.*/pkgrel=${pkgrel}/" -i PKGBUILD - svn commit -q -m"update pkg to pkgrel=${pkgrel}" >/dev/null -} - -oneTimeSetUp() { - local d - - pkgdir="$(mktemp -d)" - cp -Lr packages/* "${pkgdir}" - for d in "${pkgdir}"/*; do - pushd $d >/dev/null - buildPackage - popd >/dev/null - done -} - -oneTimeTearDown() { - rm -rf "${pkgdir}" -} - -setUp() { - local p - local pkg - local r - local a - - TMP="$(mktemp -d)" - - export DBSCRIPTS_CONFIG=${TMP}/config.local - cat < "${DBSCRIPTS_CONFIG}" - FTP_BASE="${TMP}/ftp" - SVNREPO="file://${TMP}/svn-packages-repo" - PKGREPOS=('core' 'extra' 'testing') - PKGPOOL='pool/packages' - SRCPOOL='sources/packages' - TESTING_REPO='testing' - STABLE_REPOS=('core' 'extra') - CLEANUP_DESTDIR="${TMP}/package-cleanup" - SOURCE_CLEANUP_DESTDIR="${TMP}/source-cleanup" - STAGING="${TMP}/staging" - TMPDIR="${TMP}/tmp" - CLEANUP_DRYRUN=false - SOURCE_CLEANUP_DRYRUN=false -eot - . config - - mkdir -p "${TMP}/"{ftp,tmp,staging,{package,source}-cleanup,svn-packages-{copy,repo}} - - for r in ${PKGREPOS[@]}; do - mkdir -p "${TMP}/staging/${r}" - for a in ${ARCHES[@]}; do - mkdir -p "${TMP}/ftp/${r}/os/${a}" - done - done - mkdir -p "${TMP}/ftp/${PKGPOOL}" - mkdir -p "${TMP}/ftp/${SRCPOOL}" - - svnadmin create "${TMP}/svn-packages-repo" - svn checkout -q "file://${TMP}/svn-packages-repo" "${TMP}/svn-packages-copy" - - for p in "${pkgdir}"/*; do - pkg=${p##*/} - mkdir -p "${TMP}/svn-packages-copy/${pkg}"/{trunk,repos} - cp "${p}"/* "${TMP}/svn-packages-copy"/${pkg}/trunk/ - svn add -q "${TMP}/svn-packages-copy"/${pkg} - svn commit -q -m"initial commit of ${pkg}" "${TMP}/svn-packages-copy" - done -} - -tearDown() { - rm -rf "${TMP}" -} - -releasePackage() { - local repo=$1 - local pkgbase=$2 - local arch=$3 - local a - local p - local pkgver - local pkgname - - pushd "${TMP}/svn-packages-copy"/${pkgbase}/trunk/ >/dev/null - archrelease ${repo}-${arch} >/dev/null 2>&1 - pkgver=$(. PKGBUILD; get_full_version) - pkgname=($(. PKGBUILD; echo ${pkgname[@]})) - - for a in ${arch[@]}; do - for p in ${pkgname[@]}; do - cp ${p}-${pkgver}-${a}${PKGEXT}{,.sig} "${STAGING}"/${repo}/ - done - done - - popd >/dev/null -} - -buildPackage() { - local pkgarch - local pkgver - local pkgname - local a - local p - - pkgname=($(. PKGBUILD; echo ${pkgname[@]})) - pkgver=$(. PKGBUILD; get_full_version) - pkgarch=($(. PKGBUILD; echo ${arch[@]})) - - if [ "${pkgarch[0]}" == 'any' ]; then - makepkg -cCf || fail 'makepkg failed' - else - for a in ${pkgarch[@]}; do - CARCH=${a} makepkg -cCf || fail "makepkg failed" - done - fi - - for a in ${pkgarch[@]}; do - for p in ${pkgname[@]}; do - gpg --detach-sign --no-armor --use-agent ${p}-${pkgver}-${a}* || fail 'gpg failed' - done - done -} - -updatePackage() { - local pkgbase=$1 - - pushd "${TMP}/svn-packages-copy/${pkgbase}/trunk/" >/dev/null - __updatePKGBUILD - buildPackage - popd >/dev/null -} - -updateRepoPKGBUILD() { - local pkgbase=$1 - local repo=$2 - local arch=$3 - - pushd "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo}-${arch}/" >/dev/null - __updatePKGBUILD - popd >/dev/null -} - -checkAnyPackageDB() { - local repo=$1 - local pkg=$2 - local arch - local db - - [ -r "${FTP_BASE}/${PKGPOOL}/${pkg}" ] || fail "${PKGPOOL}/${pkg} not found" - [ -r "${FTP_BASE}/${PKGPOOL}/${pkg}.sig" ] || fail "${PKGPOOL}/${pkg}.sig not found" - - for arch in i686 x86_64; do - [ -L "${FTP_BASE}/${repo}/os/${arch}/${pkg}" ] || fail "${repo}/os/${arch}/${pkg} is not a symlink" - [ "$(readlink -e "${FTP_BASE}/${repo}/os/${arch}/${pkg}")" == "${FTP_BASE}/${PKGPOOL}/${pkg}" ] \ - || fail "${repo}/os/${arch}/${pkg} does not link to ${PKGPOOL}/${pkg}" - - [ -L "${FTP_BASE}/${repo}/os/${arch}/${pkg}.sig" ] || fail "${repo}/os/${arch}/${pkg}.sig is not a symlink" - [ "$(readlink -e "${FTP_BASE}/${repo}/os/${arch}/${pkg}.sig")" == "${FTP_BASE}/${PKGPOOL}/${pkg}.sig" ] \ - || fail "${repo}/os/${arch}/${pkg}.sig does not link to ${PKGPOOL}/${pkg}.sig" - - for db in ${DBEXT} ${FILESEXT}; do - ( [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" ] \ - && bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" -O | grep -q ${pkg}) \ - || fail "${pkg} not in ${repo}/os/${arch}/${repo}${db%.tar.*}" - done - done - [ -r "${STAGING}"/${repo}/${pkg} ] && fail "${repo}/${pkg} found in staging dir" - [ -r "${STAGING}"/${repo}/${pkg}.sig ] && fail "${repo}/${pkg}.sig found in staging dir" -} - -checkAnyPackage() { - local repo=$1 - local pkg=$2 - - checkAnyPackageDB $repo $pkg - - local pkgbase=$(__getPackageBaseFromPackage "${FTP_BASE}/${PKGPOOL}/${pkg}") - svn up -q "${TMP}/svn-packages-copy/${pkgbase}" - [ -d "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo}-any" ] \ - || fail "svn-packages-copy/${pkgbase}/repos/${repo}-any does not exist" -} - -checkPackageDB() { - local repo=$1 - local pkg=$2 - local arch=$3 - local db - - [ -r "${FTP_BASE}/${PKGPOOL}/${pkg}" ] || fail "${PKGPOOL}/${pkg} not found" - [ -L "${FTP_BASE}/${repo}/os/${arch}/${pkg}" ] || fail "${repo}/os/${arch}/${pkg} not a symlink" - [ -r "${STAGING}"/${repo}/${pkg} ] && fail "${repo}/${pkg} found in staging dir" - - [ "$(readlink -e "${FTP_BASE}/${repo}/os/${arch}/${pkg}")" == "${FTP_BASE}/${PKGPOOL}/${pkg}" ] \ - || fail "${repo}/os/${arch}/${pkg} does not link to ${PKGPOOL}/${pkg}" - - [ -r "${FTP_BASE}/${PKGPOOL}/${pkg}.sig" ] || fail "${PKGPOOL}/${pkg}.sig not found" - [ -L "${FTP_BASE}/${repo}/os/${arch}/${pkg}.sig" ] || fail "${repo}/os/${arch}/${pkg}.sig is not a symlink" - [ -r "${STAGING}"/${repo}/${pkg}.sig ] && fail "${repo}/${pkg}.sig found in staging dir" - - [ "$(readlink -e "${FTP_BASE}/${repo}/os/${arch}/${pkg}.sig")" == "${FTP_BASE}/${PKGPOOL}/${pkg}.sig" ] \ - || fail "${repo}/os/${arch}/${pkg}.sig does not link to ${PKGPOOL}/${pkg}.sig" - - for db in ${DBEXT} ${FILESEXT}; do - ( [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" ] \ - && bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" -O | grep -q ${pkg}) \ - || fail "${pkg} not in ${repo}/os/${arch}/${repo}${db%.tar.*}" - done -} - -checkPackage() { - local repo=$1 - local pkg=$2 - local arch=$3 - - checkPackageDB $repo $pkg $arch - - local pkgbase=$(__getPackageBaseFromPackage "${FTP_BASE}/${PKGPOOL}/${pkg}") - svn up -q "${TMP}/svn-packages-copy/${pkgbase}" - [ -d "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo}-${arch}" ] \ - || fail "svn-packages-copy/${pkgbase}/repos/${repo}-${arch} does not exist" -} - -checkRemovedPackageDB() { - local repo=$1 - local pkgbase=$2 - local arch=$3 - local db - - for db in ${DBEXT} ${FILESEXT}; do - ( [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" ] \ - && bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" -O | grep -q ${pkgbase}) \ - && fail "${pkgbase} should not be in ${repo}/os/${arch}/${repo}${db%.tar.*}" - done -} - -checkRemovedPackage() { - local repo=$1 - local pkgbase=$2 - local arch=$3 - - checkRemovedPackageDB $repo $pkgbase $arch - - svn up -q "${TMP}/svn-packages-copy/${pkgbase}" - [ -d "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo}-${arch}" ] \ - && fail "svn-packages-copy/${pkgbase}/repos/${repo}-${arch} should not exist" -} - -checkRemovedAnyPackageDB() { - local repo=$1 - local pkgbase=$2 - local arch - local db - - for db in ${DBEXT} ${FILESEXT}; do - for arch in i686 x86_64; do - ( [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" ] \ - && bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" -O | grep -q ${pkgbase}) \ - && fail "${pkgbase} should not be in ${repo}/os/${arch}/${repo}${db%.tar.*}" - done - done -} - -checkRemovedAnyPackage() { - local repo=$1 - local pkgbase=$2 - - checkRemovedAnyPackageDB $repo $pkgbase - - svn up -q "${TMP}/svn-packages-copy/${pkgbase}" - [ -d "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo}-any" ] \ - && fail "svn-packages-copy/${pkgbase}/repos/${repo}-any should not exist" -} diff --git a/test/lib/shunit2 b/test/lib/shunit2 deleted file mode 100755 index 8862ffd..0000000 --- a/test/lib/shunit2 +++ /dev/null @@ -1,1048 +0,0 @@ -#! /bin/sh -# $Id: shunit2 335 2011-05-01 20:10:33Z kate.ward@forestent.com $ -# vim:et:ft=sh:sts=2:sw=2 -# -# Copyright 2008 Kate Ward. All Rights Reserved. -# Released under the LGPL (GNU Lesser General Public License) -# -# shUnit2 -- Unit testing framework for Unix shell scripts. -# http://code.google.com/p/shunit2/ -# -# Author: kate.ward@forestent.com (Kate Ward) -# -# shUnit2 is a xUnit based unit test framework for Bourne shell scripts. It is -# based on the popular JUnit unit testing framework for Java. - -# return if shunit already loaded -[ -n "${SHUNIT_VERSION:-}" ] && exit 0 - -SHUNIT_VERSION='2.1.6' - -SHUNIT_TRUE=0 -SHUNIT_FALSE=1 -SHUNIT_ERROR=2 - -# enable strict mode by default -SHUNIT_STRICT=${SHUNIT_STRICT:-${SHUNIT_TRUE}} - -_shunit_warn() { echo "shunit2:WARN $@" >&2; } -_shunit_error() { echo "shunit2:ERROR $@" >&2; } -_shunit_fatal() { echo "shunit2:FATAL $@" >&2; exit ${SHUNIT_ERROR}; } - -# specific shell checks -if [ -n "${ZSH_VERSION:-}" ]; then - setopt |grep "^shwordsplit$" >/dev/null - if [ $? -ne ${SHUNIT_TRUE} ]; then - _shunit_fatal 'zsh shwordsplit option is required for proper operation' - fi - if [ -z "${SHUNIT_PARENT:-}" ]; then - _shunit_fatal "zsh does not pass \$0 through properly. please declare \ -\"SHUNIT_PARENT=\$0\" before calling shUnit2" - fi -fi - -# -# constants -# - -__SHUNIT_ASSERT_MSG_PREFIX='ASSERT:' -__SHUNIT_MODE_SOURCED='sourced' -__SHUNIT_MODE_STANDALONE='standalone' -__SHUNIT_PARENT=${SHUNIT_PARENT:-$0} - -# set the constants readonly -shunit_constants_=`set |grep '^__SHUNIT_' |cut -d= -f1` -echo "${shunit_constants_}" |grep '^Binary file' >/dev/null && \ - shunit_constants_=`set |grep -a '^__SHUNIT_' |cut -d= -f1` -for shunit_constant_ in ${shunit_constants_}; do - shunit_ro_opts_='' - case ${ZSH_VERSION:-} in - '') ;; # this isn't zsh - [123].*) ;; # early versions (1.x, 2.x, 3.x) - *) shunit_ro_opts_='-g' ;; # all later versions. declare readonly globally - esac - readonly ${shunit_ro_opts_} ${shunit_constant_} -done -unset shunit_constant_ shunit_constants_ shunit_ro_opts_ - -# variables -__shunit_lineno='' # line number of executed test -__shunit_mode=${__SHUNIT_MODE_SOURCED} # operating mode -__shunit_reportGenerated=${SHUNIT_FALSE} # is report generated -__shunit_script='' # filename of unittest script (standalone mode) -__shunit_skip=${SHUNIT_FALSE} # is skipping enabled -__shunit_suite='' # suite of tests to execute - -# counts of tests -__shunit_testSuccess=${SHUNIT_TRUE} -__shunit_testsTotal=0 -__shunit_testsPassed=0 -__shunit_testsFailed=0 - -# counts of asserts -__shunit_assertsTotal=0 -__shunit_assertsPassed=0 -__shunit_assertsFailed=0 -__shunit_assertsSkipped=0 - -# macros -_SHUNIT_LINENO_='eval __shunit_lineno=""; if [ "${1:-}" = "--lineno" ]; then [ -n "$2" ] && __shunit_lineno="[$2] "; shift 2; fi' - -#----------------------------------------------------------------------------- -# assert functions -# - -# Assert that two values are equal to one another. -# -# Args: -# message: string: failure message [optional] -# expected: string: expected value -# actual: string: actual value -# Returns: -# integer: success (TRUE/FALSE/ERROR constant) -assertEquals() -{ - ${_SHUNIT_LINENO_} - if [ $# -lt 2 -o $# -gt 3 ]; then - _shunit_error "assertEquals() requires two or three arguments; $# given" - _shunit_error "1: ${1:+$1} 2: ${2:+$2} 3: ${3:+$3}${4:+ 4: $4}" - return ${SHUNIT_ERROR} - fi - _shunit_shouldSkip && return ${SHUNIT_TRUE} - - shunit_message_=${__shunit_lineno} - if [ $# -eq 3 ]; then - shunit_message_="${shunit_message_}$1" - shift - fi - shunit_expected_=$1 - shunit_actual_=$2 - - shunit_return=${SHUNIT_TRUE} - if [ "${shunit_expected_}" = "${shunit_actual_}" ]; then - _shunit_assertPass - else - failNotEquals "${shunit_message_}" "${shunit_expected_}" "${shunit_actual_}" - shunit_return=${SHUNIT_FALSE} - fi - - unset shunit_message_ shunit_expected_ shunit_actual_ - return ${shunit_return} -} -_ASSERT_EQUALS_='eval assertEquals --lineno "${LINENO:-}"' - -# Assert that two values are not equal to one another. -# -# Args: -# message: string: failure message [optional] -# expected: string: expected value -# actual: string: actual value -# Returns: -# integer: success (TRUE/FALSE/ERROR constant) -assertNotEquals() -{ - ${_SHUNIT_LINENO_} - if [ $# -lt 2 -o $# -gt 3 ]; then - _shunit_error "assertNotEquals() requires two or three arguments; $# given" - return ${SHUNIT_ERROR} - fi - _shunit_shouldSkip && return ${SHUNIT_TRUE} - - shunit_message_=${__shunit_lineno} - if [ $# -eq 3 ]; then - shunit_message_="${shunit_message_}$1" - shift - fi - shunit_expected_=$1 - shunit_actual_=$2 - - shunit_return=${SHUNIT_TRUE} - if [ "${shunit_expected_}" != "${shunit_actual_}" ]; then - _shunit_assertPass - else - failSame "${shunit_message_}" "$@" - shunit_return=${SHUNIT_FALSE} - fi - - unset shunit_message_ shunit_expected_ shunit_actual_ - return ${shunit_return} -} -_ASSERT_NOT_EQUALS_='eval assertNotEquals --lineno "${LINENO:-}"' - -# Assert that a value is null (i.e. an empty string) -# -# Args: -# message: string: failure message [optional] -# actual: string: actual value -# Returns: -# integer: success (TRUE/FALSE/ERROR constant) -assertNull() -{ - ${_SHUNIT_LINENO_} - if [ $# -lt 1 -o $# -gt 2 ]; then - _shunit_error "assertNull() requires one or two arguments; $# given" - return ${SHUNIT_ERROR} - fi - _shunit_shouldSkip && return ${SHUNIT_TRUE} - - shunit_message_=${__shunit_lineno} - if [ $# -eq 2 ]; then - shunit_message_="${shunit_message_}$1" - shift - fi - assertTrue "${shunit_message_}" "[ -z '$1' ]" - shunit_return=$? - - unset shunit_message_ - return ${shunit_return} -} -_ASSERT_NULL_='eval assertNull --lineno "${LINENO:-}"' - -# Assert that a value is not null (i.e. a non-empty string) -# -# Args: -# message: string: failure message [optional] -# actual: string: actual value -# Returns: -# integer: success (TRUE/FALSE/ERROR constant) -assertNotNull() -{ - ${_SHUNIT_LINENO_} - if [ $# -gt 2 ]; then # allowing 0 arguments as $1 might actually be null - _shunit_error "assertNotNull() requires one or two arguments; $# given" - return ${SHUNIT_ERROR} - fi - _shunit_shouldSkip && return ${SHUNIT_TRUE} - - shunit_message_=${__shunit_lineno} - if [ $# -eq 2 ]; then - shunit_message_="${shunit_message_}$1" - shift - fi - shunit_actual_=`_shunit_escapeCharactersInString "${1:-}"` - test -n "${shunit_actual_}" - assertTrue "${shunit_message_}" $? - shunit_return=$? - - unset shunit_actual_ shunit_message_ - return ${shunit_return} -} -_ASSERT_NOT_NULL_='eval assertNotNull --lineno "${LINENO:-}"' - -# Assert that two values are the same (i.e. equal to one another). -# -# Args: -# message: string: failure message [optional] -# expected: string: expected value -# actual: string: actual value -# Returns: -# integer: success (TRUE/FALSE/ERROR constant) -assertSame() -{ - ${_SHUNIT_LINENO_} - if [ $# -lt 2 -o $# -gt 3 ]; then - _shunit_error "assertSame() requires two or three arguments; $# given" - return ${SHUNIT_ERROR} - fi - _shunit_shouldSkip && return ${SHUNIT_TRUE} - - shunit_message_=${__shunit_lineno} - if [ $# -eq 3 ]; then - shunit_message_="${shunit_message_}$1" - shift - fi - assertEquals "${shunit_message_}" "$1" "$2" - shunit_return=$? - - unset shunit_message_ - return ${shunit_return} -} -_ASSERT_SAME_='eval assertSame --lineno "${LINENO:-}"' - -# Assert that two values are not the same (i.e. not equal to one another). -# -# Args: -# message: string: failure message [optional] -# expected: string: expected value -# actual: string: actual value -# Returns: -# integer: success (TRUE/FALSE/ERROR constant) -assertNotSame() -{ - ${_SHUNIT_LINENO_} - if [ $# -lt 2 -o $# -gt 3 ]; then - _shunit_error "assertNotSame() requires two or three arguments; $# given" - return ${SHUNIT_ERROR} - fi - _shunit_shouldSkip && return ${SHUNIT_TRUE} - - shunit_message_=${__shunit_lineno} - if [ $# -eq 3 ]; then - shunit_message_="${shunit_message_:-}$1" - shift - fi - assertNotEquals "${shunit_message_}" "$1" "$2" - shunit_return=$? - - unset shunit_message_ - return ${shunit_return} -} -_ASSERT_NOT_SAME_='eval assertNotSame --lineno "${LINENO:-}"' - -# Assert that a value or shell test condition is true. -# -# In shell, a value of 0 is true and a non-zero value is false. Any integer -# value passed can thereby be tested. -# -# Shell supports much more complicated tests though, and a means to support -# them was needed. As such, this function tests that conditions are true or -# false through evaluation rather than just looking for a true or false. -# -# The following test will succeed: -# assertTrue 0 -# assertTrue "[ 34 -gt 23 ]" -# The folloing test will fail with a message: -# assertTrue 123 -# assertTrue "test failed" "[ -r '/non/existant/file' ]" -# -# Args: -# message: string: failure message [optional] -# condition: string: integer value or shell conditional statement -# Returns: -# integer: success (TRUE/FALSE/ERROR constant) -assertTrue() -{ - ${_SHUNIT_LINENO_} - if [ $# -gt 2 ]; then - _shunit_error "assertTrue() takes one two arguments; $# given" - return ${SHUNIT_ERROR} - fi - _shunit_shouldSkip && return ${SHUNIT_TRUE} - - shunit_message_=${__shunit_lineno} - if [ $# -eq 2 ]; then - shunit_message_="${shunit_message_}$1" - shift - fi - shunit_condition_=$1 - - # see if condition is an integer, i.e. a return value - shunit_match_=`expr "${shunit_condition_}" : '\([0-9]*\)'` - shunit_return=${SHUNIT_TRUE} - if [ -z "${shunit_condition_}" ]; then - # null condition - shunit_return=${SHUNIT_FALSE} - elif [ -n "${shunit_match_}" -a "${shunit_condition_}" = "${shunit_match_}" ] - then - # possible return value. treating 0 as true, and non-zero as false. - [ ${shunit_condition_} -ne 0 ] && shunit_return=${SHUNIT_FALSE} - else - # (hopefully) a condition - ( eval ${shunit_condition_} ) >/dev/null 2>&1 - [ $? -ne 0 ] && shunit_return=${SHUNIT_FALSE} - fi - - # record the test - if [ ${shunit_return} -eq ${SHUNIT_TRUE} ]; then - _shunit_assertPass - else - _shunit_assertFail "${shunit_message_}" - fi - - unset shunit_message_ shunit_condition_ shunit_match_ - return ${shunit_return} -} -_ASSERT_TRUE_='eval assertTrue --lineno "${LINENO:-}"' - -# Assert that a value or shell test condition is false. -# -# In shell, a value of 0 is true and a non-zero value is false. Any integer -# value passed can thereby be tested. -# -# Shell supports much more complicated tests though, and a means to support -# them was needed. As such, this function tests that conditions are true or -# false through evaluation rather than just looking for a true or false. -# -# The following test will succeed: -# assertFalse 1 -# assertFalse "[ 'apples' = 'oranges' ]" -# The folloing test will fail with a message: -# assertFalse 0 -# assertFalse "test failed" "[ 1 -eq 1 -a 2 -eq 2 ]" -# -# Args: -# message: string: failure message [optional] -# condition: string: integer value or shell conditional statement -# Returns: -# integer: success (TRUE/FALSE/ERROR constant) -assertFalse() -{ - ${_SHUNIT_LINENO_} - if [ $# -lt 1 -o $# -gt 2 ]; then - _shunit_error "assertFalse() quires one or two arguments; $# given" - return ${SHUNIT_ERROR} - fi - _shunit_shouldSkip && return ${SHUNIT_TRUE} - - shunit_message_=${__shunit_lineno} - if [ $# -eq 2 ]; then - shunit_message_="${shunit_message_}$1" - shift - fi - shunit_condition_=$1 - - # see if condition is an integer, i.e. a return value - shunit_match_=`expr "${shunit_condition_}" : '\([0-9]*\)'` - shunit_return=${SHUNIT_TRUE} - if [ -z "${shunit_condition_}" ]; then - # null condition - shunit_return=${SHUNIT_FALSE} - elif [ -n "${shunit_match_}" -a "${shunit_condition_}" = "${shunit_match_}" ] - then - # possible return value. treating 0 as true, and non-zero as false. - [ ${shunit_condition_} -eq 0 ] && shunit_return=${SHUNIT_FALSE} - else - # (hopefully) a condition - ( eval ${shunit_condition_} ) >/dev/null 2>&1 - [ $? -eq 0 ] && shunit_return=${SHUNIT_FALSE} - fi - - # record the test - if [ ${shunit_return} -eq ${SHUNIT_TRUE} ]; then - _shunit_assertPass - else - _shunit_assertFail "${shunit_message_}" - fi - - unset shunit_message_ shunit_condition_ shunit_match_ - return ${shunit_return} -} -_ASSERT_FALSE_='eval assertFalse --lineno "${LINENO:-}"' - -#----------------------------------------------------------------------------- -# failure functions -# - -# Records a test failure. -# -# Args: -# message: string: failure message [optional] -# Returns: -# integer: success (TRUE/FALSE/ERROR constant) -fail() -{ - ${_SHUNIT_LINENO_} - if [ $# -gt 1 ]; then - _shunit_error "fail() requires zero or one arguments; $# given" - return ${SHUNIT_ERROR} - fi - _shunit_shouldSkip && return ${SHUNIT_TRUE} - - shunit_message_=${__shunit_lineno} - if [ $# -eq 1 ]; then - shunit_message_="${shunit_message_}$1" - shift - fi - - _shunit_assertFail "${shunit_message_}" - - unset shunit_message_ - return ${SHUNIT_FALSE} -} -_FAIL_='eval fail --lineno "${LINENO:-}"' - -# Records a test failure, stating two values were not equal. -# -# Args: -# message: string: failure message [optional] -# expected: string: expected value -# actual: string: actual value -# Returns: -# integer: success (TRUE/FALSE/ERROR constant) -failNotEquals() -{ - ${_SHUNIT_LINENO_} - if [ $# -lt 2 -o $# -gt 3 ]; then - _shunit_error "failNotEquals() requires one or two arguments; $# given" - return ${SHUNIT_ERROR} - fi - _shunit_shouldSkip && return ${SHUNIT_TRUE} - - shunit_message_=${__shunit_lineno} - if [ $# -eq 3 ]; then - shunit_message_="${shunit_message_}$1" - shift - fi - shunit_expected_=$1 - shunit_actual_=$2 - - _shunit_assertFail "${shunit_message_:+${shunit_message_} }expected:<${shunit_expected_}> but was:<${shunit_actual_}>" - - unset shunit_message_ shunit_expected_ shunit_actual_ - return ${SHUNIT_FALSE} -} -_FAIL_NOT_EQUALS_='eval failNotEquals --lineno "${LINENO:-}"' - -# Records a test failure, stating two values should have been the same. -# -# Args: -# message: string: failure message [optional] -# expected: string: expected value -# actual: string: actual value -# Returns: -# integer: success (TRUE/FALSE/ERROR constant) -failSame() -{ - ${_SHUNIT_LINENO_} - if [ $# -lt 2 -o $# -gt 3 ]; then - _shunit_error "failSame() requires two or three arguments; $# given" - return ${SHUNIT_ERROR} - fi - _shunit_shouldSkip && return ${SHUNIT_TRUE} - - shunit_message_=${__shunit_lineno} - if [ $# -eq 3 ]; then - shunit_message_="${shunit_message_}$1" - shift - fi - - _shunit_assertFail "${shunit_message_:+${shunit_message_} }expected not same" - - unset shunit_message_ - return ${SHUNIT_FALSE} -} -_FAIL_SAME_='eval failSame --lineno "${LINENO:-}"' - -# Records a test failure, stating two values were not equal. -# -# This is functionally equivalent to calling failNotEquals(). -# -# Args: -# message: string: failure message [optional] -# expected: string: expected value -# actual: string: actual value -# Returns: -# integer: success (TRUE/FALSE/ERROR constant) -failNotSame() -{ - ${_SHUNIT_LINENO_} - if [ $# -lt 2 -o $# -gt 3 ]; then - _shunit_error "failNotEquals() requires one or two arguments; $# given" - return ${SHUNIT_ERROR} - fi - _shunit_shouldSkip && return ${SHUNIT_TRUE} - - shunit_message_=${__shunit_lineno} - if [ $# -eq 3 ]; then - shunit_message_="${shunit_message_}$1" - shift - fi - failNotEquals "${shunit_message_}" "$1" "$2" - shunit_return=$? - - unset shunit_message_ - return ${shunit_return} -} -_FAIL_NOT_SAME_='eval failNotSame --lineno "${LINENO:-}"' - -#----------------------------------------------------------------------------- -# skipping functions -# - -# Force remaining assert and fail functions to be "skipped". -# -# This function forces the remaining assert and fail functions to be "skipped", -# i.e. they will have no effect. Each function skipped will be recorded so that -# the total of asserts and fails will not be altered. -# -# Args: -# None -startSkipping() -{ - __shunit_skip=${SHUNIT_TRUE} -} - -# Resume the normal recording behavior of assert and fail calls. -# -# Args: -# None -endSkipping() -{ - __shunit_skip=${SHUNIT_FALSE} -} - -# Returns the state of assert and fail call skipping. -# -# Args: -# None -# Returns: -# boolean: (TRUE/FALSE constant) -isSkipping() -{ - return ${__shunit_skip} -} - -#----------------------------------------------------------------------------- -# suite functions -# - -# Stub. This function should contains all unit test calls to be made. -# -# DEPRECATED (as of 2.1.0) -# -# This function can be optionally overridden by the user in their test suite. -# -# If this function exists, it will be called when shunit2 is sourced. If it -# does not exist, shunit2 will search the parent script for all functions -# beginning with the word 'test', and they will be added dynamically to the -# test suite. -# -# This function should be overridden by the user in their unit test suite. -# Note: see _shunit_mktempFunc() for actual implementation -# -# Args: -# None -#suite() { :; } # DO NOT UNCOMMENT THIS FUNCTION - -# Adds a function name to the list of tests schedule for execution. -# -# This function should only be called from within the suite() function. -# -# Args: -# function: string: name of a function to add to current unit test suite -suite_addTest() -{ - shunit_func_=${1:-} - - __shunit_suite="${__shunit_suite:+${__shunit_suite} }${shunit_func_}" - __shunit_testsTotal=`expr ${__shunit_testsTotal} + 1` - - unset shunit_func_ -} - -# Stub. This function will be called once before any tests are run. -# -# Common one-time environment preparation tasks shared by all tests can be -# defined here. -# -# This function should be overridden by the user in their unit test suite. -# Note: see _shunit_mktempFunc() for actual implementation -# -# Args: -# None -#oneTimeSetUp() { :; } # DO NOT UNCOMMENT THIS FUNCTION - -# Stub. This function will be called once after all tests are finished. -# -# Common one-time environment cleanup tasks shared by all tests can be defined -# here. -# -# This function should be overridden by the user in their unit test suite. -# Note: see _shunit_mktempFunc() for actual implementation -# -# Args: -# None -#oneTimeTearDown() { :; } # DO NOT UNCOMMENT THIS FUNCTION - -# Stub. This function will be called before each test is run. -# -# Common environment preparation tasks shared by all tests can be defined here. -# -# This function should be overridden by the user in their unit test suite. -# Note: see _shunit_mktempFunc() for actual implementation -# -# Args: -# None -#setUp() { :; } - -# Note: see _shunit_mktempFunc() for actual implementation -# Stub. This function will be called after each test is run. -# -# Common environment cleanup tasks shared by all tests can be defined here. -# -# This function should be overridden by the user in their unit test suite. -# Note: see _shunit_mktempFunc() for actual implementation -# -# Args: -# None -#tearDown() { :; } # DO NOT UNCOMMENT THIS FUNCTION - -#------------------------------------------------------------------------------ -# internal shUnit2 functions -# - -# Create a temporary directory to store various run-time files in. -# -# This function is a cross-platform temporary directory creation tool. Not all -# OSes have the mktemp function, so one is included here. -# -# Args: -# None -# Outputs: -# string: the temporary directory that was created -_shunit_mktempDir() -{ - # try the standard mktemp function - ( exec mktemp -dqt shunit.XXXXXX 2>/dev/null ) && return - - # the standard mktemp didn't work. doing our own. - if [ -r '/dev/urandom' -a -x '/usr/bin/od' ]; then - _shunit_random_=`/usr/bin/od -vAn -N4 -tx4 "${_shunit_file_}" -#! /bin/sh -exit ${SHUNIT_TRUE} -EOF - chmod +x "${_shunit_file_}" - done - - unset _shunit_file_ -} - -# Final cleanup function to leave things as we found them. -# -# Besides removing the temporary directory, this function is in charge of the -# final exit code of the unit test. The exit code is based on how the script -# was ended (e.g. normal exit, or via Ctrl-C). -# -# Args: -# name: string: name of the trap called (specified when trap defined) -_shunit_cleanup() -{ - _shunit_name_=$1 - - case ${_shunit_name_} in - EXIT) _shunit_signal_=0 ;; - INT) _shunit_signal_=2 ;; - TERM) _shunit_signal_=15 ;; - *) - _shunit_warn "unrecognized trap value (${_shunit_name_})" - _shunit_signal_=0 - ;; - esac - - # do our work - rm -fr "${__shunit_tmpDir}" - - # exit for all non-EXIT signals - if [ ${_shunit_name_} != 'EXIT' ]; then - _shunit_warn "trapped and now handling the (${_shunit_name_}) signal" - # disable EXIT trap - trap 0 - # add 128 to signal and exit - exit `expr ${_shunit_signal_} + 128` - elif [ ${__shunit_reportGenerated} -eq ${SHUNIT_FALSE} ] ; then - _shunit_assertFail 'Unknown failure encountered running a test' - _shunit_generateReport - exit ${SHUNIT_ERROR} - fi - - unset _shunit_name_ _shunit_signal_ -} - -# The actual running of the tests happens here. -# -# Args: -# None -_shunit_execSuite() -{ - for _shunit_test_ in ${__shunit_suite}; do - __shunit_testSuccess=${SHUNIT_TRUE} - - # disable skipping - endSkipping - - # execute the per-test setup function - setUp - - # execute the test - echo "${_shunit_test_}" - eval ${_shunit_test_} - - # execute the per-test tear-down function - tearDown - - # update stats - if [ ${__shunit_testSuccess} -eq ${SHUNIT_TRUE} ]; then - __shunit_testsPassed=`expr ${__shunit_testsPassed} + 1` - else - __shunit_testsFailed=`expr ${__shunit_testsFailed} + 1` - fi - done - - unset _shunit_test_ -} - -# Generates the user friendly report with appropriate OK/FAILED message. -# -# Args: -# None -# Output: -# string: the report of successful and failed tests, as well as totals. -_shunit_generateReport() -{ - _shunit_ok_=${SHUNIT_TRUE} - - # if no exit code was provided one, determine an appropriate one - [ ${__shunit_testsFailed} -gt 0 \ - -o ${__shunit_testSuccess} -eq ${SHUNIT_FALSE} ] \ - && _shunit_ok_=${SHUNIT_FALSE} - - echo - if [ ${__shunit_testsTotal} -eq 1 ]; then - echo "Ran ${__shunit_testsTotal} test." - else - echo "Ran ${__shunit_testsTotal} tests." - fi - - _shunit_failures_='' - _shunit_skipped_='' - [ ${__shunit_assertsFailed} -gt 0 ] \ - && _shunit_failures_="failures=${__shunit_assertsFailed}" - [ ${__shunit_assertsSkipped} -gt 0 ] \ - && _shunit_skipped_="skipped=${__shunit_assertsSkipped}" - - if [ ${_shunit_ok_} -eq ${SHUNIT_TRUE} ]; then - _shunit_msg_='OK' - [ -n "${_shunit_skipped_}" ] \ - && _shunit_msg_="${_shunit_msg_} (${_shunit_skipped_})" - else - _shunit_msg_="FAILED (${_shunit_failures_}" - [ -n "${_shunit_skipped_}" ] \ - && _shunit_msg_="${_shunit_msg_},${_shunit_skipped_}" - _shunit_msg_="${_shunit_msg_})" - fi - - echo - echo ${_shunit_msg_} - __shunit_reportGenerated=${SHUNIT_TRUE} - - unset _shunit_failures_ _shunit_msg_ _shunit_ok_ _shunit_skipped_ -} - -# Test for whether a function should be skipped. -# -# Args: -# None -# Returns: -# boolean: whether the test should be skipped (TRUE/FALSE constant) -_shunit_shouldSkip() -{ - [ ${__shunit_skip} -eq ${SHUNIT_FALSE} ] && return ${SHUNIT_FALSE} - _shunit_assertSkip -} - -# Records a successful test. -# -# Args: -# None -_shunit_assertPass() -{ - __shunit_assertsPassed=`expr ${__shunit_assertsPassed} + 1` - __shunit_assertsTotal=`expr ${__shunit_assertsTotal} + 1` -} - -# Records a test failure. -# -# Args: -# message: string: failure message to provide user -_shunit_assertFail() -{ - _shunit_msg_=$1 - - __shunit_testSuccess=${SHUNIT_FALSE} - __shunit_assertsFailed=`expr ${__shunit_assertsFailed} + 1` - __shunit_assertsTotal=`expr ${__shunit_assertsTotal} + 1` - echo "${__SHUNIT_ASSERT_MSG_PREFIX}${_shunit_msg_}" - - unset _shunit_msg_ -} - -# Records a skipped test. -# -# Args: -# None -_shunit_assertSkip() -{ - __shunit_assertsSkipped=`expr ${__shunit_assertsSkipped} + 1` - __shunit_assertsTotal=`expr ${__shunit_assertsTotal} + 1` -} - -# Prepare a script filename for sourcing. -# -# Args: -# script: string: path to a script to source -# Returns: -# string: filename prefixed with ./ (if necessary) -_shunit_prepForSourcing() -{ - _shunit_script_=$1 - case "${_shunit_script_}" in - /*|./*) echo "${_shunit_script_}" ;; - *) echo "./${_shunit_script_}" ;; - esac - unset _shunit_script_ -} - -# Escape a character in a string. -# -# Args: -# c: string: unescaped character -# s: string: to escape character in -# Returns: -# string: with escaped character(s) -_shunit_escapeCharInStr() -{ - [ -n "$2" ] || return # no point in doing work on an empty string - - # Note: using shorter variable names to prevent conflicts with - # _shunit_escapeCharactersInString(). - _shunit_c_=$1 - _shunit_s_=$2 - - - # escape the character - echo ''${_shunit_s_}'' |sed 's/\'${_shunit_c_}'/\\\'${_shunit_c_}'/g' - - unset _shunit_c_ _shunit_s_ -} - -# Escape a character in a string. -# -# Args: -# str: string: to escape characters in -# Returns: -# string: with escaped character(s) -_shunit_escapeCharactersInString() -{ - [ -n "$1" ] || return # no point in doing work on an empty string - - _shunit_str_=$1 - - # Note: using longer variable names to prevent conflicts with - # _shunit_escapeCharInStr(). - for _shunit_char_ in '"' '$' "'" '`'; do - _shunit_str_=`_shunit_escapeCharInStr "${_shunit_char_}" "${_shunit_str_}"` - done - - echo "${_shunit_str_}" - unset _shunit_char_ _shunit_str_ -} - -# Extract list of functions to run tests against. -# -# Args: -# script: string: name of script to extract functions from -# Returns: -# string: of function names -_shunit_extractTestFunctions() -{ - _shunit_script_=$1 - - # extract the lines with test function names, strip of anything besides the - # function name, and output everything on a single line. - _shunit_regex_='^[ ]*(function )*test[A-Za-z0-9_]* *\(\)' - egrep "${_shunit_regex_}" "${_shunit_script_}" \ - |sed 's/^[^A-Za-z0-9_]*//;s/^function //;s/\([A-Za-z0-9_]*\).*/\1/g' \ - |xargs - - unset _shunit_regex_ _shunit_script_ -} - -#------------------------------------------------------------------------------ -# main -# - -# determine the operating mode -if [ $# -eq 0 ]; then - __shunit_script=${__SHUNIT_PARENT} - __shunit_mode=${__SHUNIT_MODE_SOURCED} -else - __shunit_script=$1 - [ -r "${__shunit_script}" ] || \ - _shunit_fatal "unable to read from ${__shunit_script}" - __shunit_mode=${__SHUNIT_MODE_STANDALONE} -fi - -# create a temporary storage location -__shunit_tmpDir=`_shunit_mktempDir` - -# provide a public temporary directory for unit test scripts -# TODO(kward): document this -SHUNIT_TMPDIR="${__shunit_tmpDir}/tmp" -mkdir "${SHUNIT_TMPDIR}" - -# setup traps to clean up after ourselves -trap '_shunit_cleanup EXIT' 0 -trap '_shunit_cleanup INT' 2 -trap '_shunit_cleanup TERM' 15 - -# create phantom functions to work around issues with Cygwin -_shunit_mktempFunc -PATH="${__shunit_tmpDir}:${PATH}" - -# make sure phantom functions are executable. this will bite if /tmp (or the -# current $TMPDIR) points to a path on a partition that was mounted with the -# 'noexec' option. the noexec command was created with _shunit_mktempFunc(). -noexec 2>/dev/null || _shunit_fatal \ - 'please declare TMPDIR with path on partition with exec permission' - -# we must manually source the tests in standalone mode -if [ "${__shunit_mode}" = "${__SHUNIT_MODE_STANDALONE}" ]; then - . "`_shunit_prepForSourcing \"${__shunit_script}\"`" -fi - -# execute the oneTimeSetUp function (if it exists) -oneTimeSetUp - -# execute the suite function defined in the parent test script -# deprecated as of 2.1.0 -suite - -# if no suite function was defined, dynamically build a list of functions -if [ -z "${__shunit_suite}" ]; then - shunit_funcs_=`_shunit_extractTestFunctions "${__shunit_script}"` - for shunit_func_ in ${shunit_funcs_}; do - suite_addTest ${shunit_func_} - done -fi -unset shunit_func_ shunit_funcs_ - -# execute the tests -_shunit_execSuite - -# execute the oneTimeTearDown function (if it exists) -oneTimeTearDown - -# generate the report -_shunit_generateReport - -# that's it folks -[ ${__shunit_testsFailed} -eq 0 ] -exit $? diff --git a/test/test.d/db-move.bats b/test/test.d/db-move.bats new file mode 100644 index 0000000..a9a58d0 --- /dev/null +++ b/test/test.d/db-move.bats @@ -0,0 +1,117 @@ +load ../lib/common + +@test "testMoveSimplePackages" { + local arches=('i686' 'x86_64') + local pkgs=('pkg-simple-a' 'pkg-simple-b') + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + releasePackage testing ${pkgbase} ${arch} + done + done + + db-update + + db-move testing extra pkg-simple-a + + for arch in ${arches[@]}; do + checkPackage extra pkg-simple-a-1-1-${arch}.pkg.tar.xz ${arch} + checkRemovedPackage testing pkg-simple-a-1-1-${arch}.pkg.tar.xz ${arch} + + checkPackage testing pkg-simple-b-1-1-${arch}.pkg.tar.xz ${arch} + done +} + +@test "testMoveMultiplePackages" { + local arches=('i686' 'x86_64') + local pkgs=('pkg-simple-a' 'pkg-simple-b') + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + releasePackage testing ${pkgbase} ${arch} + done + done + + db-update + + db-move testing extra pkg-simple-a pkg-simple-b + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + checkPackage extra ${pkgbase}-1-1-${arch}.pkg.tar.xz ${arch} + checkRemovedPackage testing ${pkgbase}-1-1-${arch}.pkg.tar.xz ${arch} + done + done +} + +@test "testMoveEpochPackages" { + local arches=('i686' 'x86_64') + local pkgs=('pkg-simple-epoch') + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + releasePackage testing ${pkgbase} ${arch} + done + done + + db-update + + db-move testing extra pkg-simple-epoch + + for arch in ${arches[@]}; do + checkPackage extra pkg-simple-epoch-1:1-1-${arch}.pkg.tar.xz ${arch} + checkRemovedPackage testing pkg-simple-epoch-1:1-1-${arch}.pkg.tar.xz ${arch} + done +} + +@test "testMoveAnyPackages" { + local pkgs=('pkg-any-a' 'pkg-any-b') + local pkgbase + + for pkgbase in ${pkgs[@]}; do + releasePackage testing ${pkgbase} any + done + + db-update + db-move testing extra pkg-any-a + + checkAnyPackage extra pkg-any-a-1-1-any.pkg.tar.xz + checkRemovedAnyPackage testing pkg-any-a + checkAnyPackage testing pkg-any-b-1-1-any.pkg.tar.xz +} + +@test "testMoveSplitPackages" { + local arches=('i686' 'x86_64') + local pkgs=('pkg-split-a' 'pkg-split-b') + local pkg + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + releasePackage testing ${pkgbase} ${arch} + done + done + + db-update + db-move testing extra pkg-split-a + + for arch in ${arches[@]}; do + for pkg in $(getPackageNamesFromPackageBase pkg-split-a); do + checkPackage extra ${pkg} ${arch} + done + done + for arch in ${arches[@]}; do + for pkg in $(getPackageNamesFromPackageBase pkg-split-b); do + checkPackage testing ${pkg} ${arch} + done + done + + checkRemovedAnyPackage testing pkg-split-a +} diff --git a/test/test.d/db-move.sh b/test/test.d/db-move.sh deleted file mode 100755 index 4da5ae2..0000000 --- a/test/test.d/db-move.sh +++ /dev/null @@ -1,115 +0,0 @@ -testMoveSimplePackages() { - local arches=('i686' 'x86_64') - local pkgs=('pkg-simple-a' 'pkg-simple-b') - local pkgbase - local arch - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage testing ${pkgbase} ${arch} - done - done - - db-update - - db-move testing extra pkg-simple-a - - for arch in ${arches[@]}; do - checkPackage extra pkg-simple-a-1-1-${arch}.pkg.tar.xz ${arch} - checkRemovedPackage testing pkg-simple-a-1-1-${arch}.pkg.tar.xz ${arch} - - checkPackage testing pkg-simple-b-1-1-${arch}.pkg.tar.xz ${arch} - done -} - -testMoveMultiplePackages() { - local arches=('i686' 'x86_64') - local pkgs=('pkg-simple-a' 'pkg-simple-b') - local pkgbase - local arch - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage testing ${pkgbase} ${arch} - done - done - - db-update - - db-move testing extra pkg-simple-a pkg-simple-b - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - checkPackage extra ${pkgbase}-1-1-${arch}.pkg.tar.xz ${arch} - checkRemovedPackage testing ${pkgbase}-1-1-${arch}.pkg.tar.xz ${arch} - done - done -} - -testMoveEpochPackages() { - local arches=('i686' 'x86_64') - local pkgs=('pkg-simple-epoch') - local pkgbase - local arch - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage testing ${pkgbase} ${arch} - done - done - - db-update - - db-move testing extra pkg-simple-epoch - - for arch in ${arches[@]}; do - checkPackage extra pkg-simple-epoch-1:1-1-${arch}.pkg.tar.xz ${arch} - checkRemovedPackage testing pkg-simple-epoch-1:1-1-${arch}.pkg.tar.xz ${arch} - done -} - -testMoveAnyPackages() { - local pkgs=('pkg-any-a' 'pkg-any-b') - local pkgbase - - for pkgbase in ${pkgs[@]}; do - releasePackage testing ${pkgbase} any - done - - db-update - db-move testing extra pkg-any-a - - checkAnyPackage extra pkg-any-a-1-1-any.pkg.tar.xz - checkRemovedAnyPackage testing pkg-any-a - checkAnyPackage testing pkg-any-b-1-1-any.pkg.tar.xz -} - -testMoveSplitPackages() { - local arches=('i686' 'x86_64') - local pkgs=('pkg-split-a' 'pkg-split-b') - local pkg - local pkgbase - local arch - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage testing ${pkgbase} ${arch} - done - done - - db-update - db-move testing extra pkg-split-a - - for arch in ${arches[@]}; do - for pkg in "${pkgdir}/pkg-split-a"/*-${arch}${PKGEXT}; do - checkPackage extra ${pkg##*/} ${arch} - done - done - for arch in ${arches[@]}; do - for pkg in "${pkgdir}/pkg-split-b"/*-${arch}${PKGEXT}; do - checkPackage testing ${pkg##*/} ${arch} - done - done - - checkRemovedAnyPackage testing pkg-split-a -} diff --git a/test/test.d/db-remove.bats b/test/test.d/db-remove.bats new file mode 100644 index 0000000..bc4a981 --- /dev/null +++ b/test/test.d/db-remove.bats @@ -0,0 +1,72 @@ +load ../lib/common + +@test "testRemovePackages" { + local arches=('i686' 'x86_64') + local pkgs=('pkg-simple-a' 'pkg-simple-b' 'pkg-split-a' 'pkg-split-b' 'pkg-simple-epoch') + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + releasePackage extra ${pkgbase} ${arch} + done + done + + db-update + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + db-remove extra ${arch} ${pkgbase} + done + done + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + checkRemovedPackage extra ${pkgbase} ${arch} + done + done +} + +@test "testRemoveMultiplePackages" { + local arches=('i686' 'x86_64') + local pkgs=('pkg-simple-a' 'pkg-simple-b' 'pkg-split-a' 'pkg-split-b' 'pkg-simple-epoch') + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + releasePackage extra ${pkgbase} ${arch} + done + done + + db-update + + for arch in ${arches[@]}; do + db-remove extra ${arch} ${pkgs[@]} + done + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + checkRemovedPackage extra ${pkgbase} ${arch} + done + done +} + +@test "testRemoveAnyPackages" { + local pkgs=('pkg-any-a' 'pkg-any-b') + local pkgbase + + for pkgbase in ${pkgs[@]}; do + releasePackage extra ${pkgbase} any + done + + db-update + + for pkgbase in ${pkgs[@]}; do + db-remove extra any ${pkgbase} + done + + for pkgbase in ${pkgs[@]}; do + checkRemovedAnyPackage extra ${pkgbase} + done +} diff --git a/test/test.d/db-remove.sh b/test/test.d/db-remove.sh deleted file mode 100755 index eab1ef4..0000000 --- a/test/test.d/db-remove.sh +++ /dev/null @@ -1,70 +0,0 @@ -testRemovePackages() { - local arches=('i686' 'x86_64') - local pkgs=('pkg-simple-a' 'pkg-simple-b' 'pkg-split-a' 'pkg-split-b' 'pkg-simple-epoch') - local pkgbase - local arch - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage extra ${pkgbase} ${arch} - done - done - - db-update - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - db-remove extra ${arch} ${pkgbase} - done - done - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - checkRemovedPackage extra ${pkgbase} ${arch} - done - done -} - -testRemoveMultiplePackages() { - local arches=('i686' 'x86_64') - local pkgs=('pkg-simple-a' 'pkg-simple-b' 'pkg-split-a' 'pkg-split-b' 'pkg-simple-epoch') - local pkgbase - local arch - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage extra ${pkgbase} ${arch} - done - done - - db-update - - for arch in ${arches[@]}; do - db-remove extra ${arch} ${pkgs[@]} - done - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - checkRemovedPackage extra ${pkgbase} ${arch} - done - done -} - -testRemoveAnyPackages() { - local pkgs=('pkg-any-a' 'pkg-any-b') - local pkgbase - - for pkgbase in ${pkgs[@]}; do - releasePackage extra ${pkgbase} any - done - - db-update - - for pkgbase in ${pkgs[@]}; do - db-remove extra any ${pkgbase} - done - - for pkgbase in ${pkgs[@]}; do - checkRemovedAnyPackage extra ${pkgbase} - done -} diff --git a/test/test.d/db-repo-add.bats b/test/test.d/db-repo-add.bats new file mode 100644 index 0000000..d0aae60 --- /dev/null +++ b/test/test.d/db-repo-add.bats @@ -0,0 +1,72 @@ +load ../lib/common + +@test "testRepoAddSimplePackages" { + local arches=('i686' 'x86_64') + local pkgs=('pkg-simple-a' 'pkg-simple-b') + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + releasePackage extra ${pkgbase} ${arch} + mv "${STAGING}"/extra/* "${FTP_BASE}/${PKGPOOL}/" + ln -s "${FTP_BASE}/${PKGPOOL}/${pkgbase}-1-1-${arch}.pkg.tar.xz" "${FTP_BASE}/extra/os/${arch}/" + ln -s "${FTP_BASE}/${PKGPOOL}/${pkgbase}-1-1-${arch}.pkg.tar.xz.sig" "${FTP_BASE}/extra/os/${arch}/" + db-repo-add extra ${arch} ${pkgbase}-1-1-${arch}.pkg.tar.xz + done + done + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + checkPackageDB extra ${pkgbase}-1-1-${arch}.pkg.tar.xz ${arch} + done + done +} + +@test "testRepoAddMultiplePackages" { + local arches=('i686' 'x86_64') + local pkgs=('pkg-simple-a' 'pkg-simple-b') + local pkgbase + local arch + + for arch in ${arches[@]}; do + add_pkgs=() + for pkgbase in ${pkgs[@]}; do + releasePackage extra ${pkgbase} ${arch} + mv "${STAGING}"/extra/* "${FTP_BASE}/${PKGPOOL}/" + ln -s "${FTP_BASE}/${PKGPOOL}/${pkgbase}-1-1-${arch}.pkg.tar.xz" "${FTP_BASE}/extra/os/${arch}/" + ln -s "${FTP_BASE}/${PKGPOOL}/${pkgbase}-1-1-${arch}.pkg.tar.xz.sig" "${FTP_BASE}/extra/os/${arch}/" + add_pkgs[${#add_pkgs[*]}]=${pkgbase}-1-1-${arch}.pkg.tar.xz + done + db-repo-add extra ${arch} ${add_pkgs[@]} + done + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + checkPackageDB extra ${pkgbase}-1-1-${arch}.pkg.tar.xz ${arch} + done + done +} + +@test "testRepoAddAnyPackages" { + local arches=('i686' 'x86_64') + local pkgs=('pkg-any-a' 'pkg-any-b') + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + releasePackage extra ${pkgbase} any + mv "${STAGING}"/extra/* "${FTP_BASE}/${PKGPOOL}/" + for arch in ${arches[@]}; do + ln -s "${FTP_BASE}/${PKGPOOL}/${pkgbase}-1-1-any.pkg.tar.xz" "${FTP_BASE}/extra/os/${arch}/" + ln -s "${FTP_BASE}/${PKGPOOL}/${pkgbase}-1-1-any.pkg.tar.xz.sig" "${FTP_BASE}/extra/os/${arch}/" + done + db-repo-add extra any ${pkgbase}-1-1-any.pkg.tar.xz + done + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + checkPackageDB extra ${pkgbase}-1-1-any.pkg.tar.xz ${arch} + done + done +} diff --git a/test/test.d/db-repo-add.sh b/test/test.d/db-repo-add.sh deleted file mode 100755 index ed72957..0000000 --- a/test/test.d/db-repo-add.sh +++ /dev/null @@ -1,70 +0,0 @@ -testRepoAddSimplePackages() { - local arches=('i686' 'x86_64') - local pkgs=('pkg-simple-a' 'pkg-simple-b') - local pkgbase - local arch - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - cp "${pkgdir}/${pkgbase}/${pkgbase}-1-1-${arch}.pkg.tar.xz" "${FTP_BASE}/${PKGPOOL}/" - touch "${FTP_BASE}/${PKGPOOL}/${pkgbase}-1-1-${arch}.pkg.tar.xz.sig" - ln -s "${FTP_BASE}/${PKGPOOL}/${pkgbase}-1-1-${arch}.pkg.tar.xz" "${FTP_BASE}/extra/os/${arch}/" - ln -s "${FTP_BASE}/${PKGPOOL}/${pkgbase}-1-1-${arch}.pkg.tar.xz.sig" "${FTP_BASE}/extra/os/${arch}/" - db-repo-add extra ${arch} ${pkgbase}-1-1-${arch}.pkg.tar.xz - done - done - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - checkPackageDB extra ${pkgbase}-1-1-${arch}.pkg.tar.xz ${arch} - done - done -} - -testRepoAddMultiplePackages() { - local arches=('i686' 'x86_64') - local pkgs=('pkg-simple-a' 'pkg-simple-b') - local pkgbase - local arch - - for arch in ${arches[@]}; do - add_pkgs=() - for pkgbase in ${pkgs[@]}; do - cp "${pkgdir}/${pkgbase}/${pkgbase}-1-1-${arch}.pkg.tar.xz" "${FTP_BASE}/${PKGPOOL}/" - touch "${FTP_BASE}/${PKGPOOL}/${pkgbase}-1-1-${arch}.pkg.tar.xz.sig" - ln -s "${FTP_BASE}/${PKGPOOL}/${pkgbase}-1-1-${arch}.pkg.tar.xz" "${FTP_BASE}/extra/os/${arch}/" - ln -s "${FTP_BASE}/${PKGPOOL}/${pkgbase}-1-1-${arch}.pkg.tar.xz.sig" "${FTP_BASE}/extra/os/${arch}/" - add_pkgs[${#add_pkgs[*]}]=${pkgbase}-1-1-${arch}.pkg.tar.xz - done - db-repo-add extra ${arch} ${add_pkgs[@]} - done - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - checkPackageDB extra ${pkgbase}-1-1-${arch}.pkg.tar.xz ${arch} - done - done -} - -testRepoAddAnyPackages() { - local arches=('i686' 'x86_64') - local pkgs=('pkg-any-a' 'pkg-any-b') - local pkgbase - local arch - - for pkgbase in ${pkgs[@]}; do - cp "${pkgdir}/${pkgbase}/${pkgbase}-1-1-any.pkg.tar.xz" "${FTP_BASE}/${PKGPOOL}/" - touch "${FTP_BASE}/${PKGPOOL}/${pkgbase}-1-1-any.pkg.tar.xz.sig" - for arch in ${arches[@]}; do - ln -s "${FTP_BASE}/${PKGPOOL}/${pkgbase}-1-1-any.pkg.tar.xz" "${FTP_BASE}/extra/os/${arch}/" - ln -s "${FTP_BASE}/${PKGPOOL}/${pkgbase}-1-1-any.pkg.tar.xz.sig" "${FTP_BASE}/extra/os/${arch}/" - done - db-repo-add extra any ${pkgbase}-1-1-any.pkg.tar.xz - done - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - checkPackageDB extra ${pkgbase}-1-1-any.pkg.tar.xz ${arch} - done - done -} diff --git a/test/test.d/db-repo-remove.bats b/test/test.d/db-repo-remove.bats new file mode 100644 index 0000000..a41db12 --- /dev/null +++ b/test/test.d/db-repo-remove.bats @@ -0,0 +1,72 @@ +load ../lib/common + +@test "testRepoRemovePackages" { + local arches=('i686' 'x86_64') + local pkgs=('pkg-simple-a' 'pkg-simple-b' 'pkg-simple-epoch') + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + releasePackage extra ${pkgbase} ${arch} + done + done + + db-update + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + db-repo-remove extra ${arch} ${pkgbase} + done + done + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + checkRemovedPackageDB extra ${pkgbase} ${arch} + done + done +} + +@test "testRepoRemoveMultiplePackages" { + local arches=('i686' 'x86_64') + local pkgs=('pkg-simple-a' 'pkg-simple-b' 'pkg-simple-epoch') + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + releasePackage extra ${pkgbase} ${arch} + done + done + + db-update + + for arch in ${arches[@]}; do + db-repo-remove extra ${arch} ${pkgs[@]} + done + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + checkRemovedPackageDB extra ${pkgbase} ${arch} + done + done +} + +@test "testRepoRemoveAnyPackages" { + local pkgs=('pkg-any-a' 'pkg-any-b') + local pkgbase + + for pkgbase in ${pkgs[@]}; do + releasePackage extra ${pkgbase} any + done + + db-update + + for pkgbase in ${pkgs[@]}; do + db-repo-remove extra any ${pkgbase} + done + + for pkgbase in ${pkgs[@]}; do + checkRemovedAnyPackageDB extra ${pkgbase} + done +} diff --git a/test/test.d/db-repo-remove.sh b/test/test.d/db-repo-remove.sh deleted file mode 100755 index 1aced25..0000000 --- a/test/test.d/db-repo-remove.sh +++ /dev/null @@ -1,70 +0,0 @@ -testRepoRemovePackages() { - local arches=('i686' 'x86_64') - local pkgs=('pkg-simple-a' 'pkg-simple-b' 'pkg-simple-epoch') - local pkgbase - local arch - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage extra ${pkgbase} ${arch} - done - done - - db-update - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - db-repo-remove extra ${arch} ${pkgbase} - done - done - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - checkRemovedPackageDB extra ${pkgbase} ${arch} - done - done -} - -testRepoRemoveMultiplePackages() { - local arches=('i686' 'x86_64') - local pkgs=('pkg-simple-a' 'pkg-simple-b' 'pkg-simple-epoch') - local pkgbase - local arch - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage extra ${pkgbase} ${arch} - done - done - - db-update - - for arch in ${arches[@]}; do - db-repo-remove extra ${arch} ${pkgs[@]} - done - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - checkRemovedPackageDB extra ${pkgbase} ${arch} - done - done -} - -testRepoRemoveAnyPackages() { - local pkgs=('pkg-any-a' 'pkg-any-b') - local pkgbase - - for pkgbase in ${pkgs[@]}; do - releasePackage extra ${pkgbase} any - done - - db-update - - for pkgbase in ${pkgs[@]}; do - db-repo-remove extra any ${pkgbase} - done - - for pkgbase in ${pkgs[@]}; do - checkRemovedAnyPackageDB extra ${pkgbase} - done -} diff --git a/test/test.d/db-update.bats b/test/test.d/db-update.bats new file mode 100644 index 0000000..2cb2395 --- /dev/null +++ b/test/test.d/db-update.bats @@ -0,0 +1,263 @@ +load ../lib/common + +@test "testAddSimplePackages" { + local arches=('i686' 'x86_64') + local pkgs=('pkg-simple-a' 'pkg-simple-b') + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + releasePackage extra ${pkgbase} ${arch} + done + done + + db-update + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + checkPackage extra ${pkgbase}-1-1-${arch}.pkg.tar.xz ${arch} + done + done +} + +@test "testAddSingleSimplePackage" { + releasePackage extra 'pkg-simple-a' 'i686' + db-update + checkPackage extra 'pkg-simple-a-1-1-i686.pkg.tar.xz' 'i686' +} + +@test "testAddSingleEpochPackage" { + releasePackage extra 'pkg-simple-epoch' 'i686' + db-update + checkPackage extra 'pkg-simple-epoch-1:1-1-i686.pkg.tar.xz' 'i686' +} + +@test "testAddAnyPackages" { + local pkgs=('pkg-any-a' 'pkg-any-b') + local pkgbase + + for pkgbase in ${pkgs[@]}; do + releasePackage extra ${pkgbase} any + done + + db-update + + for pkgbase in ${pkgs[@]}; do + checkAnyPackage extra ${pkgbase}-1-1-any.pkg.tar.xz + done +} + +@test "testAddSplitPackages" { + local arches=('i686' 'x86_64') + local pkgs=('pkg-split-a' 'pkg-split-b') + local pkg + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + releasePackage extra ${pkgbase} ${arch} + done + done + + db-update + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + for pkg in $(getPackageNamesFromPackageBase ${pkgbase}); do + checkPackage extra ${pkg##*/} ${arch} + done + done + done +} + +@test "testUpdateAnyPackage" { + releasePackage extra pkg-any-a any + db-update + + updatePackage pkg-any-a any + + releasePackage extra pkg-any-a any + db-update + + checkAnyPackage extra pkg-any-a-1-2-any.pkg.tar.xz any +} + +@test "testUpdateAnyPackageToDifferentRepositoriesAtOnce" { + releasePackage extra pkg-any-a any + + updatePackage pkg-any-a any + + releasePackage testing pkg-any-a any + + db-update + + checkAnyPackage extra pkg-any-a-1-1-any.pkg.tar.xz any + checkAnyPackage testing pkg-any-a-1-2-any.pkg.tar.xz any +} + +@test "testUpdateSameAnyPackageToSameRepository" { + releasePackage extra pkg-any-a any + db-update + checkAnyPackage extra pkg-any-a-1-1-any.pkg.tar.xz any + + releasePackage extra pkg-any-a any + run db-update + [ "$status" -ne 0 ] +} + +@test "testUpdateSameAnyPackageToDifferentRepositories" { + local arch + + releasePackage extra pkg-any-a any + db-update + checkAnyPackage extra pkg-any-a-1-1-any.pkg.tar.xz any + + releasePackage testing pkg-any-a any + run db-update + [ "$status" -ne 0 ] + + for arch in i686 x86_64; do + if [ -r "${FTP_BASE}/testing/os/${arch}/testing${DBEXT%.tar.*}" ]; then + echo "$(bsdtar -xf "${FTP_BASE}/testing/os/${arch}/testing${DBEXT%.tar.*}" -O)" | grep -qv ${pkgbase} + fi + done +} + +@test "testAddIncompleteSplitPackage" { + local arches=('i686' 'x86_64') + local repo='extra' + local pkgbase='pkg-split-a' + local arch + + for arch in ${arches[@]}; do + releasePackage ${repo} ${pkgbase} ${arch} + done + + # remove a split package to make db-update fail + rm "${STAGING}"/extra/${pkgbase}1-* + + run db-update + [ "$status" -ne 0 ] + + for arch in ${arches[@]}; do + if [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" ]; then + echo "$(bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" -O)" | grep -qv ${pkgbase} + fi + done +} + +@test "testUnknownRepo" { + mkdir "${STAGING}/unknown/" + releasePackage extra 'pkg-simple-a' 'i686' + releasePackage unknown 'pkg-simple-b' 'i686' + db-update + checkPackage extra 'pkg-simple-a-1-1-i686.pkg.tar.xz' 'i686' + [ ! -e "${FTP_BASE}/unknown" ] + rm -rf "${STAGING}/unknown/" +} + +@test "testAddUnsignedPackageFails" { + releasePackage extra 'pkg-simple-a' 'i686' + rm "${STAGING}"/extra/*.sig + run db-update + [ "$status" -ne 0 ] + + checkRemovedPackage extra pkg-simple-a-1-1-i686.pkg.tar.xz i686 +} + +@test "testAddInvalidSignedPackageFails" { + local p + releasePackage extra 'pkg-simple-a' 'i686' + for p in "${STAGING}"/extra/*${PKGEXT}; do + unxz $p + xz -0 ${p%%.xz} + done + run db-update + [ "$status" -ne 0 ] + + checkRemovedPackage extra pkg-simple-a-1-1-i686.pkg.tar.xz i686 +} + +@test "testAddBrokenSignatureFails" { + local s + releasePackage extra 'pkg-simple-a' 'i686' + for s in "${STAGING}"/extra/*.sig; do + echo 0 > $s + done + run db-update + [ "$status" -ne 0 ] + + checkRemovedPackage extra pkg-simple-a-1-1-i686.pkg.tar.xz i686 +} + +@test "testAddPackageWithInconsistentVersionFails" { + local p + releasePackage extra 'pkg-simple-a' 'i686' + + for p in "${STAGING}"/extra/*; do + mv "${p}" "${p/pkg-simple-a-1/pkg-simple-a-2}" + done + + run db-update + [ "$status" -ne 0 ] + checkRemovedPackage extra 'pkg-simple-a-2-1-i686.pkg.tar.xz' 'i686' +} + +@test "testAddPackageWithInconsistentNameFails" { + local p + releasePackage extra 'pkg-simple-a' 'i686' + + for p in "${STAGING}"/extra/*; do + mv "${p}" "${p/pkg-/foo-pkg-}" + done + + run db-update + [ "$status" -ne 0 ] + checkRemovedPackage extra 'foo-pkg-simple-a-1-1-i686.pkg.tar.xz' 'i686' +} + +@test "testAddPackageWithInconsistentPKGBUILDFails" { + releasePackage extra 'pkg-simple-a' 'i686' + + updateRepoPKGBUILD 'pkg-simple-a' extra i686 + + run db-update + [ "$status" -ne 0 ] + checkRemovedPackage extra 'pkg-simple-a-1-1-i686.pkg.tar.xz' 'i686' +} + +@test "testAddPackageWithInsufficientPermissionsFails" { + releasePackage core 'pkg-simple-a' 'i686' + releasePackage extra 'pkg-simple-b' 'i686' + + chmod -xwr ${FTP_BASE}/core/os/i686 + run db-update + [ "$status" -ne 0 ] + chmod +xwr ${FTP_BASE}/core/os/i686 + + checkRemovedPackage core 'pkg-simple-a-1-1-i686.pkg.tar.xz' 'i686' + checkRemovedPackage extra 'pkg-simple-b-1-1-i686.pkg.tar.xz' 'i686' +} + +@test "testPackageHasToBeARegularFile" { + local p + local target=$(mktemp -d) + local arches=('i686' 'x86_64') + + for arch in ${arches[@]}; do + releasePackage extra 'pkg-simple-a' $arch + done + + for p in "${STAGING}"/extra/*i686*; do + mv "${p}" "${target}" + ln -s "${target}/${p##*/}" "${p}" + done + + run db-update + [ "$status" -ne 0 ] + for arch in ${arches[@]}; do + checkRemovedPackage extra "pkg-simple-a-1-1-${arch}.pkg.tar.xz" $arch + done +} diff --git a/test/test.d/db-update.sh b/test/test.d/db-update.sh deleted file mode 100755 index db37b82..0000000 --- a/test/test.d/db-update.sh +++ /dev/null @@ -1,251 +0,0 @@ -testAddSimplePackages() { - local arches=('i686' 'x86_64') - local pkgs=('pkg-simple-a' 'pkg-simple-b') - local pkgbase - local arch - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage extra ${pkgbase} ${arch} - done - done - - db-update - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - checkPackage extra ${pkgbase}-1-1-${arch}.pkg.tar.xz ${arch} - done - done -} - -testAddSingleSimplePackage() { - releasePackage extra 'pkg-simple-a' 'i686' - db-update - checkPackage extra 'pkg-simple-a-1-1-i686.pkg.tar.xz' 'i686' -} - -testAddSingleEpochPackage() { - releasePackage extra 'pkg-simple-epoch' 'i686' - db-update - checkPackage extra 'pkg-simple-epoch-1:1-1-i686.pkg.tar.xz' 'i686' -} - -testAddAnyPackages() { - local pkgs=('pkg-any-a' 'pkg-any-b') - local pkgbase - - for pkgbase in ${pkgs[@]}; do - releasePackage extra ${pkgbase} any - done - - db-update - - for pkgbase in ${pkgs[@]}; do - checkAnyPackage extra ${pkgbase}-1-1-any.pkg.tar.xz - done -} - -testAddSplitPackages() { - local arches=('i686' 'x86_64') - local pkgs=('pkg-split-a' 'pkg-split-b') - local pkg - local pkgbase - local arch - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage extra ${pkgbase} ${arch} - done - done - - db-update - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - for pkg in "${pkgdir}/${pkgbase}"/*-${arch}${PKGEXT}; do - checkPackage extra ${pkg##*/} ${arch} - done - done - done -} - -testUpdateAnyPackage() { - releasePackage extra pkg-any-a any - db-update - - updatePackage pkg-any-a - - releasePackage extra pkg-any-a any - db-update - - checkAnyPackage extra pkg-any-a-1-2-any.pkg.tar.xz any -} - -testUpdateAnyPackageToDifferentRepositoriesAtOnce() { - releasePackage extra pkg-any-a any - - updatePackage pkg-any-a - - releasePackage testing pkg-any-a any - - db-update - - checkAnyPackage extra pkg-any-a-1-1-any.pkg.tar.xz any - checkAnyPackage testing pkg-any-a-1-2-any.pkg.tar.xz any -} - -testUpdateSameAnyPackageToSameRepository() { - releasePackage extra pkg-any-a any - db-update - checkAnyPackage extra pkg-any-a-1-1-any.pkg.tar.xz any - - releasePackage extra pkg-any-a any - db-update >/dev/null 2>&1 && (fail 'Adding an existing package to the same repository should fail'; return 1) -} - -testUpdateSameAnyPackageToDifferentRepositories() { - releasePackage extra pkg-any-a any - db-update - checkAnyPackage extra pkg-any-a-1-1-any.pkg.tar.xz any - - releasePackage testing pkg-any-a any - db-update >/dev/null 2>&1 && (fail 'Adding an existing package to another repository should fail'; return 1) - - local arch - for arch in i686 x86_64; do - ( [ -r "${FTP_BASE}/testing/os/${arch}/testing${DBEXT%.tar.*}" ] \ - && bsdtar -xf "${FTP_BASE}/testing/os/${arch}/testing${DBEXT%.tar.*}" -O | grep -q ${pkgbase}) \ - && fail "${pkgbase} should not be in testing/os/${arch}/testing${DBEXT%.tar.*}" - done -} - -testAddIncompleteSplitPackage() { - local arches=('i686' 'x86_64') - local repo='extra' - local pkgbase='pkg-split-a' - local arch - - for arch in ${arches[@]}; do - releasePackage ${repo} ${pkgbase} ${arch} - done - - # remove a split package to make db-update fail - rm "${STAGING}"/extra/${pkgbase}1-* - - db-update >/dev/null 2>&1 && fail "db-update should fail when a split package is missing!" - - for arch in ${arches[@]}; do - ( [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" ] \ - && bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" -O | grep -q ${pkgbase}) \ - && fail "${pkgbase} should not be in ${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" - done -} - -testUnknownRepo() { - mkdir "${STAGING}/unknown/" - releasePackage extra 'pkg-simple-a' 'i686' - releasePackage unknown 'pkg-simple-b' 'i686' - db-update - checkPackage extra 'pkg-simple-a-1-1-i686.pkg.tar.xz' 'i686' - [ -e "${FTP_BASE}/unknown" ] && fail "db-update pushed a package into an unknown repository" - rm -rf "${STAGING}/unknown/" -} - -testAddUnsignedPackageFails() { - releasePackage extra 'pkg-simple-a' 'i686' - rm "${STAGING}"/extra/*.sig - db-update >/dev/null 2>&1 && fail "db-update should fail when a signature is missing!" - - checkRemovedPackage extra pkg-simple-a-1-1-i686.pkg.tar.xz i686 -} - -testAddInvalidSignedPackageFails() { - local p - releasePackage extra 'pkg-simple-a' 'i686' - for p in "${STAGING}"/extra/*${PKGEXT}; do - unxz $p - xz -0 ${p%%.xz} - done - db-update >/dev/null 2>&1 && fail "db-update should fail when a signature is invalid!" - - checkRemovedPackage extra pkg-simple-a-1-1-i686.pkg.tar.xz i686 -} - -testAddBrokenSignatureFails() { - local s - releasePackage extra 'pkg-simple-a' 'i686' - for s in "${STAGING}"/extra/*.sig; do - echo 0 > $s - done - db-update >/dev/null 2>&1 && fail "db-update should fail when a signature is broken!" - - checkRemovedPackage extra pkg-simple-a-1-1-i686.pkg.tar.xz i686 -} - -testAddPackageWithInconsistentVersionFails() { - local p - releasePackage extra 'pkg-simple-a' 'i686' - - for p in "${STAGING}"/extra/*; do - mv "${p}" "${p/pkg-simple-a-1/pkg-simple-a-2}" - done - - db-update >/dev/null 2>&1 && fail "db-update should fail when a package is not consistent!" - checkRemovedPackage extra 'pkg-simple-a-2-1-i686.pkg.tar.xz' 'i686' -} - -testAddPackageWithInconsistentNameFails() { - local p - releasePackage extra 'pkg-simple-a' 'i686' - - for p in "${STAGING}"/extra/*; do - mv "${p}" "${p/pkg-/foo-pkg-}" - done - - db-update >/dev/null 2>&1 && fail "db-update should fail when a package is not consistent!" - checkRemovedPackage extra 'foo-pkg-simple-a-1-1-i686.pkg.tar.xz' 'i686' -} - -testAddPackageWithInconsistentPKGBUILDFails() { - releasePackage extra 'pkg-simple-a' 'i686' - - updateRepoPKGBUILD 'pkg-simple-a' extra i686 - - db-update >/dev/null 2>&1 && fail "db-update should fail when a package is not consistent!" - checkRemovedPackage extra 'pkg-simple-a-1-1-i686.pkg.tar.xz' 'i686' -} - -testAddPackageWithInsufficientPermissionsFails() -{ - releasePackage core 'pkg-simple-a' 'i686' - releasePackage extra 'pkg-simple-b' 'i686' - - chmod -xwr ${FTP_BASE}/core/os/i686 - db-update >/dev/null 2>&1 && fail "db-update should fail when permissions are insufficient!" - chmod +xwr ${FTP_BASE}/core/os/i686 - - checkRemovedPackage core 'pkg-simple-a-1-1-i686.pkg.tar.xz' 'i686' - checkRemovedPackage extra 'pkg-simple-b-1-1-i686.pkg.tar.xz' 'i686' -} - -testPackageHasToBeARegularFile() -{ - local p - local target=$(mktemp -d) - local arches=('i686' 'x86_64') - - for arch in ${arches[@]}; do - releasePackage extra 'pkg-simple-a' $arch - done - - for p in "${STAGING}"/extra/*i686*; do - mv "${p}" "${target}" - ln -s "${target}/${p##*/}" "${p}" - done - - db-update >/dev/null 2>&1 && fail "db-update should fail when a package is a symlink!" - for arch in ${arches[@]}; do - checkRemovedPackage extra "pkg-simple-a-1-1-${arch}.pkg.tar.xz" $arch - done -} diff --git a/test/test.d/ftpdir-cleanup.bats b/test/test.d/ftpdir-cleanup.bats new file mode 100644 index 0000000..3e4d956 --- /dev/null +++ b/test/test.d/ftpdir-cleanup.bats @@ -0,0 +1,150 @@ +load ../lib/common + +@test "testCleanupSimplePackages" { + local arches=('i686' 'x86_64') + local pkgs=('pkg-simple-a' 'pkg-simple-b') + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + releasePackage extra ${pkgbase} ${arch} + done + done + + db-update + + for arch in ${arches[@]}; do + db-remove extra ${arch} pkg-simple-a + done + + ftpdir-cleanup + + for arch in ${arches[@]}; do + local pkg1="pkg-simple-a-1-1-${arch}.pkg.tar.xz" + checkRemovedPackage extra 'pkg-simple-a' ${arch} + [ ! -f "${FTP_BASE}/${PKGPOOL}/${pkg1}" ] + [ ! -f "${FTP_BASE}/${repo}/os/${arch}/${pkg1}" ] + + local pkg2="pkg-simple-b-1-1-${arch}.pkg.tar.xz" + checkPackage extra ${pkg2} ${arch} + done +} + +@test "testCleanupEpochPackages" { + local arches=('i686' 'x86_64') + local pkgs=('pkg-simple-epoch') + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + releasePackage extra ${pkgbase} ${arch} + done + done + + db-update + + for arch in ${arches[@]}; do + db-remove extra ${arch} pkg-simple-epoch + done + + ftpdir-cleanup + + for arch in ${arches[@]}; do + local pkg1="pkg-simple-epoch-1:1-1-${arch}.pkg.tar.xz" + checkRemovedPackage extra 'pkg-simple-epoch' ${arch} + [ ! -f "${FTP_BASE}/${PKGPOOL}/${pkg1}" ] + [ ! -f "${FTP_BASE}/${repo}/os/${arch}/${pkg1}" ] + done +} + +@test "testCleanupAnyPackages" { + local pkgs=('pkg-any-a' 'pkg-any-b') + local pkgbase + local arch='any' + + for pkgbase in ${pkgs[@]}; do + releasePackage extra ${pkgbase} any + done + + db-update + db-remove extra any pkg-any-a + ftpdir-cleanup + + local pkg1='pkg-any-a-1-1-any.pkg.tar.xz' + checkRemovedAnyPackage extra 'pkg-any-a' + [ ! -f "${FTP_BASE}/${PKGPOOL}/${pkg1}" ] + [ ! -f "${FTP_BASE}/${repo}/os/${arch}/${pkg1}" ] + + local pkg2="pkg-any-b-1-1-${arch}.pkg.tar.xz" + checkAnyPackage extra ${pkg2} +} + +@test "testCleanupSplitPackages" { + local arches=('i686' 'x86_64') + local pkgs=('pkg-split-a' 'pkg-split-b') + local pkg + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + releasePackage extra ${pkgbase} ${arch} + done + done + + db-update + + for arch in ${arches[@]}; do + db-remove extra ${arch} ${pkgs[0]} + done + + ftpdir-cleanup + + for arch in ${arches[@]}; do + for pkg in $(getPackageNamesFromPackageBase ${pkgs[0]}); do + checkRemovedPackage extra ${pkg} ${arch} + [ ! -f "${FTP_BASE}/${PKGPOOL}/${pkg}" ] + [ ! -f "${FTP_BASE}/${repo}/os/${arch}/${pkg}" ] + done + + for pkg in $(getPackageNamesFromPackageBase ${pkgs[1]}); do + checkPackage extra ${pkg##*/} ${arch} + done + done +} + +@test "testCleanupOldPackages" { + local arches=('i686' 'x86_64') + local pkgs=('pkg-simple-a' 'pkg-simple-b') + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + releasePackage extra ${pkgbase} ${arch} + done + done + + db-update + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + db-remove extra ${arch} ${pkgbase} + done + done + + ftpdir-cleanup + + local pkgfilea="pkg-simple-a-1-1-${arch}.pkg.tar.xz" + local pkgfileb="pkg-simple-b-1-1-${arch}.pkg.tar.xz" + for arch in ${arches[@]}; do + touch -d "-$(expr ${CLEANUP_KEEP} + 1)days" ${CLEANUP_DESTDIR}/${pkgfilea}{,.sig} + done + + ftpdir-cleanup + + [ ! -f ${CLEANUP_DESTDIR}/${pkgfilea} ] + [ -f ${CLEANUP_DESTDIR}/${pkgfileb} ] +} diff --git a/test/test.d/ftpdir-cleanup.sh b/test/test.d/ftpdir-cleanup.sh deleted file mode 100755 index 2749a87..0000000 --- a/test/test.d/ftpdir-cleanup.sh +++ /dev/null @@ -1,148 +0,0 @@ -testCleanupSimplePackages() { - local arches=('i686' 'x86_64') - local pkgs=('pkg-simple-a' 'pkg-simple-b') - local pkgbase - local arch - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage extra ${pkgbase} ${arch} - done - done - - db-update - - for arch in ${arches[@]}; do - db-remove extra ${arch} pkg-simple-a - done - - ftpdir-cleanup >/dev/null - - for arch in ${arches[@]}; do - local pkg1="pkg-simple-a-1-1-${arch}.pkg.tar.xz" - checkRemovedPackage extra 'pkg-simple-a' ${arch} - [ -f "${FTP_BASE}/${PKGPOOL}/${pkg1}" ] && fail "${PKGPOOL}/${pkg1} found" - [ -f "${FTP_BASE}/${repo}/os/${arch}/${pkg1}" ] && fail "${repo}/os/${arch}/${pkg1} found" - - local pkg2="pkg-simple-b-1-1-${arch}.pkg.tar.xz" - checkPackage extra ${pkg2} ${arch} - done -} - -testCleanupEpochPackages() { - local arches=('i686' 'x86_64') - local pkgs=('pkg-simple-epoch') - local pkgbase - local arch - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage extra ${pkgbase} ${arch} - done - done - - db-update - - for arch in ${arches[@]}; do - db-remove extra ${arch} pkg-simple-epoch - done - - ftpdir-cleanup >/dev/null - - for arch in ${arches[@]}; do - local pkg1="pkg-simple-epoch-1:1-1-${arch}.pkg.tar.xz" - checkRemovedPackage extra 'pkg-simple-epoch' ${arch} - [ -f "${FTP_BASE}/${PKGPOOL}/${pkg1}" ] && fail "${PKGPOOL}/${pkg1} found" - [ -f "${FTP_BASE}/${repo}/os/${arch}/${pkg1}" ] && fail "${repo}/os/${arch}/${pkg1} found" - done -} - -testCleanupAnyPackages() { - local pkgs=('pkg-any-a' 'pkg-any-b') - local pkgbase - local arch='any' - - for pkgbase in ${pkgs[@]}; do - releasePackage extra ${pkgbase} any - done - - db-update - db-remove extra any pkg-any-a - ftpdir-cleanup >/dev/null - - local pkg1='pkg-any-a-1-1-any.pkg.tar.xz' - checkRemovedAnyPackage extra 'pkg-any-a' - [ -f "${FTP_BASE}/${PKGPOOL}/${pkg1}" ] && fail "${PKGPOOL}/${pkg1} found" - [ -f "${FTP_BASE}/${repo}/os/${arch}/${pkg1}" ] && fail "${repo}/os/${arch}/${pkg1} found" - - local pkg2="pkg-any-b-1-1-${arch}.pkg.tar.xz" - checkAnyPackage extra ${pkg2} -} - -testCleanupSplitPackages() { - local arches=('i686' 'x86_64') - local pkgs=('pkg-split-a' 'pkg-split-b') - local pkg - local pkgbase - local arch - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage extra ${pkgbase} ${arch} - done - done - - db-update - - for arch in ${arches[@]}; do - db-remove extra ${arch} ${pkgs[0]} - done - - ftpdir-cleanup >/dev/null - - for arch in ${arches[@]}; do - for pkg in "${pkgdir}/${pkgs[0]}"/*-${arch}${PKGEXT}; do - checkRemovedPackage extra ${pkgs[0]} ${arch} - [ -f "${FTP_BASE}/${PKGPOOL}/${pkg}" ] && fail "${PKGPOOL}/${pkg} found" - [ -f "${FTP_BASE}/${repo}/os/${arch}/${pkg}" ] && fail "${repo}/os/${arch}/${pkg} found" - done - - for pkg in "${pkgdir}/${pkgs[1]}"/*-${arch}${PKGEXT}; do - checkPackage extra ${pkg##*/} ${arch} - done - done -} - -testCleanupOldPackages() { - local arches=('i686' 'x86_64') - local pkgs=('pkg-simple-a' 'pkg-simple-b') - local pkgbase - local arch - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage extra ${pkgbase} ${arch} - done - done - - db-update - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - db-remove extra ${arch} ${pkgbase} - done - done - - ftpdir-cleanup >/dev/null - - local pkgfilea="pkg-simple-a-1-1-${arch}.pkg.tar.xz" - local pkgfileb="pkg-simple-b-1-1-${arch}.pkg.tar.xz" - for arch in ${arches[@]}; do - touch -d "-$(expr ${CLEANUP_KEEP} + 1)days" ${CLEANUP_DESTDIR}/${pkgfilea}{,.sig} - done - - ftpdir-cleanup >/dev/null - - [ -f ${CLEANUP_DESTDIR}/${pkgfilea} ] && fail "${pkgfilea} was not removed" - [ -f ${CLEANUP_DESTDIR}/${pkgfileb} ] || fail "${pkgfileb} was removed" -} diff --git a/test/test.d/packages.bats b/test/test.d/packages.bats new file mode 100644 index 0000000..63408d0 --- /dev/null +++ b/test/test.d/packages.bats @@ -0,0 +1,34 @@ +load ../lib/common + +@test "testPackages" { + local result + local pkg + local pkgbase + local pkgarchs + local pkgarch + local tmp=$(mktemp -d) + + # FIXME: Evaluate if this test is sane and even needed + + cp -r packages/* ${tmp} + + for pkgbase in ${tmp}/*; do + pushd ${pkgbase} + # FIXME: Is overriding IFS a bats bug? + IFS=' ' + pkgarchs=($(. PKGBUILD; echo ${arch[@]})) + for pkgarch in ${pkgarchs[@]}; do + echo Building ${pkgbase} on ${pkgarch} + run namcap -e pkgnameindesc,tags PKGBUILD + [ -z "$output" ] + + CARCH=${arch} makepkg -cf + + for pkg in *${PKGEXT}; do + run namcap -e pkgnameindesc ${pkg} + [ -z "$output" ] + done + done + popd + done +} diff --git a/test/test.d/packages.sh b/test/test.d/packages.sh deleted file mode 100755 index e758c72..0000000 --- a/test/test.d/packages.sh +++ /dev/null @@ -1,7 +0,0 @@ -testPackages() { - local result - for pkg in $(find "${pkgdir}" -name "*${PKGEXT}"); do - result=$(namcap -e pkgnameindesc ${pkg}) - [[ -n "${result}" ]] && fail "${result}" - done; -} diff --git a/test/test.d/sourceballs.bats b/test/test.d/sourceballs.bats new file mode 100644 index 0000000..70836ad --- /dev/null +++ b/test/test.d/sourceballs.bats @@ -0,0 +1,79 @@ +load ../lib/common + +@test "testSourceballs" { + local arches=('i686' 'x86_64') + local pkgs=('pkg-simple-a' 'pkg-simple-b' 'pkg-simple-epoch') + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + releasePackage extra ${pkgbase} ${arch} + done + done + db-update + + sourceballs + for pkgbase in ${pkgs[@]}; do + [ -r ${FTP_BASE}/${SRCPOOL}/${pkgbase}-*${SRCEXT} ] + done +} + +@test "testAnySourceballs" { + local pkgs=('pkg-any-a' 'pkg-any-b') + local pkgbase + + for pkgbase in ${pkgs[@]}; do + releasePackage extra ${pkgbase} any + done + db-update + + sourceballs + for pkgbase in ${pkgs[@]}; do + [ -r ${FTP_BASE}/${SRCPOOL}/${pkgbase}-*${SRCEXT} ] + done +} + +@test "testSplitSourceballs" { + local arches=('i686' 'x86_64') + local pkgs=('pkg-split-a' 'pkg-split-b') + local pkg + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + releasePackage extra ${pkgbase} ${arch} + done + done + + db-update + + sourceballs + for pkgbase in ${pkgs[@]}; do + [ -r ${FTP_BASE}/${SRCPOOL}/${pkgbase}-*${SRCEXT} ] + done +} + +@test "testSourceballsCleanup" { + local arches=('i686' 'x86_64') + local pkgs=('pkg-simple-a' 'pkg-simple-b') + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + releasePackage extra ${pkgbase} ${arch} + done + done + db-update + sourceballs + + for arch in ${arches[@]}; do + db-remove extra ${arch} pkg-simple-a + done + + sourceballs + [ ! -r ${FTP_BASE}/${SRCPOOL}/pkg-simple-a-*${SRCEXT} ] + [ -r ${FTP_BASE}/${SRCPOOL}/pkg-simple-b-*${SRCEXT} ] +} diff --git a/test/test.d/sourceballs.sh b/test/test.d/sourceballs.sh deleted file mode 100755 index 04d0fd3..0000000 --- a/test/test.d/sourceballs.sh +++ /dev/null @@ -1,77 +0,0 @@ -testSourceballs() { - local arches=('i686' 'x86_64') - local pkgs=('pkg-simple-a' 'pkg-simple-b' 'pkg-simple-epoch') - local pkgbase - local arch - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage extra ${pkgbase} ${arch} - done - done - db-update - - sourceballs - for pkgbase in ${pkgs[@]}; do - [ ! -r ${FTP_BASE}/${SRCPOOL}/${pkgbase}-*${SRCEXT} ] && fail "source package not found!" - done -} - -testAnySourceballs() { - local pkgs=('pkg-any-a' 'pkg-any-b') - local pkgbase - - for pkgbase in ${pkgs[@]}; do - releasePackage extra ${pkgbase} any - done - db-update - - sourceballs - for pkgbase in ${pkgs[@]}; do - [ ! -r ${FTP_BASE}/${SRCPOOL}/${pkgbase}-*${SRCEXT} ] && fail "source package not found!" - done -} - -testSplitSourceballs() { - local arches=('i686' 'x86_64') - local pkgs=('pkg-split-a' 'pkg-split-b') - local pkg - local pkgbase - local arch - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage extra ${pkgbase} ${arch} - done - done - - db-update - - sourceballs - for pkgbase in ${pkgs[@]}; do - [ ! -r ${FTP_BASE}/${SRCPOOL}/${pkgbase}-*${SRCEXT} ] && fail "source package not found!" - done -} - -testSourceballsCleanup() { - local arches=('i686' 'x86_64') - local pkgs=('pkg-simple-a' 'pkg-simple-b') - local pkgbase - local arch - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage extra ${pkgbase} ${arch} - done - done - db-update - sourceballs - - for arch in ${arches[@]}; do - db-remove extra ${arch} pkg-simple-a - done - - sourceballs - [ -r ${FTP_BASE}/${SRCPOOL}/pkg-simple-a-*${SRCEXT} ] && fail "source package was not removed!" - [ ! -r ${FTP_BASE}/${SRCPOOL}/pkg-simple-b-*${SRCEXT} ] && fail "source package not found!" -} diff --git a/test/test.d/testing2x.bats b/test/test.d/testing2x.bats new file mode 100644 index 0000000..01f3f24 --- /dev/null +++ b/test/test.d/testing2x.bats @@ -0,0 +1,16 @@ +load ../lib/common + +@test "testTesting2xAnyPackage" { + releasePackage core pkg-any-a any + db-update + + updatePackage pkg-any-a any + + releasePackage testing pkg-any-a any + db-update + + testing2x pkg-any-a + + checkAnyPackage core pkg-any-a-1-2-any.pkg.tar.xz + checkRemovedAnyPackage testing pkg-any-a +} diff --git a/test/test.d/testing2x.sh b/test/test.d/testing2x.sh deleted file mode 100755 index cecf5c2..0000000 --- a/test/test.d/testing2x.sh +++ /dev/null @@ -1,14 +0,0 @@ -testTesting2xAnyPackage() { - releasePackage core pkg-any-a any - db-update - - updatePackage pkg-any-a - - releasePackage testing pkg-any-a any - db-update - - testing2x pkg-any-a - - checkAnyPackage core pkg-any-a-1-2-any.pkg.tar.xz any - checkRemovedAnyPackage testing pkg-any-a -} -- cgit v1.2.2 From 73082d2dde58ed8c7bc74de8e3ee58375ac88fc4 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Wed, 19 Apr 2017 12:38:44 +0200 Subject: Re-add docker image dependency --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 92b3fec..51dd430 100644 --- a/Makefile +++ b/Makefile @@ -4,10 +4,10 @@ RUN_OPTIONS:=-t --rm --network=none -v $(PWD):/dbscripts:ro --tmpfs=/tmp:exec -w test-image: docker build --pull -t $(IMAGE) test -test: +test: test-image docker run $(RUN_OPTIONS) $(IMAGE) make test -test-coverage: +test-coverage: test-image rm -rf ${PWD}/coverage mkdir -m 777 ${PWD}/coverage docker run $(RUN_OPTIONS) -v ${PWD}/coverage:/coverage -e COVERAGE_DIR=/coverage $(IMAGE) make test-coverage -- cgit v1.2.2 From 60f0691a945b69b4ede50a18a642c8df1bdc3559 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Wed, 19 Apr 2017 12:48:37 +0200 Subject: Disable docker terminal to avoid issues with the CI system --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 51dd430..988d335 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ IMAGE:=dbscripts/test -RUN_OPTIONS:=-t --rm --network=none -v $(PWD):/dbscripts:ro --tmpfs=/tmp:exec -w /dbscripts/test +RUN_OPTIONS:=--rm --network=none -v $(PWD):/dbscripts:ro --tmpfs=/tmp:exec -w /dbscripts/test test-image: docker build --pull -t $(IMAGE) test -- cgit v1.2.2 From 59d495b55f52253518fe1a184627e7259034f978 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Wed, 19 Apr 2017 17:23:59 +0200 Subject: Cache already built packages --- test/Makefile | 6 ++++-- test/lib/common.bash | 24 +++++++++++++++++++----- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/test/Makefile b/test/Makefile index afc3cff..c4cc021 100644 --- a/test/Makefile +++ b/test/Makefile @@ -1,8 +1,10 @@ +PACKAGE_CACHE:=$(shell mktemp -d) + test: - PATH=$(CURDIR)/../:$(CURDIR)/../cron-jobs/:$(PATH) bats test.d + PATH=$(CURDIR)/../:$(CURDIR)/../cron-jobs/:$(PATH) PACKAGE_CACHE=$(PACKAGE_CACHE) bats test.d test-coverage: - PATH=$(CURDIR)/../:$(CURDIR)/../cron-jobs/:$(PATH) kcov \ + PATH=$(CURDIR)/../:$(CURDIR)/../cron-jobs/:$(PATH) PACKAGE_CACHE=$(PACKAGE_CACHE) kcov \ --include-path=$(CURDIR)/../ \ --exclude-path=$(CURDIR)/../test,$(CURDIR)/../cron-jobs/makepkg.conf,$(CURDIR)/../config \ $(COVERAGE_DIR) \ diff --git a/test/lib/common.bash b/test/lib/common.bash index aba3336..187875b 100644 --- a/test/lib/common.bash +++ b/test/lib/common.bash @@ -25,21 +25,30 @@ __updatePKGBUILD() { svn commit -q -m"update pkg to pkgrel=${pkgrel}" } +__getCheckSum() { + local result=($(sha1sum $1)) + echo ${result[0]} +} + __buildPackage() { local arch=$1 local pkgver local pkgname local a local p + local checkSum + + if [[ -n ${PACKAGE_CACHE} ]]; then + checkSum=$(__getCheckSum PKGBUILD) + # TODO: Be more specific + if cp -av ${PACKAGE_CACHE}/${checkSum}/*-${arch}${PKGEXT}{,.sig} .; then + return 0 + fi + fi pkgname=($(. PKGBUILD; echo ${pkgname[@]})) pkgver=$(. PKGBUILD; get_full_version) - # FIXME: Avoid rebuilding of the same package - for p in ${pkgname[@]}; do - [ -f ${p}-${pkgver}-${arch}${PKGEXT} ] && return 0 - done - if [ "${arch}" == 'any' ]; then makepkg -c else @@ -49,6 +58,11 @@ __buildPackage() { for p in ${pkgname[@]}; do gpg --detach-sign --no-armor --use-agent ${p}-${pkgver}-${arch}* done + + if [[ -n ${PACKAGE_CACHE} ]]; then + mkdir -p ${PACKAGE_CACHE}/${checkSum} + cp -av *-${arch}${PKGEXT}{,.sig} ${PACKAGE_CACHE}/${checkSum}/ + fi } setup() { -- cgit v1.2.2 From 4751408429ba67625d8a54c6758e979cd20b4c5f Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Thu, 20 Apr 2017 12:10:34 +0200 Subject: Extend check functions to handle 'any' packages as well --- test/lib/common.bash | 87 ++++++++++------------------------------- test/test.d/db-move.bats | 9 +++-- test/test.d/db-remove.bats | 2 +- test/test.d/db-repo-remove.bats | 2 +- test/test.d/db-update.bats | 12 +++--- test/test.d/ftpdir-cleanup.bats | 4 +- test/test.d/testing2x.bats | 4 +- 7 files changed, 38 insertions(+), 82 deletions(-) diff --git a/test/lib/common.bash b/test/lib/common.bash index 187875b..3435ff4 100644 --- a/test/lib/common.bash +++ b/test/lib/common.bash @@ -167,16 +167,25 @@ getPackageNamesFromPackageBase() { $(. "packages/${pkgbase}/PKGBUILD"; echo ${pkgname[@]}) } -checkAnyPackageDB() { +checkPackageDB() { local repo=$1 local pkg=$2 - local arch + local arch=$3 local db + local tarches [ -r "${FTP_BASE}/${PKGPOOL}/${pkg}" ] [ -r "${FTP_BASE}/${PKGPOOL}/${pkg}.sig" ] + [ ! -r "${STAGING}"/${repo}/${pkg} ] + [ ! -r "${STAGING}"/${repo}/${pkg}.sig ] - for arch in i686 x86_64; do + if [[ $arch == any ]]; then + tarches=(${ARCHES[@]}) + else + tarches=(${arch}) + fi + + for arch in ${tarches[@]}; do [ -L "${FTP_BASE}/${repo}/os/${arch}/${pkg}" ] [ "$(readlink -e "${FTP_BASE}/${repo}/os/${arch}/${pkg}")" == "${FTP_BASE}/${PKGPOOL}/${pkg}" ] @@ -188,44 +197,6 @@ checkAnyPackageDB() { bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" -O | grep -q ${pkg} done done - - [ ! -r "${STAGING}"/${repo}/${pkg} ] - [ ! -r "${STAGING}"/${repo}/${pkg}.sig ] -} - -checkAnyPackage() { - local repo=$1 - local pkg=$2 - - checkAnyPackageDB $repo $pkg - - local pkgbase=$(__getPackageBaseFromPackage "${FTP_BASE}/${PKGPOOL}/${pkg}") - svn up -q "${TMP}/svn-packages-copy/${pkgbase}" - [ -d "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo}-any" ] -} - -checkPackageDB() { - local repo=$1 - local pkg=$2 - local arch=$3 - local db - - [ -r "${FTP_BASE}/${PKGPOOL}/${pkg}" ] - [ -L "${FTP_BASE}/${repo}/os/${arch}/${pkg}" ] - [ ! -r "${STAGING}"/${repo}/${pkg} ] - - [ "$(readlink -e "${FTP_BASE}/${repo}/os/${arch}/${pkg}")" == "${FTP_BASE}/${PKGPOOL}/${pkg}" ] - - [ -r "${FTP_BASE}/${PKGPOOL}/${pkg}.sig" ] - [ -L "${FTP_BASE}/${repo}/os/${arch}/${pkg}.sig" ] - [ ! -r "${STAGING}"/${repo}/${pkg}.sig ] - - [ "$(readlink -e "${FTP_BASE}/${repo}/os/${arch}/${pkg}.sig")" == "${FTP_BASE}/${PKGPOOL}/${pkg}.sig" ] - - for db in ${DBEXT} ${FILESEXT}; do - [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" ] - bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" -O | grep -q ${pkg} - done } checkPackage() { @@ -240,19 +211,6 @@ checkPackage() { [ -d "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo}-${arch}" ] } -checkRemovedPackageDB() { - local repo=$1 - local pkgbase=$2 - local arch=$3 - local db - - for db in ${DBEXT} ${FILESEXT}; do - if [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" ]; then - echo "$(bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" -O)" | grep -qv ${pkgbase} - fi - done -} - checkRemovedPackage() { local repo=$1 local pkgbase=$2 @@ -264,11 +222,18 @@ checkRemovedPackage() { [ ! -d "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo}-${arch}" ] } -checkRemovedAnyPackageDB() { +checkRemovedPackageDB() { local repo=$1 local pkgbase=$2 - local arch + local arch=$3 local db + local tarches + + if [[ $arch == any ]]; then + tarches=(${ARCHES[@]}) + else + tarches=(${arch}) + fi for db in ${DBEXT} ${FILESEXT}; do for arch in i686 x86_64; do @@ -278,13 +243,3 @@ checkRemovedAnyPackageDB() { done done } - -checkRemovedAnyPackage() { - local repo=$1 - local pkgbase=$2 - - checkRemovedAnyPackageDB $repo $pkgbase - - svn up -q "${TMP}/svn-packages-copy/${pkgbase}" - [ ! -d "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo}-any" ] -} diff --git a/test/test.d/db-move.bats b/test/test.d/db-move.bats index a9a58d0..d12c0e5 100644 --- a/test/test.d/db-move.bats +++ b/test/test.d/db-move.bats @@ -81,9 +81,9 @@ load ../lib/common db-update db-move testing extra pkg-any-a - checkAnyPackage extra pkg-any-a-1-1-any.pkg.tar.xz - checkRemovedAnyPackage testing pkg-any-a - checkAnyPackage testing pkg-any-b-1-1-any.pkg.tar.xz + checkPackage extra pkg-any-a-1-1-any.pkg.tar.xz any + checkRemovedPackage testing pkg-any-a any + checkPackage testing pkg-any-b-1-1-any.pkg.tar.xz any } @test "testMoveSplitPackages" { @@ -113,5 +113,6 @@ load ../lib/common done done - checkRemovedAnyPackage testing pkg-split-a + # FIXME: Why any? + checkRemovedPackage testing pkg-split-a any } diff --git a/test/test.d/db-remove.bats b/test/test.d/db-remove.bats index bc4a981..e70745f 100644 --- a/test/test.d/db-remove.bats +++ b/test/test.d/db-remove.bats @@ -67,6 +67,6 @@ load ../lib/common done for pkgbase in ${pkgs[@]}; do - checkRemovedAnyPackage extra ${pkgbase} + checkRemovedPackage extra ${pkgbase} any done } diff --git a/test/test.d/db-repo-remove.bats b/test/test.d/db-repo-remove.bats index a41db12..35ff0e7 100644 --- a/test/test.d/db-repo-remove.bats +++ b/test/test.d/db-repo-remove.bats @@ -67,6 +67,6 @@ load ../lib/common done for pkgbase in ${pkgs[@]}; do - checkRemovedAnyPackageDB extra ${pkgbase} + checkRemovedPackageDB extra ${pkgbase} any done } diff --git a/test/test.d/db-update.bats b/test/test.d/db-update.bats index 2cb2395..c481838 100644 --- a/test/test.d/db-update.bats +++ b/test/test.d/db-update.bats @@ -44,7 +44,7 @@ load ../lib/common db-update for pkgbase in ${pkgs[@]}; do - checkAnyPackage extra ${pkgbase}-1-1-any.pkg.tar.xz + checkPackage extra ${pkgbase}-1-1-any.pkg.tar.xz any done } @@ -81,7 +81,7 @@ load ../lib/common releasePackage extra pkg-any-a any db-update - checkAnyPackage extra pkg-any-a-1-2-any.pkg.tar.xz any + checkPackage extra pkg-any-a-1-2-any.pkg.tar.xz any } @test "testUpdateAnyPackageToDifferentRepositoriesAtOnce" { @@ -93,14 +93,14 @@ load ../lib/common db-update - checkAnyPackage extra pkg-any-a-1-1-any.pkg.tar.xz any - checkAnyPackage testing pkg-any-a-1-2-any.pkg.tar.xz any + checkPackage extra pkg-any-a-1-1-any.pkg.tar.xz any + checkPackage testing pkg-any-a-1-2-any.pkg.tar.xz any } @test "testUpdateSameAnyPackageToSameRepository" { releasePackage extra pkg-any-a any db-update - checkAnyPackage extra pkg-any-a-1-1-any.pkg.tar.xz any + checkPackage extra pkg-any-a-1-1-any.pkg.tar.xz any releasePackage extra pkg-any-a any run db-update @@ -112,7 +112,7 @@ load ../lib/common releasePackage extra pkg-any-a any db-update - checkAnyPackage extra pkg-any-a-1-1-any.pkg.tar.xz any + checkPackage extra pkg-any-a-1-1-any.pkg.tar.xz any releasePackage testing pkg-any-a any run db-update diff --git a/test/test.d/ftpdir-cleanup.bats b/test/test.d/ftpdir-cleanup.bats index 3e4d956..809c136 100644 --- a/test/test.d/ftpdir-cleanup.bats +++ b/test/test.d/ftpdir-cleanup.bats @@ -73,12 +73,12 @@ load ../lib/common ftpdir-cleanup local pkg1='pkg-any-a-1-1-any.pkg.tar.xz' - checkRemovedAnyPackage extra 'pkg-any-a' + checkRemovedPackage extra 'pkg-any-a' any [ ! -f "${FTP_BASE}/${PKGPOOL}/${pkg1}" ] [ ! -f "${FTP_BASE}/${repo}/os/${arch}/${pkg1}" ] local pkg2="pkg-any-b-1-1-${arch}.pkg.tar.xz" - checkAnyPackage extra ${pkg2} + checkPackage extra ${pkg2} any } @test "testCleanupSplitPackages" { diff --git a/test/test.d/testing2x.bats b/test/test.d/testing2x.bats index 01f3f24..2314bbd 100644 --- a/test/test.d/testing2x.bats +++ b/test/test.d/testing2x.bats @@ -11,6 +11,6 @@ load ../lib/common testing2x pkg-any-a - checkAnyPackage core pkg-any-a-1-2-any.pkg.tar.xz - checkRemovedAnyPackage testing pkg-any-a + checkPackage core pkg-any-a-1-2-any.pkg.tar.xz any + checkRemovedPackage testing pkg-any-a any } -- cgit v1.2.2 From a53b874b6ed85097382d8d4889dc50cc98fbfdc6 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Thu, 20 Apr 2017 13:11:47 +0200 Subject: Move common checks to functions --- test/lib/common.bash | 22 ++++++++++++---------- test/test.d/db-repo-add.bats | 35 ++++++++++++++++++++++++----------- test/test.d/db-update.bats | 10 ++-------- test/test.d/ftpdir-cleanup.bats | 24 ++++++++++++++---------- test/test.d/sourceballs.bats | 20 +++++++++++++++----- 5 files changed, 67 insertions(+), 44 deletions(-) diff --git a/test/lib/common.bash b/test/lib/common.bash index 3435ff4..2343136 100644 --- a/test/lib/common.bash +++ b/test/lib/common.bash @@ -172,6 +172,7 @@ checkPackageDB() { local pkg=$2 local arch=$3 local db + local tarch local tarches [ -r "${FTP_BASE}/${PKGPOOL}/${pkg}" ] @@ -185,16 +186,16 @@ checkPackageDB() { tarches=(${arch}) fi - for arch in ${tarches[@]}; do - [ -L "${FTP_BASE}/${repo}/os/${arch}/${pkg}" ] - [ "$(readlink -e "${FTP_BASE}/${repo}/os/${arch}/${pkg}")" == "${FTP_BASE}/${PKGPOOL}/${pkg}" ] + for tarch in ${tarches[@]}; do + [ -L "${FTP_BASE}/${repo}/os/${tarch}/${pkg}" ] + [ "$(readlink -e "${FTP_BASE}/${repo}/os/${tarch}/${pkg}")" == "${FTP_BASE}/${PKGPOOL}/${pkg}" ] - [ -L "${FTP_BASE}/${repo}/os/${arch}/${pkg}.sig" ] - [ "$(readlink -e "${FTP_BASE}/${repo}/os/${arch}/${pkg}.sig")" == "${FTP_BASE}/${PKGPOOL}/${pkg}.sig" ] + [ -L "${FTP_BASE}/${repo}/os/${tarch}/${pkg}.sig" ] + [ "$(readlink -e "${FTP_BASE}/${repo}/os/${tarch}/${pkg}.sig")" == "${FTP_BASE}/${PKGPOOL}/${pkg}.sig" ] for db in ${DBEXT} ${FILESEXT}; do - [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" ] - bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" -O | grep -q ${pkg} + [ -r "${FTP_BASE}/${repo}/os/${tarch}/${repo}${db%.tar.*}" ] + bsdtar -xf "${FTP_BASE}/${repo}/os/${tarch}/${repo}${db%.tar.*}" -O | grep -q ${pkg} done done } @@ -227,6 +228,7 @@ checkRemovedPackageDB() { local pkgbase=$2 local arch=$3 local db + local tarch local tarches if [[ $arch == any ]]; then @@ -236,9 +238,9 @@ checkRemovedPackageDB() { fi for db in ${DBEXT} ${FILESEXT}; do - for arch in i686 x86_64; do - if [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" ]; then - echo "$(bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${db%.tar.*}" -O)" | grep -qv ${pkgbase} + for tarch in ${tarches[@]}; do + if [ -r "${FTP_BASE}/${repo}/os/${tarch}/${repo}${db%.tar.*}" ]; then + echo "$(bsdtar -xf "${FTP_BASE}/${repo}/os/${tarch}/${repo}${db%.tar.*}" -O)" | grep -qv ${pkgbase} fi done done diff --git a/test/test.d/db-repo-add.bats b/test/test.d/db-repo-add.bats index d0aae60..eb0c4aa 100644 --- a/test/test.d/db-repo-add.bats +++ b/test/test.d/db-repo-add.bats @@ -1,5 +1,26 @@ load ../lib/common +__movePackageToRepo() { + local repo=$1 + local pkgbase=$2 + local arch=$3 + local tarch + local tarches + + if [[ $arch == any ]]; then + tarches=(${ARCHES[@]}) + else + tarches=(${arch}) + fi + + # FIXME: pkgbase might not be part of the package filename + mv -v "${STAGING}"/${repo}/${pkgbase}-*-*-${arch}${PKGEXT}{,.sig} "${FTP_BASE}/${PKGPOOL}/" + for tarch in ${tarches[@]}; do + ln -sv ${FTP_BASE}/${PKGPOOL}/${pkgbase}-*-*-${arch}${PKGEXT} "${FTP_BASE}/${repo}/os/${tarch}/" + ln -sv ${FTP_BASE}/${PKGPOOL}/${pkgbase}-*-*-${arch}${PKGEXT}.sig "${FTP_BASE}/${repo}/os/${tarch}/" + done +} + @test "testRepoAddSimplePackages" { local arches=('i686' 'x86_64') local pkgs=('pkg-simple-a' 'pkg-simple-b') @@ -9,9 +30,7 @@ load ../lib/common for pkgbase in ${pkgs[@]}; do for arch in ${arches[@]}; do releasePackage extra ${pkgbase} ${arch} - mv "${STAGING}"/extra/* "${FTP_BASE}/${PKGPOOL}/" - ln -s "${FTP_BASE}/${PKGPOOL}/${pkgbase}-1-1-${arch}.pkg.tar.xz" "${FTP_BASE}/extra/os/${arch}/" - ln -s "${FTP_BASE}/${PKGPOOL}/${pkgbase}-1-1-${arch}.pkg.tar.xz.sig" "${FTP_BASE}/extra/os/${arch}/" + __movePackageToRepo extra ${pkgbase} ${arch} db-repo-add extra ${arch} ${pkgbase}-1-1-${arch}.pkg.tar.xz done done @@ -33,9 +52,7 @@ load ../lib/common add_pkgs=() for pkgbase in ${pkgs[@]}; do releasePackage extra ${pkgbase} ${arch} - mv "${STAGING}"/extra/* "${FTP_BASE}/${PKGPOOL}/" - ln -s "${FTP_BASE}/${PKGPOOL}/${pkgbase}-1-1-${arch}.pkg.tar.xz" "${FTP_BASE}/extra/os/${arch}/" - ln -s "${FTP_BASE}/${PKGPOOL}/${pkgbase}-1-1-${arch}.pkg.tar.xz.sig" "${FTP_BASE}/extra/os/${arch}/" + __movePackageToRepo extra ${pkgbase} ${arch} add_pkgs[${#add_pkgs[*]}]=${pkgbase}-1-1-${arch}.pkg.tar.xz done db-repo-add extra ${arch} ${add_pkgs[@]} @@ -56,11 +73,7 @@ load ../lib/common for pkgbase in ${pkgs[@]}; do releasePackage extra ${pkgbase} any - mv "${STAGING}"/extra/* "${FTP_BASE}/${PKGPOOL}/" - for arch in ${arches[@]}; do - ln -s "${FTP_BASE}/${PKGPOOL}/${pkgbase}-1-1-any.pkg.tar.xz" "${FTP_BASE}/extra/os/${arch}/" - ln -s "${FTP_BASE}/${PKGPOOL}/${pkgbase}-1-1-any.pkg.tar.xz.sig" "${FTP_BASE}/extra/os/${arch}/" - done + __movePackageToRepo extra ${pkgbase} any db-repo-add extra any ${pkgbase}-1-1-any.pkg.tar.xz done diff --git a/test/test.d/db-update.bats b/test/test.d/db-update.bats index c481838..d2e8b9f 100644 --- a/test/test.d/db-update.bats +++ b/test/test.d/db-update.bats @@ -118,11 +118,7 @@ load ../lib/common run db-update [ "$status" -ne 0 ] - for arch in i686 x86_64; do - if [ -r "${FTP_BASE}/testing/os/${arch}/testing${DBEXT%.tar.*}" ]; then - echo "$(bsdtar -xf "${FTP_BASE}/testing/os/${arch}/testing${DBEXT%.tar.*}" -O)" | grep -qv ${pkgbase} - fi - done + checkRemovedPackageDB testing pkg-any-a any } @test "testAddIncompleteSplitPackage" { @@ -142,9 +138,7 @@ load ../lib/common [ "$status" -ne 0 ] for arch in ${arches[@]}; do - if [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" ]; then - echo "$(bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" -O)" | grep -qv ${pkgbase} - fi + checkRemovedPackageDB ${repo} ${pkgbase} ${arch} done } diff --git a/test/test.d/ftpdir-cleanup.bats b/test/test.d/ftpdir-cleanup.bats index 809c136..18465d3 100644 --- a/test/test.d/ftpdir-cleanup.bats +++ b/test/test.d/ftpdir-cleanup.bats @@ -1,5 +1,15 @@ load ../lib/common +__checkRepoRemovedPackage() { + local repo=$1 + local pkgbase=$2 + local arch=$3 + + # FIXME: pkgbase might not be part of the package filename + [[ ! -f ${FTP_BASE}/${PKGPOOL}/${pkgbase}*${PKGEXT} ]] + [[ ! -f ${FTP_BASE}/${repo}/os/${arch}/${pkgbase}*${PKGEXT} ]] +} + @test "testCleanupSimplePackages" { local arches=('i686' 'x86_64') local pkgs=('pkg-simple-a' 'pkg-simple-b') @@ -21,10 +31,8 @@ load ../lib/common ftpdir-cleanup for arch in ${arches[@]}; do - local pkg1="pkg-simple-a-1-1-${arch}.pkg.tar.xz" checkRemovedPackage extra 'pkg-simple-a' ${arch} - [ ! -f "${FTP_BASE}/${PKGPOOL}/${pkg1}" ] - [ ! -f "${FTP_BASE}/${repo}/os/${arch}/${pkg1}" ] + __checkRepoRemovedPackage extra 'pkg-simple-a' ${arch} local pkg2="pkg-simple-b-1-1-${arch}.pkg.tar.xz" checkPackage extra ${pkg2} ${arch} @@ -52,10 +60,8 @@ load ../lib/common ftpdir-cleanup for arch in ${arches[@]}; do - local pkg1="pkg-simple-epoch-1:1-1-${arch}.pkg.tar.xz" checkRemovedPackage extra 'pkg-simple-epoch' ${arch} - [ ! -f "${FTP_BASE}/${PKGPOOL}/${pkg1}" ] - [ ! -f "${FTP_BASE}/${repo}/os/${arch}/${pkg1}" ] + __checkRepoRemovedPackage extra 'pkg-simple-epoch' ${arch} done } @@ -74,8 +80,7 @@ load ../lib/common local pkg1='pkg-any-a-1-1-any.pkg.tar.xz' checkRemovedPackage extra 'pkg-any-a' any - [ ! -f "${FTP_BASE}/${PKGPOOL}/${pkg1}" ] - [ ! -f "${FTP_BASE}/${repo}/os/${arch}/${pkg1}" ] + __checkRepoRemovedPackage extra 'pkg-any-a' any local pkg2="pkg-any-b-1-1-${arch}.pkg.tar.xz" checkPackage extra ${pkg2} any @@ -105,8 +110,7 @@ load ../lib/common for arch in ${arches[@]}; do for pkg in $(getPackageNamesFromPackageBase ${pkgs[0]}); do checkRemovedPackage extra ${pkg} ${arch} - [ ! -f "${FTP_BASE}/${PKGPOOL}/${pkg}" ] - [ ! -f "${FTP_BASE}/${repo}/os/${arch}/${pkg}" ] + __checkRepoRemovedPackage extra ${pkg} ${arch} done for pkg in $(getPackageNamesFromPackageBase ${pkgs[1]}); do diff --git a/test/test.d/sourceballs.bats b/test/test.d/sourceballs.bats index 70836ad..7ccbd75 100644 --- a/test/test.d/sourceballs.bats +++ b/test/test.d/sourceballs.bats @@ -1,5 +1,15 @@ load ../lib/common +__checkSourcePackage() { + local pkgbase=$1 + [ -r ${FTP_BASE}/${SRCPOOL}/${pkgbase}-*${SRCEXT} ] +} + +__checkRemovedSourcePackage() { + local pkgbase=$1 + [ ! -r ${FTP_BASE}/${SRCPOOL}/${pkgbase}-*${SRCEXT} ] +} + @test "testSourceballs" { local arches=('i686' 'x86_64') local pkgs=('pkg-simple-a' 'pkg-simple-b' 'pkg-simple-epoch') @@ -15,7 +25,7 @@ load ../lib/common sourceballs for pkgbase in ${pkgs[@]}; do - [ -r ${FTP_BASE}/${SRCPOOL}/${pkgbase}-*${SRCEXT} ] + __checkSourcePackage ${pkgbase} done } @@ -30,7 +40,7 @@ load ../lib/common sourceballs for pkgbase in ${pkgs[@]}; do - [ -r ${FTP_BASE}/${SRCPOOL}/${pkgbase}-*${SRCEXT} ] + __checkSourcePackage ${pkgbase} done } @@ -51,7 +61,7 @@ load ../lib/common sourceballs for pkgbase in ${pkgs[@]}; do - [ -r ${FTP_BASE}/${SRCPOOL}/${pkgbase}-*${SRCEXT} ] + __checkSourcePackage ${pkgbase} done } @@ -74,6 +84,6 @@ load ../lib/common done sourceballs - [ ! -r ${FTP_BASE}/${SRCPOOL}/pkg-simple-a-*${SRCEXT} ] - [ -r ${FTP_BASE}/${SRCPOOL}/pkg-simple-b-*${SRCEXT} ] + __checkRemovedSourcePackage pkg-simple-a + __checkSourcePackage pkg-simple-b } -- cgit v1.2.2 From b1c039e2d2a63fa305eb7f16130bdb7273bd88e2 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Thu, 20 Apr 2017 13:32:18 +0200 Subject: Remove useless code --- test/test.d/db-move.bats | 5 ----- 1 file changed, 5 deletions(-) diff --git a/test/test.d/db-move.bats b/test/test.d/db-move.bats index d12c0e5..ccab292 100644 --- a/test/test.d/db-move.bats +++ b/test/test.d/db-move.bats @@ -106,13 +106,8 @@ load ../lib/common for pkg in $(getPackageNamesFromPackageBase pkg-split-a); do checkPackage extra ${pkg} ${arch} done - done - for arch in ${arches[@]}; do for pkg in $(getPackageNamesFromPackageBase pkg-split-b); do checkPackage testing ${pkg} ${arch} done done - - # FIXME: Why any? - checkRemovedPackage testing pkg-split-a any } -- cgit v1.2.2 From 7767d890385219cee6b7243e224861b547151ac8 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Thu, 20 Apr 2017 15:24:02 +0200 Subject: Use tmpfs as build dir --- Makefile | 2 +- test/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 988d335..677bc39 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ IMAGE:=dbscripts/test -RUN_OPTIONS:=--rm --network=none -v $(PWD):/dbscripts:ro --tmpfs=/tmp:exec -w /dbscripts/test +RUN_OPTIONS:=--rm --network=none -v $(PWD):/dbscripts:ro --tmpfs=/tmp:exec --tmpfs=/build -w /dbscripts/test test-image: docker build --pull -t $(IMAGE) test diff --git a/test/Dockerfile b/test/Dockerfile index e428f0a..3668d5b 100644 --- a/test/Dockerfile +++ b/test/Dockerfile @@ -3,7 +3,7 @@ RUN pacman -Syu --noconfirm --needed base-devel devtools kcov bash-bats RUN pacman-key --init RUN echo '%wheel ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/wheel RUN useradd -N -g users -G wheel -m tester -RUN install -d -o tester -g users /build +RUN install -d -m 1777 /build USER tester RUN echo -e "\ %echo Generating signing key...\n\ -- cgit v1.2.2 From 76f95dd139094dc2985b6733f23fcb098f07c64e Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Thu, 20 Apr 2017 21:55:23 +0200 Subject: Allways build packages for all supported architectures --- db-functions | 4 +- test/Dockerfile | 3 +- test/Makefile | 6 +-- test/lib/common.bash | 84 +++++++++++++++---------------- test/packages/pkg-single-arch/PKGBUILD | 12 +++++ test/packages/pkg-single-epoch/PKGBUILD | 13 +++++ test/test.d/db-move.bats | 18 ++----- test/test.d/db-remove.bats | 10 ++-- test/test.d/db-repo-add.bats | 9 ++-- test/test.d/db-repo-remove.bats | 10 ++-- test/test.d/db-update.bats | 88 +++++++++++++++------------------ test/test.d/ftpdir-cleanup.bats | 18 ++----- test/test.d/sourceballs.bats | 14 ++---- test/test.d/testing2x.bats | 6 +-- 14 files changed, 139 insertions(+), 156 deletions(-) create mode 100644 test/packages/pkg-single-arch/PKGBUILD create mode 100644 test/packages/pkg-single-epoch/PKGBUILD diff --git a/db-functions b/db-functions index 45e8d38..38b3c2f 100644 --- a/db-functions +++ b/db-functions @@ -1,8 +1,6 @@ #!/bin/bash -. /usr/share/makepkg/util/message.sh -. /usr/share/makepkg/util/util.sh -. /usr/share/makepkg/util/pkgbuild.sh +. /usr/share/makepkg/util.sh # Some PKGBUILDs need CARCH to be set CARCH="x86_64" diff --git a/test/Dockerfile b/test/Dockerfile index 3668d5b..d01350c 100644 --- a/test/Dockerfile +++ b/test/Dockerfile @@ -19,4 +19,5 @@ Expire-Date: 0\n\ | gpg --quiet --batch --no-tty --no-permission-warning --gen-key RUN gpg --export | sudo pacman-key -a - RUN sudo pacman-key --lsign-key tester@localhost -RUN echo -e 'BUILDDIR=/build\nPACKAGER="Bob Tester "' > /home/tester/.makepkg.conf +ENV BUILDDIR=/build +ENV PACKAGER="Bob Tester " diff --git a/test/Makefile b/test/Makefile index c4cc021..afc3cff 100644 --- a/test/Makefile +++ b/test/Makefile @@ -1,10 +1,8 @@ -PACKAGE_CACHE:=$(shell mktemp -d) - test: - PATH=$(CURDIR)/../:$(CURDIR)/../cron-jobs/:$(PATH) PACKAGE_CACHE=$(PACKAGE_CACHE) bats test.d + PATH=$(CURDIR)/../:$(CURDIR)/../cron-jobs/:$(PATH) bats test.d test-coverage: - PATH=$(CURDIR)/../:$(CURDIR)/../cron-jobs/:$(PATH) PACKAGE_CACHE=$(PACKAGE_CACHE) kcov \ + PATH=$(CURDIR)/../:$(CURDIR)/../cron-jobs/:$(PATH) kcov \ --include-path=$(CURDIR)/../ \ --exclude-path=$(CURDIR)/../test,$(CURDIR)/../cron-jobs/makepkg.conf,$(CURDIR)/../config \ $(COVERAGE_DIR) \ diff --git a/test/lib/common.bash b/test/lib/common.bash index 2343136..3388ff5 100644 --- a/test/lib/common.bash +++ b/test/lib/common.bash @@ -1,4 +1,4 @@ -. /usr/share/makepkg/util/pkgbuild.sh +. /usr/share/makepkg/util.sh __getPackageBaseFromPackage() { local _base @@ -31,38 +31,43 @@ __getCheckSum() { } __buildPackage() { - local arch=$1 - local pkgver - local pkgname - local a + local pkgdest=${1:-.} local p - local checkSum - - if [[ -n ${PACKAGE_CACHE} ]]; then - checkSum=$(__getCheckSum PKGBUILD) - # TODO: Be more specific - if cp -av ${PACKAGE_CACHE}/${checkSum}/*-${arch}${PKGEXT}{,.sig} .; then - return 0 - fi + local cache + local pkgarches + local tarch + local pkgnames + + if [[ -n ${BUILDDIR} ]]; then + cache=${BUILDDIR}/$(__getCheckSum PKGBUILD) + if [[ -d ${cache} ]]; then + cp -Lv ${cache}/*${PKGEXT}{,.sig} ${pkgdest} + return 0 + else + mkdir -p ${cache} + fi fi - pkgname=($(. PKGBUILD; echo ${pkgname[@]})) - pkgver=$(. PKGBUILD; get_full_version) + pkgarches=($(. PKGBUILD; echo ${arch[@]})) + for tarch in ${pkgarches[@]}; do + if [ "${tarch}" == 'any' ]; then + PKGDEST=${pkgdest} makepkg -c + else + PKGDEST=${pkgdest} CARCH=${tarch} makepkg -c + fi + done - if [ "${arch}" == 'any' ]; then - makepkg -c - else - CARCH=${arch} makepkg -c - fi + pkgnames=($(. PKGBUILD; print_all_package_names)) + pushd ${pkgdest} + for p in ${pkgnames[@]/%/${PKGEXT}}; do + # Manually sign packages as "makepkg --sign" is buggy + gpg -v --detach-sign --no-armor --use-agent ${p} - for p in ${pkgname[@]}; do - gpg --detach-sign --no-armor --use-agent ${p}-${pkgver}-${arch}* + if [[ -n ${BUILDDIR} ]]; then + cp -Lv ${p}{,.sig} ${cache}/ + fi done - - if [[ -n ${PACKAGE_CACHE} ]]; then - mkdir -p ${PACKAGE_CACHE}/${checkSum} - cp -av *-${arch}${PKGEXT}{,.sig} ${PACKAGE_CACHE}/${checkSum}/ - fi + popd } setup() { @@ -113,11 +118,8 @@ teardown() { releasePackage() { local repo=$1 local pkgbase=$2 - local arch=$3 - local a - local p - local pkgver - local pkgname + local pkgarches + local tarch if [ ! -d "${TMP}/svn-packages-copy/${pkgbase}/trunk" ]; then mkdir -p "${TMP}/svn-packages-copy/${pkgbase}"/{trunk,repos} @@ -127,27 +129,21 @@ releasePackage() { fi pushd "${TMP}/svn-packages-copy"/${pkgbase}/trunk/ - __buildPackage ${arch} - archrelease -f ${repo}-${arch} - pkgver=$(. PKGBUILD; get_full_version) - pkgname=($(. PKGBUILD; echo ${pkgname[@]})) - - for a in ${arch[@]}; do - for p in ${pkgname[@]}; do - cp ${p}-${pkgver}-${a}${PKGEXT}{,.sig} "${STAGING}"/${repo}/ - done - done + __buildPackage "${STAGING}"/${repo} + pkgarches=($(. PKGBUILD; echo ${arch[@]})) + for tarch in ${pkgarches[@]}; do + archrelease -f ${repo}-${tarch} + done popd } updatePackage() { local pkgbase=$1 - local arch=$2 pushd "${TMP}/svn-packages-copy/${pkgbase}/trunk/" __updatePKGBUILD - __buildPackage ${arch} + __buildPackage popd } diff --git a/test/packages/pkg-single-arch/PKGBUILD b/test/packages/pkg-single-arch/PKGBUILD new file mode 100644 index 0000000..581efb4 --- /dev/null +++ b/test/packages/pkg-single-arch/PKGBUILD @@ -0,0 +1,12 @@ +pkgname=pkg-single-arch +pkgver=1 +pkgrel=1 +pkgdesc="A package called ${pkgname}" +arch=('x86_64') +url='http://www.archlinux.org/' +license=('GPL') +depends=('glibc') + +package() { + install -D -m755 /bin/true ${pkgdir}/usr/bin/${pkgname} +} diff --git a/test/packages/pkg-single-epoch/PKGBUILD b/test/packages/pkg-single-epoch/PKGBUILD new file mode 100644 index 0000000..c1bc3d6 --- /dev/null +++ b/test/packages/pkg-single-epoch/PKGBUILD @@ -0,0 +1,13 @@ +pkgname=pkg-single-epoch +pkgver=1 +pkgrel=1 +epoch=1 +pkgdesc="A package called ${pkgname}" +arch=('x86_64') +url='http://www.archlinux.org/' +license=('GPL') +depends=('glibc') + +package() { + install -D -m755 /bin/true ${pkgdir}/usr/bin/${pkgname} +} diff --git a/test/test.d/db-move.bats b/test/test.d/db-move.bats index ccab292..a154183 100644 --- a/test/test.d/db-move.bats +++ b/test/test.d/db-move.bats @@ -7,9 +7,7 @@ load ../lib/common local arch for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage testing ${pkgbase} ${arch} - done + releasePackage testing ${pkgbase} done db-update @@ -31,9 +29,7 @@ load ../lib/common local arch for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage testing ${pkgbase} ${arch} - done + releasePackage testing ${pkgbase} done db-update @@ -55,9 +51,7 @@ load ../lib/common local arch for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage testing ${pkgbase} ${arch} - done + releasePackage testing ${pkgbase} done db-update @@ -75,7 +69,7 @@ load ../lib/common local pkgbase for pkgbase in ${pkgs[@]}; do - releasePackage testing ${pkgbase} any + releasePackage testing ${pkgbase} done db-update @@ -94,9 +88,7 @@ load ../lib/common local arch for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage testing ${pkgbase} ${arch} - done + releasePackage testing ${pkgbase} done db-update diff --git a/test/test.d/db-remove.bats b/test/test.d/db-remove.bats index e70745f..3e0d1f4 100644 --- a/test/test.d/db-remove.bats +++ b/test/test.d/db-remove.bats @@ -7,9 +7,7 @@ load ../lib/common local arch for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage extra ${pkgbase} ${arch} - done + releasePackage extra ${pkgbase} done db-update @@ -34,9 +32,7 @@ load ../lib/common local arch for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage extra ${pkgbase} ${arch} - done + releasePackage extra ${pkgbase} done db-update @@ -57,7 +53,7 @@ load ../lib/common local pkgbase for pkgbase in ${pkgs[@]}; do - releasePackage extra ${pkgbase} any + releasePackage extra ${pkgbase} done db-update diff --git a/test/test.d/db-repo-add.bats b/test/test.d/db-repo-add.bats index eb0c4aa..98fa001 100644 --- a/test/test.d/db-repo-add.bats +++ b/test/test.d/db-repo-add.bats @@ -28,8 +28,8 @@ __movePackageToRepo() { local arch for pkgbase in ${pkgs[@]}; do + releasePackage extra ${pkgbase} for arch in ${arches[@]}; do - releasePackage extra ${pkgbase} ${arch} __movePackageToRepo extra ${pkgbase} ${arch} db-repo-add extra ${arch} ${pkgbase}-1-1-${arch}.pkg.tar.xz done @@ -48,10 +48,13 @@ __movePackageToRepo() { local pkgbase local arch + for pkgbase in ${pkgs[@]}; do + releasePackage extra ${pkgbase} + done + for arch in ${arches[@]}; do add_pkgs=() for pkgbase in ${pkgs[@]}; do - releasePackage extra ${pkgbase} ${arch} __movePackageToRepo extra ${pkgbase} ${arch} add_pkgs[${#add_pkgs[*]}]=${pkgbase}-1-1-${arch}.pkg.tar.xz done @@ -72,7 +75,7 @@ __movePackageToRepo() { local arch for pkgbase in ${pkgs[@]}; do - releasePackage extra ${pkgbase} any + releasePackage extra ${pkgbase} __movePackageToRepo extra ${pkgbase} any db-repo-add extra any ${pkgbase}-1-1-any.pkg.tar.xz done diff --git a/test/test.d/db-repo-remove.bats b/test/test.d/db-repo-remove.bats index 35ff0e7..b3a0b3b 100644 --- a/test/test.d/db-repo-remove.bats +++ b/test/test.d/db-repo-remove.bats @@ -7,9 +7,7 @@ load ../lib/common local arch for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage extra ${pkgbase} ${arch} - done + releasePackage extra ${pkgbase} done db-update @@ -34,9 +32,7 @@ load ../lib/common local arch for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage extra ${pkgbase} ${arch} - done + releasePackage extra ${pkgbase} done db-update @@ -57,7 +53,7 @@ load ../lib/common local pkgbase for pkgbase in ${pkgs[@]}; do - releasePackage extra ${pkgbase} any + releasePackage extra ${pkgbase} done db-update diff --git a/test/test.d/db-update.bats b/test/test.d/db-update.bats index d2e8b9f..55eeee7 100644 --- a/test/test.d/db-update.bats +++ b/test/test.d/db-update.bats @@ -7,9 +7,7 @@ load ../lib/common local arch for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage extra ${pkgbase} ${arch} - done + releasePackage extra ${pkgbase} done db-update @@ -22,15 +20,15 @@ load ../lib/common } @test "testAddSingleSimplePackage" { - releasePackage extra 'pkg-simple-a' 'i686' + releasePackage extra 'pkg-single-arch' db-update - checkPackage extra 'pkg-simple-a-1-1-i686.pkg.tar.xz' 'i686' + checkPackage extra 'pkg-single-arch-1-1-x86_64.pkg.tar.xz' 'x86_64' } @test "testAddSingleEpochPackage" { - releasePackage extra 'pkg-simple-epoch' 'i686' + releasePackage extra 'pkg-single-epoch' db-update - checkPackage extra 'pkg-simple-epoch-1:1-1-i686.pkg.tar.xz' 'i686' + checkPackage extra 'pkg-single-epoch-1:1-1-x86_64.pkg.tar.xz' 'x86_64' } @test "testAddAnyPackages" { @@ -38,7 +36,7 @@ load ../lib/common local pkgbase for pkgbase in ${pkgs[@]}; do - releasePackage extra ${pkgbase} any + releasePackage extra ${pkgbase} done db-update @@ -56,9 +54,7 @@ load ../lib/common local arch for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage extra ${pkgbase} ${arch} - done + releasePackage extra ${pkgbase} done db-update @@ -73,23 +69,23 @@ load ../lib/common } @test "testUpdateAnyPackage" { - releasePackage extra pkg-any-a any + releasePackage extra pkg-any-a db-update - updatePackage pkg-any-a any + updatePackage pkg-any-a - releasePackage extra pkg-any-a any + releasePackage extra pkg-any-a db-update checkPackage extra pkg-any-a-1-2-any.pkg.tar.xz any } @test "testUpdateAnyPackageToDifferentRepositoriesAtOnce" { - releasePackage extra pkg-any-a any + releasePackage extra pkg-any-a - updatePackage pkg-any-a any + updatePackage pkg-any-a - releasePackage testing pkg-any-a any + releasePackage testing pkg-any-a db-update @@ -98,11 +94,11 @@ load ../lib/common } @test "testUpdateSameAnyPackageToSameRepository" { - releasePackage extra pkg-any-a any + releasePackage extra pkg-any-a db-update checkPackage extra pkg-any-a-1-1-any.pkg.tar.xz any - releasePackage extra pkg-any-a any + releasePackage extra pkg-any-a run db-update [ "$status" -ne 0 ] } @@ -110,11 +106,11 @@ load ../lib/common @test "testUpdateSameAnyPackageToDifferentRepositories" { local arch - releasePackage extra pkg-any-a any + releasePackage extra pkg-any-a db-update checkPackage extra pkg-any-a-1-1-any.pkg.tar.xz any - releasePackage testing pkg-any-a any + releasePackage testing pkg-any-a run db-update [ "$status" -ne 0 ] @@ -127,9 +123,7 @@ load ../lib/common local pkgbase='pkg-split-a' local arch - for arch in ${arches[@]}; do - releasePackage ${repo} ${pkgbase} ${arch} - done + releasePackage ${repo} ${pkgbase} # remove a split package to make db-update fail rm "${STAGING}"/extra/${pkgbase}1-* @@ -144,26 +138,26 @@ load ../lib/common @test "testUnknownRepo" { mkdir "${STAGING}/unknown/" - releasePackage extra 'pkg-simple-a' 'i686' - releasePackage unknown 'pkg-simple-b' 'i686' + releasePackage extra 'pkg-any-a' + releasePackage unknown 'pkg-any-b' db-update - checkPackage extra 'pkg-simple-a-1-1-i686.pkg.tar.xz' 'i686' + checkPackage extra 'pkg-any-a-1-1-any.pkg.tar.xz' any [ ! -e "${FTP_BASE}/unknown" ] rm -rf "${STAGING}/unknown/" } @test "testAddUnsignedPackageFails" { - releasePackage extra 'pkg-simple-a' 'i686' + releasePackage extra 'pkg-any-a' rm "${STAGING}"/extra/*.sig run db-update [ "$status" -ne 0 ] - checkRemovedPackage extra pkg-simple-a-1-1-i686.pkg.tar.xz i686 + checkRemovedPackage extra pkg-any-a-1-1-any.pkg.tar.xz any } @test "testAddInvalidSignedPackageFails" { local p - releasePackage extra 'pkg-simple-a' 'i686' + releasePackage extra 'pkg-any-a' for p in "${STAGING}"/extra/*${PKGEXT}; do unxz $p xz -0 ${p%%.xz} @@ -171,37 +165,37 @@ load ../lib/common run db-update [ "$status" -ne 0 ] - checkRemovedPackage extra pkg-simple-a-1-1-i686.pkg.tar.xz i686 + checkRemovedPackage extra pkg-any-a-1-1-any.pkg.tar.xz any } @test "testAddBrokenSignatureFails" { local s - releasePackage extra 'pkg-simple-a' 'i686' + releasePackage extra 'pkg-any-a' for s in "${STAGING}"/extra/*.sig; do echo 0 > $s done run db-update [ "$status" -ne 0 ] - checkRemovedPackage extra pkg-simple-a-1-1-i686.pkg.tar.xz i686 + checkRemovedPackage extra pkg-any-a-1-1-any.pkg.tar.xz any } @test "testAddPackageWithInconsistentVersionFails" { local p - releasePackage extra 'pkg-simple-a' 'i686' + releasePackage extra 'pkg-any-a' for p in "${STAGING}"/extra/*; do - mv "${p}" "${p/pkg-simple-a-1/pkg-simple-a-2}" + mv "${p}" "${p/pkg-any-a-1/pkg-any-a-2}" done run db-update [ "$status" -ne 0 ] - checkRemovedPackage extra 'pkg-simple-a-2-1-i686.pkg.tar.xz' 'i686' + checkRemovedPackage extra 'pkg-any-a-2-1-any.pkg.tar.xz' 'any' } @test "testAddPackageWithInconsistentNameFails" { local p - releasePackage extra 'pkg-simple-a' 'i686' + releasePackage extra 'pkg-any-a' for p in "${STAGING}"/extra/*; do mv "${p}" "${p/pkg-/foo-pkg-}" @@ -209,30 +203,30 @@ load ../lib/common run db-update [ "$status" -ne 0 ] - checkRemovedPackage extra 'foo-pkg-simple-a-1-1-i686.pkg.tar.xz' 'i686' + checkRemovedPackage extra 'foo-pkg-any-a-1-1-any.pkg.tar.xz' 'any' } @test "testAddPackageWithInconsistentPKGBUILDFails" { - releasePackage extra 'pkg-simple-a' 'i686' + releasePackage extra 'pkg-any-a' - updateRepoPKGBUILD 'pkg-simple-a' extra i686 + updateRepoPKGBUILD 'pkg-any-a' extra any run db-update [ "$status" -ne 0 ] - checkRemovedPackage extra 'pkg-simple-a-1-1-i686.pkg.tar.xz' 'i686' + checkRemovedPackage extra 'pkg-any-a-1-1-any.pkg.tar.xz' 'any' } @test "testAddPackageWithInsufficientPermissionsFails" { - releasePackage core 'pkg-simple-a' 'i686' - releasePackage extra 'pkg-simple-b' 'i686' + releasePackage core 'pkg-any-a' + releasePackage extra 'pkg-any-b' chmod -xwr ${FTP_BASE}/core/os/i686 run db-update [ "$status" -ne 0 ] chmod +xwr ${FTP_BASE}/core/os/i686 - checkRemovedPackage core 'pkg-simple-a-1-1-i686.pkg.tar.xz' 'i686' - checkRemovedPackage extra 'pkg-simple-b-1-1-i686.pkg.tar.xz' 'i686' + checkRemovedPackage core 'pkg-any-a-1-1-any.pkg.tar.xz' 'any' + checkRemovedPackage extra 'pkg-any-b-1-1-any.pkg.tar.xz' 'any' } @test "testPackageHasToBeARegularFile" { @@ -240,9 +234,7 @@ load ../lib/common local target=$(mktemp -d) local arches=('i686' 'x86_64') - for arch in ${arches[@]}; do - releasePackage extra 'pkg-simple-a' $arch - done + releasePackage extra 'pkg-simple-a' for p in "${STAGING}"/extra/*i686*; do mv "${p}" "${target}" diff --git a/test/test.d/ftpdir-cleanup.bats b/test/test.d/ftpdir-cleanup.bats index 18465d3..444c3c5 100644 --- a/test/test.d/ftpdir-cleanup.bats +++ b/test/test.d/ftpdir-cleanup.bats @@ -17,9 +17,7 @@ __checkRepoRemovedPackage() { local arch for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage extra ${pkgbase} ${arch} - done + releasePackage extra ${pkgbase} done db-update @@ -46,9 +44,7 @@ __checkRepoRemovedPackage() { local arch for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage extra ${pkgbase} ${arch} - done + releasePackage extra ${pkgbase} done db-update @@ -71,7 +67,7 @@ __checkRepoRemovedPackage() { local arch='any' for pkgbase in ${pkgs[@]}; do - releasePackage extra ${pkgbase} any + releasePackage extra ${pkgbase} done db-update @@ -94,9 +90,7 @@ __checkRepoRemovedPackage() { local arch for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage extra ${pkgbase} ${arch} - done + releasePackage extra ${pkgbase} done db-update @@ -126,9 +120,7 @@ __checkRepoRemovedPackage() { local arch for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage extra ${pkgbase} ${arch} - done + releasePackage extra ${pkgbase} done db-update diff --git a/test/test.d/sourceballs.bats b/test/test.d/sourceballs.bats index 7ccbd75..0edc594 100644 --- a/test/test.d/sourceballs.bats +++ b/test/test.d/sourceballs.bats @@ -17,9 +17,7 @@ __checkRemovedSourcePackage() { local arch for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage extra ${pkgbase} ${arch} - done + releasePackage extra ${pkgbase} done db-update @@ -34,7 +32,7 @@ __checkRemovedSourcePackage() { local pkgbase for pkgbase in ${pkgs[@]}; do - releasePackage extra ${pkgbase} any + releasePackage extra ${pkgbase} done db-update @@ -52,9 +50,7 @@ __checkRemovedSourcePackage() { local arch for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage extra ${pkgbase} ${arch} - done + releasePackage extra ${pkgbase} done db-update @@ -72,9 +68,7 @@ __checkRemovedSourcePackage() { local arch for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage extra ${pkgbase} ${arch} - done + releasePackage extra ${pkgbase} done db-update sourceballs diff --git a/test/test.d/testing2x.bats b/test/test.d/testing2x.bats index 2314bbd..bac75de 100644 --- a/test/test.d/testing2x.bats +++ b/test/test.d/testing2x.bats @@ -1,12 +1,12 @@ load ../lib/common @test "testTesting2xAnyPackage" { - releasePackage core pkg-any-a any + releasePackage core pkg-any-a db-update - updatePackage pkg-any-a any + updatePackage pkg-any-a - releasePackage testing pkg-any-a any + releasePackage testing pkg-any-a db-update testing2x pkg-any-a -- cgit v1.2.2 From 3c1a10948081da550fdf59c519a57489ec774c69 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Thu, 20 Apr 2017 23:04:06 +0200 Subject: Remove dependency on devtools --- test/lib/common.bash | 32 +++++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/test/lib/common.bash b/test/lib/common.bash index 3388ff5..76c389d 100644 --- a/test/lib/common.bash +++ b/test/lib/common.bash @@ -70,6 +70,30 @@ __buildPackage() { popd } +__archrelease() { + local repo=$1 + local pkgarches + local tarch + local tag + + pkgarches=($(. PKGBUILD; echo ${arch[@]})) + pushd .. + for tarch in ${pkgarches[@]}; do + tag=${repo}-${tarch} + + if [[ -d repos/$tag ]]; then + svn rm repos/$tag/PKGBUILD + else + mkdir -p repos/$tag + svn add repos/$tag + fi + + svn copy -r HEAD trunk/PKGBUILD repos/$tag/ + done + svn commit -m "__archrelease" + popd +} + setup() { local p local pkg @@ -118,8 +142,6 @@ teardown() { releasePackage() { local repo=$1 local pkgbase=$2 - local pkgarches - local tarch if [ ! -d "${TMP}/svn-packages-copy/${pkgbase}/trunk" ]; then mkdir -p "${TMP}/svn-packages-copy/${pkgbase}"/{trunk,repos} @@ -129,12 +151,8 @@ releasePackage() { fi pushd "${TMP}/svn-packages-copy"/${pkgbase}/trunk/ - __buildPackage "${STAGING}"/${repo} - pkgarches=($(. PKGBUILD; echo ${arch[@]})) - for tarch in ${pkgarches[@]}; do - archrelease -f ${repo}-${tarch} - done + __archrelease ${repo} popd } -- cgit v1.2.2 From c39624b25a16ed260b420d5689f0a73378fa1372 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Thu, 20 Apr 2017 23:15:34 +0200 Subject: Reduce test dependencies --- test/Dockerfile | 6 ++---- test/packages/pkg-any-a/PKGBUILD | 1 + test/packages/pkg-any-b/PKGBUILD | 1 + test/packages/pkg-simple-a/PKGBUILD | 1 + test/packages/pkg-simple-b/PKGBUILD | 1 + test/packages/pkg-simple-epoch/PKGBUILD | 1 + test/packages/pkg-single-arch/PKGBUILD | 1 + test/packages/pkg-single-epoch/PKGBUILD | 1 + test/packages/pkg-split-a/PKGBUILD | 1 + test/packages/pkg-split-b/PKGBUILD | 1 + test/test.d/packages.bats | 34 --------------------------------- 11 files changed, 11 insertions(+), 38 deletions(-) delete mode 100644 test/test.d/packages.bats diff --git a/test/Dockerfile b/test/Dockerfile index d01350c..3fa9aaf 100644 --- a/test/Dockerfile +++ b/test/Dockerfile @@ -1,12 +1,11 @@ FROM pierres/archlinux -RUN pacman -Syu --noconfirm --needed base-devel devtools kcov bash-bats +RUN pacman -Syu --noconfirm sudo fakeroot awk subversion make kcov bash-bats RUN pacman-key --init RUN echo '%wheel ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/wheel RUN useradd -N -g users -G wheel -m tester RUN install -d -m 1777 /build USER tester RUN echo -e "\ -%echo Generating signing key...\n\ Key-Type: RSA\n\ Key-Length: 1024\n\ Key-Usage: sign\n\ @@ -14,8 +13,7 @@ Name-Real: Bob Tester\n\ Name-Email: tester@localhost\n\ Expire-Date: 0\n\ %no-protection\n\ -%commit\n\ -%echo Done\n"\ +%commit\n"\ | gpg --quiet --batch --no-tty --no-permission-warning --gen-key RUN gpg --export | sudo pacman-key -a - RUN sudo pacman-key --lsign-key tester@localhost diff --git a/test/packages/pkg-any-a/PKGBUILD b/test/packages/pkg-any-a/PKGBUILD index 3026919..cf2774b 100644 --- a/test/packages/pkg-any-a/PKGBUILD +++ b/test/packages/pkg-any-a/PKGBUILD @@ -5,6 +5,7 @@ pkgdesc="A package called ${pkgname}" arch=('any') url='http://www.archlinux.org/' license=('GPL') +options=(!strip) package() { install -d ${pkgdir}/usr/share/${pkgname} diff --git a/test/packages/pkg-any-b/PKGBUILD b/test/packages/pkg-any-b/PKGBUILD index cca1859..05dfd10 100644 --- a/test/packages/pkg-any-b/PKGBUILD +++ b/test/packages/pkg-any-b/PKGBUILD @@ -5,6 +5,7 @@ pkgdesc="A package called ${pkgname}" arch=('any') url='http://www.archlinux.org/' license=('GPL') +options=(!strip) package() { install -d ${pkgdir}/usr/share/${pkgname} diff --git a/test/packages/pkg-simple-a/PKGBUILD b/test/packages/pkg-simple-a/PKGBUILD index 870339d..329a4a3 100644 --- a/test/packages/pkg-simple-a/PKGBUILD +++ b/test/packages/pkg-simple-a/PKGBUILD @@ -6,6 +6,7 @@ arch=('i686' 'x86_64') url='http://www.archlinux.org/' license=('GPL') depends=('glibc') +options=(!strip) package() { install -D -m755 /bin/true ${pkgdir}/usr/bin/${pkgname} diff --git a/test/packages/pkg-simple-b/PKGBUILD b/test/packages/pkg-simple-b/PKGBUILD index 22f7ece..287eee7 100644 --- a/test/packages/pkg-simple-b/PKGBUILD +++ b/test/packages/pkg-simple-b/PKGBUILD @@ -6,6 +6,7 @@ arch=('i686' 'x86_64') url='http://www.archlinux.org/' license=('GPL') depends=('glibc') +options=(!strip) package() { install -D -m755 /bin/true ${pkgdir}/usr/bin/${pkgname} diff --git a/test/packages/pkg-simple-epoch/PKGBUILD b/test/packages/pkg-simple-epoch/PKGBUILD index f81d067..c624439 100644 --- a/test/packages/pkg-simple-epoch/PKGBUILD +++ b/test/packages/pkg-simple-epoch/PKGBUILD @@ -7,6 +7,7 @@ arch=('i686' 'x86_64') url='http://www.archlinux.org/' license=('GPL') depends=('glibc') +options=(!strip) package() { install -D -m755 /bin/true ${pkgdir}/usr/bin/${pkgname} diff --git a/test/packages/pkg-single-arch/PKGBUILD b/test/packages/pkg-single-arch/PKGBUILD index 581efb4..de5dbc2 100644 --- a/test/packages/pkg-single-arch/PKGBUILD +++ b/test/packages/pkg-single-arch/PKGBUILD @@ -6,6 +6,7 @@ arch=('x86_64') url='http://www.archlinux.org/' license=('GPL') depends=('glibc') +options=(!strip) package() { install -D -m755 /bin/true ${pkgdir}/usr/bin/${pkgname} diff --git a/test/packages/pkg-single-epoch/PKGBUILD b/test/packages/pkg-single-epoch/PKGBUILD index c1bc3d6..95898ac 100644 --- a/test/packages/pkg-single-epoch/PKGBUILD +++ b/test/packages/pkg-single-epoch/PKGBUILD @@ -7,6 +7,7 @@ arch=('x86_64') url='http://www.archlinux.org/' license=('GPL') depends=('glibc') +options=(!strip) package() { install -D -m755 /bin/true ${pkgdir}/usr/bin/${pkgname} diff --git a/test/packages/pkg-split-a/PKGBUILD b/test/packages/pkg-split-a/PKGBUILD index 2acfd92..2c91390 100644 --- a/test/packages/pkg-split-a/PKGBUILD +++ b/test/packages/pkg-split-a/PKGBUILD @@ -7,6 +7,7 @@ arch=('i686' 'x86_64') url='http://www.archlinux.org/' license=('GPL') depends=('glibc') +options=(!strip) package_pkg-split-a1() { install -D -m755 /bin/true ${pkgdir}/usr/bin/${pkgname} diff --git a/test/packages/pkg-split-b/PKGBUILD b/test/packages/pkg-split-b/PKGBUILD index 8b11f73..1d07827 100644 --- a/test/packages/pkg-split-b/PKGBUILD +++ b/test/packages/pkg-split-b/PKGBUILD @@ -7,6 +7,7 @@ arch=('i686' 'x86_64') url='http://www.archlinux.org/' license=('GPL') depends=('glibc') +options=(!strip) package_pkg-split-b1() { install -D -m755 /bin/true ${pkgdir}/usr/bin/${pkgname} diff --git a/test/test.d/packages.bats b/test/test.d/packages.bats deleted file mode 100644 index 63408d0..0000000 --- a/test/test.d/packages.bats +++ /dev/null @@ -1,34 +0,0 @@ -load ../lib/common - -@test "testPackages" { - local result - local pkg - local pkgbase - local pkgarchs - local pkgarch - local tmp=$(mktemp -d) - - # FIXME: Evaluate if this test is sane and even needed - - cp -r packages/* ${tmp} - - for pkgbase in ${tmp}/*; do - pushd ${pkgbase} - # FIXME: Is overriding IFS a bats bug? - IFS=' ' - pkgarchs=($(. PKGBUILD; echo ${arch[@]})) - for pkgarch in ${pkgarchs[@]}; do - echo Building ${pkgbase} on ${pkgarch} - run namcap -e pkgnameindesc,tags PKGBUILD - [ -z "$output" ] - - CARCH=${arch} makepkg -cf - - for pkg in *${PKGEXT}; do - run namcap -e pkgnameindesc ${pkg} - [ -z "$output" ] - done - done - popd - done -} -- cgit v1.2.2 From e0ee502955948e980c711112cdb0c3eb249d0c7b Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Fri, 21 Apr 2017 08:12:18 +0200 Subject: Correctly use checkRemovedPackage --- test/test.d/db-move.bats | 6 +++--- test/test.d/db-update.bats | 18 +++++++++--------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/test/test.d/db-move.bats b/test/test.d/db-move.bats index a154183..3aae9e8 100644 --- a/test/test.d/db-move.bats +++ b/test/test.d/db-move.bats @@ -16,7 +16,7 @@ load ../lib/common for arch in ${arches[@]}; do checkPackage extra pkg-simple-a-1-1-${arch}.pkg.tar.xz ${arch} - checkRemovedPackage testing pkg-simple-a-1-1-${arch}.pkg.tar.xz ${arch} + checkRemovedPackage testing pkg-simple-a ${arch} checkPackage testing pkg-simple-b-1-1-${arch}.pkg.tar.xz ${arch} done @@ -39,7 +39,7 @@ load ../lib/common for pkgbase in ${pkgs[@]}; do for arch in ${arches[@]}; do checkPackage extra ${pkgbase}-1-1-${arch}.pkg.tar.xz ${arch} - checkRemovedPackage testing ${pkgbase}-1-1-${arch}.pkg.tar.xz ${arch} + checkRemovedPackage testing ${pkgbase} ${arch} done done } @@ -60,7 +60,7 @@ load ../lib/common for arch in ${arches[@]}; do checkPackage extra pkg-simple-epoch-1:1-1-${arch}.pkg.tar.xz ${arch} - checkRemovedPackage testing pkg-simple-epoch-1:1-1-${arch}.pkg.tar.xz ${arch} + checkRemovedPackage testing pkg-simple-epoch ${arch} done } diff --git a/test/test.d/db-update.bats b/test/test.d/db-update.bats index 55eeee7..3cae9e5 100644 --- a/test/test.d/db-update.bats +++ b/test/test.d/db-update.bats @@ -152,7 +152,7 @@ load ../lib/common run db-update [ "$status" -ne 0 ] - checkRemovedPackage extra pkg-any-a-1-1-any.pkg.tar.xz any + checkRemovedPackageDB extra pkg-any-a any } @test "testAddInvalidSignedPackageFails" { @@ -165,7 +165,7 @@ load ../lib/common run db-update [ "$status" -ne 0 ] - checkRemovedPackage extra pkg-any-a-1-1-any.pkg.tar.xz any + checkRemovedPackageDB extra pkg-any-a any } @test "testAddBrokenSignatureFails" { @@ -177,7 +177,7 @@ load ../lib/common run db-update [ "$status" -ne 0 ] - checkRemovedPackage extra pkg-any-a-1-1-any.pkg.tar.xz any + checkRemovedPackageDB extra pkg-any-a any } @test "testAddPackageWithInconsistentVersionFails" { @@ -190,7 +190,7 @@ load ../lib/common run db-update [ "$status" -ne 0 ] - checkRemovedPackage extra 'pkg-any-a-2-1-any.pkg.tar.xz' 'any' + checkRemovedPackageDB extra 'pkg-any-a' 'any' } @test "testAddPackageWithInconsistentNameFails" { @@ -203,7 +203,7 @@ load ../lib/common run db-update [ "$status" -ne 0 ] - checkRemovedPackage extra 'foo-pkg-any-a-1-1-any.pkg.tar.xz' 'any' + checkRemovedPackage extra 'foo-pkg-any-a' 'any' } @test "testAddPackageWithInconsistentPKGBUILDFails" { @@ -213,7 +213,7 @@ load ../lib/common run db-update [ "$status" -ne 0 ] - checkRemovedPackage extra 'pkg-any-a-1-1-any.pkg.tar.xz' 'any' + checkRemovedPackageDB extra 'pkg-any-a' 'any' } @test "testAddPackageWithInsufficientPermissionsFails" { @@ -225,8 +225,8 @@ load ../lib/common [ "$status" -ne 0 ] chmod +xwr ${FTP_BASE}/core/os/i686 - checkRemovedPackage core 'pkg-any-a-1-1-any.pkg.tar.xz' 'any' - checkRemovedPackage extra 'pkg-any-b-1-1-any.pkg.tar.xz' 'any' + checkRemovedPackageDB core 'pkg-any-a' 'any' + checkRemovedPackageDB extra 'pkg-any-b' 'any' } @test "testPackageHasToBeARegularFile" { @@ -244,6 +244,6 @@ load ../lib/common run db-update [ "$status" -ne 0 ] for arch in ${arches[@]}; do - checkRemovedPackage extra "pkg-simple-a-1-1-${arch}.pkg.tar.xz" $arch + checkRemovedPackageDB extra "pkg-simple-a" $arch done } -- cgit v1.2.2 From 7628525156110022fa70ad91e4bc13ee8a3cceb0 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 23 Apr 2017 12:05:43 +0200 Subject: Move package and db checks from test cases to library --- test/lib/common.bash | 135 ++++++++++++++++++++++------------------ test/test.d/db-move.bats | 37 ++++------- test/test.d/db-remove.bats | 10 +-- test/test.d/db-repo-add.bats | 12 +--- test/test.d/db-repo-remove.bats | 10 +-- test/test.d/db-update.bats | 54 +++++++--------- test/test.d/ftpdir-cleanup.bats | 47 +++++++------- test/test.d/testing2x.bats | 4 +- 8 files changed, 146 insertions(+), 163 deletions(-) diff --git a/test/lib/common.bash b/test/lib/common.bash index 76c389d..8e1f428 100644 --- a/test/lib/common.bash +++ b/test/lib/common.bash @@ -1,22 +1,5 @@ . /usr/share/makepkg/util.sh -__getPackageBaseFromPackage() { - local _base - _grep_pkginfo() { - local _ret - - _ret="$(/usr/bin/bsdtar -xOqf "$1" .PKGINFO | grep -m 1 "^${2} = ")" - echo "${_ret#${2} = }" - } - - _base="$(_grep_pkginfo "$1" "pkgbase")" - if [ -z "$_base" ]; then - _grep_pkginfo "$1" "pkgname" - else - echo "$_base" - fi -} - __updatePKGBUILD() { local pkgrel @@ -175,86 +158,118 @@ updateRepoPKGBUILD() { popd } -getPackageNamesFromPackageBase() { - local pkgbase=$1 - - $(. "packages/${pkgbase}/PKGBUILD"; echo ${pkgname[@]}) -} - checkPackageDB() { local repo=$1 - local pkg=$2 - local arch=$3 + local pkgbase=$2 local db - local tarch - local tarches - - [ -r "${FTP_BASE}/${PKGPOOL}/${pkg}" ] - [ -r "${FTP_BASE}/${PKGPOOL}/${pkg}.sig" ] - [ ! -r "${STAGING}"/${repo}/${pkg} ] - [ ! -r "${STAGING}"/${repo}/${pkg}.sig ] - - if [[ $arch == any ]]; then - tarches=(${ARCHES[@]}) + local pkgarch + local repoarch + local repoarches + local pkgfile + local pkgname + + # FIXME: We guess the location of the PKGBUILD used for this repo + # We cannot read from trunk as __updatePKGBUILD() might have bumped the version + # and different repos can have different versions of the same package + local pkgbuildPaths=($(compgen -G "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo}-*")) + local pkgbuildPath="${pkgbuildPaths[0]}" + echo Repo is $repo + echo pkgbuildPaths = ${pkgbuildPaths[@]} + echo pkgbuildPath = ${pkgbuildPath} + ls -ahl ${TMP}/svn-packages-copy/${pkgbase}/repos/ + [ -r "${pkgbuildPath}/PKGBUILD" ] + + local pkgarches=($(. "${pkgbuildPath}/PKGBUILD"; echo ${arch[@]})) + local pkgnames=($(. "${pkgbuildPath}/PKGBUILD"; echo ${pkgname[@]})) + local pkgver=$(. "${pkgbuildPath}/PKGBUILD"; get_full_version) + + if [[ ${pkgarches[@]} == any ]]; then + repoarches=(${ARCHES[@]}) else - tarches=(${arch}) + repoarches=(${pkgarches[@]}) fi - for tarch in ${tarches[@]}; do - [ -L "${FTP_BASE}/${repo}/os/${tarch}/${pkg}" ] - [ "$(readlink -e "${FTP_BASE}/${repo}/os/${tarch}/${pkg}")" == "${FTP_BASE}/${PKGPOOL}/${pkg}" ] - - [ -L "${FTP_BASE}/${repo}/os/${tarch}/${pkg}.sig" ] - [ "$(readlink -e "${FTP_BASE}/${repo}/os/${tarch}/${pkg}.sig")" == "${FTP_BASE}/${PKGPOOL}/${pkg}.sig" ] - - for db in ${DBEXT} ${FILESEXT}; do - [ -r "${FTP_BASE}/${repo}/os/${tarch}/${repo}${db%.tar.*}" ] - bsdtar -xf "${FTP_BASE}/${repo}/os/${tarch}/${repo}${db%.tar.*}" -O | grep -q ${pkg} + for pkgarch in ${pkgarches[@]}; do + for pkgname in ${pkgnames[@]}; do + pkgfile="${pkgname}-${pkgver}-${pkgarch}${PKGEXT}" + + [ -r ${FTP_BASE}/${PKGPOOL}/${pkgfile} ] + [ -r ${FTP_BASE}/${PKGPOOL}/${pkgfile}.sig ] + [ ! -r ${STAGING}/${repo}/${pkgfile} ] + [ ! -r ${STAGING}/${repo}/${pkgfile}.sig ] + + for repoarch in ${repoarches[@]}; do + # Only 'any' packages can be found in repos of both arches + if [[ $pkgarch != any ]]; then + if [[ $pkgarch != ${repoarch} ]]; then + continue + fi + fi + + [ -L ${FTP_BASE}/${repo}/os/${repoarch}/${pkgfile} ] + [ "$(readlink -e ${FTP_BASE}/${repo}/os/${repoarch}/${pkgfile})" == ${FTP_BASE}/${PKGPOOL}/${pkgfile} ] + + [ -L ${FTP_BASE}/${repo}/os/${repoarch}/${pkgfile}.sig ] + [ "$(readlink -e ${FTP_BASE}/${repo}/os/${repoarch}/${pkgfile}.sig)" == ${FTP_BASE}/${PKGPOOL}/${pkgfile}.sig ] + + for db in ${DBEXT} ${FILESEXT}; do + [ -r "${FTP_BASE}/${repo}/os/${repoarch}/${repo}${db%.tar.*}" ] + bsdtar -xf "${FTP_BASE}/${repo}/os/${repoarch}/${repo}${db%.tar.*}" -O | grep -q "${pkgfile%${PKGEXT}}" + done + done done done } checkPackage() { local repo=$1 - local pkg=$2 - local arch=$3 - - checkPackageDB $repo $pkg $arch + local pkgbase=$2 - local pkgbase=$(__getPackageBaseFromPackage "${FTP_BASE}/${PKGPOOL}/${pkg}") svn up -q "${TMP}/svn-packages-copy/${pkgbase}" - [ -d "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo}-${arch}" ] + # TODO: Does not fail if one arch is missing + compgen -G "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo}-*" >/dev/null + + checkPackageDB $repo $pkgbase } checkRemovedPackage() { local repo=$1 local pkgbase=$2 - local arch=$3 - - checkRemovedPackageDB $repo $pkgbase $arch svn up -q "${TMP}/svn-packages-copy/${pkgbase}" - [ ! -d "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo}-${arch}" ] + ! compgen -G "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo}-*" >/dev/null + + checkRemovedPackageDB $repo $pkgbase } checkRemovedPackageDB() { local repo=$1 local pkgbase=$2 - local arch=$3 + local arch local db local tarch local tarches + local pkgarches + local pkgnames + local pkgname + + local pkgbuildPath="${TMP}/svn-packages-copy/${pkgbase}/trunk/PKGBUILD" + [[ -r ${pkgbuildPath} ]] + pkgarches=($(. "${pkgbuildPath}"; echo ${arch[@]})) + pkgnames=($(. "${pkgbuildPath}"; echo ${pkgname[@]})) - if [[ $arch == any ]]; then + if [[ ${pkgarches[@]} == any ]]; then tarches=(${ARCHES[@]}) else - tarches=(${arch}) + tarches=($pkgarches[@]) fi for db in ${DBEXT} ${FILESEXT}; do for tarch in ${tarches[@]}; do if [ -r "${FTP_BASE}/${repo}/os/${tarch}/${repo}${db%.tar.*}" ]; then - echo "$(bsdtar -xf "${FTP_BASE}/${repo}/os/${tarch}/${repo}${db%.tar.*}" -O)" | grep -qv ${pkgbase} + for pkgname in ${pkgnames[@]}; do + echo "$(bsdtar -xf "${FTP_BASE}/${repo}/os/${tarch}/${repo}${db%.tar.*}" -O)" | grep -qv ${pkgname} + done fi done done diff --git a/test/test.d/db-move.bats b/test/test.d/db-move.bats index 3aae9e8..4c3dfed 100644 --- a/test/test.d/db-move.bats +++ b/test/test.d/db-move.bats @@ -14,12 +14,9 @@ load ../lib/common db-move testing extra pkg-simple-a - for arch in ${arches[@]}; do - checkPackage extra pkg-simple-a-1-1-${arch}.pkg.tar.xz ${arch} - checkRemovedPackage testing pkg-simple-a ${arch} - - checkPackage testing pkg-simple-b-1-1-${arch}.pkg.tar.xz ${arch} - done + checkRemovedPackage testing pkg-simple-a + checkPackage extra pkg-simple-a + checkPackage testing pkg-simple-b } @test "testMoveMultiplePackages" { @@ -37,10 +34,8 @@ load ../lib/common db-move testing extra pkg-simple-a pkg-simple-b for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - checkPackage extra ${pkgbase}-1-1-${arch}.pkg.tar.xz ${arch} - checkRemovedPackage testing ${pkgbase} ${arch} - done + checkRemovedPackage testing ${pkgbase} + checkPackage extra ${pkgbase} done } @@ -58,10 +53,8 @@ load ../lib/common db-move testing extra pkg-simple-epoch - for arch in ${arches[@]}; do - checkPackage extra pkg-simple-epoch-1:1-1-${arch}.pkg.tar.xz ${arch} - checkRemovedPackage testing pkg-simple-epoch ${arch} - done + checkRemovedPackage testing pkg-simple-epoch + checkPackage extra pkg-simple-epoch } @test "testMoveAnyPackages" { @@ -75,9 +68,9 @@ load ../lib/common db-update db-move testing extra pkg-any-a - checkPackage extra pkg-any-a-1-1-any.pkg.tar.xz any - checkRemovedPackage testing pkg-any-a any - checkPackage testing pkg-any-b-1-1-any.pkg.tar.xz any + checkPackage extra pkg-any-a + checkRemovedPackage testing pkg-any-a + checkPackage testing pkg-any-b } @test "testMoveSplitPackages" { @@ -94,12 +87,6 @@ load ../lib/common db-update db-move testing extra pkg-split-a - for arch in ${arches[@]}; do - for pkg in $(getPackageNamesFromPackageBase pkg-split-a); do - checkPackage extra ${pkg} ${arch} - done - for pkg in $(getPackageNamesFromPackageBase pkg-split-b); do - checkPackage testing ${pkg} ${arch} - done - done + checkPackage extra pkg-split-a + checkPackage testing pkg-split-b } diff --git a/test/test.d/db-remove.bats b/test/test.d/db-remove.bats index 3e0d1f4..379966a 100644 --- a/test/test.d/db-remove.bats +++ b/test/test.d/db-remove.bats @@ -19,9 +19,7 @@ load ../lib/common done for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - checkRemovedPackage extra ${pkgbase} ${arch} - done + checkRemovedPackage extra ${pkgbase} done } @@ -42,9 +40,7 @@ load ../lib/common done for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - checkRemovedPackage extra ${pkgbase} ${arch} - done + checkRemovedPackage extra ${pkgbase} done } @@ -63,6 +59,6 @@ load ../lib/common done for pkgbase in ${pkgs[@]}; do - checkRemovedPackage extra ${pkgbase} any + checkRemovedPackage extra ${pkgbase} done } diff --git a/test/test.d/db-repo-add.bats b/test/test.d/db-repo-add.bats index 98fa001..bbf0e8f 100644 --- a/test/test.d/db-repo-add.bats +++ b/test/test.d/db-repo-add.bats @@ -36,9 +36,7 @@ __movePackageToRepo() { done for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - checkPackageDB extra ${pkgbase}-1-1-${arch}.pkg.tar.xz ${arch} - done + checkPackageDB extra ${pkgbase} done } @@ -62,9 +60,7 @@ __movePackageToRepo() { done for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - checkPackageDB extra ${pkgbase}-1-1-${arch}.pkg.tar.xz ${arch} - done + checkPackageDB extra ${pkgbase} done } @@ -81,8 +77,6 @@ __movePackageToRepo() { done for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - checkPackageDB extra ${pkgbase}-1-1-any.pkg.tar.xz ${arch} - done + checkPackageDB extra ${pkgbase} done } diff --git a/test/test.d/db-repo-remove.bats b/test/test.d/db-repo-remove.bats index b3a0b3b..e8df35b 100644 --- a/test/test.d/db-repo-remove.bats +++ b/test/test.d/db-repo-remove.bats @@ -19,9 +19,7 @@ load ../lib/common done for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - checkRemovedPackageDB extra ${pkgbase} ${arch} - done + checkRemovedPackageDB extra ${pkgbase} done } @@ -42,9 +40,7 @@ load ../lib/common done for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - checkRemovedPackageDB extra ${pkgbase} ${arch} - done + checkRemovedPackageDB extra ${pkgbase} done } @@ -63,6 +59,6 @@ load ../lib/common done for pkgbase in ${pkgs[@]}; do - checkRemovedPackageDB extra ${pkgbase} any + checkRemovedPackageDB extra ${pkgbase} done } diff --git a/test/test.d/db-update.bats b/test/test.d/db-update.bats index 3cae9e5..c9db68f 100644 --- a/test/test.d/db-update.bats +++ b/test/test.d/db-update.bats @@ -13,22 +13,20 @@ load ../lib/common db-update for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - checkPackage extra ${pkgbase}-1-1-${arch}.pkg.tar.xz ${arch} - done + checkPackage extra ${pkgbase} done } @test "testAddSingleSimplePackage" { releasePackage extra 'pkg-single-arch' db-update - checkPackage extra 'pkg-single-arch-1-1-x86_64.pkg.tar.xz' 'x86_64' + checkPackage extra 'pkg-single-arch' } @test "testAddSingleEpochPackage" { releasePackage extra 'pkg-single-epoch' db-update - checkPackage extra 'pkg-single-epoch-1:1-1-x86_64.pkg.tar.xz' 'x86_64' + checkPackage extra 'pkg-single-epoch' } @test "testAddAnyPackages" { @@ -42,7 +40,7 @@ load ../lib/common db-update for pkgbase in ${pkgs[@]}; do - checkPackage extra ${pkgbase}-1-1-any.pkg.tar.xz any + checkPackage extra ${pkgbase} done } @@ -60,11 +58,7 @@ load ../lib/common db-update for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - for pkg in $(getPackageNamesFromPackageBase ${pkgbase}); do - checkPackage extra ${pkg##*/} ${arch} - done - done + checkPackage extra ${pkgbase} done } @@ -77,7 +71,7 @@ load ../lib/common releasePackage extra pkg-any-a db-update - checkPackage extra pkg-any-a-1-2-any.pkg.tar.xz any + checkPackage extra pkg-any-a } @test "testUpdateAnyPackageToDifferentRepositoriesAtOnce" { @@ -89,14 +83,14 @@ load ../lib/common db-update - checkPackage extra pkg-any-a-1-1-any.pkg.tar.xz any - checkPackage testing pkg-any-a-1-2-any.pkg.tar.xz any + checkPackage extra pkg-any-a + checkPackage testing pkg-any-a } @test "testUpdateSameAnyPackageToSameRepository" { releasePackage extra pkg-any-a db-update - checkPackage extra pkg-any-a-1-1-any.pkg.tar.xz any + checkPackage extra pkg-any-a releasePackage extra pkg-any-a run db-update @@ -108,13 +102,13 @@ load ../lib/common releasePackage extra pkg-any-a db-update - checkPackage extra pkg-any-a-1-1-any.pkg.tar.xz any + checkPackage extra pkg-any-a releasePackage testing pkg-any-a run db-update [ "$status" -ne 0 ] - checkRemovedPackageDB testing pkg-any-a any + checkRemovedPackageDB testing pkg-any-a } @test "testAddIncompleteSplitPackage" { @@ -131,9 +125,7 @@ load ../lib/common run db-update [ "$status" -ne 0 ] - for arch in ${arches[@]}; do - checkRemovedPackageDB ${repo} ${pkgbase} ${arch} - done + checkRemovedPackageDB ${repo} ${pkgbase} } @test "testUnknownRepo" { @@ -141,7 +133,7 @@ load ../lib/common releasePackage extra 'pkg-any-a' releasePackage unknown 'pkg-any-b' db-update - checkPackage extra 'pkg-any-a-1-1-any.pkg.tar.xz' any + checkPackage extra 'pkg-any-a' [ ! -e "${FTP_BASE}/unknown" ] rm -rf "${STAGING}/unknown/" } @@ -152,7 +144,7 @@ load ../lib/common run db-update [ "$status" -ne 0 ] - checkRemovedPackageDB extra pkg-any-a any + checkRemovedPackageDB extra pkg-any-a } @test "testAddInvalidSignedPackageFails" { @@ -165,7 +157,7 @@ load ../lib/common run db-update [ "$status" -ne 0 ] - checkRemovedPackageDB extra pkg-any-a any + checkRemovedPackageDB extra pkg-any-a } @test "testAddBrokenSignatureFails" { @@ -177,7 +169,7 @@ load ../lib/common run db-update [ "$status" -ne 0 ] - checkRemovedPackageDB extra pkg-any-a any + checkRemovedPackageDB extra pkg-any-a } @test "testAddPackageWithInconsistentVersionFails" { @@ -190,7 +182,7 @@ load ../lib/common run db-update [ "$status" -ne 0 ] - checkRemovedPackageDB extra 'pkg-any-a' 'any' + checkRemovedPackageDB extra 'pkg-any-a' } @test "testAddPackageWithInconsistentNameFails" { @@ -203,7 +195,7 @@ load ../lib/common run db-update [ "$status" -ne 0 ] - checkRemovedPackage extra 'foo-pkg-any-a' 'any' + checkRemovedPackage extra 'pkg-any-a' } @test "testAddPackageWithInconsistentPKGBUILDFails" { @@ -213,7 +205,7 @@ load ../lib/common run db-update [ "$status" -ne 0 ] - checkRemovedPackageDB extra 'pkg-any-a' 'any' + checkRemovedPackageDB extra 'pkg-any-a' } @test "testAddPackageWithInsufficientPermissionsFails" { @@ -225,8 +217,8 @@ load ../lib/common [ "$status" -ne 0 ] chmod +xwr ${FTP_BASE}/core/os/i686 - checkRemovedPackageDB core 'pkg-any-a' 'any' - checkRemovedPackageDB extra 'pkg-any-b' 'any' + checkRemovedPackageDB core 'pkg-any-a' + checkRemovedPackageDB extra 'pkg-any-b' } @test "testPackageHasToBeARegularFile" { @@ -243,7 +235,5 @@ load ../lib/common run db-update [ "$status" -ne 0 ] - for arch in ${arches[@]}; do - checkRemovedPackageDB extra "pkg-simple-a" $arch - done + checkRemovedPackageDB extra "pkg-simple-a" } diff --git a/test/test.d/ftpdir-cleanup.bats b/test/test.d/ftpdir-cleanup.bats index 444c3c5..e3adeee 100644 --- a/test/test.d/ftpdir-cleanup.bats +++ b/test/test.d/ftpdir-cleanup.bats @@ -1,13 +1,21 @@ load ../lib/common +__getPackageNamesFromPackageBase() { + local pkgbase=$1 + + (. "packages/${pkgbase}/PKGBUILD"; echo ${pkgname[@]}) +} + __checkRepoRemovedPackage() { local repo=$1 local pkgbase=$2 - local arch=$3 + local repoarch=$3 + local pkgname - # FIXME: pkgbase might not be part of the package filename - [[ ! -f ${FTP_BASE}/${PKGPOOL}/${pkgbase}*${PKGEXT} ]] - [[ ! -f ${FTP_BASE}/${repo}/os/${arch}/${pkgbase}*${PKGEXT} ]] + for pkgname in $(__getPackageNamesFromPackageBase ${pkgbase}); do + [[ ! -f ${FTP_BASE}/${PKGPOOL}/${pkgname}-*${PKGEXT} ]] + [[ ! -f ${FTP_BASE}/${repo}/os/${repoarch}/${pkgname}-*${PKGEXT} ]] + done } @test "testCleanupSimplePackages" { @@ -28,13 +36,12 @@ __checkRepoRemovedPackage() { ftpdir-cleanup + checkRemovedPackage extra 'pkg-simple-a' for arch in ${arches[@]}; do - checkRemovedPackage extra 'pkg-simple-a' ${arch} __checkRepoRemovedPackage extra 'pkg-simple-a' ${arch} - - local pkg2="pkg-simple-b-1-1-${arch}.pkg.tar.xz" - checkPackage extra ${pkg2} ${arch} done + + checkPackage extra pkg-simple-b } @test "testCleanupEpochPackages" { @@ -55,13 +62,14 @@ __checkRepoRemovedPackage() { ftpdir-cleanup + checkRemovedPackage extra 'pkg-simple-epoch' for arch in ${arches[@]}; do - checkRemovedPackage extra 'pkg-simple-epoch' ${arch} __checkRepoRemovedPackage extra 'pkg-simple-epoch' ${arch} done } @test "testCleanupAnyPackages" { + local arches=('i686' 'x86_64') local pkgs=('pkg-any-a' 'pkg-any-b') local pkgbase local arch='any' @@ -75,11 +83,12 @@ __checkRepoRemovedPackage() { ftpdir-cleanup local pkg1='pkg-any-a-1-1-any.pkg.tar.xz' - checkRemovedPackage extra 'pkg-any-a' any - __checkRepoRemovedPackage extra 'pkg-any-a' any + checkRemovedPackage extra 'pkg-any-a' + for arch in ${arches[@]}; do + __checkRepoRemovedPackage extra 'pkg-any-a' ${arch} + done - local pkg2="pkg-any-b-1-1-${arch}.pkg.tar.xz" - checkPackage extra ${pkg2} any + checkPackage extra pkg-any-b } @test "testCleanupSplitPackages" { @@ -102,15 +111,11 @@ __checkRepoRemovedPackage() { ftpdir-cleanup for arch in ${arches[@]}; do - for pkg in $(getPackageNamesFromPackageBase ${pkgs[0]}); do - checkRemovedPackage extra ${pkg} ${arch} - __checkRepoRemovedPackage extra ${pkg} ${arch} - done - - for pkg in $(getPackageNamesFromPackageBase ${pkgs[1]}); do - checkPackage extra ${pkg##*/} ${arch} - done + __checkRepoRemovedPackage extra ${pkg[0]} ${arch} done + + checkRemovedPackage extra ${pkgs[0]} + checkPackage extra ${pkgs[1]} } @test "testCleanupOldPackages" { diff --git a/test/test.d/testing2x.bats b/test/test.d/testing2x.bats index bac75de..e849799 100644 --- a/test/test.d/testing2x.bats +++ b/test/test.d/testing2x.bats @@ -11,6 +11,6 @@ load ../lib/common testing2x pkg-any-a - checkPackage core pkg-any-a-1-2-any.pkg.tar.xz any - checkRemovedPackage testing pkg-any-a any + checkPackage core pkg-any-a + checkRemovedPackage testing pkg-any-a } -- cgit v1.2.2 From 685bba9fff21b7d6902fd484384223ce966b24f8 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 23 Apr 2017 12:19:35 +0200 Subject: Move test cases and fixtures to more meaningful directories --- .gitignore | 1 - test/Makefile | 4 +- test/cases/db-move.bats | 92 ++++++++++++ test/cases/db-remove.bats | 64 +++++++++ test/cases/db-repo-add.bats | 82 +++++++++++ test/cases/db-repo-remove.bats | 64 +++++++++ test/cases/db-update.bats | 239 ++++++++++++++++++++++++++++++++ test/cases/ftpdir-cleanup.bats | 151 ++++++++++++++++++++ test/cases/sourceballs.bats | 83 +++++++++++ test/cases/testing2x.bats | 16 +++ test/fixtures/pkg-any-a/PKGBUILD | 13 ++ test/fixtures/pkg-any-b/PKGBUILD | 13 ++ test/fixtures/pkg-simple-a/PKGBUILD | 13 ++ test/fixtures/pkg-simple-b/PKGBUILD | 13 ++ test/fixtures/pkg-simple-epoch/PKGBUILD | 14 ++ test/fixtures/pkg-single-arch/PKGBUILD | 13 ++ test/fixtures/pkg-single-epoch/PKGBUILD | 14 ++ test/fixtures/pkg-split-a/PKGBUILD | 18 +++ test/fixtures/pkg-split-b/PKGBUILD | 18 +++ test/lib/common.bash | 2 +- test/packages/pkg-any-a/PKGBUILD | 13 -- test/packages/pkg-any-b/PKGBUILD | 13 -- test/packages/pkg-simple-a/PKGBUILD | 13 -- test/packages/pkg-simple-b/PKGBUILD | 13 -- test/packages/pkg-simple-epoch/PKGBUILD | 14 -- test/packages/pkg-single-arch/PKGBUILD | 13 -- test/packages/pkg-single-epoch/PKGBUILD | 14 -- test/packages/pkg-split-a/PKGBUILD | 18 --- test/packages/pkg-split-b/PKGBUILD | 18 --- test/test.d/db-move.bats | 92 ------------ test/test.d/db-remove.bats | 64 --------- test/test.d/db-repo-add.bats | 82 ----------- test/test.d/db-repo-remove.bats | 64 --------- test/test.d/db-update.bats | 239 -------------------------------- test/test.d/ftpdir-cleanup.bats | 151 -------------------- test/test.d/sourceballs.bats | 83 ----------- test/test.d/testing2x.bats | 16 --- 37 files changed, 923 insertions(+), 924 deletions(-) create mode 100644 test/cases/db-move.bats create mode 100644 test/cases/db-remove.bats create mode 100644 test/cases/db-repo-add.bats create mode 100644 test/cases/db-repo-remove.bats create mode 100644 test/cases/db-update.bats create mode 100644 test/cases/ftpdir-cleanup.bats create mode 100644 test/cases/sourceballs.bats create mode 100644 test/cases/testing2x.bats create mode 100644 test/fixtures/pkg-any-a/PKGBUILD create mode 100644 test/fixtures/pkg-any-b/PKGBUILD create mode 100644 test/fixtures/pkg-simple-a/PKGBUILD create mode 100644 test/fixtures/pkg-simple-b/PKGBUILD create mode 100644 test/fixtures/pkg-simple-epoch/PKGBUILD create mode 100644 test/fixtures/pkg-single-arch/PKGBUILD create mode 100644 test/fixtures/pkg-single-epoch/PKGBUILD create mode 100644 test/fixtures/pkg-split-a/PKGBUILD create mode 100644 test/fixtures/pkg-split-b/PKGBUILD delete mode 100644 test/packages/pkg-any-a/PKGBUILD delete mode 100644 test/packages/pkg-any-b/PKGBUILD delete mode 100644 test/packages/pkg-simple-a/PKGBUILD delete mode 100644 test/packages/pkg-simple-b/PKGBUILD delete mode 100644 test/packages/pkg-simple-epoch/PKGBUILD delete mode 100644 test/packages/pkg-single-arch/PKGBUILD delete mode 100644 test/packages/pkg-single-epoch/PKGBUILD delete mode 100644 test/packages/pkg-split-a/PKGBUILD delete mode 100644 test/packages/pkg-split-b/PKGBUILD delete mode 100644 test/test.d/db-move.bats delete mode 100644 test/test.d/db-remove.bats delete mode 100644 test/test.d/db-repo-add.bats delete mode 100644 test/test.d/db-repo-remove.bats delete mode 100644 test/test.d/db-update.bats delete mode 100644 test/test.d/ftpdir-cleanup.bats delete mode 100644 test/test.d/sourceballs.bats delete mode 100644 test/test.d/testing2x.bats diff --git a/.gitignore b/.gitignore index 4e3ce99..293ca9f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,4 @@ *~ /config.local -/test/packages/*/*.pkg.tar.?z /.idea /coverage diff --git a/test/Makefile b/test/Makefile index afc3cff..d9437bf 100644 --- a/test/Makefile +++ b/test/Makefile @@ -1,11 +1,11 @@ test: - PATH=$(CURDIR)/../:$(CURDIR)/../cron-jobs/:$(PATH) bats test.d + PATH=$(CURDIR)/../:$(CURDIR)/../cron-jobs/:$(PATH) bats cases test-coverage: PATH=$(CURDIR)/../:$(CURDIR)/../cron-jobs/:$(PATH) kcov \ --include-path=$(CURDIR)/../ \ --exclude-path=$(CURDIR)/../test,$(CURDIR)/../cron-jobs/makepkg.conf,$(CURDIR)/../config \ $(COVERAGE_DIR) \ - bats test.d + bats cases .PHONY: test test-coverage diff --git a/test/cases/db-move.bats b/test/cases/db-move.bats new file mode 100644 index 0000000..4c3dfed --- /dev/null +++ b/test/cases/db-move.bats @@ -0,0 +1,92 @@ +load ../lib/common + +@test "testMoveSimplePackages" { + local arches=('i686' 'x86_64') + local pkgs=('pkg-simple-a' 'pkg-simple-b') + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + releasePackage testing ${pkgbase} + done + + db-update + + db-move testing extra pkg-simple-a + + checkRemovedPackage testing pkg-simple-a + checkPackage extra pkg-simple-a + checkPackage testing pkg-simple-b +} + +@test "testMoveMultiplePackages" { + local arches=('i686' 'x86_64') + local pkgs=('pkg-simple-a' 'pkg-simple-b') + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + releasePackage testing ${pkgbase} + done + + db-update + + db-move testing extra pkg-simple-a pkg-simple-b + + for pkgbase in ${pkgs[@]}; do + checkRemovedPackage testing ${pkgbase} + checkPackage extra ${pkgbase} + done +} + +@test "testMoveEpochPackages" { + local arches=('i686' 'x86_64') + local pkgs=('pkg-simple-epoch') + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + releasePackage testing ${pkgbase} + done + + db-update + + db-move testing extra pkg-simple-epoch + + checkRemovedPackage testing pkg-simple-epoch + checkPackage extra pkg-simple-epoch +} + +@test "testMoveAnyPackages" { + local pkgs=('pkg-any-a' 'pkg-any-b') + local pkgbase + + for pkgbase in ${pkgs[@]}; do + releasePackage testing ${pkgbase} + done + + db-update + db-move testing extra pkg-any-a + + checkPackage extra pkg-any-a + checkRemovedPackage testing pkg-any-a + checkPackage testing pkg-any-b +} + +@test "testMoveSplitPackages" { + local arches=('i686' 'x86_64') + local pkgs=('pkg-split-a' 'pkg-split-b') + local pkg + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + releasePackage testing ${pkgbase} + done + + db-update + db-move testing extra pkg-split-a + + checkPackage extra pkg-split-a + checkPackage testing pkg-split-b +} diff --git a/test/cases/db-remove.bats b/test/cases/db-remove.bats new file mode 100644 index 0000000..379966a --- /dev/null +++ b/test/cases/db-remove.bats @@ -0,0 +1,64 @@ +load ../lib/common + +@test "testRemovePackages" { + local arches=('i686' 'x86_64') + local pkgs=('pkg-simple-a' 'pkg-simple-b' 'pkg-split-a' 'pkg-split-b' 'pkg-simple-epoch') + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + releasePackage extra ${pkgbase} + done + + db-update + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + db-remove extra ${arch} ${pkgbase} + done + done + + for pkgbase in ${pkgs[@]}; do + checkRemovedPackage extra ${pkgbase} + done +} + +@test "testRemoveMultiplePackages" { + local arches=('i686' 'x86_64') + local pkgs=('pkg-simple-a' 'pkg-simple-b' 'pkg-split-a' 'pkg-split-b' 'pkg-simple-epoch') + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + releasePackage extra ${pkgbase} + done + + db-update + + for arch in ${arches[@]}; do + db-remove extra ${arch} ${pkgs[@]} + done + + for pkgbase in ${pkgs[@]}; do + checkRemovedPackage extra ${pkgbase} + done +} + +@test "testRemoveAnyPackages" { + local pkgs=('pkg-any-a' 'pkg-any-b') + local pkgbase + + for pkgbase in ${pkgs[@]}; do + releasePackage extra ${pkgbase} + done + + db-update + + for pkgbase in ${pkgs[@]}; do + db-remove extra any ${pkgbase} + done + + for pkgbase in ${pkgs[@]}; do + checkRemovedPackage extra ${pkgbase} + done +} diff --git a/test/cases/db-repo-add.bats b/test/cases/db-repo-add.bats new file mode 100644 index 0000000..bbf0e8f --- /dev/null +++ b/test/cases/db-repo-add.bats @@ -0,0 +1,82 @@ +load ../lib/common + +__movePackageToRepo() { + local repo=$1 + local pkgbase=$2 + local arch=$3 + local tarch + local tarches + + if [[ $arch == any ]]; then + tarches=(${ARCHES[@]}) + else + tarches=(${arch}) + fi + + # FIXME: pkgbase might not be part of the package filename + mv -v "${STAGING}"/${repo}/${pkgbase}-*-*-${arch}${PKGEXT}{,.sig} "${FTP_BASE}/${PKGPOOL}/" + for tarch in ${tarches[@]}; do + ln -sv ${FTP_BASE}/${PKGPOOL}/${pkgbase}-*-*-${arch}${PKGEXT} "${FTP_BASE}/${repo}/os/${tarch}/" + ln -sv ${FTP_BASE}/${PKGPOOL}/${pkgbase}-*-*-${arch}${PKGEXT}.sig "${FTP_BASE}/${repo}/os/${tarch}/" + done +} + +@test "testRepoAddSimplePackages" { + local arches=('i686' 'x86_64') + local pkgs=('pkg-simple-a' 'pkg-simple-b') + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + releasePackage extra ${pkgbase} + for arch in ${arches[@]}; do + __movePackageToRepo extra ${pkgbase} ${arch} + db-repo-add extra ${arch} ${pkgbase}-1-1-${arch}.pkg.tar.xz + done + done + + for pkgbase in ${pkgs[@]}; do + checkPackageDB extra ${pkgbase} + done +} + +@test "testRepoAddMultiplePackages" { + local arches=('i686' 'x86_64') + local pkgs=('pkg-simple-a' 'pkg-simple-b') + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + releasePackage extra ${pkgbase} + done + + for arch in ${arches[@]}; do + add_pkgs=() + for pkgbase in ${pkgs[@]}; do + __movePackageToRepo extra ${pkgbase} ${arch} + add_pkgs[${#add_pkgs[*]}]=${pkgbase}-1-1-${arch}.pkg.tar.xz + done + db-repo-add extra ${arch} ${add_pkgs[@]} + done + + for pkgbase in ${pkgs[@]}; do + checkPackageDB extra ${pkgbase} + done +} + +@test "testRepoAddAnyPackages" { + local arches=('i686' 'x86_64') + local pkgs=('pkg-any-a' 'pkg-any-b') + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + releasePackage extra ${pkgbase} + __movePackageToRepo extra ${pkgbase} any + db-repo-add extra any ${pkgbase}-1-1-any.pkg.tar.xz + done + + for pkgbase in ${pkgs[@]}; do + checkPackageDB extra ${pkgbase} + done +} diff --git a/test/cases/db-repo-remove.bats b/test/cases/db-repo-remove.bats new file mode 100644 index 0000000..e8df35b --- /dev/null +++ b/test/cases/db-repo-remove.bats @@ -0,0 +1,64 @@ +load ../lib/common + +@test "testRepoRemovePackages" { + local arches=('i686' 'x86_64') + local pkgs=('pkg-simple-a' 'pkg-simple-b' 'pkg-simple-epoch') + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + releasePackage extra ${pkgbase} + done + + db-update + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + db-repo-remove extra ${arch} ${pkgbase} + done + done + + for pkgbase in ${pkgs[@]}; do + checkRemovedPackageDB extra ${pkgbase} + done +} + +@test "testRepoRemoveMultiplePackages" { + local arches=('i686' 'x86_64') + local pkgs=('pkg-simple-a' 'pkg-simple-b' 'pkg-simple-epoch') + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + releasePackage extra ${pkgbase} + done + + db-update + + for arch in ${arches[@]}; do + db-repo-remove extra ${arch} ${pkgs[@]} + done + + for pkgbase in ${pkgs[@]}; do + checkRemovedPackageDB extra ${pkgbase} + done +} + +@test "testRepoRemoveAnyPackages" { + local pkgs=('pkg-any-a' 'pkg-any-b') + local pkgbase + + for pkgbase in ${pkgs[@]}; do + releasePackage extra ${pkgbase} + done + + db-update + + for pkgbase in ${pkgs[@]}; do + db-repo-remove extra any ${pkgbase} + done + + for pkgbase in ${pkgs[@]}; do + checkRemovedPackageDB extra ${pkgbase} + done +} diff --git a/test/cases/db-update.bats b/test/cases/db-update.bats new file mode 100644 index 0000000..c9db68f --- /dev/null +++ b/test/cases/db-update.bats @@ -0,0 +1,239 @@ +load ../lib/common + +@test "testAddSimplePackages" { + local arches=('i686' 'x86_64') + local pkgs=('pkg-simple-a' 'pkg-simple-b') + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + releasePackage extra ${pkgbase} + done + + db-update + + for pkgbase in ${pkgs[@]}; do + checkPackage extra ${pkgbase} + done +} + +@test "testAddSingleSimplePackage" { + releasePackage extra 'pkg-single-arch' + db-update + checkPackage extra 'pkg-single-arch' +} + +@test "testAddSingleEpochPackage" { + releasePackage extra 'pkg-single-epoch' + db-update + checkPackage extra 'pkg-single-epoch' +} + +@test "testAddAnyPackages" { + local pkgs=('pkg-any-a' 'pkg-any-b') + local pkgbase + + for pkgbase in ${pkgs[@]}; do + releasePackage extra ${pkgbase} + done + + db-update + + for pkgbase in ${pkgs[@]}; do + checkPackage extra ${pkgbase} + done +} + +@test "testAddSplitPackages" { + local arches=('i686' 'x86_64') + local pkgs=('pkg-split-a' 'pkg-split-b') + local pkg + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + releasePackage extra ${pkgbase} + done + + db-update + + for pkgbase in ${pkgs[@]}; do + checkPackage extra ${pkgbase} + done +} + +@test "testUpdateAnyPackage" { + releasePackage extra pkg-any-a + db-update + + updatePackage pkg-any-a + + releasePackage extra pkg-any-a + db-update + + checkPackage extra pkg-any-a +} + +@test "testUpdateAnyPackageToDifferentRepositoriesAtOnce" { + releasePackage extra pkg-any-a + + updatePackage pkg-any-a + + releasePackage testing pkg-any-a + + db-update + + checkPackage extra pkg-any-a + checkPackage testing pkg-any-a +} + +@test "testUpdateSameAnyPackageToSameRepository" { + releasePackage extra pkg-any-a + db-update + checkPackage extra pkg-any-a + + releasePackage extra pkg-any-a + run db-update + [ "$status" -ne 0 ] +} + +@test "testUpdateSameAnyPackageToDifferentRepositories" { + local arch + + releasePackage extra pkg-any-a + db-update + checkPackage extra pkg-any-a + + releasePackage testing pkg-any-a + run db-update + [ "$status" -ne 0 ] + + checkRemovedPackageDB testing pkg-any-a +} + +@test "testAddIncompleteSplitPackage" { + local arches=('i686' 'x86_64') + local repo='extra' + local pkgbase='pkg-split-a' + local arch + + releasePackage ${repo} ${pkgbase} + + # remove a split package to make db-update fail + rm "${STAGING}"/extra/${pkgbase}1-* + + run db-update + [ "$status" -ne 0 ] + + checkRemovedPackageDB ${repo} ${pkgbase} +} + +@test "testUnknownRepo" { + mkdir "${STAGING}/unknown/" + releasePackage extra 'pkg-any-a' + releasePackage unknown 'pkg-any-b' + db-update + checkPackage extra 'pkg-any-a' + [ ! -e "${FTP_BASE}/unknown" ] + rm -rf "${STAGING}/unknown/" +} + +@test "testAddUnsignedPackageFails" { + releasePackage extra 'pkg-any-a' + rm "${STAGING}"/extra/*.sig + run db-update + [ "$status" -ne 0 ] + + checkRemovedPackageDB extra pkg-any-a +} + +@test "testAddInvalidSignedPackageFails" { + local p + releasePackage extra 'pkg-any-a' + for p in "${STAGING}"/extra/*${PKGEXT}; do + unxz $p + xz -0 ${p%%.xz} + done + run db-update + [ "$status" -ne 0 ] + + checkRemovedPackageDB extra pkg-any-a +} + +@test "testAddBrokenSignatureFails" { + local s + releasePackage extra 'pkg-any-a' + for s in "${STAGING}"/extra/*.sig; do + echo 0 > $s + done + run db-update + [ "$status" -ne 0 ] + + checkRemovedPackageDB extra pkg-any-a +} + +@test "testAddPackageWithInconsistentVersionFails" { + local p + releasePackage extra 'pkg-any-a' + + for p in "${STAGING}"/extra/*; do + mv "${p}" "${p/pkg-any-a-1/pkg-any-a-2}" + done + + run db-update + [ "$status" -ne 0 ] + checkRemovedPackageDB extra 'pkg-any-a' +} + +@test "testAddPackageWithInconsistentNameFails" { + local p + releasePackage extra 'pkg-any-a' + + for p in "${STAGING}"/extra/*; do + mv "${p}" "${p/pkg-/foo-pkg-}" + done + + run db-update + [ "$status" -ne 0 ] + checkRemovedPackage extra 'pkg-any-a' +} + +@test "testAddPackageWithInconsistentPKGBUILDFails" { + releasePackage extra 'pkg-any-a' + + updateRepoPKGBUILD 'pkg-any-a' extra any + + run db-update + [ "$status" -ne 0 ] + checkRemovedPackageDB extra 'pkg-any-a' +} + +@test "testAddPackageWithInsufficientPermissionsFails" { + releasePackage core 'pkg-any-a' + releasePackage extra 'pkg-any-b' + + chmod -xwr ${FTP_BASE}/core/os/i686 + run db-update + [ "$status" -ne 0 ] + chmod +xwr ${FTP_BASE}/core/os/i686 + + checkRemovedPackageDB core 'pkg-any-a' + checkRemovedPackageDB extra 'pkg-any-b' +} + +@test "testPackageHasToBeARegularFile" { + local p + local target=$(mktemp -d) + local arches=('i686' 'x86_64') + + releasePackage extra 'pkg-simple-a' + + for p in "${STAGING}"/extra/*i686*; do + mv "${p}" "${target}" + ln -s "${target}/${p##*/}" "${p}" + done + + run db-update + [ "$status" -ne 0 ] + checkRemovedPackageDB extra "pkg-simple-a" +} diff --git a/test/cases/ftpdir-cleanup.bats b/test/cases/ftpdir-cleanup.bats new file mode 100644 index 0000000..cb1ca45 --- /dev/null +++ b/test/cases/ftpdir-cleanup.bats @@ -0,0 +1,151 @@ +load ../lib/common + +__getPackageNamesFromPackageBase() { + local pkgbase=$1 + + (. "fixtures/${pkgbase}/PKGBUILD"; echo ${pkgname[@]}) +} + +__checkRepoRemovedPackage() { + local repo=$1 + local pkgbase=$2 + local repoarch=$3 + local pkgname + + for pkgname in $(__getPackageNamesFromPackageBase ${pkgbase}); do + [[ ! -f ${FTP_BASE}/${PKGPOOL}/${pkgname}-*${PKGEXT} ]] + [[ ! -f ${FTP_BASE}/${repo}/os/${repoarch}/${pkgname}-*${PKGEXT} ]] + done +} + +@test "testCleanupSimplePackages" { + local arches=('i686' 'x86_64') + local pkgs=('pkg-simple-a' 'pkg-simple-b') + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + releasePackage extra ${pkgbase} + done + + db-update + + for arch in ${arches[@]}; do + db-remove extra ${arch} pkg-simple-a + done + + ftpdir-cleanup + + checkRemovedPackage extra 'pkg-simple-a' + for arch in ${arches[@]}; do + __checkRepoRemovedPackage extra 'pkg-simple-a' ${arch} + done + + checkPackage extra pkg-simple-b +} + +@test "testCleanupEpochPackages" { + local arches=('i686' 'x86_64') + local pkgs=('pkg-simple-epoch') + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + releasePackage extra ${pkgbase} + done + + db-update + + for arch in ${arches[@]}; do + db-remove extra ${arch} pkg-simple-epoch + done + + ftpdir-cleanup + + checkRemovedPackage extra 'pkg-simple-epoch' + for arch in ${arches[@]}; do + __checkRepoRemovedPackage extra 'pkg-simple-epoch' ${arch} + done +} + +@test "testCleanupAnyPackages" { + local arches=('i686' 'x86_64') + local pkgs=('pkg-any-a' 'pkg-any-b') + local pkgbase + local arch='any' + + for pkgbase in ${pkgs[@]}; do + releasePackage extra ${pkgbase} + done + + db-update + db-remove extra any pkg-any-a + ftpdir-cleanup + + local pkg1='pkg-any-a-1-1-any.pkg.tar.xz' + checkRemovedPackage extra 'pkg-any-a' + for arch in ${arches[@]}; do + __checkRepoRemovedPackage extra 'pkg-any-a' ${arch} + done + + checkPackage extra pkg-any-b +} + +@test "testCleanupSplitPackages" { + local arches=('i686' 'x86_64') + local pkgs=('pkg-split-a' 'pkg-split-b') + local pkg + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + releasePackage extra ${pkgbase} + done + + db-update + + for arch in ${arches[@]}; do + db-remove extra ${arch} ${pkgs[0]} + done + + ftpdir-cleanup + + for arch in ${arches[@]}; do + __checkRepoRemovedPackage extra ${pkg[0]} ${arch} + done + + checkRemovedPackage extra ${pkgs[0]} + checkPackage extra ${pkgs[1]} +} + +@test "testCleanupOldPackages" { + local arches=('i686' 'x86_64') + local pkgs=('pkg-simple-a' 'pkg-simple-b') + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + releasePackage extra ${pkgbase} + done + + db-update + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + db-remove extra ${arch} ${pkgbase} + done + done + + ftpdir-cleanup + + local pkgfilea="pkg-simple-a-1-1-${arch}.pkg.tar.xz" + local pkgfileb="pkg-simple-b-1-1-${arch}.pkg.tar.xz" + for arch in ${arches[@]}; do + touch -d "-$(expr ${CLEANUP_KEEP} + 1)days" ${CLEANUP_DESTDIR}/${pkgfilea}{,.sig} + done + + ftpdir-cleanup + + [ ! -f ${CLEANUP_DESTDIR}/${pkgfilea} ] + [ -f ${CLEANUP_DESTDIR}/${pkgfileb} ] +} diff --git a/test/cases/sourceballs.bats b/test/cases/sourceballs.bats new file mode 100644 index 0000000..0edc594 --- /dev/null +++ b/test/cases/sourceballs.bats @@ -0,0 +1,83 @@ +load ../lib/common + +__checkSourcePackage() { + local pkgbase=$1 + [ -r ${FTP_BASE}/${SRCPOOL}/${pkgbase}-*${SRCEXT} ] +} + +__checkRemovedSourcePackage() { + local pkgbase=$1 + [ ! -r ${FTP_BASE}/${SRCPOOL}/${pkgbase}-*${SRCEXT} ] +} + +@test "testSourceballs" { + local arches=('i686' 'x86_64') + local pkgs=('pkg-simple-a' 'pkg-simple-b' 'pkg-simple-epoch') + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + releasePackage extra ${pkgbase} + done + db-update + + sourceballs + for pkgbase in ${pkgs[@]}; do + __checkSourcePackage ${pkgbase} + done +} + +@test "testAnySourceballs" { + local pkgs=('pkg-any-a' 'pkg-any-b') + local pkgbase + + for pkgbase in ${pkgs[@]}; do + releasePackage extra ${pkgbase} + done + db-update + + sourceballs + for pkgbase in ${pkgs[@]}; do + __checkSourcePackage ${pkgbase} + done +} + +@test "testSplitSourceballs" { + local arches=('i686' 'x86_64') + local pkgs=('pkg-split-a' 'pkg-split-b') + local pkg + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + releasePackage extra ${pkgbase} + done + + db-update + + sourceballs + for pkgbase in ${pkgs[@]}; do + __checkSourcePackage ${pkgbase} + done +} + +@test "testSourceballsCleanup" { + local arches=('i686' 'x86_64') + local pkgs=('pkg-simple-a' 'pkg-simple-b') + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + releasePackage extra ${pkgbase} + done + db-update + sourceballs + + for arch in ${arches[@]}; do + db-remove extra ${arch} pkg-simple-a + done + + sourceballs + __checkRemovedSourcePackage pkg-simple-a + __checkSourcePackage pkg-simple-b +} diff --git a/test/cases/testing2x.bats b/test/cases/testing2x.bats new file mode 100644 index 0000000..e849799 --- /dev/null +++ b/test/cases/testing2x.bats @@ -0,0 +1,16 @@ +load ../lib/common + +@test "testTesting2xAnyPackage" { + releasePackage core pkg-any-a + db-update + + updatePackage pkg-any-a + + releasePackage testing pkg-any-a + db-update + + testing2x pkg-any-a + + checkPackage core pkg-any-a + checkRemovedPackage testing pkg-any-a +} diff --git a/test/fixtures/pkg-any-a/PKGBUILD b/test/fixtures/pkg-any-a/PKGBUILD new file mode 100644 index 0000000..cf2774b --- /dev/null +++ b/test/fixtures/pkg-any-a/PKGBUILD @@ -0,0 +1,13 @@ +pkgname=pkg-any-a +pkgver=1 +pkgrel=1 +pkgdesc="A package called ${pkgname}" +arch=('any') +url='http://www.archlinux.org/' +license=('GPL') +options=(!strip) + +package() { + install -d ${pkgdir}/usr/share/${pkgname} + :>${pkgdir}/usr/share/${pkgname}/test +} diff --git a/test/fixtures/pkg-any-b/PKGBUILD b/test/fixtures/pkg-any-b/PKGBUILD new file mode 100644 index 0000000..05dfd10 --- /dev/null +++ b/test/fixtures/pkg-any-b/PKGBUILD @@ -0,0 +1,13 @@ +pkgname=pkg-any-b +pkgver=1 +pkgrel=1 +pkgdesc="A package called ${pkgname}" +arch=('any') +url='http://www.archlinux.org/' +license=('GPL') +options=(!strip) + +package() { + install -d ${pkgdir}/usr/share/${pkgname} + :>${pkgdir}/usr/share/${pkgname}/test +} diff --git a/test/fixtures/pkg-simple-a/PKGBUILD b/test/fixtures/pkg-simple-a/PKGBUILD new file mode 100644 index 0000000..329a4a3 --- /dev/null +++ b/test/fixtures/pkg-simple-a/PKGBUILD @@ -0,0 +1,13 @@ +pkgname=pkg-simple-a +pkgver=1 +pkgrel=1 +pkgdesc="A package called ${pkgname}" +arch=('i686' 'x86_64') +url='http://www.archlinux.org/' +license=('GPL') +depends=('glibc') +options=(!strip) + +package() { + install -D -m755 /bin/true ${pkgdir}/usr/bin/${pkgname} +} diff --git a/test/fixtures/pkg-simple-b/PKGBUILD b/test/fixtures/pkg-simple-b/PKGBUILD new file mode 100644 index 0000000..287eee7 --- /dev/null +++ b/test/fixtures/pkg-simple-b/PKGBUILD @@ -0,0 +1,13 @@ +pkgname=pkg-simple-b +pkgver=1 +pkgrel=1 +pkgdesc="A package called ${pkgname}" +arch=('i686' 'x86_64') +url='http://www.archlinux.org/' +license=('GPL') +depends=('glibc') +options=(!strip) + +package() { + install -D -m755 /bin/true ${pkgdir}/usr/bin/${pkgname} +} diff --git a/test/fixtures/pkg-simple-epoch/PKGBUILD b/test/fixtures/pkg-simple-epoch/PKGBUILD new file mode 100644 index 0000000..c624439 --- /dev/null +++ b/test/fixtures/pkg-simple-epoch/PKGBUILD @@ -0,0 +1,14 @@ +pkgname=pkg-simple-epoch +pkgver=1 +pkgrel=1 +epoch=1 +pkgdesc="A package called ${pkgname}" +arch=('i686' 'x86_64') +url='http://www.archlinux.org/' +license=('GPL') +depends=('glibc') +options=(!strip) + +package() { + install -D -m755 /bin/true ${pkgdir}/usr/bin/${pkgname} +} diff --git a/test/fixtures/pkg-single-arch/PKGBUILD b/test/fixtures/pkg-single-arch/PKGBUILD new file mode 100644 index 0000000..de5dbc2 --- /dev/null +++ b/test/fixtures/pkg-single-arch/PKGBUILD @@ -0,0 +1,13 @@ +pkgname=pkg-single-arch +pkgver=1 +pkgrel=1 +pkgdesc="A package called ${pkgname}" +arch=('x86_64') +url='http://www.archlinux.org/' +license=('GPL') +depends=('glibc') +options=(!strip) + +package() { + install -D -m755 /bin/true ${pkgdir}/usr/bin/${pkgname} +} diff --git a/test/fixtures/pkg-single-epoch/PKGBUILD b/test/fixtures/pkg-single-epoch/PKGBUILD new file mode 100644 index 0000000..95898ac --- /dev/null +++ b/test/fixtures/pkg-single-epoch/PKGBUILD @@ -0,0 +1,14 @@ +pkgname=pkg-single-epoch +pkgver=1 +pkgrel=1 +epoch=1 +pkgdesc="A package called ${pkgname}" +arch=('x86_64') +url='http://www.archlinux.org/' +license=('GPL') +depends=('glibc') +options=(!strip) + +package() { + install -D -m755 /bin/true ${pkgdir}/usr/bin/${pkgname} +} diff --git a/test/fixtures/pkg-split-a/PKGBUILD b/test/fixtures/pkg-split-a/PKGBUILD new file mode 100644 index 0000000..2c91390 --- /dev/null +++ b/test/fixtures/pkg-split-a/PKGBUILD @@ -0,0 +1,18 @@ +pkgbase=pkg-split-a +pkgname=('pkg-split-a1' 'pkg-split-a2') +pkgver=1 +pkgrel=1 +pkgdesc="A split package called ${pkgbase}" +arch=('i686' 'x86_64') +url='http://www.archlinux.org/' +license=('GPL') +depends=('glibc') +options=(!strip) + +package_pkg-split-a1() { + install -D -m755 /bin/true ${pkgdir}/usr/bin/${pkgname} +} + +package_pkg-split-a2() { + install -D -m755 /bin/true ${pkgdir}/usr/bin/${pkgname} +} diff --git a/test/fixtures/pkg-split-b/PKGBUILD b/test/fixtures/pkg-split-b/PKGBUILD new file mode 100644 index 0000000..1d07827 --- /dev/null +++ b/test/fixtures/pkg-split-b/PKGBUILD @@ -0,0 +1,18 @@ +pkgbase=pkg-split-b +pkgname=('pkg-split-b1' 'pkg-split-b2') +pkgver=1 +pkgrel=1 +pkgdesc="A split package called ${pkgbase}" +arch=('i686' 'x86_64') +url='http://www.archlinux.org/' +license=('GPL') +depends=('glibc') +options=(!strip) + +package_pkg-split-b1() { + install -D -m755 /bin/true ${pkgdir}/usr/bin/${pkgname} +} + +package_pkg-split-b2() { + install -D -m755 /bin/true ${pkgdir}/usr/bin/${pkgname} +} diff --git a/test/lib/common.bash b/test/lib/common.bash index 8e1f428..7589f51 100644 --- a/test/lib/common.bash +++ b/test/lib/common.bash @@ -128,7 +128,7 @@ releasePackage() { if [ ! -d "${TMP}/svn-packages-copy/${pkgbase}/trunk" ]; then mkdir -p "${TMP}/svn-packages-copy/${pkgbase}"/{trunk,repos} - cp -r "packages/${pkgbase}"/* "${TMP}/svn-packages-copy"/${pkgbase}/trunk/ + cp -r "fixtures/${pkgbase}"/* "${TMP}/svn-packages-copy"/${pkgbase}/trunk/ svn add -q "${TMP}/svn-packages-copy"/${pkgbase} svn commit -q -m"initial commit of ${pkgbase}" "${TMP}/svn-packages-copy" fi diff --git a/test/packages/pkg-any-a/PKGBUILD b/test/packages/pkg-any-a/PKGBUILD deleted file mode 100644 index cf2774b..0000000 --- a/test/packages/pkg-any-a/PKGBUILD +++ /dev/null @@ -1,13 +0,0 @@ -pkgname=pkg-any-a -pkgver=1 -pkgrel=1 -pkgdesc="A package called ${pkgname}" -arch=('any') -url='http://www.archlinux.org/' -license=('GPL') -options=(!strip) - -package() { - install -d ${pkgdir}/usr/share/${pkgname} - :>${pkgdir}/usr/share/${pkgname}/test -} diff --git a/test/packages/pkg-any-b/PKGBUILD b/test/packages/pkg-any-b/PKGBUILD deleted file mode 100644 index 05dfd10..0000000 --- a/test/packages/pkg-any-b/PKGBUILD +++ /dev/null @@ -1,13 +0,0 @@ -pkgname=pkg-any-b -pkgver=1 -pkgrel=1 -pkgdesc="A package called ${pkgname}" -arch=('any') -url='http://www.archlinux.org/' -license=('GPL') -options=(!strip) - -package() { - install -d ${pkgdir}/usr/share/${pkgname} - :>${pkgdir}/usr/share/${pkgname}/test -} diff --git a/test/packages/pkg-simple-a/PKGBUILD b/test/packages/pkg-simple-a/PKGBUILD deleted file mode 100644 index 329a4a3..0000000 --- a/test/packages/pkg-simple-a/PKGBUILD +++ /dev/null @@ -1,13 +0,0 @@ -pkgname=pkg-simple-a -pkgver=1 -pkgrel=1 -pkgdesc="A package called ${pkgname}" -arch=('i686' 'x86_64') -url='http://www.archlinux.org/' -license=('GPL') -depends=('glibc') -options=(!strip) - -package() { - install -D -m755 /bin/true ${pkgdir}/usr/bin/${pkgname} -} diff --git a/test/packages/pkg-simple-b/PKGBUILD b/test/packages/pkg-simple-b/PKGBUILD deleted file mode 100644 index 287eee7..0000000 --- a/test/packages/pkg-simple-b/PKGBUILD +++ /dev/null @@ -1,13 +0,0 @@ -pkgname=pkg-simple-b -pkgver=1 -pkgrel=1 -pkgdesc="A package called ${pkgname}" -arch=('i686' 'x86_64') -url='http://www.archlinux.org/' -license=('GPL') -depends=('glibc') -options=(!strip) - -package() { - install -D -m755 /bin/true ${pkgdir}/usr/bin/${pkgname} -} diff --git a/test/packages/pkg-simple-epoch/PKGBUILD b/test/packages/pkg-simple-epoch/PKGBUILD deleted file mode 100644 index c624439..0000000 --- a/test/packages/pkg-simple-epoch/PKGBUILD +++ /dev/null @@ -1,14 +0,0 @@ -pkgname=pkg-simple-epoch -pkgver=1 -pkgrel=1 -epoch=1 -pkgdesc="A package called ${pkgname}" -arch=('i686' 'x86_64') -url='http://www.archlinux.org/' -license=('GPL') -depends=('glibc') -options=(!strip) - -package() { - install -D -m755 /bin/true ${pkgdir}/usr/bin/${pkgname} -} diff --git a/test/packages/pkg-single-arch/PKGBUILD b/test/packages/pkg-single-arch/PKGBUILD deleted file mode 100644 index de5dbc2..0000000 --- a/test/packages/pkg-single-arch/PKGBUILD +++ /dev/null @@ -1,13 +0,0 @@ -pkgname=pkg-single-arch -pkgver=1 -pkgrel=1 -pkgdesc="A package called ${pkgname}" -arch=('x86_64') -url='http://www.archlinux.org/' -license=('GPL') -depends=('glibc') -options=(!strip) - -package() { - install -D -m755 /bin/true ${pkgdir}/usr/bin/${pkgname} -} diff --git a/test/packages/pkg-single-epoch/PKGBUILD b/test/packages/pkg-single-epoch/PKGBUILD deleted file mode 100644 index 95898ac..0000000 --- a/test/packages/pkg-single-epoch/PKGBUILD +++ /dev/null @@ -1,14 +0,0 @@ -pkgname=pkg-single-epoch -pkgver=1 -pkgrel=1 -epoch=1 -pkgdesc="A package called ${pkgname}" -arch=('x86_64') -url='http://www.archlinux.org/' -license=('GPL') -depends=('glibc') -options=(!strip) - -package() { - install -D -m755 /bin/true ${pkgdir}/usr/bin/${pkgname} -} diff --git a/test/packages/pkg-split-a/PKGBUILD b/test/packages/pkg-split-a/PKGBUILD deleted file mode 100644 index 2c91390..0000000 --- a/test/packages/pkg-split-a/PKGBUILD +++ /dev/null @@ -1,18 +0,0 @@ -pkgbase=pkg-split-a -pkgname=('pkg-split-a1' 'pkg-split-a2') -pkgver=1 -pkgrel=1 -pkgdesc="A split package called ${pkgbase}" -arch=('i686' 'x86_64') -url='http://www.archlinux.org/' -license=('GPL') -depends=('glibc') -options=(!strip) - -package_pkg-split-a1() { - install -D -m755 /bin/true ${pkgdir}/usr/bin/${pkgname} -} - -package_pkg-split-a2() { - install -D -m755 /bin/true ${pkgdir}/usr/bin/${pkgname} -} diff --git a/test/packages/pkg-split-b/PKGBUILD b/test/packages/pkg-split-b/PKGBUILD deleted file mode 100644 index 1d07827..0000000 --- a/test/packages/pkg-split-b/PKGBUILD +++ /dev/null @@ -1,18 +0,0 @@ -pkgbase=pkg-split-b -pkgname=('pkg-split-b1' 'pkg-split-b2') -pkgver=1 -pkgrel=1 -pkgdesc="A split package called ${pkgbase}" -arch=('i686' 'x86_64') -url='http://www.archlinux.org/' -license=('GPL') -depends=('glibc') -options=(!strip) - -package_pkg-split-b1() { - install -D -m755 /bin/true ${pkgdir}/usr/bin/${pkgname} -} - -package_pkg-split-b2() { - install -D -m755 /bin/true ${pkgdir}/usr/bin/${pkgname} -} diff --git a/test/test.d/db-move.bats b/test/test.d/db-move.bats deleted file mode 100644 index 4c3dfed..0000000 --- a/test/test.d/db-move.bats +++ /dev/null @@ -1,92 +0,0 @@ -load ../lib/common - -@test "testMoveSimplePackages" { - local arches=('i686' 'x86_64') - local pkgs=('pkg-simple-a' 'pkg-simple-b') - local pkgbase - local arch - - for pkgbase in ${pkgs[@]}; do - releasePackage testing ${pkgbase} - done - - db-update - - db-move testing extra pkg-simple-a - - checkRemovedPackage testing pkg-simple-a - checkPackage extra pkg-simple-a - checkPackage testing pkg-simple-b -} - -@test "testMoveMultiplePackages" { - local arches=('i686' 'x86_64') - local pkgs=('pkg-simple-a' 'pkg-simple-b') - local pkgbase - local arch - - for pkgbase in ${pkgs[@]}; do - releasePackage testing ${pkgbase} - done - - db-update - - db-move testing extra pkg-simple-a pkg-simple-b - - for pkgbase in ${pkgs[@]}; do - checkRemovedPackage testing ${pkgbase} - checkPackage extra ${pkgbase} - done -} - -@test "testMoveEpochPackages" { - local arches=('i686' 'x86_64') - local pkgs=('pkg-simple-epoch') - local pkgbase - local arch - - for pkgbase in ${pkgs[@]}; do - releasePackage testing ${pkgbase} - done - - db-update - - db-move testing extra pkg-simple-epoch - - checkRemovedPackage testing pkg-simple-epoch - checkPackage extra pkg-simple-epoch -} - -@test "testMoveAnyPackages" { - local pkgs=('pkg-any-a' 'pkg-any-b') - local pkgbase - - for pkgbase in ${pkgs[@]}; do - releasePackage testing ${pkgbase} - done - - db-update - db-move testing extra pkg-any-a - - checkPackage extra pkg-any-a - checkRemovedPackage testing pkg-any-a - checkPackage testing pkg-any-b -} - -@test "testMoveSplitPackages" { - local arches=('i686' 'x86_64') - local pkgs=('pkg-split-a' 'pkg-split-b') - local pkg - local pkgbase - local arch - - for pkgbase in ${pkgs[@]}; do - releasePackage testing ${pkgbase} - done - - db-update - db-move testing extra pkg-split-a - - checkPackage extra pkg-split-a - checkPackage testing pkg-split-b -} diff --git a/test/test.d/db-remove.bats b/test/test.d/db-remove.bats deleted file mode 100644 index 379966a..0000000 --- a/test/test.d/db-remove.bats +++ /dev/null @@ -1,64 +0,0 @@ -load ../lib/common - -@test "testRemovePackages" { - local arches=('i686' 'x86_64') - local pkgs=('pkg-simple-a' 'pkg-simple-b' 'pkg-split-a' 'pkg-split-b' 'pkg-simple-epoch') - local pkgbase - local arch - - for pkgbase in ${pkgs[@]}; do - releasePackage extra ${pkgbase} - done - - db-update - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - db-remove extra ${arch} ${pkgbase} - done - done - - for pkgbase in ${pkgs[@]}; do - checkRemovedPackage extra ${pkgbase} - done -} - -@test "testRemoveMultiplePackages" { - local arches=('i686' 'x86_64') - local pkgs=('pkg-simple-a' 'pkg-simple-b' 'pkg-split-a' 'pkg-split-b' 'pkg-simple-epoch') - local pkgbase - local arch - - for pkgbase in ${pkgs[@]}; do - releasePackage extra ${pkgbase} - done - - db-update - - for arch in ${arches[@]}; do - db-remove extra ${arch} ${pkgs[@]} - done - - for pkgbase in ${pkgs[@]}; do - checkRemovedPackage extra ${pkgbase} - done -} - -@test "testRemoveAnyPackages" { - local pkgs=('pkg-any-a' 'pkg-any-b') - local pkgbase - - for pkgbase in ${pkgs[@]}; do - releasePackage extra ${pkgbase} - done - - db-update - - for pkgbase in ${pkgs[@]}; do - db-remove extra any ${pkgbase} - done - - for pkgbase in ${pkgs[@]}; do - checkRemovedPackage extra ${pkgbase} - done -} diff --git a/test/test.d/db-repo-add.bats b/test/test.d/db-repo-add.bats deleted file mode 100644 index bbf0e8f..0000000 --- a/test/test.d/db-repo-add.bats +++ /dev/null @@ -1,82 +0,0 @@ -load ../lib/common - -__movePackageToRepo() { - local repo=$1 - local pkgbase=$2 - local arch=$3 - local tarch - local tarches - - if [[ $arch == any ]]; then - tarches=(${ARCHES[@]}) - else - tarches=(${arch}) - fi - - # FIXME: pkgbase might not be part of the package filename - mv -v "${STAGING}"/${repo}/${pkgbase}-*-*-${arch}${PKGEXT}{,.sig} "${FTP_BASE}/${PKGPOOL}/" - for tarch in ${tarches[@]}; do - ln -sv ${FTP_BASE}/${PKGPOOL}/${pkgbase}-*-*-${arch}${PKGEXT} "${FTP_BASE}/${repo}/os/${tarch}/" - ln -sv ${FTP_BASE}/${PKGPOOL}/${pkgbase}-*-*-${arch}${PKGEXT}.sig "${FTP_BASE}/${repo}/os/${tarch}/" - done -} - -@test "testRepoAddSimplePackages" { - local arches=('i686' 'x86_64') - local pkgs=('pkg-simple-a' 'pkg-simple-b') - local pkgbase - local arch - - for pkgbase in ${pkgs[@]}; do - releasePackage extra ${pkgbase} - for arch in ${arches[@]}; do - __movePackageToRepo extra ${pkgbase} ${arch} - db-repo-add extra ${arch} ${pkgbase}-1-1-${arch}.pkg.tar.xz - done - done - - for pkgbase in ${pkgs[@]}; do - checkPackageDB extra ${pkgbase} - done -} - -@test "testRepoAddMultiplePackages" { - local arches=('i686' 'x86_64') - local pkgs=('pkg-simple-a' 'pkg-simple-b') - local pkgbase - local arch - - for pkgbase in ${pkgs[@]}; do - releasePackage extra ${pkgbase} - done - - for arch in ${arches[@]}; do - add_pkgs=() - for pkgbase in ${pkgs[@]}; do - __movePackageToRepo extra ${pkgbase} ${arch} - add_pkgs[${#add_pkgs[*]}]=${pkgbase}-1-1-${arch}.pkg.tar.xz - done - db-repo-add extra ${arch} ${add_pkgs[@]} - done - - for pkgbase in ${pkgs[@]}; do - checkPackageDB extra ${pkgbase} - done -} - -@test "testRepoAddAnyPackages" { - local arches=('i686' 'x86_64') - local pkgs=('pkg-any-a' 'pkg-any-b') - local pkgbase - local arch - - for pkgbase in ${pkgs[@]}; do - releasePackage extra ${pkgbase} - __movePackageToRepo extra ${pkgbase} any - db-repo-add extra any ${pkgbase}-1-1-any.pkg.tar.xz - done - - for pkgbase in ${pkgs[@]}; do - checkPackageDB extra ${pkgbase} - done -} diff --git a/test/test.d/db-repo-remove.bats b/test/test.d/db-repo-remove.bats deleted file mode 100644 index e8df35b..0000000 --- a/test/test.d/db-repo-remove.bats +++ /dev/null @@ -1,64 +0,0 @@ -load ../lib/common - -@test "testRepoRemovePackages" { - local arches=('i686' 'x86_64') - local pkgs=('pkg-simple-a' 'pkg-simple-b' 'pkg-simple-epoch') - local pkgbase - local arch - - for pkgbase in ${pkgs[@]}; do - releasePackage extra ${pkgbase} - done - - db-update - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - db-repo-remove extra ${arch} ${pkgbase} - done - done - - for pkgbase in ${pkgs[@]}; do - checkRemovedPackageDB extra ${pkgbase} - done -} - -@test "testRepoRemoveMultiplePackages" { - local arches=('i686' 'x86_64') - local pkgs=('pkg-simple-a' 'pkg-simple-b' 'pkg-simple-epoch') - local pkgbase - local arch - - for pkgbase in ${pkgs[@]}; do - releasePackage extra ${pkgbase} - done - - db-update - - for arch in ${arches[@]}; do - db-repo-remove extra ${arch} ${pkgs[@]} - done - - for pkgbase in ${pkgs[@]}; do - checkRemovedPackageDB extra ${pkgbase} - done -} - -@test "testRepoRemoveAnyPackages" { - local pkgs=('pkg-any-a' 'pkg-any-b') - local pkgbase - - for pkgbase in ${pkgs[@]}; do - releasePackage extra ${pkgbase} - done - - db-update - - for pkgbase in ${pkgs[@]}; do - db-repo-remove extra any ${pkgbase} - done - - for pkgbase in ${pkgs[@]}; do - checkRemovedPackageDB extra ${pkgbase} - done -} diff --git a/test/test.d/db-update.bats b/test/test.d/db-update.bats deleted file mode 100644 index c9db68f..0000000 --- a/test/test.d/db-update.bats +++ /dev/null @@ -1,239 +0,0 @@ -load ../lib/common - -@test "testAddSimplePackages" { - local arches=('i686' 'x86_64') - local pkgs=('pkg-simple-a' 'pkg-simple-b') - local pkgbase - local arch - - for pkgbase in ${pkgs[@]}; do - releasePackage extra ${pkgbase} - done - - db-update - - for pkgbase in ${pkgs[@]}; do - checkPackage extra ${pkgbase} - done -} - -@test "testAddSingleSimplePackage" { - releasePackage extra 'pkg-single-arch' - db-update - checkPackage extra 'pkg-single-arch' -} - -@test "testAddSingleEpochPackage" { - releasePackage extra 'pkg-single-epoch' - db-update - checkPackage extra 'pkg-single-epoch' -} - -@test "testAddAnyPackages" { - local pkgs=('pkg-any-a' 'pkg-any-b') - local pkgbase - - for pkgbase in ${pkgs[@]}; do - releasePackage extra ${pkgbase} - done - - db-update - - for pkgbase in ${pkgs[@]}; do - checkPackage extra ${pkgbase} - done -} - -@test "testAddSplitPackages" { - local arches=('i686' 'x86_64') - local pkgs=('pkg-split-a' 'pkg-split-b') - local pkg - local pkgbase - local arch - - for pkgbase in ${pkgs[@]}; do - releasePackage extra ${pkgbase} - done - - db-update - - for pkgbase in ${pkgs[@]}; do - checkPackage extra ${pkgbase} - done -} - -@test "testUpdateAnyPackage" { - releasePackage extra pkg-any-a - db-update - - updatePackage pkg-any-a - - releasePackage extra pkg-any-a - db-update - - checkPackage extra pkg-any-a -} - -@test "testUpdateAnyPackageToDifferentRepositoriesAtOnce" { - releasePackage extra pkg-any-a - - updatePackage pkg-any-a - - releasePackage testing pkg-any-a - - db-update - - checkPackage extra pkg-any-a - checkPackage testing pkg-any-a -} - -@test "testUpdateSameAnyPackageToSameRepository" { - releasePackage extra pkg-any-a - db-update - checkPackage extra pkg-any-a - - releasePackage extra pkg-any-a - run db-update - [ "$status" -ne 0 ] -} - -@test "testUpdateSameAnyPackageToDifferentRepositories" { - local arch - - releasePackage extra pkg-any-a - db-update - checkPackage extra pkg-any-a - - releasePackage testing pkg-any-a - run db-update - [ "$status" -ne 0 ] - - checkRemovedPackageDB testing pkg-any-a -} - -@test "testAddIncompleteSplitPackage" { - local arches=('i686' 'x86_64') - local repo='extra' - local pkgbase='pkg-split-a' - local arch - - releasePackage ${repo} ${pkgbase} - - # remove a split package to make db-update fail - rm "${STAGING}"/extra/${pkgbase}1-* - - run db-update - [ "$status" -ne 0 ] - - checkRemovedPackageDB ${repo} ${pkgbase} -} - -@test "testUnknownRepo" { - mkdir "${STAGING}/unknown/" - releasePackage extra 'pkg-any-a' - releasePackage unknown 'pkg-any-b' - db-update - checkPackage extra 'pkg-any-a' - [ ! -e "${FTP_BASE}/unknown" ] - rm -rf "${STAGING}/unknown/" -} - -@test "testAddUnsignedPackageFails" { - releasePackage extra 'pkg-any-a' - rm "${STAGING}"/extra/*.sig - run db-update - [ "$status" -ne 0 ] - - checkRemovedPackageDB extra pkg-any-a -} - -@test "testAddInvalidSignedPackageFails" { - local p - releasePackage extra 'pkg-any-a' - for p in "${STAGING}"/extra/*${PKGEXT}; do - unxz $p - xz -0 ${p%%.xz} - done - run db-update - [ "$status" -ne 0 ] - - checkRemovedPackageDB extra pkg-any-a -} - -@test "testAddBrokenSignatureFails" { - local s - releasePackage extra 'pkg-any-a' - for s in "${STAGING}"/extra/*.sig; do - echo 0 > $s - done - run db-update - [ "$status" -ne 0 ] - - checkRemovedPackageDB extra pkg-any-a -} - -@test "testAddPackageWithInconsistentVersionFails" { - local p - releasePackage extra 'pkg-any-a' - - for p in "${STAGING}"/extra/*; do - mv "${p}" "${p/pkg-any-a-1/pkg-any-a-2}" - done - - run db-update - [ "$status" -ne 0 ] - checkRemovedPackageDB extra 'pkg-any-a' -} - -@test "testAddPackageWithInconsistentNameFails" { - local p - releasePackage extra 'pkg-any-a' - - for p in "${STAGING}"/extra/*; do - mv "${p}" "${p/pkg-/foo-pkg-}" - done - - run db-update - [ "$status" -ne 0 ] - checkRemovedPackage extra 'pkg-any-a' -} - -@test "testAddPackageWithInconsistentPKGBUILDFails" { - releasePackage extra 'pkg-any-a' - - updateRepoPKGBUILD 'pkg-any-a' extra any - - run db-update - [ "$status" -ne 0 ] - checkRemovedPackageDB extra 'pkg-any-a' -} - -@test "testAddPackageWithInsufficientPermissionsFails" { - releasePackage core 'pkg-any-a' - releasePackage extra 'pkg-any-b' - - chmod -xwr ${FTP_BASE}/core/os/i686 - run db-update - [ "$status" -ne 0 ] - chmod +xwr ${FTP_BASE}/core/os/i686 - - checkRemovedPackageDB core 'pkg-any-a' - checkRemovedPackageDB extra 'pkg-any-b' -} - -@test "testPackageHasToBeARegularFile" { - local p - local target=$(mktemp -d) - local arches=('i686' 'x86_64') - - releasePackage extra 'pkg-simple-a' - - for p in "${STAGING}"/extra/*i686*; do - mv "${p}" "${target}" - ln -s "${target}/${p##*/}" "${p}" - done - - run db-update - [ "$status" -ne 0 ] - checkRemovedPackageDB extra "pkg-simple-a" -} diff --git a/test/test.d/ftpdir-cleanup.bats b/test/test.d/ftpdir-cleanup.bats deleted file mode 100644 index e3adeee..0000000 --- a/test/test.d/ftpdir-cleanup.bats +++ /dev/null @@ -1,151 +0,0 @@ -load ../lib/common - -__getPackageNamesFromPackageBase() { - local pkgbase=$1 - - (. "packages/${pkgbase}/PKGBUILD"; echo ${pkgname[@]}) -} - -__checkRepoRemovedPackage() { - local repo=$1 - local pkgbase=$2 - local repoarch=$3 - local pkgname - - for pkgname in $(__getPackageNamesFromPackageBase ${pkgbase}); do - [[ ! -f ${FTP_BASE}/${PKGPOOL}/${pkgname}-*${PKGEXT} ]] - [[ ! -f ${FTP_BASE}/${repo}/os/${repoarch}/${pkgname}-*${PKGEXT} ]] - done -} - -@test "testCleanupSimplePackages" { - local arches=('i686' 'x86_64') - local pkgs=('pkg-simple-a' 'pkg-simple-b') - local pkgbase - local arch - - for pkgbase in ${pkgs[@]}; do - releasePackage extra ${pkgbase} - done - - db-update - - for arch in ${arches[@]}; do - db-remove extra ${arch} pkg-simple-a - done - - ftpdir-cleanup - - checkRemovedPackage extra 'pkg-simple-a' - for arch in ${arches[@]}; do - __checkRepoRemovedPackage extra 'pkg-simple-a' ${arch} - done - - checkPackage extra pkg-simple-b -} - -@test "testCleanupEpochPackages" { - local arches=('i686' 'x86_64') - local pkgs=('pkg-simple-epoch') - local pkgbase - local arch - - for pkgbase in ${pkgs[@]}; do - releasePackage extra ${pkgbase} - done - - db-update - - for arch in ${arches[@]}; do - db-remove extra ${arch} pkg-simple-epoch - done - - ftpdir-cleanup - - checkRemovedPackage extra 'pkg-simple-epoch' - for arch in ${arches[@]}; do - __checkRepoRemovedPackage extra 'pkg-simple-epoch' ${arch} - done -} - -@test "testCleanupAnyPackages" { - local arches=('i686' 'x86_64') - local pkgs=('pkg-any-a' 'pkg-any-b') - local pkgbase - local arch='any' - - for pkgbase in ${pkgs[@]}; do - releasePackage extra ${pkgbase} - done - - db-update - db-remove extra any pkg-any-a - ftpdir-cleanup - - local pkg1='pkg-any-a-1-1-any.pkg.tar.xz' - checkRemovedPackage extra 'pkg-any-a' - for arch in ${arches[@]}; do - __checkRepoRemovedPackage extra 'pkg-any-a' ${arch} - done - - checkPackage extra pkg-any-b -} - -@test "testCleanupSplitPackages" { - local arches=('i686' 'x86_64') - local pkgs=('pkg-split-a' 'pkg-split-b') - local pkg - local pkgbase - local arch - - for pkgbase in ${pkgs[@]}; do - releasePackage extra ${pkgbase} - done - - db-update - - for arch in ${arches[@]}; do - db-remove extra ${arch} ${pkgs[0]} - done - - ftpdir-cleanup - - for arch in ${arches[@]}; do - __checkRepoRemovedPackage extra ${pkg[0]} ${arch} - done - - checkRemovedPackage extra ${pkgs[0]} - checkPackage extra ${pkgs[1]} -} - -@test "testCleanupOldPackages" { - local arches=('i686' 'x86_64') - local pkgs=('pkg-simple-a' 'pkg-simple-b') - local pkgbase - local arch - - for pkgbase in ${pkgs[@]}; do - releasePackage extra ${pkgbase} - done - - db-update - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - db-remove extra ${arch} ${pkgbase} - done - done - - ftpdir-cleanup - - local pkgfilea="pkg-simple-a-1-1-${arch}.pkg.tar.xz" - local pkgfileb="pkg-simple-b-1-1-${arch}.pkg.tar.xz" - for arch in ${arches[@]}; do - touch -d "-$(expr ${CLEANUP_KEEP} + 1)days" ${CLEANUP_DESTDIR}/${pkgfilea}{,.sig} - done - - ftpdir-cleanup - - [ ! -f ${CLEANUP_DESTDIR}/${pkgfilea} ] - [ -f ${CLEANUP_DESTDIR}/${pkgfileb} ] -} diff --git a/test/test.d/sourceballs.bats b/test/test.d/sourceballs.bats deleted file mode 100644 index 0edc594..0000000 --- a/test/test.d/sourceballs.bats +++ /dev/null @@ -1,83 +0,0 @@ -load ../lib/common - -__checkSourcePackage() { - local pkgbase=$1 - [ -r ${FTP_BASE}/${SRCPOOL}/${pkgbase}-*${SRCEXT} ] -} - -__checkRemovedSourcePackage() { - local pkgbase=$1 - [ ! -r ${FTP_BASE}/${SRCPOOL}/${pkgbase}-*${SRCEXT} ] -} - -@test "testSourceballs" { - local arches=('i686' 'x86_64') - local pkgs=('pkg-simple-a' 'pkg-simple-b' 'pkg-simple-epoch') - local pkgbase - local arch - - for pkgbase in ${pkgs[@]}; do - releasePackage extra ${pkgbase} - done - db-update - - sourceballs - for pkgbase in ${pkgs[@]}; do - __checkSourcePackage ${pkgbase} - done -} - -@test "testAnySourceballs" { - local pkgs=('pkg-any-a' 'pkg-any-b') - local pkgbase - - for pkgbase in ${pkgs[@]}; do - releasePackage extra ${pkgbase} - done - db-update - - sourceballs - for pkgbase in ${pkgs[@]}; do - __checkSourcePackage ${pkgbase} - done -} - -@test "testSplitSourceballs" { - local arches=('i686' 'x86_64') - local pkgs=('pkg-split-a' 'pkg-split-b') - local pkg - local pkgbase - local arch - - for pkgbase in ${pkgs[@]}; do - releasePackage extra ${pkgbase} - done - - db-update - - sourceballs - for pkgbase in ${pkgs[@]}; do - __checkSourcePackage ${pkgbase} - done -} - -@test "testSourceballsCleanup" { - local arches=('i686' 'x86_64') - local pkgs=('pkg-simple-a' 'pkg-simple-b') - local pkgbase - local arch - - for pkgbase in ${pkgs[@]}; do - releasePackage extra ${pkgbase} - done - db-update - sourceballs - - for arch in ${arches[@]}; do - db-remove extra ${arch} pkg-simple-a - done - - sourceballs - __checkRemovedSourcePackage pkg-simple-a - __checkSourcePackage pkg-simple-b -} diff --git a/test/test.d/testing2x.bats b/test/test.d/testing2x.bats deleted file mode 100644 index e849799..0000000 --- a/test/test.d/testing2x.bats +++ /dev/null @@ -1,16 +0,0 @@ -load ../lib/common - -@test "testTesting2xAnyPackage" { - releasePackage core pkg-any-a - db-update - - updatePackage pkg-any-a - - releasePackage testing pkg-any-a - db-update - - testing2x pkg-any-a - - checkPackage core pkg-any-a - checkRemovedPackage testing pkg-any-a -} -- cgit v1.2.2 From 876da50e3f06d134d45043f1372543b4dc93a1ec Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 23 Apr 2017 13:23:58 +0200 Subject: Document test cases in a more readable format --- test/cases/db-move.bats | 10 +++++----- test/cases/db-remove.bats | 6 +++--- test/cases/db-repo-add.bats | 6 +++--- test/cases/db-repo-remove.bats | 6 +++--- test/cases/db-update.bats | 38 +++++++++++++++++++------------------- test/cases/ftpdir-cleanup.bats | 10 +++++----- test/cases/sourceballs.bats | 8 ++++---- test/cases/testing2x.bats | 2 +- 8 files changed, 43 insertions(+), 43 deletions(-) diff --git a/test/cases/db-move.bats b/test/cases/db-move.bats index 4c3dfed..df3dbdf 100644 --- a/test/cases/db-move.bats +++ b/test/cases/db-move.bats @@ -1,6 +1,6 @@ load ../lib/common -@test "testMoveSimplePackages" { +@test "move single packages" { local arches=('i686' 'x86_64') local pkgs=('pkg-simple-a' 'pkg-simple-b') local pkgbase @@ -19,7 +19,7 @@ load ../lib/common checkPackage testing pkg-simple-b } -@test "testMoveMultiplePackages" { +@test "move multiple packages" { local arches=('i686' 'x86_64') local pkgs=('pkg-simple-a' 'pkg-simple-b') local pkgbase @@ -39,7 +39,7 @@ load ../lib/common done } -@test "testMoveEpochPackages" { +@test "move epoch packages" { local arches=('i686' 'x86_64') local pkgs=('pkg-simple-epoch') local pkgbase @@ -57,7 +57,7 @@ load ../lib/common checkPackage extra pkg-simple-epoch } -@test "testMoveAnyPackages" { +@test "move any packages" { local pkgs=('pkg-any-a' 'pkg-any-b') local pkgbase @@ -73,7 +73,7 @@ load ../lib/common checkPackage testing pkg-any-b } -@test "testMoveSplitPackages" { +@test "move split packages" { local arches=('i686' 'x86_64') local pkgs=('pkg-split-a' 'pkg-split-b') local pkg diff --git a/test/cases/db-remove.bats b/test/cases/db-remove.bats index 379966a..fe373bc 100644 --- a/test/cases/db-remove.bats +++ b/test/cases/db-remove.bats @@ -1,6 +1,6 @@ load ../lib/common -@test "testRemovePackages" { +@test "remove single packages" { local arches=('i686' 'x86_64') local pkgs=('pkg-simple-a' 'pkg-simple-b' 'pkg-split-a' 'pkg-split-b' 'pkg-simple-epoch') local pkgbase @@ -23,7 +23,7 @@ load ../lib/common done } -@test "testRemoveMultiplePackages" { +@test "remove multiple packages" { local arches=('i686' 'x86_64') local pkgs=('pkg-simple-a' 'pkg-simple-b' 'pkg-split-a' 'pkg-split-b' 'pkg-simple-epoch') local pkgbase @@ -44,7 +44,7 @@ load ../lib/common done } -@test "testRemoveAnyPackages" { +@test "remove any packages" { local pkgs=('pkg-any-a' 'pkg-any-b') local pkgbase diff --git a/test/cases/db-repo-add.bats b/test/cases/db-repo-add.bats index bbf0e8f..4fd13e4 100644 --- a/test/cases/db-repo-add.bats +++ b/test/cases/db-repo-add.bats @@ -21,7 +21,7 @@ __movePackageToRepo() { done } -@test "testRepoAddSimplePackages" { +@test "add single packages" { local arches=('i686' 'x86_64') local pkgs=('pkg-simple-a' 'pkg-simple-b') local pkgbase @@ -40,7 +40,7 @@ __movePackageToRepo() { done } -@test "testRepoAddMultiplePackages" { +@test "add multiple packages" { local arches=('i686' 'x86_64') local pkgs=('pkg-simple-a' 'pkg-simple-b') local pkgbase @@ -64,7 +64,7 @@ __movePackageToRepo() { done } -@test "testRepoAddAnyPackages" { +@test "add any packages" { local arches=('i686' 'x86_64') local pkgs=('pkg-any-a' 'pkg-any-b') local pkgbase diff --git a/test/cases/db-repo-remove.bats b/test/cases/db-repo-remove.bats index e8df35b..77bcc24 100644 --- a/test/cases/db-repo-remove.bats +++ b/test/cases/db-repo-remove.bats @@ -1,6 +1,6 @@ load ../lib/common -@test "testRepoRemovePackages" { +@test "remove single packages" { local arches=('i686' 'x86_64') local pkgs=('pkg-simple-a' 'pkg-simple-b' 'pkg-simple-epoch') local pkgbase @@ -23,7 +23,7 @@ load ../lib/common done } -@test "testRepoRemoveMultiplePackages" { +@test "remove multiple packages" { local arches=('i686' 'x86_64') local pkgs=('pkg-simple-a' 'pkg-simple-b' 'pkg-simple-epoch') local pkgbase @@ -44,7 +44,7 @@ load ../lib/common done } -@test "testRepoRemoveAnyPackages" { +@test "remove any packages" { local pkgs=('pkg-any-a' 'pkg-any-b') local pkgbase diff --git a/test/cases/db-update.bats b/test/cases/db-update.bats index c9db68f..1da7eef 100644 --- a/test/cases/db-update.bats +++ b/test/cases/db-update.bats @@ -1,6 +1,6 @@ load ../lib/common -@test "testAddSimplePackages" { +@test "add simple packages" { local arches=('i686' 'x86_64') local pkgs=('pkg-simple-a' 'pkg-simple-b') local pkgbase @@ -17,19 +17,19 @@ load ../lib/common done } -@test "testAddSingleSimplePackage" { +@test "add single simple package" { releasePackage extra 'pkg-single-arch' db-update checkPackage extra 'pkg-single-arch' } -@test "testAddSingleEpochPackage" { +@test "add single epoch package" { releasePackage extra 'pkg-single-epoch' db-update checkPackage extra 'pkg-single-epoch' } -@test "testAddAnyPackages" { +@test "add any packages" { local pkgs=('pkg-any-a' 'pkg-any-b') local pkgbase @@ -44,7 +44,7 @@ load ../lib/common done } -@test "testAddSplitPackages" { +@test "add split packages" { local arches=('i686' 'x86_64') local pkgs=('pkg-split-a' 'pkg-split-b') local pkg @@ -62,7 +62,7 @@ load ../lib/common done } -@test "testUpdateAnyPackage" { +@test "update any package" { releasePackage extra pkg-any-a db-update @@ -74,7 +74,7 @@ load ../lib/common checkPackage extra pkg-any-a } -@test "testUpdateAnyPackageToDifferentRepositoriesAtOnce" { +@test "update any package to different repositories at once" { releasePackage extra pkg-any-a updatePackage pkg-any-a @@ -87,7 +87,7 @@ load ../lib/common checkPackage testing pkg-any-a } -@test "testUpdateSameAnyPackageToSameRepository" { +@test "update same any package to same repository fails" { releasePackage extra pkg-any-a db-update checkPackage extra pkg-any-a @@ -97,7 +97,7 @@ load ../lib/common [ "$status" -ne 0 ] } -@test "testUpdateSameAnyPackageToDifferentRepositories" { +@test "update same any package to different repositories fails" { local arch releasePackage extra pkg-any-a @@ -111,7 +111,7 @@ load ../lib/common checkRemovedPackageDB testing pkg-any-a } -@test "testAddIncompleteSplitPackage" { +@test "add incomplete split package fails" { local arches=('i686' 'x86_64') local repo='extra' local pkgbase='pkg-split-a' @@ -128,7 +128,7 @@ load ../lib/common checkRemovedPackageDB ${repo} ${pkgbase} } -@test "testUnknownRepo" { +@test "add package to unknown repo fails" { mkdir "${STAGING}/unknown/" releasePackage extra 'pkg-any-a' releasePackage unknown 'pkg-any-b' @@ -138,7 +138,7 @@ load ../lib/common rm -rf "${STAGING}/unknown/" } -@test "testAddUnsignedPackageFails" { +@test "add unsigned package fails" { releasePackage extra 'pkg-any-a' rm "${STAGING}"/extra/*.sig run db-update @@ -147,7 +147,7 @@ load ../lib/common checkRemovedPackageDB extra pkg-any-a } -@test "testAddInvalidSignedPackageFails" { +@test "add invalid signed package fails" { local p releasePackage extra 'pkg-any-a' for p in "${STAGING}"/extra/*${PKGEXT}; do @@ -160,7 +160,7 @@ load ../lib/common checkRemovedPackageDB extra pkg-any-a } -@test "testAddBrokenSignatureFails" { +@test "add broken signature fails" { local s releasePackage extra 'pkg-any-a' for s in "${STAGING}"/extra/*.sig; do @@ -172,7 +172,7 @@ load ../lib/common checkRemovedPackageDB extra pkg-any-a } -@test "testAddPackageWithInconsistentVersionFails" { +@test "add package with inconsistent version fails" { local p releasePackage extra 'pkg-any-a' @@ -185,7 +185,7 @@ load ../lib/common checkRemovedPackageDB extra 'pkg-any-a' } -@test "testAddPackageWithInconsistentNameFails" { +@test "add package with inconsistent name fails" { local p releasePackage extra 'pkg-any-a' @@ -198,7 +198,7 @@ load ../lib/common checkRemovedPackage extra 'pkg-any-a' } -@test "testAddPackageWithInconsistentPKGBUILDFails" { +@test "add package with inconsistent pkgbuild fails" { releasePackage extra 'pkg-any-a' updateRepoPKGBUILD 'pkg-any-a' extra any @@ -208,7 +208,7 @@ load ../lib/common checkRemovedPackageDB extra 'pkg-any-a' } -@test "testAddPackageWithInsufficientPermissionsFails" { +@test "add package with insufficient permissions fails" { releasePackage core 'pkg-any-a' releasePackage extra 'pkg-any-b' @@ -221,7 +221,7 @@ load ../lib/common checkRemovedPackageDB extra 'pkg-any-b' } -@test "testPackageHasToBeARegularFile" { +@test "package has to be aregular file" { local p local target=$(mktemp -d) local arches=('i686' 'x86_64') diff --git a/test/cases/ftpdir-cleanup.bats b/test/cases/ftpdir-cleanup.bats index cb1ca45..6280ce0 100644 --- a/test/cases/ftpdir-cleanup.bats +++ b/test/cases/ftpdir-cleanup.bats @@ -18,7 +18,7 @@ __checkRepoRemovedPackage() { done } -@test "testCleanupSimplePackages" { +@test "cleanup simple packages" { local arches=('i686' 'x86_64') local pkgs=('pkg-simple-a' 'pkg-simple-b') local pkgbase @@ -44,7 +44,7 @@ __checkRepoRemovedPackage() { checkPackage extra pkg-simple-b } -@test "testCleanupEpochPackages" { +@test "cleanup epoch packages" { local arches=('i686' 'x86_64') local pkgs=('pkg-simple-epoch') local pkgbase @@ -68,7 +68,7 @@ __checkRepoRemovedPackage() { done } -@test "testCleanupAnyPackages" { +@test "cleanup any packages" { local arches=('i686' 'x86_64') local pkgs=('pkg-any-a' 'pkg-any-b') local pkgbase @@ -91,7 +91,7 @@ __checkRepoRemovedPackage() { checkPackage extra pkg-any-b } -@test "testCleanupSplitPackages" { +@test "cleanup split packages" { local arches=('i686' 'x86_64') local pkgs=('pkg-split-a' 'pkg-split-b') local pkg @@ -118,7 +118,7 @@ __checkRepoRemovedPackage() { checkPackage extra ${pkgs[1]} } -@test "testCleanupOldPackages" { +@test "cleanup old packages" { local arches=('i686' 'x86_64') local pkgs=('pkg-simple-a' 'pkg-simple-b') local pkgbase diff --git a/test/cases/sourceballs.bats b/test/cases/sourceballs.bats index 0edc594..a0a2999 100644 --- a/test/cases/sourceballs.bats +++ b/test/cases/sourceballs.bats @@ -10,7 +10,7 @@ __checkRemovedSourcePackage() { [ ! -r ${FTP_BASE}/${SRCPOOL}/${pkgbase}-*${SRCEXT} ] } -@test "testSourceballs" { +@test "create simple package sourceballs" { local arches=('i686' 'x86_64') local pkgs=('pkg-simple-a' 'pkg-simple-b' 'pkg-simple-epoch') local pkgbase @@ -27,7 +27,7 @@ __checkRemovedSourcePackage() { done } -@test "testAnySourceballs" { +@test "create any package sourceballs" { local pkgs=('pkg-any-a' 'pkg-any-b') local pkgbase @@ -42,7 +42,7 @@ __checkRemovedSourcePackage() { done } -@test "testSplitSourceballs" { +@test "create split package sourceballs" { local arches=('i686' 'x86_64') local pkgs=('pkg-split-a' 'pkg-split-b') local pkg @@ -61,7 +61,7 @@ __checkRemovedSourcePackage() { done } -@test "testSourceballsCleanup" { +@test "cleanup sourceballs" { local arches=('i686' 'x86_64') local pkgs=('pkg-simple-a' 'pkg-simple-b') local pkgbase diff --git a/test/cases/testing2x.bats b/test/cases/testing2x.bats index e849799..f9f04e2 100644 --- a/test/cases/testing2x.bats +++ b/test/cases/testing2x.bats @@ -1,6 +1,6 @@ load ../lib/common -@test "testTesting2xAnyPackage" { +@test "move any package" { releasePackage core pkg-any-a db-update -- cgit v1.2.2 From e810a1b0b48b1dd95b0847e716b76011f1d323a2 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 23 Apr 2017 13:36:06 +0200 Subject: Add initial README file --- README.md | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..cb5be7b --- /dev/null +++ b/README.md @@ -0,0 +1,9 @@ +# Arch Linux repository management scripts +## Configuration +* The default configuration can be found in `config`. +* An optional `config.local` may override the default configuration. +* The path and name of the local configuration file can be overriden by setting the `DBSCRIPTS_CONFIG` environment variable. +## Testing +* Install the `make` and `docker` packages. Start the docker daemon by issuing `systemctl start docker`. +* The test suite can now be run with `make test`. +* A coverage report can be generated with `make test-coverage`. Open `coverage/index.html` in your web browser to inspect the results. -- cgit v1.2.2 From 6d932c2e61a94a00025cfd5b6f4ae53c6e8533a2 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 23 Apr 2017 13:57:57 +0200 Subject: Add travis build status to README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index cb5be7b..753b3c1 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Arch Linux repository management scripts +# Arch Linux repository management scripts [![Build Status](https://travis-ci.org/archlinux/dbscripts.svg?branch=master)](https://travis-ci.org/archlinux/dbscripts) ## Configuration * The default configuration can be found in `config`. * An optional `config.local` may override the default configuration. -- cgit v1.2.2 From f4085eaa30142ed778291c4e7dd6694c087bfaef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20L=C3=B6thberg?= Date: Tue, 30 May 2017 15:50:43 +0200 Subject: config: add GPL3/LGPL3 to ALLOWED_LICENSES MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Johannes Löthberg --- config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config b/config index 6d76ef3..92cc4a5 100644 --- a/config +++ b/config @@ -29,7 +29,7 @@ PKGEXT=".pkg.tar.?z" SRCEXT=".src.tar.gz" # Allowed licenses: get sourceballs only for licenses in this array -ALLOWED_LICENSES=('GPL' 'GPL1' 'GPL2' 'LGPL' 'LGPL1' 'LGPL2' 'LGPL2.1') +ALLOWED_LICENSES=('GPL' 'GPL1' 'GPL2' 'GPL3' 'LGPL' 'LGPL1' 'LGPL2' 'LGPL2.1' 'LGPL3') # Override default config with config.local LOCAL_CONFIG=${DBSCRIPTS_CONFIG:-"$(dirname ${BASH_SOURCE[0]})/config.local"} -- cgit v1.2.2 From 8aa1244d2b3d2320dd46c15c633170fa05126e6a Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Tue, 11 Jul 2017 21:03:09 +0200 Subject: Use archlinux/base image --- test/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Dockerfile b/test/Dockerfile index 3fa9aaf..cffebf6 100644 --- a/test/Dockerfile +++ b/test/Dockerfile @@ -1,4 +1,4 @@ -FROM pierres/archlinux +FROM archlinux/base RUN pacman -Syu --noconfirm sudo fakeroot awk subversion make kcov bash-bats RUN pacman-key --init RUN echo '%wheel ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/wheel -- cgit v1.2.2 From 1cad51c5af4b0aa1e1196d737edaa00938af937f Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Wed, 1 Nov 2017 11:03:46 +0100 Subject: Install required packages for testing --- test/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Dockerfile b/test/Dockerfile index cffebf6..3913f00 100644 --- a/test/Dockerfile +++ b/test/Dockerfile @@ -1,5 +1,5 @@ FROM archlinux/base -RUN pacman -Syu --noconfirm sudo fakeroot awk subversion make kcov bash-bats +RUN pacman -Syu --noconfirm --needed sudo fakeroot awk subversion make kcov bash-bats gettext grep RUN pacman-key --init RUN echo '%wheel ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/wheel RUN useradd -N -g users -G wheel -m tester -- cgit v1.2.2 From eec8e35eba84658bdd03230c83f449d2bb437b10 Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Mon, 12 Feb 2018 17:21:39 -0500 Subject: Use return codes properly when checking for failed commands. Rather than doing a test(1) on the numerical value of the return code, use the `if !` and `||` operators during control flow. --- db-functions | 36 ++++++++++++------------------------ db-update | 3 +-- 2 files changed, 13 insertions(+), 26 deletions(-) diff --git a/db-functions b/db-functions index 38b3c2f..f0bf43f 100644 --- a/db-functions +++ b/db-functions @@ -292,12 +292,9 @@ getpkgfiles() { check_pkgfile() { local pkgfile=$1 - local pkgname="$(getpkgname ${pkgfile})" - [ $? -ge 1 ] && return 1 - local pkgver="$(getpkgver ${pkgfile})" - [ $? -ge 1 ] && return 1 - local pkgarch="$(getpkgarch ${pkgfile})" - [ $? -ge 1 ] && return 1 + local pkgname="$(getpkgname ${pkgfile})" || return 1 + local pkgver="$(getpkgver ${pkgfile})" || return 1 + local pkgarch="$(getpkgarch ${pkgfile})" || return 1 in_array "${pkgarch}" ${ARCHES[@]} 'any' || return 1 @@ -310,14 +307,10 @@ check_pkgfile() { check_pkgsvn() { local pkgfile="${1}" - local _pkgbase="$(getpkgbase ${pkgfile})" - [ $? -ge 1 ] && return 1 - local _pkgname="$(getpkgname ${pkgfile})" - [ $? -ge 1 ] && return 1 - local _pkgver="$(getpkgver ${pkgfile})" - [ $? -ge 1 ] && return 1 - local _pkgarch="$(getpkgarch ${pkgfile})" - [ $? -ge 1 ] && return 1 + local _pkgbase="$(getpkgbase ${pkgfile})" || return 1 + local _pkgname="$(getpkgname ${pkgfile})" || return 1 + local _pkgver="$(getpkgver ${pkgfile})" || return 1 + local _pkgarch="$(getpkgarch ${pkgfile})" || return 1 local repo="${2}" in_array "${repo}" ${PKGREPOS[@]} || return 1 @@ -325,8 +318,7 @@ check_pkgsvn() { if [ ! -f "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}" ]; then mkdir -p "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}" arch_svn export -q "${SVNREPO}/${_pkgbase}/repos/${repo}-${_pkgarch}/PKGBUILD" \ - "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}" >/dev/null - [ $? -ge 1 ] && return 1 + "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}" >/dev/null || return 1 fi local svnver="$(. "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}"; get_full_version)" @@ -360,8 +352,7 @@ check_splitpkgs() { if [ ! -f "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}" ]; then mkdir -p "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}" arch_svn export -q "${SVNREPO}/${_pkgbase}/repos/${repo}-${_pkgarch}/PKGBUILD" \ - "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}" >/dev/null - [ $? -ge 1 ] && return 1 + "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}" >/dev/null || return 1 fi local svnnames=($(. "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}"; echo ${pkgname[@]})) @@ -386,12 +377,9 @@ check_splitpkgs() { check_pkgrepos() { local pkgfile=$1 - local pkgname="$(getpkgname ${pkgfile})" - [ $? -ge 1 ] && return 1 - local pkgver="$(getpkgver ${pkgfile})" - [ $? -ge 1 ] && return 1 - local pkgarch="$(getpkgarch ${pkgfile})" - [ $? -ge 1 ] && return 1 + local pkgname="$(getpkgname ${pkgfile})" || return 1 + local pkgver="$(getpkgver ${pkgfile})" || return 1 + local pkgarch="$(getpkgarch ${pkgfile})" || return 1 [ -f "${FTP_BASE}/${PKGPOOL}/${pkgname}-${pkgver}-${pkgarch}"${PKGEXT} ] && return 1 [ -f "${FTP_BASE}/${PKGPOOL}/${pkgname}-${pkgver}-${pkgarch}"${PKGEXT}.sig ] && return 1 diff --git a/db-update b/db-update index 54e60cb..f473f45 100755 --- a/db-update +++ b/db-update @@ -9,8 +9,7 @@ if [ $# -ge 1 ]; then fi # Find repos with packages to release -staging_repos=($(find "${STAGING}" -mindepth 1 -type f -name "*${PKGEXT}" -printf '%h\n' | sort -u)) -if [ $? -ge 1 ]; then +if ! staging_repos=($(find "${STAGING}" -mindepth 1 -type f -name "*${PKGEXT}" -printf '%h\n' | sort -u)); then die "Could not read ${STAGING}" fi -- cgit v1.2.2 From 5afac1ed83479c5ff7aab134b54245ec4f5b59fe Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Mon, 12 Feb 2018 17:53:10 -0500 Subject: Update messages to make full use of printf formatters libmakepkg messaging functions provide automatic access to gettext (which we do not currently make use of) in addition to cleanly separating data from message strings. In order for this to work properly, pass argv correctly from die() to libmakepkg's error() --- cron-jobs/ftpdir-cleanup | 4 ++-- cron-jobs/sourceballs | 4 ++-- cron-jobs/update-web-db | 4 ++-- db-functions | 48 ++++++++++++++++++++++++------------------------ db-move | 6 +++--- db-remove | 8 ++++---- db-repo-add | 6 +++--- db-repo-remove | 4 ++-- db-update | 28 ++++++++++++++-------------- testing2x | 4 ++-- 10 files changed, 58 insertions(+), 58 deletions(-) diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index 2683676..2f3d5aa 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -44,7 +44,7 @@ for repo in ${PKGREPOS[@]}; do missing_pkgs=($(comm -13 "${WORKDIR}/repo-${repo}-${arch}" "${WORKDIR}/db-${repo}-${arch}")) if [ ${#missing_pkgs[@]} -ge 1 ]; then - error "Missing packages in [${repo}] (${arch})..." + error "Missing packages in [%s] (%s)..." "$repo" "$arch" for missing_pkg in ${missing_pkgs[@]}; do msg2 "${missing_pkg}" done @@ -52,7 +52,7 @@ for repo in ${PKGREPOS[@]}; do old_pkgs=($(comm -23 "${WORKDIR}/repo-${repo}-${arch}" "${WORKDIR}/db-${repo}-${arch}")) if [ ${#old_pkgs[@]} -ge 1 ]; then - msg "Removing old packages from [${repo}] (${arch})..." + msg "Removing old packages from [%s] (%s)..." "$repo" "$arch" for old_pkg in ${old_pkgs[@]}; do msg2 "${old_pkg}" clean_pkg "${FTP_BASE}/${repo}/os/${arch}/${old_pkg}" diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 4126c37..e9c5586 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -103,13 +103,13 @@ for repo in ${PKGREPOS[@]}; do done < "${WORKDIR}/db-${repo}" if [ ${#newpkgs[@]} -ge 1 ]; then - msg "Adding source packages for [${repo}]..." + msg "Adding source packages for [%s]..." "$repo" for new_pkg in ${newpkgs[@]}; do msg2 "${new_pkg}" done fi if [ ${#failedpkgs[@]} -ge 1 ]; then - msg "Failed to create source packages for [${repo}]..." + msg "Failed to create source packages for [%s]..." "$repo" for failed_pkg in ${failedpkgs[@]}; do msg2 "${failed_pkg}" done diff --git a/cron-jobs/update-web-db b/cron-jobs/update-web-db index 195d1fc..23af067 100755 --- a/cron-jobs/update-web-db +++ b/cron-jobs/update-web-db @@ -16,7 +16,7 @@ LOGOUT="/tmp/archweb_update.log" # figure out what operation to perform cmd="${0##*/}" if [[ $cmd != "update-web-db" && $cmd != "update-web-files-db" ]]; then - die "Invalid command name '$cmd' specified!" + die "Invalid command name '%s' specified!" "$cmd" fi script_lock @@ -24,7 +24,7 @@ script_lock # run at nice 5. it can churn quite a bit of cpu after all. renice +5 -p $$ > /dev/null -echo "$cmd: Updating DB at $(date)" >> "${LOGOUT}" +echo "%s: Updating DB at %s" "$cmd" "$(date)" >> "${LOGOUT}" # source our virtualenv if it exists if [ -f "$ENVPATH" ]; then diff --git a/db-functions b/db-functions index f0bf43f..f8e058a 100644 --- a/db-functions +++ b/db-functions @@ -40,7 +40,7 @@ script_lock() { local LOCKDIR="$TMPDIR/.scriptlock.${0##*/}" if ! mkdir "$LOCKDIR" >/dev/null 2>&1 ; then local _owner="$(/usr/bin/stat -c %U $LOCKDIR)" - error "Script ${0##*/} is already locked by $_owner." + error "Script %s is already locked by %s." "${0##*/}" "$_owner" exit 1 else set_umask @@ -51,7 +51,7 @@ script_lock() { script_unlock() { local LOCKDIR="$TMPDIR/.scriptlock.${0##*/}" if [ ! -d "$LOCKDIR" ]; then - warning "Script ${0##*/} was not locked!" + warning "Script %s was not locked!" "${0##*/}" restore_umask return 1 else @@ -71,12 +71,12 @@ cleanup() { repo=${l%.*} arch=${l#*.} if [ -d "$TMPDIR/.repolock.$repo.$arch" ]; then - msg "Removing left over lock from [${repo}] (${arch})" + msg "Removing left over lock from [%s] (%s)" "$repo" "$arch" repo_unlock $repo $arch fi done if [ -d "$TMPDIR/.scriptlock.${0##*/}" ]; then - msg "Removing left over lock from ${0##*/}" + msg "Removing left over lock from %s" "${0##*/}" script_unlock fi rm -rf "$WORKDIR" @@ -94,7 +94,7 @@ abort() { } die() { - error "$*" + error "$@" cleanup 1 } @@ -114,7 +114,7 @@ repo_lock () { # This is the lock file used by repo-add and repo-remove if [ -f "${DBLOCKFILE}" ]; then - error "Repo [${1}] (${2}) is already locked by repo-{add,remove} process $(cat $DBLOCKFILE)" + error "Repo [%s] (%s) is already locked by repo-{add,remove} process %s" "$1" "$2" "$(cat $DBLOCKFILE)" return 1 fi @@ -131,8 +131,8 @@ repo_lock () { while [ $_count -le $_trial ] || $_lockblock ; do if ! mkdir "$LOCKDIR" >/dev/null 2>&1 ; then _owner="$(/usr/bin/stat -c %U $LOCKDIR)" - warning "Repo [${1}] (${2}) is already locked by $_owner. " - msg2 "Retrying in $LOCK_DELAY seconds..." + warning "Repo [%s] (%s) is already locked by %s." "$1" "$2" "$_owner" + msg2 "Retrying in %s seconds..." "$LOCK_DELAY" else LOCKS[${#LOCKS[*]}]="$1.$2" set_umask @@ -142,14 +142,14 @@ repo_lock () { let _count=$_count+1 done - error "Repo [${1}] (${2}) is already locked by $_owner. Giving up!" + error "Repo [%s] (%s) is already locked by %s. Giving up!" "$1" "$2" "$_owner" return 1 } repo_unlock () { #repo_unlock local LOCKDIR="$TMPDIR/.repolock.$1.$2" if [ ! -d "$LOCKDIR" ]; then - warning "Repo lock [${1}] (${2}) was not locked!" + warning "Repo lock [%s] (%s) was not locked!" "$1" "$2" restore_umask return 1 else @@ -204,7 +204,7 @@ getpkgname() { _name="$(_grep_pkginfo "$1" "pkgname")" if [ -z "$_name" ]; then - error "Package '$1' has no pkgname in the PKGINFO. Fail!" + error "Package '%s' has no pkgname in the PKGINFO. Fail!" "$1" exit 1 fi @@ -217,7 +217,7 @@ getpkgver() { _ver="$(_grep_pkginfo "$1" "pkgver")" if [ -z "$_ver" ]; then - error "Package '$1' has no pkgver in the PKGINFO. Fail!" + error "Package '%s' has no pkgver in the PKGINFO. Fail!" "$1" exit 1 fi @@ -229,7 +229,7 @@ getpkgarch() { _ver="$(_grep_pkginfo "$1" "arch")" if [ -z "$_ver" ]; then - error "Package '$1' has no arch in the PKGINFO. Fail!" + error "Package '%s' has no arch in the PKGINFO. Fail!" "$1" exit 1 fi @@ -259,10 +259,10 @@ getpkgfile() { error 'No canonical package found!' exit 1 elif [ ! -f "${1}" ]; then - error "Package ${1} not found!" + error "Package %s not found!" "$1" exit 1 elif [ ! -f "${1}.sig" ]; then - error "Package signature ${1}.sig not found!" + error "Package signature %s.sig not found!" "$1" exit 1 fi @@ -278,10 +278,10 @@ getpkgfiles() { for f in ${@}; do if [ ! -f "${f}" ]; then - error "Package ${f} not found!" + error "Package %s not found!" "$f" exit 1 elif [ ! -f "${f}.sig" ]; then - error "Package signature ${f}.sig not found!" + error "Package signature %s.sig not found!" "$f" exit 1 fi done @@ -392,7 +392,7 @@ check_pkgrepos() { #usage: chk_license ${license[@]}" chk_license() { local l - for l in ${@}; do + for l in "${@}"; do in_array ${l} ${ALLOWED_LICENSES[@]} && return 0 done @@ -428,12 +428,12 @@ set_repo_permission() { if [ -w "${dbfile}" ]; then local group=$(/usr/bin/stat --printf='%G' "$(dirname "${dbfile}")") - chgrp $group "${dbfile}" || error "Could not change group of ${dbfile} to $group" - chgrp $group "${filesfile}" || error "Could not change group of ${filesfile} to $group" - chmod g+w "${dbfile}" || error "Could not set write permission for group $group to ${dbfile}" - chmod g+w "${filesfile}" || error "Could not set write permission for group $group to ${filesfile}" + chgrp $group "${dbfile}" || error "Could not change group of %s to %s" "$dbfile" "$group" + chgrp $group "${filesfile}" || error "Could not change group of %s to %s" "$filesfile" "$group" + chmod g+w "${dbfile}" || error "Could not set write permission for group %s to %s" "$group" "$dbfile" + chmod g+w "${filesfile}" || error "Could not set write permission for group %s to %s" "$group" "$filesfile" else - error "You don't have permission to change ${dbfile}" + error "You don't have permission to change %s" "$dbfile" fi } @@ -459,7 +459,7 @@ arch_repo_remove() { local dbfile="${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" if [ ! -f "${dbfile}" ]; then - error "No database found at '${dbfile}'" + error "No database found at '%s'" "$dbfile" return 1 fi /usr/bin/repo-remove -q "${dbfile}" ${pkgs[@]} \ diff --git a/db-move b/db-move index ce3fcf9..e41e488 100755 --- a/db-move +++ b/db-move @@ -15,7 +15,7 @@ ftppath_from="${FTP_BASE}/${repo_from}/os/" ftppath_to="${FTP_BASE}/${repo_to}/os/" if ! check_repo_permission $repo_to || ! check_repo_permission $repo_from; then - die "You don't have permission to move packages from ${repo_from} to ${repo_to}" + die "You don't have permission to move packages from %s to %s" "$repo_from" "$repo_to" fi # TODO: this might lock too much (architectures) @@ -55,10 +55,10 @@ for pkgbase in ${args[@]:2}; do continue 2 fi done - die "${pkgbase} not found in ${repo_from}" + die "%s not found in %s" "$pkgbase" "$repo_from" done -msg "Moving packages from [${repo_from}] to [${repo_to}]..." +msg "Moving packages from [%s] to [%s]..." "$repo_from" "$repo_to" declare -A add_pkgs declare -A remove_pkgs diff --git a/db-remove b/db-remove index 25cb9a7..b1d6409 100755 --- a/db-remove +++ b/db-remove @@ -16,7 +16,7 @@ ftppath="$FTP_BASE/$repo/os" svnrepo="$repo-$arch" if ! check_repo_permission $repo; then - die "You don't have permission to remove packages from ${repo}" + die "You don't have permission to remove packages from %s" "$repo" fi if [ "$arch" == "any" ]; then @@ -31,7 +31,7 @@ done remove_pkgs=() for pkgbase in ${pkgbases[@]}; do - msg "Removing $pkgbase from [$repo]..." + msg "Removing %s from [%s]..." "$pkgbase" "$repo" arch_svn checkout -q "${SVNREPO}/${pkgbase}" "${WORKDIR}/svn/${pkgbase}" >/dev/null if [ -d "${WORKDIR}/svn/$pkgbase/repos/$svnrepo" ]; then @@ -39,8 +39,8 @@ for pkgbase in ${pkgbases[@]}; do arch_svn rm --force -q "${WORKDIR}/svn/$pkgbase/repos/$svnrepo" arch_svn commit -q "${WORKDIR}/svn/$pkgbase" -m "${0##*/}: $pkgbase removed by $(id -un)" else - warning "$pkgbase not found in $svnrepo" - warning "Removing only $pkgbase from the repo" + warning "%s not found in %s" "$pkgbase" "$svnrepo" + warning "Removing only %s from the repo" "$pkgbase" warning "If it was a split package you have to remove the others yourself!" remove_pkgs[${#remove_pkgs[*]}]=$pkgbase fi diff --git a/db-repo-add b/db-repo-add index 78f387f..f6a3d70 100755 --- a/db-repo-add +++ b/db-repo-add @@ -15,7 +15,7 @@ pkgfiles=(${@:3}) ftppath="$FTP_BASE/$repo/os" if ! check_repo_permission $repo; then - die "You don't have permission to add packages to ${repo}" + die "You don't have permission to add packages to %s" "$repo" fi if [ "$arch" == "any" ]; then @@ -31,9 +31,9 @@ done for tarch in ${tarches[@]}; do for pkgfile in ${pkgfiles[@]}; do if [[ ! -f "${FTP_BASE}/${repo}/os/${tarch}/${pkgfile##*/}" ]]; then - die "Package file ${pkgfile##*/} not found in ${FTP_BASE}/${repo}/os/${tarch}/" + die "Package file %s not found in %s" "${pkgfile##*/}" "${FTP_BASE}/${repo}/os/${tarch}/" else - msg "Adding $pkgfile to [$repo]..." + msg "Adding %s to [%s]..." "$pkgfile" "$repo" fi done arch_repo_add "${repo}" "${tarch}" ${pkgfiles[@]} diff --git a/db-repo-remove b/db-repo-remove index 2a693f4..2f08ebc 100755 --- a/db-repo-remove +++ b/db-repo-remove @@ -15,7 +15,7 @@ pkgnames=(${@:3}) ftppath="$FTP_BASE/$repo/os" if ! check_repo_permission $repo; then - die "You don't have permission to remove packages from ${repo}" + die "You don't have permission to remove packages from %s" "$repo" fi if [ "$arch" == "any" ]; then @@ -30,7 +30,7 @@ done for tarch in ${tarches[@]}; do for pkgname in ${pkgnames[@]}; do - msg "Removing $pkgname from [$repo]..." + msg "Removing %s from [%s]..." "$pkgname" "$repo" done arch_repo_remove "${repo}" "${tarch}" ${pkgnames[@]} repo_unlock $repo $tarch diff --git a/db-update b/db-update index f473f45..4640b05 100755 --- a/db-update +++ b/db-update @@ -4,13 +4,13 @@ . "$(dirname $0)/db-functions" if [ $# -ge 1 ]; then - warning "Calling ${0##*/} with a specific repository is no longer supported" + warning "Calling %s with a specific repository is no longer supported" "${0##*/}" exit 1 fi # Find repos with packages to release if ! staging_repos=($(find "${STAGING}" -mindepth 1 -type f -name "*${PKGEXT}" -printf '%h\n' | sort -u)); then - die "Could not read ${STAGING}" + die "Could not read %s" "$STAGING" fi repos=() @@ -30,46 +30,46 @@ done # check if packages are valid for repo in ${repos[@]}; do if ! check_repo_permission "${repo}"; then - die "You don't have permission to update packages in ${repo}" + die "You don't have permission to update packages in %s" "$repo" fi pkgs=($(getpkgfiles "${STAGING}/${repo}/"*${PKGEXT})) if [ $? -eq 0 ]; then for pkg in ${pkgs[@]}; do if [ -h "${pkg}" ]; then - die "Package ${repo}/${pkg##*/} is a symbolic link" + die "Package %s is a symbolic link" "$repo/${pkg##*/}" fi if ! check_pkgfile "${pkg}"; then - die "Package ${repo}/${pkg##*/} is not consistent with its meta data" + die "Package %s is not consistent with its meta data" "$repo/${pkg##*/}" fi if ! pacman-key -v "${pkg}.sig" >/dev/null 2>&1; then - die "Package ${repo}/${pkg##*/} does not have a valid signature" + die "Package %s does not have a valid signature" "$repo/${pkg##*/}" fi if ! check_pkgsvn "${pkg}" "${repo}"; then - die "Package ${repo}/${pkg##*/} is not consistent with svn repository" + die "Package %s is not consistent with svn repository" "$repo/${pkg##*/}" fi if ! check_pkgrepos "${pkg}"; then - die "Package ${repo}/${pkg##*/} already exists in another repository" + die "Package %s already exists in another repository" "$repo/${pkg##*/}" fi if ! check_packager "${pkg}"; then - die "Package ${repo}/${pkg##*/} does not have a valid packager" + die "Package %s does not have a valid packager" "$repo/${pkg##*/}" fi if ! check_buildinfo "${pkg}"; then - die "Package ${repo}/${pkg##*/} does not have a .BUILDINFO file" + die "Package %s does not have a .BUILDINFO file" "$repo/${pkg##*/}" fi if ! check_builddir "${pkg}"; then - die "Package ${repo}/${pkg##*/} was not built in a chroot" + die "Package %s was not built in a chroot" "$repo/${pkg##*/}" fi done if ! check_splitpkgs ${repo} ${pkgs[@]}; then - die "Missing split packages for ${repo}" + die "Missing split packages for %s" "$repo" fi else - die "Could not read ${STAGING}" + die "Could not read %s" "$STAGING" fi done for repo in ${repos[@]}; do - msg "Updating [${repo}]..." + msg "Updating [%s]..." "$repo" any_pkgs=($(getpkgfiles "${STAGING}/${repo}/"*-any${PKGEXT} 2>/dev/null)) for pkgarch in ${ARCHES[@]}; do add_pkgs=() diff --git a/testing2x b/testing2x index 369857f..f51c4c0 100755 --- a/testing2x +++ b/testing2x @@ -30,7 +30,7 @@ for pkgbase in $*; do break fi done - ${found_source} || die "${pkgbase} not found in [${TESTING_REPO}]" + ${found_source} || die "%s not found in [%s]" "$pkgbase" "$TESTING_REPO" found_target=false for pkgarch in ${ARCHES[@]} 'any'; do for repo in ${STABLE_REPOS[@]}; do @@ -42,7 +42,7 @@ for pkgbase in $*; do fi done done - ${found_target} || die "${pkgbase} not found in any of these repos: ${STABLE_REPOS[@]}" + ${found_target} || die "%s not found in any of these repos: %s" "$pkgbase" "${STABLE_REPOS[*]}" fi done -- cgit v1.2.2 From a7b497e8347fc964f8738d4dfc3c3ef23806fdc7 Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Mon, 12 Feb 2018 19:41:29 -0500 Subject: ARCHES is an array, do not attempt to call it as a string This meant only the first array element was checked in check_repo_permission(). Although arguably this should never cause real issues as something else would have to be broken if multiple architectures have different permissions, we should catch this now anyway. --- db-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db-functions b/db-functions index f8e058a..6d6beea 100644 --- a/db-functions +++ b/db-functions @@ -410,7 +410,7 @@ check_repo_permission() { [ -w "$FTP_BASE/${PKGPOOL}" ] || return 1 local arch - for arch in ${ARCHES}; do + for arch in ${ARCHES[@]}; do local dir="${FTP_BASE}/${repo}/os/${arch}/" [ -w "${dir}" ] || return 1 [ -f "${dir}"${repo}${DBEXT} -a ! -w "${dir}"${repo}${DBEXT} ] && return 1 -- cgit v1.2.2 From f4f9d1a0099c1f784c4a964e2b5651b56f629b82 Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Mon, 12 Feb 2018 20:39:30 -0500 Subject: Use modern bash to append to strings/arrays. Rather than using ugly hacks like arr[${#arr[*]}]="foo", bash 3.1 has the += operator. Update strings to use the same operator while we are at it. --- cron-jobs/sourceballs | 6 +++--- db-functions | 2 +- db-move | 2 +- db-remove | 4 ++-- db-update | 2 +- test/cases/db-repo-add.bats | 2 +- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index e9c5586..9ab4e98 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -82,7 +82,7 @@ for repo in ${PKGREPOS[@]}; do arch_svn export -q "${SVNREPO}/${pkgbase}/repos/${repo}-${pkgarch}" \ "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" >/dev/null 2>&1 if [ $? -ge 1 ]; then - failedpkgs[${#failedpkgs[*]}]="${pkgbase}-${pkgver}${SRCEXT}" + failedpkgs+=("${pkgbase}-${pkgver}${SRCEXT}") continue fi @@ -93,9 +93,9 @@ for repo in ${PKGREPOS[@]}; do mv_acl "${pkgbase}-${pkgver}${SRCEXT}" "${FTP_BASE}/${SRCPOOL}/${pkgbase}-${pkgver}${SRCEXT}" # Avoid creating the same source package for every arch echo "${pkgbase}-${pkgver}${SRCEXT}" >> "${WORKDIR}/available-src-pkgs" - newpkgs[${#newpkgs[*]}]="${pkgbase}-${pkgver}${SRCEXT}" + newpkgs+=("${pkgbase}-${pkgver}${SRCEXT}") else - failedpkgs[${#failedpkgs[*]}]="${pkgbase}-${pkgver}${SRCEXT}" + failedpkgs+=("${pkgbase}-${pkgver}${SRCEXT}") cat "${WORKDIR}/${pkgbase}.log" >> "${WORKDIR}/makepkg-fail.log" fi popd >/dev/null diff --git a/db-functions b/db-functions index 6d6beea..d66955b 100644 --- a/db-functions +++ b/db-functions @@ -134,7 +134,7 @@ repo_lock () { warning "Repo [%s] (%s) is already locked by %s." "$1" "$2" "$_owner" msg2 "Retrying in %s seconds..." "$LOCK_DELAY" else - LOCKS[${#LOCKS[*]}]="$1.$2" + LOCKS+=("$1.$2") set_umask return 0 fi diff --git a/db-move b/db-move index e41e488..71b62ea 100755 --- a/db-move +++ b/db-move @@ -91,7 +91,7 @@ for pkgbase in ${args[@]:2}; do arch_svn mv -q -r HEAD "${svnrepo_from}/$file@" "${svnrepo_to}/" done arch_svn rm --force -q "${svnrepo_from}" - tag_list="$tag_list, $pkgarch" + tag_list+=", $pkgarch" for pkgname in ${pkgnames[@]}; do for tarch in ${tarches[@]}; do diff --git a/db-remove b/db-remove index b1d6409..4f7a0f7 100755 --- a/db-remove +++ b/db-remove @@ -35,14 +35,14 @@ for pkgbase in ${pkgbases[@]}; do arch_svn checkout -q "${SVNREPO}/${pkgbase}" "${WORKDIR}/svn/${pkgbase}" >/dev/null if [ -d "${WORKDIR}/svn/$pkgbase/repos/$svnrepo" ]; then - remove_pkgs=(${remove_pkgs[@]} $(. "${WORKDIR}/svn/$pkgbase/repos/$svnrepo/PKGBUILD"; echo ${pkgname[@]})) + remove_pkgs+=($(. "${WORKDIR}/svn/$pkgbase/repos/$svnrepo/PKGBUILD"; echo ${pkgname[@]})) arch_svn rm --force -q "${WORKDIR}/svn/$pkgbase/repos/$svnrepo" arch_svn commit -q "${WORKDIR}/svn/$pkgbase" -m "${0##*/}: $pkgbase removed by $(id -un)" else warning "%s not found in %s" "$pkgbase" "$svnrepo" warning "Removing only %s from the repo" "$pkgbase" warning "If it was a split package you have to remove the others yourself!" - remove_pkgs[${#remove_pkgs[*]}]=$pkgbase + remove_pkgs+=("$pkgbase") fi done diff --git a/db-update b/db-update index 4640b05..57c0813 100755 --- a/db-update +++ b/db-update @@ -89,7 +89,7 @@ for repo in ${repos[@]}; do if [ -f "$FTP_BASE/${PKGPOOL}/${pkgfile}.sig" ]; then ln -s "../../../${PKGPOOL}/${pkgfile}.sig" "$FTP_BASE/$repo/os/${pkgarch}" fi - add_pkgs[${#add_pkgs[*]}]=${pkgfile} + add_pkgs+=("${pkgfile}") done if [ ${#add_pkgs[@]} -ge 1 ]; then arch_repo_add "${repo}" "${pkgarch}" ${add_pkgs[@]} diff --git a/test/cases/db-repo-add.bats b/test/cases/db-repo-add.bats index 4fd13e4..9fde381 100644 --- a/test/cases/db-repo-add.bats +++ b/test/cases/db-repo-add.bats @@ -54,7 +54,7 @@ __movePackageToRepo() { add_pkgs=() for pkgbase in ${pkgs[@]}; do __movePackageToRepo extra ${pkgbase} ${arch} - add_pkgs[${#add_pkgs[*]}]=${pkgbase}-1-1-${arch}.pkg.tar.xz + add_pkgs+=("${pkgbase}-1-1-${arch}.pkg.tar.xz") done db-repo-add extra ${arch} ${add_pkgs[@]} done -- cgit v1.2.2 From b61a7148eaf546a31597b74d9dd8948e4a39dea1 Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Mon, 12 Feb 2018 20:50:57 -0500 Subject: Use more bashisms Fix numerous instances of POSIX `[ ... ]`, including reliance on ugly deprecated constructs like POSIX `-a`. Since we require bash regardless, it makes sense to take full advantage of it. bash `[[ ... ]]` does not require quoting variables as the shell natively recognizes them as variables rather than expanded strings. Use shell arithmetic rather than test, when checking numerical values. --- config | 2 +- db-functions | 83 +++++++++++++++++++++++++++++----------------------------- db-move | 20 +++++++------- db-remove | 6 ++--- db-repo-add | 6 ++--- db-repo-remove | 4 +-- db-update | 14 +++++----- testing2x | 12 ++++----- 8 files changed, 73 insertions(+), 74 deletions(-) diff --git a/config b/config index 92cc4a5..90aa8c0 100644 --- a/config +++ b/config @@ -33,4 +33,4 @@ ALLOWED_LICENSES=('GPL' 'GPL1' 'GPL2' 'GPL3' 'LGPL' 'LGPL1' 'LGPL2' 'LGPL2.1' 'L # Override default config with config.local LOCAL_CONFIG=${DBSCRIPTS_CONFIG:-"$(dirname ${BASH_SOURCE[0]})/config.local"} -[ -f "${LOCAL_CONFIG}" ] && . "${LOCAL_CONFIG}" +[[ -f "${LOCAL_CONFIG}" ]] && . "${LOCAL_CONFIG}" diff --git a/db-functions b/db-functions index d66955b..c0af03c 100644 --- a/db-functions +++ b/db-functions @@ -8,8 +8,7 @@ CARCH="x86_64" # Useful functions UMASK="" set_umask () { - [ "$UMASK" == "" ] && UMASK="$(umask)" - export UMASK + export UMASK="${UMASK:-$(umask)}" umask 002 } @@ -28,7 +27,7 @@ mv_acl() { # set up general environment WORKDIR=$(mktemp -d "${TMPDIR}/${0##*/}.XXXXXXXXXX") -if [ -n "${SVNUSER}" ]; then +if [[ -n ${SVNUSER} ]]; then setfacl -m u:"${SVNUSER}":rwx "${WORKDIR}" setfacl -m d:u:"${USER}":rwx "${WORKDIR}" setfacl -m d:u:"${SVNUSER}":rwx "${WORKDIR}" @@ -50,7 +49,7 @@ script_lock() { script_unlock() { local LOCKDIR="$TMPDIR/.scriptlock.${0##*/}" - if [ ! -d "$LOCKDIR" ]; then + if [[ ! -d $LOCKDIR ]]; then warning "Script %s was not locked!" "${0##*/}" restore_umask return 1 @@ -70,12 +69,12 @@ cleanup() { for l in ${LOCKS[@]}; do repo=${l%.*} arch=${l#*.} - if [ -d "$TMPDIR/.repolock.$repo.$arch" ]; then + if [[ -d $TMPDIR/.repolock.$repo.$arch ]]; then msg "Removing left over lock from [%s] (%s)" "$repo" "$arch" repo_unlock $repo $arch fi done - if [ -d "$TMPDIR/.scriptlock.${0##*/}" ]; then + if [[ -d $TMPDIR/.scriptlock.${0##*/} ]]; then msg "Removing left over lock from %s" "${0##*/}" script_unlock fi @@ -85,7 +84,7 @@ cleanup() { date +%s > "${FTP_BASE}/lastupdate" fi - [ "$1" ] && exit $1 + [[ -n $1 ]] && exit $1 } abort() { @@ -113,22 +112,22 @@ repo_lock () { local _owner # This is the lock file used by repo-add and repo-remove - if [ -f "${DBLOCKFILE}" ]; then + if [[ -f ${DBLOCKFILE} ]]; then error "Repo [%s] (%s) is already locked by repo-{add,remove} process %s" "$1" "$2" "$(cat $DBLOCKFILE)" return 1 fi - if [ $# -eq 2 ]; then + if (( $# == 2 )); then _lockblock=true _trial=0 - elif [ $# -eq 3 ]; then + elif (( $# == 3 )); then _lockblock=false _timeout=$3 let _trial=$_timeout/$LOCK_DELAY fi _count=0 - while [ $_count -le $_trial ] || $_lockblock ; do + while (( _count <= _trial )) || $_lockblock ; do if ! mkdir "$LOCKDIR" >/dev/null 2>&1 ; then _owner="$(/usr/bin/stat -c %U $LOCKDIR)" warning "Repo [%s] (%s) is already locked by %s." "$1" "$2" "$_owner" @@ -148,7 +147,7 @@ repo_lock () { repo_unlock () { #repo_unlock local LOCKDIR="$TMPDIR/.repolock.$1.$2" - if [ ! -d "$LOCKDIR" ]; then + if [[ ! -d $LOCKDIR ]]; then warning "Repo lock [%s] (%s) was not locked!" "$1" "$2" restore_umask return 1 @@ -180,7 +179,7 @@ getpkgbase() { local _base _base="$(_grep_pkginfo "$1" "pkgbase")" - if [ -z "$_base" ]; then + if [[ -z $_base ]]; then getpkgname "$1" else echo "$_base" @@ -191,7 +190,7 @@ issplitpkg() { local _base _base="$(_grep_pkginfo "$1" "pkgbase")" - if [ -z "$_base" ]; then + if [[ -z $_base ]]; then return 1 else return 0 @@ -203,7 +202,7 @@ getpkgname() { local _name _name="$(_grep_pkginfo "$1" "pkgname")" - if [ -z "$_name" ]; then + if [[ -z $_name ]]; then error "Package '%s' has no pkgname in the PKGINFO. Fail!" "$1" exit 1 fi @@ -216,7 +215,7 @@ getpkgver() { local _ver _ver="$(_grep_pkginfo "$1" "pkgver")" - if [ -z "$_ver" ]; then + if [[ -z $_ver ]]; then error "Package '%s' has no pkgver in the PKGINFO. Fail!" "$1" exit 1 fi @@ -228,7 +227,7 @@ getpkgarch() { local _ver _ver="$(_grep_pkginfo "$1" "arch")" - if [ -z "$_ver" ]; then + if [[ -z $_ver ]]; then error "Package '%s' has no arch in the PKGINFO. Fail!" "$1" exit 1 fi @@ -240,7 +239,7 @@ check_packager() { local _packager _packager=$(_grep_pkginfo "$1" "packager") - [[ $_packager && $_packager != 'Unknown Packager' ]] + [[ -n $_packager && $_packager != 'Unknown Packager' ]] } check_buildinfo() { @@ -251,17 +250,17 @@ check_builddir() { local _builddir _builddir=$(_grep_buildinfo "$1" "builddir") - [[ $_builddir && $_builddir = '/build' ]] + [[ -n $_builddir && $_builddir = '/build' ]] } getpkgfile() { - if [[ ${#} -ne 1 ]]; then + if (( $# != 1 )); then error 'No canonical package found!' exit 1 - elif [ ! -f "${1}" ]; then + elif [[ ! -f ${1} ]]; then error "Package %s not found!" "$1" exit 1 - elif [ ! -f "${1}.sig" ]; then + elif [[ ! -f ${1}.sig ]]; then error "Package signature %s.sig not found!" "$1" exit 1 fi @@ -270,8 +269,8 @@ getpkgfile() { } getpkgfiles() { - local f - if [ ! -z "$(echo ${@%\.*} | sed "s/ /\n/g" | sort | uniq -D)" ]; then + local f files + if [[ ! -z "$(echo ${@%\.*} | sed "s/ /\n/g" | sort | uniq -D)" ]]; then error 'Duplicate packages found!' exit 1 fi @@ -315,14 +314,14 @@ check_pkgsvn() { in_array "${repo}" ${PKGREPOS[@]} || return 1 - if [ ! -f "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}" ]; then + if [[ ! -f ${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase} ]]; then mkdir -p "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}" arch_svn export -q "${SVNREPO}/${_pkgbase}/repos/${repo}-${_pkgarch}/PKGBUILD" \ "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}" >/dev/null || return 1 fi local svnver="$(. "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}"; get_full_version)" - [ "${svnver}" == "${_pkgver}" ] || return 1 + [[ "${svnver}" = "${_pkgver}" ]] || return 1 local svnnames=($(. "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}"; echo ${pkgname[@]})) in_array "${_pkgname}" ${svnnames[@]} || return 1 @@ -349,7 +348,7 @@ check_splitpkgs() { mkdir -p "${repo}/${_pkgarch}/${_pkgbase}" echo "${_pkgname}" >> "${repo}/${_pkgarch}/${_pkgbase}/staging" - if [ ! -f "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}" ]; then + if [[ ! -f ${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase} ]]; then mkdir -p "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}" arch_svn export -q "${SVNREPO}/${_pkgbase}/repos/${repo}-${_pkgarch}/PKGBUILD" \ "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}" >/dev/null || return 1 @@ -363,10 +362,10 @@ check_splitpkgs() { popd >/dev/null for pkgdir in "${WORKDIR}/check_splitpkgs/${repo}"/*/*; do - [ ! -d "${pkgdir}" ] && continue + [[ ! -d ${pkgdir} ]] && continue sort -u "${pkgdir}/staging" -o "${pkgdir}/staging" sort -u "${pkgdir}/svn" -o "${pkgdir}/svn" - if [ ! -z "$(comm -13 "${pkgdir}/staging" "${pkgdir}/svn")" ]; then + if [[ ! -z "$(comm -13 "${pkgdir}/staging" "${pkgdir}/svn")" ]]; then return 1 fi done @@ -381,10 +380,10 @@ check_pkgrepos() { local pkgver="$(getpkgver ${pkgfile})" || return 1 local pkgarch="$(getpkgarch ${pkgfile})" || return 1 - [ -f "${FTP_BASE}/${PKGPOOL}/${pkgname}-${pkgver}-${pkgarch}"${PKGEXT} ] && return 1 - [ -f "${FTP_BASE}/${PKGPOOL}/${pkgname}-${pkgver}-${pkgarch}"${PKGEXT}.sig ] && return 1 - [ -f "${FTP_BASE}/${PKGPOOL}/${pkgfile##*/}" ] && return 1 - [ -f "${FTP_BASE}/${PKGPOOL}/${pkgfile##*/}.sig" ] && return 1 + [[ -f ${FTP_BASE}/${PKGPOOL}/${pkgname}-${pkgver}-${pkgarch}${PKGEXT} ]] && return 1 + [[ -f ${FTP_BASE}/${PKGPOOL}/${pkgname}-${pkgver}-${pkgarch}${PKGEXT}.sig ]] && return 1 + [[ -f ${FTP_BASE}/${PKGPOOL}/${pkgfile##*/} ]] && return 1 + [[ -f ${FTP_BASE}/${PKGPOOL}/${pkgfile##*/}.sig ]] && return 1 return 0 } @@ -402,19 +401,19 @@ chk_license() { check_repo_permission() { local repo=$1 - [ ${#PKGREPOS[@]} -eq 0 ] && return 1 - [ -z "${PKGPOOL}" ] && return 1 + (( ${#PKGREPOS[@]} == 0 )) && return 1 + [[ -z "${PKGPOOL}" ]] && return 1 in_array "${repo}" ${PKGREPOS[@]} || return 1 - [ -w "$FTP_BASE/${PKGPOOL}" ] || return 1 + [[ -w $FTP_BASE/${PKGPOOL} ]] || return 1 local arch for arch in ${ARCHES[@]}; do local dir="${FTP_BASE}/${repo}/os/${arch}/" - [ -w "${dir}" ] || return 1 - [ -f "${dir}"${repo}${DBEXT} -a ! -w "${dir}"${repo}${DBEXT} ] && return 1 - [ -f "${dir}"${repo}${FILESEXT} -a ! -w "${dir}"${repo}${FILESEXT} ] && return 1 + [[ -w ${dir} ]] || return 1 + [[ -f ${dir}${repo}${DBEXT} && ! -w ${dir}${repo}${DBEXT} ]] && return 1 + [[ -f ${dir}${repo}${FILESEXT} && ! -w ${dir}${repo}${FILESEXT} ]] && return 1 done return 0 @@ -426,7 +425,7 @@ set_repo_permission() { local dbfile="${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" local filesfile="${FTP_BASE}/${repo}/os/${arch}/${repo}${FILESEXT}" - if [ -w "${dbfile}" ]; then + if [[ -w ${dbfile} ]]; then local group=$(/usr/bin/stat --printf='%G' "$(dirname "${dbfile}")") chgrp $group "${dbfile}" || error "Could not change group of %s to %s" "$dbfile" "$group" chgrp $group "${filesfile}" || error "Could not change group of %s to %s" "$filesfile" "$group" @@ -458,7 +457,7 @@ arch_repo_remove() { local pkgs=(${@:3}) local dbfile="${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" - if [ ! -f "${dbfile}" ]; then + if [[ ! -f ${dbfile} ]]; then error "No database found at '%s'" "$dbfile" return 1 fi @@ -470,7 +469,7 @@ arch_repo_remove() { } arch_svn() { - if [ -z "${SVNUSER}" ]; then + if [[ -z ${SVNUSER} ]]; then /usr/bin/svn "${@}" else sudo -u "${SVNUSER}" -- /usr/bin/svn --username "${USER}" "${@}" diff --git a/db-move b/db-move index 71b62ea..37a9884 100755 --- a/db-move +++ b/db-move @@ -3,7 +3,7 @@ . "$(dirname $0)/config" . "$(dirname $0)/db-functions" -if [ $# -lt 3 ]; then +if (( $# < 3 )); then msg "usage: ${0##*/} ..." exit 1 fi @@ -30,18 +30,18 @@ for pkgbase in ${args[@]:2}; do arch_svn up -q "${WORKDIR}/svn/${pkgbase}" >/dev/null for pkgarch in ${ARCHES[@]} 'any'; do svnrepo_from="${WORKDIR}/svn/${pkgbase}/repos/${repo_from}-${pkgarch}" - if [ -r "${svnrepo_from}/PKGBUILD" ]; then + if [[ -r ${svnrepo_from}/PKGBUILD ]]; then pkgnames=($(. "${svnrepo_from}/PKGBUILD"; echo ${pkgname[@]})) - if [ ${#pkgnames[@]} -lt 1 ]; then + if (( ${#pkgnames[@]} < 1 )); then die "Could not read pkgname" fi pkgver=$(. "${svnrepo_from}/PKGBUILD"; get_full_version) - if [ -z "${pkgver}" ]; then + if [[ -z ${pkgver} ]]; then die "Could not read pkgver" fi - if [ "${pkgarch}" == 'any' ]; then + if [[ ${pkgarch} = any ]]; then tarches=(${ARCHES[@]}) else tarches=("${pkgarch}") @@ -68,8 +68,8 @@ for pkgbase in ${args[@]:2}; do svnrepo_from="${WORKDIR}/svn/${pkgbase}/repos/${repo_from}-${pkgarch}" svnrepo_to="${WORKDIR}/svn/${pkgbase}/repos/${repo_to}-${pkgarch}" - if [ -f "${svnrepo_from}/PKGBUILD" ]; then - if [ "${pkgarch}" == 'any' ]; then + if [[ -f ${svnrepo_from}/PKGBUILD ]]; then + if [[ ${pkgarch} = any ]]; then tarches=(${ARCHES[@]}) else tarches=("${pkgarch}") @@ -78,7 +78,7 @@ for pkgbase in ${args[@]:2}; do pkgnames=($(. "${svnrepo_from}/PKGBUILD"; echo ${pkgname[@]})) pkgver=$(. "${svnrepo_from}/PKGBUILD"; get_full_version) - if [ -d "${svnrepo_to}" ]; then + if [[ -d ${svnrepo_to} ]]; then for file in $(arch_svn ls "${svnrepo_to}"); do arch_svn rm -q "${svnrepo_to}/$file@" done @@ -99,7 +99,7 @@ for pkgbase in ${args[@]:2}; do pkgfile="${pkgpath##*/}" ln -s "../../../${PKGPOOL}/${pkgfile}" ${ftppath_to}/${tarch}/ - if [ -f ${FTP_BASE}/${PKGPOOL}/${pkgfile}.sig ]; then + if [[ -f ${FTP_BASE}/${PKGPOOL}/${pkgfile}.sig ]]; then ln -s "../../../${PKGPOOL}/${pkgfile}.sig" ${ftppath_to}/${tarch}/ fi add_pkgs[${tarch}]+="${FTP_BASE}/${PKGPOOL}/${pkgfile} " @@ -113,7 +113,7 @@ for pkgbase in ${args[@]:2}; do done for tarch in ${ARCHES[@]}; do - if [ -n "${add_pkgs[${tarch}]}" ]; then + if [[ -n ${add_pkgs[${tarch}]} ]]; then arch_repo_add "${repo_to}" "${tarch}" ${add_pkgs[${tarch}]} arch_repo_remove "${repo_from}" "${tarch}" ${remove_pkgs[${tarch}]} fi diff --git a/db-remove b/db-remove index 4f7a0f7..70502bc 100755 --- a/db-remove +++ b/db-remove @@ -3,7 +3,7 @@ . "$(dirname $0)/config" . "$(dirname $0)/db-functions" -if [ $# -lt 3 ]; then +if (( $# < 3 )); then msg "usage: ${0##*/} ..." exit 1 fi @@ -19,7 +19,7 @@ if ! check_repo_permission $repo; then die "You don't have permission to remove packages from %s" "$repo" fi -if [ "$arch" == "any" ]; then +if [[ $arch = any ]]; then tarches=(${ARCHES[@]}) else tarches=("$arch") @@ -34,7 +34,7 @@ for pkgbase in ${pkgbases[@]}; do msg "Removing %s from [%s]..." "$pkgbase" "$repo" arch_svn checkout -q "${SVNREPO}/${pkgbase}" "${WORKDIR}/svn/${pkgbase}" >/dev/null - if [ -d "${WORKDIR}/svn/$pkgbase/repos/$svnrepo" ]; then + if [[ -d ${WORKDIR}/svn/$pkgbase/repos/$svnrepo ]]; then remove_pkgs+=($(. "${WORKDIR}/svn/$pkgbase/repos/$svnrepo/PKGBUILD"; echo ${pkgname[@]})) arch_svn rm --force -q "${WORKDIR}/svn/$pkgbase/repos/$svnrepo" arch_svn commit -q "${WORKDIR}/svn/$pkgbase" -m "${0##*/}: $pkgbase removed by $(id -un)" diff --git a/db-repo-add b/db-repo-add index f6a3d70..d7be302 100755 --- a/db-repo-add +++ b/db-repo-add @@ -3,7 +3,7 @@ . "$(dirname $0)/config" . "$(dirname $0)/db-functions" -if [ $# -lt 3 ]; then +if (( $# < 3 )); then msg "usage: ${0##*/} ..." exit 1 fi @@ -18,7 +18,7 @@ if ! check_repo_permission $repo; then die "You don't have permission to add packages to %s" "$repo" fi -if [ "$arch" == "any" ]; then +if [[ $arch = any ]]; then tarches=(${ARCHES[@]}) else tarches=("$arch") @@ -30,7 +30,7 @@ done for tarch in ${tarches[@]}; do for pkgfile in ${pkgfiles[@]}; do - if [[ ! -f "${FTP_BASE}/${repo}/os/${tarch}/${pkgfile##*/}" ]]; then + if [[ ! -f ${FTP_BASE}/${repo}/os/${tarch}/${pkgfile##*/} ]]; then die "Package file %s not found in %s" "${pkgfile##*/}" "${FTP_BASE}/${repo}/os/${tarch}/" else msg "Adding %s to [%s]..." "$pkgfile" "$repo" diff --git a/db-repo-remove b/db-repo-remove index 2f08ebc..32d167e 100755 --- a/db-repo-remove +++ b/db-repo-remove @@ -3,7 +3,7 @@ . "$(dirname $0)/config" . "$(dirname $0)/db-functions" -if [ $# -lt 3 ]; then +if (( $# < 3 )); then msg "usage: ${0##*/} ..." exit 1 fi @@ -18,7 +18,7 @@ if ! check_repo_permission $repo; then die "You don't have permission to remove packages from %s" "$repo" fi -if [ "$arch" == "any" ]; then +if [[ $arch = any ]]; then tarches=(${ARCHES[@]}) else tarches=("$arch") diff --git a/db-update b/db-update index 57c0813..45755a4 100755 --- a/db-update +++ b/db-update @@ -3,7 +3,7 @@ . "$(dirname $0)/config" . "$(dirname $0)/db-functions" -if [ $# -ge 1 ]; then +if (( $# >= 1 )); then warning "Calling %s with a specific repository is no longer supported" "${0##*/}" exit 1 fi @@ -33,9 +33,9 @@ for repo in ${repos[@]}; do die "You don't have permission to update packages in %s" "$repo" fi pkgs=($(getpkgfiles "${STAGING}/${repo}/"*${PKGEXT})) - if [ $? -eq 0 ]; then + if (( $? == 0 )); then for pkg in ${pkgs[@]}; do - if [ -h "${pkg}" ]; then + if [[ -h ${pkg} ]]; then die "Package %s is a symbolic link" "$repo/${pkg##*/}" fi if ! check_pkgfile "${pkg}"; then @@ -78,20 +78,20 @@ for repo in ${repos[@]}; do pkgfile="${pkg##*/}" msg2 "${pkgfile} (${pkgarch})" # any packages might have been moved by the previous run - if [ -f "${pkg}" ]; then + if [[ -f ${pkg} ]]; then mv "${pkg}" "$FTP_BASE/${PKGPOOL}" fi ln -s "../../../${PKGPOOL}/${pkgfile}" "$FTP_BASE/$repo/os/${pkgarch}" # also move signatures - if [ -f "${pkg}.sig" ]; then + if [[ -f ${pkg}.sig ]]; then mv "${pkg}.sig" "$FTP_BASE/${PKGPOOL}" fi - if [ -f "$FTP_BASE/${PKGPOOL}/${pkgfile}.sig" ]; then + if [[ -f $FTP_BASE/${PKGPOOL}/${pkgfile}.sig ]]; then ln -s "../../../${PKGPOOL}/${pkgfile}.sig" "$FTP_BASE/$repo/os/${pkgarch}" fi add_pkgs+=("${pkgfile}") done - if [ ${#add_pkgs[@]} -ge 1 ]; then + if (( ${#add_pkgs[@]} >= 1 )); then arch_repo_add "${repo}" "${pkgarch}" ${add_pkgs[@]} fi done diff --git a/testing2x b/testing2x index f51c4c0..f0d77cb 100755 --- a/testing2x +++ b/testing2x @@ -3,7 +3,7 @@ . "$(dirname $0)/config" . "$(dirname $0)/db-functions" -if [ $# -lt 1 ]; then +if (( $# < 1 )); then msg "usage: ${0##*/} ..." exit 1 fi @@ -19,13 +19,13 @@ done declare -A pkgs for pkgbase in $*; do - if [ ! -d "${WORKDIR}/${pkgbase}" ]; then + if [[ ! -d ${WORKDIR}/${pkgbase} ]]; then arch_svn export -q "${SVNREPO}/${pkgbase}/repos" "${WORKDIR}/${pkgbase}" >/dev/null found_source=false for pkgarch in ${ARCHES[@]} 'any'; do svnrepo_from="${WORKDIR}/${pkgbase}/${TESTING_REPO}-${pkgarch}" - if [ -r "${svnrepo_from}/PKGBUILD" ]; then + if [[ -r ${svnrepo_from}/PKGBUILD ]]; then found_source=true break fi @@ -35,7 +35,7 @@ for pkgbase in $*; do for pkgarch in ${ARCHES[@]} 'any'; do for repo in ${STABLE_REPOS[@]}; do svnrepo_to="${WORKDIR}/${pkgbase}/${repo}-${pkgarch}" - if [ -r "${svnrepo_to}/PKGBUILD" ]; then + if [[ -r ${svnrepo_to}/PKGBUILD ]]; then found_target=true pkgs[${repo}]+="${pkgbase} " break 2 @@ -49,11 +49,11 @@ done for pkgarch in ${ARCHES[@]}; do repo_unlock ${TESTING_REPO} ${pkgarch} done -for repo in ${STABLE_REPOS[@]}; do +for repo in ${STABLE_REPOS[@]}; do for pkgarch in ${ARCHES[@]}; do repo_unlock ${repo} ${pkgarch} done - if [ -n "${pkgs[${repo}]}" ]; then + if [[ -n ${pkgs[${repo}]} ]]; then "$(dirname $0)/db-move" ${TESTING_REPO} "${repo}" ${pkgs[${repo}]} fi done -- cgit v1.2.2 From 36b71d3231aca071ad635b995342b786676ef8fe Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Mon, 12 Feb 2018 20:53:06 -0500 Subject: db-functions: deduplicate some repeated logic reuse getpkgfile() in getpkgfiles() --- db-functions | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/db-functions b/db-functions index c0af03c..e8949d7 100644 --- a/db-functions +++ b/db-functions @@ -275,17 +275,11 @@ getpkgfiles() { exit 1 fi - for f in ${@}; do - if [ ! -f "${f}" ]; then - error "Package %s not found!" "$f" - exit 1 - elif [ ! -f "${f}.sig" ]; then - error "Package signature %s.sig not found!" "$f" - exit 1 - fi + for f in "$@"; do + files+=("$(getpkgfile "$f")") || exit 1 done - echo ${@} + echo ${files[@]} } check_pkgfile() { -- cgit v1.2.2 From 883005f5450ed6a278b5cef46be1245783840b29 Mon Sep 17 00:00:00 2001 From: Florian Pritz Date: Thu, 15 Feb 2018 16:01:07 +0100 Subject: Remove i686 from config We dropped it long ago and this should be reflected here rather than being changed on our servers only. Signed-off-by: Florian Pritz --- config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config b/config index 90aa8c0..d2c1942 100644 --- a/config +++ b/config @@ -22,7 +22,7 @@ LOCK_TIMEOUT=300 STAGING="$HOME/staging" TMPDIR="/var/tmp" -ARCHES=(i686 x86_64) +ARCHES=(x86_64) DBEXT=".db.tar.gz" FILESEXT=".files.tar.gz" PKGEXT=".pkg.tar.?z" -- cgit v1.2.2 From 24671e6cb2fa343fd6f15fe54b16840aed091ad4 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Thu, 15 Feb 2018 17:04:29 +0100 Subject: PR-2: Allow i686 for tests --- test/lib/common.bash | 1 + 1 file changed, 1 insertion(+) diff --git a/test/lib/common.bash b/test/lib/common.bash index 7589f51..540e403 100644 --- a/test/lib/common.bash +++ b/test/lib/common.bash @@ -98,6 +98,7 @@ setup() { SOURCE_CLEANUP_DESTDIR="${TMP}/source-cleanup" STAGING="${TMP}/staging" TMPDIR="${TMP}/tmp" + ARCHES=(x86_64 i686) CLEANUP_DRYRUN=false SOURCE_CLEANUP_DRYRUN=false eot -- cgit v1.2.2 From c7f1d85c0e11aaf3c9acd944388f664aea40037b Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Thu, 15 Feb 2018 17:07:02 +0100 Subject: PR-2: Default to x86_64 for check_packages --- cron-jobs/check_archlinux/check_packages.py | 8 ++++---- cron-jobs/check_archlinux/parse_pkgbuilds.sh | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cron-jobs/check_archlinux/check_packages.py b/cron-jobs/check_archlinux/check_packages.py index d233bf6..a77b7ba 100755 --- a/cron-jobs/check_archlinux/check_packages.py +++ b/cron-jobs/check_archlinux/check_packages.py @@ -366,15 +366,15 @@ def print_usage(): print "Options:" print " --abs-tree= Check the specified tree(s) (default : /var/abs)" print " --repos= Check the specified repos (default : core,extra)" - print " --arch= Check the specified arch (default : i686)" + print " --arch= Check the specified arch (default : x86_64)" print " --repo-dir= Check the dbs at the specified path (default : /srv/ftp)" print " -h, --help Show this help and exit" print "" print "Examples:" print "\n Check core and extra in existing abs tree:" - print " ./check_packages.py --abs-tree=/var/abs --repos=core,extra --arch=i686" + print " ./check_packages.py --abs-tree=/var/abs --repos=core,extra --arch=x86_64" print "\n Check community:" - print " ./check_packages.py --abs-tree=/var/abs --repos=community --arch=i686" + print " ./check_packages.py --abs-tree=/var/abs --repos=community --arch=x86_64" print "" if __name__ == "__main__": @@ -383,7 +383,7 @@ if __name__ == "__main__": ## Default list of repos to check repos = ['core', 'extra'] ## Default arch - arch = "i686" + arch = "x86_64" ## Default repodir repodir = "/srv/ftp" diff --git a/cron-jobs/check_archlinux/parse_pkgbuilds.sh b/cron-jobs/check_archlinux/parse_pkgbuilds.sh index 3f92169..10457fe 100755 --- a/cron-jobs/check_archlinux/parse_pkgbuilds.sh +++ b/cron-jobs/check_archlinux/parse_pkgbuilds.sh @@ -1,7 +1,7 @@ #!/bin/bash # Usage : parse_pkgbuilds.sh arch -# Example : parse_pkgbuilds.sh i686 /var/abs/core /var/abs/extra +# Example : parse_pkgbuilds.sh x86_64 /var/abs/core /var/abs/extra exit() { return; } -- cgit v1.2.2 From 134aff74d16331231ed94f34d8417d1325d4600f Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Thu, 15 Feb 2018 17:29:57 +0100 Subject: Add usage tests --- test/cases/common.bats | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 test/cases/common.bats diff --git a/test/cases/common.bats b/test/cases/common.bats new file mode 100644 index 0000000..4b798f8 --- /dev/null +++ b/test/cases/common.bats @@ -0,0 +1,10 @@ +load ../lib/common + +@test "commands display usage message by default" { + for cmd in db-move db-remove db-repo-add db-repo-remove testing2x; do + echo Testing $cmd + run $cmd + (( $status == 1 )) + [[ $output == *'usage: '* ]] + done +} -- cgit v1.2.2 From b45650c3e3870b975620dd89bb6aa77585c8ff02 Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Mon, 19 Feb 2018 11:42:22 -0500 Subject: Use even more bashisms. Catch some cases that were missed in the previous run. --- cron-jobs/devlist-mailer | 6 +++--- cron-jobs/ftpdir-cleanup | 14 +++++++------- cron-jobs/integrity-check | 2 +- cron-jobs/sourceballs | 12 ++++++------ cron-jobs/update-web-db | 6 +++--- 5 files changed, 20 insertions(+), 20 deletions(-) diff --git a/cron-jobs/devlist-mailer b/cron-jobs/devlist-mailer index ca2e46b..65a5483 100755 --- a/cron-jobs/devlist-mailer +++ b/cron-jobs/devlist-mailer @@ -7,17 +7,17 @@ LIST="arch-dev-public@archlinux.org" FROM="repomaint@archlinux.org" SUBJECT="Repository Maintenance $(date +"%d-%m-%Y")" -if [ $# -ge 1 ]; then +if (( $# >= 1 )); then SUBJECT="$1 $(date +"%d-%m-%Y")" fi -if [ $# -ge 2 ]; then +if (( $# >= 2 )); then LIST="$2" fi stdin="$(cat)" #echo used to strip whitespace for checking for actual data -if [ -n "$(echo $stdin)" ]; then +if [[ -n "$(echo $stdin)" ]]; then echo "Subject: $SUBJECT To: $LIST diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index 2f3d5aa..c771950 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -9,11 +9,11 @@ clean_pkg() { if ! ${CLEANUP_DRYRUN}; then for pkg in "$@"; do - if [ -h "$pkg" ]; then + if [[ -h $pkg ]]; then rm -f "$pkg" "$pkg.sig" else mv_acl "$pkg" "$CLEANUP_DESTDIR/${pkg##*/}" - if [ -e "$pkg.sig" ]; then + if [[ -e $pkg.sig ]]; then mv_acl "$pkg.sig" "$CLEANUP_DESTDIR/${pkg##*/}.sig" fi touch "${CLEANUP_DESTDIR}/${pkg##*/}" @@ -34,7 +34,7 @@ ${CLEANUP_DRYRUN} && warning 'dry run mode is active' for repo in ${PKGREPOS[@]}; do for arch in ${ARCHES[@]}; do - if [ ! -f "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" ]; then + if [[ ! -f ${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT} ]]; then continue fi # get a list of actual available package files @@ -43,7 +43,7 @@ for repo in ${PKGREPOS[@]}; do bsdtar -xOf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" | awk '/^%FILENAME%/{getline;print}' | sort > "${WORKDIR}/db-${repo}-${arch}" missing_pkgs=($(comm -13 "${WORKDIR}/repo-${repo}-${arch}" "${WORKDIR}/db-${repo}-${arch}")) - if [ ${#missing_pkgs[@]} -ge 1 ]; then + if (( ${#missing_pkgs[@]} >= 1 )); then error "Missing packages in [%s] (%s)..." "$repo" "$arch" for missing_pkg in ${missing_pkgs[@]}; do msg2 "${missing_pkg}" @@ -51,7 +51,7 @@ for repo in ${PKGREPOS[@]}; do fi old_pkgs=($(comm -23 "${WORKDIR}/repo-${repo}-${arch}" "${WORKDIR}/db-${repo}-${arch}")) - if [ ${#old_pkgs[@]} -ge 1 ]; then + if (( ${#old_pkgs[@]} >= 1 )); then msg "Removing old packages from [%s] (%s)..." "$repo" "$arch" for old_pkg in ${old_pkgs[@]}; do msg2 "${old_pkg}" @@ -67,7 +67,7 @@ find "$FTP_BASE/${PKGPOOL}" -name "*${PKGEXT}" -printf '%f\n' | sort > "${WORKDI find "${WORKDIR}" -maxdepth 1 -type f -name 'db-*' -exec cat {} \; | sort -u > "${WORKDIR}/db" old_pkgs=($(comm -23 "${WORKDIR}/pool" "${WORKDIR}/db")) -if [ ${#old_pkgs[@]} -ge 1 ]; then +if (( ${#old_pkgs[@]} >= 1 )); then msg "Removing old packages from package pool..." for old_pkg in ${old_pkgs[@]}; do msg2 "${old_pkg}" @@ -76,7 +76,7 @@ if [ ${#old_pkgs[@]} -ge 1 ]; then fi old_pkgs=($(find ${CLEANUP_DESTDIR} -type f -name "*${PKGEXT}" -mtime +${CLEANUP_KEEP} -printf '%f\n')) -if [ ${#old_pkgs[@]} -ge 1 ]; then +if (( ${#old_pkgs[@]} >= 1 )); then msg "Removing old packages from the cleanup directory..." for old_pkg in ${old_pkgs[@]}; do msg2 "${old_pkg}" diff --git a/cron-jobs/integrity-check b/cron-jobs/integrity-check index 211a24b..f1e75ab 100755 --- a/cron-jobs/integrity-check +++ b/cron-jobs/integrity-check @@ -7,7 +7,7 @@ dirname="$(dirname $0)" script_lock -if [ $# -ne 1 ]; then +if (( $# != 1 )); then die "usage: ${0##*/} " fi mailto=$1 diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 9ab4e98..25a8abb 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -21,7 +21,7 @@ renice +10 -p $$ > /dev/null for repo in ${PKGREPOS[@]}; do for arch in ${ARCHES[@]}; do # Repo does not exist; skip it - if [ ! -f "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" ]; then + if [[ ! -f ${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT} ]]; then continue fi bsdtar -xOf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" \ @@ -81,7 +81,7 @@ for repo in ${PKGREPOS[@]}; do mkdir -p -m0770 "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}" arch_svn export -q "${SVNREPO}/${pkgbase}/repos/${repo}-${pkgarch}" \ "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" >/dev/null 2>&1 - if [ $? -ge 1 ]; then + if (( $? >= 1 )); then failedpkgs+=("${pkgbase}-${pkgver}${SRCEXT}") continue fi @@ -89,7 +89,7 @@ for repo in ${PKGREPOS[@]}; do # Build the actual source package pushd "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" >/dev/null makepkg --nocolor --allsource --ignorearch --skippgpcheck --config "${dirname}/makepkg.conf" >"${WORKDIR}/${pkgbase}.log" 2>&1 - if [ $? -eq 0 ] && [ -f "${pkgbase}-${pkgver}${SRCEXT}" ]; then + if (( $? == 0 )) && [[ -f ${pkgbase}-${pkgver}${SRCEXT} ]]; then mv_acl "${pkgbase}-${pkgver}${SRCEXT}" "${FTP_BASE}/${SRCPOOL}/${pkgbase}-${pkgver}${SRCEXT}" # Avoid creating the same source package for every arch echo "${pkgbase}-${pkgver}${SRCEXT}" >> "${WORKDIR}/available-src-pkgs" @@ -121,7 +121,7 @@ find "${WORKDIR}" -maxdepth 1 -type f -name 'expected-src-pkgs' -exec cat {} \; find "${WORKDIR}" -maxdepth 1 -type f -name 'available-src-pkgs' -exec cat {} \; | sort -u > "${WORKDIR}/available-src-pkgs.sort" old_pkgs=($(comm -23 "${WORKDIR}/available-src-pkgs.sort" "${WORKDIR}/expected-src-pkgs.sort")) -if [ ${#old_pkgs[@]} -ge 1 ]; then +if (( ${#old_pkgs[@]} >= 1 )); then msg "Removing old source packages..." ${SOURCE_CLEANUP_DRYRUN} && warning 'dry run mode is active' for old_pkg in ${old_pkgs[@]}; do @@ -134,7 +134,7 @@ if [ ${#old_pkgs[@]} -ge 1 ]; then fi old_pkgs=($(find ${SOURCE_CLEANUP_DESTDIR} -type f -name "*${SRCEXT}" -mtime +${SOURCE_CLEANUP_KEEP} -printf '%f\n')) -if [ ${#old_pkgs[@]} -ge 1 ]; then +if (( ${#old_pkgs[@]} >= 1 )); then msg "Removing old source packages from the cleanup directory..." for old_pkg in ${old_pkgs[@]}; do msg2 "${old_pkg}" @@ -142,7 +142,7 @@ if [ ${#old_pkgs[@]} -ge 1 ]; then done fi -if [ -f "${WORKDIR}/makepkg-fail.log" ]; then +if [[ -f ${WORKDIR}/makepkg-fail.log ]]; then msg "Log of failed packages" cat "${WORKDIR}/makepkg-fail.log" fi diff --git a/cron-jobs/update-web-db b/cron-jobs/update-web-db index 23af067..c859ce0 100755 --- a/cron-jobs/update-web-db +++ b/cron-jobs/update-web-db @@ -27,7 +27,7 @@ renice +5 -p $$ > /dev/null echo "%s: Updating DB at %s" "$cmd" "$(date)" >> "${LOGOUT}" # source our virtualenv if it exists -if [ -f "$ENVPATH" ]; then +if [[ -f "$ENVPATH" ]]; then . "$ENVPATH" fi @@ -47,7 +47,7 @@ for repo in ${REPOS[@]}; do for arch in ${ARCHES[@]}; do repo_lock ${repo} ${arch} || exit 1 dbfile="/srv/ftp/${repo}/os/${arch}/${repo}${dbfileext}" - if [ -f "${dbfile}" ]; then + if [[ -f ${dbfile} ]]; then mkdir -p "${WORKDIR}/${repo}/${arch}" cp "${dbfile}" "${WORKDIR}/${repo}/${arch}/${repo}${dbfileext}" fi @@ -60,7 +60,7 @@ pushd $SPATH >/dev/null for repo in ${REPOS[@]}; do for arch in ${ARCHES[@]}; do dbcopy="${WORKDIR}/${repo}/${arch}/${repo}${dbfileext}" - if [ -f "${dbcopy}" ]; then + if [[ -f ${dbcopy} ]]; then echo "Updating ${repo}-${arch}" >> "${LOGOUT}" ./manage.py reporead ${flags} ${arch} "${dbcopy}" >> "${LOGOUT}" 2>&1 echo "" >> "${LOGOUT}" -- cgit v1.2.2 From 07e6a91da1331ae61b667d878092c5b03aa49ced Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Mon, 19 Feb 2018 11:12:02 -0500 Subject: Fix overloading PKGEXT to mean two things. PKGEXT is a makepkg variable referring to a fixed filename suffix, but we were also using it to mean a bash glob referring to candidate filenames. This is wrong, so rename it to PKGEXTS which is more descriptive of its purpose. Exclude the testsuite from this change, as the testsuite actually uses PKGEXT for its intended purpose. Fix the testsuite to consistently use PKGEXT, as it hardcoded the file extension in several cases, and pin its value to .pkg.tar.xz --- config | 2 +- cron-jobs/ftpdir-cleanup | 6 +++--- db-functions | 4 ++-- db-move | 4 ++-- db-update | 8 ++++---- test/cases/db-repo-add.bats | 6 +++--- test/cases/db-update.bats | 3 +-- test/cases/ftpdir-cleanup.bats | 6 +++--- test/lib/common.bash | 5 +++-- 9 files changed, 22 insertions(+), 22 deletions(-) diff --git a/config b/config index d2c1942..5bb3b16 100644 --- a/config +++ b/config @@ -25,7 +25,7 @@ TMPDIR="/var/tmp" ARCHES=(x86_64) DBEXT=".db.tar.gz" FILESEXT=".files.tar.gz" -PKGEXT=".pkg.tar.?z" +PKGEXTS=".pkg.tar.?z" SRCEXT=".src.tar.gz" # Allowed licenses: get sourceballs only for licenses in this array diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index c771950..630efa8 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -38,7 +38,7 @@ for repo in ${PKGREPOS[@]}; do continue fi # get a list of actual available package files - find "${FTP_BASE}/${repo}/os/${arch}" -xtype f -name "*${PKGEXT}" -printf '%f\n' | sort > "${WORKDIR}/repo-${repo}-${arch}" + find "${FTP_BASE}/${repo}/os/${arch}" -xtype f -name "*${PKGEXTS}" -printf '%f\n' | sort > "${WORKDIR}/repo-${repo}-${arch}" # get a list of package files defined in the repo db bsdtar -xOf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" | awk '/^%FILENAME%/{getline;print}' | sort > "${WORKDIR}/db-${repo}-${arch}" @@ -62,7 +62,7 @@ for repo in ${PKGREPOS[@]}; do done # get a list of all available packages in the pacakge pool -find "$FTP_BASE/${PKGPOOL}" -name "*${PKGEXT}" -printf '%f\n' | sort > "${WORKDIR}/pool" +find "$FTP_BASE/${PKGPOOL}" -name "*${PKGEXTS}" -printf '%f\n' | sort > "${WORKDIR}/pool" # create a list of packages in our db find "${WORKDIR}" -maxdepth 1 -type f -name 'db-*' -exec cat {} \; | sort -u > "${WORKDIR}/db" @@ -75,7 +75,7 @@ if (( ${#old_pkgs[@]} >= 1 )); then done fi -old_pkgs=($(find ${CLEANUP_DESTDIR} -type f -name "*${PKGEXT}" -mtime +${CLEANUP_KEEP} -printf '%f\n')) +old_pkgs=($(find ${CLEANUP_DESTDIR} -type f -name "*${PKGEXTS}" -mtime +${CLEANUP_KEEP} -printf '%f\n')) if (( ${#old_pkgs[@]} >= 1 )); then msg "Removing old packages from the cleanup directory..." for old_pkg in ${old_pkgs[@]}; do diff --git a/db-functions b/db-functions index e8949d7..e8eb2bc 100644 --- a/db-functions +++ b/db-functions @@ -374,8 +374,8 @@ check_pkgrepos() { local pkgver="$(getpkgver ${pkgfile})" || return 1 local pkgarch="$(getpkgarch ${pkgfile})" || return 1 - [[ -f ${FTP_BASE}/${PKGPOOL}/${pkgname}-${pkgver}-${pkgarch}${PKGEXT} ]] && return 1 - [[ -f ${FTP_BASE}/${PKGPOOL}/${pkgname}-${pkgver}-${pkgarch}${PKGEXT}.sig ]] && return 1 + [[ -f ${FTP_BASE}/${PKGPOOL}/${pkgname}-${pkgver}-${pkgarch}${PKGEXTS} ]] && return 1 + [[ -f ${FTP_BASE}/${PKGPOOL}/${pkgname}-${pkgver}-${pkgarch}${PKGEXTS}.sig ]] && return 1 [[ -f ${FTP_BASE}/${PKGPOOL}/${pkgfile##*/} ]] && return 1 [[ -f ${FTP_BASE}/${PKGPOOL}/${pkgfile##*/}.sig ]] && return 1 diff --git a/db-move b/db-move index 37a9884..fb7ebac 100755 --- a/db-move +++ b/db-move @@ -49,7 +49,7 @@ for pkgbase in ${args[@]:2}; do for pkgname in ${pkgnames[@]}; do for tarch in ${tarches[@]}; do - getpkgfile "${ftppath_from}/${tarch}/"${pkgname}-${pkgver}-${pkgarch}${PKGEXT} >/dev/null + getpkgfile "${ftppath_from}/${tarch}/"${pkgname}-${pkgver}-${pkgarch}${PKGEXTS} >/dev/null done done continue 2 @@ -95,7 +95,7 @@ for pkgbase in ${args[@]:2}; do for pkgname in ${pkgnames[@]}; do for tarch in ${tarches[@]}; do - pkgpath=$(getpkgfile "${ftppath_from}/${tarch}/"${pkgname}-${pkgver}-${pkgarch}${PKGEXT}) + pkgpath=$(getpkgfile "${ftppath_from}/${tarch}/"${pkgname}-${pkgver}-${pkgarch}${PKGEXTS}) pkgfile="${pkgpath##*/}" ln -s "../../../${PKGPOOL}/${pkgfile}" ${ftppath_to}/${tarch}/ diff --git a/db-update b/db-update index 45755a4..a8d885a 100755 --- a/db-update +++ b/db-update @@ -9,7 +9,7 @@ if (( $# >= 1 )); then fi # Find repos with packages to release -if ! staging_repos=($(find "${STAGING}" -mindepth 1 -type f -name "*${PKGEXT}" -printf '%h\n' | sort -u)); then +if ! staging_repos=($(find "${STAGING}" -mindepth 1 -type f -name "*${PKGEXTS}" -printf '%h\n' | sort -u)); then die "Could not read %s" "$STAGING" fi @@ -32,7 +32,7 @@ for repo in ${repos[@]}; do if ! check_repo_permission "${repo}"; then die "You don't have permission to update packages in %s" "$repo" fi - pkgs=($(getpkgfiles "${STAGING}/${repo}/"*${PKGEXT})) + pkgs=($(getpkgfiles "${STAGING}/${repo}/"*${PKGEXTS})) if (( $? == 0 )); then for pkg in ${pkgs[@]}; do if [[ -h ${pkg} ]]; then @@ -70,10 +70,10 @@ done for repo in ${repos[@]}; do msg "Updating [%s]..." "$repo" - any_pkgs=($(getpkgfiles "${STAGING}/${repo}/"*-any${PKGEXT} 2>/dev/null)) + any_pkgs=($(getpkgfiles "${STAGING}/${repo}/"*-any${PKGEXTS} 2>/dev/null)) for pkgarch in ${ARCHES[@]}; do add_pkgs=() - arch_pkgs=($(getpkgfiles "${STAGING}/${repo}/"*-${pkgarch}${PKGEXT} 2>/dev/null)) + arch_pkgs=($(getpkgfiles "${STAGING}/${repo}/"*-${pkgarch}${PKGEXTS} 2>/dev/null)) for pkg in ${arch_pkgs[@]} ${any_pkgs[@]}; do pkgfile="${pkg##*/}" msg2 "${pkgfile} (${pkgarch})" diff --git a/test/cases/db-repo-add.bats b/test/cases/db-repo-add.bats index 9fde381..ac91058 100644 --- a/test/cases/db-repo-add.bats +++ b/test/cases/db-repo-add.bats @@ -31,7 +31,7 @@ __movePackageToRepo() { releasePackage extra ${pkgbase} for arch in ${arches[@]}; do __movePackageToRepo extra ${pkgbase} ${arch} - db-repo-add extra ${arch} ${pkgbase}-1-1-${arch}.pkg.tar.xz + db-repo-add extra ${arch} ${pkgbase}-1-1-${arch}${PKGEXT} done done @@ -54,7 +54,7 @@ __movePackageToRepo() { add_pkgs=() for pkgbase in ${pkgs[@]}; do __movePackageToRepo extra ${pkgbase} ${arch} - add_pkgs+=("${pkgbase}-1-1-${arch}.pkg.tar.xz") + add_pkgs+=("${pkgbase}-1-1-${arch}${PKGEXT}") done db-repo-add extra ${arch} ${add_pkgs[@]} done @@ -73,7 +73,7 @@ __movePackageToRepo() { for pkgbase in ${pkgs[@]}; do releasePackage extra ${pkgbase} __movePackageToRepo extra ${pkgbase} any - db-repo-add extra any ${pkgbase}-1-1-any.pkg.tar.xz + db-repo-add extra any ${pkgbase}-1-1-any${PKGEXT} done for pkgbase in ${pkgs[@]}; do diff --git a/test/cases/db-update.bats b/test/cases/db-update.bats index 1da7eef..349b195 100644 --- a/test/cases/db-update.bats +++ b/test/cases/db-update.bats @@ -151,8 +151,7 @@ load ../lib/common local p releasePackage extra 'pkg-any-a' for p in "${STAGING}"/extra/*${PKGEXT}; do - unxz $p - xz -0 ${p%%.xz} + printf '%s\n' "Not a real package" | gpg -v --detach-sign --no-armor --use-agent - > "${p}.sig" done run db-update [ "$status" -ne 0 ] diff --git a/test/cases/ftpdir-cleanup.bats b/test/cases/ftpdir-cleanup.bats index 6280ce0..fd485f3 100644 --- a/test/cases/ftpdir-cleanup.bats +++ b/test/cases/ftpdir-cleanup.bats @@ -82,7 +82,7 @@ __checkRepoRemovedPackage() { db-remove extra any pkg-any-a ftpdir-cleanup - local pkg1='pkg-any-a-1-1-any.pkg.tar.xz' + local pkg1="pkg-any-a-1-1-any${PKGEXT}" checkRemovedPackage extra 'pkg-any-a' for arch in ${arches[@]}; do __checkRepoRemovedPackage extra 'pkg-any-a' ${arch} @@ -138,8 +138,8 @@ __checkRepoRemovedPackage() { ftpdir-cleanup - local pkgfilea="pkg-simple-a-1-1-${arch}.pkg.tar.xz" - local pkgfileb="pkg-simple-b-1-1-${arch}.pkg.tar.xz" + local pkgfilea="pkg-simple-a-1-1-${arch}${PKGEXT}" + local pkgfileb="pkg-simple-b-1-1-${arch}${PKGEXT}" for arch in ${arches[@]}; do touch -d "-$(expr ${CLEANUP_KEEP} + 1)days" ${CLEANUP_DESTDIR}/${pkgfilea}{,.sig} done diff --git a/test/lib/common.bash b/test/lib/common.bash index 540e403..24b2378 100644 --- a/test/lib/common.bash +++ b/test/lib/common.bash @@ -34,9 +34,9 @@ __buildPackage() { pkgarches=($(. PKGBUILD; echo ${arch[@]})) for tarch in ${pkgarches[@]}; do if [ "${tarch}" == 'any' ]; then - PKGDEST=${pkgdest} makepkg -c + PKGDEST=${pkgdest} PKGEXT=${PKGEXT} makepkg -c else - PKGDEST=${pkgdest} CARCH=${tarch} makepkg -c + PKGDEST=${pkgdest} PKGEXT=${PKGEXT} CARCH=${tarch} makepkg -c fi done @@ -82,6 +82,7 @@ setup() { local pkg local r local a + PKGEXT=".pkg.tar.xz" TMP="$(mktemp -d)" -- cgit v1.2.2 From 8757c459089c0db12e57db66561f640f657e34ac Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Fri, 16 Feb 2018 12:10:25 -0500 Subject: test: db-update: verify that PKGEXT(S) is treated as a glob Commit b61a714 introduced a regression that in a db-functions function called by db-update, config:PKGEXT is treated like a fixed string, instead of as a glob. Because of inadequacy of the test-suite, this did not cause a test failure. https://lists.archlinux.org/pipermail/arch-projects/2018-February/004742.html The broken function checks if the repo already contains a package that matches the one being released. This did not trigger a test-suite failure because right after the broken check there is another check that checks for the exact filename of the new package file. In the test-suite, all package files have the same extension, so the latter check always "saved" us. So, modify a relevant test case (a test case that verifies that we can't release the same package twice) such that the second time we release the package, we set PKGEXT to something different (.gz, instead of the normal .xz). This way, we can be sure that when db-update rejects the operation, it's because of the glob check, not the exact-filename check. This is made slightly tricky because the __buildPackage() test helper routine tries to cache builds, to speed up the tests. Without modification, this means that it will just ignore the new PKGEXT. So, modify it to only use the cache if it can successfully copy build artifacts out, in a way that respects PKGEXT. --- test/cases/db-update.bats | 2 +- test/lib/common.bash | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/test/cases/db-update.bats b/test/cases/db-update.bats index 349b195..e7e4489 100644 --- a/test/cases/db-update.bats +++ b/test/cases/db-update.bats @@ -92,7 +92,7 @@ load ../lib/common db-update checkPackage extra pkg-any-a - releasePackage extra pkg-any-a + PKGEXT=.pkg.tar.gz releasePackage extra pkg-any-a run db-update [ "$status" -ne 0 ] } diff --git a/test/lib/common.bash b/test/lib/common.bash index 24b2378..568a541 100644 --- a/test/lib/common.bash +++ b/test/lib/common.bash @@ -23,8 +23,7 @@ __buildPackage() { if [[ -n ${BUILDDIR} ]]; then cache=${BUILDDIR}/$(__getCheckSum PKGBUILD) - if [[ -d ${cache} ]]; then - cp -Lv ${cache}/*${PKGEXT}{,.sig} ${pkgdest} + if cp -Lv ${cache}/*${PKGEXT}{,.sig} ${pkgdest} 2>/dev/null; then return 0 else mkdir -p ${cache} -- cgit v1.2.2 From 3e01ba9a82234c1d78a14be095c18a626591ad71 Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Thu, 15 Feb 2018 22:26:24 -0500 Subject: db-update: replace external find command with bash globbing Don't bother emitting errors. bash doesn't show globbing errors if it cannot read a directory to try globbing there. And the former code never aborted on errors anyway, as without `set -o pipefail` the sort command swallowed the return code. --- db-functions | 4 ++++ db-update | 8 +++++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/db-functions b/db-functions index e8eb2bc..394c7a2 100644 --- a/db-functions +++ b/db-functions @@ -2,6 +2,10 @@ . /usr/share/makepkg/util.sh +# global shell options for enhanced bash scripting +shopt -s globstar nullglob + + # Some PKGBUILDs need CARCH to be set CARCH="x86_64" diff --git a/db-update b/db-update index a8d885a..4e17184 100755 --- a/db-update +++ b/db-update @@ -9,9 +9,11 @@ if (( $# >= 1 )); then fi # Find repos with packages to release -if ! staging_repos=($(find "${STAGING}" -mindepth 1 -type f -name "*${PKGEXTS}" -printf '%h\n' | sort -u)); then - die "Could not read %s" "$STAGING" -fi +mapfile -t -d '' staging_repos < <( + for f in "${STAGING}"/**/*${PKGEXTS}; do + printf '%s\0' "${f%/*}" + done | sort -uz +) repos=() for staging_repo in ${staging_repos[@]##*/}; do -- cgit v1.2.2 From c82f4372440b42f879344f105e12fb358c4c0ccf Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Thu, 15 Feb 2018 22:32:04 -0500 Subject: ftpdir-cleanup,sourceballs: replace external find command with bash globbing This fully removes the use of find from the codebase, leads to a micro-optimization in a couple cases, and ensures that $PKGEXT is consistently treated as a shell globbing character (which is important because it is used as one). Of the eight instances in these files: - One was unnecessary as `cat` can natively consume all files passed to it and no directory traversal was in use. - Two were unnecessary as they were hardcoded to read a single file.... - Another four were only being used to strip leading directory paths, and can be replaced by globstar and ${filepath##*/} - The final two were checking the modification time of the files, and can be replaced with touch(1) and [[ -nt ]]. Although this introduces an additional temporary file, this is not such a big deal. --- cron-jobs/ftpdir-cleanup | 22 +++++++++++++++++----- cron-jobs/sourceballs | 19 +++++++++++++++---- 2 files changed, 32 insertions(+), 9 deletions(-) diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index 630efa8..ff65d46 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -38,7 +38,11 @@ for repo in ${PKGREPOS[@]}; do continue fi # get a list of actual available package files - find "${FTP_BASE}/${repo}/os/${arch}" -xtype f -name "*${PKGEXTS}" -printf '%f\n' | sort > "${WORKDIR}/repo-${repo}-${arch}" + for f in "${FTP_BASE}"/${repo}/os/${arch}/*${PKGEXTS}; do + if [[ -f $f ]]; then + printf '%s\n' "${f##*/}" + fi + done | sort > "${WORKDIR}/repo-${repo}-${arch}" # get a list of package files defined in the repo db bsdtar -xOf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" | awk '/^%FILENAME%/{getline;print}' | sort > "${WORKDIR}/db-${repo}-${arch}" @@ -61,10 +65,12 @@ for repo in ${PKGREPOS[@]}; do done done -# get a list of all available packages in the pacakge pool -find "$FTP_BASE/${PKGPOOL}" -name "*${PKGEXTS}" -printf '%f\n' | sort > "${WORKDIR}/pool" +# get a list of all available packages in the package pool +for f in "$FTP_BASE/${PKGPOOL}"/*${PKGEXTS}; do + printf '%s\n' "${f##*/}" +done | sort > "${WORKDIR}/pool" # create a list of packages in our db -find "${WORKDIR}" -maxdepth 1 -type f -name 'db-*' -exec cat {} \; | sort -u > "${WORKDIR}/db" +cat "${WORKDIR}"/db-* 2>/dev/null | sort -u > "${WORKDIR}/db" old_pkgs=($(comm -23 "${WORKDIR}/pool" "${WORKDIR}/db")) if (( ${#old_pkgs[@]} >= 1 )); then @@ -75,7 +81,13 @@ if (( ${#old_pkgs[@]} >= 1 )); then done fi -old_pkgs=($(find ${CLEANUP_DESTDIR} -type f -name "*${PKGEXTS}" -mtime +${CLEANUP_KEEP} -printf '%f\n')) +unset old_pkgs +touch -d "${CLEANUP_KEEP} days ago" "${WORKDIR}/cleanup_timestamp" +for f in "${CLEANUP_DESTDIR}"/**/*${PKGEXTS}; do + if [[ ${WORKDIR}/cleanup_timestamp -nt $f ]]; then + old_pkgs+=("${f##*/}") + fi +done if (( ${#old_pkgs[@]} >= 1 )); then msg "Removing old packages from the cleanup directory..." for old_pkg in ${old_pkgs[@]}; do diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 25a8abb..8f089b3 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -46,7 +46,11 @@ for repo in ${PKGREPOS[@]}; do done # Create a list of all available source package file names -find "${FTP_BASE}/${SRCPOOL}" -xtype f -name "*${SRCEXT}" -printf '%f\n' | sort -u > "${WORKDIR}/available-src-pkgs" +for f in "${FTP_BASE}"/${SRCPOOL}/*${SRCEXT}; do + if [[ -f $f ]]; then + printf '%s\n' "${f##*/}" + fi +done | sort -u > "${WORKDIR}/available-src-pkgs" # Check for all packages if we need to build a source package for repo in ${PKGREPOS[@]}; do @@ -117,8 +121,8 @@ for repo in ${PKGREPOS[@]}; do done # Cleanup old source packages -find "${WORKDIR}" -maxdepth 1 -type f -name 'expected-src-pkgs' -exec cat {} \; | sort -u > "${WORKDIR}/expected-src-pkgs.sort" -find "${WORKDIR}" -maxdepth 1 -type f -name 'available-src-pkgs' -exec cat {} \; | sort -u > "${WORKDIR}/available-src-pkgs.sort" +sort -u "${WORKDIR}/expected-src-pkgs" > "${WORKDIR}/expected-src-pkgs.sort" +sort -u "${WORKDIR}/available-src-pkgs" > "${WORKDIR}/available-src-pkgs.sort" old_pkgs=($(comm -23 "${WORKDIR}/available-src-pkgs.sort" "${WORKDIR}/expected-src-pkgs.sort")) if (( ${#old_pkgs[@]} >= 1 )); then @@ -133,7 +137,14 @@ if (( ${#old_pkgs[@]} >= 1 )); then done fi -old_pkgs=($(find ${SOURCE_CLEANUP_DESTDIR} -type f -name "*${SRCEXT}" -mtime +${SOURCE_CLEANUP_KEEP} -printf '%f\n')) +unset old_pkgs +touch -d "${SOURCE_CLEANUP_KEEP} days ago" "${WORKDIR}/cleanup_timestamp" +for f in "${SOURCE_CLEANUP_DESTDIR}"/*${SRCEXT}; do + if [[ ${WORKDIR}/cleanup_timestamp -nt $f ]]; then + old_pkgs+=("${f##*/}") + fi +done + if (( ${#old_pkgs[@]} >= 1 )); then msg "Removing old source packages from the cleanup directory..." for old_pkg in ${old_pkgs[@]}; do -- cgit v1.2.2 From e53cad6e4a8284165c6d0b2c7c86f6c077be693b Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Thu, 15 Feb 2018 22:34:36 -0500 Subject: Globally set $PKGEXT to a bash extended glob representing valid choices. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The current glob `*.pkg.tar.?z` is both less restrictive and more restrictive than makepkg, as it accepts any valid unicode character. To be more exact, it's almost completely orthogonal to the one in makepkg. makepkg only accepts .tar.gz, .tar.bz2, .tar.xz, .tar.lzo, .tar.lrz, and .tar.Z and most of those fail to match against a two-char compression type. dbscripts accepts .pkg.tar.💩z which incidentally is what I think of cherry-picking xz and gz as supported methods. Since this can be anything makepkg.conf accepts, it needs to be able to match all that, unless we decide to perform additional restrictions in which case we should still explicitly list each allowed extension. Using bash extended globbing allows us to do this relatively painlessly. Document the fact that this has *always* been some sort of glob, and update the two cases where this was (not!) being evaluated by bash [[ ... ]], to use a not-elegant-at-all proxy function is_globfile() to evaluate globs *before* testing if they exist. --- config | 4 +++- db-functions | 14 +++++++++++--- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/config b/config index 5bb3b16..c03fe50 100644 --- a/config +++ b/config @@ -25,9 +25,11 @@ TMPDIR="/var/tmp" ARCHES=(x86_64) DBEXT=".db.tar.gz" FILESEXT=".files.tar.gz" -PKGEXTS=".pkg.tar.?z" SRCEXT=".src.tar.gz" +# bash glob listing allowed extensions. Note that db-functions turns on extglob. +PKGEXTS=".pkg.tar.@(gz|bz2|xz|lzo|lrz|Z)" + # Allowed licenses: get sourceballs only for licenses in this array ALLOWED_LICENSES=('GPL' 'GPL1' 'GPL2' 'GPL3' 'LGPL' 'LGPL1' 'LGPL2' 'LGPL2.1' 'LGPL3') diff --git a/db-functions b/db-functions index 394c7a2..8b71cae 100644 --- a/db-functions +++ b/db-functions @@ -3,7 +3,7 @@ . /usr/share/makepkg/util.sh # global shell options for enhanced bash scripting -shopt -s globstar nullglob +shopt -s extglob globstar nullglob # Some PKGBUILDs need CARCH to be set @@ -20,6 +20,14 @@ restore_umask () { umask $UMASK >/dev/null } +# Proxy function to check if a file exists. Using [[ -f ... ]] directly is not +# always wanted because we might want to expand bash globs first. This way we +# can pass unquoted globs to is_globfile() and have them expanded as function +# arguments before being checked. +is_globfile() { + [[ -f $1 ]] +} + # just like mv -f, but we touch the file and then copy the content so # default ACLs in the target dir will be applied mv_acl() { @@ -378,8 +386,8 @@ check_pkgrepos() { local pkgver="$(getpkgver ${pkgfile})" || return 1 local pkgarch="$(getpkgarch ${pkgfile})" || return 1 - [[ -f ${FTP_BASE}/${PKGPOOL}/${pkgname}-${pkgver}-${pkgarch}${PKGEXTS} ]] && return 1 - [[ -f ${FTP_BASE}/${PKGPOOL}/${pkgname}-${pkgver}-${pkgarch}${PKGEXTS}.sig ]] && return 1 + is_globfile "${FTP_BASE}/${PKGPOOL}/${pkgname}-${pkgver}-${pkgarch}"${PKGEXTS} && return 1 + is_globfile "${FTP_BASE}/${PKGPOOL}/${pkgname}-${pkgver}-${pkgarch}"${PKGEXTS}.sig && return 1 [[ -f ${FTP_BASE}/${PKGPOOL}/${pkgfile##*/} ]] && return 1 [[ -f ${FTP_BASE}/${PKGPOOL}/${pkgfile##*/}.sig ]] && return 1 -- cgit v1.2.2 From f22f1554720da245955337782ce554d5c3f93c8c Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Mon, 26 Feb 2018 17:57:31 -0500 Subject: db-functions: unify the repo-add/repo-remove caller Use less duplicated logic, opening the way to more elegantly change the interface for the repo-add/repo-remove scripts which are themselves symlinked together and mostly have the same interface. --- db-functions | 34 ++++++++++++---------------------- db-move | 4 ++-- db-remove | 2 +- db-repo-add | 2 +- db-repo-remove | 2 +- db-update | 2 +- 6 files changed, 18 insertions(+), 28 deletions(-) diff --git a/db-functions b/db-functions index 8b71cae..653f130 100644 --- a/db-functions +++ b/db-functions @@ -442,34 +442,24 @@ set_repo_permission() { fi } -arch_repo_add() { - local repo=$1 - local arch=$2 - local pkgs=(${@:3}) - - # package files might be relative to repo dir - pushd "${FTP_BASE}/${repo}/os/${arch}" >/dev/null - /usr/bin/repo-add -q "${repo}${DBEXT}" ${pkgs[@]} \ - || error "repo-add ${repo}${DBEXT} ${pkgs[@]}" - popd >/dev/null - set_repo_permission "${repo}" "${arch}" - - REPO_MODIFIED=1 -} - -arch_repo_remove() { - local repo=$1 - local arch=$2 - local pkgs=(${@:3}) +arch_repo_modify() { + local action=$1 + local repo=$2 + local arch=$3 + local pkgs=("${@:4}") local dbfile="${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" - if [[ ! -f ${dbfile} ]]; then + if [[ ${action} = remove && ! -f ${dbfile} ]]; then error "No database found at '%s'" "$dbfile" return 1 fi - /usr/bin/repo-remove -q "${dbfile}" ${pkgs[@]} \ - || error "repo-remove ${dbfile} ${pkgs[@]}" + + # package files for repo-add might be relative to repo dir + pushd "${dbfile%/*}" >/dev/null + /usr/bin/repo-${action} -q "${dbfile}" ${pkgs[@]} \ + || error "repo-${action} ${dbfile} ${pkgs[@]}" set_repo_permission "${repo}" "${arch}" + popd >/dev/null REPO_MODIFIED=1 } diff --git a/db-move b/db-move index fb7ebac..e9c9fa7 100755 --- a/db-move +++ b/db-move @@ -114,8 +114,8 @@ done for tarch in ${ARCHES[@]}; do if [[ -n ${add_pkgs[${tarch}]} ]]; then - arch_repo_add "${repo_to}" "${tarch}" ${add_pkgs[${tarch}]} - arch_repo_remove "${repo_from}" "${tarch}" ${remove_pkgs[${tarch}]} + arch_repo_modify add "${repo_to}" "${tarch}" ${add_pkgs[${tarch}]} + arch_repo_modify remove "${repo_from}" "${tarch}" ${remove_pkgs[${tarch}]} fi done diff --git a/db-remove b/db-remove index 70502bc..08a95f9 100755 --- a/db-remove +++ b/db-remove @@ -47,6 +47,6 @@ for pkgbase in ${pkgbases[@]}; do done for tarch in ${tarches[@]}; do - arch_repo_remove "${repo}" "${tarch}" ${remove_pkgs[@]} + arch_repo_modify remove "${repo}" "${tarch}" ${remove_pkgs[@]} repo_unlock $repo $tarch done diff --git a/db-repo-add b/db-repo-add index d7be302..4b16f7e 100755 --- a/db-repo-add +++ b/db-repo-add @@ -36,6 +36,6 @@ for tarch in ${tarches[@]}; do msg "Adding %s to [%s]..." "$pkgfile" "$repo" fi done - arch_repo_add "${repo}" "${tarch}" ${pkgfiles[@]} + arch_repo_modify add "${repo}" "${tarch}" ${pkgfiles[@]} repo_unlock $repo $tarch done diff --git a/db-repo-remove b/db-repo-remove index 32d167e..3077786 100755 --- a/db-repo-remove +++ b/db-repo-remove @@ -32,6 +32,6 @@ for tarch in ${tarches[@]}; do for pkgname in ${pkgnames[@]}; do msg "Removing %s from [%s]..." "$pkgname" "$repo" done - arch_repo_remove "${repo}" "${tarch}" ${pkgnames[@]} + arch_repo_modify remove "${repo}" "${tarch}" ${pkgnames[@]} repo_unlock $repo $tarch done diff --git a/db-update b/db-update index 4e17184..e8ae08d 100755 --- a/db-update +++ b/db-update @@ -94,7 +94,7 @@ for repo in ${repos[@]}; do add_pkgs+=("${pkgfile}") done if (( ${#add_pkgs[@]} >= 1 )); then - arch_repo_add "${repo}" "${pkgarch}" ${add_pkgs[@]} + arch_repo_modify add "${repo}" "${pkgarch}" ${add_pkgs[@]} fi done done -- cgit v1.2.2 From 33aae318542016f66d2f0e09654606649d404ff2 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Thu, 22 Feb 2018 21:15:41 -0500 Subject: Update messages to make fuller use of printf formatters These are things that were (IMO) missed in 5afac1e. I found them using: git grep -E '(plain|msg|msg2|warning|error|die) "[^"]*\$' I went a little above-and-beyond for escaping strings for the error messages in db-functions' arch_repo_add and arch_repo_remove. The code should explain itself, but I wanted to point it out, as it's more than the usual "slap %s in there, and move the ${...} to the right". --- cron-jobs/ftpdir-cleanup | 8 ++++---- cron-jobs/integrity-check | 2 +- cron-jobs/sourceballs | 8 ++++---- db-functions | 2 +- db-move | 4 ++-- db-remove | 2 +- db-repo-add | 2 +- db-repo-remove | 2 +- db-update | 2 +- testing2x | 2 +- 10 files changed, 17 insertions(+), 17 deletions(-) diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index ff65d46..e24e614 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -50,7 +50,7 @@ for repo in ${PKGREPOS[@]}; do if (( ${#missing_pkgs[@]} >= 1 )); then error "Missing packages in [%s] (%s)..." "$repo" "$arch" for missing_pkg in ${missing_pkgs[@]}; do - msg2 "${missing_pkg}" + msg2 '%s' "${missing_pkg}" done fi @@ -58,7 +58,7 @@ for repo in ${PKGREPOS[@]}; do if (( ${#old_pkgs[@]} >= 1 )); then msg "Removing old packages from [%s] (%s)..." "$repo" "$arch" for old_pkg in ${old_pkgs[@]}; do - msg2 "${old_pkg}" + msg2 '%s' "${old_pkg}" clean_pkg "${FTP_BASE}/${repo}/os/${arch}/${old_pkg}" done fi @@ -76,7 +76,7 @@ old_pkgs=($(comm -23 "${WORKDIR}/pool" "${WORKDIR}/db")) if (( ${#old_pkgs[@]} >= 1 )); then msg "Removing old packages from package pool..." for old_pkg in ${old_pkgs[@]}; do - msg2 "${old_pkg}" + msg2 '%s' "${old_pkg}" clean_pkg "$FTP_BASE/${PKGPOOL}/${old_pkg}" done fi @@ -91,7 +91,7 @@ done if (( ${#old_pkgs[@]} >= 1 )); then msg "Removing old packages from the cleanup directory..." for old_pkg in ${old_pkgs[@]}; do - msg2 "${old_pkg}" + msg2 '%s' "${old_pkg}" if ! ${CLEANUP_DRYRUN}; then rm -f "${CLEANUP_DESTDIR}/${old_pkg}" rm -f "${CLEANUP_DESTDIR}/${old_pkg}.sig" diff --git a/cron-jobs/integrity-check b/cron-jobs/integrity-check index f1e75ab..47cf856 100755 --- a/cron-jobs/integrity-check +++ b/cron-jobs/integrity-check @@ -8,7 +8,7 @@ dirname="$(dirname $0)" script_lock if (( $# != 1 )); then - die "usage: ${0##*/} " + die "usage: %s " "${0##*/}" fi mailto=$1 diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 8f089b3..7370594 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -109,13 +109,13 @@ for repo in ${PKGREPOS[@]}; do if [ ${#newpkgs[@]} -ge 1 ]; then msg "Adding source packages for [%s]..." "$repo" for new_pkg in ${newpkgs[@]}; do - msg2 "${new_pkg}" + msg2 '%s' "${new_pkg}" done fi if [ ${#failedpkgs[@]} -ge 1 ]; then msg "Failed to create source packages for [%s]..." "$repo" for failed_pkg in ${failedpkgs[@]}; do - msg2 "${failed_pkg}" + msg2 '%s' "${failed_pkg}" done fi done @@ -129,7 +129,7 @@ if (( ${#old_pkgs[@]} >= 1 )); then msg "Removing old source packages..." ${SOURCE_CLEANUP_DRYRUN} && warning 'dry run mode is active' for old_pkg in ${old_pkgs[@]}; do - msg2 "${old_pkg}" + msg2 '%s' "${old_pkg}" if ! ${SOURCE_CLEANUP_DRYRUN}; then mv_acl "$FTP_BASE/${SRCPOOL}/${old_pkg}" "${SOURCE_CLEANUP_DESTDIR}/${old_pkg}" touch "${SOURCE_CLEANUP_DESTDIR}/${old_pkg}" @@ -148,7 +148,7 @@ done if (( ${#old_pkgs[@]} >= 1 )); then msg "Removing old source packages from the cleanup directory..." for old_pkg in ${old_pkgs[@]}; do - msg2 "${old_pkg}" + msg2 '%s' "${old_pkg}" ${SOURCE_CLEANUP_DRYRUN} || rm -f "${SOURCE_CLEANUP_DESTDIR}/${old_pkg}" done fi diff --git a/db-functions b/db-functions index 653f130..d052740 100644 --- a/db-functions +++ b/db-functions @@ -457,7 +457,7 @@ arch_repo_modify() { # package files for repo-add might be relative to repo dir pushd "${dbfile%/*}" >/dev/null /usr/bin/repo-${action} -q "${dbfile}" ${pkgs[@]} \ - || error "repo-${action} ${dbfile} ${pkgs[@]}" + || error "repo-${action} %s %s " "${dbfile@Q}" "${pkgs[@]@Q}" set_repo_permission "${repo}" "${arch}" popd >/dev/null diff --git a/db-move b/db-move index e9c9fa7..e686d5a 100755 --- a/db-move +++ b/db-move @@ -4,7 +4,7 @@ . "$(dirname $0)/db-functions" if (( $# < 3 )); then - msg "usage: ${0##*/} ..." + msg "usage: %s ..." "${0##*/}" exit 1 fi @@ -74,7 +74,7 @@ for pkgbase in ${args[@]:2}; do else tarches=("${pkgarch}") fi - msg2 "${pkgbase} ($(echo ${tarches[@]}))" + msg2 "%s (%s)" "$pkgbase" "${tarches[*]}" pkgnames=($(. "${svnrepo_from}/PKGBUILD"; echo ${pkgname[@]})) pkgver=$(. "${svnrepo_from}/PKGBUILD"; get_full_version) diff --git a/db-remove b/db-remove index 08a95f9..b34cda5 100755 --- a/db-remove +++ b/db-remove @@ -4,7 +4,7 @@ . "$(dirname $0)/db-functions" if (( $# < 3 )); then - msg "usage: ${0##*/} ..." + msg "usage: %s ..." "${0##*/}" exit 1 fi diff --git a/db-repo-add b/db-repo-add index 4b16f7e..90e94cc 100755 --- a/db-repo-add +++ b/db-repo-add @@ -4,7 +4,7 @@ . "$(dirname $0)/db-functions" if (( $# < 3 )); then - msg "usage: ${0##*/} ..." + msg "usage: %s ..." "${0##*/}" exit 1 fi diff --git a/db-repo-remove b/db-repo-remove index 3077786..efe3828 100755 --- a/db-repo-remove +++ b/db-repo-remove @@ -4,7 +4,7 @@ . "$(dirname $0)/db-functions" if (( $# < 3 )); then - msg "usage: ${0##*/} ..." + msg "usage: %s ..." "${0##*/}" exit 1 fi diff --git a/db-update b/db-update index e8ae08d..b974a30 100755 --- a/db-update +++ b/db-update @@ -78,7 +78,7 @@ for repo in ${repos[@]}; do arch_pkgs=($(getpkgfiles "${STAGING}/${repo}/"*-${pkgarch}${PKGEXTS} 2>/dev/null)) for pkg in ${arch_pkgs[@]} ${any_pkgs[@]}; do pkgfile="${pkg##*/}" - msg2 "${pkgfile} (${pkgarch})" + msg2 '%s (%s)' "$pkgfile" "$pkgarch" # any packages might have been moved by the previous run if [[ -f ${pkg} ]]; then mv "${pkg}" "$FTP_BASE/${PKGPOOL}" diff --git a/testing2x b/testing2x index f0d77cb..d68e405 100755 --- a/testing2x +++ b/testing2x @@ -4,7 +4,7 @@ . "$(dirname $0)/db-functions" if (( $# < 1 )); then - msg "usage: ${0##*/} ..." + msg "usage: %s ..." "${0##*/}" exit 1 fi -- cgit v1.2.2 From 4ae3ea2f71344045fa4990c0524b4c662ab83cfc Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Thu, 22 Feb 2018 21:15:39 -0500 Subject: test: common.bash:__getCheckSum: Don't rely on IFS I managed to stumble across a bug in BATS where the run() function screwed with the global IFS. The bug has been fixed in git, but isn't in a release yet. https://github.com/sstephenson/bats/issues/89 Anyway, this bug breaks __getCheckSum(). Fortunately, we have avoided tripping it so far because luck has it that we never call __getCheckSum() after run() in the same test. So, there's nothing actually broken here, but it makes me nervous. So go ahead and modify __getCheckSum to not rely on IFS. And, while we're at it: declare the result variable and set it as separate commands. Doing both in the same command masks the exit code of the subshell expansion. We don't explicitly check the exit code, but BATS runs the test suite with `set -e`, so splitting it does mean that BATS will now detect errors from sha1sum. We don't really expect that to happen, but if BATS will give us error checking on it for free, why not? --- test/lib/common.bash | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/lib/common.bash b/test/lib/common.bash index 568a541..5411641 100644 --- a/test/lib/common.bash +++ b/test/lib/common.bash @@ -9,8 +9,9 @@ __updatePKGBUILD() { } __getCheckSum() { - local result=($(sha1sum $1)) - echo ${result[0]} + local result + result="$(sha1sum "$1")" + echo "${result%% *}" } __buildPackage() { -- cgit v1.2.2 From da49ea616e162d3cfcb4b6a4242f497e94c36564 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Thu, 22 Feb 2018 21:15:40 -0500 Subject: test: Fixup glob matching - ftpdir-cleanup.bats: Glob expansion does not occur in [[ -f ]] tests. The [[ ! -f .../${pkgname}-*${PKGEXT} ]] checks were checking that there were no files containing a literal '*' for that part of their name. Obviously, this isn't what was intended. - sourceballs.bats: [ -r ] checks explode if the glob returns >1 file. Avoid using them if the path being checked contains a glob. --- test/cases/ftpdir-cleanup.bats | 4 ++-- test/cases/sourceballs.bats | 4 ++-- test/lib/common.bash | 10 ++++++++++ 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/test/cases/ftpdir-cleanup.bats b/test/cases/ftpdir-cleanup.bats index fd485f3..8c713c6 100644 --- a/test/cases/ftpdir-cleanup.bats +++ b/test/cases/ftpdir-cleanup.bats @@ -13,8 +13,8 @@ __checkRepoRemovedPackage() { local pkgname for pkgname in $(__getPackageNamesFromPackageBase ${pkgbase}); do - [[ ! -f ${FTP_BASE}/${PKGPOOL}/${pkgname}-*${PKGEXT} ]] - [[ ! -f ${FTP_BASE}/${repo}/os/${repoarch}/${pkgname}-*${PKGEXT} ]] + ! __isGlobfile "${FTP_BASE}/${PKGPOOL}/${pkgname}"-*"${PKGEXT}" + ! __isGlobfile "${FTP_BASE}/${repo}/os/${repoarch}/${pkgname}"-*"${PKGEXT}" done } diff --git a/test/cases/sourceballs.bats b/test/cases/sourceballs.bats index a0a2999..df7ddd4 100644 --- a/test/cases/sourceballs.bats +++ b/test/cases/sourceballs.bats @@ -2,12 +2,12 @@ load ../lib/common __checkSourcePackage() { local pkgbase=$1 - [ -r ${FTP_BASE}/${SRCPOOL}/${pkgbase}-*${SRCEXT} ] + __isGlobfile "${FTP_BASE}/${SRCPOOL}/${pkgbase}"-*"${SRCEXT}" } __checkRemovedSourcePackage() { local pkgbase=$1 - [ ! -r ${FTP_BASE}/${SRCPOOL}/${pkgbase}-*${SRCEXT} ] + ! __isGlobfile "${FTP_BASE}/${SRCPOOL}/${pkgbase}"-*"${SRCEXT}" } @test "create simple package sourceballs" { diff --git a/test/lib/common.bash b/test/lib/common.bash index 5411641..f9057f0 100644 --- a/test/lib/common.bash +++ b/test/lib/common.bash @@ -14,6 +14,16 @@ __getCheckSum() { echo "${result%% *}" } +# Proxy function to check if a file exists. Using [[ -f ... ]] directly is not +# always wanted because we might want to expand bash globs first. This way we +# can pass unquoted globs to __isGlobfile() and have them expanded as function +# arguments before being checked. +# +# This is a copy of db-functions is_globfile +__isGlobfile() { + [[ -f $1 ]] +} + __buildPackage() { local pkgdest=${1:-.} local p -- cgit v1.2.2 From 5c867ea3b857cba0993fc18e064c6f02f70ee099 Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Fri, 16 Mar 2018 00:14:17 -0400 Subject: fixup! Update messages to make fuller use of printf formatters pkgs should be passed as a single argument --- db-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db-functions b/db-functions index d052740..e5b57a7 100644 --- a/db-functions +++ b/db-functions @@ -457,7 +457,7 @@ arch_repo_modify() { # package files for repo-add might be relative to repo dir pushd "${dbfile%/*}" >/dev/null /usr/bin/repo-${action} -q "${dbfile}" ${pkgs[@]} \ - || error "repo-${action} %s %s " "${dbfile@Q}" "${pkgs[@]@Q}" + || error "repo-${action} %s %s " "${dbfile@Q}" "${pkgs[*]@Q}" set_repo_permission "${repo}" "${arch}" popd >/dev/null -- cgit v1.2.2 From 41333413f19e025d800e02c83511b85bee65ac60 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Fri, 17 Jan 2014 13:06:53 -0500 Subject: Export TMPDIR, and use mktemp -t instead of making it part of the template --- config | 2 +- db-functions | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config b/config index c03fe50..10b9668 100644 --- a/config +++ b/config @@ -21,7 +21,7 @@ LOCK_DELAY=10 LOCK_TIMEOUT=300 STAGING="$HOME/staging" -TMPDIR="/var/tmp" +export TMPDIR="/var/tmp" ARCHES=(x86_64) DBEXT=".db.tar.gz" FILESEXT=".files.tar.gz" diff --git a/db-functions b/db-functions index e5b57a7..81c2ece 100644 --- a/db-functions +++ b/db-functions @@ -38,7 +38,7 @@ mv_acl() { } # set up general environment -WORKDIR=$(mktemp -d "${TMPDIR}/${0##*/}.XXXXXXXXXX") +WORKDIR=$(mktemp -dt "${0##*/}.XXXXXXXXXX") if [[ -n ${SVNUSER} ]]; then setfacl -m u:"${SVNUSER}":rwx "${WORKDIR}" setfacl -m d:u:"${USER}":rwx "${WORKDIR}" -- cgit v1.2.2 From 9672d6ec64ff5773df4b22bca9f2da44303dc03f Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 18 Jun 2014 21:04:02 -0400 Subject: Normalize to tab indent --- cron-jobs/check_archlinux/parse_pkgbuilds.sh | 10 +++++----- cron-jobs/devlist-mailer | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/cron-jobs/check_archlinux/parse_pkgbuilds.sh b/cron-jobs/check_archlinux/parse_pkgbuilds.sh index 10457fe..45f2a45 100755 --- a/cron-jobs/check_archlinux/parse_pkgbuilds.sh +++ b/cron-jobs/check_archlinux/parse_pkgbuilds.sh @@ -118,11 +118,11 @@ source_pkgbuild() { } find_pkgbuilds() { - #Skip over some dirs - local d="${1##*/}" - if [ "$d" = "CVS" -o "$d" = ".svn" ]; then - return - fi + #Skip over some dirs + local d="${1##*/}" + if [ "$d" = "CVS" -o "$d" = ".svn" ]; then + return + fi if [ -f $1/PKGBUILD ]; then source_pkgbuild $1 diff --git a/cron-jobs/devlist-mailer b/cron-jobs/devlist-mailer index 65a5483..712b38b 100755 --- a/cron-jobs/devlist-mailer +++ b/cron-jobs/devlist-mailer @@ -8,18 +8,18 @@ FROM="repomaint@archlinux.org" SUBJECT="Repository Maintenance $(date +"%d-%m-%Y")" if (( $# >= 1 )); then - SUBJECT="$1 $(date +"%d-%m-%Y")" + SUBJECT="$1 $(date +"%d-%m-%Y")" fi if (( $# >= 2 )); then - LIST="$2" + LIST="$2" fi stdin="$(cat)" #echo used to strip whitespace for checking for actual data if [[ -n "$(echo $stdin)" ]]; then -echo "Subject: $SUBJECT + echo "Subject: $SUBJECT To: $LIST From: $FROM -- cgit v1.2.2 From 97e17a5996425e8b7c3c1765a3c5074e1c4ff38d Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Sun, 17 Apr 2016 02:07:26 -0400 Subject: Add "#!/hint/bash" to the beginning of several files It is a method of notifying text-editors that a file is in Bash syntax without giving it a propper shebang (which would be confusing, as it would suggest that the file should be executable), as well as working across virtually all text-editors (unlike "-*- Mode: Bash -*-" or whatever). --- config | 2 ++ config.local.svn-community | 2 ++ config.local.svn-packages | 2 ++ db-functions | 2 +- 4 files changed, 7 insertions(+), 1 deletion(-) diff --git a/config b/config index 10b9668..e5db66a 100644 --- a/config +++ b/config @@ -1,3 +1,5 @@ +#!/hint/bash + FTP_BASE="/srv/ftp" SVNREPO='' SVNUSER='' diff --git a/config.local.svn-community b/config.local.svn-community index 105ea66..5d61b5e 100644 --- a/config.local.svn-community +++ b/config.local.svn-community @@ -1,3 +1,5 @@ +#!/hint/bash + PKGREPOS=('community' 'community-testing' 'community-staging' 'multilib' 'multilib-testing' 'multilib-staging') PKGPOOL='pool/community' SRCPOOL='sources/community' diff --git a/config.local.svn-packages b/config.local.svn-packages index 958a483..34aab35 100644 --- a/config.local.svn-packages +++ b/config.local.svn-packages @@ -1,3 +1,5 @@ +#!/hint/bash + PKGREPOS=('core' 'extra' 'testing' 'staging' 'kde-unstable' 'gnome-unstable') PKGPOOL='pool/packages' SRCPOOL='sources/packages' diff --git a/db-functions b/db-functions index 81c2ece..52faea2 100644 --- a/db-functions +++ b/db-functions @@ -1,4 +1,4 @@ -#!/bin/bash +#!/hint/bash . /usr/share/makepkg/util.sh -- cgit v1.2.2 From 23c2b82c336bf19b7a29a90d19bca4423d8b8839 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Sun, 17 Apr 2016 12:18:02 -0400 Subject: Don't use `grep -q` when operating on piped stdin (By default, prefer `grep &>/dev/null`) `grep -q` may exit as soon as it finds a match; this is a good optimization for when the input is a file. However, if the input is the output of another program, then that other program will receive SIGPIPE, and further writes will fail. When this happens, it might (bsdtar does) print a message about a "write error" to stderr. Which is going to confuse and alarm the user. In one of the cases (in common.bash, in the test suite), this had already been mitigated by wrapping bsdtar in "echo "$(bsdtar ...)", as Bash builtin echo doesn't complain if it gets SIGPIPE. However, that means we're storing the entire output of bsdtar in memory, which is silly. Additionally, the way it was implemented is also wrong; because it was being used with `grep -qv` instead of just `grep -q`, it *always* found a non-matching line (even something inconsequential like `%NAME%`), and *never* triggered a test failure. Looking at a few of these cases, it might also make sense to switch to using `bsdtar tf` instead of `bsdtar xf` when checking membership, but that's work for another day. --- db-functions | 6 +----- test/lib/common.bash | 6 ++++-- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/db-functions b/db-functions index 52faea2..4731f36 100644 --- a/db-functions +++ b/db-functions @@ -303,11 +303,7 @@ check_pkgfile() { in_array "${pkgarch}" ${ARCHES[@]} 'any' || return 1 - if echo "${pkgfile##*/}" | grep -q "^${pkgname}-${pkgver}-${pkgarch}"; then - return 0 - else - return 1 - fi + [[ ${pkgfile##*/} = ${pkgname}-${pkgver}-${pkgarch}* ]] } check_pkgsvn() { diff --git a/test/lib/common.bash b/test/lib/common.bash index f9057f0..36c735f 100644 --- a/test/lib/common.bash +++ b/test/lib/common.bash @@ -226,7 +226,7 @@ checkPackageDB() { for db in ${DBEXT} ${FILESEXT}; do [ -r "${FTP_BASE}/${repo}/os/${repoarch}/${repo}${db%.tar.*}" ] - bsdtar -xf "${FTP_BASE}/${repo}/os/${repoarch}/${repo}${db%.tar.*}" -O | grep -q "${pkgfile%${PKGEXT}}" + bsdtar -xf "${FTP_BASE}/${repo}/os/${repoarch}/${repo}${db%.tar.*}" -O | grep "${pkgfile%${PKGEXT}}" &>/dev/null done done done @@ -280,7 +280,9 @@ checkRemovedPackageDB() { for tarch in ${tarches[@]}; do if [ -r "${FTP_BASE}/${repo}/os/${tarch}/${repo}${db%.tar.*}" ]; then for pkgname in ${pkgnames[@]}; do - echo "$(bsdtar -xf "${FTP_BASE}/${repo}/os/${tarch}/${repo}${db%.tar.*}" -O)" | grep -qv ${pkgname} + if bsdtar -xf "${FTP_BASE}/${repo}/os/${tarch}/${repo}${db%.tar.*}" -O | grep ${pkgname} &>/dev/null; then + return 1 + fi done fi done -- cgit v1.2.2 From 36087fbd8b030fa6f908fdbb667292e3c078b615 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 14 Mar 2018 16:52:05 -0400 Subject: Remove uses of the "v=true; if $v ..." anti-pattern Instead, compare the value of $v to 'true'. --- cron-jobs/ftpdir-cleanup | 6 +++--- cron-jobs/sourceballs | 6 +++--- db-functions | 2 +- testing2x | 4 ++-- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index e24e614..d54cdec 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -7,7 +7,7 @@ clean_pkg() { local pkg local target - if ! ${CLEANUP_DRYRUN}; then + if [[ $CLEANUP_DRYRUN != true ]]; then for pkg in "$@"; do if [[ -h $pkg ]]; then rm -f "$pkg" "$pkg.sig" @@ -30,7 +30,7 @@ for repo in ${PKGREPOS[@]}; do done done -${CLEANUP_DRYRUN} && warning 'dry run mode is active' +[[ $CLEANUP_DRYRUN = true ]] && warning 'dry run mode is active' for repo in ${PKGREPOS[@]}; do for arch in ${ARCHES[@]}; do @@ -92,7 +92,7 @@ if (( ${#old_pkgs[@]} >= 1 )); then msg "Removing old packages from the cleanup directory..." for old_pkg in ${old_pkgs[@]}; do msg2 '%s' "${old_pkg}" - if ! ${CLEANUP_DRYRUN}; then + if [[ $CLEANUP_DRYRUN != true ]]; then rm -f "${CLEANUP_DESTDIR}/${old_pkg}" rm -f "${CLEANUP_DESTDIR}/${old_pkg}.sig" fi diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 7370594..5908758 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -127,10 +127,10 @@ old_pkgs=($(comm -23 "${WORKDIR}/available-src-pkgs.sort" "${WORKDIR}/expected-s if (( ${#old_pkgs[@]} >= 1 )); then msg "Removing old source packages..." - ${SOURCE_CLEANUP_DRYRUN} && warning 'dry run mode is active' + [[ $SOURCE_CLEANUP_DRYRUN = true ]] && warning 'dry run mode is active' for old_pkg in ${old_pkgs[@]}; do msg2 '%s' "${old_pkg}" - if ! ${SOURCE_CLEANUP_DRYRUN}; then + if [[ $SOURCE_CLEANUP_DRYRUN != true ]]; then mv_acl "$FTP_BASE/${SRCPOOL}/${old_pkg}" "${SOURCE_CLEANUP_DESTDIR}/${old_pkg}" touch "${SOURCE_CLEANUP_DESTDIR}/${old_pkg}" fi @@ -149,7 +149,7 @@ if (( ${#old_pkgs[@]} >= 1 )); then msg "Removing old source packages from the cleanup directory..." for old_pkg in ${old_pkgs[@]}; do msg2 '%s' "${old_pkg}" - ${SOURCE_CLEANUP_DRYRUN} || rm -f "${SOURCE_CLEANUP_DESTDIR}/${old_pkg}" + [[ $SOURCE_CLEANUP_DRYRUN = true ]] || rm -f "${SOURCE_CLEANUP_DESTDIR}/${old_pkg}" done fi diff --git a/db-functions b/db-functions index 4731f36..cb44ba9 100644 --- a/db-functions +++ b/db-functions @@ -139,7 +139,7 @@ repo_lock () { fi _count=0 - while (( _count <= _trial )) || $_lockblock ; do + while (( _count <= _trial )) || [[ $_lockblock = true ]]; do if ! mkdir "$LOCKDIR" >/dev/null 2>&1 ; then _owner="$(/usr/bin/stat -c %U $LOCKDIR)" warning "Repo [%s] (%s) is already locked by %s." "$1" "$2" "$_owner" diff --git a/testing2x b/testing2x index d68e405..f0f1e3a 100755 --- a/testing2x +++ b/testing2x @@ -30,7 +30,7 @@ for pkgbase in $*; do break fi done - ${found_source} || die "%s not found in [%s]" "$pkgbase" "$TESTING_REPO" + [[ $found_source = true ]] || die "%s not found in [%s]" "$pkgbase" "$TESTING_REPO" found_target=false for pkgarch in ${ARCHES[@]} 'any'; do for repo in ${STABLE_REPOS[@]}; do @@ -42,7 +42,7 @@ for pkgbase in $*; do fi done done - ${found_target} || die "%s not found in any of these repos: %s" "$pkgbase" "${STABLE_REPOS[*]}" + [[ $found_target = true ]] || die "%s not found in any of these repos: %s" "$pkgbase" "${STABLE_REPOS[*]}" fi done -- cgit v1.2.2 From 0432cffc42040f852b9a55c27316248da9064b67 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 18 Jun 2014 12:07:09 -0400 Subject: Fixups near unquoted variables Using the following command to find unquoted variables (and ignoring more than a a few false positives), grep -Prn --exclude-dir=.git '(?/dev/null || printf 'unset %q\n' "$var" done } @@ -72,6 +60,7 @@ print_info() { } source_pkgbuild() { + local restore_package_variables ret=0 dir=$1 pkgbuild=$dir/PKGBUILD @@ -93,7 +82,7 @@ source_pkgbuild() { echo -e "%INVALID%\n$pkgbuild\n" return 1 else - backup_package_variables + restore_package_variables=$(backup_package_variables) pkgname=$pkg while IFS= read -r line; do var=${line%%=*} @@ -106,7 +95,7 @@ source_pkgbuild() { done done < <(type package_${pkg}) print_info - restore_package_variables + eval "$restore_package_variables" fi done else diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 5908758..14947ab 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -56,19 +56,15 @@ done | sort -u > "${WORKDIR}/available-src-pkgs" for repo in ${PKGREPOS[@]}; do newpkgs=() failedpkgs=() - while read line; do - pkginfo=(${line}) - pkgbase=${pkginfo[0]} - pkgver=${pkginfo[1]} - pkgarch=${pkginfo[2]} - pkglicense=(${pkginfo[@]:3}) + while read -r pkgbase pkgver pkgarch pkglicense; do + read -ra pkglicense <<<"$pkglicense" # Should this package be skipped? if grep -Fqx "${pkgbase}" "${dirname}/sourceballs.skip"; then continue fi # Check if the license or .force file does not enforce creating a source package - if ! ([[ -z ${ALLOWED_LICENSES[@]} ]] || chk_license ${pkglicense[@]} || grep -Fqx "${pkgbase}" "${dirname}/sourceballs.force"); then + if ! ((( ${#ALLOWED_LICENSES[@]} == 0 )) || chk_license "${pkglicense[@]}" || grep -Fqx "${pkgbase}" "${dirname}/sourceballs.force"); then continue fi # Store the expected file name of the source package diff --git a/db-functions b/db-functions index cb44ba9..b4d8e7f 100644 --- a/db-functions +++ b/db-functions @@ -282,7 +282,7 @@ getpkgfile() { getpkgfiles() { local f files - if [[ ! -z "$(echo ${@%\.*} | sed "s/ /\n/g" | sort | uniq -D)" ]]; then + if ! printf '%s\n' "${@%\.*}" | awk 'a[$0]++{exit 1}'; then error 'Duplicate packages found!' exit 1 fi @@ -357,9 +357,7 @@ check_splitpkgs() { fi local svnnames=($(. "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}"; echo ${pkgname[@]})) - for svnname in ${svnnames[@]}; do - echo "${svnname}" >> "${repo}/${_pkgarch}/${_pkgbase}/svn" - done + printf '%s\n' "${svnnames[@]}" >> "${repo}/${_pkgarch}/${_pkgbase}/svn" done popd >/dev/null @@ -429,9 +427,9 @@ set_repo_permission() { if [[ -w ${dbfile} ]]; then local group=$(/usr/bin/stat --printf='%G' "$(dirname "${dbfile}")") - chgrp $group "${dbfile}" || error "Could not change group of %s to %s" "$dbfile" "$group" - chgrp $group "${filesfile}" || error "Could not change group of %s to %s" "$filesfile" "$group" - chmod g+w "${dbfile}" || error "Could not set write permission for group %s to %s" "$group" "$dbfile" + chgrp "$group" "${dbfile}" || error "Could not change group of %s to %s" "$dbfile" "$group" + chgrp "$group" "${filesfile}" || error "Could not change group of %s to %s" "$filesfile" "$group" + chmod g+w "${dbfile}" || error "Could not set write permission for group %s to %s" "$group" "$dbfile" chmod g+w "${filesfile}" || error "Could not set write permission for group %s to %s" "$group" "$filesfile" else error "You don't have permission to change %s" "$dbfile" diff --git a/test/fixtures/pkg-simple-a/PKGBUILD b/test/fixtures/pkg-simple-a/PKGBUILD index 329a4a3..eaa841b 100644 --- a/test/fixtures/pkg-simple-a/PKGBUILD +++ b/test/fixtures/pkg-simple-a/PKGBUILD @@ -4,7 +4,7 @@ pkgrel=1 pkgdesc="A package called ${pkgname}" arch=('i686' 'x86_64') url='http://www.archlinux.org/' -license=('GPL') +license=('GPL' 'LGPL') depends=('glibc') options=(!strip) diff --git a/testing2x b/testing2x index f0f1e3a..3ac687d 100755 --- a/testing2x +++ b/testing2x @@ -10,7 +10,7 @@ fi # Lock everything to reduce possibility of interfering task between the different repo-updates script_lock -for repo in ${TESTING_REPO} ${STABLE_REPOS[@]}; do +for repo in "${TESTING_REPO}" "${STABLE_REPOS[@]}"; do for pkgarch in ${ARCHES[@]}; do repo_lock ${repo} ${pkgarch} || exit 1 done @@ -18,7 +18,7 @@ done declare -A pkgs -for pkgbase in $*; do +for pkgbase in "$@"; do if [[ ! -d ${WORKDIR}/${pkgbase} ]]; then arch_svn export -q "${SVNREPO}/${pkgbase}/repos" "${WORKDIR}/${pkgbase}" >/dev/null -- cgit v1.2.2 From 8cc8e9cbbefdf6fac368031d3dacc74d001f23b7 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 18 Jun 2014 20:33:06 -0400 Subject: Clean up printf-formatters for user messages - db-functions: getpkgfile: The .sig file extension should be part of the filename parameter, not part of the message format string. - db-functions: arch_repo_modify: Shouldn't use ${action} string-interpolation in the message format string. Since the entire message is a command, and we're using @Q to escape arguments anyway, go ahead and just construct the entire command as a single string that way, and feed it to '%s'. --- db-functions | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/db-functions b/db-functions index b4d8e7f..a4a5522 100644 --- a/db-functions +++ b/db-functions @@ -273,7 +273,7 @@ getpkgfile() { error "Package %s not found!" "$1" exit 1 elif [[ ! -f ${1}.sig ]]; then - error "Package signature %s.sig not found!" "$1" + error "Package signature %s not found!" "$1.sig" exit 1 fi @@ -450,8 +450,8 @@ arch_repo_modify() { # package files for repo-add might be relative to repo dir pushd "${dbfile%/*}" >/dev/null - /usr/bin/repo-${action} -q "${dbfile}" ${pkgs[@]} \ - || error "repo-${action} %s %s " "${dbfile@Q}" "${pkgs[*]@Q}" + /usr/bin/repo-${action} -q "${dbfile}" "${pkgs[@]}" \ + || error '%s' "repo-${action} ${dbfile@Q} ${pkgs[*]@Q}" set_repo_permission "${repo}" "${arch}" popd >/dev/null -- cgit v1.2.2 From 4dbcc003b6aa8fcf009be1e98c8ff13fe1def2fb Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Thu, 15 Mar 2018 23:54:14 -0400 Subject: README.md: fix spelling mistake: "overriden"->"overridden" --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 753b3c1..7a94c43 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ ## Configuration * The default configuration can be found in `config`. * An optional `config.local` may override the default configuration. -* The path and name of the local configuration file can be overriden by setting the `DBSCRIPTS_CONFIG` environment variable. +* The path and name of the local configuration file can be overridden by setting the `DBSCRIPTS_CONFIG` environment variable. ## Testing * Install the `make` and `docker` packages. Start the docker daemon by issuing `systemctl start docker`. * The test suite can now be run with `make test`. -- cgit v1.2.2 From 7134da63690e0798d498fd15a09786db42091bfd Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Sun, 17 Apr 2016 18:56:14 -0400 Subject: README.md: Write an overview of the programs included --- README.md | 55 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/README.md b/README.md index 7a94c43..c672998 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,61 @@ * The default configuration can be found in `config`. * An optional `config.local` may override the default configuration. * The path and name of the local configuration file can be overridden by setting the `DBSCRIPTS_CONFIG` environment variable. +## Overview +The executables that you (might) care about are: + + dbscripts/ + ├── cron-jobs/ + │   ├── devlist-mailer + │   ├── ftpdir-cleanup + │   ├── integrity-check + │   ├── sourceballs + │   ├── update-web-db + │   └── update-web-files-db + ├── db-move + ├── db-remove + ├── db-repo-add + ├── db-repo-remove + ├── db-update + └── testing2x + +Ok, now let's talk about what those are. + +There are 3 "main" programs: + + - `db-update` : add packages to repositories + - `db-remove` : remove packages from repositories + - `db-move` : move packages from one repository to another + +Moving packages from testing to stable repositories is such a common +task that we have a wrapper around `db-move` to make it easier: + + - `testing2x` + +Of course, sometimes things go wrong, and you need to drop to a +lower-level, but you don't want to go all the way down to pacman's +`repo-add`/`repo-remove`. So, we have: + + - `db-repo-add` + - `db-repo-remove` + +Now, we'd like to be able to check that the repos are all OK, so we +have + + - `cron-jobs/integrity-check` + +When we remove a package from a repository, it stays in the package +"pool". We would like to be able to eventually remove packages from +the pool, to reclaim the disk space: + + - `cron-jobs/ftpdir-cleanup` + +Things that haven't been mentioned yet: + + - `cron-jobs/devlist-mailer` + - `cron-jobs/sourceballs` + - `cron-jobs/update-web-db` + - `cron-jobs/update-web-files-db` ## Testing * Install the `make` and `docker` packages. Start the docker daemon by issuing `systemctl start docker`. * The test suite can now be run with `make test`. -- cgit v1.2.2 From a852b2822620b6294e2efa411ff88a3b0afe1754 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 18 Jun 2014 12:07:09 -0400 Subject: test/: Verify that db-update won't release duplicate packages in the same transaction This is a leftover change from the patch that became 0432cff; v2 of that patch broke db-functions:getpkgfiles, but the testsuite didn't catch it. This patch adds a testcase that catches this type of breakage. --- test/cases/db-update.bats | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test/cases/db-update.bats b/test/cases/db-update.bats index e7e4489..ec58458 100644 --- a/test/cases/db-update.bats +++ b/test/cases/db-update.bats @@ -97,6 +97,13 @@ load ../lib/common [ "$status" -ne 0 ] } +@test "update duplicate package fails" { + PKGEXT=.pkg.tar.xz releasePackage extra pkg-any-a + PKGEXT=.pkg.tar.gz releasePackage extra pkg-any-a + run db-update + [ "$status" -ne 0 ] +} + @test "update same any package to different repositories fails" { local arch -- cgit v1.2.2 From 5b4b19cd7f4c0ef7a479306b73a4db3733df9c44 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Thu, 15 Mar 2018 21:47:58 -0400 Subject: Fixup quoting around variables, especially arrays Search for unquoted variables using the command: grep -Prn --exclude-dir=.git '(?/dev/null || ret=$? + source "$pkgbuild" &>/dev/null || ret=$? # ensure $pkgname and $pkgver variables were found if [ $ret -ne 0 -o -z "$pkgname" -o -z "$pkgver" ]; then @@ -77,8 +77,8 @@ source_pkgbuild() { if [ "${#pkgname[@]}" -gt "1" ]; then pkgbase=${pkgbase:-${pkgname[0]}} - for pkg in ${pkgname[@]}; do - if [ "$(type -t package_${pkg})" != "function" ]; then + for pkg in "${pkgname[@]}"; do + if [ "$(type -t "package_${pkg}")" != "function" ]; then echo -e "%INVALID%\n$pkgbuild\n" return 1 else @@ -87,13 +87,13 @@ source_pkgbuild() { while IFS= read -r line; do var=${line%%=*} var="${var#"${var%%[![:space:]]*}"}" # remove leading whitespace characters - for realvar in ${variables[@]}; do + for realvar in "${variables[@]}"; do if [ "$var" == "$realvar" ]; then eval $line break fi done - done < <(type package_${pkg}) + done < <(type "package_${pkg}") print_info eval "$restore_package_variables" fi @@ -113,14 +113,14 @@ find_pkgbuilds() { return fi - if [ -f $1/PKGBUILD ]; then - source_pkgbuild $1 + if [ -f "$1/PKGBUILD" ]; then + source_pkgbuild "$1" return fi empty=1 - for dir in $1/*; do - if [ -d $dir ]; then - find_pkgbuilds $dir + for dir in "$1"/*; do + if [ -d "$dir" ]; then + find_pkgbuilds "$dir" unset empty fi done @@ -136,7 +136,7 @@ fi CARCH=$1 shift for dir in "$@"; do - find_pkgbuilds $dir + find_pkgbuilds "$dir" done exit 0 diff --git a/cron-jobs/devlist-mailer b/cron-jobs/devlist-mailer index 712b38b..1b59cb3 100755 --- a/cron-jobs/devlist-mailer +++ b/cron-jobs/devlist-mailer @@ -23,6 +23,6 @@ if [[ -n "$(echo $stdin)" ]]; then To: $LIST From: $FROM -$stdin" | /usr/sbin/sendmail -F$FROM "$LIST" +$stdin" | /usr/sbin/sendmail -F"$FROM" "$LIST" fi diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index d54cdec..e742536 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -1,7 +1,7 @@ #!/bin/bash -. "$(dirname $0)/../config" -. "$(dirname $0)/../db-functions" +. "$(dirname "$0")/../config" +. "$(dirname "$0")/../db-functions" clean_pkg() { local pkg @@ -24,16 +24,16 @@ clean_pkg() { script_lock -for repo in ${PKGREPOS[@]}; do - for arch in ${ARCHES[@]}; do - repo_lock ${repo} ${arch} || exit 1 +for repo in "${PKGREPOS[@]}"; do + for arch in "${ARCHES[@]}"; do + repo_lock "${repo}" "${arch}" || exit 1 done done [[ $CLEANUP_DRYRUN = true ]] && warning 'dry run mode is active' -for repo in ${PKGREPOS[@]}; do - for arch in ${ARCHES[@]}; do +for repo in "${PKGREPOS[@]}"; do + for arch in "${ARCHES[@]}"; do if [[ ! -f ${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT} ]]; then continue fi @@ -49,7 +49,7 @@ for repo in ${PKGREPOS[@]}; do missing_pkgs=($(comm -13 "${WORKDIR}/repo-${repo}-${arch}" "${WORKDIR}/db-${repo}-${arch}")) if (( ${#missing_pkgs[@]} >= 1 )); then error "Missing packages in [%s] (%s)..." "$repo" "$arch" - for missing_pkg in ${missing_pkgs[@]}; do + for missing_pkg in "${missing_pkgs[@]}"; do msg2 '%s' "${missing_pkg}" done fi @@ -57,7 +57,7 @@ for repo in ${PKGREPOS[@]}; do old_pkgs=($(comm -23 "${WORKDIR}/repo-${repo}-${arch}" "${WORKDIR}/db-${repo}-${arch}")) if (( ${#old_pkgs[@]} >= 1 )); then msg "Removing old packages from [%s] (%s)..." "$repo" "$arch" - for old_pkg in ${old_pkgs[@]}; do + for old_pkg in "${old_pkgs[@]}"; do msg2 '%s' "${old_pkg}" clean_pkg "${FTP_BASE}/${repo}/os/${arch}/${old_pkg}" done @@ -75,7 +75,7 @@ cat "${WORKDIR}"/db-* 2>/dev/null | sort -u > "${WORKDIR}/db" old_pkgs=($(comm -23 "${WORKDIR}/pool" "${WORKDIR}/db")) if (( ${#old_pkgs[@]} >= 1 )); then msg "Removing old packages from package pool..." - for old_pkg in ${old_pkgs[@]}; do + for old_pkg in "${old_pkgs[@]}"; do msg2 '%s' "${old_pkg}" clean_pkg "$FTP_BASE/${PKGPOOL}/${old_pkg}" done @@ -90,7 +90,7 @@ for f in "${CLEANUP_DESTDIR}"/**/*${PKGEXTS}; do done if (( ${#old_pkgs[@]} >= 1 )); then msg "Removing old packages from the cleanup directory..." - for old_pkg in ${old_pkgs[@]}; do + for old_pkg in "${old_pkgs[@]}"; do msg2 '%s' "${old_pkg}" if [[ $CLEANUP_DRYRUN != true ]]; then rm -f "${CLEANUP_DESTDIR}/${old_pkg}" @@ -99,9 +99,9 @@ if (( ${#old_pkgs[@]} >= 1 )); then done fi -for repo in ${PKGREPOS[@]}; do - for arch in ${ARCHES[@]}; do - repo_unlock ${repo} ${arch} +for repo in "${PKGREPOS[@]}"; do + for arch in "${ARCHES[@]}"; do + repo_unlock "${repo}" "${arch}" done done diff --git a/cron-jobs/integrity-check b/cron-jobs/integrity-check index 47cf856..17153c7 100755 --- a/cron-jobs/integrity-check +++ b/cron-jobs/integrity-check @@ -1,6 +1,6 @@ #!/bin/bash -dirname="$(dirname $0)" +dirname="$(dirname "$0")" . "${dirname}/../config" . "${dirname}/../db-functions" @@ -13,12 +13,12 @@ fi mailto=$1 check() { - ${dirname}/check_archlinux/check_packages.py \ + "${dirname}"/check_archlinux/check_packages.py \ --repos="${repos}" \ --abs-tree="/srv/abs/rsync/${arch},/srv/abs/rsync/any" \ --repo-dir="${FTP_BASE}" \ --arch="${arch}" \ - 2>&1 | ${dirname}/devlist-mailer "Integrity Check ${arch}: ${repos}" "${mailto}" + 2>&1 | "${dirname}"/devlist-mailer "Integrity Check ${arch}: ${repos}" "${mailto}" } repos='core,extra,community,multilib' diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 14947ab..6be28ab 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -1,15 +1,15 @@ #!/bin/bash -dirname="$(dirname $(readlink -e $0))" +dirname="$(dirname "$(readlink -e "$0")")" . "${dirname}/../config" . "${dirname}/../db-functions" pushd "${WORKDIR}" >/dev/null script_lock -for repo in ${PKGREPOS[@]}; do - for arch in ${ARCHES[@]}; do - repo_lock ${repo} ${arch} || exit 1 +for repo in "${PKGREPOS[@]}"; do + for arch in "${ARCHES[@]}"; do + repo_lock "${repo}" "${arch}" || exit 1 done done @@ -18,8 +18,8 @@ renice +10 -p $$ > /dev/null # Create a readable file for each repo with the following format # - [ ] -for repo in ${PKGREPOS[@]}; do - for arch in ${ARCHES[@]}; do +for repo in "${PKGREPOS[@]}"; do + for arch in "${ARCHES[@]}"; do # Repo does not exist; skip it if [[ ! -f ${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT} ]]; then continue @@ -39,9 +39,9 @@ for repo in ${PKGREPOS[@]}; do done | sort -u > "${WORKDIR}/db-${repo}" done -for repo in ${PKGREPOS[@]}; do - for arch in ${ARCHES[@]}; do - repo_unlock ${repo} ${arch} +for repo in "${PKGREPOS[@]}"; do + for arch in "${ARCHES[@]}"; do + repo_unlock "${repo}" "${arch}" done done @@ -53,7 +53,7 @@ for f in "${FTP_BASE}"/${SRCPOOL}/*${SRCEXT}; do done | sort -u > "${WORKDIR}/available-src-pkgs" # Check for all packages if we need to build a source package -for repo in ${PKGREPOS[@]}; do +for repo in "${PKGREPOS[@]}"; do newpkgs=() failedpkgs=() while read -r pkgbase pkgver pkgarch pkglicense; do @@ -73,7 +73,7 @@ for repo in ${PKGREPOS[@]}; do # Build the source package if its not already there if ! grep -Fqx "${pkgbase}-${pkgver}${SRCEXT}" "${WORKDIR}/available-src-pkgs"; then # Check if we had failed before - if in_array "${pkgbase}-${pkgver}${SRCEXT}" ${failedpkgs[@]}; then + if in_array "${pkgbase}-${pkgver}${SRCEXT}" "${failedpkgs[@]}"; then continue fi @@ -104,13 +104,13 @@ for repo in ${PKGREPOS[@]}; do if [ ${#newpkgs[@]} -ge 1 ]; then msg "Adding source packages for [%s]..." "$repo" - for new_pkg in ${newpkgs[@]}; do + for new_pkg in "${newpkgs[@]}"; do msg2 '%s' "${new_pkg}" done fi if [ ${#failedpkgs[@]} -ge 1 ]; then msg "Failed to create source packages for [%s]..." "$repo" - for failed_pkg in ${failedpkgs[@]}; do + for failed_pkg in "${failedpkgs[@]}"; do msg2 '%s' "${failed_pkg}" done fi @@ -124,7 +124,7 @@ old_pkgs=($(comm -23 "${WORKDIR}/available-src-pkgs.sort" "${WORKDIR}/expected-s if (( ${#old_pkgs[@]} >= 1 )); then msg "Removing old source packages..." [[ $SOURCE_CLEANUP_DRYRUN = true ]] && warning 'dry run mode is active' - for old_pkg in ${old_pkgs[@]}; do + for old_pkg in "${old_pkgs[@]}"; do msg2 '%s' "${old_pkg}" if [[ $SOURCE_CLEANUP_DRYRUN != true ]]; then mv_acl "$FTP_BASE/${SRCPOOL}/${old_pkg}" "${SOURCE_CLEANUP_DESTDIR}/${old_pkg}" @@ -143,7 +143,7 @@ done if (( ${#old_pkgs[@]} >= 1 )); then msg "Removing old source packages from the cleanup directory..." - for old_pkg in ${old_pkgs[@]}; do + for old_pkg in "${old_pkgs[@]}"; do msg2 '%s' "${old_pkg}" [[ $SOURCE_CLEANUP_DRYRUN = true ]] || rm -f "${SOURCE_CLEANUP_DESTDIR}/${old_pkg}" done diff --git a/cron-jobs/update-web-db b/cron-jobs/update-web-db index c859ce0..87e8bb0 100755 --- a/cron-jobs/update-web-db +++ b/cron-jobs/update-web-db @@ -1,7 +1,7 @@ #!/bin/bash -. "$(dirname $0)/../config" -. "$(dirname $0)/../db-functions" +. "$(dirname "$0")/../config" +. "$(dirname "$0")/../db-functions" # setup paths SPATH="/srv/http/archweb" @@ -43,26 +43,26 @@ case "$cmd" in esac # Lock the repos and get a copy of the db files to work on -for repo in ${REPOS[@]}; do - for arch in ${ARCHES[@]}; do - repo_lock ${repo} ${arch} || exit 1 +for repo in "${REPOS[@]}"; do + for arch in "${ARCHES[@]}"; do + repo_lock "${repo}" "${arch}" || exit 1 dbfile="/srv/ftp/${repo}/os/${arch}/${repo}${dbfileext}" if [[ -f ${dbfile} ]]; then mkdir -p "${WORKDIR}/${repo}/${arch}" cp "${dbfile}" "${WORKDIR}/${repo}/${arch}/${repo}${dbfileext}" fi - repo_unlock ${repo} ${arch} + repo_unlock "${repo}" "${arch}" done done # Run reporead on our db copy -pushd $SPATH >/dev/null -for repo in ${REPOS[@]}; do - for arch in ${ARCHES[@]}; do +pushd "$SPATH" >/dev/null +for repo in "${REPOS[@]}"; do + for arch in "${ARCHES[@]}"; do dbcopy="${WORKDIR}/${repo}/${arch}/${repo}${dbfileext}" if [[ -f ${dbcopy} ]]; then echo "Updating ${repo}-${arch}" >> "${LOGOUT}" - ./manage.py reporead ${flags} ${arch} "${dbcopy}" >> "${LOGOUT}" 2>&1 + ./manage.py reporead "${flags}" "${arch}" "${dbcopy}" >> "${LOGOUT}" 2>&1 echo "" >> "${LOGOUT}" fi done diff --git a/db-functions b/db-functions index a4a5522..0491c22 100644 --- a/db-functions +++ b/db-functions @@ -17,7 +17,7 @@ set_umask () { } restore_umask () { - umask $UMASK >/dev/null + umask "$UMASK" >/dev/null } # Proxy function to check if a file exists. Using [[ -f ... ]] directly is not @@ -50,7 +50,7 @@ REPO_MODIFIED=0 script_lock() { local LOCKDIR="$TMPDIR/.scriptlock.${0##*/}" if ! mkdir "$LOCKDIR" >/dev/null 2>&1 ; then - local _owner="$(/usr/bin/stat -c %U $LOCKDIR)" + local _owner="$(/usr/bin/stat -c %U "$LOCKDIR")" error "Script %s is already locked by %s." "${0##*/}" "$_owner" exit 1 else @@ -78,12 +78,12 @@ cleanup() { local arch trap - EXIT INT QUIT TERM - for l in ${LOCKS[@]}; do + for l in "${LOCKS[@]}"; do repo=${l%.*} arch=${l#*.} if [[ -d $TMPDIR/.repolock.$repo.$arch ]]; then msg "Removing left over lock from [%s] (%s)" "$repo" "$arch" - repo_unlock $repo $arch + repo_unlock "$repo" "$arch" fi done if [[ -d $TMPDIR/.scriptlock.${0##*/} ]]; then @@ -96,7 +96,7 @@ cleanup() { date +%s > "${FTP_BASE}/lastupdate" fi - [[ -n $1 ]] && exit $1 + [[ -n $1 ]] && exit "$1" } abort() { @@ -125,7 +125,7 @@ repo_lock () { # This is the lock file used by repo-add and repo-remove if [[ -f ${DBLOCKFILE} ]]; then - error "Repo [%s] (%s) is already locked by repo-{add,remove} process %s" "$1" "$2" "$(cat $DBLOCKFILE)" + error "Repo [%s] (%s) is already locked by repo-{add,remove} process %s" "$1" "$2" "$(cat "$DBLOCKFILE")" return 1 fi @@ -141,7 +141,7 @@ repo_lock () { _count=0 while (( _count <= _trial )) || [[ $_lockblock = true ]]; do if ! mkdir "$LOCKDIR" >/dev/null 2>&1 ; then - _owner="$(/usr/bin/stat -c %U $LOCKDIR)" + _owner="$(/usr/bin/stat -c %U "$LOCKDIR")" warning "Repo [%s] (%s) is already locked by %s." "$1" "$2" "$_owner" msg2 "Retrying in %s seconds..." "$LOCK_DELAY" else @@ -149,7 +149,7 @@ repo_lock () { set_umask return 0 fi - sleep $LOCK_DELAY + sleep "$LOCK_DELAY" let _count=$_count+1 done @@ -277,7 +277,7 @@ getpkgfile() { exit 1 fi - echo ${1} + echo "${1}" } getpkgfiles() { @@ -291,30 +291,30 @@ getpkgfiles() { files+=("$(getpkgfile "$f")") || exit 1 done - echo ${files[@]} + echo "${files[@]}" } check_pkgfile() { local pkgfile=$1 - local pkgname="$(getpkgname ${pkgfile})" || return 1 - local pkgver="$(getpkgver ${pkgfile})" || return 1 - local pkgarch="$(getpkgarch ${pkgfile})" || return 1 + local pkgname="$(getpkgname "${pkgfile}")" || return 1 + local pkgver="$(getpkgver "${pkgfile}")" || return 1 + local pkgarch="$(getpkgarch "${pkgfile}")" || return 1 - in_array "${pkgarch}" ${ARCHES[@]} 'any' || return 1 + in_array "${pkgarch}" "${ARCHES[@]}" 'any' || return 1 - [[ ${pkgfile##*/} = ${pkgname}-${pkgver}-${pkgarch}* ]] + [[ ${pkgfile##*/} = "${pkgname}-${pkgver}-${pkgarch}"* ]] } check_pkgsvn() { local pkgfile="${1}" - local _pkgbase="$(getpkgbase ${pkgfile})" || return 1 - local _pkgname="$(getpkgname ${pkgfile})" || return 1 - local _pkgver="$(getpkgver ${pkgfile})" || return 1 - local _pkgarch="$(getpkgarch ${pkgfile})" || return 1 + local _pkgbase="$(getpkgbase "${pkgfile}")" || return 1 + local _pkgname="$(getpkgname "${pkgfile}")" || return 1 + local _pkgver="$(getpkgver "${pkgfile}")" || return 1 + local _pkgarch="$(getpkgarch "${pkgfile}")" || return 1 local repo="${2}" - in_array "${repo}" ${PKGREPOS[@]} || return 1 + in_array "${repo}" "${PKGREPOS[@]}" || return 1 if [[ ! -f ${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase} ]]; then mkdir -p "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}" @@ -325,8 +325,8 @@ check_pkgsvn() { local svnver="$(. "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}"; get_full_version)" [[ "${svnver}" = "${_pkgver}" ]] || return 1 - local svnnames=($(. "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}"; echo ${pkgname[@]})) - in_array "${_pkgname}" ${svnnames[@]} || return 1 + local svnnames=($(. "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}"; echo "${pkgname[@]}")) + in_array "${_pkgname}" "${svnnames[@]}" || return 1 return 0 } @@ -334,7 +334,7 @@ check_pkgsvn() { check_splitpkgs() { local repo="${1}" shift - local pkgfiles=(${@}) + local pkgfiles=("${@}") local pkgfile local pkgdir local svnname @@ -342,11 +342,11 @@ check_splitpkgs() { mkdir -p "${WORKDIR}/check_splitpkgs/" pushd "${WORKDIR}/check_splitpkgs" >/dev/null - for pkgfile in ${pkgfiles[@]}; do + for pkgfile in "${pkgfiles[@]}"; do issplitpkg "${pkgfile}" || continue - local _pkgbase="$(getpkgbase ${pkgfile})" - local _pkgname="$(getpkgname ${pkgfile})" - local _pkgarch="$(getpkgarch ${pkgfile})" + local _pkgbase="$(getpkgbase "${pkgfile}")" + local _pkgname="$(getpkgname "${pkgfile}")" + local _pkgarch="$(getpkgarch "${pkgfile}")" mkdir -p "${repo}/${_pkgarch}/${_pkgbase}" echo "${_pkgname}" >> "${repo}/${_pkgarch}/${_pkgbase}/staging" @@ -356,7 +356,7 @@ check_splitpkgs() { "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}" >/dev/null || return 1 fi - local svnnames=($(. "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}"; echo ${pkgname[@]})) + local svnnames=($(. "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}"; echo "${pkgname[@]}")) printf '%s\n' "${svnnames[@]}" >> "${repo}/${_pkgarch}/${_pkgbase}/svn" done popd >/dev/null @@ -376,9 +376,9 @@ check_splitpkgs() { check_pkgrepos() { local pkgfile=$1 - local pkgname="$(getpkgname ${pkgfile})" || return 1 - local pkgver="$(getpkgver ${pkgfile})" || return 1 - local pkgarch="$(getpkgarch ${pkgfile})" || return 1 + local pkgname="$(getpkgname "${pkgfile}")" || return 1 + local pkgver="$(getpkgver "${pkgfile}")" || return 1 + local pkgarch="$(getpkgarch "${pkgfile}")" || return 1 is_globfile "${FTP_BASE}/${PKGPOOL}/${pkgname}-${pkgver}-${pkgarch}"${PKGEXTS} && return 1 is_globfile "${FTP_BASE}/${PKGPOOL}/${pkgname}-${pkgver}-${pkgarch}"${PKGEXTS}.sig && return 1 @@ -392,7 +392,7 @@ check_pkgrepos() { chk_license() { local l for l in "${@}"; do - in_array ${l} ${ALLOWED_LICENSES[@]} && return 0 + in_array "${l}" "${ALLOWED_LICENSES[@]}" && return 0 done return 1 @@ -404,12 +404,12 @@ check_repo_permission() { (( ${#PKGREPOS[@]} == 0 )) && return 1 [[ -z "${PKGPOOL}" ]] && return 1 - in_array "${repo}" ${PKGREPOS[@]} || return 1 + in_array "${repo}" "${PKGREPOS[@]}" || return 1 [[ -w $FTP_BASE/${PKGPOOL} ]] || return 1 local arch - for arch in ${ARCHES[@]}; do + for arch in "${ARCHES[@]}"; do local dir="${FTP_BASE}/${repo}/os/${arch}/" [[ -w ${dir} ]] || return 1 [[ -f ${dir}${repo}${DBEXT} && ! -w ${dir}${repo}${DBEXT} ]] && return 1 @@ -450,7 +450,7 @@ arch_repo_modify() { # package files for repo-add might be relative to repo dir pushd "${dbfile%/*}" >/dev/null - /usr/bin/repo-${action} -q "${dbfile}" "${pkgs[@]}" \ + /usr/bin/"repo-${action}" -q "${dbfile}" "${pkgs[@]}" \ || error '%s' "repo-${action} ${dbfile@Q} ${pkgs[*]@Q}" set_repo_permission "${repo}" "${arch}" popd >/dev/null diff --git a/db-move b/db-move index e686d5a..1fa2055 100755 --- a/db-move +++ b/db-move @@ -1,37 +1,37 @@ #!/bin/bash -. "$(dirname $0)/config" -. "$(dirname $0)/db-functions" +. "$(dirname "$0")/config" +. "$(dirname "$0")/db-functions" if (( $# < 3 )); then msg "usage: %s ..." "${0##*/}" exit 1 fi -args=(${@}) +args=("${@}") repo_from="${args[0]}" repo_to="${args[1]}" ftppath_from="${FTP_BASE}/${repo_from}/os/" ftppath_to="${FTP_BASE}/${repo_to}/os/" -if ! check_repo_permission $repo_to || ! check_repo_permission $repo_from; then +if ! check_repo_permission "$repo_to" || ! check_repo_permission "$repo_from"; then die "You don't have permission to move packages from %s to %s" "$repo_from" "$repo_to" fi # TODO: this might lock too much (architectures) -for pkgarch in ${ARCHES[@]}; do - repo_lock ${repo_to} ${pkgarch} || exit 1 - repo_lock ${repo_from} ${pkgarch} || exit 1 +for pkgarch in "${ARCHES[@]}"; do + repo_lock "${repo_to}" "${pkgarch}" || exit 1 + repo_lock "${repo_from}" "${pkgarch}" || exit 1 done # check if packages to be moved exist in svn and ftp dir arch_svn checkout -q -N "${SVNREPO}" "${WORKDIR}/svn" >/dev/null -for pkgbase in ${args[@]:2}; do +for pkgbase in "${args[@]:2}"; do arch_svn up -q "${WORKDIR}/svn/${pkgbase}" >/dev/null - for pkgarch in ${ARCHES[@]} 'any'; do + for pkgarch in "${ARCHES[@]}" 'any'; do svnrepo_from="${WORKDIR}/svn/${pkgbase}/repos/${repo_from}-${pkgarch}" if [[ -r ${svnrepo_from}/PKGBUILD ]]; then - pkgnames=($(. "${svnrepo_from}/PKGBUILD"; echo ${pkgname[@]})) + pkgnames=($(. "${svnrepo_from}/PKGBUILD"; echo "${pkgname[@]}")) if (( ${#pkgnames[@]} < 1 )); then die "Could not read pkgname" fi @@ -41,15 +41,15 @@ for pkgbase in ${args[@]:2}; do die "Could not read pkgver" fi - if [[ ${pkgarch} = any ]]; then - tarches=(${ARCHES[@]}) + if [[ "${pkgarch}" = any ]]; then + tarches=("${ARCHES[@]}") else tarches=("${pkgarch}") fi - for pkgname in ${pkgnames[@]}; do - for tarch in ${tarches[@]}; do - getpkgfile "${ftppath_from}/${tarch}/"${pkgname}-${pkgver}-${pkgarch}${PKGEXTS} >/dev/null + for pkgname in "${pkgnames[@]}"; do + for tarch in "${tarches[@]}"; do + getpkgfile "${ftppath_from}/${tarch}/${pkgname}-${pkgver}-${pkgarch}"${PKGEXTS} >/dev/null done done continue 2 @@ -62,20 +62,20 @@ msg "Moving packages from [%s] to [%s]..." "$repo_from" "$repo_to" declare -A add_pkgs declare -A remove_pkgs -for pkgbase in ${args[@]:2}; do +for pkgbase in "${args[@]:2}"; do tag_list="" - for pkgarch in ${ARCHES[@]} 'any'; do + for pkgarch in "${ARCHES[@]}" 'any'; do svnrepo_from="${WORKDIR}/svn/${pkgbase}/repos/${repo_from}-${pkgarch}" svnrepo_to="${WORKDIR}/svn/${pkgbase}/repos/${repo_to}-${pkgarch}" if [[ -f ${svnrepo_from}/PKGBUILD ]]; then if [[ ${pkgarch} = any ]]; then - tarches=(${ARCHES[@]}) + tarches=("${ARCHES[@]}") else tarches=("${pkgarch}") fi msg2 "%s (%s)" "$pkgbase" "${tarches[*]}" - pkgnames=($(. "${svnrepo_from}/PKGBUILD"; echo ${pkgname[@]})) + pkgnames=($(. "${svnrepo_from}/PKGBUILD"; echo "${pkgname[@]}")) pkgver=$(. "${svnrepo_from}/PKGBUILD"; get_full_version) if [[ -d ${svnrepo_to} ]]; then @@ -93,14 +93,14 @@ for pkgbase in ${args[@]:2}; do arch_svn rm --force -q "${svnrepo_from}" tag_list+=", $pkgarch" - for pkgname in ${pkgnames[@]}; do - for tarch in ${tarches[@]}; do - pkgpath=$(getpkgfile "${ftppath_from}/${tarch}/"${pkgname}-${pkgver}-${pkgarch}${PKGEXTS}) + for pkgname in "${pkgnames[@]}"; do + for tarch in "${tarches[@]}"; do + pkgpath=$(getpkgfile "${ftppath_from}/${tarch}/${pkgname}-${pkgver}-${pkgarch}"${PKGEXTS}) pkgfile="${pkgpath##*/}" - ln -s "../../../${PKGPOOL}/${pkgfile}" ${ftppath_to}/${tarch}/ + ln -s "../../../${PKGPOOL}/${pkgfile}" "${ftppath_to}/${tarch}/" if [[ -f ${FTP_BASE}/${PKGPOOL}/${pkgfile}.sig ]]; then - ln -s "../../../${PKGPOOL}/${pkgfile}.sig" ${ftppath_to}/${tarch}/ + ln -s "../../../${PKGPOOL}/${pkgfile}.sig" "${ftppath_to}/${tarch}/" fi add_pkgs[${tarch}]+="${FTP_BASE}/${PKGPOOL}/${pkgfile} " remove_pkgs[${tarch}]+="${pkgname} " @@ -112,14 +112,14 @@ for pkgbase in ${args[@]:2}; do arch_svn commit -q "${WORKDIR}/svn/${pkgbase}" -m "${0##*/}: moved ${pkgbase} from [${repo_from}] to [${repo_to}] (${tag_list})" done -for tarch in ${ARCHES[@]}; do +for tarch in "${ARCHES[@]}"; do if [[ -n ${add_pkgs[${tarch}]} ]]; then arch_repo_modify add "${repo_to}" "${tarch}" ${add_pkgs[${tarch}]} arch_repo_modify remove "${repo_from}" "${tarch}" ${remove_pkgs[${tarch}]} fi done -for pkgarch in ${ARCHES[@]}; do - repo_unlock ${repo_from} ${pkgarch} - repo_unlock ${repo_to} ${pkgarch} +for pkgarch in "${ARCHES[@]}"; do + repo_unlock "${repo_from}" "${pkgarch}" + repo_unlock "${repo_to}" "${pkgarch}" done diff --git a/db-remove b/db-remove index b34cda5..e42a977 100755 --- a/db-remove +++ b/db-remove @@ -1,7 +1,7 @@ #!/bin/bash -. "$(dirname $0)/config" -. "$(dirname $0)/db-functions" +. "$(dirname "$0")/config" +. "$(dirname "$0")/db-functions" if (( $# < 3 )); then msg "usage: %s ..." "${0##*/}" @@ -10,27 +10,27 @@ fi repo="$1" arch="$2" -pkgbases=(${@:3}) +pkgbases=("${@:3}") ftppath="$FTP_BASE/$repo/os" svnrepo="$repo-$arch" -if ! check_repo_permission $repo; then +if ! check_repo_permission "$repo"; then die "You don't have permission to remove packages from %s" "$repo" fi if [[ $arch = any ]]; then - tarches=(${ARCHES[@]}) + tarches=("${ARCHES[@]}") else tarches=("$arch") fi -for tarch in ${tarches[@]}; do - repo_lock $repo $tarch || exit 1 +for tarch in "${tarches[@]}"; do + repo_lock "$repo" "$tarch" || exit 1 done remove_pkgs=() -for pkgbase in ${pkgbases[@]}; do +for pkgbase in "${pkgbases[@]}"; do msg "Removing %s from [%s]..." "$pkgbase" "$repo" arch_svn checkout -q "${SVNREPO}/${pkgbase}" "${WORKDIR}/svn/${pkgbase}" >/dev/null @@ -46,7 +46,7 @@ for pkgbase in ${pkgbases[@]}; do fi done -for tarch in ${tarches[@]}; do - arch_repo_modify remove "${repo}" "${tarch}" ${remove_pkgs[@]} - repo_unlock $repo $tarch +for tarch in "${tarches[@]}"; do + arch_repo_modify remove "${repo}" "${tarch}" "${remove_pkgs[@]}" + repo_unlock "$repo" "$tarch" done diff --git a/db-repo-add b/db-repo-add index 90e94cc..1cc63af 100755 --- a/db-repo-add +++ b/db-repo-add @@ -1,7 +1,7 @@ #!/bin/bash -. "$(dirname $0)/config" -. "$(dirname $0)/db-functions" +. "$(dirname "$0")/config" +. "$(dirname "$0")/db-functions" if (( $# < 3 )); then msg "usage: %s ..." "${0##*/}" @@ -10,32 +10,32 @@ fi repo="$1" arch="$2" -pkgfiles=(${@:3}) +pkgfiles=("${@:3}") ftppath="$FTP_BASE/$repo/os" -if ! check_repo_permission $repo; then +if ! check_repo_permission "$repo"; then die "You don't have permission to add packages to %s" "$repo" fi if [[ $arch = any ]]; then - tarches=(${ARCHES[@]}) + tarches=("${ARCHES[@]}") else tarches=("$arch") fi -for tarch in ${tarches[@]}; do - repo_lock $repo $tarch || exit 1 +for tarch in "${tarches[@]}"; do + repo_lock "$repo" "$tarch" || exit 1 done -for tarch in ${tarches[@]}; do - for pkgfile in ${pkgfiles[@]}; do +for tarch in "${tarches[@]}"; do + for pkgfile in "${pkgfiles[@]}"; do if [[ ! -f ${FTP_BASE}/${repo}/os/${tarch}/${pkgfile##*/} ]]; then die "Package file %s not found in %s" "${pkgfile##*/}" "${FTP_BASE}/${repo}/os/${tarch}/" else msg "Adding %s to [%s]..." "$pkgfile" "$repo" fi done - arch_repo_modify add "${repo}" "${tarch}" ${pkgfiles[@]} - repo_unlock $repo $tarch + arch_repo_modify add "${repo}" "${tarch}" "${pkgfiles[@]}" + repo_unlock "$repo" "$tarch" done diff --git a/db-repo-remove b/db-repo-remove index efe3828..40c1ac8 100755 --- a/db-repo-remove +++ b/db-repo-remove @@ -1,7 +1,7 @@ #!/bin/bash -. "$(dirname $0)/config" -. "$(dirname $0)/db-functions" +. "$(dirname "$0")/config" +. "$(dirname "$0")/db-functions" if (( $# < 3 )); then msg "usage: %s ..." "${0##*/}" @@ -10,28 +10,28 @@ fi repo="$1" arch="$2" -pkgnames=(${@:3}) +pkgnames=("${@:3}") ftppath="$FTP_BASE/$repo/os" -if ! check_repo_permission $repo; then +if ! check_repo_permission "$repo"; then die "You don't have permission to remove packages from %s" "$repo" fi if [[ $arch = any ]]; then - tarches=(${ARCHES[@]}) + tarches=("${ARCHES[@]}") else tarches=("$arch") fi -for tarch in ${tarches[@]}; do - repo_lock $repo $tarch || exit 1 +for tarch in "${tarches[@]}"; do + repo_lock "$repo" "$tarch" || exit 1 done -for tarch in ${tarches[@]}; do - for pkgname in ${pkgnames[@]}; do +for tarch in "${tarches[@]}"; do + for pkgname in "${pkgnames[@]}"; do msg "Removing %s from [%s]..." "$pkgname" "$repo" done - arch_repo_modify remove "${repo}" "${tarch}" ${pkgnames[@]} - repo_unlock $repo $tarch + arch_repo_modify remove "${repo}" "${tarch}" "${pkgnames[@]}" + repo_unlock "$repo" "$tarch" done diff --git a/db-update b/db-update index b974a30..536f7fd 100755 --- a/db-update +++ b/db-update @@ -1,7 +1,7 @@ #!/bin/bash -. "$(dirname $0)/config" -. "$(dirname $0)/db-functions" +. "$(dirname "$0")/config" +. "$(dirname "$0")/db-functions" if (( $# >= 1 )); then warning "Calling %s with a specific repository is no longer supported" "${0##*/}" @@ -16,27 +16,27 @@ mapfile -t -d '' staging_repos < <( ) repos=() -for staging_repo in ${staging_repos[@]##*/}; do - if in_array ${staging_repo} ${PKGREPOS[@]}; then - repos+=(${staging_repo}) +for staging_repo in "${staging_repos[@]##*/}"; do + if in_array "${staging_repo}" "${PKGREPOS[@]}"; then + repos+=("${staging_repo}") fi done # TODO: this might lock too much (architectures) -for repo in ${repos[@]}; do - for pkgarch in ${ARCHES[@]}; do - repo_lock ${repo} ${pkgarch} || exit 1 +for repo in "${repos[@]}"; do + for pkgarch in "${ARCHES[@]}"; do + repo_lock "${repo}" "${pkgarch}" || exit 1 done done # check if packages are valid -for repo in ${repos[@]}; do +for repo in "${repos[@]}"; do if ! check_repo_permission "${repo}"; then die "You don't have permission to update packages in %s" "$repo" fi pkgs=($(getpkgfiles "${STAGING}/${repo}/"*${PKGEXTS})) if (( $? == 0 )); then - for pkg in ${pkgs[@]}; do + for pkg in "${pkgs[@]}"; do if [[ -h ${pkg} ]]; then die "Package %s is a symbolic link" "$repo/${pkg##*/}" fi @@ -62,7 +62,7 @@ for repo in ${repos[@]}; do die "Package %s was not built in a chroot" "$repo/${pkg##*/}" fi done - if ! check_splitpkgs ${repo} ${pkgs[@]}; then + if ! check_splitpkgs "${repo}" "${pkgs[@]}"; then die "Missing split packages for %s" "$repo" fi else @@ -70,13 +70,13 @@ for repo in ${repos[@]}; do fi done -for repo in ${repos[@]}; do +for repo in "${repos[@]}"; do msg "Updating [%s]..." "$repo" any_pkgs=($(getpkgfiles "${STAGING}/${repo}/"*-any${PKGEXTS} 2>/dev/null)) - for pkgarch in ${ARCHES[@]}; do + for pkgarch in "${ARCHES[@]}"; do add_pkgs=() - arch_pkgs=($(getpkgfiles "${STAGING}/${repo}/"*-${pkgarch}${PKGEXTS} 2>/dev/null)) - for pkg in ${arch_pkgs[@]} ${any_pkgs[@]}; do + arch_pkgs=($(getpkgfiles "${STAGING}/${repo}/"*"-${pkgarch}"${PKGEXTS} 2>/dev/null)) + for pkg in "${arch_pkgs[@]}" "${any_pkgs[@]}"; do pkgfile="${pkg##*/}" msg2 '%s (%s)' "$pkgfile" "$pkgarch" # any packages might have been moved by the previous run @@ -99,8 +99,8 @@ for repo in ${repos[@]}; do done done -for repo in ${repos[@]}; do - for pkgarch in ${ARCHES[@]}; do - repo_unlock ${repo} ${pkgarch} +for repo in "${repos[@]}"; do + for pkgarch in "${ARCHES[@]}"; do + repo_unlock "${repo}" "${pkgarch}" done done diff --git a/testing2x b/testing2x index 3ac687d..c2bd6d4 100755 --- a/testing2x +++ b/testing2x @@ -1,7 +1,7 @@ #!/bin/bash -. "$(dirname $0)/config" -. "$(dirname $0)/db-functions" +. "$(dirname "$0")/config" +. "$(dirname "$0")/db-functions" if (( $# < 1 )); then msg "usage: %s ..." "${0##*/}" @@ -11,8 +11,8 @@ fi # Lock everything to reduce possibility of interfering task between the different repo-updates script_lock for repo in "${TESTING_REPO}" "${STABLE_REPOS[@]}"; do - for pkgarch in ${ARCHES[@]}; do - repo_lock ${repo} ${pkgarch} || exit 1 + for pkgarch in "${ARCHES[@]}"; do + repo_lock "${repo}" "${pkgarch}" || exit 1 done done @@ -23,7 +23,7 @@ for pkgbase in "$@"; do arch_svn export -q "${SVNREPO}/${pkgbase}/repos" "${WORKDIR}/${pkgbase}" >/dev/null found_source=false - for pkgarch in ${ARCHES[@]} 'any'; do + for pkgarch in "${ARCHES[@]}" 'any'; do svnrepo_from="${WORKDIR}/${pkgbase}/${TESTING_REPO}-${pkgarch}" if [[ -r ${svnrepo_from}/PKGBUILD ]]; then found_source=true @@ -32,8 +32,8 @@ for pkgbase in "$@"; do done [[ $found_source = true ]] || die "%s not found in [%s]" "$pkgbase" "$TESTING_REPO" found_target=false - for pkgarch in ${ARCHES[@]} 'any'; do - for repo in ${STABLE_REPOS[@]}; do + for pkgarch in "${ARCHES[@]}" 'any'; do + for repo in "${STABLE_REPOS[@]}"; do svnrepo_to="${WORKDIR}/${pkgbase}/${repo}-${pkgarch}" if [[ -r ${svnrepo_to}/PKGBUILD ]]; then found_target=true @@ -46,15 +46,15 @@ for pkgbase in "$@"; do fi done -for pkgarch in ${ARCHES[@]}; do - repo_unlock ${TESTING_REPO} ${pkgarch} +for pkgarch in "${ARCHES[@]}"; do + repo_unlock "${TESTING_REPO}" "${pkgarch}" done -for repo in ${STABLE_REPOS[@]}; do - for pkgarch in ${ARCHES[@]}; do - repo_unlock ${repo} ${pkgarch} +for repo in "${STABLE_REPOS[@]}"; do + for pkgarch in "${ARCHES[@]}"; do + repo_unlock "${repo}" "${pkgarch}" done if [[ -n ${pkgs[${repo}]} ]]; then - "$(dirname $0)/db-move" ${TESTING_REPO} "${repo}" ${pkgs[${repo}]} + "$(dirname "$0")/db-move" "${TESTING_REPO}" "${repo}" ${pkgs[${repo}]} fi done -- cgit v1.2.2 From 0cf2ac49c18a71c050f5a9a9ff42e8dff89c2375 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Tue, 17 Jun 2014 23:47:21 -0400 Subject: Consistently use "$(dirname "$(readlink -e "$0")")" This does correct handling of - executing a program by symlink - any weird characters in the full path - I'm sure there's another case I thought about when I originally did this. --- cron-jobs/ftpdir-cleanup | 4 ++-- cron-jobs/integrity-check | 2 +- cron-jobs/update-web-db | 4 ++-- db-move | 4 ++-- db-remove | 4 ++-- db-repo-add | 4 ++-- db-repo-remove | 4 ++-- db-update | 4 ++-- testing2x | 6 +++--- 9 files changed, 18 insertions(+), 18 deletions(-) diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index e742536..9df5f99 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -1,7 +1,7 @@ #!/bin/bash -. "$(dirname "$0")/../config" -. "$(dirname "$0")/../db-functions" +. "$(dirname "$(readlink -e "$0")")/../config" +. "$(dirname "$(readlink -e "$0")")/../db-functions" clean_pkg() { local pkg diff --git a/cron-jobs/integrity-check b/cron-jobs/integrity-check index 17153c7..904dfc1 100755 --- a/cron-jobs/integrity-check +++ b/cron-jobs/integrity-check @@ -1,6 +1,6 @@ #!/bin/bash -dirname="$(dirname "$0")" +dirname="$(dirname "$(readlink -e "$0")")" . "${dirname}/../config" . "${dirname}/../db-functions" diff --git a/cron-jobs/update-web-db b/cron-jobs/update-web-db index 87e8bb0..39ed765 100755 --- a/cron-jobs/update-web-db +++ b/cron-jobs/update-web-db @@ -1,7 +1,7 @@ #!/bin/bash -. "$(dirname "$0")/../config" -. "$(dirname "$0")/../db-functions" +. "$(dirname "$(readlink -e "$0")")/../config" +. "$(dirname "$(readlink -e "$0")")/../db-functions" # setup paths SPATH="/srv/http/archweb" diff --git a/db-move b/db-move index 1fa2055..63e5c14 100755 --- a/db-move +++ b/db-move @@ -1,7 +1,7 @@ #!/bin/bash -. "$(dirname "$0")/config" -. "$(dirname "$0")/db-functions" +. "$(dirname "$(readlink -e "$0")")/config" +. "$(dirname "$(readlink -e "$0")")/db-functions" if (( $# < 3 )); then msg "usage: %s ..." "${0##*/}" diff --git a/db-remove b/db-remove index e42a977..ac9a168 100755 --- a/db-remove +++ b/db-remove @@ -1,7 +1,7 @@ #!/bin/bash -. "$(dirname "$0")/config" -. "$(dirname "$0")/db-functions" +. "$(dirname "$(readlink -e "$0")")/config" +. "$(dirname "$(readlink -e "$0")")/db-functions" if (( $# < 3 )); then msg "usage: %s ..." "${0##*/}" diff --git a/db-repo-add b/db-repo-add index 1cc63af..6109d84 100755 --- a/db-repo-add +++ b/db-repo-add @@ -1,7 +1,7 @@ #!/bin/bash -. "$(dirname "$0")/config" -. "$(dirname "$0")/db-functions" +. "$(dirname "$(readlink -e "$0")")/config" +. "$(dirname "$(readlink -e "$0")")/db-functions" if (( $# < 3 )); then msg "usage: %s ..." "${0##*/}" diff --git a/db-repo-remove b/db-repo-remove index 40c1ac8..8d2c6c7 100755 --- a/db-repo-remove +++ b/db-repo-remove @@ -1,7 +1,7 @@ #!/bin/bash -. "$(dirname "$0")/config" -. "$(dirname "$0")/db-functions" +. "$(dirname "$(readlink -e "$0")")/config" +. "$(dirname "$(readlink -e "$0")")/db-functions" if (( $# < 3 )); then msg "usage: %s ..." "${0##*/}" diff --git a/db-update b/db-update index 536f7fd..8eb27df 100755 --- a/db-update +++ b/db-update @@ -1,7 +1,7 @@ #!/bin/bash -. "$(dirname "$0")/config" -. "$(dirname "$0")/db-functions" +. "$(dirname "$(readlink -e "$0")")/config" +. "$(dirname "$(readlink -e "$0")")/db-functions" if (( $# >= 1 )); then warning "Calling %s with a specific repository is no longer supported" "${0##*/}" diff --git a/testing2x b/testing2x index c2bd6d4..d3b096b 100755 --- a/testing2x +++ b/testing2x @@ -1,7 +1,7 @@ #!/bin/bash -. "$(dirname "$0")/config" -. "$(dirname "$0")/db-functions" +. "$(dirname "$(readlink -e "$0")")/config" +. "$(dirname "$(readlink -e "$0")")/db-functions" if (( $# < 1 )); then msg "usage: %s ..." "${0##*/}" @@ -54,7 +54,7 @@ for repo in "${STABLE_REPOS[@]}"; do repo_unlock "${repo}" "${pkgarch}" done if [[ -n ${pkgs[${repo}]} ]]; then - "$(dirname "$0")/db-move" "${TESTING_REPO}" "${repo}" ${pkgs[${repo}]} + "$(dirname "$(readlink -e "$0")")/db-move" "${TESTING_REPO}" "${repo}" ${pkgs[${repo}]} fi done -- cgit v1.2.2 From 656fd6d5d8f455492b1e51add9e6e1da8f911a7c Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Mon, 25 May 2015 01:00:46 -0600 Subject: devlist-mailer: Make LIST and FROM configurable --- config | 5 +++++ cron-jobs/devlist-mailer | 5 ++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/config b/config index d703e7b..fd79b6e 100644 --- a/config +++ b/config @@ -35,6 +35,11 @@ PKGEXTS=".pkg.tar.@(gz|bz2|xz|lzo|lrz|Z)" # Allowed licenses: get sourceballs only for licenses in this array ALLOWED_LICENSES=('GPL' 'GPL1' 'GPL2' 'GPL3' 'LGPL' 'LGPL1' 'LGPL2' 'LGPL2.1' 'LGPL3') +# Where to send error emails, and who they are from +LIST="arch-dev-public@archlinux.org" +#LIST="aaronmgriffin@gmail.com" +FROM="repomaint@archlinux.org" + # Override default config with config.local LOCAL_CONFIG=${DBSCRIPTS_CONFIG:-"$(dirname "${BASH_SOURCE[0]}")/config.local"} [[ -f "${LOCAL_CONFIG}" ]] && . "${LOCAL_CONFIG}" diff --git a/cron-jobs/devlist-mailer b/cron-jobs/devlist-mailer index 1b59cb3..578d23e 100755 --- a/cron-jobs/devlist-mailer +++ b/cron-jobs/devlist-mailer @@ -2,9 +2,8 @@ #Dummy helper to send email to arch-dev # It does nothing if no output -LIST="arch-dev-public@archlinux.org" -#LIST="aaronmgriffin@gmail.com" -FROM="repomaint@archlinux.org" +# Load $LIST and $FROM from the config file +. "$(dirname "$(readlink -e "$0")")/../config" SUBJECT="Repository Maintenance $(date +"%d-%m-%Y")" if (( $# >= 1 )); then -- cgit v1.2.2 From 2c68135f4ab9db048db460bc949322ad7bb418eb Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Wed, 21 Mar 2018 12:09:21 -0400 Subject: pacman 5.1 compatibility The print_all_package_names function changed its behavior, adapt --- test/lib/common.bash | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/test/lib/common.bash b/test/lib/common.bash index 36c735f..738cc97 100644 --- a/test/lib/common.bash +++ b/test/lib/common.bash @@ -30,7 +30,7 @@ __buildPackage() { local cache local pkgarches local tarch - local pkgnames + local pkgfiles if [[ -n ${BUILDDIR} ]]; then cache=${BUILDDIR}/$(__getCheckSum PKGBUILD) @@ -45,14 +45,14 @@ __buildPackage() { for tarch in ${pkgarches[@]}; do if [ "${tarch}" == 'any' ]; then PKGDEST=${pkgdest} PKGEXT=${PKGEXT} makepkg -c + mapfile -tO "${#pkgfiles[@]}" pkgfiles < <(PKGDEST=${pkgdest} PKGEXT=${PKGEXT} makepkg --packagelist) else PKGDEST=${pkgdest} PKGEXT=${PKGEXT} CARCH=${tarch} makepkg -c + mapfile -tO "${#pkgfiles[@]}" pkgfiles < <(PKGDEST=${pkgdest} PKGEXT=${PKGEXT} CARCH=${tarch} makepkg --packagelist) fi done - pkgnames=($(. PKGBUILD; print_all_package_names)) - pushd ${pkgdest} - for p in ${pkgnames[@]/%/${PKGEXT}}; do + for p in ${pkgfiles[@]}; do # Manually sign packages as "makepkg --sign" is buggy gpg -v --detach-sign --no-armor --use-agent ${p} @@ -60,7 +60,6 @@ __buildPackage() { cp -Lv ${p}{,.sig} ${cache}/ fi done - popd } __archrelease() { -- cgit v1.2.2 From 05dd9be0db06d7f7cea0eff9f9f6f1993eb6d463 Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Mon, 28 May 2018 19:45:15 -0400 Subject: test: BUILDDIR must be owned by build user pacman 5.1 enforces this restriction. OTOH it is a simpler setup to set this as the user homedir directly in account creation (just like makechrootpkg has always done) than to create an additional, world-writable, directory. dockerfile: don't use tmpfs for /build --- Makefile | 2 +- test/Dockerfile | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 677bc39..988d335 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ IMAGE:=dbscripts/test -RUN_OPTIONS:=--rm --network=none -v $(PWD):/dbscripts:ro --tmpfs=/tmp:exec --tmpfs=/build -w /dbscripts/test +RUN_OPTIONS:=--rm --network=none -v $(PWD):/dbscripts:ro --tmpfs=/tmp:exec -w /dbscripts/test test-image: docker build --pull -t $(IMAGE) test diff --git a/test/Dockerfile b/test/Dockerfile index 3913f00..83c8449 100644 --- a/test/Dockerfile +++ b/test/Dockerfile @@ -2,8 +2,7 @@ FROM archlinux/base RUN pacman -Syu --noconfirm --needed sudo fakeroot awk subversion make kcov bash-bats gettext grep RUN pacman-key --init RUN echo '%wheel ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/wheel -RUN useradd -N -g users -G wheel -m tester -RUN install -d -m 1777 /build +RUN useradd -N -g users -G wheel -d /build -m tester USER tester RUN echo -e "\ Key-Type: RSA\n\ -- cgit v1.2.2 From c802479271d694608116bc33e7913322ce82511f Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Wed, 4 Apr 2018 16:22:20 -0400 Subject: test: Fix typoed variables - ftpdir-cleanup: ${pkg[0]} -> ${pkgs[0]} This mistake was introduced in 7628525156110022fa70ad91e4bc13ee8a3cceb0 - test: common.bash: $pkgarches[@] -> ${pkgarches[@]} This mistake was introduced in 7628525156110022fa70ad91e4bc13ee8a3cceb0 --- test/cases/ftpdir-cleanup.bats | 2 +- test/lib/common.bash | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/cases/ftpdir-cleanup.bats b/test/cases/ftpdir-cleanup.bats index 8c713c6..74215d0 100644 --- a/test/cases/ftpdir-cleanup.bats +++ b/test/cases/ftpdir-cleanup.bats @@ -111,7 +111,7 @@ __checkRepoRemovedPackage() { ftpdir-cleanup for arch in ${arches[@]}; do - __checkRepoRemovedPackage extra ${pkg[0]} ${arch} + __checkRepoRemovedPackage extra ${pkgs[0]} ${arch} done checkRemovedPackage extra ${pkgs[0]} diff --git a/test/lib/common.bash b/test/lib/common.bash index 738cc97..b843817 100644 --- a/test/lib/common.bash +++ b/test/lib/common.bash @@ -272,7 +272,7 @@ checkRemovedPackageDB() { if [[ ${pkgarches[@]} == any ]]; then tarches=(${ARCHES[@]}) else - tarches=($pkgarches[@]) + tarches=(${pkgarches[@]}) fi for db in ${DBEXT} ${FILESEXT}; do -- cgit v1.2.2 From 37954a353fef96c82990e0d80982d700574cb26c Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Thu, 21 Jun 2018 22:01:54 -0400 Subject: test: db-move: Verify that it works on single-arch packages --- test/cases/db-move.bats | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/test/cases/db-move.bats b/test/cases/db-move.bats index df3dbdf..27fe8d1 100644 --- a/test/cases/db-move.bats +++ b/test/cases/db-move.bats @@ -39,6 +39,25 @@ load ../lib/common done } +@test "move single-arch packages" { + local arches=('i686' 'x86_64') + local pkgs=('pkg-single-arch' 'pkg-simple-b') + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + releasePackage testing ${pkgbase} + done + + db-update + + db-move testing extra pkg-single-arch + + checkRemovedPackage testing pkg-single-arch + checkPackage extra pkg-single-arch + checkPackage testing pkg-simple-b +} + @test "move epoch packages" { local arches=('i686' 'x86_64') local pkgs=('pkg-simple-epoch') -- cgit v1.2.2 From f563f6b1be3e2df1c482d29ae451dd608e53ef0e Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Sat, 16 Jun 2018 00:03:40 -0400 Subject: db-move, db-remove: Don't parse PKGBUILD files Don't try to parse PKGBUILD files from SVN; all of the information we need is already in the DBEXT files. Several programs use [[ -f PKGBUILD ]] or [[ -r PKGBUILD ]] on files from SVN; those checks can stay, just remove all instances of actually trying to *parse* those files. As an exception, don't modify parse_pkgbuilds.sh (which is called by check_packages.py, which is called by cron-jobs/integrity-check). Most of the attributes we need have been present in the DBEXT files for as long as `repo-add` has been part of pacman: attribute | git | git date | ver | ver date ----------+----------+------------+-------+----------- FILENAME | aa1c0ba9 | 2006-11-20 | 3.0.0 | 2007-03-25 NAME | aa1c0ba9 | 2006-11-20 | 3.0.0 | 2007-03-25 VERSION | aa1c0ba9 | 2006-11-20 | 3.0.0 | 2007-03-25 ARCH | aa1c0ba9 | 2006-11-20 | 3.0.0 | 2007-03-25 BASE | 4b21504f | 2009-07-22 | 3.3.0 | 2009-08-02 However, `BASE` for split packages is slightly newer, and its presence relies not only on the `repo-add` version, but also the `makepkg` version used to build the package. As of 2018-06-15, the oldest package in any of the Arch Linux repos is community/gimp-refocus, built on 2013-07-22; I don't believe it is necessary to handle packages from before that change was made (before `BASE` was set). "Ignore space change" might be useful when viewing this diff. --- db-functions | 38 +++++++++++++++++++++-- db-move | 99 +++++++++++++++++++++--------------------------------------- db-remove | 11 +++---- 3 files changed, 74 insertions(+), 74 deletions(-) diff --git a/db-functions b/db-functions index 0491c22..69f35b4 100644 --- a/db-functions +++ b/db-functions @@ -6,9 +6,6 @@ shopt -s extglob globstar nullglob -# Some PKGBUILDs need CARCH to be set -CARCH="x86_64" - # Useful functions UMASK="" set_umask () { @@ -294,6 +291,41 @@ getpkgfiles() { echo "${files[@]}" } +# usage: getdbinfo repo arch fields +getdbinfo() { + local repo=$1 + local arch=$2 + local pkgbase=$3 + local fields=$4 + + local dbfile="${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" + local dbdir="${WORKDIR}/dbfiles/${repo}-${arch}" + if ! [[ -d $dbdir ]]; then + mkdir -p "$dbdir" + bsdtar -xf "$dbfile" -C "$dbdir" --include='*/desc' + fi + + # The grep/xargs is just a fast filter with possible + # false-postives (like a Bloom filter), because awk is + # comparatively slow. You could remove the grep/xargs line, + # and append `"$dbdir"/*/desc` to the awk command, and it + # would do the same thing; just several times slower. + grep -r -lZ -Fx -e "$pkgbase" -- "$dbdir" | xargs -r0 \ + awk -v pkgbase="$pkgbase" -v fields="$fields" -vFS='\n' -vRS='' ' + { dat[gensub(/^%(.*)%$/, "\\1", 1, $1)] = gensub(FS, ",", "g", gensub($1 FS, "", 1)) } + ENDFILE { + if (dat["BASE"] == pkgbase || dat["NAME"] == pkgbase) { + n=split(fields, fieldlist, ",") + str = "" + for (i=1; i<=n; i++) + str = str (i==1?"":" ") dat[fieldlist[i]] + print str + } + delete dat + } + ' +} + check_pkgfile() { local pkgfile=$1 diff --git a/db-move b/db-move index 63e5c14..03debfc 100755 --- a/db-move +++ b/db-move @@ -28,32 +28,14 @@ done arch_svn checkout -q -N "${SVNREPO}" "${WORKDIR}/svn" >/dev/null for pkgbase in "${args[@]:2}"; do arch_svn up -q "${WORKDIR}/svn/${pkgbase}" >/dev/null - for pkgarch in "${ARCHES[@]}" 'any'; do - svnrepo_from="${WORKDIR}/svn/${pkgbase}/repos/${repo_from}-${pkgarch}" - if [[ -r ${svnrepo_from}/PKGBUILD ]]; then - pkgnames=($(. "${svnrepo_from}/PKGBUILD"; echo "${pkgname[@]}")) - if (( ${#pkgnames[@]} < 1 )); then - die "Could not read pkgname" + for tarch in "${ARCHES[@]}"; do + while read -r pkgarch pkgfile; do + svnrepo_from="${WORKDIR}/svn/${pkgbase}/repos/${repo_from}-${pkgarch}" + if [[ -r ${svnrepo_from}/PKGBUILD ]]; then + getpkgfile "${ftppath_from}/${tarch}/${pkgfile}" >/dev/null + continue 3 fi - - pkgver=$(. "${svnrepo_from}/PKGBUILD"; get_full_version) - if [[ -z ${pkgver} ]]; then - die "Could not read pkgver" - fi - - if [[ "${pkgarch}" = any ]]; then - tarches=("${ARCHES[@]}") - else - tarches=("${pkgarch}") - fi - - for pkgname in "${pkgnames[@]}"; do - for tarch in "${tarches[@]}"; do - getpkgfile "${ftppath_from}/${tarch}/${pkgname}-${pkgver}-${pkgarch}"${PKGEXTS} >/dev/null - done - done - continue 2 - fi + done < <(getdbinfo "$repo_from" "$tarch" "$pkgbase" ARCH,FILENAME) done die "%s not found in %s" "$pkgbase" "$repo_from" done @@ -64,49 +46,36 @@ declare -A add_pkgs declare -A remove_pkgs for pkgbase in "${args[@]:2}"; do tag_list="" - for pkgarch in "${ARCHES[@]}" 'any'; do - svnrepo_from="${WORKDIR}/svn/${pkgbase}/repos/${repo_from}-${pkgarch}" - svnrepo_to="${WORKDIR}/svn/${pkgbase}/repos/${repo_to}-${pkgarch}" - - if [[ -f ${svnrepo_from}/PKGBUILD ]]; then - if [[ ${pkgarch} = any ]]; then - tarches=("${ARCHES[@]}") - else - tarches=("${pkgarch}") - fi - msg2 "%s (%s)" "$pkgbase" "${tarches[*]}" - pkgnames=($(. "${svnrepo_from}/PKGBUILD"; echo "${pkgname[@]}")) - pkgver=$(. "${svnrepo_from}/PKGBUILD"; get_full_version) - - if [[ -d ${svnrepo_to} ]]; then - for file in $(arch_svn ls "${svnrepo_to}"); do - arch_svn rm -q "${svnrepo_to}/$file@" + for tarch in "${ARCHES[@]}"; do + while read -r pkgname pkgver pkgarch pkgfile; do + svnrepo_from="${WORKDIR}/svn/${pkgbase}/repos/${repo_from}-${pkgarch}" + svnrepo_to="${WORKDIR}/svn/${pkgbase}/repos/${repo_to}-${pkgarch}" + if [[ -f ${svnrepo_from}/PKGBUILD ]]; then + msg2 "%s (%s)" "$pkgbase" "$pkgarch" + + if [[ -d ${svnrepo_to} ]]; then + for file in $(arch_svn ls "${svnrepo_to}"); do + arch_svn rm -q "${svnrepo_to}/$file@" + done + else + mkdir "${svnrepo_to}" + arch_svn add -q "${svnrepo_to}" + fi + + for file in $(arch_svn ls "${svnrepo_from}"); do + arch_svn mv -q -r HEAD "${svnrepo_from}/$file@" "${svnrepo_to}/" done - else - mkdir "${svnrepo_to}" - arch_svn add -q "${svnrepo_to}" + arch_svn rm --force -q "${svnrepo_from}" + tag_list+=", $pkgarch" fi - for file in $(arch_svn ls "${svnrepo_from}"); do - arch_svn mv -q -r HEAD "${svnrepo_from}/$file@" "${svnrepo_to}/" - done - arch_svn rm --force -q "${svnrepo_from}" - tag_list+=", $pkgarch" - - for pkgname in "${pkgnames[@]}"; do - for tarch in "${tarches[@]}"; do - pkgpath=$(getpkgfile "${ftppath_from}/${tarch}/${pkgname}-${pkgver}-${pkgarch}"${PKGEXTS}) - pkgfile="${pkgpath##*/}" - - ln -s "../../../${PKGPOOL}/${pkgfile}" "${ftppath_to}/${tarch}/" - if [[ -f ${FTP_BASE}/${PKGPOOL}/${pkgfile}.sig ]]; then - ln -s "../../../${PKGPOOL}/${pkgfile}.sig" "${ftppath_to}/${tarch}/" - fi - add_pkgs[${tarch}]+="${FTP_BASE}/${PKGPOOL}/${pkgfile} " - remove_pkgs[${tarch}]+="${pkgname} " - done - done - fi + ln -s "../../../${PKGPOOL}/${pkgfile}" "${ftppath_to}/${tarch}/" + if [[ -f ${FTP_BASE}/${PKGPOOL}/${pkgfile}.sig ]]; then + ln -s "../../../${PKGPOOL}/${pkgfile}.sig" "${ftppath_to}/${tarch}/" + fi + add_pkgs[${tarch}]+="${FTP_BASE}/${PKGPOOL}/${pkgfile} " + remove_pkgs[${tarch}]+="${pkgname} " + done < <(getdbinfo "$repo_from" "$tarch" "$pkgbase" NAME,VERSION,ARCH,FILENAME) done tag_list="${tag_list#, }" arch_svn commit -q "${WORKDIR}/svn/${pkgbase}" -m "${0##*/}: moved ${pkgbase} from [${repo_from}] to [${repo_to}] (${tag_list})" diff --git a/db-remove b/db-remove index ac9a168..3017026 100755 --- a/db-remove +++ b/db-remove @@ -32,17 +32,16 @@ done remove_pkgs=() for pkgbase in "${pkgbases[@]}"; do msg "Removing %s from [%s]..." "$pkgbase" "$repo" - arch_svn checkout -q "${SVNREPO}/${pkgbase}" "${WORKDIR}/svn/${pkgbase}" >/dev/null + mapfile -t pkgnames < <(getdbinfo "$repo" "${tarches[0]}" "$pkgbase" NAME) + remove_pkgs+=("${pkgnames[@]}") + + arch_svn checkout -q "${SVNREPO}/${pkgbase}" "${WORKDIR}/svn/${pkgbase}" >/dev/null if [[ -d ${WORKDIR}/svn/$pkgbase/repos/$svnrepo ]]; then - remove_pkgs+=($(. "${WORKDIR}/svn/$pkgbase/repos/$svnrepo/PKGBUILD"; echo ${pkgname[@]})) arch_svn rm --force -q "${WORKDIR}/svn/$pkgbase/repos/$svnrepo" arch_svn commit -q "${WORKDIR}/svn/$pkgbase" -m "${0##*/}: $pkgbase removed by $(id -un)" else - warning "%s not found in %s" "$pkgbase" "$svnrepo" - warning "Removing only %s from the repo" "$pkgbase" - warning "If it was a split package you have to remove the others yourself!" - remove_pkgs+=("$pkgbase") + warning "pkgbase '%s' not found in svn; unable to commit removal to svn" "$pkgbase" fi done -- cgit v1.2.2 From 400ca768221dbfca1555ec011adb3ca64b6c0f3a Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Sun, 17 Jun 2018 11:28:20 -0400 Subject: db-move: Clarify the flow of the preflight check This should have the same result, but be a bit less confusing to read. Also, have a slightly more specific error message on a missing PKGBUILD. --- db-move | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/db-move b/db-move index 03debfc..2627b9a 100755 --- a/db-move +++ b/db-move @@ -28,16 +28,23 @@ done arch_svn checkout -q -N "${SVNREPO}" "${WORKDIR}/svn" >/dev/null for pkgbase in "${args[@]:2}"; do arch_svn up -q "${WORKDIR}/svn/${pkgbase}" >/dev/null + found=false for tarch in "${ARCHES[@]}"; do while read -r pkgarch pkgfile; do + svnrepo_from="${WORKDIR}/svn/${pkgbase}/repos/${repo_from}-${pkgarch}" - if [[ -r ${svnrepo_from}/PKGBUILD ]]; then - getpkgfile "${ftppath_from}/${tarch}/${pkgfile}" >/dev/null - continue 3 + if ! [[ -r ${svnrepo_from}/PKGBUILD ]]; then + die "%s not found in %s-%s" "$pkgbase" "$repo_from" "$pkgarch" fi + + # getpkgfile will `exit` for us if it fails; + # no need to check its result + getpkgfile "${ftppath_from}/${tarch}/${pkgfile}" >/dev/null + + found=true done < <(getdbinfo "$repo_from" "$tarch" "$pkgbase" ARCH,FILENAME) done - die "%s not found in %s" "$pkgbase" "$repo_from" + [[ $found = true ]] || die "%s not found in %s" "$pkgbase" "$repo_from" done msg "Moving packages from [%s] to [%s]..." "$repo_from" "$repo_to" -- cgit v1.2.2 From 7600e7b98e721e3d0baecf0c8fd5812ae6eb4540 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Sun, 17 Jun 2018 11:43:45 -0400 Subject: testing2x: Discover correct repos based on the DBEXT files, not SVN testing2x needs to: 1. Verify that the new version of the package is found in TESTING_REPO 2. Determine which of STABLE_REPOS the old version of the package is in Currently, to do those things, it checks for the existence of PKGBUILD files in SVN. That information is already stored in the DBEXT files; get it from there, instead of talking to SVN. "Ignore space change" might be useful when viewing this diff. --- testing2x | 40 +++++++++++++++++----------------------- 1 file changed, 17 insertions(+), 23 deletions(-) diff --git a/testing2x b/testing2x index d3b096b..f455a93 100755 --- a/testing2x +++ b/testing2x @@ -19,31 +19,25 @@ done declare -A pkgs for pkgbase in "$@"; do - if [[ ! -d ${WORKDIR}/${pkgbase} ]]; then - arch_svn export -q "${SVNREPO}/${pkgbase}/repos" "${WORKDIR}/${pkgbase}" >/dev/null - - found_source=false - for pkgarch in "${ARCHES[@]}" 'any'; do - svnrepo_from="${WORKDIR}/${pkgbase}/${TESTING_REPO}-${pkgarch}" - if [[ -r ${svnrepo_from}/PKGBUILD ]]; then - found_source=true - break + found_source=false + for tarch in "${ARCHES[@]}"; do + if [[ -n $(getdbinfo "$TESTING_REPO" "$tarch" "$pkgbase" NAME) ]]; then + found_source=true + break + fi + done + [[ $found_source = true ]] || die "%s not found in [%s]" "$pkgbase" "$TESTING_REPO" + found_target=false + for tarch in "${ARCHES[@]}"; do + for repo in "${STABLE_REPOS[@]}"; do + if [[ -n $(getdbinfo "$repo" "$tarch" "$pkgbase" NAME) ]]; then + found_target=true + pkgs[${repo}]+="${pkgbase} " + break 2 fi done - [[ $found_source = true ]] || die "%s not found in [%s]" "$pkgbase" "$TESTING_REPO" - found_target=false - for pkgarch in "${ARCHES[@]}" 'any'; do - for repo in "${STABLE_REPOS[@]}"; do - svnrepo_to="${WORKDIR}/${pkgbase}/${repo}-${pkgarch}" - if [[ -r ${svnrepo_to}/PKGBUILD ]]; then - found_target=true - pkgs[${repo}]+="${pkgbase} " - break 2 - fi - done - done - [[ $found_target = true ]] || die "%s not found in any of these repos: %s" "$pkgbase" "${STABLE_REPOS[*]}" - fi + done + [[ $found_target = true ]] || die "%s not found in any of these repos: %s" "$pkgbase" "${STABLE_REPOS[*]}" done for pkgarch in "${ARCHES[@]}"; do -- cgit v1.2.2 From b954d619658ba55385d75b4c5d543de6db6904a7 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Sun, 17 Jun 2018 11:42:27 -0400 Subject: Centralize all SVN access into 'abs_*' functions in a new 'db-abs' file Moving all SVN code in to a separate file means both that: 1. It is easier to identify the interactions with SVN, when considering a replacement. 2. It is easier to swap out the one file if/when replacing SVN with something else. Put another way: try to be less tightly coupled with SVN. --- cron-jobs/sourceballs | 4 +- db-abs | 106 ++++++++++++++++++++++++++++++++++++++++++++++++++ db-functions | 23 ++++------- db-move | 31 ++------------- db-remove | 9 +---- 5 files changed, 121 insertions(+), 52 deletions(-) create mode 100644 db-abs diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 6be28ab..f56f0cf 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -79,8 +79,8 @@ for repo in "${PKGREPOS[@]}"; do # Get the sources from svn mkdir -p -m0770 "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}" - arch_svn export -q "${SVNREPO}/${pkgbase}/repos/${repo}-${pkgarch}" \ - "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" >/dev/null 2>&1 + abs_export "$repo" "$pkgarch" "$pkgbase" \ + "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" if (( $? >= 1 )); then failedpkgs+=("${pkgbase}-${pkgver}${SRCEXT}") continue diff --git a/db-abs b/db-abs new file mode 100644 index 0000000..5a34b84 --- /dev/null +++ b/db-abs @@ -0,0 +1,106 @@ +#!/hint/bash + +arch_svn() { + if [[ -z ${SVNUSER} ]]; then + /usr/bin/svn "${@}" + else + sudo -u "${SVNUSER}" -- /usr/bin/svn --username "${USER}" "${@}" + fi +} + +_abs_checkout() { + local pkgbase=$1 + if ! [[ -d ${WORKDIR}/svn ]]; then + arch_svn checkout -q -N "${SVNREPO}" "${WORKDIR}/svn" >/dev/null + fi + if ! [[ -d ${WORKDIR}/svn/${pkgbase} ]]; then + arch_svn up -q "${WORKDIR}/svn/${pkgbase}" >/dev/null + fi +} + +abs_move_preflight_check() { + local repo_from=$1 + local pkgarch=$2 + local pkgbase=$3 + + _abs_checkout "$pkgbase" + local svnrepo_from="${WORKDIR}/svn/${pkgbase}/repos/${repo_from}-${pkgarch}" + [[ -r ${svnrepo_from}/PKGBUILD ]] +} + +abs_move_start() { + abs_move_repo_from=$1 + abs_move_repo_to=$2 + abs_move_pkgbase=$3 + + abs_move_tag_list="" +} + +# If the "from" PKGBUILD doesn't exist, this is a no-op (not an +# error), so that it can be run for each arch, and the invoker doesn't +# need to worry about hoisting it out of the loop if arch=(any). If +# the nonexistence is such that it should be an error, we count on +# abs_move_preflight_check having already caught that. +abs_move_arch() { + local pkgarch=$1 + + local repo_from=$abs_move_repo_from + local repo_to=$abs_move_repo_to + local pkgbase=$abs_move_pkgbase + + local svnrepo_from="${WORKDIR}/svn/${pkgbase}/repos/${repo_from}-${pkgarch}" + local svnrepo_to="${WORKDIR}/svn/${pkgbase}/repos/${repo_to}-${pkgarch}" + if [[ -f ${svnrepo_from}/PKGBUILD ]]; then + msg2 "%s (%s)" "$pkgbase" "$pkgarch" + + if [[ -d ${svnrepo_to} ]]; then + for file in $(arch_svn ls "${svnrepo_to}"); do + arch_svn rm -q "${svnrepo_to}/$file@" + done + else + mkdir "${svnrepo_to}" + arch_svn add -q "${svnrepo_to}" + fi + + for file in $(arch_svn ls "${svnrepo_from}"); do + arch_svn mv -q -r HEAD "${svnrepo_from}/$file@" "${svnrepo_to}/" + done + arch_svn rm --force -q "${svnrepo_from}" + abs_move_tag_list+=", $pkgarch" + fi +} + +abs_move_finish() { + local repo_from=$abs_move_repo_from + local repo_to=$abs_move_repo_to + local pkgbase=$abs_move_pkgbase + + local tag_list="${abs_move_tag_list#, }" + arch_svn commit -q "${WORKDIR}/svn/${pkgbase}" -m "${0##*/}: moved ${pkgbase} from [${repo_from}] to [${repo_to}] (${tag_list})" +} + +abs_remove() { + local repo=$1 + local arch=$2 + local pkgbase=$3 + + local svnrepo="$repo-$arch" + + _abs_checkout "$pkgbase" + if [[ -d ${WORKDIR}/svn/$pkgbase/repos/$svnrepo ]]; then + arch_svn rm --force -q "${WORKDIR}/svn/$pkgbase/repos/$svnrepo" + arch_svn commit -q "${WORKDIR}/svn/$pkgbase" -m "${0##*/}: $pkgbase removed by $(id -un)" + else + warning "pkgbase '%s' not found in svn; unable to commit removal to svn" "$pkgbase" + fi +} + +abs_export() { + local repo=$1 + local pkgarch=$2 + local pkgbase=$3 + local dest=$4 + + arch_svn export -q "${SVNREPO}/${pkgbase}/repos/${repo}-${pkgarch}" \ + "${dest}" >/dev/null 2>&1 +} diff --git a/db-functions b/db-functions index 69f35b4..340c794 100644 --- a/db-functions +++ b/db-functions @@ -1,6 +1,7 @@ #!/hint/bash . /usr/share/makepkg/util.sh +. "$(dirname "${BASH_SOURCE[0]}")/db-abs" # global shell options for enhanced bash scripting shopt -s extglob globstar nullglob @@ -348,16 +349,16 @@ check_pkgsvn() { in_array "${repo}" "${PKGREPOS[@]}" || return 1 - if [[ ! -f ${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase} ]]; then + if [[ ! -f ${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}/PKGBUILD ]]; then mkdir -p "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}" - arch_svn export -q "${SVNREPO}/${_pkgbase}/repos/${repo}-${_pkgarch}/PKGBUILD" \ + abs_export "$repo" "$_pkgarch" "$_pkgbase" \ "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}" >/dev/null || return 1 fi - local svnver="$(. "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}"; get_full_version)" + local svnver="$(. "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}/PKGBUILD"; get_full_version)" [[ "${svnver}" = "${_pkgver}" ]] || return 1 - local svnnames=($(. "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}"; echo "${pkgname[@]}")) + local svnnames=($(. "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}/PKGBUILD"; echo "${pkgname[@]}")) in_array "${_pkgname}" "${svnnames[@]}" || return 1 return 0 @@ -382,13 +383,13 @@ check_splitpkgs() { mkdir -p "${repo}/${_pkgarch}/${_pkgbase}" echo "${_pkgname}" >> "${repo}/${_pkgarch}/${_pkgbase}/staging" - if [[ ! -f ${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase} ]]; then + if [[ ! -f ${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}/PKGBUILD ]]; then mkdir -p "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}" - arch_svn export -q "${SVNREPO}/${_pkgbase}/repos/${repo}-${_pkgarch}/PKGBUILD" \ + abs_export "$repo" "$_pkgarch" "$_pkgbase" \ "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}" >/dev/null || return 1 fi - local svnnames=($(. "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}"; echo "${pkgname[@]}")) + local svnnames=($(. "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}/PKGBUILD"; echo "${pkgname[@]}")) printf '%s\n' "${svnnames[@]}" >> "${repo}/${_pkgarch}/${_pkgbase}/svn" done popd >/dev/null @@ -489,11 +490,3 @@ arch_repo_modify() { REPO_MODIFIED=1 } - -arch_svn() { - if [[ -z ${SVNUSER} ]]; then - /usr/bin/svn "${@}" - else - sudo -u "${SVNUSER}" -- /usr/bin/svn --username "${USER}" "${@}" - fi -} diff --git a/db-move b/db-move index 2627b9a..94e6352 100755 --- a/db-move +++ b/db-move @@ -25,15 +25,12 @@ for pkgarch in "${ARCHES[@]}"; do done # check if packages to be moved exist in svn and ftp dir -arch_svn checkout -q -N "${SVNREPO}" "${WORKDIR}/svn" >/dev/null for pkgbase in "${args[@]:2}"; do - arch_svn up -q "${WORKDIR}/svn/${pkgbase}" >/dev/null found=false for tarch in "${ARCHES[@]}"; do while read -r pkgarch pkgfile; do - svnrepo_from="${WORKDIR}/svn/${pkgbase}/repos/${repo_from}-${pkgarch}" - if ! [[ -r ${svnrepo_from}/PKGBUILD ]]; then + if ! abs_move_preflight_check "$repo_from" "$pkgarch" "$pkgbase"; then die "%s not found in %s-%s" "$pkgbase" "$repo_from" "$pkgarch" fi @@ -52,29 +49,10 @@ msg "Moving packages from [%s] to [%s]..." "$repo_from" "$repo_to" declare -A add_pkgs declare -A remove_pkgs for pkgbase in "${args[@]:2}"; do - tag_list="" + abs_move_start "$repo_from" "$repo_to" "$pkgbase" for tarch in "${ARCHES[@]}"; do while read -r pkgname pkgver pkgarch pkgfile; do - svnrepo_from="${WORKDIR}/svn/${pkgbase}/repos/${repo_from}-${pkgarch}" - svnrepo_to="${WORKDIR}/svn/${pkgbase}/repos/${repo_to}-${pkgarch}" - if [[ -f ${svnrepo_from}/PKGBUILD ]]; then - msg2 "%s (%s)" "$pkgbase" "$pkgarch" - - if [[ -d ${svnrepo_to} ]]; then - for file in $(arch_svn ls "${svnrepo_to}"); do - arch_svn rm -q "${svnrepo_to}/$file@" - done - else - mkdir "${svnrepo_to}" - arch_svn add -q "${svnrepo_to}" - fi - - for file in $(arch_svn ls "${svnrepo_from}"); do - arch_svn mv -q -r HEAD "${svnrepo_from}/$file@" "${svnrepo_to}/" - done - arch_svn rm --force -q "${svnrepo_from}" - tag_list+=", $pkgarch" - fi + abs_move_arch "$pkgarch" ln -s "../../../${PKGPOOL}/${pkgfile}" "${ftppath_to}/${tarch}/" if [[ -f ${FTP_BASE}/${PKGPOOL}/${pkgfile}.sig ]]; then @@ -84,8 +62,7 @@ for pkgbase in "${args[@]:2}"; do remove_pkgs[${tarch}]+="${pkgname} " done < <(getdbinfo "$repo_from" "$tarch" "$pkgbase" NAME,VERSION,ARCH,FILENAME) done - tag_list="${tag_list#, }" - arch_svn commit -q "${WORKDIR}/svn/${pkgbase}" -m "${0##*/}: moved ${pkgbase} from [${repo_from}] to [${repo_to}] (${tag_list})" + abs_move_finish done for tarch in "${ARCHES[@]}"; do diff --git a/db-remove b/db-remove index 3017026..93430b6 100755 --- a/db-remove +++ b/db-remove @@ -13,7 +13,6 @@ arch="$2" pkgbases=("${@:3}") ftppath="$FTP_BASE/$repo/os" -svnrepo="$repo-$arch" if ! check_repo_permission "$repo"; then die "You don't have permission to remove packages from %s" "$repo" @@ -36,13 +35,7 @@ for pkgbase in "${pkgbases[@]}"; do mapfile -t pkgnames < <(getdbinfo "$repo" "${tarches[0]}" "$pkgbase" NAME) remove_pkgs+=("${pkgnames[@]}") - arch_svn checkout -q "${SVNREPO}/${pkgbase}" "${WORKDIR}/svn/${pkgbase}" >/dev/null - if [[ -d ${WORKDIR}/svn/$pkgbase/repos/$svnrepo ]]; then - arch_svn rm --force -q "${WORKDIR}/svn/$pkgbase/repos/$svnrepo" - arch_svn commit -q "${WORKDIR}/svn/$pkgbase" -m "${0##*/}: $pkgbase removed by $(id -un)" - else - warning "pkgbase '%s' not found in svn; unable to commit removal to svn" "$pkgbase" - fi + abs_remove "$repo" "$arch" "$pkgbase" done for tarch in "${tarches[@]}"; do -- cgit v1.2.2 From 2fdf0f2a61615c14597e24d89263b7897c599610 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Sun, 17 Jun 2018 13:22:45 -0400 Subject: db-move, db-abs: Simplify calling the abs_move_* functions Currently, abs_move_preflight_check and abs_move_arch require the exact pkgarch (i.e. "any" or "x86_64"). Modify them so that they take the tarch (i.e. "x86_64") and figure out themselves if it needs to set pkgarch=$tarch or pkgarch=any. abs_export could probably stand to be modified to do something similar, but it's more work to adjust its callers. "Ignore space change" might be useful when viewing this diff. --- db-abs | 56 ++++++++++++++++++++++++++++++++------------------------ db-move | 15 +++++++-------- 2 files changed, 39 insertions(+), 32 deletions(-) diff --git a/db-abs b/db-abs index 5a34b84..c510bad 100644 --- a/db-abs +++ b/db-abs @@ -20,12 +20,12 @@ _abs_checkout() { abs_move_preflight_check() { local repo_from=$1 - local pkgarch=$2 + local tarch=$2 local pkgbase=$3 _abs_checkout "$pkgbase" - local svnrepo_from="${WORKDIR}/svn/${pkgbase}/repos/${repo_from}-${pkgarch}" - [[ -r ${svnrepo_from}/PKGBUILD ]] + local reposdir="${WORKDIR}/svn/${pkgbase}/repos" + [[ -r ${reposdir}/${repo_from}-${tarch}/PKGBUILD || -r ${reposdir}/${repo_from}-any/PKGBUILD ]] } abs_move_start() { @@ -37,37 +37,45 @@ abs_move_start() { } # If the "from" PKGBUILD doesn't exist, this is a no-op (not an -# error), so that it can be run for each arch, and the invoker doesn't -# need to worry about hoisting it out of the loop if arch=(any). If -# the nonexistence is such that it should be an error, we count on -# abs_move_preflight_check having already caught that. +# error). This is because we expect to be called exactly once for +# each tarch (eg: x86_64, i686), but if arch=(any) then we only need +# do the work once; on the subsequent calls the "from" PKGBUILD won't +# exist anymore. If the source PKGBUILD never existed, we expect that +# to have already been caught by abs_move_preflight_check. abs_move_arch() { - local pkgarch=$1 + local tarch=$1 local repo_from=$abs_move_repo_from local repo_to=$abs_move_repo_to local pkgbase=$abs_move_pkgbase + local pkgarch + if [[ -r "${WORKDIR}/svn/${pkgbase}/repos/${repo_from}-${tarch}/PKGBUILD" ]]; then + pkgarch=$tarch + elif [[ -r "${WORKDIR}/svn/${pkgbase}/repos/${repo_from}-any/PKGBUILD" ]]; then + pkgarch=any + else + return 0 + fi + local svnrepo_from="${WORKDIR}/svn/${pkgbase}/repos/${repo_from}-${pkgarch}" local svnrepo_to="${WORKDIR}/svn/${pkgbase}/repos/${repo_to}-${pkgarch}" - if [[ -f ${svnrepo_from}/PKGBUILD ]]; then - msg2 "%s (%s)" "$pkgbase" "$pkgarch" - - if [[ -d ${svnrepo_to} ]]; then - for file in $(arch_svn ls "${svnrepo_to}"); do - arch_svn rm -q "${svnrepo_to}/$file@" - done - else - mkdir "${svnrepo_to}" - arch_svn add -q "${svnrepo_to}" - fi - - for file in $(arch_svn ls "${svnrepo_from}"); do - arch_svn mv -q -r HEAD "${svnrepo_from}/$file@" "${svnrepo_to}/" + msg2 "%s (%s)" "$pkgbase" "$pkgarch" + + if [[ -d ${svnrepo_to} ]]; then + for file in $(arch_svn ls "${svnrepo_to}"); do + arch_svn rm -q "${svnrepo_to}/$file@" done - arch_svn rm --force -q "${svnrepo_from}" - abs_move_tag_list+=", $pkgarch" + else + mkdir "${svnrepo_to}" + arch_svn add -q "${svnrepo_to}" fi + + for file in $(arch_svn ls "${svnrepo_from}"); do + arch_svn mv -q -r HEAD "${svnrepo_from}/$file@" "${svnrepo_to}/" + done + arch_svn rm --force -q "${svnrepo_from}" + abs_move_tag_list+=", $pkgarch" } abs_move_finish() { diff --git a/db-move b/db-move index 94e6352..40af7c5 100755 --- a/db-move +++ b/db-move @@ -28,10 +28,10 @@ done for pkgbase in "${args[@]:2}"; do found=false for tarch in "${ARCHES[@]}"; do - while read -r pkgarch pkgfile; do + while read -r pkgfile; do - if ! abs_move_preflight_check "$repo_from" "$pkgarch" "$pkgbase"; then - die "%s not found in %s-%s" "$pkgbase" "$repo_from" "$pkgarch" + if ! abs_move_preflight_check "$repo_from" "$tarch" "$pkgbase"; then + die "%s not found in %s-%s" "$pkgbase" "$repo_from" "$tarch" fi # getpkgfile will `exit` for us if it fails; @@ -39,7 +39,7 @@ for pkgbase in "${args[@]:2}"; do getpkgfile "${ftppath_from}/${tarch}/${pkgfile}" >/dev/null found=true - done < <(getdbinfo "$repo_from" "$tarch" "$pkgbase" ARCH,FILENAME) + done < <(getdbinfo "$repo_from" "$tarch" "$pkgbase" FILENAME) done [[ $found = true ]] || die "%s not found in %s" "$pkgbase" "$repo_from" done @@ -51,16 +51,15 @@ declare -A remove_pkgs for pkgbase in "${args[@]:2}"; do abs_move_start "$repo_from" "$repo_to" "$pkgbase" for tarch in "${ARCHES[@]}"; do - while read -r pkgname pkgver pkgarch pkgfile; do - abs_move_arch "$pkgarch" - + abs_move_arch "$tarch" + while read -r pkgname pkgver pkgfile; do ln -s "../../../${PKGPOOL}/${pkgfile}" "${ftppath_to}/${tarch}/" if [[ -f ${FTP_BASE}/${PKGPOOL}/${pkgfile}.sig ]]; then ln -s "../../../${PKGPOOL}/${pkgfile}.sig" "${ftppath_to}/${tarch}/" fi add_pkgs[${tarch}]+="${FTP_BASE}/${PKGPOOL}/${pkgfile} " remove_pkgs[${tarch}]+="${pkgname} " - done < <(getdbinfo "$repo_from" "$tarch" "$pkgbase" NAME,VERSION,ARCH,FILENAME) + done < <(getdbinfo "$repo_from" "$tarch" "$pkgbase" NAME,VERSION,FILENAME) done abs_move_finish done -- cgit v1.2.2 From 8871594e983151f33dd7bdd71a7df4aa33c42576 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Mon, 19 Feb 2018 17:01:28 -0500 Subject: Improve quoting You may verify that the only differences between the lines removed and lines added are double-quotes: diff -u \ <(git show|sed -n 's/^-//p'|grep -v '^-- a/'|sed 's/"//g') \ <(git show|sed -n 's/^+//p'|grep -v '^++ b/'|sed 's/"//g') --- config | 2 +- db-functions | 2 +- db-update | 2 +- test/cases/db-move.bats | 26 ++++++++++---------- test/cases/db-remove.bats | 34 +++++++++++++------------- test/cases/db-repo-add.bats | 22 ++++++++--------- test/cases/db-repo-remove.bats | 22 ++++++++--------- test/cases/db-update.bats | 30 +++++++++++------------ test/cases/ftpdir-cleanup.bats | 26 ++++++++++---------- test/cases/sourceballs.bats | 24 +++++++++---------- test/fixtures/pkg-any-a/PKGBUILD | 4 ++-- test/fixtures/pkg-any-b/PKGBUILD | 4 ++-- test/fixtures/pkg-simple-a/PKGBUILD | 2 +- test/fixtures/pkg-simple-b/PKGBUILD | 2 +- test/fixtures/pkg-simple-epoch/PKGBUILD | 2 +- test/fixtures/pkg-split-a/PKGBUILD | 4 ++-- test/fixtures/pkg-split-b/PKGBUILD | 4 ++-- test/lib/common.bash | 42 ++++++++++++++++----------------- 18 files changed, 127 insertions(+), 127 deletions(-) mode change 100644 => 100755 test/cases/db-move.bats mode change 100644 => 100755 test/cases/db-remove.bats mode change 100644 => 100755 test/cases/db-repo-add.bats mode change 100644 => 100755 test/cases/db-repo-remove.bats mode change 100644 => 100755 test/cases/db-update.bats mode change 100644 => 100755 test/cases/ftpdir-cleanup.bats mode change 100644 => 100755 test/cases/sourceballs.bats diff --git a/config b/config index fd79b6e..6b8781d 100644 --- a/config +++ b/config @@ -41,5 +41,5 @@ LIST="arch-dev-public@archlinux.org" FROM="repomaint@archlinux.org" # Override default config with config.local -LOCAL_CONFIG=${DBSCRIPTS_CONFIG:-"$(dirname "${BASH_SOURCE[0]}")/config.local"} +LOCAL_CONFIG=${DBSCRIPTS_CONFIG:-"$(dirname ${BASH_SOURCE[0]})/config.local"} [[ -f "${LOCAL_CONFIG}" ]] && . "${LOCAL_CONFIG}" diff --git a/db-functions b/db-functions index 340c794..7c54121 100644 --- a/db-functions +++ b/db-functions @@ -483,7 +483,7 @@ arch_repo_modify() { # package files for repo-add might be relative to repo dir pushd "${dbfile%/*}" >/dev/null - /usr/bin/"repo-${action}" -q "${dbfile}" "${pkgs[@]}" \ + /usr/bin/repo-${action} -q "${dbfile}" "${pkgs[@]}" \ || error '%s' "repo-${action} ${dbfile@Q} ${pkgs[*]@Q}" set_repo_permission "${repo}" "${arch}" popd >/dev/null diff --git a/db-update b/db-update index 8eb27df..8ce7b2d 100755 --- a/db-update +++ b/db-update @@ -94,7 +94,7 @@ for repo in "${repos[@]}"; do add_pkgs+=("${pkgfile}") done if (( ${#add_pkgs[@]} >= 1 )); then - arch_repo_modify add "${repo}" "${pkgarch}" ${add_pkgs[@]} + arch_repo_modify add "${repo}" "${pkgarch}" "${add_pkgs[@]}" fi done done diff --git a/test/cases/db-move.bats b/test/cases/db-move.bats old mode 100644 new mode 100755 index 27fe8d1..ad92c57 --- a/test/cases/db-move.bats +++ b/test/cases/db-move.bats @@ -6,8 +6,8 @@ load ../lib/common local pkgbase local arch - for pkgbase in ${pkgs[@]}; do - releasePackage testing ${pkgbase} + for pkgbase in "${pkgs[@]}"; do + releasePackage testing "${pkgbase}" done db-update @@ -25,17 +25,17 @@ load ../lib/common local pkgbase local arch - for pkgbase in ${pkgs[@]}; do - releasePackage testing ${pkgbase} + for pkgbase in "${pkgs[@]}"; do + releasePackage testing "${pkgbase}" done db-update db-move testing extra pkg-simple-a pkg-simple-b - for pkgbase in ${pkgs[@]}; do - checkRemovedPackage testing ${pkgbase} - checkPackage extra ${pkgbase} + for pkgbase in "${pkgs[@]}"; do + checkRemovedPackage testing "${pkgbase}" + checkPackage extra "${pkgbase}" done } @@ -64,8 +64,8 @@ load ../lib/common local pkgbase local arch - for pkgbase in ${pkgs[@]}; do - releasePackage testing ${pkgbase} + for pkgbase in "${pkgs[@]}"; do + releasePackage testing "${pkgbase}" done db-update @@ -80,8 +80,8 @@ load ../lib/common local pkgs=('pkg-any-a' 'pkg-any-b') local pkgbase - for pkgbase in ${pkgs[@]}; do - releasePackage testing ${pkgbase} + for pkgbase in "${pkgs[@]}"; do + releasePackage testing "${pkgbase}" done db-update @@ -99,8 +99,8 @@ load ../lib/common local pkgbase local arch - for pkgbase in ${pkgs[@]}; do - releasePackage testing ${pkgbase} + for pkgbase in "${pkgs[@]}"; do + releasePackage testing "${pkgbase}" done db-update diff --git a/test/cases/db-remove.bats b/test/cases/db-remove.bats old mode 100644 new mode 100755 index fe373bc..03e9493 --- a/test/cases/db-remove.bats +++ b/test/cases/db-remove.bats @@ -6,20 +6,20 @@ load ../lib/common local pkgbase local arch - for pkgbase in ${pkgs[@]}; do - releasePackage extra ${pkgbase} + for pkgbase in "${pkgs[@]}"; do + releasePackage extra "${pkgbase}" done db-update - for pkgbase in ${pkgs[@]}; do + for pkgbase in "${pkgs[@]}"; do for arch in ${arches[@]}; do - db-remove extra ${arch} ${pkgbase} + db-remove extra "${arch}" "${pkgbase}" done done - for pkgbase in ${pkgs[@]}; do - checkRemovedPackage extra ${pkgbase} + for pkgbase in "${pkgs[@]}"; do + checkRemovedPackage extra "${pkgbase}" done } @@ -29,18 +29,18 @@ load ../lib/common local pkgbase local arch - for pkgbase in ${pkgs[@]}; do - releasePackage extra ${pkgbase} + for pkgbase in "${pkgs[@]}"; do + releasePackage extra "${pkgbase}" done db-update for arch in ${arches[@]}; do - db-remove extra ${arch} ${pkgs[@]} + db-remove extra "${arch}" "${pkgs[@]}" done - for pkgbase in ${pkgs[@]}; do - checkRemovedPackage extra ${pkgbase} + for pkgbase in "${pkgs[@]}"; do + checkRemovedPackage extra "${pkgbase}" done } @@ -48,17 +48,17 @@ load ../lib/common local pkgs=('pkg-any-a' 'pkg-any-b') local pkgbase - for pkgbase in ${pkgs[@]}; do - releasePackage extra ${pkgbase} + for pkgbase in "${pkgs[@]}"; do + releasePackage extra "${pkgbase}" done db-update - for pkgbase in ${pkgs[@]}; do - db-remove extra any ${pkgbase} + for pkgbase in "${pkgs[@]}"; do + db-remove extra any "${pkgbase}" done - for pkgbase in ${pkgs[@]}; do - checkRemovedPackage extra ${pkgbase} + for pkgbase in "${pkgs[@]}"; do + checkRemovedPackage extra "${pkgbase}" done } diff --git a/test/cases/db-repo-add.bats b/test/cases/db-repo-add.bats old mode 100644 new mode 100755 index ac91058..638cc08 --- a/test/cases/db-repo-add.bats +++ b/test/cases/db-repo-add.bats @@ -27,16 +27,16 @@ __movePackageToRepo() { local pkgbase local arch - for pkgbase in ${pkgs[@]}; do - releasePackage extra ${pkgbase} + for pkgbase in "${pkgs[@]}"; do + releasePackage extra "$pkgbase" for arch in ${arches[@]}; do __movePackageToRepo extra ${pkgbase} ${arch} - db-repo-add extra ${arch} ${pkgbase}-1-1-${arch}${PKGEXT} + db-repo-add extra "${arch}" "${pkgbase}-1-1-${arch}${PKGEXT}" done done - for pkgbase in ${pkgs[@]}; do - checkPackageDB extra ${pkgbase} + for pkgbase in "${pkgs[@]}"; do + checkPackageDB extra "${pkgbase}" done } @@ -46,21 +46,21 @@ __movePackageToRepo() { local pkgbase local arch - for pkgbase in ${pkgs[@]}; do - releasePackage extra ${pkgbase} + for pkgbase in "${pkgs[@]}"; do + releasePackage extra "$pkgbase" done for arch in ${arches[@]}; do add_pkgs=() - for pkgbase in ${pkgs[@]}; do + for pkgbase in "${pkgs[@]}"; do __movePackageToRepo extra ${pkgbase} ${arch} add_pkgs+=("${pkgbase}-1-1-${arch}${PKGEXT}") done - db-repo-add extra ${arch} ${add_pkgs[@]} + db-repo-add extra "${arch}" "${add_pkgs[@]}" done - for pkgbase in ${pkgs[@]}; do - checkPackageDB extra ${pkgbase} + for pkgbase in "${pkgs[@]}"; do + checkPackageDB extra "${pkgbase}" done } diff --git a/test/cases/db-repo-remove.bats b/test/cases/db-repo-remove.bats old mode 100644 new mode 100755 index 77bcc24..ddf78aa --- a/test/cases/db-repo-remove.bats +++ b/test/cases/db-repo-remove.bats @@ -6,20 +6,20 @@ load ../lib/common local pkgbase local arch - for pkgbase in ${pkgs[@]}; do - releasePackage extra ${pkgbase} + for pkgbase in "${pkgs[@]}"; do + releasePackage extra "${pkgbase}" done db-update - for pkgbase in ${pkgs[@]}; do + for pkgbase in "${pkgs[@]}"; do for arch in ${arches[@]}; do - db-repo-remove extra ${arch} ${pkgbase} + db-repo-remove extra "${arch}" "${pkgbase}" done done - for pkgbase in ${pkgs[@]}; do - checkRemovedPackageDB extra ${pkgbase} + for pkgbase in "${pkgs[@]}"; do + checkRemovedPackageDB extra "${pkgbase}" done } @@ -29,18 +29,18 @@ load ../lib/common local pkgbase local arch - for pkgbase in ${pkgs[@]}; do - releasePackage extra ${pkgbase} + for pkgbase in "${pkgs[@]}"; do + releasePackage extra "${pkgbase}" done db-update for arch in ${arches[@]}; do - db-repo-remove extra ${arch} ${pkgs[@]} + db-repo-remove extra "${arch}" "${pkgs[@]}" done - for pkgbase in ${pkgs[@]}; do - checkRemovedPackageDB extra ${pkgbase} + for pkgbase in "${pkgs[@]}"; do + checkRemovedPackageDB extra "${pkgbase}" done } diff --git a/test/cases/db-update.bats b/test/cases/db-update.bats old mode 100644 new mode 100755 index ec58458..edaa48e --- a/test/cases/db-update.bats +++ b/test/cases/db-update.bats @@ -6,14 +6,14 @@ load ../lib/common local pkgbase local arch - for pkgbase in ${pkgs[@]}; do - releasePackage extra ${pkgbase} + for pkgbase in "${pkgs[@]}"; do + releasePackage extra "${pkgbase}" done db-update - for pkgbase in ${pkgs[@]}; do - checkPackage extra ${pkgbase} + for pkgbase in "${pkgs[@]}"; do + checkPackage extra "${pkgbase}" done } @@ -33,14 +33,14 @@ load ../lib/common local pkgs=('pkg-any-a' 'pkg-any-b') local pkgbase - for pkgbase in ${pkgs[@]}; do - releasePackage extra ${pkgbase} + for pkgbase in "${pkgs[@]}"; do + releasePackage extra "${pkgbase}" done db-update - for pkgbase in ${pkgs[@]}; do - checkPackage extra ${pkgbase} + for pkgbase in "${pkgs[@]}"; do + checkPackage extra "${pkgbase}" done } @@ -51,14 +51,14 @@ load ../lib/common local pkgbase local arch - for pkgbase in ${pkgs[@]}; do - releasePackage extra ${pkgbase} + for pkgbase in "${pkgs[@]}"; do + releasePackage extra "${pkgbase}" done db-update - for pkgbase in ${pkgs[@]}; do - checkPackage extra ${pkgbase} + for pkgbase in "${pkgs[@]}"; do + checkPackage extra "${pkgbase}" done } @@ -124,10 +124,10 @@ load ../lib/common local pkgbase='pkg-split-a' local arch - releasePackage ${repo} ${pkgbase} + releasePackage "${repo}" "${pkgbase}" # remove a split package to make db-update fail - rm "${STAGING}"/extra/${pkgbase}1-* + rm "${STAGING}/extra/${pkgbase}1-"* run db-update [ "$status" -ne 0 ] @@ -170,7 +170,7 @@ load ../lib/common local s releasePackage extra 'pkg-any-a' for s in "${STAGING}"/extra/*.sig; do - echo 0 > $s + echo 0 > "$s" done run db-update [ "$status" -ne 0 ] diff --git a/test/cases/ftpdir-cleanup.bats b/test/cases/ftpdir-cleanup.bats old mode 100644 new mode 100755 index 74215d0..aac4c93 --- a/test/cases/ftpdir-cleanup.bats +++ b/test/cases/ftpdir-cleanup.bats @@ -24,14 +24,14 @@ __checkRepoRemovedPackage() { local pkgbase local arch - for pkgbase in ${pkgs[@]}; do - releasePackage extra ${pkgbase} + for pkgbase in "${pkgs[@]}"; do + releasePackage extra "${pkgbase}" done db-update for arch in ${arches[@]}; do - db-remove extra ${arch} pkg-simple-a + db-remove extra "${arch}" pkg-simple-a done ftpdir-cleanup @@ -50,14 +50,14 @@ __checkRepoRemovedPackage() { local pkgbase local arch - for pkgbase in ${pkgs[@]}; do - releasePackage extra ${pkgbase} + for pkgbase in "${pkgs[@]}"; do + releasePackage extra "${pkgbase}" done db-update for arch in ${arches[@]}; do - db-remove extra ${arch} pkg-simple-epoch + db-remove extra "${arch}" pkg-simple-epoch done ftpdir-cleanup @@ -74,8 +74,8 @@ __checkRepoRemovedPackage() { local pkgbase local arch='any' - for pkgbase in ${pkgs[@]}; do - releasePackage extra ${pkgbase} + for pkgbase in "${pkgs[@]}"; do + releasePackage extra "${pkgbase}" done db-update @@ -98,14 +98,14 @@ __checkRepoRemovedPackage() { local pkgbase local arch - for pkgbase in ${pkgs[@]}; do - releasePackage extra ${pkgbase} + for pkgbase in "${pkgs[@]}"; do + releasePackage extra "${pkgbase}" done db-update for arch in ${arches[@]}; do - db-remove extra ${arch} ${pkgs[0]} + db-remove extra "${arch}" "${pkgs[0]}" done ftpdir-cleanup @@ -114,8 +114,8 @@ __checkRepoRemovedPackage() { __checkRepoRemovedPackage extra ${pkgs[0]} ${arch} done - checkRemovedPackage extra ${pkgs[0]} - checkPackage extra ${pkgs[1]} + checkRemovedPackage extra "${pkgs[0]}" + checkPackage extra "${pkgs[1]}" } @test "cleanup old packages" { diff --git a/test/cases/sourceballs.bats b/test/cases/sourceballs.bats old mode 100644 new mode 100755 index df7ddd4..f8b0e3c --- a/test/cases/sourceballs.bats +++ b/test/cases/sourceballs.bats @@ -16,13 +16,13 @@ __checkRemovedSourcePackage() { local pkgbase local arch - for pkgbase in ${pkgs[@]}; do - releasePackage extra ${pkgbase} + for pkgbase in "${pkgs[@]}"; do + releasePackage extra "${pkgbase}" done db-update sourceballs - for pkgbase in ${pkgs[@]}; do + for pkgbase in "${pkgs[@]}"; do __checkSourcePackage ${pkgbase} done } @@ -31,13 +31,13 @@ __checkRemovedSourcePackage() { local pkgs=('pkg-any-a' 'pkg-any-b') local pkgbase - for pkgbase in ${pkgs[@]}; do - releasePackage extra ${pkgbase} + for pkgbase in "${pkgs[@]}"; do + releasePackage extra "${pkgbase}" done db-update sourceballs - for pkgbase in ${pkgs[@]}; do + for pkgbase in "${pkgs[@]}"; do __checkSourcePackage ${pkgbase} done } @@ -49,14 +49,14 @@ __checkRemovedSourcePackage() { local pkgbase local arch - for pkgbase in ${pkgs[@]}; do - releasePackage extra ${pkgbase} + for pkgbase in "${pkgs[@]}"; do + releasePackage extra "${pkgbase}" done db-update sourceballs - for pkgbase in ${pkgs[@]}; do + for pkgbase in "${pkgs[@]}"; do __checkSourcePackage ${pkgbase} done } @@ -67,14 +67,14 @@ __checkRemovedSourcePackage() { local pkgbase local arch - for pkgbase in ${pkgs[@]}; do - releasePackage extra ${pkgbase} + for pkgbase in "${pkgs[@]}"; do + releasePackage extra "${pkgbase}" done db-update sourceballs for arch in ${arches[@]}; do - db-remove extra ${arch} pkg-simple-a + db-remove extra "${arch}" pkg-simple-a done sourceballs diff --git a/test/fixtures/pkg-any-a/PKGBUILD b/test/fixtures/pkg-any-a/PKGBUILD index cf2774b..9996cdc 100644 --- a/test/fixtures/pkg-any-a/PKGBUILD +++ b/test/fixtures/pkg-any-a/PKGBUILD @@ -8,6 +8,6 @@ license=('GPL') options=(!strip) package() { - install -d ${pkgdir}/usr/share/${pkgname} - :>${pkgdir}/usr/share/${pkgname}/test + install -d "${pkgdir}"/usr/share/${pkgname} + :>"${pkgdir}"/usr/share/${pkgname}/test } diff --git a/test/fixtures/pkg-any-b/PKGBUILD b/test/fixtures/pkg-any-b/PKGBUILD index 05dfd10..ce30461 100644 --- a/test/fixtures/pkg-any-b/PKGBUILD +++ b/test/fixtures/pkg-any-b/PKGBUILD @@ -8,6 +8,6 @@ license=('GPL') options=(!strip) package() { - install -d ${pkgdir}/usr/share/${pkgname} - :>${pkgdir}/usr/share/${pkgname}/test + install -d "${pkgdir}"/usr/share/${pkgname} + :>"${pkgdir}"/usr/share/${pkgname}/test } diff --git a/test/fixtures/pkg-simple-a/PKGBUILD b/test/fixtures/pkg-simple-a/PKGBUILD index eaa841b..3cfad00 100644 --- a/test/fixtures/pkg-simple-a/PKGBUILD +++ b/test/fixtures/pkg-simple-a/PKGBUILD @@ -9,5 +9,5 @@ depends=('glibc') options=(!strip) package() { - install -D -m755 /bin/true ${pkgdir}/usr/bin/${pkgname} + install -D -m755 /bin/true "${pkgdir}"/usr/bin/${pkgname} } diff --git a/test/fixtures/pkg-simple-b/PKGBUILD b/test/fixtures/pkg-simple-b/PKGBUILD index 287eee7..e16dbc5 100644 --- a/test/fixtures/pkg-simple-b/PKGBUILD +++ b/test/fixtures/pkg-simple-b/PKGBUILD @@ -9,5 +9,5 @@ depends=('glibc') options=(!strip) package() { - install -D -m755 /bin/true ${pkgdir}/usr/bin/${pkgname} + install -D -m755 /bin/true "${pkgdir}"/usr/bin/${pkgname} } diff --git a/test/fixtures/pkg-simple-epoch/PKGBUILD b/test/fixtures/pkg-simple-epoch/PKGBUILD index c624439..a19572b 100644 --- a/test/fixtures/pkg-simple-epoch/PKGBUILD +++ b/test/fixtures/pkg-simple-epoch/PKGBUILD @@ -10,5 +10,5 @@ depends=('glibc') options=(!strip) package() { - install -D -m755 /bin/true ${pkgdir}/usr/bin/${pkgname} + install -D -m755 /bin/true "${pkgdir}"/usr/bin/${pkgname} } diff --git a/test/fixtures/pkg-split-a/PKGBUILD b/test/fixtures/pkg-split-a/PKGBUILD index 2c91390..05d48d3 100644 --- a/test/fixtures/pkg-split-a/PKGBUILD +++ b/test/fixtures/pkg-split-a/PKGBUILD @@ -10,9 +10,9 @@ depends=('glibc') options=(!strip) package_pkg-split-a1() { - install -D -m755 /bin/true ${pkgdir}/usr/bin/${pkgname} + install -D -m755 /bin/true "${pkgdir}"/usr/bin/${pkgname} } package_pkg-split-a2() { - install -D -m755 /bin/true ${pkgdir}/usr/bin/${pkgname} + install -D -m755 /bin/true "${pkgdir}"/usr/bin/${pkgname} } diff --git a/test/fixtures/pkg-split-b/PKGBUILD b/test/fixtures/pkg-split-b/PKGBUILD index 1d07827..cf455fd 100644 --- a/test/fixtures/pkg-split-b/PKGBUILD +++ b/test/fixtures/pkg-split-b/PKGBUILD @@ -10,9 +10,9 @@ depends=('glibc') options=(!strip) package_pkg-split-b1() { - install -D -m755 /bin/true ${pkgdir}/usr/bin/${pkgname} + install -D -m755 /bin/true "${pkgdir}"/usr/bin/${pkgname} } package_pkg-split-b2() { - install -D -m755 /bin/true ${pkgdir}/usr/bin/${pkgname} + install -D -m755 /bin/true "${pkgdir}"/usr/bin/${pkgname} } diff --git a/test/lib/common.bash b/test/lib/common.bash index b843817..fbd9492 100644 --- a/test/lib/common.bash +++ b/test/lib/common.bash @@ -116,9 +116,9 @@ eot mkdir -p "${TMP}/"{ftp,tmp,staging,{package,source}-cleanup,svn-packages-{copy,repo}} - for r in ${PKGREPOS[@]}; do + for r in "${PKGREPOS[@]}"; do mkdir -p "${TMP}/staging/${r}" - for a in ${ARCHES[@]}; do + for a in "${ARCHES[@]}"; do mkdir -p "${TMP}/ftp/${r}/os/${a}" done done @@ -144,7 +144,7 @@ releasePackage() { svn commit -q -m"initial commit of ${pkgbase}" "${TMP}/svn-packages-copy" fi - pushd "${TMP}/svn-packages-copy"/${pkgbase}/trunk/ + pushd "${TMP}/svn-packages-copy/${pkgbase}/trunk/" __buildPackage "${STAGING}"/${repo} __archrelease ${repo} popd @@ -195,35 +195,35 @@ checkPackageDB() { local pkgver=$(. "${pkgbuildPath}/PKGBUILD"; get_full_version) if [[ ${pkgarches[@]} == any ]]; then - repoarches=(${ARCHES[@]}) + repoarches=("${ARCHES[@]}") else - repoarches=(${pkgarches[@]}) + repoarches=("${pkgarches[@]}") fi for pkgarch in ${pkgarches[@]}; do for pkgname in ${pkgnames[@]}; do pkgfile="${pkgname}-${pkgver}-${pkgarch}${PKGEXT}" - [ -r ${FTP_BASE}/${PKGPOOL}/${pkgfile} ] - [ -r ${FTP_BASE}/${PKGPOOL}/${pkgfile}.sig ] - [ ! -r ${STAGING}/${repo}/${pkgfile} ] - [ ! -r ${STAGING}/${repo}/${pkgfile}.sig ] + [ -r "${FTP_BASE}/${PKGPOOL}/${pkgfile}" ] + [ -r "${FTP_BASE}/${PKGPOOL}/${pkgfile}.sig" ] + [ ! -r "${STAGING}/${repo}/${pkgfile}" ] + [ ! -r "${STAGING}/${repo}/${pkgfile}.sig" ] - for repoarch in ${repoarches[@]}; do + for repoarch in "${repoarches[@]}"; do # Only 'any' packages can be found in repos of both arches if [[ $pkgarch != any ]]; then - if [[ $pkgarch != ${repoarch} ]]; then + if [[ $pkgarch != "$repoarch" ]]; then continue fi fi - [ -L ${FTP_BASE}/${repo}/os/${repoarch}/${pkgfile} ] - [ "$(readlink -e ${FTP_BASE}/${repo}/os/${repoarch}/${pkgfile})" == ${FTP_BASE}/${PKGPOOL}/${pkgfile} ] + [ -L "${FTP_BASE}/${repo}/os/${repoarch}/${pkgfile}" ] + [ "$(readlink -e "${FTP_BASE}/${repo}/os/${repoarch}/${pkgfile}")" == "$(readlink -e "${FTP_BASE}/${PKGPOOL}/${pkgfile}")" ] - [ -L ${FTP_BASE}/${repo}/os/${repoarch}/${pkgfile}.sig ] - [ "$(readlink -e ${FTP_BASE}/${repo}/os/${repoarch}/${pkgfile}.sig)" == ${FTP_BASE}/${PKGPOOL}/${pkgfile}.sig ] + [ -L "${FTP_BASE}/${repo}/os/${repoarch}/${pkgfile}.sig" ] + [ "$(readlink -e "${FTP_BASE}/${repo}/os/${repoarch}/${pkgfile}.sig")" == "$(readlink -e "${FTP_BASE}/${PKGPOOL}/${pkgfile}.sig")" ] - for db in ${DBEXT} ${FILESEXT}; do + for db in "${DBEXT}" "${FILESEXT}"; do [ -r "${FTP_BASE}/${repo}/os/${repoarch}/${repo}${db%.tar.*}" ] bsdtar -xf "${FTP_BASE}/${repo}/os/${repoarch}/${repo}${db%.tar.*}" -O | grep "${pkgfile%${PKGEXT}}" &>/dev/null done @@ -240,7 +240,7 @@ checkPackage() { # TODO: Does not fail if one arch is missing compgen -G "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo}-*" >/dev/null - checkPackageDB $repo $pkgbase + checkPackageDB "$repo" "$pkgbase" } checkRemovedPackage() { @@ -250,7 +250,7 @@ checkRemovedPackage() { svn up -q "${TMP}/svn-packages-copy/${pkgbase}" ! compgen -G "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo}-*" >/dev/null - checkRemovedPackageDB $repo $pkgbase + checkRemovedPackageDB "$repo" "$pkgbase" } checkRemovedPackageDB() { @@ -275,11 +275,11 @@ checkRemovedPackageDB() { tarches=(${pkgarches[@]}) fi - for db in ${DBEXT} ${FILESEXT}; do - for tarch in ${tarches[@]}; do + for db in "${DBEXT}" "${FILESEXT}"; do + for tarch in "${tarches[@]}"; do if [ -r "${FTP_BASE}/${repo}/os/${tarch}/${repo}${db%.tar.*}" ]; then for pkgname in ${pkgnames[@]}; do - if bsdtar -xf "${FTP_BASE}/${repo}/os/${tarch}/${repo}${db%.tar.*}" -O | grep ${pkgname} &>/dev/null; then + if bsdtar -xf "${FTP_BASE}/${repo}/os/${tarch}/${repo}${db%.tar.*}" -O | grep "${pkgname}" &>/dev/null; then return 1 fi done -- cgit v1.2.2 From 6ae6f8c1605661542a04ad6ead64eb2f222db669 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Fri, 22 Jun 2018 12:55:37 -0400 Subject: Remove unused variables --- db-remove | 2 -- test/cases/db-move.bats | 4 ---- test/cases/db-repo-add.bats | 1 - test/cases/db-update.bats | 6 ------ test/cases/sourceballs.bats | 2 -- 5 files changed, 15 deletions(-) diff --git a/db-remove b/db-remove index 93430b6..5893f4d 100755 --- a/db-remove +++ b/db-remove @@ -12,8 +12,6 @@ repo="$1" arch="$2" pkgbases=("${@:3}") -ftppath="$FTP_BASE/$repo/os" - if ! check_repo_permission "$repo"; then die "You don't have permission to remove packages from %s" "$repo" fi diff --git a/test/cases/db-move.bats b/test/cases/db-move.bats index ad92c57..3006106 100755 --- a/test/cases/db-move.bats +++ b/test/cases/db-move.bats @@ -1,7 +1,6 @@ load ../lib/common @test "move single packages" { - local arches=('i686' 'x86_64') local pkgs=('pkg-simple-a' 'pkg-simple-b') local pkgbase local arch @@ -20,7 +19,6 @@ load ../lib/common } @test "move multiple packages" { - local arches=('i686' 'x86_64') local pkgs=('pkg-simple-a' 'pkg-simple-b') local pkgbase local arch @@ -59,7 +57,6 @@ load ../lib/common } @test "move epoch packages" { - local arches=('i686' 'x86_64') local pkgs=('pkg-simple-epoch') local pkgbase local arch @@ -93,7 +90,6 @@ load ../lib/common } @test "move split packages" { - local arches=('i686' 'x86_64') local pkgs=('pkg-split-a' 'pkg-split-b') local pkg local pkgbase diff --git a/test/cases/db-repo-add.bats b/test/cases/db-repo-add.bats index 638cc08..0ef9836 100755 --- a/test/cases/db-repo-add.bats +++ b/test/cases/db-repo-add.bats @@ -65,7 +65,6 @@ __movePackageToRepo() { } @test "add any packages" { - local arches=('i686' 'x86_64') local pkgs=('pkg-any-a' 'pkg-any-b') local pkgbase local arch diff --git a/test/cases/db-update.bats b/test/cases/db-update.bats index edaa48e..6c500c4 100755 --- a/test/cases/db-update.bats +++ b/test/cases/db-update.bats @@ -1,7 +1,6 @@ load ../lib/common @test "add simple packages" { - local arches=('i686' 'x86_64') local pkgs=('pkg-simple-a' 'pkg-simple-b') local pkgbase local arch @@ -45,7 +44,6 @@ load ../lib/common } @test "add split packages" { - local arches=('i686' 'x86_64') local pkgs=('pkg-split-a' 'pkg-split-b') local pkg local pkgbase @@ -105,8 +103,6 @@ load ../lib/common } @test "update same any package to different repositories fails" { - local arch - releasePackage extra pkg-any-a db-update checkPackage extra pkg-any-a @@ -119,7 +115,6 @@ load ../lib/common } @test "add incomplete split package fails" { - local arches=('i686' 'x86_64') local repo='extra' local pkgbase='pkg-split-a' local arch @@ -230,7 +225,6 @@ load ../lib/common @test "package has to be aregular file" { local p local target=$(mktemp -d) - local arches=('i686' 'x86_64') releasePackage extra 'pkg-simple-a' diff --git a/test/cases/sourceballs.bats b/test/cases/sourceballs.bats index f8b0e3c..49b2233 100755 --- a/test/cases/sourceballs.bats +++ b/test/cases/sourceballs.bats @@ -11,7 +11,6 @@ __checkRemovedSourcePackage() { } @test "create simple package sourceballs" { - local arches=('i686' 'x86_64') local pkgs=('pkg-simple-a' 'pkg-simple-b' 'pkg-simple-epoch') local pkgbase local arch @@ -43,7 +42,6 @@ __checkRemovedSourcePackage() { } @test "create split package sourceballs" { - local arches=('i686' 'x86_64') local pkgs=('pkg-split-a' 'pkg-split-b') local pkg local pkgbase -- cgit v1.2.2 From 3f3fe2125cea7b20c5633b736e8e858c41c5d377 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Fri, 22 Jun 2018 12:49:31 -0400 Subject: test/lib/common.bash: #!/hint/bash --- test/lib/common.bash | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/lib/common.bash b/test/lib/common.bash index fbd9492..febc458 100644 --- a/test/lib/common.bash +++ b/test/lib/common.bash @@ -1,3 +1,5 @@ +#!/hint/bash + . /usr/share/makepkg/util.sh __updatePKGBUILD() { -- cgit v1.2.2 From d6974819b704c4b3bf3f095a6e34a14aa6072ccd Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Fri, 22 Jun 2018 12:49:56 -0400 Subject: config: Be friendly with `set -e` programs --- config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config b/config index 6b8781d..4e23283 100644 --- a/config +++ b/config @@ -42,4 +42,4 @@ FROM="repomaint@archlinux.org" # Override default config with config.local LOCAL_CONFIG=${DBSCRIPTS_CONFIG:-"$(dirname ${BASH_SOURCE[0]})/config.local"} -[[ -f "${LOCAL_CONFIG}" ]] && . "${LOCAL_CONFIG}" +if [[ -f "${LOCAL_CONFIG}" ]]; then . "${LOCAL_CONFIG}"; fi -- cgit v1.2.2 From 7b0a9c48b418fc86f457e66acb66fe2ed16c6412 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Mon, 25 May 2015 01:19:43 -0600 Subject: db-functions: use librelib instead of having copies here --- db-functions | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/db-functions b/db-functions index 7c54121..39a6905 100644 --- a/db-functions +++ b/db-functions @@ -3,6 +3,20 @@ . /usr/share/makepkg/util.sh . "$(dirname "${BASH_SOURCE[0]}")/db-abs" +# Used: +# libmakepkg/util/message.sh: +# plain, msg, msg2, warning, error +# libmakepkg/util/pkgbuild.sh: +# get_full_version +# libmakepkg/util/util.sh: +# in_array +# devtools/common.sh: +# abort, die +# Overwritten: +# devtools/common.sh: +# cleanup +. "$(librelib common)" + # global shell options for enhanced bash scripting shopt -s extglob globstar nullglob @@ -97,16 +111,6 @@ cleanup() { [[ -n $1 ]] && exit "$1" } -abort() { - msg 'Aborting...' - cleanup 0 -} - -die() { - error "$@" - cleanup 1 -} - trap abort INT QUIT TERM HUP trap cleanup EXIT -- cgit v1.2.2 From 58eac16f89b94cd6cc71de9caba19329af41b970 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Sat, 17 Mar 2018 00:18:46 -0400 Subject: rm testing2x cron-jobs/update-web-* --- README.md | 13 ++------ config | 2 -- cron-jobs/update-web-db | 78 ------------------------------------------- cron-jobs/update-web-files-db | 1 - test/cases/common.bats | 2 +- test/cases/testing2x.bats | 16 --------- testing2x | 55 ------------------------------ 7 files changed, 4 insertions(+), 163 deletions(-) delete mode 100755 cron-jobs/update-web-db delete mode 120000 cron-jobs/update-web-files-db delete mode 100644 test/cases/testing2x.bats delete mode 100755 testing2x diff --git a/README.md b/README.md index c672998..87bbb10 100644 --- a/README.md +++ b/README.md @@ -12,14 +12,14 @@ The executables that you (might) care about are: │   ├── ftpdir-cleanup │   ├── integrity-check │   ├── sourceballs - │   ├── update-web-db - │   └── update-web-files-db + │   ├── update-web-db [Arch Linux only] + │   └── update-web-files-db [Arch Linux only] ├── db-move ├── db-remove ├── db-repo-add ├── db-repo-remove ├── db-update - └── testing2x + └── testing2x [Arch Linux only] Ok, now let's talk about what those are. @@ -29,11 +29,6 @@ There are 3 "main" programs: - `db-remove` : remove packages from repositories - `db-move` : move packages from one repository to another -Moving packages from testing to stable repositories is such a common -task that we have a wrapper around `db-move` to make it easier: - - - `testing2x` - Of course, sometimes things go wrong, and you need to drop to a lower-level, but you don't want to go all the way down to pacman's `repo-add`/`repo-remove`. So, we have: @@ -56,8 +51,6 @@ Things that haven't been mentioned yet: - `cron-jobs/devlist-mailer` - `cron-jobs/sourceballs` - - `cron-jobs/update-web-db` - - `cron-jobs/update-web-files-db` ## Testing * Install the `make` and `docker` packages. Start the docker daemon by issuing `systemctl start docker`. * The test suite can now be run with `make test`. diff --git a/config b/config index 4e23283..769a69f 100644 --- a/config +++ b/config @@ -6,8 +6,6 @@ SVNUSER='' PKGREPOS=() PKGPOOL='' SRCPOOL='' -TESTING_REPO='' -STABLE_REPOS=() CLEANUP_DESTDIR="/var/tmp" CLEANUP_DRYRUN=false diff --git a/cron-jobs/update-web-db b/cron-jobs/update-web-db deleted file mode 100755 index 39ed765..0000000 --- a/cron-jobs/update-web-db +++ /dev/null @@ -1,78 +0,0 @@ -#!/bin/bash - -. "$(dirname "$(readlink -e "$0")")/../config" -. "$(dirname "$(readlink -e "$0")")/../db-functions" - -# setup paths -SPATH="/srv/http/archweb" -ENVPATH="/srv/http/archweb-env/bin/activate" - -# having "more important repos" last should make [core] trickle to the top of -# the updates list each hour rather than being overwhelmed by big [extra] and -# [community] updates -REPOS=('community-testing' 'multilib-testing' 'multilib' 'community' 'extra' 'testing' 'core') -LOGOUT="/tmp/archweb_update.log" - -# figure out what operation to perform -cmd="${0##*/}" -if [[ $cmd != "update-web-db" && $cmd != "update-web-files-db" ]]; then - die "Invalid command name '%s' specified!" "$cmd" -fi - -script_lock - -# run at nice 5. it can churn quite a bit of cpu after all. -renice +5 -p $$ > /dev/null - -echo "%s: Updating DB at %s" "$cmd" "$(date)" >> "${LOGOUT}" - -# source our virtualenv if it exists -if [[ -f "$ENVPATH" ]]; then - . "$ENVPATH" -fi - -case "$cmd" in - update-web-db) - dbfileext="${DBEXT}" - flags="" - ;; - update-web-files-db) - dbfileext="${FILESEXT}" - flags="--filesonly" - ;; -esac - -# Lock the repos and get a copy of the db files to work on -for repo in "${REPOS[@]}"; do - for arch in "${ARCHES[@]}"; do - repo_lock "${repo}" "${arch}" || exit 1 - dbfile="/srv/ftp/${repo}/os/${arch}/${repo}${dbfileext}" - if [[ -f ${dbfile} ]]; then - mkdir -p "${WORKDIR}/${repo}/${arch}" - cp "${dbfile}" "${WORKDIR}/${repo}/${arch}/${repo}${dbfileext}" - fi - repo_unlock "${repo}" "${arch}" - done -done - -# Run reporead on our db copy -pushd "$SPATH" >/dev/null -for repo in "${REPOS[@]}"; do - for arch in "${ARCHES[@]}"; do - dbcopy="${WORKDIR}/${repo}/${arch}/${repo}${dbfileext}" - if [[ -f ${dbcopy} ]]; then - echo "Updating ${repo}-${arch}" >> "${LOGOUT}" - ./manage.py reporead "${flags}" "${arch}" "${dbcopy}" >> "${LOGOUT}" 2>&1 - echo "" >> "${LOGOUT}" - fi - done -done -popd >/dev/null -echo "" >> "${LOGOUT}" - -# rotate the file if it is getting big (> 10M), overwriting any old backup -if [[ $(stat -c%s "${LOGOUT}") -gt 10485760 ]]; then - mv "${LOGOUT}" "${LOGOUT}.old" -fi - -script_unlock diff --git a/cron-jobs/update-web-files-db b/cron-jobs/update-web-files-db deleted file mode 120000 index 0c2c4fa..0000000 --- a/cron-jobs/update-web-files-db +++ /dev/null @@ -1 +0,0 @@ -update-web-db \ No newline at end of file diff --git a/test/cases/common.bats b/test/cases/common.bats index 4b798f8..8b4c3a0 100644 --- a/test/cases/common.bats +++ b/test/cases/common.bats @@ -1,7 +1,7 @@ load ../lib/common @test "commands display usage message by default" { - for cmd in db-move db-remove db-repo-add db-repo-remove testing2x; do + for cmd in db-move db-remove db-repo-add db-repo-remove; do echo Testing $cmd run $cmd (( $status == 1 )) diff --git a/test/cases/testing2x.bats b/test/cases/testing2x.bats deleted file mode 100644 index f9f04e2..0000000 --- a/test/cases/testing2x.bats +++ /dev/null @@ -1,16 +0,0 @@ -load ../lib/common - -@test "move any package" { - releasePackage core pkg-any-a - db-update - - updatePackage pkg-any-a - - releasePackage testing pkg-any-a - db-update - - testing2x pkg-any-a - - checkPackage core pkg-any-a - checkRemovedPackage testing pkg-any-a -} diff --git a/testing2x b/testing2x deleted file mode 100755 index f455a93..0000000 --- a/testing2x +++ /dev/null @@ -1,55 +0,0 @@ -#!/bin/bash - -. "$(dirname "$(readlink -e "$0")")/config" -. "$(dirname "$(readlink -e "$0")")/db-functions" - -if (( $# < 1 )); then - msg "usage: %s ..." "${0##*/}" - exit 1 -fi - -# Lock everything to reduce possibility of interfering task between the different repo-updates -script_lock -for repo in "${TESTING_REPO}" "${STABLE_REPOS[@]}"; do - for pkgarch in "${ARCHES[@]}"; do - repo_lock "${repo}" "${pkgarch}" || exit 1 - done -done - -declare -A pkgs - -for pkgbase in "$@"; do - found_source=false - for tarch in "${ARCHES[@]}"; do - if [[ -n $(getdbinfo "$TESTING_REPO" "$tarch" "$pkgbase" NAME) ]]; then - found_source=true - break - fi - done - [[ $found_source = true ]] || die "%s not found in [%s]" "$pkgbase" "$TESTING_REPO" - found_target=false - for tarch in "${ARCHES[@]}"; do - for repo in "${STABLE_REPOS[@]}"; do - if [[ -n $(getdbinfo "$repo" "$tarch" "$pkgbase" NAME) ]]; then - found_target=true - pkgs[${repo}]+="${pkgbase} " - break 2 - fi - done - done - [[ $found_target = true ]] || die "%s not found in any of these repos: %s" "$pkgbase" "${STABLE_REPOS[*]}" -done - -for pkgarch in "${ARCHES[@]}"; do - repo_unlock "${TESTING_REPO}" "${pkgarch}" -done -for repo in "${STABLE_REPOS[@]}"; do - for pkgarch in "${ARCHES[@]}"; do - repo_unlock "${repo}" "${pkgarch}" - done - if [[ -n ${pkgs[${repo}]} ]]; then - "$(dirname "$(readlink -e "$0")")/db-move" "${TESTING_REPO}" "${repo}" ${pkgs[${repo}]} - fi -done - -script_unlock -- cgit v1.2.2 From be7ac4a0e0a081a81f2d8191556c6ce581006815 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Thu, 15 Mar 2018 23:56:07 -0400 Subject: Don't use Docker for the test suite / use libremakepkg --- .travis.yml | 3 --- Makefile | 18 +++++++----------- README.md | 2 +- test/Dockerfile | 20 -------------------- test/Makefile | 5 ++++- test/lib/common.bash | 10 +++++----- 6 files changed, 17 insertions(+), 41 deletions(-) delete mode 100644 test/Dockerfile diff --git a/.travis.yml b/.travis.yml index 0b0519c..ef3d669 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,8 +2,5 @@ sudo: required language: bash -services: - - docker - script: - make test diff --git a/Makefile b/Makefile index 988d335..ecb78c9 100644 --- a/Makefile +++ b/Makefile @@ -1,15 +1,11 @@ -IMAGE:=dbscripts/test -RUN_OPTIONS:=--rm --network=none -v $(PWD):/dbscripts:ro --tmpfs=/tmp:exec -w /dbscripts/test +all: +.PHONY: all -test-image: - docker build --pull -t $(IMAGE) test +test: + $(MAKE) -C test test -test: test-image - docker run $(RUN_OPTIONS) $(IMAGE) make test - -test-coverage: test-image +test-coverage: rm -rf ${PWD}/coverage - mkdir -m 777 ${PWD}/coverage - docker run $(RUN_OPTIONS) -v ${PWD}/coverage:/coverage -e COVERAGE_DIR=/coverage $(IMAGE) make test-coverage + $(MAKE) -C test test-coverage -.PHONY: test-image test test-coverage +.PHONY: test test-coverage diff --git a/README.md b/README.md index 87bbb10..9f5a415 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,6 @@ Things that haven't been mentioned yet: - `cron-jobs/devlist-mailer` - `cron-jobs/sourceballs` ## Testing -* Install the `make` and `docker` packages. Start the docker daemon by issuing `systemctl start docker`. +* Install the `base-devel` package group, as well as the `bash-bats`, `kcov`, `librelib`, and `subversion` packages. * The test suite can now be run with `make test`. * A coverage report can be generated with `make test-coverage`. Open `coverage/index.html` in your web browser to inspect the results. diff --git a/test/Dockerfile b/test/Dockerfile deleted file mode 100644 index 83c8449..0000000 --- a/test/Dockerfile +++ /dev/null @@ -1,20 +0,0 @@ -FROM archlinux/base -RUN pacman -Syu --noconfirm --needed sudo fakeroot awk subversion make kcov bash-bats gettext grep -RUN pacman-key --init -RUN echo '%wheel ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/wheel -RUN useradd -N -g users -G wheel -d /build -m tester -USER tester -RUN echo -e "\ -Key-Type: RSA\n\ -Key-Length: 1024\n\ -Key-Usage: sign\n\ -Name-Real: Bob Tester\n\ -Name-Email: tester@localhost\n\ -Expire-Date: 0\n\ -%no-protection\n\ -%commit\n"\ -| gpg --quiet --batch --no-tty --no-permission-warning --gen-key -RUN gpg --export | sudo pacman-key -a - -RUN sudo pacman-key --lsign-key tester@localhost -ENV BUILDDIR=/build -ENV PACKAGER="Bob Tester " diff --git a/test/Makefile b/test/Makefile index d9437bf..3252374 100644 --- a/test/Makefile +++ b/test/Makefile @@ -1,10 +1,13 @@ +export BUILDDIR ?= $(or $(TMPDIR),/tmp)/dbscripts-build +COVERAGE_DIR ?= $(CURDIR)/../coverage + test: PATH=$(CURDIR)/../:$(CURDIR)/../cron-jobs/:$(PATH) bats cases test-coverage: PATH=$(CURDIR)/../:$(CURDIR)/../cron-jobs/:$(PATH) kcov \ --include-path=$(CURDIR)/../ \ - --exclude-path=$(CURDIR)/../test,$(CURDIR)/../cron-jobs/makepkg.conf,$(CURDIR)/../config \ + --exclude-path=$(CURDIR)/../test,$(CURDIR)/../config \ $(COVERAGE_DIR) \ bats cases diff --git a/test/lib/common.bash b/test/lib/common.bash index febc458..6957992 100644 --- a/test/lib/common.bash +++ b/test/lib/common.bash @@ -46,12 +46,12 @@ __buildPackage() { pkgarches=($(. PKGBUILD; echo ${arch[@]})) for tarch in ${pkgarches[@]}; do if [ "${tarch}" == 'any' ]; then - PKGDEST=${pkgdest} PKGEXT=${PKGEXT} makepkg -c - mapfile -tO "${#pkgfiles[@]}" pkgfiles < <(PKGDEST=${pkgdest} PKGEXT=${PKGEXT} makepkg --packagelist) - else - PKGDEST=${pkgdest} PKGEXT=${PKGEXT} CARCH=${tarch} makepkg -c - mapfile -tO "${#pkgfiles[@]}" pkgfiles < <(PKGDEST=${pkgdest} PKGEXT=${PKGEXT} CARCH=${tarch} makepkg --packagelist) + tarch=$(source "$(librelib conf)" && load_conf makepkg.conf CARCH && printf '%s\n' "$CARCH") fi + sudo librechroot -n "dbscripts@${tarch}" -A "$tarch" sync + mapfile -tO "${#pkgfiles[@]}" pkgfiles < <(PKGDEST=${pkgdest} PKGEXT=${PKGEXT} CARCH=${tarch} makepkg --packagelist) + sudo librechroot -n "dbscripts@${tarch}" run bash -c "$(printf '%q ' echo "PKGEXT=${PKGEXT@Q}") >> /etc/makepkg.conf" + sudo PKGDEST="${pkgdest}" libremakepkg -n "dbscripts@${tarch}" done for p in ${pkgfiles[@]}; do -- cgit v1.2.2 From 04915096f0da6832d60125623cf85df63ecb15ff Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Fri, 22 Jun 2018 12:54:14 -0400 Subject: xbs instead of svn --- config | 2 -- db-abs | 99 +++++++++++----------------------------------------- test/lib/common.bash | 43 ++++++++++------------- 3 files changed, 38 insertions(+), 106 deletions(-) diff --git a/config b/config index 769a69f..4519a07 100644 --- a/config +++ b/config @@ -1,8 +1,6 @@ #!/hint/bash FTP_BASE="/srv/ftp" -SVNREPO='' -SVNUSER='' PKGREPOS=() PKGPOOL='' SRCPOOL='' diff --git a/db-abs b/db-abs index c510bad..02cf76b 100644 --- a/db-abs +++ b/db-abs @@ -1,105 +1,46 @@ #!/hint/bash -arch_svn() { - if [[ -z ${SVNUSER} ]]; then - /usr/bin/svn "${@}" - else - sudo -u "${SVNUSER}" -- /usr/bin/svn --username "${USER}" "${@}" - fi -} - -_abs_checkout() { - local pkgbase=$1 - if ! [[ -d ${WORKDIR}/svn ]]; then - arch_svn checkout -q -N "${SVNREPO}" "${WORKDIR}/svn" >/dev/null - fi - if ! [[ -d ${WORKDIR}/svn/${pkgbase} ]]; then - arch_svn up -q "${WORKDIR}/svn/${pkgbase}" >/dev/null - fi -} - abs_move_preflight_check() { local repo_from=$1 local tarch=$2 local pkgbase=$3 + local dir_tarch dir_any - _abs_checkout "$pkgbase" - local reposdir="${WORKDIR}/svn/${pkgbase}/repos" - [[ -r ${reposdir}/${repo_from}-${tarch}/PKGBUILD || -r ${reposdir}/${repo_from}-any/PKGBUILD ]] + dir_tarch="$(xbs releasepath "${pkgbase}" "${repo_from}" "${tarch}")" + dir_any="$(xbs releasepath "${pkgbase}" "${repo_from}" any)" + [[ -r ${dir_tarch}/PKGBUILD || -r ${dir_any}/PKGBUILD ]] } abs_move_start() { - abs_move_repo_from=$1 - abs_move_repo_to=$2 - abs_move_pkgbase=$3 + local repo_from=$1 + local repo_to=$2 + local pkgbase=$3 - abs_move_tag_list="" + xbs move "${repo_from}" "${repo_to}" "${pkgbase}" } -# If the "from" PKGBUILD doesn't exist, this is a no-op (not an -# error). This is because we expect to be called exactly once for -# each tarch (eg: x86_64, i686), but if arch=(any) then we only need -# do the work once; on the subsequent calls the "from" PKGBUILD won't -# exist anymore. If the source PKGBUILD never existed, we expect that -# to have already been caught by abs_move_preflight_check. abs_move_arch() { - local tarch=$1 - - local repo_from=$abs_move_repo_from - local repo_to=$abs_move_repo_to - local pkgbase=$abs_move_pkgbase - - local pkgarch - if [[ -r "${WORKDIR}/svn/${pkgbase}/repos/${repo_from}-${tarch}/PKGBUILD" ]]; then - pkgarch=$tarch - elif [[ -r "${WORKDIR}/svn/${pkgbase}/repos/${repo_from}-any/PKGBUILD" ]]; then - pkgarch=any - else - return 0 - fi - - local svnrepo_from="${WORKDIR}/svn/${pkgbase}/repos/${repo_from}-${pkgarch}" - local svnrepo_to="${WORKDIR}/svn/${pkgbase}/repos/${repo_to}-${pkgarch}" - msg2 "%s (%s)" "$pkgbase" "$pkgarch" - - if [[ -d ${svnrepo_to} ]]; then - for file in $(arch_svn ls "${svnrepo_to}"); do - arch_svn rm -q "${svnrepo_to}/$file@" - done - else - mkdir "${svnrepo_to}" - arch_svn add -q "${svnrepo_to}" - fi - - for file in $(arch_svn ls "${svnrepo_from}"); do - arch_svn mv -q -r HEAD "${svnrepo_from}/$file@" "${svnrepo_to}/" - done - arch_svn rm --force -q "${svnrepo_from}" - abs_move_tag_list+=", $pkgarch" + local pkgarch=$1 + # no-op, everything was taken care of in _start } abs_move_finish() { - local repo_from=$abs_move_repo_from - local repo_to=$abs_move_repo_to - local pkgbase=$abs_move_pkgbase - - local tag_list="${abs_move_tag_list#, }" - arch_svn commit -q "${WORKDIR}/svn/${pkgbase}" -m "${0##*/}: moved ${pkgbase} from [${repo_from}] to [${repo_to}] (${tag_list})" + : + # no-op, everything was taken care of in _start } abs_remove() { local repo=$1 local arch=$2 local pkgbase=$3 + local path - local svnrepo="$repo-$arch" - - _abs_checkout "$pkgbase" - if [[ -d ${WORKDIR}/svn/$pkgbase/repos/$svnrepo ]]; then - arch_svn rm --force -q "${WORKDIR}/svn/$pkgbase/repos/$svnrepo" - arch_svn commit -q "${WORKDIR}/svn/$pkgbase" -m "${0##*/}: $pkgbase removed by $(id -un)" + path="$(xbs releasepath "$pkgbase" "$repo" "$arch")" + if [[ -d $path ]]; then + xbs unrelease "$pkgbase" "$repo" "$arch" else - warning "pkgbase '%s' not found in svn; unable to commit removal to svn" "$pkgbase" + warning "pkgbase '%s' not found in %s for %s; unable to commit removal there" \ + "$pkgbase" "$(xbs name)" "$repo-$arch" fi } @@ -109,6 +50,6 @@ abs_export() { local pkgbase=$3 local dest=$4 - arch_svn export -q "${SVNREPO}/${pkgbase}/repos/${repo}-${pkgarch}" \ - "${dest}" >/dev/null 2>&1 + cp -a "$(xbs releasepath "${pkgbase}" "${repo}" "${pkgarch}")" \ + "$dest" } diff --git a/test/lib/common.bash b/test/lib/common.bash index 6957992..d0ee671 100644 --- a/test/lib/common.bash +++ b/test/lib/common.bash @@ -64,30 +64,6 @@ __buildPackage() { done } -__archrelease() { - local repo=$1 - local pkgarches - local tarch - local tag - - pkgarches=($(. PKGBUILD; echo ${arch[@]})) - pushd .. - for tarch in ${pkgarches[@]}; do - tag=${repo}-${tarch} - - if [[ -d repos/$tag ]]; then - svn rm repos/$tag/PKGBUILD - else - mkdir -p repos/$tag - svn add repos/$tag - fi - - svn copy -r HEAD trunk/PKGBUILD repos/$tag/ - done - svn commit -m "__archrelease" - popd -} - setup() { local p local pkg @@ -129,6 +105,17 @@ eot svnadmin create "${TMP}/svn-packages-repo" svn checkout -q "file://${TMP}/svn-packages-repo" "${TMP}/svn-packages-copy" + + mkdir -p "${TMP}/home/.config/xbs" + export XDG_CONFIG_HOME="${TMP}/home/.config" + cat < "$XDG_CONFIG_HOME/xbs/xbs-abs.conf" + SVNDIR="${TMP}" + SVNREPOS=( + "svn-packages-copy file://${TMP}/svn-packages-repo core extra testing" + ) + ARCHES=(${ARCHES[*]@Q}) +eot + echo 'BUILDSYSTEM=abs' > "$XDG_CONFIG_HOME/xbs/xbs.conf" } teardown() { @@ -138,6 +125,8 @@ teardown() { releasePackage() { local repo=$1 local pkgbase=$2 + local pkgarches + local tarch if [ ! -d "${TMP}/svn-packages-copy/${pkgbase}/trunk" ]; then mkdir -p "${TMP}/svn-packages-copy/${pkgbase}"/{trunk,repos} @@ -147,8 +136,12 @@ releasePackage() { fi pushd "${TMP}/svn-packages-copy/${pkgbase}/trunk/" + __buildPackage "${STAGING}"/${repo} - __archrelease ${repo} + pkgarches=($(. PKGBUILD; echo ${arch[@]})) + for tarch in "${pkgarches[@]}"; do + xbs release-client "${repo}" "${tarch}" + done popd } -- cgit v1.2.2 From ec06d25a0524e1fb533c68f48d0df61cc98e6bbf Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Fri, 22 Jun 2018 12:53:28 -0400 Subject: s/svn/xbs/g diff -u \ <(git show|sed -n 's/^-//p'|grep -v '^-- a/'|sed 's/svn/xbs/g') \ <(git show|sed -n 's/^+//p'|grep -v '^++ b/'|sed 's/svn/xbs/g') --- cron-jobs/sourceballs | 2 +- db-functions | 20 ++++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index f56f0cf..8d14323 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -77,7 +77,7 @@ for repo in "${PKGREPOS[@]}"; do continue fi - # Get the sources from svn + # Get the sources from xbs mkdir -p -m0770 "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}" abs_export "$repo" "$pkgarch" "$pkgbase" \ "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" diff --git a/db-functions b/db-functions index 39a6905..ce1bed6 100644 --- a/db-functions +++ b/db-functions @@ -343,7 +343,7 @@ check_pkgfile() { [[ ${pkgfile##*/} = "${pkgname}-${pkgver}-${pkgarch}"* ]] } -check_pkgsvn() { +check_pkgxbs() { local pkgfile="${1}" local _pkgbase="$(getpkgbase "${pkgfile}")" || return 1 local _pkgname="$(getpkgname "${pkgfile}")" || return 1 @@ -359,11 +359,11 @@ check_pkgsvn() { "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}" >/dev/null || return 1 fi - local svnver="$(. "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}/PKGBUILD"; get_full_version)" - [[ "${svnver}" = "${_pkgver}" ]] || return 1 + local xbsver="$(. "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}/PKGBUILD"; get_full_version)" + [[ "${xbsver}" = "${_pkgver}" ]] || return 1 - local svnnames=($(. "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}/PKGBUILD"; echo "${pkgname[@]}")) - in_array "${_pkgname}" "${svnnames[@]}" || return 1 + local xbsnames=($(. "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}/PKGBUILD"; echo "${pkgname[@]}")) + in_array "${_pkgname}" "${xbsnames[@]}" || return 1 return 0 } @@ -374,7 +374,7 @@ check_splitpkgs() { local pkgfiles=("${@}") local pkgfile local pkgdir - local svnname + local xbsname mkdir -p "${WORKDIR}/check_splitpkgs/" pushd "${WORKDIR}/check_splitpkgs" >/dev/null @@ -393,16 +393,16 @@ check_splitpkgs() { "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}" >/dev/null || return 1 fi - local svnnames=($(. "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}/PKGBUILD"; echo "${pkgname[@]}")) - printf '%s\n' "${svnnames[@]}" >> "${repo}/${_pkgarch}/${_pkgbase}/svn" + local xbsnames=($(. "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}/PKGBUILD"; echo "${pkgname[@]}")) + printf '%s\n' "${xbsnames[@]}" >> "${repo}/${_pkgarch}/${_pkgbase}/xbs" done popd >/dev/null for pkgdir in "${WORKDIR}/check_splitpkgs/${repo}"/*/*; do [[ ! -d ${pkgdir} ]] && continue sort -u "${pkgdir}/staging" -o "${pkgdir}/staging" - sort -u "${pkgdir}/svn" -o "${pkgdir}/svn" - if [[ ! -z "$(comm -13 "${pkgdir}/staging" "${pkgdir}/svn")" ]]; then + sort -u "${pkgdir}/xbs" -o "${pkgdir}/xbs" + if [[ ! -z "$(comm -13 "${pkgdir}/staging" "${pkgdir}/xbs")" ]]; then return 1 fi done -- cgit v1.2.2 From d6f7431b1eb840814618d3730b40dbd09925e714 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Fri, 22 Jun 2018 12:57:36 -0400 Subject: Parabola documentation --- .gitignore | 8 + COPYING | 674 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ HACKING.org | 15 ++ README.md | 94 ++++++++- config | 4 + 5 files changed, 790 insertions(+), 5 deletions(-) create mode 100644 COPYING create mode 100644 HACKING.org diff --git a/.gitignore b/.gitignore index 293ca9f..7c92465 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,12 @@ *~ /config.local +\#*# +.#* +*.pyc /.idea /coverage + +yftime +src* +pkg* +!/test/fixtures/pkg-*/ diff --git a/COPYING b/COPYING new file mode 100644 index 0000000..94a9ed0 --- /dev/null +++ b/COPYING @@ -0,0 +1,674 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. diff --git a/HACKING.org b/HACKING.org new file mode 100644 index 0000000..2fb3e6c --- /dev/null +++ b/HACKING.org @@ -0,0 +1,15 @@ +* README-WORKING-ON.txt +Message from aurelien. + +Working on #bug 5 https://labs.parabola.nu/issues/5 + +db-update~old is the untouched file + +db-update is the testing file + + +db-update-mailer is another (testing) one for the moment. in cron/ +* TODO.txt +** Fix db-move + + - Make it use abslibre diff --git a/README.md b/README.md index 9f5a415..c23ca93 100644 --- a/README.md +++ b/README.md @@ -1,24 +1,36 @@ -# Arch Linux repository management scripts [![Build Status](https://travis-ci.org/archlinux/dbscripts.svg?branch=master)](https://travis-ci.org/archlinux/dbscripts) +# Parabola GNU/Linux-libre repository management scripts ## Configuration * The default configuration can be found in `config`. * An optional `config.local` may override the default configuration. -* The path and name of the local configuration file can be overridden by setting the `DBSCRIPTS_CONFIG` environment variable. +* The path and name of the local configuration file can be overridden + by setting the `DBSCRIPTS_CONFIG` environment variable. ## Overview The executables that you (might) care about are: dbscripts/ ├── cron-jobs/ + │   ├── db-cleanup [Parabola only] │   ├── devlist-mailer │   ├── ftpdir-cleanup │   ├── integrity-check + │   ├── make_repo_torrents [Parabola only] │   ├── sourceballs │   ├── update-web-db [Arch Linux only] │   └── update-web-files-db [Arch Linux only] + ├── db-check-nonfree [Parabola only] + ├── db-check-package-libraries [Parabola only] + ├── db-check-repo-sanity [Parabola only] + ├── db-check-unsigned-packages [Parabola only] + ├── db-check-unsigned-packages.py [Parabola only] + ├── db-import-archlinuxarm-src [Parabola only] + ├── db-import-pkg [Parabola only] + ├── db-init [Parabola only] ├── db-move ├── db-remove ├── db-repo-add ├── db-repo-remove ├── db-update + ├── make_individual_torrent [Parabola only] └── testing2x [Arch Linux only] Ok, now let's talk about what those are. @@ -41,17 +53,89 @@ have - `cron-jobs/integrity-check` +Instead of enhancing `integrity-check`, Parabola developers have +decided to write multiple stand-alone tools that should probably be +merged into `integrity-check` + + - `db-check-*` + When we remove a package from a repository, it stays in the package "pool". We would like to be able to eventually remove packages from the pool, to reclaim the disk space: - `cron-jobs/ftpdir-cleanup` + - `cron-jobs/db-cleanup` + +Both of these programs do the exact same thing. Parabola developers +decided to write their own from scratch, instead of modifying +`ftpdir-cleanup`. They should eventually be merged. + +But, Parabola doesn't just publish our own packages, we also import +packages from elsewhere: + + - `db-import-*` + +Unfortunately, these import scripts fiddle with the repos directly, +rather than calling `db-{update,move,remove}`, and are prone to break +things. Things that haven't been mentioned yet: - `cron-jobs/devlist-mailer` + - `cron-jobs/make_repo_torrents` - `cron-jobs/sourceballs` + - `db-init` + - `make_individual_torrent` + ## Testing -* Install the `base-devel` package group, as well as the `bash-bats`, `kcov`, `librelib`, and `subversion` packages. -* The test suite can now be run with `make test`. -* A coverage report can be generated with `make test-coverage`. Open `coverage/index.html` in your web browser to inspect the results. + +### Requirements + +Packages: + + - `base-devel` package group + - `bash-bats` package + - `kcov` package + - `libretools` package + - `subversion` package + - `qemu-user-static-binfmt` package + +Other setup: + + - Arrange for `gpg` to sign files using a key that is trusted by the + pacman keyring. This is easy if you are already a Parabola + packager--your PGP key is already trusted by the pacman keyring, as + it is included in the `parabola-keyring` package. + +### Running the tests + +The command to run the test suite is + + [BUILDIR=/path/to/cache] [COVERAGE_DIR=/path/to/output] make check[-coverage] + + - The default `BUILDDIR` is `${TMPDIR:-/tmp}/dbscripts-build`; + packages built as part of test setup are cached here. This can be + shared between test suite runs (as libremakepkg isn't the target of + our tests). + - Using the `check-coverage` target instead of the `check` target + also generates a coverage report when running the tests; open + `${COVERAGE_DIR}/index.html` in your web browser to view the + results. The default `COVERAGE_DIR` is `$PWD/coverage`. + +### Other testing notes + + - The test suite will use `sudo` to run `unshare -m`, `librechroot`, + and `libremakepkg`. It should be safe, but if this makes you + nervous, run the test suite in a VM or container. + +Things outside of the dbscripts directory that are modified: + + - The test suite creates and uses the `librechroot` chroots + `dbscripts@{any,armv7h,i686,x86_64}`. It does not ever update + these after creation or clean these up, and leaves them around to + speed up future runs. + - Built packages are cached in `BUILDDIR` (see above). This + directory is never cleaned up, and is left around to speed up + future runs. + +You may want to manually clean these up periodically. diff --git a/config b/config index 4519a07..43c9362 100644 --- a/config +++ b/config @@ -1,4 +1,8 @@ #!/hint/bash +# Please try to refrain from adding new variables to this file. +# Instead, create separate ${toolname}.conf files. Only add a +# variable here if multiple tools start needing the option. + FTP_BASE="/srv/ftp" PKGREPOS=() -- cgit v1.2.2 From 889d50e5381f808a9b126d4821489196e7a51c5f Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Fri, 22 Jun 2018 13:23:27 -0400 Subject: Minor formatting improvement --- db-functions | 2 +- db-move | 3 ++- test/cases/db-update.bats | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/db-functions b/db-functions index ce1bed6..776c74a 100644 --- a/db-functions +++ b/db-functions @@ -464,7 +464,7 @@ set_repo_permission() { if [[ -w ${dbfile} ]]; then local group=$(/usr/bin/stat --printf='%G' "$(dirname "${dbfile}")") - chgrp "$group" "${dbfile}" || error "Could not change group of %s to %s" "$dbfile" "$group" + chgrp "$group" "${dbfile}" || error "Could not change group of %s to %s" "$dbfile" "$group" chgrp "$group" "${filesfile}" || error "Could not change group of %s to %s" "$filesfile" "$group" chmod g+w "${dbfile}" || error "Could not set write permission for group %s to %s" "$group" "$dbfile" chmod g+w "${filesfile}" || error "Could not set write permission for group %s to %s" "$group" "$filesfile" diff --git a/db-move b/db-move index 40af7c5..6a23e9c 100755 --- a/db-move +++ b/db-move @@ -24,7 +24,7 @@ for pkgarch in "${ARCHES[@]}"; do repo_lock "${repo_from}" "${pkgarch}" || exit 1 done -# check if packages to be moved exist in svn and ftp dir +# First loop is to check that all necessary files exist for pkgbase in "${args[@]:2}"; do found=false for tarch in "${ARCHES[@]}"; do @@ -51,6 +51,7 @@ declare -A remove_pkgs for pkgbase in "${args[@]:2}"; do abs_move_start "$repo_from" "$repo_to" "$pkgbase" for tarch in "${ARCHES[@]}"; do + msg2 "%s (%s)" "$pkgbase" "$tarch" abs_move_arch "$tarch" while read -r pkgname pkgver pkgfile; do ln -s "../../../${PKGPOOL}/${pkgfile}" "${ftppath_to}/${tarch}/" diff --git a/test/cases/db-update.bats b/test/cases/db-update.bats index 6c500c4..f3801a8 100755 --- a/test/cases/db-update.bats +++ b/test/cases/db-update.bats @@ -222,7 +222,7 @@ load ../lib/common checkRemovedPackageDB extra 'pkg-any-b' } -@test "package has to be aregular file" { +@test "package has to be a regular file" { local p local target=$(mktemp -d) -- cgit v1.2.2 From e76bd26d0a69ce6e97aab60f236b4c67fcd3f805 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Fri, 22 Jun 2018 13:25:16 -0400 Subject: xbs is broken --- db-update | 10 ++++------ test/cases/db-update.bats | 2 ++ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/db-update b/db-update index 8ce7b2d..156cb91 100755 --- a/db-update +++ b/db-update @@ -46,9 +46,6 @@ for repo in "${repos[@]}"; do if ! pacman-key -v "${pkg}.sig" >/dev/null 2>&1; then die "Package %s does not have a valid signature" "$repo/${pkg##*/}" fi - if ! check_pkgsvn "${pkg}" "${repo}"; then - die "Package %s is not consistent with svn repository" "$repo/${pkg##*/}" - fi if ! check_pkgrepos "${pkg}"; then die "Package %s already exists in another repository" "$repo/${pkg##*/}" fi @@ -62,9 +59,10 @@ for repo in "${repos[@]}"; do die "Package %s was not built in a chroot" "$repo/${pkg##*/}" fi done - if ! check_splitpkgs "${repo}" "${pkgs[@]}"; then - die "Missing split packages for %s" "$repo" - fi + # This is fucking obnoxious + #if ! check_splitpkgs ${repo} "${pkgs[@]}"; then + # die "Missing split packages for %s" "${repo}" + #fi else die "Could not read %s" "$STAGING" fi diff --git a/test/cases/db-update.bats b/test/cases/db-update.bats index f3801a8..4bd1c4f 100755 --- a/test/cases/db-update.bats +++ b/test/cases/db-update.bats @@ -115,6 +115,7 @@ load ../lib/common } @test "add incomplete split package fails" { + skip # commented out with "This is fucking obnoxious" -- abslibre is broken local repo='extra' local pkgbase='pkg-split-a' local arch @@ -200,6 +201,7 @@ load ../lib/common } @test "add package with inconsistent pkgbuild fails" { + skip # abslibre is broken releasePackage extra 'pkg-any-a' updateRepoPKGBUILD 'pkg-any-a' extra any -- cgit v1.2.2 From 4078d10963eac177dad70bc05f4f1dca95e51d52 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Fri, 22 Jun 2018 12:48:16 -0400 Subject: systemd --- cron-jobs/update-lastsync | 4 ++++ systemd/update-lastsync.service | 7 +++++++ systemd/update-lastsync.timer | 10 ++++++++++ 3 files changed, 21 insertions(+) create mode 100755 cron-jobs/update-lastsync create mode 100644 systemd/update-lastsync.service create mode 100644 systemd/update-lastsync.timer diff --git a/cron-jobs/update-lastsync b/cron-jobs/update-lastsync new file mode 100755 index 0000000..799c761 --- /dev/null +++ b/cron-jobs/update-lastsync @@ -0,0 +1,4 @@ +#!/bin/bash + +. "$(dirname "$(readlink -e "$0")")/../config" +date -u +'%s' > "${FTP_BASE}/lastsync" diff --git a/systemd/update-lastsync.service b/systemd/update-lastsync.service new file mode 100644 index 0000000..33abd45 --- /dev/null +++ b/systemd/update-lastsync.service @@ -0,0 +1,7 @@ +[Unit] +Description=Update ${FTP_BASE}/lastsync + +[Service] +Type=oneshot +User=repo +ExecStart=/opt/dbscripts/cron-jobs/update-lastsync diff --git a/systemd/update-lastsync.timer b/systemd/update-lastsync.timer new file mode 100644 index 0000000..428c0e2 --- /dev/null +++ b/systemd/update-lastsync.timer @@ -0,0 +1,10 @@ +[Unit] +Description=Periodic update of ${FTP_BASE}/lastsync + +[Timer] +OnCalendar=*-*-* *:00/10:00 +AccuracySec=1us +Persistent=true + +[Install] +WantedBy=timers.target -- cgit v1.2.2 From 527618a398c0640382b51b56ca87c18700a46c58 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Fri, 22 Jun 2018 12:48:27 -0400 Subject: db-cleanup --- cron-jobs/db-cleanup | 70 ++++++++++++++++++++++++++++++++++++++++++++++ db-cleanup.conf | 21 ++++++++++++++ systemd/db-cleanup.service | 7 +++++ systemd/db-cleanup.timer | 9 ++++++ 4 files changed, 107 insertions(+) create mode 100755 cron-jobs/db-cleanup create mode 100644 db-cleanup.conf create mode 100644 systemd/db-cleanup.service create mode 100644 systemd/db-cleanup.timer diff --git a/cron-jobs/db-cleanup b/cron-jobs/db-cleanup new file mode 100755 index 0000000..ca6ec4e --- /dev/null +++ b/cron-jobs/db-cleanup @@ -0,0 +1,70 @@ +#!/bin/bash +# Syncs pools against themselves using database contents as filter to cleanup +# them up +# License: GPLv3 + +# Principles +# * Get repos dbs contents +# * Make them a include list +# * Rsync pools against themselves removing excluded files +# * Instant cleanup! + +trap_exit() { + echo + error "$@" + exit 1 +} + +source "$(dirname $(dirname "$(readlink -e "$0")"))/config" +source "$(dirname $(dirname "$(readlink -e "$0")"))/db-cleanup.conf" +source "$(librelib messages)" + +# From makepkg +set -E + +trap 'trap_exit "$(gettext "TERM signal caught. Exiting...")"' TERM HUP QUIT +trap 'trap_exit "$(gettext "Aborted by user! Exiting...")"' INT +trap 'trap_exit "$(gettext "An unknown error has occurred. Exiting...")"' ERR + +EXTRAFLAGS=() +"${CLEANUP_DRYRUN}" && EXTRAFLAGS+=(--dry-run) + +filter=$(mktemp -t "${0##*/}.XXXXXXXXXX") +trap "rm -f -- ${filter@Q}" EXIT + +for _repo in "${PKGREPOS[@]}"; do + for _arch in "${ARCHES[@]}"; do + msg "Getting %s-%s database" "${_repo}" "${_arch}" + + dbfile="${FTP_BASE}/${_repo}/os/${_arch}/${_repo}${DBEXT}" + + if [ ! -r "${dbfile}" ]; then + warning "Not found" + continue + fi + + # Echo the contents into a filter file + bsdtar tf "${dbfile}" | \ + cut -d'/' -f1 | \ + sort -u | \ + sed "s|$|*|" >> "$filter" + + done +done + +msg "Removing old files:" + +for POOL in "${PKGPOOLS[@]}" "${SRCPOOLS[@]}"; do + msg2 '%s' "${POOL}" + + rsync "${EXTRAFLAGS[@]}" -va --delete-excluded \ + --include-from="$filter" \ + --exclude="*" \ + "${FTP_BASE}/${POOL}/" \ + "${FTP_BASE}/${POOL}/" +done + +msg "Removing dead symlinks:" +actions=(-print) +"${CLEANUP_DRYRUN}" || actions+=(-delete) +find -L "${FTP_BASE}/" -type l "${actions[@]}" diff --git a/db-cleanup.conf b/db-cleanup.conf new file mode 100644 index 0000000..12d65f9 --- /dev/null +++ b/db-cleanup.conf @@ -0,0 +1,21 @@ +#!/hint/bash + +# Both PKGPOOLS and SRCPOOLS are relative to `config:FTP_BASE`. + +# Directories where packages are shared between repos +PKGPOOLS=( + pool/parabola # Parabola GNU/Linux-libre + pool/alarm # Arch Linux ARM + pool/archlinux32 # Arch Linux 32-bits + pool/packages # Arch Linux (project) + pool/community # Arch Linux (community) +) + +# Directories where sources are stored +SRCPOOLS=( + sources/parabola # Parabola GNU/Linux-libre + sources/alarm # Arch Linux ARM + sources/archlinux32 # Arch Linux 32-bits + sources/packages # Arch Linux (project) + sources/community # Arch Linux (community) +) diff --git a/systemd/db-cleanup.service b/systemd/db-cleanup.service new file mode 100644 index 0000000..669f87e --- /dev/null +++ b/systemd/db-cleanup.service @@ -0,0 +1,7 @@ +[Unit] +Description=Clean up old files + +[Service] +Type=oneshot +User=repo +ExecStart=/opt/dbscripts/cron-jobs/db-cleanup diff --git a/systemd/db-cleanup.timer b/systemd/db-cleanup.timer new file mode 100644 index 0000000..8a1c5da --- /dev/null +++ b/systemd/db-cleanup.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Periodic cleaning of old packages + +[Timer] +OnCalendar=Sun,Tue,Thu,Sat *-*-* 14:13:00 +Persistent=true + +[Install] +WantedBy=timers.target -- cgit v1.2.2 From 4852331439733769c019e3f154d23f30a67f2bba Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Fri, 22 Jun 2018 12:47:19 -0400 Subject: db-import --- config-list-mirrors-archlinux | 61 ++++ config-list-mirrors-archlinuxarm | 63 ++++ db-import-archlinux32.conf | 21 ++ db-import-archlinuxarm-src | 125 +++++++ db-import-archlinuxarm.conf | 41 +++ db-import-community.conf | 25 ++ db-import-packages.conf | 25 ++ db-import-pkg | 396 +++++++++++++++++++++++ systemd/db-import@.service | 12 + systemd/db-import@.timer | 8 + systemd/db-import@archlinux32.timer.d_time.conf | 2 + systemd/db-import@archlinuxarm.timer.d_time.conf | 2 + systemd/db-import@community.timer.d_time.conf | 2 + systemd/db-import@packages.timer.d_time.conf | 2 + test/cases/common.bats | 4 +- test/cases/db-import-pkg.bats | 367 +++++++++++++++++++++ test/fixtures/pkg-any32/PKGBUILD | 13 + test/fixtures/pkg-any64/PKGBUILD | 13 + test/fixtures/pkg-simple-c/PKGBUILD | 13 + test/fixtures/slavery/PKGBUILD | 13 + test/lib/common.bash | 1 - test/lib/runserver | 39 +++ 22 files changed, 1245 insertions(+), 3 deletions(-) create mode 100755 config-list-mirrors-archlinux create mode 100755 config-list-mirrors-archlinuxarm create mode 100644 db-import-archlinux32.conf create mode 100755 db-import-archlinuxarm-src create mode 100644 db-import-archlinuxarm.conf create mode 100644 db-import-community.conf create mode 100644 db-import-packages.conf create mode 100755 db-import-pkg create mode 100644 systemd/db-import@.service create mode 100644 systemd/db-import@.timer create mode 100644 systemd/db-import@archlinux32.timer.d_time.conf create mode 100644 systemd/db-import@archlinuxarm.timer.d_time.conf create mode 100644 systemd/db-import@community.timer.d_time.conf create mode 100644 systemd/db-import@packages.timer.d_time.conf create mode 100644 test/cases/db-import-pkg.bats create mode 100644 test/fixtures/pkg-any32/PKGBUILD create mode 100644 test/fixtures/pkg-any64/PKGBUILD create mode 100644 test/fixtures/pkg-simple-c/PKGBUILD create mode 100644 test/fixtures/slavery/PKGBUILD create mode 100755 test/lib/runserver diff --git a/config-list-mirrors-archlinux b/config-list-mirrors-archlinux new file mode 100755 index 0000000..e59ea2c --- /dev/null +++ b/config-list-mirrors-archlinux @@ -0,0 +1,61 @@ +#!/usr/bin/env ruby +# 2013, 2016, 2018 Luke Shumaker + +require 'json' +require 'net/http' +require 'optparse' +require 'set' +devnull = open("/dev/null") + +requires = Set.new() +begin + OptionParser.new do |parser| + parser.on("--require=DIR", "Optional directory that the mirror needs ('archive', 'iso', 'other', or 'sources')") do |dir| + requires.add(dir) + end + parser.on("-h", "--help", "Display this help") do + puts parser + exit + end + end.parse! +rescue OptionParser::ParseError => e + STDERR.puts "#{$0}: #{e}" + STDERR.puts "Try '#{$0} --help' for more information." + exit 2 +end +unless ARGV.empty? + STDERR.puts "#{$0}: extra argument '#{ARGV[0]}'" + STDERR.puts "Try '#{$0} --help' for more information." + exit 2 +end + +data = JSON::parse(Net::HTTP.get(URI("https://www.archlinux.org/mirrors/status/tier/1/json/"))) + +if data["version"] != 3 + print "Data format version != 3" + exit 1 +end + +# Filter out URLs with incomplete information +urls = data["urls"].select{|a| a.none?{|k,v|v.nil?}} + +# Filter based on our criteria +rsync_urls = urls.select{|a| a["protocol"]=="rsync"} +if requires.include?("iso") + rsync_urls = rsync_urls.select{|a| a["isos"]==true} + requires.delete("iso") +end +unless requires.empty? + listing_procs = {} + rsync_urls.each do |a| + # trailing slash is important; ensure we have one + listing_procs[a["url"]] ||= IO.popen(["rsync", "--no-motd", "--list-only", "--", a["url"]+"/"], :err => devnull) + end + listings = {} + listing_procs.map.each do |url,listing_proc| + listings[url] = listing_proc.read.lines.map{|line|line.chomp.sub(/.*\s/,'')} + listing_proc.close + end + rsync_urls = rsync_urls.select{|a|requires.all?{|dir|listings[a["url"]].include?(dir)}} +end +puts JSON.dump(rsync_urls) diff --git a/config-list-mirrors-archlinuxarm b/config-list-mirrors-archlinuxarm new file mode 100755 index 0000000..58c2557 --- /dev/null +++ b/config-list-mirrors-archlinuxarm @@ -0,0 +1,63 @@ +#!/usr/bin/env ruby +# 2018 Luke Shumaker + +require 'json' +require 'net/http' +require 'optparse' +require 'set' +devnull = open("/dev/null") + +begin + OptionParser.new do |parser| + parser.on("-h", "--help", "Display this help") do + puts parser + exit + end + end.parse! +rescue OptionParser::ParseError => e + STDERR.puts "#{$0}: #{e}" + STDERR.puts "Try '#{$0} --help' for more information." + exit 2 +end +unless ARGV.empty? + STDERR.puts "#{$0}: extra argument '#{ARGV[0]}'" + STDERR.puts "Try '#{$0} --help' for more information." + exit 2 +end + +data = JSON::parse(Net::HTTP.post( + URI("https://archlinuxarm.org/data/mirrors/list"), "", { + 'Referer' => 'https://archlinuxarm.org/about/mirrors', + 'X-Requested-With' => 'XMLHttpRequest', + }).body) + +hosts = data["data"].map{|m|m[2]} + +# Get a listing of rsync modules for each host +modulelist_procs = {} +hosts.each do |host| + modulelist_procs[host] ||= IO.popen(["timeout", "2s", "rsync", "#{host}::"], :err => devnull) +end +modulelists = {} +modulelist_procs.each do |host,modulelist_proc| + modulelists[host] = modulelist_proc.read + modulelist_proc.close +end +# Look for modules that look like ALARM mirrors +modules = Set.new() +modulelists.each do |host,listing| + hostmodules = listing.lines.select{|line| line =~ /(alarm|arch.*arm)/i }.map{|line| line.chomp.sub(/\s.*/, '') } + modules.merge(hostmodules.map{|m| "rsync://#{host}/#{m}"}) +end +# Filter the ones that we have read access for +listing_procs = {} +modules.each do |m| + listing_procs[m] ||= IO.popen(["rsync", "--no-motd", "--list-only", "--", m], :err => devnull) +end +listing_procs.map.each do |url,listing_proc| + listing_proc.read + listing_proc.close + if $?.exitstatus == 0 + puts url + end +end diff --git a/db-import-archlinux32.conf b/db-import-archlinux32.conf new file mode 100644 index 0000000..2744840 --- /dev/null +++ b/db-import-archlinux32.conf @@ -0,0 +1,21 @@ +#!/hint/bash + +# "tags" are repo-arch pairs +ARCHTAGS=({build-support,core,testing,extra,community}-i686) +ARCHPATH='$arch/$repo' +ARCHPKGPOOL='pool' + +OURPKGPOOL='pool/archlinux32' + +INHERIT=( + 'pool/packages' + 'pool/community' + 'pool/alarm' +) + +# https://git.archlinux32.org/archlinux32/packages/raw/branch/master/core/pacman-mirrorlist/mirrorlist +ARCHMIRROR='rsync://mirror.archlinux32.org/archlinux32/' + +if [[ -n ${DBIMPORT_CONFIG} ]]; then + source "${DBIMPORT_CONFIG}" +fi diff --git a/db-import-archlinuxarm-src b/db-import-archlinuxarm-src new file mode 100755 index 0000000..b0c1181 --- /dev/null +++ b/db-import-archlinuxarm-src @@ -0,0 +1,125 @@ +#!/bin/bash + +set -e + +source "$(dirname "$(readlink -e "$0")")/config" +source "$(dirname "$(readlink -e "$0")")/db-import-archlinuxarm.conf" + +# Steps +# * Sync abs +# * Download blacklist.txt +# * Sync abslibre from abs excluding from blacklist +# * Create repo.abs.tar.gz tarballs + +function sync_abs() { + # sync from ${REPO}.abs.tar.gz files in ALARM mirrors + local TMP_SUBDIR="${tmpdir}/abs-alarm/" + rm -rf -- "$TMP_SUBDIR" "$ABS_ROOT" + mkdir -- "$TMP_SUBDIR" + for ARCH in armv7h; do + for REPO in "${ARCHTAGS[@]%-*}"; do + rsync ${SYNCARGS} \ + rsync://${ABS_SERVER}/${ARCH}/${REPO}/${REPO}.abs.tar.gz \ + "$TMP_SUBDIR" || return $? + mkdir -p -- "${ABS_ROOT}/${ARCH}" + bsdtar xf "${TMP_SUBDIR}/${REPO}.abs.tar.gz" \ + -C "${ABS_ROOT}/${ARCH}" + done + done + + # add changes by ALARM since those aren't present in .abs.tar.gz files + if ! git -C "$ALARM_ROOT" pull &>/dev/null; then + rm -rf -- "$ALARM_ROOT" + if ! git clone "$ALARM_GIT" "$ALARM_ROOT"; then + printf "[FAILED]\n" + return 1 + fi + fi + # FIXME: $ALARM_ROOT doesn't separate packages by architecture + for ARCH in armv7h; do + for REPO in "${ARCHTAGS[@]%-*}"; do + rsync -mrtq --no-p --no-o --no-g --exclude='README' \ + "${ALARM_ROOT}/${REPO}/" \ + "${ABS_ROOT}/${ARCH}/${REPO}" + done + done + + # fix some permissions + find "${ABS_ROOT}" -type d -print0 | xargs -0 chmod 755 + find "${ABS_ROOT}" -type f -print0 | xargs -0 chmod 644 +} + +function get_blacklist() { + libreblacklist update + if ! libreblacklist cat | libreblacklist get-pkg | sort -u | sed "s/^/**\//" > ${BLFILE}; then + printf "[FAILED]\n" + return 1 + fi + + # Prevent using an empty blacklist + [ $(wc -l ${BLFILE} | cut -d " " -f1) -eq 0 ] && return 1 + + printf "[OK]\n" +} + +function sync_abs_libre() { + + # Clone ABSLibre git repo + rm -rf -- "$tmpdir/abslibre" + git clone "$ABSLIBRE_GIT" "$tmpdir/abslibre" + + # Sync from ABS and then sync from ABSLibre + printf ":: Syncing ABSLibre...\t" + if ! rsync ${SYNCARGS} --delete-excluded --exclude-from=${BLFILE} ${ABS_ROOT} ${ABSLIBRE_ROOT}; then + printf "[FAILED]\n" + return 1 + fi + for ARCH in armv7h; do + if ! rsync -v -mrtq --no-motd --no-p --no-o --no-g --quiet --exclude=.git/ "$tmpdir/abslibre/" ${ABSLIBRE_ROOT}/${ARCH}/; then + printf "[FAILED]\n" + return 1 + fi + done + + # fix some permissions + find "${ABSLIBRE_ROOT}" -type d -print0 | xargs -0 chmod 755 + find "${ABSLIBRE_ROOT}" -type f -print0 | xargs -0 chmod 644 + + printf "[OK]\n" +} + +# Create .abs.tar.gz tarballs +create_tarballs() { + for repo in ${ABSLIBRE_ROOT}/armv7h/*; do + baserepo=${repo##*/} + arch=$(basename $(dirname $repo)) + + # Remove the old one + mkdir -p $FTP_BASE/$baserepo/os/$arch/ + rm -fv $FTP_BASE/$baserepo/os/$arch/$baserepo.abs.tar.gz + # Create a new one joining arch and any + # Remove the first part of the path (it could be $repo but any isn't hit) + include=($repo/*) + if [[ -d ${ABSLIBRE_ROOT}/any/${baserepo}/ ]]; then + include+=(${ABSLIBRE_ROOT}/any/${baserepo}/*) + fi + bsdtar -czf $FTP_BASE/$baserepo/os/$arch/$baserepo.abs.tar.gz \ + -s ":${ABSLIBRE_ROOT}/[a-z0-9_]\+/[a-z]\+::" \ + "${include[@]}" + done +} + +main() { + trap 'rm -rf -- "$tmpdir"' EXIT + tmpdir=$(mktemp --tmpdir -d "${0##*/}.XXXXXXXXXX") + + BLFILE=${tmpdir}/blacklist.txt + mkdir -p -- "$ABSLIBRE_ROOT" "$ABS_ROOT" "$ALARM_ROOT" + + sync_abs + get_blacklist + sync_abs_libre + create_tarballs +} + +main "$@" diff --git a/db-import-archlinuxarm.conf b/db-import-archlinuxarm.conf new file mode 100644 index 0000000..5f103bd --- /dev/null +++ b/db-import-archlinuxarm.conf @@ -0,0 +1,41 @@ +#!/hint/bash + +# "tags" are repo-arch pairs +ARCHTAGS=({core,extra,community}-armv7h) +ARCHPATH='$arch/$repo' + +OURPKGPOOL='pool/alarm' + +INHERIT=( + 'pool/packages' + 'pool/community' +) + +# Of the ALARM mirrors that have public rsync: +# +# $ ./config-list-mirrors-archlinuxarm +# rsync://nl.mirror.archlinuxarm.org/alarmro +# rsync://dk.mirror.archlinuxarm.org/archlinuxarm +# rsync://de3.mirror.archlinuxarm.org/archlinux-arm +# rsync://sg.mirror.archlinuxarm.org/archlinuxarm +# rsync://ru.mirror.archlinuxarm.org/archlinuxarm +# +# the ones in Denmark and the Netherlands have the best ping time from +# winston.parabola.nu (Iceland). +ARCHMIRROR='rsync://nl.mirror.archlinuxarm.org/alarmro/' + +# The following is for db-import-archlinuxarm-src + +# NB: I am unsure if the presence or absence of a trailing slash in +# the _ROOT variables is significant. -- lukeshu +SYNCARGS='-mrtv --no-motd --delete-after --no-p --no-o --no-g --quiet --exclude .~tmp~/' +ABSLIBRE_ROOT=/srv/repo/db-import-archlinuxarm-src/abslibre +ABSLIBRE_GIT=https://git.parabola.nu/abslibre.git/ +ABS_ROOT='/srv/repo/db-import-archlinuxarm-src/abs/' +ABS_SERVER="${ARCHMIRROR#rsync://}/" +ALARM_GIT='https://github.com/archlinuxarm/PKGBUILDs' +ALARM_ROOT='/srv/repo/db-import-archlinuxarm-src/alarm/' + +if [[ -n ${DBIMPORT_CONFIG} ]]; then + source "${DBIMPORT_CONFIG}" +fi diff --git a/db-import-community.conf b/db-import-community.conf new file mode 100644 index 0000000..7705a5a --- /dev/null +++ b/db-import-community.conf @@ -0,0 +1,25 @@ +#!/hint/bash + +# "tags" are repo-arch pairs +ARCHTAGS=({community,multilib,multilib-testing}-x86_64) +ARCHPATH='$repo/os/$arch' +ARCHPKGPOOL='pool/community' +ARCHSRCPOOL='sources/community' + +OURPKGPOOL='pool/community' +OURSRCPOOL='sources/community' + +# Of the few tier-1 mirrors with the 'sources/' folder: +# +# $ ./config-list-mirrors-archlinux --require=sources | jq -r '.[].url' +# rsync://archlinux.c3sl.ufpr.br/archlinux/ +# rsync://mirrors.kernel.org/archlinux/ +# rsync://ftp.acc.umu.se/mirror/archlinux/ +# +# ftp.acc.umu.se (Sweden) has (by far) the best ping time from +# winston.parabola.nu (Iceland). +ARCHMIRROR='rsync://ftp.acc.umu.se/mirror/archlinux/' + +if [[ -n ${DBIMPORT_CONFIG} ]]; then + source "${DBIMPORT_CONFIG}" +fi diff --git a/db-import-packages.conf b/db-import-packages.conf new file mode 100644 index 0000000..3f1f4c5 --- /dev/null +++ b/db-import-packages.conf @@ -0,0 +1,25 @@ +#!/hint/bash + +# "tags" are repo-arch pairs +ARCHTAGS=({core,testing,extra}-x86_64) +ARCHPATH='$repo/os/$arch' +ARCHPKGPOOL='pool/packages' +ARCHSRCPOOL='sources/packages' + +OURPKGPOOL='pool/packages' +OURSRCPOOL='sources/packages' + +# Of the few tier-1 mirrors with the 'sources/' folder: +# +# $ ./config-list-mirrors-archlinux --require=sources | jq -r '.[].url' +# rsync://archlinux.c3sl.ufpr.br/archlinux/ +# rsync://mirrors.kernel.org/archlinux/ +# rsync://ftp.acc.umu.se/mirror/archlinux/ +# +# ftp.acc.umu.se (Sweden) has (by far) the best ping time from +# winston.parabola.nu (Iceland). +ARCHMIRROR='rsync://ftp.acc.umu.se/mirror/archlinux/' + +if [[ -n ${DBIMPORT_CONFIG} ]]; then + source "${DBIMPORT_CONFIG}" +fi diff --git a/db-import-pkg b/db-import-pkg new file mode 100755 index 0000000..6915a4d --- /dev/null +++ b/db-import-pkg @@ -0,0 +1,396 @@ +#!/bin/bash +# Syncs Arch, ALARM or Arch32 repos based on info contained in the +# accompanying .conf files. +# License: GPLv3 + +set -eE +shopt -s extglob globstar nullglob +source "$(librelib messages)" +source "$(librelib blacklist)" +source "$(librelib conf)" +setup_traps + +readonly -a UPSTREAMS=(packages community archlinux{32,arm}) + +# usage: fetch_dbs +# +# Fetch excluding everything but db files +# TODO: we could be doing without things other than what is in +# ${ARCHTAGS[@]} +fetch_dbs() { + rsync "${extra[@]}" --no-motd -mrtlH --no-p \ + --include="*/" \ + --include="*.db" \ + --include="*${DBEXT}" \ + --include="*.files" \ + --include="*${FILESEXT}" \ + --exclude="*" \ + --delete-after \ + "$1" "$2" +} + +# usage: get_repo_dir +# +# Prints repo directory path for the given combination, +# relative to the rsync root. +get_repo_dir() { + repo=$1 arch=$2 envsubst '$repo $arch' <<<"$ARCHPATH" +} + +# usage: db_list_pkgs +# +# Prints a list of packages within a given , one-per-line, +# in the format: +# +# pkgname [epoch:]pkgver-pkgrel +db_list_pkgs() { + bsdtar tf "$1" | + cut -d "/" -f 1 | + sed -r 's/-([^-]*-[^-]*)$/ \1/' | + sort -u +} + +# usage: filter_blacklisted FILTERED_LIST +# +# Given a list of packages in the format: +# +# pkgname [epoch:]pkgver-pkgrel +# +# filter out all of the packages named in blacklist.txt. +filter_blacklisted() { + sort -u | join -v1 \ + - \ + <(blacklist-cat | blacklist-get-pkg | sort -u) +} + +# usage: filter_duplicates FILTERED_LIST +# +# Given a list of packages in the format: +# +# pkgname [epoch:]pkgver-pkgrel +# +# filter out arch=(any) packages present elsewhere, as it confuses +# parabolaweb, librechroot, and who-knows-what-else. This only +# filters exact pkgname/epoch/pkgver/pkgrel matches. +filter_duplicates() { + sort -u | comm -23 - <( + for pool in "${INHERIT[@]}"; do + for f in "${FTP_BASE}/${pool}"/*-any${PKGEXTS}; do + f=${f##*/} + f=${f%-any$PKGEXTS} + pkgname=${f%-*-*} + fullpkgver=${f#"${pkgname}-"} + printf '%s %s\n' "$pkgname" "$fullpkgver" + done + done | sort -u + ) +} + +# usage: sync_pool +# +# Sync excluding everything but whitelist +sync_pool() { + local -r _from=$1 _whitelist=$2 _into=$3 + + mkdir -p -- "$_into" + msg2 "Retrieving %d packages from %s pool" \ + "$(wc -l < "$_whitelist")" \ + "$(basename "$_into")" + + # *Don't delete-after*, this is the job of + # cleanup scripts. It will remove our packages too + rsync "${extra[@]}" --no-motd -rtlH --no-t \ + --delay-updates \ + --safe-links \ + --include-from="$_whitelist" \ + --exclude="*" \ + "$_from" \ + "$_into" +} + +# usage: poolify +# +# Given a list of packages in the format: +# +# pkgname [epoch:]pkgver-pkgrel +# +# Resolve each to a file in `${FTP_BASE}/pool/`. The output is +# relative to `${FTP_BASE}/pool/`. That is, something along the lines +# of: +# +# poolname/pkgname-[epoch:]pkgver-pkgrel-arch.pkg.tar.xz +# archlinux32/zip-3.0-7-i686.pkg.tar.xz +# packages/rhino-1.7.7.1-1-any.pkg.tar.xz +poolify() { + local -r arch=$1 + + local pkgname fullpkgver + local restore paths path + while read -r pkgname fullpkgver; do + paths=() + for pool in "${INHERIT[@]}" "$OURPKGPOOL"; do + paths+=( + "${FTP_BASE}/${pool}/${pkgname}-${fullpkgver}-any"${PKGEXTS} + "${FTP_BASE}/${pool}/${pkgname}-${fullpkgver}-${arch}"${PKGEXTS} + ) + done + path="${paths[0]:-}" + if ! [[ -f "$path" && -f "${path}.sig" ]]; then + error "No file was found for %q=%q, aborting" "$pkgname" "$fullpkgver" + exit 1 + fi + printf '%s\n' "${path#"${FTP_BASE}/pool/"}" + done +} + +# usage: make_repo_symlinks TAG +make_repo_dbs() { + local -r from=${WORKDIR}/staging-rsync/${1}/os/${2} + local -r into=${FTP_BASE}/${1}/os/${2}/ + local -r db_file=${from}/${1}${DBEXT} + local -r files_file=${from}/${1}${FILESEXT} + + # create fresh databases to reflect actual `any.pkg.tar.xz` packages. + # this also avoids corrupt upstream metadata (ALARM) + msg2 "Adding whitelisted packages to clean DBs ..." + + pushd "${from}" + local -r UMASK=$(umask) + umask 002 + repo-add "${db_file##*/}" *${PKGEXTS} + umask "$UMASK" >/dev/null + popd >/dev/null + + mkdir -p -- "$into" + # This bit is based on db-functions:set_repo_permission() + local -r group=$(/usr/bin/stat --printf='%G' "${into}") + chgrp "$group" "${db_file}" + chgrp "$group" "${files_file}" + chmod g+w "${db_file}" + chmod g+w "${files_file}" + + msg2 "Updating %s-%s databases" "$2" "$1" + rsync "${extra[@]}" --no-motd -rtlpH --no-t \ + --delay-updates \ + --delete-after \ + --links \ + "$from/" "$into" +} + +# Main function. Process the databases and get the libre packages +# Outline: +# 1. Fetch package info +# * Get blacklist.txt +# * Get repo.db from an Arch-like repo +# 2. Figure out what we want +# * Generate textfiles describing the current repo state, and +# (using blacklist.txt) the desired repo state +# 3. Fetch the packages we want +# * Create sync whitelist (based on package blacklist) +# * Call sync_pool to fetch packages and signatures +# 4. Put the packages in the repos +# * Create new repo.db with them (repo-add) +# * rsync scratch directory => repos +main() { + ############################################################## + # 0. Initialization # + ############################################################## + + # Run as `V=true db-import-pkg` to get verbose output + readonly VERBOSE=${V} + extra=() + ${VERBOSE} && extra+=(-v) + readonly extra + readonly UPSTREAM=$1 + + # Print usage message + if [[ $# -ne 1 ]] || ! in_array "$UPSTREAM" "${UPSTREAMS[@]}" ; then + IFS='|' + msg 'usage: [V=true] %s {%s}' "${0##*/}" "${UPSTREAMS[*]}" + exit $EXIT_INVALIDARGUMENT + fi + + load_conf "$(dirname "$(readlink -e "$0")")/config" DBEXT FILESEXT FTP_BASE + load_conf "$(dirname "$(readlink -e "$0")")/db-import-${UPSTREAM}.conf" \ + ARCHMIRROR ARCHTAGS ARCHPATH OURPKGPOOL # optional: OURSRCPOOL ARCH{PKG,SRC}POOL INHERIT + if [[ -n ${ARCHSRCPOOL:-} && -z ${OURSRCPOOL:-} ]]; then + print 'If you set %s, then you must set %s' {ARCH,OUR}SRCPOOL + exit $EXIT_NOTCONFIGURED + elif [[ -n ${OURSRCPOOL:-} && -z ${ARCHSRCPOOL:-} ]]; then + print 'If you set %s, then you must set %s' {OUR,ARCH}SRCPOOL + exit $EXIT_NOTCONFIGURED + fi + if [[ -n ${ARCHSRCPOOL:-} && -z ${ARCHPKGPOOL:-} ]]; then + print '%s requires that %s is also set' ARCH{SRC,PKG}POOL + exit $EXIT_NOTCONFIGURED + fi + + WORKDIR=$(mktemp -dt "${0##*/}.XXXXXXXXXX") + trap "rm -rf -- ${WORKDIR@Q}" EXIT + + ############################################################## + # 1. Fetch package info # + ############################################################## + + # Get the blacklisted packages + blacklist-update + + # Sync the repos databases + msg 'Downloading .db and .files files to import' + mkdir "${WORKDIR}/rsync" + fetch_dbs "${ARCHMIRROR}/" "$WORKDIR/rsync" + + ############################################################## + # 2. Figure out what we want # + ############################################################## + + mkdir "${WORKDIR}"/{old,new,dif} + local _tag _repo _arch db_file + for _tag in "${ARCHTAGS[@]}"; do + _repo=${_tag%-*} + _arch=${_tag##*-} + # FIXME: this assumes that the local DBEXT and the + # imported DBEXT are the same, which is potentially + # not true. + # + # FIXME: this should use db-functions to lock the + # repos while we read them. + db_file="${FTP_BASE}/${_repo}/os/${_arch}/${_repo}${DBEXT}" + db_list_pkgs "$db_file" > "${WORKDIR}/old/${_tag}.txt" + + db_file="${WORKDIR}/rsync/$(get_repo_dir "${_repo}" "${_arch}")/${_repo}${DBEXT}" + db_list_pkgs "$db_file" | filter_blacklisted > "${WORKDIR}/new/${_tag}.txt" + done + + # We now have $WORKDIR/old/ describing the way the repos are, + # and $WORKDIR/new/ describing the way we want them to be. We + # now create $WORKDIR/dif/ describing how to get from point A + # to point B. + # + # TODO: finish this section + for _tag in "${ARCHTAGS[@]}"; do + comm -23 "${WORKDIR}"/{old,new}/"${_tag}.txt" # take packages that have been "removed" + done | grep -rFx -f /dev/stdin "${WORKDIR}/new/" | # but now appear in another repo + sort -u > "${WORKDIR}/dif/moved.txt" + comm -23 \ + <(cat "${WORKDIR}"/old/* | cut -d' ' -f1 | sort -u) \ + <(cat "${WORKDIR}"/new/* | cut -d' ' -f1 | sort -u) \ + > "${WORKDIR}/dif/removed.txt" + + ############################################################## + # 3. Fetch the packages we want # + ############################################################## + + # OK, now we have $WORKDIR/old/ describing the way the repos + # are, $WORKDIR/new/ describing the way we want them to be, + # and $WORKDIR/dif/ describing how to get from `old` to `new`. + # We should (TODO) now use db-move, db-update, and db-remove + # to apply that diff. + # + # But, + # - db-move is broken + # - The code that populates /dif/ isn't finished + # So, just nuke the current repos and entirely re-create + # everything from /new/. + + local whitelists=() + for _tag in "${ARCHTAGS[@]}"; do + msg "Processing %s" "$_tag" + _repo=${_tag%-*} + _arch=${_tag##*-} + # Create a whitelist, add * wildcard to end. + # + # FIXME: due to lack of -arch suffix, the pool sync + # retrieves every arch even if we aren't syncing them. + # + # IMPORTANT: the . in the sed command is needed + # because an empty whitelist would consist of a single + # * allowing any package to pass through. + filter_duplicates \ + <"${WORKDIR}/new/${_tag}.txt" \ + | sed -e 's/ /-/' -e 's|.$|&*|g' \ + > "${WORKDIR}/${_tag}.whitelist" + if [[ -n ${ARCHPKGPOOL:-} ]]; then + # Append to whitelists array so that we can + # later sync_pool() all packages + whitelists+=("${WORKDIR}/${_tag}.whitelist") + else + # Upstream doesn't use an $ARCHPKGPOOL + sync_pool \ + "${ARCHMIRROR}/$(get_repo_dir "${_repo}" "${_arch}")/" \ + "${WORKDIR}/${_tag}.whitelist" \ + "${FTP_BASE}/${OURPKGPOOL}/" + poolify "${_arch}" \ + <"${WORKDIR}/new/${_tag}.txt" \ + >"${WORKDIR}/${_tag}.pool" + make_repo_symlinks "$_tag" \ + <"${WORKDIR}/${_tag}.pool" + fi + done + + if (( ${#whitelists[@]} > 0 )); then + # Concatenate all whitelists, check for single *s just in case + cat "${whitelists[@]}" | grep -v "^\*$" | + sort -u > "${WORKDIR}/all.whitelist" + # FIXME: make_whitelist() wildcards should be narrowed + # down to respect the architecture of the tag + + msg "Syncing package pool" + sync_pool \ + "${ARCHMIRROR}/${ARCHPKGPOOL}/" \ + "${WORKDIR}/all.whitelist" \ + "${FTP_BASE}/${OURPKGPOOL}/" + for _tag in "${ARCHTAGS[@]}"; do + _repo=${_tag%-*} + _arch=${_tag##*-} + poolify "${_arch}" "${OURPKGPOOL}" \ + <"${WORKDIR}/new/${_tag}.txt" \ + >"${WORKDIR}/${_tag}.pool" + make_repo_symlinks "$_tag" \ + <"${WORKDIR}/${_tag}.pool" + done + + if [[ -n ${ARCHSRCPOOL:-} ]]; then + msg "Syncing source pool" + sync_pool \ + "${ARCHMIRROR}/${ARCHSRCPOOL}/" \ + "${WORKDIR}/all.whitelist" \ + "${FTP_BASE}/${OURSRCPOOL}/" + fi + fi + + ############################################################## + # 4. Put the packages in the repos # + ############################################################## + + msg "Putting databases back in place" + + # FIXME: all repo DBs should be replaced at once (per architecture) + ln -srT "$FTP_BASE/pool" "${WORKDIR}/staging-rsync/pool" + for _tag in "${ARCHTAGS[@]}"; do + _repo=${_tag%-*} + _arch=${_tag##*-} + make_repo_dbs "$_repo" "$_arch" + done + date +%s > "${FTP_BASE}/lastupdate" +} + +main "$@" diff --git a/systemd/db-import@.service b/systemd/db-import@.service new file mode 100644 index 0000000..884d4af --- /dev/null +++ b/systemd/db-import@.service @@ -0,0 +1,12 @@ +[Unit] +Description=db-import %I +Wants=network-online.target +After=network-online.target + +[Service] +Type=oneshot +User=repo +Environment=DBSCRIPTS_CONFIG=/etc/dbscripts/config.local.%I +ExecStart=/usr/bin/db-import-pkg %I + +PrivateTmp=true diff --git a/systemd/db-import@.timer b/systemd/db-import@.timer new file mode 100644 index 0000000..77570d6 --- /dev/null +++ b/systemd/db-import@.timer @@ -0,0 +1,8 @@ +[Unit] +Description=Daily db-import %I + +[Timer] +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/systemd/db-import@archlinux32.timer.d_time.conf b/systemd/db-import@archlinux32.timer.d_time.conf new file mode 100644 index 0000000..028665c --- /dev/null +++ b/systemd/db-import@archlinux32.timer.d_time.conf @@ -0,0 +1,2 @@ +[Timer] +OnCalendar=*-*-* 08:13:00 diff --git a/systemd/db-import@archlinuxarm.timer.d_time.conf b/systemd/db-import@archlinuxarm.timer.d_time.conf new file mode 100644 index 0000000..455b0bc --- /dev/null +++ b/systemd/db-import@archlinuxarm.timer.d_time.conf @@ -0,0 +1,2 @@ +[Timer] +OnCalendar=*-*-* 16:13:00 diff --git a/systemd/db-import@community.timer.d_time.conf b/systemd/db-import@community.timer.d_time.conf new file mode 100644 index 0000000..3ec5e02 --- /dev/null +++ b/systemd/db-import@community.timer.d_time.conf @@ -0,0 +1,2 @@ +[Timer] +OnCalendar=*-*-* 04:13:00 diff --git a/systemd/db-import@packages.timer.d_time.conf b/systemd/db-import@packages.timer.d_time.conf new file mode 100644 index 0000000..2561a96 --- /dev/null +++ b/systemd/db-import@packages.timer.d_time.conf @@ -0,0 +1,2 @@ +[Timer] +OnCalendar=*-*-* 00:13:00 diff --git a/test/cases/common.bats b/test/cases/common.bats index 8b4c3a0..337df48 100644 --- a/test/cases/common.bats +++ b/test/cases/common.bats @@ -1,10 +1,10 @@ load ../lib/common @test "commands display usage message by default" { - for cmd in db-move db-remove db-repo-add db-repo-remove; do + for cmd in db-move db-remove db-repo-add db-repo-remove db-import-pkg; do echo Testing $cmd run $cmd - (( $status == 1 )) + (( $status != 0 )) [[ $output == *'usage: '* ]] done } diff --git a/test/cases/db-import-pkg.bats b/test/cases/db-import-pkg.bats new file mode 100644 index 0000000..8e51646 --- /dev/null +++ b/test/cases/db-import-pkg.bats @@ -0,0 +1,367 @@ +load ../lib/common + +httpserver() { + read -r verb path version || exit + if [[ $verb != GET ]]; then + printf "HTTP/1.1 405 Method Not Allowed\r\nContent-Type: text/plain\r\nContent-length: 0\r\n\r\n" + exit + fi + path="$(cd / && realpath -ms "$path")" + if ! [[ -f "$1/$path" ]]; then + printf "HTTP/1.1 404 Not found\r\nContent-Type: text/plain\r\nContent-length: 0\r\n\r\n" + exit + fi + printf "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-length: %d\r\n\r\n" $(stat -L -c %s "$1/$path") + cat "$1/$path" +} + +eval "__common_$(declare -f setup)" +setup() { + __common_setup + + # Set up rsync server + cat <<-eot >"${TMP}/rsyncd.conf" + use chroot = no + [rsyncd] + path = ${TMP}/rsyncd + eot + local rsyncport + rsyncport=$(./lib/runserver "$TMP/rsyncd.pid" \ + rsync --daemon --config "${TMP}/rsyncd.conf") + + # Set up rsync contents + mkdir -p -- "${TMP}/rsyncd/archlinux/core/os/x86_64" + touch -- "${TMP}/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz" + ln -s core.db.tar.gz "${TMP}/rsyncd/archlinux/core/os/x86_64/core.db" + mkdir -p -- "${TMP}/rsyncd/archlinux"/{pool,sources}/{packages,community} + date +%s > "${TMP}/rsyncd/archlinux/lastupdate" + date +%s > "${TMP}/rsyncd/archlinux/lastsync" + + mkdir -p -- "${TMP}/rsyncd/archlinux32/i686/core" + touch -- "${TMP}/rsyncd/archlinux32/i686/core/core.db.tar.gz" + ln -s core.db.tar.gz "${TMP}/rsyncd/archlinux32/i686/core/core.db" + mkdir -p -- "${TMP}/rsyncd/archlinux32/pool" + date +%s > "${TMP}/rsyncd/archlinux32/lastupdate" + date +%s > "${TMP}/rsyncd/archlinux32/lastsync" + + mkdir -p -- "${TMP}/rsyncd/archlinuxarm/armv7h/core" + touch -- "${TMP}/rsyncd/archlinuxarm/armv7h/core/core.db.tar.gz" + ln -s core.db.tar.gz "${TMP}/rsyncd/archlinuxarm/armv7h/core/core.db" + date +%s > "${TMP}/rsyncd/archlinuxarm/lastupdate" + date +%s > "${TMP}/rsyncd/archlinuxarm/lastsync" + + # Configure db-import to use that rsyncd server + cat <<-eot >"${TMP}/db-import-archlinux.local.conf" + ARCHTAGS=('core-x86_64') + ARCHMIRROR=rsync://localhost:${rsyncport@Q}/rsyncd/archlinux/ + eot + cat <<-eot >"${TMP}/db-import-archlinux32.local.conf" + ARCHTAGS=('core-i686') + ARCHMIRROR=rsync://localhost:${rsyncport@Q}/rsyncd/archlinux32/ + eot + cat <<-eot >"${TMP}/db-import-archlinuxarm.local.conf" + ARCHTAGS=('core-armv7h') + ARCHMIRROR=rsync://localhost:${rsyncport@Q}/rsyncd/archlinuxarm/ + eot + + # Set up HTTP server + local httpport + httpport=$(./lib/runserver "$TMP/httpd.pid" \ + bash -c "$(declare -f httpserver); httpserver \"\$@\"" -- "$TMP/httpd") + + # Set up HTTP contents + mkdir -- "$TMP/httpd" + cat <<-eot >"$TMP/httpd/blacklist.txt" + slavery:freedom:fsf:slavekit:Obviously + eot + + # Configure db-import to use that HTTP server + mkdir "$XDG_CONFIG_HOME"/libretools + cat <<-eot >"$XDG_CONFIG_HOME"/libretools/libretools.conf + BLACKLIST=http://localhost:${httpport@Q}/blacklist.txt + eot + + # Set up repo contents + mkdir -p -- "${TMP}/ftp/core/os"/{x86_64,i686,armv7h} + touch -- "${TMP}"/ftp/core/os/{x86_64,i686,armv7h}/core.db.tar.gz + ln -s core.db.tar.gz "${TMP}/ftp/core/os/x86_64/core.db" + ln -s core.db.tar.gz "${TMP}/ftp/core/os/i686/core.db" + ln -s core.db.tar.gz "${TMP}/ftp/core/os/armv7h/core.db" + mkdir -p -- "${TMP}/ftp"/{pool,sources}/{packages,community,archlinux32,alarm} + date +%s > "${TMP}/ftp/lastupdate" + date +%s > "${TMP}/ftp/lastsync" +} +eval "__common_$(declare -f teardown)" +teardown() { + xargs -a "${TMP}/httpd.pid" kill -- + xargs -a "${TMP}/rsyncd.pid" kill -- + __common_teardown +} + +###################################################################### + +# Run the command in a new mount namespace with /tmp remounted +# read-only, but with $TMP (which might be under /tmp) still writable. +# +# Arguments are passed as arguments to `sudo`. +__withRoTmp() { + local mount="mount -o bind ${TMP@Q}{,} && mount -o bind,remount,ro /tmp{,}" + local env=( + "DBIMPORT_CONFIG=${DBIMPORT_CONFIG}" + "DBSCRIPTS_CONFIG=${DBSCRIPTS_CONFIG}" + "XDG_CONFIG_HOME=${XDG_CONFIG_HOME}" + ) + sudo -- unshare -m -- sh -c "${mount} && sudo -u ${USER@Q} ${env[*]@Q} \$@" -- "$@" +} + +__db-import-pkg() { + local ret=0 + # Since common.bash->config.local sets TMPDIR=${TMP}/tmp, + # TMPDIR is necessarily != /tmp. + # Which means that if we try to write anything directly under /tmp, + # then we are erroneously disregarding TMPDIR. + # So, make /tmp read-only to make that be an error. + __withRoTmp db-import-pkg "$@" || ret=$? + # Verify that it cleaned up after itself and TMPDIR is empty + find "$TMPDIR" -mindepth 1 | diff - /dev/null + return $ret +} + +# releaseImportedPackage PKGBASE ARCH DBFILE [POOLDIR] +# +# This is different from common.bash:releasePackage because +# - it doesn't mess with SVN +# - it adds the package to the .db file +__releaseImportedPackage() { + local pkgbase=$1 + local arch=$2 + local dbfile=$3 + local pooldir=$4 + local repodir="${dbfile%/*}" + local dir restore pkgfiles pkgfile pkgs + + dir="$TMP/import-build/$pkgbase" + if ! [[ -d "$dir" ]]; then + mkdir -p -- "$dir" + cp -t "$dir" -- "fixtures/${pkgbase}"/* + fi + pushd "$dir" + __buildPackage + restore="$(shopt -p nullglob || true)" + shopt -s nullglob + pkgfiles=(*-{"$arch",any}$PKGEXT{,.sig}) + $restore + popd + + mkdir -p "$repodir" + if [[ -z $pooldir ]]; then + mv -t "$repodir" -- "${pkgfiles[@]/#/"$dir/"}" + else + mkdir -p "$pooldir" + mv -t "$pooldir" -- "${pkgfiles[@]/#/"$dir/"}" + ln -sr -t "$repodir" -- "${pkgfiles[@]/#/"$pooldir/"}" + fi + + pushd "$repodir" + pkgs=() + for pkgfile in "${pkgfiles[@]}"; do + if [[ "$pkgfile" = *.sig ]]; then + continue + fi + pkgs+=("$pkgfile") + done + repo-add -q "${dbfile##*/}" "${pkgs[@]}" + popd +} + +__updateImportedPackage() { + pushd "$TMP/import-build/$1" + local pkgrel + pkgrel=$(. PKGBUILD; expr ${pkgrel} + 1) + sed "s/pkgrel=.*/pkgrel=${pkgrel}/" -i PKGBUILD + popd +} + +__isLinkTo() { + [[ -L $1 ]] + [[ $1 -ef $2 ]] +} + +__doesNotExist() { + local file + for file in "$@"; do + if stat "$file" 2>/dev/null; then + echo "TEST ERROR: File shouldn't exist, but does: $file" + return 1 + fi + done +} + +###################################################################### + +@test "import no blacklisted packages (x86_64)" { + __releaseImportedPackage slavery x86_64 "$TMP/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz" "$TMP/rsyncd/archlinux/pool/packages" + __releaseImportedPackage pkg-simple-c x86_64 "$TMP/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz" "$TMP/rsyncd/archlinux/pool/packages" + + DBIMPORT_CONFIG="${TMP}/db-import-archlinux.local.conf" __db-import-pkg packages + + __isLinkTo "$TMP/ftp/core/os/x86_64/pkg-simple-c-1-1-x86_64.pkg.tar.xz" "$TMP/ftp/pool/packages/pkg-simple-c-1-1-x86_64.pkg.tar.xz" + __doesNotExist "$TMP"/ftp/{core/os/x86_64,pool/packages,sources/packages}/slavery-* +} + +@test "import no blacklisted packages (i686)" { + __releaseImportedPackage slavery i686 "$TMP/rsyncd/archlinux32/i686/core/core.db.tar.gz" "$TMP/rsyncd/archlinux32/pool" + __releaseImportedPackage pkg-simple-c i686 "$TMP/rsyncd/archlinux32/i686/core/core.db.tar.gz" "$TMP/rsyncd/archlinux32/pool" + + DBIMPORT_CONFIG="${TMP}/db-import-archlinux32.local.conf" __db-import-pkg archlinux32 + + __isLinkTo "$TMP/ftp/core/os/i686/pkg-simple-c-1-1-i686.pkg.tar.xz" "$TMP/ftp/pool/archlinux32/pkg-simple-c-1-1-i686.pkg.tar.xz" + __doesNotExist "$TMP"/ftp/{core/os/i686,pool/archlinux32,sources/archlinux32}/slavery-* +} + +@test "import no blacklisted packages (armv7h)" { + __releaseImportedPackage slavery armv7h "$TMP/rsyncd/archlinuxarm/armv7h/core/core.db.tar.gz" + __releaseImportedPackage pkg-simple-c armv7h "$TMP/rsyncd/archlinuxarm/armv7h/core/core.db.tar.gz" + + DBIMPORT_CONFIG="${TMP}/db-import-archlinuxarm.local.conf" __db-import-pkg archlinuxarm + + __isLinkTo "$TMP/ftp/core/os/armv7h/pkg-simple-c-1-1-armv7h.pkg.tar.xz" "$TMP/ftp/pool/alarm/pkg-simple-c-1-1-armv7h.pkg.tar.xz" + __doesNotExist "$TMP"/ftp/{core/os/alarm,pool/alarm,sources/alarm}/slavery-* +} + +@test "import DBs with no blacklisted packages" { + __releaseImportedPackage pkg-simple-c x86_64 "$TMP/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz" "$TMP/rsyncd/archlinux/pool/packages" + + DBIMPORT_CONFIG="${TMP}/db-import-archlinux.local.conf" __db-import-pkg packages + + __isLinkTo "$TMP/ftp/core/os/x86_64/pkg-simple-c-1-1-x86_64.pkg.tar.xz" "$TMP/ftp/pool/packages/pkg-simple-c-1-1-x86_64.pkg.tar.xz" +} + +@test "import updated packages" { + __releaseImportedPackage slavery x86_64 "$TMP/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz" "$TMP/rsyncd/archlinux/pool/packages" + __releaseImportedPackage pkg-simple-c x86_64 "$TMP/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz" "$TMP/rsyncd/archlinux/pool/packages" + + DBIMPORT_CONFIG="${TMP}/db-import-archlinux.local.conf" __db-import-pkg packages + + __isLinkTo "$TMP/ftp/core/os/x86_64/pkg-simple-c-1-1-x86_64.pkg.tar.xz" "$TMP/ftp/pool/packages/pkg-simple-c-1-1-x86_64.pkg.tar.xz" + + __updateImportedPackage pkg-simple-c + __releaseImportedPackage pkg-simple-c x86_64 "$TMP/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz" "$TMP/rsyncd/archlinux/pool/packages" + + DBIMPORT_CONFIG="${TMP}/db-import-archlinux.local.conf" __db-import-pkg packages + + __isLinkTo "$TMP/ftp/core/os/x86_64/pkg-simple-c-1-2-x86_64.pkg.tar.xz" "$TMP/ftp/pool/packages/pkg-simple-c-1-2-x86_64.pkg.tar.xz" +} + +@test "import .db files as 0664 (x86_64)" { + __releaseImportedPackage slavery x86_64 "$TMP/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz" "$TMP/rsyncd/archlinux/pool/packages" + __releaseImportedPackage pkg-simple-c x86_64 "$TMP/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz" "$TMP/rsyncd/archlinux/pool/packages" + + DBIMPORT_CONFIG="${TMP}/db-import-archlinux.local.conf" __db-import-pkg packages + + __isLinkTo "$TMP/ftp/core/os/x86_64/pkg-simple-c-1-1-x86_64.pkg.tar.xz" "$TMP/ftp/pool/packages/pkg-simple-c-1-1-x86_64.pkg.tar.xz" + __doesNotExist "$TMP"/ftp/{core/os/x86_64,pool/packages,sources/packages}/slavery-* + [[ "$(stat -c '%a' -- "$TMP/ftp/core/os/x86_64/core.db.tar.gz")" = 664 ]] +} + +@test "import .db files as 0664 (i686)" { + __releaseImportedPackage slavery i686 "$TMP/rsyncd/archlinux32/i686/core/core.db.tar.gz" "$TMP/rsyncd/archlinux32/pool" + __releaseImportedPackage pkg-simple-c i686 "$TMP/rsyncd/archlinux32/i686/core/core.db.tar.gz" "$TMP/rsyncd/archlinux32/pool" + + DBIMPORT_CONFIG="${TMP}/db-import-archlinux32.local.conf" __db-import-pkg archlinux32 + + __isLinkTo "$TMP/ftp/core/os/i686/pkg-simple-c-1-1-i686.pkg.tar.xz" "$TMP/ftp/pool/archlinux32/pkg-simple-c-1-1-i686.pkg.tar.xz" + __doesNotExist "$TMP"/ftp/{core/os/i686,pool/archlinux32,sources/archlinux32}/slavery-* + stat -- "$TMP/ftp/core/os/i686/core.db.tar.gz" + [[ "$(stat -c '%a' -- "$TMP/ftp/core/os/i686/core.db.tar.gz")" = 664 ]] +} + +@test "import .db files as 0664 (armv7h)" { + __releaseImportedPackage slavery armv7h "$TMP/rsyncd/archlinuxarm/armv7h/core/core.db.tar.gz" + __releaseImportedPackage pkg-simple-c armv7h "$TMP/rsyncd/archlinuxarm/armv7h/core/core.db.tar.gz" + + DBIMPORT_CONFIG="${TMP}/db-import-archlinuxarm.local.conf" __db-import-pkg archlinuxarm + + __isLinkTo "$TMP/ftp/core/os/armv7h/pkg-simple-c-1-1-armv7h.pkg.tar.xz" "$TMP/ftp/pool/alarm/pkg-simple-c-1-1-armv7h.pkg.tar.xz" + __doesNotExist "$TMP"/ftp/{core/os/armv7h,pool/alarm,sources/alarm}/slavery-* + stat -- "$TMP/ftp/core/os/armv7h/core.db.tar.gz" + [[ "$(stat -c '%a' -- "$TMP/ftp/core/os/armv7h/core.db.tar.gz")" = 664 ]] +} + +@test "import fully-masked upstream" { + __releaseImportedPackage pkg-any-a x86_64 "$TMP/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz" "$TMP/rsyncd/archlinux/pool/packages" + __releaseImportedPackage pkg-any-a i686 "$TMP/rsyncd/archlinux32/i686/core/core.db.tar.gz" "$TMP/rsyncd/archlinux32/pool" + __releaseImportedPackage pkg-any-a armv7h "$TMP/rsyncd/archlinuxarm/armv7h/core/core.db.tar.gz" + + DBIMPORT_CONFIG="${TMP}/db-import-archlinux.local.conf" __db-import-pkg packages + DBIMPORT_CONFIG="${TMP}/db-import-archlinux32.local.conf" __db-import-pkg archlinux32 + DBIMPORT_CONFIG="${TMP}/db-import-archlinuxarm.local.conf" __db-import-pkg archlinuxarm + + __isLinkTo "$TMP/ftp/core/os/x86_64/pkg-any-a-1-1-any.pkg.tar.xz" "$TMP/ftp/pool/packages/pkg-any-a-1-1-any.pkg.tar.xz" + __isLinkTo "$TMP/ftp/core/os/i686/pkg-any-a-1-1-any.pkg.tar.xz" "$TMP/ftp/pool/packages/pkg-any-a-1-1-any.pkg.tar.xz" + __isLinkTo "$TMP/ftp/core/os/armv7h/pkg-any-a-1-1-any.pkg.tar.xz" "$TMP/ftp/pool/packages/pkg-any-a-1-1-any.pkg.tar.xz" +} + +@test "import errors on pkgpool selection failures" { + # I believe that there's no way to get pkgpool selection to + # fail anymore... maybe a race condition? But that's hard to + # test for. + skip + + # pkg-simple-c is just to make sure that the "fully-masked + # upstream" bug isn't being tested here + __releaseImportedPackage pkg-any-a x86_64 "$TMP/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz" "$TMP/rsyncd/archlinux/pool/packages" + __releaseImportedPackage pkg-simple-c x86_64 "$TMP/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz" "$TMP/rsyncd/archlinux/pool/packages" + __updateImportedPackage pkg-any-a + __releaseImportedPackage pkg-any-a x86_64 "$TMP/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz" "$TMP/rsyncd/archlinux/pool/packages" + DBIMPORT_CONFIG="${TMP}/db-import-archlinux.local.conf" __db-import-pkg packages + __isLinkTo "$TMP/ftp/core/os/x86_64/pkg-any-a-1-2-any.pkg.tar.xz" "$TMP/ftp/pool/packages/pkg-any-a-1-2-any.pkg.tar.xz" + + # This assumes that a package nested too deelply under /pool/ + # is filtered from being downloaded, but isn't found when + # poolifying. + mkdir -- "$TMP/ftp/pool/nested" + mv -T -- "$TMP/ftp/pool/packages" "$TMP/ftp/pool/nested/packages" + __releaseImportedPackage pkg-any-a i686 "$TMP/rsyncd/archlinux32/i686/core/core.db.tar.gz" "$TMP/rsyncd/archlinux32/pool" + __releaseImportedPackage pkg-simple-c i686 "$TMP/rsyncd/archlinux32/i686/core/core.db.tar.gz" "$TMP/rsyncd/archlinux32/pool" + + local status=0 + DBIMPORT_CONFIG="${TMP}/db-import-archlinux32.local.conf" __db-import-pkg archlinux32 || status=$? + [[ $status != 0 ]] + __doesNotExist "$TMP/ftp/core/os/i686/pkg-any-a-1-2-any.pkg.tar.xz" +} + +@test "import arch=any packages with sub-pkgrel" { + # This is modeled after the situation with 'asp' and 'asp32' + + __releaseImportedPackage pkg-any64 x86_64 "$TMP/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz" "$TMP/rsyncd/archlinux/pool/packages" + DBIMPORT_CONFIG="${TMP}/db-import-archlinux.local.conf" __db-import-pkg packages + __isLinkTo "$TMP/ftp/core/os/x86_64/pkg-any-2-1-any.pkg.tar.xz" "$TMP/ftp/pool/packages/pkg-any-2-1-any.pkg.tar.xz" + + __releaseImportedPackage pkg-any32 i686 "$TMP/rsyncd/archlinux32/i686/core/core.db.tar.gz" "$TMP/rsyncd/archlinux32/pool" + DBIMPORT_CONFIG="${TMP}/db-import-archlinux32.local.conf" __db-import-pkg archlinux32 + __isLinkTo "$TMP/ftp/core/os/i686/pkg-any-1-1.2-any.pkg.tar.xz" "$TMP/ftp/pool/archlinux32/pkg-any-1-1.2-any.pkg.tar.xz" +} + +@test "import respects INHERIT precedence" { + __releaseImportedPackage pkg-any-a armv7h "$TMP/rsyncd/archlinuxarm/armv7h/core/core.db.tar.gz" + DBIMPORT_CONFIG="${TMP}/db-import-archlinux32.local.conf" __db-import-pkg archlinux32 + + __releaseImportedPackage pkg-any-a x86_64 "$TMP/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz" "$TMP/rsyncd/archlinux/pool/packages" + DBIMPORT_CONFIG="${TMP}/db-import-archlinux.local.conf" __db-import-pkg packages + DBIMPORT_CONFIG="${TMP}/db-import-archlinuxarm.local.conf" __db-import-pkg archlinuxarm + + __isLinkTo "$TMP/ftp/core/os/x86_64/pkg-any-a-1-1-any.pkg.tar.xz" "$TMP/ftp/pool/packages/pkg-any-a-1-1-any.pkg.tar.xz" + __isLinkTo "$TMP/ftp/core/os/armv7h/pkg-any-a-1-1-any.pkg.tar.xz" "$TMP/ftp/pool/packages/pkg-any-a-1-1-any.pkg.tar.xz" +} + +@test "import doesn't backdate packages" { + __releaseImportedPackage pkg-simple-c x86_64 "$TMP/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz" "$TMP/rsyncd/archlinux/pool/packages" + + touch "$TMP/stamp" + DBIMPORT_CONFIG="${TMP}/db-import-archlinux.local.conf" __db-import-pkg packages + + __isLinkTo "$TMP/ftp/core/os/x86_64/pkg-simple-c-1-1-x86_64.pkg.tar.xz" "$TMP/ftp/pool/packages/pkg-simple-c-1-1-x86_64.pkg.tar.xz" + [[ "$TMP/ftp/pool/packages/pkg-simple-c-1-1-x86_64.pkg.tar.xz" -nt "$TMP/stamp" ]] + [[ "$TMP/ftp/lastupdate" -nt "$TMP/stamp" ]] +} diff --git a/test/fixtures/pkg-any32/PKGBUILD b/test/fixtures/pkg-any32/PKGBUILD new file mode 100644 index 0000000..e0421da --- /dev/null +++ b/test/fixtures/pkg-any32/PKGBUILD @@ -0,0 +1,13 @@ +pkgname=pkg-any +pkgver=1 +pkgrel=1.2 +pkgdesc="A package called ${pkgname} (32)" +arch=('any') +url='http://www.archlinux.org/' +license=('GPL') +options=(!strip) + +package() { + install -d -m755 "${pkgdir}"/usr/share/${pkgname} + echo 'test' > "${pkgdir}"/usr/share/${pkgname}/test +} diff --git a/test/fixtures/pkg-any64/PKGBUILD b/test/fixtures/pkg-any64/PKGBUILD new file mode 100644 index 0000000..1290ebd --- /dev/null +++ b/test/fixtures/pkg-any64/PKGBUILD @@ -0,0 +1,13 @@ +pkgname=pkg-any +pkgver=2 +pkgrel=1 +pkgdesc="A package called ${pkgname} (64)" +arch=('any') +url='http://www.archlinux.org/' +license=('GPL') +options=(!strip) + +package() { + install -d -m755 "${pkgdir}"/usr/share/${pkgname} + echo 'test' > "${pkgdir}"/usr/share/${pkgname}/test +} diff --git a/test/fixtures/pkg-simple-c/PKGBUILD b/test/fixtures/pkg-simple-c/PKGBUILD new file mode 100644 index 0000000..1e8361d --- /dev/null +++ b/test/fixtures/pkg-simple-c/PKGBUILD @@ -0,0 +1,13 @@ +pkgname=pkg-simple-c +pkgver=1 +pkgrel=1 +pkgdesc="A package called ${pkgname}" +arch=('i686' 'x86_64' 'armv7h') +url='http://www.archlinux.org/' +license=('GPL' 'LGPL') +depends=('glibc') +options=(!strip) + +package() { + install -D -m755 /bin/true "${pkgdir}"/usr/bin/${pkgname} +} diff --git a/test/fixtures/slavery/PKGBUILD b/test/fixtures/slavery/PKGBUILD new file mode 100644 index 0000000..93cc079 --- /dev/null +++ b/test/fixtures/slavery/PKGBUILD @@ -0,0 +1,13 @@ +pkgname=slavery +pkgver=1 +pkgrel=1 +pkgdesc="A package called ${pkgname}" +arch=('any') +url='http://www.archlinux.org/' +license=('GPL') +options=(!strip) + +package() { + install -d -m755 "${pkgdir}"/usr/share/${pkgname} + echo 'test' > "${pkgdir}"/usr/share/${pkgname}/test +} diff --git a/test/lib/common.bash b/test/lib/common.bash index d0ee671..76061f6 100644 --- a/test/lib/common.bash +++ b/test/lib/common.bash @@ -86,7 +86,6 @@ setup() { SOURCE_CLEANUP_DESTDIR="${TMP}/source-cleanup" STAGING="${TMP}/staging" TMPDIR="${TMP}/tmp" - ARCHES=(x86_64 i686) CLEANUP_DRYRUN=false SOURCE_CLEANUP_DRYRUN=false eot diff --git a/test/lib/runserver b/test/lib/runserver new file mode 100755 index 0000000..7705e40 --- /dev/null +++ b/test/lib/runserver @@ -0,0 +1,39 @@ +#!/usr/bin/env perl +use strict; +use warnings; + +use IO::Socket::INET; +use Fcntl; + +my $socket = IO::Socket::INET->new( + LocalAddr => "localhost:0", + Listen => 1 +); + +sysopen(my $fh, $ARGV[0], O_WRONLY|O_CREAT|O_EXCL, 0666) or die "open: $ARGV[0]: $!"; +my $pid = fork(); +if ($pid > 0) { + print $fh $pid; + print $socket->sockport()."\n"; + exit; +} +close($fh); +open(STDIN, 'accept(); + my $worker = fork(); + if ($worker == 0) { + open(STDIN, '<&', $conn); + open(STDOUT, '>&', $conn); + close($conn); + close($socket); + exec @cmd; + exit + } + close($conn); +} -- cgit v1.2.2 From 3d6efa9892d85c66741469e90de78686bc023201 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Fri, 22 Jun 2018 12:47:57 -0400 Subject: torrents --- cron-jobs/make_repo_torrents | 92 ++++++++++++++++++++++++++++++++++++++ make_individual_torrent | 52 +++++++++++++++++++++ systemd/make_repo_torrents.service | 7 +++ systemd/make_repo_torrents.timer | 9 ++++ 4 files changed, 160 insertions(+) create mode 100755 cron-jobs/make_repo_torrents create mode 100755 make_individual_torrent create mode 100644 systemd/make_repo_torrents.service create mode 100644 systemd/make_repo_torrents.timer diff --git a/cron-jobs/make_repo_torrents b/cron-jobs/make_repo_torrents new file mode 100755 index 0000000..767e182 --- /dev/null +++ b/cron-jobs/make_repo_torrents @@ -0,0 +1,92 @@ +#!/bin/bash +# Copyright (C) 2014, 2017 Joseph Graham +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# This script finds any updated packages and calls +# `make_indivudual_torrent' for each of them. Run every hour from cron. + +username=$( id -un ) + +case "${username}" in + repo | root ) + true + ;; + * ) + echo "This script must be run as repo user or root user." + echo "ByeBye!" + exit 1 + ;; +esac + +# pacman doesn't support multiple different packages of the same name, +# so it's OK to just stuff all the torrents into a single directory. +script_directory="$(dirname "$(readlink -e "$0")")/.." +. "$(dirname "$(readlink -e "$0")")/../config" +public_location="$FTP_BASE/" +torrent_location="$FTP_BASE/torrents/" + +cd "${torrent_location}" + +pkgfilelist=$(mktemp) + +# Find any directories that might have packages in them +find "${public_location}" -name 'os' -type 'd' | + while read dir + do + # Find any packages + find "${dir}" -regex '[^ ]+\.pkg\.tar\.xz' + done > "${pkgfilelist}" + +while read pkg +do + pkg_name="${pkg##*/}" + + if [[ -h "${pkg}" ]] # check if it's a symbolic link + then + # We get the target of the symlink + pkg=$( readlink -f "${pkg}" ) + fi + + # If a .torrent file does not already exist for this package, we call + # `make_individual_torrent' to make it. + if ! [[ -f "${torrent_location}${pkg_name}.torrent" ]] + then + "$script_directory/make_individual_torrent" "${pkg}" "${public_location}" + fi +done < "${pkgfilelist}" + +# For torrents older than 1 year, we check if it's package still +# exists, else clean it up. This shouldn't be done every hour, but it +# is good for it to be in this cronjob so it can re-use the +# $pkgfilelist we made earlier. So we do it with a 1-in-30 +# probability. +if ! (( $(shuf -e {0..29} | head -1) )) +then + find -H "${torrent_location}" -mtime +365 -name '*.torrent' -type f | + while read oldtorrent + do + oldtorrentnm="${oldtorrent##*/}" + correspackagenm="${oldtorrentnm%.torrent}" + + grep "${correspackagenm}" "${pkgfilelist}" &> /dev/null || rm "${oldtorrent}" + done +fi + +if [[ "${username}" == root ]] +then + chown repo * +fi + +rm -f "${pkgfilelist}" diff --git a/make_individual_torrent b/make_individual_torrent new file mode 100755 index 0000000..0200e9a --- /dev/null +++ b/make_individual_torrent @@ -0,0 +1,52 @@ +#!/bin/bash +# Copyright (C) 2014, 2017 Joseph Graham +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# This script is called by `make_repo_torrents' to make a torrent. It +# depends on `mktorrent'. It takes the following args: +# $1 - path of package +# $2 - public location + +# Comma seperated list of trackers, no spaces +# taskenizer.crabdance.com is run by Xylon. the others come from https://github.com/ngosang/trackerslist +trackers='http://taskenizer.crabdance.com:6969/announce,http://tracker.opentrackr.org:1337/announce,udp://tracker.pirateparty.gr:6969/announce' + +# This mirror is put as a webseed. Which mirror we use for a webseed +# doesn't really matter since it's re-written on the client machine by +# pacman2pacman so it won't normally be used anyway. +seed_url='http://repo.parabolagnulinux.org/' + +if [[ -z "${1}" ]] +then + echo "Error. First arg must be the path of the package." + echo 1 +fi + +if [[ -z "${2}" ]] +then + echo "Error. Second arg must be the public location." + echo 1 +fi + +pkg="${1}" +public_location="${2}" + +pkg_name="${pkg##*/}" + +# URL of the actual package for the webseed +webseed="${seed_url}${pkg#${public_location}}" + +mktorrent -a "${trackers}" "${pkg}" -w "${webseed}" >/dev/null || +echo "Error making torrent for \"${pkg}\"" diff --git a/systemd/make_repo_torrents.service b/systemd/make_repo_torrents.service new file mode 100644 index 0000000..fc084ee --- /dev/null +++ b/systemd/make_repo_torrents.service @@ -0,0 +1,7 @@ +[Unit] +Description=Create pacman2pacman torrent files + +[Service] +Type=oneshot +User=repo +ExecStart=/opt/dbscripts/cron-jobs/make_repo_torrents diff --git a/systemd/make_repo_torrents.timer b/systemd/make_repo_torrents.timer new file mode 100644 index 0000000..9d02cb4 --- /dev/null +++ b/systemd/make_repo_torrents.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Periodic creation of pacman2pacman torrent files + +[Timer] +OnCalendar=hourly +Persistent=true + +[Install] +WantedBy=timers.target -- cgit v1.2.2 From 975d1ec6f19cd4e44c4b5fbd93fc2a1a76bcb8b9 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Fri, 22 Jun 2018 13:22:44 -0400 Subject: db-check-* --- db-check-nonfree | 47 ++++++++++ db-check-package-libraries | 203 ++++++++++++++++++++++++++++++++++++++++++ db-check-repo-sanity | 57 ++++++++++++ db-check-unsigned-packages | 38 ++++++++ db-check-unsigned-packages.py | 96 ++++++++++++++++++++ 5 files changed, 441 insertions(+) create mode 100755 db-check-nonfree create mode 100755 db-check-package-libraries create mode 100755 db-check-repo-sanity create mode 100755 db-check-unsigned-packages create mode 100755 db-check-unsigned-packages.py diff --git a/db-check-nonfree b/db-check-nonfree new file mode 100755 index 0000000..7cbeb8f --- /dev/null +++ b/db-check-nonfree @@ -0,0 +1,47 @@ +#!/bin/bash + +. "$(dirname "$(readlink -e "$0")")/config" +. "$(dirname "$(readlink -e "$0")")/db-functions" + +if [ $# -ge 1 ]; then + error "Calling %s with a specific repository is not supported" "${0##*/}" + exit 1 +fi + +# TODO: this might lock too much (architectures) +for repo in "${repos[@]}"; do + for pkgarch in "${ARCHES[@]}"; do + repo_lock "${repo}" "${pkgarch}" || exit 1 + done +done + +msg "Check nonfree in repo:" +libreblacklist update +nonfree=($(libreblacklist cat | libreblacklist get-pkg | sort -u)) +for repo in "${ARCHREPOS[@]}"; do + for pkgarch in "${ARCHES[@]}"; do + msg2 "%s %s" "$repo" "$pkgarch" + if [ ! -f "${FTP_BASE}/${repo}/os/${pkgarch}/${repo}${DBEXT}" ]; then + continue + fi + unset dbpkgs + unset cleanpkgs + cleanpkgs=() + dbpkgs=($(bsdtar -xOf "${FTP_BASE}/${repo}/os/${pkgarch}/${repo}${DBEXT}" | awk '/^%NAME%/{getline;print}' | sort -u )) + for pkgname in "${dbpkgs[@]}"; do + if in_array "${pkgname}" "${nonfree[@]}"; then + cleanpkgs+=("${pkgname}") + fi + done + if [ ${#cleanpkgs[@]} -ge 1 ]; then + msg2 "Nonfree: %s" "${cleanpkgs[*]}" + arch_repo_remove "${repo}" "${pkgarch}" "${cleanpkgs[@]}" + fi + done +done + +for repo in "${repos[@]}"; do + for pkgarch in "${ARCHES[@]}"; do + repo_unlock "${repo}" "${pkgarch}" + done +done diff --git a/db-check-package-libraries b/db-check-package-libraries new file mode 100755 index 0000000..e24b58e --- /dev/null +++ b/db-check-package-libraries @@ -0,0 +1,203 @@ +#!/usr/bin/env python3 +# Copyright (C) 2012 Michał Masłowski +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +""" +Check which libraries are provided or required by a package, store +this in a database, update and list broken packages. + +Dependencies: + +- Python 3.2 or later with SQLite 3 support + +- ``bsdtar`` + +- ``readelf`` +""" + + +import os.path +import re +import sqlite3 +import subprocess +import tempfile + + +#: Regexp matching an interesting dynamic entry. +_DYNAMIC = re.compile(r"^\s*[0-9a-fx]+" + "\s*\((NEEDED|SONAME)\)[^:]*:\s*\[(.+)\]$") + + +def make_db(path): + """Make a new, empty, library database at *path*.""" + con = sqlite3.connect(path) + con.executescript(""" +create table provided( + library varchar not null, + package varchar not null +); +create table used( + library varchar not null, + package varchar not null +); +""") + con.close() + + +def begin(database): + """Connect to *database* and start a transaction.""" + con = sqlite3.connect(database) + con.execute("begin exclusive") + return con + + +def add_provided(con, package, libraries): + """Write that *package* provides *libraries*.""" + for library in libraries: + con.execute("insert into provided (package, library) values (?,?)", + (package, library)) + + +def add_used(con, package, libraries): + """Write that *package* uses *libraries*.""" + for library in libraries: + con.execute("insert into used (package, library) values (?,?)", + (package, library)) + + +def remove_package(con, package): + """Remove all entries for a package.""" + con.execute("delete from provided where package=?", (package,)) + con.execute("delete from used where package=?", (package,)) + + +def add_package(con, package): + """Add entries from a named *package*.""" + # Extract to a temporary directory. This could be done more + # efficiently, since there is no need to store more than one file + # at once. + print("adding package:", package) + with tempfile.TemporaryDirectory(None, "db-check-package-libraries."+os.path.basename(package)+".") as temp: + subprocess.Popen(("bsdtar", "xf", package, "-C", temp)).communicate() + subprocess.Popen(('find', temp, + '-type', 'd', + '(', '-not', '-readable', '-o', '-not', '-executable', ')', + '-exec', 'chmod', '755', '--', '{}', ';')).communicate() + subprocess.Popen(('find', temp, + '-type', 'f', + '-not', '-readable', + '-exec', 'chmod', '644', '--', '{}', ';')).communicate() + with open(os.path.join(temp, ".PKGINFO")) as pkginfo: + for line in pkginfo: + if line.startswith("pkgname ="): + pkgname = line[len("pkgname ="):].strip() + break + # Don't list previously removed libraries. + remove_package(con, pkgname) + provided = set() + used = set() + # Search for ELFs. + for dirname, dirnames, filenames in os.walk(temp): + assert dirnames is not None # unused, avoid pylint warning + for file_name in filenames: + path = os.path.join(dirname, file_name) + if os.path.islink(path) or not os.path.isfile(path): + continue + with open(path, "rb") as file_object: + if file_object.read(4) != b"\177ELF": + continue + readelf = subprocess.Popen(("readelf", "-d", path), + stdout=subprocess.PIPE) + for line in readelf.communicate()[0].split(b"\n"): + match = _DYNAMIC.match(line.decode("ascii")) + if match: + if match.group(1) == "SONAME": + provided.add(match.group(2)) + elif match.group(1) == "NEEDED": + used.add(match.group(2)) + else: + raise AssertionError("unknown entry type " + + match.group(1)) + add_provided(con, pkgname, provided) + add_used(con, pkgname, used) + + +def init(arguments): + """Initialize.""" + make_db(arguments.database) + + +def add(arguments): + """Add packages.""" + con = begin(arguments.database) + for package in arguments.packages: + add_package(con, package) + con.commit() + con.close() + + +def remove(arguments): + """Remove packages.""" + con = begin(arguments.database) + for package in arguments.packages: + remove_package(con, package) + con.commit() + con.close() + + +def check(arguments): + """List broken packages.""" + con = begin(arguments.database) + available = set(row[0] for row + in con.execute("select library from provided")) + for package, library in con.execute("select package, library from used"): + if library not in available: + print(package, "needs", library) + con.close() + + +def main(): + """Get arguments and run the command.""" + from argparse import ArgumentParser + parser = ArgumentParser(prog="db-check-package-libraries", + description="Check packages for " + "provided/needed libraries") + parser.add_argument("-d", "--database", type=str, + help="Database file to use", + default="package-libraries.sqlite") + subparsers = parser.add_subparsers() + subparser = subparsers.add_parser(name="init", + help="initialize the database") + subparser.set_defaults(command=init) + subparser = subparsers.add_parser(name="add", + help="add packages to database") + subparser.add_argument("packages", nargs="+", type=str, + help="package files to add") + subparser.set_defaults(command=add) + subparser = subparsers.add_parser(name="remove", + help="remove packages from database") + subparser.add_argument("packages", nargs="+", type=str, + help="package names to remove") + subparser.set_defaults(command=remove) + subparser = subparsers.add_parser(name="check", + help="list broken packages") + subparser.set_defaults(command=check) + arguments = parser.parse_args() + arguments.command(arguments) + + +if __name__ == "__main__": + main() diff --git a/db-check-repo-sanity b/db-check-repo-sanity new file mode 100755 index 0000000..239f042 --- /dev/null +++ b/db-check-repo-sanity @@ -0,0 +1,57 @@ +#!/bin/bash +# Solves issue165... on the old roundup install. From the database +# backups, the title was "Older/deprecated packages never leave the +# repo", I don't know how the body of the issue is stored in the DB, +# but the title says enough, I think. + +. "$(dirname "$(readlink -e "$0")")/../config" +. "$(dirname "$(readlink -e "$0")")/../db-functions" + +# Traverse all repos +for _repo in "${PKGREPOS[@]}"; do + msg "Cleaning up [%s]" "${_repo}" + + # Find all pkgnames on this repo's abs + on_abs=($( + find "${SVNREPO}/${_repo}" -name PKGBUILD | \ + while read pkgbuild; do + source "${pkgbuild}" >/dev/null 2>&1 + # cleanup to save memory + unset build package source md5sums pkgdesc pkgver pkgrel epoch \ + url license arch depends makedepends optdepends options \ + >/dev/null 2>&1 + + # also cleanup package functions + for _pkg in "${pkgname[@]}"; do + unset "package_${pkg}" >/dev/null 2>&1 + done + + # this fills the on_abs array + echo "${pkgname[@]}" + done + )) + + # quit if abs is empty + if [ ${#on_abs[*]} -eq 0 ]; then + warning "[%s]'s ABS tree is empty, skipping" "${_repo}" + break + fi + + # Find all pkgnames on repos + on_repo=($( + find "${FTP_BASE}/${_repo}" -name "*.pkg.tar.?z" \ + -printf "%f\n" | sed "s/^\(.\+\)-[^-]\+-[^-]\+-[^-]\+$/\1/" + )) + + # Compares them, whatever is on repos but not on abs should be removed + remove=($(comm -13 \ + <(printf '%s\n' "${on_abs[@]}" | sort -u) \ + <(printf '%s\n' "${on_repo[@]}" | sort -u) )) + + # Remove them from databases, ftpdir-cleanup will take care of the rest + find "${FTP_BASE}/${_repo}" -name "*.db.tar.?z" \ + -exec repo-remove {} "${remove[@]}" \; >/dev/null 2>&1 + + msg2 "Removed the following packages:" + plain '%s' "${remove[@]}" +done diff --git a/db-check-unsigned-packages b/db-check-unsigned-packages new file mode 100755 index 0000000..0fc053b --- /dev/null +++ b/db-check-unsigned-packages @@ -0,0 +1,38 @@ +#!/bin/bash +# Copyright (C) 2012 Michał Masłowski +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +set -e + +# Output a list of repo/package-name-and-version pairs representing +# unsigned packages available for architecture $1 and specified for +# architecture $2 (usually $1 or any, default is to list all). + +. "$(dirname "$(readlink -e "$0")")/config" +. "$(dirname "$(readlink -e "$0")")/db-functions" + +if [ $# -lt 1 ]; then + msg "usage: %s " "${0##*/}" + exit 1 +fi + +arch=$1 +shift + +for repo in "${PKGREPOS[@]}" +do + db="${FTP_BASE}/${repo}/os/${arch}/${repo}.db" + [ -f "$db" ] && "$(dirname "$(readlink -e "$0")")/db-check-unsigned-packages.py" "$repo" "$@" < "$db" +done diff --git a/db-check-unsigned-packages.py b/db-check-unsigned-packages.py new file mode 100755 index 0000000..80cff51 --- /dev/null +++ b/db-check-unsigned-packages.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +# Copyright (C) 2012 Michał Masłowski +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +""" +Output a list of repo/package-name-and-version pairs representing +unsigned packages in the database at standard input of repo named in +the first argument and specified for architectures listed in the +following arguments (usually the one of the database or any, default +is to list all). + +If the --keyset argument is passed, print the key fingerprint of every +signed package. +""" + + +import base64 +import subprocess +import sys +import tarfile + + +def main(): + """Do the job.""" + check_keys = False + if "--keyset" in sys.argv: + sys.argv.remove("--keyset") + check_keys = True + repo = sys.argv[1] + pkgarches = frozenset(name.encode("utf-8") for name in sys.argv[2:]) + packages = [] + keys = [] + with tarfile.open(fileobj=sys.stdin.buffer) as archive: + for entry in archive: + if entry.name.endswith("/desc"): + content = archive.extractfile(entry) + skip = False + is_arch = False + key = None + for line in content: + if is_arch: + is_arch = False + if pkgarches and line.strip() not in pkgarches: + skip = True # different architecture + break + if line == b"%PGPSIG%\n": + skip = True # signed + key = b"" + if check_keys: + continue + else: + break + if line == b"%ARCH%\n": + is_arch = True + continue + if key is not None: + if line.strip(): + key += line.strip() + else: + break + if check_keys and key: + key_binary = base64.b64decode(key) + keys.append(key_binary) + packages.append(repo + "/" + entry.name[:-5]) + if skip: + continue + print(repo + "/" + entry.name[:-5]) + if check_keys and keys: + # We have collected all signed package names in packages and + # all keys in keys. Let's now ask gpg to list all signatures + # and find which keys made them. + packets = subprocess.check_output(("gpg", "--list-packets"), + input=b"".join(keys)) + i = 0 + for line in packets.decode("latin1").split("\n"): + if line.startswith(":signature packet:"): + keyid = line[line.index("keyid ") + len("keyid "):] + print(packages[i], keyid) + i += 1 + + +if __name__ == "__main__": + main() -- cgit v1.2.2 From 25b31f79b5c149ef137073fba607db3debe18a42 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Fri, 22 Jun 2018 13:22:49 -0400 Subject: db-init --- db-init | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100755 db-init diff --git a/db-init b/db-init new file mode 100755 index 0000000..e25dbff --- /dev/null +++ b/db-init @@ -0,0 +1,6 @@ +#!/bin/bash +# Creates the repo structure defined in config + +source "$(dirname "$(readlink -e "$0")")/config" + +mkdir -p -- "${FTP_BASE}"/{"${PKGPOOL}","${SRCPOOL}"} "${CLEANUP_DESTDIR}" "${SOURCE_CLEANUP_DESTDIR}" "${STAGING}" -- cgit v1.2.2 From b4473f46cfa42376cf4dd673270f7ad9ac79b3cf Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Fri, 22 Jun 2018 12:28:48 -0400 Subject: Change config* to match Parabola --- .gitignore | 1 - config | 25 ++++++++++--------------- config.local | 1 + config.local.archlinux32 | 5 +++++ config.local.archlinuxarm | 5 +++++ config.local.community | 5 +++++ config.local.packages | 5 +++++ config.local.parabola | 14 ++++++++++++++ config.local.svn-community | 13 ------------- config.local.svn-packages | 13 ------------- 10 files changed, 45 insertions(+), 42 deletions(-) create mode 120000 config.local create mode 100644 config.local.archlinux32 create mode 100644 config.local.archlinuxarm create mode 100644 config.local.community create mode 100644 config.local.packages create mode 100644 config.local.parabola delete mode 100644 config.local.svn-community delete mode 100644 config.local.svn-packages diff --git a/.gitignore b/.gitignore index 7c92465..d29e370 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,4 @@ *~ -/config.local \#*# .#* *.pyc diff --git a/config b/config index 43c9362..d5f489b 100644 --- a/config +++ b/config @@ -3,28 +3,27 @@ # Instead, create separate ${toolname}.conf files. Only add a # variable here if multiple tools start needing the option. +FTP_BASE="/srv/repo/main" -FTP_BASE="/srv/ftp" PKGREPOS=() PKGPOOL='' SRCPOOL='' -CLEANUP_DESTDIR="/var/tmp" +CLEANUP_DESTDIR="/srv/repo/package-cleanup" CLEANUP_DRYRUN=false # Time in days to keep moved packages CLEANUP_KEEP=30 -SOURCE_CLEANUP_DESTDIR="/var/tmp" -SOURCE_CLEANUP_DRYRUN=false +SOURCE_CLEANUP_DESTDIR="/srv/repo/source-cleanup" +SOURCE_CLEANUP_DRYRUN=true # Time in days to keep moved sourcepackages -SOURCE_CLEANUP_KEEP=14 +SOURCE_CLEANUP_KEEP=30 LOCK_DELAY=10 -LOCK_TIMEOUT=300 -STAGING="$HOME/staging" -export TMPDIR="/var/tmp" -ARCHES=(x86_64) +[ -n "${STAGING:-}" ] || STAGING="$HOME/staging/unknown/staging" +export TMPDIR="/tmp" +ARCHES=(x86_64 i686 armv7h) DBEXT=".db.tar.gz" FILESEXT=".files.tar.gz" SRCEXT=".src.tar.gz" @@ -32,13 +31,9 @@ SRCEXT=".src.tar.gz" # bash glob listing allowed extensions. Note that db-functions turns on extglob. PKGEXTS=".pkg.tar.@(gz|bz2|xz|lzo|lrz|Z)" -# Allowed licenses: get sourceballs only for licenses in this array -ALLOWED_LICENSES=('GPL' 'GPL1' 'GPL2' 'GPL3' 'LGPL' 'LGPL1' 'LGPL2' 'LGPL2.1' 'LGPL3') - # Where to send error emails, and who they are from -LIST="arch-dev-public@archlinux.org" -#LIST="aaronmgriffin@gmail.com" -FROM="repomaint@archlinux.org" +LIST="maintenance@lists.parabola.nu" +FROM="repo@repo.parabola.nu" # Override default config with config.local LOCAL_CONFIG=${DBSCRIPTS_CONFIG:-"$(dirname ${BASH_SOURCE[0]})/config.local"} diff --git a/config.local b/config.local new file mode 120000 index 0000000..906ba68 --- /dev/null +++ b/config.local @@ -0,0 +1 @@ +config.local.parabola \ No newline at end of file diff --git a/config.local.archlinux32 b/config.local.archlinux32 new file mode 100644 index 0000000..094bf6d --- /dev/null +++ b/config.local.archlinux32 @@ -0,0 +1,5 @@ +#!/hint/bash + +PKGREPOS=(build-support community community-staging community-testing core extra gnome-unstable kde-unstable staging testing) +PKGPOOL='pool/archlinux32' +SRCPOOL='sources/archlinux32' diff --git a/config.local.archlinuxarm b/config.local.archlinuxarm new file mode 100644 index 0000000..42861fe --- /dev/null +++ b/config.local.archlinuxarm @@ -0,0 +1,5 @@ +#!/hint/bash + +PKGREPOS=(alarm aur community core extra) +PKGPOOL='pool/alarm' +SRCPOOL='sources/alarm' diff --git a/config.local.community b/config.local.community new file mode 100644 index 0000000..b4a0f91 --- /dev/null +++ b/config.local.community @@ -0,0 +1,5 @@ +#!/hint/bash + +PKGREPOS=('community' 'community-testing' 'community-staging' 'multilib' 'multilib-testing' 'multilib-staging') +PKGPOOL='pool/community' +SRCPOOL='sources/community' diff --git a/config.local.packages b/config.local.packages new file mode 100644 index 0000000..017f034 --- /dev/null +++ b/config.local.packages @@ -0,0 +1,5 @@ +#!/hint/bash + +PKGREPOS=('core' 'extra' 'testing' 'staging' 'kde-unstable' 'gnome-unstable') +PKGPOOL='pool/packages' +SRCPOOL='sources/packages' diff --git a/config.local.parabola b/config.local.parabola new file mode 100644 index 0000000..ddfc4d7 --- /dev/null +++ b/config.local.parabola @@ -0,0 +1,14 @@ +#!/hint/bash + +PKGREPOS=( + 'core' 'testing' 'extra' 'community' 'multilib' 'multilib-testing' + 'libre' 'libre-testing' 'libre-multilib' 'libre-multilib-testing' + '~smv' '~xihh' '~brendan' '~lukeshu' '~emulatorman' '~aurelien' + '~jorginho' '~coadde' '~drtan' 'nonsystemd' 'nonsystemd-testing' + 'nonprism' 'nonprism-testing' 'nonprism-multilib' + 'nonprism-multilib-testing' 'pcr' 'pcr-testing' 'pcr-multilib' + 'pcr-multilib-testing' 'pcr-testing' 'kernels' 'kernels-testing' + 'cross' 'java' 'unmaintained' 'build-support' +) +PKGPOOL='pool/parabola' +SRCPOOL='sources/parabola' diff --git a/config.local.svn-community b/config.local.svn-community deleted file mode 100644 index 5d61b5e..0000000 --- a/config.local.svn-community +++ /dev/null @@ -1,13 +0,0 @@ -#!/hint/bash - -PKGREPOS=('community' 'community-testing' 'community-staging' 'multilib' 'multilib-testing' 'multilib-staging') -PKGPOOL='pool/community' -SRCPOOL='sources/community' -SVNREPO='file:///srv/repos/svn-community/svn' -SVNUSER='svn-community' -TESTING_REPO='community-testing' -STABLE_REPOS=('community') - -CLEANUP_DESTDIR="/srv/repos/svn-community/package-cleanup" -SOURCE_CLEANUP_DESTDIR="/srv/repos/svn-community/source-cleanup" -TMPDIR="/srv/repos/svn-community/tmp" diff --git a/config.local.svn-packages b/config.local.svn-packages deleted file mode 100644 index 34aab35..0000000 --- a/config.local.svn-packages +++ /dev/null @@ -1,13 +0,0 @@ -#!/hint/bash - -PKGREPOS=('core' 'extra' 'testing' 'staging' 'kde-unstable' 'gnome-unstable') -PKGPOOL='pool/packages' -SRCPOOL='sources/packages' -SVNREPO='file:///srv/repos/svn-packages/svn' -SVNUSER='svn-packages' -TESTING_REPO='testing' -STABLE_REPOS=('core' 'extra') - -CLEANUP_DESTDIR="/srv/repos/svn-packages/package-cleanup" -SOURCE_CLEANUP_DESTDIR="/srv/repos/svn-packages/source-cleanup" -TMPDIR="/srv/repos/svn-packages/tmp" -- cgit v1.2.2 From c63fafb1afdc77a5564a459899035519fed72cd5 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Fri, 22 Jun 2018 13:33:20 -0400 Subject: Makefile: `make check` instead of `make test`, per GNU conventions --- Makefile | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index ecb78c9..66913dc 100644 --- a/Makefile +++ b/Makefile @@ -1,11 +1,13 @@ all: .PHONY: all -test: +check: $(MAKE) -C test test - -test-coverage: +check-coverage: rm -rf ${PWD}/coverage $(MAKE) -C test test-coverage +.PHONY: check check-coverage +test: check +test-coverage: check-coverage .PHONY: test test-coverage -- cgit v1.2.2 From b5e9f9d489307eb7e044115ed9f055a48d80eff0 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Fri, 22 Jun 2018 13:25:23 -0400 Subject: misc --- cron-jobs/integrity-check | 4 ++ cron-jobs/makepkg.conf | 147 ---------------------------------------------- cron-jobs/sourceballs | 2 +- db-functions | 22 +++---- db-update | 12 ++++ test/lib/common.bash | 2 +- 6 files changed, 26 insertions(+), 163 deletions(-) delete mode 100644 cron-jobs/makepkg.conf diff --git a/cron-jobs/integrity-check b/cron-jobs/integrity-check index 904dfc1..5d9d0d0 100755 --- a/cron-jobs/integrity-check +++ b/cron-jobs/integrity-check @@ -21,6 +21,10 @@ check() { 2>&1 | "${dirname}"/devlist-mailer "Integrity Check ${arch}: ${repos}" "${mailto}" } +repos='core,extra,community' +arch='i686' +check + repos='core,extra,community,multilib' arch='x86_64' check diff --git a/cron-jobs/makepkg.conf b/cron-jobs/makepkg.conf deleted file mode 100644 index d8512fb..0000000 --- a/cron-jobs/makepkg.conf +++ /dev/null @@ -1,147 +0,0 @@ -# -# /etc/makepkg.conf -# - -######################################################################### -# SOURCE ACQUISITION -######################################################################### -# -#-- The download utilities that makepkg should use to acquire sources -# Format: 'protocol::agent' -DLAGENTS=('ftp::/usr/bin/curl -sS -fC - --ftp-pasv --retry 3 --retry-delay 3 -o %o %u' - 'http::/usr/bin/curl -sS -fLC - --retry 3 --retry-delay 3 -o %o %u' - 'https::/usr/bin/curl -sS -fLC - --retry 3 --retry-delay 3 -o %o %u' - 'rsync::/usr/bin/rsync -q --no-motd -z %u %o' - 'scp::/usr/bin/scp -q -C %u %o') - -# Other common tools: -# /usr/bin/snarf -# /usr/bin/lftpget -c -# /usr/bin/wget - -#-- The package required by makepkg to download VCS sources -# Format: 'protocol::package' -VCSCLIENTS=('bzr::bzr' - 'git::git' - 'hg::mercurial' - 'svn::subversion') - -######################################################################### -# ARCHITECTURE, COMPILE FLAGS -######################################################################### -# -CARCH="x86_64" -CHOST="x86_64-unknown-linux-gnu" - -#-- Compiler and Linker Flags -# -march (or -mcpu) builds exclusively for an architecture -# -mtune optimizes for an architecture, but builds for whole processor family -CPPFLAGS="-D_FORTIFY_SOURCE=2" -CFLAGS="-march=x86-64 -mtune=generic -O2 -pipe -fstack-protector-strong" -CXXFLAGS="-march=x86-64 -mtune=generic -O2 -pipe -fstack-protector-strong" -LDFLAGS="-Wl,-O1,--sort-common,--as-needed,-z,relro" -#-- Make Flags: change this for DistCC/SMP systems -#MAKEFLAGS="-j2" -#-- Debugging flags -DEBUG_CFLAGS="-g -fvar-tracking-assignments" -DEBUG_CXXFLAGS="-g -fvar-tracking-assignments" - -######################################################################### -# BUILD ENVIRONMENT -######################################################################### -# -# Defaults: BUILDENV=(!distcc color !ccache check !sign) -# A negated environment option will do the opposite of the comments below. -# -#-- distcc: Use the Distributed C/C++/ObjC compiler -#-- color: Colorize output messages -#-- ccache: Use ccache to cache compilation -#-- check: Run the check() function if present in the PKGBUILD -#-- sign: Generate PGP signature file -# -BUILDENV=(!distcc color !ccache check !sign) -# -#-- If using DistCC, your MAKEFLAGS will also need modification. In addition, -#-- specify a space-delimited list of hosts running in the DistCC cluster. -#DISTCC_HOSTS="" -# -#-- Specify a directory for package building. -#BUILDDIR=/tmp/makepkg - -######################################################################### -# GLOBAL PACKAGE OPTIONS -# These are default values for the options=() settings -######################################################################### -# -# Default: OPTIONS=(strip docs !libtool !staticlibs emptydirs zipman purge !optipng !upx !debug) -# A negated option will do the opposite of the comments below. -# -#-- strip: Strip symbols from binaries/libraries -#-- docs: Save doc directories specified by DOC_DIRS -#-- libtool: Leave libtool (.la) files in packages -#-- staticlibs: Leave static library (.a) files in packages -#-- emptydirs: Leave empty directories in packages -#-- zipman: Compress manual (man and info) pages in MAN_DIRS with gzip -#-- purge: Remove files specified by PURGE_TARGETS -#-- upx: Compress binary executable files using UPX -#-- optipng: Optimize PNG images with optipng -#-- debug: Add debugging flags as specified in DEBUG_* variables -# -OPTIONS=(strip docs !libtool !staticlibs emptydirs zipman purge !optipng !upx !debug) - -#-- File integrity checks to use. Valid: md5, sha1, sha256, sha384, sha512 -INTEGRITY_CHECK=(md5) -#-- Options to be used when stripping binaries. See `man strip' for details. -STRIP_BINARIES="--strip-all" -#-- Options to be used when stripping shared libraries. See `man strip' for details. -STRIP_SHARED="--strip-unneeded" -#-- Options to be used when stripping static libraries. See `man strip' for details. -STRIP_STATIC="--strip-debug" -#-- Manual (man and info) directories to compress (if zipman is specified) -MAN_DIRS=({usr{,/local}{,/share},opt/*}/{man,info}) -#-- Doc directories to remove (if !docs is specified) -DOC_DIRS=(usr/{,local/}{,share/}{doc,gtk-doc} opt/*/{doc,gtk-doc}) -#-- Files to be removed from all packages (if purge is specified) -PURGE_TARGETS=(usr/{,share}/info/dir .packlist *.pod) - -######################################################################### -# PACKAGE OUTPUT -######################################################################### -# -# Default: put built package and cached source in build directory -# -#-- Destination: specify a fixed directory where all packages will be placed -#PKGDEST=/home/packages -#-- Source cache: specify a fixed directory where source files will be cached -#SRCDEST=/home/sources -#-- Source packages: specify a fixed directory where all src packages will be placed -#SRCPKGDEST=/home/srcpackages -#-- Log files: specify a fixed directory where all log files will be placed -#LOGDEST=/home/makepkglogs -#-- Packager: name/email of the person or organization building packages -#PACKAGER="John Doe " -#-- Specify a key to use for package signing -#GPGKEY="" - -######################################################################### -# COMPRESSION DEFAULTS -######################################################################### -# -COMPRESSGZ=(gzip -c -f -n) -COMPRESSBZ2=(bzip2 -c -f) -COMPRESSXZ=(xz -c -z -) -COMPRESSLRZ=(lrzip -q) -COMPRESSLZO=(lzop -q) -COMPRESSZ=(compress -c -f) - -######################################################################### -# EXTENSION DEFAULTS -######################################################################### -# -# WARNING: Do NOT modify these variables unless you know what you are -# doing. -# -PKGEXT='.pkg.tar.xz' -SRCEXT='.src.tar.gz' - -# vim: set ft=sh ts=2 sw=2 et: diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 8d14323..15829ec 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -88,7 +88,7 @@ for repo in "${PKGREPOS[@]}"; do # Build the actual source package pushd "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" >/dev/null - makepkg --nocolor --allsource --ignorearch --skippgpcheck --config "${dirname}/makepkg.conf" >"${WORKDIR}/${pkgbase}.log" 2>&1 + SRCPKGDEST=. makepkg --nocolor --allsource --ignorearch --skippgpcheck >"${WORKDIR}/${pkgbase}.log" 2>&1 if (( $? == 0 )) && [[ -f ${pkgbase}-${pkgver}${SRCEXT} ]]; then mv_acl "${pkgbase}-${pkgver}${SRCEXT}" "${FTP_BASE}/${SRCPOOL}/${pkgbase}-${pkgver}${SRCEXT}" # Avoid creating the same source package for every arch diff --git a/db-functions b/db-functions index 776c74a..fe1bfd9 100644 --- a/db-functions +++ b/db-functions @@ -3,20 +3,6 @@ . /usr/share/makepkg/util.sh . "$(dirname "${BASH_SOURCE[0]}")/db-abs" -# Used: -# libmakepkg/util/message.sh: -# plain, msg, msg2, warning, error -# libmakepkg/util/pkgbuild.sh: -# get_full_version -# libmakepkg/util/util.sh: -# in_array -# devtools/common.sh: -# abort, die -# Overwritten: -# devtools/common.sh: -# cleanup -. "$(librelib common)" - # global shell options for enhanced bash scripting shopt -s extglob globstar nullglob @@ -59,6 +45,14 @@ fi LOCKS=() REPO_MODIFIED=0 +# Used: plain, msg, msg2, warning, error, in_array, get_full_version, abort, die +# Overwritten: cleanup +# Ignored: stat_busy, stat_done, +# setup_workdir, trap_abort, trap_exit, +# lock, slock, lock_close +# pkgver_equal, find_cached_package, check_root +. "$(librelib common)" + script_lock() { local LOCKDIR="$TMPDIR/.scriptlock.${0##*/}" if ! mkdir "$LOCKDIR" >/dev/null 2>&1 ; then diff --git a/db-update b/db-update index 156cb91..7161972 100755 --- a/db-update +++ b/db-update @@ -21,6 +21,7 @@ for staging_repo in "${staging_repos[@]##*/}"; do repos+=("${staging_repo}") fi done +repos=($(echo "${repos[@]}" | tr " " "\n" | sort -u)) # TODO: this might lock too much (architectures) for repo in "${repos[@]}"; do @@ -102,3 +103,14 @@ for repo in "${repos[@]}"; do repo_unlock "${repo}" "${pkgarch}" done done + +cd "${STAGING}" +while read -r file; do + pub="${FTP_BASE}/${file}" + if [[ -f "$pub" ]]; then + warning "file already exists: %s" "$file" + else + mkdir -p -- "${pub%/*}" + mv -vn "$file" "$pub" + fi +done < <(find other sources -type f 2>/dev/null) diff --git a/test/lib/common.bash b/test/lib/common.bash index 76061f6..a91fa7b 100644 --- a/test/lib/common.bash +++ b/test/lib/common.bash @@ -129,7 +129,7 @@ releasePackage() { if [ ! -d "${TMP}/svn-packages-copy/${pkgbase}/trunk" ]; then mkdir -p "${TMP}/svn-packages-copy/${pkgbase}"/{trunk,repos} - cp -r "fixtures/${pkgbase}"/* "${TMP}/svn-packages-copy"/${pkgbase}/trunk/ + cp "fixtures/${pkgbase}"/* "${TMP}/svn-packages-copy"/${pkgbase}/trunk/ svn add -q "${TMP}/svn-packages-copy"/${pkgbase} svn commit -q -m"initial commit of ${pkgbase}" "${TMP}/svn-packages-copy" fi -- cgit v1.2.2