From ced789360b76e705fed538cf61329e604996cc42 Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Wed, 21 Mar 2018 12:09:21 -0400 Subject: pacman 5.1 compatibility The print_all_package_names function changed its behavior, adapt (cherry picked from commit 2c68135f4ab9db048db460bc949322ad7bb418eb) --- test/lib/common.bash | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/test/lib/common.bash b/test/lib/common.bash index 73b0c54..b1e5a37 100644 --- a/test/lib/common.bash +++ b/test/lib/common.bash @@ -23,7 +23,7 @@ __buildPackage() { local cache local pkgarches local tarch - local pkgnames + local pkgfiles if [[ -n ${BUILDDIR} ]]; then cache=${BUILDDIR}/$(__getCheckSum PKGBUILD) @@ -39,15 +39,15 @@ __buildPackage() { for tarch in ${pkgarches[@]}; do if [ "${tarch}" == 'any' ]; then sudo librechroot -n "dbscripts@${tarch}" make + mapfile -tO "${#pkgfiles[@]}" pkgfiles < <(PKGDEST=${pkgdest} PKGEXT=${PKGEXT} makepkg --packagelist) else sudo librechroot -n "dbscripts@${tarch}" -A "$tarch" make + mapfile -tO "${#pkgfiles[@]}" pkgfiles < <(PKGDEST=${pkgdest} PKGEXT=${PKGEXT} CARCH=${tarch} makepkg --packagelist) fi sudo PKGDEST="${pkgdest}" libremakepkg -n "dbscripts@${tarch}" done - pkgnames=($(. PKGBUILD; print_all_package_names)) - pushd ${pkgdest} - for p in ${pkgnames[@]/%/${PKGEXT}}; do + for p in ${pkgfiles[@]}; do # Manually sign packages as "makepkg --sign" is buggy gpg -v --detach-sign --no-armor --use-agent ${p} @@ -55,7 +55,6 @@ __buildPackage() { cp -Lv ${p}{,.sig} ${cache}/ fi done - popd } setup() { -- cgit v1.2.2 From 46313ebca5d75d637dd1055dd27e9a4dea068de2 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Fri, 1 Jun 2018 12:12:29 -0400 Subject: pacman 5.1 compatibility (db-import-pkg) --- db-import-pkg | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/db-import-pkg b/db-import-pkg index e8b4432..f88ad0a 100755 --- a/db-import-pkg +++ b/db-import-pkg @@ -105,8 +105,10 @@ make_whitelist() { # Remove blacklisted packages and count them # TODO: capture all removed packages for printing on debug mode msg2 "Removing blacklisted packages from %s ..." "${db_file##*/}" - LC_ALL=C repo-remove "$db_file" "${blacklist[@]}" |& - sed -n 's/-> Removing/ &/p' + local package + for package in "${blacklist[@]}"; do + LC_ALL=C repo-remove "$db_file" "$package" || true + done |& sed -n 's/-> Removing/ &/p' # Get db contents local -r db=($(get_repo_content "${db_file}")) -- cgit v1.2.2 From 9834f87ea7e5830f305ca4d01d8ad9282b698723 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Thu, 31 May 2018 11:35:08 -0400 Subject: test: db-import-pkg: Add some tests --- db-import-archlinux.conf | 4 + db-import-archlinux32.conf | 4 + db-import-archlinuxarm.conf | 4 + test/cases/db-import-pkg.bats | 197 +++++++++++++++++++++++++++++++++++++++++ test/fixtures/slavery/PKGBUILD | 13 +++ test/lib/runserver | 39 ++++++++ 6 files changed, 261 insertions(+) create mode 100644 test/cases/db-import-pkg.bats create mode 100644 test/fixtures/slavery/PKGBUILD create mode 100755 test/lib/runserver diff --git a/db-import-archlinux.conf b/db-import-archlinux.conf index e230a2b..14a1221 100644 --- a/db-import-archlinux.conf +++ b/db-import-archlinux.conf @@ -26,3 +26,7 @@ ABSLIBRE_ROOT=/srv/repo/db-import-archlinux-src/abslibre ABSLIBRE_GIT=https://git.parabola.nu/abslibre.git/ ABS_ROOT="/srv/repo/db-import-archlinux-src/abs/" ABS_SERVER="rsync.archlinux.org" + +if [[ -n ${DBIMPORT_CONFIG} ]]; then + source "${DBIMPORT_CONFIG}" +fi diff --git a/db-import-archlinux32.conf b/db-import-archlinux32.conf index f2cb3dc..c56beeb 100644 --- a/db-import-archlinux32.conf +++ b/db-import-archlinux32.conf @@ -24,3 +24,7 @@ ABS_ROOT='/srv/repo/db-import-archlinux32-src/abs/' ABS_SERVER="${mirror}/${mirrorpath}/" ALARM_GIT='https://github.com/archlinux32/packages' ALARM_ROOT='/srv/repo/db-import-archlinux32-src/archlinux32/' + +if [[ -n ${DBIMPORT_CONFIG} ]]; then + source "${DBIMPORT_CONFIG}" +fi diff --git a/db-import-archlinuxarm.conf b/db-import-archlinuxarm.conf index 5e0dd4f..c9e1ef3 100644 --- a/db-import-archlinuxarm.conf +++ b/db-import-archlinuxarm.conf @@ -27,3 +27,7 @@ ABS_ROOT='/srv/repo/db-import-archlinuxarm-src/abs/' ABS_SERVER="${mirror}/${mirrorpath}/" ALARM_GIT='https://github.com/archlinuxarm/PKGBUILDs' ALARM_ROOT='/srv/repo/db-import-archlinuxarm-src/alarm/' + +if [[ -n ${DBIMPORT_CONFIG} ]]; then + source "${DBIMPORT_CONFIG}" +fi diff --git a/test/cases/db-import-pkg.bats b/test/cases/db-import-pkg.bats new file mode 100644 index 0000000..fdd1ba5 --- /dev/null +++ b/test/cases/db-import-pkg.bats @@ -0,0 +1,197 @@ +load ../lib/common + +httpserver() { + read -r verb path version || exit + if [[ $verb != GET ]]; then + printf "HTTP/1.1 405 Method Not Allowed\r\nContent-Type: text/plain\r\nContent-length: 0\r\n\r\n" + exit + fi + path="$(cd / && realpath -ms "$path")" + if ! [[ -f "$1/$path" ]]; then + printf "HTTP/1.1 404 Not found\r\nContent-Type: text/plain\r\nContent-length: 0\r\n\r\n" + exit + fi + printf "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-length: %d\r\n\r\n" $(stat -L -c %s "$1/$path") + cat "$1/$path" +} + +eval "__common_$(declare -f setup)" +setup() { + __common_setup + + # Set up rsync server + cat <<-eot >"${TMP}/rsyncd.conf" + use chroot = no + [archlinux] + path = ${TMP}/rsyncd/archlinux + [archlinux32] + path = ${TMP}/rsyncd/archlinux32 + eot + local rsyncport + rsyncport=$(./lib/runserver "$TMP/rsyncd.pid" \ + rsync --daemon --config "${TMP}/rsyncd.conf") + + # Set up rsync contents + mkdir -p -- "${TMP}/rsyncd/archlinux/core/os/x86_64" + touch -- "${TMP}/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz" + ln -s core.db.tar.gz "${TMP}/rsyncd/archlinux/core/os/x86_64/core.db" + mkdir -p -- "${TMP}/rsyncd/archlinux"/{pool,sources}/{packages,community} + date +%s > "${TMP}/rsyncd/archlinux/lastupdate" + date +%s > "${TMP}/rsyncd/archlinux/lastsync" + + mkdir -p -- "${TMP}/rsyncd/archlinux32/i686/core" + touch -- "${TMP}/rsyncd/archlinux32/i686/core/core.db.tar.gz" + ln -s core.db.tar.gz "${TMP}/rsyncd/archlinux32/i686/core/core.db" + date +%s > "${TMP}/rsyncd/archlinux32/lastupdate" + date +%s > "${TMP}/rsyncd/archlinux32/lastsync" + + # Configure db-import to use that rsyncd server + cat <<-eot >"${TMP}/db-import-archlinux.local.conf" + ARCHREPOS=('core') + mirror=localhost:${rsyncport@Q} + mirrorpath=archlinux + eot + cat <<-eot >"${TMP}/db-import-archlinux32.local.conf" + ARCHREPOS=('core') + mirror=localhost:${rsyncport@Q} + mirrorpath=archlinux32 + eot + + # Set up HTTP server + local httpport + httpport=$(./lib/runserver "$TMP/httpd.pid" \ + bash -c "$(declare -f httpserver); httpserver \"\$@\"" -- "$TMP/httpd") + + # Set up HTTP contents + mkdir -- "$TMP/httpd" + cat <<-eot >"$TMP/httpd/blacklist.txt" + slavery:freedom:fsf:slavekit:Obviously + eot + + # Configure db-import to use that HTTP server + mkdir "$XDG_CONFIG_HOME"/libretools + cat <<-eot >"$XDG_CONFIG_HOME"/libretools/libretools.conf + BLACKLIST=http://localhost:${httpport@Q}/blacklist.txt + eot +} +eval "__common_$(declare -f teardown)" +teardown() { + xargs -a "${TMP}/httpd.pid" kill -- + xargs -a "${TMP}/rsyncd.pid" kill -- + __common_teardown +} + +###################################################################### + +# releaseImportedPackage PKGBASE ARCH DBFILE [POOLDIR] +# +# This is different from common.bash:releasePackage because +# - it doesn't mess with SVN +# - it adds the package to the .db file +__releaseImportedPackage() { + local pkgbase=$1 + local arch=$2 + local dbfile=$3 + local pooldir=$4 + local repodir="${dbfile%/*}" + local dir restore pkgfiles pkgfile pkgs + + dir="$TMP/import-build/$pkgbase" + if ! [[ -d "$dir" ]]; then + mkdir -p -- "$dir" + cp -t "$dir" -- "fixtures/${pkgbase}"/* + fi + pushd "$dir" + __buildPackage + restore="$(shopt -p nullglob || true)" + shopt -s nullglob + pkgfiles=(*-{"$arch",any}$PKGEXT{,.sig}) + $restore + popd + + mkdir -p "$repodir" + if [[ -z $pooldir ]]; then + mv -t "$repodir" -- "${pkgfiles[@]/#/"$dir/"}" + else + mkdir -p "$pooldir" + mv -t "$pooldir" -- "${pkgfiles[@]/#/"$dir/"}" + ln -sr -t "$repodir" -- "${pkgfiles[@]/#/"$pooldir/"}" + fi + + pushd "$repodir" + pkgs=() + for pkgfile in "${pkgfiles[@]}"; do + if [[ "$pkgfile" = *.sig ]]; then + continue + fi + pkgs+=("$pkgfile") + done + repo-add -q "${dbfile##*/}" "${pkgs[@]}" + popd +} + +__updateImportedPackage() { + pushd "$TMP/import-build/$1" + local pkgrel + pkgrel=$(. PKGBUILD; expr ${pkgrel} + 1) + sed "s/pkgrel=.*/pkgrel=${pkgrel}/" -i PKGBUILD + popd +} + +__isLinkTo() { + [[ -L $1 ]] + [[ $1 -ef $2 ]] +} + +__doesNotExist() { + local file + for file in "$@"; do + ! stat "$file" + done +} + +###################################################################### + +@test "imports no blacklisted packages (x86_64)" { + __releaseImportedPackage slavery x86_64 "$TMP/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz" "$TMP/rsyncd/archlinux/pool/packages" + __releaseImportedPackage pkg-simple-a x86_64 "$TMP/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz" "$TMP/rsyncd/archlinux/pool/packages" + + DBIMPORT_CONFIG="${TMP}/db-import-archlinux.local.conf" db-import-pkg archlinux + + __isLinkTo "$TMP/ftp/core/os/x86_64/pkg-simple-a-1-1-x86_64.pkg.tar.xz" "$TMP/ftp/pool/packages/pkg-simple-a-1-1-x86_64.pkg.tar.xz" + __doesNotExist "$TMP"/ftp/{core/os/x86_64,pool/packages,sources/packages}/slavery-* +} + +@test "imports no blacklisted packages (i686)" { + __releaseImportedPackage slavery i686 "$TMP/rsyncd/archlinux32/i686/core/core.db.tar.gz" + __releaseImportedPackage pkg-simple-a i686 "$TMP/rsyncd/archlinux32/i686/core/core.db.tar.gz" + + DBIMPORT_CONFIG="${TMP}/db-import-archlinux32.local.conf" db-import-pkg archlinux32 + + __isLinkTo "$TMP/ftp/core/os/i686/pkg-simple-a-1-1-i686.pkg.tar.xz" "$TMP/ftp/pool/archlinux32/pkg-simple-a-1-1-i686.pkg.tar.xz" + __doesNotExist "$TMP"/ftp/{core/os/i686,pool/archlinux32,sources/archlinux32}/slavery-* +} + +@test "imports DBs with no blacklisted packages" { + __releaseImportedPackage pkg-simple-a x86_64 "$TMP/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz" "$TMP/rsyncd/archlinux/pool/packages" + + DBIMPORT_CONFIG="${TMP}/db-import-archlinux.local.conf" db-import-pkg archlinux + + __isLinkTo "$TMP/ftp/core/os/x86_64/pkg-simple-a-1-1-x86_64.pkg.tar.xz" "$TMP/ftp/pool/packages/pkg-simple-a-1-1-x86_64.pkg.tar.xz" +} + +@test "imports updated packages" { + __releaseImportedPackage slavery x86_64 "$TMP/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz" "$TMP/rsyncd/archlinux/pool/packages" + __releaseImportedPackage pkg-simple-a x86_64 "$TMP/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz" "$TMP/rsyncd/archlinux/pool/packages" + + DBIMPORT_CONFIG="${TMP}/db-import-archlinux.local.conf" db-import-pkg archlinux + + __isLinkTo "$TMP/ftp/core/os/x86_64/pkg-simple-a-1-1-x86_64.pkg.tar.xz" "$TMP/ftp/pool/packages/pkg-simple-a-1-1-x86_64.pkg.tar.xz" + + __updateImportedPackage pkg-simple-a + __releaseImportedPackage pkg-simple-a x86_64 "$TMP/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz" "$TMP/rsyncd/archlinux/pool/packages" + + DBIMPORT_CONFIG="${TMP}/db-import-archlinux.local.conf" db-import-pkg archlinux + + __isLinkTo "$TMP/ftp/core/os/x86_64/pkg-simple-a-1-2-x86_64.pkg.tar.xz" "$TMP/ftp/pool/packages/pkg-simple-a-1-2-x86_64.pkg.tar.xz" +} diff --git a/test/fixtures/slavery/PKGBUILD b/test/fixtures/slavery/PKGBUILD new file mode 100644 index 0000000..93cc079 --- /dev/null +++ b/test/fixtures/slavery/PKGBUILD @@ -0,0 +1,13 @@ +pkgname=slavery +pkgver=1 +pkgrel=1 +pkgdesc="A package called ${pkgname}" +arch=('any') +url='http://www.archlinux.org/' +license=('GPL') +options=(!strip) + +package() { + install -d -m755 "${pkgdir}"/usr/share/${pkgname} + echo 'test' > "${pkgdir}"/usr/share/${pkgname}/test +} diff --git a/test/lib/runserver b/test/lib/runserver new file mode 100755 index 0000000..7705e40 --- /dev/null +++ b/test/lib/runserver @@ -0,0 +1,39 @@ +#!/usr/bin/env perl +use strict; +use warnings; + +use IO::Socket::INET; +use Fcntl; + +my $socket = IO::Socket::INET->new( + LocalAddr => "localhost:0", + Listen => 1 +); + +sysopen(my $fh, $ARGV[0], O_WRONLY|O_CREAT|O_EXCL, 0666) or die "open: $ARGV[0]: $!"; +my $pid = fork(); +if ($pid > 0) { + print $fh $pid; + print $socket->sockport()."\n"; + exit; +} +close($fh); +open(STDIN, 'accept(); + my $worker = fork(); + if ($worker == 0) { + open(STDIN, '<&', $conn); + open(STDOUT, '>&', $conn); + close($conn); + close($socket); + exec @cmd; + exit + } + close($conn); +} -- cgit v1.2.2 From 30956797cbca4d2924a17796cf7a7ca7138768b6 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Fri, 1 Jun 2018 11:41:05 -0400 Subject: Post-merge --- db-import-packages.conf | 4 ++++ test/cases/db-import-pkg.bats | 8 ++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/db-import-packages.conf b/db-import-packages.conf index f971f74..4b3ef91 100644 --- a/db-import-packages.conf +++ b/db-import-packages.conf @@ -26,3 +26,7 @@ ABSLIBRE_ROOT=/srv/repo/db-import-archlinux-src/abslibre ABSLIBRE_GIT=https://git.parabola.nu/abslibre.git/ ABS_ROOT="/srv/repo/db-import-archlinux-src/abs/" ABS_SERVER="rsync.archlinux.org" + +if [[ -n ${DBIMPORT_CONFIG} ]]; then + source "${DBIMPORT_CONFIG}" +fi diff --git a/test/cases/db-import-pkg.bats b/test/cases/db-import-pkg.bats index fdd1ba5..817c53f 100644 --- a/test/cases/db-import-pkg.bats +++ b/test/cases/db-import-pkg.bats @@ -156,7 +156,7 @@ __doesNotExist() { __releaseImportedPackage slavery x86_64 "$TMP/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz" "$TMP/rsyncd/archlinux/pool/packages" __releaseImportedPackage pkg-simple-a x86_64 "$TMP/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz" "$TMP/rsyncd/archlinux/pool/packages" - DBIMPORT_CONFIG="${TMP}/db-import-archlinux.local.conf" db-import-pkg archlinux + DBIMPORT_CONFIG="${TMP}/db-import-archlinux.local.conf" db-import-pkg packages __isLinkTo "$TMP/ftp/core/os/x86_64/pkg-simple-a-1-1-x86_64.pkg.tar.xz" "$TMP/ftp/pool/packages/pkg-simple-a-1-1-x86_64.pkg.tar.xz" __doesNotExist "$TMP"/ftp/{core/os/x86_64,pool/packages,sources/packages}/slavery-* @@ -175,7 +175,7 @@ __doesNotExist() { @test "imports DBs with no blacklisted packages" { __releaseImportedPackage pkg-simple-a x86_64 "$TMP/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz" "$TMP/rsyncd/archlinux/pool/packages" - DBIMPORT_CONFIG="${TMP}/db-import-archlinux.local.conf" db-import-pkg archlinux + DBIMPORT_CONFIG="${TMP}/db-import-archlinux.local.conf" db-import-pkg packages __isLinkTo "$TMP/ftp/core/os/x86_64/pkg-simple-a-1-1-x86_64.pkg.tar.xz" "$TMP/ftp/pool/packages/pkg-simple-a-1-1-x86_64.pkg.tar.xz" } @@ -184,14 +184,14 @@ __doesNotExist() { __releaseImportedPackage slavery x86_64 "$TMP/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz" "$TMP/rsyncd/archlinux/pool/packages" __releaseImportedPackage pkg-simple-a x86_64 "$TMP/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz" "$TMP/rsyncd/archlinux/pool/packages" - DBIMPORT_CONFIG="${TMP}/db-import-archlinux.local.conf" db-import-pkg archlinux + DBIMPORT_CONFIG="${TMP}/db-import-archlinux.local.conf" db-import-pkg packages __isLinkTo "$TMP/ftp/core/os/x86_64/pkg-simple-a-1-1-x86_64.pkg.tar.xz" "$TMP/ftp/pool/packages/pkg-simple-a-1-1-x86_64.pkg.tar.xz" __updateImportedPackage pkg-simple-a __releaseImportedPackage pkg-simple-a x86_64 "$TMP/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz" "$TMP/rsyncd/archlinux/pool/packages" - DBIMPORT_CONFIG="${TMP}/db-import-archlinux.local.conf" db-import-pkg archlinux + DBIMPORT_CONFIG="${TMP}/db-import-archlinux.local.conf" db-import-pkg packages __isLinkTo "$TMP/ftp/core/os/x86_64/pkg-simple-a-1-2-x86_64.pkg.tar.xz" "$TMP/ftp/pool/packages/pkg-simple-a-1-2-x86_64.pkg.tar.xz" } -- cgit v1.2.2 From 98fea4efd1417eb6246dd4d3a63e7d9f0756f63c Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Fri, 1 Jun 2018 11:55:47 -0400 Subject: Post-merge --- test/cases/db-import-pkg.bats | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/test/cases/db-import-pkg.bats b/test/cases/db-import-pkg.bats index 817c53f..1f1f2c2 100644 --- a/test/cases/db-import-pkg.bats +++ b/test/cases/db-import-pkg.bats @@ -47,14 +47,12 @@ setup() { # Configure db-import to use that rsyncd server cat <<-eot >"${TMP}/db-import-archlinux.local.conf" - ARCHREPOS=('core') - mirror=localhost:${rsyncport@Q} - mirrorpath=archlinux + ARCHTAGS=('core-x86_64') + ARCHMIRROR=rsync://localhost:${rsyncport@Q}/archlinux eot cat <<-eot >"${TMP}/db-import-archlinux32.local.conf" - ARCHREPOS=('core') - mirror=localhost:${rsyncport@Q} - mirrorpath=archlinux32 + ARCHTAGS=('core-i686') + ARCHMIRROR=rsync://localhost:${rsyncport@Q}/archlinux32 eot # Set up HTTP server -- cgit v1.2.2 From ee3c583ac965191d99787329e68b3d2a696e0f9b Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Fri, 1 Jun 2018 16:58:23 -0400 Subject: test: db-import-pkg: Verify that .db files are mode=0664 (and not 0644) --- test/cases/db-import-pkg.bats | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/test/cases/db-import-pkg.bats b/test/cases/db-import-pkg.bats index 1f1f2c2..d038031 100644 --- a/test/cases/db-import-pkg.bats +++ b/test/cases/db-import-pkg.bats @@ -193,3 +193,26 @@ __doesNotExist() { __isLinkTo "$TMP/ftp/core/os/x86_64/pkg-simple-a-1-2-x86_64.pkg.tar.xz" "$TMP/ftp/pool/packages/pkg-simple-a-1-2-x86_64.pkg.tar.xz" } + +@test "imports .db files as 0664 (x86_64)" { + __releaseImportedPackage slavery x86_64 "$TMP/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz" "$TMP/rsyncd/archlinux/pool/packages" + __releaseImportedPackage pkg-simple-a x86_64 "$TMP/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz" "$TMP/rsyncd/archlinux/pool/packages" + + DBIMPORT_CONFIG="${TMP}/db-import-archlinux.local.conf" db-import-pkg packages + + __isLinkTo "$TMP/ftp/core/os/x86_64/pkg-simple-a-1-1-x86_64.pkg.tar.xz" "$TMP/ftp/pool/packages/pkg-simple-a-1-1-x86_64.pkg.tar.xz" + __doesNotExist "$TMP"/ftp/{core/os/x86_64,pool/packages,sources/packages}/slavery-* + [[ "$(stat -c '%a' -- "$TMP/ftp/core/os/x86_64/core.db.tar.gz")" = 664 ]] +} + +@test "imports .db files as 0664 (i686)" { + __releaseImportedPackage slavery i686 "$TMP/rsyncd/archlinux32/i686/core/core.db.tar.gz" + __releaseImportedPackage pkg-simple-a i686 "$TMP/rsyncd/archlinux32/i686/core/core.db.tar.gz" + + DBIMPORT_CONFIG="${TMP}/db-import-archlinux32.local.conf" db-import-pkg archlinux32 + + __isLinkTo "$TMP/ftp/core/os/i686/pkg-simple-a-1-1-i686.pkg.tar.xz" "$TMP/ftp/pool/archlinux32/pkg-simple-a-1-1-i686.pkg.tar.xz" + __doesNotExist "$TMP"/ftp/{core/os/i686,pool/archlinux32,sources/archlinux32}/slavery-* + stat -- "$TMP/ftp/core/os/i686/core.db.tar.gz" + [[ "$(stat -c '%a' -- "$TMP/ftp/core/os/i686/core.db.tar.gz")" = 664 ]] +} -- cgit v1.2.2 From 6bdc99f7a02f4506243f59da44dd81d9159e9fd2 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Fri, 1 Jun 2018 17:25:29 -0400 Subject: test: db-import-pkg: Improve failure output just a bit --- test/cases/db-import-pkg.bats | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/cases/db-import-pkg.bats b/test/cases/db-import-pkg.bats index d038031..7c8e0af 100644 --- a/test/cases/db-import-pkg.bats +++ b/test/cases/db-import-pkg.bats @@ -144,7 +144,10 @@ __isLinkTo() { __doesNotExist() { local file for file in "$@"; do - ! stat "$file" + if stat "$file" 2>/dev/null; then + echo "TEST ERROR: File shouldn't exist, but does: $file" + return 1 + fi done } -- cgit v1.2.2 From 0e430a56a24064d2556f9637c321adbecf680927 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Thu, 10 May 2018 12:42:46 -0400 Subject: (Post-merge) db-import-pkg: Don't bother with hardlinking prior to sync_repo When implementing 8393137cd49de14f0f74caee8e93fa43ca881e2b, I had thought that being able to create a hardlink-shadow of the repo was necessary to avoid huge amounts of extra network traffic. Creating hardlinks required TMPDIR to be on the same filesystem as FTP_BASE. However, I now realize that sync_repo *only* syncs symlinks, which are cheap, network-wise. So, partially revert "db-import-pkg: Don't do any work directly in $FTP_BASE/$repo or $WORKDIR/rsync" Revert the bit that created hardlinks, and revert the changes to TMPDIR. (cherry picked from commit 5189a00cd7e3cc415f13fb8aa867fd7c8c6711fa) --- db-import-pkg | 2 -- systemd/db-import@.service | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/db-import-pkg b/db-import-pkg index 3591419..be21610 100755 --- a/db-import-pkg +++ b/db-import-pkg @@ -335,8 +335,6 @@ main() { whitelists+=("${WORKDIR}/${reponame}-${_arch}.whitelist") # Get repo packages (symlinks) mkdir "${WORKDIR}/staging-rsync/${_tag}" - ln -t "${WORKDIR}/staging-rsync/${_tag}" \ - "${FTP_BASE}/${reponame}/os/${_arch}"/* sync_repo \ "${ARCHMIRROR_fullmodule}/$(get_repo_dir "${reponame}" "${_arch}")/" \ "${WORKDIR}/${reponame}-${_arch}.whitelist" \ diff --git a/systemd/db-import@.service b/systemd/db-import@.service index f8d0679..884d4af 100644 --- a/systemd/db-import@.service +++ b/systemd/db-import@.service @@ -8,3 +8,5 @@ Type=oneshot User=repo Environment=DBSCRIPTS_CONFIG=/etc/dbscripts/config.local.%I ExecStart=/usr/bin/db-import-pkg %I + +PrivateTmp=true -- cgit v1.2.2 From b24bbcba5fe63d04bbfed531232d64b176071e7b Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Fri, 1 Jun 2018 17:50:58 -0400 Subject: Post-merge --- test/cases/db-import-pkg.bats | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/test/cases/db-import-pkg.bats b/test/cases/db-import-pkg.bats index 7c8e0af..25c78f4 100644 --- a/test/cases/db-import-pkg.bats +++ b/test/cases/db-import-pkg.bats @@ -48,11 +48,11 @@ setup() { # Configure db-import to use that rsyncd server cat <<-eot >"${TMP}/db-import-archlinux.local.conf" ARCHTAGS=('core-x86_64') - ARCHMIRROR=rsync://localhost:${rsyncport@Q}/archlinux + ARCHMIRROR=rsync://localhost:${rsyncport@Q}/archlinux/"\${ARCHMIRROR#rsync://*/*/}" eot cat <<-eot >"${TMP}/db-import-archlinux32.local.conf" ARCHTAGS=('core-i686') - ARCHMIRROR=rsync://localhost:${rsyncport@Q}/archlinux32 + ARCHMIRROR=rsync://localhost:${rsyncport@Q}/archlinux32/"\${ARCHMIRROR#rsync://*/*/}" eot # Set up HTTP server @@ -71,6 +71,15 @@ setup() { cat <<-eot >"$XDG_CONFIG_HOME"/libretools/libretools.conf BLACKLIST=http://localhost:${httpport@Q}/blacklist.txt eot + + # Set up repo contents + mkdir -p -- "${TMP}/ftp/core/os"/{x86_64,i686} + touch -- "${TMP}"/ftp/core/os/{x86_64,i686}/core.db.tar.gz + ln -s core.db.tar.gz "${TMP}/ftp/core/os/x86_64/core.db" + ln -s core.db.tar.gz "${TMP}/ftp/core/os/i686/core.db" + mkdir -p -- "${TMP}/ftp"/{pool,sources}/{packages,community,archlinux32} + date +%s > "${TMP}/ftp/lastupdate" + date +%s > "${TMP}/ftp/lastsync" } eval "__common_$(declare -f teardown)" teardown() { -- cgit v1.2.2 From 529789c5e2154007f2ca143fc40b1c5308251bb0 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Fri, 1 Jun 2018 18:15:21 -0400 Subject: test: db-import-pkg: Verify that it obeys config:TMPDIR --- test/cases/db-import-pkg.bats | 39 ++++++++++++++++++++++++++++++++------- 1 file changed, 32 insertions(+), 7 deletions(-) diff --git a/test/cases/db-import-pkg.bats b/test/cases/db-import-pkg.bats index 7c8e0af..2e04a75 100644 --- a/test/cases/db-import-pkg.bats +++ b/test/cases/db-import-pkg.bats @@ -81,6 +81,31 @@ teardown() { ###################################################################### +# Run the command in a new mount namespace with /tmp remounted +# read-only, but with $TMP (which might be under /tmp) still writable. +# +# Arguments are passed as arguments to `sudo`. +__withRoTmp() { + local mount="mount -o bind ${TMP@Q}{,} && mount -o bind,remount,ro /tmp{,}" + local env=( + "DBIMPORT_CONFIG=${DBIMPORT_CONFIG}" + "DBSCRIPTS_CONFIG=${DBSCRIPTS_CONFIG}" + "XDG_CONFIG_HOME=${XDG_CONFIG_HOME}" + ) + sudo -- unshare -m -- sh -c "${mount} && sudo -u ${USER@Q} ${env[*]@Q} \$@" -- "$@" +} + +__db-import-pkg() { + # Since common.bash->config.local sets TMPDIR=${TMP}/tmp, + # TMPDIR is necessarily != /tmp. + # Which means that if we try to write anything directly under /tmp, + # then we are erroneously disregarding TMPDIR. + # So, make /tmp read-only to make that be an error. + __withRoTmp db-import-pkg "$@" + # Verify that it cleaned up after itself and TMPDIR is empty + find "$TMPDIR" -mindepth 1 | diff - /dev/null +} + # releaseImportedPackage PKGBASE ARCH DBFILE [POOLDIR] # # This is different from common.bash:releasePackage because @@ -157,7 +182,7 @@ __doesNotExist() { __releaseImportedPackage slavery x86_64 "$TMP/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz" "$TMP/rsyncd/archlinux/pool/packages" __releaseImportedPackage pkg-simple-a x86_64 "$TMP/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz" "$TMP/rsyncd/archlinux/pool/packages" - DBIMPORT_CONFIG="${TMP}/db-import-archlinux.local.conf" db-import-pkg packages + DBIMPORT_CONFIG="${TMP}/db-import-archlinux.local.conf" __db-import-pkg packages __isLinkTo "$TMP/ftp/core/os/x86_64/pkg-simple-a-1-1-x86_64.pkg.tar.xz" "$TMP/ftp/pool/packages/pkg-simple-a-1-1-x86_64.pkg.tar.xz" __doesNotExist "$TMP"/ftp/{core/os/x86_64,pool/packages,sources/packages}/slavery-* @@ -167,7 +192,7 @@ __doesNotExist() { __releaseImportedPackage slavery i686 "$TMP/rsyncd/archlinux32/i686/core/core.db.tar.gz" __releaseImportedPackage pkg-simple-a i686 "$TMP/rsyncd/archlinux32/i686/core/core.db.tar.gz" - DBIMPORT_CONFIG="${TMP}/db-import-archlinux32.local.conf" db-import-pkg archlinux32 + DBIMPORT_CONFIG="${TMP}/db-import-archlinux32.local.conf" __db-import-pkg archlinux32 __isLinkTo "$TMP/ftp/core/os/i686/pkg-simple-a-1-1-i686.pkg.tar.xz" "$TMP/ftp/pool/archlinux32/pkg-simple-a-1-1-i686.pkg.tar.xz" __doesNotExist "$TMP"/ftp/{core/os/i686,pool/archlinux32,sources/archlinux32}/slavery-* @@ -176,7 +201,7 @@ __doesNotExist() { @test "imports DBs with no blacklisted packages" { __releaseImportedPackage pkg-simple-a x86_64 "$TMP/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz" "$TMP/rsyncd/archlinux/pool/packages" - DBIMPORT_CONFIG="${TMP}/db-import-archlinux.local.conf" db-import-pkg packages + DBIMPORT_CONFIG="${TMP}/db-import-archlinux.local.conf" __db-import-pkg packages __isLinkTo "$TMP/ftp/core/os/x86_64/pkg-simple-a-1-1-x86_64.pkg.tar.xz" "$TMP/ftp/pool/packages/pkg-simple-a-1-1-x86_64.pkg.tar.xz" } @@ -185,14 +210,14 @@ __doesNotExist() { __releaseImportedPackage slavery x86_64 "$TMP/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz" "$TMP/rsyncd/archlinux/pool/packages" __releaseImportedPackage pkg-simple-a x86_64 "$TMP/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz" "$TMP/rsyncd/archlinux/pool/packages" - DBIMPORT_CONFIG="${TMP}/db-import-archlinux.local.conf" db-import-pkg packages + DBIMPORT_CONFIG="${TMP}/db-import-archlinux.local.conf" __db-import-pkg packages __isLinkTo "$TMP/ftp/core/os/x86_64/pkg-simple-a-1-1-x86_64.pkg.tar.xz" "$TMP/ftp/pool/packages/pkg-simple-a-1-1-x86_64.pkg.tar.xz" __updateImportedPackage pkg-simple-a __releaseImportedPackage pkg-simple-a x86_64 "$TMP/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz" "$TMP/rsyncd/archlinux/pool/packages" - DBIMPORT_CONFIG="${TMP}/db-import-archlinux.local.conf" db-import-pkg packages + DBIMPORT_CONFIG="${TMP}/db-import-archlinux.local.conf" __db-import-pkg packages __isLinkTo "$TMP/ftp/core/os/x86_64/pkg-simple-a-1-2-x86_64.pkg.tar.xz" "$TMP/ftp/pool/packages/pkg-simple-a-1-2-x86_64.pkg.tar.xz" } @@ -201,7 +226,7 @@ __doesNotExist() { __releaseImportedPackage slavery x86_64 "$TMP/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz" "$TMP/rsyncd/archlinux/pool/packages" __releaseImportedPackage pkg-simple-a x86_64 "$TMP/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz" "$TMP/rsyncd/archlinux/pool/packages" - DBIMPORT_CONFIG="${TMP}/db-import-archlinux.local.conf" db-import-pkg packages + DBIMPORT_CONFIG="${TMP}/db-import-archlinux.local.conf" __db-import-pkg packages __isLinkTo "$TMP/ftp/core/os/x86_64/pkg-simple-a-1-1-x86_64.pkg.tar.xz" "$TMP/ftp/pool/packages/pkg-simple-a-1-1-x86_64.pkg.tar.xz" __doesNotExist "$TMP"/ftp/{core/os/x86_64,pool/packages,sources/packages}/slavery-* @@ -212,7 +237,7 @@ __doesNotExist() { __releaseImportedPackage slavery i686 "$TMP/rsyncd/archlinux32/i686/core/core.db.tar.gz" __releaseImportedPackage pkg-simple-a i686 "$TMP/rsyncd/archlinux32/i686/core/core.db.tar.gz" - DBIMPORT_CONFIG="${TMP}/db-import-archlinux32.local.conf" db-import-pkg archlinux32 + DBIMPORT_CONFIG="${TMP}/db-import-archlinux32.local.conf" __db-import-pkg archlinux32 __isLinkTo "$TMP/ftp/core/os/i686/pkg-simple-a-1-1-i686.pkg.tar.xz" "$TMP/ftp/pool/archlinux32/pkg-simple-a-1-1-i686.pkg.tar.xz" __doesNotExist "$TMP"/ftp/{core/os/i686,pool/archlinux32,sources/archlinux32}/slavery-* -- cgit v1.2.2 From 55ab486759719d83d79a0b19cd26a02aeb6beade Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Sat, 9 Jun 2018 22:51:22 -0400 Subject: test: db-import-pkg: Verify that pkgpool selection errors are caught --- test/cases/db-import-pkg.bats | 39 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/test/cases/db-import-pkg.bats b/test/cases/db-import-pkg.bats index 7a83ce2..93aaac7 100644 --- a/test/cases/db-import-pkg.bats +++ b/test/cases/db-import-pkg.bats @@ -105,14 +105,16 @@ __withRoTmp() { } __db-import-pkg() { + local ret=0 # Since common.bash->config.local sets TMPDIR=${TMP}/tmp, # TMPDIR is necessarily != /tmp. # Which means that if we try to write anything directly under /tmp, # then we are erroneously disregarding TMPDIR. # So, make /tmp read-only to make that be an error. - __withRoTmp db-import-pkg "$@" + __withRoTmp db-import-pkg "$@" || ret=$? # Verify that it cleaned up after itself and TMPDIR is empty find "$TMPDIR" -mindepth 1 | diff - /dev/null + return $ret } # releaseImportedPackage PKGBASE ARCH DBFILE [POOLDIR] @@ -253,3 +255,38 @@ __doesNotExist() { stat -- "$TMP/ftp/core/os/i686/core.db.tar.gz" [[ "$(stat -c '%a' -- "$TMP/ftp/core/os/i686/core.db.tar.gz")" = 664 ]] } + +@test "imports fully-masked upstream" { + __releaseImportedPackage pkg-any-a x86_64 "$TMP/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz" "$TMP/rsyncd/archlinux/pool/packages" + __releaseImportedPackage pkg-any-a i686 "$TMP/rsyncd/archlinux32/i686/core/core.db.tar.gz" + + DBIMPORT_CONFIG="${TMP}/db-import-archlinux.local.conf" __db-import-pkg packages + DBIMPORT_CONFIG="${TMP}/db-import-archlinux32.local.conf" __db-import-pkg archlinux32 + + __isLinkTo "$TMP/ftp/core/os/x86_64/pkg-any-a-1-1-any.pkg.tar.xz" "$TMP/ftp/pool/packages/pkg-any-a-1-1-any.pkg.tar.xz" + __isLinkTo "$TMP/ftp/core/os/i686/pkg-any-a-1-1-any.pkg.tar.xz" "$TMP/ftp/pool/packages/pkg-any-a-1-1-any.pkg.tar.xz" +} + +@test "import errors on pkgpool selection failures" { + # pkg-simple-a is just to make sure that the "fully-masked + # upstream" bug isn't being tested here + __releaseImportedPackage pkg-any-a x86_64 "$TMP/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz" "$TMP/rsyncd/archlinux/pool/packages" + __releaseImportedPackage pkg-simple-a x86_64 "$TMP/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz" "$TMP/rsyncd/archlinux/pool/packages" + __updateImportedPackage pkg-any-a + __releaseImportedPackage pkg-any-a x86_64 "$TMP/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz" "$TMP/rsyncd/archlinux/pool/packages" + DBIMPORT_CONFIG="${TMP}/db-import-archlinux.local.conf" __db-import-pkg packages + __isLinkTo "$TMP/ftp/core/os/x86_64/pkg-any-a-1-2-any.pkg.tar.xz" "$TMP/ftp/pool/packages/pkg-any-a-1-2-any.pkg.tar.xz" + + # This assumes that a package nested too deelply under /pool/ + # is filtered from being downloaded, but isn't found when + # poolifying. + mkdir -- "$TMP/ftp/pool/nested" + mv -T -- "$TMP/ftp/pool/packages" "$TMP/ftp/pool/nested/packages" + __releaseImportedPackage pkg-any-a i686 "$TMP/rsyncd/archlinux32/i686/core/core.db.tar.gz" + __releaseImportedPackage pkg-simple-a i686 "$TMP/rsyncd/archlinux32/i686/core/core.db.tar.gz" + + local status=0 + DBIMPORT_CONFIG="${TMP}/db-import-archlinux32.local.conf" __db-import-pkg archlinux32 || status=$? + [[ $status != 0 ]] + __doesNotExist "$TMP/ftp/core/os/i686/pkg-any-a-1-2-any.pkg.tar.xz" +} -- cgit v1.2.2 From e81c529584a2d10b80f50d7f52b0727e2a494d55 Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Sun, 10 Jun 2018 11:50:33 -0400 Subject: test: db-import-pkg: Add a case for asp on i686 --- .gitignore | 1 + test/cases/db-import-pkg.bats | 12 ++++++++++++ test/fixtures/pkg-any32/PKGBUILD | 13 +++++++++++++ test/fixtures/pkg-any64/PKGBUILD | 13 +++++++++++++ 4 files changed, 39 insertions(+) create mode 100644 test/fixtures/pkg-any32/PKGBUILD create mode 100644 test/fixtures/pkg-any64/PKGBUILD diff --git a/.gitignore b/.gitignore index c354a4a..d29e370 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ yftime src* pkg* +!/test/fixtures/pkg-*/ diff --git a/test/cases/db-import-pkg.bats b/test/cases/db-import-pkg.bats index 93aaac7..da5ecce 100644 --- a/test/cases/db-import-pkg.bats +++ b/test/cases/db-import-pkg.bats @@ -290,3 +290,15 @@ __doesNotExist() { [[ $status != 0 ]] __doesNotExist "$TMP/ftp/core/os/i686/pkg-any-a-1-2-any.pkg.tar.xz" } + +@test "imports arch=any packages with sub-pkgrel" { + # This is modeled after the situation with 'asp' and 'asp32' + + __releaseImportedPackage pkg-any64 x86_64 "$TMP/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz" "$TMP/rsyncd/archlinux/pool/packages" + DBIMPORT_CONFIG="${TMP}/db-import-archlinux.local.conf" __db-import-pkg packages + __isLinkTo "$TMP/ftp/core/os/x86_64/pkg-any-2-1-any.pkg.tar.xz" "$TMP/ftp/pool/packages/pkg-any-2-1-any.pkg.tar.xz" + + __releaseImportedPackage pkg-any32 i686 "$TMP/rsyncd/archlinux32/i686/core/core.db.tar.gz" + DBIMPORT_CONFIG="${TMP}/db-import-archlinux32.local.conf" __db-import-pkg archlinux32 + __isLinkTo "$TMP/ftp/core/os/i686/pkg-any-1-1.2-any.pkg.tar.xz" "$TMP/ftp/pool/archlinux32/pkg-any-1-1.2-any.pkg.tar.xz" +} diff --git a/test/fixtures/pkg-any32/PKGBUILD b/test/fixtures/pkg-any32/PKGBUILD new file mode 100644 index 0000000..e0421da --- /dev/null +++ b/test/fixtures/pkg-any32/PKGBUILD @@ -0,0 +1,13 @@ +pkgname=pkg-any +pkgver=1 +pkgrel=1.2 +pkgdesc="A package called ${pkgname} (32)" +arch=('any') +url='http://www.archlinux.org/' +license=('GPL') +options=(!strip) + +package() { + install -d -m755 "${pkgdir}"/usr/share/${pkgname} + echo 'test' > "${pkgdir}"/usr/share/${pkgname}/test +} diff --git a/test/fixtures/pkg-any64/PKGBUILD b/test/fixtures/pkg-any64/PKGBUILD new file mode 100644 index 0000000..1290ebd --- /dev/null +++ b/test/fixtures/pkg-any64/PKGBUILD @@ -0,0 +1,13 @@ +pkgname=pkg-any +pkgver=2 +pkgrel=1 +pkgdesc="A package called ${pkgname} (64)" +arch=('any') +url='http://www.archlinux.org/' +license=('GPL') +options=(!strip) + +package() { + install -d -m755 "${pkgdir}"/usr/share/${pkgname} + echo 'test' > "${pkgdir}"/usr/share/${pkgname}/test +} -- cgit v1.2.2