From 4c798bbe91c98149672f9088c4b120a8ca96e95a Mon Sep 17 00:00:00 2001 From: Judd Vinet Date: Sat, 16 Mar 2002 08:01:37 +0000 Subject: Added script genpkglist --- genpkglist | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100755 genpkglist diff --git a/genpkglist b/genpkglist new file mode 100755 index 0000000..0999592 --- /dev/null +++ b/genpkglist @@ -0,0 +1,15 @@ +#!/bin/bash + +tl=`pwd` + +for category in {base,opt,contrib}; do + echo $category >>packages.txt + for pkg in `ls $category`; do + cd $tl/$category/$pkg + . PKGBUILD + cd $tl + echo " $pkgname $pkgver" >>packages.txt + done + cd $tl + echo "" >>packages.txt +done -- cgit v1.2.2 From e322cffb28883e85593b6e458ccd2feae417b1b8 Mon Sep 17 00:00:00 2001 From: Judd Vinet Date: Tue, 19 Mar 2002 00:17:28 +0000 Subject: Updated packages --- genpkglist | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/genpkglist b/genpkglist index 0999592..5abe496 100755 --- a/genpkglist +++ b/genpkglist @@ -4,11 +4,13 @@ tl=`pwd` for category in {base,opt,contrib}; do echo $category >>packages.txt - for pkg in `ls $category`; do + for pkg in `ls $category | sort`; do cd $tl/$category/$pkg - . PKGBUILD - cd $tl - echo " $pkgname $pkgver" >>packages.txt + if [ -f PKGBUILD ]; then + . PKGBUILD + cd $tl + echo " $pkgname $pkgver" >>packages.txt + fi done cd $tl echo "" >>packages.txt -- cgit v1.2.2 From 660ea7e41b092ce286bcda9d4a6fcae105d98da7 Mon Sep 17 00:00:00 2001 From: Judd Vinet Date: Fri, 24 May 2002 18:08:43 +0000 Subject: Fixed pkglist generator scripts so they aren't hardcoded to use base,opt,contrib package categories anymore --- genpkglist | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/genpkglist b/genpkglist index 5abe496..21e18d0 100755 --- a/genpkglist +++ b/genpkglist @@ -2,7 +2,7 @@ tl=`pwd` -for category in {base,opt,contrib}; do +for category in `find * -type d -maxdepth 0 | grep -v CVS`; do echo $category >>packages.txt for pkg in `ls $category | sort`; do cd $tl/$category/$pkg -- cgit v1.2.2 From 3a035c17142e854afc355082cd5344821b27235e Mon Sep 17 00:00:00 2001 From: Judd Vinet Date: Wed, 26 Jun 2002 18:59:49 +0000 Subject: Added cvs tags to scripts Added pacsync_update to scripts Added ftpmaint to scripts --- genpkglist | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/genpkglist b/genpkglist index 21e18d0..94a9336 100755 --- a/genpkglist +++ b/genpkglist @@ -1,7 +1,18 @@ #!/bin/bash +# $Id: genpkglist,v 1.4 2002/06/26 18:59:49 judd Exp $ tl=`pwd` +cat >packages.txt <<_EOF + + + + + + + +_EOF + for category in `find * -type d -maxdepth 0 | grep -v CVS`; do echo $category >>packages.txt for pkg in `ls $category | sort`; do -- cgit v1.2.2 From 8b3ecd497c4f514031e768913a345019bc763899 Mon Sep 17 00:00:00 2001 From: Judd Vinet Date: Wed, 21 Aug 2002 18:13:14 +0000 Subject: added new db generation scripts --- db-arch | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100755 db-arch diff --git a/db-arch b/db-arch new file mode 100755 index 0000000..ab2a700 --- /dev/null +++ b/db-arch @@ -0,0 +1,23 @@ +#!/bin/bash +# $Id: db-arch,v 1.1 2002/08/21 18:13:14 judd Exp $ + +echo "==> Updating ABS Tree..." >&2 +cd /usr +CVSROOT=:ext:judd@archlinux.org:/home/cvs-arch cvs -q export -r CURRENT -f arch/build +rm -rf /usr/abs && mv arch/build /usr/abs +rmdir arch + +echo "==> Rebuilding HTML Package List..." >&2 +cd /usr/abs +rm -f packages.txt +/root/bin/htmlpkglist +mv pkgs_name.html pkgs_cat.html /home/httpd/html/ + +echo "==> Generating pacsync1/setup Database..." >&2 +/root/bin/gensyncdb + +cp /home/ftp/current/pacsync/pacsync.db /home/ftp/current/setup/packages.txt + +echo "==> Generating pacsync2 Database for ARCH..." >&2 +/usr/bin/gensync /usr/abs /tmp/current.db.tar.gz +[ -f /tmp/current.db.tar.gz ] && mv -f /tmp/current.db.tar.gz /home/ftp/current -- cgit v1.2.2 From 39101b32c119eb1508ca76ae30b7f36cd7d781fe Mon Sep 17 00:00:00 2001 From: Judd Vinet Date: Thu, 28 Nov 2002 08:28:13 +0000 Subject: removed old gensyncdb script (used for pacsync) replaced old genpkglist with a new one --- genpkglist | 46 ++++++++++++++++++++++++---------------------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/genpkglist b/genpkglist index 94a9336..67f384e 100755 --- a/genpkglist +++ b/genpkglist @@ -1,28 +1,30 @@ #!/bin/bash -# $Id: genpkglist,v 1.4 2002/06/26 18:59:49 judd Exp $ +# $Id: genpkglist,v 1.5 2002/11/28 08:28:13 judd Exp $ -tl=`pwd` +# +# genpkglist +# +# Generates a text package database for use with the setup script +# -cat >packages.txt <<_EOF -
PackageVersionCollectionDescription
- - - - - - -_EOF +pkgfile="`pwd`/packages.txt" +cd /usr/abs +rm -f $pkgfile for category in `find * -type d -maxdepth 0 | grep -v CVS`; do - echo $category >>packages.txt - for pkg in `ls $category | sort`; do - cd $tl/$category/$pkg - if [ -f PKGBUILD ]; then - . PKGBUILD - cd $tl - echo " $pkgname $pkgver" >>packages.txt - fi - done - cd $tl - echo "" >>packages.txt + cd $category + for pkg in `/bin/ls`; do + cd $pkg + if [ -f PKGBUILD ]; then + . PKGBUILD + if [ -f /home/ftp/current/$pkgname-$pkgver-$pkgrel.pkg.tar.gz ]; then + echo "$category/$pkgname-$pkgver-$pkgrel.pkg.tar.gz" >>$pkgfile + else + echo "notice: Missing $pkgname-$pkgver-$pkgrel.pkg.tar.gz in ftp site" >&2 + fi + fi + cd .. + done + cd .. done + -- cgit v1.2.2 From eff770d4ca0c3d15cc4360b7fbb7b9e867271f7d Mon Sep 17 00:00:00 2001 From: Judd Vinet Date: Thu, 28 Nov 2002 08:33:44 +0000 Subject: updated paths in db-arch script --- db-arch | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/db-arch b/db-arch index ab2a700..e8114c8 100755 --- a/db-arch +++ b/db-arch @@ -1,9 +1,9 @@ #!/bin/bash -# $Id: db-arch,v 1.1 2002/08/21 18:13:14 judd Exp $ +# $Id: db-arch,v 1.2 2002/11/28 08:33:44 judd Exp $ echo "==> Updating ABS Tree..." >&2 cd /usr -CVSROOT=:ext:judd@archlinux.org:/home/cvs-arch cvs -q export -r CURRENT -f arch/build +CVSROOT=:ext:judd@cvs.archlinux.org:/home/cvs-arch cvs -q export -r CURRENT -f arch/build rm -rf /usr/abs && mv arch/build /usr/abs rmdir arch @@ -11,12 +11,11 @@ echo "==> Rebuilding HTML Package List..." >&2 cd /usr/abs rm -f packages.txt /root/bin/htmlpkglist -mv pkgs_name.html pkgs_cat.html /home/httpd/html/ +mv pkgs_name.html pkgs_cat.html /home/sites/archlinux/htdocs/ -echo "==> Generating pacsync1/setup Database..." >&2 -/root/bin/gensyncdb - -cp /home/ftp/current/pacsync/pacsync.db /home/ftp/current/setup/packages.txt +echo "==> Generating Text Package List..." >&2 +/root/bin/genpkglist +mv packages.txt /home/ftp/current/setup/packages.txt echo "==> Generating pacsync2 Database for ARCH..." >&2 /usr/bin/gensync /usr/abs /tmp/current.db.tar.gz -- cgit v1.2.2 From 9e5739be85cbfa5b8b1acb7543e03e1aed1cd490 Mon Sep 17 00:00:00 2001 From: Judd Vinet Date: Thu, 19 Dec 2002 19:09:28 +0000 Subject: added generalized package list generation so we can use it with unofficial --- db-arch | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/db-arch b/db-arch index e8114c8..b34ef23 100755 --- a/db-arch +++ b/db-arch @@ -1,5 +1,5 @@ #!/bin/bash -# $Id: db-arch,v 1.2 2002/11/28 08:33:44 judd Exp $ +# $Id: db-arch,v 1.3 2002/12/19 19:09:28 judd Exp $ echo "==> Updating ABS Tree..." >&2 cd /usr @@ -10,8 +10,8 @@ rmdir arch echo "==> Rebuilding HTML Package List..." >&2 cd /usr/abs rm -f packages.txt -/root/bin/htmlpkglist -mv pkgs_name.html pkgs_cat.html /home/sites/archlinux/htdocs/ +/root/bin/htmlpkglist arch +mv pkgs_arch_name.html pkgs_arch_cat.html /home/sites/archlinux/htdocs/pkglists echo "==> Generating Text Package List..." >&2 /root/bin/genpkglist -- cgit v1.2.2 From e7b97f41b420767f651912c967898411fcde6b92 Mon Sep 17 00:00:00 2001 From: Judd Vinet Date: Fri, 24 Jan 2003 19:27:35 +0000 Subject: Updated to use the /tmp area instead of /usr/abs --- db-arch | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/db-arch b/db-arch index b34ef23..95f5d71 100755 --- a/db-arch +++ b/db-arch @@ -1,14 +1,12 @@ #!/bin/bash -# $Id: db-arch,v 1.3 2002/12/19 19:09:28 judd Exp $ +# $Id: db-arch,v 1.4 2003/01/24 19:27:35 judd Exp $ echo "==> Updating ABS Tree..." >&2 -cd /usr +cd /tmp CVSROOT=:ext:judd@cvs.archlinux.org:/home/cvs-arch cvs -q export -r CURRENT -f arch/build -rm -rf /usr/abs && mv arch/build /usr/abs -rmdir arch echo "==> Rebuilding HTML Package List..." >&2 -cd /usr/abs +cd /tmp/arch/build rm -f packages.txt /root/bin/htmlpkglist arch mv pkgs_arch_name.html pkgs_arch_cat.html /home/sites/archlinux/htdocs/pkglists @@ -18,5 +16,7 @@ echo "==> Generating Text Package List..." >&2 mv packages.txt /home/ftp/current/setup/packages.txt echo "==> Generating pacsync2 Database for ARCH..." >&2 -/usr/bin/gensync /usr/abs /tmp/current.db.tar.gz +/usr/bin/gensync /tmp/arch/build /tmp/current.db.tar.gz [ -f /tmp/current.db.tar.gz ] && mv -f /tmp/current.db.tar.gz /home/ftp/current + +rm -rf /tmp/arch -- cgit v1.2.2 From dc73d5af70bbcd1cfa8515e34478c3eb33d15749 Mon Sep 17 00:00:00 2001 From: Judd Vinet Date: Mon, 24 Feb 2003 00:54:29 +0000 Subject: added db-unstable script --- db-unstable | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100755 db-unstable diff --git a/db-unstable b/db-unstable new file mode 100755 index 0000000..7e3431f --- /dev/null +++ b/db-unstable @@ -0,0 +1,19 @@ +#!/bin/bash +# $Id: db-unstable,v 1.1 2003/02/24 00:54:29 judd Exp $ + +echo "==> Generating pacsync2 Database for UNSTABLE..." >&2 +cd /tmp +CVS_RSH=ssh CVSROOT=:ext:cvs.archlinux.org:/home/cvs-unstable cvs -q export -r CURRENT unstable +/usr/bin/gensync /tmp/unstable /tmp/unstable.db.tar.gz +[ -f /tmp/unstable.db.tar.gz ] && mv -f /tmp/unstable.db.tar.gz /home/ftp/unstable + +echo "==> Rebuilding HTML Package List..." >&2 +cd /tmp/unstable +rm -f packages.txt +$HOME/htmlpkglist unstable +cp -f pkgs_unstable_name.html /home/sites/archlinux/htdocs/pkglists +cp -f pkgs_unstable_cat.html /home/sites/archlinux/htdocs/pkglists +rm pkgs_unstable_name.html pkgs_unstable_cat.html + +rm -rf /tmp/unstable + -- cgit v1.2.2 From c6a78c714e6dc0a814d954eb55ec0028e897f1f9 Mon Sep 17 00:00:00 2001 From: Judd Vinet Date: Wed, 21 May 2003 23:07:33 +0000 Subject: added/updated scripts for the new website --- db-arch | 8 ++++--- db-unstable | 5 +++-- genpkglist | 4 ++-- pkgdb | 70 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 80 insertions(+), 7 deletions(-) create mode 100755 pkgdb diff --git a/db-arch b/db-arch index 95f5d71..c6a7339 100755 --- a/db-arch +++ b/db-arch @@ -1,18 +1,20 @@ #!/bin/bash -# $Id: db-arch,v 1.4 2003/01/24 19:27:35 judd Exp $ +# $Id: db-arch,v 1.5 2003/05/21 23:07:33 judd Exp $ echo "==> Updating ABS Tree..." >&2 cd /tmp +rm -rf arch CVSROOT=:ext:judd@cvs.archlinux.org:/home/cvs-arch cvs -q export -r CURRENT -f arch/build echo "==> Rebuilding HTML Package List..." >&2 cd /tmp/arch/build rm -f packages.txt -/root/bin/htmlpkglist arch +/arch/htmlpkglist arch mv pkgs_arch_name.html pkgs_arch_cat.html /home/sites/archlinux/htdocs/pkglists +/arch/pkgdb 1 echo "==> Generating Text Package List..." >&2 -/root/bin/genpkglist +/arch/genpkglist mv packages.txt /home/ftp/current/setup/packages.txt echo "==> Generating pacsync2 Database for ARCH..." >&2 diff --git a/db-unstable b/db-unstable index 7e3431f..4d76d9e 100755 --- a/db-unstable +++ b/db-unstable @@ -1,5 +1,5 @@ #!/bin/bash -# $Id: db-unstable,v 1.1 2003/02/24 00:54:29 judd Exp $ +# $Id: db-unstable,v 1.2 2003/05/21 23:07:33 judd Exp $ echo "==> Generating pacsync2 Database for UNSTABLE..." >&2 cd /tmp @@ -10,10 +10,11 @@ CVS_RSH=ssh CVSROOT=:ext:cvs.archlinux.org:/home/cvs-unstable cvs -q export -r C echo "==> Rebuilding HTML Package List..." >&2 cd /tmp/unstable rm -f packages.txt -$HOME/htmlpkglist unstable +/arch/htmlpkglist unstable cp -f pkgs_unstable_name.html /home/sites/archlinux/htdocs/pkglists cp -f pkgs_unstable_cat.html /home/sites/archlinux/htdocs/pkglists rm pkgs_unstable_name.html pkgs_unstable_cat.html +/arch/pkgdb 3 rm -rf /tmp/unstable diff --git a/genpkglist b/genpkglist index 67f384e..ee4298b 100755 --- a/genpkglist +++ b/genpkglist @@ -1,5 +1,5 @@ #!/bin/bash -# $Id: genpkglist,v 1.5 2002/11/28 08:28:13 judd Exp $ +# $Id: genpkglist,v 1.6 2003/05/21 23:07:33 judd Exp $ # # genpkglist @@ -9,7 +9,7 @@ pkgfile="`pwd`/packages.txt" -cd /usr/abs +cd /tmp/arch/build rm -f $pkgfile for category in `find * -type d -maxdepth 0 | grep -v CVS`; do cd $category diff --git a/pkgdb b/pkgdb new file mode 100755 index 0000000..b063da7 --- /dev/null +++ b/pkgdb @@ -0,0 +1,70 @@ +#!/bin/bash +# $Id: pkgdb,v 1.1 2003/05/21 23:07:33 judd Exp $ + +tl=`pwd` +TMPFILE=/tmp/.mysqltmp +repoid=$1 +dbuser='archweb' +dbname='archweb' +dbpass='14xuk0Gi' + +if [ "$repoid" = "" ]; then + me=`basename $0` + echo "usage: $me " + exit 1 +fi + +echo "==> Updating Package Entries in Database..." + +for category in `find * -type d -maxdepth 0 | grep -v CVS | grep -v "^unofficial" | grep -v "^unstable"`; do + for pkg in `ls $category | sort`; do + cd $tl/$category/$pkg + if [ -f PKGBUILD ]; then + . PKGBUILD + deplist=${depends[@]} + deplist=`php -r "echo addslashes(\"$deplist\");"` + sources=${source[@]} + sources=`php -r "echo addslashes(\"$sources\");"` + pkgname=`php -r "echo addslashes(\"$pkgname\");"` + pkgver=`php -r "echo addslashes(\"$pkgver\");"` + pkgrel=`php -r "echo addslashes(\"$pkgrel\");"` + pkgdesc=`php -r "echo addslashes(\"$pkgdesc\");"` + mysql -u $dbuser -p$dbpass $dbname >$TMPFILE <<_EOF +SELECT id FROM packages WHERE repoid=$repoid AND pkgname='$pkgname'; +_EOF + id=`tail -1 $TMPFILE` + if [ "$id" != "" ]; then + mysql -u $dbuser -p$dbpass $dbname >$TMPFILE <<_EOF +SELECT pkgver,pkgrel FROM packages WHERE repoid=$repoid AND pkgname='$pkgname'; +_EOF + oldver=`tail -1 $TMPFILE | awk '{print $1}'` + oldrel=`tail -1 $TMPFILE | awk '{print $2}'` + if [ "$oldver" != "$pkgver" -o "$oldrel" != "$pkgrel" ]; then + mysql -u $dbuser -p$dbpass $dbname <<_EOF +UPDATE packages SET category='$category',pkgname='$pkgname',pkgver='$pkgver', +pkgrel='$pkgrel',pkgdesc='$pkgdesc',url='$url',sources='$sources', +depends='$deplist',lastupdate=NOW() WHERE id='$id'; +_EOF + else + mysql -u $dbuser -p$dbpass $dbname <<_EOF +UPDATE packages SET category='$category',pkgname='$pkgname',pkgver='$pkgver', +pkgrel='$pkgrel',pkgdesc='$pkgdesc',url='$url',sources='$sources', +depends='$deplist' WHERE id='$id'; +_EOF + + fi + else + mysql -u $dbuser -p$dbpass $dbname <<_EOF +INSERT INTO packages (id,repoid,category,pkgname,pkgver,pkgrel, +pkgdesc,url,sources,depends,lastupdate) VALUES (NULL,$repoid, +'$category','$pkgname','$pkgver','$pkgrel','$pkgdesc','$url', +'$sources','$deplist',NOW()); +_EOF + fi + fi + done + cd $tl +done + +rm -f $TMPFILE + -- cgit v1.2.2 From 9656502b826f7e7b22658a7091b30c73baeda958 Mon Sep 17 00:00:00 2001 From: Judd Vinet Date: Fri, 23 May 2003 05:08:57 +0000 Subject: removed calls to htmlpkglist (no longer needed) --- db-arch | 7 +------ db-unstable | 9 +-------- pkgdb | 23 ++++++++--------------- 3 files changed, 10 insertions(+), 29 deletions(-) diff --git a/db-arch b/db-arch index c6a7339..510637d 100755 --- a/db-arch +++ b/db-arch @@ -1,16 +1,11 @@ #!/bin/bash -# $Id: db-arch,v 1.5 2003/05/21 23:07:33 judd Exp $ +# $Id: db-arch,v 1.6 2003/05/23 05:08:57 judd Exp $ echo "==> Updating ABS Tree..." >&2 cd /tmp rm -rf arch CVSROOT=:ext:judd@cvs.archlinux.org:/home/cvs-arch cvs -q export -r CURRENT -f arch/build -echo "==> Rebuilding HTML Package List..." >&2 -cd /tmp/arch/build -rm -f packages.txt -/arch/htmlpkglist arch -mv pkgs_arch_name.html pkgs_arch_cat.html /home/sites/archlinux/htdocs/pkglists /arch/pkgdb 1 echo "==> Generating Text Package List..." >&2 diff --git a/db-unstable b/db-unstable index 4d76d9e..45c69c7 100755 --- a/db-unstable +++ b/db-unstable @@ -1,5 +1,5 @@ #!/bin/bash -# $Id: db-unstable,v 1.2 2003/05/21 23:07:33 judd Exp $ +# $Id: db-unstable,v 1.3 2003/05/23 05:08:57 judd Exp $ echo "==> Generating pacsync2 Database for UNSTABLE..." >&2 cd /tmp @@ -7,13 +7,6 @@ CVS_RSH=ssh CVSROOT=:ext:cvs.archlinux.org:/home/cvs-unstable cvs -q export -r C /usr/bin/gensync /tmp/unstable /tmp/unstable.db.tar.gz [ -f /tmp/unstable.db.tar.gz ] && mv -f /tmp/unstable.db.tar.gz /home/ftp/unstable -echo "==> Rebuilding HTML Package List..." >&2 -cd /tmp/unstable -rm -f packages.txt -/arch/htmlpkglist unstable -cp -f pkgs_unstable_name.html /home/sites/archlinux/htdocs/pkglists -cp -f pkgs_unstable_cat.html /home/sites/archlinux/htdocs/pkglists -rm pkgs_unstable_name.html pkgs_unstable_cat.html /arch/pkgdb 3 rm -rf /tmp/unstable diff --git a/pkgdb b/pkgdb index b063da7..b394e3a 100755 --- a/pkgdb +++ b/pkgdb @@ -1,5 +1,5 @@ #!/bin/bash -# $Id: pkgdb,v 1.1 2003/05/21 23:07:33 judd Exp $ +# $Id: pkgdb,v 1.2 2003/05/23 05:08:57 judd Exp $ tl=`pwd` TMPFILE=/tmp/.mysqltmp @@ -20,6 +20,8 @@ for category in `find * -type d -maxdepth 0 | grep -v CVS | grep -v "^unofficial for pkg in `ls $category | sort`; do cd $tl/$category/$pkg if [ -f PKGBUILD ]; then + unset depends source deplist sources + unset pkgname pkgver pkgrel pkgdesc url . PKGBUILD deplist=${depends[@]} deplist=`php -r "echo addslashes(\"$deplist\");"` @@ -29,29 +31,20 @@ for category in `find * -type d -maxdepth 0 | grep -v CVS | grep -v "^unofficial pkgver=`php -r "echo addslashes(\"$pkgver\");"` pkgrel=`php -r "echo addslashes(\"$pkgrel\");"` pkgdesc=`php -r "echo addslashes(\"$pkgdesc\");"` + url=`php -r "echo addslashes(\"$url\");"` mysql -u $dbuser -p$dbpass $dbname >$TMPFILE <<_EOF -SELECT id FROM packages WHERE repoid=$repoid AND pkgname='$pkgname'; +SELECT id,pkgver,pkgrel FROM packages WHERE repoid=$repoid AND pkgname='$pkgname'; _EOF - id=`tail -1 $TMPFILE` + id=`tail -1 $TMPFILE | awk '{print $1}'` if [ "$id" != "" ]; then - mysql -u $dbuser -p$dbpass $dbname >$TMPFILE <<_EOF -SELECT pkgver,pkgrel FROM packages WHERE repoid=$repoid AND pkgname='$pkgname'; -_EOF - oldver=`tail -1 $TMPFILE | awk '{print $1}'` - oldrel=`tail -1 $TMPFILE | awk '{print $2}'` + oldver=`tail -1 $TMPFILE | awk '{print $2}'` + oldrel=`tail -1 $TMPFILE | awk '{print $3}'` if [ "$oldver" != "$pkgver" -o "$oldrel" != "$pkgrel" ]; then mysql -u $dbuser -p$dbpass $dbname <<_EOF UPDATE packages SET category='$category',pkgname='$pkgname',pkgver='$pkgver', pkgrel='$pkgrel',pkgdesc='$pkgdesc',url='$url',sources='$sources', depends='$deplist',lastupdate=NOW() WHERE id='$id'; _EOF - else - mysql -u $dbuser -p$dbpass $dbname <<_EOF -UPDATE packages SET category='$category',pkgname='$pkgname',pkgver='$pkgver', -pkgrel='$pkgrel',pkgdesc='$pkgdesc',url='$url',sources='$sources', -depends='$deplist' WHERE id='$id'; -_EOF - fi else mysql -u $dbuser -p$dbpass $dbname <<_EOF -- cgit v1.2.2 From 3d078d5f874b5410267cc6beb73349ffeffce42c Mon Sep 17 00:00:00 2001 From: Judd Vinet Date: Sat, 24 May 2003 00:25:59 +0000 Subject: fixed bugs --- db-arch | 4 ++-- db-unstable | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/db-arch b/db-arch index 510637d..a6b9118 100755 --- a/db-arch +++ b/db-arch @@ -1,12 +1,12 @@ #!/bin/bash -# $Id: db-arch,v 1.6 2003/05/23 05:08:57 judd Exp $ +# $Id: db-arch,v 1.7 2003/05/24 00:25:59 judd Exp $ echo "==> Updating ABS Tree..." >&2 cd /tmp rm -rf arch CVSROOT=:ext:judd@cvs.archlinux.org:/home/cvs-arch cvs -q export -r CURRENT -f arch/build -/arch/pkgdb 1 +cd /tmp/arch/build && /arch/pkgdb 1 echo "==> Generating Text Package List..." >&2 /arch/genpkglist diff --git a/db-unstable b/db-unstable index 45c69c7..8fa0388 100755 --- a/db-unstable +++ b/db-unstable @@ -1,5 +1,5 @@ #!/bin/bash -# $Id: db-unstable,v 1.3 2003/05/23 05:08:57 judd Exp $ +# $Id: db-unstable,v 1.4 2003/05/24 00:25:59 judd Exp $ echo "==> Generating pacsync2 Database for UNSTABLE..." >&2 cd /tmp @@ -7,7 +7,7 @@ CVS_RSH=ssh CVSROOT=:ext:cvs.archlinux.org:/home/cvs-unstable cvs -q export -r C /usr/bin/gensync /tmp/unstable /tmp/unstable.db.tar.gz [ -f /tmp/unstable.db.tar.gz ] && mv -f /tmp/unstable.db.tar.gz /home/ftp/unstable -/arch/pkgdb 3 +cd /tmp/unstable && /arch/pkgdb 3 rm -rf /tmp/unstable -- cgit v1.2.2 From a1b1d65283834f9108e33b5a4e2945f8259f9d5a Mon Sep 17 00:00:00 2001 From: Judd Vinet Date: Sun, 25 May 2003 23:50:02 +0000 Subject: improved arch db scripts --- db-arch | 27 +++++++++++++++++---------- db-unstable | 24 +++++++++++++++++------- genpkglist | 3 +-- 3 files changed, 35 insertions(+), 19 deletions(-) diff --git a/db-arch b/db-arch index a6b9118..1e24f88 100755 --- a/db-arch +++ b/db-arch @@ -1,19 +1,26 @@ #!/bin/bash -# $Id: db-arch,v 1.7 2003/05/24 00:25:59 judd Exp $ +# $Id: db-arch,v 1.8 2003/05/25 23:50:02 judd Exp $ -echo "==> Updating ABS Tree..." >&2 -cd /tmp -rm -rf arch +uid=`id -u` +TMPDIR="/tmp/archpkg.$uid" + +# RedHat's mktemp is broken... +if [ -d /tmp/archpkg.$uid ]; then + echo "==> Removing old temp dir..." >&2 + rm -rf /tmp/archpkg.$uid || exit 1 +fi +mkdir /tmp/archpkg.$uid; [ $? -gt 0 ] && exit 1 + +echo "==> Generating Pacman Database for OFFICIAL..." >&2 +cd $TMPDIR CVSROOT=:ext:judd@cvs.archlinux.org:/home/cvs-arch cvs -q export -r CURRENT -f arch/build +/usr/bin/gensync $TMPDIR/arch/build $TMPDIR/current.db.tar.gz +[ -f $TMPDIR/current.db.tar.gz ] && mv -f $TMPDIR/current.db.tar.gz /home/ftp/current -cd /tmp/arch/build && /arch/pkgdb 1 +cd $TMPDIR/arch/build && /arch/pkgdb 1 echo "==> Generating Text Package List..." >&2 /arch/genpkglist mv packages.txt /home/ftp/current/setup/packages.txt -echo "==> Generating pacsync2 Database for ARCH..." >&2 -/usr/bin/gensync /tmp/arch/build /tmp/current.db.tar.gz -[ -f /tmp/current.db.tar.gz ] && mv -f /tmp/current.db.tar.gz /home/ftp/current - -rm -rf /tmp/arch +rm -rf $TMPDIR diff --git a/db-unstable b/db-unstable index 8fa0388..18554c0 100755 --- a/db-unstable +++ b/db-unstable @@ -1,13 +1,23 @@ #!/bin/bash -# $Id: db-unstable,v 1.4 2003/05/24 00:25:59 judd Exp $ +# $Id: db-unstable,v 1.5 2003/05/25 23:50:02 judd Exp $ -echo "==> Generating pacsync2 Database for UNSTABLE..." >&2 -cd /tmp +uid=`id -u` +TMPDIR="/tmp/archpkg.$uid" + +# RedHat's mktemp is broken... +if [ -d $TMPDIR ]; then + echo "==> Removing old temp dir..." >&2 + rm -rf $TMPDIR || exit 1 +fi +mkdir $TMPDIR; [ $? -gt 0 ] && exit 1 + +echo "==> Generating Pacman Database for UNSTABLE..." >&2 +cd $TMPDIR CVS_RSH=ssh CVSROOT=:ext:cvs.archlinux.org:/home/cvs-unstable cvs -q export -r CURRENT unstable -/usr/bin/gensync /tmp/unstable /tmp/unstable.db.tar.gz -[ -f /tmp/unstable.db.tar.gz ] && mv -f /tmp/unstable.db.tar.gz /home/ftp/unstable +/usr/bin/gensync $TMPDIR/unstable $TMPDIR/unstable.db.tar.gz +[ -f $TMPDIR/unstable.db.tar.gz ] && mv -f $TMPDIR/unstable.db.tar.gz /home/ftp/unstable -cd /tmp/unstable && /arch/pkgdb 3 +cd $TMPDIR/unstable && /arch/pkgdb 3 -rm -rf /tmp/unstable +rm -rf $TMPDIR diff --git a/genpkglist b/genpkglist index ee4298b..6af0bed 100755 --- a/genpkglist +++ b/genpkglist @@ -1,5 +1,5 @@ #!/bin/bash -# $Id: genpkglist,v 1.6 2003/05/21 23:07:33 judd Exp $ +# $Id: genpkglist,v 1.7 2003/05/25 23:50:02 judd Exp $ # # genpkglist @@ -9,7 +9,6 @@ pkgfile="`pwd`/packages.txt" -cd /tmp/arch/build rm -f $pkgfile for category in `find * -type d -maxdepth 0 | grep -v CVS`; do cd $category -- cgit v1.2.2 From 1131750e8972cde558e3d6dc451023f940e66044 Mon Sep 17 00:00:00 2001 From: Judd Vinet Date: Sun, 25 May 2003 23:51:25 +0000 Subject: improved package db handling to delete packages --- pkgdb | 132 +++++++++++++++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 102 insertions(+), 30 deletions(-) diff --git a/pkgdb b/pkgdb index b394e3a..1f71162 100755 --- a/pkgdb +++ b/pkgdb @@ -1,28 +1,80 @@ #!/bin/bash -# $Id: pkgdb,v 1.2 2003/05/23 05:08:57 judd Exp $ +# $Id: pkgdb,v 1.3 2003/05/25 23:51:25 judd Exp $ tl=`pwd` -TMPFILE=/tmp/.mysqltmp + +# we use this instead of mktemp cuz the RH server's mktemp is broken :( +uid=`id -u` +TMPFILE="/tmp/.mysqltmp.$uid" + repoid=$1 dbuser='archweb' dbname='archweb' dbpass='14xuk0Gi' +dbuser='arch' +dbname='arch' +dbpass='arch' + if [ "$repoid" = "" ]; then me=`basename $0` echo "usage: $me " exit 1 fi -echo "==> Updating Package Entries in Database..." +deletepkg() +{ + echo "Deleting $pkgname" + mysql -u $dbuser -p$dbpass $dbname <<_EOF +DELETE FROM packages WHERE id='$id'; +_EOF +} + +updatepkg() +{ + echo "Updating $pkgname" + mysql -u $dbuser -p$dbpass $dbname <<_EOF +UPDATE packages SET category='$category',pkgname='$pkgname',pkgver='$pkgver', +pkgrel='$pkgrel',pkgdesc='$pkgdesc',url='$url',sources='$sources', +depends='$deplist',lastupdate=NOW() WHERE id='$id'; +_EOF +} + +insertpkg() +{ + echo "Inserting $pkgname" + mysql -u $dbuser -p$dbpass $dbname <<_EOF +INSERT INTO packages (id,repoid,category,pkgname,pkgver,pkgrel, +pkgdesc,url,sources,depends,lastupdate) VALUES (NULL,$repoid, +'$category','$pkgname','$pkgver','$pkgrel','$pkgdesc','$url', +'$sources','$deplist',NOW()); +_EOF +} -for category in `find * -type d -maxdepth 0 | grep -v CVS | grep -v "^unofficial" | grep -v "^unstable"`; do - for pkg in `ls $category | sort`; do - cd $tl/$category/$pkg - if [ -f PKGBUILD ]; then +process() { + pkg=$1 + + id=`echo $pkg | awk '{print $1}'` + pkgname=`echo $pkg | awk '{print $2}'` + oldver=`echo $pkg | awk '{print $3}'` + oldrel=`echo $pkg | awk '{print $4}'` + [ "$pkgname" = "" ] && return; + cd $tl + pkgdir=`find . -type d -name "$pkgname" -maxdepth 2 -mindepth 1 | head -1` + if [ "$pkgdir" = "" ]; then + deletepkg + else + cd $pkgdir + if [ ! -f PKGBUILD ]; then + echo "WARNING: missing PKGBUILD in `pwd`" >&2 + else unset depends source deplist sources unset pkgname pkgver pkgrel pkgdesc url - . PKGBUILD + . PKGBUILD + # tag it + touch .tag + category=`pwd | sed 's|^.*/||g'` + cd - &>/dev/null deplist=${depends[@]} deplist=`php -r "echo addslashes(\"$deplist\");"` sources=${source[@]} @@ -32,32 +84,52 @@ for category in `find * -type d -maxdepth 0 | grep -v CVS | grep -v "^unofficial pkgrel=`php -r "echo addslashes(\"$pkgrel\");"` pkgdesc=`php -r "echo addslashes(\"$pkgdesc\");"` url=`php -r "echo addslashes(\"$url\");"` - mysql -u $dbuser -p$dbpass $dbname >$TMPFILE <<_EOF -SELECT id,pkgver,pkgrel FROM packages WHERE repoid=$repoid AND pkgname='$pkgname'; -_EOF - id=`tail -1 $TMPFILE | awk '{print $1}'` - if [ "$id" != "" ]; then - oldver=`tail -1 $TMPFILE | awk '{print $2}'` - oldrel=`tail -1 $TMPFILE | awk '{print $3}'` - if [ "$oldver" != "$pkgver" -o "$oldrel" != "$pkgrel" ]; then - mysql -u $dbuser -p$dbpass $dbname <<_EOF -UPDATE packages SET category='$category',pkgname='$pkgname',pkgver='$pkgver', -pkgrel='$pkgrel',pkgdesc='$pkgdesc',url='$url',sources='$sources', -depends='$deplist',lastupdate=NOW() WHERE id='$id'; -_EOF - fi - else - mysql -u $dbuser -p$dbpass $dbname <<_EOF -INSERT INTO packages (id,repoid,category,pkgname,pkgver,pkgrel, -pkgdesc,url,sources,depends,lastupdate) VALUES (NULL,$repoid, -'$category','$pkgname','$pkgver','$pkgrel','$pkgdesc','$url', -'$sources','$deplist',NOW()); + + if [ "$oldver" != "$pkgver" -o "$oldrel" != "$pkgrel" ]; then + updatepkg + fi + fi + fi +} + +echo "==> Updating Package Entries in Database..." +mysql --skip-column-names -u $dbuser -p$dbpass $dbname >$TMPFILE <<_EOF +SELECT id,pkgname,pkgver,pkgrel FROM packages WHERE repoid=$repoid _EOF + +cat $TMPFILE | ( \ + ln='a' + while [ "$ln" != "" ]; do + read ln + process "$ln" + done +) + +cd $tl +for category in `find . -type d -maxdepth 0 | grep -v CVS | grep -v "^unofficial" | grep -v "^unstable"`; do + for pkg in `ls $category | sort`; do + cd $tl/$category/$pkg + if [ -f .tag ]; then + # already processed + rm -f .tag + else + if [ -f PKGBUILD ]; then + . PKGBUILD + deplist=${depends[@]} + deplist=`php -r "echo addslashes(\"$deplist\");"` + sources=${source[@]} + sources=`php -r "echo addslashes(\"$sources\");"` + pkgname=`php -r "echo addslashes(\"$pkgname\");"` + pkgver=`php -r "echo addslashes(\"$pkgver\");"` + pkgrel=`php -r "echo addslashes(\"$pkgrel\");"` + pkgdesc=`php -r "echo addslashes(\"$pkgdesc\");"` + url=`php -r "echo addslashes(\"$url\");"` + insertpkg fi fi - done - cd $tl + done done rm -f $TMPFILE +exit 0 -- cgit v1.2.2 From c4e3bd62a57454cfe3b759a4207e5977bc10c1eb Mon Sep 17 00:00:00 2001 From: Judd Vinet Date: Sun, 25 May 2003 23:56:21 +0000 Subject: removed debug settings --- pkgdb | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/pkgdb b/pkgdb index 1f71162..365e4e3 100755 --- a/pkgdb +++ b/pkgdb @@ -1,5 +1,5 @@ #!/bin/bash -# $Id: pkgdb,v 1.3 2003/05/25 23:51:25 judd Exp $ +# $Id: pkgdb,v 1.4 2003/05/25 23:56:21 judd Exp $ tl=`pwd` @@ -12,10 +12,6 @@ dbuser='archweb' dbname='archweb' dbpass='14xuk0Gi' -dbuser='arch' -dbname='arch' -dbpass='arch' - if [ "$repoid" = "" ]; then me=`basename $0` echo "usage: $me " -- cgit v1.2.2 From 27acb4d0c95667fa8c26f78f015ee6cc9c009a34 Mon Sep 17 00:00:00 2001 From: Judd Vinet Date: Mon, 26 May 2003 00:13:44 +0000 Subject: fixed a little inserting bug --- pkgdb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkgdb b/pkgdb index 365e4e3..3bf9f5d 100755 --- a/pkgdb +++ b/pkgdb @@ -1,5 +1,5 @@ #!/bin/bash -# $Id: pkgdb,v 1.4 2003/05/25 23:56:21 judd Exp $ +# $Id: pkgdb,v 1.5 2003/05/26 00:13:44 judd Exp $ tl=`pwd` @@ -102,7 +102,7 @@ cat $TMPFILE | ( \ ) cd $tl -for category in `find . -type d -maxdepth 0 | grep -v CVS | grep -v "^unofficial" | grep -v "^unstable"`; do +for category in `find . -type d -maxdepth 1 | grep -v CVS | grep -v "^unofficial" | grep -v "^unstable"`; do for pkg in `ls $category | sort`; do cd $tl/$category/$pkg if [ -f .tag ]; then @@ -123,6 +123,7 @@ for category in `find . -type d -maxdepth 0 | grep -v CVS | grep -v "^unofficial insertpkg fi fi + cd $tl done done -- cgit v1.2.2 From dddb8b09af8bcd4666edc492a6ad178e74018e43 Mon Sep 17 00:00:00 2001 From: Judd Vinet Date: Mon, 26 May 2003 17:10:12 +0000 Subject: fixed another insert bug --- pkgdb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkgdb b/pkgdb index 3bf9f5d..ae14660 100755 --- a/pkgdb +++ b/pkgdb @@ -1,5 +1,5 @@ #!/bin/bash -# $Id: pkgdb,v 1.5 2003/05/26 00:13:44 judd Exp $ +# $Id: pkgdb,v 1.6 2003/05/26 17:10:12 judd Exp $ tl=`pwd` @@ -104,6 +104,7 @@ cat $TMPFILE | ( \ cd $tl for category in `find . -type d -maxdepth 1 | grep -v CVS | grep -v "^unofficial" | grep -v "^unstable"`; do for pkg in `ls $category | sort`; do + category=`echo $category | sed 's|.*/||'` cd $tl/$category/$pkg if [ -f .tag ]; then # already processed -- cgit v1.2.2 From 0748b95685f4041631fbfa1a8d6d5c79d0c95022 Mon Sep 17 00:00:00 2001 From: Judd Vinet Date: Mon, 26 May 2003 17:25:59 +0000 Subject: category bug --- pkgdb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgdb b/pkgdb index ae14660..13c3163 100755 --- a/pkgdb +++ b/pkgdb @@ -1,5 +1,5 @@ #!/bin/bash -# $Id: pkgdb,v 1.6 2003/05/26 17:10:12 judd Exp $ +# $Id: pkgdb,v 1.7 2003/05/26 17:25:59 judd Exp $ tl=`pwd` @@ -69,7 +69,7 @@ process() { . PKGBUILD # tag it touch .tag - category=`pwd | sed 's|^.*/||g'` + category=`pwd | sed "s|$tl/||" | sed 's|/.*$||'` cd - &>/dev/null deplist=${depends[@]} deplist=`php -r "echo addslashes(\"$deplist\");"` -- cgit v1.2.2 From 401e01f8411179d8aa9598a07e4621b2fb19e986 Mon Sep 17 00:00:00 2001 From: Judd Vinet Date: Thu, 5 Jun 2003 00:01:24 +0000 Subject: removed my username --- db-arch | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/db-arch b/db-arch index 1e24f88..0310555 100755 --- a/db-arch +++ b/db-arch @@ -1,5 +1,5 @@ #!/bin/bash -# $Id: db-arch,v 1.8 2003/05/25 23:50:02 judd Exp $ +# $Id: db-arch,v 1.9 2003/06/05 00:01:24 judd Exp $ uid=`id -u` TMPDIR="/tmp/archpkg.$uid" @@ -13,7 +13,7 @@ mkdir /tmp/archpkg.$uid; [ $? -gt 0 ] && exit 1 echo "==> Generating Pacman Database for OFFICIAL..." >&2 cd $TMPDIR -CVSROOT=:ext:judd@cvs.archlinux.org:/home/cvs-arch cvs -q export -r CURRENT -f arch/build +CVS_RSH=ssh CVSROOT=:ext:cvs.archlinux.org:/home/cvs-arch cvs -q export -r CURRENT -f arch/build /usr/bin/gensync $TMPDIR/arch/build $TMPDIR/current.db.tar.gz [ -f $TMPDIR/current.db.tar.gz ] && mv -f $TMPDIR/current.db.tar.gz /home/ftp/current -- cgit v1.2.2 From cd3590946ac66b7adcfa95f51f00f917327bf20a Mon Sep 17 00:00:00 2001 From: Judd Vinet Date: Sat, 16 Aug 2003 23:56:46 +0000 Subject: added locking --- db-arch | 15 ++++++++++++++- db-unstable | 15 ++++++++++++++- pkgdb | 14 +++++++++----- 3 files changed, 37 insertions(+), 7 deletions(-) diff --git a/db-arch b/db-arch index 0310555..74fa5ed 100755 --- a/db-arch +++ b/db-arch @@ -1,8 +1,19 @@ #!/bin/bash -# $Id: db-arch,v 1.9 2003/06/05 00:01:24 judd Exp $ +# $Id: db-arch,v 1.10 2003/08/16 23:56:46 judd Exp $ uid=`id -u` TMPDIR="/tmp/archpkg.$uid" +repoid=1 + +# check for locks +if [ -f /tmp/.repolck.$repoid ]; then + owner=`/bin/ls -l /tmp/.repolck.$repoid | awk '{print $3}'` + echo "error: db generation is already in progress (started by $owner)" + exit 1 +fi + +# lock +touch /tmp/.repolck.$repoid # RedHat's mktemp is broken... if [ -d /tmp/archpkg.$uid ]; then @@ -24,3 +35,5 @@ echo "==> Generating Text Package List..." >&2 mv packages.txt /home/ftp/current/setup/packages.txt rm -rf $TMPDIR +# unlock +rm -f /tmp/.repolck.$repoid diff --git a/db-unstable b/db-unstable index 18554c0..93ea565 100755 --- a/db-unstable +++ b/db-unstable @@ -1,8 +1,19 @@ #!/bin/bash -# $Id: db-unstable,v 1.5 2003/05/25 23:50:02 judd Exp $ +# $Id: db-unstable,v 1.6 2003/08/16 23:56:46 judd Exp $ uid=`id -u` TMPDIR="/tmp/archpkg.$uid" +repoid=3 + +# check for locks +if [ -f /tmp/.repolck.$repoid ]; then + owner=`/bin/ls -l /tmp/.repolck.$repoid | awk '{print $3}'` + echo "error: db generation is already in progress (started by $owner)" + exit 1 +fi + +# lock +touch /tmp/.repolck.$repoid # RedHat's mktemp is broken... if [ -d $TMPDIR ]; then @@ -20,4 +31,6 @@ CVS_RSH=ssh CVSROOT=:ext:cvs.archlinux.org:/home/cvs-unstable cvs -q export -r C cd $TMPDIR/unstable && /arch/pkgdb 3 rm -rf $TMPDIR +# unlock +rm -f /tmp/.repolck.$repoid diff --git a/pkgdb b/pkgdb index 13c3163..df847ec 100755 --- a/pkgdb +++ b/pkgdb @@ -1,5 +1,5 @@ #!/bin/bash -# $Id: pkgdb,v 1.7 2003/05/26 17:25:59 judd Exp $ +# $Id: pkgdb,v 1.8 2003/08/16 23:56:46 judd Exp $ tl=`pwd` @@ -10,7 +10,7 @@ TMPFILE="/tmp/.mysqltmp.$uid" repoid=$1 dbuser='archweb' dbname='archweb' -dbpass='14xuk0Gi' +dbpass='YWkrOjqm' if [ "$repoid" = "" ]; then me=`basename $0` @@ -30,7 +30,7 @@ updatepkg() { echo "Updating $pkgname" mysql -u $dbuser -p$dbpass $dbname <<_EOF -UPDATE packages SET category='$category',pkgname='$pkgname',pkgver='$pkgver', +UPDATE packages SET categoryid='$categoryid',pkgname='$pkgname',pkgver='$pkgver', pkgrel='$pkgrel',pkgdesc='$pkgdesc',url='$url',sources='$sources', depends='$deplist',lastupdate=NOW() WHERE id='$id'; _EOF @@ -40,9 +40,9 @@ insertpkg() { echo "Inserting $pkgname" mysql -u $dbuser -p$dbpass $dbname <<_EOF -INSERT INTO packages (id,repoid,category,pkgname,pkgver,pkgrel, +INSERT INTO packages (id,repoid,categoryid,pkgname,pkgver,pkgrel, pkgdesc,url,sources,depends,lastupdate) VALUES (NULL,$repoid, -'$category','$pkgname','$pkgver','$pkgrel','$pkgdesc','$url', +'$categoryid','$pkgname','$pkgver','$pkgrel','$pkgdesc','$url', '$sources','$deplist',NOW()); _EOF } @@ -70,6 +70,10 @@ process() { # tag it touch .tag category=`pwd | sed "s|$tl/||" | sed 's|/.*$||'` + mysql --skip-column-names -u $dbuser -p$dbpass $dbname >$TMPFILE <<_EOF +SELECT id FROM categories WHERE category='$category'; +_EOF + categoryid=`cat $TMPFILE | awk '{print $1}'` cd - &>/dev/null deplist=${depends[@]} deplist=`php -r "echo addslashes(\"$deplist\");"` -- cgit v1.2.2 From 6cac5b93a08fce0f30308bcf96f85ebd08df68fd Mon Sep 17 00:00:00 2001 From: Judd Vinet Date: Sun, 17 Aug 2003 00:58:05 +0000 Subject: reset needupdate --- pkgdb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgdb b/pkgdb index df847ec..2578220 100755 --- a/pkgdb +++ b/pkgdb @@ -1,5 +1,5 @@ #!/bin/bash -# $Id: pkgdb,v 1.8 2003/08/16 23:56:46 judd Exp $ +# $Id: pkgdb,v 1.9 2003/08/17 00:58:05 judd Exp $ tl=`pwd` @@ -32,7 +32,7 @@ updatepkg() mysql -u $dbuser -p$dbpass $dbname <<_EOF UPDATE packages SET categoryid='$categoryid',pkgname='$pkgname',pkgver='$pkgver', pkgrel='$pkgrel',pkgdesc='$pkgdesc',url='$url',sources='$sources', -depends='$deplist',lastupdate=NOW() WHERE id='$id'; +depends='$deplist',needupdate=0,lastupdate=NOW() WHERE id='$id'; _EOF } -- cgit v1.2.2 From 3c4909c323aadad6be0aa600f0534beb0bd413b3 Mon Sep 17 00:00:00 2001 From: Judd Vinet Date: Mon, 18 Aug 2003 18:14:53 +0000 Subject: fixed a dupe bug --- pkgdb | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/pkgdb b/pkgdb index 2578220..6d6a483 100755 --- a/pkgdb +++ b/pkgdb @@ -1,11 +1,12 @@ #!/bin/bash -# $Id: pkgdb,v 1.9 2003/08/17 00:58:05 judd Exp $ +# $Id: pkgdb,v 1.10 2003/08/18 18:14:53 judd Exp $ tl=`pwd` # we use this instead of mktemp cuz the RH server's mktemp is broken :( uid=`id -u` -TMPFILE="/tmp/.mysqltmp.$uid" +TMPFILE="/tmp/.mysqltmp.$uid.1" +TMPFILE2="/tmp/.mysqltmp.$uid.2" repoid=$1 dbuser='archweb' @@ -70,10 +71,13 @@ process() { # tag it touch .tag category=`pwd | sed "s|$tl/||" | sed 's|/.*$||'` - mysql --skip-column-names -u $dbuser -p$dbpass $dbname >$TMPFILE <<_EOF + mysql --skip-column-names -u $dbuser -p$dbpass $dbname >$TMPFILE2 <<_EOF SELECT id FROM categories WHERE category='$category'; _EOF - categoryid=`cat $TMPFILE | awk '{print $1}'` + categoryid=`cat $TMPFILE2 | awk '{print $1}'` + if [ "$categoryid" = "" ]; then + echo "WARNING: no categoryid found for '$category'" >&2 + fi cd - &>/dev/null deplist=${depends[@]} deplist=`php -r "echo addslashes(\"$deplist\");"` @@ -94,9 +98,10 @@ _EOF echo "==> Updating Package Entries in Database..." mysql --skip-column-names -u $dbuser -p$dbpass $dbname >$TMPFILE <<_EOF -SELECT id,pkgname,pkgver,pkgrel FROM packages WHERE repoid=$repoid +SELECT id,pkgname,pkgver,pkgrel FROM packages WHERE repoid='$repoid'; _EOF +echo " -> Scanning for Updated/Deleted packages..." cat $TMPFILE | ( \ ln='a' while [ "$ln" != "" ]; do @@ -105,10 +110,21 @@ cat $TMPFILE | ( \ done ) +echo " -> Scanning for New packages..." cd $tl for category in `find . -type d -maxdepth 1 | grep -v CVS | grep -v "^unofficial" | grep -v "^unstable"`; do + category=`echo $category | sed 's|./||'` + if [ "$category" = "." ]; then + continue + fi + mysql --skip-column-names -u $dbuser -p$dbpass $dbname >$TMPFILE <<_EOF +SELECT id FROM categories WHERE category='$category'; +_EOF + categoryid=`cat $TMPFILE | awk '{print $1}'` + if [ "$categoryid" = "" ]; then + echo "WARNING: no categoryid found for '$category'" >&2 + fi for pkg in `ls $category | sort`; do - category=`echo $category | sed 's|.*/||'` cd $tl/$category/$pkg if [ -f .tag ]; then # already processed -- cgit v1.2.2 From fdf319926e30942672a3cc57ad6f1bc173d81a6d Mon Sep 17 00:00:00 2001 From: Judd Vinet Date: Sun, 12 Oct 2003 18:41:26 +0000 Subject: updated to work with other repos --- genpkglist | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/genpkglist b/genpkglist index 6af0bed..c18c61f 100755 --- a/genpkglist +++ b/genpkglist @@ -1,13 +1,15 @@ #!/bin/bash -# $Id: genpkglist,v 1.7 2003/05/25 23:50:02 judd Exp $ +# $Id: genpkglist,v 1.8 2003/10/12 18:41:26 judd Exp $ # # genpkglist # # Generates a text package database for use with the setup script +# (also used to check for missing packages in the download directory) # pkgfile="`pwd`/packages.txt" +repodir=$1 rm -f $pkgfile for category in `find * -type d -maxdepth 0 | grep -v CVS`; do @@ -16,7 +18,7 @@ for category in `find * -type d -maxdepth 0 | grep -v CVS`; do cd $pkg if [ -f PKGBUILD ]; then . PKGBUILD - if [ -f /home/ftp/current/$pkgname-$pkgver-$pkgrel.pkg.tar.gz ]; then + if [ -f /home/ftp/$repodir/$pkgname-$pkgver-$pkgrel.pkg.tar.gz ]; then echo "$category/$pkgname-$pkgver-$pkgrel.pkg.tar.gz" >>$pkgfile else echo "notice: Missing $pkgname-$pkgver-$pkgrel.pkg.tar.gz in ftp site" >&2 -- cgit v1.2.2 From a1f715792fe75aba32625d998f8d6a27605704bc Mon Sep 17 00:00:00 2001 From: Judd Vinet Date: Sun, 12 Oct 2003 18:42:55 +0000 Subject: updated db generation scripts to use genpkglist --- db-arch | 4 ++-- db-unstable | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/db-arch b/db-arch index 74fa5ed..587aaeb 100755 --- a/db-arch +++ b/db-arch @@ -1,5 +1,5 @@ #!/bin/bash -# $Id: db-arch,v 1.10 2003/08/16 23:56:46 judd Exp $ +# $Id: db-arch,v 1.11 2003/10/12 18:42:55 judd Exp $ uid=`id -u` TMPDIR="/tmp/archpkg.$uid" @@ -31,7 +31,7 @@ CVS_RSH=ssh CVSROOT=:ext:cvs.archlinux.org:/home/cvs-arch cvs -q export -r CURRE cd $TMPDIR/arch/build && /arch/pkgdb 1 echo "==> Generating Text Package List..." >&2 -/arch/genpkglist +/arch/genpkglist current mv packages.txt /home/ftp/current/setup/packages.txt rm -rf $TMPDIR diff --git a/db-unstable b/db-unstable index 93ea565..c6471ef 100755 --- a/db-unstable +++ b/db-unstable @@ -1,5 +1,5 @@ #!/bin/bash -# $Id: db-unstable,v 1.6 2003/08/16 23:56:46 judd Exp $ +# $Id: db-unstable,v 1.7 2003/10/12 18:42:55 judd Exp $ uid=`id -u` TMPDIR="/tmp/archpkg.$uid" @@ -30,6 +30,9 @@ CVS_RSH=ssh CVSROOT=:ext:cvs.archlinux.org:/home/cvs-unstable cvs -q export -r C cd $TMPDIR/unstable && /arch/pkgdb 3 +echo "==> Scanning for missing packages..." >&2 +/arch/genpkglist unstable + rm -rf $TMPDIR # unlock rm -f /tmp/.repolck.$repoid -- cgit v1.2.2 From 14ed62289b8026a2a0487cf86c8b0a58c8b7c120 Mon Sep 17 00:00:00 2001 From: Judd Vinet Date: Sun, 16 Nov 2003 03:50:23 +0000 Subject: added Dale's dupe checking --- genpkglist | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/genpkglist b/genpkglist index c18c61f..83e16a6 100755 --- a/genpkglist +++ b/genpkglist @@ -1,5 +1,5 @@ #!/bin/bash -# $Id: genpkglist,v 1.8 2003/10/12 18:41:26 judd Exp $ +# $Id: genpkglist,v 1.9 2003/11/16 03:50:23 judd Exp $ # # genpkglist @@ -29,3 +29,11 @@ for category in `find * -type d -maxdepth 0 | grep -v CVS`; do cd .. done +DUPES=`ls -1 /home/ftp/$1 | rev | cut -d- -f 3- | rev | sort | uniq -c | egrep -v '^ 1' | awk '{print $2}'` + +if [ -n "$DUPES" ]; then + echo "Possible Dupes for $1 (please remove old versions)." + echo "Date Filename" + for dupe in $DUPES; do ls -l /home/ftp/$1/$dupe* | awk '{print $6" "$7" "$8" "$9}'; done +fi + -- cgit v1.2.2 From c29d56987d44169e63345012c6532436a643b63b Mon Sep 17 00:00:00 2001 From: Judd Vinet Date: Tue, 25 Nov 2003 01:28:26 +0000 Subject: modifed to work with the new gensync --- db-arch | 8 ++++++-- db-unstable | 8 ++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/db-arch b/db-arch index 587aaeb..e0dcf0e 100755 --- a/db-arch +++ b/db-arch @@ -1,5 +1,5 @@ #!/bin/bash -# $Id: db-arch,v 1.11 2003/10/12 18:42:55 judd Exp $ +# $Id: db-arch,v 1.12 2003/11/25 01:28:26 judd Exp $ uid=`id -u` TMPDIR="/tmp/archpkg.$uid" @@ -11,6 +11,10 @@ if [ -f /tmp/.repolck.$repoid ]; then echo "error: db generation is already in progress (started by $owner)" exit 1 fi +if [ -d /tmp/.gensync ]; then + echo "error: someone else is currently running gensync!" + exit 1 +fi # lock touch /tmp/.repolck.$repoid @@ -25,7 +29,7 @@ mkdir /tmp/archpkg.$uid; [ $? -gt 0 ] && exit 1 echo "==> Generating Pacman Database for OFFICIAL..." >&2 cd $TMPDIR CVS_RSH=ssh CVSROOT=:ext:cvs.archlinux.org:/home/cvs-arch cvs -q export -r CURRENT -f arch/build -/usr/bin/gensync $TMPDIR/arch/build $TMPDIR/current.db.tar.gz +/usr/bin/gensync $TMPDIR/arch/build $TMPDIR/current.db.tar.gz /home/ftp/current [ -f $TMPDIR/current.db.tar.gz ] && mv -f $TMPDIR/current.db.tar.gz /home/ftp/current cd $TMPDIR/arch/build && /arch/pkgdb 1 diff --git a/db-unstable b/db-unstable index c6471ef..e090c88 100755 --- a/db-unstable +++ b/db-unstable @@ -1,5 +1,5 @@ #!/bin/bash -# $Id: db-unstable,v 1.7 2003/10/12 18:42:55 judd Exp $ +# $Id: db-unstable,v 1.8 2003/11/25 01:28:26 judd Exp $ uid=`id -u` TMPDIR="/tmp/archpkg.$uid" @@ -11,6 +11,10 @@ if [ -f /tmp/.repolck.$repoid ]; then echo "error: db generation is already in progress (started by $owner)" exit 1 fi +if [ -d /tmp/.gensync ]; then + echo "error: someone else is currently running gensync!" + exit 1 +fi # lock touch /tmp/.repolck.$repoid @@ -25,7 +29,7 @@ mkdir $TMPDIR; [ $? -gt 0 ] && exit 1 echo "==> Generating Pacman Database for UNSTABLE..." >&2 cd $TMPDIR CVS_RSH=ssh CVSROOT=:ext:cvs.archlinux.org:/home/cvs-unstable cvs -q export -r CURRENT unstable -/usr/bin/gensync $TMPDIR/unstable $TMPDIR/unstable.db.tar.gz +/usr/bin/gensync $TMPDIR/unstable $TMPDIR/unstable.db.tar.gz /home/ftp/unstable [ -f $TMPDIR/unstable.db.tar.gz ] && mv -f $TMPDIR/unstable.db.tar.gz /home/ftp/unstable cd $TMPDIR/unstable && /arch/pkgdb 3 -- cgit v1.2.2 From b708026bb365fd1904c779b9e8b46eb70e05781f Mon Sep 17 00:00:00 2001 From: Judd Vinet Date: Sun, 21 Dec 2003 03:22:27 +0000 Subject: renamed db-unofficial to db-extra --- db-extra | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100755 db-extra diff --git a/db-extra b/db-extra new file mode 100755 index 0000000..399c004 --- /dev/null +++ b/db-extra @@ -0,0 +1,42 @@ +#!/bin/bash +# $Id: db-extra,v 1.1 2003/12/21 03:22:27 judd Exp $ + +uid=`id -u` +TMPDIR="/tmp/archpkg.$uid" +repoid=2 + +# check for locks +if [ -f /tmp/.repolck.$repoid ]; then + owner=`/bin/ls -l /tmp/.repolck.$repoid | awk '{print $3}'` + echo "error: db generation is already in progress (started by $owner)" + exit 1 +fi +if [ -d /tmp/.gensync ]; then + echo "error: someone else is currently running gensync!" + exit 1 +fi + +# lock +touch /tmp/.repolck.$repoid + +# RedHat's mktemp is broken... +if [ -d $TMPDIR ]; then + echo "==> Removing old temp dir..." >&2 + rm -rf $TMPDIR || exit 1 +fi +mkdir $TMPDIR; [ $? -gt 0 ] && exit 1 + +echo "==> Generating Pacman Database for UNOFFICIAL..." >&2 +cd $TMPDIR +CVS_RSH=ssh CVSROOT=:ext:cvs.archlinux.org:/home/cvs-extra cvs -q export -r CURRENT extra +/usr/bin/gensync $TMPDIR/extra $TMPDIR/extra.db.tar.gz /home/ftp/extra +[ -f $TMPDIR/extra.db.tar.gz ] && mv -f $TMPDIR/extra.db.tar.gz /home/ftp/extra + +cd $TMPDIR/extra && /arch/pkgdb 2 + +echo "==> Scanning for missing packages..." >&2 +/arch/genpkglist extra + +rm -rf $TMPDIR +# unlock +rm -f /tmp/.repolck.$repoid -- cgit v1.2.2 From 57508048a376fd31a30d839c1ef16ef7d4fae1e1 Mon Sep 17 00:00:00 2001 From: Judd Vinet Date: Sun, 21 Dec 2003 03:23:28 +0000 Subject: missed a spot --- db-extra | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/db-extra b/db-extra index 399c004..27ad735 100755 --- a/db-extra +++ b/db-extra @@ -1,5 +1,5 @@ #!/bin/bash -# $Id: db-extra,v 1.1 2003/12/21 03:22:27 judd Exp $ +# $Id: db-extra,v 1.2 2003/12/21 03:23:28 judd Exp $ uid=`id -u` TMPDIR="/tmp/archpkg.$uid" @@ -26,7 +26,7 @@ if [ -d $TMPDIR ]; then fi mkdir $TMPDIR; [ $? -gt 0 ] && exit 1 -echo "==> Generating Pacman Database for UNOFFICIAL..." >&2 +echo "==> Generating Pacman Database for EXTRAL..." >&2 cd $TMPDIR CVS_RSH=ssh CVSROOT=:ext:cvs.archlinux.org:/home/cvs-extra cvs -q export -r CURRENT extra /usr/bin/gensync $TMPDIR/extra $TMPDIR/extra.db.tar.gz /home/ftp/extra -- cgit v1.2.2 From 905bcd8474b725d24881ca11b9ecc1adfe98b444 Mon Sep 17 00:00:00 2001 From: Judd Vinet Date: Sun, 21 Dec 2003 03:24:09 +0000 Subject: third times the charm :/ --- db-extra | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/db-extra b/db-extra index 27ad735..506aea6 100755 --- a/db-extra +++ b/db-extra @@ -1,5 +1,5 @@ #!/bin/bash -# $Id: db-extra,v 1.2 2003/12/21 03:23:28 judd Exp $ +# $Id: db-extra,v 1.3 2003/12/21 03:24:09 judd Exp $ uid=`id -u` TMPDIR="/tmp/archpkg.$uid" @@ -26,7 +26,7 @@ if [ -d $TMPDIR ]; then fi mkdir $TMPDIR; [ $? -gt 0 ] && exit 1 -echo "==> Generating Pacman Database for EXTRAL..." >&2 +echo "==> Generating Pacman Database for EXTRA..." >&2 cd $TMPDIR CVS_RSH=ssh CVSROOT=:ext:cvs.archlinux.org:/home/cvs-extra cvs -q export -r CURRENT extra /usr/bin/gensync $TMPDIR/extra $TMPDIR/extra.db.tar.gz /home/ftp/extra -- cgit v1.2.2 From 63b99d1562607f89d4a3edc0eb0f5c971312d8db Mon Sep 17 00:00:00 2001 From: Judd Vinet Date: Tue, 6 Jan 2004 02:50:53 +0000 Subject: trapped SIGINT to cleanly remove lock files --- db-arch | 19 +++++++++++++------ db-extra | 17 ++++++++++++----- db-unstable | 18 ++++++++++++------ 3 files changed, 37 insertions(+), 17 deletions(-) diff --git a/db-arch b/db-arch index e0dcf0e..d2c93b8 100755 --- a/db-arch +++ b/db-arch @@ -1,10 +1,17 @@ #!/bin/bash -# $Id: db-arch,v 1.12 2003/11/25 01:28:26 judd Exp $ +# $Id: db-arch,v 1.13 2004/01/06 02:50:53 judd Exp $ uid=`id -u` TMPDIR="/tmp/archpkg.$uid" repoid=1 +cleanup() { + rm -rf $TMPDIR + # unlock + rm -f /tmp/.repolck.$repoid + exit 0 +} + # check for locks if [ -f /tmp/.repolck.$repoid ]; then owner=`/bin/ls -l /tmp/.repolck.$repoid | awk '{print $3}'` @@ -16,6 +23,8 @@ if [ -d /tmp/.gensync ]; then exit 1 fi +# catch ^C breaks +trap cleanup SIGINT # lock touch /tmp/.repolck.$repoid @@ -26,18 +35,16 @@ if [ -d /tmp/archpkg.$uid ]; then fi mkdir /tmp/archpkg.$uid; [ $? -gt 0 ] && exit 1 -echo "==> Generating Pacman Database for OFFICIAL..." >&2 +echo "==> Generating Pacman Database for CURRENT..." >&2 cd $TMPDIR CVS_RSH=ssh CVSROOT=:ext:cvs.archlinux.org:/home/cvs-arch cvs -q export -r CURRENT -f arch/build /usr/bin/gensync $TMPDIR/arch/build $TMPDIR/current.db.tar.gz /home/ftp/current [ -f $TMPDIR/current.db.tar.gz ] && mv -f $TMPDIR/current.db.tar.gz /home/ftp/current -cd $TMPDIR/arch/build && /arch/pkgdb 1 +cd $TMPDIR/arch/build && /arch/pkgdb $repoid echo "==> Generating Text Package List..." >&2 /arch/genpkglist current mv packages.txt /home/ftp/current/setup/packages.txt -rm -rf $TMPDIR -# unlock -rm -f /tmp/.repolck.$repoid +cleanup diff --git a/db-extra b/db-extra index 506aea6..b50c7a6 100755 --- a/db-extra +++ b/db-extra @@ -1,10 +1,17 @@ #!/bin/bash -# $Id: db-extra,v 1.3 2003/12/21 03:24:09 judd Exp $ +# $Id: db-extra,v 1.4 2004/01/06 02:50:53 judd Exp $ uid=`id -u` TMPDIR="/tmp/archpkg.$uid" repoid=2 +cleanup() { + rm -rf $TMPDIR + # unlock + rm -f /tmp/.repolck.$repoid + exit 0 +} + # check for locks if [ -f /tmp/.repolck.$repoid ]; then owner=`/bin/ls -l /tmp/.repolck.$repoid | awk '{print $3}'` @@ -16,6 +23,8 @@ if [ -d /tmp/.gensync ]; then exit 1 fi +# catch ^C breaks +trap cleanup SIGINT # lock touch /tmp/.repolck.$repoid @@ -32,11 +41,9 @@ CVS_RSH=ssh CVSROOT=:ext:cvs.archlinux.org:/home/cvs-extra cvs -q export -r CURR /usr/bin/gensync $TMPDIR/extra $TMPDIR/extra.db.tar.gz /home/ftp/extra [ -f $TMPDIR/extra.db.tar.gz ] && mv -f $TMPDIR/extra.db.tar.gz /home/ftp/extra -cd $TMPDIR/extra && /arch/pkgdb 2 +cd $TMPDIR/extra && /arch/pkgdb $repoid echo "==> Scanning for missing packages..." >&2 /arch/genpkglist extra -rm -rf $TMPDIR -# unlock -rm -f /tmp/.repolck.$repoid +cleanup diff --git a/db-unstable b/db-unstable index e090c88..93e8465 100755 --- a/db-unstable +++ b/db-unstable @@ -1,10 +1,17 @@ #!/bin/bash -# $Id: db-unstable,v 1.8 2003/11/25 01:28:26 judd Exp $ +# $Id: db-unstable,v 1.9 2004/01/06 02:50:53 judd Exp $ uid=`id -u` TMPDIR="/tmp/archpkg.$uid" repoid=3 +cleanup() { + rm -rf $TMPDIR + # unlock + rm -f /tmp/.repolck.$repoid + exit 0 +} + # check for locks if [ -f /tmp/.repolck.$repoid ]; then owner=`/bin/ls -l /tmp/.repolck.$repoid | awk '{print $3}'` @@ -16,6 +23,8 @@ if [ -d /tmp/.gensync ]; then exit 1 fi +# catch ^C breaks +trap cleanup SIGINT # lock touch /tmp/.repolck.$repoid @@ -32,12 +41,9 @@ CVS_RSH=ssh CVSROOT=:ext:cvs.archlinux.org:/home/cvs-unstable cvs -q export -r C /usr/bin/gensync $TMPDIR/unstable $TMPDIR/unstable.db.tar.gz /home/ftp/unstable [ -f $TMPDIR/unstable.db.tar.gz ] && mv -f $TMPDIR/unstable.db.tar.gz /home/ftp/unstable -cd $TMPDIR/unstable && /arch/pkgdb 3 +cd $TMPDIR/unstable && /arch/pkgdb $repoid echo "==> Scanning for missing packages..." >&2 /arch/genpkglist unstable -rm -rf $TMPDIR -# unlock -rm -f /tmp/.repolck.$repoid - +cleanup -- cgit v1.2.2 From c05e86b8d0fbef859f3f76d8897f1333d0ae49a1 Mon Sep 17 00:00:00 2001 From: Judd Vinet Date: Fri, 20 Feb 2004 09:20:32 +0000 Subject: updates for 0.6 --- pkgdb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgdb b/pkgdb index 6d6a483..69affd0 100755 --- a/pkgdb +++ b/pkgdb @@ -1,5 +1,5 @@ #!/bin/bash -# $Id: pkgdb,v 1.10 2003/08/18 18:14:53 judd Exp $ +# $Id: pkgdb,v 1.11 2004/02/20 09:20:32 judd Exp $ tl=`pwd` @@ -112,7 +112,7 @@ cat $TMPFILE | ( \ echo " -> Scanning for New packages..." cd $tl -for category in `find . -type d -maxdepth 1 | grep -v CVS | grep -v "^unofficial" | grep -v "^unstable"`; do +for category in `find . -type d -maxdepth 1 | grep -v CVS | grep -v "^extra" | grep -v "^unstable"`; do category=`echo $category | sed 's|./||'` if [ "$category" = "." ]; then continue -- cgit v1.2.2 From c73bc36772e5811700cc80cc24c8a759b16cbc1e Mon Sep 17 00:00:00 2001 From: Judd Vinet Date: Wed, 17 Mar 2004 18:14:15 +0000 Subject: Dale's fix for a bug in dupe-checking --- genpkglist | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/genpkglist b/genpkglist index 83e16a6..b740a2c 100755 --- a/genpkglist +++ b/genpkglist @@ -1,5 +1,5 @@ #!/bin/bash -# $Id: genpkglist,v 1.9 2003/11/16 03:50:23 judd Exp $ +# $Id: genpkglist,v 1.10 2004/03/17 18:14:15 judd Exp $ # # genpkglist @@ -34,6 +34,6 @@ DUPES=`ls -1 /home/ftp/$1 | rev | cut -d- -f 3- | rev | sort | uniq -c | egrep - if [ -n "$DUPES" ]; then echo "Possible Dupes for $1 (please remove old versions)." echo "Date Filename" - for dupe in $DUPES; do ls -l /home/ftp/$1/$dupe* | awk '{print $6" "$7" "$8" "$9}'; done + for dupe in $DUPES; do ls -l /home/ftp/$1/${dupe}* | awk '{print $6" "$7" "$8" "$9}'; done fi -- cgit v1.2.2 From f277673d7e083f2cb32537a1764c1819edecb923 Mon Sep 17 00:00:00 2001 From: Judd Vinet Date: Thu, 1 Apr 2004 21:04:14 +0000 Subject: changed mysql temp file so one user can run multiple db-gen scripts concurrently --- db-arch | 9 +++++++-- db-extra | 9 +++++++-- db-unstable | 9 +++++++-- pkgdb | 8 ++++---- 4 files changed, 25 insertions(+), 10 deletions(-) diff --git a/db-arch b/db-arch index d2c93b8..506346b 100755 --- a/db-arch +++ b/db-arch @@ -1,9 +1,9 @@ #!/bin/bash -# $Id: db-arch,v 1.13 2004/01/06 02:50:53 judd Exp $ +# $Id: db-arch,v 1.14 2004/04/01 21:09:13 judd Exp $ uid=`id -u` -TMPDIR="/tmp/archpkg.$uid" repoid=1 +TMPDIR="/tmp/archpkg.$repoid.$uid" cleanup() { rm -rf $TMPDIR @@ -38,6 +38,11 @@ mkdir /tmp/archpkg.$uid; [ $? -gt 0 ] && exit 1 echo "==> Generating Pacman Database for CURRENT..." >&2 cd $TMPDIR CVS_RSH=ssh CVSROOT=:ext:cvs.archlinux.org:/home/cvs-arch cvs -q export -r CURRENT -f arch/build +# check again +if [ -d /tmp/.gensync ]; then + echo "error: someone else is currently running gensync!" + exit 1 +fi /usr/bin/gensync $TMPDIR/arch/build $TMPDIR/current.db.tar.gz /home/ftp/current [ -f $TMPDIR/current.db.tar.gz ] && mv -f $TMPDIR/current.db.tar.gz /home/ftp/current diff --git a/db-extra b/db-extra index b50c7a6..204366f 100755 --- a/db-extra +++ b/db-extra @@ -1,9 +1,9 @@ #!/bin/bash -# $Id: db-extra,v 1.4 2004/01/06 02:50:53 judd Exp $ +# $Id: db-extra,v 1.5 2004/04/01 21:09:13 judd Exp $ uid=`id -u` -TMPDIR="/tmp/archpkg.$uid" repoid=2 +TMPDIR="/tmp/archpkg.$repoid.$uid" cleanup() { rm -rf $TMPDIR @@ -38,6 +38,11 @@ mkdir $TMPDIR; [ $? -gt 0 ] && exit 1 echo "==> Generating Pacman Database for EXTRA..." >&2 cd $TMPDIR CVS_RSH=ssh CVSROOT=:ext:cvs.archlinux.org:/home/cvs-extra cvs -q export -r CURRENT extra +# check again +if [ -d /tmp/.gensync ]; then + echo "error: someone else is currently running gensync!" + exit 1 +fi /usr/bin/gensync $TMPDIR/extra $TMPDIR/extra.db.tar.gz /home/ftp/extra [ -f $TMPDIR/extra.db.tar.gz ] && mv -f $TMPDIR/extra.db.tar.gz /home/ftp/extra diff --git a/db-unstable b/db-unstable index 93e8465..6e0efa5 100755 --- a/db-unstable +++ b/db-unstable @@ -1,9 +1,9 @@ #!/bin/bash -# $Id: db-unstable,v 1.9 2004/01/06 02:50:53 judd Exp $ +# $Id: db-unstable,v 1.10 2004/04/01 21:09:13 judd Exp $ uid=`id -u` -TMPDIR="/tmp/archpkg.$uid" repoid=3 +TMPDIR="/tmp/archpkg.$repoid.$uid" cleanup() { rm -rf $TMPDIR @@ -38,6 +38,11 @@ mkdir $TMPDIR; [ $? -gt 0 ] && exit 1 echo "==> Generating Pacman Database for UNSTABLE..." >&2 cd $TMPDIR CVS_RSH=ssh CVSROOT=:ext:cvs.archlinux.org:/home/cvs-unstable cvs -q export -r CURRENT unstable +# check again +if [ -d /tmp/.gensync ]; then + echo "error: someone else is currently running gensync!" + exit 1 +fi /usr/bin/gensync $TMPDIR/unstable $TMPDIR/unstable.db.tar.gz /home/ftp/unstable [ -f $TMPDIR/unstable.db.tar.gz ] && mv -f $TMPDIR/unstable.db.tar.gz /home/ftp/unstable diff --git a/pkgdb b/pkgdb index 69affd0..4ccb54f 100755 --- a/pkgdb +++ b/pkgdb @@ -1,14 +1,14 @@ #!/bin/bash -# $Id: pkgdb,v 1.11 2004/02/20 09:20:32 judd Exp $ +# $Id: pkgdb,v 1.12 2004/04/01 21:04:14 judd Exp $ tl=`pwd` +repoid=$1 # we use this instead of mktemp cuz the RH server's mktemp is broken :( uid=`id -u` -TMPFILE="/tmp/.mysqltmp.$uid.1" -TMPFILE2="/tmp/.mysqltmp.$uid.2" +TMPFILE="/tmp/.mysqltmp.$uid.$repoid.1" +TMPFILE2="/tmp/.mysqltmp.$uid.$repoid.2" -repoid=$1 dbuser='archweb' dbname='archweb' dbpass='YWkrOjqm' -- cgit v1.2.2 From abe20f6ff58f4ef9ed4a6a4670d8c46332ce56cc Mon Sep 17 00:00:00 2001 From: Judd Vinet Date: Fri, 2 Apr 2004 00:48:50 +0000 Subject: fixed a temp name bug --- db-arch | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/db-arch b/db-arch index 506346b..3c201c2 100755 --- a/db-arch +++ b/db-arch @@ -1,5 +1,5 @@ #!/bin/bash -# $Id: db-arch,v 1.14 2004/04/01 21:09:13 judd Exp $ +# $Id: db-arch,v 1.15 2004/04/02 00:48:50 judd Exp $ uid=`id -u` repoid=1 @@ -29,11 +29,11 @@ trap cleanup SIGINT touch /tmp/.repolck.$repoid # RedHat's mktemp is broken... -if [ -d /tmp/archpkg.$uid ]; then +if [ -d $TMPDIR ]; then echo "==> Removing old temp dir..." >&2 - rm -rf /tmp/archpkg.$uid || exit 1 + rm -rf $TMPDIR || exit 1 fi -mkdir /tmp/archpkg.$uid; [ $? -gt 0 ] && exit 1 +mkdir $TMPDIR; [ $? -gt 0 ] && exit 1 echo "==> Generating Pacman Database for CURRENT..." >&2 cd $TMPDIR -- cgit v1.2.2 From 3982aa170c4e66ab6bc13b1bfa23493d21838611 Mon Sep 17 00:00:00 2001 From: Judd Vinet Date: Tue, 13 Apr 2004 01:47:04 +0000 Subject: added db-testing --- db-testing | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100755 db-testing diff --git a/db-testing b/db-testing new file mode 100755 index 0000000..a515f25 --- /dev/null +++ b/db-testing @@ -0,0 +1,54 @@ +#!/bin/bash +# $Id: db-testing,v 1.1 2004/04/13 01:47:04 judd Exp $ + +uid=`id -u` +repoid=4 +TMPDIR="/tmp/archpkg.$repoid.$uid" + +cleanup() { + rm -rf $TMPDIR + # unlock + rm -f /tmp/.repolck.$repoid + exit 0 +} + +# check for locks +if [ -f /tmp/.repolck.$repoid ]; then + owner=`/bin/ls -l /tmp/.repolck.$repoid | awk '{print $3}'` + echo "error: db generation is already in progress (started by $owner)" + exit 1 +fi +if [ -d /tmp/.gensync ]; then + echo "error: someone else is currently running gensync!" + exit 1 +fi + +# catch ^C breaks +trap cleanup SIGINT +# lock +touch /tmp/.repolck.$repoid + +# RedHat's mktemp is broken... +if [ -d $TMPDIR ]; then + echo "==> Removing old temp dir..." >&2 + rm -rf $TMPDIR || exit 1 +fi +mkdir $TMPDIR; [ $? -gt 0 ] && exit 1 + +echo "==> Generating Pacman Database for TESTING..." >&2 +cd $TMPDIR +CVS_RSH=ssh CVSROOT=:ext:cvs.archlinux.org:/home/cvs-arch cvs -q export -r TESTING -f arch/build +# check again +if [ -d /tmp/.gensync ]; then + echo "error: someone else is currently running gensync!" + exit 1 +fi +/usr/bin/gensync $TMPDIR/arch/build $TMPDIR/testing.db.tar.gz /home/ftp/testing +[ -f $TMPDIR/testing.db.tar.gz ] && mv -f $TMPDIR/testing.db.tar.gz /home/ftp/testing + +cd $TMPDIR/arch/build && /arch/pkgdb $repoid + +echo "==> Generating Text Package List..." >&2 +/arch/genpkglist testing + +cleanup -- cgit v1.2.2 From 95305bef22f815471521522d94a010b0cd4e608b Mon Sep 17 00:00:00 2001 From: Judd Vinet Date: Tue, 13 Apr 2004 02:06:49 +0000 Subject: fix --- db-testing | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/db-testing b/db-testing index a515f25..b576f27 100755 --- a/db-testing +++ b/db-testing @@ -1,5 +1,5 @@ #!/bin/bash -# $Id: db-testing,v 1.1 2004/04/13 01:47:04 judd Exp $ +# $Id: db-testing,v 1.2 2004/04/13 02:06:49 judd Exp $ uid=`id -u` repoid=4 @@ -37,7 +37,7 @@ mkdir $TMPDIR; [ $? -gt 0 ] && exit 1 echo "==> Generating Pacman Database for TESTING..." >&2 cd $TMPDIR -CVS_RSH=ssh CVSROOT=:ext:cvs.archlinux.org:/home/cvs-arch cvs -q export -r TESTING -f arch/build +CVS_RSH=ssh CVSROOT=:ext:cvs.archlinux.org:/home/cvs-arch cvs -q export -r TESTING arch # check again if [ -d /tmp/.gensync ]; then echo "error: someone else is currently running gensync!" -- cgit v1.2.2 From 5bfca7d4025c014f2c1ffbe917c32a03889502a5 Mon Sep 17 00:00:00 2001 From: Judd Vinet Date: Tue, 13 Apr 2004 02:26:39 +0000 Subject: removed -f switch to cvs export --- db-arch | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/db-arch b/db-arch index 3c201c2..723cd46 100755 --- a/db-arch +++ b/db-arch @@ -1,5 +1,5 @@ #!/bin/bash -# $Id: db-arch,v 1.15 2004/04/02 00:48:50 judd Exp $ +# $Id: db-arch,v 1.16 2004/04/13 02:26:39 judd Exp $ uid=`id -u` repoid=1 @@ -37,7 +37,7 @@ mkdir $TMPDIR; [ $? -gt 0 ] && exit 1 echo "==> Generating Pacman Database for CURRENT..." >&2 cd $TMPDIR -CVS_RSH=ssh CVSROOT=:ext:cvs.archlinux.org:/home/cvs-arch cvs -q export -r CURRENT -f arch/build +CVS_RSH=ssh CVSROOT=:ext:cvs.archlinux.org:/home/cvs-arch cvs -q export -r CURRENT arch/build # check again if [ -d /tmp/.gensync ]; then echo "error: someone else is currently running gensync!" -- cgit v1.2.2 From 05b2f60d5ecaebc0cac22692a35bdaa7a7f7f0fa Mon Sep 17 00:00:00 2001 From: Judd Vinet Date: Fri, 16 Apr 2004 18:13:46 +0000 Subject: added support for all 3 repos --- db-testing | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/db-testing b/db-testing index b576f27..7965d95 100755 --- a/db-testing +++ b/db-testing @@ -1,5 +1,5 @@ #!/bin/bash -# $Id: db-testing,v 1.2 2004/04/13 02:06:49 judd Exp $ +# $Id: db-testing,v 1.3 2004/04/16 18:13:46 judd Exp $ uid=`id -u` repoid=4 @@ -38,6 +38,11 @@ mkdir $TMPDIR; [ $? -gt 0 ] && exit 1 echo "==> Generating Pacman Database for TESTING..." >&2 cd $TMPDIR CVS_RSH=ssh CVSROOT=:ext:cvs.archlinux.org:/home/cvs-arch cvs -q export -r TESTING arch +mv arch/build/* . && rm -rf arch +CVS_RSH=ssh CVSROOT=:ext:cvs.archlinux.org:/home/cvs-extra cvs -q export -r TESTING extra +cp -a extra/* . && rm -rf extra +CVS_RSH=ssh CVSROOT=:ext:cvs.archlinux.org:/home/cvs-unstable cvs -q export -r TESTING unstable +cp -a unstable/* . && rm -rf unstable # check again if [ -d /tmp/.gensync ]; then echo "error: someone else is currently running gensync!" -- cgit v1.2.2 From 416d13a0b7da0f12056af2dadd90e1c6c6874ee7 Mon Sep 17 00:00:00 2001 From: Judd Vinet Date: Fri, 16 Apr 2004 18:14:18 +0000 Subject: added support for all 3 repos --- db-testing | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/db-testing b/db-testing index 7965d95..2e36f88 100755 --- a/db-testing +++ b/db-testing @@ -1,5 +1,5 @@ #!/bin/bash -# $Id: db-testing,v 1.3 2004/04/16 18:13:46 judd Exp $ +# $Id: db-testing,v 1.4 2004/04/16 18:14:18 judd Exp $ uid=`id -u` repoid=4 @@ -48,10 +48,10 @@ if [ -d /tmp/.gensync ]; then echo "error: someone else is currently running gensync!" exit 1 fi -/usr/bin/gensync $TMPDIR/arch/build $TMPDIR/testing.db.tar.gz /home/ftp/testing +/usr/bin/gensync $TMPDIR $TMPDIR/testing.db.tar.gz /home/ftp/testing [ -f $TMPDIR/testing.db.tar.gz ] && mv -f $TMPDIR/testing.db.tar.gz /home/ftp/testing -cd $TMPDIR/arch/build && /arch/pkgdb $repoid +cd $TMPDIR && /arch/pkgdb $repoid echo "==> Generating Text Package List..." >&2 /arch/genpkglist testing -- cgit v1.2.2 From bada0b43fd3c682ad1cfd2b9733649bdc1dd3a49 Mon Sep 17 00:00:00 2001 From: Judd Vinet Date: Fri, 16 Apr 2004 18:15:38 +0000 Subject: added support for all 3 repos --- db-testing | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/db-testing b/db-testing index 2e36f88..ab4850d 100755 --- a/db-testing +++ b/db-testing @@ -1,5 +1,5 @@ #!/bin/bash -# $Id: db-testing,v 1.4 2004/04/16 18:14:18 judd Exp $ +# $Id: db-testing,v 1.5 2004/04/16 18:15:38 judd Exp $ uid=`id -u` repoid=4 @@ -37,10 +37,13 @@ mkdir $TMPDIR; [ $? -gt 0 ] && exit 1 echo "==> Generating Pacman Database for TESTING..." >&2 cd $TMPDIR +echo "--- CURRENT ---" CVS_RSH=ssh CVSROOT=:ext:cvs.archlinux.org:/home/cvs-arch cvs -q export -r TESTING arch mv arch/build/* . && rm -rf arch +echo "--- EXTRA ---" CVS_RSH=ssh CVSROOT=:ext:cvs.archlinux.org:/home/cvs-extra cvs -q export -r TESTING extra cp -a extra/* . && rm -rf extra +echo "--- UNSTABLE ---" CVS_RSH=ssh CVSROOT=:ext:cvs.archlinux.org:/home/cvs-unstable cvs -q export -r TESTING unstable cp -a unstable/* . && rm -rf unstable # check again -- cgit v1.2.2 From 985b10cd15a230a461a8482de6b7fe134c0046e3 Mon Sep 17 00:00:00 2001 From: Judd Vinet Date: Thu, 6 May 2004 01:39:42 +0000 Subject: package updates now automatically mark todo list items completed --- pkgdb | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pkgdb b/pkgdb index 4ccb54f..5d05a65 100755 --- a/pkgdb +++ b/pkgdb @@ -1,5 +1,5 @@ #!/bin/bash -# $Id: pkgdb,v 1.12 2004/04/01 21:04:14 judd Exp $ +# $Id: pkgdb,v 1.13 2004/05/06 01:39:42 judd Exp $ tl=`pwd` @@ -34,6 +34,9 @@ updatepkg() UPDATE packages SET categoryid='$categoryid',pkgname='$pkgname',pkgver='$pkgver', pkgrel='$pkgrel',pkgdesc='$pkgdesc',url='$url',sources='$sources', depends='$deplist',needupdate=0,lastupdate=NOW() WHERE id='$id'; +_EOF + mysql -u $dbuser -p$dbpass $dbname <<_EOF +UPDATE todolist_pkgs SET complete=1 WHERE pkgid='$id'; _EOF } -- cgit v1.2.2 From af7a8885d8095ac78c8d4f4a5f183bb038c5e17a Mon Sep 17 00:00:00 2001 From: Judd Vinet Date: Thu, 6 May 2004 02:27:40 +0000 Subject: fixed package deletion with new todolist_pkgs table --- pkgdb | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pkgdb b/pkgdb index 5d05a65..d2cf510 100755 --- a/pkgdb +++ b/pkgdb @@ -1,5 +1,5 @@ #!/bin/bash -# $Id: pkgdb,v 1.13 2004/05/06 01:39:42 judd Exp $ +# $Id: pkgdb,v 1.14 2004/05/06 02:27:40 judd Exp $ tl=`pwd` @@ -24,6 +24,9 @@ deletepkg() echo "Deleting $pkgname" mysql -u $dbuser -p$dbpass $dbname <<_EOF DELETE FROM packages WHERE id='$id'; +_EOF + mysql -u $dbuser -p$dbpass $dbname <<_EOF +DELETE FROM todolist_pkgs WHERE pkgid='$id'; _EOF } -- cgit v1.2.2 From 93e510e081756fb2aebc9079a52cd9a046fae69b Mon Sep 17 00:00:00 2001 From: Judd Vinet Date: Thu, 1 Jul 2004 19:41:21 +0000 Subject: added new pkgdb scripts --- db-arch | 6 +- pkgdb1 | 39 +++++++++++ pkgdb2.c | 233 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 276 insertions(+), 2 deletions(-) create mode 100755 pkgdb1 create mode 100644 pkgdb2.c diff --git a/db-arch b/db-arch index 723cd46..3dbd3dc 100755 --- a/db-arch +++ b/db-arch @@ -1,5 +1,5 @@ #!/bin/bash -# $Id: db-arch,v 1.16 2004/04/13 02:26:39 judd Exp $ +# $Id: db-arch,v 1.17 2004/07/01 19:41:21 judd Exp $ uid=`id -u` repoid=1 @@ -46,7 +46,9 @@ fi /usr/bin/gensync $TMPDIR/arch/build $TMPDIR/current.db.tar.gz /home/ftp/current [ -f $TMPDIR/current.db.tar.gz ] && mv -f $TMPDIR/current.db.tar.gz /home/ftp/current -cd $TMPDIR/arch/build && /arch/pkgdb $repoid +echo "==> Scanning for New/Updated/Deleted packages..." >&2 +cd $TMPDIR/arch/build +/arch/pkgdb1 $repoid | /arch/pkgdb2 $repoid echo "==> Generating Text Package List..." >&2 /arch/genpkglist current diff --git a/pkgdb1 b/pkgdb1 new file mode 100755 index 0000000..5c5c93d --- /dev/null +++ b/pkgdb1 @@ -0,0 +1,39 @@ +#!/bin/bash +# $Id: pkgdb1,v 1.1 2004/07/01 19:41:21 judd Exp $ + +tl=`pwd` + +repoid=$1 +if [ "$repoid" = "" ]; then + me=`basename $0` + echo "usage: $me " + exit 1 +fi + +cd $tl +for category in `find . -type d -maxdepth 1 | grep -v CVS | grep -v "^extra" | grep -v "^unstable"`; do + category=`echo $category | sed 's|./||'` + if [ "$category" = "." ]; then + continue + fi + for pkg in `ls $category | sort`; do + cd $tl/$category/$pkg + if [ -f PKGBUILD ]; then + . PKGBUILD + deplist=${depends[@]} + sources=${source[@]} + echo $pkgname + echo $pkgver + echo $pkgrel + echo $pkgdesc + echo $category + echo $url + echo $sources + echo $deplist + fi + cd $tl + done +done + +exit 0 + diff --git a/pkgdb2.c b/pkgdb2.c new file mode 100644 index 0000000..3576e30 --- /dev/null +++ b/pkgdb2.c @@ -0,0 +1,233 @@ +/* $Id: pkgdb2.c,v 1.1 2004/07/01 19:41:21 judd Exp $ */ + +#include +#include +#include +#include + +typedef struct pkg { + unsigned int id; + char *name; + char *ver; + char *rel; + struct pkg *next; +} pkg_t; + +MYSQL_RES *doquery(MYSQL *m, const char* q) +{ + MYSQL_RES *res; + if(mysql_query(m, q)) { + fprintf(stderr, "mysql_query: %s\n", mysql_error(m)); + return(NULL); + } + res = mysql_store_result(m); + return(res); +} + +/* this function is ugly -- it malloc's for each string it + * returns, and they probably won't be freed by the caller. + */ +char* addslashes(const char *s) { + char slashed[4096]; + char *p; + + slashed[0] = '\0'; + p = slashed; + while(*s) { + if(*s == '\'' || *s == '"' || *s == '\\') { + *p++ = '\\'; + } + *p++ = *s++; + } + *p = '\0'; + return(strdup(slashed)); +} + +char* trim(char *str) +{ + char *pch = str; + while(isspace(*pch)) { + pch++; + } + if(pch != str) { + memmove(str, pch, (strlen(pch) + 1)); + } + + pch = (char*)(str + (strlen(str) - 1)); + while(isspace(*pch)) { + pch--; + } + *++pch = '\0'; + + return str; +} + +int main(int argc, char **argv) +{ + MYSQL db; + MYSQL_RES *result; + MYSQL_ROW row; + char query[4096]; + int repoid; + pkg_t *dblist = NULL; + pkg_t *pkglist = NULL; + pkg_t *pkgptr, *ptr; + + if(argc < 2) { + printf("usage: pkgdb2 \n"); + printf("\nWARNING: Do not run this manually! It is intended to be" + " run with pkgdb1 only.\n"); + return(1); + } + repoid = atoi(argv[1]); + + if(mysql_init(&db) == NULL) { + fprintf(stderr, "could not initialize\n"); + return(1); + } + if(mysql_real_connect(&db, "localhost", "archweb", "passwords-are-cool", + "archweb", 0, NULL, 0) == NULL) { + fprintf(stderr, "failed to connect to database: %s\n", mysql_error(&db)); + return(1); + } + snprintf(query, sizeof(query), "SELECT id,pkgname,pkgver,pkgrel FROM packages " + "WHERE repoid='%d'", repoid); + result = doquery(&db, query); + while(row = mysql_fetch_row(result)) { + int i; + /*unsigned long *lengths; + lengths = mysql_fetch_lengths(result);*/ + /* add the node to the list */ + if(dblist == NULL) { + dblist = (pkg_t*)malloc(sizeof(pkg_t)); + if(dblist == NULL) { + fprintf(stderr, "error: out of memory!\n"); + return(1); + } + ptr = dblist; + } else { + ptr->next = (pkg_t*)malloc(sizeof(pkg_t)); + if(ptr->next == NULL) { + fprintf(stderr, "error: out of memory!\n"); + return(1); + } + ptr = ptr->next; + } + ptr->next = NULL; + /* pick out the fields */ + ptr->id = atoi(row[0]); + ptr->name = strdup(row[1]); + ptr->ver = strdup(row[2]); + ptr->rel = strdup(row[3]); + } + mysql_free_result(result); + + while(!feof(stdin)) { + int found = 0; + unsigned int catid = 0; + char name[256], ver[256], rel[256], desc[256]; + char cat[256], url[256], sources[4096], deplist[4096]; + /* get package data from stdin */ + fgets(name, 256, stdin); trim(name); if(feof(stdin)) continue; + fgets(ver, 256, stdin); trim(ver); if(feof(stdin)) continue; + fgets(rel, 256, stdin); trim(rel); if(feof(stdin)) continue; + fgets(desc, 256, stdin); trim(desc); if(feof(stdin)) continue; + fgets(cat, 256, stdin); trim(cat); if(feof(stdin)) continue; + fgets(url, 256, stdin); trim(url); if(feof(stdin)) continue; + fgets(sources, 4096, stdin); trim(sources);if(feof(stdin)) continue; + fgets(deplist, 4096, stdin); trim(deplist);if(feof(stdin)) continue; + /* add the node to the list */ + if(pkglist == NULL) { + pkglist = (pkg_t*)malloc(sizeof(pkg_t)); + if(pkglist == NULL) { + fprintf(stderr, "error: out of memory!\n"); + return(1); + } + pkgptr = pkglist; + } else { + pkgptr->next = (pkg_t*)malloc(sizeof(pkg_t)); + if(pkgptr->next == NULL) { + fprintf(stderr, "error: out of memory!\n"); + return(1); + } + pkgptr = pkgptr->next; + } + pkgptr->next = NULL; + pkgptr->name = strdup(name); + /* look it up in our cache */ + for(ptr = dblist; ptr; ptr = ptr->next) { + if(!strcmp(name, ptr->name)) { + found = 1; + break; + } + } + /* get the category */ + snprintf(query, sizeof(query), + "SELECT id FROM categories WHERE category='%s'", cat); + result = doquery(&db, query); + if(mysql_num_rows(result) == 0) { + fprintf(stderr, "pkgdb2: no db category found for '%s'\n", cat); + /* + snprintf(query, sizeof(query), "INSERT INTO categories (id,category) " + " VALUES (NULL,'%s')", addslashes(cat)); + doquery(&db, query); + catid = (unsigned int)mysql_insert_id(&db); + */ + } else { + row = mysql_fetch_row(result); + catid = (unsigned int)atoi(row[0]); + } + if(!found) { + /* Insert... */ + printf("pkgdb2: inserting %s\n", name); + snprintf(query, sizeof(query), "INSERT INTO packages (id,repoid," + "categoryid,pkgname,pkgver,pkgrel,pkgdesc,url,sources,depends," + "lastupdate) VALUES (NULL,'%d','%d','%s','%s','%s','%s'," + "'%s','%s','%s',NOW())", + repoid, catid, addslashes(name), addslashes(ver), addslashes(rel), + addslashes(desc), addslashes(url), addslashes(sources), + addslashes(deplist)); + doquery(&db, query); + continue; + } else if(strcmp(ptr->ver, ver) || strcmp(ptr->rel, rel)) { + /* ...or Update */ + printf("pkgdb2: updating %s (%s-%s ==> %s-%s)\n", + ptr->name, ptr->ver, ptr->rel, ver, rel); + snprintf(query, sizeof(query), "UPDATE packages SET categoryid='%d'," + "pkgname='%s',pkgver='%s',pkgrel='%s',pkgdesc='%s',url='%s'," + "sources='%s',depends='%s',needupdate=0,lastupdate=NOW() " + "WHERE id='%d'", + catid, addslashes(name), addslashes(ver), addslashes(rel), + addslashes(desc), addslashes(url), addslashes(sources), + addslashes(deplist), ptr->id); + doquery(&db, query); + snprintf(query, sizeof(query), "UPDATE todolist_pkgs SET complete=1 " + "WHERE pkgid='%d'", ptr->id); + doquery(&db, query); + } + } + + /* look for delete packages */ + for(ptr = dblist; ptr; ptr = ptr->next) { + int found = 0; + for(pkgptr = pkglist; pkgptr; pkgptr = pkgptr->next) { + if(!strcmp(ptr->name, pkgptr->name)) { + found = 1; + break; + } + } + if(!found) { + /* delete from db */ + printf("pkgdb2: deleting %s\n", ptr->name); + snprintf(query, sizeof(query), "DELETE FROM packages WHERE id='%d'", + ptr->id); + doquery(&db, query); + snprintf(query, sizeof(query), "DELETE FROM todolist_pkgs WHERE id='%d'", + ptr->id); + doquery(&db, query); + } + } + + mysql_close(&db); + return(0); +} -- cgit v1.2.2 From d2686502b928b98f6dad9ec13786973ef095ef8d Mon Sep 17 00:00:00 2001 From: Judd Vinet Date: Thu, 1 Jul 2004 20:26:27 +0000 Subject: lengthened the pkgdesc string --- pkgdb2.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/pkgdb2.c b/pkgdb2.c index 3576e30..84bab60 100644 --- a/pkgdb2.c +++ b/pkgdb2.c @@ -1,4 +1,4 @@ -/* $Id: pkgdb2.c,v 1.1 2004/07/01 19:41:21 judd Exp $ */ +/* $Id: pkgdb2.c,v 1.2 2004/07/01 20:26:27 judd Exp $ */ #include #include @@ -125,17 +125,17 @@ int main(int argc, char **argv) while(!feof(stdin)) { int found = 0; unsigned int catid = 0; - char name[256], ver[256], rel[256], desc[256]; + char name[256], ver[256], rel[256], desc[4096]; char cat[256], url[256], sources[4096], deplist[4096]; /* get package data from stdin */ - fgets(name, 256, stdin); trim(name); if(feof(stdin)) continue; - fgets(ver, 256, stdin); trim(ver); if(feof(stdin)) continue; - fgets(rel, 256, stdin); trim(rel); if(feof(stdin)) continue; - fgets(desc, 256, stdin); trim(desc); if(feof(stdin)) continue; - fgets(cat, 256, stdin); trim(cat); if(feof(stdin)) continue; - fgets(url, 256, stdin); trim(url); if(feof(stdin)) continue; - fgets(sources, 4096, stdin); trim(sources);if(feof(stdin)) continue; - fgets(deplist, 4096, stdin); trim(deplist);if(feof(stdin)) continue; + fgets(name, 256, stdin); trim(name); if(feof(stdin)) continue; + fgets(ver, 256, stdin); trim(ver); if(feof(stdin)) continue; + fgets(rel, 256, stdin); trim(rel); if(feof(stdin)) continue; + fgets(desc, 4096, stdin); trim(desc); if(feof(stdin)) continue; + fgets(cat, 256, stdin); trim(cat); if(feof(stdin)) continue; + fgets(url, 256, stdin); trim(url); if(feof(stdin)) continue; + fgets(sources, 4096, stdin); trim(sources); if(feof(stdin)) continue; + fgets(deplist, 4096, stdin); trim(deplist); if(feof(stdin)) continue; /* add the node to the list */ if(pkglist == NULL) { pkglist = (pkg_t*)malloc(sizeof(pkg_t)); -- cgit v1.2.2 From b2290a1dd4ca481986fcd1f83f999828c405caa0 Mon Sep 17 00:00:00 2001 From: Judd Vinet Date: Fri, 2 Jul 2004 00:17:52 +0000 Subject: enforced line lengths from stdin --- pkgdb2.c | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/pkgdb2.c b/pkgdb2.c index 84bab60..7b21600 100644 --- a/pkgdb2.c +++ b/pkgdb2.c @@ -1,4 +1,4 @@ -/* $Id: pkgdb2.c,v 1.2 2004/07/01 20:26:27 judd Exp $ */ +/* $Id: pkgdb2.c,v 1.3 2004/07/02 00:17:52 judd Exp $ */ #include #include @@ -28,7 +28,7 @@ MYSQL_RES *doquery(MYSQL *m, const char* q) * returns, and they probably won't be freed by the caller. */ char* addslashes(const char *s) { - char slashed[4096]; + char slashed[8192]; char *p; slashed[0] = '\0'; @@ -136,6 +136,15 @@ int main(int argc, char **argv) fgets(url, 256, stdin); trim(url); if(feof(stdin)) continue; fgets(sources, 4096, stdin); trim(sources); if(feof(stdin)) continue; fgets(deplist, 4096, stdin); trim(deplist); if(feof(stdin)) continue; + /* check for overruns */ + if(strlen(name) > 254 || strlen(ver) >= 254 || strlen(rel) > 254 || + strlen(desc) > 4094 || strlen(cat) >= 254 || strlen(url) > 254 || + strlen(sources) > 4094 || strlen(deplist) > 4094) { + fprintf(stderr, "pkgdb2: one or more fields are too long in package '%s'\n", name); + fprintf(stderr, "pkgdb2: check the lengths of your strings, most are limited " + "to 255 chars, some are 4095\n"); + return(1); + } /* add the node to the list */ if(pkglist == NULL) { pkglist = (pkg_t*)malloc(sizeof(pkg_t)); -- cgit v1.2.2 From b59fb0b6a14a71bc8e29c6974eb2b77612209bb2 Mon Sep 17 00:00:00 2001 From: Judd Vinet Date: Fri, 2 Jul 2004 17:50:32 +0000 Subject: added unsets to fix bad url problems in updates --- pkgdb1 | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkgdb1 b/pkgdb1 index 5c5c93d..b32f0db 100755 --- a/pkgdb1 +++ b/pkgdb1 @@ -1,5 +1,5 @@ #!/bin/bash -# $Id: pkgdb1,v 1.1 2004/07/01 19:41:21 judd Exp $ +# $Id: pkgdb1,v 1.2 2004/07/02 17:50:32 judd Exp $ tl=`pwd` @@ -19,6 +19,8 @@ for category in `find . -type d -maxdepth 1 | grep -v CVS | grep -v "^extra" | g for pkg in `ls $category | sort`; do cd $tl/$category/$pkg if [ -f PKGBUILD ]; then + unset depends source deplist sources + unset pkgname pkgver pkgrel pkgdesc url . PKGBUILD deplist=${depends[@]} sources=${source[@]} -- cgit v1.2.2 From 543c4e812bcc6df5d8e437b4accfbc7e1a273534 Mon Sep 17 00:00:00 2001 From: Judd Vinet Date: Sat, 3 Jul 2004 20:28:11 +0000 Subject: updated scripts for new ftp paths, moved over to pkgdb1/pkgdb2 --- db-arch | 8 ++++---- db-extra | 10 ++++++---- db-testing | 10 ++++++---- db-unstable | 10 ++++++---- genpkglist | 8 ++++---- 5 files changed, 26 insertions(+), 20 deletions(-) diff --git a/db-arch b/db-arch index 3dbd3dc..7abc604 100755 --- a/db-arch +++ b/db-arch @@ -1,5 +1,5 @@ #!/bin/bash -# $Id: db-arch,v 1.17 2004/07/01 19:41:21 judd Exp $ +# $Id: db-arch,v 1.18 2004/07/03 20:28:11 judd Exp $ uid=`id -u` repoid=1 @@ -43,8 +43,8 @@ if [ -d /tmp/.gensync ]; then echo "error: someone else is currently running gensync!" exit 1 fi -/usr/bin/gensync $TMPDIR/arch/build $TMPDIR/current.db.tar.gz /home/ftp/current -[ -f $TMPDIR/current.db.tar.gz ] && mv -f $TMPDIR/current.db.tar.gz /home/ftp/current +/usr/bin/gensync $TMPDIR/arch/build $TMPDIR/current.db.tar.gz /home/ftp/current/os/i686 +[ -f $TMPDIR/current.db.tar.gz ] && mv -f $TMPDIR/current.db.tar.gz /home/ftp/current/os/i686 echo "==> Scanning for New/Updated/Deleted packages..." >&2 cd $TMPDIR/arch/build @@ -52,6 +52,6 @@ cd $TMPDIR/arch/build echo "==> Generating Text Package List..." >&2 /arch/genpkglist current -mv packages.txt /home/ftp/current/setup/packages.txt +mv packages.txt /home/ftp/current/os/i686/setup/packages.txt cleanup diff --git a/db-extra b/db-extra index 204366f..689ac3f 100755 --- a/db-extra +++ b/db-extra @@ -1,5 +1,5 @@ #!/bin/bash -# $Id: db-extra,v 1.5 2004/04/01 21:09:13 judd Exp $ +# $Id: db-extra,v 1.6 2004/07/03 20:28:11 judd Exp $ uid=`id -u` repoid=2 @@ -43,10 +43,12 @@ if [ -d /tmp/.gensync ]; then echo "error: someone else is currently running gensync!" exit 1 fi -/usr/bin/gensync $TMPDIR/extra $TMPDIR/extra.db.tar.gz /home/ftp/extra -[ -f $TMPDIR/extra.db.tar.gz ] && mv -f $TMPDIR/extra.db.tar.gz /home/ftp/extra +/usr/bin/gensync $TMPDIR/extra $TMPDIR/extra.db.tar.gz /home/ftp/extra/os/i686 +[ -f $TMPDIR/extra.db.tar.gz ] && mv -f $TMPDIR/extra.db.tar.gz /home/ftp/extra/os/i686 -cd $TMPDIR/extra && /arch/pkgdb $repoid +echo "==> Scanning for New/Updated/Deleted packages..." >&2 +cd $TMPDIR/extra +/arch/pkgdb1 $repoid | /arch/pkgdb2 $repoid echo "==> Scanning for missing packages..." >&2 /arch/genpkglist extra diff --git a/db-testing b/db-testing index ab4850d..d0991a7 100755 --- a/db-testing +++ b/db-testing @@ -1,5 +1,5 @@ #!/bin/bash -# $Id: db-testing,v 1.5 2004/04/16 18:15:38 judd Exp $ +# $Id: db-testing,v 1.6 2004/07/03 20:28:11 judd Exp $ uid=`id -u` repoid=4 @@ -51,10 +51,12 @@ if [ -d /tmp/.gensync ]; then echo "error: someone else is currently running gensync!" exit 1 fi -/usr/bin/gensync $TMPDIR $TMPDIR/testing.db.tar.gz /home/ftp/testing -[ -f $TMPDIR/testing.db.tar.gz ] && mv -f $TMPDIR/testing.db.tar.gz /home/ftp/testing +/usr/bin/gensync $TMPDIR $TMPDIR/testing.db.tar.gz /home/ftp/testing/os/i686 +[ -f $TMPDIR/testing.db.tar.gz ] && mv -f $TMPDIR/testing.db.tar.gz /home/ftp/testing/os/i686 -cd $TMPDIR && /arch/pkgdb $repoid +echo "==> Scanning for New/Updated/Deleted packages..." >&2 +cd $TMPDIR +/arch/pkgdb1 $repoid | /arch/pkgdb2 $repoid echo "==> Generating Text Package List..." >&2 /arch/genpkglist testing diff --git a/db-unstable b/db-unstable index 6e0efa5..0ab9983 100755 --- a/db-unstable +++ b/db-unstable @@ -1,5 +1,5 @@ #!/bin/bash -# $Id: db-unstable,v 1.10 2004/04/01 21:09:13 judd Exp $ +# $Id: db-unstable,v 1.11 2004/07/03 20:28:11 judd Exp $ uid=`id -u` repoid=3 @@ -43,10 +43,12 @@ if [ -d /tmp/.gensync ]; then echo "error: someone else is currently running gensync!" exit 1 fi -/usr/bin/gensync $TMPDIR/unstable $TMPDIR/unstable.db.tar.gz /home/ftp/unstable -[ -f $TMPDIR/unstable.db.tar.gz ] && mv -f $TMPDIR/unstable.db.tar.gz /home/ftp/unstable +/usr/bin/gensync $TMPDIR/unstable $TMPDIR/unstable.db.tar.gz /home/ftp/unstable/os/i686 +[ -f $TMPDIR/unstable.db.tar.gz ] && mv -f $TMPDIR/unstable.db.tar.gz /home/ftp/unstable/os/i686 -cd $TMPDIR/unstable && /arch/pkgdb $repoid +echo "==> Scanning for New/Updated/Deleted packages..." >&2 +cd $TMPDIR/unstable +/arch/pkgdb1 $repoid | /arch/pkgdb2 $repoid echo "==> Scanning for missing packages..." >&2 /arch/genpkglist unstable diff --git a/genpkglist b/genpkglist index b740a2c..ec50488 100755 --- a/genpkglist +++ b/genpkglist @@ -1,5 +1,5 @@ #!/bin/bash -# $Id: genpkglist,v 1.10 2004/03/17 18:14:15 judd Exp $ +# $Id: genpkglist,v 1.11 2004/07/03 20:28:11 judd Exp $ # # genpkglist @@ -18,7 +18,7 @@ for category in `find * -type d -maxdepth 0 | grep -v CVS`; do cd $pkg if [ -f PKGBUILD ]; then . PKGBUILD - if [ -f /home/ftp/$repodir/$pkgname-$pkgver-$pkgrel.pkg.tar.gz ]; then + if [ -f /home/ftp/$repodir/os/i686/$pkgname-$pkgver-$pkgrel.pkg.tar.gz ]; then echo "$category/$pkgname-$pkgver-$pkgrel.pkg.tar.gz" >>$pkgfile else echo "notice: Missing $pkgname-$pkgver-$pkgrel.pkg.tar.gz in ftp site" >&2 @@ -29,11 +29,11 @@ for category in `find * -type d -maxdepth 0 | grep -v CVS`; do cd .. done -DUPES=`ls -1 /home/ftp/$1 | rev | cut -d- -f 3- | rev | sort | uniq -c | egrep -v '^ 1' | awk '{print $2}'` +DUPES=`ls -1 /home/ftp/$1/os/i686 | rev | cut -d- -f 3- | rev | sort | uniq -c | egrep -v '^ 1' | awk '{print $2}'` if [ -n "$DUPES" ]; then echo "Possible Dupes for $1 (please remove old versions)." echo "Date Filename" - for dupe in $DUPES; do ls -l /home/ftp/$1/${dupe}* | awk '{print $6" "$7" "$8" "$9}'; done + for dupe in $DUPES; do ls -l /home/ftp/$1/os/i686/${dupe}* | awk '{print $6" "$7" "$8" "$9}'; done fi -- cgit v1.2.2 From de737c171f2d57baa3e01abe01f7afd588ec1753 Mon Sep 17 00:00:00 2001 From: Judd Vinet Date: Sun, 11 Jul 2004 20:45:21 +0000 Subject: fixed todolist_pkgs table ref in pkgdb2 --- pkgdb2.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgdb2.c b/pkgdb2.c index 7b21600..127c31c 100644 --- a/pkgdb2.c +++ b/pkgdb2.c @@ -1,4 +1,4 @@ -/* $Id: pkgdb2.c,v 1.3 2004/07/02 00:17:52 judd Exp $ */ +/* $Id: pkgdb2.c,v 1.4 2004/07/11 20:45:21 judd Exp $ */ #include #include @@ -231,7 +231,7 @@ int main(int argc, char **argv) snprintf(query, sizeof(query), "DELETE FROM packages WHERE id='%d'", ptr->id); doquery(&db, query); - snprintf(query, sizeof(query), "DELETE FROM todolist_pkgs WHERE id='%d'", + snprintf(query, sizeof(query), "DELETE FROM todolist_pkgs WHERE listid='%d'", ptr->id); doquery(&db, query); } -- cgit v1.2.2 From 356342c65bcec5ab91680bd0118e566b6f0d90b2 Mon Sep 17 00:00:00 2001 From: Judd Vinet Date: Tue, 3 Aug 2004 00:24:55 +0000 Subject: disabled the auto-mark-as-complete for todo lists --- pkgdb2.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkgdb2.c b/pkgdb2.c index 127c31c..a113a87 100644 --- a/pkgdb2.c +++ b/pkgdb2.c @@ -1,4 +1,4 @@ -/* $Id: pkgdb2.c,v 1.4 2004/07/11 20:45:21 judd Exp $ */ +/* $Id: pkgdb2.c,v 1.5 2004/08/03 00:24:55 judd Exp $ */ #include #include @@ -210,9 +210,11 @@ int main(int argc, char **argv) addslashes(desc), addslashes(url), addslashes(sources), addslashes(deplist), ptr->id); doquery(&db, query); + /* snprintf(query, sizeof(query), "UPDATE todolist_pkgs SET complete=1 " "WHERE pkgid='%d'", ptr->id); doquery(&db, query); + */ } } -- cgit v1.2.2 From 5c8c26b650008d7af1305d5e304fb5a2c8f3e85c Mon Sep 17 00:00:00 2001 From: Judd Vinet Date: Tue, 17 Aug 2004 19:17:36 +0000 Subject: added some sanity checks around the cvs export calls --- db-arch | 6 +++++- db-extra | 6 +++++- db-testing | 14 +++++++++++++- db-unstable | 6 +++++- 4 files changed, 28 insertions(+), 4 deletions(-) diff --git a/db-arch b/db-arch index 7abc604..823de44 100755 --- a/db-arch +++ b/db-arch @@ -1,5 +1,5 @@ #!/bin/bash -# $Id: db-arch,v 1.18 2004/07/03 20:28:11 judd Exp $ +# $Id: db-arch,v 1.19 2004/08/17 19:17:36 judd Exp $ uid=`id -u` repoid=1 @@ -38,6 +38,10 @@ mkdir $TMPDIR; [ $? -gt 0 ] && exit 1 echo "==> Generating Pacman Database for CURRENT..." >&2 cd $TMPDIR CVS_RSH=ssh CVSROOT=:ext:cvs.archlinux.org:/home/cvs-arch cvs -q export -r CURRENT arch/build +if [ $? -gt 0 ]; then + echo "==> CVS export failed!" + exit 1 +fi # check again if [ -d /tmp/.gensync ]; then echo "error: someone else is currently running gensync!" diff --git a/db-extra b/db-extra index 689ac3f..04f7307 100755 --- a/db-extra +++ b/db-extra @@ -1,5 +1,5 @@ #!/bin/bash -# $Id: db-extra,v 1.6 2004/07/03 20:28:11 judd Exp $ +# $Id: db-extra,v 1.7 2004/08/17 19:17:36 judd Exp $ uid=`id -u` repoid=2 @@ -38,6 +38,10 @@ mkdir $TMPDIR; [ $? -gt 0 ] && exit 1 echo "==> Generating Pacman Database for EXTRA..." >&2 cd $TMPDIR CVS_RSH=ssh CVSROOT=:ext:cvs.archlinux.org:/home/cvs-extra cvs -q export -r CURRENT extra +if [ $? -gt 0 ]; then + echo "==> CVS export failed!" + exit 1 +fi # check again if [ -d /tmp/.gensync ]; then echo "error: someone else is currently running gensync!" diff --git a/db-testing b/db-testing index d0991a7..255ce13 100755 --- a/db-testing +++ b/db-testing @@ -1,5 +1,5 @@ #!/bin/bash -# $Id: db-testing,v 1.6 2004/07/03 20:28:11 judd Exp $ +# $Id: db-testing,v 1.7 2004/08/17 19:17:36 judd Exp $ uid=`id -u` repoid=4 @@ -39,12 +39,24 @@ echo "==> Generating Pacman Database for TESTING..." >&2 cd $TMPDIR echo "--- CURRENT ---" CVS_RSH=ssh CVSROOT=:ext:cvs.archlinux.org:/home/cvs-arch cvs -q export -r TESTING arch +if [ $? -gt 0 ]; then + echo "==> CVS export failed!" + exit 1 +fi mv arch/build/* . && rm -rf arch echo "--- EXTRA ---" CVS_RSH=ssh CVSROOT=:ext:cvs.archlinux.org:/home/cvs-extra cvs -q export -r TESTING extra +if [ $? -gt 0 ]; then + echo "==> CVS export failed!" + exit 1 +fi cp -a extra/* . && rm -rf extra echo "--- UNSTABLE ---" CVS_RSH=ssh CVSROOT=:ext:cvs.archlinux.org:/home/cvs-unstable cvs -q export -r TESTING unstable +if [ $? -gt 0 ]; then + echo "==> CVS export failed!" + exit 1 +fi cp -a unstable/* . && rm -rf unstable # check again if [ -d /tmp/.gensync ]; then diff --git a/db-unstable b/db-unstable index 0ab9983..e562e58 100755 --- a/db-unstable +++ b/db-unstable @@ -1,5 +1,5 @@ #!/bin/bash -# $Id: db-unstable,v 1.11 2004/07/03 20:28:11 judd Exp $ +# $Id: db-unstable,v 1.12 2004/08/17 19:17:36 judd Exp $ uid=`id -u` repoid=3 @@ -38,6 +38,10 @@ mkdir $TMPDIR; [ $? -gt 0 ] && exit 1 echo "==> Generating Pacman Database for UNSTABLE..." >&2 cd $TMPDIR CVS_RSH=ssh CVSROOT=:ext:cvs.archlinux.org:/home/cvs-unstable cvs -q export -r CURRENT unstable +if [ $? -gt 0 ]; then + echo "==> CVS export failed!" + exit 1 +fi # check again if [ -d /tmp/.gensync ]; then echo "error: someone else is currently running gensync!" -- cgit v1.2.2 From e11622c10fa4b75d264c78c668282f5b21e6f158 Mon Sep 17 00:00:00 2001 From: Judd Vinet Date: Thu, 19 Aug 2004 23:47:03 +0000 Subject: added cleanup calls to db- scripts, removed export-failure checks from db-testing, as its allowed for tesitng --- db-arch | 4 +++- db-extra | 4 +++- db-testing | 15 ++------------- db-unstable | 4 +++- 4 files changed, 11 insertions(+), 16 deletions(-) diff --git a/db-arch b/db-arch index 823de44..a3938c3 100755 --- a/db-arch +++ b/db-arch @@ -1,5 +1,5 @@ #!/bin/bash -# $Id: db-arch,v 1.19 2004/08/17 19:17:36 judd Exp $ +# $Id: db-arch,v 1.20 2004/08/19 23:47:03 judd Exp $ uid=`id -u` repoid=1 @@ -40,11 +40,13 @@ cd $TMPDIR CVS_RSH=ssh CVSROOT=:ext:cvs.archlinux.org:/home/cvs-arch cvs -q export -r CURRENT arch/build if [ $? -gt 0 ]; then echo "==> CVS export failed!" + cleanup exit 1 fi # check again if [ -d /tmp/.gensync ]; then echo "error: someone else is currently running gensync!" + cleanup exit 1 fi /usr/bin/gensync $TMPDIR/arch/build $TMPDIR/current.db.tar.gz /home/ftp/current/os/i686 diff --git a/db-extra b/db-extra index 04f7307..1413b80 100755 --- a/db-extra +++ b/db-extra @@ -1,5 +1,5 @@ #!/bin/bash -# $Id: db-extra,v 1.7 2004/08/17 19:17:36 judd Exp $ +# $Id: db-extra,v 1.8 2004/08/19 23:47:03 judd Exp $ uid=`id -u` repoid=2 @@ -40,11 +40,13 @@ cd $TMPDIR CVS_RSH=ssh CVSROOT=:ext:cvs.archlinux.org:/home/cvs-extra cvs -q export -r CURRENT extra if [ $? -gt 0 ]; then echo "==> CVS export failed!" + cleanup exit 1 fi # check again if [ -d /tmp/.gensync ]; then echo "error: someone else is currently running gensync!" + cleanup exit 1 fi /usr/bin/gensync $TMPDIR/extra $TMPDIR/extra.db.tar.gz /home/ftp/extra/os/i686 diff --git a/db-testing b/db-testing index 255ce13..a75ad97 100755 --- a/db-testing +++ b/db-testing @@ -1,5 +1,5 @@ #!/bin/bash -# $Id: db-testing,v 1.7 2004/08/17 19:17:36 judd Exp $ +# $Id: db-testing,v 1.8 2004/08/19 23:47:03 judd Exp $ uid=`id -u` repoid=4 @@ -39,28 +39,17 @@ echo "==> Generating Pacman Database for TESTING..." >&2 cd $TMPDIR echo "--- CURRENT ---" CVS_RSH=ssh CVSROOT=:ext:cvs.archlinux.org:/home/cvs-arch cvs -q export -r TESTING arch -if [ $? -gt 0 ]; then - echo "==> CVS export failed!" - exit 1 -fi mv arch/build/* . && rm -rf arch echo "--- EXTRA ---" CVS_RSH=ssh CVSROOT=:ext:cvs.archlinux.org:/home/cvs-extra cvs -q export -r TESTING extra -if [ $? -gt 0 ]; then - echo "==> CVS export failed!" - exit 1 -fi cp -a extra/* . && rm -rf extra echo "--- UNSTABLE ---" CVS_RSH=ssh CVSROOT=:ext:cvs.archlinux.org:/home/cvs-unstable cvs -q export -r TESTING unstable -if [ $? -gt 0 ]; then - echo "==> CVS export failed!" - exit 1 -fi cp -a unstable/* . && rm -rf unstable # check again if [ -d /tmp/.gensync ]; then echo "error: someone else is currently running gensync!" + cleanup exit 1 fi /usr/bin/gensync $TMPDIR $TMPDIR/testing.db.tar.gz /home/ftp/testing/os/i686 diff --git a/db-unstable b/db-unstable index e562e58..9826222 100755 --- a/db-unstable +++ b/db-unstable @@ -1,5 +1,5 @@ #!/bin/bash -# $Id: db-unstable,v 1.12 2004/08/17 19:17:36 judd Exp $ +# $Id: db-unstable,v 1.13 2004/08/19 23:47:03 judd Exp $ uid=`id -u` repoid=3 @@ -40,11 +40,13 @@ cd $TMPDIR CVS_RSH=ssh CVSROOT=:ext:cvs.archlinux.org:/home/cvs-unstable cvs -q export -r CURRENT unstable if [ $? -gt 0 ]; then echo "==> CVS export failed!" + cleanup exit 1 fi # check again if [ -d /tmp/.gensync ]; then echo "error: someone else is currently running gensync!" + cleanup exit 1 fi /usr/bin/gensync $TMPDIR/unstable $TMPDIR/unstable.db.tar.gz /home/ftp/unstable/os/i686 -- cgit v1.2.2 From f468dc2d2519a54667393c6b18b50ef17c4e27a5 Mon Sep 17 00:00:00 2001 From: Judd Vinet Date: Mon, 3 Jan 2005 08:09:21 +0000 Subject: re-arranged /bin/find options to get rid of an annoying warning message --- genpkglist | 4 ++-- pkgdb | 6 +++--- pkgdb1 | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/genpkglist b/genpkglist index ec50488..08111ce 100755 --- a/genpkglist +++ b/genpkglist @@ -1,5 +1,5 @@ #!/bin/bash -# $Id: genpkglist,v 1.11 2004/07/03 20:28:11 judd Exp $ +# $Id: genpkglist,v 1.12 2005/01/03 08:09:21 judd Exp $ # # genpkglist @@ -12,7 +12,7 @@ pkgfile="`pwd`/packages.txt" repodir=$1 rm -f $pkgfile -for category in `find * -type d -maxdepth 0 | grep -v CVS`; do +for category in `find * -maxdepth 0 -type d | grep -v CVS`; do cd $category for pkg in `/bin/ls`; do cd $pkg diff --git a/pkgdb b/pkgdb index d2cf510..e73c455 100755 --- a/pkgdb +++ b/pkgdb @@ -1,5 +1,5 @@ #!/bin/bash -# $Id: pkgdb,v 1.14 2004/05/06 02:27:40 judd Exp $ +# $Id: pkgdb,v 1.15 2005/01/03 08:09:21 judd Exp $ tl=`pwd` @@ -63,7 +63,7 @@ process() { oldrel=`echo $pkg | awk '{print $4}'` [ "$pkgname" = "" ] && return; cd $tl - pkgdir=`find . -type d -name "$pkgname" -maxdepth 2 -mindepth 1 | head -1` + pkgdir=`find . -maxdepth 2 -mindepth 1 -type d -name "$pkgname" | head -1` if [ "$pkgdir" = "" ]; then deletepkg else @@ -118,7 +118,7 @@ cat $TMPFILE | ( \ echo " -> Scanning for New packages..." cd $tl -for category in `find . -type d -maxdepth 1 | grep -v CVS | grep -v "^extra" | grep -v "^unstable"`; do +for category in `find . -maxdepth 1 -type d | grep -v CVS | grep -v "^extra" | grep -v "^unstable"`; do category=`echo $category | sed 's|./||'` if [ "$category" = "." ]; then continue diff --git a/pkgdb1 b/pkgdb1 index b32f0db..dd4e5d3 100755 --- a/pkgdb1 +++ b/pkgdb1 @@ -1,5 +1,5 @@ #!/bin/bash -# $Id: pkgdb1,v 1.2 2004/07/02 17:50:32 judd Exp $ +# $Id: pkgdb1,v 1.3 2005/01/03 08:11:19 judd Exp $ tl=`pwd` @@ -11,7 +11,7 @@ if [ "$repoid" = "" ]; then fi cd $tl -for category in `find . -type d -maxdepth 1 | grep -v CVS | grep -v "^extra" | grep -v "^unstable"`; do +for category in `find . -maxdepth 1 -type d | grep -v CVS | grep -v "^extra" | grep -v "^unstable"`; do category=`echo $category | sed 's|./||'` if [ "$category" = "." ]; then continue -- cgit v1.2.2 From ebd86ff4b54d7dbce2cc8c77293937cd688a801b Mon Sep 17 00:00:00 2001 From: Judd Vinet Date: Tue, 1 Feb 2005 03:00:47 +0000 Subject: added packages.txt to all repos --- db-extra | 3 ++- db-testing | 3 ++- db-unstable | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/db-extra b/db-extra index 1413b80..4c25389 100755 --- a/db-extra +++ b/db-extra @@ -1,5 +1,5 @@ #!/bin/bash -# $Id: db-extra,v 1.8 2004/08/19 23:47:03 judd Exp $ +# $Id: db-extra,v 1.9 2005/02/01 03:00:47 judd Exp $ uid=`id -u` repoid=2 @@ -58,5 +58,6 @@ cd $TMPDIR/extra echo "==> Scanning for missing packages..." >&2 /arch/genpkglist extra +mv packages.txt /home/ftp/extra/os/i686/packages.txt cleanup diff --git a/db-testing b/db-testing index a75ad97..af94104 100755 --- a/db-testing +++ b/db-testing @@ -1,5 +1,5 @@ #!/bin/bash -# $Id: db-testing,v 1.8 2004/08/19 23:47:03 judd Exp $ +# $Id: db-testing,v 1.9 2005/02/01 03:00:47 judd Exp $ uid=`id -u` repoid=4 @@ -61,5 +61,6 @@ cd $TMPDIR echo "==> Generating Text Package List..." >&2 /arch/genpkglist testing +mv packages.txt /home/ftp/testing/os/i686/packages.txt cleanup diff --git a/db-unstable b/db-unstable index 9826222..90f640d 100755 --- a/db-unstable +++ b/db-unstable @@ -1,5 +1,5 @@ #!/bin/bash -# $Id: db-unstable,v 1.13 2004/08/19 23:47:03 judd Exp $ +# $Id: db-unstable,v 1.14 2005/02/01 03:00:47 judd Exp $ uid=`id -u` repoid=3 @@ -58,5 +58,6 @@ cd $TMPDIR/unstable echo "==> Scanning for missing packages..." >&2 /arch/genpkglist unstable +mv packages.txt /home/ftp/unstable/os/i686/packages.txt cleanup -- cgit v1.2.2 From abccc814a02f7be0e97bade0892f80eb4c71f0de Mon Sep 17 00:00:00 2001 From: Judd Vinet Date: Fri, 25 Feb 2005 21:27:44 +0000 Subject: added filelists to the web package db --- db-arch | 12 ++++++---- db-extra | 12 ++++++---- db-testing | 12 ++++++---- db-unstable | 12 ++++++---- pkgdb2.c | 76 +++++++++++++++++++++++++++++++++++++++++++++++++++++-------- 5 files changed, 94 insertions(+), 30 deletions(-) diff --git a/db-arch b/db-arch index a3938c3..1f42a96 100755 --- a/db-arch +++ b/db-arch @@ -1,8 +1,10 @@ #!/bin/bash -# $Id: db-arch,v 1.20 2004/08/19 23:47:03 judd Exp $ +# $Id: db-arch,v 1.21 2005/02/25 21:27:44 judd Exp $ uid=`id -u` repoid=1 +ftppath="/home/ftp/current/os/i686" + TMPDIR="/tmp/archpkg.$repoid.$uid" cleanup() { @@ -49,15 +51,15 @@ if [ -d /tmp/.gensync ]; then cleanup exit 1 fi -/usr/bin/gensync $TMPDIR/arch/build $TMPDIR/current.db.tar.gz /home/ftp/current/os/i686 -[ -f $TMPDIR/current.db.tar.gz ] && mv -f $TMPDIR/current.db.tar.gz /home/ftp/current/os/i686 +/usr/bin/gensync $TMPDIR/arch/build $TMPDIR/current.db.tar.gz $ftppath +[ -f $TMPDIR/current.db.tar.gz ] && mv -f $TMPDIR/current.db.tar.gz $ftppath echo "==> Scanning for New/Updated/Deleted packages..." >&2 cd $TMPDIR/arch/build -/arch/pkgdb1 $repoid | /arch/pkgdb2 $repoid +/arch/pkgdb1 $repoid | /arch/pkgdb2 $repoid $ftppath echo "==> Generating Text Package List..." >&2 /arch/genpkglist current -mv packages.txt /home/ftp/current/os/i686/setup/packages.txt +mv packages.txt $ftppath/setup/packages.txt cleanup diff --git a/db-extra b/db-extra index 4c25389..c2f3574 100755 --- a/db-extra +++ b/db-extra @@ -1,8 +1,10 @@ #!/bin/bash -# $Id: db-extra,v 1.9 2005/02/01 03:00:47 judd Exp $ +# $Id: db-extra,v 1.10 2005/02/25 21:27:44 judd Exp $ uid=`id -u` repoid=2 +ftppath="/home/ftp/extra/os/i686" + TMPDIR="/tmp/archpkg.$repoid.$uid" cleanup() { @@ -49,15 +51,15 @@ if [ -d /tmp/.gensync ]; then cleanup exit 1 fi -/usr/bin/gensync $TMPDIR/extra $TMPDIR/extra.db.tar.gz /home/ftp/extra/os/i686 -[ -f $TMPDIR/extra.db.tar.gz ] && mv -f $TMPDIR/extra.db.tar.gz /home/ftp/extra/os/i686 +/usr/bin/gensync $TMPDIR/extra $TMPDIR/extra.db.tar.gz $ftppath +[ -f $TMPDIR/extra.db.tar.gz ] && mv -f $TMPDIR/extra.db.tar.gz $ftppath echo "==> Scanning for New/Updated/Deleted packages..." >&2 cd $TMPDIR/extra -/arch/pkgdb1 $repoid | /arch/pkgdb2 $repoid +/arch/pkgdb1 $repoid | /arch/pkgdb2 $repoid $ftppath echo "==> Scanning for missing packages..." >&2 /arch/genpkglist extra -mv packages.txt /home/ftp/extra/os/i686/packages.txt +mv packages.txt $ftppath/packages.txt cleanup diff --git a/db-testing b/db-testing index af94104..230c0f7 100755 --- a/db-testing +++ b/db-testing @@ -1,8 +1,10 @@ #!/bin/bash -# $Id: db-testing,v 1.9 2005/02/01 03:00:47 judd Exp $ +# $Id: db-testing,v 1.10 2005/02/25 21:27:44 judd Exp $ uid=`id -u` repoid=4 +ftppath="/home/ftp/testing/os/i686" + TMPDIR="/tmp/archpkg.$repoid.$uid" cleanup() { @@ -52,15 +54,15 @@ if [ -d /tmp/.gensync ]; then cleanup exit 1 fi -/usr/bin/gensync $TMPDIR $TMPDIR/testing.db.tar.gz /home/ftp/testing/os/i686 -[ -f $TMPDIR/testing.db.tar.gz ] && mv -f $TMPDIR/testing.db.tar.gz /home/ftp/testing/os/i686 +/usr/bin/gensync $TMPDIR $TMPDIR/testing.db.tar.gz $ftppath +[ -f $TMPDIR/testing.db.tar.gz ] && mv -f $TMPDIR/testing.db.tar.gz $ftppath echo "==> Scanning for New/Updated/Deleted packages..." >&2 cd $TMPDIR -/arch/pkgdb1 $repoid | /arch/pkgdb2 $repoid +/arch/pkgdb1 $repoid | /arch/pkgdb2 $repoid $ftppath echo "==> Generating Text Package List..." >&2 /arch/genpkglist testing -mv packages.txt /home/ftp/testing/os/i686/packages.txt +mv packages.txt $ftppath/packages.txt cleanup diff --git a/db-unstable b/db-unstable index 90f640d..5b5edad 100755 --- a/db-unstable +++ b/db-unstable @@ -1,8 +1,10 @@ #!/bin/bash -# $Id: db-unstable,v 1.14 2005/02/01 03:00:47 judd Exp $ +# $Id: db-unstable,v 1.15 2005/02/25 21:27:44 judd Exp $ uid=`id -u` repoid=3 +ftppath="/home/ftp/unstable/os/i686" + TMPDIR="/tmp/archpkg.$repoid.$uid" cleanup() { @@ -49,15 +51,15 @@ if [ -d /tmp/.gensync ]; then cleanup exit 1 fi -/usr/bin/gensync $TMPDIR/unstable $TMPDIR/unstable.db.tar.gz /home/ftp/unstable/os/i686 -[ -f $TMPDIR/unstable.db.tar.gz ] && mv -f $TMPDIR/unstable.db.tar.gz /home/ftp/unstable/os/i686 +/usr/bin/gensync $TMPDIR/unstable $TMPDIR/unstable.db.tar.gz $ftppath +[ -f $TMPDIR/unstable.db.tar.gz ] && mv -f $TMPDIR/unstable.db.tar.gz $ftppath echo "==> Scanning for New/Updated/Deleted packages..." >&2 cd $TMPDIR/unstable -/arch/pkgdb1 $repoid | /arch/pkgdb2 $repoid +/arch/pkgdb1 $repoid | /arch/pkgdb2 $repoid $ftppath echo "==> Scanning for missing packages..." >&2 /arch/genpkglist unstable -mv packages.txt /home/ftp/unstable/os/i686/packages.txt +mv packages.txt $ftppath/packages.txt cleanup diff --git a/pkgdb2.c b/pkgdb2.c index a113a87..9710cc5 100644 --- a/pkgdb2.c +++ b/pkgdb2.c @@ -1,9 +1,14 @@ -/* $Id: pkgdb2.c,v 1.5 2004/08/03 00:24:55 judd Exp $ */ +/* $Id: pkgdb2.c,v 1.6 2005/02/25 21:27:45 judd Exp $ */ #include #include #include #include +#include + +#define DB_USER "archweb" +#define DB_NAME "archweb" +#define DB_PASS "passwords-are-cool" typedef struct pkg { unsigned int id; @@ -62,31 +67,73 @@ char* trim(char *str) return str; } +/* scan a .pkg.tar.gz file and put all files listed into the database. + * + * this function is hacky and should be done properly, but this route is + * easier than reading the file with libtar. + */ +void updatefilelist(MYSQL *db, unsigned long id, char *fn) +{ + FILE *fp; + char *tmp; + char cmd[PATH_MAX]; + char line[PATH_MAX]; + char query[PATH_MAX]; + + tmp = tempnam("/tmp", "pkgdb"); + snprintf(cmd, PATH_MAX-1, "/bin/tar tzvf %s | awk '{print $6}' >%s", fn, tmp); + system(cmd); + fp = fopen(tmp, "r"); + if(fp == NULL) { + fprintf(stderr, "pkgdb2: could not open tempfile: %s\n", tmp); + return; + } + snprintf(query, sizeof(query), "DELETE FROM packages_files WHERE id='%d'", id); + doquery(db, query); + while(fgets(line, sizeof(line)-1, fp)) { + char *fixedfn = addslashes(trim(line)); + if(!strcmp(fixedfn, ".FILELIST") || !strcmp(fixedfn, ".PKGINFO") || !strcmp(fixedfn, ".INSTALL")) { + free(fixedfn); + continue; + } + /* varchars aren't case-sensitive but filesystems are, so we use REPLACE INTO */ + snprintf(query, sizeof(query), "REPLACE INTO packages_files (id,path) VALUES " + "('%d', '%s')", id, fixedfn); + free(fixedfn); + doquery(db, query); + } + fclose(fp); + unlink(tmp); +} + int main(int argc, char **argv) { MYSQL db; MYSQL_RES *result; MYSQL_ROW row; char query[4096]; + char fn[PATH_MAX]; + char ftppath[PATH_MAX]; int repoid; pkg_t *dblist = NULL; pkg_t *pkglist = NULL; pkg_t *pkgptr, *ptr; - if(argc < 2) { - printf("usage: pkgdb2 \n"); + if(argc < 3) { + printf("usage: pkgdb2 \n"); printf("\nWARNING: Do not run this manually! It is intended to be" " run with pkgdb1 only.\n"); return(1); } repoid = atoi(argv[1]); + strncpy(ftppath, argv[2], PATH_MAX-1); if(mysql_init(&db) == NULL) { fprintf(stderr, "could not initialize\n"); return(1); } - if(mysql_real_connect(&db, "localhost", "archweb", "passwords-are-cool", - "archweb", 0, NULL, 0) == NULL) { + if(mysql_real_connect(&db, "localhost", DB_USER, DB_PASS, DB_NAME, + 0, NULL, 0) == NULL) { fprintf(stderr, "failed to connect to database: %s\n", mysql_error(&db)); return(1); } @@ -188,7 +235,8 @@ int main(int argc, char **argv) } if(!found) { /* Insert... */ - printf("pkgdb2: inserting %s\n", name); + unsigned long id; + fprintf(stderr, "pkgdb2: inserting %s\n", name); snprintf(query, sizeof(query), "INSERT INTO packages (id,repoid," "categoryid,pkgname,pkgver,pkgrel,pkgdesc,url,sources,depends," "lastupdate) VALUES (NULL,'%d','%d','%s','%s','%s','%s'," @@ -197,10 +245,13 @@ int main(int argc, char **argv) addslashes(desc), addslashes(url), addslashes(sources), addslashes(deplist)); doquery(&db, query); + id = mysql_insert_id(&db); + snprintf(fn, PATH_MAX-1, "%s/%s-%s-%s.pkg.tar.gz", ftppath, name, ver, rel); + updatefilelist(&db, id, fn); continue; } else if(strcmp(ptr->ver, ver) || strcmp(ptr->rel, rel)) { /* ...or Update */ - printf("pkgdb2: updating %s (%s-%s ==> %s-%s)\n", + fprintf(stderr, "pkgdb2: updating %s (%s-%s ==> %s-%s)\n", ptr->name, ptr->ver, ptr->rel, ver, rel); snprintf(query, sizeof(query), "UPDATE packages SET categoryid='%d'," "pkgname='%s',pkgver='%s',pkgrel='%s',pkgdesc='%s',url='%s'," @@ -210,6 +261,8 @@ int main(int argc, char **argv) addslashes(desc), addslashes(url), addslashes(sources), addslashes(deplist), ptr->id); doquery(&db, query); + snprintf(fn, PATH_MAX-1, "%s/%s-%s-%s.pkg.tar.gz", ftppath, name, ver, rel); + updatefilelist(&db, ptr->id, fn); /* snprintf(query, sizeof(query), "UPDATE todolist_pkgs SET complete=1 " "WHERE pkgid='%d'", ptr->id); @@ -218,7 +271,7 @@ int main(int argc, char **argv) } } - /* look for delete packages */ + /* look for deleted packages */ for(ptr = dblist; ptr; ptr = ptr->next) { int found = 0; for(pkgptr = pkglist; pkgptr; pkgptr = pkgptr->next) { @@ -229,11 +282,14 @@ int main(int argc, char **argv) } if(!found) { /* delete from db */ - printf("pkgdb2: deleting %s\n", ptr->name); + fprintf(stderr, "pkgdb2: deleting %s\n", ptr->name); snprintf(query, sizeof(query), "DELETE FROM packages WHERE id='%d'", ptr->id); doquery(&db, query); - snprintf(query, sizeof(query), "DELETE FROM todolist_pkgs WHERE listid='%d'", + snprintf(query, sizeof(query), "DELETE FROM packages_files WHERE id='%d'", + ptr->id); + doquery(&db, query); + snprintf(query, sizeof(query), "DELETE FROM todolist_pkgs WHERE pkgid='%d'", ptr->id); doquery(&db, query); } -- cgit v1.2.2 From bdcdf586bdb095e833084c092d1cb79d3ed63523 Mon Sep 17 00:00:00 2001 From: Judd Vinet Date: Tue, 21 Jun 2005 03:44:57 +0000 Subject: nixed all the redundant code in the db- scripts --- db-arch | 65 +++++++------------------------------------------------------ db-extra | 65 +++++++------------------------------------------------------ db-inc | 59 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ db-unstable | 65 +++++++------------------------------------------------------ 4 files changed, 80 insertions(+), 174 deletions(-) create mode 100644 db-inc diff --git a/db-arch b/db-arch index 1f42a96..f237b3b 100755 --- a/db-arch +++ b/db-arch @@ -1,65 +1,14 @@ #!/bin/bash -# $Id: db-arch,v 1.21 2005/02/25 21:27:44 judd Exp $ +# $Id: db-arch,v 1.22 2005/06/21 03:44:57 judd Exp $ -uid=`id -u` repoid=1 -ftppath="/home/ftp/current/os/i686" - -TMPDIR="/tmp/archpkg.$repoid.$uid" - -cleanup() { - rm -rf $TMPDIR - # unlock - rm -f /tmp/.repolck.$repoid - exit 0 -} - -# check for locks -if [ -f /tmp/.repolck.$repoid ]; then - owner=`/bin/ls -l /tmp/.repolck.$repoid | awk '{print $3}'` - echo "error: db generation is already in progress (started by $owner)" - exit 1 -fi -if [ -d /tmp/.gensync ]; then - echo "error: someone else is currently running gensync!" - exit 1 -fi +reponame="current" -# catch ^C breaks -trap cleanup SIGINT -# lock -touch /tmp/.repolck.$repoid - -# RedHat's mktemp is broken... -if [ -d $TMPDIR ]; then - echo "==> Removing old temp dir..." >&2 - rm -rf $TMPDIR || exit 1 -fi -mkdir $TMPDIR; [ $? -gt 0 ] && exit 1 - -echo "==> Generating Pacman Database for CURRENT..." >&2 -cd $TMPDIR -CVS_RSH=ssh CVSROOT=:ext:cvs.archlinux.org:/home/cvs-arch cvs -q export -r CURRENT arch/build -if [ $? -gt 0 ]; then - echo "==> CVS export failed!" - cleanup - exit 1 -fi -# check again -if [ -d /tmp/.gensync ]; then - echo "error: someone else is currently running gensync!" - cleanup - exit 1 -fi -/usr/bin/gensync $TMPDIR/arch/build $TMPDIR/current.db.tar.gz $ftppath -[ -f $TMPDIR/current.db.tar.gz ] && mv -f $TMPDIR/current.db.tar.gz $ftppath +ftppath="/home/ftp/current/os/i686" -echo "==> Scanning for New/Updated/Deleted packages..." >&2 -cd $TMPDIR/arch/build -/arch/pkgdb1 $repoid | /arch/pkgdb2 $repoid $ftppath +cvspath="/home/cvs-arch" +cvsmod="arch/build" -echo "==> Generating Text Package List..." >&2 -/arch/genpkglist current -mv packages.txt $ftppath/setup/packages.txt +# call the real script +. `dirname $0`/db-inc -cleanup diff --git a/db-extra b/db-extra index c2f3574..be5fc3a 100755 --- a/db-extra +++ b/db-extra @@ -1,65 +1,14 @@ #!/bin/bash -# $Id: db-extra,v 1.10 2005/02/25 21:27:44 judd Exp $ +# $Id: db-extra,v 1.11 2005/06/21 03:44:57 judd Exp $ -uid=`id -u` repoid=2 -ftppath="/home/ftp/extra/os/i686" - -TMPDIR="/tmp/archpkg.$repoid.$uid" - -cleanup() { - rm -rf $TMPDIR - # unlock - rm -f /tmp/.repolck.$repoid - exit 0 -} - -# check for locks -if [ -f /tmp/.repolck.$repoid ]; then - owner=`/bin/ls -l /tmp/.repolck.$repoid | awk '{print $3}'` - echo "error: db generation is already in progress (started by $owner)" - exit 1 -fi -if [ -d /tmp/.gensync ]; then - echo "error: someone else is currently running gensync!" - exit 1 -fi +reponame="extra" -# catch ^C breaks -trap cleanup SIGINT -# lock -touch /tmp/.repolck.$repoid - -# RedHat's mktemp is broken... -if [ -d $TMPDIR ]; then - echo "==> Removing old temp dir..." >&2 - rm -rf $TMPDIR || exit 1 -fi -mkdir $TMPDIR; [ $? -gt 0 ] && exit 1 - -echo "==> Generating Pacman Database for EXTRA..." >&2 -cd $TMPDIR -CVS_RSH=ssh CVSROOT=:ext:cvs.archlinux.org:/home/cvs-extra cvs -q export -r CURRENT extra -if [ $? -gt 0 ]; then - echo "==> CVS export failed!" - cleanup - exit 1 -fi -# check again -if [ -d /tmp/.gensync ]; then - echo "error: someone else is currently running gensync!" - cleanup - exit 1 -fi -/usr/bin/gensync $TMPDIR/extra $TMPDIR/extra.db.tar.gz $ftppath -[ -f $TMPDIR/extra.db.tar.gz ] && mv -f $TMPDIR/extra.db.tar.gz $ftppath +ftppath="/home/ftp/extra/os/i686" -echo "==> Scanning for New/Updated/Deleted packages..." >&2 -cd $TMPDIR/extra -/arch/pkgdb1 $repoid | /arch/pkgdb2 $repoid $ftppath +cvspath="/home/cvs-extra" +cvsmod="extra" -echo "==> Scanning for missing packages..." >&2 -/arch/genpkglist extra -mv packages.txt $ftppath/packages.txt +# call the real script +. `dirname $0`/db-inc -cleanup diff --git a/db-inc b/db-inc new file mode 100644 index 0000000..bc059d6 --- /dev/null +++ b/db-inc @@ -0,0 +1,59 @@ +# $Id: db-inc,v 1.1 2005/06/21 03:44:57 judd Exp $ + +UID=`id -u` +TMPDIR="/tmp/archpkg.$repoid.$UID" + +cleanup() { + rm -rf $TMPDIR + # unlock + rm -f /tmp/.repolck.$repoid + exit 0 +} + +# check for locks +if [ -f /tmp/.repolck.$repoid ]; then + owner=`/bin/ls -l /tmp/.repolck.$repoid | awk '{print $3}'` + echo "error: db generation is already in progress (started by $owner)" + exit 1 +fi + +# catch ^C breaks +trap cleanup SIGINT +# lock +touch /tmp/.repolck.$repoid + +# RedHat's mktemp is broken... +if [ -d $TMPDIR ]; then + echo "==> Removing old temp dir..." >&2 + rm -rf $TMPDIR || exit 1 +fi +mkdir $TMPDIR; [ $? -gt 0 ] && exit 1 + +echo "==> Generating Pacman Database for repository '$reponame'..." >&2 +cd $TMPDIR +CVS_RSH=ssh CVSROOT=:ext:cvs.archlinux.org:$cvspath cvs -q export -r CURRENT $cvsmod +if [ $? -gt 0 ]; then + echo "==> CVS export failed!" + cleanup + exit 1 +fi +/usr/bin/gensync $TMPDIR/$cvsmod $TMPDIR/$reponame.db.tar.gz $ftppath +[ -f $TMPDIR/$reponame.db.tar.gz ] && mv -f $TMPDIR/$reponame.db.tar.gz $ftppath + +echo "==> Scanning for New/Updated/Deleted packages..." >&2 +cd $TMPDIR/$cvsmod +/arch/pkgdb1 $repoid | /arch/pkgdb2 $repoid $ftppath + +echo "==> Generating Text Package List..." >&2 +/arch/genpkglist $reponame + +# hack -- only Current's packages.txt goes in a "setup" subdir +if [ "$reponame" = "current" ]; then + mv packages.txt $ftppath/setup/packages.txt +else + mv packages.txt $ftppath/packages.txt +fi + +cleanup + +# vim: set ft=sh: diff --git a/db-unstable b/db-unstable index 5b5edad..a22188b 100755 --- a/db-unstable +++ b/db-unstable @@ -1,65 +1,14 @@ #!/bin/bash -# $Id: db-unstable,v 1.15 2005/02/25 21:27:44 judd Exp $ +# $Id: db-unstable,v 1.16 2005/06/21 03:44:57 judd Exp $ -uid=`id -u` repoid=3 -ftppath="/home/ftp/unstable/os/i686" - -TMPDIR="/tmp/archpkg.$repoid.$uid" - -cleanup() { - rm -rf $TMPDIR - # unlock - rm -f /tmp/.repolck.$repoid - exit 0 -} - -# check for locks -if [ -f /tmp/.repolck.$repoid ]; then - owner=`/bin/ls -l /tmp/.repolck.$repoid | awk '{print $3}'` - echo "error: db generation is already in progress (started by $owner)" - exit 1 -fi -if [ -d /tmp/.gensync ]; then - echo "error: someone else is currently running gensync!" - exit 1 -fi +reponame="unstable" -# catch ^C breaks -trap cleanup SIGINT -# lock -touch /tmp/.repolck.$repoid - -# RedHat's mktemp is broken... -if [ -d $TMPDIR ]; then - echo "==> Removing old temp dir..." >&2 - rm -rf $TMPDIR || exit 1 -fi -mkdir $TMPDIR; [ $? -gt 0 ] && exit 1 - -echo "==> Generating Pacman Database for UNSTABLE..." >&2 -cd $TMPDIR -CVS_RSH=ssh CVSROOT=:ext:cvs.archlinux.org:/home/cvs-unstable cvs -q export -r CURRENT unstable -if [ $? -gt 0 ]; then - echo "==> CVS export failed!" - cleanup - exit 1 -fi -# check again -if [ -d /tmp/.gensync ]; then - echo "error: someone else is currently running gensync!" - cleanup - exit 1 -fi -/usr/bin/gensync $TMPDIR/unstable $TMPDIR/unstable.db.tar.gz $ftppath -[ -f $TMPDIR/unstable.db.tar.gz ] && mv -f $TMPDIR/unstable.db.tar.gz $ftppath +ftppath="/home/ftp/unstable/os/i686" -echo "==> Scanning for New/Updated/Deleted packages..." >&2 -cd $TMPDIR/unstable -/arch/pkgdb1 $repoid | /arch/pkgdb2 $repoid $ftppath +cvspath="/home/cvs-unstable" +cvsmod="unstable" -echo "==> Scanning for missing packages..." >&2 -/arch/genpkglist unstable -mv packages.txt $ftppath/packages.txt +# call the real script +. `dirname $0`/db-inc -cleanup -- cgit v1.2.2 From 8c4c7c7ce6324bf1aa8ffb0b9c5e378d446166ef Mon Sep 17 00:00:00 2001 From: Judd Vinet Date: Tue, 21 Jun 2005 03:47:38 +0000 Subject: fixed a little bug in db-inc - UID is already present --- db-inc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/db-inc b/db-inc index bc059d6..9ca9852 100644 --- a/db-inc +++ b/db-inc @@ -1,6 +1,6 @@ -# $Id: db-inc,v 1.1 2005/06/21 03:44:57 judd Exp $ +# $Id: db-inc,v 1.2 2005/06/21 03:47:38 judd Exp $ -UID=`id -u` +[ "$UID" = "" ] && UID=`uid` TMPDIR="/tmp/archpkg.$repoid.$UID" cleanup() { -- cgit v1.2.2 From 7c99f3e12b92f5112f3f79bf55deebcab6c3fc11 Mon Sep 17 00:00:00 2001 From: Judd Vinet Date: Thu, 30 Jun 2005 22:30:36 +0000 Subject: run gensync through fakeroot first, so all files are owned by root.root --- db-inc | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/db-inc b/db-inc index 9ca9852..3c7e954 100644 --- a/db-inc +++ b/db-inc @@ -1,8 +1,13 @@ -# $Id: db-inc,v 1.2 2005/06/21 03:47:38 judd Exp $ +# $Id: db-inc,v 1.3 2005/06/30 22:30:36 judd Exp $ [ "$UID" = "" ] && UID=`uid` TMPDIR="/tmp/archpkg.$repoid.$UID" +if [ ! `type -p fakeroot` ]; then + echo "error: fakeroot is missing" + exit 1 +fi + cleanup() { rm -rf $TMPDIR # unlock @@ -37,7 +42,11 @@ if [ $? -gt 0 ]; then cleanup exit 1 fi -/usr/bin/gensync $TMPDIR/$cvsmod $TMPDIR/$reponame.db.tar.gz $ftppath + +echo "chown -R root.root $TMPDIR/$cvsmod;" \ + "/usr/bin/gensync $TMPDIR/$cvsmod $TMPDIR/$reponame.db.tar.gz $ftppath" \ + | fakeroot + [ -f $TMPDIR/$reponame.db.tar.gz ] && mv -f $TMPDIR/$reponame.db.tar.gz $ftppath echo "==> Scanning for New/Updated/Deleted packages..." >&2 @@ -56,4 +65,4 @@ fi cleanup -# vim: set ft=sh: +# vim: set ts=2 noet ft=sh: -- cgit v1.2.2 From a49feb74bba7a0dfe7f5a546fc4a188ded208c61 Mon Sep 17 00:00:00 2001 From: Judd Vinet Date: Mon, 20 Feb 2006 06:33:40 +0000 Subject: re-worked db-gen scripts -- new and improved --- db-arch | 4 +- db-extra | 4 +- db-inc | 136 +++++++++++++++++++++++++------ db-testing | 82 ++++++++----------- db-unstable | 4 +- pkgdb1 | 73 ++++++++++------- pkgdb2-add | Bin 0 -> 13039 bytes pkgdb2-del | Bin 0 -> 8692 bytes updatesync-many | 248 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 9 files changed, 448 insertions(+), 103 deletions(-) create mode 100755 pkgdb2-add create mode 100755 pkgdb2-del create mode 100755 updatesync-many diff --git a/db-arch b/db-arch index f237b3b..e348228 100755 --- a/db-arch +++ b/db-arch @@ -1,5 +1,5 @@ #!/bin/bash -# $Id: db-arch,v 1.22 2005/06/21 03:44:57 judd Exp $ +# $Id: db-arch,v 1.23 2006/02/20 06:33:40 judd Exp $ repoid=1 reponame="current" @@ -9,6 +9,8 @@ ftppath="/home/ftp/current/os/i686" cvspath="/home/cvs-arch" cvsmod="arch/build" +stagedir="$HOME/staging/arch" + # call the real script . `dirname $0`/db-inc diff --git a/db-extra b/db-extra index be5fc3a..5ee7564 100755 --- a/db-extra +++ b/db-extra @@ -1,5 +1,5 @@ #!/bin/bash -# $Id: db-extra,v 1.11 2005/06/21 03:44:57 judd Exp $ +# $Id: db-extra,v 1.12 2006/02/20 06:33:40 judd Exp $ repoid=2 reponame="extra" @@ -9,6 +9,8 @@ ftppath="/home/ftp/extra/os/i686" cvspath="/home/cvs-extra" cvsmod="extra" +stagedir="$HOME/staging/extra" + # call the real script . `dirname $0`/db-inc diff --git a/db-inc b/db-inc index 3c7e954..99cbcee 100644 --- a/db-inc +++ b/db-inc @@ -1,10 +1,18 @@ -# $Id: db-inc,v 1.3 2005/06/30 22:30:36 judd Exp $ +# $Id: db-inc,v 1.4 2006/02/20 06:33:40 judd Exp $ [ "$UID" = "" ] && UID=`uid` TMPDIR="/tmp/archpkg.$repoid.$UID" +# where are the arch scripts located? +ARCHDIR="/arch-new" + if [ ! `type -p fakeroot` ]; then - echo "error: fakeroot is missing" + echo "error: fakeroot is missing" >&2 + exit 1 +fi + +if [ ! -d $stagedir ]; then + echo "error: staging directory missing: $stagedir" >&2 exit 1 fi @@ -12,7 +20,17 @@ cleanup() { rm -rf $TMPDIR # unlock rm -f /tmp/.repolck.$repoid - exit 0 + [ "$1" ] && exit $1 +} + +ctrl_c() { + echo "Interrupted" >&2 + cleanup 0 +} + +die() { + echo "$*" >&2 + cleanup 1 } # check for locks @@ -23,7 +41,7 @@ if [ -f /tmp/.repolck.$repoid ]; then fi # catch ^C breaks -trap cleanup SIGINT +trap ctrl_c SIGINT # lock touch /tmp/.repolck.$repoid @@ -34,33 +52,103 @@ if [ -d $TMPDIR ]; then fi mkdir $TMPDIR; [ $? -gt 0 ] && exit 1 -echo "==> Generating Pacman Database for repository '$reponame'..." >&2 cd $TMPDIR -CVS_RSH=ssh CVSROOT=:ext:cvs.archlinux.org:$cvspath cvs -q export -r CURRENT $cvsmod -if [ $? -gt 0 ]; then - echo "==> CVS export failed!" - cleanup - exit 1 + +# Checkout the CVS module if we need to +updatelists= +if [ "`ls $stagedir/add`" -o "`ls $stagedir/del`" ]; then + # if $cvsdir is set, then use that instead of doing our own cvs checkout + if [ "$cvsdir" ]; then + mv $cvsdir $TMPDIR/$cvsmod + else + echo "==> Checking out module: $cvsmod" + CVS_RSH=ssh CVSROOT=:ext:cvs.archlinux.org:$cvspath cvs -q export -r CURRENT $cvsmod + if [ $? -gt 0 ]; then + die "==> CVS export failed!" + fi + fi + updatelists=1 +else + echo "No files to process" + cleanup 0 fi -echo "chown -R root.root $TMPDIR/$cvsmod;" \ - "/usr/bin/gensync $TMPDIR/$cvsmod $TMPDIR/$reponame.db.tar.gz $ftppath" \ - | fakeroot +# Right-O, now we look through the "add" and "del" subdirectories of +# $stagedir and process the packages there accordingly -- all packages +# in the "add" dir are added/updated, all packages in the "del" dir +# are removed. +# +# This means the sync db could actually be unpacked/repacked twice in +# one db-* invocation, but it's not a huge performance hit. -[ -f $TMPDIR/$reponame.db.tar.gz ] && mv -f $TMPDIR/$reponame.db.tar.gz $ftppath +if [ "`ls $stagedir/add`" ]; then + cd $TMPDIR + echo "==> Processing new/updated packages for repository '$reponame'..." >&2 -echo "==> Scanning for New/Updated/Deleted packages..." >&2 -cd $TMPDIR/$cvsmod -/arch/pkgdb1 $repoid | /arch/pkgdb2 $repoid $ftppath + # copy the db file into our working area + cp $ftppath/$reponame.db.tar.gz . -echo "==> Generating Text Package List..." >&2 -/arch/genpkglist $reponame + cd $stagedir/add + # run it thru fakeroot make sure everything is owned by root.root + echo "$ARCHDIR/updatesync-many add $TMPDIR/$reponame.db.tar.gz $TMPDIR/$cvsmod" \ + | fakeroot -# hack -- only Current's packages.txt goes in a "setup" subdir -if [ "$reponame" = "current" ]; then - mv packages.txt $ftppath/setup/packages.txt -else - mv packages.txt $ftppath/packages.txt + if [ $? -ne 0 ]; then + die "==> Error returned from updatesync-many" + fi + + cp $TMPDIR/$reponame.db.tar.gz $ftppath + + echo "==> Scanning for New/Updated packages..." >&2 + cd $stagedir/add + $ARCHDIR/pkgdb1 $TMPDIR/$cvsmod | $ARCHDIR/pkgdb2-add $repoid $stagedir/add + + # move the package files into the ftp directory + mv -f $stagedir/add/*.pkg.tar.gz $ftppath +fi + +if [ "`ls $stagedir/del`" ]; then + cd $TMPDIR + echo "==> Processing deleted packages for repository '$reponame'..." >&2 + + # copy the db file into our working area + cp $ftppath/$reponame.db.tar.gz . + + cd $stagedir/del + # run it thru fakeroot make sure everything is owned by root.root + echo "$ARCHDIR/updatesync-many del $TMPDIR/$reponame.db.tar.gz NOT-USED" \ + | fakeroot + + if [ $? -ne 0 ]; then + die "==> Error returned from updatesync-many" + fi + + cp $TMPDIR/$reponame.db.tar.gz $ftppath + + echo "==> Scanning for Deleted packages..." >&2 + cd $stagedir/del + ( + for i in *.pkg.tar.gz; do + pkgname=${i%-*-*} + echo $pkgname + done + ) | $ARCHDIR/pkgdb2-del $repoid $stagedir/del + + # remove the package files + rm -f $stagedir/del/*.pkg.tar.gz +fi + +if [ "$updatelists" ]; then + echo "==> Generating Text Package List..." >&2 + cd $TMPDIR/$cvsmod + $ARCHDIR/genpkglist $reponame + + # hack -- only Current's packages.txt goes in a "setup" subdir + if [ "$reponame" = "current" ]; then + mv packages.txt $ftppath/setup/packages.txt + else + mv packages.txt $ftppath/packages.txt + fi fi cleanup diff --git a/db-testing b/db-testing index 230c0f7..9107353 100755 --- a/db-testing +++ b/db-testing @@ -1,68 +1,54 @@ #!/bin/bash -# $Id: db-testing,v 1.10 2005/02/25 21:27:44 judd Exp $ +# $Id: db-testing,v 1.11 2006/02/20 06:33:40 judd Exp $ -uid=`id -u` repoid=4 +reponame="testing" + ftppath="/home/ftp/testing/os/i686" -TMPDIR="/tmp/archpkg.$repoid.$uid" +cvspath="/home/cvs-arch" +cvsmod="testing" + +stagedir="$HOME/staging/testing" -cleanup() { - rm -rf $TMPDIR - # unlock - rm -f /tmp/.repolck.$repoid - exit 0 +testing_cleanup() { + rm -rf $testingtmp } -# check for locks +# catch ^C +trap cleanup SIGINT + if [ -f /tmp/.repolck.$repoid ]; then owner=`/bin/ls -l /tmp/.repolck.$repoid | awk '{print $3}'` echo "error: db generation is already in progress (started by $owner)" exit 1 fi -if [ -d /tmp/.gensync ]; then - echo "error: someone else is currently running gensync!" - exit 1 -fi - -# catch ^C breaks -trap cleanup SIGINT # lock touch /tmp/.repolck.$repoid -# RedHat's mktemp is broken... -if [ -d $TMPDIR ]; then - echo "==> Removing old temp dir..." >&2 - rm -rf $TMPDIR || exit 1 -fi -mkdir $TMPDIR; [ $? -gt 0 ] && exit 1 - -echo "==> Generating Pacman Database for TESTING..." >&2 -cd $TMPDIR -echo "--- CURRENT ---" -CVS_RSH=ssh CVSROOT=:ext:cvs.archlinux.org:/home/cvs-arch cvs -q export -r TESTING arch -mv arch/build/* . && rm -rf arch -echo "--- EXTRA ---" -CVS_RSH=ssh CVSROOT=:ext:cvs.archlinux.org:/home/cvs-extra cvs -q export -r TESTING extra -cp -a extra/* . && rm -rf extra -echo "--- UNSTABLE ---" -CVS_RSH=ssh CVSROOT=:ext:cvs.archlinux.org:/home/cvs-unstable cvs -q export -r TESTING unstable -cp -a unstable/* . && rm -rf unstable -# check again -if [ -d /tmp/.gensync ]; then - echo "error: someone else is currently running gensync!" - cleanup - exit 1 +# testing is special, so we have to do the CVS checkout stuff here +testingtmp=$(mktemp -d /tmp/testtmp-XXXXXX) || exit 1 +cvsdir=$testingtmp/testing +mkdir $cvsdir +cd $cvsdir + +if [ "`ls $stagedir/add`" -o "`ls $stagedir/del`" ]; then + echo "==> Checking out TESTING branches..." >&2 + echo "--- CURRENT ---" + CVS_RSH=ssh CVSROOT=:ext:cvs.archlinux.org:/home/cvs-arch cvs -q export -r TESTING arch + mv arch/build/* . && rm -rf arch + echo "--- EXTRA ---" + CVS_RSH=ssh CVSROOT=:ext:cvs.archlinux.org:/home/cvs-extra cvs -q export -r TESTING extra + cp -a extra/* . && rm -rf extra + echo "--- UNSTABLE ---" + CVS_RSH=ssh CVSROOT=:ext:cvs.archlinux.org:/home/cvs-unstable cvs -q export -r TESTING unstable + cp -a unstable/* . && rm -rf unstable fi -/usr/bin/gensync $TMPDIR $TMPDIR/testing.db.tar.gz $ftppath -[ -f $TMPDIR/testing.db.tar.gz ] && mv -f $TMPDIR/testing.db.tar.gz $ftppath -echo "==> Scanning for New/Updated/Deleted packages..." >&2 -cd $TMPDIR -/arch/pkgdb1 $repoid | /arch/pkgdb2 $repoid $ftppath +# unlock +rm -f /tmp/.repolck.$repoid -echo "==> Generating Text Package List..." >&2 -/arch/genpkglist testing -mv packages.txt $ftppath/packages.txt +# call the real script +. `dirname $0`/db-inc -cleanup +testing_cleanup diff --git a/db-unstable b/db-unstable index a22188b..194bee5 100755 --- a/db-unstable +++ b/db-unstable @@ -1,5 +1,5 @@ #!/bin/bash -# $Id: db-unstable,v 1.16 2005/06/21 03:44:57 judd Exp $ +# $Id: db-unstable,v 1.17 2006/02/20 06:33:40 judd Exp $ repoid=3 reponame="unstable" @@ -9,6 +9,8 @@ ftppath="/home/ftp/unstable/os/i686" cvspath="/home/cvs-unstable" cvsmod="unstable" +stagedir="$HOME/staging/unstable" + # call the real script . `dirname $0`/db-inc diff --git a/pkgdb1 b/pkgdb1 index dd4e5d3..5b2f09a 100755 --- a/pkgdb1 +++ b/pkgdb1 @@ -1,40 +1,57 @@ #!/bin/bash -# $Id: pkgdb1,v 1.3 2005/01/03 08:11:19 judd Exp $ +# $Id: pkgdb1,v 1.4 2006/02/20 06:33:40 judd Exp $ -tl=`pwd` +STAGEDIR=`pwd` -repoid=$1 -if [ "$repoid" = "" ]; then +ABSDIR=$1 +if [ "$ABSDIR" = "" ]; then me=`basename $0` - echo "usage: $me " + echo "usage: $me " >&2 exit 1 fi -cd $tl -for category in `find . -maxdepth 1 -type d | grep -v CVS | grep -v "^extra" | grep -v "^unstable"`; do - category=`echo $category | sed 's|./||'` - if [ "$category" = "." ]; then +if [ ! "`ls $STAGEDIR/*.pkg.tar.gz 2>/dev/null`" ]; then + exit +fi + +cd $STAGEDIR +for pkgfile in `ls $STAGEDIR/*.pkg.tar.gz`; do + tmp=${pkgfile##*/} + pkgname=${tmp%-*-*}; + fullname=${tmp%.pkg.tar.gz} + # find the matching PKGBUILD + tmpf=$(mktemp /tmp/pkgdb1.XXXXXXXXXX) || exit 1 + find $ABSDIR -type d -name "$pkgname" >$tmpf + if [ "`cat $tmpf | wc -l`" != "1" ]; then + echo "WARNING: could not find PKGBUILD for $pkgname, cannot update this entry" >&2 + rm $tmpf continue fi - for pkg in `ls $category | sort`; do - cd $tl/$category/$pkg - if [ -f PKGBUILD ]; then - unset depends source deplist sources - unset pkgname pkgver pkgrel pkgdesc url - . PKGBUILD - deplist=${depends[@]} - sources=${source[@]} - echo $pkgname - echo $pkgver - echo $pkgrel - echo $pkgdesc - echo $category - echo $url - echo $sources - echo $deplist - fi - cd $tl - done + pkgbuild="`cat $tmpf`/PKGBUILD" + rm $tmpf + if [ ! -f $pkgbuild ]; then + echo "WARNING: could not find PKGBUILD for $fullname, cannot update this entry" >&2 + continue + fi + # pick out the category from the pathname + catpath=$(cd `dirname $pkgbuild`/.. && pwd) + category=${catpath##*/} + # now read the PKGBUILD and output the data for pkgdb2 + unset pkgname pkgver pkgrel pkgdesc license groups provides md5sums force + unset replaces depends conflicts backup source install build makedepends + unset options + source $pkgbuild || continue + + deplist=${depends[@]} + sources=${source[@]} + echo $pkgname + echo $pkgver + echo $pkgrel + echo $pkgdesc + echo $category + echo $url + echo $sources + echo $deplist done exit 0 diff --git a/pkgdb2-add b/pkgdb2-add new file mode 100755 index 0000000..e6a87c0 Binary files /dev/null and b/pkgdb2-add differ diff --git a/pkgdb2-del b/pkgdb2-del new file mode 100755 index 0000000..af44fe1 Binary files /dev/null and b/pkgdb2-del differ diff --git a/updatesync-many b/updatesync-many new file mode 100755 index 0000000..150cd66 --- /dev/null +++ b/updatesync-many @@ -0,0 +1,248 @@ +#!/bin/bash +# +# updatesync-many +# +# Copyright (c) 2004 by Jason Chu +# Derived from gensync (c) 2002-2006 Judd Vinet +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, +# USA. +# + +usage() { + echo "updatesync-many" + echo "usage: $0 " + echo + echo "This should probably only be run from the Arch db-generation scripts" + echo + echo "Caveats:" + echo " - Make sure you run it from the staging directory" + echo " - Use absolute pathnames for dbfile and abs_dir" + echo + exit 0 +} + +die() +{ + echo "updatesync-many: $*" >&2 + rm -rf $TMPDIR + exit 1 +} + +msg() +{ + echo "updatesync-many: $*" >&2 +} + +check_option() { + local i + for i in ${options[@]}; do + local uc=`echo $i | tr [:lower:] [:upper:]` + local lc=`echo $i | tr [:upper:] [:lower:]` + if [ "$uc" = "$1" -o "$lc" = "$1" ]; then + echo $1 + return + fi + done +} + +get_md5checksum() +{ + md5line=`md5sum $1` + [ ! -z "$md5line" ] && pkgmd5sum=${md5line% *} + echo $pkgmd5sum +} + +db_write_entry() +{ + unset pkgname pkgver pkgrel pkgdesc license groups provides md5sums force + unset replaces depends conflicts backup source install build makedepends + unset options + source $1 || return 1 + cd $TMPDIR + mkdir $pkgname-$pkgver-$pkgrel || return 1 + cd $pkgname-$pkgver-$pkgrel + # desc + : >desc + echo "%NAME%" >>desc + echo "$pkgname" >>desc + echo "" >>desc + echo "%VERSION%" >>desc + echo "$pkgver-$pkgrel" >>desc + echo "" >>desc + echo "%DESC%" >>desc + echo "$pkgdesc" >>desc + echo "" >>desc + echo "%CSIZE%" >>desc + echo "$csize" >>desc + echo "" >>desc + if [ ! -z $pkgmd5sum ]; then + echo "%MD5SUM%" >>desc + echo "$pkgmd5sum" >>desc + echo "" >>desc + fi + if [ ${#groups[*]} -gt 0 ]; then + echo "%GROUPS%" >>desc + for it in "${groups[@]}"; do + echo "$it" >>desc + done + echo "" >>desc + fi + if [ ${#replaces[*]} -gt 0 ]; then + echo "%REPLACES%" >>desc + for it in "${replaces[@]}"; do + echo "$it" >>desc + done + echo "" >>desc + fi + if [ "$force" = "y" -o "$force" = "Y" -o "`check_option FORCE`" ]; then + echo "%FORCE%" >>desc + echo "" >>desc + fi + # depends + : >depends + if [ ${#depends[*]} -gt 0 ]; then + echo "%DEPENDS%" >>depends + for it in "${depends[@]}"; do + echo "$it" >>depends + done + echo "" >>depends + fi + if [ ${#conflicts[*]} -gt 0 ]; then + echo "%CONFLICTS%" >>depends + for it in "${conflicts[@]}"; do + echo "$it" >>depends + done + echo "" >>depends + fi + if [ ${#provides[*]} -gt 0 ]; then + echo "%PROVIDES%" >>depends + for it in "${provides[@]}"; do + echo "$it" >>depends + done + echo "" >>depends + fi +} + +delete_entry() +{ + # strip to the basename + tmp=${1##*/} + # grab the pkgname + pkgname=${tmp%-*-*} + for i in *; do + if [ "${i%-*-*}" = "$pkgname" ]; then + rm -rf $i + fi + done +} + +update_entry() +{ + pkgfile=$1 + tmp=${pkgfile##*/} + pkgname=${tmp%-*-*}; + fullname=${tmp%.pkg.tar.gz} + + # find the matching PKGBUILD + tmpf=$(mktemp /tmp/updatesync-many.XXXXXXXXXX) || exit 1 + find $ABSDIR -type d -name "$pkgname" >$tmpf + if [ "`cat $tmpf | wc -l`" != "1" ]; then + msg "WARNING: could not find PKGBUILD for $pkgname, cannot update this entry" + rm $tmpf + return + fi + pkgbuild="`cat $tmpf`/PKGBUILD" + rm $tmpf + if [ ! -f $pkgbuild ]; then + msg "WARNING: could not find PKGBUILD for $fullname, cannot update this entry" + return + fi + + source $pkgbuild + if [ $? -ne 0 ]; then + msg "WARNING: PKGBUILD for $fullname has errors, cannot update this entry" + return + fi + + # all good so far - delete the old entry + cd $TMPDIR + delete_entry $pkgfile + + csize=`du -b $pkgfile | cut -f1` + pkgmd5sum=`get_md5checksum $pkgfile` + [ -z $pkgmd5sum ] && die "error generating checksum for $pkgfile" + + db_write_entry $pkgbuild || die "error writing entry for $pkgname" + cd - >/dev/null +} + + +if [ $# -lt 3 ]; then + usage + exit 1 +fi + +if [ "$1" = "-h" -o "$1" = "--help" ]; then + usage + exit 0 +fi + +ACTION=$1 +PKGDB=$2 +ABSDIR=$3 +STAGEDIR="`pwd`" +PKGDIR="`dirname $PKGDB`" +if [ "$PKGDIR" = "." ]; then + PKGDIR=$STAGEDIR +fi + +if [ "$ACTION" != "add" -a "$ACTION" != "del" ]; then + usage + exit 1 +fi + +# Prepare the sync db for modifications +TMPDIR=$(mktemp -d /tmp/updatesync-many.XXXXXXXXXX) || exit 1 +cd $TMPDIR +if [ ! -f $PKGDB ]; then + die "$PKGDB not found" +fi +msg "Unpacking db: $PKGDB" +tar zxf $PKGDB || die "error unpacking $PKGDB" + +# Process packages in the staging directory +for pkgfile in $STAGEDIR/*.pkg.tar.gz; do + tmp=${pkgfile##*/} + pkgname=${tmp%-*-*}; + fullname=${tmp%.pkg.tar.gz} + if [ "$ACTION" = "del" ]; then + msg "Deleting entry: $pkgname" + delete_entry $pkgfile + else + msg "Updating entry: $pkgname" + update_entry $pkgfile + fi +done + +# Repackage the DB +msg "Repacking db: $PKGDB" +cd $TMPDIR +tar c * | gzip -9 >$PKGDB || die "error writing to $PKGDB" + +cd / +rm -rf $TMPDIR + +exit 0 -- cgit v1.2.2 From 5128cca4a76c5133f3c98db79b9c66a7b55916c6 Mon Sep 17 00:00:00 2001 From: Judd Vinet Date: Wed, 22 Feb 2006 07:56:11 +0000 Subject: fixed dupe package listings --- genpkglist | 36 ++++++++++++++++++++++++++++++------ 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/genpkglist b/genpkglist index 08111ce..1d25201 100755 --- a/genpkglist +++ b/genpkglist @@ -1,5 +1,5 @@ #!/bin/bash -# $Id: genpkglist,v 1.12 2005/01/03 08:09:21 judd Exp $ +# $Id: genpkglist,v 1.13 2006/02/22 07:56:11 judd Exp $ # # genpkglist @@ -29,11 +29,35 @@ for category in `find * -maxdepth 0 -type d | grep -v CVS`; do cd .. done -DUPES=`ls -1 /home/ftp/$1/os/i686 | rev | cut -d- -f 3- | rev | sort | uniq -c | egrep -v '^ 1' | awk '{print $2}'` +cd /home/ftp/$1/os/i686 +unset DUPES DUPEFILES last +for pkg in *.pkg.tar.gz; do + pkgname=${pkg%-*-*} + if [ "$last" = "$pkgname" ]; then + DUPES="$DUPES $pkgname" + DUPEFILES="$DUPEFILES $pkg" + fi + last=$pkgname +done -if [ -n "$DUPES" ]; then - echo "Possible Dupes for $1 (please remove old versions)." - echo "Date Filename" - for dupe in $DUPES; do ls -l /home/ftp/$1/os/i686/${dupe}* | awk '{print $6" "$7" "$8" "$9}'; done +showdupes() { + done= + for i in *.pkg.tar.gz; do + pkgname=${i%-*-*} + if [ "$pkgname" = "$1" ]; then + ls -l $i | awk '{print $6" "$7" "$8" "$9}' + done=1 + else + [ "$done" = "1" ] && return + fi + done +} + +if [ "$DUPES" ]; then + echo "Possible Dupes for $1 (please remove old versions)" + echo "Date Filename" + for dupe in $((for d in `echo $DUPES`; do echo $d; done) | sort -u); do + showdupes $dupe + done fi -- cgit v1.2.2 From f3d8a756a29d26d91bfe191e717b2a656d39c176 Mon Sep 17 00:00:00 2001 From: Judd Vinet Date: Fri, 10 Mar 2006 23:17:48 +0000 Subject: l'il fix --- db-testing | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/db-testing b/db-testing index 9107353..97039a3 100755 --- a/db-testing +++ b/db-testing @@ -1,5 +1,5 @@ #!/bin/bash -# $Id: db-testing,v 1.11 2006/02/20 06:33:40 judd Exp $ +# $Id: db-testing,v 1.12 2006/03/10 23:17:48 judd Exp $ repoid=4 reponame="testing" @@ -16,7 +16,7 @@ testing_cleanup() { } # catch ^C -trap cleanup SIGINT +trap testing_cleanup SIGINT if [ -f /tmp/.repolck.$repoid ]; then owner=`/bin/ls -l /tmp/.repolck.$repoid | awk '{print $3}'` -- cgit v1.2.2 From 2d8cc96bf6eada340975827b8f631496da79e63b Mon Sep 17 00:00:00 2001 From: Judd Vinet Date: Fri, 5 May 2006 01:38:33 +0000 Subject: updates for multi-architecture support --- db-arch | 4 +++- db-arch64 | 18 ++++++++++++++++++ db-extra | 4 +++- db-extra64 | 18 ++++++++++++++++++ db-inc | 52 +++++++++++++++++++++++++++++----------------------- db-testing | 12 +++++++----- db-unstable | 4 +++- db-unstable64 | 18 ++++++++++++++++++ genpkglist | 10 +++++----- updatesync-many | 16 ++++++++++++++-- 10 files changed, 118 insertions(+), 38 deletions(-) create mode 100755 db-arch64 create mode 100755 db-extra64 create mode 100755 db-unstable64 diff --git a/db-arch b/db-arch index e348228..9b61eee 100755 --- a/db-arch +++ b/db-arch @@ -1,13 +1,15 @@ #!/bin/bash -# $Id: db-arch,v 1.23 2006/02/20 06:33:40 judd Exp $ +# $Id: db-arch,v 1.24 2006/05/05 01:38:33 judd Exp $ repoid=1 reponame="current" +arch="i686" ftppath="/home/ftp/current/os/i686" cvspath="/home/cvs-arch" cvsmod="arch/build" +cvstag="CURRENT" stagedir="$HOME/staging/arch" diff --git a/db-arch64 b/db-arch64 new file mode 100755 index 0000000..ac2cbb1 --- /dev/null +++ b/db-arch64 @@ -0,0 +1,18 @@ +#!/bin/bash +# $Id: db-arch64,v 1.1 2006/05/05 01:38:33 judd Exp $ + +repoid=1 +reponame="current" +arch="x86_64" + +ftppath="/home/ftp/current/os/x86_64" + +cvspath="/home/cvs-arch" +cvsmod="arch/build" +cvstag="CURRENT-64" + +stagedir="/home/arch64/staging/arch" + +# call the real script +. `dirname $0`/db-inc + diff --git a/db-extra b/db-extra index 5ee7564..c24a26f 100755 --- a/db-extra +++ b/db-extra @@ -1,13 +1,15 @@ #!/bin/bash -# $Id: db-extra,v 1.12 2006/02/20 06:33:40 judd Exp $ +# $Id: db-extra,v 1.13 2006/05/05 01:38:33 judd Exp $ repoid=2 reponame="extra" +arch="i686" ftppath="/home/ftp/extra/os/i686" cvspath="/home/cvs-extra" cvsmod="extra" +cvstag="CURRENT" stagedir="$HOME/staging/extra" diff --git a/db-extra64 b/db-extra64 new file mode 100755 index 0000000..8dce59b --- /dev/null +++ b/db-extra64 @@ -0,0 +1,18 @@ +#!/bin/bash +# $Id: db-extra64,v 1.1 2006/05/05 01:38:33 judd Exp $ + +repoid=2 +reponame="extra" +arch="x86_64" + +ftppath="/home/ftp/extra/os/x86_64" + +cvspath="/home/cvs-extra" +cvsmod="extra" +cvstag="CURRENT-64" + +stagedir="/home/arch64/staging/extra" + +# call the real script +. `dirname $0`/db-inc + diff --git a/db-inc b/db-inc index 99cbcee..5e2eff0 100644 --- a/db-inc +++ b/db-inc @@ -1,10 +1,10 @@ -# $Id: db-inc,v 1.4 2006/02/20 06:33:40 judd Exp $ +# $Id: db-inc,v 1.5 2006/05/05 01:38:33 judd Exp $ [ "$UID" = "" ] && UID=`uid` -TMPDIR="/tmp/archpkg.$repoid.$UID" +TMPDIR="/tmp/archpkg.$arch.$repoid.$UID" # where are the arch scripts located? -ARCHDIR="/arch-new" +ARCHDIR="/arch" if [ ! `type -p fakeroot` ]; then echo "error: fakeroot is missing" >&2 @@ -19,7 +19,7 @@ fi cleanup() { rm -rf $TMPDIR # unlock - rm -f /tmp/.repolck.$repoid + rm -f /tmp/.repolck.$arch.$repoid [ "$1" ] && exit $1 } @@ -34,8 +34,8 @@ die() { } # check for locks -if [ -f /tmp/.repolck.$repoid ]; then - owner=`/bin/ls -l /tmp/.repolck.$repoid | awk '{print $3}'` +if [ -f /tmp/.repolck.$arch.$repoid ]; then + owner=`/bin/ls -l /tmp/.repolck.$arch.$repoid | awk '{print $3}'` echo "error: db generation is already in progress (started by $owner)" exit 1 fi @@ -43,7 +43,7 @@ fi # catch ^C breaks trap ctrl_c SIGINT # lock -touch /tmp/.repolck.$repoid +touch /tmp/.repolck.$arch.$repoid # RedHat's mktemp is broken... if [ -d $TMPDIR ]; then @@ -61,8 +61,8 @@ if [ "`ls $stagedir/add`" -o "`ls $stagedir/del`" ]; then if [ "$cvsdir" ]; then mv $cvsdir $TMPDIR/$cvsmod else - echo "==> Checking out module: $cvsmod" - CVS_RSH=ssh CVSROOT=:ext:cvs.archlinux.org:$cvspath cvs -q export -r CURRENT $cvsmod + echo "==> Checking out module: $cvsmod ($cvstag)" + CVS_RSH=ssh CVSROOT=:ext:cvs.archlinux.org:$cvspath cvs -q export -r $cvstag $cvsmod if [ $? -gt 0 ]; then die "==> CVS export failed!" fi @@ -81,7 +81,7 @@ fi # This means the sync db could actually be unpacked/repacked twice in # one db-* invocation, but it's not a huge performance hit. -if [ "`ls $stagedir/add`" ]; then +if [ -d $stagedir/add -a "`ls $stagedir/add`" ]; then cd $TMPDIR echo "==> Processing new/updated packages for repository '$reponame'..." >&2 @@ -99,15 +99,18 @@ if [ "`ls $stagedir/add`" ]; then cp $TMPDIR/$reponame.db.tar.gz $ftppath - echo "==> Scanning for New/Updated packages..." >&2 - cd $stagedir/add - $ARCHDIR/pkgdb1 $TMPDIR/$cvsmod | $ARCHDIR/pkgdb2-add $repoid $stagedir/add + # only for i686 (for now) + if [ "$arch" = "i686" ]; then + echo "==> Scanning for New/Updated packages..." >&2 + cd $stagedir/add + $ARCHDIR/pkgdb1 $TMPDIR/$cvsmod | $ARCHDIR/pkgdb2-add $repoid $stagedir/add + fi # move the package files into the ftp directory mv -f $stagedir/add/*.pkg.tar.gz $ftppath fi -if [ "`ls $stagedir/del`" ]; then +if [ -d $stagedir/del -a "`ls $stagedir/del`" ]; then cd $TMPDIR echo "==> Processing deleted packages for repository '$reponame'..." >&2 @@ -125,14 +128,17 @@ if [ "`ls $stagedir/del`" ]; then cp $TMPDIR/$reponame.db.tar.gz $ftppath - echo "==> Scanning for Deleted packages..." >&2 - cd $stagedir/del - ( - for i in *.pkg.tar.gz; do - pkgname=${i%-*-*} - echo $pkgname - done - ) | $ARCHDIR/pkgdb2-del $repoid $stagedir/del + # only for i686 (for now) + if [ "$arch" = "i686" ]; then + echo "==> Scanning for Deleted packages..." >&2 + cd $stagedir/del + ( + for i in *.pkg.tar.gz; do + pkgname=${i%-*-*} + echo $pkgname + done + ) | $ARCHDIR/pkgdb2-del $repoid $stagedir/del + fi # remove the package files rm -f $stagedir/del/*.pkg.tar.gz @@ -141,7 +147,7 @@ fi if [ "$updatelists" ]; then echo "==> Generating Text Package List..." >&2 cd $TMPDIR/$cvsmod - $ARCHDIR/genpkglist $reponame + $ARCHDIR/genpkglist $ftppath # hack -- only Current's packages.txt goes in a "setup" subdir if [ "$reponame" = "current" ]; then diff --git a/db-testing b/db-testing index 97039a3..1baad46 100755 --- a/db-testing +++ b/db-testing @@ -1,13 +1,15 @@ #!/bin/bash -# $Id: db-testing,v 1.12 2006/03/10 23:17:48 judd Exp $ +# $Id: db-testing,v 1.13 2006/05/05 01:38:33 judd Exp $ repoid=4 reponame="testing" +arch="i686" ftppath="/home/ftp/testing/os/i686" cvspath="/home/cvs-arch" cvsmod="testing" +cvstag="TESTING" stagedir="$HOME/staging/testing" @@ -18,13 +20,13 @@ testing_cleanup() { # catch ^C trap testing_cleanup SIGINT -if [ -f /tmp/.repolck.$repoid ]; then - owner=`/bin/ls -l /tmp/.repolck.$repoid | awk '{print $3}'` +if [ -f /tmp/.repolck.$arch.$repoid ]; then + owner=`/bin/ls -l /tmp/.repolck.$arch.$repoid | awk '{print $3}'` echo "error: db generation is already in progress (started by $owner)" exit 1 fi # lock -touch /tmp/.repolck.$repoid +touch /tmp/.repolck.$arch.$repoid # testing is special, so we have to do the CVS checkout stuff here testingtmp=$(mktemp -d /tmp/testtmp-XXXXXX) || exit 1 @@ -46,7 +48,7 @@ if [ "`ls $stagedir/add`" -o "`ls $stagedir/del`" ]; then fi # unlock -rm -f /tmp/.repolck.$repoid +rm -f /tmp/.repolck.$arch.$repoid # call the real script . `dirname $0`/db-inc diff --git a/db-unstable b/db-unstable index 194bee5..e37e664 100755 --- a/db-unstable +++ b/db-unstable @@ -1,13 +1,15 @@ #!/bin/bash -# $Id: db-unstable,v 1.17 2006/02/20 06:33:40 judd Exp $ +# $Id: db-unstable,v 1.18 2006/05/05 01:38:33 judd Exp $ repoid=3 reponame="unstable" +arch="i686" ftppath="/home/ftp/unstable/os/i686" cvspath="/home/cvs-unstable" cvsmod="unstable" +cvstag="CURRENT" stagedir="$HOME/staging/unstable" diff --git a/db-unstable64 b/db-unstable64 new file mode 100755 index 0000000..38904d3 --- /dev/null +++ b/db-unstable64 @@ -0,0 +1,18 @@ +#!/bin/bash +# $Id: db-unstable64,v 1.1 2006/05/05 01:38:33 judd Exp $ + +repoid=3 +reponame="unstable" +arch="x86_64" + +ftppath="/home/ftp/unstable/os/x86_64" + +cvspath="/home/cvs-unstable" +cvsmod="unstable" +cvstag="CURRENT-64" + +stagedir="/home/arch64/staging/unstable" + +# call the real script +. `dirname $0`/db-inc + diff --git a/genpkglist b/genpkglist index 1d25201..7dba4d6 100755 --- a/genpkglist +++ b/genpkglist @@ -1,5 +1,5 @@ #!/bin/bash -# $Id: genpkglist,v 1.13 2006/02/22 07:56:11 judd Exp $ +# $Id: genpkglist,v 1.14 2006/05/05 01:38:33 judd Exp $ # # genpkglist @@ -9,7 +9,7 @@ # pkgfile="`pwd`/packages.txt" -repodir=$1 +ftppath=$1 rm -f $pkgfile for category in `find * -maxdepth 0 -type d | grep -v CVS`; do @@ -18,7 +18,7 @@ for category in `find * -maxdepth 0 -type d | grep -v CVS`; do cd $pkg if [ -f PKGBUILD ]; then . PKGBUILD - if [ -f /home/ftp/$repodir/os/i686/$pkgname-$pkgver-$pkgrel.pkg.tar.gz ]; then + if [ -f $ftppath/$pkgname-$pkgver-$pkgrel.pkg.tar.gz ]; then echo "$category/$pkgname-$pkgver-$pkgrel.pkg.tar.gz" >>$pkgfile else echo "notice: Missing $pkgname-$pkgver-$pkgrel.pkg.tar.gz in ftp site" >&2 @@ -29,7 +29,7 @@ for category in `find * -maxdepth 0 -type d | grep -v CVS`; do cd .. done -cd /home/ftp/$1/os/i686 +cd $ftppath unset DUPES DUPEFILES last for pkg in *.pkg.tar.gz; do pkgname=${pkg%-*-*} @@ -54,7 +54,7 @@ showdupes() { } if [ "$DUPES" ]; then - echo "Possible Dupes for $1 (please remove old versions)" + echo "Possible Dupes for $ftppath (remove old versions)" echo "Date Filename" for dupe in $((for d in `echo $DUPES`; do echo $d; done) | sort -u); do showdupes $dupe diff --git a/updatesync-many b/updatesync-many index 150cd66..3899632 100755 --- a/updatesync-many +++ b/updatesync-many @@ -2,8 +2,8 @@ # # updatesync-many # -# Copyright (c) 2004 by Jason Chu -# Derived from gensync (c) 2002-2006 Judd Vinet +# Copyright (c) 2004-2006 by Jason Chu and Judd Vinet +# Contact: and # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -189,6 +189,10 @@ update_entry() cd - >/dev/null } +if [ ! "`type -p lsof`" ]; then + echo "ERROR: lsof is needed to run updatesync-many!" + exit 1 +fi if [ $# -lt 3 ]; then usage @@ -225,6 +229,14 @@ tar zxf $PKGDB || die "error unpacking $PKGDB" # Process packages in the staging directory for pkgfile in $STAGEDIR/*.pkg.tar.gz; do + # Make sure this file isn't currently in use by any processes... + # This is our cheap way of (mostly) making sure the file isn't being + # uploaded at this very time (and thus incomplete). + # Of course, if an upload failed and the scp connection terminated, then + # this check will fail us. + lsof $pkgfile &>/dev/null + [ $? -ne 1 ] && continue + tmp=${pkgfile##*/} pkgname=${tmp%-*-*}; fullname=${tmp%.pkg.tar.gz} -- cgit v1.2.2 From 4536b36df32bd4d5b2a22c667d6faae21be2c87e Mon Sep 17 00:00:00 2001 From: Judd Vinet Date: Sun, 7 May 2006 18:02:42 +0000 Subject: suppress errors in first ls check --- db-inc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/db-inc b/db-inc index 5e2eff0..a984ca9 100644 --- a/db-inc +++ b/db-inc @@ -1,4 +1,4 @@ -# $Id: db-inc,v 1.5 2006/05/05 01:38:33 judd Exp $ +# $Id: db-inc,v 1.6 2006/05/07 18:02:42 judd Exp $ [ "$UID" = "" ] && UID=`uid` TMPDIR="/tmp/archpkg.$arch.$repoid.$UID" @@ -56,7 +56,7 @@ cd $TMPDIR # Checkout the CVS module if we need to updatelists= -if [ "`ls $stagedir/add`" -o "`ls $stagedir/del`" ]; then +if [ "`ls $stagedir/add 2>/dev/null`" -o "`ls $stagedir/del 2>/dev/null`" ]; then # if $cvsdir is set, then use that instead of doing our own cvs checkout if [ "$cvsdir" ]; then mv $cvsdir $TMPDIR/$cvsmod -- cgit v1.2.2 From bcb6418ab38e36505cbdec8205574c0aecdb4c2b Mon Sep 17 00:00:00 2001 From: Judd Vinet Date: Thu, 6 Jul 2006 03:37:01 +0000 Subject: added new pkgdb2 programs --- pkgdb2-add.c | 276 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ pkgdb2-del.c | 137 +++++++++++++++++++++++++++++ 2 files changed, 413 insertions(+) create mode 100644 pkgdb2-add.c create mode 100644 pkgdb2-del.c diff --git a/pkgdb2-add.c b/pkgdb2-add.c new file mode 100644 index 0000000..630d006 --- /dev/null +++ b/pkgdb2-add.c @@ -0,0 +1,276 @@ +/* $Id: pkgdb2-add.c,v 1.1 2006/07/06 03:37:01 judd Exp $ */ + +#include +#include +#include +#include +#include + +#define DB_USER "archweb" +#define DB_NAME "archweb" +#define DB_PASS "passwords-are-cool" + +typedef struct pkg { + unsigned int id; + char *name; + char *ver; + char *rel; + struct pkg *next; +} pkg_t; + +MYSQL_RES *doquery(MYSQL *m, const char* q) +{ + MYSQL_RES *res; + if(mysql_query(m, q)) { + fprintf(stderr, "mysql_query: %s\n", mysql_error(m)); + return(NULL); + } + res = mysql_store_result(m); + return(res); +} + +/* this function is ugly -- it malloc's for each string it + * returns, and they probably won't be freed by the caller. + */ +char* addslashes(const char *s) { + char slashed[8192]; + char *p; + + slashed[0] = '\0'; + p = slashed; + while(*s) { + if(*s == '\'' || *s == '"' || *s == '\\') { + *p++ = '\\'; + } + *p++ = *s++; + } + *p = '\0'; + return(strdup(slashed)); +} + +char* trim(char *str) +{ + char *pch = str; + while(isspace(*pch)) { + pch++; + } + if(pch != str) { + memmove(str, pch, (strlen(pch) + 1)); + } + + pch = (char*)(str + (strlen(str) - 1)); + while(isspace(*pch)) { + pch--; + } + *++pch = '\0'; + + return str; +} + +/* scan a .pkg.tar.gz file and put all files listed into the database. + * + * this function is hacky and should be done properly, but this route is + * easier than reading the file with libtar. + */ +void updatefilelist(MYSQL *db, unsigned long id, char *fn) +{ + FILE *fp; + char *tmp; + char cmd[PATH_MAX]; + char line[PATH_MAX]; + char query[PATH_MAX]; + + tmp = tempnam("/tmp", "pkgdb"); + snprintf(cmd, PATH_MAX-1, "/bin/tar tzvf %s | awk '{print $6}' >%s", fn, tmp); + system(cmd); + fp = fopen(tmp, "r"); + if(fp == NULL) { + fprintf(stderr, "pkgdb2-add: could not open tempfile: %s\n", tmp); + return; + } + snprintf(query, sizeof(query), "DELETE FROM packages_files WHERE id='%d'", id); + doquery(db, query); + while(fgets(line, sizeof(line)-1, fp)) { + char *fixedfn = addslashes(trim(line)); + if(!strcmp(fixedfn, ".FILELIST") || !strcmp(fixedfn, ".PKGINFO") || !strcmp(fixedfn, ".INSTALL")) { + free(fixedfn); + continue; + } + /* varchars aren't case-sensitive but filesystems are, so we use REPLACE INTO */ + snprintf(query, sizeof(query), "REPLACE INTO packages_files (id,path) VALUES " + "('%d', '%s')", id, fixedfn); + free(fixedfn); + doquery(db, query); + } + fclose(fp); + unlink(tmp); +} + +int main(int argc, char **argv) +{ + MYSQL db; + MYSQL_RES *result; + MYSQL_ROW row; + char query[4096]; + char fn[PATH_MAX]; + char ftppath[PATH_MAX]; + int repoid; + pkg_t *dblist = NULL; + pkg_t *pkglist = NULL; + pkg_t *pkgptr, *ptr; + + if(argc < 3) { + printf("usage: pkgdb2-add \n"); + printf("\nWARNING: Do not run this manually! It is intended to be run from\n" + "the Arch db-generation scripts.\n\n"); + return(1); + } + repoid = atoi(argv[1]); + strncpy(ftppath, argv[2], PATH_MAX-1); + + if(mysql_init(&db) == NULL) { + fprintf(stderr, "could not initialize\n"); + return(1); + } + if(mysql_real_connect(&db, "localhost", DB_USER, DB_PASS, DB_NAME, + 0, NULL, 0) == NULL) { + fprintf(stderr, "failed to connect to database: %s\n", mysql_error(&db)); + return(1); + } + snprintf(query, sizeof(query), "SELECT id,pkgname,pkgver,pkgrel FROM packages " + "WHERE repoid='%d'", repoid); + result = doquery(&db, query); + while(row = mysql_fetch_row(result)) { + int i; + /*unsigned long *lengths; + lengths = mysql_fetch_lengths(result);*/ + /* add the node to the list */ + if(dblist == NULL) { + dblist = (pkg_t*)malloc(sizeof(pkg_t)); + if(dblist == NULL) { + fprintf(stderr, "error: out of memory!\n"); + return(1); + } + ptr = dblist; + } else { + ptr->next = (pkg_t*)malloc(sizeof(pkg_t)); + if(ptr->next == NULL) { + fprintf(stderr, "error: out of memory!\n"); + return(1); + } + ptr = ptr->next; + } + ptr->next = NULL; + /* pick out the fields */ + ptr->id = atoi(row[0]); + ptr->name = strdup(row[1]); + ptr->ver = strdup(row[2]); + ptr->rel = strdup(row[3]); + } + mysql_free_result(result); + + while(!feof(stdin)) { + int found = 0; + unsigned int catid = 0; + char name[256], ver[256], rel[256], desc[4096]; + char cat[256], url[256], sources[4096], deplist[4096]; + /* get package data from stdin */ + fgets(name, 256, stdin); trim(name); if(feof(stdin)) continue; + fgets(ver, 256, stdin); trim(ver); if(feof(stdin)) continue; + fgets(rel, 256, stdin); trim(rel); if(feof(stdin)) continue; + fgets(desc, 4096, stdin); trim(desc); if(feof(stdin)) continue; + fgets(cat, 256, stdin); trim(cat); if(feof(stdin)) continue; + fgets(url, 256, stdin); trim(url); if(feof(stdin)) continue; + fgets(sources, 4096, stdin); trim(sources); if(feof(stdin)) continue; + fgets(deplist, 4096, stdin); trim(deplist); if(feof(stdin)) continue; + /* check for overruns */ + if(strlen(name) > 254 || strlen(ver) >= 254 || strlen(rel) > 254 || + strlen(desc) > 4094 || strlen(cat) >= 254 || strlen(url) > 254 || + strlen(sources) > 4094 || strlen(deplist) > 4094) { + fprintf(stderr, "pkgdb2-add: one or more fields are too long in package '%s'\n", name); + fprintf(stderr, "pkgdb2-add: check the lengths of your strings, most are limited " + "to 255 chars, some are 4095\n"); + return(1); + } + /* add the node to the list */ + if(pkglist == NULL) { + pkglist = (pkg_t*)malloc(sizeof(pkg_t)); + if(pkglist == NULL) { + fprintf(stderr, "error: out of memory!\n"); + return(1); + } + pkgptr = pkglist; + } else { + pkgptr->next = (pkg_t*)malloc(sizeof(pkg_t)); + if(pkgptr->next == NULL) { + fprintf(stderr, "error: out of memory!\n"); + return(1); + } + pkgptr = pkgptr->next; + } + pkgptr->next = NULL; + pkgptr->name = strdup(name); + /* look it up in our cache */ + for(ptr = dblist; ptr; ptr = ptr->next) { + if(!strcmp(name, ptr->name)) { + found = 1; + break; + } + } + /* get the category */ + snprintf(query, sizeof(query), + "SELECT id FROM categories WHERE category='%s'", cat); + result = doquery(&db, query); + if(mysql_num_rows(result) == 0) { + fprintf(stderr, "pkgdb2-add: no db category found for '%s'\n", cat); + /* + snprintf(query, sizeof(query), "INSERT INTO categories (id,category) " + " VALUES (NULL,'%s')", addslashes(cat)); + doquery(&db, query); + catid = (unsigned int)mysql_insert_id(&db); + */ + } else { + row = mysql_fetch_row(result); + catid = (unsigned int)atoi(row[0]); + } + if(!found) { + /* Insert... */ + unsigned long id; + fprintf(stderr, "pkgdb2-add: inserting %s\n", name); + snprintf(query, sizeof(query), "INSERT INTO packages (id,repoid," + "categoryid,pkgname,pkgver,pkgrel,pkgdesc,url,sources,depends," + "lastupdate) VALUES (NULL,'%d','%d','%s','%s','%s','%s'," + "'%s','%s','%s',NOW())", + repoid, catid, addslashes(name), addslashes(ver), addslashes(rel), + addslashes(desc), addslashes(url), addslashes(sources), + addslashes(deplist)); + doquery(&db, query); + id = mysql_insert_id(&db); + snprintf(fn, PATH_MAX-1, "%s/%s-%s-%s.pkg.tar.gz", ftppath, name, ver, rel); + updatefilelist(&db, id, fn); + continue; + } else if(strcmp(ptr->ver, ver) || strcmp(ptr->rel, rel)) { + /* ...or Update */ + fprintf(stderr, "pkgdb2-add: updating %s (%s-%s ==> %s-%s)\n", + ptr->name, ptr->ver, ptr->rel, ver, rel); + snprintf(query, sizeof(query), "UPDATE packages SET categoryid='%d'," + "pkgname='%s',pkgver='%s',pkgrel='%s',pkgdesc='%s',url='%s'," + "sources='%s',depends='%s',needupdate=0,lastupdate=NOW() " + "WHERE id='%d'", + catid, addslashes(name), addslashes(ver), addslashes(rel), + addslashes(desc), addslashes(url), addslashes(sources), + addslashes(deplist), ptr->id); + doquery(&db, query); + snprintf(fn, PATH_MAX-1, "%s/%s-%s-%s.pkg.tar.gz", ftppath, name, ver, rel); + updatefilelist(&db, ptr->id, fn); + /* + snprintf(query, sizeof(query), "UPDATE todolist_pkgs SET complete=1 " + "WHERE pkgid='%d'", ptr->id); + doquery(&db, query); + */ + } + } + + mysql_close(&db); + return(0); +} diff --git a/pkgdb2-del.c b/pkgdb2-del.c new file mode 100644 index 0000000..36c8553 --- /dev/null +++ b/pkgdb2-del.c @@ -0,0 +1,137 @@ +/* $Id: pkgdb2-del.c,v 1.1 2006/07/06 03:37:01 judd Exp $ */ + +#include +#include +#include +#include +#include + +#define DB_USER "archweb" +#define DB_NAME "archweb" +#define DB_PASS "passwords-are-cool" + +typedef struct pkg { + unsigned int id; + char *name; + char *ver; + char *rel; + struct pkg *next; +} pkg_t; + +MYSQL_RES *doquery(MYSQL *m, const char* q) +{ + MYSQL_RES *res; + if(mysql_query(m, q)) { + fprintf(stderr, "mysql_query: %s\n", mysql_error(m)); + return(NULL); + } + res = mysql_store_result(m); + return(res); +} + +/* this function is ugly -- it malloc's for each string it + * returns, and they probably won't be freed by the caller. + */ +char* addslashes(const char *s) { + char slashed[8192]; + char *p; + + slashed[0] = '\0'; + p = slashed; + while(*s) { + if(*s == '\'' || *s == '"' || *s == '\\') { + *p++ = '\\'; + } + *p++ = *s++; + } + *p = '\0'; + return(strdup(slashed)); +} + +char* trim(char *str) +{ + char *pch = str; + while(isspace(*pch)) { + pch++; + } + if(pch != str) { + memmove(str, pch, (strlen(pch) + 1)); + } + + pch = (char*)(str + (strlen(str) - 1)); + while(isspace(*pch)) { + pch--; + } + *++pch = '\0'; + + return str; +} + +int main(int argc, char **argv) +{ + MYSQL db; + MYSQL_RES *result; + MYSQL_ROW row; + char query[4096]; + char fn[PATH_MAX]; + char ftppath[PATH_MAX]; + int repoid; + pkg_t *pkglist = NULL; + pkg_t *pkgptr, *ptr; + + if(argc < 3) { + printf("usage: pkgdb2-del \n"); + printf("\nWARNING: Do not run this manually! It is intended to be run from\n" + "the Arch db-generation scripts.\n\n"); + return(1); + } + repoid = atoi(argv[1]); + strncpy(ftppath, argv[2], PATH_MAX-1); + + if(mysql_init(&db) == NULL) { + fprintf(stderr, "could not initialize\n"); + return(1); + } + if(mysql_real_connect(&db, "localhost", DB_USER, DB_PASS, DB_NAME, + 0, NULL, 0) == NULL) { + fprintf(stderr, "failed to connect to database: %s\n", mysql_error(&db)); + return(1); + } + + while(!feof(stdin)) { + int found = 0; + unsigned int catid = 0; + unsigned int pkgid = 0; + char name[256]; + /* get package data from stdin */ + fgets(name, 256, stdin); + trim(name); + if(feof(stdin)) continue; + /* check for overruns */ + if(strlen(name) > 254) { + fprintf(stderr, "pkgdb2-del: one or more fields are too long in package '%s'\n", name); + return(1); + } + /* get the package id */ + snprintf(query, sizeof(query), "SELECT id FROM packages WHERE " + "repoid='%d' AND pkgname='%s'", repoid, addslashes(name)); + result = doquery(&db, query); + if(mysql_num_rows(result) == 0) { + fprintf(stderr, "pkgdb2-del: %s was not found in repo %d\n", name, repoid); + continue; + } + row = mysql_fetch_row(result); + pkgid = (unsigned int)atoi(row[0]); + /* delete from db */ + fprintf(stderr, "pkgdb2-del: deleting %s (id %d)\n", name, pkgid); + snprintf(query, sizeof(query), "DELETE FROM packages WHERE id='%d'", pkgid); + doquery(&db, query); + snprintf(query, sizeof(query), "DELETE FROM packages_files WHERE id='%d'", pkgid); + doquery(&db, query); + snprintf(query, sizeof(query), "DELETE FROM todolist_pkgs WHERE pkgid='%d'", pkgid); + doquery(&db, query); + } + + mysql_close(&db); + return(0); +} -- cgit v1.2.2 From ab3b5e0eac5c557843716ab44ba44e49603f5d65 Mon Sep 17 00:00:00 2001 From: Judd Vinet Date: Thu, 6 Jul 2006 03:52:28 +0000 Subject: updated for new arch db schema --- pkgdb2-add.c | 18 +++++++++--------- pkgdb2-del.c | 8 ++++---- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/pkgdb2-add.c b/pkgdb2-add.c index 630d006..408a3e4 100644 --- a/pkgdb2-add.c +++ b/pkgdb2-add.c @@ -1,4 +1,4 @@ -/* $Id: pkgdb2-add.c,v 1.1 2006/07/06 03:37:01 judd Exp $ */ +/* $Id: pkgdb2-add.c,v 1.2 2006/07/06 03:52:28 judd Exp $ */ #include #include @@ -88,7 +88,7 @@ void updatefilelist(MYSQL *db, unsigned long id, char *fn) fprintf(stderr, "pkgdb2-add: could not open tempfile: %s\n", tmp); return; } - snprintf(query, sizeof(query), "DELETE FROM packages_files WHERE id='%d'", id); + snprintf(query, sizeof(query), "DELETE FROM packages_files WHERE pkg_id='%d'", id); doquery(db, query); while(fgets(line, sizeof(line)-1, fp)) { char *fixedfn = addslashes(trim(line)); @@ -97,7 +97,7 @@ void updatefilelist(MYSQL *db, unsigned long id, char *fn) continue; } /* varchars aren't case-sensitive but filesystems are, so we use REPLACE INTO */ - snprintf(query, sizeof(query), "REPLACE INTO packages_files (id,path) VALUES " + snprintf(query, sizeof(query), "REPLACE INTO packages_files (pkg_id,path) VALUES " "('%d', '%s')", id, fixedfn); free(fixedfn); doquery(db, query); @@ -138,7 +138,7 @@ int main(int argc, char **argv) return(1); } snprintf(query, sizeof(query), "SELECT id,pkgname,pkgver,pkgrel FROM packages " - "WHERE repoid='%d'", repoid); + "WHERE repo_id='%d'", repoid); result = doquery(&db, query); while(row = mysql_fetch_row(result)) { int i; @@ -237,9 +237,9 @@ int main(int argc, char **argv) /* Insert... */ unsigned long id; fprintf(stderr, "pkgdb2-add: inserting %s\n", name); - snprintf(query, sizeof(query), "INSERT INTO packages (id,repoid," - "categoryid,pkgname,pkgver,pkgrel,pkgdesc,url,sources,depends," - "lastupdate) VALUES (NULL,'%d','%d','%s','%s','%s','%s'," + snprintf(query, sizeof(query), "INSERT INTO packages (id,repo_id," + "category_id,pkgname,pkgver,pkgrel,pkgdesc,url,sources,depends," + "last_update) VALUES (NULL,'%d','%d','%s','%s','%s','%s'," "'%s','%s','%s',NOW())", repoid, catid, addslashes(name), addslashes(ver), addslashes(rel), addslashes(desc), addslashes(url), addslashes(sources), @@ -253,9 +253,9 @@ int main(int argc, char **argv) /* ...or Update */ fprintf(stderr, "pkgdb2-add: updating %s (%s-%s ==> %s-%s)\n", ptr->name, ptr->ver, ptr->rel, ver, rel); - snprintf(query, sizeof(query), "UPDATE packages SET categoryid='%d'," + snprintf(query, sizeof(query), "UPDATE packages SET category_id='%d'," "pkgname='%s',pkgver='%s',pkgrel='%s',pkgdesc='%s',url='%s'," - "sources='%s',depends='%s',needupdate=0,lastupdate=NOW() " + "sources='%s',depends='%s',needupdate=0,last_update=NOW() " "WHERE id='%d'", catid, addslashes(name), addslashes(ver), addslashes(rel), addslashes(desc), addslashes(url), addslashes(sources), diff --git a/pkgdb2-del.c b/pkgdb2-del.c index 36c8553..78b81fd 100644 --- a/pkgdb2-del.c +++ b/pkgdb2-del.c @@ -1,4 +1,4 @@ -/* $Id: pkgdb2-del.c,v 1.1 2006/07/06 03:37:01 judd Exp $ */ +/* $Id: pkgdb2-del.c,v 1.2 2006/07/06 03:52:28 judd Exp $ */ #include #include @@ -114,7 +114,7 @@ int main(int argc, char **argv) } /* get the package id */ snprintf(query, sizeof(query), "SELECT id FROM packages WHERE " - "repoid='%d' AND pkgname='%s'", repoid, addslashes(name)); + "repo_id='%d' AND pkgname='%s'", repoid, addslashes(name)); result = doquery(&db, query); if(mysql_num_rows(result) == 0) { fprintf(stderr, "pkgdb2-del: %s was not found in repo %d\n", name, repoid); @@ -126,9 +126,9 @@ int main(int argc, char **argv) fprintf(stderr, "pkgdb2-del: deleting %s (id %d)\n", name, pkgid); snprintf(query, sizeof(query), "DELETE FROM packages WHERE id='%d'", pkgid); doquery(&db, query); - snprintf(query, sizeof(query), "DELETE FROM packages_files WHERE id='%d'", pkgid); + snprintf(query, sizeof(query), "DELETE FROM packages_files WHERE pkg_id='%d'", pkgid); doquery(&db, query); - snprintf(query, sizeof(query), "DELETE FROM todolist_pkgs WHERE pkgid='%d'", pkgid); + snprintf(query, sizeof(query), "DELETE FROM todolists_pkgs WHERE pkg_id='%d'", pkgid); doquery(&db, query); } -- cgit v1.2.2 From a1d4355f6cae8275d4da6ee39f52095909cf11a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20B=C3=A4chler?= Date: Fri, 14 Sep 2007 16:38:41 +0000 Subject: adjusted scripts for pacman 3 naming scheme, should still work with the old one --- db-inc | 18 +++++++++++++++--- genpkglist | 21 ++++++++++++++++++--- pkgdb1 | 19 +++++++++++++++---- updatesync-many | 31 +++++++++++++++++++++---------- 4 files changed, 69 insertions(+), 20 deletions(-) diff --git a/db-inc b/db-inc index a984ca9..18eecda 100644 --- a/db-inc +++ b/db-inc @@ -1,4 +1,4 @@ -# $Id: db-inc,v 1.6 2006/05/07 18:02:42 judd Exp $ +# $Id: db-inc,v 1.7 2007/09/14 16:38:41 thomas Exp $ [ "$UID" = "" ] && UID=`uid` TMPDIR="/tmp/archpkg.$arch.$repoid.$UID" @@ -16,6 +16,18 @@ if [ ! -d $stagedir ]; then exit 1 fi +# Get the package name from the filename +# hackish, but should work for now +getpkgname() { + local tmp + + tmp=${1##*/} + tmp=${tmp%.pkg.tar.gz} + tmp=${tmp%-i686} + tmp=${tmp%-x86_64} + echo ${tmp%-*-*} +} + cleanup() { rm -rf $TMPDIR # unlock @@ -134,7 +146,7 @@ if [ -d $stagedir/del -a "`ls $stagedir/del`" ]; then cd $stagedir/del ( for i in *.pkg.tar.gz; do - pkgname=${i%-*-*} + pkgname=$(getpkgname $i) echo $pkgname done ) | $ARCHDIR/pkgdb2-del $repoid $stagedir/del @@ -147,7 +159,7 @@ fi if [ "$updatelists" ]; then echo "==> Generating Text Package List..." >&2 cd $TMPDIR/$cvsmod - $ARCHDIR/genpkglist $ftppath + $ARCHDIR/genpkglist $ftppath $arch # hack -- only Current's packages.txt goes in a "setup" subdir if [ "$reponame" = "current" ]; then diff --git a/genpkglist b/genpkglist index 7dba4d6..79224d8 100755 --- a/genpkglist +++ b/genpkglist @@ -1,5 +1,5 @@ #!/bin/bash -# $Id: genpkglist,v 1.14 2006/05/05 01:38:33 judd Exp $ +# $Id: genpkglist,v 1.15 2007/09/14 16:38:41 thomas Exp $ # # genpkglist @@ -8,8 +8,21 @@ # (also used to check for missing packages in the download directory) # +# Get the package name from the filename +# hackish, but should work for now +getpkgname() { + local tmp + + tmp=${1##*/} + tmp=${tmp%.pkg.tar.gz} + tmp=${tmp%-i686} + tmp=${tmp%-x86_64} + echo ${tmp%-*-*} +} + pkgfile="`pwd`/packages.txt" ftppath=$1 +dbarch=$2 rm -f $pkgfile for category in `find * -maxdepth 0 -type d | grep -v CVS`; do @@ -20,8 +33,10 @@ for category in `find * -maxdepth 0 -type d | grep -v CVS`; do . PKGBUILD if [ -f $ftppath/$pkgname-$pkgver-$pkgrel.pkg.tar.gz ]; then echo "$category/$pkgname-$pkgver-$pkgrel.pkg.tar.gz" >>$pkgfile + elif [ -f $ftppath/$pkgname-$pkgver-$pkgrel-$dbarch.pkg.tar.gz ]; then + echo "$category/$pkgname-$pkgver-$pkgrel-$dbarch.pkg.tar.gz" >>$pkgfile else - echo "notice: Missing $pkgname-$pkgver-$pkgrel.pkg.tar.gz in ftp site" >&2 + echo "notice: Missing $pkgname-$pkgver-$pkgrel-$dbarch.pkg.tar.gz in ftp site" >&2 fi fi cd .. @@ -32,7 +47,7 @@ done cd $ftppath unset DUPES DUPEFILES last for pkg in *.pkg.tar.gz; do - pkgname=${pkg%-*-*} + pkgname=$(getpkgname $pkg) if [ "$last" = "$pkgname" ]; then DUPES="$DUPES $pkgname" DUPEFILES="$DUPEFILES $pkg" diff --git a/pkgdb1 b/pkgdb1 index 5b2f09a..2c86ef8 100755 --- a/pkgdb1 +++ b/pkgdb1 @@ -1,5 +1,17 @@ #!/bin/bash -# $Id: pkgdb1,v 1.4 2006/02/20 06:33:40 judd Exp $ +# $Id: pkgdb1,v 1.5 2007/09/14 16:38:41 thomas Exp $ + +# Get the package name from the filename +# hackish, but should work for now +getpkgname() { + local tmp + + tmp=${1##*/} + tmp=${tmp%.pkg.tar.gz} + tmp=${tmp%-i686} + tmp=${tmp%-x86_64} + echo ${tmp%-*-*} +} STAGEDIR=`pwd` @@ -16,9 +28,8 @@ fi cd $STAGEDIR for pkgfile in `ls $STAGEDIR/*.pkg.tar.gz`; do - tmp=${pkgfile##*/} - pkgname=${tmp%-*-*}; - fullname=${tmp%.pkg.tar.gz} + pkgname=$(getpkgname $pkgfile); + fullname=$(basename $pkgfile) # find the matching PKGBUILD tmpf=$(mktemp /tmp/pkgdb1.XXXXXXXXXX) || exit 1 find $ABSDIR -type d -name "$pkgname" >$tmpf diff --git a/updatesync-many b/updatesync-many index 3899632..7833944 100755 --- a/updatesync-many +++ b/updatesync-many @@ -1,4 +1,5 @@ #!/bin/bash +# $Id: updatesync-many,v 1.3 2007/09/14 16:38:41 thomas Exp $ # # updatesync-many # @@ -46,6 +47,18 @@ msg() echo "updatesync-many: $*" >&2 } +# Get the package name from the filename +# hackish, but should work for now +getpkgname() { + local tmp + + tmp=${1##*/} + tmp=${tmp%.pkg.tar.gz} + tmp=${tmp%-i686} + tmp=${tmp%-x86_64} + echo ${tmp%-*-*} +} + check_option() { local i for i in ${options[@]}; do @@ -76,6 +89,9 @@ db_write_entry() cd $pkgname-$pkgver-$pkgrel # desc : >desc + echo "%FILENAME%" >>desc + echo "$2" >>desc + echo "" >>desc echo "%NAME%" >>desc echo "$pkgname" >>desc echo "" >>desc @@ -138,10 +154,8 @@ db_write_entry() delete_entry() { - # strip to the basename - tmp=${1##*/} # grab the pkgname - pkgname=${tmp%-*-*} + pkgname=$(getpkgname $1) for i in *; do if [ "${i%-*-*}" = "$pkgname" ]; then rm -rf $i @@ -152,9 +166,8 @@ delete_entry() update_entry() { pkgfile=$1 - tmp=${pkgfile##*/} - pkgname=${tmp%-*-*}; - fullname=${tmp%.pkg.tar.gz} + pkgname=$(getpkgname ${pkgfile}) + fullname=$(basename ${pkgfile}) # find the matching PKGBUILD tmpf=$(mktemp /tmp/updatesync-many.XXXXXXXXXX) || exit 1 @@ -185,7 +198,7 @@ update_entry() pkgmd5sum=`get_md5checksum $pkgfile` [ -z $pkgmd5sum ] && die "error generating checksum for $pkgfile" - db_write_entry $pkgbuild || die "error writing entry for $pkgname" + db_write_entry ${pkgbuild} ${fullname} || die "error writing entry for $pkgname" cd - >/dev/null } @@ -237,9 +250,7 @@ for pkgfile in $STAGEDIR/*.pkg.tar.gz; do lsof $pkgfile &>/dev/null [ $? -ne 1 ] && continue - tmp=${pkgfile##*/} - pkgname=${tmp%-*-*}; - fullname=${tmp%.pkg.tar.gz} + pkgname=$(getpkgname ${pkgfile}) if [ "$ACTION" = "del" ]; then msg "Deleting entry: $pkgname" delete_entry $pkgfile -- cgit v1.2.2 From 9292c7fd2e84315975110e7966434c5614aa605f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20B=C3=A4chler?= Date: Fri, 14 Sep 2007 22:57:54 +0000 Subject: fixed genpkglist --- genpkglist | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/genpkglist b/genpkglist index 79224d8..9207a2f 100755 --- a/genpkglist +++ b/genpkglist @@ -1,5 +1,5 @@ #!/bin/bash -# $Id: genpkglist,v 1.15 2007/09/14 16:38:41 thomas Exp $ +# $Id: genpkglist,v 1.16 2007/09/14 22:57:54 thomas Exp $ # # genpkglist @@ -58,7 +58,7 @@ done showdupes() { done= for i in *.pkg.tar.gz; do - pkgname=${i%-*-*} + pkgname=$(getpkgname $i) if [ "$pkgname" = "$1" ]; then ls -l $i | awk '{print $6" "$7" "$8" "$9}' done=1 -- cgit v1.2.2 From 872267523c5cdea86b65237632b8b369d47c8233 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20B=C3=A4chler?= Date: Fri, 14 Sep 2007 23:23:38 +0000 Subject: hopefully fixed filelists in mysql --- pkgdb1 | 3 ++- pkgdb2-add | Bin 13039 -> 13346 bytes pkgdb2-add.c | 16 +++++++++------- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/pkgdb1 b/pkgdb1 index 2c86ef8..ea6cb9b 100755 --- a/pkgdb1 +++ b/pkgdb1 @@ -1,5 +1,5 @@ #!/bin/bash -# $Id: pkgdb1,v 1.5 2007/09/14 16:38:41 thomas Exp $ +# $Id: pkgdb1,v 1.6 2007/09/14 23:23:38 thomas Exp $ # Get the package name from the filename # hackish, but should work for now @@ -55,6 +55,7 @@ for pkgfile in `ls $STAGEDIR/*.pkg.tar.gz`; do deplist=${depends[@]} sources=${source[@]} + echo $fullname echo $pkgname echo $pkgver echo $pkgrel diff --git a/pkgdb2-add b/pkgdb2-add index e6a87c0..c827bb0 100755 Binary files a/pkgdb2-add and b/pkgdb2-add differ diff --git a/pkgdb2-add.c b/pkgdb2-add.c index 408a3e4..53e0041 100644 --- a/pkgdb2-add.c +++ b/pkgdb2-add.c @@ -1,4 +1,4 @@ -/* $Id: pkgdb2-add.c,v 1.2 2006/07/06 03:52:28 judd Exp $ */ +/* $Id: pkgdb2-add.c,v 1.3 2007/09/14 23:23:38 thomas Exp $ */ #include #include @@ -88,7 +88,7 @@ void updatefilelist(MYSQL *db, unsigned long id, char *fn) fprintf(stderr, "pkgdb2-add: could not open tempfile: %s\n", tmp); return; } - snprintf(query, sizeof(query), "DELETE FROM packages_files WHERE pkg_id='%d'", id); + snprintf(query, sizeof(query), "DELETE FROM packages_files WHERE pkg_id='%lu'", id); doquery(db, query); while(fgets(line, sizeof(line)-1, fp)) { char *fixedfn = addslashes(trim(line)); @@ -98,7 +98,7 @@ void updatefilelist(MYSQL *db, unsigned long id, char *fn) } /* varchars aren't case-sensitive but filesystems are, so we use REPLACE INTO */ snprintf(query, sizeof(query), "REPLACE INTO packages_files (pkg_id,path) VALUES " - "('%d', '%s')", id, fixedfn); + "('%lu', '%s')", id, fixedfn); free(fixedfn); doquery(db, query); } @@ -174,8 +174,10 @@ int main(int argc, char **argv) unsigned int catid = 0; char name[256], ver[256], rel[256], desc[4096]; char cat[256], url[256], sources[4096], deplist[4096]; + char pkgfile[4096]; /* get package data from stdin */ - fgets(name, 256, stdin); trim(name); if(feof(stdin)) continue; + fgets(pkgfile, 4096, stdin); trim(pkgfile); if(feof(stdin)) continue; + fgets(name, 256, stdin); trim(name); if(feof(stdin)) continue; fgets(ver, 256, stdin); trim(ver); if(feof(stdin)) continue; fgets(rel, 256, stdin); trim(rel); if(feof(stdin)) continue; fgets(desc, 4096, stdin); trim(desc); if(feof(stdin)) continue; @@ -186,7 +188,7 @@ int main(int argc, char **argv) /* check for overruns */ if(strlen(name) > 254 || strlen(ver) >= 254 || strlen(rel) > 254 || strlen(desc) > 4094 || strlen(cat) >= 254 || strlen(url) > 254 || - strlen(sources) > 4094 || strlen(deplist) > 4094) { + strlen(sources) > 4094 || strlen(deplist) > 4094 || strlen(pkgfile) > 4094) { fprintf(stderr, "pkgdb2-add: one or more fields are too long in package '%s'\n", name); fprintf(stderr, "pkgdb2-add: check the lengths of your strings, most are limited " "to 255 chars, some are 4095\n"); @@ -246,7 +248,7 @@ int main(int argc, char **argv) addslashes(deplist)); doquery(&db, query); id = mysql_insert_id(&db); - snprintf(fn, PATH_MAX-1, "%s/%s-%s-%s.pkg.tar.gz", ftppath, name, ver, rel); + snprintf(fn, PATH_MAX-1, "%s/%s", ftppath, pkgfile); updatefilelist(&db, id, fn); continue; } else if(strcmp(ptr->ver, ver) || strcmp(ptr->rel, rel)) { @@ -261,7 +263,7 @@ int main(int argc, char **argv) addslashes(desc), addslashes(url), addslashes(sources), addslashes(deplist), ptr->id); doquery(&db, query); - snprintf(fn, PATH_MAX-1, "%s/%s-%s-%s.pkg.tar.gz", ftppath, name, ver, rel); + snprintf(fn, PATH_MAX-1, "%s/%s", ftppath, pkgfile); updatefilelist(&db, ptr->id, fn); /* snprintf(query, sizeof(query), "UPDATE todolist_pkgs SET complete=1 " -- cgit v1.2.2 From 1389aa84cfc4f327e2a2214dda9f4cf9605e40f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20B=C3=A4chler?= Date: Sun, 16 Sep 2007 11:43:14 +0000 Subject: new db-testing scripts --- db-testing | 8 ++++---- db-testing64 | 59 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 4 deletions(-) create mode 100755 db-testing64 diff --git a/db-testing b/db-testing index 1baad46..f631ad7 100755 --- a/db-testing +++ b/db-testing @@ -1,5 +1,5 @@ #!/bin/bash -# $Id: db-testing,v 1.13 2006/05/05 01:38:33 judd Exp $ +# $Id: db-testing,v 1.14 2007/09/16 11:43:14 thomas Exp $ repoid=4 reponame="testing" @@ -36,9 +36,9 @@ cd $cvsdir if [ "`ls $stagedir/add`" -o "`ls $stagedir/del`" ]; then echo "==> Checking out TESTING branches..." >&2 - echo "--- CURRENT ---" - CVS_RSH=ssh CVSROOT=:ext:cvs.archlinux.org:/home/cvs-arch cvs -q export -r TESTING arch - mv arch/build/* . && rm -rf arch + echo "--- CORE ---" + CVS_RSH=ssh CVSROOT=:ext:cvs.archlinux.org:/home/cvs-core cvs -q export -r TESTING core + cp -a core/* . && rm -rf core echo "--- EXTRA ---" CVS_RSH=ssh CVSROOT=:ext:cvs.archlinux.org:/home/cvs-extra cvs -q export -r TESTING extra cp -a extra/* . && rm -rf extra diff --git a/db-testing64 b/db-testing64 new file mode 100755 index 0000000..0525e2f --- /dev/null +++ b/db-testing64 @@ -0,0 +1,59 @@ +#!/bin/bash +# $Id: db-testing64,v 1.1 2007/09/16 11:43:14 thomas Exp $ + +export CARCH=x86_64 + +repoid=4 +reponame="testing" +arch="x86_64" + +ftppath="/home/ftp/testing/os/x86_64" + +cvspath="/home/cvs-arch" +cvsmod="testing" +cvstag="TESTING-64" + +#stagedir="/home/arch64/staging/testing" +stagedir="$HOME/staging/testing64" + +testing_cleanup() { + rm -rf $testingtmp +} + +# catch ^C +trap testing_cleanup SIGINT + +if [ -f /tmp/.repolck.$arch.$repoid ]; then + owner=`/bin/ls -l /tmp/.repolck.$arch.$repoid | awk '{print $3}'` + echo "error: db generation is already in progress (started by $owner)" + exit 1 +fi +# lock +touch /tmp/.repolck.$arch.$repoid + +# testing is special, so we have to do the CVS checkout stuff here +testingtmp=$(mktemp -d /tmp/testtmp-XXXXXX) || exit 1 +cvsdir=$testingtmp/testing +mkdir $cvsdir +cd $cvsdir + +if [ "`ls $stagedir/add`" -o "`ls $stagedir/del`" ]; then + echo "==> Checking out TESTING-64 branches..." >&2 + echo "--- CORE ---" + CVS_RSH=ssh CVSROOT=:ext:cvs.archlinux.org:/home/cvs-core cvs -q export -r TESTING-64 core + cp -a core/* . && rm -rf core + echo "--- EXTRA ---" + CVS_RSH=ssh CVSROOT=:ext:cvs.archlinux.org:/home/cvs-extra cvs -q export -r TESTING-64 extra + cp -a extra/* . && rm -rf extra + echo "--- UNSTABLE ---" + CVS_RSH=ssh CVSROOT=:ext:cvs.archlinux.org:/home/cvs-unstable cvs -q export -r TESTING-64 unstable + cp -a unstable/* . && rm -rf unstable +fi + +# unlock +rm -f /tmp/.repolck.$arch.$repoid + +# call the real script +. `dirname $0`/db-inc + +testing_cleanup -- cgit v1.2.2 From aa0f9ebe47906f7fe2aa29cdb0ccf74aed771708 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Thu, 29 Nov 2007 09:28:28 -0600 Subject: Remove CVS Id lines Signed-off-by: Dan McGee --- db-arch | 1 - db-arch64 | 1 - db-extra | 1 - db-extra64 | 1 - db-inc | 2 -- db-testing | 1 - db-testing64 | 1 - db-unstable | 1 - db-unstable64 | 1 - genpkglist | 2 -- pkgdb | 1 - pkgdb1 | 1 - pkgdb2-add.c | 2 -- pkgdb2-del.c | 2 -- pkgdb2.c | 2 -- updatesync-many | 1 - 16 files changed, 21 deletions(-) diff --git a/db-arch b/db-arch index 9b61eee..34a3176 100755 --- a/db-arch +++ b/db-arch @@ -1,5 +1,4 @@ #!/bin/bash -# $Id: db-arch,v 1.24 2006/05/05 01:38:33 judd Exp $ repoid=1 reponame="current" diff --git a/db-arch64 b/db-arch64 index ac2cbb1..a16a6df 100755 --- a/db-arch64 +++ b/db-arch64 @@ -1,5 +1,4 @@ #!/bin/bash -# $Id: db-arch64,v 1.1 2006/05/05 01:38:33 judd Exp $ repoid=1 reponame="current" diff --git a/db-extra b/db-extra index c24a26f..af78c98 100755 --- a/db-extra +++ b/db-extra @@ -1,5 +1,4 @@ #!/bin/bash -# $Id: db-extra,v 1.13 2006/05/05 01:38:33 judd Exp $ repoid=2 reponame="extra" diff --git a/db-extra64 b/db-extra64 index 8dce59b..20c1a9b 100755 --- a/db-extra64 +++ b/db-extra64 @@ -1,5 +1,4 @@ #!/bin/bash -# $Id: db-extra64,v 1.1 2006/05/05 01:38:33 judd Exp $ repoid=2 reponame="extra" diff --git a/db-inc b/db-inc index 18eecda..32fc9f1 100644 --- a/db-inc +++ b/db-inc @@ -1,5 +1,3 @@ -# $Id: db-inc,v 1.7 2007/09/14 16:38:41 thomas Exp $ - [ "$UID" = "" ] && UID=`uid` TMPDIR="/tmp/archpkg.$arch.$repoid.$UID" diff --git a/db-testing b/db-testing index f631ad7..70451a2 100755 --- a/db-testing +++ b/db-testing @@ -1,5 +1,4 @@ #!/bin/bash -# $Id: db-testing,v 1.14 2007/09/16 11:43:14 thomas Exp $ repoid=4 reponame="testing" diff --git a/db-testing64 b/db-testing64 index 0525e2f..8b306ac 100755 --- a/db-testing64 +++ b/db-testing64 @@ -1,5 +1,4 @@ #!/bin/bash -# $Id: db-testing64,v 1.1 2007/09/16 11:43:14 thomas Exp $ export CARCH=x86_64 diff --git a/db-unstable b/db-unstable index e37e664..0b4c524 100755 --- a/db-unstable +++ b/db-unstable @@ -1,5 +1,4 @@ #!/bin/bash -# $Id: db-unstable,v 1.18 2006/05/05 01:38:33 judd Exp $ repoid=3 reponame="unstable" diff --git a/db-unstable64 b/db-unstable64 index 38904d3..6879056 100755 --- a/db-unstable64 +++ b/db-unstable64 @@ -1,5 +1,4 @@ #!/bin/bash -# $Id: db-unstable64,v 1.1 2006/05/05 01:38:33 judd Exp $ repoid=3 reponame="unstable" diff --git a/genpkglist b/genpkglist index 9207a2f..ed91b0c 100755 --- a/genpkglist +++ b/genpkglist @@ -1,6 +1,4 @@ #!/bin/bash -# $Id: genpkglist,v 1.16 2007/09/14 22:57:54 thomas Exp $ - # # genpkglist # diff --git a/pkgdb b/pkgdb index e73c455..6a21f1e 100755 --- a/pkgdb +++ b/pkgdb @@ -1,5 +1,4 @@ #!/bin/bash -# $Id: pkgdb,v 1.15 2005/01/03 08:09:21 judd Exp $ tl=`pwd` diff --git a/pkgdb1 b/pkgdb1 index ea6cb9b..fee01fc 100755 --- a/pkgdb1 +++ b/pkgdb1 @@ -1,5 +1,4 @@ #!/bin/bash -# $Id: pkgdb1,v 1.6 2007/09/14 23:23:38 thomas Exp $ # Get the package name from the filename # hackish, but should work for now diff --git a/pkgdb2-add.c b/pkgdb2-add.c index 53e0041..292d0dc 100644 --- a/pkgdb2-add.c +++ b/pkgdb2-add.c @@ -1,5 +1,3 @@ -/* $Id: pkgdb2-add.c,v 1.3 2007/09/14 23:23:38 thomas Exp $ */ - #include #include #include diff --git a/pkgdb2-del.c b/pkgdb2-del.c index 78b81fd..c81e542 100644 --- a/pkgdb2-del.c +++ b/pkgdb2-del.c @@ -1,5 +1,3 @@ -/* $Id: pkgdb2-del.c,v 1.2 2006/07/06 03:52:28 judd Exp $ */ - #include #include #include diff --git a/pkgdb2.c b/pkgdb2.c index 9710cc5..0fc0cc9 100644 --- a/pkgdb2.c +++ b/pkgdb2.c @@ -1,5 +1,3 @@ -/* $Id: pkgdb2.c,v 1.6 2005/02/25 21:27:45 judd Exp $ */ - #include #include #include diff --git a/updatesync-many b/updatesync-many index 7833944..40c5e4d 100755 --- a/updatesync-many +++ b/updatesync-many @@ -1,5 +1,4 @@ #!/bin/bash -# $Id: updatesync-many,v 1.3 2007/09/14 16:38:41 thomas Exp $ # # updatesync-many # -- cgit v1.2.2 From 8370508de93460fc1c2860ba8d1e18de1f271847 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Thu, 29 Nov 2007 09:31:59 -0600 Subject: Remove db-arch scripts, add db-core scripts Signed-off-by: Dan McGee --- db-arch | 17 ----------------- db-arch64 | 17 ----------------- db-core | 17 +++++++++++++++++ db-core64 | 20 ++++++++++++++++++++ 4 files changed, 37 insertions(+), 34 deletions(-) delete mode 100755 db-arch delete mode 100755 db-arch64 create mode 100755 db-core create mode 100755 db-core64 diff --git a/db-arch b/db-arch deleted file mode 100755 index 34a3176..0000000 --- a/db-arch +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash - -repoid=1 -reponame="current" -arch="i686" - -ftppath="/home/ftp/current/os/i686" - -cvspath="/home/cvs-arch" -cvsmod="arch/build" -cvstag="CURRENT" - -stagedir="$HOME/staging/arch" - -# call the real script -. `dirname $0`/db-inc - diff --git a/db-arch64 b/db-arch64 deleted file mode 100755 index a16a6df..0000000 --- a/db-arch64 +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash - -repoid=1 -reponame="current" -arch="x86_64" - -ftppath="/home/ftp/current/os/x86_64" - -cvspath="/home/cvs-arch" -cvsmod="arch/build" -cvstag="CURRENT-64" - -stagedir="/home/arch64/staging/arch" - -# call the real script -. `dirname $0`/db-inc - diff --git a/db-core b/db-core new file mode 100755 index 0000000..34a24f1 --- /dev/null +++ b/db-core @@ -0,0 +1,17 @@ +#!/bin/bash + +repoid=5 +reponame="core" +arch="i686" + +ftppath="/home/ftp/core/os/i686" + +cvspath="/home/cvs-core" +cvsmod="core" +cvstag="CURRENT" + +stagedir="$HOME/staging/core" + +# call the real script +. `dirname $0`/db-inc + diff --git a/db-core64 b/db-core64 new file mode 100755 index 0000000..a96aa75 --- /dev/null +++ b/db-core64 @@ -0,0 +1,20 @@ +#!/bin/bash + +export CARCH=x86_64 + +repoid=5 +reponame="core" +arch="x86_64" + +ftppath="/home/ftp/core/os/x86_64" + +cvspath="/home/cvs-core" +cvsmod="core" +cvstag="CURRENT-64" + +#stagedir="/home/arch64/staging/extra" +stagedir="$HOME/staging/core64" + +# call the real script +. `dirname $0`/db-inc + -- cgit v1.2.2 From 9a4215ec61c7ae438695d51651665cc0e2fa0a28 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Thu, 29 Nov 2007 09:35:05 -0600 Subject: Get GIT repo in line with the current dbscripts Signed-off-by: Dan McGee --- db-extra64 | 5 ++++- db-unstable64 | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/db-extra64 b/db-extra64 index 20c1a9b..3cce034 100755 --- a/db-extra64 +++ b/db-extra64 @@ -1,5 +1,7 @@ #!/bin/bash +export CARCH=x86_64 + repoid=2 reponame="extra" arch="x86_64" @@ -10,7 +12,8 @@ cvspath="/home/cvs-extra" cvsmod="extra" cvstag="CURRENT-64" -stagedir="/home/arch64/staging/extra" +#stagedir="/home/arch64/staging/extra" +stagedir="$HOME/staging/extra64" # call the real script . `dirname $0`/db-inc diff --git a/db-unstable64 b/db-unstable64 index 6879056..dece93a 100755 --- a/db-unstable64 +++ b/db-unstable64 @@ -1,5 +1,7 @@ #!/bin/bash +export CARCH=x86_64 + repoid=3 reponame="unstable" arch="x86_64" @@ -10,7 +12,8 @@ cvspath="/home/cvs-unstable" cvsmod="unstable" cvstag="CURRENT-64" -stagedir="/home/arch64/staging/unstable" +#stagedir="/home/arch64/staging/unstable" +stagedir="$HOME/staging/unstable64" # call the real script . `dirname $0`/db-inc -- cgit v1.2.2 From 410d14a1f9f22bf3f52a58f732305020c29e3141 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20B=C3=A4chler?= Date: Sun, 3 Feb 2008 18:27:29 +0100 Subject: Adding cleanup scripts so they don't get lost: cleanup.sh: Iterates through the db file and lists dupes and missing files. cleanup2.sh: Iterates through the ftp directory and finds files that don't correspond to any package in the db. They currently only list files, but don't delete anything. --- cleanup-scripts/cleanup.sh | 53 +++++++++++++++++++++++++++++++++++++++++++++ cleanup-scripts/cleanup2.sh | 46 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 99 insertions(+) create mode 100755 cleanup-scripts/cleanup.sh create mode 100755 cleanup-scripts/cleanup2.sh diff --git a/cleanup-scripts/cleanup.sh b/cleanup-scripts/cleanup.sh new file mode 100755 index 0000000..cdb0cef --- /dev/null +++ b/cleanup-scripts/cleanup.sh @@ -0,0 +1,53 @@ +#!/bin/bash + +usage() { + echo "Usage: $0 repo architecture" +} + +getpkgname() { + local tmp + + tmp=${1##*/} + tmp=${tmp%.pkg.tar.gz} + tmp=${tmp%-i686} + tmp=${tmp%-x86_64} + echo ${tmp%-*-*} +} + +FTPBASEDIR="/home/ftp" +FTPDIR=${FTPBASEDIR}/${1}/os/${2} +DBFILE=${FTPDIR}/${1}.db.tar.gz +MISSINGFILES="" +DELETEFILES="" + +if [ $# -lt 2 -o ! -f ${DBFILE} ]; then + usage + exit 1 +fi + +TMPDIR=$(mktemp -d /tmp/cleanup.XXXXXX) || exit 1 + +cd ${TMPDIR} +tar xzf ${DBFILE} +for pkg in *; do + filename=$(grep -A1 '^%FILENAME%$' ${pkg}/desc | tail -n1) + [ -z "${filename}" ] && filename="${pkg}.pkg.tar.gz" + if [ ! -f ${FTPDIR}/${filename} ]; then + MISSINGFILES="${MISSINGFILES} ${filename}" + else + pkgname="$(getpkgname ${filename})" + for otherfile in ${FTPDIR}/${pkgname}-*; do + otherfile="$(basename ${otherfile})" + if [ "${otherfile}" != "${filename}" -a "${pkgname}" = "$(getpkgname ${otherfile})" ]; then + DELETEFILES="${DELETEFILES} ${otherfile}" + fi + done + fi +done + +cd - >/dev/null +rm -rf ${TMPDIR} + +echo -ne "DIRECTORY:\n${FTPDIR}\n\n" +echo -ne "DELETEFILES:\n${DELETEFILES}\n\n" +echo -ne "MISSINGFILES:\n${MISSINGFILES}\n\n" diff --git a/cleanup-scripts/cleanup2.sh b/cleanup-scripts/cleanup2.sh new file mode 100755 index 0000000..4a79654 --- /dev/null +++ b/cleanup-scripts/cleanup2.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +usage() { + echo "Usage: $0 repo architecture" +} + +getpkgname() { + local tmp + + tmp=${1##*/} + tmp=${tmp%.pkg.tar.gz} + tmp=${tmp%-i686} + tmp=${tmp%-x86_64} + echo ${tmp%-*-*} +} + +FTPBASEDIR="/home/ftp" +FTPDIR=${FTPBASEDIR}/${1}/os/${2} +DBFILE=${FTPDIR}/${1}.db.tar.gz +OBSOLETEFILES="" + +if [ $# -lt 2 -o ! -f ${DBFILE} ]; then + usage + exit 1 +fi + +TMPDIR=$(mktemp -d /tmp/cleanup.XXXXXX) || exit 1 +cd ${TMPDIR} +tar xzf ${DBFILE} + +cd ${FTPDIR} +for pkgfile in *.pkg.tar.gz; do + pkgname="$(getpkgname ${pkgfile})" + for p in ${FTPDIR}/${pkgname}-*; do + if [ "$(getpkgname $(basename ${p}))" = "${pkgname}" ]; then + continue 2 + fi + done + OBSOLETEFILES="${OBSOLETEFILES} ${pkgfile}" +done + +cd - >/dev/null +rm -rf ${TMPDIR} + +echo -ne "DIRECTORY:\n${FTPDIR}\n\n" +echo -ne "OBSOLETEFILES:\n${OBSOLETEFILES}\n\n" -- cgit v1.2.2 From 29a77254307e42ba0959fe7660aa3d0d76285e6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20B=C3=A4chler?= Date: Sat, 16 Feb 2008 13:50:25 +0100 Subject: Commited cleanup script to db-inc/genpkglist --- cleanup-scripts/cleanup.sh | 53 ------------------------------------------ db-inc | 2 ++ ftpdir-cleanup | 57 ++++++++++++++++++++++++++++++++++++++++++++++ genpkglist | 33 --------------------------- 4 files changed, 59 insertions(+), 86 deletions(-) delete mode 100755 cleanup-scripts/cleanup.sh create mode 100755 ftpdir-cleanup diff --git a/cleanup-scripts/cleanup.sh b/cleanup-scripts/cleanup.sh deleted file mode 100755 index cdb0cef..0000000 --- a/cleanup-scripts/cleanup.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/bin/bash - -usage() { - echo "Usage: $0 repo architecture" -} - -getpkgname() { - local tmp - - tmp=${1##*/} - tmp=${tmp%.pkg.tar.gz} - tmp=${tmp%-i686} - tmp=${tmp%-x86_64} - echo ${tmp%-*-*} -} - -FTPBASEDIR="/home/ftp" -FTPDIR=${FTPBASEDIR}/${1}/os/${2} -DBFILE=${FTPDIR}/${1}.db.tar.gz -MISSINGFILES="" -DELETEFILES="" - -if [ $# -lt 2 -o ! -f ${DBFILE} ]; then - usage - exit 1 -fi - -TMPDIR=$(mktemp -d /tmp/cleanup.XXXXXX) || exit 1 - -cd ${TMPDIR} -tar xzf ${DBFILE} -for pkg in *; do - filename=$(grep -A1 '^%FILENAME%$' ${pkg}/desc | tail -n1) - [ -z "${filename}" ] && filename="${pkg}.pkg.tar.gz" - if [ ! -f ${FTPDIR}/${filename} ]; then - MISSINGFILES="${MISSINGFILES} ${filename}" - else - pkgname="$(getpkgname ${filename})" - for otherfile in ${FTPDIR}/${pkgname}-*; do - otherfile="$(basename ${otherfile})" - if [ "${otherfile}" != "${filename}" -a "${pkgname}" = "$(getpkgname ${otherfile})" ]; then - DELETEFILES="${DELETEFILES} ${otherfile}" - fi - done - fi -done - -cd - >/dev/null -rm -rf ${TMPDIR} - -echo -ne "DIRECTORY:\n${FTPDIR}\n\n" -echo -ne "DELETEFILES:\n${DELETEFILES}\n\n" -echo -ne "MISSINGFILES:\n${MISSINGFILES}\n\n" diff --git a/db-inc b/db-inc index 32fc9f1..e86f87f 100644 --- a/db-inc +++ b/db-inc @@ -167,6 +167,8 @@ if [ "$updatelists" ]; then fi fi +$ARCHDIR/ftpdir-cleanup $reponame $arch + cleanup # vim: set ts=2 noet ft=sh: diff --git a/ftpdir-cleanup b/ftpdir-cleanup new file mode 100755 index 0000000..df400bd --- /dev/null +++ b/ftpdir-cleanup @@ -0,0 +1,57 @@ +#!/bin/bash + +usage() { + echo "Usage: $0 repo architecture" +} + +getpkgname() { + local tmp + + tmp=${1##*/} + tmp=${tmp%.pkg.tar.gz} + tmp=${tmp%-i686} + tmp=${tmp%-x86_64} + echo ${tmp%-*-*} +} + +FTPBASEDIR="/home/ftp" +FTPDIR=${FTPBASEDIR}/${1}/os/${2} +DBFILE=${FTPDIR}/${1}.db.tar.gz +MISSINGFILES="" +DELETEFILES="" + +if [ $# -lt 2 -o ! -f ${DBFILE} ]; then + usage + exit 1 +fi + +TMPDIR=$(mktemp -d /tmp/cleanup.XXXXXX) || exit 1 + +cd ${TMPDIR} +tar xzf ${DBFILE} +for pkg in *; do + filename=$(grep -A1 '^%FILENAME%$' ${pkg}/desc | tail -n1) + [ -z "${filename}" ] && filename="${pkg}.pkg.tar.gz" + if [ ! -f ${FTPDIR}/${filename} ]; then + MISSINGFILES="${MISSINGFILES} ${filename}" + else + pkgname="$(getpkgname ${filename})" + for otherfile in ${FTPDIR}/${pkgname}-*; do + otherfile="$(basename ${otherfile})" + if [ "${otherfile}" != "${filename}" -a "${pkgname}" = "$(getpkgname ${otherfile})" ]; then + DELETEFILES="${DELETEFILES} ${otherfile}" + fi + done + fi +done + +cd - >/dev/null +rm -rf ${TMPDIR} + +echo -ne "DIRECTORY:\n${FTPDIR}\n\n" +echo -ne "DELETEFILES:\n${DELETEFILES}\n\n" +echo -ne "MISSINGFILES:\n${MISSINGFILES}\n\n" + +cd ${FTPDIR} +mv ${DELETEFILES} /home/package-cleanup/ +cd - diff --git a/genpkglist b/genpkglist index ed91b0c..d026916 100755 --- a/genpkglist +++ b/genpkglist @@ -41,36 +41,3 @@ for category in `find * -maxdepth 0 -type d | grep -v CVS`; do done cd .. done - -cd $ftppath -unset DUPES DUPEFILES last -for pkg in *.pkg.tar.gz; do - pkgname=$(getpkgname $pkg) - if [ "$last" = "$pkgname" ]; then - DUPES="$DUPES $pkgname" - DUPEFILES="$DUPEFILES $pkg" - fi - last=$pkgname -done - -showdupes() { - done= - for i in *.pkg.tar.gz; do - pkgname=$(getpkgname $i) - if [ "$pkgname" = "$1" ]; then - ls -l $i | awk '{print $6" "$7" "$8" "$9}' - done=1 - else - [ "$done" = "1" ] && return - fi - done -} - -if [ "$DUPES" ]; then - echo "Possible Dupes for $ftppath (remove old versions)" - echo "Date Filename" - for dupe in $((for d in `echo $DUPES`; do echo $d; done) | sort -u); do - showdupes $dupe - done -fi - -- cgit v1.2.2 From 4bc106f1a60a6172b707bfaee5aff5030234466e Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Mon, 7 Apr 2008 22:52:55 -0500 Subject: Giant SVN changes Some other fun stuff snuck in there, like indentation changes, but this should get us mostly working under SVN, and at least allow us to build DB files at the very least Signed-off-by: Aaron Griffin --- db-core | 12 +- db-core64 | 15 +- db-extra | 12 +- db-extra64 | 15 +- db-inc | 198 ++++++++++++------------ db-testing | 50 +----- db-testing64 | 53 +------ db-unstable | 11 +- db-unstable64 | 14 +- pkgdb1 | 101 ++++++------ pkgdb2-add.c | 464 ++++++++++++++++++++++++++++---------------------------- updatesync-many | 16 +- 12 files changed, 401 insertions(+), 560 deletions(-) diff --git a/db-core b/db-core index 34a24f1..ccbfb61 100755 --- a/db-core +++ b/db-core @@ -4,14 +4,4 @@ repoid=5 reponame="core" arch="i686" -ftppath="/home/ftp/core/os/i686" - -cvspath="/home/cvs-core" -cvsmod="core" -cvstag="CURRENT" - -stagedir="$HOME/staging/core" - -# call the real script -. `dirname $0`/db-inc - +. $(dirname $0)/db-inc diff --git a/db-core64 b/db-core64 index a96aa75..d15be60 100755 --- a/db-core64 +++ b/db-core64 @@ -1,20 +1,7 @@ #!/bin/bash -export CARCH=x86_64 - repoid=5 reponame="core" arch="x86_64" -ftppath="/home/ftp/core/os/x86_64" - -cvspath="/home/cvs-core" -cvsmod="core" -cvstag="CURRENT-64" - -#stagedir="/home/arch64/staging/extra" -stagedir="$HOME/staging/core64" - -# call the real script -. `dirname $0`/db-inc - +. $(dirname $0)/db-inc diff --git a/db-extra b/db-extra index af78c98..8d2c5b8 100755 --- a/db-extra +++ b/db-extra @@ -4,14 +4,4 @@ repoid=2 reponame="extra" arch="i686" -ftppath="/home/ftp/extra/os/i686" - -cvspath="/home/cvs-extra" -cvsmod="extra" -cvstag="CURRENT" - -stagedir="$HOME/staging/extra" - -# call the real script -. `dirname $0`/db-inc - +. $(dirname $0)/db-inc diff --git a/db-extra64 b/db-extra64 index 3cce034..19eef9f 100755 --- a/db-extra64 +++ b/db-extra64 @@ -1,20 +1,7 @@ #!/bin/bash -export CARCH=x86_64 - repoid=2 reponame="extra" arch="x86_64" -ftppath="/home/ftp/extra/os/x86_64" - -cvspath="/home/cvs-extra" -cvsmod="extra" -cvstag="CURRENT-64" - -#stagedir="/home/arch64/staging/extra" -stagedir="$HOME/staging/extra64" - -# call the real script -. `dirname $0`/db-inc - +. $(dirname $0)/db-inc diff --git a/db-inc b/db-inc index e86f87f..4f64dfe 100644 --- a/db-inc +++ b/db-inc @@ -1,53 +1,69 @@ -[ "$UID" = "" ] && UID=`uid` -TMPDIR="/tmp/archpkg.$arch.$repoid.$UID" +#!/bin/bash # where are the arch scripts located? ARCHDIR="/arch" +#All this fun stuff used to be in the db-(whatever) files +# Let's make it cleaner +export CARCH="$arch" +ftppath="/home/ftp/$reponame/os/$arch/" +svnpath="/home/svn-packages" +svnrepo="$reponame-$arch" + +#Hacky for now +if [ "$arch" = "x86_64" ]; then + stagedir="$HOME/staging/$reponame-64" +else + stagedir="$HOME/staging/$reponame" +fi + +[ "$UID" = "" ] && UID=$(uid) +TMPDIR="/tmp/archpkg.$arch.$repoid.$UID" + if [ ! `type -p fakeroot` ]; then - echo "error: fakeroot is missing" >&2 - exit 1 + echo "error: fakeroot is missing" >&2 + exit 1 fi if [ ! -d $stagedir ]; then - echo "error: staging directory missing: $stagedir" >&2 - exit 1 + echo "error: staging directory missing: $stagedir" >&2 + exit 1 fi # Get the package name from the filename # hackish, but should work for now getpkgname() { - local tmp + local tmp - tmp=${1##*/} - tmp=${tmp%.pkg.tar.gz} - tmp=${tmp%-i686} - tmp=${tmp%-x86_64} - echo ${tmp%-*-*} + tmp=${1##*/} + tmp=${tmp%.pkg.tar.gz} + tmp=${tmp%-i686} + tmp=${tmp%-x86_64} + echo ${tmp%-*-*} } cleanup() { - rm -rf $TMPDIR - # unlock - rm -f /tmp/.repolck.$arch.$repoid - [ "$1" ] && exit $1 + rm -rf $TMPDIR + # unlock + rm -f /tmp/.repolck.$arch.$repoid + [ "$1" ] && exit $1 } ctrl_c() { - echo "Interrupted" >&2 - cleanup 0 + echo "Interrupted" >&2 + cleanup 0 } die() { - echo "$*" >&2 - cleanup 1 + echo "$*" >&2 + cleanup 1 } # check for locks if [ -f /tmp/.repolck.$arch.$repoid ]; then - owner=`/bin/ls -l /tmp/.repolck.$arch.$repoid | awk '{print $3}'` - echo "error: db generation is already in progress (started by $owner)" - exit 1 + owner=`/bin/ls -l /tmp/.repolck.$arch.$repoid | awk '{print $3}'` + echo "error: db generation is already in progress (started by $owner)" + exit 1 fi # catch ^C breaks @@ -57,30 +73,30 @@ touch /tmp/.repolck.$arch.$repoid # RedHat's mktemp is broken... if [ -d $TMPDIR ]; then - echo "==> Removing old temp dir..." >&2 - rm -rf $TMPDIR || exit 1 + echo "==> Removing old temp dir..." >&2 + rm -rf $TMPDIR || exit 1 fi mkdir $TMPDIR; [ $? -gt 0 ] && exit 1 cd $TMPDIR -# Checkout the CVS module if we need to +# Checkout the SVN module if we need to updatelists= if [ "`ls $stagedir/add 2>/dev/null`" -o "`ls $stagedir/del 2>/dev/null`" ]; then - # if $cvsdir is set, then use that instead of doing our own cvs checkout - if [ "$cvsdir" ]; then - mv $cvsdir $TMPDIR/$cvsmod - else - echo "==> Checking out module: $cvsmod ($cvstag)" - CVS_RSH=ssh CVSROOT=:ext:cvs.archlinux.org:$cvspath cvs -q export -r $cvstag $cvsmod - if [ $? -gt 0 ]; then - die "==> CVS export failed!" + # if $svndir is set, then use that instead of doing our own cvs checkout + if [ "$svndir" ]; then + mv $svndir $TMPDIR/checkout + else + echo "==> Checking out repo: $svnrepo ($arch)" + svn export file://$svnpath $TMPDIR/checkout + if [ $? -gt 0 ]; then + die "==> SVN export failed!" + fi fi - fi - updatelists=1 + updatelists=1 else - echo "No files to process" - cleanup 0 + echo "No files to process" + cleanup 0 fi # Right-O, now we look through the "add" and "del" subdirectories of @@ -92,83 +108,75 @@ fi # one db-* invocation, but it's not a huge performance hit. if [ -d $stagedir/add -a "`ls $stagedir/add`" ]; then - cd $TMPDIR - echo "==> Processing new/updated packages for repository '$reponame'..." >&2 + cd $TMPDIR + echo "==> Processing new/updated packages for repository '$reponame'..." >&2 - # copy the db file into our working area - cp $ftppath/$reponame.db.tar.gz . + # copy the db file into our working area + cp $ftppath/$reponame.db.tar.gz . - cd $stagedir/add - # run it thru fakeroot make sure everything is owned by root.root - echo "$ARCHDIR/updatesync-many add $TMPDIR/$reponame.db.tar.gz $TMPDIR/$cvsmod" \ + cd $stagedir/add + # run it thru fakeroot make sure everything is owned by root.root + echo "$ARCHDIR/updatesync-many add $TMPDIR/$reponame.db.tar.gz $TMPDIR/checkout $svnrepo" \ | fakeroot - if [ $? -ne 0 ]; then - die "==> Error returned from updatesync-many" - fi + if [ $? -ne 0 ]; then + die "==> Error returned from updatesync-many" + fi - cp $TMPDIR/$reponame.db.tar.gz $ftppath + cp $TMPDIR/$reponame.db.tar.gz $ftppath - # only for i686 (for now) - if [ "$arch" = "i686" ]; then - echo "==> Scanning for New/Updated packages..." >&2 - cd $stagedir/add - $ARCHDIR/pkgdb1 $TMPDIR/$cvsmod | $ARCHDIR/pkgdb2-add $repoid $stagedir/add - fi + # only for i686 (for now) + if [ "$arch" = "i686" ]; then + echo "==> Scanning for New/Updated packages..." >&2 + cd $stagedir/add + $ARCHDIR/pkgdb1 $TMPDIR/checkout $svnrepo | $ARCHDIR/pkgdb2-add $repoid $stagedir/add + fi - # move the package files into the ftp directory - mv -f $stagedir/add/*.pkg.tar.gz $ftppath + # move the package files into the ftp directory + mv -f $stagedir/add/*.pkg.tar.gz $ftppath fi - + if [ -d $stagedir/del -a "`ls $stagedir/del`" ]; then - cd $TMPDIR - echo "==> Processing deleted packages for repository '$reponame'..." >&2 + cd $TMPDIR + echo "==> Processing deleted packages for repository '$reponame'..." >&2 - # copy the db file into our working area - cp $ftppath/$reponame.db.tar.gz . + # copy the db file into our working area + cp $ftppath/$reponame.db.tar.gz . - cd $stagedir/del - # run it thru fakeroot make sure everything is owned by root.root - echo "$ARCHDIR/updatesync-many del $TMPDIR/$reponame.db.tar.gz NOT-USED" \ + cd $stagedir/del + # run it thru fakeroot make sure everything is owned by root.root + echo "$ARCHDIR/updatesync-many del $TMPDIR/$reponame.db.tar.gz NOT-USED ZOMGWOO" \ | fakeroot - if [ $? -ne 0 ]; then - die "==> Error returned from updatesync-many" - fi + if [ $? -ne 0 ]; then + die "==> Error returned from updatesync-many" + fi - cp $TMPDIR/$reponame.db.tar.gz $ftppath + cp $TMPDIR/$reponame.db.tar.gz $ftppath + + # only for i686 (for now) + if [ "$arch" = "i686" ]; then + echo "==> Scanning for Deleted packages..." >&2 + cd $stagedir/del + ( + for i in *.pkg.tar.gz; do + pkgname=$(getpkgname $i) + echo $pkgname + done + ) | $ARCHDIR/pkgdb2-del $repoid $stagedir/del + fi - # only for i686 (for now) - if [ "$arch" = "i686" ]; then - echo "==> Scanning for Deleted packages..." >&2 - cd $stagedir/del - ( - for i in *.pkg.tar.gz; do - pkgname=$(getpkgname $i) - echo $pkgname - done - ) | $ARCHDIR/pkgdb2-del $repoid $stagedir/del - fi - - # remove the package files - rm -f $stagedir/del/*.pkg.tar.gz + # remove the package files + rm -f $stagedir/del/*.pkg.tar.gz fi if [ "$updatelists" ]; then - echo "==> Generating Text Package List..." >&2 - cd $TMPDIR/$cvsmod - $ARCHDIR/genpkglist $ftppath $arch - - # hack -- only Current's packages.txt goes in a "setup" subdir - if [ "$reponame" = "current" ]; then - mv packages.txt $ftppath/setup/packages.txt - else + echo "==> Generating Text Package List..." >&2 + cd $TMPDIR/checkout + $ARCHDIR/genpkglist $ftppath $arch mv packages.txt $ftppath/packages.txt - fi fi -$ARCHDIR/ftpdir-cleanup $reponame $arch - cleanup -# vim: set ts=2 noet ft=sh: +# vim: set ts=4 sw=4 noet ft=sh: diff --git a/db-testing b/db-testing index 70451a2..15205a2 100755 --- a/db-testing +++ b/db-testing @@ -4,52 +4,4 @@ repoid=4 reponame="testing" arch="i686" -ftppath="/home/ftp/testing/os/i686" - -cvspath="/home/cvs-arch" -cvsmod="testing" -cvstag="TESTING" - -stagedir="$HOME/staging/testing" - -testing_cleanup() { - rm -rf $testingtmp -} - -# catch ^C -trap testing_cleanup SIGINT - -if [ -f /tmp/.repolck.$arch.$repoid ]; then - owner=`/bin/ls -l /tmp/.repolck.$arch.$repoid | awk '{print $3}'` - echo "error: db generation is already in progress (started by $owner)" - exit 1 -fi -# lock -touch /tmp/.repolck.$arch.$repoid - -# testing is special, so we have to do the CVS checkout stuff here -testingtmp=$(mktemp -d /tmp/testtmp-XXXXXX) || exit 1 -cvsdir=$testingtmp/testing -mkdir $cvsdir -cd $cvsdir - -if [ "`ls $stagedir/add`" -o "`ls $stagedir/del`" ]; then - echo "==> Checking out TESTING branches..." >&2 - echo "--- CORE ---" - CVS_RSH=ssh CVSROOT=:ext:cvs.archlinux.org:/home/cvs-core cvs -q export -r TESTING core - cp -a core/* . && rm -rf core - echo "--- EXTRA ---" - CVS_RSH=ssh CVSROOT=:ext:cvs.archlinux.org:/home/cvs-extra cvs -q export -r TESTING extra - cp -a extra/* . && rm -rf extra - echo "--- UNSTABLE ---" - CVS_RSH=ssh CVSROOT=:ext:cvs.archlinux.org:/home/cvs-unstable cvs -q export -r TESTING unstable - cp -a unstable/* . && rm -rf unstable -fi - -# unlock -rm -f /tmp/.repolck.$arch.$repoid - -# call the real script -. `dirname $0`/db-inc - -testing_cleanup +. $(dirname $0)/db-inc diff --git a/db-testing64 b/db-testing64 index 8b306ac..c34355b 100755 --- a/db-testing64 +++ b/db-testing64 @@ -1,58 +1,7 @@ #!/bin/bash -export CARCH=x86_64 - repoid=4 reponame="testing" arch="x86_64" -ftppath="/home/ftp/testing/os/x86_64" - -cvspath="/home/cvs-arch" -cvsmod="testing" -cvstag="TESTING-64" - -#stagedir="/home/arch64/staging/testing" -stagedir="$HOME/staging/testing64" - -testing_cleanup() { - rm -rf $testingtmp -} - -# catch ^C -trap testing_cleanup SIGINT - -if [ -f /tmp/.repolck.$arch.$repoid ]; then - owner=`/bin/ls -l /tmp/.repolck.$arch.$repoid | awk '{print $3}'` - echo "error: db generation is already in progress (started by $owner)" - exit 1 -fi -# lock -touch /tmp/.repolck.$arch.$repoid - -# testing is special, so we have to do the CVS checkout stuff here -testingtmp=$(mktemp -d /tmp/testtmp-XXXXXX) || exit 1 -cvsdir=$testingtmp/testing -mkdir $cvsdir -cd $cvsdir - -if [ "`ls $stagedir/add`" -o "`ls $stagedir/del`" ]; then - echo "==> Checking out TESTING-64 branches..." >&2 - echo "--- CORE ---" - CVS_RSH=ssh CVSROOT=:ext:cvs.archlinux.org:/home/cvs-core cvs -q export -r TESTING-64 core - cp -a core/* . && rm -rf core - echo "--- EXTRA ---" - CVS_RSH=ssh CVSROOT=:ext:cvs.archlinux.org:/home/cvs-extra cvs -q export -r TESTING-64 extra - cp -a extra/* . && rm -rf extra - echo "--- UNSTABLE ---" - CVS_RSH=ssh CVSROOT=:ext:cvs.archlinux.org:/home/cvs-unstable cvs -q export -r TESTING-64 unstable - cp -a unstable/* . && rm -rf unstable -fi - -# unlock -rm -f /tmp/.repolck.$arch.$repoid - -# call the real script -. `dirname $0`/db-inc - -testing_cleanup +. $(dirname $0)/db-inc diff --git a/db-unstable b/db-unstable index 0b4c524..b0231b4 100755 --- a/db-unstable +++ b/db-unstable @@ -4,14 +4,5 @@ repoid=3 reponame="unstable" arch="i686" -ftppath="/home/ftp/unstable/os/i686" - -cvspath="/home/cvs-unstable" -cvsmod="unstable" -cvstag="CURRENT" - -stagedir="$HOME/staging/unstable" - -# call the real script -. `dirname $0`/db-inc +. $(dirname $0)/db-inc diff --git a/db-unstable64 b/db-unstable64 index dece93a..225cf97 100755 --- a/db-unstable64 +++ b/db-unstable64 @@ -1,20 +1,8 @@ #!/bin/bash -export CARCH=x86_64 - repoid=3 reponame="unstable" arch="x86_64" -ftppath="/home/ftp/unstable/os/x86_64" - -cvspath="/home/cvs-unstable" -cvsmod="unstable" -cvstag="CURRENT-64" - -#stagedir="/home/arch64/staging/unstable" -stagedir="$HOME/staging/unstable64" - -# call the real script -. `dirname $0`/db-inc +. $(dirname $0)/db-inc diff --git a/pkgdb1 b/pkgdb1 index fee01fc..a421007 100755 --- a/pkgdb1 +++ b/pkgdb1 @@ -3,66 +3,67 @@ # Get the package name from the filename # hackish, but should work for now getpkgname() { - local tmp + local tmp - tmp=${1##*/} - tmp=${tmp%.pkg.tar.gz} - tmp=${tmp%-i686} - tmp=${tmp%-x86_64} - echo ${tmp%-*-*} + tmp=${1##*/} + tmp=${tmp%.pkg.tar.gz} + tmp=${tmp%-i686} + tmp=${tmp%-x86_64} + echo ${tmp%-*-*} } -STAGEDIR=`pwd` +STAGEDIR=$(pwd) -ABSDIR=$1 -if [ "$ABSDIR" = "" ]; then - me=`basename $0` - echo "usage: $me " >&2 - exit 1 +#This is our "unknown" category. We will have to do something about this later. +# It is a stop-gap +CATEGORY=25 + +SVNCO=$1 +REPOTAG=$2 +if [ "$SVNCO" = "" ]; then + me=$(basename $0) + echo "usage: $me " >&2 + exit 1 fi -if [ ! "`ls $STAGEDIR/*.pkg.tar.gz 2>/dev/null`" ]; then - exit +if [ ! "$(ls $STAGEDIR/*.pkg.tar.gz 2>/dev/null)" ]; then + exit fi cd $STAGEDIR -for pkgfile in `ls $STAGEDIR/*.pkg.tar.gz`; do - pkgname=$(getpkgname $pkgfile); - fullname=$(basename $pkgfile) - # find the matching PKGBUILD - tmpf=$(mktemp /tmp/pkgdb1.XXXXXXXXXX) || exit 1 - find $ABSDIR -type d -name "$pkgname" >$tmpf - if [ "`cat $tmpf | wc -l`" != "1" ]; then - echo "WARNING: could not find PKGBUILD for $pkgname, cannot update this entry" >&2 - rm $tmpf - continue - fi - pkgbuild="`cat $tmpf`/PKGBUILD" - rm $tmpf - if [ ! -f $pkgbuild ]; then - echo "WARNING: could not find PKGBUILD for $fullname, cannot update this entry" >&2 - continue - fi - # pick out the category from the pathname - catpath=$(cd `dirname $pkgbuild`/.. && pwd) - category=${catpath##*/} - # now read the PKGBUILD and output the data for pkgdb2 - unset pkgname pkgver pkgrel pkgdesc license groups provides md5sums force - unset replaces depends conflicts backup source install build makedepends - unset options - source $pkgbuild || continue +for pkgfile in $STAGEDIR/*.pkg.tar.gz; do + pkgname=$(getpkgname $pkgfile); + fullname=$(basename $pkgfile) + pkgpath="$SVNCO/$pkgname/repos/$REPOTAG" + + # find the matching PKGBUILD + if [ ! -d "$pkgpath" ]; then + msg "WARNING: could not find PKGBUILD for $pkgname, cannot update this entry" + return + fi + pkgbuild="${pkgpath}/PKGBUILD" + if [ ! -f $pkgbuild ]; then + msg "WARNING: could not find PKGBUILD for $fullname, cannot update this entry" + return + fi + + # pick out the category from the pathname + unset pkgname pkgver pkgrel pkgdesc license groups provides md5sums force + unset replaces depends conflicts backup source install build makedepends + unset options + source $pkgbuild || continue - deplist=${depends[@]} - sources=${source[@]} - echo $fullname - echo $pkgname - echo $pkgver - echo $pkgrel - echo $pkgdesc - echo $category - echo $url - echo $sources - echo $deplist + deplist=${depends[@]} + sources=${source[@]} + echo $fullname + echo $pkgname + echo $pkgver + echo $pkgrel + echo $pkgdesc + echo $CATEGORY + echo $url + echo $sources + echo $deplist done exit 0 diff --git a/pkgdb2-add.c b/pkgdb2-add.c index 292d0dc..32f2d83 100644 --- a/pkgdb2-add.c +++ b/pkgdb2-add.c @@ -1,6 +1,6 @@ #include #include -#include +#include #include #include @@ -9,60 +9,60 @@ #define DB_PASS "passwords-are-cool" typedef struct pkg { - unsigned int id; - char *name; - char *ver; - char *rel; - struct pkg *next; + unsigned int id; + char *name; + char *ver; + char *rel; + struct pkg *next; } pkg_t; MYSQL_RES *doquery(MYSQL *m, const char* q) { - MYSQL_RES *res; - if(mysql_query(m, q)) { - fprintf(stderr, "mysql_query: %s\n", mysql_error(m)); - return(NULL); - } - res = mysql_store_result(m); - return(res); + MYSQL_RES *res; + if(mysql_query(m, q)) { + fprintf(stderr, "mysql_query: %s\n", mysql_error(m)); + return(NULL); + } + res = mysql_store_result(m); + return(res); } /* this function is ugly -- it malloc's for each string it * returns, and they probably won't be freed by the caller. */ char* addslashes(const char *s) { - char slashed[8192]; - char *p; - - slashed[0] = '\0'; - p = slashed; - while(*s) { - if(*s == '\'' || *s == '"' || *s == '\\') { - *p++ = '\\'; - } - *p++ = *s++; - } - *p = '\0'; - return(strdup(slashed)); + char slashed[8192]; + char *p; + + slashed[0] = '\0'; + p = slashed; + while(*s) { + if(*s == '\'' || *s == '"' || *s == '\\') { + *p++ = '\\'; + } + *p++ = *s++; + } + *p = '\0'; + return(strdup(slashed)); } char* trim(char *str) { - char *pch = str; - while(isspace(*pch)) { - pch++; - } - if(pch != str) { - memmove(str, pch, (strlen(pch) + 1)); - } - - pch = (char*)(str + (strlen(str) - 1)); - while(isspace(*pch)) { - pch--; - } - *++pch = '\0'; + char *pch = str; + while(isspace(*pch)) { + pch++; + } + if(pch != str) { + memmove(str, pch, (strlen(pch) + 1)); + } + + pch = (char*)(str + (strlen(str) - 1)); + while(isspace(*pch)) { + pch--; + } + *++pch = '\0'; - return str; + return str; } /* scan a .pkg.tar.gz file and put all files listed into the database. @@ -72,205 +72,205 @@ char* trim(char *str) */ void updatefilelist(MYSQL *db, unsigned long id, char *fn) { - FILE *fp; - char *tmp; - char cmd[PATH_MAX]; - char line[PATH_MAX]; - char query[PATH_MAX]; + FILE *fp; + char *tmp; + char cmd[PATH_MAX]; + char line[PATH_MAX]; + char query[PATH_MAX]; - tmp = tempnam("/tmp", "pkgdb"); - snprintf(cmd, PATH_MAX-1, "/bin/tar tzvf %s | awk '{print $6}' >%s", fn, tmp); - system(cmd); - fp = fopen(tmp, "r"); - if(fp == NULL) { - fprintf(stderr, "pkgdb2-add: could not open tempfile: %s\n", tmp); - return; - } - snprintf(query, sizeof(query), "DELETE FROM packages_files WHERE pkg_id='%lu'", id); - doquery(db, query); - while(fgets(line, sizeof(line)-1, fp)) { - char *fixedfn = addslashes(trim(line)); - if(!strcmp(fixedfn, ".FILELIST") || !strcmp(fixedfn, ".PKGINFO") || !strcmp(fixedfn, ".INSTALL")) { - free(fixedfn); - continue; - } - /* varchars aren't case-sensitive but filesystems are, so we use REPLACE INTO */ - snprintf(query, sizeof(query), "REPLACE INTO packages_files (pkg_id,path) VALUES " - "('%lu', '%s')", id, fixedfn); - free(fixedfn); - doquery(db, query); - } - fclose(fp); - unlink(tmp); + tmp = tempnam("/tmp", "pkgdb"); + snprintf(cmd, PATH_MAX-1, "/bin/tar tzvf %s | awk '{print $6}' >%s", fn, tmp); + system(cmd); + fp = fopen(tmp, "r"); + if(fp == NULL) { + fprintf(stderr, "pkgdb2-add: could not open tempfile: %s\n", tmp); + return; + } + snprintf(query, sizeof(query), "DELETE FROM packages_files WHERE pkg_id='%lu'", id); + doquery(db, query); + while(fgets(line, sizeof(line)-1, fp)) { + char *fixedfn = addslashes(trim(line)); + if(!strcmp(fixedfn, ".FILELIST") || !strcmp(fixedfn, ".PKGINFO") || !strcmp(fixedfn, ".INSTALL")) { + free(fixedfn); + continue; + } + /* varchars aren't case-sensitive but filesystems are, so we use REPLACE INTO */ + snprintf(query, sizeof(query), "REPLACE INTO packages_files (pkg_id,path) VALUES " + "('%lu', '%s')", id, fixedfn); + free(fixedfn); + doquery(db, query); + } + fclose(fp); + unlink(tmp); } int main(int argc, char **argv) { - MYSQL db; - MYSQL_RES *result; - MYSQL_ROW row; - char query[4096]; - char fn[PATH_MAX]; - char ftppath[PATH_MAX]; - int repoid; - pkg_t *dblist = NULL; - pkg_t *pkglist = NULL; - pkg_t *pkgptr, *ptr; + MYSQL db; + MYSQL_RES *result; + MYSQL_ROW row; + char query[4096]; + char fn[PATH_MAX]; + char ftppath[PATH_MAX]; + int repoid; + pkg_t *dblist = NULL; + pkg_t *pkglist = NULL; + pkg_t *pkgptr, *ptr; - if(argc < 3) { - printf("usage: pkgdb2-add \n"); - printf("\nWARNING: Do not run this manually! It is intended to be run from\n" - "the Arch db-generation scripts.\n\n"); - return(1); - } - repoid = atoi(argv[1]); - strncpy(ftppath, argv[2], PATH_MAX-1); + if(argc < 3) { + printf("usage: pkgdb2-add \n"); + printf("\nWARNING: Do not run this manually! It is intended to be run from\n" + "the Arch db-generation scripts.\n\n"); + return(1); + } + repoid = atoi(argv[1]); + strncpy(ftppath, argv[2], PATH_MAX-1); - if(mysql_init(&db) == NULL) { - fprintf(stderr, "could not initialize\n"); - return(1); - } - if(mysql_real_connect(&db, "localhost", DB_USER, DB_PASS, DB_NAME, - 0, NULL, 0) == NULL) { - fprintf(stderr, "failed to connect to database: %s\n", mysql_error(&db)); - return(1); - } - snprintf(query, sizeof(query), "SELECT id,pkgname,pkgver,pkgrel FROM packages " - "WHERE repo_id='%d'", repoid); - result = doquery(&db, query); - while(row = mysql_fetch_row(result)) { - int i; - /*unsigned long *lengths; - lengths = mysql_fetch_lengths(result);*/ - /* add the node to the list */ - if(dblist == NULL) { - dblist = (pkg_t*)malloc(sizeof(pkg_t)); - if(dblist == NULL) { - fprintf(stderr, "error: out of memory!\n"); - return(1); - } - ptr = dblist; - } else { - ptr->next = (pkg_t*)malloc(sizeof(pkg_t)); - if(ptr->next == NULL) { - fprintf(stderr, "error: out of memory!\n"); - return(1); - } - ptr = ptr->next; - } - ptr->next = NULL; - /* pick out the fields */ - ptr->id = atoi(row[0]); - ptr->name = strdup(row[1]); - ptr->ver = strdup(row[2]); - ptr->rel = strdup(row[3]); - } - mysql_free_result(result); + if(mysql_init(&db) == NULL) { + fprintf(stderr, "could not initialize\n"); + return(1); + } + if(mysql_real_connect(&db, "localhost", DB_USER, DB_PASS, DB_NAME, + 0, NULL, 0) == NULL) { + fprintf(stderr, "failed to connect to database: %s\n", mysql_error(&db)); + return(1); + } + snprintf(query, sizeof(query), "SELECT id,pkgname,pkgver,pkgrel FROM packages " + "WHERE repo_id='%d'", repoid); + result = doquery(&db, query); + while(row = mysql_fetch_row(result)) { + int i; + /*unsigned long *lengths; + lengths = mysql_fetch_lengths(result);*/ + /* add the node to the list */ + if(dblist == NULL) { + dblist = (pkg_t*)malloc(sizeof(pkg_t)); + if(dblist == NULL) { + fprintf(stderr, "error: out of memory!\n"); + return(1); + } + ptr = dblist; + } else { + ptr->next = (pkg_t*)malloc(sizeof(pkg_t)); + if(ptr->next == NULL) { + fprintf(stderr, "error: out of memory!\n"); + return(1); + } + ptr = ptr->next; + } + ptr->next = NULL; + /* pick out the fields */ + ptr->id = atoi(row[0]); + ptr->name = strdup(row[1]); + ptr->ver = strdup(row[2]); + ptr->rel = strdup(row[3]); + } + mysql_free_result(result); - while(!feof(stdin)) { - int found = 0; - unsigned int catid = 0; - char name[256], ver[256], rel[256], desc[4096]; - char cat[256], url[256], sources[4096], deplist[4096]; - char pkgfile[4096]; - /* get package data from stdin */ - fgets(pkgfile, 4096, stdin); trim(pkgfile); if(feof(stdin)) continue; - fgets(name, 256, stdin); trim(name); if(feof(stdin)) continue; - fgets(ver, 256, stdin); trim(ver); if(feof(stdin)) continue; - fgets(rel, 256, stdin); trim(rel); if(feof(stdin)) continue; - fgets(desc, 4096, stdin); trim(desc); if(feof(stdin)) continue; - fgets(cat, 256, stdin); trim(cat); if(feof(stdin)) continue; - fgets(url, 256, stdin); trim(url); if(feof(stdin)) continue; - fgets(sources, 4096, stdin); trim(sources); if(feof(stdin)) continue; - fgets(deplist, 4096, stdin); trim(deplist); if(feof(stdin)) continue; - /* check for overruns */ - if(strlen(name) > 254 || strlen(ver) >= 254 || strlen(rel) > 254 || - strlen(desc) > 4094 || strlen(cat) >= 254 || strlen(url) > 254 || - strlen(sources) > 4094 || strlen(deplist) > 4094 || strlen(pkgfile) > 4094) { - fprintf(stderr, "pkgdb2-add: one or more fields are too long in package '%s'\n", name); - fprintf(stderr, "pkgdb2-add: check the lengths of your strings, most are limited " - "to 255 chars, some are 4095\n"); - return(1); - } - /* add the node to the list */ - if(pkglist == NULL) { - pkglist = (pkg_t*)malloc(sizeof(pkg_t)); - if(pkglist == NULL) { - fprintf(stderr, "error: out of memory!\n"); - return(1); - } - pkgptr = pkglist; - } else { - pkgptr->next = (pkg_t*)malloc(sizeof(pkg_t)); - if(pkgptr->next == NULL) { - fprintf(stderr, "error: out of memory!\n"); - return(1); - } - pkgptr = pkgptr->next; - } - pkgptr->next = NULL; - pkgptr->name = strdup(name); - /* look it up in our cache */ - for(ptr = dblist; ptr; ptr = ptr->next) { - if(!strcmp(name, ptr->name)) { - found = 1; - break; - } - } - /* get the category */ - snprintf(query, sizeof(query), - "SELECT id FROM categories WHERE category='%s'", cat); - result = doquery(&db, query); - if(mysql_num_rows(result) == 0) { - fprintf(stderr, "pkgdb2-add: no db category found for '%s'\n", cat); - /* - snprintf(query, sizeof(query), "INSERT INTO categories (id,category) " - " VALUES (NULL,'%s')", addslashes(cat)); - doquery(&db, query); - catid = (unsigned int)mysql_insert_id(&db); - */ - } else { - row = mysql_fetch_row(result); - catid = (unsigned int)atoi(row[0]); - } - if(!found) { - /* Insert... */ - unsigned long id; - fprintf(stderr, "pkgdb2-add: inserting %s\n", name); - snprintf(query, sizeof(query), "INSERT INTO packages (id,repo_id," - "category_id,pkgname,pkgver,pkgrel,pkgdesc,url,sources,depends," - "last_update) VALUES (NULL,'%d','%d','%s','%s','%s','%s'," - "'%s','%s','%s',NOW())", - repoid, catid, addslashes(name), addslashes(ver), addslashes(rel), - addslashes(desc), addslashes(url), addslashes(sources), - addslashes(deplist)); - doquery(&db, query); - id = mysql_insert_id(&db); - snprintf(fn, PATH_MAX-1, "%s/%s", ftppath, pkgfile); - updatefilelist(&db, id, fn); - continue; - } else if(strcmp(ptr->ver, ver) || strcmp(ptr->rel, rel)) { - /* ...or Update */ - fprintf(stderr, "pkgdb2-add: updating %s (%s-%s ==> %s-%s)\n", - ptr->name, ptr->ver, ptr->rel, ver, rel); - snprintf(query, sizeof(query), "UPDATE packages SET category_id='%d'," - "pkgname='%s',pkgver='%s',pkgrel='%s',pkgdesc='%s',url='%s'," - "sources='%s',depends='%s',needupdate=0,last_update=NOW() " - "WHERE id='%d'", - catid, addslashes(name), addslashes(ver), addslashes(rel), - addslashes(desc), addslashes(url), addslashes(sources), - addslashes(deplist), ptr->id); - doquery(&db, query); - snprintf(fn, PATH_MAX-1, "%s/%s", ftppath, pkgfile); - updatefilelist(&db, ptr->id, fn); - /* - snprintf(query, sizeof(query), "UPDATE todolist_pkgs SET complete=1 " - "WHERE pkgid='%d'", ptr->id); - doquery(&db, query); - */ - } - } + while(!feof(stdin)) { + int found = 0; + unsigned int catid = 0; + char name[256], ver[256], rel[256], desc[4096]; + char cat[256], url[256], sources[4096], deplist[4096]; + char pkgfile[4096]; + /* get package data from stdin */ + fgets(pkgfile, 4096, stdin); trim(pkgfile); if(feof(stdin)) continue; + fgets(name, 256, stdin); trim(name); if(feof(stdin)) continue; + fgets(ver, 256, stdin); trim(ver); if(feof(stdin)) continue; + fgets(rel, 256, stdin); trim(rel); if(feof(stdin)) continue; + fgets(desc, 4096, stdin); trim(desc); if(feof(stdin)) continue; + fgets(cat, 256, stdin); trim(cat); if(feof(stdin)) continue; + fgets(url, 256, stdin); trim(url); if(feof(stdin)) continue; + fgets(sources, 4096, stdin); trim(sources); if(feof(stdin)) continue; + fgets(deplist, 4096, stdin); trim(deplist); if(feof(stdin)) continue; + /* check for overruns */ + if(strlen(name) > 254 || strlen(ver) >= 254 || strlen(rel) > 254 || + strlen(desc) > 4094 || strlen(cat) >= 254 || strlen(url) > 254 || + strlen(sources) > 4094 || strlen(deplist) > 4094 || strlen(pkgfile) > 4094) { + fprintf(stderr, "pkgdb2-add: one or more fields are too long in package '%s'\n", name); + fprintf(stderr, "pkgdb2-add: check the lengths of your strings, most are limited " + "to 255 chars, some are 4095\n"); + return(1); + } + /* add the node to the list */ + if(pkglist == NULL) { + pkglist = (pkg_t*)malloc(sizeof(pkg_t)); + if(pkglist == NULL) { + fprintf(stderr, "error: out of memory!\n"); + return(1); + } + pkgptr = pkglist; + } else { + pkgptr->next = (pkg_t*)malloc(sizeof(pkg_t)); + if(pkgptr->next == NULL) { + fprintf(stderr, "error: out of memory!\n"); + return(1); + } + pkgptr = pkgptr->next; + } + pkgptr->next = NULL; + pkgptr->name = strdup(name); + /* look it up in our cache */ + for(ptr = dblist; ptr; ptr = ptr->next) { + if(!strcmp(name, ptr->name)) { + found = 1; + break; + } + } + /* get the category */ + snprintf(query, sizeof(query), + "SELECT id FROM categories WHERE category='%s'", cat); + result = doquery(&db, query); + if(mysql_num_rows(result) == 0) { + fprintf(stderr, "pkgdb2-add: no db category found for '%s'\n", cat); + /* + snprintf(query, sizeof(query), "INSERT INTO categories (id,category) " + " VALUES (NULL,'%s')", addslashes(cat)); + doquery(&db, query); + catid = (unsigned int)mysql_insert_id(&db); + */ + } else { + row = mysql_fetch_row(result); + catid = (unsigned int)atoi(row[0]); + } + if(!found) { + /* Insert... */ + unsigned long id; + fprintf(stderr, "pkgdb2-add: inserting %s\n", name); + snprintf(query, sizeof(query), "INSERT INTO packages (id,repo_id," + "category_id,pkgname,pkgver,pkgrel,pkgdesc,url,sources,depends," + "last_update) VALUES (NULL,'%d','%d','%s','%s','%s','%s'," + "'%s','%s','%s',NOW())", + repoid, catid, addslashes(name), addslashes(ver), addslashes(rel), + addslashes(desc), addslashes(url), addslashes(sources), + addslashes(deplist)); + doquery(&db, query); + id = mysql_insert_id(&db); + snprintf(fn, PATH_MAX-1, "%s/%s", ftppath, pkgfile); + updatefilelist(&db, id, fn); + continue; + } else if(strcmp(ptr->ver, ver) || strcmp(ptr->rel, rel)) { + /* ...or Update */ + fprintf(stderr, "pkgdb2-add: updating %s (%s-%s ==> %s-%s)\n", + ptr->name, ptr->ver, ptr->rel, ver, rel); + snprintf(query, sizeof(query), "UPDATE packages SET " + "pkgname='%s',pkgver='%s',pkgrel='%s',pkgdesc='%s',url='%s'," + "sources='%s',depends='%s',needupdate=0,last_update=NOW() " + "WHERE id='%d'", + addslashes(name), addslashes(ver), addslashes(rel), + addslashes(desc), addslashes(url), addslashes(sources), + addslashes(deplist), ptr->id); + doquery(&db, query); + snprintf(fn, PATH_MAX-1, "%s/%s", ftppath, pkgfile); + updatefilelist(&db, ptr->id, fn); + /* + snprintf(query, sizeof(query), "UPDATE todolist_pkgs SET complete=1 " + "WHERE pkgid='%d'", ptr->id); + doquery(&db, query); + */ + } + } - mysql_close(&db); - return(0); + mysql_close(&db); + return(0); } diff --git a/updatesync-many b/updatesync-many index 40c5e4d..8060585 100755 --- a/updatesync-many +++ b/updatesync-many @@ -23,13 +23,13 @@ usage() { echo "updatesync-many" - echo "usage: $0 " + echo "usage: $0 " echo echo "This should probably only be run from the Arch db-generation scripts" echo echo "Caveats:" echo " - Make sure you run it from the staging directory" - echo " - Use absolute pathnames for dbfile and abs_dir" + echo " - Use absolute pathnames for dbfile and svn_checkout" echo exit 0 } @@ -167,17 +167,14 @@ update_entry() pkgfile=$1 pkgname=$(getpkgname ${pkgfile}) fullname=$(basename ${pkgfile}) + pkgpath="$SVNCO/$pkgname/repos/$REPOTAG" # find the matching PKGBUILD - tmpf=$(mktemp /tmp/updatesync-many.XXXXXXXXXX) || exit 1 - find $ABSDIR -type d -name "$pkgname" >$tmpf - if [ "`cat $tmpf | wc -l`" != "1" ]; then + if [ ! -d "$pkgpath" ]; then msg "WARNING: could not find PKGBUILD for $pkgname, cannot update this entry" - rm $tmpf return fi - pkgbuild="`cat $tmpf`/PKGBUILD" - rm $tmpf + pkgbuild="${pkgpath}/PKGBUILD" if [ ! -f $pkgbuild ]; then msg "WARNING: could not find PKGBUILD for $fullname, cannot update this entry" return @@ -218,7 +215,8 @@ fi ACTION=$1 PKGDB=$2 -ABSDIR=$3 +SVNCO=$3 +REPOTAG=$4 STAGEDIR="`pwd`" PKGDIR="`dirname $PKGDB`" if [ "$PKGDIR" = "." ]; then -- cgit v1.2.2 From 6b7792a5a600cf410701e55e1271f88ede12ec43 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Tue, 8 Apr 2008 00:21:21 -0400 Subject: Recompile for new passwords Why are the binaries in source control? No clue Signed-off-by: Aaron Griffin --- pkgdb2-add | Bin 13346 -> 12929 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/pkgdb2-add b/pkgdb2-add index c827bb0..180e0be 100755 Binary files a/pkgdb2-add and b/pkgdb2-add differ -- cgit v1.2.2 From 3207e38d0f65b569402b1497920be2c47df2fbb1 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Tue, 8 Apr 2008 00:46:37 -0400 Subject: Make SVN output quiet Way too much output, and it's useless... Signed-off-by: Aaron Griffin --- db-inc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/db-inc b/db-inc index 4f64dfe..6c0b78c 100644 --- a/db-inc +++ b/db-inc @@ -87,8 +87,8 @@ if [ "`ls $stagedir/add 2>/dev/null`" -o "`ls $stagedir/del 2>/dev/null`" ]; the if [ "$svndir" ]; then mv $svndir $TMPDIR/checkout else - echo "==> Checking out repo: $svnrepo ($arch)" - svn export file://$svnpath $TMPDIR/checkout + echo "==> Checking out repo: $svnrepo ($arch) - Please be patient" + svn export -q file://$svnpath $TMPDIR/checkout if [ $? -gt 0 ]; then die "==> SVN export failed!" fi -- cgit v1.2.2 From 3cfa0be926efae73e0044ae7746917d8d9d6c874 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Tue, 8 Apr 2008 00:47:59 -0400 Subject: Make genpkglist work again I somehow delete the dupe checking. This also works based on svn repo tag, and doesn't give stupid errors if packages.txt is missing Signed-off-by: Aaron Griffin --- db-inc | 6 ++++-- genpkglist | 53 ++++++++++++++++++++++++++++++++++++++++++----------- 2 files changed, 46 insertions(+), 13 deletions(-) diff --git a/db-inc b/db-inc index 6c0b78c..7e2e61a 100644 --- a/db-inc +++ b/db-inc @@ -173,8 +173,10 @@ fi if [ "$updatelists" ]; then echo "==> Generating Text Package List..." >&2 cd $TMPDIR/checkout - $ARCHDIR/genpkglist $ftppath $arch - mv packages.txt $ftppath/packages.txt + $ARCHDIR/genpkglist $ftppath $svnrepo + if [ -f packages.txt ]; then + mv packages.txt $ftppath/packages.txt + fi fi cleanup diff --git a/genpkglist b/genpkglist index d026916..31b7c78 100755 --- a/genpkglist +++ b/genpkglist @@ -18,26 +18,57 @@ getpkgname() { echo ${tmp%-*-*} } -pkgfile="`pwd`/packages.txt" +pkgfile="$(pwd)/packages.txt" ftppath=$1 -dbarch=$2 +repotag=$2 rm -f $pkgfile -for category in `find * -maxdepth 0 -type d | grep -v CVS`; do - cd $category - for pkg in `/bin/ls`; do - cd $pkg +for package in *; do + if [ -d "$package/$repotag/" ]; then + cd "$package/$repotag/" if [ -f PKGBUILD ]; then . PKGBUILD if [ -f $ftppath/$pkgname-$pkgver-$pkgrel.pkg.tar.gz ]; then - echo "$category/$pkgname-$pkgver-$pkgrel.pkg.tar.gz" >>$pkgfile + echo "$pkgname-$pkgver-$pkgrel.pkg.tar.gz" >>$pkgfile elif [ -f $ftppath/$pkgname-$pkgver-$pkgrel-$dbarch.pkg.tar.gz ]; then - echo "$category/$pkgname-$pkgver-$pkgrel-$dbarch.pkg.tar.gz" >>$pkgfile + echo "$pkgname-$pkgver-$pkgrel-$dbarch.pkg.tar.gz" >>$pkgfile else echo "notice: Missing $pkgname-$pkgver-$pkgrel-$dbarch.pkg.tar.gz in ftp site" >&2 fi - fi + fi cd .. - done - cd .. + fi done + +cd $ftppath +unset DUPES DUPEFILES last +for pkg in *.pkg.tar.gz; do + pkgname=$(getpkgname $pkg) + if [ "$last" = "$pkgname" ]; then + DUPES="$DUPES $pkgname" + DUPEFILES="$DUPEFILES $pkg" + fi + last=$pkgname +done + +showdupes() { + done= + for i in *.pkg.tar.gz; do + pkgname=$(getpkgname $i) + if [ "$pkgname" = "$1" ]; then + ls -l $i | awk '{print $6" "$7" "$8" "$9}' + done=1 + else + [ "$done" = "1" ] && return + fi + done +} + +if [ "$DUPES" ]; then + DUPES=$((for d in $DUPES; do echo $d; done) | sort -u) + echo "Possible Dupes for $ftppath (remove old versions)" + echo "Date Filename" + for dupe in $DUPES; do + showdupes $dupe + done +fi -- cgit v1.2.2 From c8056b5a90ba2ef5351d9ccc3ffecee3f50211eb Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Tue, 8 Apr 2008 00:49:08 -0400 Subject: Fix "unknown" category handling More minor fixes for my previous SVN commits Now with less password Signed-off-by: Aaron Griffin --- pkgdb1 | 4 ++-- pkgdb2-add | Bin 12929 -> 12893 bytes pkgdb2-add.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkgdb1 b/pkgdb1 index a421007..72dc3ac 100755 --- a/pkgdb1 +++ b/pkgdb1 @@ -15,8 +15,8 @@ getpkgname() { STAGEDIR=$(pwd) #This is our "unknown" category. We will have to do something about this later. -# It is a stop-gap -CATEGORY=25 +# It is a stop-gap. We will insert new packages in this way +CATEGORY="unknown" SVNCO=$1 REPOTAG=$2 diff --git a/pkgdb2-add b/pkgdb2-add index 180e0be..be29872 100755 Binary files a/pkgdb2-add and b/pkgdb2-add differ diff --git a/pkgdb2-add.c b/pkgdb2-add.c index 32f2d83..489074c 100644 --- a/pkgdb2-add.c +++ b/pkgdb2-add.c @@ -222,7 +222,7 @@ int main(int argc, char **argv) "SELECT id FROM categories WHERE category='%s'", cat); result = doquery(&db, query); if(mysql_num_rows(result) == 0) { - fprintf(stderr, "pkgdb2-add: no db category found for '%s'\n", cat); + catid = 25; /* snprintf(query, sizeof(query), "INSERT INTO categories (id,category) " " VALUES (NULL,'%s')", addslashes(cat)); -- cgit v1.2.2 From 930bf250e7866b962dae4e9fbc174c4d234f9270 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Tue, 8 Apr 2008 02:45:12 -0400 Subject: Properly point to the 64bit staging dirs I mistyped it. Whoops, silly me Signed-off-by: Aaron Griffin --- db-inc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db-inc b/db-inc index 7e2e61a..fde6f2e 100644 --- a/db-inc +++ b/db-inc @@ -12,7 +12,7 @@ svnrepo="$reponame-$arch" #Hacky for now if [ "$arch" = "x86_64" ]; then - stagedir="$HOME/staging/$reponame-64" + stagedir="$HOME/staging/${reponame}64" else stagedir="$HOME/staging/$reponame" fi -- cgit v1.2.2 From 0e94987238ee78221892da19ab3219c49e621901 Mon Sep 17 00:00:00 2001 From: Simo Leone Date: Tue, 8 Apr 2008 12:03:08 -0400 Subject: Delete binaries from repo I don't even want to know. Signed-off-by: Simo Leone Signed-off-by: Aaron Griffin --- pkgdb2-add | Bin 12893 -> 0 bytes pkgdb2-del | Bin 8692 -> 0 bytes 2 files changed, 0 insertions(+), 0 deletions(-) delete mode 100755 pkgdb2-add delete mode 100755 pkgdb2-del diff --git a/pkgdb2-add b/pkgdb2-add deleted file mode 100755 index be29872..0000000 Binary files a/pkgdb2-add and /dev/null differ diff --git a/pkgdb2-del b/pkgdb2-del deleted file mode 100755 index af44fe1..0000000 Binary files a/pkgdb2-del and /dev/null differ -- cgit v1.2.2 From 8a92487774b8d15be627f230a4f8d864cfcc9b8d Mon Sep 17 00:00:00 2001 From: Simo Leone Date: Tue, 8 Apr 2008 12:10:09 -0400 Subject: Added a Makefile Also fixed mysql include in pkgdb2-del. Rigged some makefile defines for mysql info instead of hardcoding into the source. Signed-off-by: Simo Leone Aaron: Added -I for mysql includes Signed-off-by: Aaron Griffin --- Makefile | 20 ++++++++++++++++++++ pkgdb2-add.c | 6 +++--- pkgdb2-del.c | 6 +++--- 3 files changed, 26 insertions(+), 6 deletions(-) create mode 100644 Makefile diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..b66e271 --- /dev/null +++ b/Makefile @@ -0,0 +1,20 @@ +# mysql db info. +DB_NAME:="archlinux" +DB_USER:="archlinux" +DB_PASS:="passwords-are-cool" + +MYSQL_DEFS:=-DDB_NAME=\"$(DB_NAME)\" -DDB_USER=\"$(DB_USER)\" -DDB_PASS=\"$(DB_PASS)\" +MYSQL_FLAGS:=-I/usr/include/mysql -L/usr/lib/mysql -lmysqlclient + +all: pkgdb2-add pkgdb2-del + +pkgdb2-add: pkgdb2-add.c + gcc $(MYSQL_DEFS) $(MYSQL_FLAGS) -o pkgdb2-add pkgdb2-add.c + +pkgdb2-del: pkgdb2-del.c + gcc $(MYSQL_DEFS) $(MYSQL_FLAGS) -o pkgdb2-del pkgdb2-del.c + +clean: + rm -f pkgdb2-add pkgdb2-del + +.PHONY: all clean diff --git a/pkgdb2-add.c b/pkgdb2-add.c index 489074c..92d1339 100644 --- a/pkgdb2-add.c +++ b/pkgdb2-add.c @@ -4,9 +4,9 @@ #include #include -#define DB_USER "archweb" -#define DB_NAME "archweb" -#define DB_PASS "passwords-are-cool" +#ifndef DB_PASS + #error "MySQL connection info undefined" +#endif typedef struct pkg { unsigned int id; diff --git a/pkgdb2-del.c b/pkgdb2-del.c index c81e542..3ceb580 100644 --- a/pkgdb2-del.c +++ b/pkgdb2-del.c @@ -4,9 +4,9 @@ #include #include -#define DB_USER "archweb" -#define DB_NAME "archweb" -#define DB_PASS "passwords-are-cool" +#ifndef DB_PASS + #error "MySQL connection info undefined" +#endif typedef struct pkg { unsigned int id; -- cgit v1.2.2 From ab088d5af41a216768becd35dc4f9b84400d4afa Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Tue, 8 Apr 2008 12:22:18 -0400 Subject: Remove 'svndir' usage This was a holdover from the CVS scripts used to import and existing checkout Signed-off-by: Aaron Griffin --- Makefile | 2 +- db-inc | 17 ++++++----------- 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/Makefile b/Makefile index b66e271..8c24840 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # mysql db info. DB_NAME:="archlinux" DB_USER:="archlinux" -DB_PASS:="passwords-are-cool" +DB_PASS:="passwords-are-NOT-cool" MYSQL_DEFS:=-DDB_NAME=\"$(DB_NAME)\" -DDB_USER=\"$(DB_USER)\" -DDB_PASS=\"$(DB_PASS)\" MYSQL_FLAGS:=-I/usr/include/mysql -L/usr/lib/mysql -lmysqlclient diff --git a/db-inc b/db-inc index fde6f2e..dcd8105 100644 --- a/db-inc +++ b/db-inc @@ -83,17 +83,12 @@ cd $TMPDIR # Checkout the SVN module if we need to updatelists= if [ "`ls $stagedir/add 2>/dev/null`" -o "`ls $stagedir/del 2>/dev/null`" ]; then - # if $svndir is set, then use that instead of doing our own cvs checkout - if [ "$svndir" ]; then - mv $svndir $TMPDIR/checkout - else - echo "==> Checking out repo: $svnrepo ($arch) - Please be patient" - svn export -q file://$svnpath $TMPDIR/checkout - if [ $? -gt 0 ]; then - die "==> SVN export failed!" - fi - fi - updatelists=1 + echo "==> Checking out repo: $svnrepo ($arch) - Please be patient" + svn export -q file://$svnpath $TMPDIR/checkout + if [ $? -gt 0 ]; then + die "==> SVN export failed!" + fi + updatelists=1 else echo "No files to process" cleanup 0 -- cgit v1.2.2 From d2fa8ca9ef201b25d0a7af2c8ac5dc8ae8051162 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Tue, 8 Apr 2008 12:53:25 -0400 Subject: Use local user dirs for checkouts Signed-off-by: Aaron Griffin --- db-inc | 63 +++++++++++++++++++++++++++++++-------------------------------- 1 file changed, 31 insertions(+), 32 deletions(-) diff --git a/db-inc b/db-inc index dcd8105..7f5a66a 100644 --- a/db-inc +++ b/db-inc @@ -17,8 +17,10 @@ else stagedir="$HOME/staging/$reponame" fi -[ "$UID" = "" ] && UID=$(uid) -TMPDIR="/tmp/archpkg.$arch.$repoid.$UID" +WORKDIR="~/.dbscripts" +CHECKOUTDIR="$WORKDIR/checkout" +LOCKFILE="$WORKDIR/lock" +DBFILE="$WORKDIR/$reponame.db.tar.gz" if [ ! `type -p fakeroot` ]; then echo "error: fakeroot is missing" >&2 @@ -43,9 +45,8 @@ getpkgname() { } cleanup() { - rm -rf $TMPDIR # unlock - rm -f /tmp/.repolck.$arch.$repoid + rm -f "~/.dbscripts/lock" [ "$1" ] && exit $1 } @@ -60,38 +61,39 @@ die() { } # check for locks -if [ -f /tmp/.repolck.$arch.$repoid ]; then - owner=`/bin/ls -l /tmp/.repolck.$arch.$repoid | awk '{print $3}'` - echo "error: db generation is already in progress (started by $owner)" +if [ -f ~/.dbscripts/lock ]; then + echo "error: db generation is already in progress" exit 1 fi # catch ^C breaks trap ctrl_c SIGINT # lock -touch /tmp/.repolck.$arch.$repoid +touch ~/.dbscripts/lock -# RedHat's mktemp is broken... -if [ -d $TMPDIR ]; then - echo "==> Removing old temp dir..." >&2 - rm -rf $TMPDIR || exit 1 +if [ -d $CHECKOUTDIR ]; then + cd $CHECKOUTDIR + svn update + if [ $? -gt 0 ]; then + die "==> SVN update failed, aborting!" + fi +else + echo "==> Checking out repo: $svnrepo ($arch) - Please be patient" + svn checkout file://$svnpath $CHECKOUTDIR + if [ $? -gt 0 ]; then + die "==> SVN checkout failed, aborting!" + fi fi -mkdir $TMPDIR; [ $? -gt 0 ] && exit 1 -cd $TMPDIR +cd $CHECKOUTDIR # Checkout the SVN module if we need to updatelists= if [ "`ls $stagedir/add 2>/dev/null`" -o "`ls $stagedir/del 2>/dev/null`" ]; then - echo "==> Checking out repo: $svnrepo ($arch) - Please be patient" - svn export -q file://$svnpath $TMPDIR/checkout - if [ $? -gt 0 ]; then - die "==> SVN export failed!" - fi updatelists=1 else - echo "No files to process" - cleanup 0 + echo "No files to process" + cleanup 0 fi # Right-O, now we look through the "add" and "del" subdirectories of @@ -103,28 +105,26 @@ fi # one db-* invocation, but it's not a huge performance hit. if [ -d $stagedir/add -a "`ls $stagedir/add`" ]; then - cd $TMPDIR echo "==> Processing new/updated packages for repository '$reponame'..." >&2 # copy the db file into our working area - cp $ftppath/$reponame.db.tar.gz . + cp $ftppath/$reponame.db.tar.gz $DBFILE cd $stagedir/add # run it thru fakeroot make sure everything is owned by root.root - echo "$ARCHDIR/updatesync-many add $TMPDIR/$reponame.db.tar.gz $TMPDIR/checkout $svnrepo" \ - | fakeroot + echo "$ARCHDIR/updatesync-many add $DBFILE $CHECKOUTDIR $svnrepo" | fakeroot if [ $? -ne 0 ]; then die "==> Error returned from updatesync-many" fi - cp $TMPDIR/$reponame.db.tar.gz $ftppath + cp $DBFILE $ftppath # only for i686 (for now) if [ "$arch" = "i686" ]; then echo "==> Scanning for New/Updated packages..." >&2 cd $stagedir/add - $ARCHDIR/pkgdb1 $TMPDIR/checkout $svnrepo | $ARCHDIR/pkgdb2-add $repoid $stagedir/add + $ARCHDIR/pkgdb1 $CHECKOUTDIR $svnrepo | $ARCHDIR/pkgdb2-add $repoid $stagedir/add fi # move the package files into the ftp directory @@ -132,22 +132,21 @@ if [ -d $stagedir/add -a "`ls $stagedir/add`" ]; then fi if [ -d $stagedir/del -a "`ls $stagedir/del`" ]; then - cd $TMPDIR echo "==> Processing deleted packages for repository '$reponame'..." >&2 # copy the db file into our working area - cp $ftppath/$reponame.db.tar.gz . + cp $ftppath/$reponame.db.tar.gz $DBFILE cd $stagedir/del # run it thru fakeroot make sure everything is owned by root.root - echo "$ARCHDIR/updatesync-many del $TMPDIR/$reponame.db.tar.gz NOT-USED ZOMGWOO" \ + echo "$ARCHDIR/updatesync-many del $DBFILE NOT-USED ZOMGWOO" \ | fakeroot if [ $? -ne 0 ]; then die "==> Error returned from updatesync-many" fi - cp $TMPDIR/$reponame.db.tar.gz $ftppath + cp $DBFILE $ftppath # only for i686 (for now) if [ "$arch" = "i686" ]; then @@ -167,7 +166,7 @@ fi if [ "$updatelists" ]; then echo "==> Generating Text Package List..." >&2 - cd $TMPDIR/checkout + cd $CHECKOUTDIR $ARCHDIR/genpkglist $ftppath $svnrepo if [ -f packages.txt ]; then mv packages.txt $ftppath/packages.txt -- cgit v1.2.2 From 662fb163936fbe55b67159496ab86c5b34ff6351 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Tue, 8 Apr 2008 13:07:18 -0400 Subject: Correct lockfile usage We want a lock per repo not per user. Signed-off-by: Aaron Griffin --- db-inc | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/db-inc b/db-inc index 7f5a66a..3f0e286 100644 --- a/db-inc +++ b/db-inc @@ -19,7 +19,7 @@ fi WORKDIR="~/.dbscripts" CHECKOUTDIR="$WORKDIR/checkout" -LOCKFILE="$WORKDIR/lock" +LOCKFILE="/tmp/.repolck.$arch.$repoid" DBFILE="$WORKDIR/$reponame.db.tar.gz" if [ ! `type -p fakeroot` ]; then @@ -46,7 +46,7 @@ getpkgname() { cleanup() { # unlock - rm -f "~/.dbscripts/lock" + rm -f "$LOCKFILE" [ "$1" ] && exit $1 } @@ -61,15 +61,16 @@ die() { } # check for locks -if [ -f ~/.dbscripts/lock ]; then - echo "error: db generation is already in progress" - exit 1 +if [ -f "$LOCKFILE" ]; then + owner=$(/bin/ls -l $LOCKFILE | /bin/awk '{print $3}') + echo "error: db generation is already in progress (started by $owner)" + exit 1 fi # catch ^C breaks trap ctrl_c SIGINT # lock -touch ~/.dbscripts/lock +touch "$LOCKFILE" if [ -d $CHECKOUTDIR ]; then cd $CHECKOUTDIR -- cgit v1.2.2 From e685ac2accb50e575492372c3f73e500039a7a6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20B=C3=A4chler?= Date: Tue, 8 Apr 2008 20:32:13 +0200 Subject: Small fix for ftpdir-cleanup --- ftpdir-cleanup | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ftpdir-cleanup b/ftpdir-cleanup index df400bd..98f6f7c 100755 --- a/ftpdir-cleanup +++ b/ftpdir-cleanup @@ -53,5 +53,5 @@ echo -ne "DELETEFILES:\n${DELETEFILES}\n\n" echo -ne "MISSINGFILES:\n${MISSINGFILES}\n\n" cd ${FTPDIR} -mv ${DELETEFILES} /home/package-cleanup/ -cd - +[ -n "${DELETEFILES}" ] && mv ${DELETEFILES} /home/package-cleanup/ +cd - >/dev/null -- cgit v1.2.2 From c3c08ee41c01b0beb2f53d7e75cf089dfbae9f99 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Tue, 15 Apr 2008 13:19:30 -0400 Subject: Switch to proper script usage Switch to a callable script in place of an include file Signed-off-by: Aaron Griffin --- db-core | 6 +- db-core64 | 6 +- db-extra | 6 +- db-extra64 | 6 +- db-inc | 179 ------------------------------------------------------- db-testing | 6 +- db-testing64 | 6 +- db-unstable | 6 +- db-unstable64 | 6 +- db-update | 187 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 10 files changed, 195 insertions(+), 219 deletions(-) delete mode 100644 db-inc create mode 100644 db-update diff --git a/db-core b/db-core index ccbfb61..f14bfab 100755 --- a/db-core +++ b/db-core @@ -1,7 +1,3 @@ #!/bin/bash -repoid=5 -reponame="core" -arch="i686" - -. $(dirname $0)/db-inc +$(dirname $0)/db-update "i686" "core" diff --git a/db-core64 b/db-core64 index d15be60..f14bfab 100755 --- a/db-core64 +++ b/db-core64 @@ -1,7 +1,3 @@ #!/bin/bash -repoid=5 -reponame="core" -arch="x86_64" - -. $(dirname $0)/db-inc +$(dirname $0)/db-update "i686" "core" diff --git a/db-extra b/db-extra index 8d2c5b8..99c7b8f 100755 --- a/db-extra +++ b/db-extra @@ -1,7 +1,3 @@ #!/bin/bash -repoid=2 -reponame="extra" -arch="i686" - -. $(dirname $0)/db-inc +$(dirname $0)/db-update "i686" "extra" diff --git a/db-extra64 b/db-extra64 index 19eef9f..5c183e9 100755 --- a/db-extra64 +++ b/db-extra64 @@ -1,7 +1,3 @@ #!/bin/bash -repoid=2 -reponame="extra" -arch="x86_64" - -. $(dirname $0)/db-inc +$(dirname $0)/db-update "x86_64" "extra" diff --git a/db-inc b/db-inc deleted file mode 100644 index 3f0e286..0000000 --- a/db-inc +++ /dev/null @@ -1,179 +0,0 @@ -#!/bin/bash - -# where are the arch scripts located? -ARCHDIR="/arch" - -#All this fun stuff used to be in the db-(whatever) files -# Let's make it cleaner -export CARCH="$arch" -ftppath="/home/ftp/$reponame/os/$arch/" -svnpath="/home/svn-packages" -svnrepo="$reponame-$arch" - -#Hacky for now -if [ "$arch" = "x86_64" ]; then - stagedir="$HOME/staging/${reponame}64" -else - stagedir="$HOME/staging/$reponame" -fi - -WORKDIR="~/.dbscripts" -CHECKOUTDIR="$WORKDIR/checkout" -LOCKFILE="/tmp/.repolck.$arch.$repoid" -DBFILE="$WORKDIR/$reponame.db.tar.gz" - -if [ ! `type -p fakeroot` ]; then - echo "error: fakeroot is missing" >&2 - exit 1 -fi - -if [ ! -d $stagedir ]; then - echo "error: staging directory missing: $stagedir" >&2 - exit 1 -fi - -# Get the package name from the filename -# hackish, but should work for now -getpkgname() { - local tmp - - tmp=${1##*/} - tmp=${tmp%.pkg.tar.gz} - tmp=${tmp%-i686} - tmp=${tmp%-x86_64} - echo ${tmp%-*-*} -} - -cleanup() { - # unlock - rm -f "$LOCKFILE" - [ "$1" ] && exit $1 -} - -ctrl_c() { - echo "Interrupted" >&2 - cleanup 0 -} - -die() { - echo "$*" >&2 - cleanup 1 -} - -# check for locks -if [ -f "$LOCKFILE" ]; then - owner=$(/bin/ls -l $LOCKFILE | /bin/awk '{print $3}') - echo "error: db generation is already in progress (started by $owner)" - exit 1 -fi - -# catch ^C breaks -trap ctrl_c SIGINT -# lock -touch "$LOCKFILE" - -if [ -d $CHECKOUTDIR ]; then - cd $CHECKOUTDIR - svn update - if [ $? -gt 0 ]; then - die "==> SVN update failed, aborting!" - fi -else - echo "==> Checking out repo: $svnrepo ($arch) - Please be patient" - svn checkout file://$svnpath $CHECKOUTDIR - if [ $? -gt 0 ]; then - die "==> SVN checkout failed, aborting!" - fi -fi - -cd $CHECKOUTDIR - -# Checkout the SVN module if we need to -updatelists= -if [ "`ls $stagedir/add 2>/dev/null`" -o "`ls $stagedir/del 2>/dev/null`" ]; then - updatelists=1 -else - echo "No files to process" - cleanup 0 -fi - -# Right-O, now we look through the "add" and "del" subdirectories of -# $stagedir and process the packages there accordingly -- all packages -# in the "add" dir are added/updated, all packages in the "del" dir -# are removed. -# -# This means the sync db could actually be unpacked/repacked twice in -# one db-* invocation, but it's not a huge performance hit. - -if [ -d $stagedir/add -a "`ls $stagedir/add`" ]; then - echo "==> Processing new/updated packages for repository '$reponame'..." >&2 - - # copy the db file into our working area - cp $ftppath/$reponame.db.tar.gz $DBFILE - - cd $stagedir/add - # run it thru fakeroot make sure everything is owned by root.root - echo "$ARCHDIR/updatesync-many add $DBFILE $CHECKOUTDIR $svnrepo" | fakeroot - - if [ $? -ne 0 ]; then - die "==> Error returned from updatesync-many" - fi - - cp $DBFILE $ftppath - - # only for i686 (for now) - if [ "$arch" = "i686" ]; then - echo "==> Scanning for New/Updated packages..." >&2 - cd $stagedir/add - $ARCHDIR/pkgdb1 $CHECKOUTDIR $svnrepo | $ARCHDIR/pkgdb2-add $repoid $stagedir/add - fi - - # move the package files into the ftp directory - mv -f $stagedir/add/*.pkg.tar.gz $ftppath -fi - -if [ -d $stagedir/del -a "`ls $stagedir/del`" ]; then - echo "==> Processing deleted packages for repository '$reponame'..." >&2 - - # copy the db file into our working area - cp $ftppath/$reponame.db.tar.gz $DBFILE - - cd $stagedir/del - # run it thru fakeroot make sure everything is owned by root.root - echo "$ARCHDIR/updatesync-many del $DBFILE NOT-USED ZOMGWOO" \ - | fakeroot - - if [ $? -ne 0 ]; then - die "==> Error returned from updatesync-many" - fi - - cp $DBFILE $ftppath - - # only for i686 (for now) - if [ "$arch" = "i686" ]; then - echo "==> Scanning for Deleted packages..." >&2 - cd $stagedir/del - ( - for i in *.pkg.tar.gz; do - pkgname=$(getpkgname $i) - echo $pkgname - done - ) | $ARCHDIR/pkgdb2-del $repoid $stagedir/del - fi - - # remove the package files - rm -f $stagedir/del/*.pkg.tar.gz -fi - -if [ "$updatelists" ]; then - echo "==> Generating Text Package List..." >&2 - cd $CHECKOUTDIR - $ARCHDIR/genpkglist $ftppath $svnrepo - if [ -f packages.txt ]; then - mv packages.txt $ftppath/packages.txt - fi -fi - -cleanup - -# vim: set ts=4 sw=4 noet ft=sh: diff --git a/db-testing b/db-testing index 15205a2..e2fe01d 100755 --- a/db-testing +++ b/db-testing @@ -1,7 +1,3 @@ #!/bin/bash -repoid=4 -reponame="testing" -arch="i686" - -. $(dirname $0)/db-inc +$(dirname $0)/db-update "i686" "testing" diff --git a/db-testing64 b/db-testing64 index c34355b..244372b 100755 --- a/db-testing64 +++ b/db-testing64 @@ -1,7 +1,3 @@ #!/bin/bash -repoid=4 -reponame="testing" -arch="x86_64" - -. $(dirname $0)/db-inc +$(dirname $0)/db-update "x86_64" "testing" diff --git a/db-unstable b/db-unstable index b0231b4..a9aa49c 100755 --- a/db-unstable +++ b/db-unstable @@ -1,8 +1,4 @@ #!/bin/bash -repoid=3 -reponame="unstable" -arch="i686" - -. $(dirname $0)/db-inc +$(dirname $0)/db-update "i686" "unstable" diff --git a/db-unstable64 b/db-unstable64 index 225cf97..d9448ab 100755 --- a/db-unstable64 +++ b/db-unstable64 @@ -1,8 +1,4 @@ #!/bin/bash -repoid=3 -reponame="unstable" -arch="x86_64" - -. $(dirname $0)/db-inc +$(dirname $0)/db-update "x86_64" "unstable" diff --git a/db-update b/db-update new file mode 100644 index 0000000..24ca489 --- /dev/null +++ b/db-update @@ -0,0 +1,187 @@ +#!/bin/bash + +# where are the arch scripts located? +ARCHDIR="/arch" + +if [ $# -ne 2 ]; then + echo "usage: $(basename $0) " + exit 1 +fi + +#All this fun stuff used to be in the db-(whatever) files +# Let's make it cleaner +arch="$1" +reponame="$2" + +export CARCH="$arch" +ftppath="/home/ftp/$reponame/os/$arch/" +svnpath="/home/svn-packages" +svnrepo="$reponame-$arch" + +#Hacky for now +if [ "$arch" = "x86_64" ]; then + stagedir="$HOME/staging/${reponame}64" +else + stagedir="$HOME/staging/$reponame" +fi + +WORKDIR="~/.dbscripts" +CHECKOUTDIR="$WORKDIR/checkout" +LOCKFILE="/tmp/.repolck.$arch.$repoid" +DBFILE="$WORKDIR/$reponame.db.tar.gz" + +if [ ! `type -p fakeroot` ]; then + echo "error: fakeroot is missing" >&2 + exit 1 +fi + +if [ ! -d $stagedir ]; then + echo "error: staging directory missing: $stagedir" >&2 + exit 1 +fi + +# Get the package name from the filename +# hackish, but should work for now +getpkgname() { + local tmp + + tmp=${1##*/} + tmp=${tmp%.pkg.tar.gz} + tmp=${tmp%-i686} + tmp=${tmp%-x86_64} + echo ${tmp%-*-*} +} + +cleanup() { + # unlock + rm -f "$LOCKFILE" + [ "$1" ] && exit $1 +} + +ctrl_c() { + echo "Interrupted" >&2 + cleanup 0 +} + +die() { + echo "$*" >&2 + cleanup 1 +} + +# check for locks +if [ -f "$LOCKFILE" ]; then + owner=$(/bin/ls -l $LOCKFILE | /bin/awk '{print $3}') + echo "error: db generation is already in progress (started by $owner)" + exit 1 +fi + +# catch ^C breaks +trap ctrl_c SIGINT +# lock +touch "$LOCKFILE" + +if [ -d $CHECKOUTDIR ]; then + cd $CHECKOUTDIR + svn update + if [ $? -gt 0 ]; then + die "==> SVN update failed, aborting!" + fi +else + echo "==> Checking out repo: $svnrepo ($arch) - Please be patient" + svn checkout file://$svnpath $CHECKOUTDIR + if [ $? -gt 0 ]; then + die "==> SVN checkout failed, aborting!" + fi +fi + +cd $CHECKOUTDIR + +# Checkout the SVN module if we need to +updatelists= +if [ "`ls $stagedir/add 2>/dev/null`" -o "`ls $stagedir/del 2>/dev/null`" ]; then + updatelists=1 +else + echo "No files to process" + cleanup 0 +fi + +# Right-O, now we look through the "add" and "del" subdirectories of +# $stagedir and process the packages there accordingly -- all packages +# in the "add" dir are added/updated, all packages in the "del" dir +# are removed. +# +# This means the sync db could actually be unpacked/repacked twice in +# one db-* invocation, but it's not a huge performance hit. + +if [ -d $stagedir/add -a "`ls $stagedir/add`" ]; then + echo "==> Processing new/updated packages for repository '$reponame'..." >&2 + + # copy the db file into our working area + cp $ftppath/$reponame.db.tar.gz $DBFILE + + cd $stagedir/add + # run it thru fakeroot make sure everything is owned by root.root + echo "$ARCHDIR/updatesync-many add $DBFILE $CHECKOUTDIR $svnrepo" | fakeroot + + if [ $? -ne 0 ]; then + die "==> Error returned from updatesync-many" + fi + + cp $DBFILE $ftppath + + # only for i686 (for now) + if [ "$arch" = "i686" ]; then + echo "==> Scanning for New/Updated packages..." >&2 + cd $stagedir/add + $ARCHDIR/pkgdb1 $CHECKOUTDIR $svnrepo | $ARCHDIR/pkgdb2-add $repoid $stagedir/add + fi + + # move the package files into the ftp directory + mv -f $stagedir/add/*.pkg.tar.gz $ftppath +fi + +if [ -d $stagedir/del -a "`ls $stagedir/del`" ]; then + echo "==> Processing deleted packages for repository '$reponame'..." >&2 + + # copy the db file into our working area + cp $ftppath/$reponame.db.tar.gz $DBFILE + + cd $stagedir/del + # run it thru fakeroot make sure everything is owned by root.root + echo "$ARCHDIR/updatesync-many del $DBFILE NOT-USED ZOMGWOO" \ + | fakeroot + + if [ $? -ne 0 ]; then + die "==> Error returned from updatesync-many" + fi + + cp $DBFILE $ftppath + + # only for i686 (for now) + if [ "$arch" = "i686" ]; then + echo "==> Scanning for Deleted packages..." >&2 + cd $stagedir/del + ( + for i in *.pkg.tar.gz; do + pkgname=$(getpkgname $i) + echo $pkgname + done + ) | $ARCHDIR/pkgdb2-del $repoid $stagedir/del + fi + + # remove the package files + rm -f $stagedir/del/*.pkg.tar.gz +fi + +if [ "$updatelists" ]; then + echo "==> Generating Text Package List..." >&2 + cd $CHECKOUTDIR + $ARCHDIR/genpkglist $ftppath $svnrepo + if [ -f packages.txt ]; then + mv packages.txt $ftppath/packages.txt + fi +fi + +cleanup + +# vim: set ts=4 sw=4 noet ft=sh: -- cgit v1.2.2 From 2e0364b61c96598191e75fb9556fe454d6a1fb5f Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Tue, 15 Apr 2008 13:20:32 -0400 Subject: Remove SQL database related tasks from these scripts Updating a SQL database should NOT be a task of these scripts. They should deal ONLY with the act of maintaining a pacman repo Signed-off-by: Aaron Griffin --- Makefile | 20 ---- db-update | 21 +---- pkgdb | 158 ------------------------------- pkgdb1 | 70 -------------- pkgdb2-add.c | 276 ------------------------------------------------------ pkgdb2-del.c | 135 --------------------------- pkgdb2.c | 298 ----------------------------------------------------------- 7 files changed, 1 insertion(+), 977 deletions(-) delete mode 100644 Makefile mode change 100644 => 100755 db-update delete mode 100755 pkgdb delete mode 100755 pkgdb1 delete mode 100644 pkgdb2-add.c delete mode 100644 pkgdb2-del.c delete mode 100644 pkgdb2.c diff --git a/Makefile b/Makefile deleted file mode 100644 index 8c24840..0000000 --- a/Makefile +++ /dev/null @@ -1,20 +0,0 @@ -# mysql db info. -DB_NAME:="archlinux" -DB_USER:="archlinux" -DB_PASS:="passwords-are-NOT-cool" - -MYSQL_DEFS:=-DDB_NAME=\"$(DB_NAME)\" -DDB_USER=\"$(DB_USER)\" -DDB_PASS=\"$(DB_PASS)\" -MYSQL_FLAGS:=-I/usr/include/mysql -L/usr/lib/mysql -lmysqlclient - -all: pkgdb2-add pkgdb2-del - -pkgdb2-add: pkgdb2-add.c - gcc $(MYSQL_DEFS) $(MYSQL_FLAGS) -o pkgdb2-add pkgdb2-add.c - -pkgdb2-del: pkgdb2-del.c - gcc $(MYSQL_DEFS) $(MYSQL_FLAGS) -o pkgdb2-del pkgdb2-del.c - -clean: - rm -f pkgdb2-add pkgdb2-del - -.PHONY: all clean diff --git a/db-update b/db-update old mode 100644 new mode 100755 index 24ca489..d17fab4 --- a/db-update +++ b/db-update @@ -27,7 +27,7 @@ fi WORKDIR="~/.dbscripts" CHECKOUTDIR="$WORKDIR/checkout" -LOCKFILE="/tmp/.repolck.$arch.$repoid" +LOCKFILE="/tmp/.repolck.$arch.$reponame" DBFILE="$WORKDIR/$reponame.db.tar.gz" if [ ! `type -p fakeroot` ]; then @@ -129,13 +129,6 @@ if [ -d $stagedir/add -a "`ls $stagedir/add`" ]; then cp $DBFILE $ftppath - # only for i686 (for now) - if [ "$arch" = "i686" ]; then - echo "==> Scanning for New/Updated packages..." >&2 - cd $stagedir/add - $ARCHDIR/pkgdb1 $CHECKOUTDIR $svnrepo | $ARCHDIR/pkgdb2-add $repoid $stagedir/add - fi - # move the package files into the ftp directory mv -f $stagedir/add/*.pkg.tar.gz $ftppath fi @@ -157,18 +150,6 @@ if [ -d $stagedir/del -a "`ls $stagedir/del`" ]; then cp $DBFILE $ftppath - # only for i686 (for now) - if [ "$arch" = "i686" ]; then - echo "==> Scanning for Deleted packages..." >&2 - cd $stagedir/del - ( - for i in *.pkg.tar.gz; do - pkgname=$(getpkgname $i) - echo $pkgname - done - ) | $ARCHDIR/pkgdb2-del $repoid $stagedir/del - fi - # remove the package files rm -f $stagedir/del/*.pkg.tar.gz fi diff --git a/pkgdb b/pkgdb deleted file mode 100755 index 6a21f1e..0000000 --- a/pkgdb +++ /dev/null @@ -1,158 +0,0 @@ -#!/bin/bash - -tl=`pwd` - -repoid=$1 -# we use this instead of mktemp cuz the RH server's mktemp is broken :( -uid=`id -u` -TMPFILE="/tmp/.mysqltmp.$uid.$repoid.1" -TMPFILE2="/tmp/.mysqltmp.$uid.$repoid.2" - -dbuser='archweb' -dbname='archweb' -dbpass='YWkrOjqm' - -if [ "$repoid" = "" ]; then - me=`basename $0` - echo "usage: $me " - exit 1 -fi - -deletepkg() -{ - echo "Deleting $pkgname" - mysql -u $dbuser -p$dbpass $dbname <<_EOF -DELETE FROM packages WHERE id='$id'; -_EOF - mysql -u $dbuser -p$dbpass $dbname <<_EOF -DELETE FROM todolist_pkgs WHERE pkgid='$id'; -_EOF -} - -updatepkg() -{ - echo "Updating $pkgname" - mysql -u $dbuser -p$dbpass $dbname <<_EOF -UPDATE packages SET categoryid='$categoryid',pkgname='$pkgname',pkgver='$pkgver', -pkgrel='$pkgrel',pkgdesc='$pkgdesc',url='$url',sources='$sources', -depends='$deplist',needupdate=0,lastupdate=NOW() WHERE id='$id'; -_EOF - mysql -u $dbuser -p$dbpass $dbname <<_EOF -UPDATE todolist_pkgs SET complete=1 WHERE pkgid='$id'; -_EOF -} - -insertpkg() -{ - echo "Inserting $pkgname" - mysql -u $dbuser -p$dbpass $dbname <<_EOF -INSERT INTO packages (id,repoid,categoryid,pkgname,pkgver,pkgrel, -pkgdesc,url,sources,depends,lastupdate) VALUES (NULL,$repoid, -'$categoryid','$pkgname','$pkgver','$pkgrel','$pkgdesc','$url', -'$sources','$deplist',NOW()); -_EOF -} - -process() { - pkg=$1 - - id=`echo $pkg | awk '{print $1}'` - pkgname=`echo $pkg | awk '{print $2}'` - oldver=`echo $pkg | awk '{print $3}'` - oldrel=`echo $pkg | awk '{print $4}'` - [ "$pkgname" = "" ] && return; - cd $tl - pkgdir=`find . -maxdepth 2 -mindepth 1 -type d -name "$pkgname" | head -1` - if [ "$pkgdir" = "" ]; then - deletepkg - else - cd $pkgdir - if [ ! -f PKGBUILD ]; then - echo "WARNING: missing PKGBUILD in `pwd`" >&2 - else - unset depends source deplist sources - unset pkgname pkgver pkgrel pkgdesc url - . PKGBUILD - # tag it - touch .tag - category=`pwd | sed "s|$tl/||" | sed 's|/.*$||'` - mysql --skip-column-names -u $dbuser -p$dbpass $dbname >$TMPFILE2 <<_EOF -SELECT id FROM categories WHERE category='$category'; -_EOF - categoryid=`cat $TMPFILE2 | awk '{print $1}'` - if [ "$categoryid" = "" ]; then - echo "WARNING: no categoryid found for '$category'" >&2 - fi - cd - &>/dev/null - deplist=${depends[@]} - deplist=`php -r "echo addslashes(\"$deplist\");"` - sources=${source[@]} - sources=`php -r "echo addslashes(\"$sources\");"` - pkgname=`php -r "echo addslashes(\"$pkgname\");"` - pkgver=`php -r "echo addslashes(\"$pkgver\");"` - pkgrel=`php -r "echo addslashes(\"$pkgrel\");"` - pkgdesc=`php -r "echo addslashes(\"$pkgdesc\");"` - url=`php -r "echo addslashes(\"$url\");"` - - if [ "$oldver" != "$pkgver" -o "$oldrel" != "$pkgrel" ]; then - updatepkg - fi - fi - fi -} - -echo "==> Updating Package Entries in Database..." -mysql --skip-column-names -u $dbuser -p$dbpass $dbname >$TMPFILE <<_EOF -SELECT id,pkgname,pkgver,pkgrel FROM packages WHERE repoid='$repoid'; -_EOF - -echo " -> Scanning for Updated/Deleted packages..." -cat $TMPFILE | ( \ - ln='a' - while [ "$ln" != "" ]; do - read ln - process "$ln" - done -) - -echo " -> Scanning for New packages..." -cd $tl -for category in `find . -maxdepth 1 -type d | grep -v CVS | grep -v "^extra" | grep -v "^unstable"`; do - category=`echo $category | sed 's|./||'` - if [ "$category" = "." ]; then - continue - fi - mysql --skip-column-names -u $dbuser -p$dbpass $dbname >$TMPFILE <<_EOF -SELECT id FROM categories WHERE category='$category'; -_EOF - categoryid=`cat $TMPFILE | awk '{print $1}'` - if [ "$categoryid" = "" ]; then - echo "WARNING: no categoryid found for '$category'" >&2 - fi - for pkg in `ls $category | sort`; do - cd $tl/$category/$pkg - if [ -f .tag ]; then - # already processed - rm -f .tag - else - if [ -f PKGBUILD ]; then - . PKGBUILD - deplist=${depends[@]} - deplist=`php -r "echo addslashes(\"$deplist\");"` - sources=${source[@]} - sources=`php -r "echo addslashes(\"$sources\");"` - pkgname=`php -r "echo addslashes(\"$pkgname\");"` - pkgver=`php -r "echo addslashes(\"$pkgver\");"` - pkgrel=`php -r "echo addslashes(\"$pkgrel\");"` - pkgdesc=`php -r "echo addslashes(\"$pkgdesc\");"` - url=`php -r "echo addslashes(\"$url\");"` - insertpkg - fi - fi - cd $tl - done -done - -rm -f $TMPFILE -exit 0 - diff --git a/pkgdb1 b/pkgdb1 deleted file mode 100755 index 72dc3ac..0000000 --- a/pkgdb1 +++ /dev/null @@ -1,70 +0,0 @@ -#!/bin/bash - -# Get the package name from the filename -# hackish, but should work for now -getpkgname() { - local tmp - - tmp=${1##*/} - tmp=${tmp%.pkg.tar.gz} - tmp=${tmp%-i686} - tmp=${tmp%-x86_64} - echo ${tmp%-*-*} -} - -STAGEDIR=$(pwd) - -#This is our "unknown" category. We will have to do something about this later. -# It is a stop-gap. We will insert new packages in this way -CATEGORY="unknown" - -SVNCO=$1 -REPOTAG=$2 -if [ "$SVNCO" = "" ]; then - me=$(basename $0) - echo "usage: $me " >&2 - exit 1 -fi - -if [ ! "$(ls $STAGEDIR/*.pkg.tar.gz 2>/dev/null)" ]; then - exit -fi - -cd $STAGEDIR -for pkgfile in $STAGEDIR/*.pkg.tar.gz; do - pkgname=$(getpkgname $pkgfile); - fullname=$(basename $pkgfile) - pkgpath="$SVNCO/$pkgname/repos/$REPOTAG" - - # find the matching PKGBUILD - if [ ! -d "$pkgpath" ]; then - msg "WARNING: could not find PKGBUILD for $pkgname, cannot update this entry" - return - fi - pkgbuild="${pkgpath}/PKGBUILD" - if [ ! -f $pkgbuild ]; then - msg "WARNING: could not find PKGBUILD for $fullname, cannot update this entry" - return - fi - - # pick out the category from the pathname - unset pkgname pkgver pkgrel pkgdesc license groups provides md5sums force - unset replaces depends conflicts backup source install build makedepends - unset options - source $pkgbuild || continue - - deplist=${depends[@]} - sources=${source[@]} - echo $fullname - echo $pkgname - echo $pkgver - echo $pkgrel - echo $pkgdesc - echo $CATEGORY - echo $url - echo $sources - echo $deplist -done - -exit 0 - diff --git a/pkgdb2-add.c b/pkgdb2-add.c deleted file mode 100644 index 92d1339..0000000 --- a/pkgdb2-add.c +++ /dev/null @@ -1,276 +0,0 @@ -#include -#include -#include -#include -#include - -#ifndef DB_PASS - #error "MySQL connection info undefined" -#endif - -typedef struct pkg { - unsigned int id; - char *name; - char *ver; - char *rel; - struct pkg *next; -} pkg_t; - -MYSQL_RES *doquery(MYSQL *m, const char* q) -{ - MYSQL_RES *res; - if(mysql_query(m, q)) { - fprintf(stderr, "mysql_query: %s\n", mysql_error(m)); - return(NULL); - } - res = mysql_store_result(m); - return(res); -} - -/* this function is ugly -- it malloc's for each string it - * returns, and they probably won't be freed by the caller. - */ -char* addslashes(const char *s) { - char slashed[8192]; - char *p; - - slashed[0] = '\0'; - p = slashed; - while(*s) { - if(*s == '\'' || *s == '"' || *s == '\\') { - *p++ = '\\'; - } - *p++ = *s++; - } - *p = '\0'; - return(strdup(slashed)); -} - -char* trim(char *str) -{ - char *pch = str; - while(isspace(*pch)) { - pch++; - } - if(pch != str) { - memmove(str, pch, (strlen(pch) + 1)); - } - - pch = (char*)(str + (strlen(str) - 1)); - while(isspace(*pch)) { - pch--; - } - *++pch = '\0'; - - return str; -} - -/* scan a .pkg.tar.gz file and put all files listed into the database. - * - * this function is hacky and should be done properly, but this route is - * easier than reading the file with libtar. - */ -void updatefilelist(MYSQL *db, unsigned long id, char *fn) -{ - FILE *fp; - char *tmp; - char cmd[PATH_MAX]; - char line[PATH_MAX]; - char query[PATH_MAX]; - - tmp = tempnam("/tmp", "pkgdb"); - snprintf(cmd, PATH_MAX-1, "/bin/tar tzvf %s | awk '{print $6}' >%s", fn, tmp); - system(cmd); - fp = fopen(tmp, "r"); - if(fp == NULL) { - fprintf(stderr, "pkgdb2-add: could not open tempfile: %s\n", tmp); - return; - } - snprintf(query, sizeof(query), "DELETE FROM packages_files WHERE pkg_id='%lu'", id); - doquery(db, query); - while(fgets(line, sizeof(line)-1, fp)) { - char *fixedfn = addslashes(trim(line)); - if(!strcmp(fixedfn, ".FILELIST") || !strcmp(fixedfn, ".PKGINFO") || !strcmp(fixedfn, ".INSTALL")) { - free(fixedfn); - continue; - } - /* varchars aren't case-sensitive but filesystems are, so we use REPLACE INTO */ - snprintf(query, sizeof(query), "REPLACE INTO packages_files (pkg_id,path) VALUES " - "('%lu', '%s')", id, fixedfn); - free(fixedfn); - doquery(db, query); - } - fclose(fp); - unlink(tmp); -} - -int main(int argc, char **argv) -{ - MYSQL db; - MYSQL_RES *result; - MYSQL_ROW row; - char query[4096]; - char fn[PATH_MAX]; - char ftppath[PATH_MAX]; - int repoid; - pkg_t *dblist = NULL; - pkg_t *pkglist = NULL; - pkg_t *pkgptr, *ptr; - - if(argc < 3) { - printf("usage: pkgdb2-add \n"); - printf("\nWARNING: Do not run this manually! It is intended to be run from\n" - "the Arch db-generation scripts.\n\n"); - return(1); - } - repoid = atoi(argv[1]); - strncpy(ftppath, argv[2], PATH_MAX-1); - - if(mysql_init(&db) == NULL) { - fprintf(stderr, "could not initialize\n"); - return(1); - } - if(mysql_real_connect(&db, "localhost", DB_USER, DB_PASS, DB_NAME, - 0, NULL, 0) == NULL) { - fprintf(stderr, "failed to connect to database: %s\n", mysql_error(&db)); - return(1); - } - snprintf(query, sizeof(query), "SELECT id,pkgname,pkgver,pkgrel FROM packages " - "WHERE repo_id='%d'", repoid); - result = doquery(&db, query); - while(row = mysql_fetch_row(result)) { - int i; - /*unsigned long *lengths; - lengths = mysql_fetch_lengths(result);*/ - /* add the node to the list */ - if(dblist == NULL) { - dblist = (pkg_t*)malloc(sizeof(pkg_t)); - if(dblist == NULL) { - fprintf(stderr, "error: out of memory!\n"); - return(1); - } - ptr = dblist; - } else { - ptr->next = (pkg_t*)malloc(sizeof(pkg_t)); - if(ptr->next == NULL) { - fprintf(stderr, "error: out of memory!\n"); - return(1); - } - ptr = ptr->next; - } - ptr->next = NULL; - /* pick out the fields */ - ptr->id = atoi(row[0]); - ptr->name = strdup(row[1]); - ptr->ver = strdup(row[2]); - ptr->rel = strdup(row[3]); - } - mysql_free_result(result); - - while(!feof(stdin)) { - int found = 0; - unsigned int catid = 0; - char name[256], ver[256], rel[256], desc[4096]; - char cat[256], url[256], sources[4096], deplist[4096]; - char pkgfile[4096]; - /* get package data from stdin */ - fgets(pkgfile, 4096, stdin); trim(pkgfile); if(feof(stdin)) continue; - fgets(name, 256, stdin); trim(name); if(feof(stdin)) continue; - fgets(ver, 256, stdin); trim(ver); if(feof(stdin)) continue; - fgets(rel, 256, stdin); trim(rel); if(feof(stdin)) continue; - fgets(desc, 4096, stdin); trim(desc); if(feof(stdin)) continue; - fgets(cat, 256, stdin); trim(cat); if(feof(stdin)) continue; - fgets(url, 256, stdin); trim(url); if(feof(stdin)) continue; - fgets(sources, 4096, stdin); trim(sources); if(feof(stdin)) continue; - fgets(deplist, 4096, stdin); trim(deplist); if(feof(stdin)) continue; - /* check for overruns */ - if(strlen(name) > 254 || strlen(ver) >= 254 || strlen(rel) > 254 || - strlen(desc) > 4094 || strlen(cat) >= 254 || strlen(url) > 254 || - strlen(sources) > 4094 || strlen(deplist) > 4094 || strlen(pkgfile) > 4094) { - fprintf(stderr, "pkgdb2-add: one or more fields are too long in package '%s'\n", name); - fprintf(stderr, "pkgdb2-add: check the lengths of your strings, most are limited " - "to 255 chars, some are 4095\n"); - return(1); - } - /* add the node to the list */ - if(pkglist == NULL) { - pkglist = (pkg_t*)malloc(sizeof(pkg_t)); - if(pkglist == NULL) { - fprintf(stderr, "error: out of memory!\n"); - return(1); - } - pkgptr = pkglist; - } else { - pkgptr->next = (pkg_t*)malloc(sizeof(pkg_t)); - if(pkgptr->next == NULL) { - fprintf(stderr, "error: out of memory!\n"); - return(1); - } - pkgptr = pkgptr->next; - } - pkgptr->next = NULL; - pkgptr->name = strdup(name); - /* look it up in our cache */ - for(ptr = dblist; ptr; ptr = ptr->next) { - if(!strcmp(name, ptr->name)) { - found = 1; - break; - } - } - /* get the category */ - snprintf(query, sizeof(query), - "SELECT id FROM categories WHERE category='%s'", cat); - result = doquery(&db, query); - if(mysql_num_rows(result) == 0) { - catid = 25; - /* - snprintf(query, sizeof(query), "INSERT INTO categories (id,category) " - " VALUES (NULL,'%s')", addslashes(cat)); - doquery(&db, query); - catid = (unsigned int)mysql_insert_id(&db); - */ - } else { - row = mysql_fetch_row(result); - catid = (unsigned int)atoi(row[0]); - } - if(!found) { - /* Insert... */ - unsigned long id; - fprintf(stderr, "pkgdb2-add: inserting %s\n", name); - snprintf(query, sizeof(query), "INSERT INTO packages (id,repo_id," - "category_id,pkgname,pkgver,pkgrel,pkgdesc,url,sources,depends," - "last_update) VALUES (NULL,'%d','%d','%s','%s','%s','%s'," - "'%s','%s','%s',NOW())", - repoid, catid, addslashes(name), addslashes(ver), addslashes(rel), - addslashes(desc), addslashes(url), addslashes(sources), - addslashes(deplist)); - doquery(&db, query); - id = mysql_insert_id(&db); - snprintf(fn, PATH_MAX-1, "%s/%s", ftppath, pkgfile); - updatefilelist(&db, id, fn); - continue; - } else if(strcmp(ptr->ver, ver) || strcmp(ptr->rel, rel)) { - /* ...or Update */ - fprintf(stderr, "pkgdb2-add: updating %s (%s-%s ==> %s-%s)\n", - ptr->name, ptr->ver, ptr->rel, ver, rel); - snprintf(query, sizeof(query), "UPDATE packages SET " - "pkgname='%s',pkgver='%s',pkgrel='%s',pkgdesc='%s',url='%s'," - "sources='%s',depends='%s',needupdate=0,last_update=NOW() " - "WHERE id='%d'", - addslashes(name), addslashes(ver), addslashes(rel), - addslashes(desc), addslashes(url), addslashes(sources), - addslashes(deplist), ptr->id); - doquery(&db, query); - snprintf(fn, PATH_MAX-1, "%s/%s", ftppath, pkgfile); - updatefilelist(&db, ptr->id, fn); - /* - snprintf(query, sizeof(query), "UPDATE todolist_pkgs SET complete=1 " - "WHERE pkgid='%d'", ptr->id); - doquery(&db, query); - */ - } - } - - mysql_close(&db); - return(0); -} diff --git a/pkgdb2-del.c b/pkgdb2-del.c deleted file mode 100644 index 3ceb580..0000000 --- a/pkgdb2-del.c +++ /dev/null @@ -1,135 +0,0 @@ -#include -#include -#include -#include -#include - -#ifndef DB_PASS - #error "MySQL connection info undefined" -#endif - -typedef struct pkg { - unsigned int id; - char *name; - char *ver; - char *rel; - struct pkg *next; -} pkg_t; - -MYSQL_RES *doquery(MYSQL *m, const char* q) -{ - MYSQL_RES *res; - if(mysql_query(m, q)) { - fprintf(stderr, "mysql_query: %s\n", mysql_error(m)); - return(NULL); - } - res = mysql_store_result(m); - return(res); -} - -/* this function is ugly -- it malloc's for each string it - * returns, and they probably won't be freed by the caller. - */ -char* addslashes(const char *s) { - char slashed[8192]; - char *p; - - slashed[0] = '\0'; - p = slashed; - while(*s) { - if(*s == '\'' || *s == '"' || *s == '\\') { - *p++ = '\\'; - } - *p++ = *s++; - } - *p = '\0'; - return(strdup(slashed)); -} - -char* trim(char *str) -{ - char *pch = str; - while(isspace(*pch)) { - pch++; - } - if(pch != str) { - memmove(str, pch, (strlen(pch) + 1)); - } - - pch = (char*)(str + (strlen(str) - 1)); - while(isspace(*pch)) { - pch--; - } - *++pch = '\0'; - - return str; -} - -int main(int argc, char **argv) -{ - MYSQL db; - MYSQL_RES *result; - MYSQL_ROW row; - char query[4096]; - char fn[PATH_MAX]; - char ftppath[PATH_MAX]; - int repoid; - pkg_t *pkglist = NULL; - pkg_t *pkgptr, *ptr; - - if(argc < 3) { - printf("usage: pkgdb2-del \n"); - printf("\nWARNING: Do not run this manually! It is intended to be run from\n" - "the Arch db-generation scripts.\n\n"); - return(1); - } - repoid = atoi(argv[1]); - strncpy(ftppath, argv[2], PATH_MAX-1); - - if(mysql_init(&db) == NULL) { - fprintf(stderr, "could not initialize\n"); - return(1); - } - if(mysql_real_connect(&db, "localhost", DB_USER, DB_PASS, DB_NAME, - 0, NULL, 0) == NULL) { - fprintf(stderr, "failed to connect to database: %s\n", mysql_error(&db)); - return(1); - } - - while(!feof(stdin)) { - int found = 0; - unsigned int catid = 0; - unsigned int pkgid = 0; - char name[256]; - /* get package data from stdin */ - fgets(name, 256, stdin); - trim(name); - if(feof(stdin)) continue; - /* check for overruns */ - if(strlen(name) > 254) { - fprintf(stderr, "pkgdb2-del: one or more fields are too long in package '%s'\n", name); - return(1); - } - /* get the package id */ - snprintf(query, sizeof(query), "SELECT id FROM packages WHERE " - "repo_id='%d' AND pkgname='%s'", repoid, addslashes(name)); - result = doquery(&db, query); - if(mysql_num_rows(result) == 0) { - fprintf(stderr, "pkgdb2-del: %s was not found in repo %d\n", name, repoid); - continue; - } - row = mysql_fetch_row(result); - pkgid = (unsigned int)atoi(row[0]); - /* delete from db */ - fprintf(stderr, "pkgdb2-del: deleting %s (id %d)\n", name, pkgid); - snprintf(query, sizeof(query), "DELETE FROM packages WHERE id='%d'", pkgid); - doquery(&db, query); - snprintf(query, sizeof(query), "DELETE FROM packages_files WHERE pkg_id='%d'", pkgid); - doquery(&db, query); - snprintf(query, sizeof(query), "DELETE FROM todolists_pkgs WHERE pkg_id='%d'", pkgid); - doquery(&db, query); - } - - mysql_close(&db); - return(0); -} diff --git a/pkgdb2.c b/pkgdb2.c deleted file mode 100644 index 0fc0cc9..0000000 --- a/pkgdb2.c +++ /dev/null @@ -1,298 +0,0 @@ -#include -#include -#include -#include -#include - -#define DB_USER "archweb" -#define DB_NAME "archweb" -#define DB_PASS "passwords-are-cool" - -typedef struct pkg { - unsigned int id; - char *name; - char *ver; - char *rel; - struct pkg *next; -} pkg_t; - -MYSQL_RES *doquery(MYSQL *m, const char* q) -{ - MYSQL_RES *res; - if(mysql_query(m, q)) { - fprintf(stderr, "mysql_query: %s\n", mysql_error(m)); - return(NULL); - } - res = mysql_store_result(m); - return(res); -} - -/* this function is ugly -- it malloc's for each string it - * returns, and they probably won't be freed by the caller. - */ -char* addslashes(const char *s) { - char slashed[8192]; - char *p; - - slashed[0] = '\0'; - p = slashed; - while(*s) { - if(*s == '\'' || *s == '"' || *s == '\\') { - *p++ = '\\'; - } - *p++ = *s++; - } - *p = '\0'; - return(strdup(slashed)); -} - -char* trim(char *str) -{ - char *pch = str; - while(isspace(*pch)) { - pch++; - } - if(pch != str) { - memmove(str, pch, (strlen(pch) + 1)); - } - - pch = (char*)(str + (strlen(str) - 1)); - while(isspace(*pch)) { - pch--; - } - *++pch = '\0'; - - return str; -} - -/* scan a .pkg.tar.gz file and put all files listed into the database. - * - * this function is hacky and should be done properly, but this route is - * easier than reading the file with libtar. - */ -void updatefilelist(MYSQL *db, unsigned long id, char *fn) -{ - FILE *fp; - char *tmp; - char cmd[PATH_MAX]; - char line[PATH_MAX]; - char query[PATH_MAX]; - - tmp = tempnam("/tmp", "pkgdb"); - snprintf(cmd, PATH_MAX-1, "/bin/tar tzvf %s | awk '{print $6}' >%s", fn, tmp); - system(cmd); - fp = fopen(tmp, "r"); - if(fp == NULL) { - fprintf(stderr, "pkgdb2: could not open tempfile: %s\n", tmp); - return; - } - snprintf(query, sizeof(query), "DELETE FROM packages_files WHERE id='%d'", id); - doquery(db, query); - while(fgets(line, sizeof(line)-1, fp)) { - char *fixedfn = addslashes(trim(line)); - if(!strcmp(fixedfn, ".FILELIST") || !strcmp(fixedfn, ".PKGINFO") || !strcmp(fixedfn, ".INSTALL")) { - free(fixedfn); - continue; - } - /* varchars aren't case-sensitive but filesystems are, so we use REPLACE INTO */ - snprintf(query, sizeof(query), "REPLACE INTO packages_files (id,path) VALUES " - "('%d', '%s')", id, fixedfn); - free(fixedfn); - doquery(db, query); - } - fclose(fp); - unlink(tmp); -} - -int main(int argc, char **argv) -{ - MYSQL db; - MYSQL_RES *result; - MYSQL_ROW row; - char query[4096]; - char fn[PATH_MAX]; - char ftppath[PATH_MAX]; - int repoid; - pkg_t *dblist = NULL; - pkg_t *pkglist = NULL; - pkg_t *pkgptr, *ptr; - - if(argc < 3) { - printf("usage: pkgdb2 \n"); - printf("\nWARNING: Do not run this manually! It is intended to be" - " run with pkgdb1 only.\n"); - return(1); - } - repoid = atoi(argv[1]); - strncpy(ftppath, argv[2], PATH_MAX-1); - - if(mysql_init(&db) == NULL) { - fprintf(stderr, "could not initialize\n"); - return(1); - } - if(mysql_real_connect(&db, "localhost", DB_USER, DB_PASS, DB_NAME, - 0, NULL, 0) == NULL) { - fprintf(stderr, "failed to connect to database: %s\n", mysql_error(&db)); - return(1); - } - snprintf(query, sizeof(query), "SELECT id,pkgname,pkgver,pkgrel FROM packages " - "WHERE repoid='%d'", repoid); - result = doquery(&db, query); - while(row = mysql_fetch_row(result)) { - int i; - /*unsigned long *lengths; - lengths = mysql_fetch_lengths(result);*/ - /* add the node to the list */ - if(dblist == NULL) { - dblist = (pkg_t*)malloc(sizeof(pkg_t)); - if(dblist == NULL) { - fprintf(stderr, "error: out of memory!\n"); - return(1); - } - ptr = dblist; - } else { - ptr->next = (pkg_t*)malloc(sizeof(pkg_t)); - if(ptr->next == NULL) { - fprintf(stderr, "error: out of memory!\n"); - return(1); - } - ptr = ptr->next; - } - ptr->next = NULL; - /* pick out the fields */ - ptr->id = atoi(row[0]); - ptr->name = strdup(row[1]); - ptr->ver = strdup(row[2]); - ptr->rel = strdup(row[3]); - } - mysql_free_result(result); - - while(!feof(stdin)) { - int found = 0; - unsigned int catid = 0; - char name[256], ver[256], rel[256], desc[4096]; - char cat[256], url[256], sources[4096], deplist[4096]; - /* get package data from stdin */ - fgets(name, 256, stdin); trim(name); if(feof(stdin)) continue; - fgets(ver, 256, stdin); trim(ver); if(feof(stdin)) continue; - fgets(rel, 256, stdin); trim(rel); if(feof(stdin)) continue; - fgets(desc, 4096, stdin); trim(desc); if(feof(stdin)) continue; - fgets(cat, 256, stdin); trim(cat); if(feof(stdin)) continue; - fgets(url, 256, stdin); trim(url); if(feof(stdin)) continue; - fgets(sources, 4096, stdin); trim(sources); if(feof(stdin)) continue; - fgets(deplist, 4096, stdin); trim(deplist); if(feof(stdin)) continue; - /* check for overruns */ - if(strlen(name) > 254 || strlen(ver) >= 254 || strlen(rel) > 254 || - strlen(desc) > 4094 || strlen(cat) >= 254 || strlen(url) > 254 || - strlen(sources) > 4094 || strlen(deplist) > 4094) { - fprintf(stderr, "pkgdb2: one or more fields are too long in package '%s'\n", name); - fprintf(stderr, "pkgdb2: check the lengths of your strings, most are limited " - "to 255 chars, some are 4095\n"); - return(1); - } - /* add the node to the list */ - if(pkglist == NULL) { - pkglist = (pkg_t*)malloc(sizeof(pkg_t)); - if(pkglist == NULL) { - fprintf(stderr, "error: out of memory!\n"); - return(1); - } - pkgptr = pkglist; - } else { - pkgptr->next = (pkg_t*)malloc(sizeof(pkg_t)); - if(pkgptr->next == NULL) { - fprintf(stderr, "error: out of memory!\n"); - return(1); - } - pkgptr = pkgptr->next; - } - pkgptr->next = NULL; - pkgptr->name = strdup(name); - /* look it up in our cache */ - for(ptr = dblist; ptr; ptr = ptr->next) { - if(!strcmp(name, ptr->name)) { - found = 1; - break; - } - } - /* get the category */ - snprintf(query, sizeof(query), - "SELECT id FROM categories WHERE category='%s'", cat); - result = doquery(&db, query); - if(mysql_num_rows(result) == 0) { - fprintf(stderr, "pkgdb2: no db category found for '%s'\n", cat); - /* - snprintf(query, sizeof(query), "INSERT INTO categories (id,category) " - " VALUES (NULL,'%s')", addslashes(cat)); - doquery(&db, query); - catid = (unsigned int)mysql_insert_id(&db); - */ - } else { - row = mysql_fetch_row(result); - catid = (unsigned int)atoi(row[0]); - } - if(!found) { - /* Insert... */ - unsigned long id; - fprintf(stderr, "pkgdb2: inserting %s\n", name); - snprintf(query, sizeof(query), "INSERT INTO packages (id,repoid," - "categoryid,pkgname,pkgver,pkgrel,pkgdesc,url,sources,depends," - "lastupdate) VALUES (NULL,'%d','%d','%s','%s','%s','%s'," - "'%s','%s','%s',NOW())", - repoid, catid, addslashes(name), addslashes(ver), addslashes(rel), - addslashes(desc), addslashes(url), addslashes(sources), - addslashes(deplist)); - doquery(&db, query); - id = mysql_insert_id(&db); - snprintf(fn, PATH_MAX-1, "%s/%s-%s-%s.pkg.tar.gz", ftppath, name, ver, rel); - updatefilelist(&db, id, fn); - continue; - } else if(strcmp(ptr->ver, ver) || strcmp(ptr->rel, rel)) { - /* ...or Update */ - fprintf(stderr, "pkgdb2: updating %s (%s-%s ==> %s-%s)\n", - ptr->name, ptr->ver, ptr->rel, ver, rel); - snprintf(query, sizeof(query), "UPDATE packages SET categoryid='%d'," - "pkgname='%s',pkgver='%s',pkgrel='%s',pkgdesc='%s',url='%s'," - "sources='%s',depends='%s',needupdate=0,lastupdate=NOW() " - "WHERE id='%d'", - catid, addslashes(name), addslashes(ver), addslashes(rel), - addslashes(desc), addslashes(url), addslashes(sources), - addslashes(deplist), ptr->id); - doquery(&db, query); - snprintf(fn, PATH_MAX-1, "%s/%s-%s-%s.pkg.tar.gz", ftppath, name, ver, rel); - updatefilelist(&db, ptr->id, fn); - /* - snprintf(query, sizeof(query), "UPDATE todolist_pkgs SET complete=1 " - "WHERE pkgid='%d'", ptr->id); - doquery(&db, query); - */ - } - } - - /* look for deleted packages */ - for(ptr = dblist; ptr; ptr = ptr->next) { - int found = 0; - for(pkgptr = pkglist; pkgptr; pkgptr = pkgptr->next) { - if(!strcmp(ptr->name, pkgptr->name)) { - found = 1; - break; - } - } - if(!found) { - /* delete from db */ - fprintf(stderr, "pkgdb2: deleting %s\n", ptr->name); - snprintf(query, sizeof(query), "DELETE FROM packages WHERE id='%d'", - ptr->id); - doquery(&db, query); - snprintf(query, sizeof(query), "DELETE FROM packages_files WHERE id='%d'", - ptr->id); - doquery(&db, query); - snprintf(query, sizeof(query), "DELETE FROM todolist_pkgs WHERE pkgid='%d'", - ptr->id); - doquery(&db, query); - } - } - - mysql_close(&db); - return(0); -} -- cgit v1.2.2 From 6cb8979c4e48ec86df7b0740cb49f0fe42997f1a Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Fri, 2 May 2008 19:07:06 -0400 Subject: Massive rewrite to make this script uber Yeah too much to write out. I basically rewrote it from scratch to use repo-add and all that fun stuff. --- db-update | 227 ++++++++++++++++++++++++++++++++++---------------------------- 1 file changed, 123 insertions(+), 104 deletions(-) diff --git a/db-update b/db-update index d17fab4..766a2d5 100755 --- a/db-update +++ b/db-update @@ -1,168 +1,187 @@ #!/bin/bash -# where are the arch scripts located? -ARCHDIR="/arch" - if [ $# -ne 2 ]; then - echo "usage: $(basename $0) " + echo "usage: $(basename $0) " + exit 1 +fi + +if [ -f "/etc/makepkg.conf" ]; then + #Get some config info + . /etc/makepkg.conf +else + echo "/etc/makepkg.conf does not exist!" exit 1 fi -#All this fun stuff used to be in the db-(whatever) files -# Let's make it cleaner arch="$1" reponame="$2" export CARCH="$arch" -ftppath="/home/ftp/$reponame/os/$arch/" -svnpath="/home/svn-packages" + +##### Arch specific stuff. TODO make this configurable ##### +#ftppath="/home/ftp/$reponame/os/$arch/" +ftppath="/home/aaron/dbscripts/ftp-$reponame" +svnpath="file:///home/svn-packages" svnrepo="$reponame-$arch" +#stagedir="$HOME/staging/$reponame" +stagedir="/home/aaron/dbscripts/staging/$reponame" +############################################################ -#Hacky for now -if [ "$arch" = "x86_64" ]; then - stagedir="$HOME/staging/${reponame}64" -else - stagedir="$HOME/staging/$reponame" -fi +[ "$UID" = "" ] && UID=$(uid) -WORKDIR="~/.dbscripts" -CHECKOUTDIR="$WORKDIR/checkout" +WORKDIR="/tmp/db-update.$svnrepo.$UID" LOCKFILE="/tmp/.repolck.$arch.$reponame" -DBFILE="$WORKDIR/$reponame.db.tar.gz" +ADDPKGS="" +REMPKGS="" if [ ! `type -p fakeroot` ]; then - echo "error: fakeroot is missing" >&2 - exit 1 + echo "error: fakeroot is missing" >&2 + exit 1 fi if [ ! -d $stagedir ]; then - echo "error: staging directory missing: $stagedir" >&2 - exit 1 + echo "error: staging directory missing: $stagedir" >&2 + exit 1 fi # Get the package name from the filename # hackish, but should work for now getpkgname() { - local tmp + local tmp - tmp=${1##*/} - tmp=${tmp%.pkg.tar.gz} - tmp=${tmp%-i686} - tmp=${tmp%-x86_64} - echo ${tmp%-*-*} + tmp=${1##*/} + tmp=${tmp%$PKGEXT} + tmp=${tmp%-$CARCH} + echo ${tmp%-*-*} } cleanup() { - # unlock - rm -f "$LOCKFILE" - [ "$1" ] && exit $1 + # unlock + rm -f "$LOCKFILE" + #rm -rf "$WORKDIR" + [ "$1" ] && exit $1 } ctrl_c() { - echo "Interrupted" >&2 - cleanup 0 + echo "Interrupted" >&2 + cleanup 0 } die() { - echo "$*" >&2 - cleanup 1 + echo "$*" >&2 + cleanup 1 } # check for locks if [ -f "$LOCKFILE" ]; then - owner=$(/bin/ls -l $LOCKFILE | /bin/awk '{print $3}') + owner="$(/usr/bin/stat -c %U $LOCKFILE)" echo "error: db generation is already in progress (started by $owner)" exit 1 fi -# catch ^C breaks -trap ctrl_c SIGINT -# lock -touch "$LOCKFILE" +trap ctrl_c 2 +trap cleanup 0 -if [ -d $CHECKOUTDIR ]; then - cd $CHECKOUTDIR - svn update - if [ $? -gt 0 ]; then - die "==> SVN update failed, aborting!" - fi -else - echo "==> Checking out repo: $svnrepo ($arch) - Please be patient" - svn checkout file://$svnpath $CHECKOUTDIR - if [ $? -gt 0 ]; then - die "==> SVN checkout failed, aborting!" - fi -fi +touch "$LOCKFILE" -cd $CHECKOUTDIR +mkdir -p "$WORKDIR" -# Checkout the SVN module if we need to -updatelists= -if [ "`ls $stagedir/add 2>/dev/null`" -o "`ls $stagedir/del 2>/dev/null`" ]; then - updatelists=1 -else - echo "No files to process" - cleanup 0 +to_add="" +if [ -d "$stagedir/add" ]; then + ADDPKGS="$(ls $stagedir{64,}/add/*-${arch}$PKGEXT 2>/dev/null)" fi -# Right-O, now we look through the "add" and "del" subdirectories of -# $stagedir and process the packages there accordingly -- all packages -# in the "add" dir are added/updated, all packages in the "del" dir -# are removed. -# -# This means the sync db could actually be unpacked/repacked twice in -# one db-* invocation, but it's not a huge performance hit. - -if [ -d $stagedir/add -a "`ls $stagedir/add`" ]; then - echo "==> Processing new/updated packages for repository '$reponame'..." >&2 - - # copy the db file into our working area - cp $ftppath/$reponame.db.tar.gz $DBFILE +if [ -n "$ADDPKGS" ]; then + echo "==> Processing new/updated packages for repository '$reponame'..." >&2 + + cd "$WORKDIR" + svn checkout -N $svnpath checkout + cd checkout + + for pkg in $ADDPKGS; do + _pkgfile=$(basename $pkg) + _pkgname="$(getpkgname $pkg)" + echo " Checking SVN for $_pkgname" + svn up -q $_pkgname + if [ -d "$_pkgname/repos/$svnrepo" ]; then + . "$_pkgname/repos/$svnrepo/$BUILDSCRIPT" + if [ "$_pkgfile" = "$pkgname-$pkgver-$pkgrel-$CARCH$PKGEXT" ]; then + to_add="$to_add $pkg" + else + echo " WARNING: $_pkgfile does not match $BUILDSCRIPT in $svnrepo" + fi + else + echo " WARNING: Package $_pkgname not found in $svnrepo" + fi + done - cd $stagedir/add - # run it thru fakeroot make sure everything is owned by root.root - echo "$ARCHDIR/updatesync-many add $DBFILE $CHECKOUTDIR $svnrepo" | fakeroot + cd "$WORKDIR" + [ -d build/ ] || mkdir build - if [ $? -ne 0 ]; then - die "==> Error returned from updatesync-many" - fi + # copy the db file into our working area + [ -f "$ftppath/$reponame.db.tar.gz" ] && cp "$ftppath/$reponame.db.tar.gz" build/ + cp $to_add build/ - cp $DBFILE $ftppath + cd build/ + /usr/bin/repo-add "$reponame.db.tar.$DB_COMPRESSION" $to_add +else + echo "No packages to add" +fi - # move the package files into the ftp directory - mv -f $stagedir/add/*.pkg.tar.gz $ftppath +to_rem="" +if [ -d "$stagedir/del" ]; then + REMPKGS="$(ls $stagedir{64,}/del/*-${arch}$PKGEXT 2>/dev/null)" fi -if [ -d $stagedir/del -a "`ls $stagedir/del`" ]; then - echo "==> Processing deleted packages for repository '$reponame'..." >&2 +if [ -n "$REMPKGS" ]; then + echo "==> Processing deleted packages for repository '$reponame'..." >&2 - # copy the db file into our working area - cp $ftppath/$reponame.db.tar.gz $DBFILE + if [ ! -d "$WORKDIR/checkout" ]; then + cd "$WORKDIR" + svn checkout -N $svnpath checkout + fi + cd "$WORKDIR/checkout" + + #TODO removal shouldn't require a package file + for pkg in $REMPKGS; do + _pkgfile=$(basename $pkg) + _pkgname="$(getpkgname $pkg)" + echo " Checking SVN for $_pkgname" + svn up -q $_pkgname + if [ -d "$_pkgname/repos/$svnrepo" ]; then + echo " WARNING: $_pkgname still exists in $svnrepo" + else + to_rem="$to_rem $pkg" + fi + done - cd $stagedir/del - # run it thru fakeroot make sure everything is owned by root.root - echo "$ARCHDIR/updatesync-many del $DBFILE NOT-USED ZOMGWOO" \ - | fakeroot + cd "$WORKDIR" + [ -d build/ ] || mkdir build - if [ $? -ne 0 ]; then - die "==> Error returned from updatesync-many" + # copy the db file into our working area + [ -f "$ftppath/$reponame.db.tar.gz" ] && cp "$ftppath/$reponame.db.tar.gz" build/ + for rem in $to_rem; do + if [ -f "build/$rem" ]; then + rm "build/$rem" fi + done - cp $DBFILE $ftppath + cd build/ + /usr/bin/repo-remove "$reponame.db.tar.$DB_COMPRESSION" $to_rem +else + echo "No packages to delete" +fi - # remove the package files - rm -f $stagedir/del/*.pkg.tar.gz +if [ $(ls "$WORKDIR/build/" 2>/dev/null | wc -l) != 0 ]; then + cp -r "$WORKDIR/build/"* "$ftppath" fi if [ "$updatelists" ]; then - echo "==> Generating Text Package List..." >&2 - cd $CHECKOUTDIR - $ARCHDIR/genpkglist $ftppath $svnrepo - if [ -f packages.txt ]; then - mv packages.txt $ftppath/packages.txt - fi + echo "==> Generating Text Package List..." >&2 + cd "$WORKDIR/checkout" + $ARCHDIR/genpkglist $ftppath $svnrepo + if [ -f packages.txt ]; then + mv packages.txt $ftppath/packages.txt + fi fi - -cleanup - # vim: set ts=4 sw=4 noet ft=sh: -- cgit v1.2.2 From 79e7365c6afdd0010cc5b64d63099c673320fe6e Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Mon, 5 May 2008 17:18:39 -0400 Subject: Switch the params to be inline with archrelease Signed-off-by: Aaron Griffin --- db-core | 2 +- db-core64 | 2 +- db-extra | 2 +- db-extra64 | 2 +- db-testing | 2 +- db-testing64 | 2 +- db-unstable | 2 +- db-unstable64 | 2 +- db-update | 4 ++-- 9 files changed, 10 insertions(+), 10 deletions(-) diff --git a/db-core b/db-core index f14bfab..b88fcae 100755 --- a/db-core +++ b/db-core @@ -1,3 +1,3 @@ #!/bin/bash -$(dirname $0)/db-update "i686" "core" +$(dirname $0)/db-update "core" "i686" diff --git a/db-core64 b/db-core64 index f14bfab..e080004 100755 --- a/db-core64 +++ b/db-core64 @@ -1,3 +1,3 @@ #!/bin/bash -$(dirname $0)/db-update "i686" "core" +$(dirname $0)/db-update "core" "x86_64" diff --git a/db-extra b/db-extra index 99c7b8f..1375b13 100755 --- a/db-extra +++ b/db-extra @@ -1,3 +1,3 @@ #!/bin/bash -$(dirname $0)/db-update "i686" "extra" +$(dirname $0)/db-update "extra" "i686" diff --git a/db-extra64 b/db-extra64 index 5c183e9..cfa20a2 100755 --- a/db-extra64 +++ b/db-extra64 @@ -1,3 +1,3 @@ #!/bin/bash -$(dirname $0)/db-update "x86_64" "extra" +$(dirname $0)/db-update "extra" "x86_64" diff --git a/db-testing b/db-testing index e2fe01d..69b5376 100755 --- a/db-testing +++ b/db-testing @@ -1,3 +1,3 @@ #!/bin/bash -$(dirname $0)/db-update "i686" "testing" +$(dirname $0)/db-update "testing" "i686" diff --git a/db-testing64 b/db-testing64 index 244372b..d75a456 100755 --- a/db-testing64 +++ b/db-testing64 @@ -1,3 +1,3 @@ #!/bin/bash -$(dirname $0)/db-update "x86_64" "testing" +$(dirname $0)/db-update "testing" "x86_64" diff --git a/db-unstable b/db-unstable index a9aa49c..2a2df43 100755 --- a/db-unstable +++ b/db-unstable @@ -1,4 +1,4 @@ #!/bin/bash -$(dirname $0)/db-update "i686" "unstable" +$(dirname $0)/db-update "unstable" "i686" diff --git a/db-unstable64 b/db-unstable64 index d9448ab..cd9c3a0 100755 --- a/db-unstable64 +++ b/db-unstable64 @@ -1,4 +1,4 @@ #!/bin/bash -$(dirname $0)/db-update "x86_64" "unstable" +$(dirname $0)/db-update "unstable" "x86_64" diff --git a/db-update b/db-update index 766a2d5..6c9af1a 100755 --- a/db-update +++ b/db-update @@ -13,8 +13,8 @@ else exit 1 fi -arch="$1" -reponame="$2" +reponame="$1" +arch="$2" export CARCH="$arch" -- cgit v1.2.2 From feac9cc83037b01d7e7904433086e57056fe0bc4 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Mon, 5 May 2008 18:41:47 -0400 Subject: Make the params slightly more clear Real super minor Signed-off-by: Aaron Griffin --- ftpdir-cleanup | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/ftpdir-cleanup b/ftpdir-cleanup index 98f6f7c..18e26f2 100755 --- a/ftpdir-cleanup +++ b/ftpdir-cleanup @@ -14,9 +14,12 @@ getpkgname() { echo ${tmp%-*-*} } +reponame=$1 +arch=$2 + FTPBASEDIR="/home/ftp" -FTPDIR=${FTPBASEDIR}/${1}/os/${2} -DBFILE=${FTPDIR}/${1}.db.tar.gz +FTPDIR=${FTPBASEDIR}/$reponame/os/$arch +DBFILE=${FTPDIR}/$reponame.db.tar.gz MISSINGFILES="" DELETEFILES="" -- cgit v1.2.2 From 382a2fd7c25dfa9769f1a945f692e938168a21b6 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Tue, 6 May 2008 18:21:54 -0400 Subject: Remove testing paths These snuck in in a previous commit Signed-off-by: Aaron Griffin --- db-update | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/db-update b/db-update index 6c9af1a..f9452b4 100755 --- a/db-update +++ b/db-update @@ -19,12 +19,10 @@ arch="$2" export CARCH="$arch" ##### Arch specific stuff. TODO make this configurable ##### -#ftppath="/home/ftp/$reponame/os/$arch/" -ftppath="/home/aaron/dbscripts/ftp-$reponame" +ftppath="/home/ftp/$reponame/os/$arch/" svnpath="file:///home/svn-packages" svnrepo="$reponame-$arch" -#stagedir="$HOME/staging/$reponame" -stagedir="/home/aaron/dbscripts/staging/$reponame" +stagedir="$HOME/staging/$reponame" ############################################################ [ "$UID" = "" ] && UID=$(uid) -- cgit v1.2.2 From 1bdf5e2109a24997ec5f786d12486718492df0ef Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Tue, 6 May 2008 18:22:50 -0400 Subject: Remove usage of the 64bit dirs from the staging path Packages can be differentiated now by $CARCH in the filename. No need for this extra dir. Also, using absolute paths for binaries Signed-off-by: Aaron Griffin --- db-update | 35 ++++++++++++++++++++++------------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/db-update b/db-update index f9452b4..c3b4593 100755 --- a/db-update +++ b/db-update @@ -80,27 +80,36 @@ fi trap ctrl_c 2 trap cleanup 0 -touch "$LOCKFILE" +/bin/touch "$LOCKFILE" -mkdir -p "$WORKDIR" +/bin/mkdir -p "$WORKDIR" + +echo "Updating DB for $reponame $arch" + +if [ -d "${stagedir}64" ]; then + echo "It looks like you have an old staging dir" + echo "Packages are now differentiated by the arch in the filename." + echo "Please delete '${stagedir}64'" + /bin/cp -r "${stagedir}64/" "$stagedir" +fi to_add="" if [ -d "$stagedir/add" ]; then - ADDPKGS="$(ls $stagedir{64,}/add/*-${arch}$PKGEXT 2>/dev/null)" + ADDPKGS="$(/bin/ls $stagedir/add/*-${arch}$PKGEXT 2>/dev/null)" fi if [ -n "$ADDPKGS" ]; then echo "==> Processing new/updated packages for repository '$reponame'..." >&2 - cd "$WORKDIR" - svn checkout -N $svnpath checkout - cd checkout + /bin/cd "$WORKDIR" + /usr/bin/svn checkout -N $svnpath checkout + /bin/cd checkout for pkg in $ADDPKGS; do _pkgfile=$(basename $pkg) _pkgname="$(getpkgname $pkg)" echo " Checking SVN for $_pkgname" - svn up -q $_pkgname + /usr/bin/svn up -q $_pkgname if [ -d "$_pkgname/repos/$svnrepo" ]; then . "$_pkgname/repos/$svnrepo/$BUILDSCRIPT" if [ "$_pkgfile" = "$pkgname-$pkgver-$pkgrel-$CARCH$PKGEXT" ]; then @@ -118,7 +127,7 @@ if [ -n "$ADDPKGS" ]; then # copy the db file into our working area [ -f "$ftppath/$reponame.db.tar.gz" ] && cp "$ftppath/$reponame.db.tar.gz" build/ - cp $to_add build/ + /bin/cp $to_add build/ cd build/ /usr/bin/repo-add "$reponame.db.tar.$DB_COMPRESSION" $to_add @@ -128,17 +137,17 @@ fi to_rem="" if [ -d "$stagedir/del" ]; then - REMPKGS="$(ls $stagedir{64,}/del/*-${arch}$PKGEXT 2>/dev/null)" + REMPKGS="$(/bin/ls $stagedir/del/*-${arch}$PKGEXT 2>/dev/null)" fi if [ -n "$REMPKGS" ]; then echo "==> Processing deleted packages for repository '$reponame'..." >&2 if [ ! -d "$WORKDIR/checkout" ]; then - cd "$WORKDIR" - svn checkout -N $svnpath checkout + /bin/cd "$WORKDIR" + /usr/bin/svn checkout -N $svnpath checkout fi - cd "$WORKDIR/checkout" + /bin/cd "$WORKDIR/checkout" #TODO removal shouldn't require a package file for pkg in $REMPKGS; do @@ -160,7 +169,7 @@ if [ -n "$REMPKGS" ]; then [ -f "$ftppath/$reponame.db.tar.gz" ] && cp "$ftppath/$reponame.db.tar.gz" build/ for rem in $to_rem; do if [ -f "build/$rem" ]; then - rm "build/$rem" + /bin/rm "build/$rem" fi done -- cgit v1.2.2 From 26b2f70e1d82f2a0e9a2127604a5bf610d37f404 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Tue, 6 May 2008 18:25:18 -0400 Subject: More absolute pathing Also, added a pretty comment. It's so pretty! Signed-off-by: Aaron Griffin --- db-update | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/db-update b/db-update index c3b4593..1134b2d 100755 --- a/db-update +++ b/db-update @@ -179,8 +179,9 @@ else echo "No packages to delete" fi -if [ $(ls "$WORKDIR/build/" 2>/dev/null | wc -l) != 0 ]; then - cp -r "$WORKDIR/build/"* "$ftppath" +# if non empty, move all build dirs +if [ $(/bin/ls "$WORKDIR/build/" 2>/dev/null | wc -l) != 0 ]; then + /bin/cp -r "$WORKDIR/build/"* "$ftppath" fi if [ "$updatelists" ]; then -- cgit v1.2.2 From 2d5b42fdd5ed2fa981c1122e040fd3974e79525e Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Tue, 6 May 2008 18:27:12 -0400 Subject: Remove genpkglists from the dbscripts This is silly and broken anyway. We can do this with a cron script later Signed-off-by: Aaron Griffin --- db-update | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/db-update b/db-update index 1134b2d..db96704 100755 --- a/db-update +++ b/db-update @@ -181,15 +181,9 @@ fi # if non empty, move all build dirs if [ $(/bin/ls "$WORKDIR/build/" 2>/dev/null | wc -l) != 0 ]; then + echo "Copying new files to '$ftppath'" /bin/cp -r "$WORKDIR/build/"* "$ftppath" -fi - -if [ "$updatelists" ]; then - echo "==> Generating Text Package List..." >&2 - cd "$WORKDIR/checkout" - $ARCHDIR/genpkglist $ftppath $svnrepo - if [ -f packages.txt ]; then - mv packages.txt $ftppath/packages.txt - fi +else + echo "Nothing to copy, no work done" fi # vim: set ts=4 sw=4 noet ft=sh: -- cgit v1.2.2 From e259c23bb739c3129e23e053cf1c5a846c6c3e76 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Tue, 6 May 2008 18:28:34 -0400 Subject: Fix file deletion on package removal Signed-off-by: Aaron Griffin --- db-update | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/db-update b/db-update index db96704..2dfda2a 100755 --- a/db-update +++ b/db-update @@ -167,14 +167,16 @@ if [ -n "$REMPKGS" ]; then # copy the db file into our working area [ -f "$ftppath/$reponame.db.tar.gz" ] && cp "$ftppath/$reponame.db.tar.gz" build/ - for rem in $to_rem; do - if [ -f "build/$rem" ]; then - /bin/rm "build/$rem" - fi - done cd build/ /usr/bin/repo-remove "$reponame.db.tar.$DB_COMPRESSION" $to_rem + + for rem in $to_rem; do + if [ -f "$ftppath/$rem" ]; then + /bin/rm "$ftppath/$rem" + /bin/rm "$rem" + fi + done else echo "No packages to delete" fi -- cgit v1.2.2 From 516aece0d26fc3b36e49c2739286f2be5cc0aa01 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Wed, 7 May 2008 01:18:31 -0400 Subject: Whoops, 'cd' is a built in - who knew? I sure didn't Signed-off-by: Aaron Griffin --- db-update | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/db-update b/db-update index 2dfda2a..833abca 100755 --- a/db-update +++ b/db-update @@ -101,9 +101,9 @@ fi if [ -n "$ADDPKGS" ]; then echo "==> Processing new/updated packages for repository '$reponame'..." >&2 - /bin/cd "$WORKDIR" + cd "$WORKDIR" /usr/bin/svn checkout -N $svnpath checkout - /bin/cd checkout + cd checkout for pkg in $ADDPKGS; do _pkgfile=$(basename $pkg) @@ -144,10 +144,10 @@ if [ -n "$REMPKGS" ]; then echo "==> Processing deleted packages for repository '$reponame'..." >&2 if [ ! -d "$WORKDIR/checkout" ]; then - /bin/cd "$WORKDIR" + cd "$WORKDIR" /usr/bin/svn checkout -N $svnpath checkout fi - /bin/cd "$WORKDIR/checkout" + cd "$WORKDIR/checkout" #TODO removal shouldn't require a package file for pkg in $REMPKGS; do -- cgit v1.2.2 From 054f2b100691c83ac9f99bfae0a823144c5460d9 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Wed, 7 May 2008 01:32:38 -0400 Subject: Make sure we copy old staging dir entries properly Signed-off-by: Aaron Griffin --- db-update | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db-update b/db-update index 833abca..791207d 100755 --- a/db-update +++ b/db-update @@ -90,7 +90,7 @@ if [ -d "${stagedir}64" ]; then echo "It looks like you have an old staging dir" echo "Packages are now differentiated by the arch in the filename." echo "Please delete '${stagedir}64'" - /bin/cp -r "${stagedir}64/" "$stagedir" + /bin/cp -r "${stagedir}64/"* "$stagedir/" fi to_add="" -- cgit v1.2.2 From aca168b4f23826d3dd5ad15d22ad9a564f69a40d Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Wed, 7 May 2008 01:34:19 -0400 Subject: Make sure we're actually adding/removing files We accidentally were calling repo-add/repo-remove with invalid (empty) params. Signed-off-by: Aaron Griffin --- db-update | 49 ++++++++++++++++++++++++++++++------------------- 1 file changed, 30 insertions(+), 19 deletions(-) diff --git a/db-update b/db-update index 791207d..bde0ee9 100755 --- a/db-update +++ b/db-update @@ -122,15 +122,20 @@ if [ -n "$ADDPKGS" ]; then fi done - cd "$WORKDIR" - [ -d build/ ] || mkdir build + if [ -n "$to_add" ]; then + + cd "$WORKDIR" + [ -d build/ ] || mkdir build - # copy the db file into our working area - [ -f "$ftppath/$reponame.db.tar.gz" ] && cp "$ftppath/$reponame.db.tar.gz" build/ - /bin/cp $to_add build/ + # copy the db file into our working area + [ -f "$ftppath/$reponame.db.tar.gz" ] && cp "$ftppath/$reponame.db.tar.gz" build/ + /bin/cp $to_add build/ - cd build/ - /usr/bin/repo-add "$reponame.db.tar.$DB_COMPRESSION" $to_add + cd build/ + /usr/bin/repo-add "$reponame.db.tar.$DB_COMPRESSION" $to_add + else + echo "Errors found when adding packages" + fi else echo "No packages to add" fi @@ -162,21 +167,25 @@ if [ -n "$REMPKGS" ]; then fi done - cd "$WORKDIR" - [ -d build/ ] || mkdir build + if [ -n "$to_rem" ]; then + cd "$WORKDIR" + [ -d build/ ] || mkdir build - # copy the db file into our working area - [ -f "$ftppath/$reponame.db.tar.gz" ] && cp "$ftppath/$reponame.db.tar.gz" build/ + # copy the db file into our working area + [ -f "$ftppath/$reponame.db.tar.gz" ] && cp "$ftppath/$reponame.db.tar.gz" build/ - cd build/ - /usr/bin/repo-remove "$reponame.db.tar.$DB_COMPRESSION" $to_rem + cd build/ + /usr/bin/repo-remove "$reponame.db.tar.$DB_COMPRESSION" $to_rem - for rem in $to_rem; do - if [ -f "$ftppath/$rem" ]; then - /bin/rm "$ftppath/$rem" - /bin/rm "$rem" - fi - done + for rem in $to_rem; do + if [ -f "$ftppath/$rem" ]; then + /bin/rm "$ftppath/$rem" + /bin/rm "$rem" + fi + done + else + echo "Errors found when removing packages" + fi else echo "No packages to delete" fi @@ -188,4 +197,6 @@ if [ $(/bin/ls "$WORKDIR/build/" 2>/dev/null | wc -l) != 0 ]; then else echo "Nothing to copy, no work done" fi + +cleanup # vim: set ts=4 sw=4 noet ft=sh: -- cgit v1.2.2 From 0cb6e7ab0840196c8b9c0dda64e50fa535b17707 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Wed, 7 May 2008 01:36:03 -0400 Subject: Fail loudly if ftp dir is missing This allows us to intuitively handle on-the-fly repos by calling `db-update foobar i686` for the foobar-i686 tag, and let us fail on the server side if the repo hasn't been approved/created by someone with admin rights. Signed-off-by: Aaron Griffin --- db-update | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/db-update b/db-update index bde0ee9..5d492fd 100755 --- a/db-update +++ b/db-update @@ -32,7 +32,13 @@ LOCKFILE="/tmp/.repolck.$arch.$reponame" ADDPKGS="" REMPKGS="" -if [ ! `type -p fakeroot` ]; then +if [ ! -d "$ftppath" ]; then + echo "FTP path for this repo ($reponame) is missing" + echo "Please contact a system administrator" + exit 1 +fi + +if [ ! $(type -p fakeroot) ]; then echo "error: fakeroot is missing" >&2 exit 1 fi -- cgit v1.2.2 From cbd6533816b1c27aab268b5c8f823345be8e12ed Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Wed, 7 May 2008 01:40:02 -0400 Subject: Properly clean up stating dirs after a success Remove all package file we added or removed Signed-off-by: Aaron Griffin --- db-update | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/db-update b/db-update index 5d492fd..451d206 100755 --- a/db-update +++ b/db-update @@ -186,7 +186,6 @@ if [ -n "$REMPKGS" ]; then for rem in $to_rem; do if [ -f "$ftppath/$rem" ]; then /bin/rm "$ftppath/$rem" - /bin/rm "$rem" fi done else @@ -200,9 +199,14 @@ fi if [ $(/bin/ls "$WORKDIR/build/" 2>/dev/null | wc -l) != 0 ]; then echo "Copying new files to '$ftppath'" /bin/cp -r "$WORKDIR/build/"* "$ftppath" + echo "Cleaning staging dir" + echo "/bin/rm $to_add $to_rem" + /bin/rm $to_add $to_rem else echo "Nothing to copy, no work done" fi + + cleanup # vim: set ts=4 sw=4 noet ft=sh: -- cgit v1.2.2 From 511759504993c858d977b649cd02cf7969811b57 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Wed, 7 May 2008 02:06:21 -0400 Subject: Moving some files around, organizational Signed-off-by: Aaron Griffin --- cleanup-scripts/cleanup2.sh | 46 -------- cron-jobs/cleanup2.sh | 46 ++++++++ cron-jobs/ftpdir-cleanup | 60 ++++++++++ cron-jobs/genpkglist | 74 ++++++++++++ ftpdir-cleanup | 60 ---------- genpkglist | 74 ------------ updatesync-many | 268 -------------------------------------------- 7 files changed, 180 insertions(+), 448 deletions(-) delete mode 100755 cleanup-scripts/cleanup2.sh create mode 100755 cron-jobs/cleanup2.sh create mode 100755 cron-jobs/ftpdir-cleanup create mode 100755 cron-jobs/genpkglist delete mode 100755 ftpdir-cleanup delete mode 100755 genpkglist delete mode 100755 updatesync-many diff --git a/cleanup-scripts/cleanup2.sh b/cleanup-scripts/cleanup2.sh deleted file mode 100755 index 4a79654..0000000 --- a/cleanup-scripts/cleanup2.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/bash - -usage() { - echo "Usage: $0 repo architecture" -} - -getpkgname() { - local tmp - - tmp=${1##*/} - tmp=${tmp%.pkg.tar.gz} - tmp=${tmp%-i686} - tmp=${tmp%-x86_64} - echo ${tmp%-*-*} -} - -FTPBASEDIR="/home/ftp" -FTPDIR=${FTPBASEDIR}/${1}/os/${2} -DBFILE=${FTPDIR}/${1}.db.tar.gz -OBSOLETEFILES="" - -if [ $# -lt 2 -o ! -f ${DBFILE} ]; then - usage - exit 1 -fi - -TMPDIR=$(mktemp -d /tmp/cleanup.XXXXXX) || exit 1 -cd ${TMPDIR} -tar xzf ${DBFILE} - -cd ${FTPDIR} -for pkgfile in *.pkg.tar.gz; do - pkgname="$(getpkgname ${pkgfile})" - for p in ${FTPDIR}/${pkgname}-*; do - if [ "$(getpkgname $(basename ${p}))" = "${pkgname}" ]; then - continue 2 - fi - done - OBSOLETEFILES="${OBSOLETEFILES} ${pkgfile}" -done - -cd - >/dev/null -rm -rf ${TMPDIR} - -echo -ne "DIRECTORY:\n${FTPDIR}\n\n" -echo -ne "OBSOLETEFILES:\n${OBSOLETEFILES}\n\n" diff --git a/cron-jobs/cleanup2.sh b/cron-jobs/cleanup2.sh new file mode 100755 index 0000000..4a79654 --- /dev/null +++ b/cron-jobs/cleanup2.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +usage() { + echo "Usage: $0 repo architecture" +} + +getpkgname() { + local tmp + + tmp=${1##*/} + tmp=${tmp%.pkg.tar.gz} + tmp=${tmp%-i686} + tmp=${tmp%-x86_64} + echo ${tmp%-*-*} +} + +FTPBASEDIR="/home/ftp" +FTPDIR=${FTPBASEDIR}/${1}/os/${2} +DBFILE=${FTPDIR}/${1}.db.tar.gz +OBSOLETEFILES="" + +if [ $# -lt 2 -o ! -f ${DBFILE} ]; then + usage + exit 1 +fi + +TMPDIR=$(mktemp -d /tmp/cleanup.XXXXXX) || exit 1 +cd ${TMPDIR} +tar xzf ${DBFILE} + +cd ${FTPDIR} +for pkgfile in *.pkg.tar.gz; do + pkgname="$(getpkgname ${pkgfile})" + for p in ${FTPDIR}/${pkgname}-*; do + if [ "$(getpkgname $(basename ${p}))" = "${pkgname}" ]; then + continue 2 + fi + done + OBSOLETEFILES="${OBSOLETEFILES} ${pkgfile}" +done + +cd - >/dev/null +rm -rf ${TMPDIR} + +echo -ne "DIRECTORY:\n${FTPDIR}\n\n" +echo -ne "OBSOLETEFILES:\n${OBSOLETEFILES}\n\n" diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup new file mode 100755 index 0000000..18e26f2 --- /dev/null +++ b/cron-jobs/ftpdir-cleanup @@ -0,0 +1,60 @@ +#!/bin/bash + +usage() { + echo "Usage: $0 repo architecture" +} + +getpkgname() { + local tmp + + tmp=${1##*/} + tmp=${tmp%.pkg.tar.gz} + tmp=${tmp%-i686} + tmp=${tmp%-x86_64} + echo ${tmp%-*-*} +} + +reponame=$1 +arch=$2 + +FTPBASEDIR="/home/ftp" +FTPDIR=${FTPBASEDIR}/$reponame/os/$arch +DBFILE=${FTPDIR}/$reponame.db.tar.gz +MISSINGFILES="" +DELETEFILES="" + +if [ $# -lt 2 -o ! -f ${DBFILE} ]; then + usage + exit 1 +fi + +TMPDIR=$(mktemp -d /tmp/cleanup.XXXXXX) || exit 1 + +cd ${TMPDIR} +tar xzf ${DBFILE} +for pkg in *; do + filename=$(grep -A1 '^%FILENAME%$' ${pkg}/desc | tail -n1) + [ -z "${filename}" ] && filename="${pkg}.pkg.tar.gz" + if [ ! -f ${FTPDIR}/${filename} ]; then + MISSINGFILES="${MISSINGFILES} ${filename}" + else + pkgname="$(getpkgname ${filename})" + for otherfile in ${FTPDIR}/${pkgname}-*; do + otherfile="$(basename ${otherfile})" + if [ "${otherfile}" != "${filename}" -a "${pkgname}" = "$(getpkgname ${otherfile})" ]; then + DELETEFILES="${DELETEFILES} ${otherfile}" + fi + done + fi +done + +cd - >/dev/null +rm -rf ${TMPDIR} + +echo -ne "DIRECTORY:\n${FTPDIR}\n\n" +echo -ne "DELETEFILES:\n${DELETEFILES}\n\n" +echo -ne "MISSINGFILES:\n${MISSINGFILES}\n\n" + +cd ${FTPDIR} +[ -n "${DELETEFILES}" ] && mv ${DELETEFILES} /home/package-cleanup/ +cd - >/dev/null diff --git a/cron-jobs/genpkglist b/cron-jobs/genpkglist new file mode 100755 index 0000000..31b7c78 --- /dev/null +++ b/cron-jobs/genpkglist @@ -0,0 +1,74 @@ +#!/bin/bash +# +# genpkglist +# +# Generates a text package database for use with the setup script +# (also used to check for missing packages in the download directory) +# + +# Get the package name from the filename +# hackish, but should work for now +getpkgname() { + local tmp + + tmp=${1##*/} + tmp=${tmp%.pkg.tar.gz} + tmp=${tmp%-i686} + tmp=${tmp%-x86_64} + echo ${tmp%-*-*} +} + +pkgfile="$(pwd)/packages.txt" +ftppath=$1 +repotag=$2 + +rm -f $pkgfile +for package in *; do + if [ -d "$package/$repotag/" ]; then + cd "$package/$repotag/" + if [ -f PKGBUILD ]; then + . PKGBUILD + if [ -f $ftppath/$pkgname-$pkgver-$pkgrel.pkg.tar.gz ]; then + echo "$pkgname-$pkgver-$pkgrel.pkg.tar.gz" >>$pkgfile + elif [ -f $ftppath/$pkgname-$pkgver-$pkgrel-$dbarch.pkg.tar.gz ]; then + echo "$pkgname-$pkgver-$pkgrel-$dbarch.pkg.tar.gz" >>$pkgfile + else + echo "notice: Missing $pkgname-$pkgver-$pkgrel-$dbarch.pkg.tar.gz in ftp site" >&2 + fi + fi + cd .. + fi +done + +cd $ftppath +unset DUPES DUPEFILES last +for pkg in *.pkg.tar.gz; do + pkgname=$(getpkgname $pkg) + if [ "$last" = "$pkgname" ]; then + DUPES="$DUPES $pkgname" + DUPEFILES="$DUPEFILES $pkg" + fi + last=$pkgname +done + +showdupes() { + done= + for i in *.pkg.tar.gz; do + pkgname=$(getpkgname $i) + if [ "$pkgname" = "$1" ]; then + ls -l $i | awk '{print $6" "$7" "$8" "$9}' + done=1 + else + [ "$done" = "1" ] && return + fi + done +} + +if [ "$DUPES" ]; then + DUPES=$((for d in $DUPES; do echo $d; done) | sort -u) + echo "Possible Dupes for $ftppath (remove old versions)" + echo "Date Filename" + for dupe in $DUPES; do + showdupes $dupe + done +fi diff --git a/ftpdir-cleanup b/ftpdir-cleanup deleted file mode 100755 index 18e26f2..0000000 --- a/ftpdir-cleanup +++ /dev/null @@ -1,60 +0,0 @@ -#!/bin/bash - -usage() { - echo "Usage: $0 repo architecture" -} - -getpkgname() { - local tmp - - tmp=${1##*/} - tmp=${tmp%.pkg.tar.gz} - tmp=${tmp%-i686} - tmp=${tmp%-x86_64} - echo ${tmp%-*-*} -} - -reponame=$1 -arch=$2 - -FTPBASEDIR="/home/ftp" -FTPDIR=${FTPBASEDIR}/$reponame/os/$arch -DBFILE=${FTPDIR}/$reponame.db.tar.gz -MISSINGFILES="" -DELETEFILES="" - -if [ $# -lt 2 -o ! -f ${DBFILE} ]; then - usage - exit 1 -fi - -TMPDIR=$(mktemp -d /tmp/cleanup.XXXXXX) || exit 1 - -cd ${TMPDIR} -tar xzf ${DBFILE} -for pkg in *; do - filename=$(grep -A1 '^%FILENAME%$' ${pkg}/desc | tail -n1) - [ -z "${filename}" ] && filename="${pkg}.pkg.tar.gz" - if [ ! -f ${FTPDIR}/${filename} ]; then - MISSINGFILES="${MISSINGFILES} ${filename}" - else - pkgname="$(getpkgname ${filename})" - for otherfile in ${FTPDIR}/${pkgname}-*; do - otherfile="$(basename ${otherfile})" - if [ "${otherfile}" != "${filename}" -a "${pkgname}" = "$(getpkgname ${otherfile})" ]; then - DELETEFILES="${DELETEFILES} ${otherfile}" - fi - done - fi -done - -cd - >/dev/null -rm -rf ${TMPDIR} - -echo -ne "DIRECTORY:\n${FTPDIR}\n\n" -echo -ne "DELETEFILES:\n${DELETEFILES}\n\n" -echo -ne "MISSINGFILES:\n${MISSINGFILES}\n\n" - -cd ${FTPDIR} -[ -n "${DELETEFILES}" ] && mv ${DELETEFILES} /home/package-cleanup/ -cd - >/dev/null diff --git a/genpkglist b/genpkglist deleted file mode 100755 index 31b7c78..0000000 --- a/genpkglist +++ /dev/null @@ -1,74 +0,0 @@ -#!/bin/bash -# -# genpkglist -# -# Generates a text package database for use with the setup script -# (also used to check for missing packages in the download directory) -# - -# Get the package name from the filename -# hackish, but should work for now -getpkgname() { - local tmp - - tmp=${1##*/} - tmp=${tmp%.pkg.tar.gz} - tmp=${tmp%-i686} - tmp=${tmp%-x86_64} - echo ${tmp%-*-*} -} - -pkgfile="$(pwd)/packages.txt" -ftppath=$1 -repotag=$2 - -rm -f $pkgfile -for package in *; do - if [ -d "$package/$repotag/" ]; then - cd "$package/$repotag/" - if [ -f PKGBUILD ]; then - . PKGBUILD - if [ -f $ftppath/$pkgname-$pkgver-$pkgrel.pkg.tar.gz ]; then - echo "$pkgname-$pkgver-$pkgrel.pkg.tar.gz" >>$pkgfile - elif [ -f $ftppath/$pkgname-$pkgver-$pkgrel-$dbarch.pkg.tar.gz ]; then - echo "$pkgname-$pkgver-$pkgrel-$dbarch.pkg.tar.gz" >>$pkgfile - else - echo "notice: Missing $pkgname-$pkgver-$pkgrel-$dbarch.pkg.tar.gz in ftp site" >&2 - fi - fi - cd .. - fi -done - -cd $ftppath -unset DUPES DUPEFILES last -for pkg in *.pkg.tar.gz; do - pkgname=$(getpkgname $pkg) - if [ "$last" = "$pkgname" ]; then - DUPES="$DUPES $pkgname" - DUPEFILES="$DUPEFILES $pkg" - fi - last=$pkgname -done - -showdupes() { - done= - for i in *.pkg.tar.gz; do - pkgname=$(getpkgname $i) - if [ "$pkgname" = "$1" ]; then - ls -l $i | awk '{print $6" "$7" "$8" "$9}' - done=1 - else - [ "$done" = "1" ] && return - fi - done -} - -if [ "$DUPES" ]; then - DUPES=$((for d in $DUPES; do echo $d; done) | sort -u) - echo "Possible Dupes for $ftppath (remove old versions)" - echo "Date Filename" - for dupe in $DUPES; do - showdupes $dupe - done -fi diff --git a/updatesync-many b/updatesync-many deleted file mode 100755 index 8060585..0000000 --- a/updatesync-many +++ /dev/null @@ -1,268 +0,0 @@ -#!/bin/bash -# -# updatesync-many -# -# Copyright (c) 2004-2006 by Jason Chu and Judd Vinet -# Contact: and -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, -# USA. -# - -usage() { - echo "updatesync-many" - echo "usage: $0 " - echo - echo "This should probably only be run from the Arch db-generation scripts" - echo - echo "Caveats:" - echo " - Make sure you run it from the staging directory" - echo " - Use absolute pathnames for dbfile and svn_checkout" - echo - exit 0 -} - -die() -{ - echo "updatesync-many: $*" >&2 - rm -rf $TMPDIR - exit 1 -} - -msg() -{ - echo "updatesync-many: $*" >&2 -} - -# Get the package name from the filename -# hackish, but should work for now -getpkgname() { - local tmp - - tmp=${1##*/} - tmp=${tmp%.pkg.tar.gz} - tmp=${tmp%-i686} - tmp=${tmp%-x86_64} - echo ${tmp%-*-*} -} - -check_option() { - local i - for i in ${options[@]}; do - local uc=`echo $i | tr [:lower:] [:upper:]` - local lc=`echo $i | tr [:upper:] [:lower:]` - if [ "$uc" = "$1" -o "$lc" = "$1" ]; then - echo $1 - return - fi - done -} - -get_md5checksum() -{ - md5line=`md5sum $1` - [ ! -z "$md5line" ] && pkgmd5sum=${md5line% *} - echo $pkgmd5sum -} - -db_write_entry() -{ - unset pkgname pkgver pkgrel pkgdesc license groups provides md5sums force - unset replaces depends conflicts backup source install build makedepends - unset options - source $1 || return 1 - cd $TMPDIR - mkdir $pkgname-$pkgver-$pkgrel || return 1 - cd $pkgname-$pkgver-$pkgrel - # desc - : >desc - echo "%FILENAME%" >>desc - echo "$2" >>desc - echo "" >>desc - echo "%NAME%" >>desc - echo "$pkgname" >>desc - echo "" >>desc - echo "%VERSION%" >>desc - echo "$pkgver-$pkgrel" >>desc - echo "" >>desc - echo "%DESC%" >>desc - echo "$pkgdesc" >>desc - echo "" >>desc - echo "%CSIZE%" >>desc - echo "$csize" >>desc - echo "" >>desc - if [ ! -z $pkgmd5sum ]; then - echo "%MD5SUM%" >>desc - echo "$pkgmd5sum" >>desc - echo "" >>desc - fi - if [ ${#groups[*]} -gt 0 ]; then - echo "%GROUPS%" >>desc - for it in "${groups[@]}"; do - echo "$it" >>desc - done - echo "" >>desc - fi - if [ ${#replaces[*]} -gt 0 ]; then - echo "%REPLACES%" >>desc - for it in "${replaces[@]}"; do - echo "$it" >>desc - done - echo "" >>desc - fi - if [ "$force" = "y" -o "$force" = "Y" -o "`check_option FORCE`" ]; then - echo "%FORCE%" >>desc - echo "" >>desc - fi - # depends - : >depends - if [ ${#depends[*]} -gt 0 ]; then - echo "%DEPENDS%" >>depends - for it in "${depends[@]}"; do - echo "$it" >>depends - done - echo "" >>depends - fi - if [ ${#conflicts[*]} -gt 0 ]; then - echo "%CONFLICTS%" >>depends - for it in "${conflicts[@]}"; do - echo "$it" >>depends - done - echo "" >>depends - fi - if [ ${#provides[*]} -gt 0 ]; then - echo "%PROVIDES%" >>depends - for it in "${provides[@]}"; do - echo "$it" >>depends - done - echo "" >>depends - fi -} - -delete_entry() -{ - # grab the pkgname - pkgname=$(getpkgname $1) - for i in *; do - if [ "${i%-*-*}" = "$pkgname" ]; then - rm -rf $i - fi - done -} - -update_entry() -{ - pkgfile=$1 - pkgname=$(getpkgname ${pkgfile}) - fullname=$(basename ${pkgfile}) - pkgpath="$SVNCO/$pkgname/repos/$REPOTAG" - - # find the matching PKGBUILD - if [ ! -d "$pkgpath" ]; then - msg "WARNING: could not find PKGBUILD for $pkgname, cannot update this entry" - return - fi - pkgbuild="${pkgpath}/PKGBUILD" - if [ ! -f $pkgbuild ]; then - msg "WARNING: could not find PKGBUILD for $fullname, cannot update this entry" - return - fi - - source $pkgbuild - if [ $? -ne 0 ]; then - msg "WARNING: PKGBUILD for $fullname has errors, cannot update this entry" - return - fi - - # all good so far - delete the old entry - cd $TMPDIR - delete_entry $pkgfile - - csize=`du -b $pkgfile | cut -f1` - pkgmd5sum=`get_md5checksum $pkgfile` - [ -z $pkgmd5sum ] && die "error generating checksum for $pkgfile" - - db_write_entry ${pkgbuild} ${fullname} || die "error writing entry for $pkgname" - cd - >/dev/null -} - -if [ ! "`type -p lsof`" ]; then - echo "ERROR: lsof is needed to run updatesync-many!" - exit 1 -fi - -if [ $# -lt 3 ]; then - usage - exit 1 -fi - -if [ "$1" = "-h" -o "$1" = "--help" ]; then - usage - exit 0 -fi - -ACTION=$1 -PKGDB=$2 -SVNCO=$3 -REPOTAG=$4 -STAGEDIR="`pwd`" -PKGDIR="`dirname $PKGDB`" -if [ "$PKGDIR" = "." ]; then - PKGDIR=$STAGEDIR -fi - -if [ "$ACTION" != "add" -a "$ACTION" != "del" ]; then - usage - exit 1 -fi - -# Prepare the sync db for modifications -TMPDIR=$(mktemp -d /tmp/updatesync-many.XXXXXXXXXX) || exit 1 -cd $TMPDIR -if [ ! -f $PKGDB ]; then - die "$PKGDB not found" -fi -msg "Unpacking db: $PKGDB" -tar zxf $PKGDB || die "error unpacking $PKGDB" - -# Process packages in the staging directory -for pkgfile in $STAGEDIR/*.pkg.tar.gz; do - # Make sure this file isn't currently in use by any processes... - # This is our cheap way of (mostly) making sure the file isn't being - # uploaded at this very time (and thus incomplete). - # Of course, if an upload failed and the scp connection terminated, then - # this check will fail us. - lsof $pkgfile &>/dev/null - [ $? -ne 1 ] && continue - - pkgname=$(getpkgname ${pkgfile}) - if [ "$ACTION" = "del" ]; then - msg "Deleting entry: $pkgname" - delete_entry $pkgfile - else - msg "Updating entry: $pkgname" - update_entry $pkgfile - fi -done - -# Repackage the DB -msg "Repacking db: $PKGDB" -cd $TMPDIR -tar c * | gzip -9 >$PKGDB || die "error writing to $PKGDB" - -cd / -rm -rf $TMPDIR - -exit 0 -- cgit v1.2.2 From ebff7202e4c1c4d7e5f208659e9878082df73ced Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Wed, 7 May 2008 04:19:47 -0400 Subject: Cleanup some debugging output Leftovers - accidental commit Signed-off-by: Aaron Griffin --- db-update | 3 --- 1 file changed, 3 deletions(-) diff --git a/db-update b/db-update index 451d206..e88ab40 100755 --- a/db-update +++ b/db-update @@ -200,13 +200,10 @@ if [ $(/bin/ls "$WORKDIR/build/" 2>/dev/null | wc -l) != 0 ]; then echo "Copying new files to '$ftppath'" /bin/cp -r "$WORKDIR/build/"* "$ftppath" echo "Cleaning staging dir" - echo "/bin/rm $to_add $to_rem" /bin/rm $to_add $to_rem else echo "Nothing to copy, no work done" fi - - cleanup # vim: set ts=4 sw=4 noet ft=sh: -- cgit v1.2.2 From 39256c685d1aee1e92e1b8f4595fa15d6ba94a23 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Wed, 14 May 2008 13:23:00 -0400 Subject: Remove a check for fakeroot We don't use fakeroot anymore in these scripts Signed-off-by: Aaron Griffin --- db-update | 5 ----- 1 file changed, 5 deletions(-) diff --git a/db-update b/db-update index e88ab40..e616aba 100755 --- a/db-update +++ b/db-update @@ -38,11 +38,6 @@ if [ ! -d "$ftppath" ]; then exit 1 fi -if [ ! $(type -p fakeroot) ]; then - echo "error: fakeroot is missing" >&2 - exit 1 -fi - if [ ! -d $stagedir ]; then echo "error: staging directory missing: $stagedir" >&2 exit 1 -- cgit v1.2.2 From 4a928ca7a0ed02f7be01655831eeac9158bacabe Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Wed, 14 May 2008 13:23:25 -0400 Subject: Make sure to clean up working temp dirs Left in a comment while debugging Signed-off-by: Aaron Griffin --- db-update | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db-update b/db-update index e616aba..2e0e632 100755 --- a/db-update +++ b/db-update @@ -57,7 +57,7 @@ getpkgname() { cleanup() { # unlock rm -f "$LOCKFILE" - #rm -rf "$WORKDIR" + rm -rf "$WORKDIR" [ "$1" ] && exit $1 } -- cgit v1.2.2 From 01a1eedbdb93a0c7312af1543658997ae7aa680c Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Wed, 14 May 2008 15:07:37 -0400 Subject: Add new db-remove script This script deletes by package NAME only. Additionally, it also removes from svn. The goal is to simplify the process of removing packages. Signed-off-by: Aaron Griffin --- db-remove | 97 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100755 db-remove diff --git a/db-remove b/db-remove new file mode 100755 index 0000000..c69e179 --- /dev/null +++ b/db-remove @@ -0,0 +1,97 @@ +#!/bin/bash + +if [ $# -ne 3 ]; then + echo "usage: $(basename $0) " + exit 1 +fi + +if [ -f "/etc/makepkg.conf" ]; then + #Get some config info + . /etc/makepkg.conf +else + echo "/etc/makepkg.conf does not exist!" + exit 1 +fi + +packagename="$1" +reponame="$2" +arch="$3" + +export CARCH="$arch" + +##### Arch specific stuff. TODO make this configurable ##### +ftppath="/home/ftp/$reponame/os/$arch/" +svnpath="file:///home/svn-packages" +svnrepo="$reponame-$arch" +############################################################ + +[ "$UID" = "" ] && UID=$(uid) + +WORKDIR="/tmp/db-remove.$svnrepo.$UID" +LOCKFILE="/tmp/.repolck.$arch.$reponame" + +cleanup() { + # unlock + rm -f "$LOCKFILE" + rm -rf "$WORKDIR" + [ "$1" ] && exit $1 +} + +ctrl_c() { + echo "Interrupted" >&2 + cleanup 0 +} + +die() { + echo "$*" >&2 + cleanup 1 +} + +# check for locks +if [ -f "$LOCKFILE" ]; then + owner="$(/usr/bin/stat -c %U $LOCKFILE)" + echo "error: db generation is already in progress (started by $owner)" + exit 1 +fi + +trap ctrl_c 2 +trap cleanup 0 + +/bin/touch "$LOCKFILE" +/bin/mkdir -p "$WORKDIR" + + +echo "==> Removing package '$packagename' from '$reponame'..." >&2 + +cd "$WORKDIR" +/usr/bin/svn checkout -N $svnpath checkout +cd checkout + +/usr/bin/svn up -q $packagename +if [ -d "$packagename/repos/$svnrepo" ]; then + echo " Removing from subversion" + /usr/bin/svn rm --force -q "$packagename/repos/$svnrepo" + /usr/bin/svn commit -q -m "$(basename $0): $packagename removed by $(id -un)" +fi + +cd "$WORKDIR" +[ -d build/ ] || mkdir build + +# copy the db file into our working area +if [ -f "$ftppath/$reponame.db.tar.$DB_COMPRESSION" ]; then + cp "$ftppath/$reponame.db.tar.$DB_COMPRESSION" build/ +else + echo "No database found at '$ftpdir', nothing more to do" + exit 0 +fi + +echo " Removing from $reponame DB file" +cd build/ +/usr/bin/repo-remove "$reponame.db.tar.$DB_COMPRESSION" $packagename + +cp "$reponame.db.tar.$DB_COMPRESSION" "$ftppath/" + +echo "Package files will be cleaned up automatically" + +cleanup +# vim: set ts=4 sw=4 noet ft=sh: -- cgit v1.2.2 From e021dfb2c476c2841606e63e6ae58dfc45cd1b47 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Wed, 14 May 2008 15:09:04 -0400 Subject: Make the staging warning stand out more Signed-off-by: Aaron Griffin --- db-update | 2 ++ 1 file changed, 2 insertions(+) diff --git a/db-update b/db-update index 2e0e632..9d0ecad 100755 --- a/db-update +++ b/db-update @@ -88,9 +88,11 @@ trap cleanup 0 echo "Updating DB for $reponame $arch" if [ -d "${stagedir}64" ]; then + echo "--------------------------------------------------" echo "It looks like you have an old staging dir" echo "Packages are now differentiated by the arch in the filename." echo "Please delete '${stagedir}64'" + echo "--------------------------------------------------" /bin/cp -r "${stagedir}64/"* "$stagedir/" fi -- cgit v1.2.2 From 39298cf09e2e88b1f6aec779170990675baf8992 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Wed, 14 May 2008 18:49:20 -0400 Subject: Cron cleanup, and only copy DB files once This would break all adds if there were any deletes. We definitely don't want that. Also, cleanup of cron scripts in the same commit because I'm lazy Signed-off-by: Aaron Griffin --- cron-jobs/genpkglist | 86 ++++++++++++------------------------------------- db-update | 50 +++++++++++++++------------- misc-scripts/find-dupes | 77 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 125 insertions(+), 88 deletions(-) create mode 100755 misc-scripts/find-dupes diff --git a/cron-jobs/genpkglist b/cron-jobs/genpkglist index 31b7c78..0b5d76f 100755 --- a/cron-jobs/genpkglist +++ b/cron-jobs/genpkglist @@ -1,74 +1,28 @@ #!/bin/bash -# -# genpkglist -# -# Generates a text package database for use with the setup script -# (also used to check for missing packages in the download directory) -# -# Get the package name from the filename -# hackish, but should work for now -getpkgname() { - local tmp +if [ $# -ne 2 ]; then + echo "usage: $(basename $0) " + exit 1 +fi - tmp=${1##*/} - tmp=${tmp%.pkg.tar.gz} - tmp=${tmp%-i686} - tmp=${tmp%-x86_64} - echo ${tmp%-*-*} -} +reponame=$1 +arch=$2 -pkgfile="$(pwd)/packages.txt" -ftppath=$1 -repotag=$2 +##### Arch specific stuff. TODO make this configurable ##### +ftppath="/home/ftp/$reponame/os/$arch/" +############################################################ -rm -f $pkgfile -for package in *; do - if [ -d "$package/$repotag/" ]; then - cd "$package/$repotag/" - if [ -f PKGBUILD ]; then - . PKGBUILD - if [ -f $ftppath/$pkgname-$pkgver-$pkgrel.pkg.tar.gz ]; then - echo "$pkgname-$pkgver-$pkgrel.pkg.tar.gz" >>$pkgfile - elif [ -f $ftppath/$pkgname-$pkgver-$pkgrel-$dbarch.pkg.tar.gz ]; then - echo "$pkgname-$pkgver-$pkgrel-$dbarch.pkg.tar.gz" >>$pkgfile - else - echo "notice: Missing $pkgname-$pkgver-$pkgrel-$dbarch.pkg.tar.gz in ftp site" >&2 - fi - fi - cd .. - fi -done +if [ ! -d "$ftppath" ]; then + echo "FTP path '$ftppath' does not exist" + exit 1 +fi -cd $ftppath -unset DUPES DUPEFILES last -for pkg in *.pkg.tar.gz; do - pkgname=$(getpkgname $pkg) - if [ "$last" = "$pkgname" ]; then - DUPES="$DUPES $pkgname" - DUPEFILES="$DUPEFILES $pkg" - fi - last=$pkgname -done +if [ ! -f /etc/makepkg.conf ]; then + echo "/etc/makepkg.conf not found! Aborting" + exit 1 +fi -showdupes() { - done= - for i in *.pkg.tar.gz; do - pkgname=$(getpkgname $i) - if [ "$pkgname" = "$1" ]; then - ls -l $i | awk '{print $6" "$7" "$8" "$9}' - done=1 - else - [ "$done" = "1" ] && return - fi - done -} +. /etc/makepkg.conf -if [ "$DUPES" ]; then - DUPES=$((for d in $DUPES; do echo $d; done) | sort -u) - echo "Possible Dupes for $ftppath (remove old versions)" - echo "Date Filename" - for dupe in $DUPES; do - showdupes $dupe - done -fi +cd "$ftppath" +ls -1 *$PKGEXT diff --git a/db-update b/db-update index 9d0ecad..93672f5 100755 --- a/db-update +++ b/db-update @@ -83,7 +83,15 @@ trap cleanup 0 /bin/touch "$LOCKFILE" -/bin/mkdir -p "$WORKDIR" +/bin/mkdir -p "$WORKDIR/build" +cd "$WORKDIR" + +# copy the db file into our working area +if [ -f "$ftppath/$reponame.db.tar.$DB_COMPRESSION" ]; then + cp "$ftppath/$reponame.db.tar.$DB_COMPRESSION" build/ +else + touch "build/$reponame.db.tar.$DB_COMPRESSION" +fi echo "Updating DB for $reponame $arch" @@ -126,17 +134,15 @@ if [ -n "$ADDPKGS" ]; then done if [ -n "$to_add" ]; then + cd "$WORKDIR/build/" + /bin/cp $to_add . - cd "$WORKDIR" - [ -d build/ ] || mkdir build + pkgs="" + for pkg in $to_add; do pkgs="$pkgs $(basename $pkg)"; done - # copy the db file into our working area - [ -f "$ftppath/$reponame.db.tar.gz" ] && cp "$ftppath/$reponame.db.tar.gz" build/ - /bin/cp $to_add build/ - - cd build/ - /usr/bin/repo-add "$reponame.db.tar.$DB_COMPRESSION" $to_add + /usr/bin/repo-add "$reponame.db.tar.$DB_COMPRESSION" $pkgs else + rm -f "build/$reponame.db.tar.$DB_COMPRESSION" echo "Errors found when adding packages" fi else @@ -166,26 +172,22 @@ if [ -n "$REMPKGS" ]; then if [ -d "$_pkgname/repos/$svnrepo" ]; then echo " WARNING: $_pkgname still exists in $svnrepo" else - to_rem="$to_rem $pkg" + to_rem="$to_rem $_pkgname" fi done if [ -n "$to_rem" ]; then - cd "$WORKDIR" - [ -d build/ ] || mkdir build - - # copy the db file into our working area - [ -f "$ftppath/$reponame.db.tar.gz" ] && cp "$ftppath/$reponame.db.tar.gz" build/ + cd "$WORKDIR/build/" - cd build/ + #NOTE: to_rem consists of package NAMES only /usr/bin/repo-remove "$reponame.db.tar.$DB_COMPRESSION" $to_rem - for rem in $to_rem; do - if [ -f "$ftppath/$rem" ]; then - /bin/rm "$ftppath/$rem" - fi + for rem in $REMPKGS; do + rem="$(basename $rem)" + /bin/rm -f "$ftppath/$rem" done else + rm -f "build/$reponame.db.tar.$DB_COMPRESSION" echo "Errors found when removing packages" fi else @@ -196,11 +198,15 @@ fi if [ $(/bin/ls "$WORKDIR/build/" 2>/dev/null | wc -l) != 0 ]; then echo "Copying new files to '$ftppath'" /bin/cp -r "$WORKDIR/build/"* "$ftppath" - echo "Cleaning staging dir" - /bin/rm $to_add $to_rem + else echo "Nothing to copy, no work done" fi +if [ -n "$ADDPKGS" -o -n "$REMPKGS" ]; then + echo "Cleaning staging dir" + /bin/rm $ADDPKGS $REMPKGS +fi + cleanup # vim: set ts=4 sw=4 noet ft=sh: diff --git a/misc-scripts/find-dupes b/misc-scripts/find-dupes new file mode 100755 index 0000000..82cd1a4 --- /dev/null +++ b/misc-scripts/find-dupes @@ -0,0 +1,77 @@ +#!/bin/bash + +if [ $# -ne 2 ]; then + echo "usage: $(basename $0) " + exit 1 +fi + +reponame=$1 +arch=$2 + +##### Arch specific stuff. TODO make this configurable ##### +ftppath="/home/ftp/$reponame/os/$arch/" +############################################################ + +if [ ! -d "$ftppath" ]; then + echo "FTP path '$ftppath' does not exist" + exit 1 +fi + +if [ ! -f /etc/makepkg.conf ]; then + echo "/etc/makepkg.conf not found! Aborting" + exit 1 +fi + +. /etc/makepkg.conf + +cd $ftppath + +# Get the package name from the filename +# hackish, but should work for now +getpkgname() { + local tmp + + tmp=${1##*/} + tmp=${tmp%$PKGEXT} + tmp=${tmp%-$CARCH} + echo ${tmp%-*-*} +} + +showdupes() { + done="" + for i in *.pkg.tar.gz; do + pkgname=$(getpkgname $i) + if [ "$pkgname" = "$1" ]; then + lastmod="$(stat -c %y $i | cut -d. -f1)" + fname="$(basename $i)" + + echo "$lastmod $fname" + + done=1 + else + if [ "$done" = "1" ]; then + return + fi + fi + done +} + +echo "Scanning for duplicate packages in '$reponame' ($arch)" +DUPES="" +lastpkg="" + +for pkg in *.pkg.tar.gz; do + pkgname="$(getpkgname $pkg)" + if [ "$lastpkg" = "$pkgname" ]; then + DUPES="$DUPES $pkgname" + fi + lastpkg=$pkgname +done + +if [ "$DUPES" ]; then + DUPES="$(echo $DUPES | sed 's| |\n|g' | sort -u)" + echo "Date Filename" + for dupe in $DUPES; do + showdupes $dupe + done +fi -- cgit v1.2.2 From c544d99f907014a5f40356d79b9fb518b559372d Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Thu, 15 May 2008 12:05:36 -0400 Subject: More cron job cleanup and fixes Signed-off-by: Aaron Griffin --- cron-jobs/cleanup2.sh | 46 ------------------- cron-jobs/createFileLists | 79 +++++++++++++++++++++++++++++++++ cron-jobs/ftpdir-cleanup | 111 +++++++++++++++++++++++++++++++++------------- 3 files changed, 159 insertions(+), 77 deletions(-) delete mode 100755 cron-jobs/cleanup2.sh create mode 100644 cron-jobs/createFileLists diff --git a/cron-jobs/cleanup2.sh b/cron-jobs/cleanup2.sh deleted file mode 100755 index 4a79654..0000000 --- a/cron-jobs/cleanup2.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/bash - -usage() { - echo "Usage: $0 repo architecture" -} - -getpkgname() { - local tmp - - tmp=${1##*/} - tmp=${tmp%.pkg.tar.gz} - tmp=${tmp%-i686} - tmp=${tmp%-x86_64} - echo ${tmp%-*-*} -} - -FTPBASEDIR="/home/ftp" -FTPDIR=${FTPBASEDIR}/${1}/os/${2} -DBFILE=${FTPDIR}/${1}.db.tar.gz -OBSOLETEFILES="" - -if [ $# -lt 2 -o ! -f ${DBFILE} ]; then - usage - exit 1 -fi - -TMPDIR=$(mktemp -d /tmp/cleanup.XXXXXX) || exit 1 -cd ${TMPDIR} -tar xzf ${DBFILE} - -cd ${FTPDIR} -for pkgfile in *.pkg.tar.gz; do - pkgname="$(getpkgname ${pkgfile})" - for p in ${FTPDIR}/${pkgname}-*; do - if [ "$(getpkgname $(basename ${p}))" = "${pkgname}" ]; then - continue 2 - fi - done - OBSOLETEFILES="${OBSOLETEFILES} ${pkgfile}" -done - -cd - >/dev/null -rm -rf ${TMPDIR} - -echo -ne "DIRECTORY:\n${FTPDIR}\n\n" -echo -ne "OBSOLETEFILES:\n${OBSOLETEFILES}\n\n" diff --git a/cron-jobs/createFileLists b/cron-jobs/createFileLists new file mode 100644 index 0000000..02af296 --- /dev/null +++ b/cron-jobs/createFileLists @@ -0,0 +1,79 @@ +#!/bin/bash + +reposdir=/home/ftp/ +targetdir=/home/pierre/public_html/test-repo/ +repos="core extra unstable testing community" +arches="i686 x86_64" + +if [ -f "/tmp/createFileList.lock" ]; then + echo "Error: createFileList allready in progress." + exit 1 +fi + +touch "/tmp/createFileList.lock" || exit 1 +TMPDIR=$(mktemp -d /tmp/createFileList.XXXXXX) || exit 1 +CACHEDIR=$(mktemp -d /tmp/createFileList.XXXXXX) || exit 1 + +getpkgname() { + local tmp + + tmp=${1##*/} + tmp=${tmp%.pkg.tar.gz} + tmp=${tmp%-i686} + tmp=${tmp%-x86_64} + tmp=${tmp%-any} + echo $tmp +} + +cd $reposdir +for repo in $repos; do + REPO_DB_FILE=${repo}.files.tar.gz + for arch in $arches; do + repodir=${repo}/os/${arch}/ + cached="no" + + # extract old file archive + if [ -f ${targetdir}${repodir}${REPO_DB_FILE} ]; then + mkdir -p ${CACHEDIR}/${repodir} + bsdtar -xf ${targetdir}${repodir}${REPO_DB_FILE} -C ${CACHEDIR}/${repodir} + cached="yes" + fi + + # create file lists + for pkg in $repodir*.pkg.tar.gz; do + basename=$(basename $pkg) + pkgname=$(getpkgname $basename) + tmppkgdir=${TMPDIR}/${repodir}${pkgname}/ + mkdir -p $tmppkgdir + if [ -f "${CACHEDIR}/${repodir}${pkgname}/files" ]; then +# echo "cache: $pkgname" + mv ${CACHEDIR}/${repodir}${pkgname}/files ${tmppkgdir}files + else +# echo "$repo/$arch: $pkgname" + echo '%FILES%' > ${tmppkgdir}files + bsdtar --exclude=.* -tf $pkg >> ${tmppkgdir}files + cached="no" + fi + done + + # create new file archive + if [ "$cached" == "no" ]; then + # at least one package has changed, so let's rebuild the archive +# echo "creating ${REPO_DB_FILE}/${arch}" + pkgdir=${targetdir}${repodir} + mkdir -p $pkgdir + cd ${TMPDIR}/${repodir} + [ -f "${pkgdir}${REPO_DB_FILE}.old" ] && rm "${pkgdir}${REPO_DB_FILE}.old" + [ -f "${pkgdir}${REPO_DB_FILE}" ] && mv "${pkgdir}${REPO_DB_FILE}" "${pkgdir}${REPO_DB_FILE}.old" + bsdtar --exclude=*.tar.gz -czf ${pkgdir}${REPO_DB_FILE} * + fi + + cd $reposdir + done +done + +cd - >/dev/null +rm -rf $TMPDIR || exit 1 +rm -rf $CACHEDIR || exit 1 +rm -f "/tmp/createFileList.lock" || exit 1 +# echo 'done' diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index 18e26f2..83c7cf1 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -1,45 +1,64 @@ #!/bin/bash -usage() { - echo "Usage: $0 repo architecture" -} +if [ $# -ne 2 ]; then + echo "usage: $(basename $0) " + exit 1 +fi + +reponame=$1 +arch=$2 + +##### Arch specific stuff. TODO make this configurable ##### +ftppath="/home/ftp/$reponame/os/$arch/" +############################################################ + +if [ ! -d "$ftppath" ]; then + echo "FTP path '$ftppath' does not exist" + exit 1 +fi + +if [ ! -f /etc/makepkg.conf ]; then + echo "/etc/makepkg.conf not found! Aborting" + exit 1 +fi + +. /etc/makepkg.conf getpkgname() { - local tmp + local tmp - tmp=${1##*/} - tmp=${tmp%.pkg.tar.gz} - tmp=${tmp%-i686} - tmp=${tmp%-x86_64} - echo ${tmp%-*-*} + tmp=${1##*/} + tmp=${tmp%$PKGEXT} + tmp=${tmp%-$arch} + echo ${tmp%-*-*} } -reponame=$1 -arch=$2 +getpkgname_ver() { + local tmp + + tmp=${1##*/} + tmp=${tmp%$PKGEXT} + echo ${tmp%-$arch} +} -FTPBASEDIR="/home/ftp" -FTPDIR=${FTPBASEDIR}/$reponame/os/$arch -DBFILE=${FTPDIR}/$reponame.db.tar.gz MISSINGFILES="" DELETEFILES="" - -if [ $# -lt 2 -o ! -f ${DBFILE} ]; then - usage - exit 1 -fi +EXTRAFILES="" TMPDIR=$(mktemp -d /tmp/cleanup.XXXXXX) || exit 1 -cd ${TMPDIR} -tar xzf ${DBFILE} +cd "${TMPDIR}" +/bin/tar xzf "$ftppath/$reponame.db.tar.$DB_COMPRESSION" + for pkg in *; do - filename=$(grep -A1 '^%FILENAME%$' ${pkg}/desc | tail -n1) + filename=$(grep -A1 '^%FILENAME%$' "${pkg}/desc" | tail -n1) [ -z "${filename}" ] && filename="${pkg}.pkg.tar.gz" - if [ ! -f ${FTPDIR}/${filename} ]; then + + if [ ! -f "${ftppath}/${filename}" ]; then MISSINGFILES="${MISSINGFILES} ${filename}" else pkgname="$(getpkgname ${filename})" - for otherfile in ${FTPDIR}/${pkgname}-*; do + for otherfile in ${ftppath}/${pkgname}-*; do otherfile="$(basename ${otherfile})" if [ "${otherfile}" != "${filename}" -a "${pkgname}" = "$(getpkgname ${otherfile})" ]; then DELETEFILES="${DELETEFILES} ${otherfile}" @@ -48,13 +67,43 @@ for pkg in *; do fi done -cd - >/dev/null +cd "$ftppath" +for pkg in *$PKGEXT; do + pkgname="$(getpkgname_ver ${filename})" + if [ ! -d "$TMPDIR/$pkgname" ]; then + EXTRAFILES="$EXTRAFILES $pkg" + fi +done + + + +cd "$ftppath" rm -rf ${TMPDIR} -echo -ne "DIRECTORY:\n${FTPDIR}\n\n" -echo -ne "DELETEFILES:\n${DELETEFILES}\n\n" -echo -ne "MISSINGFILES:\n${MISSINGFILES}\n\n" +if [ -n "${DELETEFILES}" ]; then + #rm -f ${DELETEFILES} + #mv ${DELETEFILES} /home/package-cleanup/ + echo "" +fi -cd ${FTPDIR} -[ -n "${DELETEFILES}" ] && mv ${DELETEFILES} /home/package-cleanup/ -cd - >/dev/null +echo "Scan complete for $reponame ($arch) at ${ftppath}" +if [ -n "$DELETEFILES" ]; then + echo " The following files have been moved to package-cleanup:" + for f in $DELETEFILES; do + echo " $f" + done +fi +echo "" +if [ -n "$MISSINGFILES" ]; then + echo " The following files are missing in the repo:" + for f in $MISSINGFILES; do + echo " $f" + done +fi +echo "" +if [ -n "$EXTRAFILES" ]; then + echo " The following files are in the repo but not the db:" + for f in $EXTRAFILES; do + echo " $f" + done +fi -- cgit v1.2.2 From d2ce9d0893421cb35c62edad84bbadd1bf917d9e Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Tue, 20 May 2008 23:19:02 -0400 Subject: Added a simple sendmail-ish script to send to the ML Signed-off-by: Aaron Griffin --- cron-jobs/devlist-mailer | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100755 cron-jobs/devlist-mailer diff --git a/cron-jobs/devlist-mailer b/cron-jobs/devlist-mailer new file mode 100755 index 0000000..b9a27e3 --- /dev/null +++ b/cron-jobs/devlist-mailer @@ -0,0 +1,16 @@ +#!/bin/bash +#Dummy helper to send email to arch-dev +# It does nothing if no output + +#LIST="arch-dev-public@archlinux.org" +LIST="aaronmgriffin@gmail.com" +FROM="cron@archlinux.org" + +stdin="$(cat)" +#echo used to strip whitespace for checking for actual data +if [ -n "$(echo $stdin)" ]; then + echo "Subject: Repository Maintenance $(date +"%d-%m-%Y %H:%M") + From: cron@archlinux.org + + $stdin" | /usr/sbin/sendmail -F$FROM "$LIST" +fi -- cgit v1.2.2 From 8e2d2c6df75a992cf65ab24e7f97de9a9f04174c Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Wed, 21 May 2008 00:33:12 -0400 Subject: Add db-move related scripts Used to easilly move a package from one repo to another Signed-off-by: Aaron Griffin --- db-move | 120 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ testing2core | 13 +++++++ testing2extra | 13 +++++++ 3 files changed, 146 insertions(+) create mode 100755 db-move create mode 100755 testing2core create mode 100755 testing2extra diff --git a/db-move b/db-move new file mode 100755 index 0000000..cb559d2 --- /dev/null +++ b/db-move @@ -0,0 +1,120 @@ +#!/bin/bash +# Originally from Pierre's testing2extra script + +if [ $# -ne 4 ]; then + echo "usage: $(basename $0) " + exit 1 +fi + +if [ -f "/etc/makepkg.conf" ]; then + #Get some config info + . /etc/makepkg.conf +else + echo "/etc/makepkg.conf does not exist!" + exit 1 +fi + +packagename="$1" +repofrom="$2" +repoto="$3" +arch="$4" + +export CARCH="$arch" + +##### Arch specific stuff. TODO make this configurable ##### +ftppath_from="/home/ftp/$repofrom/os/$arch/" +ftppath_to="/home/ftp/$repoto/os/$arch/" +svnpath="file:///home/svn-packages" +svnrepo_from="$repofrom-$arch" +svnrepo_to="$repoto-$arch" +############################################################ + +[ "$UID" = "" ] && UID=$(uid) + +WORKDIR="/tmp/db-move.$svnrepo_from.$svnrepo_to.$UID" +LOCKFILE="/tmp/.repolck.$arch.$reponame" + +cleanup() { + # unlock + rm -f "$LOCKFILE" + #rm -rf "$WORKDIR" + [ "$1" ] && exit $1 +} + +ctrl_c() { + echo "Interrupted" >&2 + cleanup 0 +} + +die() { + echo "$*" >&2 + cleanup 1 +} + +# check for locks +if [ -f "$LOCKFILE" ]; then + owner="$(/usr/bin/stat -c %U $LOCKFILE)" + die "error: db generation is already in progress (started by $owner)" +fi + +trap ctrl_c 2 +trap cleanup 0 + +/bin/touch "$LOCKFILE" +/bin/mkdir -p "$WORKDIR" + +cd "$WORKDIR" +/usr/bin/svn checkout -N $svnpath checkout +cd checkout + +/usr/bin/svn up -q $packagename +if [ -d "$packagename/repos/$svnrepo_from" ]; then + . "$packagename/repos/$svnrepo_from/$BUILDSCRIPT" + _pkgfile="$pkgname-$pkgver-$pkgrel-$CARCH$PKGEXT" + + if [ ! -f "$ftppath_from/$_pkgfile" ]; then + die "error: package file '$_pkgfile' not found in repo '$repofrom'" + fi + + if [ -d "$packagename/repos/$svnrepo_to" ]; then + echo " Removing existing package from subversion" + /usr/bin/svn rm --force -q "$packagename/repos/$svnrepo_to" + /usr/bin/svn commit -q -m "$(basename $0): $packagename removed by $(id -un) for move to $repoto" + fi + + echo " Moving svn entries" + /usr/bin/svn mv -r HEAD "$packagename/repos/$svnrepo_from" "$packagename/repos/$svnrepo_to" + /usr/bin/svn commit -m "$(basename $0): moved $packagename from [$repofrom] to [$repoto] ($arch)" + + echo " Moving package file and updating DBs" + cd "$WORKDIR" + [ -d build/ ] || mkdir build + cd build/ + + # copy the db file into our working area + if [ -f "$ftppath_from/$repofrom.db.tar.$DB_COMPRESSION" ]; then + cp "$ftppath_from/$repofrom.db.tar.$DB_COMPRESSION" . + else + touch "$repofrom.db.tar.$DB_COMPRESSION" + fi + + /usr/bin/repo-remove "$repofrom.db.tar.$DB_COMPRESSION" $packagename || die "Error in repo-remove" + #use '*' to move the old DB too + mv $repofrom.db.tar.$DB_COMPRESSION* $ftppath_from + echo " Package files will be cleaned up automatically" + + if [ -f "$ftppath_to/$repoto.db.tar.$DB_COMPRESSION" ]; then + cp "$ftppath_to/$repoto.db.tar.$DB_COMPRESSION" . + else + touch "$repoto.db.tar.$DB_COMPRESSION" + fi + + cp "$ftppath_from/$_pkgfile" . + /usr/bin/repo-add "$repoto.db.tar.$DB_COMPRESSION" $_pkgfile || die "Error in repo-add" + #use '*' to move the old DB too + mv $repoto.db.tar.$DB_COMPRESSION* $_pkgfile $ftppath_to +else + die "Error: $packagename is not in repo $repofrom" +fi + +cleanup diff --git a/testing2core b/testing2core new file mode 100755 index 0000000..417ab0e --- /dev/null +++ b/testing2core @@ -0,0 +1,13 @@ +#!/bin/bash + +if [ $# -le 1 ]; then + echo "usage: $(basename 0) [ [ Moving package '$pkg'" + $(dirname $0)/db-move "$pkg" "testing" "core" "$arch" +done diff --git a/testing2extra b/testing2extra new file mode 100755 index 0000000..7c6c87d --- /dev/null +++ b/testing2extra @@ -0,0 +1,13 @@ +#!/bin/bash + +if [ $# -le 1 ]; then + echo "usage: $(basename 0) [ [ Moving package '$pkg'" + $(dirname $0)/db-move "$pkg" "testing" "extra" "$arch" +done -- cgit v1.2.2 From 39eb3c90a2c021dc750467360e07a6dadf7bcbb2 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Wed, 21 May 2008 00:36:21 -0400 Subject: Make sure to cleanup WORKDIR on exit Signed-off-by: Aaron Griffin --- db-move | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db-move b/db-move index cb559d2..f8adfc1 100755 --- a/db-move +++ b/db-move @@ -37,7 +37,7 @@ LOCKFILE="/tmp/.repolck.$arch.$reponame" cleanup() { # unlock rm -f "$LOCKFILE" - #rm -rf "$WORKDIR" + rm -rf "$WORKDIR" [ "$1" ] && exit $1 } -- cgit v1.2.2 From 7fc8fedea48fccce0ceb31383061519754bdaa1d Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Wed, 21 May 2008 02:25:46 -0400 Subject: Remove arch from the testing2 scripts, add *64 version Thanks Pierre for the suggestion. This is much more in line with the existing DB scripts Signed-off-by: Aaron Griffin --- testing2core | 8 +++----- testing2core64 | 11 +++++++++++ testing2extra | 8 +++----- testing2extra64 | 11 +++++++++++ 4 files changed, 28 insertions(+), 10 deletions(-) create mode 100755 testing2core64 create mode 100755 testing2extra64 diff --git a/testing2core b/testing2core index 417ab0e..3a7acbf 100755 --- a/testing2core +++ b/testing2core @@ -1,13 +1,11 @@ #!/bin/bash -if [ $# -le 1 ]; then - echo "usage: $(basename 0) [ [ [ [ Moving package '$pkg'" - $(dirname $0)/db-move "$pkg" "testing" "core" "$arch" + $(dirname $0)/db-move "$pkg" "testing" "core" "i686" done diff --git a/testing2core64 b/testing2core64 new file mode 100755 index 0000000..4d30036 --- /dev/null +++ b/testing2core64 @@ -0,0 +1,11 @@ +#!/bin/bash + +if [ $# -le 0 ]; then + echo "usage: $(basename 0) [ [ Moving package '$pkg'" + $(dirname $0)/db-move "$pkg" "testing" "core" "x86_64" +done diff --git a/testing2extra b/testing2extra index 7c6c87d..3be108e 100755 --- a/testing2extra +++ b/testing2extra @@ -1,13 +1,11 @@ #!/bin/bash -if [ $# -le 1 ]; then - echo "usage: $(basename 0) [ [ [ [ Moving package '$pkg'" - $(dirname $0)/db-move "$pkg" "testing" "extra" "$arch" + $(dirname $0)/db-move "$pkg" "testing" "extra" "i686" done diff --git a/testing2extra64 b/testing2extra64 new file mode 100755 index 0000000..418876c --- /dev/null +++ b/testing2extra64 @@ -0,0 +1,11 @@ +#!/bin/bash + +if [ $# -le 0 ]; then + echo "usage: $(basename 0) [ [ Moving package '$pkg'" + $(dirname $0)/db-move "$pkg" "testing" "extra" "x86_64" +done -- cgit v1.2.2 From b0792658d9d6fd11c63a737037bd95ef5ccc50f2 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Wed, 21 May 2008 14:48:12 -0400 Subject: Switch to output filelists in official repos Signed-off-by: Aaron Griffin --- cron-jobs/createFileLists | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) mode change 100644 => 100755 cron-jobs/createFileLists diff --git a/cron-jobs/createFileLists b/cron-jobs/createFileLists old mode 100644 new mode 100755 index 02af296..d6cbd18 --- a/cron-jobs/createFileLists +++ b/cron-jobs/createFileLists @@ -1,7 +1,8 @@ #!/bin/bash reposdir=/home/ftp/ -targetdir=/home/pierre/public_html/test-repo/ +#targetdir=/home/pierre/public_html/test-repo/ +targetdir=$reposdir repos="core extra unstable testing community" arches="i686 x86_64" -- cgit v1.2.2 From 8685aaea284147536394969805c9b29c7da71188 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Wed, 21 May 2008 14:50:52 -0400 Subject: Copied 'ftpmaint' from root as adjust-permissions This script makes sure we all have write access to the repos Signed-off-by: Aaron Griffin --- cron-jobs/adjust-permissions | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 cron-jobs/adjust-permissions diff --git a/cron-jobs/adjust-permissions b/cron-jobs/adjust-permissions new file mode 100644 index 0000000..7f7e4c8 --- /dev/null +++ b/cron-jobs/adjust-permissions @@ -0,0 +1,24 @@ +#!/bin/sh + +if [ -f /tmp/.ftpmaint.lck ]; then + exit 0 +fi + +/bin/touch /tmp/.ftpmaint.lck + +cd /home/ftp +/bin/chown -R ftp:ftp-arch core/os/i686 +/bin/chown -R ftp:ftp-arch core/os/x86_64 +/bin/chown -R ftp:ftp-extra {extra,unstable,testing}/os/i686 +/bin/chown -R ftp:ftp-extra {extra,unstable,testing}/os/x86_64 +/bin/chown -R ftp:aur community + +/bin/chmod -R g+w /home/aur/unsupported{,-temp} + +for d in core extra unstable testing community; do + /bin/chmod -R g+w $d/os/i686 + /bin/chmod -R g+w $d/os/x86_64 +done +/bin/chmod 555 /home/ftp + +rm -f /tmp/.ftpmaint.lck -- cgit v1.2.2 From 2343985fcc46852f5b4de82c70824fa1dd702bbf Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Wed, 21 May 2008 15:33:29 -0400 Subject: Switch EXTRAFILES test to be more accurate New test suggested by Thomas via cleanup2.sh script Signed-off-by: Aaron Griffin --- cron-jobs/ftpdir-cleanup | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index 83c7cf1..576dd6e 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -69,41 +69,46 @@ done cd "$ftppath" for pkg in *$PKGEXT; do - pkgname="$(getpkgname_ver ${filename})" - if [ ! -d "$TMPDIR/$pkgname" ]; then - EXTRAFILES="$EXTRAFILES $pkg" + pkgname="$(getpkgname $pkg)" + if [ "$pkg" != "$TMPDIR/$pkgname-*" -a "$(getpkgname $(basename $pkg))" = "$pkgname" ]; then + continue 2 fi + EXTRAFILES="$EXTRAFILE $pkg" done - - cd "$ftppath" rm -rf ${TMPDIR} -if [ -n "${DELETEFILES}" ]; then - #rm -f ${DELETEFILES} - #mv ${DELETEFILES} /home/package-cleanup/ - echo "" -fi - echo "Scan complete for $reponame ($arch) at ${ftppath}" if [ -n "$DELETEFILES" ]; then - echo " The following files have been moved to package-cleanup:" + echo " The following files are out of date" + echo " They will be moved to /home/package-cleanup" for f in $DELETEFILES; do echo " $f" done fi echo "" if [ -n "$MISSINGFILES" ]; then - echo " The following files are missing in the repo:" + echo " The following files are missing in the repo" for f in $MISSINGFILES; do echo " $f" done fi echo "" if [ -n "$EXTRAFILES" ]; then - echo " The following files are in the repo but not the db:" + echo " The following files are in the repo but not the db" + echo " They will be moved to /home/package-cleanup" for f in $EXTRAFILES; do echo " $f" done fi + +if [ -n "${DELETEFILES}" ]; then + #mv ${DELETEFILES} /home/package-cleanup/ + echo "" +fi + +if [ -n "${EXTRAFILES}" ]; then + #mv ${EXTRAFILES} /home/package-cleanup/ + echo "" +fi -- cgit v1.2.2 From 603d737cd871d4195dd241c345a133989ce59d28 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Wed, 21 May 2008 15:34:15 -0400 Subject: Adjust permissions of the adjust-permissions script HAH! Signed-off-by: Aaron Griffin --- cron-jobs/adjust-permissions | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 cron-jobs/adjust-permissions diff --git a/cron-jobs/adjust-permissions b/cron-jobs/adjust-permissions old mode 100644 new mode 100755 -- cgit v1.2.2 From a0c4daf2679a56f6eaa7df0062c7184fc55f95bf Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Thu, 22 May 2008 12:52:53 -0400 Subject: Fix db-remove usage text Signed-off-by: Aaron Griffin --- db-remove | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db-remove b/db-remove index c69e179..f9495e9 100755 --- a/db-remove +++ b/db-remove @@ -1,7 +1,7 @@ #!/bin/bash if [ $# -ne 3 ]; then - echo "usage: $(basename $0) " + echo "usage: $(basename $0) " exit 1 fi -- cgit v1.2.2 From 41bf0372aacd5737f31968d6a274e813301b69cc Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Thu, 22 May 2008 12:53:47 -0400 Subject: Use 'mv' to shuffle files out of the *64 dirs This way, all package files get cleaned up properly at the end of the process. Signed-off-by: Aaron Griffin --- db-update | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/db-update b/db-update index 93672f5..7ff768e 100755 --- a/db-update +++ b/db-update @@ -101,7 +101,8 @@ if [ -d "${stagedir}64" ]; then echo "Packages are now differentiated by the arch in the filename." echo "Please delete '${stagedir}64'" echo "--------------------------------------------------" - /bin/cp -r "${stagedir}64/"* "$stagedir/" + /bin/mv "${stagedir}64/add/"* "$stagedir/add/" + /bin/mv "${stagedir}64/del/"* "$stagedir/del/" fi to_add="" -- cgit v1.2.2 From d1f0d5732167d142af48b48a84bbbbd2adb7593a Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Thu, 22 May 2008 17:18:27 -0400 Subject: Git tells me this file was modifed I don't know what changed, but I am committing to check! Signed-off-by: Aaron Griffin --- cron-jobs/ftpdir-cleanup | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index 576dd6e..d6dce11 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -104,11 +104,11 @@ if [ -n "$EXTRAFILES" ]; then fi if [ -n "${DELETEFILES}" ]; then - #mv ${DELETEFILES} /home/package-cleanup/ + mv ${DELETEFILES} /home/package-cleanup/ echo "" fi if [ -n "${EXTRAFILES}" ]; then - #mv ${EXTRAFILES} /home/package-cleanup/ + mv ${EXTRAFILES} /home/package-cleanup/ echo "" fi -- cgit v1.2.2 From 6f67fcda30b1589f1716fb64191c752864e9fab7 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Thu, 22 May 2008 17:20:32 -0400 Subject: crontab file for repo-based cron jobs Signed-off-by: Aaron Griffin --- cron-jobs/repo-maint.crontab | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 cron-jobs/repo-maint.crontab diff --git a/cron-jobs/repo-maint.crontab b/cron-jobs/repo-maint.crontab new file mode 100644 index 0000000..dadd1fb --- /dev/null +++ b/cron-jobs/repo-maint.crontab @@ -0,0 +1,12 @@ +MAILTO="root" +#################################################################### +#minute (0-59), # +#| hour (0-23), # +#| | day of the month (1-31), # +#| | | month of the year (1-12), # +#| | | | day of the week (0-6 with 0=Sunday)# +#| | | | | commands # +#################################################################### +*/5 * * * * /arch/cron-jobs/adjust-permissions +00 21 * * * /arch/cron-jobs/createFileList +00 */3 * * * /arch/cron-jobs/ftpdir-cleanup | /arch/cron-jobs/devlist-mailer -- cgit v1.2.2 From 11279e5f6a29d1b1d8061ab69deab0f4a977b891 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Thu, 22 May 2008 17:22:12 -0400 Subject: Add a script to generate source tarballs for GPL compliance This may need to be moved out of here later /me shrugs Signed-off-by: Aaron Griffin --- misc-scripts/make-sourceball | 107 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100755 misc-scripts/make-sourceball diff --git a/misc-scripts/make-sourceball b/misc-scripts/make-sourceball new file mode 100755 index 0000000..c4eb459 --- /dev/null +++ b/misc-scripts/make-sourceball @@ -0,0 +1,107 @@ +#!/bin/bash + +if [ $# -ne 3 ]; then + echo "usage: $(basename $0) " + exit 1 +fi + +if [ -f "/etc/makepkg.conf" ]; then + #Get some config info + . /etc/makepkg.conf +else + echo "/etc/makepkg.conf does not exist!" + exit 1 +fi +packagename="$1" +reponame="$2" +arch="$3" + +##### Arch specific stuff. TODO make this configurable ##### +srcpath="/home/aaron/public_html/sources/" +svnpath="file:///home/svn-packages/$packagename/repos/$reponame-$arch" +############################################################ + +WORKDIR="/tmp/make-sourceball.$svnrepo.$UID" + +cleanup() { + # unlock + rm -rf "$WORKDIR" + [ "$1" ] && exit $1 +} + +ctrl_c() { + echo "Interrupted" >&2 + cleanup 0 +} + +die() { + echo "$*" >&2 + cleanup 1 +} + +##### Copied from makepkg ##### + +strip_url() { + echo "$1" | sed 's|^.*://.*/||g' +} + +create_srcpackage() { + echo "Creating source package..." + local comp_files="$BUILDSCRIPT" + echo " Adding: $BUILDSCRIPT" + . $BUILDSCRIPT + + if [ "$install" != "" ]; then + if [ -f $install ]; then + echo " Adding: install script '$install'" + comp_files="$comp_files $install" + else + die "Install script '$install' not found." + fi + fi + + if [ -f ChangeLog ]; then + echo " Adding: ChangeLog" + comp_files="$comp_files ChangeLog" + fi + + local i + for i in ${source[@]}; do + i="$(strip_url "$i")" + if [ -f $i ]; then + echo " Adding: $i" + comp_files="$comp_files $i" + fi + done + + local pkg_file="${pkgname}-${pkgver}-${pkgrel}${SRCEXT}" + + # tar it up + echo "Compressing source package" + if ! /usr/bin/bsdtar -czf "$pkg_file" $comp_files; then + die "Failed to create source package file." + fi + + echo "Source package complete: $pkg_file" + if [ ! -d "$srcpath" ]; then + mkdir -p "$srcpath" + fi + cp $pkg_file "$srcpath" +} + +trap ctrl_c 2 +trap cleanup 0 + +/bin/mkdir -p "$WORKDIR" +cd "$WORKDIR" + +echo "Creating Source tarball for $packagename ($reponame-$arch)" + +if /usr/bin/svn checkout -N $svnpath; then + cd "$reponame-$arch" + echo "Cheating with makepkg to download sources" + /usr/bin/makepkg -g || die "Error downloading files" + create_srcpackage +else + die "Package '$packagename' does not exist in repo $reponame-$arch" +fi -- cgit v1.2.2 From 8c9b0d64cb8e84029a4a7f56ca12b24f09d1cce9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20B=C3=A4chler?= Date: Sat, 24 May 2008 12:27:07 +0200 Subject: Commit testing2x script MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Thomas Bächler Signed-off-by: Aaron Griffin --- testing2x | 58 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100755 testing2x diff --git a/testing2x b/testing2x new file mode 100755 index 0000000..7096f1d --- /dev/null +++ b/testing2x @@ -0,0 +1,58 @@ +#!/bin/bash + +if [ -f "/etc/makepkg.conf" ]; then + . /etc/makepkg.conf +else + echo "/etc/makepkg.conf does not exist!" + exit 1 +fi + +case "$0" in + *64) + arch="x86_64" + ;; + *) + arch="i686" + ;; +esac +svnpath="file:///home/svn-packages" +WORKDIR="$(mktemp -d /tmp/testing2x.XXXXXX)" + +cleanup() { + rm -rf "${WORKDIR}" + [ -n "$1" ] && exit $1 +} + +ctrl_c() { + echo "Interrupted" >&2 + cleanup 0 +} + +trap ctrl_c 2 +trap cleanup 0 + +cd "${WORKDIR}" +/usr/bin/svn checkout -N ${svnpath} checkout +cd checkout + +for pkg in $*; do + moved=0 + /usr/bin/svn up -q ${pkg} + if [ -f "${pkg}/repos/testing-${arch}/${BUILDSCRIPT}" ]; then + for repo in core extra; do + if [ -f "${pkg}/repos/${repo}-${arch}/${BUILDSCRIPT}" ]; then + echo "===> Moving package '${pkg}': testing-${arch} -> ${repo}-${arch}" + $(dirname $0)/db-move "${pkg}" "testing" "${repo}" "${arch}" + moved=1 + break + fi + done + if [ ${moved} -eq 0 ]; then + echo "===> Warning: ${pkg} is only in testing-${arch}, cannot determine where to move it" + fi + else + echo "===> Warning: ${pkg} is not in testing-${arch}" + fi +done + +cleanup -- cgit v1.2.2 From 06c8428afdeb31c5514f27def63a13dad782ff6d Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Sat, 24 May 2008 13:13:57 -0500 Subject: Link the new testing2x script to the 64 version too Signed-off-by: Aaron Griffin --- testing2x64 | 1 + 1 file changed, 1 insertion(+) create mode 120000 testing2x64 diff --git a/testing2x64 b/testing2x64 new file mode 120000 index 0000000..ff08aa7 --- /dev/null +++ b/testing2x64 @@ -0,0 +1 @@ +testing2x \ No newline at end of file -- cgit v1.2.2 From 966c53e8ce0b11b9cd7fb88a497f2c086c682676 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Tue, 27 May 2008 14:23:24 -0400 Subject: Minor changes to the make-sourceball script Signed-off-by: Aaron Griffin --- misc-scripts/make-sourceball | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/misc-scripts/make-sourceball b/misc-scripts/make-sourceball index c4eb459..e9e2152 100755 --- a/misc-scripts/make-sourceball +++ b/misc-scripts/make-sourceball @@ -18,7 +18,7 @@ arch="$3" ##### Arch specific stuff. TODO make this configurable ##### srcpath="/home/aaron/public_html/sources/" -svnpath="file:///home/svn-packages/$packagename/repos/$reponame-$arch" +svnpath="file:///home/svn-packages/$packagename/" ############################################################ WORKDIR="/tmp/make-sourceball.$svnrepo.$UID" @@ -97,11 +97,20 @@ cd "$WORKDIR" echo "Creating Source tarball for $packagename ($reponame-$arch)" -if /usr/bin/svn checkout -N $svnpath; then +if /usr/bin/svn checkout -N "$svnpath/repos/$reponame-$arch"; then cd "$reponame-$arch" - echo "Cheating with makepkg to download sources" - /usr/bin/makepkg -g || die "Error downloading files" - create_srcpackage + if /usr/bin/makepkg -g; then + create_srcpackage + else #try the trunk, it may have updates to the source URL + if /usr/bin/svn checkout -N "$svnpath/trunk"; then + cd "trunk" + if /usr/bin/makepkg -g; then + create_srcpackage + else + die "Cannot get sources properly. Dying" + fi + fi + fi else die "Package '$packagename' does not exist in repo $reponame-$arch" fi -- cgit v1.2.2 From 1c9a21bdf378ddd3572b3b8f405d99b3063521c9 Mon Sep 17 00:00:00 2001 From: Scott Horowitz Date: Tue, 27 May 2008 14:34:00 -0400 Subject: Add the check_archlinux.py repo checker This is to be used for integrity checked, generated perhaps once a day Signed-off-by: Aaron Griffin --- cron-jobs/check_archlinux.py | 724 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 724 insertions(+) create mode 100755 cron-jobs/check_archlinux.py diff --git a/cron-jobs/check_archlinux.py b/cron-jobs/check_archlinux.py new file mode 100755 index 0000000..8e39c81 --- /dev/null +++ b/cron-jobs/check_archlinux.py @@ -0,0 +1,724 @@ +#!/usr/bin/env python +# +# check_archlinux.py +# +# Written by Scott Horowitz +# Graphical dependency tree by Cesar G. Miguel +# +# This script currently checks for a number of issues in your ABS tree: +# 1. Directories with missing PKGBUILDS +# 2. Duplicate PKGBUILDs +# 3. Missing (make-)dependencies, taking into account 'provides' +# 4. Provisioned dependencies +# 5. Circular dependencies +# 6. Valid arch's in PKGBUILDS +# 7. Missing packages in Arch repos +# 8. PKGBUILD names that don't match their directory +# 9. Hierarchy of repos (e.g., that a core package doesn't depend on +# a non-core package) +# It can also, optionally, generate a graphical representation of the +# dependency tree. +# +# Todo: +# 1. Accommodate testing repo? + +abs_conf_dir = "/etc/abs" +valid_archs = ['i686', 'x86_64'] +cvs_tags = {'i686': 'CURRENT', 'x86_64': 'CURRENT-64'} +include_paths = ['core', 'extra', 'community', 'unstable'] + +base_server = "ftp.archlinux.org" # Must be ftp site +# Ensure that core repo only depends on core, and the extra repo only +# depends on core and extra, etc. +repo_hierarchy = {'core': ('core'), \ +'extra': ('core', 'extra'), \ +'unstable': ('core', 'extra', 'unstable'), \ +'community': ('core', 'extra', 'unstable', 'community')} + +pkgname_str = "pkgname=" +dep_str = "depends=(" +makedep_str = "makedepends=(" +provides_str = "provides=(" +arch_str = "arch=(" +pkgver_str = "pkgver=" +pkgrel_str = "pkgrel=" +build_str = "build()" +source_str = "source=" +url_str = "url=" +mathlist = (">=", "<=", "=", ">", "<") +sup_tag_str = "*default tag=" + +import os, sys, getopt, tempfile +from ftplib import FTP +try: + import pydot + HAS_PYDOT = True +except: + HAS_PYDOT = False + +def print_heading(heading): + print "" + print "=" * (len(heading) + 4) + print "= " + heading + " =" + print "=" * (len(heading) + 4) + +def print_subheading(subheading): + print "" + print subheading + print "-" * (len(subheading) + 2) + +def rmgeneric(path, __func__): + try: + __func__(path) + except OSError, (errno, strerror): + pass + +def removeall(path): + if not os.path.isdir(path): + return + + files=os.listdir(path) + + for x in files: + fullpath=os.path.join(path, x) + if os.path.isfile(fullpath): + f=os.remove + rmgeneric(fullpath, f) + elif os.path.isdir(fullpath): + removeall(fullpath) + f=os.rmdir + rmgeneric(fullpath, f) + +def update_var(line, user_vars, pkgpath): + if line.count("$") > 0: + export_line = "" + for var in user_vars: + if line.count(var[0]) > 0: + export_line = export_line + var[0] + "=" + var[1] + " && " + if line.startswith("(") and line.endswith(")"): + line = line[1:-1] + export_line = export_line + "echo \"" + line + "\"" + line = os.popen(export_line).read().replace("\n", "") + return line + +def split_dep_prov_symbol(dep): + # Splits 'foo>=1.2.3' into ('foo', '1.2.3', '>=') + prov = "" + symbol = "" + for char in mathlist: + pos = dep.find(char) + if pos > -1: + prov = dep[pos:].replace(char, "") + dep = dep[:pos] + symbol = char + break + return (dep, prov, symbol) + +def repo_from_path(path): + return path.split("/")[-4] + +def create_supfile(sourcefile, destfile, newtag): + o = open(sourcefile, 'r') + info = o.read() + o.close() + lines = info.split("\n") + + o = open(destfile, 'w') + for line in lines: + line = line.strip() + if line[:len(sup_tag_str)] == sup_tag_str: + line = sup_tag_str + newtag + o.write(line + "\n") + o.close() + +def get_deps_provides_etc(pkgpath): + # Parse PKGBUILD for name, depends, makedepends, provides, arch's, and version + o = open(pkgpath, 'r') + info = o.read() + o.close() + lines = info.split("\n") + + deps = [] + provides = [] + archs = [] + makedeps = [] + array = [] + user_vars = [] + continue_line = False + for line in lines: + line = line.strip() + if line.find("#") > -1: + line = line[:line.find("#")].strip() + if not continue_line: + deps_line = False + provides_line = False + arch_line = False + makedeps_line = False + if line[:len(dep_str)] == dep_str: + line = line.replace(dep_str,"") + deps_line = True + elif line[:len(makedep_str)] == makedep_str: + line = line.replace(makedep_str,"") + makedeps_line = True + elif line[:len(provides_str)] == provides_str: + line = line.replace(provides_str,"") + provides_line = True + elif line[:len(arch_str)] == arch_str: + line = line.replace(arch_str, "") + arch_line = True + elif line[:len(pkgname_str)] == pkgname_str: + pkgname = line.replace(pkgname_str, "") + if pkgname.startswith("\"") and pkgname.endswith("\""): + pkgname = pkgname[1:-1] + pkgname = update_var(pkgname, user_vars, pkgpath) + user_vars.append([pkgname_str, pkgname]) + line = "" + elif line[:len(pkgver_str)] == pkgver_str: + pkgver = line.replace(pkgver_str, "") + if pkgver.startswith("\"") and pkgver.endswith("\""): + pkgver = pkgver[1:-1] + pkgver = update_var(pkgver, user_vars, pkgpath) + user_vars.append([pkgver_str[:-1], pkgver]) + line = "" + elif line[:len(pkgrel_str)] == pkgrel_str: + pkgrel = line.replace(pkgrel_str, "") + if pkgrel.startswith("\"") and pkgrel.endswith("\""): + pkgrel = pkgrel[1:-1] + pkgrel = update_var(pkgrel, user_vars, pkgpath) + user_vars.append([pkgrel_str[:-1], pkgrel]) + line = "" + elif line[:len(build_str)] == build_str: + break + elif not continue_line: + if line.count("=") == 1 and line.count(" ") == 0 and line[:1] != "#" and \ + line[:len(source_str)] != source_str and line[:len(url_str)] != url_str: + split = line.split("=") + for item in range(len(split)): + split[item] = update_var(split[item], user_vars, pkgpath) + user_vars.append(split) + line = "" + if len(line) > 0: + pos = line.find(")") + if pos > -1: + # strip everything from closing paranthesis on + # since some PKGBUILDS have comments after the + # depends array + line = line[:pos] + line = line.split(' ') + for i in range(len(line)): + line[i] = line[i].replace("'","").replace('"','') + line[i] = update_var(line[i], user_vars, pkgpath) + if len(line[i]) > 0: + if deps_line: + deps.append(line[i]) + array=deps + elif provides_line: + provides.append(line[i]) + array=provides + elif arch_line: + archs.append(line[i]) + array=archs + elif makedeps_line: + makedeps.append(line[i]) + array=makedeps + if array and (array[-1] == "\\" or array[-1][-1] == "\\"): + # continue reading deps/provides on next line + if array[-1] == "\\": + array.pop(-1) + else: + array[-1] = array[-1].replace("\\", "") + continue_line = True + else: + continue_line = False + version = pkgver + "-" + pkgrel + return (pkgname, deps, makedeps, provides, archs, version) + +def get_pkgbuilds_in_dir(rootdir): + # Recursively populates pkgbuild_deps, pkgbuild_paths, etc. + # dicts with info from each PKGBUILD found in rootdir: + if rootdir != absroot: + if rootdir.count("/") == (absroot.count("/")+1) and rootdir not in curr_include_paths: + return + pkgfound = False + for f in os.listdir(rootdir): + fpath = rootdir + "/" + f + if os.path.isdir(fpath): + get_pkgbuilds_in_dir(fpath) + elif f == 'PKGBUILD': + pkgfound = True + name = rootdir.split("/")[-1] + if name in pkgbuild_deps: + dups.append(fpath.replace(absroot, "") + " vs. " + pkgbuild_paths[name].replace(absroot, "")) + else: + (pkgname, deps, makedeps, provides, archs, version) = get_deps_provides_etc(fpath) + pkgbuild_deps[pkgname] = deps + pkgbuild_makedeps[pkgname] = makedeps + pkgbuild_paths[pkgname] = fpath + pkgbuild_archs[pkgname] = archs + pkgbuild_versions[pkgname] = version + # We'll store the provides "backwards" compared to + # the other dicts. This will make searching for + # provides easier by being able to look up the + # provide name itself and find what provides it + for provide in provides: + pkgbuild_provides[provide] = pkgname + if pkgname != name: + mismatches.append(pkgname + " vs. " + fpath.replace(absroot, "")) + if not pkgfound and rootdir.replace(absroot, "").count("/") == 3: + misses.append(rootdir.replace(absroot, "") + "/PKGBUILD") + +def verify_depends_makedepends(verify_makedeps=False): + # Make sure all the deps we parsed are actually packages; also + # ensure we meet dep provisions. + if verify_makedeps: + array = pkgbuild_makedeps + else: + array = pkgbuild_deps + for pkgname in array: + deps = array[pkgname] + pkg_repo = repo_from_path(pkgbuild_paths[pkgname]) + deps_to_pop = [] + for i in range(len(deps)): + dep = deps[i] + (dep, prov, char) = split_dep_prov_symbol(dep) + try: + x = pkgbuild_deps[dep] + # Check that prov is met too: + if len(prov) > 0: + compare_str = "vercmp " + pkgbuild_versions[dep] + " " + prov + error = False + if char == "<=": + if int(os.popen(compare_str).read().replace("\n", "")) > 0: + error = True + elif char == ">=": + if int(os.popen(compare_str).read().replace("\n", "")) < 0: + error = True + elif char == "=": + if int(os.popen(compare_str).read().replace("\n", "")) != 0: + error = True + elif char == ">": + if int(os.popen(compare_str).read().replace("\n", "")) <= 0: + error = True + elif char == "<": + if int(os.popen(compare_str).read().replace("\n", "")) >= 0: + error = True + if error: + if verify_makedeps: + unmet_makedep_provs.append(pkgname + " --> '" + dep + char + prov + "'") + else: + unmet_dep_provs.append(pkgname + " --> '" + dep + char + prov + "'") + # Check repos fit hierarchy scheme: + dep_repo = repo_from_path(pkgbuild_paths[dep]) + try: + valid_repos = repo_hierarchy[pkg_repo] + # Make sure dep repo is one of the valid repos: + if dep_repo not in valid_repos: + if verify_makedeps: + makedep_hierarchy.append(pkg_repo + "/" + pkgname + " depends on " + dep_repo + "/" + dep) + else: + dep_hierarchy.append(pkg_repo + "/" + pkgname + " depends on " + dep_repo + "/" + dep) + except: + pass + except: + # Check if a package provides this dep: + try: + x = pkgbuild_provides[dep] + except: + if verify_makedeps: + missing_makedeps.append(pkgname + " --> '" + dep + "'") + else: + missing_deps.append(pkgname + " --> '" + dep + "'") + deps_to_pop.append(i) + # Pop deps not found from end to beginning: + while len(deps_to_pop) > 0: + deps.pop(deps_to_pop[-1]) + deps_to_pop.pop(-1) + +def verify_archs(): + for pkgname in pkgbuild_archs: + newarch = [] + archs = pkgbuild_archs[pkgname] + for arch in archs: + if arch not in valid_archs: + invalid_archs.append(pkgname + " --> " + arch) + else: + newarch.append(arch) + if len(newarch) > 0: + pkgbuild_archs[pkgname] = newarch + +def verify_packages(tree_arch): + # Make sure packages exist in Arch repo(s): + ftp = FTP(base_server) + ftp.login() + prev_wd = "" + # Find all repos/archs; marching through them in order will greatly speed + # up searching for ftp files by minimizing the number of .nlst's that we + # have to do. + repos = [] + for pkgname in pkgbuild_paths: + pkgrepo = repo_from_path(pkgbuild_paths[pkgname]) + if not pkgrepo in repos: + repos.append(pkgrepo) + archs = [] + for pkgname in pkgbuild_archs: + pkgarchs = pkgbuild_archs[pkgname] + for arch in pkgarchs: + if not arch in archs: + archs.append(arch) + for r in repos: + for pkgname in pkgbuild_archs: + repo = repo_from_path(pkgbuild_paths[pkgname]) + if repo == r: + archs = pkgbuild_archs[pkgname] + pkgver_rel = pkgbuild_versions[pkgname] + for arch in archs: + if arch == tree_arch: + # Check for file: + wd = repo + "/os/" + arch + if wd != prev_wd: + ftpfiles = ftp.nlst(wd) + prev_wd = wd + fname_new = wd + "/" + pkgname + "-" + pkgver_rel + "-" + arch + ".pkg.tar.gz" + fname_old = wd + "/" + pkgname + "-" + pkgver_rel + ".pkg.tar.gz" + if fname_old not in ftpfiles and fname_new not in ftpfiles: + missing_pkgs.append(pkgname + "-" + pkgver_rel + " in " + wd) + ftp.quit() + +def subset_superset_path(currpath, currpathnum, paths): + # If a pair of subset/superset paths are found, + # pop the superset one to show the more minimal + # case. + # + # e.g. foo > bar > baz > foo (superset) + # foo > bar > foo (subset) + # --> pop the superset + # + currdeps = currpath.split(">") + currdeps = list(set(currdeps)) + currdeps.sort() + pathnum = 0 + for path in paths: + if pathnum != currpathnum: + deps = path.split(">") + deps = list(set(deps)) + deps.sort() + if len(currdeps) < len(path): + subset = True + for d in currdeps: + if not d in path: + subset = False + break + if subset: + circular_deps.pop(pathnum) + if pathnum <= currpathnum: + currpathnum -= 1 + elif len(currdeps) > len(path): + superset = True + for d in path: + if not d in currdeps: + superset = False + break + if superset: + circular_deps.pop(currpathnum) + currpathnum -= 1 + pathnum += 1 + return True + +def unique_path(currpath, paths): + # Returns false if an equivalent path exists in + # paths. + # + # e.g. foo > bar > foo + # bar > foo > bar + # + currdeps = currpath.split(">") + currdeps = list(set(currdeps)) + currdeps.sort() + for path in paths: + deps = path.split(">") + deps = list(set(deps)) + deps.sort() + if currdeps == deps: + return False + return True + +def update_paths(paths, dep, dep2): + # Update paths by appending new paths with dep2 + # based on all the different ways we could get + # to dep2. Returns True if a path was updated. + new_path = False + for i in range(len(paths)): + array = paths[i].split(">") + newpath = paths[i] + ">" + dep2 + if array[-1] == dep and not dep2 in array and unique_path(newpath, paths): + paths.append(newpath) + new_path = True + return new_path + +def check_name_in_recursive_deps(pkgname): + # Retrieve all recursive dependencies from a package and + # determines if pkgname is found in one of its deps. + recursive_deps = [] + for dep in pkgbuild_deps[pkgname]: + dep = split_dep_prov_symbol(dep)[0] # Strip any provision + recursive_deps.append(dep) + paths = [] + for dep in recursive_deps: + dep = split_dep_prov_symbol(dep)[0] # Strip any provision + paths.append(dep) + searching = True + while searching: + searching = False + for dep in recursive_deps: + for dep2 in pkgbuild_deps[dep]: + dep2 = split_dep_prov_symbol(dep2)[0] # Strip any provision + # This is a HUGE time-saver. Instead of generating every single + # possible path that can yield a circular dep, we'll reduce + # the number greatly by throwing out paths like such: + # + # If we have a path: foo>bar>baz>blah>poop>foo + # We will throw out any: + # foo>...>bar>baz>blah>poop>foo + # foo>...>baz>blah>poop>foo + # foo>...>blah>poop>foo + # and so on. Otherwise we will find hundreds or even thousands + # of possible paths that all essentially represent the same + # circular dep. + # + # However, we will always let pkgname through in order to make + # sure we can find multiple circular deps for a given pkg. + if dep2 not in recursive_deps or dep2 == pkgname: + updated = update_paths(paths, dep, dep2) + if dep2 not in recursive_deps: + recursive_deps.append(dep2) + if updated: + searching = True + # Done searching, store circular deps: + for path in paths: + if path.split(">")[-1] == pkgname: + if unique_path(pkgname + ">" + path, circular_deps): + circular_deps.append(pkgname + ">" + path) + # Reduce any subset/superset path pairs: + pathnum = 0 + for path in circular_deps: + subset_superset_path(path, pathnum, circular_deps) + pathnum += 1 + +def circular_deps_check(): + # Check for circular dependencies: + for pkgname in pkgbuild_deps: + check_name_in_recursive_deps(pkgname) + +def visualize_repo(): + output = 'digraph G { \n \ + concentrate = true; \n \ + ordering = out; \n \ + ranksep=5.0; \n \ + node [style=filled,fontsize=8]; \n' + + # draws circular dependencies in red + for path in circular_deps: + output += '\t "'+path[0]+'"' + deps = path.split(">") + for d in deps: + output += ' -> "'+d+'"' + output += ' [color=red]\n' + + for pkg in pkgbuild_deps.keys(): + output += '\t "'+pkg+'" -> { ' + for d in pkgbuild_deps[pkg]: + d = split_dep_prov_symbol(d)[0] # Strip any provision + output += '"'+d+'"; ' + output += '}\n' + + output += '}' + + # Uncomment these lines to get a file dump called + # 'output'. This can be used to manually generate + # an image using, e.g., dot -Tsvg output tree.svg + #dump = open('output', 'w') + #dump.write(output) + #dump.close() + + fname = 'dependency_tree-' + arch + '.svg' + print "Generating " + fname + "..." + g = pydot.graph_from_dot_data(output) + g.write(fname, prog='dot', format='svg') + +def print_result(list, subheading): + if len(list) > 0: + print_subheading(subheading) + for item in list: + print item + +def print_results(): + print_result(misses, "Missing PKGBUILDs") + print_result(mismatches, "Mismatched Pkgnames") + print_result(dups, "Duplicate PKGBUILDs") + print_result(missing_deps, "Missing Dependencies") + print_result(missing_makedeps, "Missing Makedepends") + print_result(unmet_dep_provs, "Unmet Dependency Provisions") + print_result(unmet_makedep_provs, "Unmet Makedepends Provisions") + print_result(dep_hierarchy, "Repo Hierarchy for Dependencies") + print_result(makedep_hierarchy, "Repo Hierarchy for Makedepends") + print_result(invalid_archs, "Invalid Archs") + print_result(circular_deps, "Circular Dependencies") + print_result(missing_pkgs, "Missing Repo Packages") + print_subheading("Summary") + print "Dirs with missing PKGBUILDs: ", len(misses) + print "Duplicate PKGBUILDs: ", len(dups) + print "Missing (make)dependencies: ", len(missing_deps)+len(missing_makedeps) + print "Unmet provisioned (make)dependencies: ", len(unmet_dep_provs)+len(unmet_makedep_provs) + print "Circular dependencies: ", len(circular_deps) + print "Invalid archs: ", len(invalid_archs) + print "Missing packages in repos: ", len(missing_pkgs) + print "Mismatching PKGBUILD names: ", len(mismatches) + print "Repo hierarchy problems: ", len(dep_hierarchy)+len(makedep_hierarchy) + print "" + +def print_usage(): + print "" + print "Usage: check_archlinux [OPTION]" + print "" + print "Options:" + print " --abs-tree= Check specified tree (assumes the abs tree" + print " is i686 unless overridden with --arch)" + print " --arch= Use specified arch (e.g. 'x86_64')" + print " -g Generate graphical dependency tree(s)" + print " -h, --help Show this help and exit" + print "" + print "Examples:" + print "\n Check all arches and do fresh cvs checkouts:" + print " check_archlinux" + print "\n Check x_86_64 only and do fresh cvs checkout:" + print " check_archlinux --arch=x86_64" + print "\n Check existing i686 abs tree:" + print " check_archlinux --abs-tree=/var/abs" + print "\n Check existing x86_64 abs tree and also generate dep tree image:" + print " check_archlinux --abs-tree=/var/abs --arch=x86_64 -g" + print "" + +graphdeptree = False +user_absroot = "" +user_arch = "" +try: + opts, args = getopt.getopt(sys.argv[1:], "g", ["abs-tree=", "arch="]) +except getopt.GetoptError: + print_usage() + sys.exit() +if opts != []: + for o, a in opts: + if o in ("-g"): + graphdeptree = True + if not HAS_PYDOT: + print "You must install pydot to generate a graphical dependency tree. Aborting..." + sys.exit() + elif o in ("--abs-tree"): + user_absroot = a + elif o in ("--arch"): + user_arch = a + if user_arch not in valid_archs: + print "You did not specify a valid arch. Aborting..." + sys.exit() + else: + print_usage() + sys.exit() + if args != []: + for a in args: + if a in ("play", "pause", "stop", "next", "prev", "pp", "info", "status", "repeat", "shuffle"): + self.single_connect_for_passed_arg(a) + else: + self.print_usage() + sys.exit() + +if len(user_absroot) > 0 and len(user_arch) == 0: + user_arch = valid_archs[0] # i686 default.. + +if len(user_absroot) > 0: + print "Warning: Ensure your ABS tree is clean to prevent false positives." + +try: + absroots = [] + fsup = 'supfile.curr' + for arch in valid_archs: + if len(user_arch) == 0 or user_arch == arch: + print_heading(arch + " Integrity Check") + if len(user_absroot) == 0: + # Create temp dir for cvs checkout: + absroot = tempfile.mkdtemp(prefix="abs-" + arch + "-") + absroots.append(absroot) + print "\nChecking out clean abs tree" + for repo in include_paths: + print "==>", repo + # Create (and modify) supfile: + if not os.path.exists(abs_conf_dir + '/supfile.' + repo): + print "Cannot find file " + abs_conf_dir + '/supfile.' + repo + ". Aborting..." + sys.exit() + create_supfile(abs_conf_dir + '/supfile.' + repo, fsup, cvs_tags[arch]) + cmd = 'csup -L 0 -r 0 -g -b ' + absroot + ' -c .sup ' + fsup + os.system(cmd) + else: + absroot = user_absroot + absroots.append(absroot) + curr_include_paths = [] + for repo in include_paths: + curr_include_paths.append(absroot + "/" + repo) + + # Re-init vars for new abs tree: + pkgbuild_deps = {} # pkgname: [dep1, dep2>foo, dep3=bar, ...] + pkgbuild_makedeps = {} # pkgname: [dep1, dep2>foo, dep3=bar, ...] + pkgbuild_provides = {} # provide_name: pkgname + pkgbuild_paths = {} # pkgname: /var/abs/foo/bar/pkgname/PKGBUILD + pkgbuild_archs = {} # pkgname: [i686, x86_64] + pkgbuild_versions = {} # pkname: 1.0.4-2 + # circular_deps is not a dict to accommodate multiple circ deps for a given pkg + circular_deps = [] # pkgname>dep1>dep2>...>pkgname + mismatches = [] + misses = [] + dups = [] + missing_deps = [] + unmet_dep_provs = [] + dep_hierarchy = [] + missing_makedeps = [] + unmet_makedep_provs = [] + makedep_hierarchy = [] + invalid_archs = [] + missing_pkgs = [] + + # Verify stuff for abs tree: + print "\nPerforming integrity checks..." + print "==> parsing pkgbuilds" + get_pkgbuilds_in_dir(absroot) + print "==> checking dependencies" + verify_depends_makedepends() + print "==> checking makedepends" + verify_depends_makedepends(True) + print "==> checking archs" + verify_archs() + print "==> checking for circular dependencies" + circular_deps_check() + print "==> checking repo packages" + verify_packages(arch) + print_results() + if graphdeptree: + visualize_repo() + + print "\nCleaning up temporary abs tree..." + if len(user_absroot) == 0: + # Delete temp abs tree: + if os.path.exists(absroot): + removeall(absroot) + os.rmdir(absroot) +except: + # Cleanup files... + if len(user_absroot) == 0: + print "\nCleaning up temporary abs tree(s)..." + # Delete any temp abs trees: + for path in absroots: + if os.path.exists(path): + removeall(path) + os.rmdir(path) +finally: + # Remove supfile: + if os.path.exists(fsup): + os.remove(fsup) -- cgit v1.2.2 From c417aa306de4329322d99f1067f97a35013610b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20B=C3=A4chler?= Date: Thu, 29 May 2008 15:29:56 -0400 Subject: Fix EXTRAFILES in ftpdir-cleanup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Thomas Bächler Signed-off-by: Aaron Griffin --- cron-jobs/ftpdir-cleanup | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index d6dce11..f8c1a8e 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -70,10 +70,12 @@ done cd "$ftppath" for pkg in *$PKGEXT; do pkgname="$(getpkgname $pkg)" - if [ "$pkg" != "$TMPDIR/$pkgname-*" -a "$(getpkgname $(basename $pkg))" = "$pkgname" ]; then - continue 2 - fi - EXTRAFILES="$EXTRAFILE $pkg" + for p in ${TMPDIR}/${pkgname}-*; do + if [ -d "${p}" -a "$(getpkgname $(basename ${p}))" = "${pkgname}" ]; then + continue 2 + fi + done + EXTRAFILES="$EXTRAFILES $pkg" done cd "$ftppath" -- cgit v1.2.2 From fd921f8dc355ac51a8be843a4da052474b22700c Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Thu, 29 May 2008 15:31:21 -0400 Subject: Move ftpdir-cleanup to misc-scripts Signed-off-by: Aaron Griffin --- cron-jobs/ftpdir-cleanup | 116 -------------------------------------------- misc-scripts/ftpdir-cleanup | 116 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 116 insertions(+), 116 deletions(-) delete mode 100755 cron-jobs/ftpdir-cleanup create mode 100755 misc-scripts/ftpdir-cleanup diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup deleted file mode 100755 index f8c1a8e..0000000 --- a/cron-jobs/ftpdir-cleanup +++ /dev/null @@ -1,116 +0,0 @@ -#!/bin/bash - -if [ $# -ne 2 ]; then - echo "usage: $(basename $0) " - exit 1 -fi - -reponame=$1 -arch=$2 - -##### Arch specific stuff. TODO make this configurable ##### -ftppath="/home/ftp/$reponame/os/$arch/" -############################################################ - -if [ ! -d "$ftppath" ]; then - echo "FTP path '$ftppath' does not exist" - exit 1 -fi - -if [ ! -f /etc/makepkg.conf ]; then - echo "/etc/makepkg.conf not found! Aborting" - exit 1 -fi - -. /etc/makepkg.conf - -getpkgname() { - local tmp - - tmp=${1##*/} - tmp=${tmp%$PKGEXT} - tmp=${tmp%-$arch} - echo ${tmp%-*-*} -} - -getpkgname_ver() { - local tmp - - tmp=${1##*/} - tmp=${tmp%$PKGEXT} - echo ${tmp%-$arch} -} - -MISSINGFILES="" -DELETEFILES="" -EXTRAFILES="" - -TMPDIR=$(mktemp -d /tmp/cleanup.XXXXXX) || exit 1 - -cd "${TMPDIR}" -/bin/tar xzf "$ftppath/$reponame.db.tar.$DB_COMPRESSION" - -for pkg in *; do - filename=$(grep -A1 '^%FILENAME%$' "${pkg}/desc" | tail -n1) - [ -z "${filename}" ] && filename="${pkg}.pkg.tar.gz" - - if [ ! -f "${ftppath}/${filename}" ]; then - MISSINGFILES="${MISSINGFILES} ${filename}" - else - pkgname="$(getpkgname ${filename})" - for otherfile in ${ftppath}/${pkgname}-*; do - otherfile="$(basename ${otherfile})" - if [ "${otherfile}" != "${filename}" -a "${pkgname}" = "$(getpkgname ${otherfile})" ]; then - DELETEFILES="${DELETEFILES} ${otherfile}" - fi - done - fi -done - -cd "$ftppath" -for pkg in *$PKGEXT; do - pkgname="$(getpkgname $pkg)" - for p in ${TMPDIR}/${pkgname}-*; do - if [ -d "${p}" -a "$(getpkgname $(basename ${p}))" = "${pkgname}" ]; then - continue 2 - fi - done - EXTRAFILES="$EXTRAFILES $pkg" -done - -cd "$ftppath" -rm -rf ${TMPDIR} - -echo "Scan complete for $reponame ($arch) at ${ftppath}" -if [ -n "$DELETEFILES" ]; then - echo " The following files are out of date" - echo " They will be moved to /home/package-cleanup" - for f in $DELETEFILES; do - echo " $f" - done -fi -echo "" -if [ -n "$MISSINGFILES" ]; then - echo " The following files are missing in the repo" - for f in $MISSINGFILES; do - echo " $f" - done -fi -echo "" -if [ -n "$EXTRAFILES" ]; then - echo " The following files are in the repo but not the db" - echo " They will be moved to /home/package-cleanup" - for f in $EXTRAFILES; do - echo " $f" - done -fi - -if [ -n "${DELETEFILES}" ]; then - mv ${DELETEFILES} /home/package-cleanup/ - echo "" -fi - -if [ -n "${EXTRAFILES}" ]; then - mv ${EXTRAFILES} /home/package-cleanup/ - echo "" -fi diff --git a/misc-scripts/ftpdir-cleanup b/misc-scripts/ftpdir-cleanup new file mode 100755 index 0000000..f8c1a8e --- /dev/null +++ b/misc-scripts/ftpdir-cleanup @@ -0,0 +1,116 @@ +#!/bin/bash + +if [ $# -ne 2 ]; then + echo "usage: $(basename $0) " + exit 1 +fi + +reponame=$1 +arch=$2 + +##### Arch specific stuff. TODO make this configurable ##### +ftppath="/home/ftp/$reponame/os/$arch/" +############################################################ + +if [ ! -d "$ftppath" ]; then + echo "FTP path '$ftppath' does not exist" + exit 1 +fi + +if [ ! -f /etc/makepkg.conf ]; then + echo "/etc/makepkg.conf not found! Aborting" + exit 1 +fi + +. /etc/makepkg.conf + +getpkgname() { + local tmp + + tmp=${1##*/} + tmp=${tmp%$PKGEXT} + tmp=${tmp%-$arch} + echo ${tmp%-*-*} +} + +getpkgname_ver() { + local tmp + + tmp=${1##*/} + tmp=${tmp%$PKGEXT} + echo ${tmp%-$arch} +} + +MISSINGFILES="" +DELETEFILES="" +EXTRAFILES="" + +TMPDIR=$(mktemp -d /tmp/cleanup.XXXXXX) || exit 1 + +cd "${TMPDIR}" +/bin/tar xzf "$ftppath/$reponame.db.tar.$DB_COMPRESSION" + +for pkg in *; do + filename=$(grep -A1 '^%FILENAME%$' "${pkg}/desc" | tail -n1) + [ -z "${filename}" ] && filename="${pkg}.pkg.tar.gz" + + if [ ! -f "${ftppath}/${filename}" ]; then + MISSINGFILES="${MISSINGFILES} ${filename}" + else + pkgname="$(getpkgname ${filename})" + for otherfile in ${ftppath}/${pkgname}-*; do + otherfile="$(basename ${otherfile})" + if [ "${otherfile}" != "${filename}" -a "${pkgname}" = "$(getpkgname ${otherfile})" ]; then + DELETEFILES="${DELETEFILES} ${otherfile}" + fi + done + fi +done + +cd "$ftppath" +for pkg in *$PKGEXT; do + pkgname="$(getpkgname $pkg)" + for p in ${TMPDIR}/${pkgname}-*; do + if [ -d "${p}" -a "$(getpkgname $(basename ${p}))" = "${pkgname}" ]; then + continue 2 + fi + done + EXTRAFILES="$EXTRAFILES $pkg" +done + +cd "$ftppath" +rm -rf ${TMPDIR} + +echo "Scan complete for $reponame ($arch) at ${ftppath}" +if [ -n "$DELETEFILES" ]; then + echo " The following files are out of date" + echo " They will be moved to /home/package-cleanup" + for f in $DELETEFILES; do + echo " $f" + done +fi +echo "" +if [ -n "$MISSINGFILES" ]; then + echo " The following files are missing in the repo" + for f in $MISSINGFILES; do + echo " $f" + done +fi +echo "" +if [ -n "$EXTRAFILES" ]; then + echo " The following files are in the repo but not the db" + echo " They will be moved to /home/package-cleanup" + for f in $EXTRAFILES; do + echo " $f" + done +fi + +if [ -n "${DELETEFILES}" ]; then + mv ${DELETEFILES} /home/package-cleanup/ + echo "" +fi + +if [ -n "${EXTRAFILES}" ]; then + mv ${EXTRAFILES} /home/package-cleanup/ + echo "" +fi -- cgit v1.2.2 From a90b2f5dc26474c291bd15b4bbddc75afea2731f Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Thu, 29 May 2008 15:34:49 -0400 Subject: Add a real cron script to cycle over all repos Signed-off-by: Aaron Griffin --- cron-jobs/ftpdir-cleanup | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100755 cron-jobs/ftpdir-cleanup diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup new file mode 100755 index 0000000..ce91b62 --- /dev/null +++ b/cron-jobs/ftpdir-cleanup @@ -0,0 +1,35 @@ +#!/bin/bash + +#TODO add community +repos="core extra unstable testing" +arches="i686 x86_64" + +LOCKFILE="/tmp/.ftpdircleanup.lock" + +cleanup () { + rm -f "$LOCKFILE" + exit 0 +} + +ctrl_c() { + cleanup +} + +if [ -f "$LOCKFILE" ]; then + owner="$(/usr/bin/stat -c %U $LOCKFILE)" + echo "error: source tarball generation is already in progress (started by $owner)" + exit 1 +fi + +trap cleanup 0 +trap ctrl_c 2 + +/bin/touch "$LOCKFILE" + +for repo in $repos; do + for arch in $arches; do + $(dirname $0)/../misc-scripts/ftpdir-cleanup $repo $arch + done +done + +cleanup -- cgit v1.2.2 From 9dd4897c4cc8ece483b38a51f9cc9a27cda4d185 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Thu, 29 May 2008 17:01:10 -0400 Subject: make-sourceball fixes * Add a top-level directory to the tarball * Use makepkg -gc and compress the dir - simpler * Other stuff, I'm sure Signed-off-by: Aaron Griffin --- misc-scripts/make-sourceball | 94 ++++++++++++++++++++------------------------ 1 file changed, 43 insertions(+), 51 deletions(-) diff --git a/misc-scripts/make-sourceball b/misc-scripts/make-sourceball index e9e2152..5a3db6b 100755 --- a/misc-scripts/make-sourceball +++ b/misc-scripts/make-sourceball @@ -21,7 +21,7 @@ srcpath="/home/aaron/public_html/sources/" svnpath="file:///home/svn-packages/$packagename/" ############################################################ -WORKDIR="/tmp/make-sourceball.$svnrepo.$UID" +WORKDIR="/tmp/make-sourceball.$packagename.$UID" cleanup() { # unlock @@ -46,47 +46,28 @@ strip_url() { } create_srcpackage() { - echo "Creating source package..." - local comp_files="$BUILDSCRIPT" - echo " Adding: $BUILDSCRIPT" - . $BUILDSCRIPT - - if [ "$install" != "" ]; then - if [ -f $install ]; then - echo " Adding: install script '$install'" - comp_files="$comp_files $install" - else - die "Install script '$install' not found." - fi - fi - - if [ -f ChangeLog ]; then - echo " Adding: ChangeLog" - comp_files="$comp_files ChangeLog" - fi - - local i - for i in ${source[@]}; do - i="$(strip_url "$i")" - if [ -f $i ]; then - echo " Adding: $i" - comp_files="$comp_files $i" - fi - done - - local pkg_file="${pkgname}-${pkgver}-${pkgrel}${SRCEXT}" - - # tar it up - echo "Compressing source package" - if ! /usr/bin/bsdtar -czf "$pkg_file" $comp_files; then - die "Failed to create source package file." - fi - - echo "Source package complete: $pkg_file" - if [ ! -d "$srcpath" ]; then - mkdir -p "$srcpath" + if [ -d "$1" ]; then + pushd "$1" >/dev/null + . "$BUILDSCRIPT" + if ! /usr/bin/makepkg -gc >/dev/null 2>&1; then + popd >/dev/null + return 1 + fi + popd >/dev/null + + local pkg_file="${pkgname}-${pkgver}-${pkgrel}${SRCEXT}" + if ! /usr/bin/bsdtar -czf "$pkg_file" "$1"; then + return 2 + fi + + echo ":: Source package complete: $pkg_file" + if [ ! -d "$srcpath" ]; then + mkdir -p "$srcpath" + fi + cp $pkg_file "$srcpath" + + return 0 fi - cp $pkg_file "$srcpath" } trap ctrl_c 2 @@ -97,19 +78,30 @@ cd "$WORKDIR" echo "Creating Source tarball for $packagename ($reponame-$arch)" -if /usr/bin/svn checkout -N "$svnpath/repos/$reponame-$arch"; then - cd "$reponame-$arch" - if /usr/bin/makepkg -g; then - create_srcpackage - else #try the trunk, it may have updates to the source URL - if /usr/bin/svn checkout -N "$svnpath/trunk"; then - cd "trunk" - if /usr/bin/makepkg -g; then - create_srcpackage +if /usr/bin/svn export -q "$svnpath/repos/$reponame-$arch" $packagename; then + create_srcpackage "$packagename" + if [ $? -eq 0 ]; then + exit 0 + elif [ $? -eq 1 ]; then + #trunk sometimes has updated URLs + echo ":: Failed to download source, attempting trunk build" + rm -rf "$packagename" + if /usr/bin/svn export -q "$svnpath/trunk" "$packagename"; then + create_srcpackage "$packagename" + if [ $? -eq 0 ]; then + echo ":: Source package complete: $pkg_file" + exit 0 + elif [ $? -eq 1 ]; then + die ":: Failed to download source" + elif [ $? -eq 2 ]; then + die ":: Failed to compress package" else - die "Cannot get sources properly. Dying" + die ":: Unknown failure reason" fi fi + exit 1 + elif [ $? -eq 2 ]; then + die ":: Failed to compress package" fi else die "Package '$packagename' does not exist in repo $reponame-$arch" -- cgit v1.2.2 From 9b8a23b0e260dfa54deb2183866a83f8e486ff18 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Thu, 29 May 2008 17:07:01 -0400 Subject: Remove unused code from make-sourceball Signed-off-by: Aaron Griffin --- misc-scripts/make-sourceball | 6 ------ 1 file changed, 6 deletions(-) diff --git a/misc-scripts/make-sourceball b/misc-scripts/make-sourceball index 5a3db6b..71bb405 100755 --- a/misc-scripts/make-sourceball +++ b/misc-scripts/make-sourceball @@ -39,12 +39,6 @@ die() { cleanup 1 } -##### Copied from makepkg ##### - -strip_url() { - echo "$1" | sed 's|^.*://.*/||g' -} - create_srcpackage() { if [ -d "$1" ]; then pushd "$1" >/dev/null -- cgit v1.2.2 From 3d287cbad91a35158444b49840ebfc3f86e6e59a Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Wed, 18 Jun 2008 18:07:08 -0400 Subject: Correct the devlist mailer script to send real emails Signed-off-by: Aaron Griffin --- cron-jobs/devlist-mailer | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/cron-jobs/devlist-mailer b/cron-jobs/devlist-mailer index b9a27e3..094d60d 100755 --- a/cron-jobs/devlist-mailer +++ b/cron-jobs/devlist-mailer @@ -2,15 +2,16 @@ #Dummy helper to send email to arch-dev # It does nothing if no output -#LIST="arch-dev-public@archlinux.org" -LIST="aaronmgriffin@gmail.com" -FROM="cron@archlinux.org" +LIST="arch-dev-public@archlinux.org" +#LIST="aaronmgriffin@gmail.com" +FROM="repomaint" stdin="$(cat)" #echo used to strip whitespace for checking for actual data if [ -n "$(echo $stdin)" ]; then - echo "Subject: Repository Maintenance $(date +"%d-%m-%Y %H:%M") - From: cron@archlinux.org - $stdin" | /usr/sbin/sendmail -F$FROM "$LIST" +echo "Subject: Repository Maintenance $(date +"%d-%m-%Y %H:%M") + +$stdin" | /usr/sbin/sendmail -F$FROM "$LIST" + fi -- cgit v1.2.2 From df172d0deeb21640bd6d557fbd4eff9d99f97a99 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Thu, 19 Jun 2008 18:08:21 -0400 Subject: Make sure ftpdir-cleanup actually does work before outputting Signed-off-by: Aaron Griffin --- misc-scripts/ftpdir-cleanup | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/misc-scripts/ftpdir-cleanup b/misc-scripts/ftpdir-cleanup index f8c1a8e..6cbb893 100755 --- a/misc-scripts/ftpdir-cleanup +++ b/misc-scripts/ftpdir-cleanup @@ -81,7 +81,13 @@ done cd "$ftppath" rm -rf ${TMPDIR} +#Make sure we've done *something* before outputting anything +if [ -z "$DELETEFILES$MISSINGFILES$EXTRAFILES" ]; then + exit 0 +fi + echo "Scan complete for $reponame ($arch) at ${ftppath}" + if [ -n "$DELETEFILES" ]; then echo " The following files are out of date" echo " They will be moved to /home/package-cleanup" -- cgit v1.2.2 From b969bbff9e04b573a7bde42ee98890a5e84cb1f7 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Thu, 19 Jun 2008 18:09:31 -0400 Subject: Make blank lines conditional Mainly for output cleanup Signed-off-by: Aaron Griffin --- misc-scripts/ftpdir-cleanup | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/misc-scripts/ftpdir-cleanup b/misc-scripts/ftpdir-cleanup index 6cbb893..e44dd92 100755 --- a/misc-scripts/ftpdir-cleanup +++ b/misc-scripts/ftpdir-cleanup @@ -94,15 +94,17 @@ if [ -n "$DELETEFILES" ]; then for f in $DELETEFILES; do echo " $f" done + echo "" fi -echo "" + if [ -n "$MISSINGFILES" ]; then echo " The following files are missing in the repo" for f in $MISSINGFILES; do echo " $f" done + echo "" fi -echo "" + if [ -n "$EXTRAFILES" ]; then echo " The following files are in the repo but not the db" echo " They will be moved to /home/package-cleanup" -- cgit v1.2.2 From 5cf896673b4b73d6591f52acbcba5a1dfc803f6c Mon Sep 17 00:00:00 2001 From: Travis Willard Date: Sun, 1 Jun 2008 17:44:09 -0400 Subject: Patch check_archlinux.py to work on our current ABS tree. Patch this on devtools - to run against /home/abs, we'd use: ./check_archlinux.py --abs-tree=/home/abs/rsync/i686 --arch=i686 ./check_archlinux.py --abs-tree=/home/abs/rsync/x86_64 --arch=x86_64 Signed-off-by: Travis Willard Signed-off-by: Aaron Griffin --- cron-jobs/check_archlinux.py | 81 ++++++++++++++++---------------------------- 1 file changed, 30 insertions(+), 51 deletions(-) diff --git a/cron-jobs/check_archlinux.py b/cron-jobs/check_archlinux.py index 8e39c81..055c0a4 100755 --- a/cron-jobs/check_archlinux.py +++ b/cron-jobs/check_archlinux.py @@ -26,6 +26,7 @@ abs_conf_dir = "/etc/abs" valid_archs = ['i686', 'x86_64'] cvs_tags = {'i686': 'CURRENT', 'x86_64': 'CURRENT-64'} include_paths = ['core', 'extra', 'community', 'unstable'] +pkgdir_path_depth = 3 base_server = "ftp.archlinux.org" # Must be ftp site # Ensure that core repo only depends on core, and the extra repo only @@ -115,7 +116,11 @@ def split_dep_prov_symbol(dep): return (dep, prov, symbol) def repo_from_path(path): - return path.split("/")[-4] + # Community HACK: community still has old + # community/category/pkgname/PKGBUILD path - accomodate for this + if path.split("/")[-1 * (pkgdir_path_depth + 1)] == "community": + return path.split("/")[-1 * (pkgdir_path_depth + 1)] + return path.split("/")[-1 * pkgdir_path_depth] def create_supfile(sourcefile, destfile, newtag): o = open(sourcefile, 'r') @@ -264,8 +269,15 @@ def get_pkgbuilds_in_dir(rootdir): pkgbuild_provides[provide] = pkgname if pkgname != name: mismatches.append(pkgname + " vs. " + fpath.replace(absroot, "")) - if not pkgfound and rootdir.replace(absroot, "").count("/") == 3: - misses.append(rootdir.replace(absroot, "") + "/PKGBUILD") + if not pkgfound and rootdir != absroot: + repo = rootdir.replace(absroot, "").split("/")[1] + num_slashes = pkgdir_path_depth - 1 + # Community HACK: community still has old + # community/category/pkgname/PKGBUILD path - accomodate for this + if repo == "community" and rootdir.replace(absroot, "").count("/") == num_slashes + 1 and rootdir.split("/")[-1] != "CVS": + misses.append(rootdir.replace(absroot, "") + "/PKGBUILD") + if repo != "community" and rootdir.replace(absroot, "").count("/") == num_slashes: + misses.append(rootdir.replace(absroot, "") + "/PKGBUILD") def verify_depends_makedepends(verify_makedeps=False): # Make sure all the deps we parsed are actually packages; also @@ -581,17 +593,13 @@ def print_usage(): print "Usage: check_archlinux [OPTION]" print "" print "Options:" - print " --abs-tree= Check specified tree (assumes the abs tree" - print " is i686 unless overridden with --arch)" - print " --arch= Use specified arch (e.g. 'x86_64')" - print " -g Generate graphical dependency tree(s)" - print " -h, --help Show this help and exit" + print " --abs-tree= REQUIRED Check specified tree (assumes the abs tree" + print " is i686 unless overridden with --arch)" + print " --arch= OPTIONAL Use specified arch (e.g. 'x86_64')" + print " -g OPTIONAL Generate graphical dependency tree(s)" + print " -h, --help OPTIONAL Show this help and exit" print "" print "Examples:" - print "\n Check all arches and do fresh cvs checkouts:" - print " check_archlinux" - print "\n Check x_86_64 only and do fresh cvs checkout:" - print " check_archlinux --arch=x86_64" print "\n Check existing i686 abs tree:" print " check_archlinux --abs-tree=/var/abs" print "\n Check existing x86_64 abs tree and also generate dep tree image:" @@ -631,35 +639,21 @@ if opts != []: self.print_usage() sys.exit() -if len(user_absroot) > 0 and len(user_arch) == 0: +if len(user_absroot) == 0: + self.print_usage() + sys.exit() + +if len(user_arch) == 0: user_arch = valid_archs[0] # i686 default.. if len(user_absroot) > 0: print "Warning: Ensure your ABS tree is clean to prevent false positives." try: - absroots = [] - fsup = 'supfile.curr' for arch in valid_archs: if len(user_arch) == 0 or user_arch == arch: print_heading(arch + " Integrity Check") - if len(user_absroot) == 0: - # Create temp dir for cvs checkout: - absroot = tempfile.mkdtemp(prefix="abs-" + arch + "-") - absroots.append(absroot) - print "\nChecking out clean abs tree" - for repo in include_paths: - print "==>", repo - # Create (and modify) supfile: - if not os.path.exists(abs_conf_dir + '/supfile.' + repo): - print "Cannot find file " + abs_conf_dir + '/supfile.' + repo + ". Aborting..." - sys.exit() - create_supfile(abs_conf_dir + '/supfile.' + repo, fsup, cvs_tags[arch]) - cmd = 'csup -L 0 -r 0 -g -b ' + absroot + ' -c .sup ' + fsup - os.system(cmd) - else: - absroot = user_absroot - absroots.append(absroot) + absroot = user_absroot curr_include_paths = [] for repo in include_paths: curr_include_paths.append(absroot + "/" + repo) @@ -702,23 +696,8 @@ try: print_results() if graphdeptree: visualize_repo() - - print "\nCleaning up temporary abs tree..." - if len(user_absroot) == 0: - # Delete temp abs tree: - if os.path.exists(absroot): - removeall(absroot) - os.rmdir(absroot) + except: - # Cleanup files... - if len(user_absroot) == 0: - print "\nCleaning up temporary abs tree(s)..." - # Delete any temp abs trees: - for path in absroots: - if os.path.exists(path): - removeall(path) - os.rmdir(path) -finally: - # Remove supfile: - if os.path.exists(fsup): - os.remove(fsup) + sys.exit() + +# vim: set ts=2 sw=2 noet : -- cgit v1.2.2 From b3120a4c604d4151c8b778f2f4364518e1e66a95 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Fri, 20 Jun 2008 14:26:14 -0400 Subject: Remove some 'self' usage from check_archlinux.py Signed-off-by: Aaron Griffin --- cron-jobs/check_archlinux.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cron-jobs/check_archlinux.py b/cron-jobs/check_archlinux.py index 055c0a4..61b9bc2 100755 --- a/cron-jobs/check_archlinux.py +++ b/cron-jobs/check_archlinux.py @@ -636,11 +636,11 @@ if opts != []: if a in ("play", "pause", "stop", "next", "prev", "pp", "info", "status", "repeat", "shuffle"): self.single_connect_for_passed_arg(a) else: - self.print_usage() + print_usage() sys.exit() if len(user_absroot) == 0: - self.print_usage() + print_usage() sys.exit() if len(user_arch) == 0: -- cgit v1.2.2 From 547b245147b7d44930df46f553067864339f9c43 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Fri, 20 Jun 2008 15:19:52 -0400 Subject: Corrections to the devlist-mailer script * Add configurable subject * Add To: and From: headers Signed-off-by: Aaron Griffin --- cron-jobs/devlist-mailer | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/cron-jobs/devlist-mailer b/cron-jobs/devlist-mailer index 094d60d..b24d530 100755 --- a/cron-jobs/devlist-mailer +++ b/cron-jobs/devlist-mailer @@ -4,13 +4,20 @@ LIST="arch-dev-public@archlinux.org" #LIST="aaronmgriffin@gmail.com" -FROM="repomaint" +FROM="repomaint@archlinux.org" + +SUBJECT="Repository Maintenance $(date +"%d-%m-%Y %H:%M")" +if [ $# -eq 1 ]; then + SUBJECT="$1" +fi stdin="$(cat)" #echo used to strip whitespace for checking for actual data if [ -n "$(echo $stdin)" ]; then -echo "Subject: Repository Maintenance $(date +"%d-%m-%Y %H:%M") +echo "Subject: $SUBJECT +To: $LIST +From: $FROM $stdin" | /usr/sbin/sendmail -F$FROM "$LIST" -- cgit v1.2.2 From f404689fc11bbeace243779305ede5b7d7270ae8 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Mon, 30 Jun 2008 12:43:02 -0400 Subject: Remove community from the integrity check, for now To much output sent to the dev list, this needs to be sent to the aur-general list separately Signed-off-by: Aaron Griffin --- cron-jobs/check_archlinux.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cron-jobs/check_archlinux.py b/cron-jobs/check_archlinux.py index 61b9bc2..cb43351 100755 --- a/cron-jobs/check_archlinux.py +++ b/cron-jobs/check_archlinux.py @@ -25,7 +25,8 @@ abs_conf_dir = "/etc/abs" valid_archs = ['i686', 'x86_64'] cvs_tags = {'i686': 'CURRENT', 'x86_64': 'CURRENT-64'} -include_paths = ['core', 'extra', 'community', 'unstable'] +#include_paths = ['core', 'extra', 'community', 'unstable'] +include_paths = ['core', 'extra', 'unstable'] pkgdir_path_depth = 3 base_server = "ftp.archlinux.org" # Must be ftp site -- cgit v1.2.2 From 909b017c08109bda405a3e38a59cbf26211e6683 Mon Sep 17 00:00:00 2001 From: Xavier Chantry Date: Fri, 29 Aug 2008 20:23:26 +0200 Subject: Replace check_archlinux.py by check_archlinux/check_packages.py The old script had several problems so I decided to do a full rewrite. The improvements include : * better and safer parsing of PKGBUILDs It now uses separate parse_pkgbuilds.sh bash script (inspired from namcap) * much better performance A python module for calling vercmp natively, and the algorithm for checking circular dependencies was greatly improved * more accurate dependency and provision handling Now versioned dependencies and provisions are handled correctly. After building the python module and moving it next to the main script, it should be possible to use it like this : For core and extra : ./check_packages.py --abs-tree=/home/abs/rsync/i686 --repos=core,extra ./check_packages.py --abs-tree=/home/abs/rsync/x86_64 --repos=core,extra For community : ./check_packages.py --abs-tree=/home/abs/rsync/i686 --repos=community ./check_packages.py --abs-tree=/home/abs/rsync/x86_64 --repos=community Signed-off-by: Xavier Chantry Signed-off-by: Aaron Griffin --- cron-jobs/check_archlinux.py | 704 --------------------------- cron-jobs/check_archlinux/README | 8 + cron-jobs/check_archlinux/alpm.c | 40 ++ cron-jobs/check_archlinux/check_packages.py | 392 +++++++++++++++ cron-jobs/check_archlinux/parse_pkgbuilds.sh | 78 +++ cron-jobs/check_archlinux/setup.py | 10 + 6 files changed, 528 insertions(+), 704 deletions(-) delete mode 100755 cron-jobs/check_archlinux.py create mode 100644 cron-jobs/check_archlinux/README create mode 100644 cron-jobs/check_archlinux/alpm.c create mode 100755 cron-jobs/check_archlinux/check_packages.py create mode 100755 cron-jobs/check_archlinux/parse_pkgbuilds.sh create mode 100644 cron-jobs/check_archlinux/setup.py diff --git a/cron-jobs/check_archlinux.py b/cron-jobs/check_archlinux.py deleted file mode 100755 index cb43351..0000000 --- a/cron-jobs/check_archlinux.py +++ /dev/null @@ -1,704 +0,0 @@ -#!/usr/bin/env python -# -# check_archlinux.py -# -# Written by Scott Horowitz -# Graphical dependency tree by Cesar G. Miguel -# -# This script currently checks for a number of issues in your ABS tree: -# 1. Directories with missing PKGBUILDS -# 2. Duplicate PKGBUILDs -# 3. Missing (make-)dependencies, taking into account 'provides' -# 4. Provisioned dependencies -# 5. Circular dependencies -# 6. Valid arch's in PKGBUILDS -# 7. Missing packages in Arch repos -# 8. PKGBUILD names that don't match their directory -# 9. Hierarchy of repos (e.g., that a core package doesn't depend on -# a non-core package) -# It can also, optionally, generate a graphical representation of the -# dependency tree. -# -# Todo: -# 1. Accommodate testing repo? - -abs_conf_dir = "/etc/abs" -valid_archs = ['i686', 'x86_64'] -cvs_tags = {'i686': 'CURRENT', 'x86_64': 'CURRENT-64'} -#include_paths = ['core', 'extra', 'community', 'unstable'] -include_paths = ['core', 'extra', 'unstable'] -pkgdir_path_depth = 3 - -base_server = "ftp.archlinux.org" # Must be ftp site -# Ensure that core repo only depends on core, and the extra repo only -# depends on core and extra, etc. -repo_hierarchy = {'core': ('core'), \ -'extra': ('core', 'extra'), \ -'unstable': ('core', 'extra', 'unstable'), \ -'community': ('core', 'extra', 'unstable', 'community')} - -pkgname_str = "pkgname=" -dep_str = "depends=(" -makedep_str = "makedepends=(" -provides_str = "provides=(" -arch_str = "arch=(" -pkgver_str = "pkgver=" -pkgrel_str = "pkgrel=" -build_str = "build()" -source_str = "source=" -url_str = "url=" -mathlist = (">=", "<=", "=", ">", "<") -sup_tag_str = "*default tag=" - -import os, sys, getopt, tempfile -from ftplib import FTP -try: - import pydot - HAS_PYDOT = True -except: - HAS_PYDOT = False - -def print_heading(heading): - print "" - print "=" * (len(heading) + 4) - print "= " + heading + " =" - print "=" * (len(heading) + 4) - -def print_subheading(subheading): - print "" - print subheading - print "-" * (len(subheading) + 2) - -def rmgeneric(path, __func__): - try: - __func__(path) - except OSError, (errno, strerror): - pass - -def removeall(path): - if not os.path.isdir(path): - return - - files=os.listdir(path) - - for x in files: - fullpath=os.path.join(path, x) - if os.path.isfile(fullpath): - f=os.remove - rmgeneric(fullpath, f) - elif os.path.isdir(fullpath): - removeall(fullpath) - f=os.rmdir - rmgeneric(fullpath, f) - -def update_var(line, user_vars, pkgpath): - if line.count("$") > 0: - export_line = "" - for var in user_vars: - if line.count(var[0]) > 0: - export_line = export_line + var[0] + "=" + var[1] + " && " - if line.startswith("(") and line.endswith(")"): - line = line[1:-1] - export_line = export_line + "echo \"" + line + "\"" - line = os.popen(export_line).read().replace("\n", "") - return line - -def split_dep_prov_symbol(dep): - # Splits 'foo>=1.2.3' into ('foo', '1.2.3', '>=') - prov = "" - symbol = "" - for char in mathlist: - pos = dep.find(char) - if pos > -1: - prov = dep[pos:].replace(char, "") - dep = dep[:pos] - symbol = char - break - return (dep, prov, symbol) - -def repo_from_path(path): - # Community HACK: community still has old - # community/category/pkgname/PKGBUILD path - accomodate for this - if path.split("/")[-1 * (pkgdir_path_depth + 1)] == "community": - return path.split("/")[-1 * (pkgdir_path_depth + 1)] - return path.split("/")[-1 * pkgdir_path_depth] - -def create_supfile(sourcefile, destfile, newtag): - o = open(sourcefile, 'r') - info = o.read() - o.close() - lines = info.split("\n") - - o = open(destfile, 'w') - for line in lines: - line = line.strip() - if line[:len(sup_tag_str)] == sup_tag_str: - line = sup_tag_str + newtag - o.write(line + "\n") - o.close() - -def get_deps_provides_etc(pkgpath): - # Parse PKGBUILD for name, depends, makedepends, provides, arch's, and version - o = open(pkgpath, 'r') - info = o.read() - o.close() - lines = info.split("\n") - - deps = [] - provides = [] - archs = [] - makedeps = [] - array = [] - user_vars = [] - continue_line = False - for line in lines: - line = line.strip() - if line.find("#") > -1: - line = line[:line.find("#")].strip() - if not continue_line: - deps_line = False - provides_line = False - arch_line = False - makedeps_line = False - if line[:len(dep_str)] == dep_str: - line = line.replace(dep_str,"") - deps_line = True - elif line[:len(makedep_str)] == makedep_str: - line = line.replace(makedep_str,"") - makedeps_line = True - elif line[:len(provides_str)] == provides_str: - line = line.replace(provides_str,"") - provides_line = True - elif line[:len(arch_str)] == arch_str: - line = line.replace(arch_str, "") - arch_line = True - elif line[:len(pkgname_str)] == pkgname_str: - pkgname = line.replace(pkgname_str, "") - if pkgname.startswith("\"") and pkgname.endswith("\""): - pkgname = pkgname[1:-1] - pkgname = update_var(pkgname, user_vars, pkgpath) - user_vars.append([pkgname_str, pkgname]) - line = "" - elif line[:len(pkgver_str)] == pkgver_str: - pkgver = line.replace(pkgver_str, "") - if pkgver.startswith("\"") and pkgver.endswith("\""): - pkgver = pkgver[1:-1] - pkgver = update_var(pkgver, user_vars, pkgpath) - user_vars.append([pkgver_str[:-1], pkgver]) - line = "" - elif line[:len(pkgrel_str)] == pkgrel_str: - pkgrel = line.replace(pkgrel_str, "") - if pkgrel.startswith("\"") and pkgrel.endswith("\""): - pkgrel = pkgrel[1:-1] - pkgrel = update_var(pkgrel, user_vars, pkgpath) - user_vars.append([pkgrel_str[:-1], pkgrel]) - line = "" - elif line[:len(build_str)] == build_str: - break - elif not continue_line: - if line.count("=") == 1 and line.count(" ") == 0 and line[:1] != "#" and \ - line[:len(source_str)] != source_str and line[:len(url_str)] != url_str: - split = line.split("=") - for item in range(len(split)): - split[item] = update_var(split[item], user_vars, pkgpath) - user_vars.append(split) - line = "" - if len(line) > 0: - pos = line.find(")") - if pos > -1: - # strip everything from closing paranthesis on - # since some PKGBUILDS have comments after the - # depends array - line = line[:pos] - line = line.split(' ') - for i in range(len(line)): - line[i] = line[i].replace("'","").replace('"','') - line[i] = update_var(line[i], user_vars, pkgpath) - if len(line[i]) > 0: - if deps_line: - deps.append(line[i]) - array=deps - elif provides_line: - provides.append(line[i]) - array=provides - elif arch_line: - archs.append(line[i]) - array=archs - elif makedeps_line: - makedeps.append(line[i]) - array=makedeps - if array and (array[-1] == "\\" or array[-1][-1] == "\\"): - # continue reading deps/provides on next line - if array[-1] == "\\": - array.pop(-1) - else: - array[-1] = array[-1].replace("\\", "") - continue_line = True - else: - continue_line = False - version = pkgver + "-" + pkgrel - return (pkgname, deps, makedeps, provides, archs, version) - -def get_pkgbuilds_in_dir(rootdir): - # Recursively populates pkgbuild_deps, pkgbuild_paths, etc. - # dicts with info from each PKGBUILD found in rootdir: - if rootdir != absroot: - if rootdir.count("/") == (absroot.count("/")+1) and rootdir not in curr_include_paths: - return - pkgfound = False - for f in os.listdir(rootdir): - fpath = rootdir + "/" + f - if os.path.isdir(fpath): - get_pkgbuilds_in_dir(fpath) - elif f == 'PKGBUILD': - pkgfound = True - name = rootdir.split("/")[-1] - if name in pkgbuild_deps: - dups.append(fpath.replace(absroot, "") + " vs. " + pkgbuild_paths[name].replace(absroot, "")) - else: - (pkgname, deps, makedeps, provides, archs, version) = get_deps_provides_etc(fpath) - pkgbuild_deps[pkgname] = deps - pkgbuild_makedeps[pkgname] = makedeps - pkgbuild_paths[pkgname] = fpath - pkgbuild_archs[pkgname] = archs - pkgbuild_versions[pkgname] = version - # We'll store the provides "backwards" compared to - # the other dicts. This will make searching for - # provides easier by being able to look up the - # provide name itself and find what provides it - for provide in provides: - pkgbuild_provides[provide] = pkgname - if pkgname != name: - mismatches.append(pkgname + " vs. " + fpath.replace(absroot, "")) - if not pkgfound and rootdir != absroot: - repo = rootdir.replace(absroot, "").split("/")[1] - num_slashes = pkgdir_path_depth - 1 - # Community HACK: community still has old - # community/category/pkgname/PKGBUILD path - accomodate for this - if repo == "community" and rootdir.replace(absroot, "").count("/") == num_slashes + 1 and rootdir.split("/")[-1] != "CVS": - misses.append(rootdir.replace(absroot, "") + "/PKGBUILD") - if repo != "community" and rootdir.replace(absroot, "").count("/") == num_slashes: - misses.append(rootdir.replace(absroot, "") + "/PKGBUILD") - -def verify_depends_makedepends(verify_makedeps=False): - # Make sure all the deps we parsed are actually packages; also - # ensure we meet dep provisions. - if verify_makedeps: - array = pkgbuild_makedeps - else: - array = pkgbuild_deps - for pkgname in array: - deps = array[pkgname] - pkg_repo = repo_from_path(pkgbuild_paths[pkgname]) - deps_to_pop = [] - for i in range(len(deps)): - dep = deps[i] - (dep, prov, char) = split_dep_prov_symbol(dep) - try: - x = pkgbuild_deps[dep] - # Check that prov is met too: - if len(prov) > 0: - compare_str = "vercmp " + pkgbuild_versions[dep] + " " + prov - error = False - if char == "<=": - if int(os.popen(compare_str).read().replace("\n", "")) > 0: - error = True - elif char == ">=": - if int(os.popen(compare_str).read().replace("\n", "")) < 0: - error = True - elif char == "=": - if int(os.popen(compare_str).read().replace("\n", "")) != 0: - error = True - elif char == ">": - if int(os.popen(compare_str).read().replace("\n", "")) <= 0: - error = True - elif char == "<": - if int(os.popen(compare_str).read().replace("\n", "")) >= 0: - error = True - if error: - if verify_makedeps: - unmet_makedep_provs.append(pkgname + " --> '" + dep + char + prov + "'") - else: - unmet_dep_provs.append(pkgname + " --> '" + dep + char + prov + "'") - # Check repos fit hierarchy scheme: - dep_repo = repo_from_path(pkgbuild_paths[dep]) - try: - valid_repos = repo_hierarchy[pkg_repo] - # Make sure dep repo is one of the valid repos: - if dep_repo not in valid_repos: - if verify_makedeps: - makedep_hierarchy.append(pkg_repo + "/" + pkgname + " depends on " + dep_repo + "/" + dep) - else: - dep_hierarchy.append(pkg_repo + "/" + pkgname + " depends on " + dep_repo + "/" + dep) - except: - pass - except: - # Check if a package provides this dep: - try: - x = pkgbuild_provides[dep] - except: - if verify_makedeps: - missing_makedeps.append(pkgname + " --> '" + dep + "'") - else: - missing_deps.append(pkgname + " --> '" + dep + "'") - deps_to_pop.append(i) - # Pop deps not found from end to beginning: - while len(deps_to_pop) > 0: - deps.pop(deps_to_pop[-1]) - deps_to_pop.pop(-1) - -def verify_archs(): - for pkgname in pkgbuild_archs: - newarch = [] - archs = pkgbuild_archs[pkgname] - for arch in archs: - if arch not in valid_archs: - invalid_archs.append(pkgname + " --> " + arch) - else: - newarch.append(arch) - if len(newarch) > 0: - pkgbuild_archs[pkgname] = newarch - -def verify_packages(tree_arch): - # Make sure packages exist in Arch repo(s): - ftp = FTP(base_server) - ftp.login() - prev_wd = "" - # Find all repos/archs; marching through them in order will greatly speed - # up searching for ftp files by minimizing the number of .nlst's that we - # have to do. - repos = [] - for pkgname in pkgbuild_paths: - pkgrepo = repo_from_path(pkgbuild_paths[pkgname]) - if not pkgrepo in repos: - repos.append(pkgrepo) - archs = [] - for pkgname in pkgbuild_archs: - pkgarchs = pkgbuild_archs[pkgname] - for arch in pkgarchs: - if not arch in archs: - archs.append(arch) - for r in repos: - for pkgname in pkgbuild_archs: - repo = repo_from_path(pkgbuild_paths[pkgname]) - if repo == r: - archs = pkgbuild_archs[pkgname] - pkgver_rel = pkgbuild_versions[pkgname] - for arch in archs: - if arch == tree_arch: - # Check for file: - wd = repo + "/os/" + arch - if wd != prev_wd: - ftpfiles = ftp.nlst(wd) - prev_wd = wd - fname_new = wd + "/" + pkgname + "-" + pkgver_rel + "-" + arch + ".pkg.tar.gz" - fname_old = wd + "/" + pkgname + "-" + pkgver_rel + ".pkg.tar.gz" - if fname_old not in ftpfiles and fname_new not in ftpfiles: - missing_pkgs.append(pkgname + "-" + pkgver_rel + " in " + wd) - ftp.quit() - -def subset_superset_path(currpath, currpathnum, paths): - # If a pair of subset/superset paths are found, - # pop the superset one to show the more minimal - # case. - # - # e.g. foo > bar > baz > foo (superset) - # foo > bar > foo (subset) - # --> pop the superset - # - currdeps = currpath.split(">") - currdeps = list(set(currdeps)) - currdeps.sort() - pathnum = 0 - for path in paths: - if pathnum != currpathnum: - deps = path.split(">") - deps = list(set(deps)) - deps.sort() - if len(currdeps) < len(path): - subset = True - for d in currdeps: - if not d in path: - subset = False - break - if subset: - circular_deps.pop(pathnum) - if pathnum <= currpathnum: - currpathnum -= 1 - elif len(currdeps) > len(path): - superset = True - for d in path: - if not d in currdeps: - superset = False - break - if superset: - circular_deps.pop(currpathnum) - currpathnum -= 1 - pathnum += 1 - return True - -def unique_path(currpath, paths): - # Returns false if an equivalent path exists in - # paths. - # - # e.g. foo > bar > foo - # bar > foo > bar - # - currdeps = currpath.split(">") - currdeps = list(set(currdeps)) - currdeps.sort() - for path in paths: - deps = path.split(">") - deps = list(set(deps)) - deps.sort() - if currdeps == deps: - return False - return True - -def update_paths(paths, dep, dep2): - # Update paths by appending new paths with dep2 - # based on all the different ways we could get - # to dep2. Returns True if a path was updated. - new_path = False - for i in range(len(paths)): - array = paths[i].split(">") - newpath = paths[i] + ">" + dep2 - if array[-1] == dep and not dep2 in array and unique_path(newpath, paths): - paths.append(newpath) - new_path = True - return new_path - -def check_name_in_recursive_deps(pkgname): - # Retrieve all recursive dependencies from a package and - # determines if pkgname is found in one of its deps. - recursive_deps = [] - for dep in pkgbuild_deps[pkgname]: - dep = split_dep_prov_symbol(dep)[0] # Strip any provision - recursive_deps.append(dep) - paths = [] - for dep in recursive_deps: - dep = split_dep_prov_symbol(dep)[0] # Strip any provision - paths.append(dep) - searching = True - while searching: - searching = False - for dep in recursive_deps: - for dep2 in pkgbuild_deps[dep]: - dep2 = split_dep_prov_symbol(dep2)[0] # Strip any provision - # This is a HUGE time-saver. Instead of generating every single - # possible path that can yield a circular dep, we'll reduce - # the number greatly by throwing out paths like such: - # - # If we have a path: foo>bar>baz>blah>poop>foo - # We will throw out any: - # foo>...>bar>baz>blah>poop>foo - # foo>...>baz>blah>poop>foo - # foo>...>blah>poop>foo - # and so on. Otherwise we will find hundreds or even thousands - # of possible paths that all essentially represent the same - # circular dep. - # - # However, we will always let pkgname through in order to make - # sure we can find multiple circular deps for a given pkg. - if dep2 not in recursive_deps or dep2 == pkgname: - updated = update_paths(paths, dep, dep2) - if dep2 not in recursive_deps: - recursive_deps.append(dep2) - if updated: - searching = True - # Done searching, store circular deps: - for path in paths: - if path.split(">")[-1] == pkgname: - if unique_path(pkgname + ">" + path, circular_deps): - circular_deps.append(pkgname + ">" + path) - # Reduce any subset/superset path pairs: - pathnum = 0 - for path in circular_deps: - subset_superset_path(path, pathnum, circular_deps) - pathnum += 1 - -def circular_deps_check(): - # Check for circular dependencies: - for pkgname in pkgbuild_deps: - check_name_in_recursive_deps(pkgname) - -def visualize_repo(): - output = 'digraph G { \n \ - concentrate = true; \n \ - ordering = out; \n \ - ranksep=5.0; \n \ - node [style=filled,fontsize=8]; \n' - - # draws circular dependencies in red - for path in circular_deps: - output += '\t "'+path[0]+'"' - deps = path.split(">") - for d in deps: - output += ' -> "'+d+'"' - output += ' [color=red]\n' - - for pkg in pkgbuild_deps.keys(): - output += '\t "'+pkg+'" -> { ' - for d in pkgbuild_deps[pkg]: - d = split_dep_prov_symbol(d)[0] # Strip any provision - output += '"'+d+'"; ' - output += '}\n' - - output += '}' - - # Uncomment these lines to get a file dump called - # 'output'. This can be used to manually generate - # an image using, e.g., dot -Tsvg output tree.svg - #dump = open('output', 'w') - #dump.write(output) - #dump.close() - - fname = 'dependency_tree-' + arch + '.svg' - print "Generating " + fname + "..." - g = pydot.graph_from_dot_data(output) - g.write(fname, prog='dot', format='svg') - -def print_result(list, subheading): - if len(list) > 0: - print_subheading(subheading) - for item in list: - print item - -def print_results(): - print_result(misses, "Missing PKGBUILDs") - print_result(mismatches, "Mismatched Pkgnames") - print_result(dups, "Duplicate PKGBUILDs") - print_result(missing_deps, "Missing Dependencies") - print_result(missing_makedeps, "Missing Makedepends") - print_result(unmet_dep_provs, "Unmet Dependency Provisions") - print_result(unmet_makedep_provs, "Unmet Makedepends Provisions") - print_result(dep_hierarchy, "Repo Hierarchy for Dependencies") - print_result(makedep_hierarchy, "Repo Hierarchy for Makedepends") - print_result(invalid_archs, "Invalid Archs") - print_result(circular_deps, "Circular Dependencies") - print_result(missing_pkgs, "Missing Repo Packages") - print_subheading("Summary") - print "Dirs with missing PKGBUILDs: ", len(misses) - print "Duplicate PKGBUILDs: ", len(dups) - print "Missing (make)dependencies: ", len(missing_deps)+len(missing_makedeps) - print "Unmet provisioned (make)dependencies: ", len(unmet_dep_provs)+len(unmet_makedep_provs) - print "Circular dependencies: ", len(circular_deps) - print "Invalid archs: ", len(invalid_archs) - print "Missing packages in repos: ", len(missing_pkgs) - print "Mismatching PKGBUILD names: ", len(mismatches) - print "Repo hierarchy problems: ", len(dep_hierarchy)+len(makedep_hierarchy) - print "" - -def print_usage(): - print "" - print "Usage: check_archlinux [OPTION]" - print "" - print "Options:" - print " --abs-tree= REQUIRED Check specified tree (assumes the abs tree" - print " is i686 unless overridden with --arch)" - print " --arch= OPTIONAL Use specified arch (e.g. 'x86_64')" - print " -g OPTIONAL Generate graphical dependency tree(s)" - print " -h, --help OPTIONAL Show this help and exit" - print "" - print "Examples:" - print "\n Check existing i686 abs tree:" - print " check_archlinux --abs-tree=/var/abs" - print "\n Check existing x86_64 abs tree and also generate dep tree image:" - print " check_archlinux --abs-tree=/var/abs --arch=x86_64 -g" - print "" - -graphdeptree = False -user_absroot = "" -user_arch = "" -try: - opts, args = getopt.getopt(sys.argv[1:], "g", ["abs-tree=", "arch="]) -except getopt.GetoptError: - print_usage() - sys.exit() -if opts != []: - for o, a in opts: - if o in ("-g"): - graphdeptree = True - if not HAS_PYDOT: - print "You must install pydot to generate a graphical dependency tree. Aborting..." - sys.exit() - elif o in ("--abs-tree"): - user_absroot = a - elif o in ("--arch"): - user_arch = a - if user_arch not in valid_archs: - print "You did not specify a valid arch. Aborting..." - sys.exit() - else: - print_usage() - sys.exit() - if args != []: - for a in args: - if a in ("play", "pause", "stop", "next", "prev", "pp", "info", "status", "repeat", "shuffle"): - self.single_connect_for_passed_arg(a) - else: - print_usage() - sys.exit() - -if len(user_absroot) == 0: - print_usage() - sys.exit() - -if len(user_arch) == 0: - user_arch = valid_archs[0] # i686 default.. - -if len(user_absroot) > 0: - print "Warning: Ensure your ABS tree is clean to prevent false positives." - -try: - for arch in valid_archs: - if len(user_arch) == 0 or user_arch == arch: - print_heading(arch + " Integrity Check") - absroot = user_absroot - curr_include_paths = [] - for repo in include_paths: - curr_include_paths.append(absroot + "/" + repo) - - # Re-init vars for new abs tree: - pkgbuild_deps = {} # pkgname: [dep1, dep2>foo, dep3=bar, ...] - pkgbuild_makedeps = {} # pkgname: [dep1, dep2>foo, dep3=bar, ...] - pkgbuild_provides = {} # provide_name: pkgname - pkgbuild_paths = {} # pkgname: /var/abs/foo/bar/pkgname/PKGBUILD - pkgbuild_archs = {} # pkgname: [i686, x86_64] - pkgbuild_versions = {} # pkname: 1.0.4-2 - # circular_deps is not a dict to accommodate multiple circ deps for a given pkg - circular_deps = [] # pkgname>dep1>dep2>...>pkgname - mismatches = [] - misses = [] - dups = [] - missing_deps = [] - unmet_dep_provs = [] - dep_hierarchy = [] - missing_makedeps = [] - unmet_makedep_provs = [] - makedep_hierarchy = [] - invalid_archs = [] - missing_pkgs = [] - - # Verify stuff for abs tree: - print "\nPerforming integrity checks..." - print "==> parsing pkgbuilds" - get_pkgbuilds_in_dir(absroot) - print "==> checking dependencies" - verify_depends_makedepends() - print "==> checking makedepends" - verify_depends_makedepends(True) - print "==> checking archs" - verify_archs() - print "==> checking for circular dependencies" - circular_deps_check() - print "==> checking repo packages" - verify_packages(arch) - print_results() - if graphdeptree: - visualize_repo() - -except: - sys.exit() - -# vim: set ts=2 sw=2 noet : diff --git a/cron-jobs/check_archlinux/README b/cron-jobs/check_archlinux/README new file mode 100644 index 0000000..f3a1b90 --- /dev/null +++ b/cron-jobs/check_archlinux/README @@ -0,0 +1,8 @@ +1) Build the python module +$ python setup.py build + +2) copy it back to the current working directory +$ cp build/lib.*/alpm.* . + +3) run the script +$ ./check_packages.py -h diff --git a/cron-jobs/check_archlinux/alpm.c b/cron-jobs/check_archlinux/alpm.c new file mode 100644 index 0000000..0b7cd2c --- /dev/null +++ b/cron-jobs/check_archlinux/alpm.c @@ -0,0 +1,40 @@ +#include +#include + +static PyObject * +alpm_vercmp(PyObject *self, PyObject *args) +{ + const char *v1, *v2; + int ret; + + if (!PyArg_ParseTuple(args, "ss", &v1, &v2)) + return NULL; + ret = alpm_pkg_vercmp(v1, v2); + return Py_BuildValue("i", ret); +} + +static PyMethodDef AlpmMethods[] = { + {"vercmp", alpm_vercmp, METH_VARARGS, + "Execute vercmp."}, + {NULL, NULL, 0, NULL} /* Sentinel */ +}; + +PyMODINIT_FUNC +initalpm(void) +{ + (void) Py_InitModule("alpm", AlpmMethods); +} + +int +main(int argc, char *argv[]) +{ + /* Pass argv[0] to the Python interpreter */ + Py_SetProgramName(argv[0]); + + /* Initialize the Python interpreter. Required. */ + Py_Initialize(); + + /* Add a static module */ + initalpm(); + return 0; +} diff --git a/cron-jobs/check_archlinux/check_packages.py b/cron-jobs/check_archlinux/check_packages.py new file mode 100755 index 0000000..2d2efbe --- /dev/null +++ b/cron-jobs/check_archlinux/check_packages.py @@ -0,0 +1,392 @@ +#!/usr/bin/python +# +# check_archlinux.py +# +# Original script by Scott Horowitz +# Rewritten by Xavier Chantry +# +# This script currently checks for a number of issues in your ABS tree: +# 1. Directories with missing PKGBUILDS +# 2. Invalid PKGBUILDs (bash syntax error for instance) +# 3. PKGBUILD names that don't match their directory +# 4. Duplicate PKGBUILDs +# 5. Valid arch's in PKGBUILDS +# 6. Missing (make-)dependencies +# 7. Hierarchy of repos (e.g., that a core package doesn't depend on +# a non-core package) +# 8. Circular dependencies + +import os,re,commands,getopt,sys,alpm +import pdb + +packages = {} # pkgname : PacmanPackage +provisions = {} # provision : PacmanPackage +pkgdeps,makepkgdeps = {},{} # pkgname : list of the PacmanPackage dependencies +invalid_pkgbuilds = [] +missing_pkgbuilds = [] +dups = [] + +mismatches = [] +missing_deps = [] +missing_makedeps = [] +invalid_archs = [] +dep_hierarchy = [] +makedep_hierarchy = [] +circular_deps = [] # pkgname>dep1>dep2>...>pkgname +checked_deps = [] + +class PacmanPackage: + def __init__(self): + self.name,self.version = "","" + self.path,self.repo = "","" + self.deps,self.makedeps = [],[] + self.provides,self.conflicts = [],[] + self.archs = [] + +class Depend: + def __init__(self,name,version,mod): + self.name = name + self.version = version + self.mod = mod + +def parse_pkgbuilds(repos): + oldcwd = os.getcwd() + os.chdir(absroot) + for repo in repos: + data = commands.getoutput(oldcwd + '/parse_pkgbuilds.sh ' + repo) + parse_data(repo,data) + os.chdir(oldcwd) + +def parse_data(repo,data): + attrname = None + + for line in data.split('\n'): + if line.startswith('%'): + attrname = line.strip('%').lower() + elif line.strip() == '': + attrname = None + elif attrname == "invalid": + if repo in repos: + invalid_pkgbuilds.append(line) + elif attrname == "missing": + if repo in repos: + missing_pkgbuilds.append(line) + elif attrname == "name": + pkg = PacmanPackage() + pkg.name = line + pkg.repo = repo + dup = None + if packages.has_key(pkg.name): + dup = packages[pkg.name] + packages[pkg.name] = pkg + elif attrname == "version": + pkg.version = line + elif attrname == "path": + pkg.path = line + if dup != None and (pkg.repo in repos or dup.repo in repos): + dups.append(pkg.path + " vs. " + dup.path) + elif attrname == "arch": + pkg.archs.append(line) + elif attrname == "depends": + pkg.deps.append(line) + elif attrname == "makedepends": + pkg.makedeps.append(line) + elif attrname == "conflicts": + pkg.conflicts.append(line) + elif attrname == "provides": + pkg.provides.append(line) + provname=line.split("=")[0] + if not provisions.has_key(provname): + provisions[provname] = [] + provisions[provname].append(pkg) + +def splitdep(dep): + name = dep + version = "" + mod = "" + for char in (">=", "<=", "=", ">", "<"): + pos = dep.find(char) + if pos > -1: + name = dep[:pos] + version = dep[pos:].replace(char, "") + mod = char + break + return Depend(name,version,mod) + +def splitprov(prov): + name = prov + version = "" + pos = prov.find("=") + if pos > -1: + name = prov[:pos] + version = prov[pos:].replace("=", "") + return (name,version) + +def vercmp(v1,mod,v2): + res = alpm.vercmp(v1,v2) + if res == 0: + return (mod.find("=") > -1) + elif res < 0: + return (mod.find("<") > -1) + elif res > 0: + return (mod.find(">") > -1) + return False + + +def depcmp(name,version,dep): + if name != dep.name: + return False + if dep.version == "" or dep.mod == "": + return True + if version == "": + return False + return vercmp(version,dep.mod,dep.version) + +def provcmp(pkg,dep): + for prov in pkg.provides: + (provname,provver) = splitprov(prov) + if depcmp(provname,provver,dep): + return True + return False + +def verify_dep(dep): + dep = splitdep(dep) + if packages.has_key(dep.name): + pkg = packages[dep.name] + if depcmp(pkg.name,pkg.version,dep): + return [pkg] + if provisions.has_key(dep.name): + provlist = provisions[dep.name] + results = [] + for prov in provlist: + if provcmp(prov,dep): + results.append(prov) + return results + return [] + +def verify_deps(name,repo,deps): + pkg_deps = [] + missdeps = [] + hierarchy = [] + for dep in deps: + pkglist = verify_dep(dep) + if pkglist == []: + missdeps.append(name + " --> '" + dep + "'") + else: + valid_repos = get_repo_hierarchy(repo) + pkgdep = None + for pkg in pkglist: + if pkg.repo in valid_repos: + pkgdep = pkg + break + if not pkgdep: + pkgdep = pkglist[0] + hierarchy.append(repo + "/" + name + " depends on " + pkgdep.repo + "/" + pkgdep.name) + pkg_deps.append(pkgdep) + + return (pkg_deps,missdeps,hierarchy) + +def get_repo_hierarchy(repo): + repo_hierarchy = {'core': ['core'], \ + 'extra': ['core', 'extra'], \ + 'community': ['core', 'extra', 'community']} + if repo_hierarchy.has_key(repo): + return repo_hierarchy[repo] + else: + return ['core','extra','community'] + +def verify_archs(name,archs): + valid_archs = ['i686', 'x86_64'] + invalid_archs = [] + for arch in archs: + if arch not in valid_archs: + invalid_archs.append(name + " --> " + arch) + return invalid_archs + +def find_scc(packages): + # reset all variables + global index,S,pkgindex,pkglowlink + index = 0 + S = [] + pkgindex = {} + pkglowlink = {} + cycles = [] + for pkg in packages: + tarjan(pkg) + +def tarjan(pkg): + global index,S,pkgindex,pkglowlink,cycles + pkgindex[pkg] = index + pkglowlink[pkg] = index + index += 1 + checked_deps.append(pkg) + S.append(pkg) + if pkgdeps.has_key(pkg): + deps = pkgdeps[pkg] + else: + print pkg.name + deps = [] + for dep in deps: + if not pkgindex.has_key(dep): + tarjan(dep) + pkglowlink[pkg] = min(pkglowlink[pkg],pkglowlink[dep]) + elif dep in S: + pkglowlink[pkg] = min(pkglowlink[pkg],pkgindex[dep]) + if pkglowlink[pkg] == pkgindex[pkg]: + dep = S.pop() + if pkg == dep: + return + path = pkg.name + while pkg != dep: + path = dep.name + ">" + path + dep = S.pop() + path = dep.name + ">" + path + if pkg.repo in repos: + circular_deps.append(path) + +def print_heading(heading): + print "" + print "=" * (len(heading) + 4) + print "= " + heading + " =" + print "=" * (len(heading) + 4) + +def print_subheading(subheading): + print "" + print subheading + print "-" * (len(subheading) + 2) + +def print_missdeps(pkgname,missdeps) : + for d in missdeps: + print pkgname + " : " + d + +def print_result(list, subheading): + if len(list) > 0: + print_subheading(subheading) + for item in list: + print item + +def print_results(): + print_result(missing_pkgbuilds, "Missing PKGBUILDs") + print_result(invalid_pkgbuilds, "Invalid PKGBUILDs") + print_result(mismatches, "Mismatched Pkgnames") + print_result(dups, "Duplicate PKGBUILDs") + print_result(invalid_archs, "Invalid Archs") + print_result(missing_deps, "Missing Dependencies") + print_result(missing_makedeps, "Missing Makedepends") + print_result(dep_hierarchy, "Repo Hierarchy for Dependencies") + print_result(makedep_hierarchy, "Repo Hierarchy for Makedepends") + print_result(circular_deps, "Circular Dependencies") + print_subheading("Summary") + print "Missing PKGBUILDs: ", len(missing_pkgbuilds) + print "Invalid PKGBUILDs: ", len(invalid_pkgbuilds) + print "Mismatching PKGBUILD names: ", len(mismatches) + print "Duplicate PKGBUILDs: ", len(dups) + print "Invalid archs: ", len(invalid_archs) + print "Missing (make)dependencies: ", len(missing_deps)+len(missing_makedeps) + print "Repo hierarchy problems: ", len(dep_hierarchy)+len(makedep_hierarchy) + print "Circular dependencies: ", len(circular_deps) + print "" + +def print_usage(): + print "" + print "Usage: ./check_packages.py [OPTION]" + print "" + print "Options:" + print " --abs-tree= Check specified tree (default : /var/abs)" + print " --repos= Check specified repos (default : core,extra)" + print " -h, --help Show this help and exit" + print "" + print "Examples:" + print "\n Check core and extra in existing abs tree:" + print " ./check_packages.py --abs-tree=/var/abs --repos=core,extra" + print "\n Check community:" + print " ./check_packages.py --abs-tree=/var/abs --repos=community" + print "" + +## Default path to the abs root directory +absroot = "/var/abs" +## Default list of repos to check +repos = ['core', 'extra'] + +try: + opts, args = getopt.getopt(sys.argv[1:], "", ["abs-tree=", "repos="]) +except getopt.GetoptError: + print_usage() + sys.exit() +if opts != []: + for o, a in opts: + if o in ("--abs-tree"): + absroot = a + elif o in ("--repos"): + repos = a.split(",") + else: + print_usage() + sys.exit() + if args != []: + print_usage() + sys.exit() + +if not os.path.isdir(absroot): + print "Error : the abs tree " + absroot + " does not exist" + sys.exit() +for repo in repos: + repopath = absroot + "/" + repo + if not os.path.isdir(repopath): + print "Error : the repository " + repo + " does not exist in " + absroot + sys.exit() +# repos which need to be loaded +loadrepos = set([]) +for repo in repos: + loadrepos = loadrepos | set(get_repo_hierarchy(repo)) + +print_heading("Integrity Check") +print "\nPerforming integrity checks..." + +print "==> parsing pkgbuilds" +parse_pkgbuilds(loadrepos) + +repopkgs = {} +for name,pkg in packages.iteritems(): + if pkg.repo in repos: + repopkgs[name] = pkg + +print "==> checking mismatches" +for name,pkg in repopkgs.iteritems(): + pkgdirname = pkg.path.split("/")[-1] + if name != pkgdirname: + mismatches.append(name + " vs. " + pkg.path) + +print "==> checking archs" +for name,pkg in repopkgs.iteritems(): + archs = verify_archs(name,pkg.archs) + invalid_archs.extend(archs) + +# ugly hack to strip the weird kblic- deps +for name,pkg in packages.iteritems(): + p = re.compile('klibc-[A-Za-z0-9]{20,}|klibc-\*') + pkg.deps = [dep for dep in pkg.deps if not p.match(dep)] + pkg.makedeps = [dep for dep in pkg.makedeps if not p.match(dep)] + +print "==> checking dependencies" +for name,pkg in repopkgs.iteritems(): + (deps,missdeps,hierarchy) = verify_deps(name,pkg.repo,pkg.deps) + pkgdeps[pkg] = deps + missing_deps.extend(missdeps) + dep_hierarchy.extend(hierarchy) + +print "==> checking makedepends" +for name,pkg in repopkgs.iteritems(): + (makedeps,missdeps,hierarchy) = verify_deps(name,pkg.repo,pkg.makedeps) + makepkgdeps[pkg] = makedeps + missing_makedeps.extend(missdeps) + makedep_hierarchy.extend(hierarchy) + +print "==> checking for circular dependencies" +# make sure pkgdeps is filled for every package +for name,pkg in packages.iteritems(): + if not pkgdeps.has_key(pkg): + (deps,missdeps,_) = verify_deps(name,pkg.repo,pkg.deps) + pkgdeps[pkg] = deps +find_scc(repopkgs.values()) + +print_results() diff --git a/cron-jobs/check_archlinux/parse_pkgbuilds.sh b/cron-jobs/check_archlinux/parse_pkgbuilds.sh new file mode 100755 index 0000000..d4205ae --- /dev/null +++ b/cron-jobs/check_archlinux/parse_pkgbuilds.sh @@ -0,0 +1,78 @@ +#!/bin/bash + +parse() { + unset pkgname pkgver pkgrel + unset depends makedepends conflicts provides + ret=0 + dir=$1 + pkgbuild=$dir/PKGBUILD + source $pkgbuild &>/dev/null || ret=$? + + # ensure $pkgname and $pkgver variables were found + if [ $ret -ne 0 -o -z "$pkgname" -o -z "$pkgver" ]; then + echo -e "%INVALID%\n$pkgbuild\n" + return 1 + fi + + echo -e "%NAME%\n$pkgname\n" + echo -e "%VERSION%\n$pkgver-$pkgrel\n" + echo -e "%PATH%\n$dir\n" + + if [ -n "$arch" ]; then + echo "%ARCH%" + for i in ${arch[@]}; do echo $i; done + echo "" + fi + if [ -n "$depends" ]; then + echo "%DEPENDS%" + for i in ${depends[@]}; do + echo $i + done + echo "" + fi + if [ -n "$makedepends" ]; then + echo "%MAKEDEPENDS%" + for i in ${makedepends[@]}; do + echo $i + done + echo "" + fi + if [ -n "$conflicts" ]; then + echo "%CONFLICTS%" + for i in ${conflicts[@]}; do echo $i; done + echo "" + fi + if [ -n "$provides" ]; then + echo "%PROVIDES%" + for i in ${provides[@]}; do echo $i; done + echo "" + fi + return 0 +} + +find_pkgbuilds() { + if [ -f $1/PKGBUILD ]; then + parse $1 + return + fi + empty=1 + for dir in $1/*; do + if [ -d $dir ]; then + find_pkgbuilds $dir + unset empty + fi + done + if [ -n "$empty" ]; then + echo -e "%MISSING%\n$1\n" + fi +} + +if [ -z "$*" ]; then + exit 1 +fi + +for dir in "$@"; do + find_pkgbuilds $dir +done + +exit 0 diff --git a/cron-jobs/check_archlinux/setup.py b/cron-jobs/check_archlinux/setup.py new file mode 100644 index 0000000..b172752 --- /dev/null +++ b/cron-jobs/check_archlinux/setup.py @@ -0,0 +1,10 @@ +from distutils.core import setup, Extension + +alpm = Extension('alpm', + libraries = ['alpm'], + sources = ['alpm.c']) + +setup (name = 'Alpm', + version = '1.0', + description = 'Alpm bindings', + ext_modules = [alpm]) -- cgit v1.2.2 From 7f0886c7e8935352007d6ba8acc6d9e28b55f2da Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Sat, 30 Aug 2008 21:05:22 -0500 Subject: Fix an error message copy/paste error Signed-off-by: Aaron Griffin --- cron-jobs/ftpdir-cleanup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index ce91b62..65f0855 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -17,7 +17,7 @@ ctrl_c() { if [ -f "$LOCKFILE" ]; then owner="$(/usr/bin/stat -c %U $LOCKFILE)" - echo "error: source tarball generation is already in progress (started by $owner)" + echo "error: ftp cleanup is already in progress (started by $owner)" exit 1 fi -- cgit v1.2.2 From 01d2e01a129a4ded9ccba2659250a94cb3fddd36 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Sat, 30 Aug 2008 21:08:15 -0500 Subject: Prevent staging cleanup if copy to repo fails This stops files being removed from the staging area if they were not successfully copied to the repo. This can happen due to permission issues with the database when multiple people update the repo in quick succession. Original-work-by: Allan McRae Signed-off-by: Aaron Griffin --- db-update | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/db-update b/db-update index 7ff768e..4b2571d 100755 --- a/db-update +++ b/db-update @@ -198,7 +198,9 @@ fi # if non empty, move all build dirs if [ $(/bin/ls "$WORKDIR/build/" 2>/dev/null | wc -l) != 0 ]; then echo "Copying new files to '$ftppath'" - /bin/cp -r "$WORKDIR/build/"* "$ftppath" + if ! /bin/cp -r "$WORKDIR/build/"* "$ftppath"; then + die "error: failure while copying files to $ftppath" + fi else echo "Nothing to copy, no work done" -- cgit v1.2.2 From afc6b5b5977fe77e699af8f02f7c8c4c3df603de Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Sat, 30 Aug 2008 21:21:43 -0500 Subject: Create a "common functions" file for DB scripts Common functions, such as sourcing makepkg, and locking a repo Additionally, repo-locking has been unified to prevent multiple users from corrupting a repo (i.e. db-move run while db-remove in progress) Signed-off-by: Aaron Griffin --- db-move | 26 +++++++++----------------- db-remove | 25 +++++++------------------ db-update | 24 ++++++------------------ testing2x | 11 ++++------- 4 files changed, 26 insertions(+), 60 deletions(-) diff --git a/db-move b/db-move index f8adfc1..13f2c4e 100755 --- a/db-move +++ b/db-move @@ -6,13 +6,9 @@ if [ $# -ne 4 ]; then exit 1 fi -if [ -f "/etc/makepkg.conf" ]; then - #Get some config info - . /etc/makepkg.conf -else - echo "/etc/makepkg.conf does not exist!" - exit 1 -fi +. "$(dirname $0)/db-functions" + +source_makepkg packagename="$1" repofrom="$2" @@ -32,11 +28,11 @@ svnrepo_to="$repoto-$arch" [ "$UID" = "" ] && UID=$(uid) WORKDIR="/tmp/db-move.$svnrepo_from.$svnrepo_to.$UID" -LOCKFILE="/tmp/.repolck.$arch.$reponame" cleanup() { # unlock - rm -f "$LOCKFILE" + repo_unlock $repoto $arch + repo_unlock $repofrom $arch rm -rf "$WORKDIR" [ "$1" ] && exit $1 } @@ -51,16 +47,12 @@ die() { cleanup 1 } -# check for locks -if [ -f "$LOCKFILE" ]; then - owner="$(/usr/bin/stat -c %U $LOCKFILE)" - die "error: db generation is already in progress (started by $owner)" -fi - trap ctrl_c 2 -trap cleanup 0 +trap cleanup 0 1 + +repo_lock $repoto $arch +repo_lock $repofrom $arch -/bin/touch "$LOCKFILE" /bin/mkdir -p "$WORKDIR" cd "$WORKDIR" diff --git a/db-remove b/db-remove index f9495e9..f4d9419 100755 --- a/db-remove +++ b/db-remove @@ -5,13 +5,9 @@ if [ $# -ne 3 ]; then exit 1 fi -if [ -f "/etc/makepkg.conf" ]; then - #Get some config info - . /etc/makepkg.conf -else - echo "/etc/makepkg.conf does not exist!" - exit 1 -fi +. "$(dirname $0)/db-functions" + +source_makepkg packagename="$1" reponame="$2" @@ -28,11 +24,10 @@ svnrepo="$reponame-$arch" [ "$UID" = "" ] && UID=$(uid) WORKDIR="/tmp/db-remove.$svnrepo.$UID" -LOCKFILE="/tmp/.repolck.$arch.$reponame" cleanup() { # unlock - rm -f "$LOCKFILE" + repo_unlock $reponame $arch rm -rf "$WORKDIR" [ "$1" ] && exit $1 } @@ -47,17 +42,11 @@ die() { cleanup 1 } -# check for locks -if [ -f "$LOCKFILE" ]; then - owner="$(/usr/bin/stat -c %U $LOCKFILE)" - echo "error: db generation is already in progress (started by $owner)" - exit 1 -fi - trap ctrl_c 2 -trap cleanup 0 +trap cleanup 0 1 + +repo_lock $reponame $arch -/bin/touch "$LOCKFILE" /bin/mkdir -p "$WORKDIR" diff --git a/db-update b/db-update index 4b2571d..5060449 100755 --- a/db-update +++ b/db-update @@ -5,13 +5,9 @@ if [ $# -ne 2 ]; then exit 1 fi -if [ -f "/etc/makepkg.conf" ]; then - #Get some config info - . /etc/makepkg.conf -else - echo "/etc/makepkg.conf does not exist!" - exit 1 -fi +. "$(dirname $0)/db-functions" + +source_makepkg reponame="$1" arch="$2" @@ -28,7 +24,6 @@ stagedir="$HOME/staging/$reponame" [ "$UID" = "" ] && UID=$(uid) WORKDIR="/tmp/db-update.$svnrepo.$UID" -LOCKFILE="/tmp/.repolck.$arch.$reponame" ADDPKGS="" REMPKGS="" @@ -56,7 +51,7 @@ getpkgname() { cleanup() { # unlock - rm -f "$LOCKFILE" + repo_unlock $reponame $arch rm -rf "$WORKDIR" [ "$1" ] && exit $1 } @@ -71,17 +66,10 @@ die() { cleanup 1 } -# check for locks -if [ -f "$LOCKFILE" ]; then - owner="$(/usr/bin/stat -c %U $LOCKFILE)" - echo "error: db generation is already in progress (started by $owner)" - exit 1 -fi - trap ctrl_c 2 -trap cleanup 0 +trap cleanup 0 1 -/bin/touch "$LOCKFILE" +repo_lock $reponame $arch /bin/mkdir -p "$WORKDIR/build" cd "$WORKDIR" diff --git a/testing2x b/testing2x index 7096f1d..c385fd8 100755 --- a/testing2x +++ b/testing2x @@ -1,11 +1,8 @@ #!/bin/bash -if [ -f "/etc/makepkg.conf" ]; then - . /etc/makepkg.conf -else - echo "/etc/makepkg.conf does not exist!" - exit 1 -fi +. "$(dirname $0)/db-functions" + +source_makepkg case "$0" in *64) @@ -29,7 +26,7 @@ ctrl_c() { } trap ctrl_c 2 -trap cleanup 0 +trap cleanup 0 1 cd "${WORKDIR}" /usr/bin/svn checkout -N ${svnpath} checkout -- cgit v1.2.2 From 06194a8691f50ffc9130795e16837496e13dd08f Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Sun, 31 Aug 2008 00:04:32 -0500 Subject: Move source generation to /home/ftp/sources Signed-off-by: Aaron Griffin --- misc-scripts/make-sourceball | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/misc-scripts/make-sourceball b/misc-scripts/make-sourceball index 71bb405..ea5ef49 100755 --- a/misc-scripts/make-sourceball +++ b/misc-scripts/make-sourceball @@ -5,19 +5,16 @@ if [ $# -ne 3 ]; then exit 1 fi -if [ -f "/etc/makepkg.conf" ]; then - #Get some config info - . /etc/makepkg.conf -else - echo "/etc/makepkg.conf does not exist!" - exit 1 -fi +. "$(dirname $0)/../db-functions" + +source_makepkg + packagename="$1" reponame="$2" arch="$3" ##### Arch specific stuff. TODO make this configurable ##### -srcpath="/home/aaron/public_html/sources/" +srcpath="/home/ftp/sources/" svnpath="file:///home/svn-packages/$packagename/" ############################################################ @@ -43,21 +40,19 @@ create_srcpackage() { if [ -d "$1" ]; then pushd "$1" >/dev/null . "$BUILDSCRIPT" - if ! /usr/bin/makepkg -gc >/dev/null 2>&1; then + if ! /usr/bin/makepkg --allsource >/dev/null 2>&1; then popd >/dev/null return 1 fi popd >/dev/null local pkg_file="${pkgname}-${pkgver}-${pkgrel}${SRCEXT}" - if ! /usr/bin/bsdtar -czf "$pkg_file" "$1"; then - return 2 - fi echo ":: Source package complete: $pkg_file" if [ ! -d "$srcpath" ]; then mkdir -p "$srcpath" fi + rm -f "$srcpath/${pkgname}-*-*${SRCEXT}" cp $pkg_file "$srcpath" return 0 @@ -65,7 +60,7 @@ create_srcpackage() { } trap ctrl_c 2 -trap cleanup 0 +trap cleanup 0 1 /bin/mkdir -p "$WORKDIR" cd "$WORKDIR" -- cgit v1.2.2 From e1ad729fd49c630eece0055ed996b382811a836c Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Sun, 31 Aug 2008 01:07:26 -0400 Subject: Add sourceball generating cron script Signed-off-by: Aaron Griffin --- cron-jobs/sourceballs | 81 ++++++++++++++++++++++++++++++++++++++++++++++ cron-jobs/sourceballs.skip | 1 + 2 files changed, 82 insertions(+) create mode 100755 cron-jobs/sourceballs create mode 100644 cron-jobs/sourceballs.skip diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs new file mode 100755 index 0000000..f599ff1 --- /dev/null +++ b/cron-jobs/sourceballs @@ -0,0 +1,81 @@ +#!/bin/bash + +ftpbase="/home/ftp" +srcbase="/home/aaron/public_html/sources/" +repos="core extra unstable testing" +arches="i686 x86_64" + +LOCKFILE="/tmp/.sourceball.lock" + +if [ ! -f /etc/makepkg.conf ]; then + echo "/etc/makepkg.conf not found! Aborting" + exit 1 +fi + +. /etc/makepkg.conf +cleanup () { + rm -f "$LOCKFILE" + exit 0 +} + +ctrl_c() { + cleanup +} + +if [ -f "$LOCKFILE" ]; then + owner="$(/usr/bin/stat -c %U $LOCKFILE)" + echo "error: source tarball generation is already in progress (started by $owner)" + exit 1 +fi + +trap cleanup 0 +trap ctrl_c 2 + +/bin/touch "$LOCKFILE" + +dirname="$(/bin/readlink -f $(/usr/bin/dirname $0))" + +getpkgname() { + local tmp + + tmp=${1##*/} + tmp=${tmp%$PKGEXT} + for a in $arches; do + tmp=${tmp%-$a} + done + echo ${tmp%-*-*} +} + +FAILED_PKGS="" + +for repo in $repos; do + for arch in $arches; do + export CARCH="$arch" + ftppath="$ftpbase/$repo/os/$arch" + cd $ftppath + for pkg in *$PKGEXT; do + pkgname=$(getpkgname $pkg) + srcpath="$srcbase/" + srcpkg="${pkg//-$arch$PKGEXT/$SRCEXT}" + + if grep $pkgname "$dirname/sourceballs.skip" >/dev/null 2>&1; then + echo ":: Skipping package '$pkgname'" + continue + fi + + if [ ! -f "$srcpath$srcpkg" ]; then + if ! $dirname/../misc-scripts/make-sourceball $pkgname $repo $arch; then + FAILED_PKGS="$FAILED_PKGS $pkg" + fi + fi + done + done +done + +if [ -n "$FAILED_PKGS" ]; then + echo "" + echo "The following packages failed:" + echo -e $FAILED_PKGS | sed "s| |\n\t|g" | sort -u +fi + +cleanup diff --git a/cron-jobs/sourceballs.skip b/cron-jobs/sourceballs.skip new file mode 100644 index 0000000..0b5a58e --- /dev/null +++ b/cron-jobs/sourceballs.skip @@ -0,0 +1 @@ +readline -- cgit v1.2.2 From 766727fa10cbc2715baba93437fbb30b798f08c9 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Sun, 31 Aug 2008 00:08:54 -0500 Subject: Add sourceball cron job at the proper path (whoops) Also remove readline from the skip-list Signed-off-by: Aaron Griffin --- cron-jobs/sourceballs | 2 +- cron-jobs/sourceballs.skip | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index f599ff1..ee62d46 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -1,7 +1,7 @@ #!/bin/bash ftpbase="/home/ftp" -srcbase="/home/aaron/public_html/sources/" +srcbase="/home/ftp/sources" repos="core extra unstable testing" arches="i686 x86_64" diff --git a/cron-jobs/sourceballs.skip b/cron-jobs/sourceballs.skip index 0b5a58e..e69de29 100644 --- a/cron-jobs/sourceballs.skip +++ b/cron-jobs/sourceballs.skip @@ -1 +0,0 @@ -readline -- cgit v1.2.2 From 29095faf6fa2d01e65fbfebe4702ae240d1688b6 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Tue, 2 Sep 2008 00:01:03 -0500 Subject: Forgot the db-functions file. Whoops Signed-off-by: Aaron Griffin --- db-functions | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 db-functions diff --git a/db-functions b/db-functions new file mode 100644 index 0000000..bcc25bb --- /dev/null +++ b/db-functions @@ -0,0 +1,48 @@ +#!/bin/bash + +# Random integrity things +[ "$UID" = "" ] && UID=$(uid) + +# Useful functions +source_makepkg () { + if [ -f "/etc/makepkg.conf" ]; then + #Get some config info + . /etc/makepkg.conf + else + echo "/etc/makepkg.conf does not exist!" + exit 1 + fi +} + +repo_lock () { #repo_lock repo-name arch + LOCKFILE="/tmp/.repolck.$1.$2" + if [ -f "$LOCKFILE" ]; then + owner="$(/usr/bin/stat -c %U $LOCKFILE)" + echo "error: db generation is already in progress (started by $owner)" + exit 1 + else + /bin/touch "$LOCKFILE" + fi +} + +repo_unlock () { #repo_unlock repo-name arch + LOCKFILE="/tmp/.repolck.$1.$2" + if [ ! -f "$LOCKFILE" ]; then + echo "error: repo lock doesn't exist... something went terribly wrong!" + else + rm -f "$LOCKFILE" + fi +} + +# Get the package name from the filename +# hackish, but should work for now +getpkgname() { + local tmp + + tmp=${1##*/} + tmp=${tmp%$PKGEXT} + tmp=${tmp%-$CARCH} + echo ${tmp%-*-*} +} + +# vim: set ts=4 sw=4 noet ft=sh: -- cgit v1.2.2 From 6fc5ac8adc6bd66770351b1d786f5895d83b2f97 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Tue, 2 Sep 2008 00:01:54 -0500 Subject: Remove 'unstable' from sourceball creation Signed-off-by: Aaron Griffin --- cron-jobs/sourceballs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index ee62d46..fc313e3 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -2,7 +2,7 @@ ftpbase="/home/ftp" srcbase="/home/ftp/sources" -repos="core extra unstable testing" +repos="core extra testing" arches="i686 x86_64" LOCKFILE="/tmp/.sourceball.lock" -- cgit v1.2.2 From a3a43e54b6c673829a06b9bedcf8072c0358e0aa Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Tue, 2 Sep 2008 00:12:46 -0500 Subject: Remove the unstable repository Signed-off-by: Aaron Griffin --- cron-jobs/adjust-permissions | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cron-jobs/adjust-permissions b/cron-jobs/adjust-permissions index 7f7e4c8..88ec006 100755 --- a/cron-jobs/adjust-permissions +++ b/cron-jobs/adjust-permissions @@ -9,13 +9,13 @@ fi cd /home/ftp /bin/chown -R ftp:ftp-arch core/os/i686 /bin/chown -R ftp:ftp-arch core/os/x86_64 -/bin/chown -R ftp:ftp-extra {extra,unstable,testing}/os/i686 -/bin/chown -R ftp:ftp-extra {extra,unstable,testing}/os/x86_64 +/bin/chown -R ftp:ftp-extra {extra,testing}/os/i686 +/bin/chown -R ftp:ftp-extra {extra,testing}/os/x86_64 /bin/chown -R ftp:aur community /bin/chmod -R g+w /home/aur/unsupported{,-temp} -for d in core extra unstable testing community; do +for d in core extra testing community; do /bin/chmod -R g+w $d/os/i686 /bin/chmod -R g+w $d/os/x86_64 done -- cgit v1.2.2 From 4bef49c9d906a5388027eb5cbe694b0b6b93e807 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Tue, 2 Sep 2008 00:13:40 -0500 Subject: Cron-job changes Signed-off-by: Aaron Griffin --- cron-jobs/repo-maint.crontab | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cron-jobs/repo-maint.crontab b/cron-jobs/repo-maint.crontab index dadd1fb..babcea1 100644 --- a/cron-jobs/repo-maint.crontab +++ b/cron-jobs/repo-maint.crontab @@ -8,5 +8,6 @@ MAILTO="root" #| | | | | commands # #################################################################### */5 * * * * /arch/cron-jobs/adjust-permissions -00 21 * * * /arch/cron-jobs/createFileList +00 21 * * * /arch/cron-jobs/createFileList >/dev/null 2>&1 +00 23 * * * /arch/cron-jobs/sourceballs 00 */3 * * * /arch/cron-jobs/ftpdir-cleanup | /arch/cron-jobs/devlist-mailer -- cgit v1.2.2 From 0730a707da94ff30b94c1f7829c8d247ddf0eeb2 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Tue, 2 Sep 2008 00:18:28 -0500 Subject: Correctly remove old sourceballs without removing the wrong files Signed-off-by: Aaron Griffin --- db-functions | 1 + misc-scripts/make-sourceball | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/db-functions b/db-functions index bcc25bb..25bca78 100644 --- a/db-functions +++ b/db-functions @@ -41,6 +41,7 @@ getpkgname() { tmp=${1##*/} tmp=${tmp%$PKGEXT} + tmp=${tmp%$SRCEXT} tmp=${tmp%-$CARCH} echo ${tmp%-*-*} } diff --git a/misc-scripts/make-sourceball b/misc-scripts/make-sourceball index ea5ef49..7f75fe8 100755 --- a/misc-scripts/make-sourceball +++ b/misc-scripts/make-sourceball @@ -52,7 +52,13 @@ create_srcpackage() { if [ ! -d "$srcpath" ]; then mkdir -p "$srcpath" fi - rm -f "$srcpath/${pkgname}-*-*${SRCEXT}" + #Remove old sourceballs + for pkg in "$srcpath/$pkgname-*"; do + pkg="$(basename $pkg)" + if "$(getpkgname $pkg)" == "$pkgname" ]; then + rm -f "$srcpath/$pkg" + fi + done cp $pkg_file "$srcpath" return 0 -- cgit v1.2.2 From a0f73ceca409fa8995af16eee8440c1467c5f1bf Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Sat, 13 Sep 2008 17:49:59 -0500 Subject: Validate a package file's architecture Apparently, people seem to be renaming files as a "hack" and breaking things. Let's make sure they edited the PKGINFO too Signed-off-by: Aaron Griffin --- db-functions | 17 +++++++++++++++++ db-update | 24 +++++++++++++++--------- 2 files changed, 32 insertions(+), 9 deletions(-) diff --git a/db-functions b/db-functions index 25bca78..663067e 100644 --- a/db-functions +++ b/db-functions @@ -46,4 +46,21 @@ getpkgname() { echo ${tmp%-*-*} } +check_pkg_arch () { #check_pkg_arch pkgfile arch + local arch + _arch="$(/usr/bin/bsdtar -xOf "$1" .PKGINFO | /bin/grep "^arch" | /bin/sed 's|\w*\s*=\s*\(.*\)|\1|')" + + if [ -z "$_arch" ]; then + echo "ERROR: Package '$1' has no arch in the PKGINFO. Fail!" + return 1 + fi + if [ "$_arch" = "$2" ]; then + echo "yay" + return 0 + else + echo "nay" + return 1 + fi +} + # vim: set ts=4 sw=4 noet ft=sh: diff --git a/db-update b/db-update index 5060449..d3fff2c 100755 --- a/db-update +++ b/db-update @@ -108,17 +108,23 @@ if [ -n "$ADDPKGS" ]; then for pkg in $ADDPKGS; do _pkgfile=$(basename $pkg) _pkgname="$(getpkgname $pkg)" - echo " Checking SVN for $_pkgname" - /usr/bin/svn up -q $_pkgname - if [ -d "$_pkgname/repos/$svnrepo" ]; then - . "$_pkgname/repos/$svnrepo/$BUILDSCRIPT" - if [ "$_pkgfile" = "$pkgname-$pkgver-$pkgrel-$CARCH$PKGEXT" ]; then - to_add="$to_add $pkg" + + echo " Validating package arch ($arch) $_pkgname" + if ! check_pkg_arch "$pkg" "$arch"; then + echo " ERROR: $_pkgfile was built for the wrong architecture" + else + echo " Checking SVN for $_pkgname" + /usr/bin/svn up -q $_pkgname + if [ -d "$_pkgname/repos/$svnrepo" ]; then + . "$_pkgname/repos/$svnrepo/$BUILDSCRIPT" + if [ "$_pkgfile" = "$pkgname-$pkgver-$pkgrel-$CARCH$PKGEXT" ]; then + to_add="$to_add $pkg" + else + echo " WARNING: $_pkgfile does not match $BUILDSCRIPT in $svnrepo" + fi else - echo " WARNING: $_pkgfile does not match $BUILDSCRIPT in $svnrepo" + echo " WARNING: Package $_pkgname not found in $svnrepo" fi - else - echo " WARNING: Package $_pkgname not found in $svnrepo" fi done -- cgit v1.2.2 From 42bbb3dd2ef276c5204651fafb152a888c449c15 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Sat, 13 Sep 2008 20:30:27 -0500 Subject: Move temporary directories to /home/tmp This is to allow moves to /home/ftp/ to remain on the same filesystem, thus making the final moves atomic Signed-off-by: Aaron Griffin --- db-move | 2 +- db-remove | 2 +- db-update | 2 +- testing2x | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/db-move b/db-move index 13f2c4e..c8d30c4 100755 --- a/db-move +++ b/db-move @@ -27,7 +27,7 @@ svnrepo_to="$repoto-$arch" [ "$UID" = "" ] && UID=$(uid) -WORKDIR="/tmp/db-move.$svnrepo_from.$svnrepo_to.$UID" +WORKDIR="/home/tmp/db-move.$svnrepo_from.$svnrepo_to.$UID" cleanup() { # unlock diff --git a/db-remove b/db-remove index f4d9419..fdc7c5d 100755 --- a/db-remove +++ b/db-remove @@ -23,7 +23,7 @@ svnrepo="$reponame-$arch" [ "$UID" = "" ] && UID=$(uid) -WORKDIR="/tmp/db-remove.$svnrepo.$UID" +WORKDIR="/home/tmp/db-remove.$svnrepo.$UID" cleanup() { # unlock diff --git a/db-update b/db-update index d3fff2c..dbfbe51 100755 --- a/db-update +++ b/db-update @@ -23,7 +23,7 @@ stagedir="$HOME/staging/$reponame" [ "$UID" = "" ] && UID=$(uid) -WORKDIR="/tmp/db-update.$svnrepo.$UID" +WORKDIR="/home/tmp/db-update.$svnrepo.$UID" ADDPKGS="" REMPKGS="" diff --git a/testing2x b/testing2x index c385fd8..687994e 100755 --- a/testing2x +++ b/testing2x @@ -13,7 +13,7 @@ case "$0" in ;; esac svnpath="file:///home/svn-packages" -WORKDIR="$(mktemp -d /tmp/testing2x.XXXXXX)" +WORKDIR="/home/tmp/testing2x.$UID" cleanup() { rm -rf "${WORKDIR}" -- cgit v1.2.2 From 86ec8b5c9dcd5b83cee55e410d42784c830efbb6 Mon Sep 17 00:00:00 2001 From: Xavier Chantry Date: Sat, 27 Sep 2008 00:56:24 +0200 Subject: check_packages : improve the "skip klibc dep" hack. It is not possible to check the klibc dep on the PKGBUILD level, so I made a hack to skip it. This hack broke on klibc-jfflyAahxqaliwAofrf_fdf5upI because of the underscore. But we can simply use the \w regexp which matches any alpha-numeric char, including underscore. According to klibc developer, dashes are possible too. Also the length of this string is always 27 chars, so we can use that. Signed-off-by: Xavier Chantry Signed-off-by: Aaron Griffin --- cron-jobs/check_archlinux/check_packages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cron-jobs/check_archlinux/check_packages.py b/cron-jobs/check_archlinux/check_packages.py index 2d2efbe..69b14c3 100755 --- a/cron-jobs/check_archlinux/check_packages.py +++ b/cron-jobs/check_archlinux/check_packages.py @@ -363,7 +363,7 @@ for name,pkg in repopkgs.iteritems(): # ugly hack to strip the weird kblic- deps for name,pkg in packages.iteritems(): - p = re.compile('klibc-[A-Za-z0-9]{20,}|klibc-\*') + p = re.compile('klibc-[\w\-]{27}|klibc-\*') pkg.deps = [dep for dep in pkg.deps if not p.match(dep)] pkg.makedeps = [dep for dep in pkg.makedeps if not p.match(dep)] -- cgit v1.2.2 From 5d686fdacc99e34a2f1b6d510873be46433df7b2 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Thu, 16 Oct 2008 19:38:03 -0400 Subject: Fix a typo in db-update's usage output Signed-off-by: Aaron Griffin --- db-update | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db-update b/db-update index dbfbe51..81b3905 100755 --- a/db-update +++ b/db-update @@ -1,7 +1,7 @@ #!/bin/bash if [ $# -ne 2 ]; then - echo "usage: $(basename $0) " + echo "usage: $(basename $0) " exit 1 fi -- cgit v1.2.2 From 0dd49e0e02f40a1687814f15d4aea55e1cc8411a Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Tue, 21 Oct 2008 13:05:42 -0700 Subject: Remove some goofy debugging messages from db-functions Signed-off-by: Aaron Griffin --- db-functions | 2 -- 1 file changed, 2 deletions(-) diff --git a/db-functions b/db-functions index 663067e..9df678c 100644 --- a/db-functions +++ b/db-functions @@ -55,10 +55,8 @@ check_pkg_arch () { #check_pkg_arch pkgfile arch return 1 fi if [ "$_arch" = "$2" ]; then - echo "yay" return 0 else - echo "nay" return 1 fi } -- cgit v1.2.2 From 23f48e65a5f4133be8a5d2883de6ffb2f4f04962 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Tue, 21 Oct 2008 13:06:06 -0700 Subject: Add copy_helper function to ensure correct permissions This will force all files copied back and forth to have 0664 permissions so that we can attempt to do away with this permission adjusting cron job Signed-off-by: Aaron Griffin --- db-functions | 7 +++++++ db-move | 6 +++--- db-remove | 4 ++-- db-update | 12 +++++++----- 4 files changed, 19 insertions(+), 10 deletions(-) diff --git a/db-functions b/db-functions index 9df678c..a20496b 100644 --- a/db-functions +++ b/db-functions @@ -61,4 +61,11 @@ check_pkg_arch () { #check_pkg_arch pkgfile arch fi } +# Simple helper function to ensure we always +# have proper DB permissions +copy_helper () { #copy_helper file dest + /bin/cp $1 $2 || return 1 + /bin/chmod 664 "$(dirname $2)/$(basename $1)" || return 1 +} + # vim: set ts=4 sw=4 noet ft=sh: diff --git a/db-move b/db-move index c8d30c4..aee18f2 100755 --- a/db-move +++ b/db-move @@ -85,7 +85,7 @@ if [ -d "$packagename/repos/$svnrepo_from" ]; then # copy the db file into our working area if [ -f "$ftppath_from/$repofrom.db.tar.$DB_COMPRESSION" ]; then - cp "$ftppath_from/$repofrom.db.tar.$DB_COMPRESSION" . + copy_helper "$ftppath_from/$repofrom.db.tar.$DB_COMPRESSION" . else touch "$repofrom.db.tar.$DB_COMPRESSION" fi @@ -96,12 +96,12 @@ if [ -d "$packagename/repos/$svnrepo_from" ]; then echo " Package files will be cleaned up automatically" if [ -f "$ftppath_to/$repoto.db.tar.$DB_COMPRESSION" ]; then - cp "$ftppath_to/$repoto.db.tar.$DB_COMPRESSION" . + copy_helper "$ftppath_to/$repoto.db.tar.$DB_COMPRESSION" . else touch "$repoto.db.tar.$DB_COMPRESSION" fi - cp "$ftppath_from/$_pkgfile" . + copy_helper "$ftppath_from/$_pkgfile" . /usr/bin/repo-add "$repoto.db.tar.$DB_COMPRESSION" $_pkgfile || die "Error in repo-add" #use '*' to move the old DB too mv $repoto.db.tar.$DB_COMPRESSION* $_pkgfile $ftppath_to diff --git a/db-remove b/db-remove index fdc7c5d..bb91afd 100755 --- a/db-remove +++ b/db-remove @@ -68,7 +68,7 @@ cd "$WORKDIR" # copy the db file into our working area if [ -f "$ftppath/$reponame.db.tar.$DB_COMPRESSION" ]; then - cp "$ftppath/$reponame.db.tar.$DB_COMPRESSION" build/ + copy_helper "$ftppath/$reponame.db.tar.$DB_COMPRESSION" build/ else echo "No database found at '$ftpdir', nothing more to do" exit 0 @@ -78,7 +78,7 @@ echo " Removing from $reponame DB file" cd build/ /usr/bin/repo-remove "$reponame.db.tar.$DB_COMPRESSION" $packagename -cp "$reponame.db.tar.$DB_COMPRESSION" "$ftppath/" +copy_helper "$reponame.db.tar.$DB_COMPRESSION" "$ftppath/" echo "Package files will be cleaned up automatically" diff --git a/db-update b/db-update index 81b3905..d25ab2c 100755 --- a/db-update +++ b/db-update @@ -76,7 +76,7 @@ cd "$WORKDIR" # copy the db file into our working area if [ -f "$ftppath/$reponame.db.tar.$DB_COMPRESSION" ]; then - cp "$ftppath/$reponame.db.tar.$DB_COMPRESSION" build/ + copy_helper "$ftppath/$reponame.db.tar.$DB_COMPRESSION" build/ else touch "build/$reponame.db.tar.$DB_COMPRESSION" fi @@ -130,7 +130,7 @@ if [ -n "$ADDPKGS" ]; then if [ -n "$to_add" ]; then cd "$WORKDIR/build/" - /bin/cp $to_add . + for f in $to_add; do copy_helper "$f" .; done pkgs="" for pkg in $to_add; do pkgs="$pkgs $(basename $pkg)"; done @@ -192,9 +192,11 @@ fi # if non empty, move all build dirs if [ $(/bin/ls "$WORKDIR/build/" 2>/dev/null | wc -l) != 0 ]; then echo "Copying new files to '$ftppath'" - if ! /bin/cp -r "$WORKDIR/build/"* "$ftppath"; then - die "error: failure while copying files to $ftppath" - fi + for f in "$WORKDIR/build/"*; do + if ! copy_helper "$f" "$ftppath"; then + die "error: failure while copying files to $ftppath" + fi + done else echo "Nothing to copy, no work done" -- cgit v1.2.2 From 2063416ef4a160f56bb47f203257fe7903ba77e2 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Sun, 9 Nov 2008 00:27:43 -0600 Subject: Remove an erroneous '1' signal trap for cleanup() func Signed-off-by: Aaron Griffin --- db-move | 2 +- db-remove | 2 +- db-update | 2 +- testing2x | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/db-move b/db-move index aee18f2..88a8cb6 100755 --- a/db-move +++ b/db-move @@ -48,7 +48,7 @@ die() { } trap ctrl_c 2 -trap cleanup 0 1 +trap cleanup 0 repo_lock $repoto $arch repo_lock $repofrom $arch diff --git a/db-remove b/db-remove index bb91afd..01f18d6 100755 --- a/db-remove +++ b/db-remove @@ -43,7 +43,7 @@ die() { } trap ctrl_c 2 -trap cleanup 0 1 +trap cleanup 0 repo_lock $reponame $arch diff --git a/db-update b/db-update index d25ab2c..e2a6534 100755 --- a/db-update +++ b/db-update @@ -67,7 +67,7 @@ die() { } trap ctrl_c 2 -trap cleanup 0 1 +trap cleanup 0 repo_lock $reponame $arch diff --git a/testing2x b/testing2x index 687994e..0494b98 100755 --- a/testing2x +++ b/testing2x @@ -26,7 +26,7 @@ ctrl_c() { } trap ctrl_c 2 -trap cleanup 0 1 +trap cleanup 0 cd "${WORKDIR}" /usr/bin/svn checkout -N ${svnpath} checkout -- cgit v1.2.2 From bdc50e3ecd4bd0a1e02e66f331ea7084cb61882b Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Mon, 10 Nov 2008 10:43:13 -0800 Subject: Remove chmod on /home/aur/* This one portion of the adjust-permissions script takes over a minute of wall clock time to run on gerolde when no other cron jobs are running. It takes three minutes when we hit the hour mark on the clock due to everything else going at the same time. A find on /home/aur/unsupported reveals this command will need to touch ~77000 files. Signed-off-by: Dan McGee Signed-off-by: Aaron Griffin --- cron-jobs/adjust-permissions | 2 -- 1 file changed, 2 deletions(-) diff --git a/cron-jobs/adjust-permissions b/cron-jobs/adjust-permissions index 88ec006..232d48a 100755 --- a/cron-jobs/adjust-permissions +++ b/cron-jobs/adjust-permissions @@ -13,8 +13,6 @@ cd /home/ftp /bin/chown -R ftp:ftp-extra {extra,testing}/os/x86_64 /bin/chown -R ftp:aur community -/bin/chmod -R g+w /home/aur/unsupported{,-temp} - for d in core extra testing community; do /bin/chmod -R g+w $d/os/i686 /bin/chmod -R g+w $d/os/x86_64 -- cgit v1.2.2 From 0dcc6b02d0ed7cf25e9493667030ded9525473ce Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Mon, 10 Nov 2008 10:45:29 -0800 Subject: cron-jobs: use renice to lower job priority Lower the job priority of our cron jobs so they don't interfere with other more important things on the server. None of these are very CPU intensive, but priority for I/O operations should go elsewhere. Signed-off-by: Dan McGee Signed-off-by: Aaron Griffin --- cron-jobs/adjust-permissions | 3 +++ cron-jobs/createFileLists | 3 +++ cron-jobs/ftpdir-cleanup | 3 +++ 3 files changed, 9 insertions(+) diff --git a/cron-jobs/adjust-permissions b/cron-jobs/adjust-permissions index 232d48a..1493f4b 100755 --- a/cron-jobs/adjust-permissions +++ b/cron-jobs/adjust-permissions @@ -6,6 +6,9 @@ fi /bin/touch /tmp/.ftpmaint.lck +#adjust the nice level to run at a lower priority +/usr/bin/renice +10 -p $$ > /dev/null + cd /home/ftp /bin/chown -R ftp:ftp-arch core/os/i686 /bin/chown -R ftp:ftp-arch core/os/x86_64 diff --git a/cron-jobs/createFileLists b/cron-jobs/createFileLists index d6cbd18..044b6d7 100755 --- a/cron-jobs/createFileLists +++ b/cron-jobs/createFileLists @@ -15,6 +15,9 @@ touch "/tmp/createFileList.lock" || exit 1 TMPDIR=$(mktemp -d /tmp/createFileList.XXXXXX) || exit 1 CACHEDIR=$(mktemp -d /tmp/createFileList.XXXXXX) || exit 1 +#adjust the nice level to run at a lower priority +/usr/bin/renice +10 -p $$ > /dev/null + getpkgname() { local tmp diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index 65f0855..8b73956 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -26,6 +26,9 @@ trap ctrl_c 2 /bin/touch "$LOCKFILE" +#adjust the nice level to run at a lower priority +/usr/bin/renice +10 -p $$ > /dev/null + for repo in $repos; do for arch in $arches; do $(dirname $0)/../misc-scripts/ftpdir-cleanup $repo $arch -- cgit v1.2.2 From 35e5070403ddae96238bcce6356f4b9713c96675 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Tue, 11 Nov 2008 22:09:24 -0800 Subject: Move as much as we can out to a config file This should allow us to move FTP and SVN placement around Signed-off-by: Aaron Griffin --- config | 4 ++++ db-functions | 5 +++++ db-move | 9 +++------ db-remove | 7 ++----- db-update | 11 ++++------- testing2x | 3 +-- 6 files changed, 19 insertions(+), 20 deletions(-) create mode 100644 config diff --git a/config b/config new file mode 100644 index 0000000..dbed5bd --- /dev/null +++ b/config @@ -0,0 +1,4 @@ + +FTP_BASE="/home/ftp/" +SVN_PATH="file:///home/svn-packages" +STAGING="$HOME/staging/" diff --git a/db-functions b/db-functions index a20496b..9724e1c 100644 --- a/db-functions +++ b/db-functions @@ -3,6 +3,11 @@ # Random integrity things [ "$UID" = "" ] && UID=$(uid) +BASEDIR="$(dirname $0)" +if [ -f "$BASEDIR/config" ]; then + . "$BASEDIR/config" +fi + # Useful functions source_makepkg () { if [ -f "/etc/makepkg.conf" ]; then diff --git a/db-move b/db-move index 88a8cb6..5be1813 100755 --- a/db-move +++ b/db-move @@ -17,13 +17,10 @@ arch="$4" export CARCH="$arch" -##### Arch specific stuff. TODO make this configurable ##### -ftppath_from="/home/ftp/$repofrom/os/$arch/" -ftppath_to="/home/ftp/$repoto/os/$arch/" -svnpath="file:///home/svn-packages" +ftppath_from="$FTP_BASE/$repofrom/os/$arch/" +ftppath_to="$FTP_BASE/$repoto/os/$arch/" svnrepo_from="$repofrom-$arch" svnrepo_to="$repoto-$arch" -############################################################ [ "$UID" = "" ] && UID=$(uid) @@ -56,7 +53,7 @@ repo_lock $repofrom $arch /bin/mkdir -p "$WORKDIR" cd "$WORKDIR" -/usr/bin/svn checkout -N $svnpath checkout +/usr/bin/svn checkout -N $SVN_PATH checkout cd checkout /usr/bin/svn up -q $packagename diff --git a/db-remove b/db-remove index 01f18d6..bdbd254 100755 --- a/db-remove +++ b/db-remove @@ -15,11 +15,8 @@ arch="$3" export CARCH="$arch" -##### Arch specific stuff. TODO make this configurable ##### -ftppath="/home/ftp/$reponame/os/$arch/" -svnpath="file:///home/svn-packages" +ftppath="$FTP_BASE/$reponame/os/$arch/" svnrepo="$reponame-$arch" -############################################################ [ "$UID" = "" ] && UID=$(uid) @@ -53,7 +50,7 @@ repo_lock $reponame $arch echo "==> Removing package '$packagename' from '$reponame'..." >&2 cd "$WORKDIR" -/usr/bin/svn checkout -N $svnpath checkout +/usr/bin/svn checkout -N $SVN_PATH checkout cd checkout /usr/bin/svn up -q $packagename diff --git a/db-update b/db-update index e2a6534..84764eb 100755 --- a/db-update +++ b/db-update @@ -14,12 +14,9 @@ arch="$2" export CARCH="$arch" -##### Arch specific stuff. TODO make this configurable ##### -ftppath="/home/ftp/$reponame/os/$arch/" -svnpath="file:///home/svn-packages" +ftppath="$FTP_BASE/$reponame/os/$arch/" svnrepo="$reponame-$arch" -stagedir="$HOME/staging/$reponame" -############################################################ +stagedir="$STAGING/$reponame" [ "$UID" = "" ] && UID=$(uid) @@ -102,7 +99,7 @@ if [ -n "$ADDPKGS" ]; then echo "==> Processing new/updated packages for repository '$reponame'..." >&2 cd "$WORKDIR" - /usr/bin/svn checkout -N $svnpath checkout + /usr/bin/svn checkout -N $SVN_PATH checkout cd checkout for pkg in $ADDPKGS; do @@ -154,7 +151,7 @@ if [ -n "$REMPKGS" ]; then if [ ! -d "$WORKDIR/checkout" ]; then cd "$WORKDIR" - /usr/bin/svn checkout -N $svnpath checkout + /usr/bin/svn checkout -N $SVN_PATH checkout fi cd "$WORKDIR/checkout" diff --git a/testing2x b/testing2x index 0494b98..d87de11 100755 --- a/testing2x +++ b/testing2x @@ -12,7 +12,6 @@ case "$0" in arch="i686" ;; esac -svnpath="file:///home/svn-packages" WORKDIR="/home/tmp/testing2x.$UID" cleanup() { @@ -29,7 +28,7 @@ trap ctrl_c 2 trap cleanup 0 cd "${WORKDIR}" -/usr/bin/svn checkout -N ${svnpath} checkout +/usr/bin/svn checkout -N $SVN_PATH checkout cd checkout for pkg in $*; do -- cgit v1.2.2 From 1809269f1df368aab3f7f1bb615c2cdc52860a1f Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Thu, 20 Nov 2008 21:48:00 -0800 Subject: Move everything to /srv to support new server conf Additionally, make TMPDIR configurable so we can move that Signed-off-by: Aaron Griffin --- config | 5 +++-- db-functions | 4 ++-- db-move | 2 +- db-remove | 2 +- db-update | 2 +- testing2x | 2 +- 6 files changed, 9 insertions(+), 8 deletions(-) diff --git a/config b/config index dbed5bd..5fe1969 100644 --- a/config +++ b/config @@ -1,4 +1,5 @@ -FTP_BASE="/home/ftp/" -SVN_PATH="file:///home/svn-packages" +FTP_BASE="/srv/ftp/" +SVN_PATH="file:///srv/svn/packages" STAGING="$HOME/staging/" +TMPDIR="/srv/tmp" diff --git a/db-functions b/db-functions index 9724e1c..4252c59 100644 --- a/db-functions +++ b/db-functions @@ -20,7 +20,7 @@ source_makepkg () { } repo_lock () { #repo_lock repo-name arch - LOCKFILE="/tmp/.repolck.$1.$2" + LOCKFILE="$TMPDIR/.repolock.$1.$2" if [ -f "$LOCKFILE" ]; then owner="$(/usr/bin/stat -c %U $LOCKFILE)" echo "error: db generation is already in progress (started by $owner)" @@ -31,7 +31,7 @@ repo_lock () { #repo_lock repo-name arch } repo_unlock () { #repo_unlock repo-name arch - LOCKFILE="/tmp/.repolck.$1.$2" + LOCKFILE="$TMPDIR/.repolock.$1.$2" if [ ! -f "$LOCKFILE" ]; then echo "error: repo lock doesn't exist... something went terribly wrong!" else diff --git a/db-move b/db-move index 5be1813..789e536 100755 --- a/db-move +++ b/db-move @@ -24,7 +24,7 @@ svnrepo_to="$repoto-$arch" [ "$UID" = "" ] && UID=$(uid) -WORKDIR="/home/tmp/db-move.$svnrepo_from.$svnrepo_to.$UID" +WORKDIR="$TMPDIR/db-move.$svnrepo_from.$svnrepo_to.$UID" cleanup() { # unlock diff --git a/db-remove b/db-remove index bdbd254..47268ed 100755 --- a/db-remove +++ b/db-remove @@ -20,7 +20,7 @@ svnrepo="$reponame-$arch" [ "$UID" = "" ] && UID=$(uid) -WORKDIR="/home/tmp/db-remove.$svnrepo.$UID" +WORKDIR="$TMPDIR/db-remove.$svnrepo.$UID" cleanup() { # unlock diff --git a/db-update b/db-update index 84764eb..315c107 100755 --- a/db-update +++ b/db-update @@ -20,7 +20,7 @@ stagedir="$STAGING/$reponame" [ "$UID" = "" ] && UID=$(uid) -WORKDIR="/home/tmp/db-update.$svnrepo.$UID" +WORKDIR="$TMPDIR/db-update.$svnrepo.$UID" ADDPKGS="" REMPKGS="" diff --git a/testing2x b/testing2x index d87de11..812edd3 100755 --- a/testing2x +++ b/testing2x @@ -12,7 +12,7 @@ case "$0" in arch="i686" ;; esac -WORKDIR="/home/tmp/testing2x.$UID" +WORKDIR="$TMPDIR/testing2x.$UID" cleanup() { rm -rf "${WORKDIR}" -- cgit v1.2.2 From 830af506b6b1ed7f3c73d55e0771f2ded6cc795a Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Thu, 20 Nov 2008 21:51:17 -0800 Subject: Switch svn path to be proper on gerolde Signed-off-by: Aaron Griffin --- config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config b/config index 5fe1969..35fe446 100644 --- a/config +++ b/config @@ -1,5 +1,5 @@ FTP_BASE="/srv/ftp/" -SVN_PATH="file:///srv/svn/packages" +SVN_PATH="file:///srv/svn-packages" STAGING="$HOME/staging/" TMPDIR="/srv/tmp" -- cgit v1.2.2 From c7fa1913b56e7619b64b93a10e130cf90508737a Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Fri, 21 Nov 2008 00:57:46 -0500 Subject: Fix patching for chmod call dirname was failing if the target was a dir Signed-off-by: Aaron Griffin --- db-functions | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/db-functions b/db-functions index 4252c59..11ab422 100644 --- a/db-functions +++ b/db-functions @@ -70,7 +70,9 @@ check_pkg_arch () { #check_pkg_arch pkgfile arch # have proper DB permissions copy_helper () { #copy_helper file dest /bin/cp $1 $2 || return 1 - /bin/chmod 664 "$(dirname $2)/$(basename $1)" || return 1 + dir=$2 + [ -f $dir ] && dir=$(dirname $dir) + /bin/chmod 664 "$dir/$(basename $1)" || return 1 } # vim: set ts=4 sw=4 noet ft=sh: -- cgit v1.2.2 From 07ec0708d31794221bec7ee3e7f07755707fd36a Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Fri, 21 Nov 2008 01:05:09 -0500 Subject: Clear traps on cleanup This prevents us from trying to remove the lockfile twice, and calling cleanup multiple times Signed-off-by: Aaron Griffin --- db-move | 1 + db-remove | 1 + db-update | 1 + testing2x | 45 +++++++++++++++++++++++---------------------- 4 files changed, 26 insertions(+), 22 deletions(-) diff --git a/db-move b/db-move index 789e536..8d006f4 100755 --- a/db-move +++ b/db-move @@ -27,6 +27,7 @@ svnrepo_to="$repoto-$arch" WORKDIR="$TMPDIR/db-move.$svnrepo_from.$svnrepo_to.$UID" cleanup() { + trap '' 0 2 # unlock repo_unlock $repoto $arch repo_unlock $repofrom $arch diff --git a/db-remove b/db-remove index 47268ed..2e0307d 100755 --- a/db-remove +++ b/db-remove @@ -23,6 +23,7 @@ svnrepo="$reponame-$arch" WORKDIR="$TMPDIR/db-remove.$svnrepo.$UID" cleanup() { + trap '' 0 2 # unlock repo_unlock $reponame $arch rm -rf "$WORKDIR" diff --git a/db-update b/db-update index 315c107..9aba199 100755 --- a/db-update +++ b/db-update @@ -47,6 +47,7 @@ getpkgname() { } cleanup() { + trap '' 0 2 # unlock repo_unlock $reponame $arch rm -rf "$WORKDIR" diff --git a/testing2x b/testing2x index 812edd3..b576640 100755 --- a/testing2x +++ b/testing2x @@ -5,23 +5,24 @@ source_makepkg case "$0" in - *64) + *64) arch="x86_64" ;; - *) + *) arch="i686" ;; esac WORKDIR="$TMPDIR/testing2x.$UID" cleanup() { - rm -rf "${WORKDIR}" - [ -n "$1" ] && exit $1 + trap '' 0 2 + rm -rf "${WORKDIR}" + [ -n "$1" ] && exit $1 } ctrl_c() { - echo "Interrupted" >&2 - cleanup 0 + echo "Interrupted" >&2 + cleanup 0 } trap ctrl_c 2 @@ -32,23 +33,23 @@ cd "${WORKDIR}" cd checkout for pkg in $*; do - moved=0 - /usr/bin/svn up -q ${pkg} - if [ -f "${pkg}/repos/testing-${arch}/${BUILDSCRIPT}" ]; then - for repo in core extra; do - if [ -f "${pkg}/repos/${repo}-${arch}/${BUILDSCRIPT}" ]; then - echo "===> Moving package '${pkg}': testing-${arch} -> ${repo}-${arch}" - $(dirname $0)/db-move "${pkg}" "testing" "${repo}" "${arch}" - moved=1 - break - fi - done - if [ ${moved} -eq 0 ]; then - echo "===> Warning: ${pkg} is only in testing-${arch}, cannot determine where to move it" + moved=0 + /usr/bin/svn up -q ${pkg} + if [ -f "${pkg}/repos/testing-${arch}/${BUILDSCRIPT}" ]; then + for repo in core extra; do + if [ -f "${pkg}/repos/${repo}-${arch}/${BUILDSCRIPT}" ]; then + echo "===> Moving package '${pkg}': testing-${arch} -> ${repo}-${arch}" + $(dirname $0)/db-move "${pkg}" "testing" "${repo}" "${arch}" + moved=1 + break + fi + done + if [ ${moved} -eq 0 ]; then + echo "===> Warning: ${pkg} is only in testing-${arch}, cannot determine where to move it" + fi + else + echo "===> Warning: ${pkg} is not in testing-${arch}" fi - else - echo "===> Warning: ${pkg} is not in testing-${arch}" - fi done cleanup -- cgit v1.2.2 From 836152eed8893d6884796363419ce61ea7b7b417 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Fri, 21 Nov 2008 01:42:24 -0500 Subject: Remove unused cron-job 'genpkglist' Signed-off-by: Aaron Griffin --- cron-jobs/genpkglist | 28 ---------------------------- 1 file changed, 28 deletions(-) delete mode 100755 cron-jobs/genpkglist diff --git a/cron-jobs/genpkglist b/cron-jobs/genpkglist deleted file mode 100755 index 0b5d76f..0000000 --- a/cron-jobs/genpkglist +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash - -if [ $# -ne 2 ]; then - echo "usage: $(basename $0) " - exit 1 -fi - -reponame=$1 -arch=$2 - -##### Arch specific stuff. TODO make this configurable ##### -ftppath="/home/ftp/$reponame/os/$arch/" -############################################################ - -if [ ! -d "$ftppath" ]; then - echo "FTP path '$ftppath' does not exist" - exit 1 -fi - -if [ ! -f /etc/makepkg.conf ]; then - echo "/etc/makepkg.conf not found! Aborting" - exit 1 -fi - -. /etc/makepkg.conf - -cd "$ftppath" -ls -1 *$PKGEXT -- cgit v1.2.2 From eada2876aa5860edc17949154a505b8a431b1b83 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Fri, 21 Nov 2008 01:42:56 -0500 Subject: Move cron-jobs to /srv from /home Signed-off-by: Aaron Griffin --- cron-jobs/adjust-permissions | 4 ++-- cron-jobs/createFileLists | 2 +- cron-jobs/sourceballs | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cron-jobs/adjust-permissions b/cron-jobs/adjust-permissions index 1493f4b..307c319 100755 --- a/cron-jobs/adjust-permissions +++ b/cron-jobs/adjust-permissions @@ -9,7 +9,7 @@ fi #adjust the nice level to run at a lower priority /usr/bin/renice +10 -p $$ > /dev/null -cd /home/ftp +cd /srv/ftp /bin/chown -R ftp:ftp-arch core/os/i686 /bin/chown -R ftp:ftp-arch core/os/x86_64 /bin/chown -R ftp:ftp-extra {extra,testing}/os/i686 @@ -20,6 +20,6 @@ for d in core extra testing community; do /bin/chmod -R g+w $d/os/i686 /bin/chmod -R g+w $d/os/x86_64 done -/bin/chmod 555 /home/ftp +/bin/chmod 555 /srv/ftp rm -f /tmp/.ftpmaint.lck diff --git a/cron-jobs/createFileLists b/cron-jobs/createFileLists index 044b6d7..9ce108c 100755 --- a/cron-jobs/createFileLists +++ b/cron-jobs/createFileLists @@ -1,6 +1,6 @@ #!/bin/bash -reposdir=/home/ftp/ +reposdir=/srv/ftp/ #targetdir=/home/pierre/public_html/test-repo/ targetdir=$reposdir repos="core extra unstable testing community" diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index fc313e3..00ca5e3 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -1,7 +1,7 @@ #!/bin/bash -ftpbase="/home/ftp" -srcbase="/home/ftp/sources" +ftpbase="/srv/ftp" +srcbase="/srv/ftp/sources" repos="core extra testing" arches="i686 x86_64" -- cgit v1.2.2 From f100999276bb5719675d843ed85f780eefbcc249 Mon Sep 17 00:00:00 2001 From: Jan de Groot Date: Fri, 21 Nov 2008 10:49:58 -0500 Subject: Fix misc-scripts/ftpdir-cleanup pathing for /srv Signed-off-by: Aaron Griffin --- misc-scripts/ftpdir-cleanup | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/misc-scripts/ftpdir-cleanup b/misc-scripts/ftpdir-cleanup index e44dd92..2b91035 100755 --- a/misc-scripts/ftpdir-cleanup +++ b/misc-scripts/ftpdir-cleanup @@ -9,7 +9,7 @@ reponame=$1 arch=$2 ##### Arch specific stuff. TODO make this configurable ##### -ftppath="/home/ftp/$reponame/os/$arch/" +ftppath="/srv/ftp/$reponame/os/$arch/" ############################################################ if [ ! -d "$ftppath" ]; then @@ -90,7 +90,7 @@ echo "Scan complete for $reponame ($arch) at ${ftppath}" if [ -n "$DELETEFILES" ]; then echo " The following files are out of date" - echo " They will be moved to /home/package-cleanup" + echo " They will be moved to /srv/package-cleanup" for f in $DELETEFILES; do echo " $f" done @@ -107,18 +107,18 @@ fi if [ -n "$EXTRAFILES" ]; then echo " The following files are in the repo but not the db" - echo " They will be moved to /home/package-cleanup" + echo " They will be moved to /srv/package-cleanup" for f in $EXTRAFILES; do echo " $f" done fi if [ -n "${DELETEFILES}" ]; then - mv ${DELETEFILES} /home/package-cleanup/ + mv ${DELETEFILES} /srv/package-cleanup/ echo "" fi if [ -n "${EXTRAFILES}" ]; then - mv ${EXTRAFILES} /home/package-cleanup/ + mv ${EXTRAFILES} /srv/package-cleanup/ echo "" fi -- cgit v1.2.2 From 6b1f06f59fd13379841b1d3cdc5e51cbf8909e66 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Fri, 21 Nov 2008 15:22:46 -0500 Subject: Drop unstable from processing in scripts Signed-off-by: Dan McGee Signed-off-by: Aaron Griffin --- cron-jobs/createFileLists | 2 +- cron-jobs/ftpdir-cleanup | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cron-jobs/createFileLists b/cron-jobs/createFileLists index 9ce108c..697c39d 100755 --- a/cron-jobs/createFileLists +++ b/cron-jobs/createFileLists @@ -3,7 +3,7 @@ reposdir=/srv/ftp/ #targetdir=/home/pierre/public_html/test-repo/ targetdir=$reposdir -repos="core extra unstable testing community" +repos="core extra testing community" arches="i686 x86_64" if [ -f "/tmp/createFileList.lock" ]; then diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index 8b73956..1f18b17 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -1,7 +1,7 @@ #!/bin/bash #TODO add community -repos="core extra unstable testing" +repos="core extra testing" arches="i686 x86_64" LOCKFILE="/tmp/.ftpdircleanup.lock" -- cgit v1.2.2 From f41f8672d040c62714c154f6cfe7af626f3941a3 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Fri, 21 Nov 2008 15:24:25 -0500 Subject: Adjust remaining scripts for new repo locations Edited-by: Aaron Griffin Signed-off-by: Dan McGee Signed-off-by: Aaron Griffin --- misc-scripts/find-dupes | 2 +- misc-scripts/make-sourceball | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/misc-scripts/find-dupes b/misc-scripts/find-dupes index 82cd1a4..372108f 100755 --- a/misc-scripts/find-dupes +++ b/misc-scripts/find-dupes @@ -9,7 +9,7 @@ reponame=$1 arch=$2 ##### Arch specific stuff. TODO make this configurable ##### -ftppath="/home/ftp/$reponame/os/$arch/" +ftppath="/srv/ftp/$reponame/os/$arch/" ############################################################ if [ ! -d "$ftppath" ]; then diff --git a/misc-scripts/make-sourceball b/misc-scripts/make-sourceball index 7f75fe8..0006092 100755 --- a/misc-scripts/make-sourceball +++ b/misc-scripts/make-sourceball @@ -14,8 +14,8 @@ reponame="$2" arch="$3" ##### Arch specific stuff. TODO make this configurable ##### -srcpath="/home/ftp/sources/" -svnpath="file:///home/svn-packages/$packagename/" +srcpath="/srv/ftp/sources/" +svnpath="file:///srv/svn-packages/$packagename/" ############################################################ WORKDIR="/tmp/make-sourceball.$packagename.$UID" -- cgit v1.2.2 From 309e7c2e5d6f685c99307adfc78ab05939c72f8f Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Fri, 21 Nov 2008 13:43:40 -0800 Subject: Remove /del dir usage from db-update db-remove is superior and requiring package files is dumb. Kill off this dir and clean up some code, yay! Signed-off-by: Aaron Griffin --- db-update | 64 ++++++++++++++------------------------------------------------- 1 file changed, 14 insertions(+), 50 deletions(-) diff --git a/db-update b/db-update index 9aba199..397a046 100755 --- a/db-update +++ b/db-update @@ -22,7 +22,6 @@ stagedir="$STAGING/$reponame" WORKDIR="$TMPDIR/db-update.$svnrepo.$UID" ADDPKGS="" -REMPKGS="" if [ ! -d "$ftppath" ]; then echo "FTP path for this repo ($reponame) is missing" @@ -91,9 +90,19 @@ if [ -d "${stagedir}64" ]; then /bin/mv "${stagedir}64/del/"* "$stagedir/del/" fi +if [ -d "${stagedir}/add" ]; then + echo "--------------------------------------------------" + echo "It looks like you have an old staging dir" + echo "The 'add' and 'del' dirs are no longer used." + echo "Please delete staging//{add,del}" + echo " and ensure you are using the newest devtools" + echo "--------------------------------------------------" + /bin/mv "${stagedir}/add/"* "$stagedir/" +fi + to_add="" -if [ -d "$stagedir/add" ]; then - ADDPKGS="$(/bin/ls $stagedir/add/*-${arch}$PKGEXT 2>/dev/null)" +if [ -d "$stagedir" ]; then + ADDPKGS="$(/bin/ls $stagedir/*-${arch}$PKGEXT 2>/dev/null)" fi if [ -n "$ADDPKGS" ]; then @@ -142,51 +151,6 @@ else echo "No packages to add" fi -to_rem="" -if [ -d "$stagedir/del" ]; then - REMPKGS="$(/bin/ls $stagedir/del/*-${arch}$PKGEXT 2>/dev/null)" -fi - -if [ -n "$REMPKGS" ]; then - echo "==> Processing deleted packages for repository '$reponame'..." >&2 - - if [ ! -d "$WORKDIR/checkout" ]; then - cd "$WORKDIR" - /usr/bin/svn checkout -N $SVN_PATH checkout - fi - cd "$WORKDIR/checkout" - - #TODO removal shouldn't require a package file - for pkg in $REMPKGS; do - _pkgfile=$(basename $pkg) - _pkgname="$(getpkgname $pkg)" - echo " Checking SVN for $_pkgname" - svn up -q $_pkgname - if [ -d "$_pkgname/repos/$svnrepo" ]; then - echo " WARNING: $_pkgname still exists in $svnrepo" - else - to_rem="$to_rem $_pkgname" - fi - done - - if [ -n "$to_rem" ]; then - cd "$WORKDIR/build/" - - #NOTE: to_rem consists of package NAMES only - /usr/bin/repo-remove "$reponame.db.tar.$DB_COMPRESSION" $to_rem - - for rem in $REMPKGS; do - rem="$(basename $rem)" - /bin/rm -f "$ftppath/$rem" - done - else - rm -f "build/$reponame.db.tar.$DB_COMPRESSION" - echo "Errors found when removing packages" - fi -else - echo "No packages to delete" -fi - # if non empty, move all build dirs if [ $(/bin/ls "$WORKDIR/build/" 2>/dev/null | wc -l) != 0 ]; then echo "Copying new files to '$ftppath'" @@ -200,9 +164,9 @@ else echo "Nothing to copy, no work done" fi -if [ -n "$ADDPKGS" -o -n "$REMPKGS" ]; then +if [ -n "$ADDPKGS" ]; then echo "Cleaning staging dir" - /bin/rm $ADDPKGS $REMPKGS + /bin/rm $ADDPKGS fi cleanup -- cgit v1.2.2 From 9960434e090fc4a02adb2434c6f941c725642158 Mon Sep 17 00:00:00 2001 From: Allan McRae Date: Fri, 21 Nov 2008 15:20:41 -0800 Subject: Make working directory in testing2x Signed-off-by: Allan McRae Signed-off-by: Aaron Griffin --- testing2x | 1 + 1 file changed, 1 insertion(+) diff --git a/testing2x b/testing2x index b576640..a5e3098 100755 --- a/testing2x +++ b/testing2x @@ -28,6 +28,7 @@ ctrl_c() { trap ctrl_c 2 trap cleanup 0 +/bin/mkdir "${WORKDIR}" cd "${WORKDIR}" /usr/bin/svn checkout -N $SVN_PATH checkout cd checkout -- cgit v1.2.2 From 3606df775189459eef24fdec2bad5cab6ccc7ec6 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Fri, 21 Nov 2008 17:14:10 -0800 Subject: Fix a variable overwrite when sourcing PKGBUILDs The local variable 'arch' was being overwritten when we source PKGBUILDs. Change it to _arch wherever we deal with PKGBUILD files Signed-off-by: Aaron Griffin --- db-move | 24 ++++++++++++------------ db-update | 22 +++++++++++----------- testing2x | 16 ++++++++-------- 3 files changed, 31 insertions(+), 31 deletions(-) diff --git a/db-move b/db-move index 8d006f4..74e0b0b 100755 --- a/db-move +++ b/db-move @@ -13,14 +13,14 @@ source_makepkg packagename="$1" repofrom="$2" repoto="$3" -arch="$4" +_arch="$4" -export CARCH="$arch" +export CARCH="$_arch" -ftppath_from="$FTP_BASE/$repofrom/os/$arch/" -ftppath_to="$FTP_BASE/$repoto/os/$arch/" -svnrepo_from="$repofrom-$arch" -svnrepo_to="$repoto-$arch" +ftppath_from="$FTP_BASE/$repofrom/os/$_arch/" +ftppath_to="$FTP_BASE/$repoto/os/$_arch/" +svnrepo_from="$repofrom-$_arch" +svnrepo_to="$repoto-$_arch" [ "$UID" = "" ] && UID=$(uid) @@ -29,8 +29,8 @@ WORKDIR="$TMPDIR/db-move.$svnrepo_from.$svnrepo_to.$UID" cleanup() { trap '' 0 2 # unlock - repo_unlock $repoto $arch - repo_unlock $repofrom $arch + repo_unlock $repoto $_arch + repo_unlock $repofrom $_arch rm -rf "$WORKDIR" [ "$1" ] && exit $1 } @@ -48,8 +48,8 @@ die() { trap ctrl_c 2 trap cleanup 0 -repo_lock $repoto $arch -repo_lock $repofrom $arch +repo_lock $repoto $_arch +repo_lock $repofrom $_arch /bin/mkdir -p "$WORKDIR" @@ -60,7 +60,7 @@ cd checkout /usr/bin/svn up -q $packagename if [ -d "$packagename/repos/$svnrepo_from" ]; then . "$packagename/repos/$svnrepo_from/$BUILDSCRIPT" - _pkgfile="$pkgname-$pkgver-$pkgrel-$CARCH$PKGEXT" + _pkgfile="$pkgname-$pkgver-$pkgrel-$_arch$PKGEXT" if [ ! -f "$ftppath_from/$_pkgfile" ]; then die "error: package file '$_pkgfile' not found in repo '$repofrom'" @@ -74,7 +74,7 @@ if [ -d "$packagename/repos/$svnrepo_from" ]; then echo " Moving svn entries" /usr/bin/svn mv -r HEAD "$packagename/repos/$svnrepo_from" "$packagename/repos/$svnrepo_to" - /usr/bin/svn commit -m "$(basename $0): moved $packagename from [$repofrom] to [$repoto] ($arch)" + /usr/bin/svn commit -m "$(basename $0): moved $packagename from [$repofrom] to [$repoto] ($_arch)" echo " Moving package file and updating DBs" cd "$WORKDIR" diff --git a/db-update b/db-update index 397a046..d98b60d 100755 --- a/db-update +++ b/db-update @@ -10,12 +10,12 @@ fi source_makepkg reponame="$1" -arch="$2" +_arch="$2" -export CARCH="$arch" +export CARCH="$_arch" -ftppath="$FTP_BASE/$reponame/os/$arch/" -svnrepo="$reponame-$arch" +ftppath="$FTP_BASE/$reponame/os/$_arch/" +svnrepo="$reponame-$_arch" stagedir="$STAGING/$reponame" [ "$UID" = "" ] && UID=$(uid) @@ -48,7 +48,7 @@ getpkgname() { cleanup() { trap '' 0 2 # unlock - repo_unlock $reponame $arch + repo_unlock $reponame $_arch rm -rf "$WORKDIR" [ "$1" ] && exit $1 } @@ -66,7 +66,7 @@ die() { trap ctrl_c 2 trap cleanup 0 -repo_lock $reponame $arch +repo_lock $reponame $_arch /bin/mkdir -p "$WORKDIR/build" cd "$WORKDIR" @@ -78,7 +78,7 @@ else touch "build/$reponame.db.tar.$DB_COMPRESSION" fi -echo "Updating DB for $reponame $arch" +echo "Updating DB for $reponame $_arch" if [ -d "${stagedir}64" ]; then echo "--------------------------------------------------" @@ -102,7 +102,7 @@ fi to_add="" if [ -d "$stagedir" ]; then - ADDPKGS="$(/bin/ls $stagedir/*-${arch}$PKGEXT 2>/dev/null)" + ADDPKGS="$(/bin/ls $stagedir/*-${_arch}$PKGEXT 2>/dev/null)" fi if [ -n "$ADDPKGS" ]; then @@ -116,15 +116,15 @@ if [ -n "$ADDPKGS" ]; then _pkgfile=$(basename $pkg) _pkgname="$(getpkgname $pkg)" - echo " Validating package arch ($arch) $_pkgname" - if ! check_pkg_arch "$pkg" "$arch"; then + echo " Validating package arch ($_arch) $_pkgname" + if ! check_pkg_arch "$pkg" "$_arch"; then echo " ERROR: $_pkgfile was built for the wrong architecture" else echo " Checking SVN for $_pkgname" /usr/bin/svn up -q $_pkgname if [ -d "$_pkgname/repos/$svnrepo" ]; then . "$_pkgname/repos/$svnrepo/$BUILDSCRIPT" - if [ "$_pkgfile" = "$pkgname-$pkgver-$pkgrel-$CARCH$PKGEXT" ]; then + if [ "$_pkgfile" = "$pkgname-$pkgver-$pkgrel-$_arch$PKGEXT" ]; then to_add="$to_add $pkg" else echo " WARNING: $_pkgfile does not match $BUILDSCRIPT in $svnrepo" diff --git a/testing2x b/testing2x index a5e3098..6a3a70e 100755 --- a/testing2x +++ b/testing2x @@ -6,10 +6,10 @@ source_makepkg case "$0" in *64) - arch="x86_64" + _arch="x86_64" ;; *) - arch="i686" + _arch="i686" ;; esac WORKDIR="$TMPDIR/testing2x.$UID" @@ -36,20 +36,20 @@ cd checkout for pkg in $*; do moved=0 /usr/bin/svn up -q ${pkg} - if [ -f "${pkg}/repos/testing-${arch}/${BUILDSCRIPT}" ]; then + if [ -f "${pkg}/repos/testing-${_arch}/${BUILDSCRIPT}" ]; then for repo in core extra; do - if [ -f "${pkg}/repos/${repo}-${arch}/${BUILDSCRIPT}" ]; then - echo "===> Moving package '${pkg}': testing-${arch} -> ${repo}-${arch}" - $(dirname $0)/db-move "${pkg}" "testing" "${repo}" "${arch}" + if [ -f "${pkg}/repos/${repo}-${_arch}/${BUILDSCRIPT}" ]; then + echo "===> Moving package '${pkg}': testing-${_arch} -> ${repo}-${_arch}" + $(dirname $0)/db-move "${pkg}" "testing" "${repo}" "${_arch}" moved=1 break fi done if [ ${moved} -eq 0 ]; then - echo "===> Warning: ${pkg} is only in testing-${arch}, cannot determine where to move it" + echo "===> Warning: ${pkg} is only in testing-${_arch}, cannot determine where to move it" fi else - echo "===> Warning: ${pkg} is not in testing-${arch}" + echo "===> Warning: ${pkg} is not in testing-${_arch}" fi done -- cgit v1.2.2 From a7a268356fac69c94d99315553f4d61fa2744261 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Fri, 21 Nov 2008 20:31:13 -0600 Subject: Add umask setting/restoring helper functions Signed-off-by: Aaron Griffin --- db-functions | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/db-functions b/db-functions index 11ab422..8c8c667 100644 --- a/db-functions +++ b/db-functions @@ -19,6 +19,16 @@ source_makepkg () { fi } +UMASK="" +set_umask () { + UMASK="$(umask)" + umask 002 +} + +restore_umask () { + umask $UMASK +} + repo_lock () { #repo_lock repo-name arch LOCKFILE="$TMPDIR/.repolock.$1.$2" if [ -f "$LOCKFILE" ]; then -- cgit v1.2.2 From 5ed9e55f4d6a26452baaffadba407459ff96f295 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Sat, 22 Nov 2008 17:48:58 -0600 Subject: Remove 'copy_helper' as chmodding is fail The chmod doesn't work unless the user is the owner of the file Resorting to umask changes here Signed-off-by: Aaron Griffin --- db-functions | 9 --------- db-move | 6 +++--- db-remove | 4 ++-- db-update | 6 +++--- 4 files changed, 8 insertions(+), 17 deletions(-) diff --git a/db-functions b/db-functions index 8c8c667..740922f 100644 --- a/db-functions +++ b/db-functions @@ -76,13 +76,4 @@ check_pkg_arch () { #check_pkg_arch pkgfile arch fi } -# Simple helper function to ensure we always -# have proper DB permissions -copy_helper () { #copy_helper file dest - /bin/cp $1 $2 || return 1 - dir=$2 - [ -f $dir ] && dir=$(dirname $dir) - /bin/chmod 664 "$dir/$(basename $1)" || return 1 -} - # vim: set ts=4 sw=4 noet ft=sh: diff --git a/db-move b/db-move index 74e0b0b..0e0587c 100755 --- a/db-move +++ b/db-move @@ -83,7 +83,7 @@ if [ -d "$packagename/repos/$svnrepo_from" ]; then # copy the db file into our working area if [ -f "$ftppath_from/$repofrom.db.tar.$DB_COMPRESSION" ]; then - copy_helper "$ftppath_from/$repofrom.db.tar.$DB_COMPRESSION" . + /bin/cp "$ftppath_from/$repofrom.db.tar.$DB_COMPRESSION" . else touch "$repofrom.db.tar.$DB_COMPRESSION" fi @@ -94,12 +94,12 @@ if [ -d "$packagename/repos/$svnrepo_from" ]; then echo " Package files will be cleaned up automatically" if [ -f "$ftppath_to/$repoto.db.tar.$DB_COMPRESSION" ]; then - copy_helper "$ftppath_to/$repoto.db.tar.$DB_COMPRESSION" . + /bin/cp "$ftppath_to/$repoto.db.tar.$DB_COMPRESSION" . else touch "$repoto.db.tar.$DB_COMPRESSION" fi - copy_helper "$ftppath_from/$_pkgfile" . + /bin/cp "$ftppath_from/$_pkgfile" . /usr/bin/repo-add "$repoto.db.tar.$DB_COMPRESSION" $_pkgfile || die "Error in repo-add" #use '*' to move the old DB too mv $repoto.db.tar.$DB_COMPRESSION* $_pkgfile $ftppath_to diff --git a/db-remove b/db-remove index 2e0307d..c5b6819 100755 --- a/db-remove +++ b/db-remove @@ -66,7 +66,7 @@ cd "$WORKDIR" # copy the db file into our working area if [ -f "$ftppath/$reponame.db.tar.$DB_COMPRESSION" ]; then - copy_helper "$ftppath/$reponame.db.tar.$DB_COMPRESSION" build/ + /bin/cp "$ftppath/$reponame.db.tar.$DB_COMPRESSION" build/ else echo "No database found at '$ftpdir', nothing more to do" exit 0 @@ -76,7 +76,7 @@ echo " Removing from $reponame DB file" cd build/ /usr/bin/repo-remove "$reponame.db.tar.$DB_COMPRESSION" $packagename -copy_helper "$reponame.db.tar.$DB_COMPRESSION" "$ftppath/" +/bin/cp "$reponame.db.tar.$DB_COMPRESSION" "$ftppath/" echo "Package files will be cleaned up automatically" diff --git a/db-update b/db-update index d98b60d..c76337a 100755 --- a/db-update +++ b/db-update @@ -73,7 +73,7 @@ cd "$WORKDIR" # copy the db file into our working area if [ -f "$ftppath/$reponame.db.tar.$DB_COMPRESSION" ]; then - copy_helper "$ftppath/$reponame.db.tar.$DB_COMPRESSION" build/ + /bin/cp "$ftppath/$reponame.db.tar.$DB_COMPRESSION" build/ else touch "build/$reponame.db.tar.$DB_COMPRESSION" fi @@ -137,7 +137,7 @@ if [ -n "$ADDPKGS" ]; then if [ -n "$to_add" ]; then cd "$WORKDIR/build/" - for f in $to_add; do copy_helper "$f" .; done + for f in $to_add; do /bin/cp "$f" .; done pkgs="" for pkg in $to_add; do pkgs="$pkgs $(basename $pkg)"; done @@ -155,7 +155,7 @@ fi if [ $(/bin/ls "$WORKDIR/build/" 2>/dev/null | wc -l) != 0 ]; then echo "Copying new files to '$ftppath'" for f in "$WORKDIR/build/"*; do - if ! copy_helper "$f" "$ftppath"; then + if ! /bin/cp "$f" "$ftppath"; then die "error: failure while copying files to $ftppath" fi done -- cgit v1.2.2 From e4da6b3a2f2ba96e6f1ee84758519078c3efc779 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Sat, 22 Nov 2008 17:50:59 -0600 Subject: Set the umask to allow g+w This should hopefully cover all our permission bases with regard to new files Signed-off-by: Aaron Griffin --- db-functions | 2 ++ 1 file changed, 2 insertions(+) diff --git a/db-functions b/db-functions index 740922f..ae1a171 100644 --- a/db-functions +++ b/db-functions @@ -37,6 +37,7 @@ repo_lock () { #repo_lock repo-name arch exit 1 else /bin/touch "$LOCKFILE" + set_umask fi } @@ -47,6 +48,7 @@ repo_unlock () { #repo_unlock repo-name arch else rm -f "$LOCKFILE" fi + restore_umask } # Get the package name from the filename -- cgit v1.2.2 From e27f2f70a4a6490abef64c66b7a74915f2d3e2e5 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Sun, 23 Nov 2008 00:20:39 -0600 Subject: Ensure we keep track of our initial umask If we call set_umask more than once, make sure we don't overwrite our initial umask, so we can restore it right Signed-off-by: Aaron Griffin --- db-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db-functions b/db-functions index ae1a171..b196d57 100644 --- a/db-functions +++ b/db-functions @@ -21,7 +21,7 @@ source_makepkg () { UMASK="" set_umask () { - UMASK="$(umask)" + [ "$UMASK" == "" ] && UMASK="$(umask)" umask 002 } -- cgit v1.2.2 From 252e11f0e5a4eba7859108e0fb4bb3593819276f Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Sun, 23 Nov 2008 02:17:39 -0600 Subject: Remove some harmless error messages If add/del dirs exist, we try to fix it all up and it can error quite loudly Signed-off-by: Aaron Griffin --- db-update | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/db-update b/db-update index c76337a..1d2efa2 100755 --- a/db-update +++ b/db-update @@ -97,7 +97,9 @@ if [ -d "${stagedir}/add" ]; then echo "Please delete staging//{add,del}" echo " and ensure you are using the newest devtools" echo "--------------------------------------------------" - /bin/mv "${stagedir}/add/"* "$stagedir/" + if [ -e "${stagedir}/add/"* ]; then + /bin/mv "${stagedir}/add/"* "$stagedir/" + fi fi to_add="" -- cgit v1.2.2 From 3fb6203f79434d48954f6ef3c5be7067beaccd86 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Sun, 23 Nov 2008 03:22:15 -0500 Subject: Only copy DB file from repo if we have pkgs We were copying our DB file from the repo to our temp build dir all the time. We should only do this if there are packages to add Signed-off-by: Aaron Griffin --- db-update | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/db-update b/db-update index 1d2efa2..42f5430 100755 --- a/db-update +++ b/db-update @@ -71,13 +71,6 @@ repo_lock $reponame $_arch /bin/mkdir -p "$WORKDIR/build" cd "$WORKDIR" -# copy the db file into our working area -if [ -f "$ftppath/$reponame.db.tar.$DB_COMPRESSION" ]; then - /bin/cp "$ftppath/$reponame.db.tar.$DB_COMPRESSION" build/ -else - touch "build/$reponame.db.tar.$DB_COMPRESSION" -fi - echo "Updating DB for $reponame $_arch" if [ -d "${stagedir}64" ]; then @@ -108,6 +101,14 @@ if [ -d "$stagedir" ]; then fi if [ -n "$ADDPKGS" ]; then + + echo "==> Copying DB file from '$reponame'..." >&2 + if [ -f "$ftppath/$reponame.db.tar.$DB_COMPRESSION" ]; then + /bin/cp "$ftppath/$reponame.db.tar.$DB_COMPRESSION" build/ + else + touch "build/$reponame.db.tar.$DB_COMPRESSION" + fi + echo "==> Processing new/updated packages for repository '$reponame'..." >&2 cd "$WORKDIR" -- cgit v1.2.2 From 73c8c32f315a6a7c493479bbbb9c322d098e9076 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20B=C3=A9langer?= Date: Mon, 1 Dec 2008 18:57:43 -0600 Subject: Remove slashes from end of config paths Clean up output so that we don't get double slashes FS#12241 Signed-off-by: Aaron Griffin --- config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config b/config index 35fe446..b6b236a 100644 --- a/config +++ b/config @@ -1,5 +1,5 @@ -FTP_BASE="/srv/ftp/" +FTP_BASE="/srv/ftp" SVN_PATH="file:///srv/svn-packages" -STAGING="$HOME/staging/" +STAGING="$HOME/staging" TMPDIR="/srv/tmp" -- cgit v1.2.2 From 1f48eaf35eecd0b35208d29d6fe6c0a8af7796df Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Mon, 1 Dec 2008 19:02:09 -0600 Subject: Do not touch db file if it doesn't exist repo-add works fine if no db file exists Signed-off-by: Aaron Griffin --- db-update | 2 -- 1 file changed, 2 deletions(-) diff --git a/db-update b/db-update index 1d2efa2..0c2d1fe 100755 --- a/db-update +++ b/db-update @@ -74,8 +74,6 @@ cd "$WORKDIR" # copy the db file into our working area if [ -f "$ftppath/$reponame.db.tar.$DB_COMPRESSION" ]; then /bin/cp "$ftppath/$reponame.db.tar.$DB_COMPRESSION" build/ -else - touch "build/$reponame.db.tar.$DB_COMPRESSION" fi echo "Updating DB for $reponame $_arch" -- cgit v1.2.2 From 87d3436dbd6fe5f3aeb7880d1596aa7a5c6e7905 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Mon, 1 Dec 2008 19:03:03 -0600 Subject: Cleanup missing db file logic Do not touch missing db files. Additionally, no need to call db-remove if the db file doesn't exist Signed-off-by: Aaron Griffin --- db-move | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/db-move b/db-move index 0e0587c..7fab215 100755 --- a/db-move +++ b/db-move @@ -84,19 +84,14 @@ if [ -d "$packagename/repos/$svnrepo_from" ]; then # copy the db file into our working area if [ -f "$ftppath_from/$repofrom.db.tar.$DB_COMPRESSION" ]; then /bin/cp "$ftppath_from/$repofrom.db.tar.$DB_COMPRESSION" . - else - touch "$repofrom.db.tar.$DB_COMPRESSION" + /usr/bin/repo-remove "$repofrom.db.tar.$DB_COMPRESSION" $packagename || die "Error in repo-remove" + #use '*' to move the old DB too + mv $repofrom.db.tar.$DB_COMPRESSION* $ftppath_from + echo " Package files will be cleaned up automatically" fi - /usr/bin/repo-remove "$repofrom.db.tar.$DB_COMPRESSION" $packagename || die "Error in repo-remove" - #use '*' to move the old DB too - mv $repofrom.db.tar.$DB_COMPRESSION* $ftppath_from - echo " Package files will be cleaned up automatically" - if [ -f "$ftppath_to/$repoto.db.tar.$DB_COMPRESSION" ]; then /bin/cp "$ftppath_to/$repoto.db.tar.$DB_COMPRESSION" . - else - touch "$repoto.db.tar.$DB_COMPRESSION" fi /bin/cp "$ftppath_from/$_pkgfile" . -- cgit v1.2.2 From 179acc5b66a610a4e9bf584e1b613b886e12cec0 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Fri, 12 Dec 2008 00:03:52 -0600 Subject: Add potential BUILDSCRIPT config setting This may get removed from makepkg.conf in the future, so let's make sure we cover our ass Signed-off-by: Aaron Griffin --- config | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config b/config index b6b236a..5c39919 100644 --- a/config +++ b/config @@ -3,3 +3,5 @@ FTP_BASE="/srv/ftp" SVN_PATH="file:///srv/svn-packages" STAGING="$HOME/staging" TMPDIR="/srv/tmp" +# To be used if $BUILDSCRIPT is removed from makepkg.conf +#BUILDSCRIPT="PKGBUILD" -- cgit v1.2.2 From 123e7560e49e04f9fdd552f9dc47bcedea2ea128 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Fri, 12 Dec 2008 00:14:39 -0600 Subject: Quiet down repo-add just a bit This assumes the -q flag patch is accepted for pacman 8) Signed-off-by: Aaron Griffin --- db-move | 4 ++-- db-remove | 2 +- db-update | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/db-move b/db-move index 7fab215..ff9677b 100755 --- a/db-move +++ b/db-move @@ -84,7 +84,7 @@ if [ -d "$packagename/repos/$svnrepo_from" ]; then # copy the db file into our working area if [ -f "$ftppath_from/$repofrom.db.tar.$DB_COMPRESSION" ]; then /bin/cp "$ftppath_from/$repofrom.db.tar.$DB_COMPRESSION" . - /usr/bin/repo-remove "$repofrom.db.tar.$DB_COMPRESSION" $packagename || die "Error in repo-remove" + /usr/bin/repo-remove -q "$repofrom.db.tar.$DB_COMPRESSION" $packagename || die "Error in repo-remove" #use '*' to move the old DB too mv $repofrom.db.tar.$DB_COMPRESSION* $ftppath_from echo " Package files will be cleaned up automatically" @@ -95,7 +95,7 @@ if [ -d "$packagename/repos/$svnrepo_from" ]; then fi /bin/cp "$ftppath_from/$_pkgfile" . - /usr/bin/repo-add "$repoto.db.tar.$DB_COMPRESSION" $_pkgfile || die "Error in repo-add" + /usr/bin/repo-add -q "$repoto.db.tar.$DB_COMPRESSION" $_pkgfile || die "Error in repo-add" #use '*' to move the old DB too mv $repoto.db.tar.$DB_COMPRESSION* $_pkgfile $ftppath_to else diff --git a/db-remove b/db-remove index c5b6819..acc849d 100755 --- a/db-remove +++ b/db-remove @@ -74,7 +74,7 @@ fi echo " Removing from $reponame DB file" cd build/ -/usr/bin/repo-remove "$reponame.db.tar.$DB_COMPRESSION" $packagename +/usr/bin/repo-remove -q "$reponame.db.tar.$DB_COMPRESSION" $packagename /bin/cp "$reponame.db.tar.$DB_COMPRESSION" "$ftppath/" diff --git a/db-update b/db-update index 0c2d1fe..40c5539 100755 --- a/db-update +++ b/db-update @@ -142,7 +142,7 @@ if [ -n "$ADDPKGS" ]; then pkgs="" for pkg in $to_add; do pkgs="$pkgs $(basename $pkg)"; done - /usr/bin/repo-add "$reponame.db.tar.$DB_COMPRESSION" $pkgs + /usr/bin/repo-add -q "$reponame.db.tar.$DB_COMPRESSION" $pkgs else rm -f "build/$reponame.db.tar.$DB_COMPRESSION" echo "Errors found when adding packages" -- cgit v1.2.2 From b0e7b12ed43b6614b6c32f1678edf53f1c668513 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Tue, 16 Dec 2008 08:50:53 -0800 Subject: Only remove packages that were successfully added This prevents some errors where packages where removed from the staging dir when they shouldn't have been. Original-idea-by: Pierre Schmitz Signed-off-by: Aaron Griffin --- db-update | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/db-update b/db-update index 40c5539..2a9dd6e 100755 --- a/db-update +++ b/db-update @@ -164,9 +164,9 @@ else echo "Nothing to copy, no work done" fi -if [ -n "$ADDPKGS" ]; then +if [ -n "$to_add" ]; then echo "Cleaning staging dir" - /bin/rm $ADDPKGS + /bin/rm $to_add fi cleanup -- cgit v1.2.2 From 1c30298885f581f3bf3731576374da7044973747 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Mon, 29 Dec 2008 11:42:07 -0800 Subject: Fix sourceball generation to work with conf file Signed-off-by: Aaron Griffin --- misc-scripts/make-sourceball | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/misc-scripts/make-sourceball b/misc-scripts/make-sourceball index 0006092..7a02abf 100755 --- a/misc-scripts/make-sourceball +++ b/misc-scripts/make-sourceball @@ -13,10 +13,7 @@ packagename="$1" reponame="$2" arch="$3" -##### Arch specific stuff. TODO make this configurable ##### -srcpath="/srv/ftp/sources/" -svnpath="file:///srv/svn-packages/$packagename/" -############################################################ +srcpath="$FTP_BASE/sources/" WORKDIR="/tmp/make-sourceball.$packagename.$UID" @@ -73,7 +70,7 @@ cd "$WORKDIR" echo "Creating Source tarball for $packagename ($reponame-$arch)" -if /usr/bin/svn export -q "$svnpath/repos/$reponame-$arch" $packagename; then +if /usr/bin/svn export -q "$SVN_PATH/repos/$reponame-$arch" $packagename; then create_srcpackage "$packagename" if [ $? -eq 0 ]; then exit 0 @@ -81,7 +78,7 @@ if /usr/bin/svn export -q "$svnpath/repos/$reponame-$arch" $packagename; then #trunk sometimes has updated URLs echo ":: Failed to download source, attempting trunk build" rm -rf "$packagename" - if /usr/bin/svn export -q "$svnpath/trunk" "$packagename"; then + if /usr/bin/svn export -q "$SVN_PATH/trunk" "$packagename"; then create_srcpackage "$packagename" if [ $? -eq 0 ]; then echo ":: Source package complete: $pkg_file" -- cgit v1.2.2 From f0d817a0deb150f576ee7733f5be40bd981f81cf Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Mon, 29 Dec 2008 13:26:02 -0800 Subject: Misc make-sourceball fixes and updates * --allsource places packages in dirs with the pkgname * Syntax errors with if statement and globbing * Set and restore umask Signed-off-by: Aaron Griffin --- misc-scripts/make-sourceball | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/misc-scripts/make-sourceball b/misc-scripts/make-sourceball index 7a02abf..44c3815 100755 --- a/misc-scripts/make-sourceball +++ b/misc-scripts/make-sourceball @@ -18,7 +18,7 @@ srcpath="$FTP_BASE/sources/" WORKDIR="/tmp/make-sourceball.$packagename.$UID" cleanup() { - # unlock + restore_umask rm -rf "$WORKDIR" [ "$1" ] && exit $1 } @@ -50,13 +50,13 @@ create_srcpackage() { mkdir -p "$srcpath" fi #Remove old sourceballs - for pkg in "$srcpath/$pkgname-*"; do + for pkg in "$srcpath/$pkgname-"*; do pkg="$(basename $pkg)" - if "$(getpkgname $pkg)" == "$pkgname" ]; then + if [ "$(getpkgname $pkg)" == "$pkgname" ]; then rm -f "$srcpath/$pkg" fi done - cp $pkg_file "$srcpath" + cp "$pkgname/$pkg_file" "$srcpath" return 0 fi @@ -65,6 +65,7 @@ create_srcpackage() { trap ctrl_c 2 trap cleanup 0 1 +set_umask /bin/mkdir -p "$WORKDIR" cd "$WORKDIR" -- cgit v1.2.2 From 81efd3d8cd39d07930b18dcd8f904153500bfbf3 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Mon, 29 Dec 2008 13:30:00 -0800 Subject: Allow devlist-mailer to send to alternate addresses Signed-off-by: Aaron Griffin --- cron-jobs/devlist-mailer | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/cron-jobs/devlist-mailer b/cron-jobs/devlist-mailer index b24d530..671cbbb 100755 --- a/cron-jobs/devlist-mailer +++ b/cron-jobs/devlist-mailer @@ -7,8 +7,12 @@ LIST="arch-dev-public@archlinux.org" FROM="repomaint@archlinux.org" SUBJECT="Repository Maintenance $(date +"%d-%m-%Y %H:%M")" -if [ $# -eq 1 ]; then - SUBJECT="$1" +if [ $# -ge 1 ]; then + SUBJECT="$1 $(date +"%d-%m-%Y %H:%M")" +fi + +if [ $# -ge 2 ]; then + LIST="$2" fi stdin="$(cat)" -- cgit v1.2.2 From 830f8f87e1ede09a8c827c1a93aa4ed5db66f0b6 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Mon, 29 Dec 2008 13:31:33 -0800 Subject: Remove an if check for old staging 'add' dirs Signed-off-by: Aaron Griffin --- db-update | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/db-update b/db-update index 2a9dd6e..254a6f9 100755 --- a/db-update +++ b/db-update @@ -95,9 +95,7 @@ if [ -d "${stagedir}/add" ]; then echo "Please delete staging//{add,del}" echo " and ensure you are using the newest devtools" echo "--------------------------------------------------" - if [ -e "${stagedir}/add/"* ]; then - /bin/mv "${stagedir}/add/"* "$stagedir/" - fi + /bin/mv "${stagedir}/add/"* "$stagedir/" fi to_add="" -- cgit v1.2.2 From d39b68ede8567fb2ee15f45995ddab8f4710e9dc Mon Sep 17 00:00:00 2001 From: Xavier Chantry Date: Sat, 3 Jan 2009 13:22:57 +0100 Subject: check_packages : add --arch option. The parsing script didn't set CARCH previously, and the flashplugin PKGBUILD exited in this case. First override the exit function to prevent the whole script to exit, and add a --arch option to be able to set CARCH correctly. To be used like this : For core and extra : ./check_packages.py --abs-tree=/home/abs/rsync/i686 --repos=core,extra --arch=i686 ./check_packages.py --abs-tree=/home/abs/rsync/x86_64 --repos=core,extra --arch=x86_64 For community : ./check_packages.py --abs-tree=/home/abs/rsync/i686 --repos=community --arch=i686 ./check_packages.py --abs-tree=/home/abs/rsync/x86_64 --repos=community --arch=x86_64 Signed-off-by: Xavier Chantry Signed-off-by: Aaron Griffin --- cron-jobs/check_archlinux/check_packages.py | 23 ++++++++++++++--------- cron-jobs/check_archlinux/parse_pkgbuilds.sh | 9 ++++++++- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/cron-jobs/check_archlinux/check_packages.py b/cron-jobs/check_archlinux/check_packages.py index 69b14c3..f0fd4a5 100755 --- a/cron-jobs/check_archlinux/check_packages.py +++ b/cron-jobs/check_archlinux/check_packages.py @@ -49,11 +49,11 @@ class Depend: self.version = version self.mod = mod -def parse_pkgbuilds(repos): +def parse_pkgbuilds(repos,arch): oldcwd = os.getcwd() os.chdir(absroot) for repo in repos: - data = commands.getoutput(oldcwd + '/parse_pkgbuilds.sh ' + repo) + data = commands.getoutput(oldcwd + '/parse_pkgbuilds.sh ' + arch + ' ' + repo) parse_data(repo,data) os.chdir(oldcwd) @@ -292,24 +292,27 @@ def print_usage(): print "Usage: ./check_packages.py [OPTION]" print "" print "Options:" - print " --abs-tree= Check specified tree (default : /var/abs)" - print " --repos= Check specified repos (default : core,extra)" + print " --abs-tree= Check the specified tree (default : /var/abs)" + print " --repos= Check the specified repos (default : core,extra)" + print " --arch= Check the specified arch (default : i686)" print " -h, --help Show this help and exit" print "" print "Examples:" print "\n Check core and extra in existing abs tree:" - print " ./check_packages.py --abs-tree=/var/abs --repos=core,extra" + print " ./check_packages.py --abs-tree=/var/abs --repos=core,extra --arch=i686" print "\n Check community:" - print " ./check_packages.py --abs-tree=/var/abs --repos=community" + print " ./check_packages.py --abs-tree=/var/abs --repos=community --arch=i686" print "" ## Default path to the abs root directory absroot = "/var/abs" ## Default list of repos to check repos = ['core', 'extra'] +## Default arch +arch = "i686" try: - opts, args = getopt.getopt(sys.argv[1:], "", ["abs-tree=", "repos="]) + opts, args = getopt.getopt(sys.argv[1:], "", ["abs-tree=", "repos=", "arch="]) except getopt.GetoptError: print_usage() sys.exit() @@ -319,6 +322,8 @@ if opts != []: absroot = a elif o in ("--repos"): repos = a.split(",") + elif o in ("--arch"): + arch = a else: print_usage() sys.exit() @@ -339,11 +344,11 @@ loadrepos = set([]) for repo in repos: loadrepos = loadrepos | set(get_repo_hierarchy(repo)) -print_heading("Integrity Check") +print_heading("Integrity Check " + arch + " of " + ",".join(repos)) print "\nPerforming integrity checks..." print "==> parsing pkgbuilds" -parse_pkgbuilds(loadrepos) +parse_pkgbuilds(loadrepos,arch) repopkgs = {} for name,pkg in packages.iteritems(): diff --git a/cron-jobs/check_archlinux/parse_pkgbuilds.sh b/cron-jobs/check_archlinux/parse_pkgbuilds.sh index d4205ae..47aec89 100755 --- a/cron-jobs/check_archlinux/parse_pkgbuilds.sh +++ b/cron-jobs/check_archlinux/parse_pkgbuilds.sh @@ -1,5 +1,10 @@ #!/bin/bash +# Usage : parse_pkgbuilds.sh arch +# Example : parse_pkgbuilds.sh i686 /var/abs/core /var/abs/extra + +exit() { return; } + parse() { unset pkgname pkgver pkgrel unset depends makedepends conflicts provides @@ -67,10 +72,12 @@ find_pkgbuilds() { fi } -if [ -z "$*" ]; then +if [ -z "$1" -o -z "$2" ]; then exit 1 fi +CARCH=$1 +shift for dir in "$@"; do find_pkgbuilds $dir done -- cgit v1.2.2 From 34dc23953ecf218cf263bf150ecb019dee36983d Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Fri, 9 Jan 2009 18:13:41 -0500 Subject: Add support for altername BASEDIRs in db-functions This is needed because we do lots of things via relative paths, and scripts in the misc-scripts dir may need some of these functions too Signed-off-by: Aaron Griffin --- db-functions | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/db-functions b/db-functions index b196d57..a2e2f72 100644 --- a/db-functions +++ b/db-functions @@ -3,7 +3,10 @@ # Random integrity things [ "$UID" = "" ] && UID=$(uid) -BASEDIR="$(dirname $0)" +if [ -z "$BASEDIR" ]; then + BASEDIR="$(dirname $0)" +fi + if [ -f "$BASEDIR/config" ]; then . "$BASEDIR/config" fi -- cgit v1.2.2 From 671f36cb092ac5b8c8c30f53c19c05b8249bdee0 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Fri, 9 Jan 2009 18:14:49 -0500 Subject: Use new BASEDIR facility in make-sourceball Signed-off-by: Aaron Griffin --- misc-scripts/make-sourceball | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/misc-scripts/make-sourceball b/misc-scripts/make-sourceball index 44c3815..c02a84d 100755 --- a/misc-scripts/make-sourceball +++ b/misc-scripts/make-sourceball @@ -5,7 +5,8 @@ if [ $# -ne 3 ]; then exit 1 fi -. "$(dirname $0)/../db-functions" +BASEDIR="$(dirname $0)/../" +. "$BASEDIR/db-functions" source_makepkg @@ -71,7 +72,7 @@ cd "$WORKDIR" echo "Creating Source tarball for $packagename ($reponame-$arch)" -if /usr/bin/svn export -q "$SVN_PATH/repos/$reponame-$arch" $packagename; then +if /usr/bin/svn export -q "$SVN_PATH/$packagename/repos/$reponame-$arch" $packagename; then create_srcpackage "$packagename" if [ $? -eq 0 ]; then exit 0 @@ -79,7 +80,7 @@ if /usr/bin/svn export -q "$SVN_PATH/repos/$reponame-$arch" $packagename; then #trunk sometimes has updated URLs echo ":: Failed to download source, attempting trunk build" rm -rf "$packagename" - if /usr/bin/svn export -q "$SVN_PATH/trunk" "$packagename"; then + if /usr/bin/svn export -q "$SVN_PATH/$packagename/trunk" "$packagename"; then create_srcpackage "$packagename" if [ $? -eq 0 ]; then echo ":: Source package complete: $pkg_file" -- cgit v1.2.2 From a3fc31dc16b442bf492ab52f9c48d4ea230878d8 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Mon, 12 Jan 2009 17:02:53 -0500 Subject: Add license checking to make-sourceball Confirm that the license of a package requires source distribution before building the source tarball TODO: Make sure we can skip this check somehow Signed-off-by: Aaron Griffin --- misc-scripts/make-sourceball | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/misc-scripts/make-sourceball b/misc-scripts/make-sourceball index c02a84d..91dc122 100755 --- a/misc-scripts/make-sourceball +++ b/misc-scripts/make-sourceball @@ -1,5 +1,8 @@ #!/bin/bash +# Allowed licenses: build only for licenses in this array +ALLOWED_LICENSES=('GPL' 'GPL1' 'GPL2' 'LGPL' 'LGPL1' 'LGPL2') + if [ $# -ne 3 ]; then echo "usage: $(basename $0) " exit 1 @@ -34,10 +37,32 @@ die() { cleanup 1 } +#usage: chk_license ${license[@]}" +chk_license() { + local l + for l in "$@"; do + l="$(echo $l | tr '[:upper:]' '[:lower:]')" + for allowed in ${ALLOWED_LICENSES[@]}; do + allowed="$(echo $allowed | tr '[:upper:]' '[:lower:]')" + if [ "$l" = "$allowed" ]; then + return 0 + fi + done + done + + return 1 +} + create_srcpackage() { if [ -d "$1" ]; then pushd "$1" >/dev/null . "$BUILDSCRIPT" + if ! chk_license ${license[@]}; then + echo "Package license does not require source tarballs. Doing nothing" + echo " license => (${license[@]})" + cleanup 0 + fi + if ! /usr/bin/makepkg --allsource >/dev/null 2>&1; then popd >/dev/null return 1 -- cgit v1.2.2 From 5f64e9993a494040b0280ed0f1daa38f68f9c46b Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Wed, 14 Jan 2009 13:31:11 -0800 Subject: Remove HH:MM from devlist-mailer subject lines This was extraneous info. Not needed as higher granularity can be gleaned from the mail's date Signed-off-by: Aaron Griffin --- cron-jobs/devlist-mailer | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cron-jobs/devlist-mailer b/cron-jobs/devlist-mailer index 671cbbb..ca2e46b 100755 --- a/cron-jobs/devlist-mailer +++ b/cron-jobs/devlist-mailer @@ -6,9 +6,9 @@ LIST="arch-dev-public@archlinux.org" #LIST="aaronmgriffin@gmail.com" FROM="repomaint@archlinux.org" -SUBJECT="Repository Maintenance $(date +"%d-%m-%Y %H:%M")" +SUBJECT="Repository Maintenance $(date +"%d-%m-%Y")" if [ $# -ge 1 ]; then - SUBJECT="$1 $(date +"%d-%m-%Y %H:%M")" + SUBJECT="$1 $(date +"%d-%m-%Y")" fi if [ $# -ge 2 ]; then -- cgit v1.2.2 From d9df504ddd648171eeff73d63afe196e153459a2 Mon Sep 17 00:00:00 2001 From: Xavier Chantry Date: Sun, 1 Feb 2009 20:59:10 +0100 Subject: check_packages: fix stupid assumption on script location. The parse_pkgbuilds.sh script was assumed to be in the current working directory, which is quite stupid since check_packages.py can be called from anywhere. Now it only assumes that check_packages.py and parse_pkgbuilds.sh are in the same directory. This should fix the empty integrity checks on arch-dev :) Signed-off-by: Xavier Chantry Signed-off-by: Dan McGee --- cron-jobs/check_archlinux/check_packages.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/cron-jobs/check_archlinux/check_packages.py b/cron-jobs/check_archlinux/check_packages.py index f0fd4a5..0392368 100755 --- a/cron-jobs/check_archlinux/check_packages.py +++ b/cron-jobs/check_archlinux/check_packages.py @@ -50,12 +50,10 @@ class Depend: self.mod = mod def parse_pkgbuilds(repos,arch): - oldcwd = os.getcwd() - os.chdir(absroot) for repo in repos: - data = commands.getoutput(oldcwd + '/parse_pkgbuilds.sh ' + arch + ' ' + repo) + data = commands.getoutput(os.path.dirname(sys.argv[0]) + '/parse_pkgbuilds.sh ' + + arch + ' ' + absroot + '/' + repo) parse_data(repo,data) - os.chdir(oldcwd) def parse_data(repo,data): attrname = None -- cgit v1.2.2 From b22c5538ee8840e7f5f6082f7a337ce43e49ac3b Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Wed, 4 Feb 2009 16:14:47 -0500 Subject: Clean up output, remove 'trunk build' Signed-off-by: Aaron Griffin --- misc-scripts/make-sourceball | 26 +++++--------------------- 1 file changed, 5 insertions(+), 21 deletions(-) diff --git a/misc-scripts/make-sourceball b/misc-scripts/make-sourceball index 91dc122..8237176 100755 --- a/misc-scripts/make-sourceball +++ b/misc-scripts/make-sourceball @@ -33,7 +33,7 @@ ctrl_c() { } die() { - echo "$*" >&2 + echo -e "$*" >&2 cleanup 1 } @@ -58,8 +58,7 @@ create_srcpackage() { pushd "$1" >/dev/null . "$BUILDSCRIPT" if ! chk_license ${license[@]}; then - echo "Package license does not require source tarballs. Doing nothing" - echo " license => (${license[@]})" + echo -e "\tPackage license (${license[@]}) does not require source tarballs. Doing nothing" cleanup 0 fi @@ -71,7 +70,6 @@ create_srcpackage() { local pkg_file="${pkgname}-${pkgver}-${pkgrel}${SRCEXT}" - echo ":: Source package complete: $pkg_file" if [ ! -d "$srcpath" ]; then mkdir -p "$srcpath" fi @@ -103,25 +101,11 @@ if /usr/bin/svn export -q "$SVN_PATH/$packagename/repos/$reponame-$arch" $packag exit 0 elif [ $? -eq 1 ]; then #trunk sometimes has updated URLs - echo ":: Failed to download source, attempting trunk build" - rm -rf "$packagename" - if /usr/bin/svn export -q "$SVN_PATH/$packagename/trunk" "$packagename"; then - create_srcpackage "$packagename" - if [ $? -eq 0 ]; then - echo ":: Source package complete: $pkg_file" - exit 0 - elif [ $? -eq 1 ]; then - die ":: Failed to download source" - elif [ $? -eq 2 ]; then - die ":: Failed to compress package" - else - die ":: Unknown failure reason" - fi - fi + die "\tFailed to download source" exit 1 elif [ $? -eq 2 ]; then - die ":: Failed to compress package" + die "\tFailed to compress package" fi else - die "Package '$packagename' does not exist in repo $reponame-$arch" + die "\tPackage does not exist in repo" fi -- cgit v1.2.2 From 9df5198acf0088ce8818bee9b3c9fbf44a26628d Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Wed, 4 Feb 2009 16:15:31 -0500 Subject: Remove 'skipping' output. Not needed Signed-off-by: Aaron Griffin --- cron-jobs/sourceballs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 00ca5e3..03b8209 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -46,6 +46,7 @@ getpkgname() { echo ${tmp%-*-*} } + FAILED_PKGS="" for repo in $repos; do @@ -59,7 +60,6 @@ for repo in $repos; do srcpkg="${pkg//-$arch$PKGEXT/$SRCEXT}" if grep $pkgname "$dirname/sourceballs.skip" >/dev/null 2>&1; then - echo ":: Skipping package '$pkgname'" continue fi -- cgit v1.2.2 From ac14b100b2e85f7c2eea63f1d7f4307e062fa46d Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Wed, 4 Feb 2009 16:20:54 -0500 Subject: More sourceball output cleanup Signed-off-by: Aaron Griffin --- misc-scripts/make-sourceball | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/misc-scripts/make-sourceball b/misc-scripts/make-sourceball index 8237176..cdbd697 100755 --- a/misc-scripts/make-sourceball +++ b/misc-scripts/make-sourceball @@ -57,6 +57,9 @@ create_srcpackage() { if [ -d "$1" ]; then pushd "$1" >/dev/null . "$BUILDSCRIPT" + + echo "Creating source tarball for $pkgname-$pkgver-$pkgrel" + if ! chk_license ${license[@]}; then echo -e "\tPackage license (${license[@]}) does not require source tarballs. Doing nothing" cleanup 0 @@ -64,7 +67,7 @@ create_srcpackage() { if ! /usr/bin/makepkg --allsource >/dev/null 2>&1; then popd >/dev/null - return 1 + die "\tFailed to download source for $pkgname-$pkgver-$pkgrel ($reponame-$arch)" fi popd >/dev/null @@ -93,19 +96,8 @@ set_umask /bin/mkdir -p "$WORKDIR" cd "$WORKDIR" -echo "Creating Source tarball for $packagename ($reponame-$arch)" - if /usr/bin/svn export -q "$SVN_PATH/$packagename/repos/$reponame-$arch" $packagename; then create_srcpackage "$packagename" - if [ $? -eq 0 ]; then - exit 0 - elif [ $? -eq 1 ]; then - #trunk sometimes has updated URLs - die "\tFailed to download source" - exit 1 - elif [ $? -eq 2 ]; then - die "\tFailed to compress package" - fi else - die "\tPackage does not exist in repo" + die "\tPackage '$packagename' does not exist in repo '$reponame-$arch'" fi -- cgit v1.2.2 From a5eb44ed1bbdf2eabd26cc02932898719375c230 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Mon, 9 Feb 2009 15:36:04 -0600 Subject: Write out failed packages to a txt file Located at /srv/ftp/sources/failed.txt Signed-off-by: Aaron Griffin --- cron-jobs/sourceballs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 03b8209..9509715 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -73,9 +73,11 @@ for repo in $repos; do done if [ -n "$FAILED_PKGS" ]; then + echo -e $FAILED_PKGS | sed "s| |\n|g" | sort -u > "$srcbase/failed.txt" + echo "" echo "The following packages failed:" - echo -e $FAILED_PKGS | sed "s| |\n\t|g" | sort -u + cat "$srcbase/failed.txt" fi cleanup -- cgit v1.2.2 From 90f27119e2d266e96f09c241d7cc84a72974d11d Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Wed, 11 Feb 2009 11:40:36 -0500 Subject: Emit makepkg --allsource to a logfile One log file per package, in /var/log/sourceballs for easier lookup and reference. Log file is removed if the command succeeds, and gzipped on failure Signed-off-by: Aaron Griffin --- misc-scripts/make-sourceball | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/misc-scripts/make-sourceball b/misc-scripts/make-sourceball index cdbd697..f45eba8 100755 --- a/misc-scripts/make-sourceball +++ b/misc-scripts/make-sourceball @@ -18,6 +18,7 @@ reponame="$2" arch="$3" srcpath="$FTP_BASE/sources/" +logpath="/var/log/sourceballs/" WORKDIR="/tmp/make-sourceball.$packagename.$UID" @@ -65,11 +66,13 @@ create_srcpackage() { cleanup 0 fi - if ! /usr/bin/makepkg --allsource >/dev/null 2>&1; then + if ! /usr/bin/makepkg --allsource >"$logpath/$pkgname" 2>&1; then popd >/dev/null + /bin/gzip -9 "$logpath/$pkgname" die "\tFailed to download source for $pkgname-$pkgver-$pkgrel ($reponame-$arch)" fi popd >/dev/null + /bin/rm "$logpath/$pkgname" local pkg_file="${pkgname}-${pkgver}-${pkgrel}${SRCEXT}" @@ -94,6 +97,7 @@ trap cleanup 0 1 set_umask /bin/mkdir -p "$WORKDIR" +/bin/mkdir -p "$logpath" cd "$WORKDIR" if /usr/bin/svn export -q "$SVN_PATH/$packagename/repos/$reponame-$arch" $packagename; then -- cgit v1.2.2 From a21429509d1d0a150cabfeefa8177e5ca51c265f Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Wed, 11 Feb 2009 11:41:41 -0500 Subject: Fail on license mismatch, instead of succeeding This is done mainly so we can get a log of all failures and ignore normal output Signed-off-by: Aaron Griffin --- misc-scripts/make-sourceball | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/misc-scripts/make-sourceball b/misc-scripts/make-sourceball index f45eba8..c2ed576 100755 --- a/misc-scripts/make-sourceball +++ b/misc-scripts/make-sourceball @@ -62,8 +62,7 @@ create_srcpackage() { echo "Creating source tarball for $pkgname-$pkgver-$pkgrel" if ! chk_license ${license[@]}; then - echo -e "\tPackage license (${license[@]}) does not require source tarballs. Doing nothing" - cleanup 0 + die -e "\tPackage license (${license[@]}) does not require source tarballs. Doing nothing" fi if ! /usr/bin/makepkg --allsource >"$logpath/$pkgname" 2>&1; then -- cgit v1.2.2 From 75de4d131ed424bfb7c349a6b55cedf9705879bd Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Wed, 11 Feb 2009 11:46:02 -0500 Subject: Output stderr to an error file for each run Signed-off-by: Aaron Griffin --- cron-jobs/sourceballs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 9509715..31ac1af 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -49,6 +49,8 @@ getpkgname() { FAILED_PKGS="" +echo "Errors occured during run:" > "$srcbase/errors.txt" + for repo in $repos; do for arch in $arches; do export CARCH="$arch" @@ -64,7 +66,8 @@ for repo in $repos; do fi if [ ! -f "$srcpath$srcpkg" ]; then - if ! $dirname/../misc-scripts/make-sourceball $pkgname $repo $arch; then + if ! $dirname/../misc-scripts/make-sourceball \ + $pkgname $repo $arch 2>>"$srcbase/errors.txt"; then FAILED_PKGS="$FAILED_PKGS $pkg" fi fi @@ -73,11 +76,8 @@ for repo in $repos; do done if [ -n "$FAILED_PKGS" ]; then - echo -e $FAILED_PKGS | sed "s| |\n|g" | sort -u > "$srcbase/failed.txt" - - echo "" - echo "The following packages failed:" - cat "$srcbase/failed.txt" + echo "The following packages failed:" > "$srcbase/failed.txt" + echo -e $FAILED_PKGS | sed "s| |\n|g" | sort -u >> "$srcbase/failed.txt" fi cleanup -- cgit v1.2.2 From 767257f9802709ef7c5f516b29bb1f498bdcea78 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Fri, 13 Feb 2009 11:53:37 -0800 Subject: Rename local 'arch' variable to '_arch' We source the PKGBUILD, which causes issues here Signed-off-by: Aaron Griffin --- misc-scripts/make-sourceball | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/misc-scripts/make-sourceball b/misc-scripts/make-sourceball index c2ed576..72f4a1f 100755 --- a/misc-scripts/make-sourceball +++ b/misc-scripts/make-sourceball @@ -15,7 +15,7 @@ source_makepkg packagename="$1" reponame="$2" -arch="$3" +_arch="$3" srcpath="$FTP_BASE/sources/" logpath="/var/log/sourceballs/" @@ -68,7 +68,7 @@ create_srcpackage() { if ! /usr/bin/makepkg --allsource >"$logpath/$pkgname" 2>&1; then popd >/dev/null /bin/gzip -9 "$logpath/$pkgname" - die "\tFailed to download source for $pkgname-$pkgver-$pkgrel ($reponame-$arch)" + die "\tFailed to download source for $pkgname-$pkgver-$pkgrel ($reponame-$_arch)" fi popd >/dev/null /bin/rm "$logpath/$pkgname" @@ -99,8 +99,8 @@ set_umask /bin/mkdir -p "$logpath" cd "$WORKDIR" -if /usr/bin/svn export -q "$SVN_PATH/$packagename/repos/$reponame-$arch" $packagename; then +if /usr/bin/svn export -q "$SVN_PATH/$packagename/repos/$reponame-$_arch" $packagename; then create_srcpackage "$packagename" else - die "\tPackage '$packagename' does not exist in repo '$reponame-$arch'" + die "\tPackage '$packagename' does not exist in repo '$reponame-$_arch'" fi -- cgit v1.2.2 From 277c01e56e42977c01a1fe709cfe2130bf177ac2 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Fri, 13 Feb 2009 12:09:46 -0800 Subject: Correct the sourceball removal step Only remove sourceballs that do NOT exist in any repo This code is experimental Signed-off-by: Aaron Griffin --- db-functions | 11 +++++++++++ misc-scripts/make-sourceball | 38 ++++++++++++++++++++++++++++++++------ 2 files changed, 43 insertions(+), 6 deletions(-) diff --git a/db-functions b/db-functions index a2e2f72..c105dab 100644 --- a/db-functions +++ b/db-functions @@ -66,6 +66,17 @@ getpkgname() { echo ${tmp%-*-*} } +# Get the pkgver-pkgrel of this package +getpkgver() { + local tmp + + tmp=${1##*/} + tmp=${tmp%$PKGEXT} + tmp=${tmp%$SRCEXT} + tmp=${tmp%-$CARCH} + echo $tmp | sed 's|.*-\(.*-.*\)$|\1|g' +} + check_pkg_arch () { #check_pkg_arch pkgfile arch local arch _arch="$(/usr/bin/bsdtar -xOf "$1" .PKGINFO | /bin/grep "^arch" | /bin/sed 's|\w*\s*=\s*\(.*\)|\1|')" diff --git a/misc-scripts/make-sourceball b/misc-scripts/make-sourceball index 72f4a1f..b882de8 100755 --- a/misc-scripts/make-sourceball +++ b/misc-scripts/make-sourceball @@ -78,16 +78,41 @@ create_srcpackage() { if [ ! -d "$srcpath" ]; then mkdir -p "$srcpath" fi - #Remove old sourceballs + cp "$pkgname/$pkg_file" "$srcpath" + + return 0 + fi +} + +remove_old() { + if [ -d "$1" ]; then + pushd "$1" >/dev/null + PKGVERS="" + for repo in *; do + cd "$repo" + . "$BUILDSCRIPT" + PKGVERS="$PKGVERS $pkgver-$pkgrel" + cd .. + done + for pkg in "$srcpath/$pkgname-"*; do pkg="$(basename $pkg)" if [ "$(getpkgname $pkg)" == "$pkgname" ]; then - rm -f "$srcpath/$pkg" + skip=0 + pver="$(getpkgver $pkg)" + for v in $PKGVERS; do + if [ "$v" = "$pver" ]; then + skip=1 + break + fi + done + if [ $skip -ne 1 ]; then + rm -f "$srcpath/$pkg" + fi fi done - cp "$pkgname/$pkg_file" "$srcpath" - return 0 + popd >/dev/null fi } @@ -99,8 +124,9 @@ set_umask /bin/mkdir -p "$logpath" cd "$WORKDIR" -if /usr/bin/svn export -q "$SVN_PATH/$packagename/repos/$reponame-$_arch" $packagename; then - create_srcpackage "$packagename" +if /usr/bin/svn export -q "$SVN_PATH/$packagename" $packagename; then + create_srcpackage "$packagename/repos/$reponame-$_arch" + remove_old "$pkgname/repos/" else die "\tPackage '$packagename' does not exist in repo '$reponame-$_arch'" fi -- cgit v1.2.2 From 3c8878d337d59f46d37446ddde2b40af69d36330 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Fri, 13 Feb 2009 12:14:41 -0800 Subject: Allow skipping of sourceball license check Also add a whitelist of files to skip (empty right now) Signed-off-by: Aaron Griffin --- cron-jobs/sourceballs | 10 +++++++++- cron-jobs/sourceballs.force | 0 2 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 cron-jobs/sourceballs.force diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 31ac1af..b709d14 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -61,12 +61,20 @@ for repo in $repos; do srcpath="$srcbase/" srcpkg="${pkg//-$arch$PKGEXT/$SRCEXT}" + #Don't do anything for package in this 'blacklist' if grep $pkgname "$dirname/sourceballs.skip" >/dev/null 2>&1; then continue fi + #Use this file to 'whitelist' or force building some sourceballs, + # skipping the license check + force="" + if grep $pkgname "$dirname/sourceballs.force" >/dev/null 2>&1; then + force="-f" + fi + if [ ! -f "$srcpath$srcpkg" ]; then - if ! $dirname/../misc-scripts/make-sourceball \ + if ! $dirname/../misc-scripts/make-sourceball $force \ $pkgname $repo $arch 2>>"$srcbase/errors.txt"; then FAILED_PKGS="$FAILED_PKGS $pkg" fi diff --git a/cron-jobs/sourceballs.force b/cron-jobs/sourceballs.force new file mode 100644 index 0000000..e69de29 -- cgit v1.2.2 From 83093650bda19395e5611c498752c5f2647157f8 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Fri, 13 Feb 2009 12:19:21 -0800 Subject: Fix pkgname parsing for files without ARCH Split the srcpkg parsing into two separate steps, one to swap the extensions, and another to remove the architecture Signed-off-by: Aaron Griffin --- cron-jobs/sourceballs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index b709d14..07bc7aa 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -59,7 +59,8 @@ for repo in $repos; do for pkg in *$PKGEXT; do pkgname=$(getpkgname $pkg) srcpath="$srcbase/" - srcpkg="${pkg//-$arch$PKGEXT/$SRCEXT}" + srcpkg="${pkg//$PKGEXT/$SRCEXT}" + srcpkg="${srcpkg//-$arch/}" #Don't do anything for package in this 'blacklist' if grep $pkgname "$dirname/sourceballs.skip" >/dev/null 2>&1; then -- cgit v1.2.2 From 36a32f7ce64e4ca8476901b8c25299b6b8fd1f63 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Fri, 13 Feb 2009 13:57:30 -0800 Subject: Add -f flag to make-sourceball Signed-off-by: Aaron Griffin --- misc-scripts/make-sourceball | 38 +++++++++++++++++++++++++------------- 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/misc-scripts/make-sourceball b/misc-scripts/make-sourceball index b882de8..75c3db7 100755 --- a/misc-scripts/make-sourceball +++ b/misc-scripts/make-sourceball @@ -4,8 +4,9 @@ ALLOWED_LICENSES=('GPL' 'GPL1' 'GPL2' 'LGPL' 'LGPL1' 'LGPL2') if [ $# -ne 3 ]; then - echo "usage: $(basename $0) " - exit 1 + echo "usage: $(basename $0) [-f] " + echo " -f Force building. Skip license checks" + exit 1 fi BASEDIR="$(dirname $0)/../" @@ -13,6 +14,12 @@ BASEDIR="$(dirname $0)/../" source_makepkg +FORCE=0 +if [ "$1" = "-f" ]; then + FORCE=1 + shift +fi + packagename="$1" reponame="$2" _arch="$3" @@ -23,33 +30,38 @@ logpath="/var/log/sourceballs/" WORKDIR="/tmp/make-sourceball.$packagename.$UID" cleanup() { - restore_umask - rm -rf "$WORKDIR" - [ "$1" ] && exit $1 + restore_umask + rm -rf "$WORKDIR" + [ "$1" ] && exit $1 } ctrl_c() { - echo "Interrupted" >&2 - cleanup 0 + echo "Interrupted" >&2 + cleanup 0 } die() { - echo -e "$*" >&2 - cleanup 1 + echo -e "$*" >&2 + cleanup 1 } #usage: chk_license ${license[@]}" chk_license() { - local l - for l in "$@"; do - l="$(echo $l | tr '[:upper:]' '[:lower:]')" + #The -f flag skips this check + if [ $FORCE = 1 ]; then + return 0 + fi + + local l + for l in "$@"; do + l="$(echo $l | tr '[:upper:]' '[:lower:]')" for allowed in ${ALLOWED_LICENSES[@]}; do allowed="$(echo $allowed | tr '[:upper:]' '[:lower:]')" if [ "$l" = "$allowed" ]; then return 0 fi done - done + done return 1 } -- cgit v1.2.2 From 5ca5f0c705590bd5f8bd4af265ad0e753ca9394f Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Fri, 13 Feb 2009 13:58:25 -0800 Subject: Call makepkg with --ignorearch during --allsource Signed-off-by: Aaron Griffin --- misc-scripts/make-sourceball | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc-scripts/make-sourceball b/misc-scripts/make-sourceball index 75c3db7..51f28c1 100755 --- a/misc-scripts/make-sourceball +++ b/misc-scripts/make-sourceball @@ -77,7 +77,7 @@ create_srcpackage() { die -e "\tPackage license (${license[@]}) does not require source tarballs. Doing nothing" fi - if ! /usr/bin/makepkg --allsource >"$logpath/$pkgname" 2>&1; then + if ! /usr/bin/makepkg --allsource --ignorearch >"$logpath/$pkgname" 2>&1; then popd >/dev/null /bin/gzip -9 "$logpath/$pkgname" die "\tFailed to download source for $pkgname-$pkgver-$pkgrel ($reponame-$_arch)" -- cgit v1.2.2 From 918e009a6468587cc442c1ed62a5f0264bd28d32 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Fri, 13 Feb 2009 14:02:49 -0800 Subject: Keep the last failed.txt too for comparison Signed-off-by: Aaron Griffin --- cron-jobs/sourceballs | 1 + 1 file changed, 1 insertion(+) diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 07bc7aa..470dd7e 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -85,6 +85,7 @@ for repo in $repos; do done if [ -n "$FAILED_PKGS" ]; then + [ -e "$srcbase/failed.txt" ] && /bin/mv "$srcbase/failed.txt" "$srcbase/failed.txt.old" echo "The following packages failed:" > "$srcbase/failed.txt" echo -e $FAILED_PKGS | sed "s| |\n|g" | sort -u >> "$srcbase/failed.txt" fi -- cgit v1.2.2 From 17899d277a6f5219a6b0534d38c884b264d5ad89 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Sat, 14 Feb 2009 15:04:45 -0800 Subject: Fix arg number check in make-sourceball Signed-off-by: Aaron Griffin --- misc-scripts/make-sourceball | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc-scripts/make-sourceball b/misc-scripts/make-sourceball index 51f28c1..4b6bb5c 100755 --- a/misc-scripts/make-sourceball +++ b/misc-scripts/make-sourceball @@ -3,7 +3,7 @@ # Allowed licenses: build only for licenses in this array ALLOWED_LICENSES=('GPL' 'GPL1' 'GPL2' 'LGPL' 'LGPL1' 'LGPL2') -if [ $# -ne 3 ]; then +if [ $# -ne 3 -a $# -ne 4 ]; then echo "usage: $(basename $0) [-f] " echo " -f Force building. Skip license checks" exit 1 -- cgit v1.2.2 From 23578ce0f2d5ceedcdca58c58b390d090572cb4c Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Sat, 14 Feb 2009 15:06:06 -0800 Subject: make-sourceball: move popd call before return This ensures the 'cp' succeeds Signed-off-by: Aaron Griffin --- misc-scripts/make-sourceball | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/misc-scripts/make-sourceball b/misc-scripts/make-sourceball index 4b6bb5c..4308c0e 100755 --- a/misc-scripts/make-sourceball +++ b/misc-scripts/make-sourceball @@ -82,7 +82,6 @@ create_srcpackage() { /bin/gzip -9 "$logpath/$pkgname" die "\tFailed to download source for $pkgname-$pkgver-$pkgrel ($reponame-$_arch)" fi - popd >/dev/null /bin/rm "$logpath/$pkgname" local pkg_file="${pkgname}-${pkgver}-${pkgrel}${SRCEXT}" @@ -92,6 +91,8 @@ create_srcpackage() { fi cp "$pkgname/$pkg_file" "$srcpath" + popd >/dev/null + return 0 fi } -- cgit v1.2.2 From d9d3bfbb65188b8f20bee9b26e7ac8416a0e4235 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Sat, 14 Feb 2009 15:08:22 -0800 Subject: Add repo/arch to make-sourceball log path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Original-work-by: Eric Bélanger Signed-off-by: Aaron Griffin --- misc-scripts/make-sourceball | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/misc-scripts/make-sourceball b/misc-scripts/make-sourceball index 4308c0e..012049a 100755 --- a/misc-scripts/make-sourceball +++ b/misc-scripts/make-sourceball @@ -77,12 +77,13 @@ create_srcpackage() { die -e "\tPackage license (${license[@]}) does not require source tarballs. Doing nothing" fi - if ! /usr/bin/makepkg --allsource --ignorearch >"$logpath/$pkgname" 2>&1; then + local logfile="$logpath/$pkgname-$reponame-$_arch" + if ! /usr/bin/makepkg --allsource --ignorearch >"$logfile" 2>&1; then popd >/dev/null - /bin/gzip -9 "$logpath/$pkgname" + /bin/gzip -9 "$logfile" die "\tFailed to download source for $pkgname-$pkgver-$pkgrel ($reponame-$_arch)" fi - /bin/rm "$logpath/$pkgname" + /bin/rm "$logfile"{,.gz} local pkg_file="${pkgname}-${pkgver}-${pkgrel}${SRCEXT}" -- cgit v1.2.2 From ab56729c8e997a9f49218ac6bf36f07c3316c8e5 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Sun, 15 Feb 2009 21:27:31 -0800 Subject: make-sourceball: Add -f to gzip and rm Signed-off-by: Aaron Griffin --- misc-scripts/make-sourceball | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/misc-scripts/make-sourceball b/misc-scripts/make-sourceball index 012049a..3e0ea8c 100755 --- a/misc-scripts/make-sourceball +++ b/misc-scripts/make-sourceball @@ -80,10 +80,10 @@ create_srcpackage() { local logfile="$logpath/$pkgname-$reponame-$_arch" if ! /usr/bin/makepkg --allsource --ignorearch >"$logfile" 2>&1; then popd >/dev/null - /bin/gzip -9 "$logfile" + /bin/gzip -f -9 "$logfile" die "\tFailed to download source for $pkgname-$pkgver-$pkgrel ($reponame-$_arch)" fi - /bin/rm "$logfile"{,.gz} + /bin/rm -f "$logfile"{,.gz} local pkg_file="${pkgname}-${pkgver}-${pkgrel}${SRCEXT}" -- cgit v1.2.2 From fb1ddcd32626b0dc552ce4d4c6e7c0fbb0b8c887 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20B=C3=A9langer?= Date: Mon, 16 Feb 2009 14:37:58 -0800 Subject: make-sourceball: copy the proper pkg path Signed-off-by: Aaron Griffin --- misc-scripts/make-sourceball | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc-scripts/make-sourceball b/misc-scripts/make-sourceball index 3e0ea8c..cfa606a 100755 --- a/misc-scripts/make-sourceball +++ b/misc-scripts/make-sourceball @@ -90,7 +90,7 @@ create_srcpackage() { if [ ! -d "$srcpath" ]; then mkdir -p "$srcpath" fi - cp "$pkgname/$pkg_file" "$srcpath" + cp "$pkg_file" "$srcpath" popd >/dev/null -- cgit v1.2.2 From 8d32b23c487fb2cd1a27ff350547ca1a2cdb5f68 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Mon, 16 Feb 2009 19:52:36 -0800 Subject: sourceballs: copy previous errors.txt output file Signed-off-by: Aaron Griffin --- cron-jobs/sourceballs | 1 + 1 file changed, 1 insertion(+) diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 470dd7e..d36da4b 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -49,6 +49,7 @@ getpkgname() { FAILED_PKGS="" +[ -e "$srcbase/errors.txt" ] && /bin/mv "$srcbase/errors.txt" "$srcbase/errors.txt.old" echo "Errors occured during run:" > "$srcbase/errors.txt" for repo in $repos; do -- cgit v1.2.2 From 4127e3721ac8ce9358082dfe8095f057b3ec1659 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Wed, 18 Feb 2009 15:45:35 -0800 Subject: make-sourceballs: one logfile per pkgname This aids with cleanup, and we really didn't need multiple files anyway :) Signed-off-by: Aaron Griffin --- misc-scripts/make-sourceball | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc-scripts/make-sourceball b/misc-scripts/make-sourceball index cfa606a..328bafd 100755 --- a/misc-scripts/make-sourceball +++ b/misc-scripts/make-sourceball @@ -77,7 +77,7 @@ create_srcpackage() { die -e "\tPackage license (${license[@]}) does not require source tarballs. Doing nothing" fi - local logfile="$logpath/$pkgname-$reponame-$_arch" + local logfile="$logpath/$pkgname" if ! /usr/bin/makepkg --allsource --ignorearch >"$logfile" 2>&1; then popd >/dev/null /bin/gzip -f -9 "$logfile" -- cgit v1.2.2 From 299295b33a310087f631f55406795a6be9411e8b Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Fri, 20 Feb 2009 09:55:25 -0800 Subject: Remove unstable scripts Signed-off-by: Aaron Griffin --- db-unstable | 4 ---- db-unstable64 | 4 ---- 2 files changed, 8 deletions(-) delete mode 100755 db-unstable delete mode 100755 db-unstable64 diff --git a/db-unstable b/db-unstable deleted file mode 100755 index 2a2df43..0000000 --- a/db-unstable +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -$(dirname $0)/db-update "unstable" "i686" - diff --git a/db-unstable64 b/db-unstable64 deleted file mode 100755 index cd9c3a0..0000000 --- a/db-unstable64 +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -$(dirname $0)/db-update "unstable" "x86_64" - -- cgit v1.2.2 From db00ce3c343cf6eea9cb512f7bba9ea7904526b8 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Sat, 21 Feb 2009 13:43:23 -0800 Subject: Add a script to simplify the integ check call Signed-off-by: Aaron Griffin --- cron-jobs/integrity-check | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100755 cron-jobs/integrity-check diff --git a/cron-jobs/integrity-check b/cron-jobs/integrity-check new file mode 100755 index 0000000..c3b016c --- /dev/null +++ b/cron-jobs/integrity-check @@ -0,0 +1,14 @@ +#!/bin/bash + +basedir="$(dirname $0)" + +if [ $# -ne 3 ]; then + echo "usage: $(basename $0) [,,...] " >2 + exit 1 +fi + +$basedir/cron-jobs/check_archlinux/check_packages.py \ + --repos="$1" \ + --abs-tree="/srv/abs/rsync/$2" --arch="$1" |\ + $basedir/cron-jobs/devlist-mailer \ + "Integrity Check $2: $1" "$3" -- cgit v1.2.2 From d63b83d26694c531d900e29932e2f38a1aa12690 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Sat, 21 Feb 2009 13:48:21 -0800 Subject: make-sourceball: Fix license error output Remove the goofy -e and add the pkgname Signed-off-by: Aaron Griffin --- misc-scripts/make-sourceball | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc-scripts/make-sourceball b/misc-scripts/make-sourceball index 328bafd..85756de 100755 --- a/misc-scripts/make-sourceball +++ b/misc-scripts/make-sourceball @@ -74,7 +74,7 @@ create_srcpackage() { echo "Creating source tarball for $pkgname-$pkgver-$pkgrel" if ! chk_license ${license[@]}; then - die -e "\tPackage license (${license[@]}) does not require source tarballs. Doing nothing" + die "\t$pkgname license (${license[@]}) does not require source tarballs. Doing nothing" fi local logfile="$logpath/$pkgname" -- cgit v1.2.2 From 110f40248d3e365a98f82d30e2256da35689fbf7 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Sat, 21 Feb 2009 13:52:19 -0800 Subject: Remove the repo-maint crontab Our cron currently doesn't handle stand-alone crontabs too well, and we're managing all these in root's crontab anyway. Meh... deleted Signed-off-by: Aaron Griffin --- cron-jobs/repo-maint.crontab | 13 ------------- 1 file changed, 13 deletions(-) delete mode 100644 cron-jobs/repo-maint.crontab diff --git a/cron-jobs/repo-maint.crontab b/cron-jobs/repo-maint.crontab deleted file mode 100644 index babcea1..0000000 --- a/cron-jobs/repo-maint.crontab +++ /dev/null @@ -1,13 +0,0 @@ -MAILTO="root" -#################################################################### -#minute (0-59), # -#| hour (0-23), # -#| | day of the month (1-31), # -#| | | month of the year (1-12), # -#| | | | day of the week (0-6 with 0=Sunday)# -#| | | | | commands # -#################################################################### -*/5 * * * * /arch/cron-jobs/adjust-permissions -00 21 * * * /arch/cron-jobs/createFileList >/dev/null 2>&1 -00 23 * * * /arch/cron-jobs/sourceballs -00 */3 * * * /arch/cron-jobs/ftpdir-cleanup | /arch/cron-jobs/devlist-mailer -- cgit v1.2.2 From 0d3c5671c27f0b5848bb72df9af2270f72e7c988 Mon Sep 17 00:00:00 2001 From: Xavier Chantry Date: Sat, 21 Feb 2009 17:16:17 -0800 Subject: integrity-check: fix a few errors I think it will work better now :) Signed-off-by: Xavier Chantry Signed-off-by: Aaron Griffin --- cron-jobs/integrity-check | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/cron-jobs/integrity-check b/cron-jobs/integrity-check index c3b016c..0b59064 100755 --- a/cron-jobs/integrity-check +++ b/cron-jobs/integrity-check @@ -3,12 +3,11 @@ basedir="$(dirname $0)" if [ $# -ne 3 ]; then - echo "usage: $(basename $0) [,,...] " >2 + echo "usage: $(basename $0) [,,...] " >&2 exit 1 fi -$basedir/cron-jobs/check_archlinux/check_packages.py \ +$basedir/check_archlinux/check_packages.py \ --repos="$1" \ - --abs-tree="/srv/abs/rsync/$2" --arch="$1" |\ - $basedir/cron-jobs/devlist-mailer \ - "Integrity Check $2: $1" "$3" + --abs-tree="/srv/abs/rsync/$2" --arch="$2" |\ + $basedir/devlist-mailer "Integrity Check $2: $1" "$3" -- cgit v1.2.2 From 044a26cf5dede31271dbeb8f8201f1418718e62c Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Mon, 23 Feb 2009 13:45:39 -0800 Subject: Remove old sourceballs before doing anything This prevents dangling sourceballs sitting around when things start to fail or licenses are changed. Signed-off-by: Aaron Griffin --- misc-scripts/make-sourceball | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc-scripts/make-sourceball b/misc-scripts/make-sourceball index 85756de..d58d48f 100755 --- a/misc-scripts/make-sourceball +++ b/misc-scripts/make-sourceball @@ -139,8 +139,8 @@ set_umask cd "$WORKDIR" if /usr/bin/svn export -q "$SVN_PATH/$packagename" $packagename; then - create_srcpackage "$packagename/repos/$reponame-$_arch" remove_old "$pkgname/repos/" + create_srcpackage "$packagename/repos/$reponame-$_arch" else die "\tPackage '$packagename' does not exist in repo '$reponame-$_arch'" fi -- cgit v1.2.2 From 2de2859cc0fd7f9db26630dffec25fccb88f3434 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Mon, 23 Feb 2009 13:50:34 -0800 Subject: Ensure grep matches only full pkgnames Signed-off-by: Aaron Griffin --- cron-jobs/sourceballs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index d36da4b..b4085f2 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -64,14 +64,14 @@ for repo in $repos; do srcpkg="${srcpkg//-$arch/}" #Don't do anything for package in this 'blacklist' - if grep $pkgname "$dirname/sourceballs.skip" >/dev/null 2>&1; then + if grep "^$pkgname\$" "$dirname/sourceballs.skip" >/dev/null 2>&1; then continue fi #Use this file to 'whitelist' or force building some sourceballs, # skipping the license check force="" - if grep $pkgname "$dirname/sourceballs.force" >/dev/null 2>&1; then + if grep "^$pkgname\$" "$dirname/sourceballs.force" >/dev/null 2>&1; then force="-f" fi -- cgit v1.2.2 From 891e4f353d098a639c3b1d09d45987f0908b0d38 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Thu, 26 Feb 2009 11:49:26 -0800 Subject: make-sourceball: license skip is NOT a failure Signed-off-by: Aaron Griffin --- misc-scripts/make-sourceball | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/misc-scripts/make-sourceball b/misc-scripts/make-sourceball index d58d48f..e24af2d 100755 --- a/misc-scripts/make-sourceball +++ b/misc-scripts/make-sourceball @@ -74,7 +74,8 @@ create_srcpackage() { echo "Creating source tarball for $pkgname-$pkgver-$pkgrel" if ! chk_license ${license[@]}; then - die "\t$pkgname license (${license[@]}) does not require source tarballs. Doing nothing" + echo -e "\t$pkgname license (${license[@]}) does not require source tarballs" >&2 + cleanup 0 fi local logfile="$logpath/$pkgname" -- cgit v1.2.2 From 1ef058dad91a5ca82db2744776251a2ab68bcf90 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Fri, 20 Feb 2009 10:01:27 -0800 Subject: Make db-update parse all architecutres ARCHES is defined in config. db-update should now loop over all arches on each run and update whatever is in the staging dir. Remove the db-*64 scripts due to this Signed-off-by: Aaron Griffin --- config | 1 + db-core | 2 +- db-core64 | 3 - db-extra | 2 +- db-extra64 | 3 - db-testing | 2 +- db-testing64 | 3 - db-update | 206 ++++++++++++++++++++++++++++++----------------------------- 8 files changed, 110 insertions(+), 112 deletions(-) delete mode 100755 db-core64 delete mode 100755 db-extra64 delete mode 100755 db-testing64 diff --git a/config b/config index 5c39919..07cf783 100644 --- a/config +++ b/config @@ -3,5 +3,6 @@ FTP_BASE="/srv/ftp" SVN_PATH="file:///srv/svn-packages" STAGING="$HOME/staging" TMPDIR="/srv/tmp" +ARCHES=(i686 x86_64) # To be used if $BUILDSCRIPT is removed from makepkg.conf #BUILDSCRIPT="PKGBUILD" diff --git a/db-core b/db-core index b88fcae..eaa5032 100755 --- a/db-core +++ b/db-core @@ -1,3 +1,3 @@ #!/bin/bash -$(dirname $0)/db-update "core" "i686" +$(dirname $0)/db-update "core" diff --git a/db-core64 b/db-core64 deleted file mode 100755 index e080004..0000000 --- a/db-core64 +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -$(dirname $0)/db-update "core" "x86_64" diff --git a/db-extra b/db-extra index 1375b13..d14b50e 100755 --- a/db-extra +++ b/db-extra @@ -1,3 +1,3 @@ #!/bin/bash -$(dirname $0)/db-update "extra" "i686" +$(dirname $0)/db-update "extra" diff --git a/db-extra64 b/db-extra64 deleted file mode 100755 index cfa20a2..0000000 --- a/db-extra64 +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -$(dirname $0)/db-update "extra" "x86_64" diff --git a/db-testing b/db-testing index 69b5376..579ae37 100755 --- a/db-testing +++ b/db-testing @@ -1,3 +1,3 @@ #!/bin/bash -$(dirname $0)/db-update "testing" "i686" +$(dirname $0)/db-update "testing" diff --git a/db-testing64 b/db-testing64 deleted file mode 100755 index d75a456..0000000 --- a/db-testing64 +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -$(dirname $0)/db-update "testing" "x86_64" diff --git a/db-update b/db-update index 77af39f..ea57eed 100755 --- a/db-update +++ b/db-update @@ -1,7 +1,7 @@ #!/bin/bash -if [ $# -ne 2 ]; then - echo "usage: $(basename $0) " +if [ $# -ne 1 ]; then + echo "usage: $(basename $0) " exit 1 fi @@ -10,30 +10,39 @@ fi source_makepkg reponame="$1" -_arch="$2" - -export CARCH="$_arch" - -ftppath="$FTP_BASE/$reponame/os/$_arch/" -svnrepo="$reponame-$_arch" -stagedir="$STAGING/$reponame" +current_arch="" [ "$UID" = "" ] && UID=$(uid) WORKDIR="$TMPDIR/db-update.$svnrepo.$UID" ADDPKGS="" -if [ ! -d "$ftppath" ]; then - echo "FTP path for this repo ($reponame) is missing" - echo "Please contact a system administrator" - exit 1 -fi - +stagedir="$STAGING/$reponame" if [ ! -d $stagedir ]; then echo "error: staging directory missing: $stagedir" >&2 exit 1 fi +if [ -d "${stagedir}64" ]; then + echo "--------------------------------------------------" + echo "It looks like you have an old staging dir" + echo "Packages are now differentiated by the arch in the filename." + echo "Please delete '${stagedir}64'" + echo "--------------------------------------------------" + /bin/mv "${stagedir}64/add/"* "$stagedir/add/" + /bin/mv "${stagedir}64/del/"* "$stagedir/del/" +fi + +if [ -d "${stagedir}/add" ]; then + echo "--------------------------------------------------" + echo "It looks like you have an old staging dir" + echo "The 'add' and 'del' dirs are no longer used." + echo "Please delete staging//{add,del}" + echo " and ensure you are using the newest devtools" + echo "--------------------------------------------------" + /bin/mv "${stagedir}/add/"* "$stagedir/" +fi + # Get the package name from the filename # hackish, but should work for now getpkgname() { @@ -41,14 +50,14 @@ getpkgname() { tmp=${1##*/} tmp=${tmp%$PKGEXT} - tmp=${tmp%-$CARCH} + tmp=${tmp%-$current_arch} echo ${tmp%-*-*} } cleanup() { trap '' 0 2 # unlock - repo_unlock $reponame $_arch + repo_unlock $reponame $current_arch rm -rf "$WORKDIR" [ "$1" ] && exit $1 } @@ -66,114 +75,111 @@ die() { trap ctrl_c 2 trap cleanup 0 -repo_lock $reponame $_arch +for A in ${ARCHES[@]}; do + current_arch="$A" -/bin/mkdir -p "$WORKDIR/build" -cd "$WORKDIR" - -# copy the db file into our working area -if [ -f "$ftppath/$reponame.db.tar.$DB_COMPRESSION" ]; then - /bin/cp "$ftppath/$reponame.db.tar.$DB_COMPRESSION" build/ -fi + ftppath="$FTP_BASE/$reponame/os/$current_arch/" -echo "Updating DB for $reponame $_arch" - -if [ -d "${stagedir}64" ]; then - echo "--------------------------------------------------" - echo "It looks like you have an old staging dir" - echo "Packages are now differentiated by the arch in the filename." - echo "Please delete '${stagedir}64'" - echo "--------------------------------------------------" - /bin/mv "${stagedir}64/add/"* "$stagedir/add/" - /bin/mv "${stagedir}64/del/"* "$stagedir/del/" -fi + if [ ! -d "$ftppath" ]; then + echo "FTP path for this repo ($reponame) is missing" + echo " -> $ftppath" + echo "Please contact a system administrator" + exit 1 + fi -if [ -d "${stagedir}/add" ]; then - echo "--------------------------------------------------" - echo "It looks like you have an old staging dir" - echo "The 'add' and 'del' dirs are no longer used." - echo "Please delete staging//{add,del}" - echo " and ensure you are using the newest devtools" - echo "--------------------------------------------------" - /bin/mv "${stagedir}/add/"* "$stagedir/" -fi + svnrepo="$reponame-$current_arch" -to_add="" -if [ -d "$stagedir" ]; then - ADDPKGS="$(/bin/ls $stagedir/*-${_arch}$PKGEXT 2>/dev/null)" -fi + repo_lock $reponame $current_arch -if [ -n "$ADDPKGS" ]; then + /bin/mkdir -p "$WORKDIR/build" + cd "$WORKDIR" - echo "==> Copying DB file from '$reponame'..." >&2 + # copy the db file into our working area if [ -f "$ftppath/$reponame.db.tar.$DB_COMPRESSION" ]; then /bin/cp "$ftppath/$reponame.db.tar.$DB_COMPRESSION" build/ - else - touch "build/$reponame.db.tar.$DB_COMPRESSION" fi - echo "==> Processing new/updated packages for repository '$reponame'..." >&2 + echo "Updating DB for $svnrepo" - cd "$WORKDIR" - /usr/bin/svn checkout -N $SVN_PATH checkout - cd checkout + to_add="" + if [ -d "$stagedir" ]; then + ADDPKGS="$(/bin/ls $stagedir/*-${current_arch}$PKGEXT 2>/dev/null)" + fi - for pkg in $ADDPKGS; do - _pkgfile=$(basename $pkg) - _pkgname="$(getpkgname $pkg)" + if [ -n "$ADDPKGS" ]; then - echo " Validating package arch ($_arch) $_pkgname" - if ! check_pkg_arch "$pkg" "$_arch"; then - echo " ERROR: $_pkgfile was built for the wrong architecture" + echo "==> Copying DB file from '$reponame'..." >&2 + if [ -f "$ftppath/$reponame.db.tar.$DB_COMPRESSION" ]; then + /bin/cp "$ftppath/$reponame.db.tar.$DB_COMPRESSION" build/ else - echo " Checking SVN for $_pkgname" - /usr/bin/svn up -q $_pkgname - if [ -d "$_pkgname/repos/$svnrepo" ]; then - . "$_pkgname/repos/$svnrepo/$BUILDSCRIPT" - if [ "$_pkgfile" = "$pkgname-$pkgver-$pkgrel-$_arch$PKGEXT" ]; then - to_add="$to_add $pkg" + touch "build/$reponame.db.tar.$DB_COMPRESSION" + fi + + echo "==> Processing new/updated packages for repository '$reponame'..." >&2 + + cd "$WORKDIR" + /usr/bin/svn checkout -N $SVN_PATH checkout + cd checkout + + for pkg in $ADDPKGS; do + _pkgfile=$(basename $pkg) + _pkgname="$(getpkgname $pkg)" + + echo " Validating package arch ($current_arch) $_pkgname" + if ! check_pkg_arch "$pkg" "$current_arch"; then + echo " ERROR: $_pkgfile was built for the wrong architecture" + else + echo " Checking SVN for $_pkgname" + /usr/bin/svn up -q $_pkgname + if [ -d "$_pkgname/repos/$svnrepo" ]; then + . "$_pkgname/repos/$svnrepo/$BUILDSCRIPT" + if [ "$_pkgfile" = "$pkgname-$pkgver-$pkgrel-$current_arch$PKGEXT" ]; then + to_add="$to_add $pkg" + else + echo " WARNING: $_pkgfile does not match $BUILDSCRIPT in $svnrepo" + fi else - echo " WARNING: $_pkgfile does not match $BUILDSCRIPT in $svnrepo" + echo " WARNING: Package $_pkgname not found in $svnrepo" fi - else - echo " WARNING: Package $_pkgname not found in $svnrepo" fi - fi - done + done - if [ -n "$to_add" ]; then - cd "$WORKDIR/build/" - for f in $to_add; do /bin/cp "$f" .; done + if [ -n "$to_add" ]; then + cd "$WORKDIR/build/" + for f in $to_add; do /bin/cp "$f" .; done - pkgs="" - for pkg in $to_add; do pkgs="$pkgs $(basename $pkg)"; done + pkgs="" + for pkg in $to_add; do pkgs="$pkgs $(basename $pkg)"; done - /usr/bin/repo-add -q "$reponame.db.tar.$DB_COMPRESSION" $pkgs + /usr/bin/repo-add -q "$reponame.db.tar.$DB_COMPRESSION" $pkgs + else + rm -f "build/$reponame.db.tar.$DB_COMPRESSION" + echo "Errors found when adding packages" + fi else - rm -f "build/$reponame.db.tar.$DB_COMPRESSION" - echo "Errors found when adding packages" + echo "No packages to add" fi -else - echo "No packages to add" -fi -# if non empty, move all build dirs -if [ $(/bin/ls "$WORKDIR/build/" 2>/dev/null | wc -l) != 0 ]; then - echo "Copying new files to '$ftppath'" - for f in "$WORKDIR/build/"*; do - if ! /bin/cp "$f" "$ftppath"; then - die "error: failure while copying files to $ftppath" - fi - done + # if non empty, move all build dirs + if [ $(/bin/ls "$WORKDIR/build/" 2>/dev/null | wc -l) != 0 ]; then + echo "Copying new files to '$ftppath'" + for f in "$WORKDIR/build/"*; do + if ! /bin/cp "$f" "$ftppath"; then + die "error: failure while copying files to $ftppath" + fi + done -else - echo "Nothing to copy, no work done" -fi + else + echo "Nothing to copy, no work done" + fi -if [ -n "$to_add" ]; then - echo "Cleaning staging dir" - /bin/rm $to_add -fi + if [ -n "$to_add" ]; then + echo "Cleaning staging dir" + /bin/rm $to_add + fi + + repo_unlock $reponame $current_arch +done cleanup # vim: set ts=4 sw=4 noet ft=sh: -- cgit v1.2.2 From 9b7b2a7e774f307b6b5ab398908e883364af7247 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Fri, 20 Feb 2009 10:23:07 -0800 Subject: Add per-repo SVN configs This could be useful if we move community to a separate SVN repo Signed-off-by: Aaron Griffin --- config | 7 ++++++- db-functions | 15 +++++++++++++++ db-move | 7 ++++++- db-remove | 3 ++- db-update | 3 ++- testing2x | 3 ++- 6 files changed, 33 insertions(+), 5 deletions(-) diff --git a/config b/config index 07cf783..867d28e 100644 --- a/config +++ b/config @@ -1,6 +1,11 @@ FTP_BASE="/srv/ftp" -SVN_PATH="file:///srv/svn-packages" + +SVNREPO_core="file:///srv/svn-packages" +SVNREPO_extra="file:///srv/svn-packages" +SVNREPO_testing="file:///srv/svn-packages" +SVNREPO_community="file:///srv/svn-community" + STAGING="$HOME/staging" TMPDIR="/srv/tmp" ARCHES=(i686 x86_64) diff --git a/db-functions b/db-functions index c105dab..99de756 100644 --- a/db-functions +++ b/db-functions @@ -92,4 +92,19 @@ check_pkg_arch () { #check_pkg_arch pkgfile arch fi } +get_svnpath () { #get_svnpath reponame + local var + local repopath + var="\$SVNREPO_${1}" + + eval repopath=${var} + if [ -z "$repopath" ]; then + echo "ERROR: SVN path not defined for '${1}'" >2 + echo " Please check SVNREPO_${1} config setting" >2 + fi + + echo "$repopath" +} + + # vim: set ts=4 sw=4 noet ft=sh: diff --git a/db-move b/db-move index ff9677b..e4f2fef 100755 --- a/db-move +++ b/db-move @@ -21,6 +21,11 @@ ftppath_from="$FTP_BASE/$repofrom/os/$_arch/" ftppath_to="$FTP_BASE/$repoto/os/$_arch/" svnrepo_from="$repofrom-$_arch" svnrepo_to="$repoto-$_arch" +svnpath="$(get_svnpath $repoto)" +if [ "$svnpath" != "$(get_svnpath $repofrom" ]; then + echo "ERROR: Cannot move packages across SVN repos" + echo " A move must be within the same svn repo" +fi [ "$UID" = "" ] && UID=$(uid) @@ -54,7 +59,7 @@ repo_lock $repofrom $_arch /bin/mkdir -p "$WORKDIR" cd "$WORKDIR" -/usr/bin/svn checkout -N $SVN_PATH checkout +/usr/bin/svn checkout -N $svnpath checkout cd checkout /usr/bin/svn up -q $packagename diff --git a/db-remove b/db-remove index acc849d..501c70a 100755 --- a/db-remove +++ b/db-remove @@ -17,6 +17,7 @@ export CARCH="$arch" ftppath="$FTP_BASE/$reponame/os/$arch/" svnrepo="$reponame-$arch" +svnpath="$(get_svnpath $reponame)" [ "$UID" = "" ] && UID=$(uid) @@ -51,7 +52,7 @@ repo_lock $reponame $arch echo "==> Removing package '$packagename' from '$reponame'..." >&2 cd "$WORKDIR" -/usr/bin/svn checkout -N $SVN_PATH checkout +/usr/bin/svn checkout -N $svnpath checkout cd checkout /usr/bin/svn up -q $packagename diff --git a/db-update b/db-update index ea57eed..3f3b0cc 100755 --- a/db-update +++ b/db-update @@ -87,6 +87,7 @@ for A in ${ARCHES[@]}; do exit 1 fi + svnpath="$(get_svnpath $reponame)" svnrepo="$reponame-$current_arch" repo_lock $reponame $current_arch @@ -118,7 +119,7 @@ for A in ${ARCHES[@]}; do echo "==> Processing new/updated packages for repository '$reponame'..." >&2 cd "$WORKDIR" - /usr/bin/svn checkout -N $SVN_PATH checkout + /usr/bin/svn checkout -N $svnpath checkout cd checkout for pkg in $ADDPKGS; do diff --git a/testing2x b/testing2x index 6a3a70e..0314738 100755 --- a/testing2x +++ b/testing2x @@ -13,6 +13,7 @@ case "$0" in ;; esac WORKDIR="$TMPDIR/testing2x.$UID" +svnpath="$(get_svnpath testing)" cleanup() { trap '' 0 2 @@ -30,7 +31,7 @@ trap cleanup 0 /bin/mkdir "${WORKDIR}" cd "${WORKDIR}" -/usr/bin/svn checkout -N $SVN_PATH checkout +/usr/bin/svn checkout -N $svnpath checkout cd checkout for pkg in $*; do -- cgit v1.2.2 From 2c0ac7367b317e641ff0a0f228087ba3befc9eff Mon Sep 17 00:00:00 2001 From: Abhishek Dasgupta Date: Wed, 11 Mar 2009 09:27:06 -0700 Subject: Add convert-to-any script Converts an arch-specific package to the "any" arch [Aaron: converted cd to pushd/popd] Signed-off-by: Aaron Griffin --- convert-to-any | 108 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 convert-to-any diff --git a/convert-to-any b/convert-to-any new file mode 100644 index 0000000..b8d298c --- /dev/null +++ b/convert-to-any @@ -0,0 +1,108 @@ +#!/bin/bash +# Converts an existing package into an +# architecture-independent package and updates +# the repositories accordingly. + +# -- Abhishek Dasgupta + +. "$(dirname $0)/db-functions" +[ "$UID" = "" ] && UID=$(uid) + +if [ $# -ne 1 ]; then + echo "Syntax: $(basename $0) " + exit 1 +fi + +repo=$(echo $1 | sed "s#\(.*\)/.*#\1#g") +pkg=$(echo $1 | sed "s#.*/\(.*\)#\1#g") + +if [ -f "$(dirname $0)/config" ]; then + . "$(dirname $0)/config" +else + TMPDIR=/srv/tmp + FTP_BASE=/srv/ftp +fi + +if [ -f /etc/makepkg.conf ]; then + . /etc/makepkg.conf +else + echo "W: /etc/makepkg.conf does not exist." + DB_COMPRESSION=gz + PKGEXT=".pkg.tar.gz" +fi + +repo_lock $repo any +WORKDIR="$TMPDIR/convert-to-any.$pkg.$UID" +ftppath="$FTP_BASE/$repo/os" + +cleanup() { + trap '' 0 2 + # unlock + repo_unlock $repo any + rm -rf "$WORKDIR" + [ "$1" ] && exit $1 +} + +ctrl_c() { + echo "Interrupted" >&2 + cleanup 0 +} + +die() { + echo "$*" >&2 + cleanup 1 +} + +# Enter the temporary build directory +# and convert the i686 package into an +# architecture-independent package. +mkdir -p "$WORKDIR/build" +pushd "$WORKDIR/build" >/dev/null + +oldpkgname=$ftppath/i686/$pkg* +if [ -f "$oldpkgname" ]; then + cp "$oldpkgname" . +else + die "E: Package $oldpkgname not found in $ftppath/i686" +fi + +for architecture in i686 x86_64; do + if [ -f "$ftppath/$architecture/$repo.db.tar.$DB_COMPRESSION" ]; then + cp $ftppath/$architecture/$repo.db.tar.$DB_COMPRESSION \ + $repo-$architecture.db.tar.$DB_COMPRESSION + else + touch $repo-$architecture.db.tar.$DB_COMPRESSION + fi +done + +if [ ! -d "$ftppath/any" ]; then mkdir -p "$ftppath/any"; fi + +# Conversion of i686 package into "any" package. +echo -n "Extracting $pkg..." +mkdir -p package +tar zxf $pkg*i686$PKGEXT -C package +echo " done." + +sed -i "s/arch = i686/arch = any/g" package/.PKGINFO +newpkgname=$(ls $pkg*i686$PKGEXT | sed "s/i686/any/g") +pushd package >/dev/null +tar czf "$newpkgname" * .PKGINFO +popd >/dev/null +mv "package/$newpkgname" . +echo "Created $newpkgname." + +# New package is ready, move it into place and update db. +mv "$newpkgname" "$ftppath/any/" +for architecture in i686 x86_64; do + # rm -f $ftppath/$architecture/$pkg*$PKGEXT + ln -s "$ftppath/any/$newpkgname" "$ftppath/$architecture/$newpkgname" + repo-remove -q $repo-$architecture.db.tar.$DB_COMPRESSION $pkg + repo-add -q $repo-$architecture.db.tar.$DB_COMPRESSION $newpkgname + mv $repo-$architecture.db.tar.$DB_COMPRESSION "$ftppath/os/$architecture" + echo "Updated $repo-$architecture for $pkg." +done + +echo -n "Cleaning up..." +popd >/dev/null +cleanup +echo " done." -- cgit v1.2.2 From d871e78664f303f9d138d7f46f6ad0c349336752 Mon Sep 17 00:00:00 2001 From: Abhishek Dasgupta Date: Wed, 11 Mar 2009 09:33:43 -0700 Subject: db-update: Add support for 'any' arch Signed-off-by: Aaron Griffin --- db-update | 40 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 38 insertions(+), 2 deletions(-) diff --git a/db-update b/db-update index 3f3b0cc..4797586 100755 --- a/db-update +++ b/db-update @@ -16,6 +16,7 @@ current_arch="" WORKDIR="$TMPDIR/db-update.$svnrepo.$UID" ADDPKGS="" +ANYPKGS="" stagedir="$STAGING/$reponame" if [ ! -d $stagedir ]; then @@ -79,6 +80,7 @@ for A in ${ARCHES[@]}; do current_arch="$A" ftppath="$FTP_BASE/$reponame/os/$current_arch/" + ftppath_any="$FTP_BASE/$reponame/os/any/" if [ ! -d "$ftppath" ]; then echo "FTP path for this repo ($reponame) is missing" @@ -105,6 +107,7 @@ for A in ${ARCHES[@]}; do to_add="" if [ -d "$stagedir" ]; then ADDPKGS="$(/bin/ls $stagedir/*-${current_arch}$PKGEXT 2>/dev/null)" + ANYPKGS="$(/bin/ls $stagedir/*-any$PKGEXT 2>/dev/null)" fi if [ -n "$ADDPKGS" ]; then @@ -145,6 +148,29 @@ for A in ${ARCHES[@]}; do fi done + for pkg in $ANYPKGS; do + _pkgfile=$(basename $pkg) + _pkgname="$(getpkgname $pkg)" + svnrepo="$reponame-any" + echo " Validating package arch (any) $_pkgname" + if ! check_pkg_arch "$pkg" "any"; then + echo " ERROR: $_pkgfile is not architecture independent!" + else + echo " Checking SVN for $_pkgname" + /usr/bin/svn up -q $_pkgname + if [ -d "$_pkgname/repos/$svnrepo" ]; then + . "$_pkgname/repos/$svnrepo/$BUILDSCRIPT" + if [ "$_pkgfile" = "$pkgname-$pkgver-$pkgrel-any$PKGEXT" ]; then + to_add="$to_add $pkg" + else + echo " WARNING: $_pkgfile does not match $BUILDSCRIPT in $svnrepo" + fi + else + echo " WARNING: Package $_pkgname not found in $svnrepo" + fi + fi + done + if [ -n "$to_add" ]; then cd "$WORKDIR/build/" for f in $to_add; do /bin/cp "$f" .; done @@ -164,12 +190,22 @@ for A in ${ARCHES[@]}; do # if non empty, move all build dirs if [ $(/bin/ls "$WORKDIR/build/" 2>/dev/null | wc -l) != 0 ]; then echo "Copying new files to '$ftppath'" - for f in "$WORKDIR/build/"*; do + for f in "$WORKDIR/build/"*$current_arch$PKGEXT; do if ! /bin/cp "$f" "$ftppath"; then die "error: failure while copying files to $ftppath" fi done - + for f in "$WORKDIR/build/"*any.pkg.tar.gz; do + if ! /bin/cp "$f" "$ftppath_any"; then + die "error: failure while copying files to $ftppath_any" + fi + if ! ln -s "$ftppath_any/$f" "$ftppath/$f"; then + die "error: failed to make link for $f." + fi + done + if ! /bin/cp "$WORKDIR/build/$reponame.db"* "$ftppath"; then + die "failed to move repository $reponame-$A". + fi else echo "Nothing to copy, no work done" fi -- cgit v1.2.2 From 60f7e2bf28bed7d18438d9378e19a82e8b6a2d24 Mon Sep 17 00:00:00 2001 From: Abhishek Dasgupta Date: Wed, 11 Mar 2009 23:55:38 +0530 Subject: Moved arch-independent processing out of the loop. Also fixes the bug in the previous commit, in which the arch-independent packages would be deleted from the staging directory after only one iteration. --- db-update | 109 +++++++++++++++++++++++++++++++++++--------------------------- 1 file changed, 62 insertions(+), 47 deletions(-) diff --git a/db-update b/db-update index 4797586..3e9fbcf 100755 --- a/db-update +++ b/db-update @@ -76,6 +76,42 @@ die() { trap ctrl_c 2 trap cleanup 0 +# Process architecture-independent packages first. +if [ -d "$stagedir" ]; then + ANYPKGS="$(/bin/ls $stagedir/*-any$PKGEXT 2>/dev/null)" +fi + +echo "==> Processing new/updated arch-independent packages for '$reponame'..." >&2 +cd "$WORKDIR" +svnpath="$(get_svnpath $reponame)" +/usr/bin/svn checkout -N $svnpath checkout +cd checkout +to_add_any="" +if [ -n "$ANYPKGS" ]; then + for pkg in $ANYPKGS; do + _pkgfile=$(basename $pkg) + _pkgname="$(getpkgname $pkg)" + svnrepo="$reponame-any" + echo " Validating package arch (any) $_pkgname" + if ! check_pkg_arch "$pkg" "any"; then + echo " ERROR: $_pkgfile is not architecture independent!" + else + echo " Checking SVN for $_pkgname" + /usr/bin/svn up -q $_pkgname + if [ -d "$_pkgname/repos/$svnrepo" ]; then + . "$_pkgname/repos/$svnrepo/$BUILDSCRIPT" + if [ "$_pkgfile" = "$pkgname-$pkgver-$pkgrel-any$PKGEXT" ]; then + to_add_any="$to_add_any $pkg" + else + echo " WARNING: $_pkgfile does not match $BUILDSCRIPT in $svnrepo" + fi + else + echo " WARNING: Package $_pkgname not found in $svnrepo" + fi + fi + done +fi + for A in ${ARCHES[@]}; do current_arch="$A" @@ -107,10 +143,9 @@ for A in ${ARCHES[@]}; do to_add="" if [ -d "$stagedir" ]; then ADDPKGS="$(/bin/ls $stagedir/*-${current_arch}$PKGEXT 2>/dev/null)" - ANYPKGS="$(/bin/ls $stagedir/*-any$PKGEXT 2>/dev/null)" fi - if [ -n "$ADDPKGS" ]; then + if [ -n "$ADDPKGS" || -n "$ANYPKGS" ]; then echo "==> Copying DB file from '$reponame'..." >&2 if [ -f "$ftppath/$reponame.db.tar.$DB_COMPRESSION" ]; then @@ -124,59 +159,38 @@ for A in ${ARCHES[@]}; do cd "$WORKDIR" /usr/bin/svn checkout -N $svnpath checkout cd checkout - - for pkg in $ADDPKGS; do - _pkgfile=$(basename $pkg) - _pkgname="$(getpkgname $pkg)" - - echo " Validating package arch ($current_arch) $_pkgname" - if ! check_pkg_arch "$pkg" "$current_arch"; then - echo " ERROR: $_pkgfile was built for the wrong architecture" - else - echo " Checking SVN for $_pkgname" - /usr/bin/svn up -q $_pkgname - if [ -d "$_pkgname/repos/$svnrepo" ]; then - . "$_pkgname/repos/$svnrepo/$BUILDSCRIPT" - if [ "$_pkgfile" = "$pkgname-$pkgver-$pkgrel-$current_arch$PKGEXT" ]; then - to_add="$to_add $pkg" - else - echo " WARNING: $_pkgfile does not match $BUILDSCRIPT in $svnrepo" - fi + + if [ -n "$ADDPKGS" ]; then + for pkg in $ADDPKGS; do + _pkgfile=$(basename $pkg) + _pkgname="$(getpkgname $pkg)" + + echo " Validating package arch ($current_arch) $_pkgname" + if ! check_pkg_arch "$pkg" "$current_arch"; then + echo " ERROR: $_pkgfile was built for the wrong architecture" else - echo " WARNING: Package $_pkgname not found in $svnrepo" - fi - fi - done - - for pkg in $ANYPKGS; do - _pkgfile=$(basename $pkg) - _pkgname="$(getpkgname $pkg)" - svnrepo="$reponame-any" - echo " Validating package arch (any) $_pkgname" - if ! check_pkg_arch "$pkg" "any"; then - echo " ERROR: $_pkgfile is not architecture independent!" - else - echo " Checking SVN for $_pkgname" - /usr/bin/svn up -q $_pkgname - if [ -d "$_pkgname/repos/$svnrepo" ]; then - . "$_pkgname/repos/$svnrepo/$BUILDSCRIPT" - if [ "$_pkgfile" = "$pkgname-$pkgver-$pkgrel-any$PKGEXT" ]; then - to_add="$to_add $pkg" + echo " Checking SVN for $_pkgname" + /usr/bin/svn up -q $_pkgname + if [ -d "$_pkgname/repos/$svnrepo" ]; then + . "$_pkgname/repos/$svnrepo/$BUILDSCRIPT" + if [ "$_pkgfile" = "$pkgname-$pkgver-$pkgrel-$current_arch$PKGEXT" ]; then + to_add="$to_add $pkg" + else + echo " WARNING: $_pkgfile does not match $BUILDSCRIPT in $svnrepo" + fi else - echo " WARNING: $_pkgfile does not match $BUILDSCRIPT in $svnrepo" + echo " WARNING: Package $_pkgname not found in $svnrepo" fi - else - echo " WARNING: Package $_pkgname not found in $svnrepo" fi - fi - done + done + fi - if [ -n "$to_add" ]; then + if [ -n "$to_add" || -n "$to_add_any" ]; then cd "$WORKDIR/build/" - for f in $to_add; do /bin/cp "$f" .; done + for f in $to_add $to_add_any; do /bin/cp "$f" .; done pkgs="" - for pkg in $to_add; do pkgs="$pkgs $(basename $pkg)"; done + for pkg in $to_add $to_add_any; do pkgs="$pkgs $(basename $pkg)"; done /usr/bin/repo-add -q "$reponame.db.tar.$DB_COMPRESSION" $pkgs else @@ -218,5 +232,6 @@ for A in ${ARCHES[@]}; do repo_unlock $reponame $current_arch done +/bin/rm $to_add_any cleanup # vim: set ts=4 sw=4 noet ft=sh: -- cgit v1.2.2 From 67e112955482397bae1971fa60e1634a92ec361c Mon Sep 17 00:00:00 2001 From: Abhishek Dasgupta Date: Fri, 20 Mar 2009 20:56:15 +0530 Subject: 'any' architecture support for db-move, db-remove Also: changed empty variable '$ftpdir' in db-remove to '$ftppath/$architecture' --- db-move | 54 ++++++++++++++++++++++++++++++++++-------------------- db-remove | 36 ++++++++++++++++++++++-------------- 2 files changed, 56 insertions(+), 34 deletions(-) diff --git a/db-move b/db-move index e4f2fef..0759ba2 100755 --- a/db-move +++ b/db-move @@ -17,8 +17,8 @@ _arch="$4" export CARCH="$_arch" -ftppath_from="$FTP_BASE/$repofrom/os/$_arch/" -ftppath_to="$FTP_BASE/$repoto/os/$_arch/" +ftppath_from="$FTP_BASE/$repofrom/os/" +ftppath_to="$FTP_BASE/$repoto/os/" svnrepo_from="$repofrom-$_arch" svnrepo_to="$repoto-$_arch" svnpath="$(get_svnpath $repoto)" @@ -67,7 +67,7 @@ if [ -d "$packagename/repos/$svnrepo_from" ]; then . "$packagename/repos/$svnrepo_from/$BUILDSCRIPT" _pkgfile="$pkgname-$pkgver-$pkgrel-$_arch$PKGEXT" - if [ ! -f "$ftppath_from/$_pkgfile" ]; then + if [ ! -f "$ftppath_from/${_arch}/$_pkgfile" ]; then die "error: package file '$_pkgfile' not found in repo '$repofrom'" fi @@ -86,23 +86,37 @@ if [ -d "$packagename/repos/$svnrepo_from" ]; then [ -d build/ ] || mkdir build cd build/ - # copy the db file into our working area - if [ -f "$ftppath_from/$repofrom.db.tar.$DB_COMPRESSION" ]; then - /bin/cp "$ftppath_from/$repofrom.db.tar.$DB_COMPRESSION" . - /usr/bin/repo-remove -q "$repofrom.db.tar.$DB_COMPRESSION" $packagename || die "Error in repo-remove" - #use '*' to move the old DB too - mv $repofrom.db.tar.$DB_COMPRESSION* $ftppath_from - echo " Package files will be cleaned up automatically" - fi - - if [ -f "$ftppath_to/$repoto.db.tar.$DB_COMPRESSION" ]; then - /bin/cp "$ftppath_to/$repoto.db.tar.$DB_COMPRESSION" . - fi - - /bin/cp "$ftppath_from/$_pkgfile" . - /usr/bin/repo-add -q "$repoto.db.tar.$DB_COMPRESSION" $_pkgfile || die "Error in repo-add" - #use '*' to move the old DB too - mv $repoto.db.tar.$DB_COMPRESSION* $_pkgfile $ftppath_to + if [ "${_arch}" == "any" ]; then + arches="i686 x86_64" + else + arches="${_arch}" + fi + + for architecture in $arches; do + # copy the db file into our working area + if [ -f "$ftppath_from/$architecture/$repofrom.db.tar.$DB_COMPRESSION" ]; then + /bin/cp "$ftppath_from/$architecture/$repofrom.db.tar.$DB_COMPRESSION" . + /usr/bin/repo-remove -q "$repofrom.db.tar.$DB_COMPRESSION" $packagename || die "Error in repo-remove" + #use '*' to move the old DB too + mv $repofrom.db.tar.$DB_COMPRESSION* "$ftppath_from/$architecture" + echo " Package files will be cleaned up automatically" + fi + + if [ -f "$ftppath_to/$architecture/$repoto.db.tar.$DB_COMPRESSION" ]; then + /bin/cp "$ftppath_to/$architecture/$repoto.db.tar.$DB_COMPRESSION" . + fi + + /bin/cp "$ftppath_from/$architecture/$_pkgfile" . + /usr/bin/repo-add -q "$repoto.db.tar.$DB_COMPRESSION" $_pkgfile || die "Error in repo-add" + #use '*' to move the old DB too + mv $repoto.db.tar.$DB_COMPRESSION* $ftppath_to/$architecture + if [ "${_arch}" == "any" ]; then + mv ${_pkgfile} $ftppath_to/any + ln -s $ftppath_to/any/${_pkgfile} $ftppath_to/$architecture/ + else + mv ${_pkgfile} $ftppath_to/$architecture + fi + done else die "Error: $packagename is not in repo $repofrom" fi diff --git a/db-remove b/db-remove index 501c70a..4763f3d 100755 --- a/db-remove +++ b/db-remove @@ -15,7 +15,7 @@ arch="$3" export CARCH="$arch" -ftppath="$FTP_BASE/$reponame/os/$arch/" +ftppath="$FTP_BASE/$reponame/os" svnrepo="$reponame-$arch" svnpath="$(get_svnpath $reponame)" @@ -65,21 +65,29 @@ fi cd "$WORKDIR" [ -d build/ ] || mkdir build -# copy the db file into our working area -if [ -f "$ftppath/$reponame.db.tar.$DB_COMPRESSION" ]; then - /bin/cp "$ftppath/$reponame.db.tar.$DB_COMPRESSION" build/ +if [ "$arch" == "any" ]; then + arches="i686 x86_64" else - echo "No database found at '$ftpdir', nothing more to do" - exit 0 + arches="$arch" fi -echo " Removing from $reponame DB file" -cd build/ -/usr/bin/repo-remove -q "$reponame.db.tar.$DB_COMPRESSION" $packagename - -/bin/cp "$reponame.db.tar.$DB_COMPRESSION" "$ftppath/" - -echo "Package files will be cleaned up automatically" - +# copy the db file into our working area +for architecture in $arches; do + if [ -f "$ftppath/$architecture/$reponame.db.tar.$DB_COMPRESSION" ]; then + /bin/cp "$ftppath/$architecture/$reponame.db.tar.$DB_COMPRESSION" build/ + else + echo "No database found at '$ftppath/$architecture', nothing more to do" + exit 0 + fi + + echo " Removing from $reponame DB file ($architecture)" + cd build/ + /usr/bin/repo-remove -q "$reponame.db.tar.$DB_COMPRESSION" $packagename + + /bin/mv "$reponame.db.tar.$DB_COMPRESSION" "$ftppath/$architecture" + + echo "Package files will be cleaned up automatically" + cd .. +done cleanup # vim: set ts=4 sw=4 noet ft=sh: -- cgit v1.2.2 From 0959e8914c08042b1a8d1227c33f4ec77ea827f6 Mon Sep 17 00:00:00 2001 From: Abhishek Dasgupta Date: Fri, 20 Mar 2009 23:52:14 +0530 Subject: ftpdir-cleanup: remove arch-independent packages Removes architecture independent packages which are not symlinked in any of the architecture repositories. --- misc-scripts/ftpdir-cleanup | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/misc-scripts/ftpdir-cleanup b/misc-scripts/ftpdir-cleanup index 2b91035..ed90437 100755 --- a/misc-scripts/ftpdir-cleanup +++ b/misc-scripts/ftpdir-cleanup @@ -10,6 +10,7 @@ arch=$2 ##### Arch specific stuff. TODO make this configurable ##### ftppath="/srv/ftp/$reponame/os/$arch/" +ftppath_base="/srv/ftp/$reponame/os" ############################################################ if [ ! -d "$ftppath" ]; then @@ -78,11 +79,18 @@ for pkg in *$PKGEXT; do EXTRAFILES="$EXTRAFILES $pkg" done +cd "$ftppath_base/any" +for pkg in *$PKGEXT; do + if [ ! -h $ftppath_base/i686/$pkg && ! -h $ftppath_base/x86_64/$pkg ]; then + ARCHINDEPFILES="$ARCHINDEPFILES $pkg" + fi +done + cd "$ftppath" rm -rf ${TMPDIR} #Make sure we've done *something* before outputting anything -if [ -z "$DELETEFILES$MISSINGFILES$EXTRAFILES" ]; then +if [ -z "$DELETEFILES$MISSINGFILES$EXTRAFILES$ARCHINDEPFILES" ]; then exit 0 fi @@ -113,6 +121,15 @@ if [ -n "$EXTRAFILES" ]; then done fi +if [ -n "$ARCHINDEPFILES" ]; then + echo " The following architecture independent packages" + echo " are not symlinked in the architecture repositories." + echo " They will be moved to /srv/package/cleanup" + for f in $ARCHINDEPFILES; do + echo " $f" + done +fi + if [ -n "${DELETEFILES}" ]; then mv ${DELETEFILES} /srv/package-cleanup/ echo "" @@ -122,3 +139,10 @@ if [ -n "${EXTRAFILES}" ]; then mv ${EXTRAFILES} /srv/package-cleanup/ echo "" fi + +cd "$ftppath_base/any" +if [ -n "${ARCHINDEPFILES}" ]; then + mv ${ARCHINDEPFILES} /srv/package-cleanup/ + echo "" +fi + -- cgit v1.2.2 From 3ec1fcda1cc209360116b6548e29e86c22500ec0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Charette?= Date: Wed, 6 May 2009 08:56:45 -0700 Subject: db-update: Use proper variable for WORKDIR Signed-off-by: Aaron Griffin --- db-update | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db-update b/db-update index 3e9fbcf..fcd7388 100755 --- a/db-update +++ b/db-update @@ -14,7 +14,7 @@ current_arch="" [ "$UID" = "" ] && UID=$(uid) -WORKDIR="$TMPDIR/db-update.$svnrepo.$UID" +WORKDIR="$TMPDIR/db-update.$reponame.$UID" ADDPKGS="" ANYPKGS="" -- cgit v1.2.2 From 1c4c32fe1fce12b7c7ea109529dc12dd39d7798a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Charette?= Date: Wed, 6 May 2009 08:57:06 -0700 Subject: db-update: Add if check before 'any' removal Signed-off-by: Aaron Griffin --- db-update | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/db-update b/db-update index fcd7388..426a236 100755 --- a/db-update +++ b/db-update @@ -231,7 +231,8 @@ for A in ${ARCHES[@]}; do repo_unlock $reponame $current_arch done - -/bin/rm $to_add_any +if [ -n "$to_add_any" ]; then + /bin/rm $to_add_any +fi cleanup # vim: set ts=4 sw=4 noet ft=sh: -- cgit v1.2.2 From 05ee55ef20d9e9085f38d95eb303a5b2613a9563 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Wed, 6 May 2009 12:28:43 -0700 Subject: config: Add 'any' ARCH to array Signed-off-by: Aaron Griffin --- config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config b/config index 867d28e..33b7738 100644 --- a/config +++ b/config @@ -8,6 +8,6 @@ SVNREPO_community="file:///srv/svn-community" STAGING="$HOME/staging" TMPDIR="/srv/tmp" -ARCHES=(i686 x86_64) +ARCHES=(i686 x86_64 any) # To be used if $BUILDSCRIPT is removed from makepkg.conf #BUILDSCRIPT="PKGBUILD" -- cgit v1.2.2 From 787ec348cf0143b658e0252766e9e415950534a2 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Wed, 6 May 2009 12:29:04 -0700 Subject: db-update: remove 'touch' of empty db file repo-add handles this on its own Signed-off-by: Aaron Griffin --- db-update | 2 -- 1 file changed, 2 deletions(-) diff --git a/db-update b/db-update index 426a236..ae99cb0 100755 --- a/db-update +++ b/db-update @@ -150,8 +150,6 @@ for A in ${ARCHES[@]}; do echo "==> Copying DB file from '$reponame'..." >&2 if [ -f "$ftppath/$reponame.db.tar.$DB_COMPRESSION" ]; then /bin/cp "$ftppath/$reponame.db.tar.$DB_COMPRESSION" build/ - else - touch "build/$reponame.db.tar.$DB_COMPRESSION" fi echo "==> Processing new/updated packages for repository '$reponame'..." >&2 -- cgit v1.2.2 From a86adff4ee619d90b5c7d49797cc77bd550e7dd6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Charette?= Date: Wed, 6 May 2009 12:30:02 -0700 Subject: db-update: Replace || with -o in if statements Signed-off-by: Aaron Griffin --- db-update | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/db-update b/db-update index ae99cb0..507d40a 100755 --- a/db-update +++ b/db-update @@ -145,7 +145,7 @@ for A in ${ARCHES[@]}; do ADDPKGS="$(/bin/ls $stagedir/*-${current_arch}$PKGEXT 2>/dev/null)" fi - if [ -n "$ADDPKGS" || -n "$ANYPKGS" ]; then + if [ -n "$ADDPKGS" -o -n "$ANYPKGS" ]; then echo "==> Copying DB file from '$reponame'..." >&2 if [ -f "$ftppath/$reponame.db.tar.$DB_COMPRESSION" ]; then @@ -183,7 +183,7 @@ for A in ${ARCHES[@]}; do done fi - if [ -n "$to_add" || -n "$to_add_any" ]; then + if [ -n "$to_add" -o -n "$to_add_any" ]; then cd "$WORKDIR/build/" for f in $to_add $to_add_any; do /bin/cp "$f" .; done -- cgit v1.2.2 From dbca654de502b1a6fb925544256dcfc3b54cc623 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Charette?= Date: Wed, 6 May 2009 12:31:26 -0700 Subject: adjust_permissions: Add 'any' architecture Signed-off-by: Aaron Griffin --- cron-jobs/adjust-permissions | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cron-jobs/adjust-permissions b/cron-jobs/adjust-permissions index 307c319..947c021 100755 --- a/cron-jobs/adjust-permissions +++ b/cron-jobs/adjust-permissions @@ -10,13 +10,16 @@ fi /usr/bin/renice +10 -p $$ > /dev/null cd /srv/ftp +/bin/chown -R ftp:ftp-arch core/os/any /bin/chown -R ftp:ftp-arch core/os/i686 /bin/chown -R ftp:ftp-arch core/os/x86_64 +/bin/chown -R ftp:ftp-extra {extra,testing}/os/any /bin/chown -R ftp:ftp-extra {extra,testing}/os/i686 /bin/chown -R ftp:ftp-extra {extra,testing}/os/x86_64 /bin/chown -R ftp:aur community for d in core extra testing community; do + /bin/chmod -R g+w $d/os/any /bin/chmod -R g+w $d/os/i686 /bin/chmod -R g+w $d/os/x86_64 done -- cgit v1.2.2 From 8e9a5b387a76f31796837e1000188d1152bd4b00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Charette?= Date: Wed, 6 May 2009 12:32:43 -0700 Subject: check_packages.py: Add 'any' arch support Signed-off-by: Aaron Griffin --- cron-jobs/check_archlinux/check_packages.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cron-jobs/check_archlinux/check_packages.py b/cron-jobs/check_archlinux/check_packages.py index 0392368..9bdd9c1 100755 --- a/cron-jobs/check_archlinux/check_packages.py +++ b/cron-jobs/check_archlinux/check_packages.py @@ -194,7 +194,7 @@ def get_repo_hierarchy(repo): return ['core','extra','community'] def verify_archs(name,archs): - valid_archs = ['i686', 'x86_64'] + valid_archs = ['any', 'i686', 'x86_64'] invalid_archs = [] for arch in archs: if arch not in valid_archs: @@ -292,7 +292,7 @@ def print_usage(): print "Options:" print " --abs-tree= Check the specified tree (default : /var/abs)" print " --repos= Check the specified repos (default : core,extra)" - print " --arch= Check the specified arch (default : i686)" + print " --arch= Check the specified arch (default : i686)" print " -h, --help Show this help and exit" print "" print "Examples:" -- cgit v1.2.2 From 09e204f873e3f5e4c4bfe966fa1188bdefac3bb6 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Wed, 6 May 2009 12:47:09 -0700 Subject: db-update: Add some total counts to output MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Original-work-by: François Charette Signed-off-by: Aaron Griffin --- db-update | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/db-update b/db-update index 507d40a..cb00f39 100755 --- a/db-update +++ b/db-update @@ -81,7 +81,8 @@ if [ -d "$stagedir" ]; then ANYPKGS="$(/bin/ls $stagedir/*-any$PKGEXT 2>/dev/null)" fi -echo "==> Processing new/updated arch-independent packages for '$reponame'..." >&2 +pkgtotal=$(echo "$ANYPKGS" | wc -w) +echo "==> Processing $pkgtotal new/updated arch-independent packages for '$reponame'..." >&2 cd "$WORKDIR" svnpath="$(get_svnpath $reponame)" /usr/bin/svn checkout -N $svnpath checkout @@ -152,7 +153,8 @@ for A in ${ARCHES[@]}; do /bin/cp "$ftppath/$reponame.db.tar.$DB_COMPRESSION" build/ fi - echo "==> Processing new/updated packages for repository '$reponame'..." >&2 + pkgtotal=$(echo "$ADDPKGS $ANYPKGS" | wc -w) + echo "==> Processing $pkgtotal new/updated packages for repository '$reponame'..." >&2 cd "$WORKDIR" /usr/bin/svn checkout -N $svnpath checkout -- cgit v1.2.2 From 73f460e1224873d12b6e5879450d79a4b4465b0f Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Wed, 6 May 2009 14:08:14 -0700 Subject: config: Revert 'any' addition to the ARCHES array This change was made in error, as db-update does its own parsing of 'any' files independently Signed-off-by: Aaron Griffin --- config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config b/config index 33b7738..867d28e 100644 --- a/config +++ b/config @@ -8,6 +8,6 @@ SVNREPO_community="file:///srv/svn-community" STAGING="$HOME/staging" TMPDIR="/srv/tmp" -ARCHES=(i686 x86_64 any) +ARCHES=(i686 x86_64) # To be used if $BUILDSCRIPT is removed from makepkg.conf #BUILDSCRIPT="PKGBUILD" -- cgit v1.2.2 From f515029a694a11065f95a79bd83b325bf04562bb Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Fri, 8 May 2009 13:21:39 -0700 Subject: check_archlinux: Skip searching CVS and .svn dirs Prevents some errors in the community scripts due to scanning of CVS dirs. Also skipping .svn dirs for the future (dotglob may be set, in which case we'd scan .svn dirs as well) Signed-off-by: Aaron Griffin --- cron-jobs/check_archlinux/parse_pkgbuilds.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cron-jobs/check_archlinux/parse_pkgbuilds.sh b/cron-jobs/check_archlinux/parse_pkgbuilds.sh index 47aec89..7dcfbbb 100755 --- a/cron-jobs/check_archlinux/parse_pkgbuilds.sh +++ b/cron-jobs/check_archlinux/parse_pkgbuilds.sh @@ -56,6 +56,12 @@ parse() { } find_pkgbuilds() { + #Skip over some dirs + local d="$(basename $1)" + if [ "$d" = "CVS" -o "$d" = ".svn" ]; then + return + fi + if [ -f $1/PKGBUILD ]; then parse $1 return -- cgit v1.2.2 From 9925af8a8aa791026f0f51a9be601bfbc15d0a40 Mon Sep 17 00:00:00 2001 From: Francois Charette Date: Tue, 12 May 2009 10:35:33 +0200 Subject: Check whether packages exist in build dirs before attempting to copy them Signed-off-by: Aaron Griffin --- db-update | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/db-update b/db-update index cb00f39..132049f 100755 --- a/db-update +++ b/db-update @@ -204,19 +204,23 @@ for A in ${ARCHES[@]}; do # if non empty, move all build dirs if [ $(/bin/ls "$WORKDIR/build/" 2>/dev/null | wc -l) != 0 ]; then echo "Copying new files to '$ftppath'" - for f in "$WORKDIR/build/"*$current_arch$PKGEXT; do - if ! /bin/cp "$f" "$ftppath"; then - die "error: failure while copying files to $ftppath" - fi - done - for f in "$WORKDIR/build/"*any.pkg.tar.gz; do - if ! /bin/cp "$f" "$ftppath_any"; then - die "error: failure while copying files to $ftppath_any" - fi - if ! ln -s "$ftppath_any/$f" "$ftppath/$f"; then - die "error: failed to make link for $f." - fi - done + if [ $(/bin/ls "$WORKDIR/build/"*$current_arch$PKGEXT 2>/dev/null | wc -l) != 0 ]; then + for f in "$WORKDIR/build/"*$current_arch$PKGEXT; do + if ! /bin/cp "$f" "$ftppath"; then + die "error: failure while copying files to $ftppath" + fi + done + fi + if [ $(/bin/ls "$WORKDIR/build/"*any$PKGEXT 2>/dev/null | wc -l) != 0 ]; then + for f in "$WORKDIR/build/"*any$PKGEXT; do + if ! /bin/cp "$f" "$ftppath_any"; then + die "error: failure while copying files to $ftppath_any" + fi + if ! ln -s "$ftppath_any/$f" "$ftppath/$f"; then + die "error: failed to make link for $f." + fi + done + fi if ! /bin/cp "$WORKDIR/build/$reponame.db"* "$ftppath"; then die "failed to move repository $reponame-$A". fi -- cgit v1.2.2 From 61053b6c98723488454eaeaf105a77dc0c3f8bbc Mon Sep 17 00:00:00 2001 From: Francois Charette Date: Tue, 12 May 2009 11:29:44 +0200 Subject: create $WORKDIR before cding to it Signed-off-by: Aaron Griffin --- db-update | 1 + 1 file changed, 1 insertion(+) diff --git a/db-update b/db-update index 132049f..ace0d0e 100755 --- a/db-update +++ b/db-update @@ -83,6 +83,7 @@ fi pkgtotal=$(echo "$ANYPKGS" | wc -w) echo "==> Processing $pkgtotal new/updated arch-independent packages for '$reponame'..." >&2 +mkdir -p $WORKDIR cd "$WORKDIR" svnpath="$(get_svnpath $reponame)" /usr/bin/svn checkout -N $svnpath checkout -- cgit v1.2.2 From b65fd05078fab48c4352609b2d7befa118a75b0f Mon Sep 17 00:00:00 2001 From: Francois Charette Date: Tue, 12 May 2009 11:42:27 +0200 Subject: fixed filename in creation of symlinks Signed-off-by: Aaron Griffin --- db-update | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/db-update b/db-update index ace0d0e..5b011c2 100755 --- a/db-update +++ b/db-update @@ -217,8 +217,9 @@ for A in ${ARCHES[@]}; do if ! /bin/cp "$f" "$ftppath_any"; then die "error: failure while copying files to $ftppath_any" fi - if ! ln -s "$ftppath_any/$f" "$ftppath/$f"; then - die "error: failed to make link for $f." + bf=$(basename $f) + if ! ln -s "$ftppath_any/$bf" "$ftppath/$bf"; then + die "error: failed to make link for $bf." fi done fi -- cgit v1.2.2 From a7e949c0779787e3677178363b664960346449d5 Mon Sep 17 00:00:00 2001 From: Francois Charette Date: Tue, 12 May 2009 11:48:12 +0200 Subject: fixed redirection of error msgs in get_svnpath Signed-off-by: Aaron Griffin --- db-functions | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/db-functions b/db-functions index 99de756..f052bad 100644 --- a/db-functions +++ b/db-functions @@ -99,8 +99,8 @@ get_svnpath () { #get_svnpath reponame eval repopath=${var} if [ -z "$repopath" ]; then - echo "ERROR: SVN path not defined for '${1}'" >2 - echo " Please check SVNREPO_${1} config setting" >2 + echo "ERROR: SVN path not defined for '${1}'" 2> + echo " Please check SVNREPO_${1} config setting" 2> fi echo "$repopath" -- cgit v1.2.2 From b4ab204e5829b55250376f224ea41dec102eb0ae Mon Sep 17 00:00:00 2001 From: Abhishek Dasgupta Date: Wed, 13 May 2009 11:42:37 -0700 Subject: convert-to-any: Cleanup and commit removal Signed-off-by: Aaron Griffin --- convert-to-any | 75 ++++++++++++++-------------------------------------------- 1 file changed, 18 insertions(+), 57 deletions(-) diff --git a/convert-to-any b/convert-to-any index b8d298c..1c48eb3 100644 --- a/convert-to-any +++ b/convert-to-any @@ -5,24 +5,15 @@ # -- Abhishek Dasgupta -. "$(dirname $0)/db-functions" [ "$UID" = "" ] && UID=$(uid) +OUTDIR="$(pwd)" +WORKDIR="/tmp/convert-to-any.$UID" if [ $# -ne 1 ]; then - echo "Syntax: $(basename $0) " + echo "Syntax: $(basename $0) " exit 1 fi -repo=$(echo $1 | sed "s#\(.*\)/.*#\1#g") -pkg=$(echo $1 | sed "s#.*/\(.*\)#\1#g") - -if [ -f "$(dirname $0)/config" ]; then - . "$(dirname $0)/config" -else - TMPDIR=/srv/tmp - FTP_BASE=/srv/ftp -fi - if [ -f /etc/makepkg.conf ]; then . /etc/makepkg.conf else @@ -31,14 +22,8 @@ else PKGEXT=".pkg.tar.gz" fi -repo_lock $repo any -WORKDIR="$TMPDIR/convert-to-any.$pkg.$UID" -ftppath="$FTP_BASE/$repo/os" - cleanup() { trap '' 0 2 - # unlock - repo_unlock $repo any rm -rf "$WORKDIR" [ "$1" ] && exit $1 } @@ -53,56 +38,32 @@ die() { cleanup 1 } -# Enter the temporary build directory -# and convert the i686 package into an -# architecture-independent package. mkdir -p "$WORKDIR/build" -pushd "$WORKDIR/build" >/dev/null -oldpkgname=$ftppath/i686/$pkg* -if [ -f "$oldpkgname" ]; then - cp "$oldpkgname" . -else - die "E: Package $oldpkgname not found in $ftppath/i686" +oldpkgname="$1" + +if [ -z "$oldpkgname" ]; then + die "convert-to-any: which package to convert?" fi -for architecture in i686 x86_64; do - if [ -f "$ftppath/$architecture/$repo.db.tar.$DB_COMPRESSION" ]; then - cp $ftppath/$architecture/$repo.db.tar.$DB_COMPRESSION \ - $repo-$architecture.db.tar.$DB_COMPRESSION - else - touch $repo-$architecture.db.tar.$DB_COMPRESSION - fi -done +pkg="$(basename $oldpkgname)" +newpkgname=$(echo $pkg | sed "s/-\(i686\|x86_64\)$PKGEXT/-any$PKGEXT/") -if [ ! -d "$ftppath/any" ]; then mkdir -p "$ftppath/any"; fi +if ! cp "$oldpkgname" "$WORKDIR/build/$pkg"; then + die "convert-to-any: failed to copy package to $WORKDIR" +fi +pushd "$WORKDIR/build" >/dev/null # Conversion of i686 package into "any" package. -echo -n "Extracting $pkg..." mkdir -p package -tar zxf $pkg*i686$PKGEXT -C package -echo " done." +if ! tar zxf "$pkg" -C package; then + die "convert-to-any: error in extracting $oldpkgname" +fi -sed -i "s/arch = i686/arch = any/g" package/.PKGINFO -newpkgname=$(ls $pkg*i686$PKGEXT | sed "s/i686/any/g") +sed -i "s/arch = \(i686\|x86_64\)/arch = any/g" package/.PKGINFO pushd package >/dev/null -tar czf "$newpkgname" * .PKGINFO +tar czf "$OUTDIR/$newpkgname" * .PKGINFO popd >/dev/null -mv "package/$newpkgname" . -echo "Created $newpkgname." - -# New package is ready, move it into place and update db. -mv "$newpkgname" "$ftppath/any/" -for architecture in i686 x86_64; do - # rm -f $ftppath/$architecture/$pkg*$PKGEXT - ln -s "$ftppath/any/$newpkgname" "$ftppath/$architecture/$newpkgname" - repo-remove -q $repo-$architecture.db.tar.$DB_COMPRESSION $pkg - repo-add -q $repo-$architecture.db.tar.$DB_COMPRESSION $newpkgname - mv $repo-$architecture.db.tar.$DB_COMPRESSION "$ftppath/os/$architecture" - echo "Updated $repo-$architecture for $pkg." -done -echo -n "Cleaning up..." popd >/dev/null cleanup -echo " done." -- cgit v1.2.2 From 7bb77c5c993c5bd8d07fb3d737666f9cf6367c29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Charette?= Date: Wed, 13 May 2009 11:55:29 -0700 Subject: convert-to-any: Change description in header Signed-off-by: Aaron Griffin --- convert-to-any | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/convert-to-any b/convert-to-any index 1c48eb3..2d30865 100644 --- a/convert-to-any +++ b/convert-to-any @@ -1,7 +1,8 @@ #!/bin/bash -# Converts an existing package into an -# architecture-independent package and updates -# the repositories accordingly. +# +# Converts an existing architecture-independent package +# for i686 or x86_64 into a package with "arch = any" +# # -- Abhishek Dasgupta -- cgit v1.2.2 From 8276b6fefffb9f092ed8713c1cf01220e21c46eb Mon Sep 17 00:00:00 2001 From: Abhishek Dasgupta Date: Thu, 14 May 2009 00:43:34 +0530 Subject: convert-to-any: use fakeroot, put .PKGINFO at top Signed-off-by: Abhishek Dasgupta Signed-off-by: Aaron Griffin --- convert-to-any | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) mode change 100644 => 100755 convert-to-any diff --git a/convert-to-any b/convert-to-any old mode 100644 new mode 100755 index 2d30865..3d55830 --- a/convert-to-any +++ b/convert-to-any @@ -57,13 +57,13 @@ pushd "$WORKDIR/build" >/dev/null # Conversion of i686 package into "any" package. mkdir -p package -if ! tar zxf "$pkg" -C package; then +if ! fakeroot tar zxf "$pkg" -C package; then die "convert-to-any: error in extracting $oldpkgname" fi sed -i "s/arch = \(i686\|x86_64\)/arch = any/g" package/.PKGINFO pushd package >/dev/null -tar czf "$OUTDIR/$newpkgname" * .PKGINFO +fakeroot tar czf "$OUTDIR/$newpkgname" .PKGINFO * popd >/dev/null popd >/dev/null -- cgit v1.2.2 From c076b93fafbf47a85bd063c626621552f364a59d Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Wed, 13 May 2009 11:06:06 +0200 Subject: use pkgext defined in makepkg.conf Signed-off-by: Pierre Schmitz Signed-off-by: Aaron Griffin --- convert-to-any | 9 ++++----- cron-jobs/createFileLists | 16 +++++++++++----- db-functions | 2 +- misc-scripts/find-dupes | 4 ++-- misc-scripts/ftpdir-cleanup | 4 ++-- 5 files changed, 20 insertions(+), 15 deletions(-) diff --git a/convert-to-any b/convert-to-any index 3d55830..5b72aab 100755 --- a/convert-to-any +++ b/convert-to-any @@ -18,9 +18,8 @@ fi if [ -f /etc/makepkg.conf ]; then . /etc/makepkg.conf else - echo "W: /etc/makepkg.conf does not exist." - DB_COMPRESSION=gz - PKGEXT=".pkg.tar.gz" + echo "E: /etc/makepkg.conf does not exist." + exit 1 fi cleanup() { @@ -57,13 +56,13 @@ pushd "$WORKDIR/build" >/dev/null # Conversion of i686 package into "any" package. mkdir -p package -if ! fakeroot tar zxf "$pkg" -C package; then +if ! fakeroot bsdtar zxf "$pkg" -C package; then die "convert-to-any: error in extracting $oldpkgname" fi sed -i "s/arch = \(i686\|x86_64\)/arch = any/g" package/.PKGINFO pushd package >/dev/null -fakeroot tar czf "$OUTDIR/$newpkgname" .PKGINFO * +fakeroot bsdtar czf "$OUTDIR/$newpkgname" .PKGINFO * popd >/dev/null popd >/dev/null diff --git a/cron-jobs/createFileLists b/cron-jobs/createFileLists index 697c39d..6d03260 100755 --- a/cron-jobs/createFileLists +++ b/cron-jobs/createFileLists @@ -1,11 +1,17 @@ #!/bin/bash reposdir=/srv/ftp/ -#targetdir=/home/pierre/public_html/test-repo/ targetdir=$reposdir repos="core extra testing community" arches="i686 x86_64" +if [ -f /etc/makepkg.conf ]; then + . /etc/makepkg.conf +else + echo "Error: /etc/makepkg.conf does not exist." + exit 1 +fi + if [ -f "/tmp/createFileList.lock" ]; then echo "Error: createFileList allready in progress." exit 1 @@ -22,7 +28,7 @@ getpkgname() { local tmp tmp=${1##*/} - tmp=${tmp%.pkg.tar.gz} + tmp=${tmp%$PKGEXT} tmp=${tmp%-i686} tmp=${tmp%-x86_64} tmp=${tmp%-any} @@ -31,7 +37,7 @@ getpkgname() { cd $reposdir for repo in $repos; do - REPO_DB_FILE=${repo}.files.tar.gz + REPO_DB_FILE=${repo}.files.tar.${DB_COMPRESSION} for arch in $arches; do repodir=${repo}/os/${arch}/ cached="no" @@ -44,7 +50,7 @@ for repo in $repos; do fi # create file lists - for pkg in $repodir*.pkg.tar.gz; do + for pkg in $repodir*${PKGEXT}; do basename=$(basename $pkg) pkgname=$(getpkgname $basename) tmppkgdir=${TMPDIR}/${repodir}${pkgname}/ @@ -69,7 +75,7 @@ for repo in $repos; do cd ${TMPDIR}/${repodir} [ -f "${pkgdir}${REPO_DB_FILE}.old" ] && rm "${pkgdir}${REPO_DB_FILE}.old" [ -f "${pkgdir}${REPO_DB_FILE}" ] && mv "${pkgdir}${REPO_DB_FILE}" "${pkgdir}${REPO_DB_FILE}.old" - bsdtar --exclude=*.tar.gz -czf ${pkgdir}${REPO_DB_FILE} * + bsdtar --exclude=*.tar.${DB_COMPRESSION} -cf ${pkgdir}${REPO_DB_FILE} * fi cd $reposdir diff --git a/db-functions b/db-functions index f052bad..cee13ac 100644 --- a/db-functions +++ b/db-functions @@ -17,7 +17,7 @@ source_makepkg () { #Get some config info . /etc/makepkg.conf else - echo "/etc/makepkg.conf does not exist!" + echo "error: /etc/makepkg.conf does not exist!" exit 1 fi } diff --git a/misc-scripts/find-dupes b/misc-scripts/find-dupes index 372108f..16bf530 100755 --- a/misc-scripts/find-dupes +++ b/misc-scripts/find-dupes @@ -39,7 +39,7 @@ getpkgname() { showdupes() { done="" - for i in *.pkg.tar.gz; do + for i in *${PKGEXT}; do pkgname=$(getpkgname $i) if [ "$pkgname" = "$1" ]; then lastmod="$(stat -c %y $i | cut -d. -f1)" @@ -60,7 +60,7 @@ echo "Scanning for duplicate packages in '$reponame' ($arch)" DUPES="" lastpkg="" -for pkg in *.pkg.tar.gz; do +for pkg in *${PKGEXT}; do pkgname="$(getpkgname $pkg)" if [ "$lastpkg" = "$pkgname" ]; then DUPES="$DUPES $pkgname" diff --git a/misc-scripts/ftpdir-cleanup b/misc-scripts/ftpdir-cleanup index ed90437..a87ed8f 100755 --- a/misc-scripts/ftpdir-cleanup +++ b/misc-scripts/ftpdir-cleanup @@ -49,11 +49,11 @@ EXTRAFILES="" TMPDIR=$(mktemp -d /tmp/cleanup.XXXXXX) || exit 1 cd "${TMPDIR}" -/bin/tar xzf "$ftppath/$reponame.db.tar.$DB_COMPRESSION" +bsdtar xf "$ftppath/$reponame.db.tar.$DB_COMPRESSION" for pkg in *; do filename=$(grep -A1 '^%FILENAME%$' "${pkg}/desc" | tail -n1) - [ -z "${filename}" ] && filename="${pkg}.pkg.tar.gz" + [ -z "${filename}" ] && filename="${pkg}${PKGEXT}" if [ ! -f "${ftppath}/${filename}" ]; then MISSINGFILES="${MISSINGFILES} ${filename}" -- cgit v1.2.2 From 548a96ef75cd18f3645909031eda062de4546888 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Wed, 13 May 2009 14:21:17 -0700 Subject: convert-to-any: Remove 'z' from bsdtar bsdtar can detect this on its own, which allows us to use differing PKGEXTs easily Signed-off-by: Aaron Griffin --- convert-to-any | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/convert-to-any b/convert-to-any index 5b72aab..862c60b 100755 --- a/convert-to-any +++ b/convert-to-any @@ -56,13 +56,13 @@ pushd "$WORKDIR/build" >/dev/null # Conversion of i686 package into "any" package. mkdir -p package -if ! fakeroot bsdtar zxf "$pkg" -C package; then +if ! fakeroot bsdtar xf "$pkg" -C package; then die "convert-to-any: error in extracting $oldpkgname" fi sed -i "s/arch = \(i686\|x86_64\)/arch = any/g" package/.PKGINFO pushd package >/dev/null -fakeroot bsdtar czf "$OUTDIR/$newpkgname" .PKGINFO * +fakeroot bsdtar cf "$OUTDIR/$newpkgname" .PKGINFO * popd >/dev/null popd >/dev/null -- cgit v1.2.2 From f1b7557d23c601a86a03cd5eee4feb4072a3b8b2 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Tue, 2 Jun 2009 19:31:47 +0200 Subject: compute compression option from file extension bsdtar does not detect the compression type from the file extension. So we need to set the right option on our own. Signed-off-by: Aaron Griffin --- convert-to-any | 8 +++++++- cron-jobs/createFileLists | 9 ++++++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/convert-to-any b/convert-to-any index 862c60b..2ad1521 100755 --- a/convert-to-any +++ b/convert-to-any @@ -62,7 +62,13 @@ fi sed -i "s/arch = \(i686\|x86_64\)/arch = any/g" package/.PKGINFO pushd package >/dev/null -fakeroot bsdtar cf "$OUTDIR/$newpkgname" .PKGINFO * +case "$newpkgname" in + *tar.gz) TAR_OPT="z" ;; + *tar.bz2) TAR_OPT="j" ;; + *tar.xz) TAR_OPT="J" ;; + *) die "$newpkgname does not have a valid archive extension." ;; +esac +fakeroot bsdtar c${TAR_OPT}f "$OUTDIR/$newpkgname" .PKGINFO * popd >/dev/null popd >/dev/null diff --git a/cron-jobs/createFileLists b/cron-jobs/createFileLists index 6d03260..10520af 100755 --- a/cron-jobs/createFileLists +++ b/cron-jobs/createFileLists @@ -35,6 +35,13 @@ getpkgname() { echo $tmp } +case "${DB_COMPRESSION}" in + gz) TAR_OPT="z" ;; + bz2) TAR_OPT="j" ;; + xz) TAR_OPT="J" ;; + *) echo "${DB_COMPRESSION} is not a valid archive compression" && exit 1 ;; +esac + cd $reposdir for repo in $repos; do REPO_DB_FILE=${repo}.files.tar.${DB_COMPRESSION} @@ -75,7 +82,7 @@ for repo in $repos; do cd ${TMPDIR}/${repodir} [ -f "${pkgdir}${REPO_DB_FILE}.old" ] && rm "${pkgdir}${REPO_DB_FILE}.old" [ -f "${pkgdir}${REPO_DB_FILE}" ] && mv "${pkgdir}${REPO_DB_FILE}" "${pkgdir}${REPO_DB_FILE}.old" - bsdtar --exclude=*.tar.${DB_COMPRESSION} -cf ${pkgdir}${REPO_DB_FILE} * + bsdtar --exclude=*.tar.${DB_COMPRESSION} -c${TAR_OPT}f ${pkgdir}${REPO_DB_FILE} * fi cd $reposdir -- cgit v1.2.2 From 9217d0074e4962761320cc7423fc139b9d8ad382 Mon Sep 17 00:00:00 2001 From: Francois Charette Date: Thu, 14 May 2009 11:23:12 +0200 Subject: fix redirection to stderr in get_svnpath [Aaron: Use >&2 instead of /dev/stderr, just in the offchance that /dev is missing for some reason] Signed-off-by: Aaron Griffin --- db-functions | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/db-functions b/db-functions index cee13ac..830886f 100644 --- a/db-functions +++ b/db-functions @@ -99,8 +99,8 @@ get_svnpath () { #get_svnpath reponame eval repopath=${var} if [ -z "$repopath" ]; then - echo "ERROR: SVN path not defined for '${1}'" 2> - echo " Please check SVNREPO_${1} config setting" 2> + echo "ERROR: SVN path not defined for '${1}'" >&2 + echo " Please check SVNREPO_${1} config setting" >&2 fi echo "$repopath" -- cgit v1.2.2 From c330b1453f7c0d7c2261324083ba9e3a98ea130d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Charette?= Date: Fri, 12 Jun 2009 08:04:38 -0700 Subject: fixes for ftpdir-cleanup * don't add files with arch=any to EXTRAFILES * replace && by -a within [ ] Signed-off-by: Aaron Griffin --- misc-scripts/ftpdir-cleanup | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/misc-scripts/ftpdir-cleanup b/misc-scripts/ftpdir-cleanup index a87ed8f..fbf2925 100755 --- a/misc-scripts/ftpdir-cleanup +++ b/misc-scripts/ftpdir-cleanup @@ -69,7 +69,7 @@ for pkg in *; do done cd "$ftppath" -for pkg in *$PKGEXT; do +for pkg in *$arch$PKGEXT; do pkgname="$(getpkgname $pkg)" for p in ${TMPDIR}/${pkgname}-*; do if [ -d "${p}" -a "$(getpkgname $(basename ${p}))" = "${pkgname}" ]; then @@ -81,7 +81,7 @@ done cd "$ftppath_base/any" for pkg in *$PKGEXT; do - if [ ! -h $ftppath_base/i686/$pkg && ! -h $ftppath_base/x86_64/$pkg ]; then + if [ ! -h $ftppath_base/i686/$pkg -a ! -h $ftppath_base/x86_64/$pkg ]; then ARCHINDEPFILES="$ARCHINDEPFILES $pkg" fi done -- cgit v1.2.2 From 277fc975d2f9f6e6e1142af8dd38b3d3af53e903 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Thu, 9 Jul 2009 16:56:39 +0200 Subject: fix syntax error Signed-off-by: Aaron Griffin --- db-move | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db-move b/db-move index 0759ba2..0cc8aa4 100755 --- a/db-move +++ b/db-move @@ -22,7 +22,7 @@ ftppath_to="$FTP_BASE/$repoto/os/" svnrepo_from="$repofrom-$_arch" svnrepo_to="$repoto-$_arch" svnpath="$(get_svnpath $repoto)" -if [ "$svnpath" != "$(get_svnpath $repofrom" ]; then +if [ "$svnpath" != "$(get_svnpath $repofrom)" ]; then echo "ERROR: Cannot move packages across SVN repos" echo " A move must be within the same svn repo" fi -- cgit v1.2.2 From 84a845a0690d6ffb96882699cdc8e9c0e497125d Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Thu, 9 Jul 2009 17:01:50 +0200 Subject: fix syntax error Signed-off-by: Aaron Griffin --- db-functions | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/db-functions b/db-functions index 830886f..966a713 100644 --- a/db-functions +++ b/db-functions @@ -78,8 +78,8 @@ getpkgver() { } check_pkg_arch () { #check_pkg_arch pkgfile arch - local arch - _arch="$(/usr/bin/bsdtar -xOf "$1" .PKGINFO | /bin/grep "^arch" | /bin/sed 's|\w*\s*=\s*\(.*\)|\1|')" + local _arch + _arch="$(/usr/bin/bsdtar -xOqf "$1" .PKGINFO | /bin/grep "^arch" | /bin/sed 's|\w*\s*=\s*\(.*\)|\1|')" if [ -z "$_arch" ]; then echo "ERROR: Package '$1' has no arch in the PKGINFO. Fail!" -- cgit v1.2.2 From d0386030626fda74e6dd370fcd3157c4d93501af Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Thu, 9 Jul 2009 17:24:15 +0200 Subject: Read meta data from .PKGINFO instead of filename Prefer pkgbase over pkgname in getpkgbase() Signed-off-by: Aaron Griffin --- db-functions | 46 +++++++++++++++++++++++++++++++--------------- 1 file changed, 31 insertions(+), 15 deletions(-) diff --git a/db-functions b/db-functions index 966a713..11ef8e7 100644 --- a/db-functions +++ b/db-functions @@ -54,32 +54,48 @@ repo_unlock () { #repo_unlock repo-name arch restore_umask } -# Get the package name from the filename -# hackish, but should work for now +# Get the package base or name +getpkgbase() { + local _base + + _base="$(/usr/bin/bsdtar -xOqf "$1" .PKGINFO | sort | /bin/grep -m 1 -E "^pkg(base|name)" | /bin/sed 's|\w*\s*=\s*\(.*\)|\1|')" + if [ -z "$_base" ]; then + echo "ERROR: Package '$1' has no pkgbase or pkgname in the PKGINFO. Fail!" + exit 1 + fi + + echo "$_base" +} + +# Get the package name getpkgname() { - local tmp + local _name - tmp=${1##*/} - tmp=${tmp%$PKGEXT} - tmp=${tmp%$SRCEXT} - tmp=${tmp%-$CARCH} - echo ${tmp%-*-*} + _name="$(/usr/bin/bsdtar -xOqf "$1" .PKGINFO | /bin/grep -m 1 "^pkgname" | /bin/sed 's|\w*\s*=\s*\(.*\)|\1|')" + if [ -z "$_name" ]; then + echo "ERROR: Package '$1' has no pkgname in the PKGINFO. Fail!" + exit 1 + fi + + echo "$_name" } # Get the pkgver-pkgrel of this package getpkgver() { - local tmp + local _ver + + _ver="$(/usr/bin/bsdtar -xOqf "$1" .PKGINFO | /bin/grep -m 1 "^pkgver" | /bin/sed 's|\w*\s*=\s*\(.*\)|\1|')" + if [ -z "$_ver" ]; then + echo "ERROR: Package '$1' has no pkgver in the PKGINFO. Fail!" + exit 1 + fi - tmp=${1##*/} - tmp=${tmp%$PKGEXT} - tmp=${tmp%$SRCEXT} - tmp=${tmp%-$CARCH} - echo $tmp | sed 's|.*-\(.*-.*\)$|\1|g' + echo "$_ver" } check_pkg_arch () { #check_pkg_arch pkgfile arch local _arch - _arch="$(/usr/bin/bsdtar -xOqf "$1" .PKGINFO | /bin/grep "^arch" | /bin/sed 's|\w*\s*=\s*\(.*\)|\1|')" + _arch="$(/usr/bin/bsdtar -xOqf "$1" .PKGINFO | /bin/grep -m 1 "^arch" | /bin/sed 's|\w*\s*=\s*\(.*\)|\1|')" if [ -z "$_arch" ]; then echo "ERROR: Package '$1' has no arch in the PKGINFO. Fail!" -- cgit v1.2.2 From 052b35e8e0d4aec5c2936f4fea21a29fa24d233e Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Thu, 9 Jul 2009 17:29:31 +0200 Subject: remove redefined function Signed-off-by: Aaron Griffin --- db-update | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/db-update b/db-update index 5b011c2..1883a58 100755 --- a/db-update +++ b/db-update @@ -44,17 +44,6 @@ if [ -d "${stagedir}/add" ]; then /bin/mv "${stagedir}/add/"* "$stagedir/" fi -# Get the package name from the filename -# hackish, but should work for now -getpkgname() { - local tmp - - tmp=${1##*/} - tmp=${tmp%$PKGEXT} - tmp=${tmp%-$current_arch} - echo ${tmp%-*-*} -} - cleanup() { trap '' 0 2 # unlock -- cgit v1.2.2 From 45dc03f5f91f1940a83a11472ff872478aeef099 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Thu, 9 Jul 2009 17:44:15 +0200 Subject: use pkgbase to get pkgbuild in db-update Signed-off-by: Aaron Griffin --- db-update | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/db-update b/db-update index 1883a58..ba8c3b5 100755 --- a/db-update +++ b/db-update @@ -82,22 +82,23 @@ if [ -n "$ANYPKGS" ]; then for pkg in $ANYPKGS; do _pkgfile=$(basename $pkg) _pkgname="$(getpkgname $pkg)" + _pkgbase="$(getpkgbase $pkg)" svnrepo="$reponame-any" echo " Validating package arch (any) $_pkgname" if ! check_pkg_arch "$pkg" "any"; then echo " ERROR: $_pkgfile is not architecture independent!" else - echo " Checking SVN for $_pkgname" - /usr/bin/svn up -q $_pkgname - if [ -d "$_pkgname/repos/$svnrepo" ]; then - . "$_pkgname/repos/$svnrepo/$BUILDSCRIPT" - if [ "$_pkgfile" = "$pkgname-$pkgver-$pkgrel-any$PKGEXT" ]; then + echo " Checking SVN for $_pkgbase" + /usr/bin/svn up -q $_pkgbase + if [ -d "$_pkgbase/repos/$svnrepo" ]; then + . "$_pkgbase/repos/$svnrepo/$BUILDSCRIPT" + if [ "$_pkgfile" = "$_pkgname-$pkgver-$pkgrel-any$PKGEXT" ]; then to_add_any="$to_add_any $pkg" else echo " WARNING: $_pkgfile does not match $BUILDSCRIPT in $svnrepo" fi else - echo " WARNING: Package $_pkgname not found in $svnrepo" + echo " WARNING: Package $_pkgbase not found in $svnrepo" fi fi done @@ -154,22 +155,23 @@ for A in ${ARCHES[@]}; do for pkg in $ADDPKGS; do _pkgfile=$(basename $pkg) _pkgname="$(getpkgname $pkg)" + _pkgbase="$(getpkgbase $pkg)" echo " Validating package arch ($current_arch) $_pkgname" if ! check_pkg_arch "$pkg" "$current_arch"; then echo " ERROR: $_pkgfile was built for the wrong architecture" else - echo " Checking SVN for $_pkgname" - /usr/bin/svn up -q $_pkgname - if [ -d "$_pkgname/repos/$svnrepo" ]; then - . "$_pkgname/repos/$svnrepo/$BUILDSCRIPT" - if [ "$_pkgfile" = "$pkgname-$pkgver-$pkgrel-$current_arch$PKGEXT" ]; then + echo " Checking SVN for $_pkgbase" + /usr/bin/svn up -q $_pkgbase + if [ -d "$_pkgbase/repos/$svnrepo" ]; then + . "$_pkgbase/repos/$svnrepo/$BUILDSCRIPT" + if [ "$_pkgfile" = "$_pkgname-$pkgver-$pkgrel-$current_arch$PKGEXT" ]; then to_add="$to_add $pkg" else echo " WARNING: $_pkgfile does not match $BUILDSCRIPT in $svnrepo" fi else - echo " WARNING: Package $_pkgname not found in $svnrepo" + echo " WARNING: Package $_pkgbase not found in $svnrepo" fi fi done -- cgit v1.2.2 From 6226ceb54a363573eb18856c1d935197fffbc334 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Tue, 14 Jul 2009 14:49:29 -0700 Subject: Refactor PKGINFO reading into a function This is just for readability and cleanup Signed-off-by: Aaron Griffin --- db-functions | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/db-functions b/db-functions index 11ef8e7..359e4c0 100644 --- a/db-functions +++ b/db-functions @@ -54,11 +54,20 @@ repo_unlock () { #repo_unlock repo-name arch restore_umask } +# usage: _grep_pkginfo pkgfile pattern +_grep_pkginfo() { + local _ret + + _ret="$(/usr/bin/bsdtar -xOqf "$1" .PKGINFO | /bin/grep -m 1 -E "$pattern" | /bin/sed 's|\w*\s*=\s*\(.*\)|\1|')" + echo "$_ret" +} + + # Get the package base or name getpkgbase() { local _base - _base="$(/usr/bin/bsdtar -xOqf "$1" .PKGINFO | sort | /bin/grep -m 1 -E "^pkg(base|name)" | /bin/sed 's|\w*\s*=\s*\(.*\)|\1|')" + _base="$(_grep_pkginfo "$1" "^pkg(base|name)")" if [ -z "$_base" ]; then echo "ERROR: Package '$1' has no pkgbase or pkgname in the PKGINFO. Fail!" exit 1 @@ -71,7 +80,7 @@ getpkgbase() { getpkgname() { local _name - _name="$(/usr/bin/bsdtar -xOqf "$1" .PKGINFO | /bin/grep -m 1 "^pkgname" | /bin/sed 's|\w*\s*=\s*\(.*\)|\1|')" + _name="$(_grep_pkginfo "$1" "^pkgname")" if [ -z "$_name" ]; then echo "ERROR: Package '$1' has no pkgname in the PKGINFO. Fail!" exit 1 @@ -84,7 +93,7 @@ getpkgname() { getpkgver() { local _ver - _ver="$(/usr/bin/bsdtar -xOqf "$1" .PKGINFO | /bin/grep -m 1 "^pkgver" | /bin/sed 's|\w*\s*=\s*\(.*\)|\1|')" + _ver="$(_grep_pkginfo "$1" "^pkgver")" if [ -z "$_ver" ]; then echo "ERROR: Package '$1' has no pkgver in the PKGINFO. Fail!" exit 1 @@ -95,7 +104,7 @@ getpkgver() { check_pkg_arch () { #check_pkg_arch pkgfile arch local _arch - _arch="$(/usr/bin/bsdtar -xOqf "$1" .PKGINFO | /bin/grep -m 1 "^arch" | /bin/sed 's|\w*\s*=\s*\(.*\)|\1|')" + _arch="$(_grep_pkginfo "$1" "^arch")" if [ -z "$_arch" ]; then echo "ERROR: Package '$1' has no arch in the PKGINFO. Fail!" -- cgit v1.2.2 From 66d12ed4b83d623cfbda06242982f73e50a78125 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Fri, 17 Jul 2009 12:32:08 -0700 Subject: Fix adjust-permissions for gerolde and sigurd Signed-off-by: Aaron Griffin --- cron-jobs/adjust-permissions | 33 +++++++++++++++++++++------------ 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/cron-jobs/adjust-permissions b/cron-jobs/adjust-permissions index 947c021..6670a6f 100755 --- a/cron-jobs/adjust-permissions +++ b/cron-jobs/adjust-permissions @@ -10,19 +10,28 @@ fi /usr/bin/renice +10 -p $$ > /dev/null cd /srv/ftp -/bin/chown -R ftp:ftp-arch core/os/any -/bin/chown -R ftp:ftp-arch core/os/i686 -/bin/chown -R ftp:ftp-arch core/os/x86_64 -/bin/chown -R ftp:ftp-extra {extra,testing}/os/any -/bin/chown -R ftp:ftp-extra {extra,testing}/os/i686 -/bin/chown -R ftp:ftp-extra {extra,testing}/os/x86_64 -/bin/chown -R ftp:aur community +if [ -d "core" ]; then + #This is unique to gerolde (main arch server) + /bin/chown -R ftp:ftp-arch core/os/any + /bin/chown -R ftp:ftp-arch core/os/i686 + /bin/chown -R ftp:ftp-arch core/os/x86_64 + /bin/chown -R ftp:ftp-extra {extra,testing}/os/any + /bin/chown -R ftp:ftp-extra {extra,testing}/os/i686 + /bin/chown -R ftp:ftp-extra {extra,testing}/os/x86_64 + for d in core extra testing; do + /bin/chmod -R g+w $d/os/any + /bin/chmod -R g+w $d/os/i686 + /bin/chmod -R g+w $d/os/x86_64 + done +else + /bin/chown -R root:tusers {community,community-testing}/os/any + /bin/chown -R root:tusers {community,community-testing}/os/i686 + /bin/chown -R root:tusers {community,community-testing}/os/x86_64 + /bin/chmod -R g+w {community,community-testing}/os/any + /bin/chmod -R g+w {community,community-testing}/os/i686 + /bin/chmod -R g+w {community,community-testing}/os/x86_64 +fi -for d in core extra testing community; do - /bin/chmod -R g+w $d/os/any - /bin/chmod -R g+w $d/os/i686 - /bin/chmod -R g+w $d/os/x86_64 -done /bin/chmod 555 /srv/ftp rm -f /tmp/.ftpmaint.lck -- cgit v1.2.2 From 0667909b7a0a72cde7db32f7664daef7c402cd91 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Fri, 17 Jul 2009 12:33:16 -0700 Subject: Add community and 'any' arch to ftp cleanup Signed-off-by: Aaron Griffin --- cron-jobs/ftpdir-cleanup | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index 1f18b17..6c295c9 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -1,8 +1,7 @@ #!/bin/bash -#TODO add community -repos="core extra testing" -arches="i686 x86_64" +repos="core extra testing community" +arches="any i686 x86_64" LOCKFILE="/tmp/.ftpdircleanup.lock" -- cgit v1.2.2 From b354593994a96901f4b86a0fb734edb5cb1b2348 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Fri, 17 Jul 2009 12:35:35 -0700 Subject: Add community and 'any' arch to sourceballs Signed-off-by: Aaron Griffin --- cron-jobs/sourceballs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index b4085f2..625d90d 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -2,8 +2,8 @@ ftpbase="/srv/ftp" srcbase="/srv/ftp/sources" -repos="core extra testing" -arches="i686 x86_64" +repos="core extra testing community" +arches="any i686 x86_64" LOCKFILE="/tmp/.sourceball.lock" -- cgit v1.2.2 From 4b4a648c3ba4612d82ed1c6c23eb81fcef97fd14 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Fri, 17 Jul 2009 12:35:49 -0700 Subject: sourceballs: skip nonexistant FTP paths This is useful when running on separate machines that don't have all repos Signed-off-by: Aaron Griffin --- cron-jobs/sourceballs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 625d90d..d07c482 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -56,6 +56,10 @@ for repo in $repos; do for arch in $arches; do export CARCH="$arch" ftppath="$ftpbase/$repo/os/$arch" + if [ ! -d "$ftppath" ]; then + echo "FTP path does not exist: $ftppath" >2 + continue + fi cd $ftppath for pkg in *$PKGEXT; do pkgname=$(getpkgname $pkg) -- cgit v1.2.2 From 7a2109fe8645eae4bc0024c057d399d67e0f0035 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Fri, 17 Jul 2009 12:47:59 -0700 Subject: ftpdir-cleanup: Allow configurable destination dir 3rd argument to the script Signed-off-by: Aaron Griffin --- misc-scripts/ftpdir-cleanup | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/misc-scripts/ftpdir-cleanup b/misc-scripts/ftpdir-cleanup index fbf2925..fd771e7 100755 --- a/misc-scripts/ftpdir-cleanup +++ b/misc-scripts/ftpdir-cleanup @@ -1,12 +1,13 @@ #!/bin/bash -if [ $# -ne 2 ]; then - echo "usage: $(basename $0) " +if [ $# -ne 3 ]; then + echo "usage: $(basename $0) " exit 1 fi reponame=$1 arch=$2 +dest=$3 ##### Arch specific stuff. TODO make this configurable ##### ftppath="/srv/ftp/$reponame/os/$arch/" @@ -98,7 +99,7 @@ echo "Scan complete for $reponame ($arch) at ${ftppath}" if [ -n "$DELETEFILES" ]; then echo " The following files are out of date" - echo " They will be moved to /srv/package-cleanup" + echo " They will be moved to '$dest'" for f in $DELETEFILES; do echo " $f" done @@ -115,7 +116,7 @@ fi if [ -n "$EXTRAFILES" ]; then echo " The following files are in the repo but not the db" - echo " They will be moved to /srv/package-cleanup" + echo " They will be moved to '$dest'" for f in $EXTRAFILES; do echo " $f" done @@ -124,25 +125,25 @@ fi if [ -n "$ARCHINDEPFILES" ]; then echo " The following architecture independent packages" echo " are not symlinked in the architecture repositories." - echo " They will be moved to /srv/package/cleanup" + echo " They will be moved to '$dest'" for f in $ARCHINDEPFILES; do echo " $f" done fi if [ -n "${DELETEFILES}" ]; then - mv ${DELETEFILES} /srv/package-cleanup/ + mv ${DELETEFILES} "$dest" echo "" fi if [ -n "${EXTRAFILES}" ]; then - mv ${EXTRAFILES} /srv/package-cleanup/ + mv ${EXTRAFILES} "$dest" echo "" fi cd "$ftppath_base/any" if [ -n "${ARCHINDEPFILES}" ]; then - mv ${ARCHINDEPFILES} /srv/package-cleanup/ + mv ${ARCHINDEPFILES} "$dest" echo "" fi -- cgit v1.2.2 From 3bcb9d8c78857cb40110616c3fe08259c577fdca Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Fri, 17 Jul 2009 12:51:58 -0700 Subject: ftpdir-cleanup: Add configurable destdir Signed-off-by: Aaron Griffin --- config | 2 ++ cron-jobs/ftpdir-cleanup | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/config b/config index 867d28e..425709a 100644 --- a/config +++ b/config @@ -6,6 +6,8 @@ SVNREPO_extra="file:///srv/svn-packages" SVNREPO_testing="file:///srv/svn-packages" SVNREPO_community="file:///srv/svn-community" +CLEANUP_DESTDIR="/srv/package-cleanup" + STAGING="$HOME/staging" TMPDIR="/srv/tmp" ARCHES=(i686 x86_64) diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index 6c295c9..77a0843 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -28,9 +28,12 @@ trap ctrl_c 2 #adjust the nice level to run at a lower priority /usr/bin/renice +10 -p $$ > /dev/null +#Get our destination dir +. $(dirname $0)/../config + for repo in $repos; do for arch in $arches; do - $(dirname $0)/../misc-scripts/ftpdir-cleanup $repo $arch + $(dirname $0)/../misc-scripts/ftpdir-cleanup $repo $arch $CLEANUP_DESTDIR done done -- cgit v1.2.2 From 2b3b2eb5791e16a4c6086516cd1df37ed7305d8d Mon Sep 17 00:00:00 2001 From: Francois Charette Date: Mon, 20 Jul 2009 09:38:28 -0700 Subject: Bugfix in db-functions for _grep_pkginfo * undefined argument $pattern changed to $2 Signed-off-by: Aaron Griffin --- db-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db-functions b/db-functions index 359e4c0..8111c6d 100644 --- a/db-functions +++ b/db-functions @@ -58,7 +58,7 @@ repo_unlock () { #repo_unlock repo-name arch _grep_pkginfo() { local _ret - _ret="$(/usr/bin/bsdtar -xOqf "$1" .PKGINFO | /bin/grep -m 1 -E "$pattern" | /bin/sed 's|\w*\s*=\s*\(.*\)|\1|')" + _ret="$(/usr/bin/bsdtar -xOqf "$1" .PKGINFO | /bin/grep -m 1 -E "$2" | /bin/sed 's|\w*\s*=\s*\(.*\)|\1|')" echo "$_ret" } -- cgit v1.2.2 From 0f54cbe863f057eb197476c24e693441414e8c67 Mon Sep 17 00:00:00 2001 From: Francois Charette Date: Mon, 20 Jul 2009 11:41:35 +0200 Subject: Fix final "cleanup" to avoid spurious error msg * When calling cleanup at the end of db-update, the function repo_unlock is called, but the repo-locks have already been unlocked! So we get a spurious error message. This commit replaces "cleanup" by "rm -rf $WORKDIR". Signed-off-by: Aaron Griffin --- db-update | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db-update b/db-update index ba8c3b5..67f8782 100755 --- a/db-update +++ b/db-update @@ -231,5 +231,5 @@ done if [ -n "$to_add_any" ]; then /bin/rm $to_add_any fi -cleanup +rm -rf $WORKDIR # vim: set ts=4 sw=4 noet ft=sh: -- cgit v1.2.2 From e53a4351d72d2a175d5342b71c1f19304840cd8e Mon Sep 17 00:00:00 2001 From: Francois Charette Date: Mon, 20 Jul 2009 14:17:53 +0200 Subject: simple syntactic improvements [Aaron: Don't remove quotes around $WORKDIR] Signed-off-by: Aaron Griffin --- db-update | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/db-update b/db-update index 67f8782..b530987 100755 --- a/db-update +++ b/db-update @@ -196,15 +196,16 @@ for A in ${ARCHES[@]}; do # if non empty, move all build dirs if [ $(/bin/ls "$WORKDIR/build/" 2>/dev/null | wc -l) != 0 ]; then echo "Copying new files to '$ftppath'" - if [ $(/bin/ls "$WORKDIR/build/"*$current_arch$PKGEXT 2>/dev/null | wc -l) != 0 ]; then - for f in "$WORKDIR/build/"*$current_arch$PKGEXT; do + if [ $(/bin/ls "$WORKDIR/build/"*-$current_arch$PKGEXT 2>/dev/null | wc -l) != 0 ]; then + for f in "$WORKDIR/build/"*-$current_arch$PKGEXT; do if ! /bin/cp "$f" "$ftppath"; then die "error: failure while copying files to $ftppath" fi done fi - if [ $(/bin/ls "$WORKDIR/build/"*any$PKGEXT 2>/dev/null | wc -l) != 0 ]; then - for f in "$WORKDIR/build/"*any$PKGEXT; do + if [ $(/bin/ls "$WORKDIR/build/"*-any$PKGEXT 2>/dev/null | wc -l) != 0 ]; then + echo "Copying new files to '$ftppath_any' and symlinking" + for f in "$WORKDIR/build/"*-any$PKGEXT; do if ! /bin/cp "$f" "$ftppath_any"; then die "error: failure while copying files to $ftppath_any" fi @@ -214,7 +215,7 @@ for A in ${ARCHES[@]}; do fi done fi - if ! /bin/cp "$WORKDIR/build/$reponame.db"* "$ftppath"; then + if ! /bin/cp "$WORKDIR/build/$reponame.db"* "$ftppath/"; then die "failed to move repository $reponame-$A". fi else @@ -228,8 +229,10 @@ for A in ${ARCHES[@]}; do repo_unlock $reponame $current_arch done + if [ -n "$to_add_any" ]; then /bin/rm $to_add_any fi + rm -rf $WORKDIR # vim: set ts=4 sw=4 noet ft=sh: -- cgit v1.2.2 From ce9a9b1951cefbdf6b2df93b07eb9120295bec7a Mon Sep 17 00:00:00 2001 From: Francois Charette Date: Mon, 20 Jul 2009 09:47:08 -0700 Subject: Minor syntactic improvements in db-update * also added error msg for repo_lock in db-functions [Aaron: keep quotes around $WORKDIR] Signed-off-by: Aaron Griffin --- db-functions | 3 +++ db-update | 7 +++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/db-functions b/db-functions index 8111c6d..6d84d31 100644 --- a/db-functions +++ b/db-functions @@ -40,6 +40,9 @@ repo_lock () { #repo_lock repo-name arch exit 1 else /bin/touch "$LOCKFILE" + if [ ! -f "$LOCKFILE" ]; then + echo "error: could not create repo lock... something went wrong!" + fi set_umask fi } diff --git a/db-update b/db-update index b530987..9d9f7eb 100755 --- a/db-update +++ b/db-update @@ -104,8 +104,7 @@ if [ -n "$ANYPKGS" ]; then done fi -for A in ${ARCHES[@]}; do - current_arch="$A" +for current_arch in ${ARCHES[@]}; do ftppath="$FTP_BASE/$reponame/os/$current_arch/" ftppath_any="$FTP_BASE/$reponame/os/any/" @@ -195,8 +194,8 @@ for A in ${ARCHES[@]}; do # if non empty, move all build dirs if [ $(/bin/ls "$WORKDIR/build/" 2>/dev/null | wc -l) != 0 ]; then - echo "Copying new files to '$ftppath'" if [ $(/bin/ls "$WORKDIR/build/"*-$current_arch$PKGEXT 2>/dev/null | wc -l) != 0 ]; then + echo "Copying new files to '$ftppath'" for f in "$WORKDIR/build/"*-$current_arch$PKGEXT; do if ! /bin/cp "$f" "$ftppath"; then die "error: failure while copying files to $ftppath" @@ -216,7 +215,7 @@ for A in ${ARCHES[@]}; do done fi if ! /bin/cp "$WORKDIR/build/$reponame.db"* "$ftppath/"; then - die "failed to move repository $reponame-$A". + die "failed to move repository $reponame-$current_arch". fi else echo "Nothing to copy, no work done" -- cgit v1.2.2 From 9316686c4c29f92a9723d47f5137e172dea04441 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Mon, 20 Jul 2009 13:28:06 -0700 Subject: ftpdir-cleanup: Corrections for 'any' packages Signed-off-by: Aaron Griffin --- misc-scripts/ftpdir-cleanup | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/misc-scripts/ftpdir-cleanup b/misc-scripts/ftpdir-cleanup index fd771e7..9bac030 100755 --- a/misc-scripts/ftpdir-cleanup +++ b/misc-scripts/ftpdir-cleanup @@ -32,6 +32,7 @@ getpkgname() { tmp=${1##*/} tmp=${tmp%$PKGEXT} tmp=${tmp%-$arch} + tmp=${tmp%-any} echo ${tmp%-*-*} } @@ -40,7 +41,8 @@ getpkgname_ver() { tmp=${1##*/} tmp=${tmp%$PKGEXT} - echo ${tmp%-$arch} + tmp=${tmp%-$arch} + echo ${tmp%-any} } MISSINGFILES="" @@ -56,7 +58,7 @@ for pkg in *; do filename=$(grep -A1 '^%FILENAME%$' "${pkg}/desc" | tail -n1) [ -z "${filename}" ] && filename="${pkg}${PKGEXT}" - if [ ! -f "${ftppath}/${filename}" ]; then + if [ ! -e "${ftppath}/${filename}" ]; then MISSINGFILES="${MISSINGFILES} ${filename}" else pkgname="$(getpkgname ${filename})" -- cgit v1.2.2 From 2ff2ae734f8909054776f89fdb2f6fb3795aac15 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Mon, 20 Jul 2009 15:22:00 -0700 Subject: db-scripts: getpkgbase: return pkgbase by default Check for pkgbase first. If and only if this doesn't exist, return pkgname. This fixes potential ordering issues. Signed-off-by: Aaron Griffin --- db-functions | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/db-functions b/db-functions index 6d84d31..fb00d89 100644 --- a/db-functions +++ b/db-functions @@ -66,14 +66,13 @@ _grep_pkginfo() { } -# Get the package base or name +# Get the package base or name as fallback getpkgbase() { local _base - _base="$(_grep_pkginfo "$1" "^pkg(base|name)")" + _base="$(_grep_pkginfo "$1" "^pkgbase")" if [ -z "$_base" ]; then - echo "ERROR: Package '$1' has no pkgbase or pkgname in the PKGINFO. Fail!" - exit 1 + echo $(getpkgname "$1") fi echo "$_base" -- cgit v1.2.2 From 755ba2fd1198272914e540806e9bfb94416da0ca Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Mon, 20 Jul 2009 15:25:05 -0700 Subject: db-functions: minor cleanup - remove echo Signed-off-by: Aaron Griffin --- db-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db-functions b/db-functions index fb00d89..d183373 100644 --- a/db-functions +++ b/db-functions @@ -72,7 +72,7 @@ getpkgbase() { _base="$(_grep_pkginfo "$1" "^pkgbase")" if [ -z "$_base" ]; then - echo $(getpkgname "$1") + getpkgname "$1" fi echo "$_base" -- cgit v1.2.2 From 754d26d772570d02c878ab6afc3dc5f75e17e373 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Mon, 20 Jul 2009 18:51:04 -0400 Subject: ftpdir-cleanup: Fixes for 'any' support * Break out ftppath, base it on ftbpath_base * Only check ARCHINDEPFILES if the any dir exists * Attempt to restore symlinks before moving 'any' files out of the way Signed-off-by: Aaron Griffin --- misc-scripts/ftpdir-cleanup | 50 ++++++++++++++++++++++++++++++++++++--------- 1 file changed, 40 insertions(+), 10 deletions(-) diff --git a/misc-scripts/ftpdir-cleanup b/misc-scripts/ftpdir-cleanup index 9bac030..7de64b9 100755 --- a/misc-scripts/ftpdir-cleanup +++ b/misc-scripts/ftpdir-cleanup @@ -10,10 +10,11 @@ arch=$2 dest=$3 ##### Arch specific stuff. TODO make this configurable ##### -ftppath="/srv/ftp/$reponame/os/$arch/" ftppath_base="/srv/ftp/$reponame/os" ############################################################ +ftppath="$ftppath_base/$arch" + if [ ! -d "$ftppath" ]; then echo "FTP path '$ftppath' does not exist" exit 1 @@ -32,7 +33,6 @@ getpkgname() { tmp=${1##*/} tmp=${tmp%$PKGEXT} tmp=${tmp%-$arch} - tmp=${tmp%-any} echo ${tmp%-*-*} } @@ -41,8 +41,7 @@ getpkgname_ver() { tmp=${1##*/} tmp=${tmp%$PKGEXT} - tmp=${tmp%-$arch} - echo ${tmp%-any} + echo ${tmp%-$arch} } MISSINGFILES="" @@ -82,12 +81,14 @@ for pkg in *$arch$PKGEXT; do EXTRAFILES="$EXTRAFILES $pkg" done -cd "$ftppath_base/any" -for pkg in *$PKGEXT; do - if [ ! -h $ftppath_base/i686/$pkg -a ! -h $ftppath_base/x86_64/$pkg ]; then - ARCHINDEPFILES="$ARCHINDEPFILES $pkg" - fi -done +if [ -d "$ftppath_base/any" ]; then + cd "$ftppath_base/any" + for pkg in *$PKGEXT; do + if [ ! -h $ftppath_base/i686/$pkg -a ! -h $ftppath_base/x86_64/$pkg ]; then + ARCHINDEPFILES="$ARCHINDEPFILES $pkg" + fi + done +fi cd "$ftppath" rm -rf ${TMPDIR} @@ -97,6 +98,35 @@ if [ -z "$DELETEFILES$MISSINGFILES$EXTRAFILES$ARCHINDEPFILES" ]; then exit 0 fi +#Do a quick check. If we have a MISSINGFILE and an ARCHINDEPFILE of the same +#name to remove, then let's restore the symlink. + +archfiles="$ARCHINDEPFILES" +ARCHINDEPFILES="" +for af in $archfiles; do + af_satisfied=0 + for mf in $MISSINGFILES; do + if [ "${af}" = "${mf}" ]; then + ln -s "${ftppath_base}/any/${af}" "${ftppath}" + af_satisfied=1 + fi + + done + if [ $af_satisfied -eq 0 ]; then + ARCHINDEPFILES="${ARCHINDEPFILES} ${af}" + fi +done +#rescan missing files now +missfiles="$MISSINGFILES" +MISSINGFILES="" +for mf in $missfiles; do + if [ ! -e "${ftppath}/${mf}" ]; then + MISSINGFILES="${MISSINGFILES} ${mf}" + else + echo "Missing file has been restored: $mf" + fi +done + echo "Scan complete for $reponame ($arch) at ${ftppath}" if [ -n "$DELETEFILES" ]; then -- cgit v1.2.2 From f8d7e57ea25a60e2dad75a7675cfee16a167f3f5 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Mon, 20 Jul 2009 18:58:08 -0400 Subject: ftpdir-cleanup: Fix the symlink restore loop Previous logic was stupid... let's just check the missing files. If it's missing and exists in the any dir, then common sense says we deleted a symlink somewhere... Signed-off-by: Aaron Griffin --- misc-scripts/ftpdir-cleanup | 29 +++++++---------------------- 1 file changed, 7 insertions(+), 22 deletions(-) diff --git a/misc-scripts/ftpdir-cleanup b/misc-scripts/ftpdir-cleanup index 7de64b9..2573cf4 100755 --- a/misc-scripts/ftpdir-cleanup +++ b/misc-scripts/ftpdir-cleanup @@ -98,32 +98,17 @@ if [ -z "$DELETEFILES$MISSINGFILES$EXTRAFILES$ARCHINDEPFILES" ]; then exit 0 fi -#Do a quick check. If we have a MISSINGFILE and an ARCHINDEPFILE of the same -#name to remove, then let's restore the symlink. - -archfiles="$ARCHINDEPFILES" -ARCHINDEPFILES="" -for af in $archfiles; do - af_satisfied=0 - for mf in $MISSINGFILES; do - if [ "${af}" = "${mf}" ]; then - ln -s "${ftppath_base}/any/${af}" "${ftppath}" - af_satisfied=1 - fi - - done - if [ $af_satisfied -eq 0 ]; then - ARCHINDEPFILES="${ARCHINDEPFILES} ${af}" - fi -done -#rescan missing files now +# Do a quick check to see if a missing ARCHINDEPFILE is in the any dir +# If it is, and the file is MISSING, restore it missfiles="$MISSINGFILES" MISSINGFILES="" for mf in $missfiles; do - if [ ! -e "${ftppath}/${mf}" ]; then - MISSINGFILES="${MISSINGFILES} ${mf}" + af_satisfied=0 + if [ -e "${ftppath_base}/any/${mf}" ]; then + echo "Restoring missing 'any' symlink: ${mf}" + ln -s "${ftppath_base}/any/${mf}" "${ftppath}" else - echo "Missing file has been restored: $mf" + MISSINGFILES="${MISSINGFILES} ${mf}" fi done -- cgit v1.2.2 From 82cfca1d3d3e9f432678d7e687c6ca91731410c1 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Mon, 20 Jul 2009 19:02:57 -0400 Subject: ftpdir-cleanup: Don't scan 'any' dirs These are handled automatically when scanning other architecutres Signed-off-by: Aaron Griffin --- cron-jobs/ftpdir-cleanup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index 77a0843..dfa968e 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -1,7 +1,7 @@ #!/bin/bash repos="core extra testing community" -arches="any i686 x86_64" +arches="i686 x86_64" LOCKFILE="/tmp/.ftpdircleanup.lock" -- cgit v1.2.2 From c912283df84f0583496c27a0935d5946ba4dd7cf Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Mon, 20 Jul 2009 19:12:31 -0400 Subject: ftpdir-cleanup: Fix ARCHINDEP parsing Failed hard on empty 'any' directories Signed-off-by: Aaron Griffin --- misc-scripts/ftpdir-cleanup | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/misc-scripts/ftpdir-cleanup b/misc-scripts/ftpdir-cleanup index 2573cf4..ca411cf 100755 --- a/misc-scripts/ftpdir-cleanup +++ b/misc-scripts/ftpdir-cleanup @@ -84,7 +84,8 @@ done if [ -d "$ftppath_base/any" ]; then cd "$ftppath_base/any" for pkg in *$PKGEXT; do - if [ ! -h $ftppath_base/i686/$pkg -a ! -h $ftppath_base/x86_64/$pkg ]; then + [ -f "$pkg" ] || continue # in case we get a file named "*.pkg.tar.gz" + if [ ! -h "$ftppath_base/i686/$pkg" -a ! -h "$ftppath_base/x86_64/$pkg" ]; then ARCHINDEPFILES="$ARCHINDEPFILES $pkg" fi done -- cgit v1.2.2 From 798a2cb73ce68ecb1068be1f5a6e93d54150f0c4 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Mon, 20 Jul 2009 19:16:53 -0400 Subject: any support: use relative symlinks Signed-off-by: Aaron Griffin --- db-move | 2 +- db-update | 2 +- misc-scripts/ftpdir-cleanup | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/db-move b/db-move index 0cc8aa4..8200810 100755 --- a/db-move +++ b/db-move @@ -112,7 +112,7 @@ if [ -d "$packagename/repos/$svnrepo_from" ]; then mv $repoto.db.tar.$DB_COMPRESSION* $ftppath_to/$architecture if [ "${_arch}" == "any" ]; then mv ${_pkgfile} $ftppath_to/any - ln -s $ftppath_to/any/${_pkgfile} $ftppath_to/$architecture/ + ln -s ../any/${_pkgfile} $ftppath_to/$architecture/ else mv ${_pkgfile} $ftppath_to/$architecture fi diff --git a/db-update b/db-update index 9d9f7eb..1c6739a 100755 --- a/db-update +++ b/db-update @@ -209,7 +209,7 @@ for current_arch in ${ARCHES[@]}; do die "error: failure while copying files to $ftppath_any" fi bf=$(basename $f) - if ! ln -s "$ftppath_any/$bf" "$ftppath/$bf"; then + if ! ln -s "../any/$bf" "$ftppath/$bf"; then die "error: failed to make link for $bf." fi done diff --git a/misc-scripts/ftpdir-cleanup b/misc-scripts/ftpdir-cleanup index ca411cf..e319b99 100755 --- a/misc-scripts/ftpdir-cleanup +++ b/misc-scripts/ftpdir-cleanup @@ -107,7 +107,7 @@ for mf in $missfiles; do af_satisfied=0 if [ -e "${ftppath_base}/any/${mf}" ]; then echo "Restoring missing 'any' symlink: ${mf}" - ln -s "${ftppath_base}/any/${mf}" "${ftppath}" + ln -s "../any/${mf}" "${ftppath}" else MISSINGFILES="${MISSINGFILES} ${mf}" fi -- cgit v1.2.2 From 8ea36dcd83222e960086bbcd114ca538d89db09c Mon Sep 17 00:00:00 2001 From: Francois Charette Date: Mon, 20 Jul 2009 19:54:23 +0200 Subject: remove trailing slash from ftppath[_any] Signed-off-by: Aaron Griffin --- db-update | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/db-update b/db-update index 1c6739a..338fb5b 100755 --- a/db-update +++ b/db-update @@ -106,8 +106,8 @@ fi for current_arch in ${ARCHES[@]}; do - ftppath="$FTP_BASE/$reponame/os/$current_arch/" - ftppath_any="$FTP_BASE/$reponame/os/any/" + ftppath="$FTP_BASE/$reponame/os/$current_arch" + ftppath_any="$FTP_BASE/$reponame/os/any" if [ ! -d "$ftppath" ]; then echo "FTP path for this repo ($reponame) is missing" @@ -197,7 +197,7 @@ for current_arch in ${ARCHES[@]}; do if [ $(/bin/ls "$WORKDIR/build/"*-$current_arch$PKGEXT 2>/dev/null | wc -l) != 0 ]; then echo "Copying new files to '$ftppath'" for f in "$WORKDIR/build/"*-$current_arch$PKGEXT; do - if ! /bin/cp "$f" "$ftppath"; then + if ! /bin/cp "$f" "$ftppath/"; then die "error: failure while copying files to $ftppath" fi done -- cgit v1.2.2 From 820452a93f68c05ac1083d56d9e560903d8ae1cf Mon Sep 17 00:00:00 2001 From: Francois Charette Date: Tue, 21 Jul 2009 11:38:09 +0200 Subject: Refactor ftpdir-cleanup to handle all arches This may look like a rather large patch, but the changes are mostly a reshuffling of the code to loop over all arches first, and then handle the arch-indep packages. The cronjob has been changed accordingly. Added new category DELETESYMLINKS, which are deleted instead of being moved to package-cleanup. I have also fixed the arch-specific issue with the ftppath, using parameters in config instead. Signed-off-by: Aaron Griffin --- config | 1 + cron-jobs/ftpdir-cleanup | 7 +- misc-scripts/ftpdir-cleanup | 267 +++++++++++++++++++++++++------------------- 3 files changed, 153 insertions(+), 122 deletions(-) diff --git a/config b/config index 425709a..16a71ce 100644 --- a/config +++ b/config @@ -1,5 +1,6 @@ FTP_BASE="/srv/ftp" +FTP_OS_SUFFIX="os" SVNREPO_core="file:///srv/svn-packages" SVNREPO_extra="file:///srv/svn-packages" diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index dfa968e..38c7fcd 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -1,13 +1,12 @@ #!/bin/bash repos="core extra testing community" -arches="i686 x86_64" LOCKFILE="/tmp/.ftpdircleanup.lock" cleanup () { rm -f "$LOCKFILE" - exit 0 + exit 0 } ctrl_c() { @@ -32,9 +31,7 @@ trap ctrl_c 2 . $(dirname $0)/../config for repo in $repos; do - for arch in $arches; do - $(dirname $0)/../misc-scripts/ftpdir-cleanup $repo $arch $CLEANUP_DESTDIR - done + $(dirname $0)/../misc-scripts/ftpdir-cleanup $repo $CLEANUP_DESTDIR done cleanup diff --git a/misc-scripts/ftpdir-cleanup b/misc-scripts/ftpdir-cleanup index e319b99..ee1dc2e 100755 --- a/misc-scripts/ftpdir-cleanup +++ b/misc-scripts/ftpdir-cleanup @@ -1,25 +1,22 @@ #!/bin/bash -if [ $# -ne 3 ]; then - echo "usage: $(basename $0) " +if [ $# -ne 2 ]; then + echo "usage: $(basename $0) " exit 1 fi reponame=$1 -arch=$2 -dest=$3 +dest=$2 -##### Arch specific stuff. TODO make this configurable ##### -ftppath_base="/srv/ftp/$reponame/os" ############################################################ -ftppath="$ftppath_base/$arch" - -if [ ! -d "$ftppath" ]; then - echo "FTP path '$ftppath' does not exist" +if [ ! -f "$(dirname $0)/../config" ]; then + echo "$(dirname $0)/../config not found! Aborting" exit 1 fi +. "$(dirname $0)/../config" + if [ ! -f /etc/makepkg.conf ]; then echo "/etc/makepkg.conf not found! Aborting" exit 1 @@ -28,140 +25,176 @@ fi . /etc/makepkg.conf getpkgname() { - local tmp - - tmp=${1##*/} - tmp=${tmp%$PKGEXT} - tmp=${tmp%-$arch} - echo ${tmp%-*-*} + local tmp + + tmp=${1##*/} + tmp=${tmp%$PKGEXT} + for arch in ${ARCHES[@]}; do + tmp=${tmp%-$arch} + done + tmp=${tmp%-any} + echo ${tmp%-*-*} } -getpkgname_ver() { - local tmp +TMPDIR=$(mktemp -d /tmp/cleanup-.XXXXXX) || exit 1 - tmp=${1##*/} - tmp=${tmp%$PKGEXT} - echo ${tmp%-$arch} -} +ftppath_base="$FTP_BASE/$reponame/$FTP_OS_SUFFIX" -MISSINGFILES="" -DELETEFILES="" -EXTRAFILES="" +for arch in ${ARCHES[@]}; do + + ftppath="$ftppath_base/$arch" + MISSINGFILES="" + DELETEFILES="" + DELETESYMLINKS="" + EXTRAFILES="" + + if [ ! -d "$ftppath" ]; then + echo "FTP path '$ftppath' does not exist" + exit 1 + fi -TMPDIR=$(mktemp -d /tmp/cleanup.XXXXXX) || exit 1 -cd "${TMPDIR}" -bsdtar xf "$ftppath/$reponame.db.tar.$DB_COMPRESSION" + cd "${TMPDIR}" + bsdtar xf "$ftppath/$reponame.db.tar.$DB_COMPRESSION" -for pkg in *; do - filename=$(grep -A1 '^%FILENAME%$' "${pkg}/desc" | tail -n1) - [ -z "${filename}" ] && filename="${pkg}${PKGEXT}" + for pkg in *; do + filename=$(grep -A1 '^%FILENAME%$' "${pkg}/desc" | tail -n1) + [ -z "${filename}" ] && filename="${pkg}${PKGEXT}" - if [ ! -e "${ftppath}/${filename}" ]; then - MISSINGFILES="${MISSINGFILES} ${filename}" - else - pkgname="$(getpkgname ${filename})" - for otherfile in ${ftppath}/${pkgname}-*; do - otherfile="$(basename ${otherfile})" - if [ "${otherfile}" != "${filename}" -a "${pkgname}" = "$(getpkgname ${otherfile})" ]; then - DELETEFILES="${DELETEFILES} ${otherfile}" + if [ ! -e "${ftppath}/${filename}" ]; then + MISSINGFILES="${MISSINGFILES} ${filename}" + else + pkgname="$(getpkgname ${filename})" + for otherfile in ${ftppath}/${pkgname}-*; do + otherfile="$(basename ${otherfile})" + if [ "${otherfile}" != "${filename}" -a "${pkgname}" = "$(getpkgname ${otherfile})" ]; then + if [ -h "${otherfile}" ]; then + DELETESYMLINKS="${DELETESYMLINKS} ${otherfile}" + else + DELETEFILES="${DELETEFILES} ${otherfile}" + fi + fi + done + fi + done + + cd "$ftppath" + for pkg in *$arch$PKGEXT; do + pkgname="$(getpkgname $pkg)" + for p in ${TMPDIR}/${pkgname}-*; do + if [ -d "${p}" -a "$(getpkgname $(basename ${p}))" = "${pkgname}" ]; then + continue 2 + fi + done + EXTRAFILES="$EXTRAFILES $pkg" + done + + cd "$ftppath" + rm -rf ${TMPDIR} + + # Do a quick check to see if a missing ARCHINDEPFILE is in the any dir + # If it is, and the file is MISSING, restore it + missfiles="$MISSINGFILES" + MISSINGFILES="" + for mf in $missfiles; do + af_satisfied=0 + if [ -e "${ftppath_base}/any/${mf}" ]; then + echo "Restoring missing 'any' symlink: ${mf}" + ln -s "../any/${mf}" "${ftppath}" + else + MISSINGFILES="${MISSINGFILES} ${mf}" fi - done + done + + echo "Scan complete for $reponame ($arch) at ${ftppath}" + + #Make sure we've done *something* before outputting anything + if [ -z "$DELETEFILES$DELETESYMLINKS$MISSINGFILES$EXTRAFILES" ]; then + continue + fi + + if [ -n "$DELETEFILES" ]; then + echo " The following files are out of date" + echo " They will be moved to '$dest'" + for f in $DELETEFILES; do + echo " $f" + done + echo "" + fi + + if [ -n "$DELETESYMLINKS" ]; then + echo " The following symlinks are out of date" + echo " They will be deleted" + for f in $DELETESYMLINKS; do + echo " $f" + done + echo "" + fi + + if [ -n "$MISSINGFILES" ]; then + echo " The following files are missing in the repo" + for f in $MISSINGFILES; do + echo " $f" + done + echo "" + fi + + if [ -n "$EXTRAFILES" ]; then + echo " The following files are in the repo but not the db" + echo " They will be moved to '$dest'" + for f in $EXTRAFILES; do + echo " $f" + done fi + + if [ -n "${DELETEFILES}" ]; then + mv ${DELETEFILES} "$dest" + echo "" + fi + + if [ -n "${DELETESYMLINKS}" ]; then + rm -f ${DELETESYMLINKS} + echo "" + fi + + if [ -n "${EXTRAFILES}" ]; then + mv ${EXTRAFILES} "$dest" + echo "" + fi + done -cd "$ftppath" -for pkg in *$arch$PKGEXT; do - pkgname="$(getpkgname $pkg)" - for p in ${TMPDIR}/${pkgname}-*; do - if [ -d "${p}" -a "$(getpkgname $(basename ${p}))" = "${pkgname}" ]; then - continue 2 - fi - done - EXTRAFILES="$EXTRAFILES $pkg" -done +ARCHINDEPFILES="" if [ -d "$ftppath_base/any" ]; then + echo "Checking arch-independent files..." cd "$ftppath_base/any" for pkg in *$PKGEXT; do [ -f "$pkg" ] || continue # in case we get a file named "*.pkg.tar.gz" + ### TODO loop over arch in ${ARCHES[@]} ... something like this: + # teststring="" + # for arch in ${ARCHES[@]}; do + # teststring="$teststring -a ! -h $ftppath_base/$arch/$pkg" + # done + # teststring=$(echo $teststring | sed 's/^ -a//') + # if [ eval "$teststring" ]; then if [ ! -h "$ftppath_base/i686/$pkg" -a ! -h "$ftppath_base/x86_64/$pkg" ]; then ARCHINDEPFILES="$ARCHINDEPFILES $pkg" fi done fi -cd "$ftppath" -rm -rf ${TMPDIR} - -#Make sure we've done *something* before outputting anything -if [ -z "$DELETEFILES$MISSINGFILES$EXTRAFILES$ARCHINDEPFILES" ]; then - exit 0 -fi - -# Do a quick check to see if a missing ARCHINDEPFILE is in the any dir -# If it is, and the file is MISSING, restore it -missfiles="$MISSINGFILES" -MISSINGFILES="" -for mf in $missfiles; do - af_satisfied=0 - if [ -e "${ftppath_base}/any/${mf}" ]; then - echo "Restoring missing 'any' symlink: ${mf}" - ln -s "../any/${mf}" "${ftppath}" - else - MISSINGFILES="${MISSINGFILES} ${mf}" - fi -done - -echo "Scan complete for $reponame ($arch) at ${ftppath}" - -if [ -n "$DELETEFILES" ]; then - echo " The following files are out of date" - echo " They will be moved to '$dest'" - for f in $DELETEFILES; do - echo " $f" - done - echo "" -fi - -if [ -n "$MISSINGFILES" ]; then - echo " The following files are missing in the repo" - for f in $MISSINGFILES; do - echo " $f" - done - echo "" -fi - -if [ -n "$EXTRAFILES" ]; then - echo " The following files are in the repo but not the db" - echo " They will be moved to '$dest'" - for f in $EXTRAFILES; do - echo " $f" - done -fi - if [ -n "$ARCHINDEPFILES" ]; then - echo " The following architecture independent packages" - echo " are not symlinked in the architecture repositories." - echo " They will be moved to '$dest'" + echo " The following architecture independent packages" + echo " are not symlinked in the architecture repositories." + echo " They will be moved to '$dest'" for f in $ARCHINDEPFILES; do echo " $f" done fi -if [ -n "${DELETEFILES}" ]; then - mv ${DELETEFILES} "$dest" - echo "" -fi - -if [ -n "${EXTRAFILES}" ]; then - mv ${EXTRAFILES} "$dest" - echo "" +if [ -d "$ftppath_base/any" -a -n "${ARCHINDEPFILES}" ]; then + cd "$ftppath_base/any" + mv ${ARCHINDEPFILES} "$dest" + echo "" fi - -cd "$ftppath_base/any" -if [ -n "${ARCHINDEPFILES}" ]; then - mv ${ARCHINDEPFILES} "$dest" - echo "" -fi - -- cgit v1.2.2 From 9cf363c8e055a5fec8bbf51f43ac8123b2fee742 Mon Sep 17 00:00:00 2001 From: Francois Charette Date: Tue, 21 Jul 2009 13:52:21 +0200 Subject: Bugfix: make TMPDIR within the for loop Signed-off-by: Aaron Griffin --- misc-scripts/ftpdir-cleanup | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/misc-scripts/ftpdir-cleanup b/misc-scripts/ftpdir-cleanup index ee1dc2e..b28ec94 100755 --- a/misc-scripts/ftpdir-cleanup +++ b/misc-scripts/ftpdir-cleanup @@ -36,12 +36,12 @@ getpkgname() { echo ${tmp%-*-*} } -TMPDIR=$(mktemp -d /tmp/cleanup-.XXXXXX) || exit 1 ftppath_base="$FTP_BASE/$reponame/$FTP_OS_SUFFIX" for arch in ${ARCHES[@]}; do + TMPDIR=$(mktemp -d /tmp/cleanup-XXXXXX) || exit 1 ftppath="$ftppath_base/$arch" MISSINGFILES="" DELETEFILES="" @@ -53,8 +53,11 @@ for arch in ${ARCHES[@]}; do exit 1 fi + if ! cd "${TMPDIR}" ; then + echo "Failed to cd to ${TMPDIR}" + exit 1 + fi - cd "${TMPDIR}" bsdtar xf "$ftppath/$reponame.db.tar.$DB_COMPRESSION" for pkg in *; do -- cgit v1.2.2 From 868df42ab97baecb7032fc576f3595dfc02674c5 Mon Sep 17 00:00:00 2001 From: Francois Charette Date: Tue, 21 Jul 2009 14:00:51 +0200 Subject: Very minor additions Signed-off-by: Aaron Griffin --- misc-scripts/ftpdir-cleanup | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/misc-scripts/ftpdir-cleanup b/misc-scripts/ftpdir-cleanup index b28ec94..fc99c46 100755 --- a/misc-scripts/ftpdir-cleanup +++ b/misc-scripts/ftpdir-cleanup @@ -113,6 +113,7 @@ for arch in ${ARCHES[@]}; do #Make sure we've done *something* before outputting anything if [ -z "$DELETEFILES$DELETESYMLINKS$MISSINGFILES$EXTRAFILES" ]; then + echo "(nothing to do for $arch)" continue fi @@ -180,7 +181,7 @@ if [ -d "$ftppath_base/any" ]; then # teststring="$teststring -a ! -h $ftppath_base/$arch/$pkg" # done # teststring=$(echo $teststring | sed 's/^ -a//') - # if [ eval "$teststring" ]; then + # if [ $teststring ]; then if [ ! -h "$ftppath_base/i686/$pkg" -a ! -h "$ftppath_base/x86_64/$pkg" ]; then ARCHINDEPFILES="$ARCHINDEPFILES $pkg" fi -- cgit v1.2.2 From f387616777b879a0dfbe801fb609763f72fa0414 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Tue, 21 Jul 2009 18:56:13 +0200 Subject: Add support for split packages to db-move All split packages are treated as unit and can only be moved together. For split packages the pkgbase value has to be used to find the corresponding entry in the svn repository. Note: different architecures (e.g. any) is not supported by makepkg. Signed-off-by: Aaron Griffin --- db-move | 60 ++++++++++++++++++++++++++++++++++++------------------------ 1 file changed, 36 insertions(+), 24 deletions(-) diff --git a/db-move b/db-move index 8200810..be46dc8 100755 --- a/db-move +++ b/db-move @@ -1,8 +1,7 @@ #!/bin/bash -# Originally from Pierre's testing2extra script if [ $# -ne 4 ]; then - echo "usage: $(basename $0) " + echo "usage: $(basename $0) " exit 1 fi @@ -10,7 +9,7 @@ fi source_makepkg -packagename="$1" +packagebase="$1" repofrom="$2" repoto="$3" _arch="$4" @@ -62,24 +61,26 @@ cd "$WORKDIR" /usr/bin/svn checkout -N $svnpath checkout cd checkout -/usr/bin/svn up -q $packagename -if [ -d "$packagename/repos/$svnrepo_from" ]; then - . "$packagename/repos/$svnrepo_from/$BUILDSCRIPT" - _pkgfile="$pkgname-$pkgver-$pkgrel-$_arch$PKGEXT" - - if [ ! -f "$ftppath_from/${_arch}/$_pkgfile" ]; then - die "error: package file '$_pkgfile' not found in repo '$repofrom'" - fi - - if [ -d "$packagename/repos/$svnrepo_to" ]; then +/usr/bin/svn up -q $packagebase +if [ -d "$packagebase/repos/$svnrepo_from" ]; then + . "$packagebase/repos/$svnrepo_from/$BUILDSCRIPT" + + for i in ${pkgname[@]}; do + _pkgfile="$i-$pkgver-$pkgrel-$_arch$PKGEXT" + if [ ! -f "$ftppath_from/${_arch}/$_pkgfile" ]; then + die "error: package file '$_pkgfile' not found in repo '$repofrom'" + fi + done + + if [ -d "$packagebase/repos/$svnrepo_to" ]; then echo " Removing existing package from subversion" - /usr/bin/svn rm --force -q "$packagename/repos/$svnrepo_to" - /usr/bin/svn commit -q -m "$(basename $0): $packagename removed by $(id -un) for move to $repoto" + /usr/bin/svn rm --force -q "$packagebase/repos/$svnrepo_to" + /usr/bin/svn commit -q -m "$(basename $0): $packagebase removed by $(id -un) for move to $repoto" fi echo " Moving svn entries" - /usr/bin/svn mv -r HEAD "$packagename/repos/$svnrepo_from" "$packagename/repos/$svnrepo_to" - /usr/bin/svn commit -m "$(basename $0): moved $packagename from [$repofrom] to [$repoto] ($_arch)" + /usr/bin/svn mv -r HEAD "$packagebase/repos/$svnrepo_from" "$packagebase/repos/$svnrepo_to" + /usr/bin/svn commit -m "$(basename $0): moved $packagebase from [$repofrom] to [$repoto] ($_arch)" echo " Moving package file and updating DBs" cd "$WORKDIR" @@ -96,7 +97,9 @@ if [ -d "$packagename/repos/$svnrepo_from" ]; then # copy the db file into our working area if [ -f "$ftppath_from/$architecture/$repofrom.db.tar.$DB_COMPRESSION" ]; then /bin/cp "$ftppath_from/$architecture/$repofrom.db.tar.$DB_COMPRESSION" . - /usr/bin/repo-remove -q "$repofrom.db.tar.$DB_COMPRESSION" $packagename || die "Error in repo-remove" + for i in ${pkgname[@]}; do + /usr/bin/repo-remove -q "$repofrom.db.tar.$DB_COMPRESSION" $i || die "Error in repo-remove $i" + done #use '*' to move the old DB too mv $repofrom.db.tar.$DB_COMPRESSION* "$ftppath_from/$architecture" echo " Package files will be cleaned up automatically" @@ -106,19 +109,28 @@ if [ -d "$packagename/repos/$svnrepo_from" ]; then /bin/cp "$ftppath_to/$architecture/$repoto.db.tar.$DB_COMPRESSION" . fi - /bin/cp "$ftppath_from/$architecture/$_pkgfile" . - /usr/bin/repo-add -q "$repoto.db.tar.$DB_COMPRESSION" $_pkgfile || die "Error in repo-add" + for i in ${pkgname[@]}; do + _pkgfile="$i-$pkgver-$pkgrel-$_arch$PKGEXT" + /bin/cp "$ftppath_from/$architecture/$_pkgfile" . + /usr/bin/repo-add -q "$repoto.db.tar.$DB_COMPRESSION" $_pkgfile || die "Error in repo-add $_pkgfile" + done #use '*' to move the old DB too mv $repoto.db.tar.$DB_COMPRESSION* $ftppath_to/$architecture if [ "${_arch}" == "any" ]; then - mv ${_pkgfile} $ftppath_to/any - ln -s ../any/${_pkgfile} $ftppath_to/$architecture/ + for i in ${pkgname[@]}; do + _pkgfile="$i-$pkgver-$pkgrel-$_arch$PKGEXT" + mv ${_pkgfile} $ftppath_to/any + ln -s ../any/${_pkgfile} $ftppath_to/$architecture/ + done else - mv ${_pkgfile} $ftppath_to/$architecture + for i in ${pkgname[@]}; do + _pkgfile="$i-$pkgver-$pkgrel-$_arch$PKGEXT" + mv ${_pkgfile} $ftppath_to/$architecture + done fi done else - die "Error: $packagename is not in repo $repofrom" + die "Error: $packagebase is not in repo $repofrom" fi cleanup -- cgit v1.2.2 From 0691b6632e612818c52dbecfd2635fd954d42e23 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Tue, 21 Jul 2009 19:19:22 +0200 Subject: add support for split packages to db-remove Signed-off-by: Aaron Griffin --- db-remove | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/db-remove b/db-remove index 4763f3d..621154d 100755 --- a/db-remove +++ b/db-remove @@ -1,7 +1,7 @@ #!/bin/bash if [ $# -ne 3 ]; then - echo "usage: $(basename $0) " + echo "usage: $(basename $0) " exit 1 fi @@ -9,7 +9,7 @@ fi source_makepkg -packagename="$1" +packagebase="$1" reponame="$2" arch="$3" @@ -49,17 +49,23 @@ repo_lock $reponame $arch /bin/mkdir -p "$WORKDIR" -echo "==> Removing package '$packagename' from '$reponame'..." >&2 +echo "==> Removing package '$packagebase' from '$reponame'..." >&2 cd "$WORKDIR" /usr/bin/svn checkout -N $svnpath checkout cd checkout -/usr/bin/svn up -q $packagename -if [ -d "$packagename/repos/$svnrepo" ]; then +/usr/bin/svn up -q $packagebase +if [ -d "$packagebase/repos/$svnrepo" ]; then echo " Removing from subversion" - /usr/bin/svn rm --force -q "$packagename/repos/$svnrepo" - /usr/bin/svn commit -q -m "$(basename $0): $packagename removed by $(id -un)" + . "$packagebase/repos/$svnrepo/$BUILDSCRIPT" + /usr/bin/svn rm --force -q "$packagebase/repos/$svnrepo" + /usr/bin/svn commit -q -m "$(basename $0): $packagebase removed by $(id -un)" +else + echo " Warning: $packagebase not found in $svnrepo" + echo " Removing split packages is not supported" + echo " You need to specify each sub package instead" + pkgname=$packagebase fi cd "$WORKDIR" @@ -82,7 +88,7 @@ for architecture in $arches; do echo " Removing from $reponame DB file ($architecture)" cd build/ - /usr/bin/repo-remove -q "$reponame.db.tar.$DB_COMPRESSION" $packagename + /usr/bin/repo-remove -q "$reponame.db.tar.$DB_COMPRESSION" ${pkgname[@]} /bin/mv "$reponame.db.tar.$DB_COMPRESSION" "$ftppath/$architecture" -- cgit v1.2.2 From 8df23b9ed44bfffbf84522d4e49b81edb4a72b41 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Tue, 21 Jul 2009 12:53:39 -0700 Subject: db-move: reduce usage of loops Signed-off-by: Aaron Griffin --- db-move | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/db-move b/db-move index be46dc8..4dbec07 100755 --- a/db-move +++ b/db-move @@ -97,9 +97,7 @@ if [ -d "$packagebase/repos/$svnrepo_from" ]; then # copy the db file into our working area if [ -f "$ftppath_from/$architecture/$repofrom.db.tar.$DB_COMPRESSION" ]; then /bin/cp "$ftppath_from/$architecture/$repofrom.db.tar.$DB_COMPRESSION" . - for i in ${pkgname[@]}; do - /usr/bin/repo-remove -q "$repofrom.db.tar.$DB_COMPRESSION" $i || die "Error in repo-remove $i" - done + /usr/bin/repo-remove -q "$repofrom.db.tar.$DB_COMPRESSION" ${pkgname[@]} || die "Error in repo-remove" #use '*' to move the old DB too mv $repofrom.db.tar.$DB_COMPRESSION* "$ftppath_from/$architecture" echo " Package files will be cleaned up automatically" @@ -116,18 +114,15 @@ if [ -d "$packagebase/repos/$svnrepo_from" ]; then done #use '*' to move the old DB too mv $repoto.db.tar.$DB_COMPRESSION* $ftppath_to/$architecture - if [ "${_arch}" == "any" ]; then - for i in ${pkgname[@]}; do - _pkgfile="$i-$pkgver-$pkgrel-$_arch$PKGEXT" - mv ${_pkgfile} $ftppath_to/any - ln -s ../any/${_pkgfile} $ftppath_to/$architecture/ - done - else - for i in ${pkgname[@]}; do - _pkgfile="$i-$pkgver-$pkgrel-$_arch$PKGEXT" - mv ${_pkgfile} $ftppath_to/$architecture - done - fi + for i in ${pkgname[@]}; do + _pkgfile="$i-$pkgver-$pkgrel-$_arch$PKGEXT" + if [ "${_arch}" == "any" ]; then + mv ${_pkgfile} $ftppath_to/any + ln -s ../any/${_pkgfile} $ftppath_to/$architecture/ + else + mv ${_pkgfile} $ftppath_to/$architecture + fi + done done else die "Error: $packagebase is not in repo $repofrom" -- cgit v1.2.2 From fc49cf28729722a2e617f3347f70c542f834d82d Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Tue, 21 Jul 2009 18:39:48 -0400 Subject: db-update: Fix final repo_unlock Because we lock/unlock in a loop, simply remove the cleanup trap at the end of the process, and manually clean up the work dir Signed-off-by: Aaron Griffin --- db-update | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/db-update b/db-update index 338fb5b..525c21e 100755 --- a/db-update +++ b/db-update @@ -46,7 +46,6 @@ fi cleanup() { trap '' 0 2 - # unlock repo_unlock $reponame $current_arch rm -rf "$WORKDIR" [ "$1" ] && exit $1 @@ -54,7 +53,7 @@ cleanup() { ctrl_c() { echo "Interrupted" >&2 - cleanup 0 + cleanup 1 } die() { @@ -233,5 +232,6 @@ if [ -n "$to_add_any" ]; then /bin/rm $to_add_any fi +trap '' 0 2 rm -rf $WORKDIR # vim: set ts=4 sw=4 noet ft=sh: -- cgit v1.2.2 From e4346ab8404ee32a69848d4f55bb0d09664c7d75 Mon Sep 17 00:00:00 2001 From: Francois Charette Date: Tue, 21 Jul 2009 18:43:12 -0400 Subject: db-update: Move some steps inside an 'if' [Aaron: keep WORKDIR creation outside the loop, as its removed at the end] Signed-off-by: Aaron Griffin --- db-update | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/db-update b/db-update index 525c21e..ade7a39 100755 --- a/db-update +++ b/db-update @@ -69,15 +69,17 @@ if [ -d "$stagedir" ]; then ANYPKGS="$(/bin/ls $stagedir/*-any$PKGEXT 2>/dev/null)" fi -pkgtotal=$(echo "$ANYPKGS" | wc -w) -echo "==> Processing $pkgtotal new/updated arch-independent packages for '$reponame'..." >&2 mkdir -p $WORKDIR cd "$WORKDIR" -svnpath="$(get_svnpath $reponame)" -/usr/bin/svn checkout -N $svnpath checkout -cd checkout -to_add_any="" + if [ -n "$ANYPKGS" ]; then + pkgtotal=$(echo "$ANYPKGS" | wc -w) + echo "==> Processing $pkgtotal new/updated arch-independent packages for '$reponame'..." >&2 + + svnpath="$(get_svnpath $reponame)" + /usr/bin/svn checkout -N $svnpath checkout + cd checkout + to_add_any="" for pkg in $ANYPKGS; do _pkgfile=$(basename $pkg) _pkgname="$(getpkgname $pkg)" -- cgit v1.2.2 From 94c1ba6eed3fbdda8be0b7c67257d30dcabfd5c6 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Wed, 22 Jul 2009 20:02:37 +0200 Subject: remove support for different svn repos Signed-off-by: Aaron Griffin --- config | 5 +---- db-functions | 14 -------------- db-move | 7 +------ db-remove | 3 +-- db-update | 6 ++---- testing2x | 3 +-- 6 files changed, 6 insertions(+), 32 deletions(-) diff --git a/config b/config index 16a71ce..a8748c7 100644 --- a/config +++ b/config @@ -2,10 +2,7 @@ FTP_BASE="/srv/ftp" FTP_OS_SUFFIX="os" -SVNREPO_core="file:///srv/svn-packages" -SVNREPO_extra="file:///srv/svn-packages" -SVNREPO_testing="file:///srv/svn-packages" -SVNREPO_community="file:///srv/svn-community" +SVNREPO="file:///srv/svn-packages" CLEANUP_DESTDIR="/srv/package-cleanup" diff --git a/db-functions b/db-functions index d183373..bbb6abc 100644 --- a/db-functions +++ b/db-functions @@ -119,19 +119,5 @@ check_pkg_arch () { #check_pkg_arch pkgfile arch fi } -get_svnpath () { #get_svnpath reponame - local var - local repopath - var="\$SVNREPO_${1}" - - eval repopath=${var} - if [ -z "$repopath" ]; then - echo "ERROR: SVN path not defined for '${1}'" >&2 - echo " Please check SVNREPO_${1} config setting" >&2 - fi - - echo "$repopath" -} - # vim: set ts=4 sw=4 noet ft=sh: diff --git a/db-move b/db-move index 4dbec07..e3897e3 100755 --- a/db-move +++ b/db-move @@ -20,11 +20,6 @@ ftppath_from="$FTP_BASE/$repofrom/os/" ftppath_to="$FTP_BASE/$repoto/os/" svnrepo_from="$repofrom-$_arch" svnrepo_to="$repoto-$_arch" -svnpath="$(get_svnpath $repoto)" -if [ "$svnpath" != "$(get_svnpath $repofrom)" ]; then - echo "ERROR: Cannot move packages across SVN repos" - echo " A move must be within the same svn repo" -fi [ "$UID" = "" ] && UID=$(uid) @@ -58,7 +53,7 @@ repo_lock $repofrom $_arch /bin/mkdir -p "$WORKDIR" cd "$WORKDIR" -/usr/bin/svn checkout -N $svnpath checkout +/usr/bin/svn checkout -N $SVNREPO checkout cd checkout /usr/bin/svn up -q $packagebase diff --git a/db-remove b/db-remove index 621154d..bbf8044 100755 --- a/db-remove +++ b/db-remove @@ -17,7 +17,6 @@ export CARCH="$arch" ftppath="$FTP_BASE/$reponame/os" svnrepo="$reponame-$arch" -svnpath="$(get_svnpath $reponame)" [ "$UID" = "" ] && UID=$(uid) @@ -52,7 +51,7 @@ repo_lock $reponame $arch echo "==> Removing package '$packagebase' from '$reponame'..." >&2 cd "$WORKDIR" -/usr/bin/svn checkout -N $svnpath checkout +/usr/bin/svn checkout -N $SVNREPO checkout cd checkout /usr/bin/svn up -q $packagebase diff --git a/db-update b/db-update index ade7a39..939fe56 100755 --- a/db-update +++ b/db-update @@ -76,8 +76,7 @@ if [ -n "$ANYPKGS" ]; then pkgtotal=$(echo "$ANYPKGS" | wc -w) echo "==> Processing $pkgtotal new/updated arch-independent packages for '$reponame'..." >&2 - svnpath="$(get_svnpath $reponame)" - /usr/bin/svn checkout -N $svnpath checkout + /usr/bin/svn checkout -N $SVNREPO checkout cd checkout to_add_any="" for pkg in $ANYPKGS; do @@ -117,7 +116,6 @@ for current_arch in ${ARCHES[@]}; do exit 1 fi - svnpath="$(get_svnpath $reponame)" svnrepo="$reponame-$current_arch" repo_lock $reponame $current_arch @@ -148,7 +146,7 @@ for current_arch in ${ARCHES[@]}; do echo "==> Processing $pkgtotal new/updated packages for repository '$reponame'..." >&2 cd "$WORKDIR" - /usr/bin/svn checkout -N $svnpath checkout + /usr/bin/svn checkout -N $SVNREPO checkout cd checkout if [ -n "$ADDPKGS" ]; then diff --git a/testing2x b/testing2x index 0314738..e116297 100755 --- a/testing2x +++ b/testing2x @@ -13,7 +13,6 @@ case "$0" in ;; esac WORKDIR="$TMPDIR/testing2x.$UID" -svnpath="$(get_svnpath testing)" cleanup() { trap '' 0 2 @@ -31,7 +30,7 @@ trap cleanup 0 /bin/mkdir "${WORKDIR}" cd "${WORKDIR}" -/usr/bin/svn checkout -N $svnpath checkout +/usr/bin/svn checkout -N $SVNREPO checkout cd checkout for pkg in $*; do -- cgit v1.2.2 From 6ed6f4e6a482e913b01f5dc6d0b52575614742a7 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Wed, 22 Jul 2009 20:55:01 -0700 Subject: Use the top level config file for cron-jobs Source our config file, and use ARCHES from it Signed-off-by: Aaron Griffin --- cron-jobs/createFileLists | 5 +++-- cron-jobs/sourceballs | 7 ++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/cron-jobs/createFileLists b/cron-jobs/createFileLists index 10520af..61cbc21 100755 --- a/cron-jobs/createFileLists +++ b/cron-jobs/createFileLists @@ -3,7 +3,8 @@ reposdir=/srv/ftp/ targetdir=$reposdir repos="core extra testing community" -arches="i686 x86_64" + +. $(dirname $0)/../config if [ -f /etc/makepkg.conf ]; then . /etc/makepkg.conf @@ -45,7 +46,7 @@ esac cd $reposdir for repo in $repos; do REPO_DB_FILE=${repo}.files.tar.${DB_COMPRESSION} - for arch in $arches; do + for arch in ${ARCHES[@]}; do repodir=${repo}/os/${arch}/ cached="no" diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index d07c482..27976ca 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -3,7 +3,8 @@ ftpbase="/srv/ftp" srcbase="/srv/ftp/sources" repos="core extra testing community" -arches="any i686 x86_64" + +. $(dirname $0)/../config LOCKFILE="/tmp/.sourceball.lock" @@ -40,7 +41,7 @@ getpkgname() { tmp=${1##*/} tmp=${tmp%$PKGEXT} - for a in $arches; do + for a in ${ARCHES[@]}; do tmp=${tmp%-$a} done echo ${tmp%-*-*} @@ -53,7 +54,7 @@ FAILED_PKGS="" echo "Errors occured during run:" > "$srcbase/errors.txt" for repo in $repos; do - for arch in $arches; do + for arch in ${ARCHES[@]}; do export CARCH="$arch" ftppath="$ftpbase/$repo/os/$arch" if [ ! -d "$ftppath" ]; then -- cgit v1.2.2 From ada0109d158c6f657137e7e6e8403ff32b23abd4 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Wed, 22 Jul 2009 21:54:39 -0700 Subject: ftpdir-cleanup: Minimize output Clean up some output so when we do nothing, we get no output This is preferred as the mailer will simply not send a mail in this case. Signed-off-by: Aaron Griffin --- misc-scripts/ftpdir-cleanup | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/misc-scripts/ftpdir-cleanup b/misc-scripts/ftpdir-cleanup index fc99c46..22d227a 100755 --- a/misc-scripts/ftpdir-cleanup +++ b/misc-scripts/ftpdir-cleanup @@ -109,14 +109,13 @@ for arch in ${ARCHES[@]}; do fi done - echo "Scan complete for $reponame ($arch) at ${ftppath}" - #Make sure we've done *something* before outputting anything if [ -z "$DELETEFILES$DELETESYMLINKS$MISSINGFILES$EXTRAFILES" ]; then - echo "(nothing to do for $arch)" continue fi + echo "Scan complete for $reponame ($arch) at ${ftppath}" + if [ -n "$DELETEFILES" ]; then echo " The following files are out of date" echo " They will be moved to '$dest'" @@ -171,7 +170,6 @@ done ARCHINDEPFILES="" if [ -d "$ftppath_base/any" ]; then - echo "Checking arch-independent files..." cd "$ftppath_base/any" for pkg in *$PKGEXT; do [ -f "$pkg" ] || continue # in case we get a file named "*.pkg.tar.gz" -- cgit v1.2.2 From 16ab9a2b407fa500bd31848e51200f974c3ad756 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Wed, 22 Jul 2009 21:55:29 -0700 Subject: ftpdir-cleanup: Loop over arches for 'any' links Remove the hardcoded i686/x86_64 and loop over the ARCHES array in the config file, checking for valid symlinks. If none are found, remove the 'any' package Signed-off-by: Aaron Griffin --- misc-scripts/ftpdir-cleanup | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/misc-scripts/ftpdir-cleanup b/misc-scripts/ftpdir-cleanup index 22d227a..622d7ed 100755 --- a/misc-scripts/ftpdir-cleanup +++ b/misc-scripts/ftpdir-cleanup @@ -173,14 +173,16 @@ if [ -d "$ftppath_base/any" ]; then cd "$ftppath_base/any" for pkg in *$PKGEXT; do [ -f "$pkg" ] || continue # in case we get a file named "*.pkg.tar.gz" - ### TODO loop over arch in ${ARCHES[@]} ... something like this: - # teststring="" - # for arch in ${ARCHES[@]}; do - # teststring="$teststring -a ! -h $ftppath_base/$arch/$pkg" - # done - # teststring=$(echo $teststring | sed 's/^ -a//') - # if [ $teststring ]; then - if [ ! -h "$ftppath_base/i686/$pkg" -a ! -h "$ftppath_base/x86_64/$pkg" ]; then + found=0 + #check for any existing symlinks + for arch in ${ARCHES[@]}; do + if [ -h "$ftppath_base/$arch/$pkg" ]; then + found=1 + break + fi + done + if [ $found -eq 0 ]; then + # We found no symlinks to this, delete it ARCHINDEPFILES="$ARCHINDEPFILES $pkg" fi done -- cgit v1.2.2 From fcd15f215b7cb09e74e565799dd8b73748129aae Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Mon, 3 Aug 2009 12:29:11 -0400 Subject: Remove community from stock ftpdir-cleanup community repo is cleaned up on a different machine Signed-off-by: Aaron Griffin --- cron-jobs/ftpdir-cleanup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index 38c7fcd..1a9fd98 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -1,6 +1,6 @@ #!/bin/bash -repos="core extra testing community" +repos="core extra testing" LOCKFILE="/tmp/.ftpdircleanup.lock" -- cgit v1.2.2 From e2767004a5f8796aee00ad7f44d561c4a959341b Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Tue, 4 Aug 2009 02:03:55 -0400 Subject: Add db-community Signed-off-by: Aaron Griffin --- db-community | 3 +++ 1 file changed, 3 insertions(+) create mode 100755 db-community diff --git a/db-community b/db-community new file mode 100755 index 0000000..44767a5 --- /dev/null +++ b/db-community @@ -0,0 +1,3 @@ +#!/bin/bash + +$(dirname $0)/db-update "community" -- cgit v1.2.2 From 5fa1331e0949f4cbdc6cbe0a6946e3fd185979b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20B=C3=A4chler?= Date: Sun, 16 Aug 2009 14:14:34 +0200 Subject: Avoid confusing messages and potential problems when no packages are present If no packages are present in the repository, a message like this will be generated, and the corresponding mv command will be issued. Prevent this by checking if *$arch$PKGEXT equals "*$arch$PKGEXT": Scan complete for core (i686) at /srv/ftp/core/os/i686 The following files are in the repo but not the db They will be moved to '/srv/package-cleanup' *i686.pkg.tar.gz --- misc-scripts/ftpdir-cleanup | 3 +++ 1 file changed, 3 insertions(+) diff --git a/misc-scripts/ftpdir-cleanup b/misc-scripts/ftpdir-cleanup index 622d7ed..9331a4e 100755 --- a/misc-scripts/ftpdir-cleanup +++ b/misc-scripts/ftpdir-cleanup @@ -83,6 +83,9 @@ for arch in ${ARCHES[@]}; do cd "$ftppath" for pkg in *$arch$PKGEXT; do + if [ "$pkg" = "*$arch$PKGEXT" ]; then + continue + fi pkgname="$(getpkgname $pkg)" for p in ${TMPDIR}/${pkgname}-*; do if [ -d "${p}" -a "$(getpkgname $(basename ${p}))" = "${pkgname}" ]; then -- cgit v1.2.2 From a51de5f2ba853a64fc9842e337cb25e000fdbbc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20B=C3=A4chler?= Date: Sun, 16 Aug 2009 15:45:00 +0200 Subject: Make ftpdir-cleanup NOT delete all packages if databas extraction fails --- misc-scripts/ftpdir-cleanup | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/misc-scripts/ftpdir-cleanup b/misc-scripts/ftpdir-cleanup index 9331a4e..222811b 100755 --- a/misc-scripts/ftpdir-cleanup +++ b/misc-scripts/ftpdir-cleanup @@ -58,7 +58,18 @@ for arch in ${ARCHES[@]}; do exit 1 fi - bsdtar xf "$ftppath/$reponame.db.tar.$DB_COMPRESSION" + if [ ! -f "$ftppath/$reponame.db.tar.$DB_COMPRESSION" ]; then + echo "" + echo "ERROR: The file \"$ftppath/$reponame.db.tar.$DB_COMPRESSION\" could not be found, aborting." + echo "" + exit 1 + fi + + if ! bsdtar xf "$ftppath/$reponame.db.tar.$DB_COMPRESSION"; then + echo "" + echo "ERROR: Command failed: bsdtar xf \"$ftppath/$reponame.db.tar.$DB_COMPRESSION\"" + exit 1 + fi for pkg in *; do filename=$(grep -A1 '^%FILENAME%$' "${pkg}/desc" | tail -n1) -- cgit v1.2.2 From cc7f23a6e7b978c4aa66c6288904a64abb2f06ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20B=C3=A4chler?= Date: Sun, 16 Aug 2009 15:47:01 +0200 Subject: Add DB_COMPRESSION='gz', as this has been removed from makepkg.conf --- misc-scripts/ftpdir-cleanup | 1 + 1 file changed, 1 insertion(+) diff --git a/misc-scripts/ftpdir-cleanup b/misc-scripts/ftpdir-cleanup index 222811b..fa43bb0 100755 --- a/misc-scripts/ftpdir-cleanup +++ b/misc-scripts/ftpdir-cleanup @@ -7,6 +7,7 @@ fi reponame=$1 dest=$2 +DB_COMPRESSION='gz' ############################################################ -- cgit v1.2.2 From 1d94c827c8e9efe6e56bc15f993a1ce3764a675a Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 16 Aug 2009 09:04:21 -0500 Subject: Add DB_COMPRESSION var to all scripts that need it Signed-off-by: Dan McGee --- cron-jobs/createFileLists | 1 + db-move | 1 + db-remove | 1 + db-update | 1 + 4 files changed, 4 insertions(+) diff --git a/cron-jobs/createFileLists b/cron-jobs/createFileLists index 61cbc21..2e42e56 100755 --- a/cron-jobs/createFileLists +++ b/cron-jobs/createFileLists @@ -3,6 +3,7 @@ reposdir=/srv/ftp/ targetdir=$reposdir repos="core extra testing community" +DB_COMPRESSION="gz" . $(dirname $0)/../config diff --git a/db-move b/db-move index e3897e3..1b07091 100755 --- a/db-move +++ b/db-move @@ -13,6 +13,7 @@ packagebase="$1" repofrom="$2" repoto="$3" _arch="$4" +DB_COMPRESSION='gz' export CARCH="$_arch" diff --git a/db-remove b/db-remove index bbf8044..b246a8f 100755 --- a/db-remove +++ b/db-remove @@ -12,6 +12,7 @@ source_makepkg packagebase="$1" reponame="$2" arch="$3" +DB_COMPRESSION='gz' export CARCH="$arch" diff --git a/db-update b/db-update index 939fe56..8e7a0c7 100755 --- a/db-update +++ b/db-update @@ -11,6 +11,7 @@ source_makepkg reponame="$1" current_arch="" +DB_COMPRESSION='gz' [ "$UID" = "" ] && UID=$(uid) -- cgit v1.2.2 From b82a2ec63574c5fe0a0dd49b242d8c365155b828 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Mon, 17 Aug 2009 15:12:33 -0700 Subject: Replace DB_COMPRESSION with DBEXT in the config DBEXT contains the full extension for the db files (.db.tar.gz) and is recorded in the config file Signed-off-by: Aaron Griffin --- config | 1 + db-move | 17 ++++++++--------- db-remove | 9 ++++----- db-update | 13 ++++++------- 4 files changed, 19 insertions(+), 21 deletions(-) diff --git a/config b/config index a8748c7..873c2f3 100644 --- a/config +++ b/config @@ -11,3 +11,4 @@ TMPDIR="/srv/tmp" ARCHES=(i686 x86_64) # To be used if $BUILDSCRIPT is removed from makepkg.conf #BUILDSCRIPT="PKGBUILD" +DBEXT=".db.tar.gz" diff --git a/db-move b/db-move index 1b07091..e1a50bd 100755 --- a/db-move +++ b/db-move @@ -13,7 +13,6 @@ packagebase="$1" repofrom="$2" repoto="$3" _arch="$4" -DB_COMPRESSION='gz' export CARCH="$_arch" @@ -91,25 +90,25 @@ if [ -d "$packagebase/repos/$svnrepo_from" ]; then for architecture in $arches; do # copy the db file into our working area - if [ -f "$ftppath_from/$architecture/$repofrom.db.tar.$DB_COMPRESSION" ]; then - /bin/cp "$ftppath_from/$architecture/$repofrom.db.tar.$DB_COMPRESSION" . - /usr/bin/repo-remove -q "$repofrom.db.tar.$DB_COMPRESSION" ${pkgname[@]} || die "Error in repo-remove" + if [ -f "$ftppath_from/$architecture/$repofrom$DBEXT" ]; then + /bin/cp "$ftppath_from/$architecture/$repofrom$DBEXT" . + /usr/bin/repo-remove -q "$repofrom$DBEXT" ${pkgname[@]} || die "Error in repo-remove" #use '*' to move the old DB too - mv $repofrom.db.tar.$DB_COMPRESSION* "$ftppath_from/$architecture" + mv $repofrom$DBEXT"$ftppath_from/$architecture" echo " Package files will be cleaned up automatically" fi - if [ -f "$ftppath_to/$architecture/$repoto.db.tar.$DB_COMPRESSION" ]; then - /bin/cp "$ftppath_to/$architecture/$repoto.db.tar.$DB_COMPRESSION" . + if [ -f "$ftppath_to/$architecture/$repoto$DBEXT" ]; then + /bin/cp "$ftppath_to/$architecture/$repoto$DBEXT" . fi for i in ${pkgname[@]}; do _pkgfile="$i-$pkgver-$pkgrel-$_arch$PKGEXT" /bin/cp "$ftppath_from/$architecture/$_pkgfile" . - /usr/bin/repo-add -q "$repoto.db.tar.$DB_COMPRESSION" $_pkgfile || die "Error in repo-add $_pkgfile" + /usr/bin/repo-add -q "$repoto$DBEXT" $_pkgfile || die "Error in repo-add $_pkgfile" done #use '*' to move the old DB too - mv $repoto.db.tar.$DB_COMPRESSION* $ftppath_to/$architecture + mv $repoto$DBEXT* $ftppath_to/$architecture for i in ${pkgname[@]}; do _pkgfile="$i-$pkgver-$pkgrel-$_arch$PKGEXT" if [ "${_arch}" == "any" ]; then diff --git a/db-remove b/db-remove index b246a8f..b4151c7 100755 --- a/db-remove +++ b/db-remove @@ -12,7 +12,6 @@ source_makepkg packagebase="$1" reponame="$2" arch="$3" -DB_COMPRESSION='gz' export CARCH="$arch" @@ -79,8 +78,8 @@ fi # copy the db file into our working area for architecture in $arches; do - if [ -f "$ftppath/$architecture/$reponame.db.tar.$DB_COMPRESSION" ]; then - /bin/cp "$ftppath/$architecture/$reponame.db.tar.$DB_COMPRESSION" build/ + if [ -f "$ftppath/$architecture/$reponame$DBEXT" ]; then + /bin/cp "$ftppath/$architecture/$reponame$DBEXT" build/ else echo "No database found at '$ftppath/$architecture', nothing more to do" exit 0 @@ -88,9 +87,9 @@ for architecture in $arches; do echo " Removing from $reponame DB file ($architecture)" cd build/ - /usr/bin/repo-remove -q "$reponame.db.tar.$DB_COMPRESSION" ${pkgname[@]} + /usr/bin/repo-remove -q "$reponame$DBEXT" ${pkgname[@]} - /bin/mv "$reponame.db.tar.$DB_COMPRESSION" "$ftppath/$architecture" + /bin/mv "$reponame$DBEXT" "$ftppath/$architecture" echo "Package files will be cleaned up automatically" cd .. diff --git a/db-update b/db-update index 8e7a0c7..05236fe 100755 --- a/db-update +++ b/db-update @@ -11,7 +11,6 @@ source_makepkg reponame="$1" current_arch="" -DB_COMPRESSION='gz' [ "$UID" = "" ] && UID=$(uid) @@ -125,8 +124,8 @@ for current_arch in ${ARCHES[@]}; do cd "$WORKDIR" # copy the db file into our working area - if [ -f "$ftppath/$reponame.db.tar.$DB_COMPRESSION" ]; then - /bin/cp "$ftppath/$reponame.db.tar.$DB_COMPRESSION" build/ + if [ -f "$ftppath/$reponame$DBEXT" ]; then + /bin/cp "$ftppath/$reponame$DBEXT" build/ fi echo "Updating DB for $svnrepo" @@ -139,8 +138,8 @@ for current_arch in ${ARCHES[@]}; do if [ -n "$ADDPKGS" -o -n "$ANYPKGS" ]; then echo "==> Copying DB file from '$reponame'..." >&2 - if [ -f "$ftppath/$reponame.db.tar.$DB_COMPRESSION" ]; then - /bin/cp "$ftppath/$reponame.db.tar.$DB_COMPRESSION" build/ + if [ -f "$ftppath/$reponame$DBEXT" ]; then + /bin/cp "$ftppath/$reponame$DBEXT" build/ fi pkgtotal=$(echo "$ADDPKGS $ANYPKGS" | wc -w) @@ -183,9 +182,9 @@ for current_arch in ${ARCHES[@]}; do pkgs="" for pkg in $to_add $to_add_any; do pkgs="$pkgs $(basename $pkg)"; done - /usr/bin/repo-add -q "$reponame.db.tar.$DB_COMPRESSION" $pkgs + /usr/bin/repo-add -q "$reponame$DBEXT" $pkgs else - rm -f "build/$reponame.db.tar.$DB_COMPRESSION" + rm -f "build/$reponame$DBEXT" echo "Errors found when adding packages" fi else -- cgit v1.2.2 From 68bf0be302f7650a31f6c188d99c00596b3fbef1 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Mon, 17 Aug 2009 15:13:18 -0700 Subject: Uncomment BUILDSCRIPT in the config file Signed-off-by: Aaron Griffin --- config | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/config b/config index 873c2f3..66ef7e8 100644 --- a/config +++ b/config @@ -9,6 +9,5 @@ CLEANUP_DESTDIR="/srv/package-cleanup" STAGING="$HOME/staging" TMPDIR="/srv/tmp" ARCHES=(i686 x86_64) -# To be used if $BUILDSCRIPT is removed from makepkg.conf -#BUILDSCRIPT="PKGBUILD" +BUILDSCRIPT="PKGBUILD" DBEXT=".db.tar.gz" -- cgit v1.2.2 From 15de56aa8bb8104a01a983393f8a398348714a3a Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Mon, 17 Aug 2009 15:20:27 -0700 Subject: Remove dependence on makepkg.conf Removed the need for makepkg.conf * Killed off CARCH * Added PKGEXT to the config file * Remove source_makepkg function * Source config file where makepkg.conf was needed Signed-off-by: Aaron Griffin --- config | 1 + convert-to-any | 7 +------ db-functions | 9 --------- db-move | 4 ---- db-remove | 4 ---- db-update | 2 -- testing2x | 2 -- 7 files changed, 2 insertions(+), 27 deletions(-) diff --git a/config b/config index 66ef7e8..c735a91 100644 --- a/config +++ b/config @@ -11,3 +11,4 @@ TMPDIR="/srv/tmp" ARCHES=(i686 x86_64) BUILDSCRIPT="PKGBUILD" DBEXT=".db.tar.gz" +PKGEXT=".pkg.tar.gz" diff --git a/convert-to-any b/convert-to-any index 2ad1521..e21b7cd 100755 --- a/convert-to-any +++ b/convert-to-any @@ -15,12 +15,7 @@ if [ $# -ne 1 ]; then exit 1 fi -if [ -f /etc/makepkg.conf ]; then - . /etc/makepkg.conf -else - echo "E: /etc/makepkg.conf does not exist." - exit 1 -fi +. "$(dirname $0)/db-functions" cleanup() { trap '' 0 2 diff --git a/db-functions b/db-functions index bbb6abc..ccab830 100644 --- a/db-functions +++ b/db-functions @@ -12,15 +12,6 @@ if [ -f "$BASEDIR/config" ]; then fi # Useful functions -source_makepkg () { - if [ -f "/etc/makepkg.conf" ]; then - #Get some config info - . /etc/makepkg.conf - else - echo "error: /etc/makepkg.conf does not exist!" - exit 1 - fi -} UMASK="" set_umask () { diff --git a/db-move b/db-move index e1a50bd..de81e85 100755 --- a/db-move +++ b/db-move @@ -7,15 +7,11 @@ fi . "$(dirname $0)/db-functions" -source_makepkg - packagebase="$1" repofrom="$2" repoto="$3" _arch="$4" -export CARCH="$_arch" - ftppath_from="$FTP_BASE/$repofrom/os/" ftppath_to="$FTP_BASE/$repoto/os/" svnrepo_from="$repofrom-$_arch" diff --git a/db-remove b/db-remove index b4151c7..eae204e 100755 --- a/db-remove +++ b/db-remove @@ -7,14 +7,10 @@ fi . "$(dirname $0)/db-functions" -source_makepkg - packagebase="$1" reponame="$2" arch="$3" -export CARCH="$arch" - ftppath="$FTP_BASE/$reponame/os" svnrepo="$reponame-$arch" diff --git a/db-update b/db-update index 05236fe..7ff48ba 100755 --- a/db-update +++ b/db-update @@ -7,8 +7,6 @@ fi . "$(dirname $0)/db-functions" -source_makepkg - reponame="$1" current_arch="" diff --git a/testing2x b/testing2x index e116297..f3b256b 100755 --- a/testing2x +++ b/testing2x @@ -2,8 +2,6 @@ . "$(dirname $0)/db-functions" -source_makepkg - case "$0" in *64) _arch="x86_64" -- cgit v1.2.2 From fa9fe91849e46ea50cd32cc55539a4f0833eb642 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Mon, 17 Aug 2009 15:26:02 -0700 Subject: Fix an error in db-move due to DBEXT commit Signed-off-by: Aaron Griffin --- db-move | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db-move b/db-move index de81e85..b49aeda 100755 --- a/db-move +++ b/db-move @@ -90,7 +90,7 @@ if [ -d "$packagebase/repos/$svnrepo_from" ]; then /bin/cp "$ftppath_from/$architecture/$repofrom$DBEXT" . /usr/bin/repo-remove -q "$repofrom$DBEXT" ${pkgname[@]} || die "Error in repo-remove" #use '*' to move the old DB too - mv $repofrom$DBEXT"$ftppath_from/$architecture" + mv $repofrom$DBEXT* "$ftppath_from/$architecture" echo " Package files will be cleaned up automatically" fi -- cgit v1.2.2 From e3ee8f680b727a4662aec414febe4a8e82921a63 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Mon, 17 Aug 2009 16:03:34 -0700 Subject: Replace a string compare with a -f check No real reason, this is just easier to understand, and checks like this should be the norm. If file does not exist, continue Signed-off-by: Aaron Griffin --- misc-scripts/ftpdir-cleanup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc-scripts/ftpdir-cleanup b/misc-scripts/ftpdir-cleanup index fa43bb0..a9865c6 100755 --- a/misc-scripts/ftpdir-cleanup +++ b/misc-scripts/ftpdir-cleanup @@ -95,7 +95,7 @@ for arch in ${ARCHES[@]}; do cd "$ftppath" for pkg in *$arch$PKGEXT; do - if [ "$pkg" = "*$arch$PKGEXT" ]; then + if [ ! -e "$pkg" ]; then continue fi pkgname="$(getpkgname $pkg)" -- cgit v1.2.2 From ae9cf32fbdb54c40c9d0b338368e7cd272590329 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20B=C3=A9langer?= Date: Tue, 18 Aug 2009 15:55:28 -0700 Subject: Variable rename fix: SVN_PATH -> SVNREPO Signed-off-by: Aaron Griffin --- misc-scripts/make-sourceball | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc-scripts/make-sourceball b/misc-scripts/make-sourceball index e24af2d..b3bd3d4 100755 --- a/misc-scripts/make-sourceball +++ b/misc-scripts/make-sourceball @@ -139,7 +139,7 @@ set_umask /bin/mkdir -p "$logpath" cd "$WORKDIR" -if /usr/bin/svn export -q "$SVN_PATH/$packagename" $packagename; then +if /usr/bin/svn export -q "$SVNREPO/$packagename" $packagename; then remove_old "$pkgname/repos/" create_srcpackage "$packagename/repos/$reponame-$_arch" else -- cgit v1.2.2 From 37c975929d6a60b21ff9741329bc72e9523f376c Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Tue, 18 Aug 2009 16:00:20 -0700 Subject: Remove find-dupes script (unused) Signed-off-by: Aaron Griffin --- misc-scripts/find-dupes | 77 ------------------------------------------------- 1 file changed, 77 deletions(-) delete mode 100755 misc-scripts/find-dupes diff --git a/misc-scripts/find-dupes b/misc-scripts/find-dupes deleted file mode 100755 index 16bf530..0000000 --- a/misc-scripts/find-dupes +++ /dev/null @@ -1,77 +0,0 @@ -#!/bin/bash - -if [ $# -ne 2 ]; then - echo "usage: $(basename $0) " - exit 1 -fi - -reponame=$1 -arch=$2 - -##### Arch specific stuff. TODO make this configurable ##### -ftppath="/srv/ftp/$reponame/os/$arch/" -############################################################ - -if [ ! -d "$ftppath" ]; then - echo "FTP path '$ftppath' does not exist" - exit 1 -fi - -if [ ! -f /etc/makepkg.conf ]; then - echo "/etc/makepkg.conf not found! Aborting" - exit 1 -fi - -. /etc/makepkg.conf - -cd $ftppath - -# Get the package name from the filename -# hackish, but should work for now -getpkgname() { - local tmp - - tmp=${1##*/} - tmp=${tmp%$PKGEXT} - tmp=${tmp%-$CARCH} - echo ${tmp%-*-*} -} - -showdupes() { - done="" - for i in *${PKGEXT}; do - pkgname=$(getpkgname $i) - if [ "$pkgname" = "$1" ]; then - lastmod="$(stat -c %y $i | cut -d. -f1)" - fname="$(basename $i)" - - echo "$lastmod $fname" - - done=1 - else - if [ "$done" = "1" ]; then - return - fi - fi - done -} - -echo "Scanning for duplicate packages in '$reponame' ($arch)" -DUPES="" -lastpkg="" - -for pkg in *${PKGEXT}; do - pkgname="$(getpkgname $pkg)" - if [ "$lastpkg" = "$pkgname" ]; then - DUPES="$DUPES $pkgname" - fi - lastpkg=$pkgname -done - -if [ "$DUPES" ]; then - DUPES="$(echo $DUPES | sed 's| |\n|g' | sort -u)" - echo "Date Filename" - for dupe in $DUPES; do - showdupes $dupe - done -fi -- cgit v1.2.2 From 6666d28f019b1b7ea38f9bf1f4371a87d42fb927 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Tue, 18 Aug 2009 16:01:54 -0700 Subject: More makepkg.conf removal - prefer toplevel config Signed-off-by: Aaron Griffin --- cron-jobs/createFileLists | 7 ------- cron-jobs/sourceballs | 6 ------ misc-scripts/ftpdir-cleanup | 7 ------- misc-scripts/make-sourceball | 6 ++---- 4 files changed, 2 insertions(+), 24 deletions(-) diff --git a/cron-jobs/createFileLists b/cron-jobs/createFileLists index 2e42e56..610f0e1 100755 --- a/cron-jobs/createFileLists +++ b/cron-jobs/createFileLists @@ -7,13 +7,6 @@ DB_COMPRESSION="gz" . $(dirname $0)/../config -if [ -f /etc/makepkg.conf ]; then - . /etc/makepkg.conf -else - echo "Error: /etc/makepkg.conf does not exist." - exit 1 -fi - if [ -f "/tmp/createFileList.lock" ]; then echo "Error: createFileList allready in progress." exit 1 diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 27976ca..a34ae2d 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -8,12 +8,6 @@ repos="core extra testing community" LOCKFILE="/tmp/.sourceball.lock" -if [ ! -f /etc/makepkg.conf ]; then - echo "/etc/makepkg.conf not found! Aborting" - exit 1 -fi - -. /etc/makepkg.conf cleanup () { rm -f "$LOCKFILE" exit 0 diff --git a/misc-scripts/ftpdir-cleanup b/misc-scripts/ftpdir-cleanup index a9865c6..f64acfa 100755 --- a/misc-scripts/ftpdir-cleanup +++ b/misc-scripts/ftpdir-cleanup @@ -18,13 +18,6 @@ fi . "$(dirname $0)/../config" -if [ ! -f /etc/makepkg.conf ]; then - echo "/etc/makepkg.conf not found! Aborting" - exit 1 -fi - -. /etc/makepkg.conf - getpkgname() { local tmp diff --git a/misc-scripts/make-sourceball b/misc-scripts/make-sourceball index b3bd3d4..bd27e86 100755 --- a/misc-scripts/make-sourceball +++ b/misc-scripts/make-sourceball @@ -9,10 +9,8 @@ if [ $# -ne 3 -a $# -ne 4 ]; then exit 1 fi -BASEDIR="$(dirname $0)/../" -. "$BASEDIR/db-functions" - -source_makepkg +. "$(dirname $0)/../config" +. "$(dirname $0)/../db-functions" FORCE=0 if [ "$1" = "-f" ]; then -- cgit v1.2.2 From 49fb4d75ea83dca2ed4653e8454c478c1fd0555a Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Tue, 18 Aug 2009 16:11:03 -0700 Subject: More removal of DB_COMPRESSION from cron jobs Remove dependance on makepkg.conf Signed-off-by: Aaron Griffin --- cron-jobs/createFileLists | 17 +++++++++-------- misc-scripts/ftpdir-cleanup | 16 +++++----------- 2 files changed, 14 insertions(+), 19 deletions(-) diff --git a/cron-jobs/createFileLists b/cron-jobs/createFileLists index 610f0e1..4ad4c22 100755 --- a/cron-jobs/createFileLists +++ b/cron-jobs/createFileLists @@ -3,7 +3,6 @@ reposdir=/srv/ftp/ targetdir=$reposdir repos="core extra testing community" -DB_COMPRESSION="gz" . $(dirname $0)/../config @@ -30,16 +29,18 @@ getpkgname() { echo $tmp } -case "${DB_COMPRESSION}" in - gz) TAR_OPT="z" ;; - bz2) TAR_OPT="j" ;; - xz) TAR_OPT="J" ;; - *) echo "${DB_COMPRESSION} is not a valid archive compression" && exit 1 ;; +case "${DBEXT}" in + *.gz) TAR_OPT="z" ;; + *.bz2) TAR_OPT="j" ;; + *.xz) TAR_OPT="J" ;; + *) echo "Unknown compression type for DBEXT=${DBEXT}" && exit 1 ;; esac +FILESEXT="${DBEXT//db/files}" + cd $reposdir for repo in $repos; do - REPO_DB_FILE=${repo}.files.tar.${DB_COMPRESSION} + REPO_DB_FILE=${repo}$FILESEXT for arch in ${ARCHES[@]}; do repodir=${repo}/os/${arch}/ cached="no" @@ -77,7 +78,7 @@ for repo in $repos; do cd ${TMPDIR}/${repodir} [ -f "${pkgdir}${REPO_DB_FILE}.old" ] && rm "${pkgdir}${REPO_DB_FILE}.old" [ -f "${pkgdir}${REPO_DB_FILE}" ] && mv "${pkgdir}${REPO_DB_FILE}" "${pkgdir}${REPO_DB_FILE}.old" - bsdtar --exclude=*.tar.${DB_COMPRESSION} -c${TAR_OPT}f ${pkgdir}${REPO_DB_FILE} * + bsdtar --exclude=*${DBEXT//\.db/} -c${TAR_OPT}f ${pkgdir}${REPO_DB_FILE} * fi cd $reposdir diff --git a/misc-scripts/ftpdir-cleanup b/misc-scripts/ftpdir-cleanup index f64acfa..96407c0 100755 --- a/misc-scripts/ftpdir-cleanup +++ b/misc-scripts/ftpdir-cleanup @@ -7,16 +7,10 @@ fi reponame=$1 dest=$2 -DB_COMPRESSION='gz' ############################################################ -if [ ! -f "$(dirname $0)/../config" ]; then - echo "$(dirname $0)/../config not found! Aborting" - exit 1 -fi - -. "$(dirname $0)/../config" +. "$(dirname $0)/../db-functions" getpkgname() { local tmp @@ -52,16 +46,16 @@ for arch in ${ARCHES[@]}; do exit 1 fi - if [ ! -f "$ftppath/$reponame.db.tar.$DB_COMPRESSION" ]; then + if [ ! -f "$ftppath/$reponame$DBEXT" ]; then echo "" - echo "ERROR: The file \"$ftppath/$reponame.db.tar.$DB_COMPRESSION\" could not be found, aborting." + echo "ERROR: The file \"$ftppath/$reponame$DBEXT\" could not be found, aborting." echo "" exit 1 fi - if ! bsdtar xf "$ftppath/$reponame.db.tar.$DB_COMPRESSION"; then + if ! bsdtar xf "$ftppath/$reponame$DBEXT"; then echo "" - echo "ERROR: Command failed: bsdtar xf \"$ftppath/$reponame.db.tar.$DB_COMPRESSION\"" + echo "ERROR: Command failed: bsdtar xf \"$ftppath/$reponame$DBEXT\"" exit 1 fi -- cgit v1.2.2 From c7c797f0863c41f1e7ef9e17b408dab84ef493e6 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Tue, 18 Aug 2009 16:12:59 -0700 Subject: Use db-functions file everywhere This source the config file and gets us functions such as getpkgname which were duplicated elsewhere Signed-off-by: Aaron Griffin --- cron-jobs/createFileLists | 13 +------------ cron-jobs/ftpdir-cleanup | 2 +- cron-jobs/sourceballs | 14 +------------- misc-scripts/make-sourceball | 1 - 4 files changed, 3 insertions(+), 27 deletions(-) diff --git a/cron-jobs/createFileLists b/cron-jobs/createFileLists index 4ad4c22..345f9aa 100755 --- a/cron-jobs/createFileLists +++ b/cron-jobs/createFileLists @@ -4,7 +4,7 @@ reposdir=/srv/ftp/ targetdir=$reposdir repos="core extra testing community" -. $(dirname $0)/../config +. "$(dirname $0)/../db-functions" if [ -f "/tmp/createFileList.lock" ]; then echo "Error: createFileList allready in progress." @@ -18,17 +18,6 @@ CACHEDIR=$(mktemp -d /tmp/createFileList.XXXXXX) || exit 1 #adjust the nice level to run at a lower priority /usr/bin/renice +10 -p $$ > /dev/null -getpkgname() { - local tmp - - tmp=${1##*/} - tmp=${tmp%$PKGEXT} - tmp=${tmp%-i686} - tmp=${tmp%-x86_64} - tmp=${tmp%-any} - echo $tmp -} - case "${DBEXT}" in *.gz) TAR_OPT="z" ;; *.bz2) TAR_OPT="j" ;; diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index 1a9fd98..04caf93 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -28,7 +28,7 @@ trap ctrl_c 2 /usr/bin/renice +10 -p $$ > /dev/null #Get our destination dir -. $(dirname $0)/../config +. "$(dirname $0)/../db-functions" for repo in $repos; do $(dirname $0)/../misc-scripts/ftpdir-cleanup $repo $CLEANUP_DESTDIR diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index a34ae2d..02ebc66 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -4,7 +4,7 @@ ftpbase="/srv/ftp" srcbase="/srv/ftp/sources" repos="core extra testing community" -. $(dirname $0)/../config +. "$(dirname $0)/../db-functions" LOCKFILE="/tmp/.sourceball.lock" @@ -30,18 +30,6 @@ trap ctrl_c 2 dirname="$(/bin/readlink -f $(/usr/bin/dirname $0))" -getpkgname() { - local tmp - - tmp=${1##*/} - tmp=${tmp%$PKGEXT} - for a in ${ARCHES[@]}; do - tmp=${tmp%-$a} - done - echo ${tmp%-*-*} -} - - FAILED_PKGS="" [ -e "$srcbase/errors.txt" ] && /bin/mv "$srcbase/errors.txt" "$srcbase/errors.txt.old" diff --git a/misc-scripts/make-sourceball b/misc-scripts/make-sourceball index bd27e86..f93ab2a 100755 --- a/misc-scripts/make-sourceball +++ b/misc-scripts/make-sourceball @@ -9,7 +9,6 @@ if [ $# -ne 3 -a $# -ne 4 ]; then exit 1 fi -. "$(dirname $0)/../config" . "$(dirname $0)/../db-functions" FORCE=0 -- cgit v1.2.2 From b40d217f2baf30f5d7c5979618ff940e43636007 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Tue, 18 Aug 2009 16:18:22 -0700 Subject: Replace pkgname with pkgbase, for split packages Signed-off-by: Aaron Griffin --- cron-jobs/sourceballs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 02ebc66..4353935 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -45,26 +45,26 @@ for repo in $repos; do fi cd $ftppath for pkg in *$PKGEXT; do - pkgname=$(getpkgname $pkg) + pkgbase=$(getpkgbase $pkg) srcpath="$srcbase/" srcpkg="${pkg//$PKGEXT/$SRCEXT}" srcpkg="${srcpkg//-$arch/}" #Don't do anything for package in this 'blacklist' - if grep "^$pkgname\$" "$dirname/sourceballs.skip" >/dev/null 2>&1; then + if grep "^$pkgbase\$" "$dirname/sourceballs.skip" >/dev/null 2>&1; then continue fi #Use this file to 'whitelist' or force building some sourceballs, # skipping the license check force="" - if grep "^$pkgname\$" "$dirname/sourceballs.force" >/dev/null 2>&1; then + if grep "^$pkgbase\$" "$dirname/sourceballs.force" >/dev/null 2>&1; then force="-f" fi if [ ! -f "$srcpath$srcpkg" ]; then if ! $dirname/../misc-scripts/make-sourceball $force \ - $pkgname $repo $arch 2>>"$srcbase/errors.txt"; then + $pkgbase $repo $arch 2>>"$srcbase/errors.txt"; then FAILED_PKGS="$FAILED_PKGS $pkg" fi fi -- cgit v1.2.2 From a4d0a6cba591e0866c249a0c2014297bdbbfb779 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20B=C3=A9langer?= Date: Tue, 18 Aug 2009 16:37:09 -0700 Subject: Fix arch parameter handling in db-remove db-remove use the arch variable to handle the arch parameter passed to the script. As the arch variable is already used in PKGBUILD, this variable conflict cause unwanted behaviour. Signed-off-by: Aaron Griffin --- db-remove | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/db-remove b/db-remove index eae204e..3ba7dd7 100755 --- a/db-remove +++ b/db-remove @@ -9,10 +9,10 @@ fi packagebase="$1" reponame="$2" -arch="$3" +_arch="$3" ftppath="$FTP_BASE/$reponame/os" -svnrepo="$reponame-$arch" +svnrepo="$reponame-$_arch" [ "$UID" = "" ] && UID=$(uid) @@ -21,7 +21,7 @@ WORKDIR="$TMPDIR/db-remove.$svnrepo.$UID" cleanup() { trap '' 0 2 # unlock - repo_unlock $reponame $arch + repo_unlock $reponame $_arch rm -rf "$WORKDIR" [ "$1" ] && exit $1 } @@ -39,7 +39,7 @@ die() { trap ctrl_c 2 trap cleanup 0 -repo_lock $reponame $arch +repo_lock $reponame $_arch /bin/mkdir -p "$WORKDIR" @@ -66,10 +66,10 @@ fi cd "$WORKDIR" [ -d build/ ] || mkdir build -if [ "$arch" == "any" ]; then +if [ "$_arch" == "any" ]; then arches="i686 x86_64" else - arches="$arch" + arches="$_arch" fi # copy the db file into our working area -- cgit v1.2.2 From 208cf5f55addd91ea6884657737cc62ee81fe0fb Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Tue, 25 Aug 2009 13:11:59 -0700 Subject: Add SRCEXT to config Signed-off-by: Aaron Griffin --- config | 1 + 1 file changed, 1 insertion(+) diff --git a/config b/config index c735a91..86a5eca 100644 --- a/config +++ b/config @@ -12,3 +12,4 @@ ARCHES=(i686 x86_64) BUILDSCRIPT="PKGBUILD" DBEXT=".db.tar.gz" PKGEXT=".pkg.tar.gz" +SRCEXT=".src.tar.gz" -- cgit v1.2.2 From bd2345c24dd8d3c731fc4d840ddd2bedc5cb68e1 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Tue, 25 Aug 2009 13:13:25 -0700 Subject: Quiet sourceball output when license is skipped Signed-off-by: Aaron Griffin --- misc-scripts/make-sourceball | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/misc-scripts/make-sourceball b/misc-scripts/make-sourceball index f93ab2a..9f565f7 100755 --- a/misc-scripts/make-sourceball +++ b/misc-scripts/make-sourceball @@ -71,7 +71,8 @@ create_srcpackage() { echo "Creating source tarball for $pkgname-$pkgver-$pkgrel" if ! chk_license ${license[@]}; then - echo -e "\t$pkgname license (${license[@]}) does not require source tarballs" >&2 + #Removed so as not to clutter failed.txt + #echo -e "\t$pkgname license (${license[@]}) does not require source tarballs" >&2 cleanup 0 fi -- cgit v1.2.2 From d455a0b2cd72cd08b7b6cde878a29841f77a041e Mon Sep 17 00:00:00 2001 From: Xavier Chantry Date: Fri, 28 Aug 2009 01:27:11 +0200 Subject: check_packages : add support for split packages. I just found a way to support split packages, by using $(type package_${pkg}), parsing that output and running eval on the relevant lines. This is a bit ugly, and while it works fine on my machine and my current abs tree, I cannot guarantee this code is bug free :) Signed-off-by: Xavier Chantry Signed-off-by: Aaron Griffin --- cron-jobs/check_archlinux/check_packages.py | 5 +- cron-jobs/check_archlinux/parse_pkgbuilds.sh | 82 +++++++++++++++++++++++----- 2 files changed, 73 insertions(+), 14 deletions(-) diff --git a/cron-jobs/check_archlinux/check_packages.py b/cron-jobs/check_archlinux/check_packages.py index 9bdd9c1..d42feab 100755 --- a/cron-jobs/check_archlinux/check_packages.py +++ b/cron-jobs/check_archlinux/check_packages.py @@ -38,6 +38,7 @@ checked_deps = [] class PacmanPackage: def __init__(self): self.name,self.version = "","" + self.base = "" self.path,self.repo = "","" self.deps,self.makedeps = [],[] self.provides,self.conflicts = [],[] @@ -77,6 +78,8 @@ def parse_data(repo,data): if packages.has_key(pkg.name): dup = packages[pkg.name] packages[pkg.name] = pkg + elif attrname == "base": + pkg.base = line elif attrname == "version": pkg.version = line elif attrname == "path": @@ -356,7 +359,7 @@ for name,pkg in packages.iteritems(): print "==> checking mismatches" for name,pkg in repopkgs.iteritems(): pkgdirname = pkg.path.split("/")[-1] - if name != pkgdirname: + if name != pkgdirname and pkg.base != pkgdirname: mismatches.append(name + " vs. " + pkg.path) print "==> checking archs" diff --git a/cron-jobs/check_archlinux/parse_pkgbuilds.sh b/cron-jobs/check_archlinux/parse_pkgbuilds.sh index 7dcfbbb..0faa29f 100755 --- a/cron-jobs/check_archlinux/parse_pkgbuilds.sh +++ b/cron-jobs/check_archlinux/parse_pkgbuilds.sh @@ -5,24 +5,36 @@ exit() { return; } -parse() { - unset pkgname pkgver pkgrel - unset depends makedepends conflicts provides - ret=0 - dir=$1 - pkgbuild=$dir/PKGBUILD - source $pkgbuild &>/dev/null || ret=$? +variables=('pkgname' 'pkgver' 'pkgrel' 'depends' 'makedepends' 'provides' 'conflicts' ) +readonly -a variables - # ensure $pkgname and $pkgver variables were found - if [ $ret -ne 0 -o -z "$pkgname" -o -z "$pkgver" ]; then - echo -e "%INVALID%\n$pkgbuild\n" - return 1 - fi +backup_package_variables() { + for var in ${variables[@]}; do + indirect="${var}_backup" + eval "${indirect}=(\${$var[@]})" + done +} + +restore_package_variables() { + for var in ${variables[@]}; do + indirect="${var}_backup" + if [ -n "${!indirect}" ]; then + eval "${var}=(\${$indirect[@]})" + else + unset ${var} + fi + done +} +print_info() { echo -e "%NAME%\n$pkgname\n" echo -e "%VERSION%\n$pkgver-$pkgrel\n" echo -e "%PATH%\n$dir\n" + if [ -n "$pkgbase" ]; then + echo -e "%BASE%\n$pkgbase\n" + fi + if [ -n "$arch" ]; then echo "%ARCH%" for i in ${arch[@]}; do echo $i; done @@ -52,6 +64,50 @@ parse() { for i in ${provides[@]}; do echo $i; done echo "" fi +} + +source_pkgbuild() { + ret=0 + dir=$1 + pkgbuild=$dir/PKGBUILD + for var in ${variables[@]}; do + unset ${var} + done + source $pkgbuild &>/dev/null || ret=$? + + # ensure $pkgname and $pkgver variables were found + if [ $ret -ne 0 -o -z "$pkgname" -o -z "$pkgver" ]; then + echo -e "%INVALID%\n$pkgbuild\n" + return 1 + fi + + if [ "${#pkgname[@]}" -gt "1" ]; then + for pkg in ${pkgname[@]}; do + if [ "$(type -t package_${pkg})" != "function" ]; then + echo -e "%INVALID%\n$pkgbuild\n" + return 1 + else + backup_package_variables + pkgname=$pkg + while IFS= read -r line; do + var=${line%%=*} + var="${var#"${var%%[![:space:]]*}"}" # remove leading whitespace characters + for realvar in ${variables[@]}; do + if [ "$var" == "$realvar" ]; then + eval $line + break + fi + done + done < <(type package_${pkg}) + print_info + restore_package_variables + fi + done + else + echo + print_info + fi + return 0 } @@ -63,7 +119,7 @@ find_pkgbuilds() { fi if [ -f $1/PKGBUILD ]; then - parse $1 + source_pkgbuild $1 return fi empty=1 -- cgit v1.2.2 From 620b631fedbbd3596baa6752b9234249a2239a2a Mon Sep 17 00:00:00 2001 From: Xavier Chantry Date: Fri, 28 Aug 2009 01:27:12 +0200 Subject: add deplist to hierarchy problems This should help a lot figuring out whether a given hierarchy problem is fixable (if it doesn't require moving too many deps). example output core/crda depends on extra/python-m2crypto (27 extra (make)deps to pull) core/iputils depends on extra/opensp (29 extra (make)deps to pull) core/iputils depends on extra/libxslt (25 extra (make)deps to pull) core/iputils depends on extra/docbook-xsl (27 extra (make)deps to pull) core/udev depends on extra/gperf (0 extra (make)deps to pull) core/udev depends on extra/libxslt (25 extra (make)deps to pull) core/e2fsprogs depends on extra/bc (0 extra (make)deps to pull) core/sqlite3 depends on extra/tcl (0 extra (make)deps to pull) core/ca-certificates depends on extra/ruby (25 extra (make)deps to pull) the actual deps are only displayed when there are less than 10. Signed-off-by: Xavier Chantry Signed-off-by: Aaron Griffin --- cron-jobs/check_archlinux/check_packages.py | 50 +++++++++++++++++++++++++++-- 1 file changed, 47 insertions(+), 3 deletions(-) diff --git a/cron-jobs/check_archlinux/check_packages.py b/cron-jobs/check_archlinux/check_packages.py index d42feab..f2a9601 100755 --- a/cron-jobs/check_archlinux/check_packages.py +++ b/cron-jobs/check_archlinux/check_packages.py @@ -182,11 +182,50 @@ def verify_deps(name,repo,deps): break if not pkgdep: pkgdep = pkglist[0] - hierarchy.append(repo + "/" + name + " depends on " + pkgdep.repo + "/" + pkgdep.name) + hierarchy.append((repo,name,pkgdep)) + pkg_deps.append(pkgdep) return (pkg_deps,missdeps,hierarchy) +def compute_deplist_aux(pkg,deplist): + newdeplist = [] + list = [] + if pkgdeps.has_key(pkg): + list.extend(pkgdeps[pkg]) + if makepkgdeps.has_key(pkg): + list.extend(makepkgdeps[pkg]) + for dep in list: + if dep not in deplist: + newdeplist.append(dep) + deplist.append(dep) + for dep in newdeplist: + deplist2 = compute_deplist_aux(dep,deplist) + for dep2 in deplist2: + if dep2 not in deplist: + deplist.append(dep2) + return deplist + +def compute_deplist(pkg): + return compute_deplist_aux(pkg,[]) + +def check_hierarchy(deph): + hierarchy = [] + for (repo,name,pkgdep) in deph: + deplist = compute_deplist(pkgdep) + valid_repos = get_repo_hierarchy(repo) + extdeps = [] + for dep in deplist: + if dep.repo not in valid_repos: + extdeps.append(dep.name) + string = repo + "/" + name + " depends on " + pkgdep.repo + "/" + pkgdep.name + " (" + string += "%s extra (make)deps to pull" % len(extdeps) + if 0 < len(extdeps) < 10: + string += " : " + ' '.join(extdeps) + string += ")" + hierarchy.append(string) + return hierarchy + def get_repo_hierarchy(repo): repo_hierarchy = {'core': ['core'], \ 'extra': ['core', 'extra'], \ @@ -373,19 +412,24 @@ for name,pkg in packages.iteritems(): pkg.deps = [dep for dep in pkg.deps if not p.match(dep)] pkg.makedeps = [dep for dep in pkg.makedeps if not p.match(dep)] +deph,makedeph = [],[] + print "==> checking dependencies" for name,pkg in repopkgs.iteritems(): (deps,missdeps,hierarchy) = verify_deps(name,pkg.repo,pkg.deps) pkgdeps[pkg] = deps missing_deps.extend(missdeps) - dep_hierarchy.extend(hierarchy) + deph.extend(hierarchy) print "==> checking makedepends" for name,pkg in repopkgs.iteritems(): (makedeps,missdeps,hierarchy) = verify_deps(name,pkg.repo,pkg.makedeps) makepkgdeps[pkg] = makedeps missing_makedeps.extend(missdeps) - makedep_hierarchy.extend(hierarchy) + makedeph.extend(hierarchy) + +dep_hierarchy = check_hierarchy(deph) +makedep_hierarchy = check_hierarchy(makedeph) print "==> checking for circular dependencies" # make sure pkgdeps is filled for every package -- cgit v1.2.2 From 1e84b6715a688e1c2f1e50e3834511ed21d5c291 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Tue, 1 Sep 2009 09:34:21 -0700 Subject: sourceballs: explicitly add the 'any' arch Loop over all arches AND the 'any' arch. Signed-off-by: Aaron Griffin --- cron-jobs/sourceballs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 4353935..aa06aff 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -36,7 +36,7 @@ FAILED_PKGS="" echo "Errors occured during run:" > "$srcbase/errors.txt" for repo in $repos; do - for arch in ${ARCHES[@]}; do + for arch in ${ARCHES[@]} any; do export CARCH="$arch" ftppath="$ftpbase/$repo/os/$arch" if [ ! -d "$ftppath" ]; then -- cgit v1.2.2 From 5ebf55bedfa891a80170facc79f2fbfca11983c4 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Tue, 8 Sep 2009 16:39:58 -0400 Subject: Added sourceballs.force entries Signed-off-by: Aaron Griffin --- cron-jobs/sourceballs.force | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cron-jobs/sourceballs.force b/cron-jobs/sourceballs.force index e69de29..badf15d 100644 --- a/cron-jobs/sourceballs.force +++ b/cron-jobs/sourceballs.force @@ -0,0 +1,4 @@ +faad2 +wxgtk +wxpython +glhack -- cgit v1.2.2 From 2afa1063284a25ae2691b2727c45238d15025a9a Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 13 Sep 2009 18:16:55 -0500 Subject: Be more careful with config file inclusion We can't really do the basedir magic from db-functions as it is just being sourced and is not the currently executing script. Although a bit repetitive, it is a lot safer to just include the config file everywhere. Noticed this when trying to run the latest available scripts on sigurd. Signed-off-by: Dan McGee --- convert-to-any | 1 + cron-jobs/createFileLists | 1 + cron-jobs/sourceballs | 1 + db-functions | 8 -------- db-move | 1 + db-remove | 1 + db-update | 1 + misc-scripts/ftpdir-cleanup | 1 + misc-scripts/make-sourceball | 1 + testing2x | 1 + 10 files changed, 9 insertions(+), 8 deletions(-) diff --git a/convert-to-any b/convert-to-any index e21b7cd..53d1a7b 100755 --- a/convert-to-any +++ b/convert-to-any @@ -16,6 +16,7 @@ if [ $# -ne 1 ]; then fi . "$(dirname $0)/db-functions" +. "$(dirname $0)/config" cleanup() { trap '' 0 2 diff --git a/cron-jobs/createFileLists b/cron-jobs/createFileLists index 345f9aa..d936853 100755 --- a/cron-jobs/createFileLists +++ b/cron-jobs/createFileLists @@ -5,6 +5,7 @@ targetdir=$reposdir repos="core extra testing community" . "$(dirname $0)/../db-functions" +. "$(dirname $0)/../config" if [ -f "/tmp/createFileList.lock" ]; then echo "Error: createFileList allready in progress." diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index aa06aff..46c2fa3 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -5,6 +5,7 @@ srcbase="/srv/ftp/sources" repos="core extra testing community" . "$(dirname $0)/../db-functions" +. "$(dirname $0)/../config" LOCKFILE="/tmp/.sourceball.lock" diff --git a/db-functions b/db-functions index ccab830..071fb85 100644 --- a/db-functions +++ b/db-functions @@ -3,14 +3,6 @@ # Random integrity things [ "$UID" = "" ] && UID=$(uid) -if [ -z "$BASEDIR" ]; then - BASEDIR="$(dirname $0)" -fi - -if [ -f "$BASEDIR/config" ]; then - . "$BASEDIR/config" -fi - # Useful functions UMASK="" diff --git a/db-move b/db-move index b49aeda..efd54e0 100755 --- a/db-move +++ b/db-move @@ -6,6 +6,7 @@ if [ $# -ne 4 ]; then fi . "$(dirname $0)/db-functions" +. "$(dirname $0)/config" packagebase="$1" repofrom="$2" diff --git a/db-remove b/db-remove index 3ba7dd7..3190231 100755 --- a/db-remove +++ b/db-remove @@ -6,6 +6,7 @@ if [ $# -ne 3 ]; then fi . "$(dirname $0)/db-functions" +. "$(dirname $0)/config" packagebase="$1" reponame="$2" diff --git a/db-update b/db-update index 7ff48ba..577a028 100755 --- a/db-update +++ b/db-update @@ -6,6 +6,7 @@ if [ $# -ne 1 ]; then fi . "$(dirname $0)/db-functions" +. "$(dirname $0)/config" reponame="$1" current_arch="" diff --git a/misc-scripts/ftpdir-cleanup b/misc-scripts/ftpdir-cleanup index 96407c0..9349bca 100755 --- a/misc-scripts/ftpdir-cleanup +++ b/misc-scripts/ftpdir-cleanup @@ -11,6 +11,7 @@ dest=$2 ############################################################ . "$(dirname $0)/../db-functions" +. "$(dirname $0)/../config" getpkgname() { local tmp diff --git a/misc-scripts/make-sourceball b/misc-scripts/make-sourceball index 9f565f7..f2d8415 100755 --- a/misc-scripts/make-sourceball +++ b/misc-scripts/make-sourceball @@ -10,6 +10,7 @@ if [ $# -ne 3 -a $# -ne 4 ]; then fi . "$(dirname $0)/../db-functions" +. "$(dirname $0)/../config" FORCE=0 if [ "$1" = "-f" ]; then diff --git a/testing2x b/testing2x index f3b256b..f9ded9e 100755 --- a/testing2x +++ b/testing2x @@ -1,6 +1,7 @@ #!/bin/bash . "$(dirname $0)/db-functions" +. "$(dirname $0)/config" case "$0" in *64) -- cgit v1.2.2 From 0897913c0adf10a0a9ff4d8790842715e0e3b1b4 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 13 Sep 2009 18:19:59 -0500 Subject: Add new get_repos_for_host() function And use it. This allows us to have server-specific behavior in our scripts without further patching, and it also allows us to simplify some of our scripts a fair amount. Signed-off-by: Dan McGee --- cron-jobs/adjust-permissions | 50 ++++++++++++++++++++++++-------------------- cron-jobs/ftpdir-cleanup | 8 +++---- db-functions | 7 +++++++ 3 files changed, 38 insertions(+), 27 deletions(-) diff --git a/cron-jobs/adjust-permissions b/cron-jobs/adjust-permissions index 6670a6f..91f2de7 100755 --- a/cron-jobs/adjust-permissions +++ b/cron-jobs/adjust-permissions @@ -1,36 +1,40 @@ -#!/bin/sh +#!/bin/bash if [ -f /tmp/.ftpmaint.lck ]; then - exit 0 + exit 0 fi /bin/touch /tmp/.ftpmaint.lck +. "$(dirname $0)/../db-functions" +. "$(dirname $0)/../config" + +get_dir_owner() { + case $1 in + core) + echo "ftp:ftp-arch" ;; + extra) + echo "ftp:ftp-extra" ;; + testing) + echo "ftp:ftp-extra" ;; + community) + echo "root:tusers" ;; + community-testing) + echo "root:tusers" ;; + esac +} + #adjust the nice level to run at a lower priority /usr/bin/renice +10 -p $$ > /dev/null cd /srv/ftp -if [ -d "core" ]; then - #This is unique to gerolde (main arch server) - /bin/chown -R ftp:ftp-arch core/os/any - /bin/chown -R ftp:ftp-arch core/os/i686 - /bin/chown -R ftp:ftp-arch core/os/x86_64 - /bin/chown -R ftp:ftp-extra {extra,testing}/os/any - /bin/chown -R ftp:ftp-extra {extra,testing}/os/i686 - /bin/chown -R ftp:ftp-extra {extra,testing}/os/x86_64 - for d in core extra testing; do - /bin/chmod -R g+w $d/os/any - /bin/chmod -R g+w $d/os/i686 - /bin/chmod -R g+w $d/os/x86_64 - done -else - /bin/chown -R root:tusers {community,community-testing}/os/any - /bin/chown -R root:tusers {community,community-testing}/os/i686 - /bin/chown -R root:tusers {community,community-testing}/os/x86_64 - /bin/chmod -R g+w {community,community-testing}/os/any - /bin/chmod -R g+w {community,community-testing}/os/i686 - /bin/chmod -R g+w {community,community-testing}/os/x86_64 -fi +for d in $(get_repos_for_host); do + owner="$(get_dir_owner $d)" + echo /bin/chown -R $owner $d/os/{any,i686,x86_64} + #/bin/chown -R $owner $d/os/{any,i686,x86_64} + echo /bin/chmod -R g+w $d/os/{any,i686,x86_64} + #/bin/chmod -R g+w $d/os/{any,i686,x86_64} +done /bin/chmod 555 /srv/ftp diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index 04caf93..d66138c 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -1,6 +1,9 @@ #!/bin/bash -repos="core extra testing" +. "$(dirname $0)/../db-functions" +. "$(dirname $0)/../config" + +repos="$(get_repos_for_host)" LOCKFILE="/tmp/.ftpdircleanup.lock" @@ -27,9 +30,6 @@ trap ctrl_c 2 #adjust the nice level to run at a lower priority /usr/bin/renice +10 -p $$ > /dev/null -#Get our destination dir -. "$(dirname $0)/../db-functions" - for repo in $repos; do $(dirname $0)/../misc-scripts/ftpdir-cleanup $repo $CLEANUP_DESTDIR done diff --git a/db-functions b/db-functions index 071fb85..f4f9ee9 100644 --- a/db-functions +++ b/db-functions @@ -102,5 +102,12 @@ check_pkg_arch () { #check_pkg_arch pkgfile arch fi } +get_repos_for_host() { + if [ "$(hostname)" = "sigurd" ]; then + echo "community community-testing" + else + echo "core extra testing" + fi +} # vim: set ts=4 sw=4 noet ft=sh: -- cgit v1.2.2 From ade48254bbe2ee2cedc7773dd5299fa2208f66ca Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 13 Sep 2009 18:25:14 -0500 Subject: Ensure you don't update DBs on the wrong server Signed-off-by: Dan McGee --- db-update | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/db-update b/db-update index 577a028..aa1cb95 100755 --- a/db-update +++ b/db-update @@ -11,7 +11,18 @@ fi reponame="$1" current_arch="" -[ "$UID" = "" ] && UID=$(uid) +# ensure we should be playing with this DB on this server +repos="$(get_repos_for_host)" +found=0 +for r in $repos; do + if [ "$r" = "$reponame" ]; then + found=1 + fi +done +if [ $found -ne 1 ]; then + echo "error: you shouldn't be updating $reponame on this server!" + exit 1 +fi WORKDIR="$TMPDIR/db-update.$reponame.$UID" ADDPKGS="" -- cgit v1.2.2 From 326d22b63d14012da72e2c9b56109c490e98cf63 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 13 Sep 2009 18:27:28 -0500 Subject: Maybe I should make the script really adjust permissions Signed-off-by: Dan McGee --- cron-jobs/adjust-permissions | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/cron-jobs/adjust-permissions b/cron-jobs/adjust-permissions index 91f2de7..0f87075 100755 --- a/cron-jobs/adjust-permissions +++ b/cron-jobs/adjust-permissions @@ -30,10 +30,8 @@ get_dir_owner() { cd /srv/ftp for d in $(get_repos_for_host); do owner="$(get_dir_owner $d)" - echo /bin/chown -R $owner $d/os/{any,i686,x86_64} - #/bin/chown -R $owner $d/os/{any,i686,x86_64} - echo /bin/chmod -R g+w $d/os/{any,i686,x86_64} - #/bin/chmod -R g+w $d/os/{any,i686,x86_64} + /bin/chown -R $owner $d/os/{any,i686,x86_64} + /bin/chmod -R g+w $d/os/{any,i686,x86_64} done /bin/chmod 555 /srv/ftp -- cgit v1.2.2 From 0e93595fc9313f02a0b9e07d70835b8dffd251ab Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Fri, 18 Sep 2009 10:29:21 -0700 Subject: Minor: remove unused variable Signed-off-by: Aaron Griffin --- misc-scripts/ftpdir-cleanup | 1 - 1 file changed, 1 deletion(-) diff --git a/misc-scripts/ftpdir-cleanup b/misc-scripts/ftpdir-cleanup index 9349bca..236dc35 100755 --- a/misc-scripts/ftpdir-cleanup +++ b/misc-scripts/ftpdir-cleanup @@ -103,7 +103,6 @@ for arch in ${ARCHES[@]}; do missfiles="$MISSINGFILES" MISSINGFILES="" for mf in $missfiles; do - af_satisfied=0 if [ -e "${ftppath_base}/any/${mf}" ]; then echo "Restoring missing 'any' symlink: ${mf}" ln -s "../any/${mf}" "${ftppath}" -- cgit v1.2.2 From f1e779055f4e73b701d3da54321d9459f0f6e46a Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Fri, 18 Sep 2009 10:29:50 -0700 Subject: Remove '$arch' from file loop This allows us to still remove older, non architecture prefixed packages Signed-off-by: Aaron Griffin --- misc-scripts/ftpdir-cleanup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc-scripts/ftpdir-cleanup b/misc-scripts/ftpdir-cleanup index 236dc35..68af271 100755 --- a/misc-scripts/ftpdir-cleanup +++ b/misc-scripts/ftpdir-cleanup @@ -82,7 +82,7 @@ for arch in ${ARCHES[@]}; do done cd "$ftppath" - for pkg in *$arch$PKGEXT; do + for pkg in *$PKGEXT; do if [ ! -e "$pkg" ]; then continue fi -- cgit v1.2.2 From 7667c1bbbb8e4bacda38995294694acbe162789b Mon Sep 17 00:00:00 2001 From: Francois Charette Date: Tue, 15 Sep 2009 08:40:37 +0200 Subject: added testing2...-any scripts Signed-off-by: Aaron Griffin --- testing2core-any | 11 +++++++++++ testing2extra-any | 11 +++++++++++ 2 files changed, 22 insertions(+) create mode 100755 testing2core-any create mode 100755 testing2extra-any diff --git a/testing2core-any b/testing2core-any new file mode 100755 index 0000000..a0993e8 --- /dev/null +++ b/testing2core-any @@ -0,0 +1,11 @@ +#!/bin/bash + +if [ $# -le 0 ]; then + echo "usage: $(basename 0) [ [ Moving package '$pkg'" + $(dirname $0)/db-move "$pkg" "testing" "core" "any" +done diff --git a/testing2extra-any b/testing2extra-any new file mode 100755 index 0000000..46ab2f0 --- /dev/null +++ b/testing2extra-any @@ -0,0 +1,11 @@ +#!/bin/bash + +if [ $# -le 0 ]; then + echo "usage: $(basename 0) [ [ Moving package '$pkg'" + $(dirname $0)/db-move "$pkg" "testing" "extra" "any" +done -- cgit v1.2.2 From 954b699c84cd0b6211bf4ffb83bcd8affd7d8875 Mon Sep 17 00:00:00 2001 From: Francois Charette Date: Tue, 15 Sep 2009 09:20:14 +0200 Subject: Remove pkg from $stagedir if already in FTP repo Signed-off-by: Aaron Griffin --- db-update | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/db-update b/db-update index aa1cb95..c68cf84 100755 --- a/db-update +++ b/db-update @@ -74,6 +74,29 @@ die() { trap ctrl_c 2 trap cleanup 0 +# Remove any package from $stagedir that is already in the FTP repository +for f in $stagedir/*-any$PKGEXT; do + bf=$(basename $f) + ftppath_any="$FTP_BASE/$reponame/os/any" + if [[ -f $ftppath_any/$bf ]]; then + echo " WARNING: Package file $bf already exists in FTP repo" + echo " Removing from $stagedir" + /bin/rm $f + fi +done + +for current_arch in ${ARCHES[@]}; do + ftppath="$FTP_BASE/$reponame/os/$current_arch" + for f in $stagedir/*-$current_arch$PKGEXT; do + bf=$(basename $f) + if [[ -f $ftppath/$bf ]]; then + echo " WARNING: Package file $bf already exists in FTP repo" + echo " Removing from $stagedir" + /bin/rm $f + fi + done +done + # Process architecture-independent packages first. if [ -d "$stagedir" ]; then ANYPKGS="$(/bin/ls $stagedir/*-any$PKGEXT 2>/dev/null)" -- cgit v1.2.2 From 953d9d8ea616a9505c9e13813241dee455a580dc Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Fri, 18 Sep 2009 10:33:11 -0700 Subject: Collapse duplicate loops for staging removal Add 'any' to the items looped over in the ${ARCHES[@]} loop Signed-off-by: Aaron Griffin --- db-update | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/db-update b/db-update index c68cf84..17d3b71 100755 --- a/db-update +++ b/db-update @@ -75,17 +75,7 @@ trap ctrl_c 2 trap cleanup 0 # Remove any package from $stagedir that is already in the FTP repository -for f in $stagedir/*-any$PKGEXT; do - bf=$(basename $f) - ftppath_any="$FTP_BASE/$reponame/os/any" - if [[ -f $ftppath_any/$bf ]]; then - echo " WARNING: Package file $bf already exists in FTP repo" - echo " Removing from $stagedir" - /bin/rm $f - fi -done - -for current_arch in ${ARCHES[@]}; do +for current_arch in ${ARCHES[@]} any; do ftppath="$FTP_BASE/$reponame/os/$current_arch" for f in $stagedir/*-$current_arch$PKGEXT; do bf=$(basename $f) -- cgit v1.2.2 From aa87efaa0e4f973ab37bd56b9fa690eabde27d92 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Fri, 18 Sep 2009 11:40:44 -0700 Subject: ftpdir-cleanup: get rid of local getpkgname This is in the db-functions file, and does a better job. In order to do this, we had to change some path related code Signed-off-by: Aaron Griffin --- misc-scripts/ftpdir-cleanup | 24 +++++------------------- 1 file changed, 5 insertions(+), 19 deletions(-) diff --git a/misc-scripts/ftpdir-cleanup b/misc-scripts/ftpdir-cleanup index 68af271..cc01363 100755 --- a/misc-scripts/ftpdir-cleanup +++ b/misc-scripts/ftpdir-cleanup @@ -13,19 +13,6 @@ dest=$2 . "$(dirname $0)/../db-functions" . "$(dirname $0)/../config" -getpkgname() { - local tmp - - tmp=${1##*/} - tmp=${tmp%$PKGEXT} - for arch in ${ARCHES[@]}; do - tmp=${tmp%-$arch} - done - tmp=${tmp%-any} - echo ${tmp%-*-*} -} - - ftppath_base="$FTP_BASE/$reponame/$FTP_OS_SUFFIX" for arch in ${ARCHES[@]}; do @@ -60,16 +47,17 @@ for arch in ${ARCHES[@]}; do exit 1 fi - for pkg in *; do + cd "$ftppath" + + for pkg in $TMPDIR/*; do filename=$(grep -A1 '^%FILENAME%$' "${pkg}/desc" | tail -n1) [ -z "${filename}" ] && filename="${pkg}${PKGEXT}" - if [ ! -e "${ftppath}/${filename}" ]; then + if [ ! -e "${filename}" ]; then MISSINGFILES="${MISSINGFILES} ${filename}" else pkgname="$(getpkgname ${filename})" - for otherfile in ${ftppath}/${pkgname}-*; do - otherfile="$(basename ${otherfile})" + for otherfile in ${pkgname}-*; do if [ "${otherfile}" != "${filename}" -a "${pkgname}" = "$(getpkgname ${otherfile})" ]; then if [ -h "${otherfile}" ]; then DELETESYMLINKS="${DELETESYMLINKS} ${otherfile}" @@ -81,7 +69,6 @@ for arch in ${ARCHES[@]}; do fi done - cd "$ftppath" for pkg in *$PKGEXT; do if [ ! -e "$pkg" ]; then continue @@ -95,7 +82,6 @@ for arch in ${ARCHES[@]}; do EXTRAFILES="$EXTRAFILES $pkg" done - cd "$ftppath" rm -rf ${TMPDIR} # Do a quick check to see if a missing ARCHINDEPFILE is in the any dir -- cgit v1.2.2 From e436894b05b1d346455c234a7b473f2215c52858 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Fri, 18 Sep 2009 11:44:56 -0700 Subject: ftpdir-cleanup: Additional getpkgname cleanup Signed-off-by: Aaron Griffin --- misc-scripts/ftpdir-cleanup | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/misc-scripts/ftpdir-cleanup b/misc-scripts/ftpdir-cleanup index cc01363..f0f89a3 100755 --- a/misc-scripts/ftpdir-cleanup +++ b/misc-scripts/ftpdir-cleanup @@ -75,9 +75,11 @@ for arch in ${ARCHES[@]}; do fi pkgname="$(getpkgname $pkg)" for p in ${TMPDIR}/${pkgname}-*; do - if [ -d "${p}" -a "$(getpkgname $(basename ${p}))" = "${pkgname}" ]; then + [ ! -d "${p}" ] || continue 2 + dbpkgname=$(grep -A1 '^%FILENAME%$' "${p}/desc" | tail -n1) + if [ "${dbpkgname}" = "${pkgname}" ]; then continue 2 - fi + fi done EXTRAFILES="$EXTRAFILES $pkg" done -- cgit v1.2.2 From beeccefdb996084b9edef27a5bbdace4c10ba1be Mon Sep 17 00:00:00 2001 From: Daniel J Griffiths Date: Fri, 18 Sep 2009 11:47:41 -0700 Subject: Add db-community-testing Signed-off-by: Aaron Griffin --- db-community-testing | 3 +++ 1 file changed, 3 insertions(+) create mode 100755 db-community-testing diff --git a/db-community-testing b/db-community-testing new file mode 100755 index 0000000..eb4cf67 --- /dev/null +++ b/db-community-testing @@ -0,0 +1,3 @@ +#!/bin/bash + +$(dirname $0)/db-update "community-testing" -- cgit v1.2.2 From 428a876ba62be88320acb11c48a1a3f45cc209a9 Mon Sep 17 00:00:00 2001 From: Daniel J Griffiths Date: Fri, 18 Sep 2009 11:49:21 -0700 Subject: Add "testing to repo" scripts for community Signed-off-by: Aaron Griffin --- community-testing2community | 11 +++++++++++ community-testing2community64 | 11 +++++++++++ 2 files changed, 22 insertions(+) create mode 100755 community-testing2community create mode 100755 community-testing2community64 diff --git a/community-testing2community b/community-testing2community new file mode 100755 index 0000000..557b39d --- /dev/null +++ b/community-testing2community @@ -0,0 +1,11 @@ +#!/bin/bash + +if [ $# -le 0 ]; then + echo "usage: $(basename 0) [ [ Moving package '$pkg'" + $(dirname $0)/db-move "$pkg" "community-testing" "community" "i686" +done diff --git a/community-testing2community64 b/community-testing2community64 new file mode 100755 index 0000000..3f88a4d --- /dev/null +++ b/community-testing2community64 @@ -0,0 +1,11 @@ +#!/bin/bash + +if [ $# -le 0 ]; then + echo "usage: $(basename 0) [ [ Moving package '$pkg'" + $(dirname $0)/db-move "$pkg" "community-testing" "community" "x86_64" +done -- cgit v1.2.2 From 915c5ce3119c84aedf30c14d0e9a87a84510271a Mon Sep 17 00:00:00 2001 From: Daniel J Griffiths Date: Fri, 18 Sep 2009 11:50:20 -0700 Subject: Add "testing to repo" for any pkgs in community Whoops, missed this in the last commit Signed-off-by: Aaron Griffin --- community-testing2community-any | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100755 community-testing2community-any diff --git a/community-testing2community-any b/community-testing2community-any new file mode 100755 index 0000000..e63a7ec --- /dev/null +++ b/community-testing2community-any @@ -0,0 +1,11 @@ +#!/bin/bash + +if [ $# -le 0 ]; then + echo "usage: $(basename 0) [ [ Moving package '$pkg'" + $(dirname $0)/db-move "$pkg" "community-testing" "community" "any" +done -- cgit v1.2.2 From a46bca9a39b37e7574f87468ddc5fb9b9577786e Mon Sep 17 00:00:00 2001 From: Henning Garus Date: Tue, 1 Sep 2009 23:54:47 +0200 Subject: check_packages.py: Allow multiple abs-trees By parsing multiple abs trees we can add any when parsing the other trees, checking any standalone doesn't make much sense. Signed-off-by: Henning Garus Signed-off-by: Xavier Chantry Signed-off-by: Aaron Griffin --- cron-jobs/check_archlinux/check_packages.py | 32 +++++++++++++++-------------- cron-jobs/integrity-check | 2 +- 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/cron-jobs/check_archlinux/check_packages.py b/cron-jobs/check_archlinux/check_packages.py index f2a9601..c12858d 100755 --- a/cron-jobs/check_archlinux/check_packages.py +++ b/cron-jobs/check_archlinux/check_packages.py @@ -51,10 +51,11 @@ class Depend: self.mod = mod def parse_pkgbuilds(repos,arch): - for repo in repos: - data = commands.getoutput(os.path.dirname(sys.argv[0]) + '/parse_pkgbuilds.sh ' - + arch + ' ' + absroot + '/' + repo) - parse_data(repo,data) + for absroot in absroots: + for repo in repos: + data = commands.getoutput(os.path.dirname(sys.argv[0]) + '/parse_pkgbuilds.sh ' + + arch + ' ' + absroot + '/' + repo) + parse_data(repo,data) def parse_data(repo,data): attrname = None @@ -332,9 +333,9 @@ def print_usage(): print "Usage: ./check_packages.py [OPTION]" print "" print "Options:" - print " --abs-tree= Check the specified tree (default : /var/abs)" + print " --abs-tree= Check the specified tree(s) (default : /var/abs)" print " --repos= Check the specified repos (default : core,extra)" - print " --arch= Check the specified arch (default : i686)" + print " --arch= Check the specified arch (default : i686)" print " -h, --help Show this help and exit" print "" print "Examples:" @@ -345,7 +346,7 @@ def print_usage(): print "" ## Default path to the abs root directory -absroot = "/var/abs" +absroots = ["/var/abs"] ## Default list of repos to check repos = ['core', 'extra'] ## Default arch @@ -359,7 +360,7 @@ except getopt.GetoptError: if opts != []: for o, a in opts: if o in ("--abs-tree"): - absroot = a + absroot = a.split(',') elif o in ("--repos"): repos = a.split(",") elif o in ("--arch"): @@ -371,14 +372,15 @@ if opts != []: print_usage() sys.exit() -if not os.path.isdir(absroot): - print "Error : the abs tree " + absroot + " does not exist" - sys.exit() -for repo in repos: - repopath = absroot + "/" + repo - if not os.path.isdir(repopath): - print "Error : the repository " + repo + " does not exist in " + absroot +for absroot in absroots: + if not os.path.isdir(absroot): + print "Error : the abs tree " + absroot + " does not exist" sys.exit() + for repo in repos: + repopath = absroot + "/" + repo + if not os.path.isdir(repopath): + print "Error : the repository " + repo + " does not exist in " + absroot + sys.exit() # repos which need to be loaded loadrepos = set([]) for repo in repos: diff --git a/cron-jobs/integrity-check b/cron-jobs/integrity-check index 0b59064..b3185ec 100755 --- a/cron-jobs/integrity-check +++ b/cron-jobs/integrity-check @@ -9,5 +9,5 @@ fi $basedir/check_archlinux/check_packages.py \ --repos="$1" \ - --abs-tree="/srv/abs/rsync/$2" --arch="$2" |\ + --abs-tree="/srv/abs/rsync/$2,/srv/abs/rsync/any" --arch="$2" |\ $basedir/devlist-mailer "Integrity Check $2: $1" "$3" -- cgit v1.2.2 From ee0dd1ef6e9a0169ad54947758f2d7858a60d960 Mon Sep 17 00:00:00 2001 From: Henning Garus Date: Tue, 1 Sep 2009 23:57:39 +0200 Subject: Add db comparison to integrity check Compare the abs tree with the repo dbs to check if we have a PKGBUILD for each package in the dbs and vice versa. Signed-off-by: Henning Garus Signed-off-by: Xavier Chantry Signed-off-by: Aaron Griffin --- cron-jobs/check_archlinux/check_packages.py | 55 +++++++++++++++++++++++++++-- 1 file changed, 53 insertions(+), 2 deletions(-) diff --git a/cron-jobs/check_archlinux/check_packages.py b/cron-jobs/check_archlinux/check_packages.py index c12858d..feab987 100755 --- a/cron-jobs/check_archlinux/check_packages.py +++ b/cron-jobs/check_archlinux/check_packages.py @@ -16,9 +16,11 @@ # a non-core package) # 8. Circular dependencies -import os,re,commands,getopt,sys,alpm +import os,re,commands,getopt,sys,tarfile,alpm import pdb +DBEXT='.db.tar.gz' + packages = {} # pkgname : PacmanPackage provisions = {} # provision : PacmanPackage pkgdeps,makepkgdeps = {},{} # pkgname : list of the PacmanPackage dependencies @@ -26,6 +28,9 @@ invalid_pkgbuilds = [] missing_pkgbuilds = [] dups = [] +dbonly = [] +absonly = [] + mismatches = [] missing_deps = [] missing_makedeps = [] @@ -102,6 +107,17 @@ def parse_data(repo,data): provisions[provname] = [] provisions[provname].append(pkg) +def parse_dbs(repos,arch): + dbpkgs = {} + for repo in repos: + pkgs = set([]) + db = tarfile.open(os.path.join(repodir,repo,'os',arch,repo + DBEXT)) + for line in db.getnames(): + if not '/' in line: + pkgs.add(line.rsplit('-',2)[0]) + dbpkgs[repo] = pkgs + return(dbpkgs) + def splitdep(dep): name = dep version = "" @@ -317,6 +333,8 @@ def print_results(): print_result(dep_hierarchy, "Repo Hierarchy for Dependencies") print_result(makedep_hierarchy, "Repo Hierarchy for Makedepends") print_result(circular_deps, "Circular Dependencies") + print_result(dbonly, "Packages found in db, but not in tree") + print_result(absonly,"Packages found in tree, but not in db") print_subheading("Summary") print "Missing PKGBUILDs: ", len(missing_pkgbuilds) print "Invalid PKGBUILDs: ", len(invalid_pkgbuilds) @@ -326,6 +344,8 @@ def print_results(): print "Missing (make)dependencies: ", len(missing_deps)+len(missing_makedeps) print "Repo hierarchy problems: ", len(dep_hierarchy)+len(makedep_hierarchy) print "Circular dependencies: ", len(circular_deps) + print "In db, but not in tree: ", len(dbonly) + print "In tree, but not in db: ", len(absonly) print "" def print_usage(): @@ -336,6 +356,7 @@ def print_usage(): print " --abs-tree= Check the specified tree(s) (default : /var/abs)" print " --repos= Check the specified repos (default : core,extra)" print " --arch= Check the specified arch (default : i686)" + print " --repo-dir= Check the dbs at the specified path (default : /srv/ftp)" print " -h, --help Show this help and exit" print "" print "Examples:" @@ -351,9 +372,12 @@ absroots = ["/var/abs"] repos = ['core', 'extra'] ## Default arch arch = "i686" +## Default repodir +repodir = "/srv/ftp" try: - opts, args = getopt.getopt(sys.argv[1:], "", ["abs-tree=", "repos=", "arch="]) + opts, args = getopt.getopt(sys.argv[1:], "", ["abs-tree=", "repos=", + "arch=", "repo-dir="]) except getopt.GetoptError: print_usage() sys.exit() @@ -365,6 +389,8 @@ if opts != []: repos = a.split(",") elif o in ("--arch"): arch = a + elif o in ("--repo-dir"): + repodir = a else: print_usage() sys.exit() @@ -381,6 +407,17 @@ for absroot in absroots: if not os.path.isdir(repopath): print "Error : the repository " + repo + " does not exist in " + absroot sys.exit() +if not os.path.isdir(repodir): + print "Error: the repository directory %s does not exist" % repodir + sys.exit() +for repo in repos: + path = os.path.join(repodir,repo,'os',arch,repo + DBEXT) + if not os.path.isfile(path): + print "Error : repo DB %s : File not found" % path + sys.exit() + if not tarfile.is_tarfile(path): + print "Error : Cant open repo DB %s, not a valid tar file" % path + sys.exit() # repos which need to be loaded loadrepos = set([]) for repo in repos: @@ -397,6 +434,9 @@ for name,pkg in packages.iteritems(): if pkg.repo in repos: repopkgs[name] = pkg +print "==> parsing db files" +dbpkgs = parse_dbs(repos,arch) + print "==> checking mismatches" for name,pkg in repopkgs.iteritems(): pkgdirname = pkg.path.split("/")[-1] @@ -441,4 +481,15 @@ for name,pkg in packages.iteritems(): pkgdeps[pkg] = deps find_scc(repopkgs.values()) +print "==> checking for differences between db files and pkgbuilds" +for repo in repos: + for pkg in dbpkgs[repo]: + if not (pkg in repopkgs.keys() and repopkgs[pkg].repo == repo): + dbonly.append("%s/%s" % (repo,pkg)) +dbonly.sort() +for name,pkg in repopkgs.iteritems(): + if not name in dbpkgs[pkg.repo]: + absonly.append("%s/%s" % (pkg.repo,name)) +absonly.sort() + print_results() -- cgit v1.2.2 From ea827b50d0c65b4ba9f2e2d3c0761572915e6b03 Mon Sep 17 00:00:00 2001 From: Xavier Chantry Date: Wed, 2 Sep 2009 18:29:19 +0200 Subject: add the repo/ info to most results, and sort them Add repo information to most results by using the repo/pkgname syntax (instead of only pkgname) Sort all results list (with the point above, this sorts the results by repo :)) Signed-off-by: Xavier Chantry Signed-off-by: Aaron Griffin --- cron-jobs/check_archlinux/check_packages.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/cron-jobs/check_archlinux/check_packages.py b/cron-jobs/check_archlinux/check_packages.py index feab987..96f3181 100755 --- a/cron-jobs/check_archlinux/check_packages.py +++ b/cron-jobs/check_archlinux/check_packages.py @@ -189,7 +189,7 @@ def verify_deps(name,repo,deps): for dep in deps: pkglist = verify_dep(dep) if pkglist == []: - missdeps.append(name + " --> '" + dep + "'") + missdeps.append(repo + "/" + name + " --> '" + dep + "'") else: valid_repos = get_repo_hierarchy(repo) pkgdep = None @@ -252,12 +252,12 @@ def get_repo_hierarchy(repo): else: return ['core','extra','community'] -def verify_archs(name,archs): +def verify_archs(name,repo,archs): valid_archs = ['any', 'i686', 'x86_64'] invalid_archs = [] for arch in archs: if arch not in valid_archs: - invalid_archs.append(name + " --> " + arch) + invalid_archs.append(repo + "/" + name + " --> " + arch) return invalid_archs def find_scc(packages): @@ -295,7 +295,7 @@ def tarjan(pkg): return path = pkg.name while pkg != dep: - path = dep.name + ">" + path + path = dep.repo + "/" + dep.name + ">" + path dep = S.pop() path = dep.name + ">" + path if pkg.repo in repos: @@ -318,6 +318,7 @@ def print_missdeps(pkgname,missdeps) : def print_result(list, subheading): if len(list) > 0: + list.sort() print_subheading(subheading) for item in list: print item @@ -445,7 +446,7 @@ for name,pkg in repopkgs.iteritems(): print "==> checking archs" for name,pkg in repopkgs.iteritems(): - archs = verify_archs(name,pkg.archs) + archs = verify_archs(name,pkg.repo,pkg.archs) invalid_archs.extend(archs) # ugly hack to strip the weird kblic- deps @@ -486,10 +487,8 @@ for repo in repos: for pkg in dbpkgs[repo]: if not (pkg in repopkgs.keys() and repopkgs[pkg].repo == repo): dbonly.append("%s/%s" % (repo,pkg)) -dbonly.sort() for name,pkg in repopkgs.iteritems(): if not name in dbpkgs[pkg.repo]: absonly.append("%s/%s" % (pkg.repo,name)) -absonly.sort() print_results() -- cgit v1.2.2 From fe4152a6949a9ec3132d1a1c08335f370989198f Mon Sep 17 00:00:00 2001 From: Xavier Chantry Date: Thu, 3 Sep 2009 22:34:39 +0200 Subject: improve compute_dep algorithm The compute_dep function I wrote for the more informative hierarchy output was very inefficient. It's much better now (10s -> 0.5s) on my box, and I get exactly the same results :) Now the big majority of the time is again spent on parsing pkgbuilds. Signed-off-by: Xavier Chantry Signed-off-by: Aaron Griffin --- cron-jobs/check_archlinux/check_packages.py | 35 +++++++++++++---------------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/cron-jobs/check_archlinux/check_packages.py b/cron-jobs/check_archlinux/check_packages.py index 96f3181..9ffdac5 100755 --- a/cron-jobs/check_archlinux/check_packages.py +++ b/cron-jobs/check_archlinux/check_packages.py @@ -205,26 +205,22 @@ def verify_deps(name,repo,deps): return (pkg_deps,missdeps,hierarchy) -def compute_deplist_aux(pkg,deplist): - newdeplist = [] - list = [] - if pkgdeps.has_key(pkg): - list.extend(pkgdeps[pkg]) - if makepkgdeps.has_key(pkg): - list.extend(makepkgdeps[pkg]) - for dep in list: - if dep not in deplist: - newdeplist.append(dep) - deplist.append(dep) - for dep in newdeplist: - deplist2 = compute_deplist_aux(dep,deplist) - for dep2 in deplist2: - if dep2 not in deplist: - deplist.append(dep2) - return deplist - def compute_deplist(pkg): - return compute_deplist_aux(pkg,[]) + list = [] + stack = [pkg] + while stack != []: + dep = stack.pop() + if pkgdeps.has_key(dep): + for dep2 in pkgdeps[dep]: + if dep2 not in list: + list.append(dep2) + stack.append(dep2) + if makepkgdeps.has_key(dep): + for dep2 in makepkgdeps[dep]: + if dep2 not in list: + list.append(dep2) + stack.append(dep2) + return list def check_hierarchy(deph): hierarchy = [] @@ -471,6 +467,7 @@ for name,pkg in repopkgs.iteritems(): missing_makedeps.extend(missdeps) makedeph.extend(hierarchy) +print "==> checking hierarchy" dep_hierarchy = check_hierarchy(deph) makedep_hierarchy = check_hierarchy(makedeph) -- cgit v1.2.2 From b36b754497df69778273f6a6e7b0d91679646565 Mon Sep 17 00:00:00 2001 From: Xavier Chantry Date: Thu, 3 Sep 2009 23:34:24 +0200 Subject: parse_pkgbuilds : simple performance tweak this simple tweak gives a nice perf boost : from 10s to 7s to parse extra repo. indeed calling basename caused a fork to happen for the thousand of files being considered Now the major bottleneck is parsing split pkgbuilds, but I am afraid there is no magical solution for that :( Signed-off-by: Xavier Chantry Signed-off-by: Aaron Griffin --- cron-jobs/check_archlinux/parse_pkgbuilds.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cron-jobs/check_archlinux/parse_pkgbuilds.sh b/cron-jobs/check_archlinux/parse_pkgbuilds.sh index 0faa29f..c2df7d1 100755 --- a/cron-jobs/check_archlinux/parse_pkgbuilds.sh +++ b/cron-jobs/check_archlinux/parse_pkgbuilds.sh @@ -113,7 +113,7 @@ source_pkgbuild() { find_pkgbuilds() { #Skip over some dirs - local d="$(basename $1)" + local d="${1##*/}" if [ "$d" = "CVS" -o "$d" = ".svn" ]; then return fi -- cgit v1.2.2 From 7eccf95bb5d53cb1b4f8c1d363cd5645ca4e567e Mon Sep 17 00:00:00 2001 From: Xavier Chantry Date: Sun, 6 Sep 2009 21:36:42 +0200 Subject: improve the way parse_pkgbuilds.sh is called 1) check the return value. before a failed call to parse_pkgbuilds.sh was completely silent.. not good :) 2) call realpath on check_packages.py before determining the directory name, to figure out where parse_pkgbuilds.sh is. This allows to symlink check_packages.py and use the symlink without moving parse_pkgbuilds.sh . for example : /foo/bar/ contains check_packages.py and parse_pkgbuilds.sh ln -s /foo/bar/check_packages.py /bin/check_package dirname(/bin/check_package) returns /bin/ so not good dirname(realpath(/bin/check_package)) returns /foo/bar/ so good Signed-off-by: Xavier Chantry Signed-off-by: Aaron Griffin --- cron-jobs/check_archlinux/check_packages.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/cron-jobs/check_archlinux/check_packages.py b/cron-jobs/check_archlinux/check_packages.py index 9ffdac5..cb7eae4 100755 --- a/cron-jobs/check_archlinux/check_packages.py +++ b/cron-jobs/check_archlinux/check_packages.py @@ -58,9 +58,13 @@ class Depend: def parse_pkgbuilds(repos,arch): for absroot in absroots: for repo in repos: - data = commands.getoutput(os.path.dirname(sys.argv[0]) + '/parse_pkgbuilds.sh ' - + arch + ' ' + absroot + '/' + repo) - parse_data(repo,data) + cmd = os.path.dirname(os.path.realpath(sys.argv[0])) + '/parse_pkgbuilds.sh ' + cmd += arch + ' ' + absroot + '/' + repo + (status,output) = commands.getstatusoutput(cmd) + if status != 0: + print "Error : failed to run '%s'" % cmd + sys.exit() + parse_data(repo,output) def parse_data(repo,data): attrname = None -- cgit v1.2.2 From 12aba89ed2100552ddfe7d69bbd9e35a2019d098 Mon Sep 17 00:00:00 2001 From: Xavier Chantry Date: Tue, 15 Sep 2009 09:19:02 +0200 Subject: parse_pkgbuilds.sh : fix pkgbase handling add the default value for pkgbase also rework slightly how variables are handled (better distinction between splitpkg variables and the rest) Signed-off-by: Xavier Chantry Signed-off-by: Aaron Griffin --- cron-jobs/check_archlinux/parse_pkgbuilds.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/cron-jobs/check_archlinux/parse_pkgbuilds.sh b/cron-jobs/check_archlinux/parse_pkgbuilds.sh index c2df7d1..5cd17e4 100755 --- a/cron-jobs/check_archlinux/parse_pkgbuilds.sh +++ b/cron-jobs/check_archlinux/parse_pkgbuilds.sh @@ -5,18 +5,19 @@ exit() { return; } -variables=('pkgname' 'pkgver' 'pkgrel' 'depends' 'makedepends' 'provides' 'conflicts' ) -readonly -a variables +splitpkg_overrides=('depends' 'optdepends' 'provides' 'conflicts') +variables=('pkgname' 'pkgbase' 'pkgver' 'pkgrel' 'makedepends' 'arch' ${splitpkg_overrides[@]}) +readonly -a variables splitpkg_overrides backup_package_variables() { - for var in ${variables[@]}; do + for var in ${splitpkg_overrides[@]}; do indirect="${var}_backup" eval "${indirect}=(\${$var[@]})" done } restore_package_variables() { - for var in ${variables[@]}; do + for var in ${splitpkg_overrides[@]}; do indirect="${var}_backup" if [ -n "${!indirect}" ]; then eval "${var}=(\${$indirect[@]})" @@ -82,6 +83,7 @@ source_pkgbuild() { fi if [ "${#pkgname[@]}" -gt "1" ]; then + pkgbase=${pkgbase:-${pkgname[0]}} for pkg in ${pkgname[@]}; do if [ "$(type -t package_${pkg})" != "function" ]; then echo -e "%INVALID%\n$pkgbuild\n" -- cgit v1.2.2 From 43f4ada2f207ebf6d3cdd95d511e3ecdc01e6d7a Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Fri, 18 Sep 2009 17:22:55 -0400 Subject: Bump check_archlinux module for libalpm.so.4 Signed-off-by: Aaron Griffin --- cron-jobs/check_archlinux/alpm.so | Bin 0 -> 7238 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100755 cron-jobs/check_archlinux/alpm.so diff --git a/cron-jobs/check_archlinux/alpm.so b/cron-jobs/check_archlinux/alpm.so new file mode 100755 index 0000000..b87b894 Binary files /dev/null and b/cron-jobs/check_archlinux/alpm.so differ -- cgit v1.2.2 From 85eee9a95d2fcbdce483b94202b489d07d258690 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Fri, 18 Sep 2009 17:28:46 -0400 Subject: check_archlinux: absroot -> absroots Allow us to use the --abs-root flag. He he Signed-off-by: Aaron Griffin --- cron-jobs/check_archlinux/check_packages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cron-jobs/check_archlinux/check_packages.py b/cron-jobs/check_archlinux/check_packages.py index cb7eae4..25362c0 100755 --- a/cron-jobs/check_archlinux/check_packages.py +++ b/cron-jobs/check_archlinux/check_packages.py @@ -385,7 +385,7 @@ except getopt.GetoptError: if opts != []: for o, a in opts: if o in ("--abs-tree"): - absroot = a.split(',') + absroots = a.split(',') elif o in ("--repos"): repos = a.split(",") elif o in ("--arch"): -- cgit v1.2.2 From 496fc1c7f54132410af5fd84bfe10b8e68d02a1e Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Fri, 18 Sep 2009 17:33:26 -0400 Subject: integrity-check: Loop over arches THis simplifies the command line and we do both anyway Signed-off-by: Aaron Griffin --- cron-jobs/integrity-check | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/cron-jobs/integrity-check b/cron-jobs/integrity-check index b3185ec..8f23129 100755 --- a/cron-jobs/integrity-check +++ b/cron-jobs/integrity-check @@ -3,11 +3,13 @@ basedir="$(dirname $0)" if [ $# -ne 3 ]; then - echo "usage: $(basename $0) [,,...] " >&2 + echo "usage: $(basename $0) [,,...] " >&2 exit 1 fi -$basedir/check_archlinux/check_packages.py \ - --repos="$1" \ - --abs-tree="/srv/abs/rsync/$2,/srv/abs/rsync/any" --arch="$2" |\ - $basedir/devlist-mailer "Integrity Check $2: $1" "$3" +for arch in i686 x86_64; do + $basedir/check_archlinux/check_packages.py \ + --repos="$1" \ + --abs-tree="/srv/abs/rsync/$arch,/srv/abs/rsync/any" --arch="$arch" |\ + $basedir/devlist-mailer "Integrity Check $arch: $1" "$3" +done -- cgit v1.2.2 From 17e96ac3d2505ed8841ff9240492e7378472d364 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Fri, 18 Sep 2009 18:18:44 -0400 Subject: integrity-check: Forgot to decrement the arg count Signed-off-by: Aaron Griffin --- cron-jobs/integrity-check | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cron-jobs/integrity-check b/cron-jobs/integrity-check index 8f23129..c2546de 100755 --- a/cron-jobs/integrity-check +++ b/cron-jobs/integrity-check @@ -2,7 +2,7 @@ basedir="$(dirname $0)" -if [ $# -ne 3 ]; then +if [ $# -ne 2 ]; then echo "usage: $(basename $0) [,,...] " >&2 exit 1 fi -- cgit v1.2.2 From d7f1242883aea50023931447c3c2f527528084b3 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Fri, 18 Sep 2009 18:26:02 -0400 Subject: integrity-check: Rename $3 -> $2 OMFG such a simple script and I keep missing crap Signed-off-by: Aaron Griffin --- cron-jobs/integrity-check | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cron-jobs/integrity-check b/cron-jobs/integrity-check index c2546de..0b2b9c7 100755 --- a/cron-jobs/integrity-check +++ b/cron-jobs/integrity-check @@ -11,5 +11,5 @@ for arch in i686 x86_64; do $basedir/check_archlinux/check_packages.py \ --repos="$1" \ --abs-tree="/srv/abs/rsync/$arch,/srv/abs/rsync/any" --arch="$arch" |\ - $basedir/devlist-mailer "Integrity Check $arch: $1" "$3" + $basedir/devlist-mailer "Integrity Check $arch: $1" "$2" done -- cgit v1.2.2 From 4643d931f6adc4f7f55c7c2fbb37532614742c74 Mon Sep 17 00:00:00 2001 From: Xavier Chantry Date: Sun, 20 Sep 2009 14:50:34 +0200 Subject: check_packages : fix a bug when filling provisions When we had a duplicate package, we would replace the old ref by the new ref. But the provisions dict kept both the old and new ref. This was not good at all. Now, we just keep the old ref, and we only fill provisions after all packages have been parsed. This should be much more sane. Signed-off-by: Xavier Chantry Signed-off-by: Aaron Griffin --- cron-jobs/check_archlinux/check_packages.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/cron-jobs/check_archlinux/check_packages.py b/cron-jobs/check_archlinux/check_packages.py index 25362c0..e8830e0 100755 --- a/cron-jobs/check_archlinux/check_packages.py +++ b/cron-jobs/check_archlinux/check_packages.py @@ -22,8 +22,9 @@ import pdb DBEXT='.db.tar.gz' packages = {} # pkgname : PacmanPackage +repopkgs = {} # pkgname : PacmanPackage provisions = {} # provision : PacmanPackage -pkgdeps,makepkgdeps = {},{} # pkgname : list of the PacmanPackage dependencies +pkgdeps,makepkgdeps = {},{} # PacmanPackage : list of the PacmanPackage dependencies invalid_pkgbuilds = [] missing_pkgbuilds = [] dups = [] @@ -87,7 +88,8 @@ def parse_data(repo,data): dup = None if packages.has_key(pkg.name): dup = packages[pkg.name] - packages[pkg.name] = pkg + else: + packages[pkg.name] = pkg elif attrname == "base": pkg.base = line elif attrname == "version": @@ -106,10 +108,6 @@ def parse_data(repo,data): pkg.conflicts.append(line) elif attrname == "provides": pkg.provides.append(line) - provname=line.split("=")[0] - if not provisions.has_key(provname): - provisions[provname] = [] - provisions[provname].append(pkg) def parse_dbs(repos,arch): dbpkgs = {} @@ -278,11 +276,9 @@ def tarjan(pkg): index += 1 checked_deps.append(pkg) S.append(pkg) + deps = [] if pkgdeps.has_key(pkg): deps = pkgdeps[pkg] - else: - print pkg.name - deps = [] for dep in deps: if not pkgindex.has_key(dep): tarjan(dep) @@ -430,7 +426,15 @@ print "\nPerforming integrity checks..." print "==> parsing pkgbuilds" parse_pkgbuilds(loadrepos,arch) -repopkgs = {} +# fill provisions +for name,pkg in packages.iteritems(): + for prov in pkg.provides: + provname=prov.split("=")[0] + if not provisions.has_key(provname): + provisions[provname] = [] + provisions[provname].append(pkg) + +# fill repopkgs for name,pkg in packages.iteritems(): if pkg.repo in repos: repopkgs[name] = pkg -- cgit v1.2.2 From c573cb4a00969d8d81d0fdf9789e255f4d296d39 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Mon, 21 Sep 2009 16:20:46 -0400 Subject: Temporarily disable community sourceball generation Signed-off-by: Aaron Griffin --- cron-jobs/sourceballs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 46c2fa3..72baa32 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -2,7 +2,7 @@ ftpbase="/srv/ftp" srcbase="/srv/ftp/sources" -repos="core extra testing community" +repos="core extra testing" #community" . "$(dirname $0)/../db-functions" . "$(dirname $0)/../config" -- cgit v1.2.2 From 29b56cd5e289a35a24459af70dd6678b0c55cb9b Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Mon, 21 Sep 2009 18:21:33 -0400 Subject: Set/restore umask in the sourceballs cron job Signed-off-by: Aaron Griffin --- cron-jobs/sourceballs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 72baa32..0d8be62 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -11,6 +11,7 @@ LOCKFILE="/tmp/.sourceball.lock" cleanup () { rm -f "$LOCKFILE" + restore_umask exit 0 } @@ -27,6 +28,8 @@ fi trap cleanup 0 trap ctrl_c 2 +set_umask + /bin/touch "$LOCKFILE" dirname="$(/bin/readlink -f $(/usr/bin/dirname $0))" -- cgit v1.2.2 From 4498eca19bcf79cef2fd71e8ab0938371c6cba71 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Mon, 21 Sep 2009 18:24:24 -0400 Subject: Modify sourceball remove_old function The revampled getpkgname/getpkgver functions need the full path to the file now, not just the filename Signed-off-by: Aaron Griffin --- misc-scripts/make-sourceball | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/misc-scripts/make-sourceball b/misc-scripts/make-sourceball index f2d8415..11fac06 100755 --- a/misc-scripts/make-sourceball +++ b/misc-scripts/make-sourceball @@ -110,7 +110,7 @@ remove_old() { done for pkg in "$srcpath/$pkgname-"*; do - pkg="$(basename $pkg)" + [ -f "$pkg" ] || continue if [ "$(getpkgname $pkg)" == "$pkgname" ]; then skip=0 pver="$(getpkgver $pkg)" @@ -121,7 +121,7 @@ remove_old() { fi done if [ $skip -ne 1 ]; then - rm -f "$srcpath/$pkg" + rm -f "$pkg" fi fi done -- cgit v1.2.2 From 7af26d483b6987e0f3d42b25d8cb4e39f691a0a3 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Mon, 21 Sep 2009 19:11:42 -0400 Subject: Add a safety check for existence in pkg loop Signed-off-by: Aaron Griffin --- cron-jobs/sourceballs | 1 + 1 file changed, 1 insertion(+) diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 0d8be62..f285d7f 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -49,6 +49,7 @@ for repo in $repos; do fi cd $ftppath for pkg in *$PKGEXT; do + [ -f "$pkg" ] || continue pkgbase=$(getpkgbase $pkg) srcpath="$srcbase/" srcpkg="${pkg//$PKGEXT/$SRCEXT}" -- cgit v1.2.2 From 652fa7ad93bfa708f0764b64107a54f4dcaf17bd Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Tue, 22 Sep 2009 16:25:51 -0400 Subject: Renamed community-testing2* scripts We know the repo. Lets make the names nicer Signed-off-by: Aaron Griffin --- community-testing2community | 11 ----------- community-testing2community-any | 11 ----------- community-testing2community64 | 11 ----------- testing2community | 11 +++++++++++ testing2community-any | 11 +++++++++++ testing2community64 | 11 +++++++++++ 6 files changed, 33 insertions(+), 33 deletions(-) delete mode 100755 community-testing2community delete mode 100755 community-testing2community-any delete mode 100755 community-testing2community64 create mode 100755 testing2community create mode 100755 testing2community-any create mode 100755 testing2community64 diff --git a/community-testing2community b/community-testing2community deleted file mode 100755 index 557b39d..0000000 --- a/community-testing2community +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -if [ $# -le 0 ]; then - echo "usage: $(basename 0) [ [ Moving package '$pkg'" - $(dirname $0)/db-move "$pkg" "community-testing" "community" "i686" -done diff --git a/community-testing2community-any b/community-testing2community-any deleted file mode 100755 index e63a7ec..0000000 --- a/community-testing2community-any +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -if [ $# -le 0 ]; then - echo "usage: $(basename 0) [ [ Moving package '$pkg'" - $(dirname $0)/db-move "$pkg" "community-testing" "community" "any" -done diff --git a/community-testing2community64 b/community-testing2community64 deleted file mode 100755 index 3f88a4d..0000000 --- a/community-testing2community64 +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -if [ $# -le 0 ]; then - echo "usage: $(basename 0) [ [ Moving package '$pkg'" - $(dirname $0)/db-move "$pkg" "community-testing" "community" "x86_64" -done diff --git a/testing2community b/testing2community new file mode 100755 index 0000000..557b39d --- /dev/null +++ b/testing2community @@ -0,0 +1,11 @@ +#!/bin/bash + +if [ $# -le 0 ]; then + echo "usage: $(basename 0) [ [ Moving package '$pkg'" + $(dirname $0)/db-move "$pkg" "community-testing" "community" "i686" +done diff --git a/testing2community-any b/testing2community-any new file mode 100755 index 0000000..e63a7ec --- /dev/null +++ b/testing2community-any @@ -0,0 +1,11 @@ +#!/bin/bash + +if [ $# -le 0 ]; then + echo "usage: $(basename 0) [ [ Moving package '$pkg'" + $(dirname $0)/db-move "$pkg" "community-testing" "community" "any" +done diff --git a/testing2community64 b/testing2community64 new file mode 100755 index 0000000..3f88a4d --- /dev/null +++ b/testing2community64 @@ -0,0 +1,11 @@ +#!/bin/bash + +if [ $# -le 0 ]; then + echo "usage: $(basename 0) [ [ Moving package '$pkg'" + $(dirname $0)/db-move "$pkg" "community-testing" "community" "x86_64" +done -- cgit v1.2.2 From 09e556788e79fc86ce0c5ab7a6a26abcddcd0cbd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20B=E9langer?= Date: Tue, 22 Sep 2009 22:21:15 -0400 Subject: Fixed split packages support in sourceballs script MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Eric Bélanger Signed-off-by: Aaron Griffin --- misc-scripts/make-sourceball | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/misc-scripts/make-sourceball b/misc-scripts/make-sourceball index 11fac06..3d42f13 100755 --- a/misc-scripts/make-sourceball +++ b/misc-scripts/make-sourceball @@ -69,23 +69,23 @@ create_srcpackage() { pushd "$1" >/dev/null . "$BUILDSCRIPT" - echo "Creating source tarball for $pkgname-$pkgver-$pkgrel" + echo "Creating source tarball for $packagename-$pkgver-$pkgrel" if ! chk_license ${license[@]}; then #Removed so as not to clutter failed.txt - #echo -e "\t$pkgname license (${license[@]}) does not require source tarballs" >&2 + #echo -e "\t$packagename license (${license[@]}) does not require source tarballs" >&2 cleanup 0 fi - local logfile="$logpath/$pkgname" + local logfile="$logpath/$packagename" if ! /usr/bin/makepkg --allsource --ignorearch >"$logfile" 2>&1; then popd >/dev/null /bin/gzip -f -9 "$logfile" - die "\tFailed to download source for $pkgname-$pkgver-$pkgrel ($reponame-$_arch)" + die "\tFailed to download source for $packagename-$pkgver-$pkgrel ($reponame-$_arch)" fi /bin/rm -f "$logfile"{,.gz} - local pkg_file="${pkgname}-${pkgver}-${pkgrel}${SRCEXT}" + local pkg_file="${packagename}-${pkgver}-${pkgrel}${SRCEXT}" if [ ! -d "$srcpath" ]; then mkdir -p "$srcpath" @@ -109,9 +109,9 @@ remove_old() { cd .. done - for pkg in "$srcpath/$pkgname-"*; do + for pkg in "$srcpath/$packagename-"*; do [ -f "$pkg" ] || continue - if [ "$(getpkgname $pkg)" == "$pkgname" ]; then + if [ "$(getpkgname $pkg)" == "$packagename" ]; then skip=0 pver="$(getpkgver $pkg)" for v in $PKGVERS; do @@ -139,7 +139,7 @@ set_umask cd "$WORKDIR" if /usr/bin/svn export -q "$SVNREPO/$packagename" $packagename; then - remove_old "$pkgname/repos/" + remove_old "$packagename/repos/" create_srcpackage "$packagename/repos/$reponame-$_arch" else die "\tPackage '$packagename' does not exist in repo '$reponame-$_arch'" -- cgit v1.2.2 From b6ca5aa8ae15e98a01f3897fa4931319aad94732 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Wed, 23 Sep 2009 07:57:20 -0700 Subject: make-sourceballs: clean up output Only output "Creating sourceball" text when we're actually creating it Signed-off-by: Aaron Griffin --- misc-scripts/make-sourceball | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/misc-scripts/make-sourceball b/misc-scripts/make-sourceball index 3d42f13..75eee07 100755 --- a/misc-scripts/make-sourceball +++ b/misc-scripts/make-sourceball @@ -69,12 +69,12 @@ create_srcpackage() { pushd "$1" >/dev/null . "$BUILDSCRIPT" - echo "Creating source tarball for $packagename-$pkgver-$pkgrel" - if ! chk_license ${license[@]}; then #Removed so as not to clutter failed.txt #echo -e "\t$packagename license (${license[@]}) does not require source tarballs" >&2 cleanup 0 + else + echo "Creating source tarball for $packagename-$pkgver-$pkgrel" fi local logfile="$logpath/$packagename" -- cgit v1.2.2 From 588cbbff128c72ac1871c26f9503ee41ea59bdfd Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Wed, 23 Sep 2009 19:05:18 -0400 Subject: make-sourceball: Re-add file parsing of name/ver getpkgname / getpkgver were replaced with versions that scanned the .PKGINFO file. These don't work for source packages, so add the functionality back in for this case Signed-off-by: Aaron Griffin --- misc-scripts/make-sourceball | 31 ++++++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/misc-scripts/make-sourceball b/misc-scripts/make-sourceball index 75eee07..865e0e7 100755 --- a/misc-scripts/make-sourceball +++ b/misc-scripts/make-sourceball @@ -43,6 +43,27 @@ die() { cleanup 1 } +pkgname_from_src() { + local tmp + tmp=${1##*/} + tmp=${tmp%$SRCEXT} + for a in ${ARCHES[@]}; do + tmp=${tmp%-$a} + done + tmp=${tmp%-any} + echo ${tmp%-*-*} +} + +pkgver_from_src() { + tmp=${1##*/} + tmp=${tmp%$SRCEXT} + for a in ${ARCHES[@]}; do + tmp=${tmp%-$a} + done + tmp=${tmp%-any} + echo $tmp | sed 's|.*-\(.*-.*\)$|\1|g' +} + #usage: chk_license ${license[@]}" chk_license() { #The -f flag skips this check @@ -109,11 +130,11 @@ remove_old() { cd .. done - for pkg in "$srcpath/$packagename-"*; do - [ -f "$pkg" ] || continue - if [ "$(getpkgname $pkg)" == "$packagename" ]; then + for srcpkg in "$srcpath/$packagename-"*; do + [ -f "$srcpkg" ] || continue + if [ "$(pkgname_from_src $srcpkg)" == "$packagename" ]; then skip=0 - pver="$(getpkgver $pkg)" + pver="$(pkgver_from_src $srcpkg)" for v in $PKGVERS; do if [ "$v" = "$pver" ]; then skip=1 @@ -121,7 +142,7 @@ remove_old() { fi done if [ $skip -ne 1 ]; then - rm -f "$pkg" + rm -f "$srcpkg" fi fi done -- cgit v1.2.2 From de547b70c1b3f6dfc163d96318d6f915446bbc4c Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Thu, 24 Sep 2009 15:01:08 -0700 Subject: Fixed test for existence of split package sources MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This patch also removes the unused CARCH variable. Signed-off-by: Eric Bélanger [Aaron: Broke srcpkgbase logic into srcpkgname/srcpkgbase to make it more readable] Signed-off-by: Aaron Griffin --- cron-jobs/sourceballs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index f285d7f..09b1df7 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -41,7 +41,6 @@ echo "Errors occured during run:" > "$srcbase/errors.txt" for repo in $repos; do for arch in ${ARCHES[@]} any; do - export CARCH="$arch" ftppath="$ftpbase/$repo/os/$arch" if [ ! -d "$ftppath" ]; then echo "FTP path does not exist: $ftppath" >2 @@ -54,6 +53,8 @@ for repo in $repos; do srcpath="$srcbase/" srcpkg="${pkg//$PKGEXT/$SRCEXT}" srcpkg="${srcpkg//-$arch/}" + srcpkgname="${srcpkg%-*-*$SRCEXT}" + srcpkgbase="${srcpkg/$srcpkgname/$pkgbase}" #Don't do anything for package in this 'blacklist' if grep "^$pkgbase\$" "$dirname/sourceballs.skip" >/dev/null 2>&1; then @@ -67,7 +68,7 @@ for repo in $repos; do force="-f" fi - if [ ! -f "$srcpath$srcpkg" ]; then + if [ ! \( -f "$srcpath$srcpkg" -o -f "$srcpath$srcpkgbase" \) ]; then if ! $dirname/../misc-scripts/make-sourceball $force \ $pkgbase $repo $arch 2>>"$srcbase/errors.txt"; then FAILED_PKGS="$FAILED_PKGS $pkg" -- cgit v1.2.2 From 2f6846c451b1b4887fd87a2bc32945cce0b943a3 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Fri, 25 Sep 2009 15:07:49 -0700 Subject: sourceballs: skip based on pkgbase failures If one portion of a split package fails, assume that all packages from the same $pkgbase are going to fail as well and skip them for this run. Signed-off-by: Aaron Griffin --- cron-jobs/sourceballs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 09b1df7..b7a4885 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -61,6 +61,11 @@ for repo in $repos; do continue fi + #This pkgbase has already failed. No sense in trying it again this run + if echo $FAILED_PKGS | grep "\<$pkgbase\>" >&/dev/null; then + continue + fi + #Use this file to 'whitelist' or force building some sourceballs, # skipping the license check force="" @@ -71,7 +76,7 @@ for repo in $repos; do if [ ! \( -f "$srcpath$srcpkg" -o -f "$srcpath$srcpkgbase" \) ]; then if ! $dirname/../misc-scripts/make-sourceball $force \ $pkgbase $repo $arch 2>>"$srcbase/errors.txt"; then - FAILED_PKGS="$FAILED_PKGS $pkg" + FAILED_PKGS="$FAILED_PKGS $pkgbase" fi fi done @@ -80,7 +85,7 @@ done if [ -n "$FAILED_PKGS" ]; then [ -e "$srcbase/failed.txt" ] && /bin/mv "$srcbase/failed.txt" "$srcbase/failed.txt.old" - echo "The following packages failed:" > "$srcbase/failed.txt" + echo "The following package bases failed:" > "$srcbase/failed.txt" echo -e $FAILED_PKGS | sed "s| |\n|g" | sort -u >> "$srcbase/failed.txt" fi -- cgit v1.2.2 From f0e417f2be331b937dd56bf970c59370a55b6301 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20B=C3=A9langer?= Date: Wed, 14 Oct 2009 15:18:06 -0400 Subject: Moved ALLOWED_LICENSES declaration to config file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Eric Bélanger Signed-off-by: Aaron Griffin --- config | 3 +++ misc-scripts/make-sourceball | 3 --- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/config b/config index 86a5eca..24ec37a 100644 --- a/config +++ b/config @@ -13,3 +13,6 @@ BUILDSCRIPT="PKGBUILD" DBEXT=".db.tar.gz" PKGEXT=".pkg.tar.gz" SRCEXT=".src.tar.gz" + +# Allowed licenses: get sourceballs only for licenses in this array +ALLOWED_LICENSES=('GPL' 'GPL1' 'GPL2' 'LGPL' 'LGPL1' 'LGPL2') diff --git a/misc-scripts/make-sourceball b/misc-scripts/make-sourceball index 865e0e7..462549f 100755 --- a/misc-scripts/make-sourceball +++ b/misc-scripts/make-sourceball @@ -1,8 +1,5 @@ #!/bin/bash -# Allowed licenses: build only for licenses in this array -ALLOWED_LICENSES=('GPL' 'GPL1' 'GPL2' 'LGPL' 'LGPL1' 'LGPL2') - if [ $# -ne 3 -a $# -ne 4 ]; then echo "usage: $(basename $0) [-f] " echo " -f Force building. Skip license checks" -- cgit v1.2.2 From 3d45f943876853b3a745914c8a5a74ffc039f99b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20B=C3=A9langer?= Date: Wed, 14 Oct 2009 15:18:07 -0400 Subject: Moved chk_license to db-functions file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Eric Bélanger Signed-off-by: Aaron Griffin --- db-functions | 16 ++++++++++++++++ misc-scripts/make-sourceball | 24 +----------------------- 2 files changed, 17 insertions(+), 23 deletions(-) diff --git a/db-functions b/db-functions index f4f9ee9..6611a92 100644 --- a/db-functions +++ b/db-functions @@ -110,4 +110,20 @@ get_repos_for_host() { fi } +#usage: chk_license ${license[@]}" +chk_license() { + local l + for l in "$@"; do + l="$(echo $l | tr '[:upper:]' '[:lower:]')" + for allowed in ${ALLOWED_LICENSES[@]}; do + allowed="$(echo $allowed | tr '[:upper:]' '[:lower:]')" + if [ "$l" = "$allowed" ]; then + return 0 + fi + done + done + + return 1 +} + # vim: set ts=4 sw=4 noet ft=sh: diff --git a/misc-scripts/make-sourceball b/misc-scripts/make-sourceball index 462549f..1e7eabc 100755 --- a/misc-scripts/make-sourceball +++ b/misc-scripts/make-sourceball @@ -61,33 +61,11 @@ pkgver_from_src() { echo $tmp | sed 's|.*-\(.*-.*\)$|\1|g' } -#usage: chk_license ${license[@]}" -chk_license() { - #The -f flag skips this check - if [ $FORCE = 1 ]; then - return 0 - fi - - local l - for l in "$@"; do - l="$(echo $l | tr '[:upper:]' '[:lower:]')" - for allowed in ${ALLOWED_LICENSES[@]}; do - allowed="$(echo $allowed | tr '[:upper:]' '[:lower:]')" - if [ "$l" = "$allowed" ]; then - return 0 - fi - done - done - - return 1 -} - create_srcpackage() { if [ -d "$1" ]; then pushd "$1" >/dev/null . "$BUILDSCRIPT" - - if ! chk_license ${license[@]}; then + if ! [ $FORCE == 1 ] && ! chk_license ${license[@]} ; then #Removed so as not to clutter failed.txt #echo -e "\t$packagename license (${license[@]}) does not require source tarballs" >&2 cleanup 0 -- cgit v1.2.2 From 96bcc18c5df5c7ef81a38c7338467ae0c75eca08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20B=C3=A9langer?= Date: Wed, 14 Oct 2009 15:18:08 -0400 Subject: Added source-cleanup directory support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Eric Bélanger Signed-off-by: Aaron Griffin --- config | 1 + misc-scripts/make-sourceball | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/config b/config index 24ec37a..92def37 100644 --- a/config +++ b/config @@ -5,6 +5,7 @@ FTP_OS_SUFFIX="os" SVNREPO="file:///srv/svn-packages" CLEANUP_DESTDIR="/srv/package-cleanup" +SOURCE_CLEANUP_DESTDIR="/srv/source-cleanup" STAGING="$HOME/staging" TMPDIR="/srv/tmp" diff --git a/misc-scripts/make-sourceball b/misc-scripts/make-sourceball index 1e7eabc..8899fca 100755 --- a/misc-scripts/make-sourceball +++ b/misc-scripts/make-sourceball @@ -117,7 +117,7 @@ remove_old() { fi done if [ $skip -ne 1 ]; then - rm -f "$srcpkg" + mv "$srcpkg" $SOURCE_CLEANUP_DESTDIR fi fi done -- cgit v1.2.2 From ee0820aa9c71e8a76f16b11c7ad77be7cb8e1ce6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20B=C3=A9langer?= Date: Wed, 14 Oct 2009 15:18:09 -0400 Subject: Added new cronjob script to clean up unneeded sourceballs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This script is intended to be a weekly cron-job. It supplements the current sourceballs cleanup in make-sourceball that only removes the old sourceballs. This script removes the sourceballs of packages that were either removed completely from the repo or moved to a splitted package. It also checks the license of the packages which have a sourceball to see, in case of a license change, if the hosting of the sources is still necessary. Signed-off-by: Eric Bélanger Signed-off-by: Aaron Griffin --- cron-jobs/sourceballs-cleanup | 61 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100755 cron-jobs/sourceballs-cleanup diff --git a/cron-jobs/sourceballs-cleanup b/cron-jobs/sourceballs-cleanup new file mode 100755 index 0000000..c8d7b15 --- /dev/null +++ b/cron-jobs/sourceballs-cleanup @@ -0,0 +1,61 @@ +#!/bin/bash + +. "$(dirname $0)/../db-functions" +. "$(dirname $0)/../config" + +srcpath="$FTP_BASE/sources/" +logfile="$srcpath/cleanup.txt" + +LOCKFILE="/tmp/.sourceball-cleanup.lock" +WORKDIR="/tmp/sourceball-cleanup.$packagename.$UID" + +cleanup () { + restore_umask + rm -rf "$WORKDIR" + rm -f "$LOCKFILE" + exit 0 +} + +ctrl_c() { + echo "Interrupted" >&2 + cleanup 0 +} + +if [ -f "$LOCKFILE" ]; then + owner="$(/usr/bin/stat -c %U $LOCKFILE)" + echo "error: source tarball generation is already in progress (started by $owner)" + exit 1 +fi + +trap cleanup 0 +trap ctrl_c 2 + +/bin/touch "$LOCKFILE" + +#adjust the nice level to run at a lower priority +/usr/bin/renice +10 -p $$ > /dev/null + +set_umask +/bin/mkdir -p "$WORKDIR" +cd "$WORKDIR" + +[ -e "$logfile" ] && /bin/mv "$logfile" "$logfile.old" +echo "Orphaned sourceballs:" > "$logfile" + +for sourceball in "$srcpath"/*$SRCEXT; do + packagename=$(basename $sourceball) + packagename=${packagename%-*-*$SRCEXT} + + if ! /usr/bin/svn export -q --force "$SVNREPO/$packagename" "$packagename" >/dev/null 2>&1 ; then + echo "$packagename : no longer in svn. Removing sourceball." >> "$logfile" + mv $sourceball $SOURCE_CLEANUP_DESTDIR + elif [ -z "$(ls -A "$packagename/repos")" ]; then + echo "$packagename : no longer in repos but trunk is still in svn. Removing sourceball." >> "$logfile" + mv $sourceball $SOURCE_CLEANUP_DESTDIR + elif ! source "$packagename/trunk/$BUILDSCRIPT" && chk_license ${license[@]}; then + echo "$packagename : source hosting no longer required by license. Removing sourceball." >> "$logfile" + mv $sourceball $SOURCE_CLEANUP_DESTDIR + fi +done + +cleanup 0 -- cgit v1.2.2 From 41bf9fc77f4c7856a481552119b61c908743b28b Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Thu, 29 Oct 2009 19:09:28 -0700 Subject: Echo pkginfo parsing errors to stderr Signed-off-by: Aaron Griffin --- db-functions | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/db-functions b/db-functions index 6611a92..a4cffd7 100644 --- a/db-functions +++ b/db-functions @@ -67,7 +67,7 @@ getpkgname() { _name="$(_grep_pkginfo "$1" "^pkgname")" if [ -z "$_name" ]; then - echo "ERROR: Package '$1' has no pkgname in the PKGINFO. Fail!" + echo "ERROR: Package '$1' has no pkgname in the PKGINFO. Fail!" >&2 exit 1 fi @@ -80,7 +80,7 @@ getpkgver() { _ver="$(_grep_pkginfo "$1" "^pkgver")" if [ -z "$_ver" ]; then - echo "ERROR: Package '$1' has no pkgver in the PKGINFO. Fail!" + echo "ERROR: Package '$1' has no pkgver in the PKGINFO. Fail!" >&2 exit 1 fi @@ -92,7 +92,7 @@ check_pkg_arch () { #check_pkg_arch pkgfile arch _arch="$(_grep_pkginfo "$1" "^arch")" if [ -z "$_arch" ]; then - echo "ERROR: Package '$1' has no arch in the PKGINFO. Fail!" + echo "ERROR: Package '$1' has no arch in the PKGINFO. Fail!" >&2 return 1 fi if [ "$_arch" = "$2" ]; then -- cgit v1.2.2 From 3c1eb5b59792fbaafde12a7408201fb1e0a77df6 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Thu, 29 Oct 2009 19:10:26 -0700 Subject: Rename createFileLists to be more like the others Signed-off-by: Aaron Griffin --- cron-jobs/create-filelists | 82 ++++++++++++++++++++++++++++++++++++++++++++++ cron-jobs/createFileLists | 82 ---------------------------------------------- 2 files changed, 82 insertions(+), 82 deletions(-) create mode 100755 cron-jobs/create-filelists delete mode 100755 cron-jobs/createFileLists diff --git a/cron-jobs/create-filelists b/cron-jobs/create-filelists new file mode 100755 index 0000000..d936853 --- /dev/null +++ b/cron-jobs/create-filelists @@ -0,0 +1,82 @@ +#!/bin/bash + +reposdir=/srv/ftp/ +targetdir=$reposdir +repos="core extra testing community" + +. "$(dirname $0)/../db-functions" +. "$(dirname $0)/../config" + +if [ -f "/tmp/createFileList.lock" ]; then + echo "Error: createFileList allready in progress." + exit 1 +fi + +touch "/tmp/createFileList.lock" || exit 1 +TMPDIR=$(mktemp -d /tmp/createFileList.XXXXXX) || exit 1 +CACHEDIR=$(mktemp -d /tmp/createFileList.XXXXXX) || exit 1 + +#adjust the nice level to run at a lower priority +/usr/bin/renice +10 -p $$ > /dev/null + +case "${DBEXT}" in + *.gz) TAR_OPT="z" ;; + *.bz2) TAR_OPT="j" ;; + *.xz) TAR_OPT="J" ;; + *) echo "Unknown compression type for DBEXT=${DBEXT}" && exit 1 ;; +esac + +FILESEXT="${DBEXT//db/files}" + +cd $reposdir +for repo in $repos; do + REPO_DB_FILE=${repo}$FILESEXT + for arch in ${ARCHES[@]}; do + repodir=${repo}/os/${arch}/ + cached="no" + + # extract old file archive + if [ -f ${targetdir}${repodir}${REPO_DB_FILE} ]; then + mkdir -p ${CACHEDIR}/${repodir} + bsdtar -xf ${targetdir}${repodir}${REPO_DB_FILE} -C ${CACHEDIR}/${repodir} + cached="yes" + fi + + # create file lists + for pkg in $repodir*${PKGEXT}; do + basename=$(basename $pkg) + pkgname=$(getpkgname $basename) + tmppkgdir=${TMPDIR}/${repodir}${pkgname}/ + mkdir -p $tmppkgdir + if [ -f "${CACHEDIR}/${repodir}${pkgname}/files" ]; then +# echo "cache: $pkgname" + mv ${CACHEDIR}/${repodir}${pkgname}/files ${tmppkgdir}files + else +# echo "$repo/$arch: $pkgname" + echo '%FILES%' > ${tmppkgdir}files + bsdtar --exclude=.* -tf $pkg >> ${tmppkgdir}files + cached="no" + fi + done + + # create new file archive + if [ "$cached" == "no" ]; then + # at least one package has changed, so let's rebuild the archive +# echo "creating ${REPO_DB_FILE}/${arch}" + pkgdir=${targetdir}${repodir} + mkdir -p $pkgdir + cd ${TMPDIR}/${repodir} + [ -f "${pkgdir}${REPO_DB_FILE}.old" ] && rm "${pkgdir}${REPO_DB_FILE}.old" + [ -f "${pkgdir}${REPO_DB_FILE}" ] && mv "${pkgdir}${REPO_DB_FILE}" "${pkgdir}${REPO_DB_FILE}.old" + bsdtar --exclude=*${DBEXT//\.db/} -c${TAR_OPT}f ${pkgdir}${REPO_DB_FILE} * + fi + + cd $reposdir + done +done + +cd - >/dev/null +rm -rf $TMPDIR || exit 1 +rm -rf $CACHEDIR || exit 1 +rm -f "/tmp/createFileList.lock" || exit 1 +# echo 'done' diff --git a/cron-jobs/createFileLists b/cron-jobs/createFileLists deleted file mode 100755 index d936853..0000000 --- a/cron-jobs/createFileLists +++ /dev/null @@ -1,82 +0,0 @@ -#!/bin/bash - -reposdir=/srv/ftp/ -targetdir=$reposdir -repos="core extra testing community" - -. "$(dirname $0)/../db-functions" -. "$(dirname $0)/../config" - -if [ -f "/tmp/createFileList.lock" ]; then - echo "Error: createFileList allready in progress." - exit 1 -fi - -touch "/tmp/createFileList.lock" || exit 1 -TMPDIR=$(mktemp -d /tmp/createFileList.XXXXXX) || exit 1 -CACHEDIR=$(mktemp -d /tmp/createFileList.XXXXXX) || exit 1 - -#adjust the nice level to run at a lower priority -/usr/bin/renice +10 -p $$ > /dev/null - -case "${DBEXT}" in - *.gz) TAR_OPT="z" ;; - *.bz2) TAR_OPT="j" ;; - *.xz) TAR_OPT="J" ;; - *) echo "Unknown compression type for DBEXT=${DBEXT}" && exit 1 ;; -esac - -FILESEXT="${DBEXT//db/files}" - -cd $reposdir -for repo in $repos; do - REPO_DB_FILE=${repo}$FILESEXT - for arch in ${ARCHES[@]}; do - repodir=${repo}/os/${arch}/ - cached="no" - - # extract old file archive - if [ -f ${targetdir}${repodir}${REPO_DB_FILE} ]; then - mkdir -p ${CACHEDIR}/${repodir} - bsdtar -xf ${targetdir}${repodir}${REPO_DB_FILE} -C ${CACHEDIR}/${repodir} - cached="yes" - fi - - # create file lists - for pkg in $repodir*${PKGEXT}; do - basename=$(basename $pkg) - pkgname=$(getpkgname $basename) - tmppkgdir=${TMPDIR}/${repodir}${pkgname}/ - mkdir -p $tmppkgdir - if [ -f "${CACHEDIR}/${repodir}${pkgname}/files" ]; then -# echo "cache: $pkgname" - mv ${CACHEDIR}/${repodir}${pkgname}/files ${tmppkgdir}files - else -# echo "$repo/$arch: $pkgname" - echo '%FILES%' > ${tmppkgdir}files - bsdtar --exclude=.* -tf $pkg >> ${tmppkgdir}files - cached="no" - fi - done - - # create new file archive - if [ "$cached" == "no" ]; then - # at least one package has changed, so let's rebuild the archive -# echo "creating ${REPO_DB_FILE}/${arch}" - pkgdir=${targetdir}${repodir} - mkdir -p $pkgdir - cd ${TMPDIR}/${repodir} - [ -f "${pkgdir}${REPO_DB_FILE}.old" ] && rm "${pkgdir}${REPO_DB_FILE}.old" - [ -f "${pkgdir}${REPO_DB_FILE}" ] && mv "${pkgdir}${REPO_DB_FILE}" "${pkgdir}${REPO_DB_FILE}.old" - bsdtar --exclude=*${DBEXT//\.db/} -c${TAR_OPT}f ${pkgdir}${REPO_DB_FILE} * - fi - - cd $reposdir - done -done - -cd - >/dev/null -rm -rf $TMPDIR || exit 1 -rm -rf $CACHEDIR || exit 1 -rm -f "/tmp/createFileList.lock" || exit 1 -# echo 'done' -- cgit v1.2.2 From 38c7241ad2528b29a65a0b81b980e76d1046e94e Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Thu, 29 Oct 2009 19:18:41 -0700 Subject: Cleanup create-filelists * Quote absolutely everything * Move directory slashes around, for readability * Move a cd to the top of the loop Signed-off-by: Aaron Griffin --- cron-jobs/create-filelists | 55 +++++++++++++++++++++++----------------------- 1 file changed, 27 insertions(+), 28 deletions(-) diff --git a/cron-jobs/create-filelists b/cron-jobs/create-filelists index d936853..b49f722 100755 --- a/cron-jobs/create-filelists +++ b/cron-jobs/create-filelists @@ -1,7 +1,7 @@ #!/bin/bash -reposdir=/srv/ftp/ -targetdir=$reposdir +reposdir="/srv/ftp" +targetdir="/srv/ftp" repos="core extra testing community" . "$(dirname $0)/../db-functions" @@ -13,8 +13,8 @@ if [ -f "/tmp/createFileList.lock" ]; then fi touch "/tmp/createFileList.lock" || exit 1 -TMPDIR=$(mktemp -d /tmp/createFileList.XXXXXX) || exit 1 -CACHEDIR=$(mktemp -d /tmp/createFileList.XXXXXX) || exit 1 +TMPDIR="$(mktemp -d /tmp/createFileList.XXXXXX)" || exit 1 +CACHEDIR="$(mktemp -d /tmp/createFileList.XXXXXX)" || exit 1 #adjust the nice level to run at a lower priority /usr/bin/renice +10 -p $$ > /dev/null @@ -28,33 +28,34 @@ esac FILESEXT="${DBEXT//db/files}" -cd $reposdir for repo in $repos; do - REPO_DB_FILE=${repo}$FILESEXT + REPO_DB_FILE="${repo}$FILESEXT" for arch in ${ARCHES[@]}; do - repodir=${repo}/os/${arch}/ + cd "$reposdir" + + repodir="${repo}/os/${arch}" cached="no" # extract old file archive - if [ -f ${targetdir}${repodir}${REPO_DB_FILE} ]; then - mkdir -p ${CACHEDIR}/${repodir} - bsdtar -xf ${targetdir}${repodir}${REPO_DB_FILE} -C ${CACHEDIR}/${repodir} + if [ -f "${targetdir}/${repodir}/${REPO_DB_FILE}" ]; then + mkdir -p "${CACHEDIR}/${repodir}" + bsdtar -xf "${targetdir}/${repodir}/${REPO_DB_FILE}" -C "${CACHEDIR}/${repodir}" cached="yes" fi # create file lists for pkg in $repodir*${PKGEXT}; do - basename=$(basename $pkg) - pkgname=$(getpkgname $basename) - tmppkgdir=${TMPDIR}/${repodir}${pkgname}/ - mkdir -p $tmppkgdir - if [ -f "${CACHEDIR}/${repodir}${pkgname}/files" ]; then + basename="$(basename "$pkg")" + pkgname="$(getpkgname "$basename")" + tmppkgdir="${TMPDIR}/${repodir}/${pkgname}" + mkdir -p "$tmppkgdir" + if [ -f "${CACHEDIR}/${repodir}/${pkgname}/files" ]; then # echo "cache: $pkgname" - mv ${CACHEDIR}/${repodir}${pkgname}/files ${tmppkgdir}files + mv "${CACHEDIR}/${repodir}$/{pkgname}/files" "${tmppkgdir}/files" else # echo "$repo/$arch: $pkgname" - echo '%FILES%' > ${tmppkgdir}files - bsdtar --exclude=.* -tf $pkg >> ${tmppkgdir}files + echo '%FILES%' > "${tmppkgdir}/files" + bsdtar --exclude=.* -tf "$pkg" >> "${tmppkgdir}/files" cached="no" fi done @@ -63,20 +64,18 @@ for repo in $repos; do if [ "$cached" == "no" ]; then # at least one package has changed, so let's rebuild the archive # echo "creating ${REPO_DB_FILE}/${arch}" - pkgdir=${targetdir}${repodir} - mkdir -p $pkgdir - cd ${TMPDIR}/${repodir} - [ -f "${pkgdir}${REPO_DB_FILE}.old" ] && rm "${pkgdir}${REPO_DB_FILE}.old" - [ -f "${pkgdir}${REPO_DB_FILE}" ] && mv "${pkgdir}${REPO_DB_FILE}" "${pkgdir}${REPO_DB_FILE}.old" - bsdtar --exclude=*${DBEXT//\.db/} -c${TAR_OPT}f ${pkgdir}${REPO_DB_FILE} * + pkgdir="${targetdir}/${repodir}" + mkdir -p "$pkgdir" + cd "${TMPDIR}/${repodir}" + [ -f "${pkgdir}/${REPO_DB_FILE}.old" ] && rm "${pkgdir}/${REPO_DB_FILE}.old" + [ -f "${pkgdir}/${REPO_DB_FILE}" ] && mv "${pkgdir}/${REPO_DB_FILE}" "${pkgdir}/${REPO_DB_FILE}.old" + bsdtar --exclude=*${DBEXT//\.db/} -c${TAR_OPT}f "${pkgdir}/${REPO_DB_FILE}" * fi - - cd $reposdir done done cd - >/dev/null -rm -rf $TMPDIR || exit 1 -rm -rf $CACHEDIR || exit 1 +rm -rf "$TMPDIR" || exit 1 +rm -rf "$CACHEDIR" || exit 1 rm -f "/tmp/createFileList.lock" || exit 1 # echo 'done' -- cgit v1.2.2 From 78c6a7916836fd10fb232d617b6eca3ae5c48089 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Thu, 29 Oct 2009 19:19:44 -0700 Subject: create-filelists: Call getpkgname for real Signed-off-by: Aaron Griffin --- cron-jobs/create-filelists | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cron-jobs/create-filelists b/cron-jobs/create-filelists index b49f722..35325cf 100755 --- a/cron-jobs/create-filelists +++ b/cron-jobs/create-filelists @@ -45,8 +45,7 @@ for repo in $repos; do # create file lists for pkg in $repodir*${PKGEXT}; do - basename="$(basename "$pkg")" - pkgname="$(getpkgname "$basename")" + pkgname="$(getpkgname "$pkg")" tmppkgdir="${TMPDIR}/${repodir}/${pkgname}" mkdir -p "$tmppkgdir" if [ -f "${CACHEDIR}/${repodir}/${pkgname}/files" ]; then -- cgit v1.2.2 From 1857fca2d204011852cf2309b66381b2fb4f6535 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20B=C3=A4chler?= Date: Sat, 31 Oct 2009 19:06:33 +0100 Subject: Fix a typo introduced in 38c7241ad2528b29a65a0b81b980e76d1046e94e --- cron-jobs/create-filelists | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cron-jobs/create-filelists b/cron-jobs/create-filelists index 35325cf..727fe58 100755 --- a/cron-jobs/create-filelists +++ b/cron-jobs/create-filelists @@ -50,7 +50,7 @@ for repo in $repos; do mkdir -p "$tmppkgdir" if [ -f "${CACHEDIR}/${repodir}/${pkgname}/files" ]; then # echo "cache: $pkgname" - mv "${CACHEDIR}/${repodir}$/{pkgname}/files" "${tmppkgdir}/files" + mv "${CACHEDIR}/${repodir}/${pkgname}/files" "${tmppkgdir}/files" else # echo "$repo/$arch: $pkgname" echo '%FILES%' > "${tmppkgdir}/files" -- cgit v1.2.2 From 782e8d67158115e8040215bcfef7e8ba54c6d04e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20B=C3=A4chler?= Date: Sat, 31 Oct 2009 19:17:39 +0100 Subject: create-filelists: add community-testing --- cron-jobs/create-filelists | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cron-jobs/create-filelists b/cron-jobs/create-filelists index 727fe58..840535e 100755 --- a/cron-jobs/create-filelists +++ b/cron-jobs/create-filelists @@ -2,7 +2,7 @@ reposdir="/srv/ftp" targetdir="/srv/ftp" -repos="core extra testing community" +repos="core extra testing community community-testing" . "$(dirname $0)/../db-functions" . "$(dirname $0)/../config" -- cgit v1.2.2 From c54f6c91dca7d84480f22aee60dda5861c73712e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20B=C3=A4chler?= Date: Sat, 31 Oct 2009 19:28:38 +0100 Subject: create-filelists: fix another typo --- cron-jobs/create-filelists | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cron-jobs/create-filelists b/cron-jobs/create-filelists index 840535e..2a15e6a 100755 --- a/cron-jobs/create-filelists +++ b/cron-jobs/create-filelists @@ -44,7 +44,7 @@ for repo in $repos; do fi # create file lists - for pkg in $repodir*${PKGEXT}; do + for pkg in $repodir/*${PKGEXT}; do pkgname="$(getpkgname "$pkg")" tmppkgdir="${TMPDIR}/${repodir}/${pkgname}" mkdir -p "$tmppkgdir" -- cgit v1.2.2 From 27e8f1071c18b2f0cbf4deb84a1222db64a3d5dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20B=C3=A4chler?= Date: Sat, 31 Oct 2009 20:07:17 +0100 Subject: create-filelists: Add pkgver-pkgrel to directory name In the past, pkgname-pkgver-pkgrel was extracted from the filename and sadly named "pkgname". Restore this behaviour --- cron-jobs/create-filelists | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/cron-jobs/create-filelists b/cron-jobs/create-filelists index 2a15e6a..c9d7db9 100755 --- a/cron-jobs/create-filelists +++ b/cron-jobs/create-filelists @@ -46,11 +46,12 @@ for repo in $repos; do # create file lists for pkg in $repodir/*${PKGEXT}; do pkgname="$(getpkgname "$pkg")" - tmppkgdir="${TMPDIR}/${repodir}/${pkgname}" + pkgver="$(getpkgver "$pkg")" + tmppkgdir="${TMPDIR}/${repodir}/${pkgname}-${pkgver}" mkdir -p "$tmppkgdir" - if [ -f "${CACHEDIR}/${repodir}/${pkgname}/files" ]; then + if [ -f "${CACHEDIR}/${repodir}/${pkgname}-${pkgver}/files" ]; then # echo "cache: $pkgname" - mv "${CACHEDIR}/${repodir}/${pkgname}/files" "${tmppkgdir}/files" + mv "${CACHEDIR}/${repodir}/${pkgname}-${pkgver}/files" "${tmppkgdir}/files" else # echo "$repo/$arch: $pkgname" echo '%FILES%' > "${tmppkgdir}/files" -- cgit v1.2.2 From cd84eaf5a82cae5414245cc24eb94b06fc1b9b4f Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Mon, 15 Feb 2010 23:35:26 +0100 Subject: remove unused script Signed-off-by: Pierre Schmitz --- convert-to-any | 71 ---------------------------------------------------------- 1 file changed, 71 deletions(-) delete mode 100755 convert-to-any diff --git a/convert-to-any b/convert-to-any deleted file mode 100755 index 53d1a7b..0000000 --- a/convert-to-any +++ /dev/null @@ -1,71 +0,0 @@ -#!/bin/bash -# -# Converts an existing architecture-independent package -# for i686 or x86_64 into a package with "arch = any" -# - -# -- Abhishek Dasgupta - -[ "$UID" = "" ] && UID=$(uid) -OUTDIR="$(pwd)" -WORKDIR="/tmp/convert-to-any.$UID" - -if [ $# -ne 1 ]; then - echo "Syntax: $(basename $0) " - exit 1 -fi - -. "$(dirname $0)/db-functions" -. "$(dirname $0)/config" - -cleanup() { - trap '' 0 2 - rm -rf "$WORKDIR" - [ "$1" ] && exit $1 -} - -ctrl_c() { - echo "Interrupted" >&2 - cleanup 0 -} - -die() { - echo "$*" >&2 - cleanup 1 -} - -mkdir -p "$WORKDIR/build" - -oldpkgname="$1" - -if [ -z "$oldpkgname" ]; then - die "convert-to-any: which package to convert?" -fi - -pkg="$(basename $oldpkgname)" -newpkgname=$(echo $pkg | sed "s/-\(i686\|x86_64\)$PKGEXT/-any$PKGEXT/") - -if ! cp "$oldpkgname" "$WORKDIR/build/$pkg"; then - die "convert-to-any: failed to copy package to $WORKDIR" -fi -pushd "$WORKDIR/build" >/dev/null - -# Conversion of i686 package into "any" package. -mkdir -p package -if ! fakeroot bsdtar xf "$pkg" -C package; then - die "convert-to-any: error in extracting $oldpkgname" -fi - -sed -i "s/arch = \(i686\|x86_64\)/arch = any/g" package/.PKGINFO -pushd package >/dev/null -case "$newpkgname" in - *tar.gz) TAR_OPT="z" ;; - *tar.bz2) TAR_OPT="j" ;; - *tar.xz) TAR_OPT="J" ;; - *) die "$newpkgname does not have a valid archive extension." ;; -esac -fakeroot bsdtar c${TAR_OPT}f "$OUTDIR/$newpkgname" .PKGINFO * -popd >/dev/null - -popd >/dev/null -cleanup -- cgit v1.2.2 From dea6954635ff0eccd116c384065c1ad745fcd1d2 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 21 Feb 2010 16:50:34 +0100 Subject: add dry run option to ftpdir-cleanup Signed-off-by: Pierre Schmitz --- config | 1 + misc-scripts/ftpdir-cleanup | 12 +++++++----- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/config b/config index 92def37..7864532 100644 --- a/config +++ b/config @@ -5,6 +5,7 @@ FTP_OS_SUFFIX="os" SVNREPO="file:///srv/svn-packages" CLEANUP_DESTDIR="/srv/package-cleanup" +CLEANUP_DRYRUN=false SOURCE_CLEANUP_DESTDIR="/srv/source-cleanup" STAGING="$HOME/staging" diff --git a/misc-scripts/ftpdir-cleanup b/misc-scripts/ftpdir-cleanup index f0f89a3..6a69d72 100755 --- a/misc-scripts/ftpdir-cleanup +++ b/misc-scripts/ftpdir-cleanup @@ -13,6 +13,8 @@ dest=$2 . "$(dirname $0)/../db-functions" . "$(dirname $0)/../config" +${CLEANUP_DRYRUN} && echo 'dry run mode is active' + ftppath_base="$FTP_BASE/$reponame/$FTP_OS_SUFFIX" for arch in ${ARCHES[@]}; do @@ -93,7 +95,7 @@ for arch in ${ARCHES[@]}; do for mf in $missfiles; do if [ -e "${ftppath_base}/any/${mf}" ]; then echo "Restoring missing 'any' symlink: ${mf}" - ln -s "../any/${mf}" "${ftppath}" + ${CLEANUP_DRYRUN} || ln -s "../any/${mf}" "${ftppath}" else MISSINGFILES="${MISSINGFILES} ${mf}" fi @@ -141,17 +143,17 @@ for arch in ${ARCHES[@]}; do fi if [ -n "${DELETEFILES}" ]; then - mv ${DELETEFILES} "$dest" + ${CLEANUP_DRYRUN} || mv ${DELETEFILES} "$dest" echo "" fi if [ -n "${DELETESYMLINKS}" ]; then - rm -f ${DELETESYMLINKS} + ${CLEANUP_DRYRUN} || rm -f ${DELETESYMLINKS} echo "" fi if [ -n "${EXTRAFILES}" ]; then - mv ${EXTRAFILES} "$dest" + ${CLEANUP_DRYRUN} || mv ${EXTRAFILES} "$dest" echo "" fi @@ -189,6 +191,6 @@ fi if [ -d "$ftppath_base/any" -a -n "${ARCHINDEPFILES}" ]; then cd "$ftppath_base/any" - mv ${ARCHINDEPFILES} "$dest" + ${CLEANUP_DRYRUN} || mv ${ARCHINDEPFILES} "$dest" echo "" fi -- cgit v1.2.2 From de0c93a652333b242e7ea282110619f25de7be8d Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 21 Feb 2010 16:54:10 +0100 Subject: Accept any *.pkg.tar.* package file name With this patch packages with different compressions are accepted. It is ensured that one cannot have the same package with different compression extensions. The new functions getpkgfile{,s} are used to sanitize globed filenames. Signed-off-by: Pierre Schmitz --- config | 2 +- db-functions | 34 +++++++++++++++++++++++++++++++--- db-move | 11 ++++++----- db-update | 12 ++++++------ 4 files changed, 44 insertions(+), 15 deletions(-) diff --git a/config b/config index 7864532..8e582d9 100644 --- a/config +++ b/config @@ -13,7 +13,7 @@ TMPDIR="/srv/tmp" ARCHES=(i686 x86_64) BUILDSCRIPT="PKGBUILD" DBEXT=".db.tar.gz" -PKGEXT=".pkg.tar.gz" +PKGEXT=".pkg.tar.*" SRCEXT=".src.tar.gz" # Allowed licenses: get sourceballs only for licenses in this array diff --git a/db-functions b/db-functions index a4cffd7..c27366c 100644 --- a/db-functions +++ b/db-functions @@ -19,12 +19,12 @@ repo_lock () { #repo_lock repo-name arch LOCKFILE="$TMPDIR/.repolock.$1.$2" if [ -f "$LOCKFILE" ]; then owner="$(/usr/bin/stat -c %U $LOCKFILE)" - echo "error: db generation is already in progress (started by $owner)" + echo "error: db generation is already in progress (started by $owner)" >&2 exit 1 else /bin/touch "$LOCKFILE" if [ ! -f "$LOCKFILE" ]; then - echo "error: could not create repo lock... something went wrong!" + echo "error: could not create repo lock... something went wrong!" >&2 fi set_umask fi @@ -33,7 +33,7 @@ repo_lock () { #repo_lock repo-name arch repo_unlock () { #repo_unlock repo-name arch LOCKFILE="$TMPDIR/.repolock.$1.$2" if [ ! -f "$LOCKFILE" ]; then - echo "error: repo lock doesn't exist... something went terribly wrong!" + echo "error: repo lock doesn't exist... something went terribly wrong!" >&2 else rm -f "$LOCKFILE" fi @@ -87,6 +87,34 @@ getpkgver() { echo "$_ver" } +getpkgfile() { + if [[ ${#} -ne 1 ]]; then + echo 'ERROR: No canonical package found!' >&2 + exit 1 + elif [ ! -f "${1}" ]; then + echo "ERROR: Package ${1} not found!" >&2 + exit 1 + fi + + echo ${1} +} + +getpkgfiles() { + if [ ! -z "$(echo ${@%\.*} | sed "s/ /\n/g" | sort | uniq -D)" ]; then + echo 'ERROR: Duplicate packages found!'>&2 + exit 1 + fi + + for f in ${@}; do + if [ ! -f "${f}" ]; then + echo "ERROR: Package ${f} not found!" >&2 + exit 1 + fi + done + + echo ${@} +} + check_pkg_arch () { #check_pkg_arch pkgfile arch local _arch _arch="$(_grep_pkginfo "$1" "^arch")" diff --git a/db-move b/db-move index efd54e0..c5bd32e 100755 --- a/db-move +++ b/db-move @@ -59,9 +59,7 @@ if [ -d "$packagebase/repos/$svnrepo_from" ]; then for i in ${pkgname[@]}; do _pkgfile="$i-$pkgver-$pkgrel-$_arch$PKGEXT" - if [ ! -f "$ftppath_from/${_arch}/$_pkgfile" ]; then - die "error: package file '$_pkgfile' not found in repo '$repofrom'" - fi + getpkgfile "$ftppath_from/${_arch}/"$_pkgfile >/dev/null done if [ -d "$packagebase/repos/$svnrepo_to" ]; then @@ -101,13 +99,16 @@ if [ -d "$packagebase/repos/$svnrepo_from" ]; then for i in ${pkgname[@]}; do _pkgfile="$i-$pkgver-$pkgrel-$_arch$PKGEXT" - /bin/cp "$ftppath_from/$architecture/$_pkgfile" . + _cpkgfile=$(getpkgfile "$ftppath_from/$architecture/"$_pkgfile) + [ $? -gt 0 ] && die + /bin/cp $_cpkgfile . /usr/bin/repo-add -q "$repoto$DBEXT" $_pkgfile || die "Error in repo-add $_pkgfile" done #use '*' to move the old DB too mv $repoto$DBEXT* $ftppath_to/$architecture for i in ${pkgname[@]}; do - _pkgfile="$i-$pkgver-$pkgrel-$_arch$PKGEXT" + _pkgfile=$(getpkgfile "$i-$pkgver-$pkgrel-$_arch"$PKGEXT) + [ $? -gt 0 ] && die if [ "${_arch}" == "any" ]; then mv ${_pkgfile} $ftppath_to/any ln -s ../any/${_pkgfile} $ftppath_to/$architecture/ diff --git a/db-update b/db-update index 17d3b71..0973bf3 100755 --- a/db-update +++ b/db-update @@ -89,7 +89,7 @@ done # Process architecture-independent packages first. if [ -d "$stagedir" ]; then - ANYPKGS="$(/bin/ls $stagedir/*-any$PKGEXT 2>/dev/null)" + ANYPKGS="$(getpkgfiles $stagedir/*-any$PKGEXT 2>/dev/null)" fi mkdir -p $WORKDIR @@ -115,7 +115,7 @@ if [ -n "$ANYPKGS" ]; then /usr/bin/svn up -q $_pkgbase if [ -d "$_pkgbase/repos/$svnrepo" ]; then . "$_pkgbase/repos/$svnrepo/$BUILDSCRIPT" - if [ "$_pkgfile" = "$_pkgname-$pkgver-$pkgrel-any$PKGEXT" ]; then + if echo "$_pkgfile" | grep -q "$_pkgname-$pkgver-$pkgrel-any"; then to_add_any="$to_add_any $pkg" else echo " WARNING: $_pkgfile does not match $BUILDSCRIPT in $svnrepo" @@ -155,7 +155,7 @@ for current_arch in ${ARCHES[@]}; do to_add="" if [ -d "$stagedir" ]; then - ADDPKGS="$(/bin/ls $stagedir/*-${current_arch}$PKGEXT 2>/dev/null)" + ADDPKGS="$(getpkgfiles $stagedir/*-${current_arch}$PKGEXT 2>/dev/null)" fi if [ -n "$ADDPKGS" -o -n "$ANYPKGS" ]; then @@ -186,7 +186,7 @@ for current_arch in ${ARCHES[@]}; do /usr/bin/svn up -q $_pkgbase if [ -d "$_pkgbase/repos/$svnrepo" ]; then . "$_pkgbase/repos/$svnrepo/$BUILDSCRIPT" - if [ "$_pkgfile" = "$_pkgname-$pkgver-$pkgrel-$current_arch$PKGEXT" ]; then + if echo "$_pkgfile" | grep -q "$_pkgname-$pkgver-$pkgrel-$current_arch"; then to_add="$to_add $pkg" else echo " WARNING: $_pkgfile does not match $BUILDSCRIPT in $svnrepo" @@ -216,7 +216,7 @@ for current_arch in ${ARCHES[@]}; do # if non empty, move all build dirs if [ $(/bin/ls "$WORKDIR/build/" 2>/dev/null | wc -l) != 0 ]; then - if [ $(/bin/ls "$WORKDIR/build/"*-$current_arch$PKGEXT 2>/dev/null | wc -l) != 0 ]; then + if [ $(getpkgfiles "$WORKDIR/build/"*-$current_arch$PKGEXT 2>/dev/null | wc -l) != 0 ]; then echo "Copying new files to '$ftppath'" for f in "$WORKDIR/build/"*-$current_arch$PKGEXT; do if ! /bin/cp "$f" "$ftppath/"; then @@ -224,7 +224,7 @@ for current_arch in ${ARCHES[@]}; do fi done fi - if [ $(/bin/ls "$WORKDIR/build/"*-any$PKGEXT 2>/dev/null | wc -l) != 0 ]; then + if [ $(getpkgfiles "$WORKDIR/build/"*-any$PKGEXT 2>/dev/null | wc -l) != 0 ]; then echo "Copying new files to '$ftppath_any' and symlinking" for f in "$WORKDIR/build/"*-any$PKGEXT; do if ! /bin/cp "$f" "$ftppath_any"; then -- cgit v1.2.2 From 0130eca14ee619d5ffe5afeebcff2522bdfd06a2 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 21 Feb 2010 20:14:42 +0100 Subject: remove unneeded and now broken check --- misc-scripts/ftpdir-cleanup | 1 - 1 file changed, 1 deletion(-) diff --git a/misc-scripts/ftpdir-cleanup b/misc-scripts/ftpdir-cleanup index 6a69d72..ccd867e 100755 --- a/misc-scripts/ftpdir-cleanup +++ b/misc-scripts/ftpdir-cleanup @@ -53,7 +53,6 @@ for arch in ${ARCHES[@]}; do for pkg in $TMPDIR/*; do filename=$(grep -A1 '^%FILENAME%$' "${pkg}/desc" | tail -n1) - [ -z "${filename}" ] && filename="${pkg}${PKGEXT}" if [ ! -e "${filename}" ]; then MISSINGFILES="${MISSINGFILES} ${filename}" -- cgit v1.2.2 From ba7968503f18e5a2537d7d87ce2a5caa582f04f7 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Tue, 23 Feb 2010 03:56:29 +0100 Subject: Hide 'file not found' message --- misc-scripts/ftpdir-cleanup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc-scripts/ftpdir-cleanup b/misc-scripts/ftpdir-cleanup index ccd867e..a185090 100755 --- a/misc-scripts/ftpdir-cleanup +++ b/misc-scripts/ftpdir-cleanup @@ -77,7 +77,7 @@ for arch in ${ARCHES[@]}; do pkgname="$(getpkgname $pkg)" for p in ${TMPDIR}/${pkgname}-*; do [ ! -d "${p}" ] || continue 2 - dbpkgname=$(grep -A1 '^%FILENAME%$' "${p}/desc" | tail -n1) + dbpkgname=$(grep -A1 '^%FILENAME%$' "${p}/desc" 2>/dev/null| tail -n1) if [ "${dbpkgname}" = "${pkgname}" ]; then continue 2 fi -- cgit v1.2.2 From 1d6421c7a12e9a07d00ec4de999ae95f7dd0874e Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 27 Feb 2010 10:47:52 -0600 Subject: create-filelists: general cleanups * Specify lock name once * Use new script name everywhere * Clean up tabs/spaces and add a modeline. This isn't necessarily the one we wanted to standardize on, but I picked the one the entire file is written to at the moment. Signed-off-by: Dan McGee --- cron-jobs/create-filelists | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/cron-jobs/create-filelists b/cron-jobs/create-filelists index c9d7db9..e90da00 100755 --- a/cron-jobs/create-filelists +++ b/cron-jobs/create-filelists @@ -3,27 +3,28 @@ reposdir="/srv/ftp" targetdir="/srv/ftp" repos="core extra testing community community-testing" +lock="/tmp/create-filelists.lock" . "$(dirname $0)/../db-functions" . "$(dirname $0)/../config" -if [ -f "/tmp/createFileList.lock" ]; then - echo "Error: createFileList allready in progress." +if [ -f "$lock" ]; then + echo "Error: create-filelists already in progress." exit 1 fi -touch "/tmp/createFileList.lock" || exit 1 -TMPDIR="$(mktemp -d /tmp/createFileList.XXXXXX)" || exit 1 -CACHEDIR="$(mktemp -d /tmp/createFileList.XXXXXX)" || exit 1 +touch "$lock" || exit 1 +TMPDIR="$(mktemp -d /tmp/create-filelists.XXXXXX)" || exit 1 +CACHEDIR="$(mktemp -d /tmp/create-filelists.XXXXXX)" || exit 1 #adjust the nice level to run at a lower priority /usr/bin/renice +10 -p $$ > /dev/null case "${DBEXT}" in - *.gz) TAR_OPT="z" ;; - *.bz2) TAR_OPT="j" ;; - *.xz) TAR_OPT="J" ;; - *) echo "Unknown compression type for DBEXT=${DBEXT}" && exit 1 ;; + *.gz) TAR_OPT="z" ;; + *.bz2) TAR_OPT="j" ;; + *.xz) TAR_OPT="J" ;; + *) echo "Unknown compression type for DBEXT=${DBEXT}" && exit 1 ;; esac FILESEXT="${DBEXT//db/files}" @@ -77,5 +78,7 @@ done cd - >/dev/null rm -rf "$TMPDIR" || exit 1 rm -rf "$CACHEDIR" || exit 1 -rm -f "/tmp/createFileList.lock" || exit 1 +rm -f "$lock" || exit 1 # echo 'done' + +# vim: set ts=4 sw=4 et ft=sh: -- cgit v1.2.2 From 6587ebadf6b3a0d8e58be1ab7169a4c2f88ed57c Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 27 Feb 2010 10:58:22 -0600 Subject: create-filelists: s/REPO_DB_FILE/FILES_DB_FILE/g This will set up changes soon to come where we actually use the real repos DB file so I don't want variable name confusion. Signed-off-by: Dan McGee --- cron-jobs/create-filelists | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/cron-jobs/create-filelists b/cron-jobs/create-filelists index e90da00..a0b6a57 100755 --- a/cron-jobs/create-filelists +++ b/cron-jobs/create-filelists @@ -30,7 +30,7 @@ esac FILESEXT="${DBEXT//db/files}" for repo in $repos; do - REPO_DB_FILE="${repo}$FILESEXT" + FILES_DB_FILE="${repo}$FILESEXT" for arch in ${ARCHES[@]}; do cd "$reposdir" @@ -38,9 +38,9 @@ for repo in $repos; do cached="no" # extract old file archive - if [ -f "${targetdir}/${repodir}/${REPO_DB_FILE}" ]; then + if [ -f "${targetdir}/${repodir}/${FILES_DB_FILE}" ]; then mkdir -p "${CACHEDIR}/${repodir}" - bsdtar -xf "${targetdir}/${repodir}/${REPO_DB_FILE}" -C "${CACHEDIR}/${repodir}" + bsdtar -xf "${targetdir}/${repodir}/${FILES_DB_FILE}" -C "${CACHEDIR}/${repodir}" cached="yes" fi @@ -64,13 +64,13 @@ for repo in $repos; do # create new file archive if [ "$cached" == "no" ]; then # at least one package has changed, so let's rebuild the archive -# echo "creating ${REPO_DB_FILE}/${arch}" +# echo "creating ${FILES_DB_FILE}/${arch}" pkgdir="${targetdir}/${repodir}" mkdir -p "$pkgdir" cd "${TMPDIR}/${repodir}" - [ -f "${pkgdir}/${REPO_DB_FILE}.old" ] && rm "${pkgdir}/${REPO_DB_FILE}.old" - [ -f "${pkgdir}/${REPO_DB_FILE}" ] && mv "${pkgdir}/${REPO_DB_FILE}" "${pkgdir}/${REPO_DB_FILE}.old" - bsdtar --exclude=*${DBEXT//\.db/} -c${TAR_OPT}f "${pkgdir}/${REPO_DB_FILE}" * + [ -f "${pkgdir}/${FILES_DB_FILE}.old" ] && rm "${pkgdir}/${FILES_DB_FILE}.old" + [ -f "${pkgdir}/${FILES_DB_FILE}" ] && mv "${pkgdir}/${FILES_DB_FILE}" "${pkgdir}/${FILES_DB_FILE}.old" + bsdtar --exclude=*${DBEXT//\.db/} -c${TAR_OPT}f "${pkgdir}/${FILES_DB_FILE}" * fi done done -- cgit v1.2.2 From ff1530def072daf95f077ec0f8a4d984da4304d6 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 27 Feb 2010 11:27:13 -0600 Subject: create-filelists: rework the package loop completely Instead of wasting time extracting .PKGINFO twice from every single package in the repos, use the package DB to eliminate most of the heavy lifting. This way we only need to worry about looking at the packages that actually have changed since the last time we built the package database. This should give a noticeable performance increase to this job in addition to reducing IO load and unnecessary reading of every package file. Signed-off-by: Dan McGee --- cron-jobs/create-filelists | 41 ++++++++++++++++++++++++++++------------- 1 file changed, 28 insertions(+), 13 deletions(-) diff --git a/cron-jobs/create-filelists b/cron-jobs/create-filelists index a0b6a57..6091bf4 100755 --- a/cron-jobs/create-filelists +++ b/cron-jobs/create-filelists @@ -14,8 +14,12 @@ if [ -f "$lock" ]; then fi touch "$lock" || exit 1 -TMPDIR="$(mktemp -d /tmp/create-filelists.XXXXXX)" || exit 1 -CACHEDIR="$(mktemp -d /tmp/create-filelists.XXXXXX)" || exit 1 +# location where the package DB is extracted so we know what to include +DBDIR="$(mktemp -d /tmp/create-filelists.dbdir.XXXXXX)" || exit 1 +# location where the old files DB is extracted to save us some work +CACHEDIR="$(mktemp -d /tmp/create-filelists.cachedir.XXXXXX)" || exit 1 +# location where the new files DB is built up and eventually zipped +TMPDIR="$(mktemp -d /tmp/create-filelists.tmpdir.XXXXXX)" || exit 1 #adjust the nice level to run at a lower priority /usr/bin/renice +10 -p $$ > /dev/null @@ -30,33 +34,45 @@ esac FILESEXT="${DBEXT//db/files}" for repo in $repos; do + REPO_DB_FILE="${repo}$DBEXT" FILES_DB_FILE="${repo}$FILESEXT" for arch in ${ARCHES[@]}; do +# echo "Running for architecture $arch, repo $repo" cd "$reposdir" repodir="${repo}/os/${arch}" cached="no" + # extract package db archive + if [ -f "${targetdir}/${repodir}/${REPO_DB_FILE}" ]; then + mkdir -p "${DBDIR}/${repodir}" +# echo "extracting $REPO_DB_FILE" + bsdtar -xf "${targetdir}/${repodir}/${REPO_DB_FILE}" -C "${DBDIR}/${repodir}" + else + echo "Fail! Does the repo $repo with arch $arch even exist?" + continue + fi + # extract old file archive if [ -f "${targetdir}/${repodir}/${FILES_DB_FILE}" ]; then mkdir -p "${CACHEDIR}/${repodir}" +# echo "extracting $FILES_DB_FILE" bsdtar -xf "${targetdir}/${repodir}/${FILES_DB_FILE}" -C "${CACHEDIR}/${repodir}" cached="yes" fi # create file lists - for pkg in $repodir/*${PKGEXT}; do - pkgname="$(getpkgname "$pkg")" - pkgver="$(getpkgver "$pkg")" - tmppkgdir="${TMPDIR}/${repodir}/${pkgname}-${pkgver}" + for pkg in $(ls ${DBDIR}/${repodir}); do + tmppkgdir="${TMPDIR}/${repodir}/${pkg}" mkdir -p "$tmppkgdir" - if [ -f "${CACHEDIR}/${repodir}/${pkgname}-${pkgver}/files" ]; then -# echo "cache: $pkgname" - mv "${CACHEDIR}/${repodir}/${pkgname}-${pkgver}/files" "${tmppkgdir}/files" + if [ -f "${CACHEDIR}/${repodir}/${pkg}/files" ]; then +# echo "cache: $pkg" + mv "${CACHEDIR}/${repodir}/${pkg}/files" "${tmppkgdir}/files" else -# echo "$repo/$arch: $pkgname" +# echo "not cache: $repo/$arch: $pkg" + filename=$(grep -A1 '^%FILENAME%$' "${DBDIR}/${repodir}/${pkg}/desc" | tail -n1) echo '%FILES%' > "${tmppkgdir}/files" - bsdtar --exclude=.* -tf "$pkg" >> "${tmppkgdir}/files" + bsdtar --exclude=.* -tf "$repodir/$filename" >> "${tmppkgdir}/files" cached="no" fi done @@ -76,8 +92,7 @@ for repo in $repos; do done cd - >/dev/null -rm -rf "$TMPDIR" || exit 1 -rm -rf "$CACHEDIR" || exit 1 +rm -rf "$TMPDIR" "$CACHEDIR" "$DBDIR" rm -f "$lock" || exit 1 # echo 'done' -- cgit v1.2.2 From ffa88b335b2f756925b388ecbd6681bf3aa58579 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 27 Feb 2010 11:52:56 -0600 Subject: create-filelists: include desc/depends entries Make the files DB include everything the original packages DB includes instead of just being 'files' entries. This will allow tools to do more with these generated files and they can be used as a drop-in replacement for a regular package database. Signed-off-by: Dan McGee --- cron-jobs/create-filelists | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/cron-jobs/create-filelists b/cron-jobs/create-filelists index 6091bf4..84867d8 100755 --- a/cron-jobs/create-filelists +++ b/cron-jobs/create-filelists @@ -63,14 +63,18 @@ for repo in $repos; do # create file lists for pkg in $(ls ${DBDIR}/${repodir}); do + dbpkgdir="${DBDIR}/${repodir}/${pkg}" + cachepkgdir="${CACHEDIR}/${repodir}/${pkg}" tmppkgdir="${TMPDIR}/${repodir}/${pkg}" mkdir -p "$tmppkgdir" - if [ -f "${CACHEDIR}/${repodir}/${pkg}/files" ]; then + ln "${dbpkgdir}/desc" "${tmppkgdir}/desc" + ln "${dbpkgdir}/depends" "${tmppkgdir}/depends" + if [ -f "${cachepkgdir}/files" ]; then # echo "cache: $pkg" - mv "${CACHEDIR}/${repodir}/${pkg}/files" "${tmppkgdir}/files" + ln "${cachepkgdir}/files" "${tmppkgdir}/files" else # echo "not cache: $repo/$arch: $pkg" - filename=$(grep -A1 '^%FILENAME%$' "${DBDIR}/${repodir}/${pkg}/desc" | tail -n1) + filename=$(grep -A1 '^%FILENAME%$' "${dbpkgdir}/desc" | tail -n1) echo '%FILES%' > "${tmppkgdir}/files" bsdtar --exclude=.* -tf "$repodir/$filename" >> "${tmppkgdir}/files" cached="no" -- cgit v1.2.2 From 17dd60ced8eb8ca461b73dd8429be0fda83264e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20B=C3=A9langer?= Date: Mon, 1 Mar 2010 22:44:08 -0500 Subject: Make repo locking an atomic process and added timeout argument MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The repo locking function now use a lock directory instead of a lock file. This makes the lock checking and creation an atomic process. The repo_lock function will now try to obtain a lock every $LOCK_DELAY seconds until it is successful. An optional third argument can be use to give a timeout in seconds; this is intended for scripts that runs unattended. Repo locking is now used in the ftpdir-cleanup script. This should fix the problem of the ftpdir-cleanup script removing the new package instead of the old one (FS#17058). Signed-off-by: Eric Bélanger --- config | 3 +++ db-functions | 48 ++++++++++++++++++++++++++++++--------------- misc-scripts/ftpdir-cleanup | 10 +++++++--- 3 files changed, 42 insertions(+), 19 deletions(-) diff --git a/config b/config index 8e582d9..a563f35 100644 --- a/config +++ b/config @@ -8,6 +8,9 @@ CLEANUP_DESTDIR="/srv/package-cleanup" CLEANUP_DRYRUN=false SOURCE_CLEANUP_DESTDIR="/srv/source-cleanup" +LOCK_DELAY=10 +LOCK_TIMEOUT=300 + STAGING="$HOME/staging" TMPDIR="/srv/tmp" ARCHES=(i686 x86_64) diff --git a/db-functions b/db-functions index c27366c..df50f22 100644 --- a/db-functions +++ b/db-functions @@ -15,27 +15,43 @@ restore_umask () { umask $UMASK } -repo_lock () { #repo_lock repo-name arch - LOCKFILE="$TMPDIR/.repolock.$1.$2" - if [ -f "$LOCKFILE" ]; then - owner="$(/usr/bin/stat -c %U $LOCKFILE)" - echo "error: db generation is already in progress (started by $owner)" >&2 - exit 1 - else - /bin/touch "$LOCKFILE" - if [ ! -f "$LOCKFILE" ]; then - echo "error: could not create repo lock... something went wrong!" >&2 - fi - set_umask +repo_lock () { #repo_lock [timeout] + LOCKDIR="$TMPDIR/.repolock.$1.$2" + local _count + local _trial + local _timeout + local _lockblock + local _owner + + if [ $# -eq 2 ]; then + _lockblock=true + _trial=0 + elif [ $# -eq 3 ]; then + _lockblock=false + _timeout=$3 + let _trial=$_timeout/$LOCK_DELAY fi + + _count=0 + while [ $_count -le $_trial ] || $_lockblock ; do + if ! mkdir "$LOCKDIR" >/dev/null 2>&1 ; then + _owner="$(/usr/bin/stat -c %U $LOCKDIR)" + echo "error: Repo $1-$2 is already locked by $_owner. Retrying in $LOCK_DELAY seconds..." >&2 + else + set_umask + break + fi + sleep $LOCK_DELAY + let _count=$_count+1 + done } -repo_unlock () { #repo_unlock repo-name arch - LOCKFILE="$TMPDIR/.repolock.$1.$2" - if [ ! -f "$LOCKFILE" ]; then +repo_unlock () { #repo_unlock + LOCKDIR="$TMPDIR/.repolock.$1.$2" + if [ ! -d "$LOCKDIR" ]; then echo "error: repo lock doesn't exist... something went terribly wrong!" >&2 else - rm -f "$LOCKFILE" + rmdir "$LOCKDIR" fi restore_umask } diff --git a/misc-scripts/ftpdir-cleanup b/misc-scripts/ftpdir-cleanup index a185090..4c559b9 100755 --- a/misc-scripts/ftpdir-cleanup +++ b/misc-scripts/ftpdir-cleanup @@ -10,7 +10,7 @@ dest=$2 ############################################################ -. "$(dirname $0)/../db-functions" +. "$(dirname $0)/../db-functions" . "$(dirname $0)/../config" ${CLEANUP_DRYRUN} && echo 'dry run mode is active' @@ -19,6 +19,8 @@ ftppath_base="$FTP_BASE/$reponame/$FTP_OS_SUFFIX" for arch in ${ARCHES[@]}; do + repo_lock $reponame $arch $LOCK_TIMEOUT + TMPDIR=$(mktemp -d /tmp/cleanup-XXXXXX) || exit 1 ftppath="$ftppath_base/$arch" MISSINGFILES="" @@ -80,10 +82,10 @@ for arch in ${ARCHES[@]}; do dbpkgname=$(grep -A1 '^%FILENAME%$' "${p}/desc" 2>/dev/null| tail -n1) if [ "${dbpkgname}" = "${pkgname}" ]; then continue 2 - fi + fi done EXTRAFILES="$EXTRAFILES $pkg" - done + done rm -rf ${TMPDIR} @@ -100,6 +102,8 @@ for arch in ${ARCHES[@]}; do fi done + repo_unlock $reponame $arch + #Make sure we've done *something* before outputting anything if [ -z "$DELETEFILES$DELETESYMLINKS$MISSINGFILES$EXTRAFILES" ]; then continue -- cgit v1.2.2 From e7fe897a91e6376a8724c226f8e078251620a173 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20B=C3=A9langer?= Date: Mon, 8 Feb 2010 10:13:42 -0500 Subject: sourceballs: Make cleanup more efficient MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Moved all cleanup related code in sourceballs-cleanup script and moved, now common, functions in db-functions. The cleanup script is now ran after all the new sourceballs have been fetched. Signed-off-by: Eric Bélanger --- cron-jobs/sourceballs | 2 + db-functions | 21 +++++++++ misc-scripts/make-sourceball | 54 ----------------------- misc-scripts/sourceballs-cleanup | 95 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 118 insertions(+), 54 deletions(-) create mode 100755 misc-scripts/sourceballs-cleanup diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index b7a4885..ccb00ad 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -89,4 +89,6 @@ if [ -n "$FAILED_PKGS" ]; then echo -e $FAILED_PKGS | sed "s| |\n|g" | sort -u >> "$srcbase/failed.txt" fi +$dirname/../misc-scripts/sourceballs-cleanup + cleanup diff --git a/db-functions b/db-functions index df50f22..d688df6 100644 --- a/db-functions +++ b/db-functions @@ -170,4 +170,25 @@ chk_license() { return 1 } +pkgname_from_src() { + local tmp + tmp=${1##*/} + tmp=${tmp%$SRCEXT} + for a in ${ARCHES[@]}; do + tmp=${tmp%-$a} + done + tmp=${tmp%-any} + echo ${tmp%-*-*} +} + +pkgver_from_src() { + tmp=${1##*/} + tmp=${tmp%$SRCEXT} + for a in ${ARCHES[@]}; do + tmp=${tmp%-$a} + done + tmp=${tmp%-any} + echo $tmp | sed 's|.*-\(.*-.*\)$|\1|g' +} + # vim: set ts=4 sw=4 noet ft=sh: diff --git a/misc-scripts/make-sourceball b/misc-scripts/make-sourceball index 8899fca..610d7f1 100755 --- a/misc-scripts/make-sourceball +++ b/misc-scripts/make-sourceball @@ -40,27 +40,6 @@ die() { cleanup 1 } -pkgname_from_src() { - local tmp - tmp=${1##*/} - tmp=${tmp%$SRCEXT} - for a in ${ARCHES[@]}; do - tmp=${tmp%-$a} - done - tmp=${tmp%-any} - echo ${tmp%-*-*} -} - -pkgver_from_src() { - tmp=${1##*/} - tmp=${tmp%$SRCEXT} - for a in ${ARCHES[@]}; do - tmp=${tmp%-$a} - done - tmp=${tmp%-any} - echo $tmp | sed 's|.*-\(.*-.*\)$|\1|g' -} - create_srcpackage() { if [ -d "$1" ]; then pushd "$1" >/dev/null @@ -94,38 +73,6 @@ create_srcpackage() { fi } -remove_old() { - if [ -d "$1" ]; then - pushd "$1" >/dev/null - PKGVERS="" - for repo in *; do - cd "$repo" - . "$BUILDSCRIPT" - PKGVERS="$PKGVERS $pkgver-$pkgrel" - cd .. - done - - for srcpkg in "$srcpath/$packagename-"*; do - [ -f "$srcpkg" ] || continue - if [ "$(pkgname_from_src $srcpkg)" == "$packagename" ]; then - skip=0 - pver="$(pkgver_from_src $srcpkg)" - for v in $PKGVERS; do - if [ "$v" = "$pver" ]; then - skip=1 - break - fi - done - if [ $skip -ne 1 ]; then - mv "$srcpkg" $SOURCE_CLEANUP_DESTDIR - fi - fi - done - - popd >/dev/null - fi -} - trap ctrl_c 2 trap cleanup 0 1 @@ -135,7 +82,6 @@ set_umask cd "$WORKDIR" if /usr/bin/svn export -q "$SVNREPO/$packagename" $packagename; then - remove_old "$packagename/repos/" create_srcpackage "$packagename/repos/$reponame-$_arch" else die "\tPackage '$packagename' does not exist in repo '$reponame-$_arch'" diff --git a/misc-scripts/sourceballs-cleanup b/misc-scripts/sourceballs-cleanup new file mode 100755 index 0000000..0a1ac4d --- /dev/null +++ b/misc-scripts/sourceballs-cleanup @@ -0,0 +1,95 @@ +#!/bin/bash + +. "$(dirname $0)/../db-functions" +. "$(dirname $0)/../config" + +srcpath="$FTP_BASE/sources/" +logfile="$srcpath/cleanup.txt" + +LOCKFILE="/tmp/.sourceball-cleanup.lock" +WORKDIR="/tmp/sourceball-cleanup.$packagename.$UID" + +cleanup () { + restore_umask + rm -rf "$WORKDIR" + rm -f "$LOCKFILE" + exit 0 +} + +ctrl_c() { + echo "Interrupted" >&2 + cleanup 0 +} + +remove_old() { + if [ -d "$1" ]; then + pushd "$1" >/dev/null + PKGVERS="" + for repo in *; do + cd "$repo" + . "$BUILDSCRIPT" + PKGVERS="$PKGVERS $pkgver-$pkgrel" + cd .. + done + + for srcpkg in "$srcpath/$packagename-"*; do + [ -f "$srcpkg" ] || continue + if [ "$(pkgname_from_src $srcpkg)" == "$packagename" ]; then + skip=0 + pver="$(pkgver_from_src $srcpkg)" + for v in $PKGVERS; do + if [ "$v" = "$pver" ]; then + skip=1 + break + fi + done + if [ $skip -ne 1 ]; then + mv "$srcpkg" $SOURCE_CLEANUP_DESTDIR + fi + fi + done + + popd >/dev/null + fi +} + +if [ -f "$LOCKFILE" ]; then + owner="$(/usr/bin/stat -c %U $LOCKFILE)" + echo "error: source tarball generation is already in progress (started by $owner)" + exit 1 +fi + +trap cleanup 0 +trap ctrl_c 2 + +/bin/touch "$LOCKFILE" + +#adjust the nice level to run at a lower priority +/usr/bin/renice +10 -p $$ > /dev/null + +set_umask +/bin/mkdir -p "$WORKDIR" +cd "$WORKDIR" + +[ -e "$logfile" ] && /bin/mv "$logfile" "$logfile.old" +echo "Orphaned sourceballs:" > "$logfile" + +for sourceball in "$srcpath"/*$SRCEXT; do + packagename=$(basename $sourceball) + packagename=${packagename%-*-*$SRCEXT} + + if ! /usr/bin/svn export -q --force "$SVNREPO/$packagename" "$packagename" >/dev/null 2>&1 ; then + echo "$packagename : no longer in svn. Removing sourceball." >> "$logfile" + mv $sourceball $SOURCE_CLEANUP_DESTDIR + elif [ -z "$(ls -A "$packagename/repos")" ]; then + echo "$packagename : no longer in repos but trunk is still in svn. Removing sourceball." >> "$logfile" + mv $sourceball $SOURCE_CLEANUP_DESTDIR + elif ! source "$packagename/trunk/$BUILDSCRIPT" && chk_license ${license[@]}; then + echo "$packagename : source hosting no longer required by license. Removing sourceball." >> "$logfile" + mv $sourceball $SOURCE_CLEANUP_DESTDIR + else + remove_old "$packagename/repos/" + fi +done + +cleanup 0 -- cgit v1.2.2 From 7bc888905280e1b59febdc4bad475b726225b95d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20B=C3=A9langer?= Date: Mon, 8 Feb 2010 10:13:43 -0500 Subject: moving sourceballs-cleanup --- cron-jobs/sourceballs-cleanup | 61 ------------------------------------------- 1 file changed, 61 deletions(-) delete mode 100755 cron-jobs/sourceballs-cleanup diff --git a/cron-jobs/sourceballs-cleanup b/cron-jobs/sourceballs-cleanup deleted file mode 100755 index c8d7b15..0000000 --- a/cron-jobs/sourceballs-cleanup +++ /dev/null @@ -1,61 +0,0 @@ -#!/bin/bash - -. "$(dirname $0)/../db-functions" -. "$(dirname $0)/../config" - -srcpath="$FTP_BASE/sources/" -logfile="$srcpath/cleanup.txt" - -LOCKFILE="/tmp/.sourceball-cleanup.lock" -WORKDIR="/tmp/sourceball-cleanup.$packagename.$UID" - -cleanup () { - restore_umask - rm -rf "$WORKDIR" - rm -f "$LOCKFILE" - exit 0 -} - -ctrl_c() { - echo "Interrupted" >&2 - cleanup 0 -} - -if [ -f "$LOCKFILE" ]; then - owner="$(/usr/bin/stat -c %U $LOCKFILE)" - echo "error: source tarball generation is already in progress (started by $owner)" - exit 1 -fi - -trap cleanup 0 -trap ctrl_c 2 - -/bin/touch "$LOCKFILE" - -#adjust the nice level to run at a lower priority -/usr/bin/renice +10 -p $$ > /dev/null - -set_umask -/bin/mkdir -p "$WORKDIR" -cd "$WORKDIR" - -[ -e "$logfile" ] && /bin/mv "$logfile" "$logfile.old" -echo "Orphaned sourceballs:" > "$logfile" - -for sourceball in "$srcpath"/*$SRCEXT; do - packagename=$(basename $sourceball) - packagename=${packagename%-*-*$SRCEXT} - - if ! /usr/bin/svn export -q --force "$SVNREPO/$packagename" "$packagename" >/dev/null 2>&1 ; then - echo "$packagename : no longer in svn. Removing sourceball." >> "$logfile" - mv $sourceball $SOURCE_CLEANUP_DESTDIR - elif [ -z "$(ls -A "$packagename/repos")" ]; then - echo "$packagename : no longer in repos but trunk is still in svn. Removing sourceball." >> "$logfile" - mv $sourceball $SOURCE_CLEANUP_DESTDIR - elif ! source "$packagename/trunk/$BUILDSCRIPT" && chk_license ${license[@]}; then - echo "$packagename : source hosting no longer required by license. Removing sourceball." >> "$logfile" - mv $sourceball $SOURCE_CLEANUP_DESTDIR - fi -done - -cleanup 0 -- cgit v1.2.2 From 57987ae42fb58c58e044bfd7d3486010b32bb313 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Wed, 31 Mar 2010 23:09:57 +0200 Subject: don't publish *.old files --- cron-jobs/create-filelists | 3 +-- db-move | 6 ++---- db-update | 2 +- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/cron-jobs/create-filelists b/cron-jobs/create-filelists index 84867d8..a25fd1a 100755 --- a/cron-jobs/create-filelists +++ b/cron-jobs/create-filelists @@ -88,8 +88,7 @@ for repo in $repos; do pkgdir="${targetdir}/${repodir}" mkdir -p "$pkgdir" cd "${TMPDIR}/${repodir}" - [ -f "${pkgdir}/${FILES_DB_FILE}.old" ] && rm "${pkgdir}/${FILES_DB_FILE}.old" - [ -f "${pkgdir}/${FILES_DB_FILE}" ] && mv "${pkgdir}/${FILES_DB_FILE}" "${pkgdir}/${FILES_DB_FILE}.old" + [ -f "${pkgdir}/${FILES_DB_FILE}" ] && rm "${pkgdir}/${FILES_DB_FILE}" bsdtar --exclude=*${DBEXT//\.db/} -c${TAR_OPT}f "${pkgdir}/${FILES_DB_FILE}" * fi done diff --git a/db-move b/db-move index c5bd32e..ca1b5da 100755 --- a/db-move +++ b/db-move @@ -88,8 +88,7 @@ if [ -d "$packagebase/repos/$svnrepo_from" ]; then if [ -f "$ftppath_from/$architecture/$repofrom$DBEXT" ]; then /bin/cp "$ftppath_from/$architecture/$repofrom$DBEXT" . /usr/bin/repo-remove -q "$repofrom$DBEXT" ${pkgname[@]} || die "Error in repo-remove" - #use '*' to move the old DB too - mv $repofrom$DBEXT* "$ftppath_from/$architecture" + mv $repofrom$DBEXT "$ftppath_from/$architecture" echo " Package files will be cleaned up automatically" fi @@ -104,8 +103,7 @@ if [ -d "$packagebase/repos/$svnrepo_from" ]; then /bin/cp $_cpkgfile . /usr/bin/repo-add -q "$repoto$DBEXT" $_pkgfile || die "Error in repo-add $_pkgfile" done - #use '*' to move the old DB too - mv $repoto$DBEXT* $ftppath_to/$architecture + mv $repoto$DBEXT $ftppath_to/$architecture for i in ${pkgname[@]}; do _pkgfile=$(getpkgfile "$i-$pkgver-$pkgrel-$_arch"$PKGEXT) [ $? -gt 0 ] && die diff --git a/db-update b/db-update index 0973bf3..82d2e25 100755 --- a/db-update +++ b/db-update @@ -236,7 +236,7 @@ for current_arch in ${ARCHES[@]}; do fi done fi - if ! /bin/cp "$WORKDIR/build/$reponame.db"* "$ftppath/"; then + if ! /bin/cp "$WORKDIR/build/$reponame$DBEXT" "$ftppath/"; then die "failed to move repository $reponame-$current_arch". fi else -- cgit v1.2.2 From 154b5301ff08ef29ff69afc17789a4f1a6a00c20 Mon Sep 17 00:00:00 2001 From: Andrea Scarpino Date: Wed, 31 Mar 2010 23:44:06 +0200 Subject: fix permissions of incoming packages Signed-off-by: Pierre Schmitz --- db-update | 2 ++ 1 file changed, 2 insertions(+) diff --git a/db-update b/db-update index 82d2e25..e4f8020 100755 --- a/db-update +++ b/db-update @@ -219,6 +219,7 @@ for current_arch in ${ARCHES[@]}; do if [ $(getpkgfiles "$WORKDIR/build/"*-$current_arch$PKGEXT 2>/dev/null | wc -l) != 0 ]; then echo "Copying new files to '$ftppath'" for f in "$WORKDIR/build/"*-$current_arch$PKGEXT; do + /bin/chmod 664 "$f" &>/dev/null if ! /bin/cp "$f" "$ftppath/"; then die "error: failure while copying files to $ftppath" fi @@ -227,6 +228,7 @@ for current_arch in ${ARCHES[@]}; do if [ $(getpkgfiles "$WORKDIR/build/"*-any$PKGEXT 2>/dev/null | wc -l) != 0 ]; then echo "Copying new files to '$ftppath_any' and symlinking" for f in "$WORKDIR/build/"*-any$PKGEXT; do + /bin/chmod 664 "$f" &>/dev/null if ! /bin/cp "$f" "$ftppath_any"; then die "error: failure while copying files to $ftppath_any" fi -- cgit v1.2.2 From ad520b527f3c5d2471e4b471eca5439643a01939 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 11 Apr 2010 13:52:32 +0200 Subject: Fail if lock couldn't be obtained; Unlock the repos if ftpdir-cleanup fails --- db-functions | 17 ++++++++++++----- db-move | 4 ++-- db-remove | 2 +- db-update | 2 +- misc-scripts/ftpdir-cleanup | 13 ++++++++++++- 5 files changed, 28 insertions(+), 10 deletions(-) diff --git a/db-functions b/db-functions index d688df6..d0665b0 100644 --- a/db-functions +++ b/db-functions @@ -8,11 +8,12 @@ UMASK="" set_umask () { [ "$UMASK" == "" ] && UMASK="$(umask)" + export UMASK umask 002 } restore_umask () { - umask $UMASK + umask $UMASK >/dev/null } repo_lock () { #repo_lock [timeout] @@ -36,24 +37,30 @@ repo_lock () { #repo_lock [timeout] while [ $_count -le $_trial ] || $_lockblock ; do if ! mkdir "$LOCKDIR" >/dev/null 2>&1 ; then _owner="$(/usr/bin/stat -c %U $LOCKDIR)" - echo "error: Repo $1-$2 is already locked by $_owner. Retrying in $LOCK_DELAY seconds..." >&2 + echo "warning: Repo $1-$2 is already locked by $_owner. Retrying in $LOCK_DELAY seconds..." >&2 else set_umask - break + return 0 fi sleep $LOCK_DELAY let _count=$_count+1 done + + echo "error: Repo $1-$2 is already locked by $_owner. Giving up!" >&2 + return 1 } repo_unlock () { #repo_unlock LOCKDIR="$TMPDIR/.repolock.$1.$2" if [ ! -d "$LOCKDIR" ]; then - echo "error: repo lock doesn't exist... something went terribly wrong!" >&2 + echo "warning: Repo lock $1-$2 was not locked!" >&2 + restore_umask + return 1 else rmdir "$LOCKDIR" + restore_umask + return 0 fi - restore_umask } # usage: _grep_pkginfo pkgfile pattern diff --git a/db-move b/db-move index ca1b5da..ed3df69 100755 --- a/db-move +++ b/db-move @@ -44,8 +44,8 @@ die() { trap ctrl_c 2 trap cleanup 0 -repo_lock $repoto $_arch -repo_lock $repofrom $_arch +repo_lock $repoto $_arch || exit 1 +repo_lock $repofrom $_arch || exit 1 /bin/mkdir -p "$WORKDIR" diff --git a/db-remove b/db-remove index 3190231..0b77fc1 100755 --- a/db-remove +++ b/db-remove @@ -40,7 +40,7 @@ die() { trap ctrl_c 2 trap cleanup 0 -repo_lock $reponame $_arch +repo_lock $reponame $_arch || exit 1 /bin/mkdir -p "$WORKDIR" diff --git a/db-update b/db-update index e4f8020..418c680 100755 --- a/db-update +++ b/db-update @@ -141,7 +141,7 @@ for current_arch in ${ARCHES[@]}; do svnrepo="$reponame-$current_arch" - repo_lock $reponame $current_arch + repo_lock $reponame $current_arch || continue /bin/mkdir -p "$WORKDIR/build" cd "$WORKDIR" diff --git a/misc-scripts/ftpdir-cleanup b/misc-scripts/ftpdir-cleanup index 4c559b9..d5f5737 100755 --- a/misc-scripts/ftpdir-cleanup +++ b/misc-scripts/ftpdir-cleanup @@ -13,13 +13,24 @@ dest=$2 . "$(dirname $0)/../db-functions" . "$(dirname $0)/../config" +cleanup() { + trap '' 2 15 + for arch in ${ARCHES[@]}; do + repo_unlock $reponame $arch + done + exit 1 +} + +trap cleanup 2 +trap cleanup 15 + ${CLEANUP_DRYRUN} && echo 'dry run mode is active' ftppath_base="$FTP_BASE/$reponame/$FTP_OS_SUFFIX" for arch in ${ARCHES[@]}; do - repo_lock $reponame $arch $LOCK_TIMEOUT + repo_lock $reponame $arch $LOCK_TIMEOUT || continue TMPDIR=$(mktemp -d /tmp/cleanup-XXXXXX) || exit 1 ftppath="$ftppath_base/$arch" -- cgit v1.2.2 From 52b668dc79b2ca05f9eebedab0dfc7eefb263765 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 11 Apr 2010 14:12:16 +0200 Subject: Do not overwrite TMPDIR defined in config --- misc-scripts/ftpdir-cleanup | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/misc-scripts/ftpdir-cleanup b/misc-scripts/ftpdir-cleanup index d5f5737..1090d57 100755 --- a/misc-scripts/ftpdir-cleanup +++ b/misc-scripts/ftpdir-cleanup @@ -32,7 +32,7 @@ for arch in ${ARCHES[@]}; do repo_lock $reponame $arch $LOCK_TIMEOUT || continue - TMPDIR=$(mktemp -d /tmp/cleanup-XXXXXX) || exit 1 + CLEANUP_TMPDIR=$(mktemp -d ${TMPDIR}/cleanup-XXXXXX) || exit 1 ftppath="$ftppath_base/$arch" MISSINGFILES="" DELETEFILES="" @@ -44,8 +44,8 @@ for arch in ${ARCHES[@]}; do exit 1 fi - if ! cd "${TMPDIR}" ; then - echo "Failed to cd to ${TMPDIR}" + if ! cd "${CLEANUP_TMPDIR}" ; then + echo "Failed to cd to ${CLEANUP_TMPDIR}" exit 1 fi @@ -64,7 +64,7 @@ for arch in ${ARCHES[@]}; do cd "$ftppath" - for pkg in $TMPDIR/*; do + for pkg in $CLEANUP_TMPDIR/*; do filename=$(grep -A1 '^%FILENAME%$' "${pkg}/desc" | tail -n1) if [ ! -e "${filename}" ]; then @@ -88,7 +88,7 @@ for arch in ${ARCHES[@]}; do continue fi pkgname="$(getpkgname $pkg)" - for p in ${TMPDIR}/${pkgname}-*; do + for p in ${CLEANUP_TMPDIR}/${pkgname}-*; do [ ! -d "${p}" ] || continue 2 dbpkgname=$(grep -A1 '^%FILENAME%$' "${p}/desc" 2>/dev/null| tail -n1) if [ "${dbpkgname}" = "${pkgname}" ]; then @@ -98,7 +98,7 @@ for arch in ${ARCHES[@]}; do EXTRAFILES="$EXTRAFILES $pkg" done - rm -rf ${TMPDIR} + rm -rf ${CLEANUP_TMPDIR} # Do a quick check to see if a missing ARCHINDEPFILE is in the any dir # If it is, and the file is MISSING, restore it -- cgit v1.2.2 From cbf6a9877dcbc8478f41d60a55f3fa2bfca63831 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Sat, 6 Feb 2010 19:47:48 -0600 Subject: db-update: Use a master package pool and symlinks This should save lots of transfer time with the mirrors :) Signed-off-by: Aaron Griffin --- db-update | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/db-update b/db-update index 418c680..a178996 100755 --- a/db-update +++ b/db-update @@ -131,6 +131,11 @@ for current_arch in ${ARCHES[@]}; do ftppath="$FTP_BASE/$reponame/os/$current_arch" ftppath_any="$FTP_BASE/$reponame/os/any" + poolpath="$FTP_BASE/packages/os/$current_arch" + poolpath_any="$FTP_BASE/packages/os/any" + # The following is used to create relative symlinks + poolrel="../../../packages/os/$current_arch" + poolrel_any="../../../packages/os/any" if [ ! -d "$ftppath" ]; then echo "FTP path for this repo ($reponame) is missing" @@ -220,8 +225,12 @@ for current_arch in ${ARCHES[@]}; do echo "Copying new files to '$ftppath'" for f in "$WORKDIR/build/"*-$current_arch$PKGEXT; do /bin/chmod 664 "$f" &>/dev/null - if ! /bin/cp "$f" "$ftppath/"; then - die "error: failure while copying files to $ftppath" + if ! /bin/cp "$f" "$poolpath/"; then + die "error: failure while copying files to $poolpath" + fi + fname="$(basename $f)" + if ! ln -s "$poolrel/$fname" "$ftppath/$fname"; then + die "error: failure symlinking $fname to $ftppath" fi done fi @@ -229,12 +238,15 @@ for current_arch in ${ARCHES[@]}; do echo "Copying new files to '$ftppath_any' and symlinking" for f in "$WORKDIR/build/"*-any$PKGEXT; do /bin/chmod 664 "$f" &>/dev/null - if ! /bin/cp "$f" "$ftppath_any"; then - die "error: failure while copying files to $ftppath_any" + if ! /bin/cp "$f" "$poolpath_any/"; then + die "error: failure while copying files to $poolpath_any" + fi + fname="$(basename $f)" + if ! ln -s "$poolrel_any/$fname" "$ftppath_any/$fname"; then + die "error: failure symlinking $fname to $ftppath_any" fi - bf=$(basename $f) - if ! ln -s "../any/$bf" "$ftppath/$bf"; then - die "error: failed to make link for $bf." + if ! ln -s "$poolrel_any/$fname" "$ftppath/$fname"; then + die "error: failure symlinking $fname to $ftppath" fi done fi -- cgit v1.2.2 From c7e3c501f01b6429519e2b8466d6bd8e2f7e22b9 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Mon, 15 Feb 2010 13:50:31 -0800 Subject: Remove FTP_OS_SUFFIX variable This isn't needed and if "os" ever changes, we have bigger changes. Won't save us any time and it's longer to type. --- config | 1 - misc-scripts/ftpdir-cleanup | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/config b/config index a563f35..57ae64b 100644 --- a/config +++ b/config @@ -1,6 +1,5 @@ FTP_BASE="/srv/ftp" -FTP_OS_SUFFIX="os" SVNREPO="file:///srv/svn-packages" diff --git a/misc-scripts/ftpdir-cleanup b/misc-scripts/ftpdir-cleanup index 1090d57..351c0f4 100755 --- a/misc-scripts/ftpdir-cleanup +++ b/misc-scripts/ftpdir-cleanup @@ -26,7 +26,7 @@ trap cleanup 15 ${CLEANUP_DRYRUN} && echo 'dry run mode is active' -ftppath_base="$FTP_BASE/$reponame/$FTP_OS_SUFFIX" +ftppath_base="$FTP_BASE/$reponame/os" for arch in ${ARCHES[@]}; do -- cgit v1.2.2 From e284d06d6cf63665cbadcb65e3a09bbd18027bd1 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Mon, 15 Feb 2010 15:44:46 -0800 Subject: Change ftpdir-cleanup scripts for package pooling * The per repo script now simply removes files as they should all be symlinks at this time * The cron job script will do the extra job of checking the global pool. If there are no more symlinks, move the pooled package Signed-off-by: Aaron Griffin --- cron-jobs/ftpdir-cleanup | 35 ++++++++++++++++++++++++++++++++++- misc-scripts/ftpdir-cleanup | 15 ++++++--------- 2 files changed, 40 insertions(+), 10 deletions(-) diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index d66138c..1ccc654 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -31,7 +31,40 @@ trap ctrl_c 2 /usr/bin/renice +10 -p $$ > /dev/null for repo in $repos; do - $(dirname $0)/../misc-scripts/ftpdir-cleanup $repo $CLEANUP_DESTDIR + $(dirname $0)/../misc-scripts/ftpdir-cleanup $repo done +to_cleanup="" +for _arch in ${ARCHES[@]}; do + poolpath="$FTP_BASE/packages/os/$_arch/" + cd $poolpath + for pkg in *$PKGEXT; do + [ -f "$pkg" ] || continue # in case we get a file named "*.pkg.tar.gz" + LINKS="$(/bin/ls $FTP_BASE/*/os/$_arch/$pkg 2>/dev/null)" + if [ -n "$LINKS" ]; then + found=0 + for lnk in $LINKS; do + if [ -h "$lnk" ]; then + found=1 + break + fi + done + # No links found, clean it up + if [ $found -eq 0 ]; then + to_cleanup="$to_cleanup $poolpath/$pkg" + fi + fi + done +done + +if [ -n "$to_cleanup" ]; then + echo " The following packages are no longer in any repo" + echo " They will be moved to $CLEANUP_DESTDIR" + for f in $to_cleanup; do + echo " $(basename "$f")" + done + echo "" + mv "$f" "$CLEANUP_DESTDIR" +fi + cleanup diff --git a/misc-scripts/ftpdir-cleanup b/misc-scripts/ftpdir-cleanup index 351c0f4..97f46b2 100755 --- a/misc-scripts/ftpdir-cleanup +++ b/misc-scripts/ftpdir-cleanup @@ -1,12 +1,11 @@ #!/bin/bash -if [ $# -ne 2 ]; then - echo "usage: $(basename $0) " +if [ $# -ne 1 ]; then + echo "usage: $(basename $0) " exit 1 fi reponame=$1 -dest=$2 ############################################################ @@ -65,6 +64,7 @@ for arch in ${ARCHES[@]}; do cd "$ftppath" for pkg in $CLEANUP_TMPDIR/*; do + [ -f "${pkg}" ] || continue filename=$(grep -A1 '^%FILENAME%$' "${pkg}/desc" | tail -n1) if [ ! -e "${filename}" ]; then @@ -124,7 +124,6 @@ for arch in ${ARCHES[@]}; do if [ -n "$DELETEFILES" ]; then echo " The following files are out of date" - echo " They will be moved to '$dest'" for f in $DELETEFILES; do echo " $f" done @@ -150,14 +149,13 @@ for arch in ${ARCHES[@]}; do if [ -n "$EXTRAFILES" ]; then echo " The following files are in the repo but not the db" - echo " They will be moved to '$dest'" for f in $EXTRAFILES; do echo " $f" done fi if [ -n "${DELETEFILES}" ]; then - ${CLEANUP_DRYRUN} || mv ${DELETEFILES} "$dest" + ${CLEANUP_DRYRUN} || rm -f ${DELETEFILES} echo "" fi @@ -167,7 +165,7 @@ for arch in ${ARCHES[@]}; do fi if [ -n "${EXTRAFILES}" ]; then - ${CLEANUP_DRYRUN} || mv ${EXTRAFILES} "$dest" + ${CLEANUP_DRYRUN} || rm -f ${EXTRAFILES} echo "" fi @@ -197,7 +195,6 @@ fi if [ -n "$ARCHINDEPFILES" ]; then echo " The following architecture independent packages" echo " are not symlinked in the architecture repositories." - echo " They will be moved to '$dest'" for f in $ARCHINDEPFILES; do echo " $f" done @@ -205,6 +202,6 @@ fi if [ -d "$ftppath_base/any" -a -n "${ARCHINDEPFILES}" ]; then cd "$ftppath_base/any" - ${CLEANUP_DRYRUN} || mv ${ARCHINDEPFILES} "$dest" + ${CLEANUP_DRYRUN} || rm -f ${ARCHINDEPFILES} echo "" fi -- cgit v1.2.2 From 709b4a514e24620145756a312ef9aab0662a3d8f Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Wed, 28 Apr 2010 11:55:04 -0700 Subject: ftpdir-cleanup: Fix cleanup of package pool --- cron-jobs/ftpdir-cleanup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index 1ccc654..2496ced 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -64,7 +64,7 @@ if [ -n "$to_cleanup" ]; then echo " $(basename "$f")" done echo "" - mv "$f" "$CLEANUP_DESTDIR" + mv $to_cleanup "$CLEANUP_DESTDIR" fi cleanup -- cgit v1.2.2 From 22c5f235fa25ea3327d95a28c6dc8ae399d6cb9a Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Wed, 28 Apr 2010 12:00:48 -0700 Subject: ftpdir-cleanup: use 'find' for symlink detection This gets rid of a bash loop for link detection --- cron-jobs/ftpdir-cleanup | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index 2496ced..977b384 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -40,19 +40,9 @@ for _arch in ${ARCHES[@]}; do cd $poolpath for pkg in *$PKGEXT; do [ -f "$pkg" ] || continue # in case we get a file named "*.pkg.tar.gz" - LINKS="$(/bin/ls $FTP_BASE/*/os/$_arch/$pkg 2>/dev/null)" - if [ -n "$LINKS" ]; then - found=0 - for lnk in $LINKS; do - if [ -h "$lnk" ]; then - found=1 - break - fi - done - # No links found, clean it up - if [ $found -eq 0 ]; then - to_cleanup="$to_cleanup $poolpath/$pkg" - fi + LINKS="$(/usr/bin/find $FTP_BASE -type l -name "$pkg" 2>/dev/null)" + if [ -z "$LINKS" ]; then + to_cleanup="$to_cleanup $poolpath/$pkg" fi done done -- cgit v1.2.2 From 1e13a032ac0bef748150c60f3b670e57dad2aee5 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Sun, 2 May 2010 08:52:59 -0700 Subject: ftpdir-cleanup: Add clean_pkgs function This should remove files if they are symlinks (pooled packages) and move them to the cleanup dir if not --- misc-scripts/ftpdir-cleanup | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/misc-scripts/ftpdir-cleanup b/misc-scripts/ftpdir-cleanup index 97f46b2..c9a232d 100755 --- a/misc-scripts/ftpdir-cleanup +++ b/misc-scripts/ftpdir-cleanup @@ -20,6 +20,16 @@ cleanup() { exit 1 } +clean_pkgs () { + for pkg in "$@"; do + if [ -h "$pkg" ]; then + rm -f "$pkg" + else + mv "$pkg" "$CLEANUP_DESTDIR" + fi + done +} + trap cleanup 2 trap cleanup 15 @@ -155,17 +165,17 @@ for arch in ${ARCHES[@]}; do fi if [ -n "${DELETEFILES}" ]; then - ${CLEANUP_DRYRUN} || rm -f ${DELETEFILES} + ${CLEANUP_DRYRUN} || clean_pkgs ${DELETEFILES} echo "" fi if [ -n "${DELETESYMLINKS}" ]; then - ${CLEANUP_DRYRUN} || rm -f ${DELETESYMLINKS} + ${CLEANUP_DRYRUN} || clean_pkgs ${DELETESYMLINKS} echo "" fi if [ -n "${EXTRAFILES}" ]; then - ${CLEANUP_DRYRUN} || rm -f ${EXTRAFILES} + ${CLEANUP_DRYRUN} || clean_pkgs ${EXTRAFILES} echo "" fi @@ -202,6 +212,6 @@ fi if [ -d "$ftppath_base/any" -a -n "${ARCHINDEPFILES}" ]; then cd "$ftppath_base/any" - ${CLEANUP_DRYRUN} || rm -f ${ARCHINDEPFILES} + ${CLEANUP_DRYRUN} || clean_pkgs ${ARCHINDEPFILES} echo "" fi -- cgit v1.2.2 From ee4074de910df5c637691ace06ec5a9d475fb4a3 Mon Sep 17 00:00:00 2001 From: Aaron Griffin Date: Mon, 3 May 2010 09:17:04 -0700 Subject: Add PKGPOOL_DIR config variable This is a relative var, descendent from FTP_BASE. We do this because we also need a relative path to the package pool dir --- config | 3 +++ cron-jobs/ftpdir-cleanup | 2 +- db-update | 8 ++++---- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/config b/config index 57ae64b..3a232f2 100644 --- a/config +++ b/config @@ -1,6 +1,9 @@ FTP_BASE="/srv/ftp" +#This dir MUST be under FTP_BASE +PKGPOOL_DIR="packages" + SVNREPO="file:///srv/svn-packages" CLEANUP_DESTDIR="/srv/package-cleanup" diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index 977b384..a278341 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -36,7 +36,7 @@ done to_cleanup="" for _arch in ${ARCHES[@]}; do - poolpath="$FTP_BASE/packages/os/$_arch/" + poolpath="$FTP_BASE/$PKGPOOL_DIR/os/$_arch/" cd $poolpath for pkg in *$PKGEXT; do [ -f "$pkg" ] || continue # in case we get a file named "*.pkg.tar.gz" diff --git a/db-update b/db-update index a178996..bcfb882 100755 --- a/db-update +++ b/db-update @@ -131,11 +131,11 @@ for current_arch in ${ARCHES[@]}; do ftppath="$FTP_BASE/$reponame/os/$current_arch" ftppath_any="$FTP_BASE/$reponame/os/any" - poolpath="$FTP_BASE/packages/os/$current_arch" - poolpath_any="$FTP_BASE/packages/os/any" + poolpath="$FTP_BASE/$PKGPOOL_DIR/os/$current_arch" + poolpath_any="$FTP_BASE/$PKGPOOL_DIR/os/any" # The following is used to create relative symlinks - poolrel="../../../packages/os/$current_arch" - poolrel_any="../../../packages/os/any" + poolrel="../../../$PKGPOOL_DIR/os/$current_arch" + poolrel_any="../../../$PKGPOOL_DIR/os/any" if [ ! -d "$ftppath" ]; then echo "FTP path for this repo ($reponame) is missing" -- cgit v1.2.2 From 56363d7e1c3ba52ec5ff4c59c1c8c904b3384fe7 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 20 Jun 2010 17:04:34 +0200 Subject: add initial .gitignore file --- .gitignore | 1 + 1 file changed, 1 insertion(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b25c15b --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +*~ -- cgit v1.2.2 From 5034b4e644c55631dfd5c88b491873389fc52e50 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 20 Jun 2010 17:29:40 +0200 Subject: Replace PKGPOOL_DIR by a function call The package pool dir depends on the host; so we solve this similar to get_repos_for_host() Note: There is no "os" subdir for the package pool. --- config | 3 --- cron-jobs/ftpdir-cleanup | 2 +- db-functions | 8 ++++++++ db-update | 8 ++++---- 4 files changed, 13 insertions(+), 8 deletions(-) diff --git a/config b/config index 3a232f2..57ae64b 100644 --- a/config +++ b/config @@ -1,9 +1,6 @@ FTP_BASE="/srv/ftp" -#This dir MUST be under FTP_BASE -PKGPOOL_DIR="packages" - SVNREPO="file:///srv/svn-packages" CLEANUP_DESTDIR="/srv/package-cleanup" diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index a278341..2e23440 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -36,7 +36,7 @@ done to_cleanup="" for _arch in ${ARCHES[@]}; do - poolpath="$FTP_BASE/$PKGPOOL_DIR/os/$_arch/" + poolpath="$FTP_BASE/$(get_pkgpool_for_host)/$_arch/" cd $poolpath for pkg in *$PKGEXT; do [ -f "$pkg" ] || continue # in case we get a file named "*.pkg.tar.gz" diff --git a/db-functions b/db-functions index d0665b0..2fe7a49 100644 --- a/db-functions +++ b/db-functions @@ -161,6 +161,14 @@ get_repos_for_host() { fi } +get_pkgpool_for_host() { + if [ "$(hostname)" = "sigurd" ]; then + echo "packages/community" + else + echo "packages/arch" + fi +} + #usage: chk_license ${license[@]}" chk_license() { local l diff --git a/db-update b/db-update index bcfb882..28095d8 100755 --- a/db-update +++ b/db-update @@ -131,11 +131,11 @@ for current_arch in ${ARCHES[@]}; do ftppath="$FTP_BASE/$reponame/os/$current_arch" ftppath_any="$FTP_BASE/$reponame/os/any" - poolpath="$FTP_BASE/$PKGPOOL_DIR/os/$current_arch" - poolpath_any="$FTP_BASE/$PKGPOOL_DIR/os/any" + poolpath="$FTP_BASE/$(get_pkgpool_for_host)/$current_arch" + poolpath_any="$FTP_BASE/$(get_pkgpool_for_host)/any" # The following is used to create relative symlinks - poolrel="../../../$PKGPOOL_DIR/os/$current_arch" - poolrel_any="../../../$PKGPOOL_DIR/os/any" + poolrel="../../../$(get_pkgpool_for_host)/$current_arch" + poolrel_any="../../../$(get_pkgpool_for_host)/any" if [ ! -d "$ftppath" ]; then echo "FTP path for this repo ($reponame) is missing" -- cgit v1.2.2 From 0f705986ab0898a1cd23c3e6d25a118cbd804648 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 20 Jun 2010 18:16:07 +0200 Subject: ftpdir-cleanup: only search for linked packages in actual repos --- cron-jobs/ftpdir-cleanup | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index 2e23440..66733f0 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -30,8 +30,10 @@ trap ctrl_c 2 #adjust the nice level to run at a lower priority /usr/bin/renice +10 -p $$ > /dev/null +repopaths='' for repo in $repos; do $(dirname $0)/../misc-scripts/ftpdir-cleanup $repo + repopaths="${repopaths} ${FTP_BASE}/${repo}/os/" done to_cleanup="" @@ -40,7 +42,7 @@ for _arch in ${ARCHES[@]}; do cd $poolpath for pkg in *$PKGEXT; do [ -f "$pkg" ] || continue # in case we get a file named "*.pkg.tar.gz" - LINKS="$(/usr/bin/find $FTP_BASE -type l -name "$pkg" 2>/dev/null)" + LINKS="$(/usr/bin/find $repopaths -type l -name "$pkg" 2>/dev/null)" if [ -z "$LINKS" ]; then to_cleanup="$to_cleanup $poolpath/$pkg" fi -- cgit v1.2.2 From f6004585ef7ceff8e0119363f297104ac7900e09 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 20 Jun 2010 18:45:55 +0200 Subject: adjust permissions for package pool --- cron-jobs/adjust-permissions | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/cron-jobs/adjust-permissions b/cron-jobs/adjust-permissions index 0f87075..c92cb9e 100755 --- a/cron-jobs/adjust-permissions +++ b/cron-jobs/adjust-permissions @@ -21,19 +21,29 @@ get_dir_owner() { echo "root:tusers" ;; community-testing) echo "root:tusers" ;; + packages/arch) + echo "ftp:ftp-extra" ;; + packages/community) + echo "root:tusers" ;; esac } #adjust the nice level to run at a lower priority /usr/bin/renice +10 -p $$ > /dev/null -cd /srv/ftp +cd $FTP_BASE for d in $(get_repos_for_host); do owner="$(get_dir_owner $d)" /bin/chown -R $owner $d/os/{any,i686,x86_64} /bin/chmod -R g+w $d/os/{any,i686,x86_64} done -/bin/chmod 555 /srv/ftp +for p in $(get_pkgpool_for_host); do + owner="$(get_dir_owner $p)" + /bin/chown -R $owner $p/{any,i686,x86_64} + /bin/chmod -R g+w $p/{any,i686,x86_64} +done + +/bin/chmod 555 $FTP_BASE rm -f /tmp/.ftpmaint.lck -- cgit v1.2.2 From e2cc14311b1bfad0c30cef395a1a033cdef030c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20B=C3=A9langer?= Date: Sun, 27 Jun 2010 23:40:38 -0400 Subject: sourceballs: Added community and community-testing support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Eric Bélanger Signed-off-by: Pierre Schmitz --- config | 1 + cron-jobs/sourceballs | 2 +- misc-scripts/make-sourceball | 14 +++++++++++--- misc-scripts/sourceballs-cleanup | 5 +++-- 4 files changed, 16 insertions(+), 6 deletions(-) diff --git a/config b/config index 57ae64b..1cd533f 100644 --- a/config +++ b/config @@ -2,6 +2,7 @@ FTP_BASE="/srv/ftp" SVNREPO="file:///srv/svn-packages" +SVNREPOCOMMUNITY="file:///srv/svn-community" CLEANUP_DESTDIR="/srv/package-cleanup" CLEANUP_DRYRUN=false diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index ccb00ad..93cb842 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -2,7 +2,7 @@ ftpbase="/srv/ftp" srcbase="/srv/ftp/sources" -repos="core extra testing" #community" +repos="core extra testing community community-testing" . "$(dirname $0)/../db-functions" . "$(dirname $0)/../config" diff --git a/misc-scripts/make-sourceball b/misc-scripts/make-sourceball index 610d7f1..37141c7 100755 --- a/misc-scripts/make-sourceball +++ b/misc-scripts/make-sourceball @@ -81,8 +81,16 @@ set_umask /bin/mkdir -p "$logpath" cd "$WORKDIR" -if /usr/bin/svn export -q "$SVNREPO/$packagename" $packagename; then - create_srcpackage "$packagename/repos/$reponame-$_arch" +if [[ "$reponame" = "community" || "$reponame" = "community-testing" ]]; then + if /usr/bin/svn export -q "$SVNREPOCOMMUNITY/$packagename" $packagename; then + create_srcpackage "$packagename/repos/$reponame-$_arch" + else + die "\tPackage '$packagename' does not exist in repo '$reponame-$_arch'" + fi else - die "\tPackage '$packagename' does not exist in repo '$reponame-$_arch'" + if /usr/bin/svn export -q "$SVNREPO/$packagename" $packagename; then + create_srcpackage "$packagename/repos/$reponame-$_arch" + else + die "\tPackage '$packagename' does not exist in repo '$reponame-$_arch'" + fi fi diff --git a/misc-scripts/sourceballs-cleanup b/misc-scripts/sourceballs-cleanup index 0a1ac4d..c7f284d 100755 --- a/misc-scripts/sourceballs-cleanup +++ b/misc-scripts/sourceballs-cleanup @@ -43,7 +43,7 @@ remove_old() { break fi done - if [ $skip -ne 1 ]; then + if [ $skip -ne 1 ]; then mv "$srcpkg" $SOURCE_CLEANUP_DESTDIR fi fi @@ -78,7 +78,8 @@ for sourceball in "$srcpath"/*$SRCEXT; do packagename=$(basename $sourceball) packagename=${packagename%-*-*$SRCEXT} - if ! /usr/bin/svn export -q --force "$SVNREPO/$packagename" "$packagename" >/dev/null 2>&1 ; then + if ! /usr/bin/svn export -q --force "$SVNREPO/$packagename" "$packagename" >/dev/null 2>&1 \ + && ! /usr/bin/svn export -q --force "$SVNREPOCOMMUNITY/$packagename" "$packagename" >/dev/null 2>&1 ; then echo "$packagename : no longer in svn. Removing sourceball." >> "$logfile" mv $sourceball $SOURCE_CLEANUP_DESTDIR elif [ -z "$(ls -A "$packagename/repos")" ]; then -- cgit v1.2.2 From f67f54fa2412e68fc02c6048f7dcee6223ccbc18 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Tue, 29 Jun 2010 10:41:50 +0200 Subject: don't mix db files of different arches --- db-update | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/db-update b/db-update index 28095d8..64265d0 100755 --- a/db-update +++ b/db-update @@ -153,7 +153,7 @@ for current_arch in ${ARCHES[@]}; do # copy the db file into our working area if [ -f "$ftppath/$reponame$DBEXT" ]; then - /bin/cp "$ftppath/$reponame$DBEXT" build/ + /bin/cp "$ftppath/$reponame$DBEXT" build/$reponame-$current_arch$DBEXT fi echo "Updating DB for $svnrepo" @@ -166,8 +166,9 @@ for current_arch in ${ARCHES[@]}; do if [ -n "$ADDPKGS" -o -n "$ANYPKGS" ]; then echo "==> Copying DB file from '$reponame'..." >&2 + if [ -f "$ftppath/$reponame$DBEXT" ]; then - /bin/cp "$ftppath/$reponame$DBEXT" build/ + /bin/cp "$ftppath/$reponame$DBEXT" build/$reponame-$current_arch$DBEXT fi pkgtotal=$(echo "$ADDPKGS $ANYPKGS" | wc -w) @@ -210,9 +211,9 @@ for current_arch in ${ARCHES[@]}; do pkgs="" for pkg in $to_add $to_add_any; do pkgs="$pkgs $(basename $pkg)"; done - /usr/bin/repo-add -q "$reponame$DBEXT" $pkgs + /usr/bin/repo-add -q "$reponame-$current_arch$DBEXT" $pkgs else - rm -f "build/$reponame$DBEXT" + rm -f "build/$reponame-$current_arch$DBEXT" echo "Errors found when adding packages" fi else @@ -250,7 +251,7 @@ for current_arch in ${ARCHES[@]}; do fi done fi - if ! /bin/cp "$WORKDIR/build/$reponame$DBEXT" "$ftppath/"; then + if ! /bin/cp "$WORKDIR/build/$reponame-$current_arch$DBEXT" "$ftppath/$reponame$DBEXT"; then die "failed to move repository $reponame-$current_arch". fi else -- cgit v1.2.2 From d989142a17315f20f038d025008350fc413ca5f1 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Tue, 29 Jun 2010 10:59:16 +0200 Subject: don't fail if any package already exists --- db-update | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/db-update b/db-update index 64265d0..f05246b 100755 --- a/db-update +++ b/db-update @@ -239,12 +239,16 @@ for current_arch in ${ARCHES[@]}; do echo "Copying new files to '$ftppath_any' and symlinking" for f in "$WORKDIR/build/"*-any$PKGEXT; do /bin/chmod 664 "$f" &>/dev/null - if ! /bin/cp "$f" "$poolpath_any/"; then - die "error: failure while copying files to $poolpath_any" - fi fname="$(basename $f)" - if ! ln -s "$poolrel_any/$fname" "$ftppath_any/$fname"; then - die "error: failure symlinking $fname to $ftppath_any" + if [ ! -f "$poolrel_any/$fname" ]; then + if ! /bin/cp "$f" "$poolpath_any/"; then + die "error: failure while copying files to $poolpath_any" + fi + fi + if [ ! -f "$ftppath_any/$fname" ]; then + if ! ln -s "$poolrel_any/$fname" "$ftppath_any/$fname"; then + die "error: failure symlinking $fname to $ftppath_any" + fi fi if ! ln -s "$poolrel_any/$fname" "$ftppath/$fname"; then die "error: failure symlinking $fname to $ftppath" -- cgit v1.2.2 From 140c187203be9d000c0cbad02fdfb7adb972937c Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sat, 7 Aug 2010 16:14:06 +0200 Subject: Override default config with config.local This is useful for local testing --- config | 3 +++ 1 file changed, 3 insertions(+) diff --git a/config b/config index 1cd533f..5e7ab94 100644 --- a/config +++ b/config @@ -21,3 +21,6 @@ SRCEXT=".src.tar.gz" # Allowed licenses: get sourceballs only for licenses in this array ALLOWED_LICENSES=('GPL' 'GPL1' 'GPL2' 'LGPL' 'LGPL1' 'LGPL2') + +# Override default config with config.local +[ -f "$(dirname $0)/config.local" ] && . "$(dirname $0)/config.local" -- cgit v1.2.2 From e843fb3b4c5792ca3311aca311c9af6ea764c4fc Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sat, 7 Aug 2010 20:00:29 +0200 Subject: determine correct dir even if config was sourced --- config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config b/config index 5e7ab94..88e50bc 100644 --- a/config +++ b/config @@ -23,4 +23,4 @@ SRCEXT=".src.tar.gz" ALLOWED_LICENSES=('GPL' 'GPL1' 'GPL2' 'LGPL' 'LGPL1' 'LGPL2') # Override default config with config.local -[ -f "$(dirname $0)/config.local" ] && . "$(dirname $0)/config.local" +[ -f "$(dirname ${BASH_SOURCE[0]})/config.local" ] && . "$(dirname ${BASH_SOURCE[0]})/config.local" -- cgit v1.2.2 From bce7eb253a5e2344c5a3a37f53cf971c8df8ce1d Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sat, 7 Aug 2010 20:02:26 +0200 Subject: add some sample PKGBUILDs --- test/packages/pkg-any-a/PKGBUILD | 11 +++++++++++ test/packages/pkg-any-b/PKGBUILD | 11 +++++++++++ test/packages/pkg-simple-a/PKGBUILD | 11 +++++++++++ test/packages/pkg-simple-b/PKGBUILD | 11 +++++++++++ test/packages/pkg-split-a/PKGBUILD | 16 ++++++++++++++++ test/packages/pkg-split-b/PKGBUILD | 16 ++++++++++++++++ 6 files changed, 76 insertions(+) create mode 100644 test/packages/pkg-any-a/PKGBUILD create mode 100644 test/packages/pkg-any-b/PKGBUILD create mode 100644 test/packages/pkg-simple-a/PKGBUILD create mode 100644 test/packages/pkg-simple-b/PKGBUILD create mode 100644 test/packages/pkg-split-a/PKGBUILD create mode 100644 test/packages/pkg-split-b/PKGBUILD diff --git a/test/packages/pkg-any-a/PKGBUILD b/test/packages/pkg-any-a/PKGBUILD new file mode 100644 index 0000000..348bc4b --- /dev/null +++ b/test/packages/pkg-any-a/PKGBUILD @@ -0,0 +1,11 @@ +pkgname=pkg-any-a +pkgver=1 +pkgrel=1 +pkgdesc="A package called ${pkgname}" +arch=('any') +url='http://www.archlinux.org/' +license=('GPL') + +package() { + true +} diff --git a/test/packages/pkg-any-b/PKGBUILD b/test/packages/pkg-any-b/PKGBUILD new file mode 100644 index 0000000..fd42748 --- /dev/null +++ b/test/packages/pkg-any-b/PKGBUILD @@ -0,0 +1,11 @@ +pkgname=pkg-any-b +pkgver=1 +pkgrel=1 +pkgdesc="A package called ${pkgname}" +arch=('any') +url='http://www.archlinux.org/' +license=('GPL') + +package() { + true +} diff --git a/test/packages/pkg-simple-a/PKGBUILD b/test/packages/pkg-simple-a/PKGBUILD new file mode 100644 index 0000000..78f3c8e --- /dev/null +++ b/test/packages/pkg-simple-a/PKGBUILD @@ -0,0 +1,11 @@ +pkgname=pkg-simple-a +pkgver=1 +pkgrel=1 +pkgdesc="A package called ${pkgname}" +arch=('i686' 'x86_64') +url='http://www.archlinux.org/' +license=('GPL') + +package() { + true +} diff --git a/test/packages/pkg-simple-b/PKGBUILD b/test/packages/pkg-simple-b/PKGBUILD new file mode 100644 index 0000000..1ef298e --- /dev/null +++ b/test/packages/pkg-simple-b/PKGBUILD @@ -0,0 +1,11 @@ +pkgname=pkg-simple-b +pkgver=1 +pkgrel=1 +pkgdesc="A package called ${pkgname}" +arch=('i686' 'x86_64') +url='http://www.archlinux.org/' +license=('GPL') + +package() { + true +} diff --git a/test/packages/pkg-split-a/PKGBUILD b/test/packages/pkg-split-a/PKGBUILD new file mode 100644 index 0000000..6ac77ea --- /dev/null +++ b/test/packages/pkg-split-a/PKGBUILD @@ -0,0 +1,16 @@ +pkgbase=pkg-split-a +pkgname=('pkg-split-a1' 'pkg-split-a2') +pkgver=1 +pkgrel=1 +pkgdesc="A split package called ${pkgbase}" +arch=('i686' 'x86_64') +url='http://www.archlinux.org/' +license=('GPL') + +package_pkg-split-a1() { + true +} + +package_pkg-split-a2() { + true +} diff --git a/test/packages/pkg-split-b/PKGBUILD b/test/packages/pkg-split-b/PKGBUILD new file mode 100644 index 0000000..e1817b7 --- /dev/null +++ b/test/packages/pkg-split-b/PKGBUILD @@ -0,0 +1,16 @@ +pkgbase=pkg-split-b +pkgname=('pkg-split-b1' 'pkg-split-b2') +pkgver=1 +pkgrel=1 +pkgdesc="A split package called ${pkgbase}" +arch=('i686' 'x86_64') +url='http://www.archlinux.org/' +license=('GPL') + +package_pkg-split-b1() { + true +} + +package_pkg-split-b2() { + true +} -- cgit v1.2.2 From bf064423e1f5d60c76940c3b8b4130921dbaa0d7 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sat, 7 Aug 2010 20:02:42 +0200 Subject: add a local copy of shunit2 This is a local copy of shUnit 2.1.5 from http://shunit2.googlecode.com/ License: LGPL --- test/lib/shunit2 | 1116 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 1116 insertions(+) create mode 100644 test/lib/shunit2 diff --git a/test/lib/shunit2 b/test/lib/shunit2 new file mode 100644 index 0000000..d900a70 --- /dev/null +++ b/test/lib/shunit2 @@ -0,0 +1,1116 @@ +# $Id: shunit2 277 2008-10-29 21:20:22Z kate.ward@forestent.com $ +# vim:et:ft=sh:sts=2:sw=2 +# vim:foldmethod=marker:foldmarker=/**,*/ +# +#/** +# +# +# +# shUnit 2.1.5 +# Shell Unit Test Framework +# +# http://shunit2.googlecode.com/ +# +# written by Kate Ward <kate.ward@forestent.com> +# released under the LGPL +# +# This module implements a xUnit based unit test framework similar to JUnit. +# +#*/ + +SHUNIT_VERSION='2.1.5' + +SHUNIT_TRUE=0 +SHUNIT_FALSE=1 +SHUNIT_ERROR=2 + +_shunit_warn() { echo "shunit2:WARN $@" >&2; } +_shunit_error() { echo "shunit2:ERROR $@" >&2; } +_shunit_fatal() { echo "shunit2:FATAL $@" >&2; } + +# specific shell checks +if [ -n "${ZSH_VERSION:-}" ]; then + setopt |grep "^shwordsplit$" >/dev/null + if [ $? -ne ${SHUNIT_TRUE} ]; then + _shunit_fatal 'zsh shwordsplit option is required for proper operation' + exit ${SHUNIT_ERROR} + fi + if [ -z "${SHUNIT_PARENT:-}" ]; then + _shunit_fatal "zsh does not pass \$0 through properly. please declare \ +\"SHUNIT_PARENT=\$0\" before calling shUnit2" + exit ${SHUNIT_ERROR} + fi +fi + +# +# constants +# + +__SHUNIT_ASSERT_MSG_PREFIX='ASSERT:' +__SHUNIT_PARENT=${SHUNIT_PARENT:-$0} + +# set the constants readonly +shunit_constants_=`set |grep '^__SHUNIT_' |cut -d= -f1` +echo "${shunit_constants_}" |grep '^Binary file' >/dev/null \ + && shunit_constants_=`set |grep -a '^__SHUNIT_' |cut -d= -f1` +for shunit_constant_ in ${shunit_constants_}; do + shunit_ro_opts_='' + case ${ZSH_VERSION:-} in + '') ;; # this isn't zsh + [123].*) ;; # early versions (1.x, 2.x, 3.x) + *) shunit_ro_opts_='-g' ;; # all later versions. declare readonly globally + esac + readonly ${shunit_ro_opts_} ${shunit_constant_} +done +unset shunit_constant_ shunit_constants_ shunit_ro_opts_ + +# variables +__shunit_skip=${SHUNIT_FALSE} +__shunit_suite='' + +# counts of tests +__shunit_testSuccess=${SHUNIT_TRUE} +__shunit_testsTotal=0 +__shunit_testsPassed=0 +__shunit_testsFailed=0 + +# counts of asserts +__shunit_assertsTotal=0 +__shunit_assertsPassed=0 +__shunit_assertsFailed=0 +__shunit_assertsSkipped=0 + +__shunit_lineno='' +__shunit_reportGenerated=${SHUNIT_FALSE} + +# macros +_SHUNIT_LINENO_='eval __shunit_lineno=""; if [ "${1:-}" = "--lineno" ]; then [ -n "$2" ] && __shunit_lineno="[$2] "; shift 2; fi' + +#----------------------------------------------------------------------------- +# assert functions +# + +#/** +# +# +# void +# +# +# +# +# assertEquals +# string [message] +# string expected +# string actual +# +# +# Asserts that expected and +# actual are equal to one another. The message is +# optional. +# +# +#*/ +assertEquals() +{ + ${_SHUNIT_LINENO_} + if [ $# -lt 2 -o $# -gt 3 ]; then + _shunit_error "assertEquals() requires two or three arguments; $# given" + _shunit_error "1: ${1:+$1} 2: ${2:+$2} 3: ${3:+$3}" + return ${SHUNIT_ERROR} + fi + _shunit_shouldSkip && return ${SHUNIT_TRUE} + + shunit_message_=${__shunit_lineno} + if [ $# -eq 3 ]; then + shunit_message_="${shunit_message_}$1" + shift + fi + shunit_expected_=$1 + shunit_actual_=$2 + + shunit_return=${SHUNIT_TRUE} + if [ "${shunit_expected_}" = "${shunit_actual_}" ]; then + _shunit_assertPass + else + failNotEquals "${shunit_message_}" "${shunit_expected_}" "${shunit_actual_}" + shunit_return=${SHUNIT_FALSE} + fi + + unset shunit_message_ shunit_expected_ shunit_actual_ + return ${shunit_return} +} +_ASSERT_EQUALS_='eval assertEquals --lineno "${LINENO:-}"' + +#/** +# +# +# void +# +# +# +# +# assertNotEquals +# string [message] +# string unexpected +# string actual +# +# +# Asserts that unexpected and +# actual are not +# equal to one another. The message is optional. +# +# +#*/ +assertNotEquals() +{ + ${_SHUNIT_LINENO_} + if [ $# -lt 2 -o $# -gt 3 ]; then + _shunit_error "assertNotEquals() requires two or three arguments; $# given" + return ${SHUNIT_ERROR} + fi + _shunit_shouldSkip && return ${SHUNIT_TRUE} + + shunit_message_=${__shunit_lineno} + if [ $# -eq 3 ]; then + shunit_message_="${shunit_message_}$1" + shift + fi + shunit_unexpected_=$1 + shunit_actual_=$2 + + shunit_return=${SHUNIT_TRUE} + if [ "${shunit_unexpected_}" != "${shunit_actual_}" ]; then + _shunit_assertPass + else + failSame "${shunit_message_}" "$@" + shunit_return=${SHUNIT_FALSE} + fi + + unset shunit_message_ shunit_unexpected_ shunit_actual_ + return ${shunit_return} +} +_ASSERT_NOT_EQUALS_='eval assertNotEquals --lineno "${LINENO:-}"' + +#/** +# +# +# void +# +# +# +# +# assertNull +# string [message] +# string value +# +# +# Asserts that value is null, +# or in shell terms a zero-length string. The message is optional. +# +# +#*/ +assertNull() +{ + ${_SHUNIT_LINENO_} + if [ $# -lt 1 -o $# -gt 2 ]; then + _shunit_error "assertNull() requires one or two arguments; $# given" + return ${SHUNIT_ERROR} + fi + _shunit_shouldSkip && return ${SHUNIT_TRUE} + + shunit_message_=${__shunit_lineno} + if [ $# -eq 2 ]; then + shunit_message_="${shunit_message_}$1" + shift + fi + assertTrue "${shunit_message_}" "[ -z '$1' ]" + shunit_return=$? + + unset shunit_message_ + return ${shunit_return} +} +_ASSERT_NULL_='eval assertNull --lineno "${LINENO:-}"' + +#/** +# +# +# void +# +# +# +# +# assertNotNull +# string [message] +# string value +# +# +# Asserts that value is not null, or in shell terms not +# a zero-length string. The message is optional. +# +# +#*/ +assertNotNull() +{ + ${_SHUNIT_LINENO_} + if [ $# -gt 2 ]; then # allowing 0 arguments as $1 might actually be null + _shunit_error "assertNotNull() requires one or two arguments; $# given" + return ${SHUNIT_ERROR} + fi + _shunit_shouldSkip && return ${SHUNIT_TRUE} + + shunit_message_=${__shunit_lineno} + if [ $# -eq 2 ]; then + shunit_message_="${shunit_message_}$1" + shift + fi + assertTrue "${shunit_message_}" "[ -n '${1:-}' ]" + shunit_return=$? + + unset shunit_message_ + return ${shunit_return} +} +_ASSERT_NOT_NULL_='eval assertNotNull --lineno "${LINENO:-}"' + +#/** +# +# +# void +# +# +# +# +# assertSame +# string [message] +# string expected +# string actual +# +# +# This function is functionally equivalent to +# assertEquals. +# +# +#*/ +assertSame() +{ + ${_SHUNIT_LINENO_} + if [ $# -lt 2 -o $# -gt 3 ]; then + _shunit_error "assertSame() requires one or two arguments; $# given" + return ${SHUNIT_ERROR} + fi + _shunit_shouldSkip && return ${SHUNIT_TRUE} + + shunit_message_=${__shunit_lineno} + if [ $# -eq 3 ]; then + shunit_message_="${shunit_message_}$1" + shift + fi + assertEquals "${shunit_message_}" "$1" "$2" + shunit_return=$? + + unset shunit_message_ + return ${shunit_return} +} +_ASSERT_SAME_='eval assertSame --lineno "${LINENO:-}"' + +#/** +# +# +# void +# +# +# +# +# assertNotSame +# string [message] +# string unexpected +# string actual +# +# +# Asserts that unexpected and +# actual are not +# equal to one another. The message is optional. +# +# +#*/ +assertNotSame() +{ + ${_SHUNIT_LINENO_} + if [ $# -lt 2 -o $# -gt 3 ]; then + _shunit_error "assertNotSame() requires two or three arguments; $# given" + return ${SHUNIT_ERROR} + fi + _shunit_shouldSkip && return ${SHUNIT_TRUE} + + shunit_message_=${__shunit_lineno} + if [ $# -eq 3 ]; then + shunit_message_="${shunit_message_:-}$1" + shift + fi + assertNotEquals "${shunit_message_}" "$1" "$2" + shunit_return=$? + + unset shunit_message_ + return ${shunit_return} +} +_ASSERT_NOT_SAME_='eval assertNotSame --lineno "${LINENO:-}"' + +#/** +# +# +# void +# +# +# +# +# assertTrue +# string [message] +# string condition +# +# +# Asserts that a given shell test condition is true. The message is +# optional. +# Testing whether something is true or false is easy enough by using +# the assertEquals/assertNotSame functions. Shell supports much more +# complicated tests though, and a means to support them was needed. As such, +# this function tests that conditions are true or false through evaluation +# rather than just looking for a true or false. +# +# The following test will succeed: assertTrue "[ 34 -gt 23 ]" +# The folloing test will fail with a message: assertTrue "test failed" "[ -r '/non/existant/file' ]" +# +# +# +#*/ +assertTrue() +{ + ${_SHUNIT_LINENO_} + if [ $# -gt 2 ]; then + _shunit_error "assertTrue() takes one two arguments; $# given" + return ${SHUNIT_ERROR} + fi + _shunit_shouldSkip && return ${SHUNIT_TRUE} + + shunit_message_=${__shunit_lineno} + if [ $# -eq 2 ]; then + shunit_message_="${shunit_message_}$1" + shift + fi + shunit_condition_=$1 + + # see if condition is an integer, i.e. a return value + shunit_match_=`expr "${shunit_condition_}" : '\([0-9]*\)'` + shunit_return=${SHUNIT_TRUE} + if [ -z "${shunit_condition_}" ]; then + # null condition + shunit_return=${SHUNIT_FALSE} + elif [ "${shunit_condition_}" = "${shunit_match_}" ]; then + # possible return value. treating 0 as true, and non-zero as false. + [ ${shunit_condition_} -ne 0 ] && shunit_return=${SHUNIT_FALSE} + else + # (hopefully) a condition + ( eval ${shunit_condition_} ) >/dev/null 2>&1 + [ $? -ne 0 ] && shunit_return=${SHUNIT_FALSE} + fi + + # record the test + if [ ${shunit_return} -eq ${SHUNIT_TRUE} ]; then + _shunit_assertPass + else + _shunit_assertFail "${shunit_message_}" + fi + + unset shunit_message_ shunit_condition_ shunit_match_ + return ${shunit_return} +} +_ASSERT_TRUE_='eval assertTrue --lineno "${LINENO:-}"' + +#/** +# +# +# void +# +# +# +# +# assertFalse +# string [message] +# string condition +# +# +# Asserts that a given shell test condition is false. The message is +# optional. +# Testing whether something is true or false is easy enough by using +# the assertEquals/assertNotSame functions. Shell supports much more +# complicated tests though, and a means to support them was needed. As such, +# this function tests that conditions are true or false through evaluation +# rather than just looking for a true or false. +# +# The following test will succeed: assertFalse "[ 'apples' = 'oranges' ]" +# The folloing test will fail with a message: assertFalse "test failed" "[ 1 -eq 1 -a 2 -eq 2 ]" +# +# +# +#*/ +assertFalse() +{ + ${_SHUNIT_LINENO_} + if [ $# -lt 1 -o $# -gt 2 ]; then + _shunit_error "assertFalse() quires one or two arguments; $# given" + return ${SHUNIT_ERROR} + fi + _shunit_shouldSkip && return ${SHUNIT_TRUE} + + shunit_message_=${__shunit_lineno} + if [ $# -eq 2 ]; then + shunit_message_="${shunit_message_}$1" + shift + fi + shunit_condition_=$1 + + # see if condition is an integer, i.e. a return value + shunit_match_=`expr "${shunit_condition_}" : '\([0-9]*\)'` + shunit_return=${SHUNIT_TRUE} + if [ -z "${shunit_condition_}" ]; then + # null condition + shunit_return=${SHUNIT_FALSE} + elif [ "${shunit_condition_}" = "${shunit_match_}" ]; then + # possible return value. treating 0 as true, and non-zero as false. + [ ${shunit_condition_} -eq 0 ] && shunit_return=${SHUNIT_FALSE} + else + # (hopefully) a condition + ( eval ${shunit_condition_} ) >/dev/null 2>&1 + [ $? -eq 0 ] && shunit_return=${SHUNIT_FALSE} + fi + + # record the test + if [ ${shunit_return} -eq ${SHUNIT_TRUE} ]; then + _shunit_assertPass + else + _shunit_assertFail "${shunit_message_}" + fi + + unset shunit_message_ shunit_condition_ shunit_match_ + return ${shunit_return} +} +_ASSERT_FALSE_='eval assertFalse --lineno "${LINENO:-}"' + +#----------------------------------------------------------------------------- +# failure functions +# + +#/** +# +# +# void +# +# +# +# +# fail +# string [message] +# +# +# Fails the test immediately, with the optional message. +# +# +#*/ +fail() +{ + ${_SHUNIT_LINENO_} + if [ $# -gt 1 ]; then + _shunit_error "fail() requires one or two arguments; $# given" + return ${SHUNIT_ERROR} + fi + _shunit_shouldSkip && return ${SHUNIT_TRUE} + + shunit_message_=${__shunit_lineno} + if [ $# -eq 1 ]; then + shunit_message_="${shunit_message_}$1" + shift + fi + + _shunit_assertFail "${shunit_message_}" + + unset shunit_message_ + return ${SHUNIT_FALSE} +} +_FAIL_='eval fail --lineno "${LINENO:-}"' + +#/** +# +# +# void +# +# +# +# +# failNotEquals +# string [message] +# string unexpected +# string actual +# +# +# Fails the test if unexpected and +# actual are not +# equal to one another. The message is optional. +# +# +#*/ +failNotEquals() +{ + ${_SHUNIT_LINENO_} + if [ $# -lt 2 -o $# -gt 3 ]; then + _shunit_error "failNotEquals() requires one or two arguments; $# given" + return ${SHUNIT_ERROR} + fi + _shunit_shouldSkip && return ${SHUNIT_TRUE} + + shunit_message_=${__shunit_lineno} + if [ $# -eq 3 ]; then + shunit_message_="${shunit_message_}$1" + shift + fi + shunit_unexpected_=$1 + shunit_actual_=$2 + + _shunit_assertFail "${shunit_message_:+${shunit_message_} }expected:<${shunit_unexpected_}> but was:<${shunit_actual_}>" + + unset shunit_message_ shunit_unexpected_ shunit_actual_ + return ${SHUNIT_FALSE} +} +_FAIL_NOT_EQUALS_='eval failNotEquals --lineno "${LINENO:-}"' + +#/** +# +# +# void +# +# +# +# +# failSame +# string [message] +# +# +# Indicate test failure because arguments were the same. The message is +# optional. +# +# +#*/ +failSame() +{ + ${_SHUNIT_LINENO_} + if [ $# -lt 2 -o $# -gt 3 ]; then + _shunit_error "failSame() requires two or three arguments; $# given" + return ${SHUNIT_ERROR} + fi + _shunit_shouldSkip && return ${SHUNIT_TRUE} + + shunit_message_=${__shunit_lineno} + if [ $# -eq 3 ]; then + shunit_message_="${shunit_message_}$1" + shift + fi + + _shunit_assertFail "${shunit_message_:+${shunit_message_} }expected not same" + + unset shunit_message_ + return ${SHUNIT_FALSE} +} +_FAIL_SAME_='eval failSame --lineno "${LINENO:-}"' + +#/** +# +# +# void +# +# +# +# +# failNotSame +# string [message] +# string expected +# string actual +# +# +# Indicate test failure because arguments were not the same. The +# message is optional. +# +# +#*/ +failNotSame() +{ + ${_SHUNIT_LINENO_} + if [ $# -lt 2 -o $# -gt 3 ]; then + _shunit_error "failNotEquals() requires one or two arguments; $# given" + return ${SHUNIT_ERROR} + fi + _shunit_shouldSkip && return ${SHUNIT_TRUE} + + shunit_message_=${__shunit_lineno} + if [ $# -eq 3 ]; then + shunit_message_="${shunit_message_}$1" + shift + fi + failNotEquals "${shunit_message_}" "$1" "$2" + shunit_return=$? + + unset shunit_message_ + return ${shunit_return} +} +_FAIL_NOT_SAME_='eval failNotSame --lineno "${LINENO:-}"' + +#----------------------------------------------------------------------------- +# skipping functions +# + +#/** +# +# +# void +# +# +# +# +# startSkipping +# +# +# +# This function forces the remaining assert and fail functions to be +# "skipped", i.e. they will have no effect. Each function skipped will be +# recorded so that the total of asserts and fails will not be altered. +# +# +#*/ +startSkipping() +{ + __shunit_skip=${SHUNIT_TRUE} +} + +#/** +# +# +# void +# +# +# +# +# endSkipping +# +# +# +# This function returns calls to the assert and fail functions to their +# default behavior, i.e. they will be called. +# +# +#*/ +endSkipping() +{ + __shunit_skip=${SHUNIT_FALSE} +} + +#/** +# +# +# boolean +# +# +# +# +# isSkipping +# +# +# +# This function returns the state of skipping. +# +# +#*/ +isSkipping() +{ + return ${__shunit_skip} +} + +#----------------------------------------------------------------------------- +# suite functions +# + +#/** +# +# +# void +# +# +# +# +# suite +# +# +# +# This function can be optionally overridden by the user in their test +# suite. +# If this function exists, it will be called when +# shunit2 is sourced. If it does not exist, shUnit2 will +# search the parent script for all functions beginning with the word +# test, and they will be added dynamically to the test +# suite. +# +# +#*/ +# Note: see _shunit_mktempFunc() for actual implementation +# suite() { :; } + +#/** +# +# +# void +# +# +# +# +# suite_addTest +# string function +# +# +# This function adds a function name to the list of tests scheduled for +# execution as part of this test suite. This function should only be called +# from within the suite() function. +# +# +#*/ +suite_addTest() +{ + shunit_func_=${1:-} + + __shunit_suite="${__shunit_suite:+${__shunit_suite} }${shunit_func_}" + __shunit_testsTotal=`expr ${__shunit_testsTotal} + 1` + + unset shunit_func_ +} + +#/** +# +# +# void +# +# +# +# +# oneTimeSetUp +# +# +# +# This function can be be optionally overridden by the user in their +# test suite. +# If this function exists, it will be called once before any tests are +# run. It is useful to prepare a common environment for all tests. +# +# +#*/ +# Note: see _shunit_mktempFunc() for actual implementation +# oneTimeSetUp() { :; } + +#/** +# +# +# void +# +# +# +# +# oneTimeTearDown +# +# +# +# This function can be be optionally overridden by the user in their +# test suite. +# If this function exists, it will be called once after all tests are +# completed. It is useful to clean up the environment after all tests. +# +# +#*/ +# Note: see _shunit_mktempFunc() for actual implementation +# oneTimeTearDown() { :; } + +#/** +# +# +# void +# +# +# +# +# setUp +# +# +# +# This function can be be optionally overridden by the user in their +# test suite. +# If this function exists, it will be called before each test is run. +# It is useful to reset the environment before each test. +# +# +#*/ +# Note: see _shunit_mktempFunc() for actual implementation +# setUp() { :; } + +#/** +# +# +# void +# +# +# +# +# tearDown +# +# +# +# This function can be be optionally overridden by the user in their +# test suite. +# If this function exists, it will be called after each test completes. +# It is useful to clean up the environment after each test. +# +# +#*/ +# Note: see _shunit_mktempFunc() for actual implementation +# tearDown() { :; } + +#------------------------------------------------------------------------------ +# internal shUnit2 functions +# + +# this function is a cross-platform temporary directory creation tool. not all +# OSes have the mktemp function, so one is included here. +_shunit_mktempDir() +{ + # try the standard mktemp function + ( exec mktemp -dqt shunit.XXXXXX 2>/dev/null ) && return + + # the standard mktemp didn't work. doing our own. + if [ -r '/dev/urandom' ]; then + _shunit_random_=`od -vAn -N4 -tx4 "${_shunit_file_}" +#! /bin/sh +exit ${SHUNIT_TRUE} +EOF + chmod +x "${_shunit_file_}" + done + + unset _shunit_file_ +} + +_shunit_cleanup() +{ + _shunit_name_=$1 + + case ${_shunit_name_} in + EXIT) _shunit_signal_=0 ;; + INT) _shunit_signal_=2 ;; + TERM) _shunit_signal_=15 ;; + *) + _shunit_warn "unrecognized trap value (${_shunit_name_})" + _shunit_signal_=0 + ;; + esac + + # do our work + rm -fr "${__shunit_tmpDir}" + + # exit for all non-EXIT signals + if [ ${_shunit_name_} != 'EXIT' ]; then + _shunit_warn "trapped and now handling the (${_shunit_name_}) signal" + # disable EXIT trap + trap 0 + # add 128 to signal and exit + exit `expr ${_shunit_signal_} + 128` + elif [ ${__shunit_reportGenerated} -eq ${SHUNIT_FALSE} ] ; then + _shunit_assertFail 'Unknown failure encountered running a test' + _shunit_generateReport + exit ${SHUNIT_ERROR} + fi + + unset _shunit_name_ _shunit_signal_ +} + +# The actual running of the tests happens here. +_shunit_execSuite() +{ + for _shunit_test_ in ${__shunit_suite}; do + __shunit_testSuccess=${SHUNIT_TRUE} + + # disable skipping + endSkipping + + # execute the per-test setup function + setUp + + # execute the test + echo "${_shunit_test_}" + eval ${_shunit_test_} + + # execute the per-test tear-down function + tearDown + + # update stats + if [ ${__shunit_testSuccess} -eq ${SHUNIT_TRUE} ]; then + __shunit_testsPassed=`expr ${__shunit_testsPassed} + 1` + else + __shunit_testsFailed=`expr ${__shunit_testsFailed} + 1` + fi + done + + unset _shunit_test_ +} + +# This function exits shUnit2 with the appropriate error code and OK/FAILED +# message. +_shunit_generateReport() +{ + _shunit_ok_=${SHUNIT_TRUE} + + # if no exit code was provided one, determine an appropriate one + [ ${__shunit_testsFailed} -gt 0 \ + -o ${__shunit_testSuccess} -eq ${SHUNIT_FALSE} ] \ + && _shunit_ok_=${SHUNIT_FALSE} + + echo + if [ ${__shunit_testsTotal} -eq 1 ]; then + echo "Ran ${__shunit_testsTotal} test." + else + echo "Ran ${__shunit_testsTotal} tests." + fi + + _shunit_failures_='' + _shunit_skipped_='' + [ ${__shunit_assertsFailed} -gt 0 ] \ + && _shunit_failures_="failures=${__shunit_assertsFailed}" + [ ${__shunit_assertsSkipped} -gt 0 ] \ + && _shunit_skipped_="skipped=${__shunit_assertsSkipped}" + + if [ ${_shunit_ok_} -eq ${SHUNIT_TRUE} ]; then + _shunit_msg_='OK' + [ -n "${_shunit_skipped_}" ] \ + && _shunit_msg_="${_shunit_msg_} (${_shunit_skipped_})" + else + _shunit_msg_="FAILED (${_shunit_failures_}" + [ -n "${_shunit_skipped_}" ] \ + && _shunit_msg_="${_shunit_msg_},${_shunit_skipped_}" + _shunit_msg_="${_shunit_msg_})" + fi + + echo + echo ${_shunit_msg_} + __shunit_reportGenerated=${SHUNIT_TRUE} + + unset _shunit_failures_ _shunit_msg_ _shunit_ok_ _shunit_skipped_ +} + +_shunit_shouldSkip() +{ + [ ${__shunit_skip} -eq ${SHUNIT_FALSE} ] && return ${SHUNIT_FALSE} + _shunit_assertSkip +} + +_shunit_assertPass() +{ + __shunit_assertsPassed=`expr ${__shunit_assertsPassed} + 1` + __shunit_assertsTotal=`expr ${__shunit_assertsTotal} + 1` +} + +_shunit_assertFail() +{ + _shunit_msg_=$1 + + __shunit_testSuccess=${SHUNIT_FALSE} + __shunit_assertsFailed=`expr ${__shunit_assertsFailed} + 1` + __shunit_assertsTotal=`expr ${__shunit_assertsTotal} + 1` + echo "${__SHUNIT_ASSERT_MSG_PREFIX}${_shunit_msg_}" + + unset _shunit_msg_ +} + +_shunit_assertSkip() +{ + __shunit_assertsSkipped=`expr ${__shunit_assertsSkipped} + 1` + __shunit_assertsTotal=`expr ${__shunit_assertsTotal} + 1` +} + +#------------------------------------------------------------------------------ +# main +# + +# create a temporary storage location +__shunit_tmpDir=`_shunit_mktempDir` + +# provide a public temporary directory for unit test scripts +# TODO(kward): document this +shunit_tmpDir="${__shunit_tmpDir}/tmp" +mkdir "${shunit_tmpDir}" + +# setup traps to clean up after ourselves +trap '_shunit_cleanup EXIT' 0 +trap '_shunit_cleanup INT' 2 +trap '_shunit_cleanup TERM' 15 + +# create phantom functions to work around issues with Cygwin +_shunit_mktempFunc +PATH="${__shunit_tmpDir}:${PATH}" + +# execute the oneTimeSetUp function (if it exists) +oneTimeSetUp + +# execute the suite function defined in the parent test script +# deprecated as of 2.1.0 +suite + +# if no suite function was defined, dynamically build a list of functions +if [ -z "${__shunit_suite}" ]; then + shunit_funcs_=`grep "^[ \t]*test[A-Za-z0-9_]* *()" ${__SHUNIT_PARENT} \ + |sed 's/[^A-Za-z0-9_]//g'` + for shunit_func_ in ${shunit_funcs_}; do + suite_addTest ${shunit_func_} + done +fi +unset shunit_func_ shunit_funcs_ + +# execute the tests +_shunit_execSuite + +# execute the oneTimeTearDown function (if it exists) +oneTimeTearDown + +# generate the report +_shunit_generateReport + +# that's it folks +[ ${__shunit_testsFailed} -eq 0 ] +exit $? + +#/** +# +#*/ -- cgit v1.2.2 From 266d9d35240e84393b2ffc4ec40d96dcdac1f87f Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sat, 7 Aug 2010 20:04:40 +0200 Subject: add default makepkg.conf for both arches --- test/lib/makepkg-i686.conf | 115 +++++++++++++++++++++++++++++++++++++++++++ test/lib/makepkg-x86_64.conf | 115 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 230 insertions(+) create mode 100644 test/lib/makepkg-i686.conf create mode 100644 test/lib/makepkg-x86_64.conf diff --git a/test/lib/makepkg-i686.conf b/test/lib/makepkg-i686.conf new file mode 100644 index 0000000..553f943 --- /dev/null +++ b/test/lib/makepkg-i686.conf @@ -0,0 +1,115 @@ +# +# /etc/makepkg.conf +# + +######################################################################### +# SOURCE ACQUISITION +######################################################################### +# +#-- The download utilities that makepkg should use to acquire sources +# Format: 'protocol::agent' +DLAGENTS=('ftp::/usr/bin/wget -c --passive-ftp -t 3 --waitretry=3 -O %o %u' + 'http::/usr/bin/wget -c -t 3 --waitretry=3 -O %o %u' + 'https::/usr/bin/wget -c -t 3 --waitretry=3 --no-check-certificate -O %o %u' + 'rsync::/usr/bin/rsync -z %u %o' + 'scp::/usr/bin/scp -C %u %o') + +# Other common tools: +# /usr/bin/snarf +# /usr/bin/lftpget -c +# /usr/bin/curl + +######################################################################### +# ARCHITECTURE, COMPILE FLAGS +######################################################################### +# +CARCH="i686" +CHOST="i686-pc-linux-gnu" + +#-- Exclusive: will only run on i686 +# -march (or -mcpu) builds exclusively for an architecture +# -mtune optimizes for an architecture, but builds for whole processor family +CFLAGS="-march=i686 -mtune=generic -O2 -pipe" +CXXFLAGS="-march=i686 -mtune=generic -O2 -pipe" +LDFLAGS="-Wl,--hash-style=gnu -Wl,--as-needed" +#-- Make Flags: change this for DistCC/SMP systems +#MAKEFLAGS="-j2" + +######################################################################### +# BUILD ENVIRONMENT +######################################################################### +# +# Defaults: BUILDENV=(fakeroot !distcc color !ccache) +# A negated environment option will do the opposite of the comments below. +# +#-- fakeroot: Allow building packages as a non-root user +#-- distcc: Use the Distributed C/C++/ObjC compiler +#-- color: Colorize output messages +#-- ccache: Use ccache to cache compilation +# +BUILDENV=(fakeroot !distcc color !ccache) +# +#-- If using DistCC, your MAKEFLAGS will also need modification. In addition, +#-- specify a space-delimited list of hosts running in the DistCC cluster. +#DISTCC_HOSTS="" + +######################################################################### +# GLOBAL PACKAGE OPTIONS +# These are default values for the options=() settings +######################################################################### +# +# Default: OPTIONS=(strip docs libtool emptydirs zipman purge) +# A negated option will do the opposite of the comments below. +# +#-- strip: Strip symbols from binaries/libraries in STRIP_DIRS +#-- docs: Save doc directories specified by DOC_DIRS +#-- libtool: Leave libtool (.la) files in packages +#-- emptydirs: Leave empty directories in packages +#-- zipman: Compress manual (man and info) pages in MAN_DIRS with gzip +#-- purge: Remove files specified by PURGE_TARGETS +# +OPTIONS=(strip docs libtool emptydirs zipman purge) + +#-- File integrity checks to use. Valid: md5, sha1, sha256, sha384, sha512 +INTEGRITY_CHECK=(md5) +#-- Options to be used when stripping binaries. See `man strip' for details. +STRIP_BINARIES="--strip-all" +#-- Options to be used when stripping shared libraries. See `man strip' for details. +STRIP_SHARED="--strip-unneeded" +#-- Options to be used when stripping static libraries. See `man strip' for details. +STRIP_STATIC="--strip-debug" +#-- Manual (man and info) directories to compress (if zipman is specified) +MAN_DIRS=({usr{,/local}{,/share},opt/*}/{man,info}) +#-- Doc directories to remove (if !docs is specified) +DOC_DIRS=(usr/{,local/}{,share/}{doc,gtk-doc} opt/*/{doc,gtk-doc}) +#-- Directories to be searched for the strip option (if strip is specified) +STRIP_DIRS=(bin lib sbin usr/{bin,lib,sbin,local/{bin,lib,sbin}} opt/*/{bin,lib,sbin}) +#-- Files to be removed from all packages (if purge is specified) +PURGE_TARGETS=(usr/{,share}/info/dir .packlist *.pod) + +######################################################################### +# PACKAGE OUTPUT +######################################################################### +# +# Default: put built package and cached source in build directory +# +#-- Destination: specify a fixed directory where all packages will be placed +#PKGDEST=/home/packages +#-- Source cache: specify a fixed directory where source files will be cached +#SRCDEST=/home/sources +#-- Source packages: specify a fixed directory where all src packages will be placed +#SRCPKGDEST=/home/srcpackages +#-- Packager: name/email of the person or organization building packages +#PACKAGER="John Doe " + +######################################################################### +# EXTENSION DEFAULTS +######################################################################### +# +# WARNING: Do NOT modify these variables unless you know what you are +# doing. +# +PKGEXT='.pkg.tar.xz' +SRCEXT='.src.tar.gz' + +# vim: set ft=sh ts=2 sw=2 et: diff --git a/test/lib/makepkg-x86_64.conf b/test/lib/makepkg-x86_64.conf new file mode 100644 index 0000000..e53375f --- /dev/null +++ b/test/lib/makepkg-x86_64.conf @@ -0,0 +1,115 @@ +# +# /etc/makepkg.conf +# + +######################################################################### +# SOURCE ACQUISITION +######################################################################### +# +#-- The download utilities that makepkg should use to acquire sources +# Format: 'protocol::agent' +DLAGENTS=('ftp::/usr/bin/wget -c --passive-ftp -t 3 --waitretry=3 -O %o %u' + 'http::/usr/bin/wget -c -t 3 --waitretry=3 -O %o %u' + 'https::/usr/bin/wget -c -t 3 --waitretry=3 --no-check-certificate -O %o %u' + 'rsync::/usr/bin/rsync -z %u %o' + 'scp::/usr/bin/scp -C %u %o') + +# Other common tools: +# /usr/bin/snarf +# /usr/bin/lftpget -c +# /usr/bin/curl + +######################################################################### +# ARCHITECTURE, COMPILE FLAGS +######################################################################### +# +CARCH="x86_64" +CHOST="x86_64-unknown-linux-gnu" + +#-- Exclusive: will only run on x86_64 +# -march (or -mcpu) builds exclusively for an architecture +# -mtune optimizes for an architecture, but builds for whole processor family +CFLAGS="-march=x86-64 -mtune=generic -O2 -pipe" +CXXFLAGS="-march=x86-64 -mtune=generic -O2 -pipe" +LDFLAGS="-Wl,--hash-style=gnu -Wl,--as-needed" +#-- Make Flags: change this for DistCC/SMP systems +#MAKEFLAGS="-j2" + +######################################################################### +# BUILD ENVIRONMENT +######################################################################### +# +# Defaults: BUILDENV=(fakeroot !distcc color !ccache) +# A negated environment option will do the opposite of the comments below. +# +#-- fakeroot: Allow building packages as a non-root user +#-- distcc: Use the Distributed C/C++/ObjC compiler +#-- color: Colorize output messages +#-- ccache: Use ccache to cache compilation +# +BUILDENV=(fakeroot !distcc color !ccache) +# +#-- If using DistCC, your MAKEFLAGS will also need modification. In addition, +#-- specify a space-delimited list of hosts running in the DistCC cluster. +#DISTCC_HOSTS="" + +######################################################################### +# GLOBAL PACKAGE OPTIONS +# These are default values for the options=() settings +######################################################################### +# +# Default: OPTIONS=(strip docs libtool emptydirs zipman purge) +# A negated option will do the opposite of the comments below. +# +#-- strip: Strip symbols from binaries/libraries in STRIP_DIRS +#-- docs: Save doc directories specified by DOC_DIRS +#-- libtool: Leave libtool (.la) files in packages +#-- emptydirs: Leave empty directories in packages +#-- zipman: Compress manual (man and info) pages in MAN_DIRS with gzip +#-- purge: Remove files specified by PURGE_TARGETS +# +OPTIONS=(strip docs libtool emptydirs zipman purge) + +#-- File integrity checks to use. Valid: md5, sha1, sha256, sha384, sha512 +INTEGRITY_CHECK=(md5) +#-- Options to be used when stripping binaries. See `man strip' for details. +STRIP_BINARIES="--strip-all" +#-- Options to be used when stripping shared libraries. See `man strip' for details. +STRIP_SHARED="--strip-unneeded" +#-- Options to be used when stripping static libraries. See `man strip' for details. +STRIP_STATIC="--strip-debug" +#-- Manual (man and info) directories to compress (if zipman is specified) +MAN_DIRS=({usr{,/local}{,/share},opt/*}/{man,info}) +#-- Doc directories to remove (if !docs is specified) +DOC_DIRS=(usr/{,local/}{,share/}{doc,gtk-doc} opt/*/{doc,gtk-doc}) +#-- Directories to be searched for the strip option (if strip is specified) +STRIP_DIRS=(bin lib sbin usr/{bin,lib,sbin,local/{bin,lib,sbin}} opt/*/{bin,lib,sbin}) +#-- Files to be removed from all packages (if purge is specified) +PURGE_TARGETS=(usr/{,share}/info/dir .packlist *.pod) + +######################################################################### +# PACKAGE OUTPUT +######################################################################### +# +# Default: put built package and cached source in build directory +# +#-- Destination: specify a fixed directory where all packages will be placed +#PKGDEST=/home/packages +#-- Source cache: specify a fixed directory where source files will be cached +#SRCDEST=/home/sources +#-- Source packages: specify a fixed directory where all src packages will be placed +#SRCPKGDEST=/home/srcpackages +#-- Packager: name/email of the person or organization building packages +#PACKAGER="John Doe " + +######################################################################### +# EXTENSION DEFAULTS +######################################################################### +# +# WARNING: Do NOT modify these variables unless you know what you are +# doing. +# +PKGEXT='.pkg.tar.xz' +SRCEXT='.src.tar.gz' + +# vim: set ft=sh ts=2 sw=2 et: -- cgit v1.2.2 From 1d903977850d8e8a22f8246994e72b0ef101eb76 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sat, 7 Aug 2010 20:05:09 +0200 Subject: add some common functions for our test suite --- test/lib/common.inc | 113 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 113 insertions(+) create mode 100644 test/lib/common.inc diff --git a/test/lib/common.inc b/test/lib/common.inc new file mode 100644 index 0000000..5abe2ed --- /dev/null +++ b/test/lib/common.inc @@ -0,0 +1,113 @@ +pkgdir="$(dirname $0)/packages" + +oneTimeSetUp() { + local p + echo -n 'Building packages...' + for p in "${pkgdir}"/*; do + pushd $p >/dev/null + linux32 makepkg -cf --config ${curdir}/lib/makepkg-i686.conf >/dev/null 2>&1 || exit 1 + linux64 makepkg -cf --config ${curdir}/lib/makepkg-x86_64.conf >/dev/null 2>&1 || exit 1 + popd >/dev/null + done + echo 'done' + echo +} + +oneTimeTearDown() { + find "${pkgdir}" -name '*.pkg.tar.*' -delete +} + +setUp() { + local p + local pkg + + [ -f "${curdir}/../config.local" ] && exit 1 + TMP="$(mktemp -d /tmp/$(basename $0).XXXXXXXXXX)" + #echo "Using ${TMP}" + + mkdir -p "${TMP}/"{ftp,tmp,staging,{package,source}-cleanup,svn-{packages,community}-{copy,repo}} + mkdir -p "${TMP}/ftp/"{{core,extra,community,testing,community-testing}/os,packages/{arch,community}}/{i686,any,x86_64} + mkdir -p "${TMP}/staging/"{core,extra,community,testing,community-testing} + + echo -n 'Creating svn repository...' + svnadmin create "${TMP}/svn-packages-repo" + svnadmin create "${TMP}/svn-community-repo" + svn checkout -q "file://${TMP}/svn-packages-repo" "${TMP}/svn-packages-copy" + svn checkout -q "file://${TMP}/svn-community-repo" "${TMP}/svn-community-copy" + + for p in "${pkgdir}"/*; do + pkg=$(basename $p) + mkdir -p "${TMP}/svn-packages-copy/${pkg}"/{trunk,repos} + cp "${p}/PKGBUILD" "${TMP}/svn-packages-copy"/${pkg}/trunk/ + svn add -q "${TMP}/svn-packages-copy"/${pkg} + svn commit -q -m"initial commit of ${pkg}" "${TMP}/svn-packages-copy" + done + echo 'done' + + cat < "${curdir}/../config.local" + FTP_BASE="${TMP}/ftp" + SVNREPO="file://${TMP}/svn-packages-repo" + SVNREPOCOMMUNITY="file://${TMP}/svn-community-repo" + CLEANUP_DESTDIR="${TMP}/package-cleanup" + SOURCE_CLEANUP_DESTDIR="${TMP}/source-cleanup" + STAGING="${TMP}/staging" + TMPDIR="${TMP}/tmp" +eot + . "${curdir}/../config" +} + +tearDown() { + rm -rf "${TMP}" + rm -f "${curdir}/../config.local" + echo +} + +testPackages() { + local p + for p in "${pkgdir}"/*; do + pushd $p >/dev/null + namcap *.pkg.tar.* || fail 'namcap failed' + popd >/dev/null + done +} + +releasePackage() { + local repo=$1 + local pkgbase=$2 + local arch=$3 + + pushd "${TMP}/svn-packages-copy"/${pkgbase}/trunk/ >/dev/null + archrelease ${repo}-${arch} >/dev/null + popd >/dev/null + cp "${pkgdir}/${pkgbase}"/*.pkg.tar.* "${STAGING}"/${repo}/ +} + +checkAnyPackage() { + local repo=$1 + local pkg=$2 + local arch + + [ -f "${FTP_BASE}/packages/arch/any/${pkg}" ] || fail "packages/arch/any/${pkg} not found" + + for arch in any i686 x86_64; do + [ -L "${FTP_BASE}/${repo}/os/${arch}/${pkg}" ] || fail "${repo}/os/${arch}/${pkg} not a symlink" + [ "$(readlink -e "${FTP_BASE}/${repo}/os/${arch}/${pkg}")" == "${FTP_BASE}/packages/arch/any/${pkg}" ] || fail "${repo}/os/${arch}/${pkg} does not link to packages/arch/any/${pkg}" + done + [ -f "${STAGING}"/${repo}/${pkg} ] && fail "${repo}/${pkg} found in staging dir" + + bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}.db.tar.gz" -O | grep -q ${pkg} || fail "${pkg} not in ${repo}/os/${arch}/${repo}.db.tar.gz" +} + +checkPackage() { + local repo=$1 + local pkg=$2 + local arch=$3 + + [ -f "${FTP_BASE}/packages/arch/${arch}/${pkg}" ] || fail "packages/arch/${arch}/${pkg} not found" + [ -L "${FTP_BASE}/${repo}/os/${arch}/${pkg}" ] || fail "${repo}/os/${arch}/${pkg} not a symlink" + [ -f "${STAGING}"/${repo}/${pkg} ] && fail "${repo}/${pkg} found in staging dir" + + [ "$(readlink -e "${FTP_BASE}/${repo}/os/${arch}/${pkg}")" == "${FTP_BASE}/packages/arch/${arch}/${pkg}" ] || fail "${repo}/os/${arch}/${pkg} does not link to packages/arch/${arch}/${pkg}" + + bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}.db.tar.gz" -O | grep -q ${pkg} || fail "${pkg} not in ${repo}/os/${arch}/${repo}.db.tar.gz" +} -- cgit v1.2.2 From 66ae94cafb62da09da77712269f411f095dd81fc Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sat, 7 Aug 2010 20:05:58 +0200 Subject: add an initial set of tests --- test/runTest | 66 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100755 test/runTest diff --git a/test/runTest b/test/runTest new file mode 100755 index 0000000..1d54ea4 --- /dev/null +++ b/test/runTest @@ -0,0 +1,66 @@ +#!/bin/bash + +curdir=$(readlink -e $(dirname $0)) +. "${curdir}/lib/common.inc" + + +testAddSimplePackages() { + local arches=('i686' 'x86_64') + local pkgs=('pkg-simple-a' 'pkg-simple-b') + local pkgbase + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + releasePackage extra ${pkgbase} ${arch} + done + done + + ../db-update extra + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + checkPackage extra ${pkgbase}-1-1-${arch}.pkg.tar.xz ${arch} + done + done +} + +testAddAnyPackages() { + local pkgs=('pkg-any-a' 'pkg-any-b') + local pkgbase + + for pkgbase in ${pkgs[@]}; do + releasePackage extra ${pkgbase} any + done + + ../db-update extra + + for pkgbase in ${pkgs[@]}; do + checkAnyPackage extra ${pkgbase}-1-1-any.pkg.tar.xz + done +} + +testAddSplitPackages() { + local arches=('i686' 'x86_64') + local pkgs=('pkg-split-a' 'pkg-split-b') + local pkg + local pkgbase + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + releasePackage extra ${pkgbase} ${arch} + done + done + + ../db-update extra + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + for pkg in "${pkgdir}/${pkgbase}"/*-${arch}.pkg.tar.*; do + checkPackage extra $(basename ${pkg}) ${arch} + done + done + done +} + + +. "${curdir}/lib/shunit2" -- cgit v1.2.2 From bccf84211be4abda3c96ded43f03b597081fecb1 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sat, 7 Aug 2010 21:21:08 +0200 Subject: add test for db-remove --- test/lib/common.inc | 13 +++------ test/runTest | 78 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+), 10 deletions(-) diff --git a/test/lib/common.inc b/test/lib/common.inc index 5abe2ed..4052aa1 100644 --- a/test/lib/common.inc +++ b/test/lib/common.inc @@ -1,4 +1,4 @@ -pkgdir="$(dirname $0)/packages" +pkgdir="${curdir}/packages" oneTimeSetUp() { local p @@ -57,20 +57,13 @@ eot } tearDown() { + echo -n 'Cleaning up...' rm -rf "${TMP}" rm -f "${curdir}/../config.local" + echo 'done' echo } -testPackages() { - local p - for p in "${pkgdir}"/*; do - pushd $p >/dev/null - namcap *.pkg.tar.* || fail 'namcap failed' - popd >/dev/null - done -} - releasePackage() { local repo=$1 local pkgbase=$2 diff --git a/test/runTest b/test/runTest index 1d54ea4..258a181 100755 --- a/test/runTest +++ b/test/runTest @@ -3,6 +3,23 @@ curdir=$(readlink -e $(dirname $0)) . "${curdir}/lib/common.inc" +# +# self test +# + +testPackages() { + local p + for p in "${pkgdir}"/*; do + pushd $p >/dev/null + namcap *.pkg.tar.* || fail 'namcap failed' + popd >/dev/null + done +} + + +# +# db-update +# testAddSimplePackages() { local arches=('i686' 'x86_64') @@ -63,4 +80,65 @@ testAddSplitPackages() { } +# +# db-remove +# + +testRemovePackages() { + local arches=('i686' 'x86_64') + local pkgs=('pkg-simple-a' 'pkg-simple-b' 'pkg-split-a' 'pkg-split-b') + local pkgbase + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + releasePackage extra ${pkgbase} ${arch} + done + done + + ../db-update extra + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + ../db-remove ${pkgbase} extra ${arch} + done + done + + for arch in ${arches[@]}; do + [ -z "$(bsdtar -tf "${FTP_BASE}/extra/os/${arch}/extra.db.tar.gz" -O)" ] || fail "extra/os/${arch}/${repo}.db.tar.gz should be empty" + done + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + svn up -q "${TMP}/svn-packages-copy/${pkgbase}" + [ -d "${TMP}/svn-packages-copy/${pkgbase}/repos/extra-${arch}" ] && fail "svn-packages-copy/${pkgbase}/repos/extra-${arch} should not exist" + done + done +} + +testRemoveAnyPackages() { + local arches=('i686' 'x86_64') + local pkgs=('pkg-any-a' 'pkg-any-b') + local pkgbase + + for pkgbase in ${pkgs[@]}; do + releasePackage extra ${pkgbase} any + done + + ../db-update extra + + for pkgbase in ${pkgs[@]}; do + ../db-remove ${pkgbase} extra any + done + + for arch in ${arches[@]}; do + [ -z "$(bsdtar -tf "${FTP_BASE}/extra/os/${arch}/extra.db.tar.gz" -O)" ] || fail "extra/os/${arch}/${repo}.db.tar.gz should be empty" + done + + for pkgbase in ${pkgs[@]}; do + svn up -q "${TMP}/svn-packages-copy/${pkgbase}" + [ -d "${TMP}/svn-packages-copy/${pkgbase}/repos/extra-any" ] && fail "svn-packages-copy/${pkgbase}/repos/extra-any should not exist" + done +} + + . "${curdir}/lib/shunit2" -- cgit v1.2.2 From 76ef37d1b1e03b0c517f8513b943474ea379738e Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sat, 7 Aug 2010 23:48:03 +0200 Subject: Simplify tests and add tests for db-move --- test/lib/common.inc | 38 ++++++++++++++++++++--- test/runTest | 86 ++++++++++++++++++++++++++++++++++++++++++++++------- 2 files changed, 110 insertions(+), 14 deletions(-) diff --git a/test/lib/common.inc b/test/lib/common.inc index 4052aa1..3ba715e 100644 --- a/test/lib/common.inc +++ b/test/lib/common.inc @@ -1,12 +1,14 @@ -pkgdir="${curdir}/packages" +. "${curdir}/../db-functions" oneTimeSetUp() { local p + pkgdir="$(mktemp -d /dev/shm/$(basename $0).XXXXXXXXXX)" + cp -r ${curdir}/packages/* "${pkgdir}" echo -n 'Building packages...' for p in "${pkgdir}"/*; do pushd $p >/dev/null linux32 makepkg -cf --config ${curdir}/lib/makepkg-i686.conf >/dev/null 2>&1 || exit 1 - linux64 makepkg -cf --config ${curdir}/lib/makepkg-x86_64.conf >/dev/null 2>&1 || exit 1 + [ -f *-any.pkg.tar.* ] || linux64 makepkg -cf --config ${curdir}/lib/makepkg-x86_64.conf >/dev/null 2>&1 || exit 1 popd >/dev/null done echo 'done' @@ -22,7 +24,7 @@ setUp() { local pkg [ -f "${curdir}/../config.local" ] && exit 1 - TMP="$(mktemp -d /tmp/$(basename $0).XXXXXXXXXX)" + TMP="$(mktemp -d /dev/shm/$(basename $0).XXXXXXXXXX)" #echo "Using ${TMP}" mkdir -p "${TMP}/"{ftp,tmp,staging,{package,source}-cleanup,svn-{packages,community}-{copy,repo}} @@ -82,7 +84,7 @@ checkAnyPackage() { [ -f "${FTP_BASE}/packages/arch/any/${pkg}" ] || fail "packages/arch/any/${pkg} not found" - for arch in any i686 x86_64; do + for arch in i686 x86_64; do [ -L "${FTP_BASE}/${repo}/os/${arch}/${pkg}" ] || fail "${repo}/os/${arch}/${pkg} not a symlink" [ "$(readlink -e "${FTP_BASE}/${repo}/os/${arch}/${pkg}")" == "${FTP_BASE}/packages/arch/any/${pkg}" ] || fail "${repo}/os/${arch}/${pkg} does not link to packages/arch/any/${pkg}" done @@ -103,4 +105,32 @@ checkPackage() { [ "$(readlink -e "${FTP_BASE}/${repo}/os/${arch}/${pkg}")" == "${FTP_BASE}/packages/arch/${arch}/${pkg}" ] || fail "${repo}/os/${arch}/${pkg} does not link to packages/arch/${arch}/${pkg}" bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}.db.tar.gz" -O | grep -q ${pkg} || fail "${pkg} not in ${repo}/os/${arch}/${repo}.db.tar.gz" + + local pkgbase=$(getpkgbase "${FTP_BASE}/packages/arch/${arch}/${pkg}") + svn up -q "${TMP}/svn-packages-copy/${pkgbase}" + [ -d "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo}-${arch}" ] || fail "svn-packages-copy/${pkgbase}/repos/${repo}-${arch} does not exist" +} + +checkRemovedPackage() { + local repo=$1 + local pkgbase=$2 + local arch=$3 + + bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}.db.tar.gz" -O | grep -q ${pkgbase} && fail "${pkgbase} should not be in ${repo}/os/${arch}/${repo}.db.tar.gz" + + svn up -q "${TMP}/svn-packages-copy/${pkgbase}" + [ -d "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo}-${arch}" ] && fail "svn-packages-copy/${pkgbase}/repos/${repo}-${arch} should not exist" } + +checkRemovedAnyPackage() { + local repo=$1 + local pkgbase=$2 + local arch + + for arch in i686 x86_64; do + bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}.db.tar.gz" -O | grep -q ${pkgbase} && fail "${pkgbase} should not be in ${repo}/os/${arch}/${repo}.db.tar.gz" + done + + svn up -q "${TMP}/svn-packages-copy/${pkgbase}" + [ -d "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo}-any" ] && fail "svn-packages-copy/${pkgbase}/repos/${repo}-any should not exist" +} \ No newline at end of file diff --git a/test/runTest b/test/runTest index 258a181..c53b63c 100755 --- a/test/runTest +++ b/test/runTest @@ -103,20 +103,14 @@ testRemovePackages() { done done - for arch in ${arches[@]}; do - [ -z "$(bsdtar -tf "${FTP_BASE}/extra/os/${arch}/extra.db.tar.gz" -O)" ] || fail "extra/os/${arch}/${repo}.db.tar.gz should be empty" - done - for pkgbase in ${pkgs[@]}; do for arch in ${arches[@]}; do - svn up -q "${TMP}/svn-packages-copy/${pkgbase}" - [ -d "${TMP}/svn-packages-copy/${pkgbase}/repos/extra-${arch}" ] && fail "svn-packages-copy/${pkgbase}/repos/extra-${arch} should not exist" + checkRemovedPackage extra ${pkgbase} ${arch} done done } testRemoveAnyPackages() { - local arches=('i686' 'x86_64') local pkgs=('pkg-any-a' 'pkg-any-b') local pkgbase @@ -130,15 +124,87 @@ testRemoveAnyPackages() { ../db-remove ${pkgbase} extra any done + for pkgbase in ${pkgs[@]}; do + checkRemovedAnyPackage extra ${pkgbase} + done +} + + +# +# db-move +# + +testMoveSimplePackages() { + local arches=('i686' 'x86_64') + local pkgs=('pkg-simple-a' 'pkg-simple-b') + local pkgbase + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + releasePackage testing ${pkgbase} ${arch} + done + done + + ../db-update testing + + for arch in ${arches[@]}; do + ../db-move-draft pkg-simple-a testing extra ${arch} + done + for arch in ${arches[@]}; do - [ -z "$(bsdtar -tf "${FTP_BASE}/extra/os/${arch}/extra.db.tar.gz" -O)" ] || fail "extra/os/${arch}/${repo}.db.tar.gz should be empty" + checkPackage extra pkg-simple-a-1-1-${arch}.pkg.tar.xz ${arch} + checkRemovedPackage testing pkg-simple-a-1-1-${arch}.pkg.tar.xz ${arch} + + checkPackage testing pkg-simple-b-1-1-${arch}.pkg.tar.xz ${arch} done +} + +testMoveAnyPackages() { + local pkgs=('pkg-any-a' 'pkg-any-b') + local pkgbase for pkgbase in ${pkgs[@]}; do - svn up -q "${TMP}/svn-packages-copy/${pkgbase}" - [ -d "${TMP}/svn-packages-copy/${pkgbase}/repos/extra-any" ] && fail "svn-packages-copy/${pkgbase}/repos/extra-any should not exist" + releasePackage testing ${pkgbase} any done + + ../db-update testing + ../db-move-draft pkg-any-a testing extra any + + checkAnyPackage extra pkg-any-a-1-1-any.pkg.tar.xz + checkRemovedAnyPackage testing pkg-any-a + checkAnyPackage testing pkg-any-b-1-1-any.pkg.tar.xz } +testMoveSplitPackages() { + local arches=('i686' 'x86_64') + local pkgs=('pkg-split-a' 'pkg-split-b') + local pkg + local pkgbase + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + releasePackage testing ${pkgbase} ${arch} + done + done + + ../db-update testing + + for arch in ${arches[@]}; do + ../db-move-draft pkg-split-a testing extra ${arch} + done + + for arch in ${arches[@]}; do + for pkg in "${pkgdir}/pkg-split-a"/*-${arch}.pkg.tar.*; do + checkPackage extra $(basename ${pkg}) ${arch} + done + done + for arch in ${arches[@]}; do + for pkg in "${pkgdir}/pkg-split-b"/*-${arch}.pkg.tar.*; do + checkPackage testing $(basename ${pkg}) ${arch} + done + done + + checkRemovedAnyPackage testing pkg-split-a +} . "${curdir}/lib/shunit2" -- cgit v1.2.2 From c81c0e57ff41bbc51e13600b8581145b34aef55f Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sat, 7 Aug 2010 23:49:01 +0200 Subject: Rewrite db-move * Cleanup code * create symlinks instead of copies * Moving split packages is now (more) atomic * Move old packages to pool if needed --- db-move | 97 +++++++++++++++++++++++++---------------------------------------- 1 file changed, 37 insertions(+), 60 deletions(-) diff --git a/db-move b/db-move index ed3df69..407027a 100755 --- a/db-move +++ b/db-move @@ -1,7 +1,7 @@ #!/bin/bash if [ $# -ne 4 ]; then - echo "usage: $(basename $0) " + echo "usage: $(basename $0) " exit 1 fi @@ -12,6 +12,11 @@ packagebase="$1" repofrom="$2" repoto="$3" _arch="$4" +if [ "${_arch}" == 'any' ]; then + _tarches=('i686' 'x86_64') +else + _tarches=("${_arch}") +fi ftppath_from="$FTP_BASE/$repofrom/os/" ftppath_to="$FTP_BASE/$repoto/os/" @@ -23,10 +28,10 @@ svnrepo_to="$repoto-$_arch" WORKDIR="$TMPDIR/db-move.$svnrepo_from.$svnrepo_to.$UID" cleanup() { - trap '' 0 2 + trap '' 0 2 # unlock - repo_unlock $repoto $_arch - repo_unlock $repofrom $_arch + repo_unlock $repoto $_arch + repo_unlock $repofrom $_arch rm -rf "$WORKDIR" [ "$1" ] && exit $1 } @@ -55,68 +60,40 @@ cd checkout /usr/bin/svn up -q $packagebase if [ -d "$packagebase/repos/$svnrepo_from" ]; then - . "$packagebase/repos/$svnrepo_from/$BUILDSCRIPT" - - for i in ${pkgname[@]}; do - _pkgfile="$i-$pkgver-$pkgrel-$_arch$PKGEXT" - getpkgfile "$ftppath_from/${_arch}/"$_pkgfile >/dev/null - done - - if [ -d "$packagebase/repos/$svnrepo_to" ]; then - echo " Removing existing package from subversion" - /usr/bin/svn rm --force -q "$packagebase/repos/$svnrepo_to" - /usr/bin/svn commit -q -m "$(basename $0): $packagebase removed by $(id -un) for move to $repoto" - fi - - echo " Moving svn entries" - /usr/bin/svn mv -r HEAD "$packagebase/repos/$svnrepo_from" "$packagebase/repos/$svnrepo_to" - /usr/bin/svn commit -m "$(basename $0): moved $packagebase from [$repofrom] to [$repoto] ($_arch)" - - echo " Moving package file and updating DBs" - cd "$WORKDIR" - [ -d build/ ] || mkdir build - cd build/ - - if [ "${_arch}" == "any" ]; then - arches="i686 x86_64" - else - arches="${_arch}" + . "$packagebase/repos/$svnrepo_from/$BUILDSCRIPT" + + if [ -d "$packagebase/repos/$svnrepo_to" ]; then + echo " Removing existing package from subversion" + /usr/bin/svn rm --force -q "$packagebase/repos/$svnrepo_to" + /usr/bin/svn commit -q -m "$(basename $0): $packagebase removed by $(id -un) for move to $repoto" fi - for architecture in $arches; do - # copy the db file into our working area - if [ -f "$ftppath_from/$architecture/$repofrom$DBEXT" ]; then - /bin/cp "$ftppath_from/$architecture/$repofrom$DBEXT" . - /usr/bin/repo-remove -q "$repofrom$DBEXT" ${pkgname[@]} || die "Error in repo-remove" - mv $repofrom$DBEXT "$ftppath_from/$architecture" - echo " Package files will be cleaned up automatically" - fi + echo " Moving svn entries" + /usr/bin/svn mv -r HEAD "$packagebase/repos/$svnrepo_from" "$packagebase/repos/$svnrepo_to" + /usr/bin/svn commit -m "$(basename $0): moved $packagebase from [$repofrom] to [$repoto] ($_arch)" + + _pkgfiles='' + for i in ${pkgname[@]}; do + _pkgpath=$(getpkgfile "$ftppath_from/${_arch}/"$i-$pkgver-$pkgrel-$_arch$PKGEXT) + _pkgfile=$(basename "${_pkgpath}") - if [ -f "$ftppath_to/$architecture/$repoto$DBEXT" ]; then - /bin/cp "$ftppath_to/$architecture/$repoto$DBEXT" . + # copy package to pool if needed + # TODO: can be removed once every package has benn moved to the package pool + if [ ! -f $FTP_BASE/$(get_pkgpool_for_host)/${_arch}/$_pkgfile ]; then + cp $_pkgpath $FTP_BASE/$(get_pkgpool_for_host)/${_arch}/ fi - for i in ${pkgname[@]}; do - _pkgfile="$i-$pkgver-$pkgrel-$_arch$PKGEXT" - _cpkgfile=$(getpkgfile "$ftppath_from/$architecture/"$_pkgfile) - [ $? -gt 0 ] && die - /bin/cp $_cpkgfile . - /usr/bin/repo-add -q "$repoto$DBEXT" $_pkgfile || die "Error in repo-add $_pkgfile" + for _tarch in ${_tarches[@]}; do + ln -s "../../../$(get_pkgpool_for_host)/${_arch}/${_pkgfile}" $ftppath_to/${_tarch}/ done - mv $repoto$DBEXT $ftppath_to/$architecture - for i in ${pkgname[@]}; do - _pkgfile=$(getpkgfile "$i-$pkgver-$pkgrel-$_arch"$PKGEXT) - [ $? -gt 0 ] && die - if [ "${_arch}" == "any" ]; then - mv ${_pkgfile} $ftppath_to/any - ln -s ../any/${_pkgfile} $ftppath_to/$architecture/ - else - mv ${_pkgfile} $ftppath_to/$architecture - fi - done + + _pkgfiles="${_pkgfiles} $FTP_BASE/$(get_pkgpool_for_host)/${_arch}/${_pkgfile}" + done + + for _tarch in ${_tarches[@]}; do + /usr/bin/repo-add -q "$ftppath_to/${_tarch}/$repoto$DBEXT" ${_pkgfiles} || die "Error in repo-add $_pkgfiles" + /usr/bin/repo-remove -q "$ftppath_from/${_tarch}/$repofrom$DBEXT" ${pkgname[@]} || die "Error in repo-remove ${pkgname[@]}" done else - die "Error: $packagebase is not in repo $repofrom" + die "Error: $packagebase is not in repo $repofrom" fi - -cleanup -- cgit v1.2.2 From 0cfcc705c755eda1911a8868b2214c6a56f8e064 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 8 Aug 2010 00:00:34 +0200 Subject: Fix old call --- test/runTest | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/runTest b/test/runTest index c53b63c..6a4f3de 100755 --- a/test/runTest +++ b/test/runTest @@ -148,7 +148,7 @@ testMoveSimplePackages() { ../db-update testing for arch in ${arches[@]}; do - ../db-move-draft pkg-simple-a testing extra ${arch} + ../db-move pkg-simple-a testing extra ${arch} done for arch in ${arches[@]}; do @@ -168,7 +168,7 @@ testMoveAnyPackages() { done ../db-update testing - ../db-move-draft pkg-any-a testing extra any + ../db-move pkg-any-a testing extra any checkAnyPackage extra pkg-any-a-1-1-any.pkg.tar.xz checkRemovedAnyPackage testing pkg-any-a @@ -190,7 +190,7 @@ testMoveSplitPackages() { ../db-update testing for arch in ${arches[@]}; do - ../db-move-draft pkg-split-a testing extra ${arch} + ../db-move pkg-split-a testing extra ${arch} done for arch in ${arches[@]}; do -- cgit v1.2.2 From 23d4669b8e330d131b2f78cd5858b30a07478c8a Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 8 Aug 2010 00:33:46 +0200 Subject: Use package pool instead of $repo/os/any dirs --- db-move | 17 ++++++++--------- db-update | 16 +++++----------- test/lib/common.inc | 2 ++ 3 files changed, 15 insertions(+), 20 deletions(-) diff --git a/db-move b/db-move index 407027a..8609459 100755 --- a/db-move +++ b/db-move @@ -74,16 +74,15 @@ if [ -d "$packagebase/repos/$svnrepo_from" ]; then _pkgfiles='' for i in ${pkgname[@]}; do - _pkgpath=$(getpkgfile "$ftppath_from/${_arch}/"$i-$pkgver-$pkgrel-$_arch$PKGEXT) - _pkgfile=$(basename "${_pkgpath}") - - # copy package to pool if needed - # TODO: can be removed once every package has benn moved to the package pool - if [ ! -f $FTP_BASE/$(get_pkgpool_for_host)/${_arch}/$_pkgfile ]; then - cp $_pkgpath $FTP_BASE/$(get_pkgpool_for_host)/${_arch}/ - fi - for _tarch in ${_tarches[@]}; do + _pkgpath=$(getpkgfile "$ftppath_from/${_tarch}/"$i-$pkgver-$pkgrel-$_arch$PKGEXT) + _pkgfile=$(basename "${_pkgpath}") + + # copy package to pool if needed + # TODO: can be removed once every package has been moved to the package pool + if [ ! -f $FTP_BASE/$(get_pkgpool_for_host)/${_arch}/$_pkgfile ]; then + cp $_pkgpath $FTP_BASE/$(get_pkgpool_for_host)/${_arch}/ + fi ln -s "../../../$(get_pkgpool_for_host)/${_arch}/${_pkgfile}" $ftppath_to/${_tarch}/ done diff --git a/db-update b/db-update index f05246b..9e641d2 100755 --- a/db-update +++ b/db-update @@ -223,7 +223,7 @@ for current_arch in ${ARCHES[@]}; do # if non empty, move all build dirs if [ $(/bin/ls "$WORKDIR/build/" 2>/dev/null | wc -l) != 0 ]; then if [ $(getpkgfiles "$WORKDIR/build/"*-$current_arch$PKGEXT 2>/dev/null | wc -l) != 0 ]; then - echo "Copying new files to '$ftppath'" + echo "Copying new files to '$poolpath' and symlinking" for f in "$WORKDIR/build/"*-$current_arch$PKGEXT; do /bin/chmod 664 "$f" &>/dev/null if ! /bin/cp "$f" "$poolpath/"; then @@ -236,20 +236,14 @@ for current_arch in ${ARCHES[@]}; do done fi if [ $(getpkgfiles "$WORKDIR/build/"*-any$PKGEXT 2>/dev/null | wc -l) != 0 ]; then - echo "Copying new files to '$ftppath_any' and symlinking" + echo "Copying new files to '$poolpath_any' and symlinking" for f in "$WORKDIR/build/"*-any$PKGEXT; do /bin/chmod 664 "$f" &>/dev/null fname="$(basename $f)" - if [ ! -f "$poolrel_any/$fname" ]; then - if ! /bin/cp "$f" "$poolpath_any/"; then - die "error: failure while copying files to $poolpath_any" - fi - fi - if [ ! -f "$ftppath_any/$fname" ]; then - if ! ln -s "$poolrel_any/$fname" "$ftppath_any/$fname"; then - die "error: failure symlinking $fname to $ftppath_any" - fi + if ! /bin/cp "$f" "$poolpath_any/"; then + die "error: failure while copying files to $poolpath_any" fi + if ! ln -s "$poolrel_any/$fname" "$ftppath/$fname"; then die "error: failure symlinking $fname to $ftppath" fi diff --git a/test/lib/common.inc b/test/lib/common.inc index 3ba715e..4080c38 100644 --- a/test/lib/common.inc +++ b/test/lib/common.inc @@ -91,6 +91,8 @@ checkAnyPackage() { [ -f "${STAGING}"/${repo}/${pkg} ] && fail "${repo}/${pkg} found in staging dir" bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}.db.tar.gz" -O | grep -q ${pkg} || fail "${pkg} not in ${repo}/os/${arch}/${repo}.db.tar.gz" + + [ -f "${FTP_BASE}/${repo}/os/any/${pkg}" ] && fail "${repo}/os/any/${pkg} should not exist" } checkPackage() { -- cgit v1.2.2 From 1db7071b222207d2d924a03a48d0745387f11c7b Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 8 Aug 2010 01:26:11 +0200 Subject: add test for ftpdir-cleanup --- test/lib/common.inc | 3 +- test/runTest | 97 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 99 insertions(+), 1 deletion(-) diff --git a/test/lib/common.inc b/test/lib/common.inc index 4080c38..dd3022e 100644 --- a/test/lib/common.inc +++ b/test/lib/common.inc @@ -16,7 +16,7 @@ oneTimeSetUp() { } oneTimeTearDown() { - find "${pkgdir}" -name '*.pkg.tar.*' -delete + rm -rf "${pkgdir}" } setUp() { @@ -54,6 +54,7 @@ setUp() { SOURCE_CLEANUP_DESTDIR="${TMP}/source-cleanup" STAGING="${TMP}/staging" TMPDIR="${TMP}/tmp" + CLEANUP_DRYRUN=false eot . "${curdir}/../config" } diff --git a/test/runTest b/test/runTest index 6a4f3de..7136508 100755 --- a/test/runTest +++ b/test/runTest @@ -25,6 +25,7 @@ testAddSimplePackages() { local arches=('i686' 'x86_64') local pkgs=('pkg-simple-a' 'pkg-simple-b') local pkgbase + local arch for pkgbase in ${pkgs[@]}; do for arch in ${arches[@]}; do @@ -61,6 +62,7 @@ testAddSplitPackages() { local pkgs=('pkg-split-a' 'pkg-split-b') local pkg local pkgbase + local arch for pkgbase in ${pkgs[@]}; do for arch in ${arches[@]}; do @@ -88,6 +90,7 @@ testRemovePackages() { local arches=('i686' 'x86_64') local pkgs=('pkg-simple-a' 'pkg-simple-b' 'pkg-split-a' 'pkg-split-b') local pkgbase + local arch for pkgbase in ${pkgs[@]}; do for arch in ${arches[@]}; do @@ -138,6 +141,7 @@ testMoveSimplePackages() { local arches=('i686' 'x86_64') local pkgs=('pkg-simple-a' 'pkg-simple-b') local pkgbase + local arch for pkgbase in ${pkgs[@]}; do for arch in ${arches[@]}; do @@ -180,6 +184,7 @@ testMoveSplitPackages() { local pkgs=('pkg-split-a' 'pkg-split-b') local pkg local pkgbase + local arch for pkgbase in ${pkgs[@]}; do for arch in ${arches[@]}; do @@ -207,4 +212,96 @@ testMoveSplitPackages() { checkRemovedAnyPackage testing pkg-split-a } + +# +# ftpdir-cleanup +# + +testCleanupSimplePackages() { + local arches=('i686' 'x86_64') + local pkgs=('pkg-simple-a' 'pkg-simple-b') + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + releasePackage extra ${pkgbase} ${arch} + done + done + + ../db-update extra + + for arch in ${arches[@]}; do + ../db-remove pkg-simple-a extra ${arch} + done + + ../cron-jobs/ftpdir-cleanup + + for arch in ${arches[@]}; do + local pkg1="pkg-simple-a-1-1-${arch}.pkg.tar.xz" + [ -f "${FTP_BASE}/packages/arch/${arch}/${pkg1}" ] && fail "packages/arch/${arch}/${pkg1} found" + [ -f "${FTP_BASE}/${repo}/os/${arch}/${pkg1}" ] && fail "${repo}/os/${arch}/${pkg1} found" + + local pkg2="pkg-simple-b-1-1-${arch}.pkg.tar.xz" + checkPackage extra ${pkg2} ${arch} + done +} + +testCleanupAnyPackages() { + local pkgs=('pkg-any-a' 'pkg-any-b') + local pkgbase + local arch='any' + + for pkgbase in ${pkgs[@]}; do + releasePackage extra ${pkgbase} any + done + + ../db-update extra + ../db-remove pkg-any-a extra any + ../cron-jobs/ftpdir-cleanup + + local pkg1='pkg-any-a-1-1-any.pkg.tar.xz' + [ -f "${FTP_BASE}/packages/arch/${arch}/${pkg1}" ] && fail "packages/arch/${arch}/${pkg1} found" + [ -f "${FTP_BASE}/${repo}/os/${arch}/${pkg1}" ] && fail "${repo}/os/${arch}/${pkg1} found" + + local pkg2="pkg-any-b-1-1-${arch}.pkg.tar.xz" + checkAnyPackage extra ${pkg2} +} + +testCleanupSplitPackages() { + local arches=('i686' 'x86_64') + local pkgs=('pkg-split-a' 'pkg-split-b') + local pkg + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + releasePackage extra ${pkgbase} ${arch} + done + done + + ../db-update extra + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + ../db-remove pkg-split-a extra ${arch} + done + done + + ../cron-jobs/ftpdir-cleanup + + for arch in ${arches[@]}; do + for pkg in "${pkgdir}/${pkgs[0]}"/*-${arch}.pkg.tar.*; do + [ -f "${FTP_BASE}/packages/arch/${arch}/${pkg}" ] && fail "packages/arch/${arch}/${pkg} found" + [ -f "${FTP_BASE}/${repo}/os/${arch}/${pkg}" ] && fail "${repo}/os/${arch}/${pkg} found" + done + + for pkg in "${pkgdir}/${pkgs[1]}"/*-${arch}.pkg.tar.*; do + checkPackage extra $(basename ${pkg}) ${arch} + done + done +} + + . "${curdir}/lib/shunit2" -- cgit v1.2.2 From 3e08da59e3ca3849b753318b791aed8f1f8d8fb4 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 8 Aug 2010 01:26:52 +0200 Subject: fix cleaning of any packages and ignore empty repos --- cron-jobs/ftpdir-cleanup | 5 +++-- misc-scripts/ftpdir-cleanup | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index 66733f0..546afd9 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -37,9 +37,9 @@ for repo in $repos; do done to_cleanup="" -for _arch in ${ARCHES[@]}; do +for _arch in any ${ARCHES[@]}; do poolpath="$FTP_BASE/$(get_pkgpool_for_host)/$_arch/" - cd $poolpath + pushd $poolpath >/dev/null for pkg in *$PKGEXT; do [ -f "$pkg" ] || continue # in case we get a file named "*.pkg.tar.gz" LINKS="$(/usr/bin/find $repopaths -type l -name "$pkg" 2>/dev/null)" @@ -47,6 +47,7 @@ for _arch in ${ARCHES[@]}; do to_cleanup="$to_cleanup $poolpath/$pkg" fi done + popd >/dev/null done if [ -n "$to_cleanup" ]; then diff --git a/misc-scripts/ftpdir-cleanup b/misc-scripts/ftpdir-cleanup index c9a232d..ac17a8d 100755 --- a/misc-scripts/ftpdir-cleanup +++ b/misc-scripts/ftpdir-cleanup @@ -60,9 +60,10 @@ for arch in ${ARCHES[@]}; do if [ ! -f "$ftppath/$reponame$DBEXT" ]; then echo "" - echo "ERROR: The file \"$ftppath/$reponame$DBEXT\" could not be found, aborting." + echo "WARNING: The file \"$ftppath/$reponame$DBEXT\" could not be found, skipping." echo "" - exit 1 + repo_unlock $reponame $arch + continue fi if ! bsdtar xf "$ftppath/$reponame$DBEXT"; then -- cgit v1.2.2 From 918eaba387411a2f447f1669f255bcd2fb681e94 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 8 Aug 2010 02:51:51 +0200 Subject: add transitions tests Theses tests simulate the case of packages with the old repo layout and the new package pool layout. --- test/runTest | 86 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/test/runTest b/test/runTest index 7136508..d453493 100755 --- a/test/runTest +++ b/test/runTest @@ -303,5 +303,91 @@ testCleanupSplitPackages() { done } +# +# transition tests +# + +testMovePackagesWithoutPool() { + local arches=('i686' 'x86_64') + local pkgs=('pkg-simple-a' 'pkg-simple-b' 'pkg-split-a' 'pkg-split-b') + local pkgbase + local arch + local pkg + local old + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + releasePackage testing ${pkgbase} ${arch} + done + done + + ../db-update testing + + # transform two packages to old style layout + for arch in ${arches[@]}; do + for old in 0 2; do + for pkg in "${pkgdir}/${pkgs[${old}]}"/*-${arch}.pkg.tar.*; do + pkg=$(basename $pkg) + mv -f "${FTP_BASE}/packages/arch/${arch}/${pkg}" "${FTP_BASE}/testing/os/${arch}/${pkg}" + done + done + done + + ../cron-jobs/ftpdir-cleanup + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + ../db-move ${pkgbase} testing extra ${arch} + done + done + + ../cron-jobs/ftpdir-cleanup + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + for pkg in "${pkgdir}/${pkgbase}"/*-${arch}.pkg.tar.*; do + checkPackage extra $(basename ${pkg}) ${arch} + done + checkRemovedPackage testing ${pkgbase} ${arch} + done + done +} + +testMoveAnyPackagesWithoutPool() { + local pkgs=('pkg-any-a' 'pkg-any-b') + local pkgbase + local arch + local pkg + + for pkgbase in ${pkgs[@]}; do + releasePackage testing ${pkgbase} any + done + + ../db-update testing + + # transform a package to old style layout + for pkg in "${pkgdir}/${pkgs[0]}"/*-any.pkg.tar.*; do + pkg=$(basename $pkg) + mv -f "${FTP_BASE}/packages/arch/any/${pkg}" "${FTP_BASE}/testing/os/any/${pkg}" + for arch in i686 x86_64; do + ln -sf "../any/${pkg}" "${FTP_BASE}/testing/os/${arch}/${pkg}" + done + done + + ../cron-jobs/ftpdir-cleanup + + for pkgbase in ${pkgs[@]}; do + ../db-move ${pkgbase} testing extra any + done + + ../cron-jobs/ftpdir-cleanup + + for pkgbase in ${pkgs[@]}; do + for pkg in "${pkgdir}/${pkgbase}"/*-any.pkg.tar.*; do + checkAnyPackage extra $(basename ${pkg}) + done + checkRemovedAnyPackage testing ${pkgbase} + done +} . "${curdir}/lib/shunit2" -- cgit v1.2.2 From c117d9048ae591401a79037222da7bf7bda85705 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 8 Aug 2010 03:44:17 +0200 Subject: Reduce verbosity Only inform of errors and processes that might take longer. --- db-move | 15 ++++++++------- db-remove | 21 +++++++++------------ db-update | 23 +++++++---------------- test/lib/common.inc | 4 +--- test/runTest | 14 +++++++------- 5 files changed, 32 insertions(+), 45 deletions(-) diff --git a/db-move b/db-move index 8609459..7dff563 100755 --- a/db-move +++ b/db-move @@ -55,22 +55,21 @@ repo_lock $repofrom $_arch || exit 1 /bin/mkdir -p "$WORKDIR" cd "$WORKDIR" -/usr/bin/svn checkout -N $SVNREPO checkout +/usr/bin/svn checkout -q -N $SVNREPO checkout >/dev/null cd checkout /usr/bin/svn up -q $packagebase if [ -d "$packagebase/repos/$svnrepo_from" ]; then . "$packagebase/repos/$svnrepo_from/$BUILDSCRIPT" + echo -n "Moving $packagebase from $repofrom to $repoto..." if [ -d "$packagebase/repos/$svnrepo_to" ]; then - echo " Removing existing package from subversion" /usr/bin/svn rm --force -q "$packagebase/repos/$svnrepo_to" /usr/bin/svn commit -q -m "$(basename $0): $packagebase removed by $(id -un) for move to $repoto" fi - echo " Moving svn entries" - /usr/bin/svn mv -r HEAD "$packagebase/repos/$svnrepo_from" "$packagebase/repos/$svnrepo_to" - /usr/bin/svn commit -m "$(basename $0): moved $packagebase from [$repofrom] to [$repoto] ($_arch)" + /usr/bin/svn mv -q -r HEAD "$packagebase/repos/$svnrepo_from" "$packagebase/repos/$svnrepo_to" + /usr/bin/svn commit -q -m "$(basename $0): moved $packagebase from [$repofrom] to [$repoto] ($_arch)" _pkgfiles='' for i in ${pkgname[@]}; do @@ -90,9 +89,11 @@ if [ -d "$packagebase/repos/$svnrepo_from" ]; then done for _tarch in ${_tarches[@]}; do - /usr/bin/repo-add -q "$ftppath_to/${_tarch}/$repoto$DBEXT" ${_pkgfiles} || die "Error in repo-add $_pkgfiles" - /usr/bin/repo-remove -q "$ftppath_from/${_tarch}/$repofrom$DBEXT" ${pkgname[@]} || die "Error in repo-remove ${pkgname[@]}" + /usr/bin/repo-add -q "$ftppath_to/${_tarch}/$repoto$DBEXT" ${_pkgfiles} >/dev/null || die "Error in repo-add $_pkgfiles" + /usr/bin/repo-remove -q "$ftppath_from/${_tarch}/$repofrom$DBEXT" ${pkgname[@]} >/dev/null || die "Error in repo-remove ${pkgname[@]}" done + + echo 'done' else die "Error: $packagebase is not in repo $repofrom" fi diff --git a/db-remove b/db-remove index 0b77fc1..357e6c4 100755 --- a/db-remove +++ b/db-remove @@ -42,26 +42,21 @@ trap cleanup 0 repo_lock $reponame $_arch || exit 1 +echo -n "Removing $packagebase from $reponame..." /bin/mkdir -p "$WORKDIR" - - -echo "==> Removing package '$packagebase' from '$reponame'..." >&2 - cd "$WORKDIR" -/usr/bin/svn checkout -N $SVNREPO checkout +/usr/bin/svn checkout -q -N $SVNREPO checkout >/dev/null cd checkout /usr/bin/svn up -q $packagebase if [ -d "$packagebase/repos/$svnrepo" ]; then - echo " Removing from subversion" . "$packagebase/repos/$svnrepo/$BUILDSCRIPT" /usr/bin/svn rm --force -q "$packagebase/repos/$svnrepo" /usr/bin/svn commit -q -m "$(basename $0): $packagebase removed by $(id -un)" else echo " Warning: $packagebase not found in $svnrepo" - echo " Removing split packages is not supported" - echo " You need to specify each sub package instead" - pkgname=$packagebase + echo " Removing split packages is not yet supported" + exit 1 fi cd "$WORKDIR" @@ -82,14 +77,16 @@ for architecture in $arches; do exit 0 fi - echo " Removing from $reponame DB file ($architecture)" cd build/ - /usr/bin/repo-remove -q "$reponame$DBEXT" ${pkgname[@]} + /usr/bin/repo-remove -q "$reponame$DBEXT" ${pkgname[@]} >/dev/null /bin/mv "$reponame$DBEXT" "$ftppath/$architecture" - echo "Package files will be cleaned up automatically" cd .. done + +echo 'done' + cleanup + # vim: set ts=4 sw=4 noet ft=sh: diff --git a/db-update b/db-update index 9e641d2..057de39 100755 --- a/db-update +++ b/db-update @@ -74,6 +74,8 @@ die() { trap ctrl_c 2 trap cleanup 0 +echo -n "Updating $reponame..." + # Remove any package from $stagedir that is already in the FTP repository for current_arch in ${ARCHES[@]} any; do ftppath="$FTP_BASE/$reponame/os/$current_arch" @@ -97,9 +99,8 @@ cd "$WORKDIR" if [ -n "$ANYPKGS" ]; then pkgtotal=$(echo "$ANYPKGS" | wc -w) - echo "==> Processing $pkgtotal new/updated arch-independent packages for '$reponame'..." >&2 - /usr/bin/svn checkout -N $SVNREPO checkout + /usr/bin/svn checkout -q -N $SVNREPO checkout >/dev/null cd checkout to_add_any="" for pkg in $ANYPKGS; do @@ -107,11 +108,9 @@ if [ -n "$ANYPKGS" ]; then _pkgname="$(getpkgname $pkg)" _pkgbase="$(getpkgbase $pkg)" svnrepo="$reponame-any" - echo " Validating package arch (any) $_pkgname" if ! check_pkg_arch "$pkg" "any"; then echo " ERROR: $_pkgfile is not architecture independent!" else - echo " Checking SVN for $_pkgbase" /usr/bin/svn up -q $_pkgbase if [ -d "$_pkgbase/repos/$svnrepo" ]; then . "$_pkgbase/repos/$svnrepo/$BUILDSCRIPT" @@ -156,8 +155,6 @@ for current_arch in ${ARCHES[@]}; do /bin/cp "$ftppath/$reponame$DBEXT" build/$reponame-$current_arch$DBEXT fi - echo "Updating DB for $svnrepo" - to_add="" if [ -d "$stagedir" ]; then ADDPKGS="$(getpkgfiles $stagedir/*-${current_arch}$PKGEXT 2>/dev/null)" @@ -165,17 +162,14 @@ for current_arch in ${ARCHES[@]}; do if [ -n "$ADDPKGS" -o -n "$ANYPKGS" ]; then - echo "==> Copying DB file from '$reponame'..." >&2 - if [ -f "$ftppath/$reponame$DBEXT" ]; then /bin/cp "$ftppath/$reponame$DBEXT" build/$reponame-$current_arch$DBEXT fi pkgtotal=$(echo "$ADDPKGS $ANYPKGS" | wc -w) - echo "==> Processing $pkgtotal new/updated packages for repository '$reponame'..." >&2 cd "$WORKDIR" - /usr/bin/svn checkout -N $SVNREPO checkout + /usr/bin/svn checkout -N $SVNREPO checkout >/dev/null cd checkout if [ -n "$ADDPKGS" ]; then @@ -184,11 +178,9 @@ for current_arch in ${ARCHES[@]}; do _pkgname="$(getpkgname $pkg)" _pkgbase="$(getpkgbase $pkg)" - echo " Validating package arch ($current_arch) $_pkgname" if ! check_pkg_arch "$pkg" "$current_arch"; then echo " ERROR: $_pkgfile was built for the wrong architecture" else - echo " Checking SVN for $_pkgbase" /usr/bin/svn up -q $_pkgbase if [ -d "$_pkgbase/repos/$svnrepo" ]; then . "$_pkgbase/repos/$svnrepo/$BUILDSCRIPT" @@ -211,7 +203,7 @@ for current_arch in ${ARCHES[@]}; do pkgs="" for pkg in $to_add $to_add_any; do pkgs="$pkgs $(basename $pkg)"; done - /usr/bin/repo-add -q "$reponame-$current_arch$DBEXT" $pkgs + /usr/bin/repo-add -q "$reponame-$current_arch$DBEXT" $pkgs >/dev/null else rm -f "build/$reponame-$current_arch$DBEXT" echo "Errors found when adding packages" @@ -223,7 +215,6 @@ for current_arch in ${ARCHES[@]}; do # if non empty, move all build dirs if [ $(/bin/ls "$WORKDIR/build/" 2>/dev/null | wc -l) != 0 ]; then if [ $(getpkgfiles "$WORKDIR/build/"*-$current_arch$PKGEXT 2>/dev/null | wc -l) != 0 ]; then - echo "Copying new files to '$poolpath' and symlinking" for f in "$WORKDIR/build/"*-$current_arch$PKGEXT; do /bin/chmod 664 "$f" &>/dev/null if ! /bin/cp "$f" "$poolpath/"; then @@ -236,7 +227,6 @@ for current_arch in ${ARCHES[@]}; do done fi if [ $(getpkgfiles "$WORKDIR/build/"*-any$PKGEXT 2>/dev/null | wc -l) != 0 ]; then - echo "Copying new files to '$poolpath_any' and symlinking" for f in "$WORKDIR/build/"*-any$PKGEXT; do /bin/chmod 664 "$f" &>/dev/null fname="$(basename $f)" @@ -257,7 +247,6 @@ for current_arch in ${ARCHES[@]}; do fi if [ -n "$to_add" ]; then - echo "Cleaning staging dir" /bin/rm $to_add fi @@ -268,6 +257,8 @@ if [ -n "$to_add_any" ]; then /bin/rm $to_add_any fi +echo 'done' + trap '' 0 2 rm -rf $WORKDIR # vim: set ts=4 sw=4 noet ft=sh: diff --git a/test/lib/common.inc b/test/lib/common.inc index dd3022e..2d2b612 100644 --- a/test/lib/common.inc +++ b/test/lib/common.inc @@ -60,10 +60,8 @@ eot } tearDown() { - echo -n 'Cleaning up...' rm -rf "${TMP}" rm -f "${curdir}/../config.local" - echo 'done' echo } @@ -73,7 +71,7 @@ releasePackage() { local arch=$3 pushd "${TMP}/svn-packages-copy"/${pkgbase}/trunk/ >/dev/null - archrelease ${repo}-${arch} >/dev/null + archrelease ${repo}-${arch} >/dev/null 2&>1 popd >/dev/null cp "${pkgdir}/${pkgbase}"/*.pkg.tar.* "${STAGING}"/${repo}/ } diff --git a/test/runTest b/test/runTest index d453493..b24b7ea 100755 --- a/test/runTest +++ b/test/runTest @@ -235,7 +235,7 @@ testCleanupSimplePackages() { ../db-remove pkg-simple-a extra ${arch} done - ../cron-jobs/ftpdir-cleanup + ../cron-jobs/ftpdir-cleanup >/dev/null for arch in ${arches[@]}; do local pkg1="pkg-simple-a-1-1-${arch}.pkg.tar.xz" @@ -258,7 +258,7 @@ testCleanupAnyPackages() { ../db-update extra ../db-remove pkg-any-a extra any - ../cron-jobs/ftpdir-cleanup + ../cron-jobs/ftpdir-cleanup >/dev/null local pkg1='pkg-any-a-1-1-any.pkg.tar.xz' [ -f "${FTP_BASE}/packages/arch/${arch}/${pkg1}" ] && fail "packages/arch/${arch}/${pkg1} found" @@ -289,7 +289,7 @@ testCleanupSplitPackages() { done done - ../cron-jobs/ftpdir-cleanup + ../cron-jobs/ftpdir-cleanup >/dev/null for arch in ${arches[@]}; do for pkg in "${pkgdir}/${pkgs[0]}"/*-${arch}.pkg.tar.*; do @@ -333,7 +333,7 @@ testMovePackagesWithoutPool() { done done - ../cron-jobs/ftpdir-cleanup + ../cron-jobs/ftpdir-cleanup >/dev/null for pkgbase in ${pkgs[@]}; do for arch in ${arches[@]}; do @@ -341,7 +341,7 @@ testMovePackagesWithoutPool() { done done - ../cron-jobs/ftpdir-cleanup + ../cron-jobs/ftpdir-cleanup >/dev/null for pkgbase in ${pkgs[@]}; do for arch in ${arches[@]}; do @@ -374,13 +374,13 @@ testMoveAnyPackagesWithoutPool() { done done - ../cron-jobs/ftpdir-cleanup + ../cron-jobs/ftpdir-cleanup >/dev/null for pkgbase in ${pkgs[@]}; do ../db-move ${pkgbase} testing extra any done - ../cron-jobs/ftpdir-cleanup + ../cron-jobs/ftpdir-cleanup >/dev/null for pkgbase in ${pkgs[@]}; do for pkg in "${pkgdir}/${pkgbase}"/*-any.pkg.tar.*; do -- cgit v1.2.2 From cbe9204ce41a81086c7498360769a66cfe48ce34 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 8 Aug 2010 11:29:16 +0200 Subject: Fix test and add some more checks --- test/runTest | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/test/runTest b/test/runTest index b24b7ea..5bbea05 100755 --- a/test/runTest +++ b/test/runTest @@ -239,6 +239,7 @@ testCleanupSimplePackages() { for arch in ${arches[@]}; do local pkg1="pkg-simple-a-1-1-${arch}.pkg.tar.xz" + checkRemovedPackage extra 'pkg-simple-a' ${arch} [ -f "${FTP_BASE}/packages/arch/${arch}/${pkg1}" ] && fail "packages/arch/${arch}/${pkg1} found" [ -f "${FTP_BASE}/${repo}/os/${arch}/${pkg1}" ] && fail "${repo}/os/${arch}/${pkg1} found" @@ -261,6 +262,7 @@ testCleanupAnyPackages() { ../cron-jobs/ftpdir-cleanup >/dev/null local pkg1='pkg-any-a-1-1-any.pkg.tar.xz' + checkRemovedAnyPackage extra 'pkg-any-a' [ -f "${FTP_BASE}/packages/arch/${arch}/${pkg1}" ] && fail "packages/arch/${arch}/${pkg1} found" [ -f "${FTP_BASE}/${repo}/os/${arch}/${pkg1}" ] && fail "${repo}/os/${arch}/${pkg1} found" @@ -283,16 +285,15 @@ testCleanupSplitPackages() { ../db-update extra - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - ../db-remove pkg-split-a extra ${arch} - done + for arch in ${arches[@]}; do + ../db-remove ${pkgs[0]} extra ${arch} done ../cron-jobs/ftpdir-cleanup >/dev/null for arch in ${arches[@]}; do for pkg in "${pkgdir}/${pkgs[0]}"/*-${arch}.pkg.tar.*; do + checkRemovedPackage extra ${pkgs[0]} ${arch} [ -f "${FTP_BASE}/packages/arch/${arch}/${pkg}" ] && fail "packages/arch/${arch}/${pkg} found" [ -f "${FTP_BASE}/${repo}/os/${arch}/${pkg}" ] && fail "${repo}/os/${arch}/${pkg} found" done -- cgit v1.2.2 From 20f1c1c27a1691cf92ad11ebc81dfaa175ace305 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 8 Aug 2010 11:39:42 +0200 Subject: Cleanup db-remove * There is no need to copy the db into a working dir * cleanup is automatically called on exit --- db-remove | 23 +++++------------------ 1 file changed, 5 insertions(+), 18 deletions(-) diff --git a/db-remove b/db-remove index 357e6c4..bf325c4 100755 --- a/db-remove +++ b/db-remove @@ -54,13 +54,10 @@ if [ -d "$packagebase/repos/$svnrepo" ]; then /usr/bin/svn rm --force -q "$packagebase/repos/$svnrepo" /usr/bin/svn commit -q -m "$(basename $0): $packagebase removed by $(id -un)" else - echo " Warning: $packagebase not found in $svnrepo" - echo " Removing split packages is not yet supported" - exit 1 + die "Error: $packagebase not found in $svnrepo" fi cd "$WORKDIR" -[ -d build/ ] || mkdir build if [ "$_arch" == "any" ]; then arches="i686 x86_64" @@ -69,24 +66,14 @@ else fi # copy the db file into our working area -for architecture in $arches; do - if [ -f "$ftppath/$architecture/$reponame$DBEXT" ]; then - /bin/cp "$ftppath/$architecture/$reponame$DBEXT" build/ - else - echo "No database found at '$ftppath/$architecture', nothing more to do" - exit 0 +for arch in $arches; do + if [ ! -f "$ftppath/$arch/$reponame$DBEXT" ]; then + die "No database found at '$ftppath/$arch', nothing more to do" fi - cd build/ - /usr/bin/repo-remove -q "$reponame$DBEXT" ${pkgname[@]} >/dev/null - - /bin/mv "$reponame$DBEXT" "$ftppath/$architecture" - - cd .. + /usr/bin/repo-remove -q "$ftppath/$arch/$reponame$DBEXT" ${pkgname[@]} >/dev/null done echo 'done' -cleanup - # vim: set ts=4 sw=4 noet ft=sh: -- cgit v1.2.2 From 3eb0f6abdb822ae15bcc72626dcdad37c0313927 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 8 Aug 2010 11:42:27 +0200 Subject: Remove check for old staging dirs devtools take care of this anyway --- db-update | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/db-update b/db-update index 057de39..9dd5c9d 100755 --- a/db-update +++ b/db-update @@ -34,26 +34,6 @@ if [ ! -d $stagedir ]; then exit 1 fi -if [ -d "${stagedir}64" ]; then - echo "--------------------------------------------------" - echo "It looks like you have an old staging dir" - echo "Packages are now differentiated by the arch in the filename." - echo "Please delete '${stagedir}64'" - echo "--------------------------------------------------" - /bin/mv "${stagedir}64/add/"* "$stagedir/add/" - /bin/mv "${stagedir}64/del/"* "$stagedir/del/" -fi - -if [ -d "${stagedir}/add" ]; then - echo "--------------------------------------------------" - echo "It looks like you have an old staging dir" - echo "The 'add' and 'del' dirs are no longer used." - echo "Please delete staging//{add,del}" - echo " and ensure you are using the newest devtools" - echo "--------------------------------------------------" - /bin/mv "${stagedir}/add/"* "$stagedir/" -fi - cleanup() { trap '' 0 2 repo_unlock $reponame $current_arch -- cgit v1.2.2 From d0b5dcb3ce6a1ec5222fc7a0420b3e1dc4c3a27e Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 8 Aug 2010 11:53:07 +0200 Subject: Remove BUILDSCRIPT variable There is no need to have a variable for things like PKGBUILD that are very unlikely to ever change. --- config | 1 - db-move | 2 +- db-remove | 2 +- db-update | 8 ++++---- misc-scripts/make-sourceball | 2 +- misc-scripts/sourceballs-cleanup | 4 ++-- testing2x | 4 ++-- 7 files changed, 11 insertions(+), 12 deletions(-) diff --git a/config b/config index 88e50bc..f6600a6 100644 --- a/config +++ b/config @@ -14,7 +14,6 @@ LOCK_TIMEOUT=300 STAGING="$HOME/staging" TMPDIR="/srv/tmp" ARCHES=(i686 x86_64) -BUILDSCRIPT="PKGBUILD" DBEXT=".db.tar.gz" PKGEXT=".pkg.tar.*" SRCEXT=".src.tar.gz" diff --git a/db-move b/db-move index 7dff563..8347a18 100755 --- a/db-move +++ b/db-move @@ -60,7 +60,7 @@ cd checkout /usr/bin/svn up -q $packagebase if [ -d "$packagebase/repos/$svnrepo_from" ]; then - . "$packagebase/repos/$svnrepo_from/$BUILDSCRIPT" + . "$packagebase/repos/$svnrepo_from/PKGBUILD" echo -n "Moving $packagebase from $repofrom to $repoto..." if [ -d "$packagebase/repos/$svnrepo_to" ]; then diff --git a/db-remove b/db-remove index bf325c4..1314bb7 100755 --- a/db-remove +++ b/db-remove @@ -50,7 +50,7 @@ cd checkout /usr/bin/svn up -q $packagebase if [ -d "$packagebase/repos/$svnrepo" ]; then - . "$packagebase/repos/$svnrepo/$BUILDSCRIPT" + . "$packagebase/repos/$svnrepo/PKGBUILD" /usr/bin/svn rm --force -q "$packagebase/repos/$svnrepo" /usr/bin/svn commit -q -m "$(basename $0): $packagebase removed by $(id -un)" else diff --git a/db-update b/db-update index 9dd5c9d..a08bee2 100755 --- a/db-update +++ b/db-update @@ -93,11 +93,11 @@ if [ -n "$ANYPKGS" ]; then else /usr/bin/svn up -q $_pkgbase if [ -d "$_pkgbase/repos/$svnrepo" ]; then - . "$_pkgbase/repos/$svnrepo/$BUILDSCRIPT" + . "$_pkgbase/repos/$svnrepo/PKGBUILD" if echo "$_pkgfile" | grep -q "$_pkgname-$pkgver-$pkgrel-any"; then to_add_any="$to_add_any $pkg" else - echo " WARNING: $_pkgfile does not match $BUILDSCRIPT in $svnrepo" + echo " WARNING: $_pkgfile does not match PKGBUILD in $svnrepo" fi else echo " WARNING: Package $_pkgbase not found in $svnrepo" @@ -163,11 +163,11 @@ for current_arch in ${ARCHES[@]}; do else /usr/bin/svn up -q $_pkgbase if [ -d "$_pkgbase/repos/$svnrepo" ]; then - . "$_pkgbase/repos/$svnrepo/$BUILDSCRIPT" + . "$_pkgbase/repos/$svnrepo/PKGBUILD" if echo "$_pkgfile" | grep -q "$_pkgname-$pkgver-$pkgrel-$current_arch"; then to_add="$to_add $pkg" else - echo " WARNING: $_pkgfile does not match $BUILDSCRIPT in $svnrepo" + echo " WARNING: $_pkgfile does not match PKGBUILD in $svnrepo" fi else echo " WARNING: Package $_pkgbase not found in $svnrepo" diff --git a/misc-scripts/make-sourceball b/misc-scripts/make-sourceball index 37141c7..41767d0 100755 --- a/misc-scripts/make-sourceball +++ b/misc-scripts/make-sourceball @@ -43,7 +43,7 @@ die() { create_srcpackage() { if [ -d "$1" ]; then pushd "$1" >/dev/null - . "$BUILDSCRIPT" + . "PKGBUILD" if ! [ $FORCE == 1 ] && ! chk_license ${license[@]} ; then #Removed so as not to clutter failed.txt #echo -e "\t$packagename license (${license[@]}) does not require source tarballs" >&2 diff --git a/misc-scripts/sourceballs-cleanup b/misc-scripts/sourceballs-cleanup index c7f284d..d259d3a 100755 --- a/misc-scripts/sourceballs-cleanup +++ b/misc-scripts/sourceballs-cleanup @@ -27,7 +27,7 @@ remove_old() { PKGVERS="" for repo in *; do cd "$repo" - . "$BUILDSCRIPT" + . "PKGBUILD" PKGVERS="$PKGVERS $pkgver-$pkgrel" cd .. done @@ -85,7 +85,7 @@ for sourceball in "$srcpath"/*$SRCEXT; do elif [ -z "$(ls -A "$packagename/repos")" ]; then echo "$packagename : no longer in repos but trunk is still in svn. Removing sourceball." >> "$logfile" mv $sourceball $SOURCE_CLEANUP_DESTDIR - elif ! source "$packagename/trunk/$BUILDSCRIPT" && chk_license ${license[@]}; then + elif ! source "$packagename/trunk/PKGBUILD" && chk_license ${license[@]}; then echo "$packagename : source hosting no longer required by license. Removing sourceball." >> "$logfile" mv $sourceball $SOURCE_CLEANUP_DESTDIR else diff --git a/testing2x b/testing2x index f9ded9e..4826853 100755 --- a/testing2x +++ b/testing2x @@ -35,9 +35,9 @@ cd checkout for pkg in $*; do moved=0 /usr/bin/svn up -q ${pkg} - if [ -f "${pkg}/repos/testing-${_arch}/${BUILDSCRIPT}" ]; then + if [ -f "${pkg}/repos/testing-${_arch}/PKGBUILD" ]; then for repo in core extra; do - if [ -f "${pkg}/repos/${repo}-${_arch}/${BUILDSCRIPT}" ]; then + if [ -f "${pkg}/repos/${repo}-${_arch}/PKGBUILD" ]; then echo "===> Moving package '${pkg}': testing-${_arch} -> ${repo}-${_arch}" $(dirname $0)/db-move "${pkg}" "testing" "${repo}" "${_arch}" moved=1 -- cgit v1.2.2 From ff0745c50507f0cea48dbf97b8f55734698e3c13 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 8 Aug 2010 12:47:06 +0200 Subject: Prepare for variable db file compression --- db-update | 1 + test/lib/common.inc | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/db-update b/db-update index a08bee2..d08c96d 100755 --- a/db-update +++ b/db-update @@ -222,6 +222,7 @@ for current_arch in ${ARCHES[@]}; do if ! /bin/cp "$WORKDIR/build/$reponame-$current_arch$DBEXT" "$ftppath/$reponame$DBEXT"; then die "failed to move repository $reponame-$current_arch". fi + ln -sf "$reponame$DBEXT" "$ftppath/$reponame${DBEXT%.tar.*}" else echo "Nothing to copy, no work done" fi diff --git a/test/lib/common.inc b/test/lib/common.inc index 2d2b612..d28017d 100644 --- a/test/lib/common.inc +++ b/test/lib/common.inc @@ -89,7 +89,7 @@ checkAnyPackage() { done [ -f "${STAGING}"/${repo}/${pkg} ] && fail "${repo}/${pkg} found in staging dir" - bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}.db.tar.gz" -O | grep -q ${pkg} || fail "${pkg} not in ${repo}/os/${arch}/${repo}.db.tar.gz" + bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" -O | grep -q ${pkg} || fail "${pkg} not in ${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" [ -f "${FTP_BASE}/${repo}/os/any/${pkg}" ] && fail "${repo}/os/any/${pkg} should not exist" } @@ -105,7 +105,7 @@ checkPackage() { [ "$(readlink -e "${FTP_BASE}/${repo}/os/${arch}/${pkg}")" == "${FTP_BASE}/packages/arch/${arch}/${pkg}" ] || fail "${repo}/os/${arch}/${pkg} does not link to packages/arch/${arch}/${pkg}" - bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}.db.tar.gz" -O | grep -q ${pkg} || fail "${pkg} not in ${repo}/os/${arch}/${repo}.db.tar.gz" + bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" -O | grep -q ${pkg} || fail "${pkg} not in ${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" local pkgbase=$(getpkgbase "${FTP_BASE}/packages/arch/${arch}/${pkg}") svn up -q "${TMP}/svn-packages-copy/${pkgbase}" @@ -117,7 +117,7 @@ checkRemovedPackage() { local pkgbase=$2 local arch=$3 - bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}.db.tar.gz" -O | grep -q ${pkgbase} && fail "${pkgbase} should not be in ${repo}/os/${arch}/${repo}.db.tar.gz" + bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" -O | grep -q ${pkgbase} && fail "${pkgbase} should not be in ${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" svn up -q "${TMP}/svn-packages-copy/${pkgbase}" [ -d "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo}-${arch}" ] && fail "svn-packages-copy/${pkgbase}/repos/${repo}-${arch} should not exist" @@ -129,7 +129,7 @@ checkRemovedAnyPackage() { local arch for arch in i686 x86_64; do - bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}.db.tar.gz" -O | grep -q ${pkgbase} && fail "${pkgbase} should not be in ${repo}/os/${arch}/${repo}.db.tar.gz" + bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" -O | grep -q ${pkgbase} && fail "${pkgbase} should not be in ${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" done svn up -q "${TMP}/svn-packages-copy/${pkgbase}" -- cgit v1.2.2 From 388d629327d97e8d15a7a67cc87ae36edfe2f385 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 8 Aug 2010 13:08:47 +0200 Subject: use common workdir --- db-functions | 4 ++-- db-move | 6 ------ db-remove | 5 ----- db-update | 2 -- misc-scripts/make-sourceball | 3 --- misc-scripts/sourceballs-cleanup | 2 -- testing2x | 2 -- 7 files changed, 2 insertions(+), 22 deletions(-) diff --git a/db-functions b/db-functions index 2fe7a49..caaf9a0 100644 --- a/db-functions +++ b/db-functions @@ -1,7 +1,7 @@ #!/bin/bash -# Random integrity things -[ "$UID" = "" ] && UID=$(uid) +# set up general environment +WORKDIR=$(mktemp -d /tmp/$(basename $0).XXXXXXXXXX) # Useful functions diff --git a/db-move b/db-move index 8347a18..8b15831 100755 --- a/db-move +++ b/db-move @@ -23,10 +23,6 @@ ftppath_to="$FTP_BASE/$repoto/os/" svnrepo_from="$repofrom-$_arch" svnrepo_to="$repoto-$_arch" -[ "$UID" = "" ] && UID=$(uid) - -WORKDIR="$TMPDIR/db-move.$svnrepo_from.$svnrepo_to.$UID" - cleanup() { trap '' 0 2 # unlock @@ -52,8 +48,6 @@ trap cleanup 0 repo_lock $repoto $_arch || exit 1 repo_lock $repofrom $_arch || exit 1 -/bin/mkdir -p "$WORKDIR" - cd "$WORKDIR" /usr/bin/svn checkout -q -N $SVNREPO checkout >/dev/null cd checkout diff --git a/db-remove b/db-remove index 1314bb7..0da9e75 100755 --- a/db-remove +++ b/db-remove @@ -15,10 +15,6 @@ _arch="$3" ftppath="$FTP_BASE/$reponame/os" svnrepo="$reponame-$_arch" -[ "$UID" = "" ] && UID=$(uid) - -WORKDIR="$TMPDIR/db-remove.$svnrepo.$UID" - cleanup() { trap '' 0 2 # unlock @@ -43,7 +39,6 @@ trap cleanup 0 repo_lock $reponame $_arch || exit 1 echo -n "Removing $packagebase from $reponame..." -/bin/mkdir -p "$WORKDIR" cd "$WORKDIR" /usr/bin/svn checkout -q -N $SVNREPO checkout >/dev/null cd checkout diff --git a/db-update b/db-update index d08c96d..e53aee8 100755 --- a/db-update +++ b/db-update @@ -24,7 +24,6 @@ if [ $found -ne 1 ]; then exit 1 fi -WORKDIR="$TMPDIR/db-update.$reponame.$UID" ADDPKGS="" ANYPKGS="" @@ -74,7 +73,6 @@ if [ -d "$stagedir" ]; then ANYPKGS="$(getpkgfiles $stagedir/*-any$PKGEXT 2>/dev/null)" fi -mkdir -p $WORKDIR cd "$WORKDIR" if [ -n "$ANYPKGS" ]; then diff --git a/misc-scripts/make-sourceball b/misc-scripts/make-sourceball index 41767d0..e216356 100755 --- a/misc-scripts/make-sourceball +++ b/misc-scripts/make-sourceball @@ -22,8 +22,6 @@ _arch="$3" srcpath="$FTP_BASE/sources/" logpath="/var/log/sourceballs/" -WORKDIR="/tmp/make-sourceball.$packagename.$UID" - cleanup() { restore_umask rm -rf "$WORKDIR" @@ -77,7 +75,6 @@ trap ctrl_c 2 trap cleanup 0 1 set_umask -/bin/mkdir -p "$WORKDIR" /bin/mkdir -p "$logpath" cd "$WORKDIR" diff --git a/misc-scripts/sourceballs-cleanup b/misc-scripts/sourceballs-cleanup index d259d3a..c87a888 100755 --- a/misc-scripts/sourceballs-cleanup +++ b/misc-scripts/sourceballs-cleanup @@ -7,7 +7,6 @@ srcpath="$FTP_BASE/sources/" logfile="$srcpath/cleanup.txt" LOCKFILE="/tmp/.sourceball-cleanup.lock" -WORKDIR="/tmp/sourceball-cleanup.$packagename.$UID" cleanup () { restore_umask @@ -68,7 +67,6 @@ trap ctrl_c 2 /usr/bin/renice +10 -p $$ > /dev/null set_umask -/bin/mkdir -p "$WORKDIR" cd "$WORKDIR" [ -e "$logfile" ] && /bin/mv "$logfile" "$logfile.old" diff --git a/testing2x b/testing2x index 4826853..e9fbeaa 100755 --- a/testing2x +++ b/testing2x @@ -11,7 +11,6 @@ case "$0" in _arch="i686" ;; esac -WORKDIR="$TMPDIR/testing2x.$UID" cleanup() { trap '' 0 2 @@ -27,7 +26,6 @@ ctrl_c() { trap ctrl_c 2 trap cleanup 0 -/bin/mkdir "${WORKDIR}" cd "${WORKDIR}" /usr/bin/svn checkout -N $SVNREPO checkout cd checkout -- cgit v1.2.2 From 9eb1cd7b9403533c4b60ecfbbbf00a08c211059a Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 8 Aug 2010 15:03:27 +0200 Subject: Move common function to db-functions db-functions now sets an individual $WORKDIR and implements trap functinos that remove locks on exit or error. There are new functions to lock and unlock the running script. misc-scripts/ftpdir-cleanup was renamed to ftpdir-cleanup-repo as the cron-job had the same name. Script names have to be unique when using db-functions. --- cron-jobs/ftpdir-cleanup | 26 +---- db-functions | 74 +++++++++++-- db-move | 25 +---- db-remove | 23 +---- db-update | 22 ---- misc-scripts/ftpdir-cleanup | 218 --------------------------------------- misc-scripts/ftpdir-cleanup-repo | 207 +++++++++++++++++++++++++++++++++++++ misc-scripts/make-sourceball | 21 +--- misc-scripts/sourceballs-cleanup | 27 +---- test/lib/common.inc | 6 +- testing2x | 16 --- 11 files changed, 291 insertions(+), 374 deletions(-) delete mode 100755 misc-scripts/ftpdir-cleanup create mode 100755 misc-scripts/ftpdir-cleanup-repo diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index 546afd9..f5e401a 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -5,34 +5,14 @@ repos="$(get_repos_for_host)" -LOCKFILE="/tmp/.ftpdircleanup.lock" - -cleanup () { - rm -f "$LOCKFILE" - exit 0 -} - -ctrl_c() { - cleanup -} - -if [ -f "$LOCKFILE" ]; then - owner="$(/usr/bin/stat -c %U $LOCKFILE)" - echo "error: ftp cleanup is already in progress (started by $owner)" - exit 1 -fi - -trap cleanup 0 -trap ctrl_c 2 - -/bin/touch "$LOCKFILE" +script_lock #adjust the nice level to run at a lower priority /usr/bin/renice +10 -p $$ > /dev/null repopaths='' for repo in $repos; do - $(dirname $0)/../misc-scripts/ftpdir-cleanup $repo + $(dirname $0)/../misc-scripts/ftpdir-cleanup-repo $repo repopaths="${repopaths} ${FTP_BASE}/${repo}/os/" done @@ -60,4 +40,4 @@ if [ -n "$to_cleanup" ]; then mv $to_cleanup "$CLEANUP_DESTDIR" fi -cleanup +script_unlock diff --git a/db-functions b/db-functions index caaf9a0..36ffb07 100644 --- a/db-functions +++ b/db-functions @@ -1,10 +1,6 @@ #!/bin/bash -# set up general environment -WORKDIR=$(mktemp -d /tmp/$(basename $0).XXXXXXXXXX) - # Useful functions - UMASK="" set_umask () { [ "$UMASK" == "" ] && UMASK="$(umask)" @@ -16,8 +12,73 @@ restore_umask () { umask $UMASK >/dev/null } +# set up general environment +WORKDIR=$(mktemp -d /tmp/$(basename $0).XXXXXXXXXX) +LOCKS=() + +script_lock() { + local LOCKDIR="$TMPDIR/.scriptlock.$(basename $0)" + if ! mkdir "$LOCKDIR" >/dev/null 2>&1 ; then + local _owner="$(/usr/bin/stat -c %U $LOCKDIR)" + echo "Error: Script $(basename $0) is already locked by $_owner." >&2 + exit 1 + else + set_umask + return 0 + fi +} + +script_unlock() { + local LOCKDIR="$TMPDIR/.scriptlock.$(basename $0)" + if [ ! -d "$LOCKDIR" ]; then + echo "Warning: Script $(basename $0) was not locked!" >&2 + restore_umask + return 1 + else + rmdir "$LOCKDIR" + restore_umask + return 0 + fi +} + +cleanup() { + local l + local repo + local arch + + trap - EXIT INT QUIT TERM + for l in ${LOCKS[@]}; do + repo=${l%.*} + arch=${l#*.} + if [ -d "$TMPDIR/.repolock.$repo.$arch" ]; then + echo "Removing left over lock from $repo/$arch" >&2 + repo_unlock $repo $_arch + fi + done + if [ -d "$TMPDIR/.scriptlock.$(basename $0)" ]; then + echo "Removing left over lock from $(basename $0)" >&2 + script_unlock + fi + rm -rf "$WORKDIR" + [ "$1" ] && exit $1 +} + +abort() { + echo 'Aborting...' >&2 + cleanup 0 +} + +die() { + echo "$*" >&2 + cleanup 1 +} + +trap abort INT QUIT TERM +trap cleanup EXIT + + repo_lock () { #repo_lock [timeout] - LOCKDIR="$TMPDIR/.repolock.$1.$2" + local LOCKDIR="$TMPDIR/.repolock.$1.$2" local _count local _trial local _timeout @@ -39,6 +100,7 @@ repo_lock () { #repo_lock [timeout] _owner="$(/usr/bin/stat -c %U $LOCKDIR)" echo "warning: Repo $1-$2 is already locked by $_owner. Retrying in $LOCK_DELAY seconds..." >&2 else + LOCKS[${#LOCKS[*]}]="$1.$2" set_umask return 0 fi @@ -51,7 +113,7 @@ repo_lock () { #repo_lock [timeout] } repo_unlock () { #repo_unlock - LOCKDIR="$TMPDIR/.repolock.$1.$2" + local LOCKDIR="$TMPDIR/.repolock.$1.$2" if [ ! -d "$LOCKDIR" ]; then echo "warning: Repo lock $1-$2 was not locked!" >&2 restore_umask diff --git a/db-move b/db-move index 8b15831..b846e79 100755 --- a/db-move +++ b/db-move @@ -23,28 +23,6 @@ ftppath_to="$FTP_BASE/$repoto/os/" svnrepo_from="$repofrom-$_arch" svnrepo_to="$repoto-$_arch" -cleanup() { - trap '' 0 2 - # unlock - repo_unlock $repoto $_arch - repo_unlock $repofrom $_arch - rm -rf "$WORKDIR" - [ "$1" ] && exit $1 -} - -ctrl_c() { - echo "Interrupted" >&2 - cleanup 0 -} - -die() { - echo "$*" >&2 - cleanup 1 -} - -trap ctrl_c 2 -trap cleanup 0 - repo_lock $repoto $_arch || exit 1 repo_lock $repofrom $_arch || exit 1 @@ -91,3 +69,6 @@ if [ -d "$packagebase/repos/$svnrepo_from" ]; then else die "Error: $packagebase is not in repo $repofrom" fi + +repo_unlock $repoto $_arch || exit 1 +repo_unlock $repofrom $_arch || exit 1 diff --git a/db-remove b/db-remove index 0da9e75..5d61c97 100755 --- a/db-remove +++ b/db-remove @@ -15,27 +15,6 @@ _arch="$3" ftppath="$FTP_BASE/$reponame/os" svnrepo="$reponame-$_arch" -cleanup() { - trap '' 0 2 - # unlock - repo_unlock $reponame $_arch - rm -rf "$WORKDIR" - [ "$1" ] && exit $1 -} - -ctrl_c() { - echo "Interrupted" >&2 - cleanup 0 -} - -die() { - echo "$*" >&2 - cleanup 1 -} - -trap ctrl_c 2 -trap cleanup 0 - repo_lock $reponame $_arch || exit 1 echo -n "Removing $packagebase from $reponame..." @@ -71,4 +50,6 @@ done echo 'done' +repo_unlock $reponame $_arch || exit 1 + # vim: set ts=4 sw=4 noet ft=sh: diff --git a/db-update b/db-update index e53aee8..f971a6f 100755 --- a/db-update +++ b/db-update @@ -33,26 +33,6 @@ if [ ! -d $stagedir ]; then exit 1 fi -cleanup() { - trap '' 0 2 - repo_unlock $reponame $current_arch - rm -rf "$WORKDIR" - [ "$1" ] && exit $1 -} - -ctrl_c() { - echo "Interrupted" >&2 - cleanup 1 -} - -die() { - echo "$*" >&2 - cleanup 1 -} - -trap ctrl_c 2 -trap cleanup 0 - echo -n "Updating $reponame..." # Remove any package from $stagedir that is already in the FTP repository @@ -238,6 +218,4 @@ fi echo 'done' -trap '' 0 2 -rm -rf $WORKDIR # vim: set ts=4 sw=4 noet ft=sh: diff --git a/misc-scripts/ftpdir-cleanup b/misc-scripts/ftpdir-cleanup deleted file mode 100755 index ac17a8d..0000000 --- a/misc-scripts/ftpdir-cleanup +++ /dev/null @@ -1,218 +0,0 @@ -#!/bin/bash - -if [ $# -ne 1 ]; then - echo "usage: $(basename $0) " - exit 1 -fi - -reponame=$1 - -############################################################ - -. "$(dirname $0)/../db-functions" -. "$(dirname $0)/../config" - -cleanup() { - trap '' 2 15 - for arch in ${ARCHES[@]}; do - repo_unlock $reponame $arch - done - exit 1 -} - -clean_pkgs () { - for pkg in "$@"; do - if [ -h "$pkg" ]; then - rm -f "$pkg" - else - mv "$pkg" "$CLEANUP_DESTDIR" - fi - done -} - -trap cleanup 2 -trap cleanup 15 - -${CLEANUP_DRYRUN} && echo 'dry run mode is active' - -ftppath_base="$FTP_BASE/$reponame/os" - -for arch in ${ARCHES[@]}; do - - repo_lock $reponame $arch $LOCK_TIMEOUT || continue - - CLEANUP_TMPDIR=$(mktemp -d ${TMPDIR}/cleanup-XXXXXX) || exit 1 - ftppath="$ftppath_base/$arch" - MISSINGFILES="" - DELETEFILES="" - DELETESYMLINKS="" - EXTRAFILES="" - - if [ ! -d "$ftppath" ]; then - echo "FTP path '$ftppath' does not exist" - exit 1 - fi - - if ! cd "${CLEANUP_TMPDIR}" ; then - echo "Failed to cd to ${CLEANUP_TMPDIR}" - exit 1 - fi - - if [ ! -f "$ftppath/$reponame$DBEXT" ]; then - echo "" - echo "WARNING: The file \"$ftppath/$reponame$DBEXT\" could not be found, skipping." - echo "" - repo_unlock $reponame $arch - continue - fi - - if ! bsdtar xf "$ftppath/$reponame$DBEXT"; then - echo "" - echo "ERROR: Command failed: bsdtar xf \"$ftppath/$reponame$DBEXT\"" - exit 1 - fi - - cd "$ftppath" - - for pkg in $CLEANUP_TMPDIR/*; do - [ -f "${pkg}" ] || continue - filename=$(grep -A1 '^%FILENAME%$' "${pkg}/desc" | tail -n1) - - if [ ! -e "${filename}" ]; then - MISSINGFILES="${MISSINGFILES} ${filename}" - else - pkgname="$(getpkgname ${filename})" - for otherfile in ${pkgname}-*; do - if [ "${otherfile}" != "${filename}" -a "${pkgname}" = "$(getpkgname ${otherfile})" ]; then - if [ -h "${otherfile}" ]; then - DELETESYMLINKS="${DELETESYMLINKS} ${otherfile}" - else - DELETEFILES="${DELETEFILES} ${otherfile}" - fi - fi - done - fi - done - - for pkg in *$PKGEXT; do - if [ ! -e "$pkg" ]; then - continue - fi - pkgname="$(getpkgname $pkg)" - for p in ${CLEANUP_TMPDIR}/${pkgname}-*; do - [ ! -d "${p}" ] || continue 2 - dbpkgname=$(grep -A1 '^%FILENAME%$' "${p}/desc" 2>/dev/null| tail -n1) - if [ "${dbpkgname}" = "${pkgname}" ]; then - continue 2 - fi - done - EXTRAFILES="$EXTRAFILES $pkg" - done - - rm -rf ${CLEANUP_TMPDIR} - - # Do a quick check to see if a missing ARCHINDEPFILE is in the any dir - # If it is, and the file is MISSING, restore it - missfiles="$MISSINGFILES" - MISSINGFILES="" - for mf in $missfiles; do - if [ -e "${ftppath_base}/any/${mf}" ]; then - echo "Restoring missing 'any' symlink: ${mf}" - ${CLEANUP_DRYRUN} || ln -s "../any/${mf}" "${ftppath}" - else - MISSINGFILES="${MISSINGFILES} ${mf}" - fi - done - - repo_unlock $reponame $arch - - #Make sure we've done *something* before outputting anything - if [ -z "$DELETEFILES$DELETESYMLINKS$MISSINGFILES$EXTRAFILES" ]; then - continue - fi - - echo "Scan complete for $reponame ($arch) at ${ftppath}" - - if [ -n "$DELETEFILES" ]; then - echo " The following files are out of date" - for f in $DELETEFILES; do - echo " $f" - done - echo "" - fi - - if [ -n "$DELETESYMLINKS" ]; then - echo " The following symlinks are out of date" - echo " They will be deleted" - for f in $DELETESYMLINKS; do - echo " $f" - done - echo "" - fi - - if [ -n "$MISSINGFILES" ]; then - echo " The following files are missing in the repo" - for f in $MISSINGFILES; do - echo " $f" - done - echo "" - fi - - if [ -n "$EXTRAFILES" ]; then - echo " The following files are in the repo but not the db" - for f in $EXTRAFILES; do - echo " $f" - done - fi - - if [ -n "${DELETEFILES}" ]; then - ${CLEANUP_DRYRUN} || clean_pkgs ${DELETEFILES} - echo "" - fi - - if [ -n "${DELETESYMLINKS}" ]; then - ${CLEANUP_DRYRUN} || clean_pkgs ${DELETESYMLINKS} - echo "" - fi - - if [ -n "${EXTRAFILES}" ]; then - ${CLEANUP_DRYRUN} || clean_pkgs ${EXTRAFILES} - echo "" - fi - -done - -ARCHINDEPFILES="" - -if [ -d "$ftppath_base/any" ]; then - cd "$ftppath_base/any" - for pkg in *$PKGEXT; do - [ -f "$pkg" ] || continue # in case we get a file named "*.pkg.tar.gz" - found=0 - #check for any existing symlinks - for arch in ${ARCHES[@]}; do - if [ -h "$ftppath_base/$arch/$pkg" ]; then - found=1 - break - fi - done - if [ $found -eq 0 ]; then - # We found no symlinks to this, delete it - ARCHINDEPFILES="$ARCHINDEPFILES $pkg" - fi - done -fi - -if [ -n "$ARCHINDEPFILES" ]; then - echo " The following architecture independent packages" - echo " are not symlinked in the architecture repositories." - for f in $ARCHINDEPFILES; do - echo " $f" - done -fi - -if [ -d "$ftppath_base/any" -a -n "${ARCHINDEPFILES}" ]; then - cd "$ftppath_base/any" - ${CLEANUP_DRYRUN} || clean_pkgs ${ARCHINDEPFILES} - echo "" -fi diff --git a/misc-scripts/ftpdir-cleanup-repo b/misc-scripts/ftpdir-cleanup-repo new file mode 100755 index 0000000..12f726c --- /dev/null +++ b/misc-scripts/ftpdir-cleanup-repo @@ -0,0 +1,207 @@ +#!/bin/bash + +if [ $# -ne 1 ]; then + echo "usage: $(basename $0) " + exit 1 +fi + +reponame=$1 + +############################################################ + +. "$(dirname $0)/../db-functions" +. "$(dirname $0)/../config" + +clean_pkgs () { + for pkg in "$@"; do + if [ -h "$pkg" ]; then + rm -f "$pkg" + else + mv "$pkg" "$CLEANUP_DESTDIR" + fi + done +} + +${CLEANUP_DRYRUN} && echo 'dry run mode is active' + +ftppath_base="$FTP_BASE/$reponame/os" + +for arch in ${ARCHES[@]}; do + + repo_lock $reponame $arch $LOCK_TIMEOUT || continue + + CLEANUP_TMPDIR=$(mktemp -d ${WORKDIR}/cleanup-XXXXXX) || exit 1 + ftppath="$ftppath_base/$arch" + MISSINGFILES="" + DELETEFILES="" + DELETESYMLINKS="" + EXTRAFILES="" + + if [ ! -d "$ftppath" ]; then + echo "FTP path '$ftppath' does not exist" + exit 1 + fi + + if ! cd "${CLEANUP_TMPDIR}" ; then + echo "Failed to cd to ${CLEANUP_TMPDIR}" + exit 1 + fi + + if [ ! -f "$ftppath/$reponame$DBEXT" ]; then + echo "" + echo "WARNING: The file \"$ftppath/$reponame$DBEXT\" could not be found, skipping." + echo "" + repo_unlock $reponame $arch + continue + fi + + if ! bsdtar xf "$ftppath/$reponame$DBEXT"; then + echo "" + echo "ERROR: Command failed: bsdtar xf \"$ftppath/$reponame$DBEXT\"" + exit 1 + fi + + cd "$ftppath" + + for pkg in $CLEANUP_TMPDIR/*; do + [ -f "${pkg}" ] || continue + filename=$(grep -A1 '^%FILENAME%$' "${pkg}/desc" | tail -n1) + + if [ ! -e "${filename}" ]; then + MISSINGFILES="${MISSINGFILES} ${filename}" + else + pkgname="$(getpkgname ${filename})" + for otherfile in ${pkgname}-*; do + if [ "${otherfile}" != "${filename}" -a "${pkgname}" = "$(getpkgname ${otherfile})" ]; then + if [ -h "${otherfile}" ]; then + DELETESYMLINKS="${DELETESYMLINKS} ${otherfile}" + else + DELETEFILES="${DELETEFILES} ${otherfile}" + fi + fi + done + fi + done + + for pkg in *$PKGEXT; do + if [ ! -e "$pkg" ]; then + continue + fi + pkgname="$(getpkgname $pkg)" + for p in ${CLEANUP_TMPDIR}/${pkgname}-*; do + [ ! -d "${p}" ] || continue 2 + dbpkgname=$(grep -A1 '^%FILENAME%$' "${p}/desc" 2>/dev/null| tail -n1) + if [ "${dbpkgname}" = "${pkgname}" ]; then + continue 2 + fi + done + EXTRAFILES="$EXTRAFILES $pkg" + done + + rm -rf ${CLEANUP_TMPDIR} + + # Do a quick check to see if a missing ARCHINDEPFILE is in the any dir + # If it is, and the file is MISSING, restore it + missfiles="$MISSINGFILES" + MISSINGFILES="" + for mf in $missfiles; do + if [ -e "${ftppath_base}/any/${mf}" ]; then + echo "Restoring missing 'any' symlink: ${mf}" + ${CLEANUP_DRYRUN} || ln -s "../any/${mf}" "${ftppath}" + else + MISSINGFILES="${MISSINGFILES} ${mf}" + fi + done + + repo_unlock $reponame $arch + + #Make sure we've done *something* before outputting anything + if [ -z "$DELETEFILES$DELETESYMLINKS$MISSINGFILES$EXTRAFILES" ]; then + continue + fi + + echo "Scan complete for $reponame ($arch) at ${ftppath}" + + if [ -n "$DELETEFILES" ]; then + echo " The following files are out of date" + for f in $DELETEFILES; do + echo " $f" + done + echo "" + fi + + if [ -n "$DELETESYMLINKS" ]; then + echo " The following symlinks are out of date" + echo " They will be deleted" + for f in $DELETESYMLINKS; do + echo " $f" + done + echo "" + fi + + if [ -n "$MISSINGFILES" ]; then + echo " The following files are missing in the repo" + for f in $MISSINGFILES; do + echo " $f" + done + echo "" + fi + + if [ -n "$EXTRAFILES" ]; then + echo " The following files are in the repo but not the db" + for f in $EXTRAFILES; do + echo " $f" + done + fi + + if [ -n "${DELETEFILES}" ]; then + ${CLEANUP_DRYRUN} || clean_pkgs ${DELETEFILES} + echo "" + fi + + if [ -n "${DELETESYMLINKS}" ]; then + ${CLEANUP_DRYRUN} || clean_pkgs ${DELETESYMLINKS} + echo "" + fi + + if [ -n "${EXTRAFILES}" ]; then + ${CLEANUP_DRYRUN} || clean_pkgs ${EXTRAFILES} + echo "" + fi + +done + +ARCHINDEPFILES="" + +if [ -d "$ftppath_base/any" ]; then + cd "$ftppath_base/any" + for pkg in *$PKGEXT; do + [ -f "$pkg" ] || continue # in case we get a file named "*.pkg.tar.gz" + found=0 + #check for any existing symlinks + for arch in ${ARCHES[@]}; do + if [ -h "$ftppath_base/$arch/$pkg" ]; then + found=1 + break + fi + done + if [ $found -eq 0 ]; then + # We found no symlinks to this, delete it + ARCHINDEPFILES="$ARCHINDEPFILES $pkg" + fi + done +fi + +if [ -n "$ARCHINDEPFILES" ]; then + echo " The following architecture independent packages" + echo " are not symlinked in the architecture repositories." + for f in $ARCHINDEPFILES; do + echo " $f" + done +fi + +if [ -d "$ftppath_base/any" -a -n "${ARCHINDEPFILES}" ]; then + cd "$ftppath_base/any" + ${CLEANUP_DRYRUN} || clean_pkgs ${ARCHINDEPFILES} + echo "" +fi diff --git a/misc-scripts/make-sourceball b/misc-scripts/make-sourceball index e216356..6e57385 100755 --- a/misc-scripts/make-sourceball +++ b/misc-scripts/make-sourceball @@ -22,21 +22,7 @@ _arch="$3" srcpath="$FTP_BASE/sources/" logpath="/var/log/sourceballs/" -cleanup() { - restore_umask - rm -rf "$WORKDIR" - [ "$1" ] && exit $1 -} - -ctrl_c() { - echo "Interrupted" >&2 - cleanup 0 -} - -die() { - echo -e "$*" >&2 - cleanup 1 -} +script_lock create_srcpackage() { if [ -d "$1" ]; then @@ -71,9 +57,6 @@ create_srcpackage() { fi } -trap ctrl_c 2 -trap cleanup 0 1 - set_umask /bin/mkdir -p "$logpath" cd "$WORKDIR" @@ -91,3 +74,5 @@ else die "\tPackage '$packagename' does not exist in repo '$reponame-$_arch'" fi fi + +script_unlock diff --git a/misc-scripts/sourceballs-cleanup b/misc-scripts/sourceballs-cleanup index c87a888..e058c38 100755 --- a/misc-scripts/sourceballs-cleanup +++ b/misc-scripts/sourceballs-cleanup @@ -6,19 +6,7 @@ srcpath="$FTP_BASE/sources/" logfile="$srcpath/cleanup.txt" -LOCKFILE="/tmp/.sourceball-cleanup.lock" - -cleanup () { - restore_umask - rm -rf "$WORKDIR" - rm -f "$LOCKFILE" - exit 0 -} - -ctrl_c() { - echo "Interrupted" >&2 - cleanup 0 -} +script_lock remove_old() { if [ -d "$1" ]; then @@ -52,17 +40,6 @@ remove_old() { fi } -if [ -f "$LOCKFILE" ]; then - owner="$(/usr/bin/stat -c %U $LOCKFILE)" - echo "error: source tarball generation is already in progress (started by $owner)" - exit 1 -fi - -trap cleanup 0 -trap ctrl_c 2 - -/bin/touch "$LOCKFILE" - #adjust the nice level to run at a lower priority /usr/bin/renice +10 -p $$ > /dev/null @@ -91,4 +68,4 @@ for sourceball in "$srcpath"/*$SRCEXT; do fi done -cleanup 0 +script_unlock diff --git a/test/lib/common.inc b/test/lib/common.inc index d28017d..ba5be9f 100644 --- a/test/lib/common.inc +++ b/test/lib/common.inc @@ -7,8 +7,8 @@ oneTimeSetUp() { echo -n 'Building packages...' for p in "${pkgdir}"/*; do pushd $p >/dev/null - linux32 makepkg -cf --config ${curdir}/lib/makepkg-i686.conf >/dev/null 2>&1 || exit 1 - [ -f *-any.pkg.tar.* ] || linux64 makepkg -cf --config ${curdir}/lib/makepkg-x86_64.conf >/dev/null 2>&1 || exit 1 + linux32 makepkg -cf --config ${curdir}/lib/makepkg-i686.conf >/dev/null 2>&1 || die 'makepkg failed' + [ -f *-any.pkg.tar.* ] || linux64 makepkg -cf --config ${curdir}/lib/makepkg-x86_64.conf >/dev/null 2>&1 || die 'makepkg failed' popd >/dev/null done echo 'done' @@ -23,7 +23,7 @@ setUp() { local p local pkg - [ -f "${curdir}/../config.local" ] && exit 1 + [ -f "${curdir}/../config.local" ] && die "${curdir}/../config.local exists" TMP="$(mktemp -d /dev/shm/$(basename $0).XXXXXXXXXX)" #echo "Using ${TMP}" diff --git a/testing2x b/testing2x index e9fbeaa..bf48be6 100755 --- a/testing2x +++ b/testing2x @@ -12,20 +12,6 @@ case "$0" in ;; esac -cleanup() { - trap '' 0 2 - rm -rf "${WORKDIR}" - [ -n "$1" ] && exit $1 -} - -ctrl_c() { - echo "Interrupted" >&2 - cleanup 0 -} - -trap ctrl_c 2 -trap cleanup 0 - cd "${WORKDIR}" /usr/bin/svn checkout -N $SVNREPO checkout cd checkout @@ -49,5 +35,3 @@ for pkg in $*; do echo "===> Warning: ${pkg} is not in testing-${_arch}" fi done - -cleanup -- cgit v1.2.2 From 6326b0e86214a37d9f9be62bc84a79b015ff3633 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 8 Aug 2010 15:11:14 +0200 Subject: Cleanup cron-jobs/adjust-permissions --- cron-jobs/adjust-permissions | 57 ++++++++++++++++++++------------------------ 1 file changed, 26 insertions(+), 31 deletions(-) diff --git a/cron-jobs/adjust-permissions b/cron-jobs/adjust-permissions index c92cb9e..c21af0c 100755 --- a/cron-jobs/adjust-permissions +++ b/cron-jobs/adjust-permissions @@ -1,49 +1,44 @@ #!/bin/bash -if [ -f /tmp/.ftpmaint.lck ]; then - exit 0 -fi - -/bin/touch /tmp/.ftpmaint.lck +script_lock . "$(dirname $0)/../db-functions" . "$(dirname $0)/../config" get_dir_owner() { - case $1 in - core) - echo "ftp:ftp-arch" ;; - extra) - echo "ftp:ftp-extra" ;; - testing) - echo "ftp:ftp-extra" ;; - community) - echo "root:tusers" ;; - community-testing) - echo "root:tusers" ;; - packages/arch) - echo "ftp:ftp-extra" ;; - packages/community) - echo "root:tusers" ;; - esac + case $1 in + core) + echo "ftp:ftp-arch" ;; + extra) + echo "ftp:ftp-extra" ;; + testing) + echo "ftp:ftp-extra" ;; + community) + echo "root:tusers" ;; + community-testing) + echo "root:tusers" ;; + packages/arch) + echo "ftp:ftp-extra" ;; + packages/community) + echo "root:tusers" ;; + esac } #adjust the nice level to run at a lower priority /usr/bin/renice +10 -p $$ > /dev/null -cd $FTP_BASE +pushd $FTP_BASE >/dev/null for d in $(get_repos_for_host); do - owner="$(get_dir_owner $d)" - /bin/chown -R $owner $d/os/{any,i686,x86_64} - /bin/chmod -R g+w $d/os/{any,i686,x86_64} + owner="$(get_dir_owner $d)" + /bin/chown -R $owner $d/os/{any,i686,x86_64} + /bin/chmod -R g+w $d/os/{any,i686,x86_64} done - for p in $(get_pkgpool_for_host); do - owner="$(get_dir_owner $p)" - /bin/chown -R $owner $p/{any,i686,x86_64} - /bin/chmod -R g+w $p/{any,i686,x86_64} + owner="$(get_dir_owner $p)" + /bin/chown -R $owner $p/{any,i686,x86_64} + /bin/chmod -R g+w $p/{any,i686,x86_64} done - /bin/chmod 555 $FTP_BASE +popd >/dev/null -rm -f /tmp/.ftpmaint.lck +script_unlock -- cgit v1.2.2 From 8a377f398ef5a01f8b52695597152c49d7a64f96 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 8 Aug 2010 15:16:55 +0200 Subject: Cleanup db-functions * fix indention * use consistent output * set vars local --- db-functions | 115 +++++++++++++++++++++++++++++++---------------------------- 1 file changed, 60 insertions(+), 55 deletions(-) diff --git a/db-functions b/db-functions index 36ffb07..e3f3504 100644 --- a/db-functions +++ b/db-functions @@ -77,7 +77,8 @@ trap abort INT QUIT TERM trap cleanup EXIT -repo_lock () { #repo_lock [timeout] +#repo_lock [timeout] +repo_lock () { local LOCKDIR="$TMPDIR/.repolock.$1.$2" local _count local _trial @@ -86,36 +87,36 @@ repo_lock () { #repo_lock [timeout] local _owner if [ $# -eq 2 ]; then - _lockblock=true - _trial=0 + _lockblock=true + _trial=0 elif [ $# -eq 3 ]; then - _lockblock=false - _timeout=$3 - let _trial=$_timeout/$LOCK_DELAY + _lockblock=false + _timeout=$3 + let _trial=$_timeout/$LOCK_DELAY fi _count=0 while [ $_count -le $_trial ] || $_lockblock ; do - if ! mkdir "$LOCKDIR" >/dev/null 2>&1 ; then - _owner="$(/usr/bin/stat -c %U $LOCKDIR)" - echo "warning: Repo $1-$2 is already locked by $_owner. Retrying in $LOCK_DELAY seconds..." >&2 - else - LOCKS[${#LOCKS[*]}]="$1.$2" - set_umask - return 0 - fi - sleep $LOCK_DELAY - let _count=$_count+1 + if ! mkdir "$LOCKDIR" >/dev/null 2>&1 ; then + _owner="$(/usr/bin/stat -c %U $LOCKDIR)" + echo "Warning: Repo $1-$2 is already locked by $_owner. Retrying in $LOCK_DELAY seconds..." >&2 + else + LOCKS[${#LOCKS[*]}]="$1.$2" + set_umask + return 0 + fi + sleep $LOCK_DELAY + let _count=$_count+1 done - echo "error: Repo $1-$2 is already locked by $_owner. Giving up!" >&2 + echo "Error: Repo $1-$2 is already locked by $_owner. Giving up!" >&2 return 1 } repo_unlock () { #repo_unlock local LOCKDIR="$TMPDIR/.repolock.$1.$2" if [ ! -d "$LOCKDIR" ]; then - echo "warning: Repo lock $1-$2 was not locked!" >&2 + echo "Warning: Repo lock $1-$2 was not locked!" >&2 restore_umask return 1 else @@ -152,7 +153,7 @@ getpkgname() { _name="$(_grep_pkginfo "$1" "^pkgname")" if [ -z "$_name" ]; then - echo "ERROR: Package '$1' has no pkgname in the PKGINFO. Fail!" >&2 + echo "Error: Package '$1' has no pkgname in the PKGINFO. Fail!" >&2 exit 1 fi @@ -165,7 +166,7 @@ getpkgver() { _ver="$(_grep_pkginfo "$1" "^pkgver")" if [ -z "$_ver" ]; then - echo "ERROR: Package '$1' has no pkgver in the PKGINFO. Fail!" >&2 + echo "Error: Package '$1' has no pkgver in the PKGINFO. Fail!" >&2 exit 1 fi @@ -174,10 +175,10 @@ getpkgver() { getpkgfile() { if [[ ${#} -ne 1 ]]; then - echo 'ERROR: No canonical package found!' >&2 + echo 'Error: No canonical package found!' >&2 exit 1 elif [ ! -f "${1}" ]; then - echo "ERROR: Package ${1} not found!" >&2 + echo "Error: Package ${1} not found!" >&2 exit 1 fi @@ -185,14 +186,15 @@ getpkgfile() { } getpkgfiles() { + local f if [ ! -z "$(echo ${@%\.*} | sed "s/ /\n/g" | sort | uniq -D)" ]; then - echo 'ERROR: Duplicate packages found!'>&2 + echo 'Error: Duplicate packages found!'>&2 exit 1 fi for f in ${@}; do if [ ! -f "${f}" ]; then - echo "ERROR: Package ${f} not found!" >&2 + echo "Error: Package ${f} not found!" >&2 exit 1 fi done @@ -200,12 +202,13 @@ getpkgfiles() { echo ${@} } -check_pkg_arch () { #check_pkg_arch pkgfile arch +#check_pkg_arch pkgfile arch +check_pkg_arch () { local _arch _arch="$(_grep_pkginfo "$1" "^arch")" if [ -z "$_arch" ]; then - echo "ERROR: Package '$1' has no arch in the PKGINFO. Fail!" >&2 + echo "Error: Package '$1' has no arch in the PKGINFO. Fail!" >&2 return 1 fi if [ "$_arch" = "$2" ]; then @@ -233,39 +236,41 @@ get_pkgpool_for_host() { #usage: chk_license ${license[@]}" chk_license() { - local l - for l in "$@"; do - l="$(echo $l | tr '[:upper:]' '[:lower:]')" - for allowed in ${ALLOWED_LICENSES[@]}; do - allowed="$(echo $allowed | tr '[:upper:]' '[:lower:]')" - if [ "$l" = "$allowed" ]; then - return 0 - fi - done - done - - return 1 + local l + local allowed + for l in "$@"; do + l="$(echo $l | tr '[:upper:]' '[:lower:]')" + for allowed in ${ALLOWED_LICENSES[@]}; do + allowed="$(echo $allowed | tr '[:upper:]' '[:lower:]')" + if [ "$l" = "$allowed" ]; then + return 0 + fi + done + done + + return 1 } pkgname_from_src() { - local tmp - tmp=${1##*/} - tmp=${tmp%$SRCEXT} - for a in ${ARCHES[@]}; do - tmp=${tmp%-$a} - done - tmp=${tmp%-any} - echo ${tmp%-*-*} + local tmp + local a + tmp=${1##*/} + tmp=${tmp%$SRCEXT} + for a in ${ARCHES[@]}; do + tmp=${tmp%-$a} + done + tmp=${tmp%-any} + echo ${tmp%-*-*} } pkgver_from_src() { - tmp=${1##*/} - tmp=${tmp%$SRCEXT} - for a in ${ARCHES[@]}; do - tmp=${tmp%-$a} - done - tmp=${tmp%-any} - echo $tmp | sed 's|.*-\(.*-.*\)$|\1|g' + local tmp + local a + tmp=${1##*/} + tmp=${tmp%$SRCEXT} + for a in ${ARCHES[@]}; do + tmp=${tmp%-$a} + done + tmp=${tmp%-any} + echo $tmp | sed 's|.*-\(.*-.*\)$|\1|g' } - -# vim: set ts=4 sw=4 noet ft=sh: -- cgit v1.2.2 From 830c4d84d13712974ffd392cdcc58b038c0dcfd9 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 8 Aug 2010 16:03:11 +0200 Subject: Source PKGBUILD in subshells This patch sources a PKGBUILD in a subshells instead of directly. This way we don't polute our scope or overwrite our vars which might lead to unexpected behavior. --- db-move | 48 +++++++++++++++++++++------------------- db-remove | 22 +++++++++--------- db-update | 42 ++++++++++++++++++----------------- misc-scripts/make-sourceball | 15 +++++++------ misc-scripts/sourceballs-cleanup | 3 ++- 5 files changed, 68 insertions(+), 62 deletions(-) diff --git a/db-move b/db-move index b846e79..bbe7ff7 100755 --- a/db-move +++ b/db-move @@ -11,20 +11,20 @@ fi packagebase="$1" repofrom="$2" repoto="$3" -_arch="$4" -if [ "${_arch}" == 'any' ]; then - _tarches=('i686' 'x86_64') +arch="$4" +if [ "${arch}" == 'any' ]; then + tarches=('i686' 'x86_64') else - _tarches=("${_arch}") + tarches=("${arch}") fi ftppath_from="$FTP_BASE/$repofrom/os/" ftppath_to="$FTP_BASE/$repoto/os/" -svnrepo_from="$repofrom-$_arch" -svnrepo_to="$repoto-$_arch" +svnrepo_from="$repofrom-$arch" +svnrepo_to="$repoto-$arch" -repo_lock $repoto $_arch || exit 1 -repo_lock $repofrom $_arch || exit 1 +repo_lock $repoto $arch || exit 1 +repo_lock $repofrom $arch || exit 1 cd "$WORKDIR" /usr/bin/svn checkout -q -N $SVNREPO checkout >/dev/null @@ -32,7 +32,9 @@ cd checkout /usr/bin/svn up -q $packagebase if [ -d "$packagebase/repos/$svnrepo_from" ]; then - . "$packagebase/repos/$svnrepo_from/PKGBUILD" + pkgname=($(. "$packagebase/repos/$svnrepo_from/PKGBUILD"; echo ${pkgname[@]})) + pkgver=$(. "$packagebase/repos/$svnrepo_from/PKGBUILD"; echo ${pkgver}) + pkgrel=$(. "$packagebase/repos/$svnrepo_from/PKGBUILD"; echo ${pkgrel}) echo -n "Moving $packagebase from $repofrom to $repoto..." if [ -d "$packagebase/repos/$svnrepo_to" ]; then @@ -41,28 +43,28 @@ if [ -d "$packagebase/repos/$svnrepo_from" ]; then fi /usr/bin/svn mv -q -r HEAD "$packagebase/repos/$svnrepo_from" "$packagebase/repos/$svnrepo_to" - /usr/bin/svn commit -q -m "$(basename $0): moved $packagebase from [$repofrom] to [$repoto] ($_arch)" + /usr/bin/svn commit -q -m "$(basename $0): moved $packagebase from [$repofrom] to [$repoto] ($arch)" - _pkgfiles='' + pkgfiles='' for i in ${pkgname[@]}; do - for _tarch in ${_tarches[@]}; do - _pkgpath=$(getpkgfile "$ftppath_from/${_tarch}/"$i-$pkgver-$pkgrel-$_arch$PKGEXT) - _pkgfile=$(basename "${_pkgpath}") + for tarch in ${tarches[@]}; do + pkgpath=$(getpkgfile "$ftppath_from/${tarch}/"$i-$pkgver-$pkgrel-$arch$PKGEXT) + pkgfile=$(basename "${pkgpath}") # copy package to pool if needed # TODO: can be removed once every package has been moved to the package pool - if [ ! -f $FTP_BASE/$(get_pkgpool_for_host)/${_arch}/$_pkgfile ]; then - cp $_pkgpath $FTP_BASE/$(get_pkgpool_for_host)/${_arch}/ + if [ ! -f $FTP_BASE/$(get_pkgpool_for_host)/${arch}/$pkgfile ]; then + cp $pkgpath $FTP_BASE/$(get_pkgpool_for_host)/${arch}/ fi - ln -s "../../../$(get_pkgpool_for_host)/${_arch}/${_pkgfile}" $ftppath_to/${_tarch}/ + ln -s "../../../$(get_pkgpool_for_host)/${arch}/${pkgfile}" $ftppath_to/${tarch}/ done - _pkgfiles="${_pkgfiles} $FTP_BASE/$(get_pkgpool_for_host)/${_arch}/${_pkgfile}" + pkgfiles="${pkgfiles} $FTP_BASE/$(get_pkgpool_for_host)/${arch}/${pkgfile}" done - for _tarch in ${_tarches[@]}; do - /usr/bin/repo-add -q "$ftppath_to/${_tarch}/$repoto$DBEXT" ${_pkgfiles} >/dev/null || die "Error in repo-add $_pkgfiles" - /usr/bin/repo-remove -q "$ftppath_from/${_tarch}/$repofrom$DBEXT" ${pkgname[@]} >/dev/null || die "Error in repo-remove ${pkgname[@]}" + for tarch in ${tarches[@]}; do + /usr/bin/repo-add -q "$ftppath_to/${tarch}/$repoto$DBEXT" ${pkgfiles} >/dev/null || die "Error in repo-add $pkgfiles" + /usr/bin/repo-remove -q "$ftppath_from/${tarch}/$repofrom$DBEXT" ${pkgname[@]} >/dev/null || die "Error in repo-remove ${pkgname[@]}" done echo 'done' @@ -70,5 +72,5 @@ else die "Error: $packagebase is not in repo $repofrom" fi -repo_unlock $repoto $_arch || exit 1 -repo_unlock $repofrom $_arch || exit 1 +repo_unlock $repoto $arch || exit 1 +repo_unlock $repofrom $arch || exit 1 diff --git a/db-remove b/db-remove index 5d61c97..5a55849 100755 --- a/db-remove +++ b/db-remove @@ -10,12 +10,12 @@ fi packagebase="$1" reponame="$2" -_arch="$3" +arch="$3" ftppath="$FTP_BASE/$reponame/os" -svnrepo="$reponame-$_arch" +svnrepo="$reponame-$arch" -repo_lock $reponame $_arch || exit 1 +repo_lock $reponame $arch || exit 1 echo -n "Removing $packagebase from $reponame..." cd "$WORKDIR" @@ -24,7 +24,7 @@ cd checkout /usr/bin/svn up -q $packagebase if [ -d "$packagebase/repos/$svnrepo" ]; then - . "$packagebase/repos/$svnrepo/PKGBUILD" + pkgname=($(. "$packagebase/repos/$svnrepo/PKGBUILD"; echo ${pkgname[@]})) /usr/bin/svn rm --force -q "$packagebase/repos/$svnrepo" /usr/bin/svn commit -q -m "$(basename $0): $packagebase removed by $(id -un)" else @@ -33,23 +33,23 @@ fi cd "$WORKDIR" -if [ "$_arch" == "any" ]; then +if [ "$arch" == "any" ]; then arches="i686 x86_64" else - arches="$_arch" + arches="$arch" fi # copy the db file into our working area -for arch in $arches; do - if [ ! -f "$ftppath/$arch/$reponame$DBEXT" ]; then - die "No database found at '$ftppath/$arch', nothing more to do" +for tarch in $arches; do + if [ ! -f "$ftppath/$tarch/$reponame$DBEXT" ]; then + die "No database found at '$ftppath/$tarch', nothing more to do" fi - /usr/bin/repo-remove -q "$ftppath/$arch/$reponame$DBEXT" ${pkgname[@]} >/dev/null + /usr/bin/repo-remove -q "$ftppath/$tarch/$reponame$DBEXT" ${pkgname[@]} >/dev/null done echo 'done' -repo_unlock $reponame $_arch || exit 1 +repo_unlock $reponame $arch || exit 1 # vim: set ts=4 sw=4 noet ft=sh: diff --git a/db-update b/db-update index f971a6f..375ed02 100755 --- a/db-update +++ b/db-update @@ -62,23 +62,24 @@ if [ -n "$ANYPKGS" ]; then cd checkout to_add_any="" for pkg in $ANYPKGS; do - _pkgfile=$(basename $pkg) - _pkgname="$(getpkgname $pkg)" - _pkgbase="$(getpkgbase $pkg)" + pkgfile=$(basename $pkg) + pkgname="$(getpkgname $pkg)" + pkgbase="$(getpkgbase $pkg)" svnrepo="$reponame-any" if ! check_pkg_arch "$pkg" "any"; then - echo " ERROR: $_pkgfile is not architecture independent!" + echo " ERROR: $pkgfile is not architecture independent!" else - /usr/bin/svn up -q $_pkgbase - if [ -d "$_pkgbase/repos/$svnrepo" ]; then - . "$_pkgbase/repos/$svnrepo/PKGBUILD" - if echo "$_pkgfile" | grep -q "$_pkgname-$pkgver-$pkgrel-any"; then + /usr/bin/svn up -q $pkgbase + if [ -d "$pkgbase/repos/$svnrepo" ]; then + pkgver=$(. "$pkgbase/repos/$svnrepo/PKGBUILD"; echo ${pkgver}) + pkgrel=$(. "$pkgbase/repos/$svnrepo/PKGBUILD"; echo ${pkgrel}) + if echo "$pkgfile" | grep -q "$pkgname-$pkgver-$pkgrel-any"; then to_add_any="$to_add_any $pkg" else - echo " WARNING: $_pkgfile does not match PKGBUILD in $svnrepo" + echo " WARNING: $pkgfile does not match PKGBUILD in $svnrepo" fi else - echo " WARNING: Package $_pkgbase not found in $svnrepo" + echo " WARNING: Package $pkgbase not found in $svnrepo" fi fi done @@ -132,23 +133,24 @@ for current_arch in ${ARCHES[@]}; do if [ -n "$ADDPKGS" ]; then for pkg in $ADDPKGS; do - _pkgfile=$(basename $pkg) - _pkgname="$(getpkgname $pkg)" - _pkgbase="$(getpkgbase $pkg)" + pkgfile=$(basename $pkg) + pkgname="$(getpkgname $pkg)" + pkgbase="$(getpkgbase $pkg)" if ! check_pkg_arch "$pkg" "$current_arch"; then - echo " ERROR: $_pkgfile was built for the wrong architecture" + echo " ERROR: $pkgfile was built for the wrong architecture" else - /usr/bin/svn up -q $_pkgbase - if [ -d "$_pkgbase/repos/$svnrepo" ]; then - . "$_pkgbase/repos/$svnrepo/PKGBUILD" - if echo "$_pkgfile" | grep -q "$_pkgname-$pkgver-$pkgrel-$current_arch"; then + /usr/bin/svn up -q $pkgbase + if [ -d "$pkgbase/repos/$svnrepo" ]; then + pkgver=$(. "$pkgbase/repos/$svnrepo/PKGBUILD"; echo ${pkgver}) + pkgrel=$(. "$pkgbase/repos/$svnrepo/PKGBUILD"; echo ${pkgrel}) + if echo "$pkgfile" | grep -q "$pkgname-$pkgver-$pkgrel-$current_arch"; then to_add="$to_add $pkg" else - echo " WARNING: $_pkgfile does not match PKGBUILD in $svnrepo" + echo " WARNING: $pkgfile does not match PKGBUILD in $svnrepo" fi else - echo " WARNING: Package $_pkgbase not found in $svnrepo" + echo " WARNING: Package $pkgbase not found in $svnrepo" fi fi done diff --git a/misc-scripts/make-sourceball b/misc-scripts/make-sourceball index 6e57385..8e26eec 100755 --- a/misc-scripts/make-sourceball +++ b/misc-scripts/make-sourceball @@ -17,7 +17,7 @@ fi packagename="$1" reponame="$2" -_arch="$3" +arch="$3" srcpath="$FTP_BASE/sources/" logpath="/var/log/sourceballs/" @@ -27,7 +27,8 @@ script_lock create_srcpackage() { if [ -d "$1" ]; then pushd "$1" >/dev/null - . "PKGBUILD" + pkgver=$(. PKGBUILD; echo ${pkgver}) + pkgrel=$(. PKGBUILD; echo ${pkgrel}) if ! [ $FORCE == 1 ] && ! chk_license ${license[@]} ; then #Removed so as not to clutter failed.txt #echo -e "\t$packagename license (${license[@]}) does not require source tarballs" >&2 @@ -40,7 +41,7 @@ create_srcpackage() { if ! /usr/bin/makepkg --allsource --ignorearch >"$logfile" 2>&1; then popd >/dev/null /bin/gzip -f -9 "$logfile" - die "\tFailed to download source for $packagename-$pkgver-$pkgrel ($reponame-$_arch)" + die "\tFailed to download source for $packagename-$pkgver-$pkgrel ($reponame-$arch)" fi /bin/rm -f "$logfile"{,.gz} @@ -63,15 +64,15 @@ cd "$WORKDIR" if [[ "$reponame" = "community" || "$reponame" = "community-testing" ]]; then if /usr/bin/svn export -q "$SVNREPOCOMMUNITY/$packagename" $packagename; then - create_srcpackage "$packagename/repos/$reponame-$_arch" + create_srcpackage "$packagename/repos/$reponame-$arch" else - die "\tPackage '$packagename' does not exist in repo '$reponame-$_arch'" + die "\tPackage '$packagename' does not exist in repo '$reponame-$arch'" fi else if /usr/bin/svn export -q "$SVNREPO/$packagename" $packagename; then - create_srcpackage "$packagename/repos/$reponame-$_arch" + create_srcpackage "$packagename/repos/$reponame-$arch" else - die "\tPackage '$packagename' does not exist in repo '$reponame-$_arch'" + die "\tPackage '$packagename' does not exist in repo '$reponame-$arch'" fi fi diff --git a/misc-scripts/sourceballs-cleanup b/misc-scripts/sourceballs-cleanup index e058c38..e407989 100755 --- a/misc-scripts/sourceballs-cleanup +++ b/misc-scripts/sourceballs-cleanup @@ -14,7 +14,8 @@ remove_old() { PKGVERS="" for repo in *; do cd "$repo" - . "PKGBUILD" + pkgver=$(. PKGBUILD; echo ${pkgver}) + pkgrel=$(. PKGBUILD; echo ${pkgrel}) PKGVERS="$PKGVERS $pkgver-$pkgrel" cd .. done -- cgit v1.2.2 From fdca04b3f5d8e27af4829416fee3843b0fe01e36 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 8 Aug 2010 16:43:16 +0200 Subject: Use db-functions in create-filelists and sourceballs --- cron-jobs/create-filelists | 50 ++++++++++++++++++---------------------------- cron-jobs/sourceballs | 33 ++++++------------------------ 2 files changed, 25 insertions(+), 58 deletions(-) diff --git a/cron-jobs/create-filelists b/cron-jobs/create-filelists index a25fd1a..8c354fc 100755 --- a/cron-jobs/create-filelists +++ b/cron-jobs/create-filelists @@ -1,44 +1,38 @@ #!/bin/bash -reposdir="/srv/ftp" -targetdir="/srv/ftp" -repos="core extra testing community community-testing" -lock="/tmp/create-filelists.lock" - . "$(dirname $0)/../db-functions" . "$(dirname $0)/../config" -if [ -f "$lock" ]; then - echo "Error: create-filelists already in progress." - exit 1 -fi +reposdir=${FTP_BASE} +targetdir=${FTP_BASE} +repos=(core extra testing community community-testing) + +script_lock -touch "$lock" || exit 1 # location where the package DB is extracted so we know what to include -DBDIR="$(mktemp -d /tmp/create-filelists.dbdir.XXXXXX)" || exit 1 +DBDIR="$(mktemp -d ${WORKDIR}/create-filelists.dbdir.XXXXXX)" || exit 1 # location where the old files DB is extracted to save us some work -CACHEDIR="$(mktemp -d /tmp/create-filelists.cachedir.XXXXXX)" || exit 1 +CACHEDIR="$(mktemp -d ${WORKDIR}/create-filelists.cachedir.XXXXXX)" || exit 1 # location where the new files DB is built up and eventually zipped -TMPDIR="$(mktemp -d /tmp/create-filelists.tmpdir.XXXXXX)" || exit 1 +MYTMPDIR="$(mktemp -d ${WORKDIR}/create-filelists.tmpdir.XXXXXX)" || exit 1 #adjust the nice level to run at a lower priority /usr/bin/renice +10 -p $$ > /dev/null case "${DBEXT}" in - *.gz) TAR_OPT="z" ;; - *.bz2) TAR_OPT="j" ;; - *.xz) TAR_OPT="J" ;; - *) echo "Unknown compression type for DBEXT=${DBEXT}" && exit 1 ;; + *.gz) TAR_OPT="z" ;; + *.bz2) TAR_OPT="j" ;; + *.xz) TAR_OPT="J" ;; + *) echo "Unknown compression type for DBEXT=${DBEXT}" && exit 1 ;; esac FILESEXT="${DBEXT//db/files}" -for repo in $repos; do +for repo in ${repos[@]}; do REPO_DB_FILE="${repo}$DBEXT" FILES_DB_FILE="${repo}$FILESEXT" for arch in ${ARCHES[@]}; do -# echo "Running for architecture $arch, repo $repo" - cd "$reposdir" + pushd "$reposdir" >/dev/null repodir="${repo}/os/${arch}" cached="no" @@ -46,7 +40,6 @@ for repo in $repos; do # extract package db archive if [ -f "${targetdir}/${repodir}/${REPO_DB_FILE}" ]; then mkdir -p "${DBDIR}/${repodir}" -# echo "extracting $REPO_DB_FILE" bsdtar -xf "${targetdir}/${repodir}/${REPO_DB_FILE}" -C "${DBDIR}/${repodir}" else echo "Fail! Does the repo $repo with arch $arch even exist?" @@ -56,7 +49,6 @@ for repo in $repos; do # extract old file archive if [ -f "${targetdir}/${repodir}/${FILES_DB_FILE}" ]; then mkdir -p "${CACHEDIR}/${repodir}" -# echo "extracting $FILES_DB_FILE" bsdtar -xf "${targetdir}/${repodir}/${FILES_DB_FILE}" -C "${CACHEDIR}/${repodir}" cached="yes" fi @@ -65,15 +57,13 @@ for repo in $repos; do for pkg in $(ls ${DBDIR}/${repodir}); do dbpkgdir="${DBDIR}/${repodir}/${pkg}" cachepkgdir="${CACHEDIR}/${repodir}/${pkg}" - tmppkgdir="${TMPDIR}/${repodir}/${pkg}" + tmppkgdir="${MYTMPDIR}/${repodir}/${pkg}" mkdir -p "$tmppkgdir" ln "${dbpkgdir}/desc" "${tmppkgdir}/desc" ln "${dbpkgdir}/depends" "${tmppkgdir}/depends" if [ -f "${cachepkgdir}/files" ]; then -# echo "cache: $pkg" ln "${cachepkgdir}/files" "${tmppkgdir}/files" else -# echo "not cache: $repo/$arch: $pkg" filename=$(grep -A1 '^%FILENAME%$' "${dbpkgdir}/desc" | tail -n1) echo '%FILES%' > "${tmppkgdir}/files" bsdtar --exclude=.* -tf "$repodir/$filename" >> "${tmppkgdir}/files" @@ -84,19 +74,17 @@ for repo in $repos; do # create new file archive if [ "$cached" == "no" ]; then # at least one package has changed, so let's rebuild the archive -# echo "creating ${FILES_DB_FILE}/${arch}" pkgdir="${targetdir}/${repodir}" mkdir -p "$pkgdir" - cd "${TMPDIR}/${repodir}" + pushd "${MYTMPDIR}/${repodir}" >/dev/null [ -f "${pkgdir}/${FILES_DB_FILE}" ] && rm "${pkgdir}/${FILES_DB_FILE}" bsdtar --exclude=*${DBEXT//\.db/} -c${TAR_OPT}f "${pkgdir}/${FILES_DB_FILE}" * + popd >/dev/null fi + popd >/dev/null done done -cd - >/dev/null -rm -rf "$TMPDIR" "$CACHEDIR" "$DBDIR" -rm -f "$lock" || exit 1 -# echo 'done' +script_unlock # vim: set ts=4 sw=4 et ft=sh: diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 93cb842..78c3959 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -1,37 +1,16 @@ #!/bin/bash -ftpbase="/srv/ftp" -srcbase="/srv/ftp/sources" -repos="core extra testing community community-testing" - . "$(dirname $0)/../db-functions" . "$(dirname $0)/../config" -LOCKFILE="/tmp/.sourceball.lock" - -cleanup () { - rm -f "$LOCKFILE" - restore_umask - exit 0 -} - -ctrl_c() { - cleanup -} +ftpbase="${FTP_BASE}" +srcbase="${FTP_BASE}/sources" +repos=(core extra testing community community-testing) -if [ -f "$LOCKFILE" ]; then - owner="$(/usr/bin/stat -c %U $LOCKFILE)" - echo "error: source tarball generation is already in progress (started by $owner)" - exit 1 -fi - -trap cleanup 0 -trap ctrl_c 2 +script_lock set_umask -/bin/touch "$LOCKFILE" - dirname="$(/bin/readlink -f $(/usr/bin/dirname $0))" FAILED_PKGS="" @@ -39,7 +18,7 @@ FAILED_PKGS="" [ -e "$srcbase/errors.txt" ] && /bin/mv "$srcbase/errors.txt" "$srcbase/errors.txt.old" echo "Errors occured during run:" > "$srcbase/errors.txt" -for repo in $repos; do +for repo in ${repos[@]}; do for arch in ${ARCHES[@]} any; do ftppath="$ftpbase/$repo/os/$arch" if [ ! -d "$ftppath" ]; then @@ -91,4 +70,4 @@ fi $dirname/../misc-scripts/sourceballs-cleanup -cleanup +script_unlock -- cgit v1.2.2 From 9c06372cf6caa2ba25e9ab1f32cbb02b2d280b13 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 8 Aug 2010 20:13:07 +0200 Subject: Fix typo --- db-functions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db-functions b/db-functions index e3f3504..9abb694 100644 --- a/db-functions +++ b/db-functions @@ -52,7 +52,7 @@ cleanup() { arch=${l#*.} if [ -d "$TMPDIR/.repolock.$repo.$arch" ]; then echo "Removing left over lock from $repo/$arch" >&2 - repo_unlock $repo $_arch + repo_unlock $repo $arch fi done if [ -d "$TMPDIR/.scriptlock.$(basename $0)" ]; then -- cgit v1.2.2 From af4f86808e8cd45cc171f55a1ec15bf30d858a0d Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Tue, 10 Aug 2010 21:40:24 +0200 Subject: Use more consitent naming for package pool MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There are no longer architecture-specific subdirs and the structure was switch to this: ftp └── pool ├── community └── packages packages contains all packages from core, extra and testing; this naming is in sync with the svn repo naming: svn-packages and svn-community --- cron-jobs/adjust-permissions | 8 ++++---- cron-jobs/ftpdir-cleanup | 20 +++++++++----------- db-functions | 4 ++-- db-move | 8 ++++---- db-update | 12 +++++------- test/lib/common.inc | 13 +++++++------ test/runTest | 10 +++++----- 7 files changed, 36 insertions(+), 39 deletions(-) diff --git a/cron-jobs/adjust-permissions b/cron-jobs/adjust-permissions index c21af0c..c28ca7c 100755 --- a/cron-jobs/adjust-permissions +++ b/cron-jobs/adjust-permissions @@ -17,9 +17,9 @@ get_dir_owner() { echo "root:tusers" ;; community-testing) echo "root:tusers" ;; - packages/arch) + pool/packages) echo "ftp:ftp-extra" ;; - packages/community) + pool/community) echo "root:tusers" ;; esac } @@ -35,8 +35,8 @@ for d in $(get_repos_for_host); do done for p in $(get_pkgpool_for_host); do owner="$(get_dir_owner $p)" - /bin/chown -R $owner $p/{any,i686,x86_64} - /bin/chmod -R g+w $p/{any,i686,x86_64} + /bin/chown $owner $p + /bin/chmod g+w $p done /bin/chmod 555 $FTP_BASE popd >/dev/null diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index f5e401a..7668ab5 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -17,18 +17,16 @@ for repo in $repos; do done to_cleanup="" -for _arch in any ${ARCHES[@]}; do - poolpath="$FTP_BASE/$(get_pkgpool_for_host)/$_arch/" - pushd $poolpath >/dev/null - for pkg in *$PKGEXT; do - [ -f "$pkg" ] || continue # in case we get a file named "*.pkg.tar.gz" - LINKS="$(/usr/bin/find $repopaths -type l -name "$pkg" 2>/dev/null)" - if [ -z "$LINKS" ]; then - to_cleanup="$to_cleanup $poolpath/$pkg" - fi - done - popd >/dev/null +poolpath="$FTP_BASE/$(get_pkgpool_for_host)/" +pushd $poolpath >/dev/null +for pkg in *$PKGEXT; do +[ -f "$pkg" ] || continue # in case we get a file named "*.pkg.tar.gz" +LINKS="$(/usr/bin/find $repopaths -type l -name "$pkg" 2>/dev/null)" +if [ -z "$LINKS" ]; then + to_cleanup="$to_cleanup $poolpath/$pkg" +fi done +popd >/dev/null if [ -n "$to_cleanup" ]; then echo " The following packages are no longer in any repo" diff --git a/db-functions b/db-functions index 9abb694..c4fc8f3 100644 --- a/db-functions +++ b/db-functions @@ -228,9 +228,9 @@ get_repos_for_host() { get_pkgpool_for_host() { if [ "$(hostname)" = "sigurd" ]; then - echo "packages/community" + echo "pool/community" else - echo "packages/arch" + echo "pool/packages" fi } diff --git a/db-move b/db-move index bbe7ff7..d18a4fa 100755 --- a/db-move +++ b/db-move @@ -53,13 +53,13 @@ if [ -d "$packagebase/repos/$svnrepo_from" ]; then # copy package to pool if needed # TODO: can be removed once every package has been moved to the package pool - if [ ! -f $FTP_BASE/$(get_pkgpool_for_host)/${arch}/$pkgfile ]; then - cp $pkgpath $FTP_BASE/$(get_pkgpool_for_host)/${arch}/ + if [ ! -f $FTP_BASE/$(get_pkgpool_for_host)/$pkgfile ]; then + cp $pkgpath $FTP_BASE/$(get_pkgpool_for_host) fi - ln -s "../../../$(get_pkgpool_for_host)/${arch}/${pkgfile}" $ftppath_to/${tarch}/ + ln -s "../../../$(get_pkgpool_for_host)/${pkgfile}" $ftppath_to/${tarch}/ done - pkgfiles="${pkgfiles} $FTP_BASE/$(get_pkgpool_for_host)/${arch}/${pkgfile}" + pkgfiles="${pkgfiles} $FTP_BASE/$(get_pkgpool_for_host)/${pkgfile}" done for tarch in ${tarches[@]}; do diff --git a/db-update b/db-update index 375ed02..46becac 100755 --- a/db-update +++ b/db-update @@ -89,11 +89,9 @@ for current_arch in ${ARCHES[@]}; do ftppath="$FTP_BASE/$reponame/os/$current_arch" ftppath_any="$FTP_BASE/$reponame/os/any" - poolpath="$FTP_BASE/$(get_pkgpool_for_host)/$current_arch" - poolpath_any="$FTP_BASE/$(get_pkgpool_for_host)/any" + poolpath="$FTP_BASE/$(get_pkgpool_for_host)" # The following is used to create relative symlinks - poolrel="../../../$(get_pkgpool_for_host)/$current_arch" - poolrel_any="../../../$(get_pkgpool_for_host)/any" + poolrel="../../../$(get_pkgpool_for_host)" if [ ! -d "$ftppath" ]; then echo "FTP path for this repo ($reponame) is missing" @@ -190,11 +188,11 @@ for current_arch in ${ARCHES[@]}; do for f in "$WORKDIR/build/"*-any$PKGEXT; do /bin/chmod 664 "$f" &>/dev/null fname="$(basename $f)" - if ! /bin/cp "$f" "$poolpath_any/"; then - die "error: failure while copying files to $poolpath_any" + if ! /bin/cp "$f" "$poolpath/"; then + die "error: failure while copying files to $poolpath" fi - if ! ln -s "$poolrel_any/$fname" "$ftppath/$fname"; then + if ! ln -s "$poolrel/$fname" "$ftppath/$fname"; then die "error: failure symlinking $fname to $ftppath" fi done diff --git a/test/lib/common.inc b/test/lib/common.inc index ba5be9f..89155c7 100644 --- a/test/lib/common.inc +++ b/test/lib/common.inc @@ -28,7 +28,8 @@ setUp() { #echo "Using ${TMP}" mkdir -p "${TMP}/"{ftp,tmp,staging,{package,source}-cleanup,svn-{packages,community}-{copy,repo}} - mkdir -p "${TMP}/ftp/"{{core,extra,community,testing,community-testing}/os,packages/{arch,community}}/{i686,any,x86_64} + mkdir -p "${TMP}/ftp/"{core,extra,community,testing,community-testing}/os/{i686,any,x86_64} + mkdir -p "${TMP}/ftp/pool/"{packages,community} mkdir -p "${TMP}/staging/"{core,extra,community,testing,community-testing} echo -n 'Creating svn repository...' @@ -81,11 +82,11 @@ checkAnyPackage() { local pkg=$2 local arch - [ -f "${FTP_BASE}/packages/arch/any/${pkg}" ] || fail "packages/arch/any/${pkg} not found" + [ -f "${FTP_BASE}/$(get_pkgpool_for_host)/${pkg}" ] || fail "$(get_pkgpool_for_host)/${pkg} not found" for arch in i686 x86_64; do [ -L "${FTP_BASE}/${repo}/os/${arch}/${pkg}" ] || fail "${repo}/os/${arch}/${pkg} not a symlink" - [ "$(readlink -e "${FTP_BASE}/${repo}/os/${arch}/${pkg}")" == "${FTP_BASE}/packages/arch/any/${pkg}" ] || fail "${repo}/os/${arch}/${pkg} does not link to packages/arch/any/${pkg}" + [ "$(readlink -e "${FTP_BASE}/${repo}/os/${arch}/${pkg}")" == "${FTP_BASE}/$(get_pkgpool_for_host)/${pkg}" ] || fail "${repo}/os/${arch}/${pkg} does not link to $(get_pkgpool_for_host)/${pkg}" done [ -f "${STAGING}"/${repo}/${pkg} ] && fail "${repo}/${pkg} found in staging dir" @@ -99,15 +100,15 @@ checkPackage() { local pkg=$2 local arch=$3 - [ -f "${FTP_BASE}/packages/arch/${arch}/${pkg}" ] || fail "packages/arch/${arch}/${pkg} not found" + [ -f "${FTP_BASE}/$(get_pkgpool_for_host)/${pkg}" ] || fail "$(get_pkgpool_for_host)/${pkg} not found" [ -L "${FTP_BASE}/${repo}/os/${arch}/${pkg}" ] || fail "${repo}/os/${arch}/${pkg} not a symlink" [ -f "${STAGING}"/${repo}/${pkg} ] && fail "${repo}/${pkg} found in staging dir" - [ "$(readlink -e "${FTP_BASE}/${repo}/os/${arch}/${pkg}")" == "${FTP_BASE}/packages/arch/${arch}/${pkg}" ] || fail "${repo}/os/${arch}/${pkg} does not link to packages/arch/${arch}/${pkg}" + [ "$(readlink -e "${FTP_BASE}/${repo}/os/${arch}/${pkg}")" == "${FTP_BASE}/$(get_pkgpool_for_host)/${pkg}" ] || fail "${repo}/os/${arch}/${pkg} does not link to $(get_pkgpool_for_host)/${pkg}" bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" -O | grep -q ${pkg} || fail "${pkg} not in ${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" - local pkgbase=$(getpkgbase "${FTP_BASE}/packages/arch/${arch}/${pkg}") + local pkgbase=$(getpkgbase "${FTP_BASE}/$(get_pkgpool_for_host)/${pkg}") svn up -q "${TMP}/svn-packages-copy/${pkgbase}" [ -d "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo}-${arch}" ] || fail "svn-packages-copy/${pkgbase}/repos/${repo}-${arch} does not exist" } diff --git a/test/runTest b/test/runTest index 5bbea05..75c7a05 100755 --- a/test/runTest +++ b/test/runTest @@ -240,7 +240,7 @@ testCleanupSimplePackages() { for arch in ${arches[@]}; do local pkg1="pkg-simple-a-1-1-${arch}.pkg.tar.xz" checkRemovedPackage extra 'pkg-simple-a' ${arch} - [ -f "${FTP_BASE}/packages/arch/${arch}/${pkg1}" ] && fail "packages/arch/${arch}/${pkg1} found" + [ -f "${FTP_BASE}/$(get_pkgpool_for_host)/${pkg1}" ] && fail "$(get_pkgpool_for_host)/${pkg1} found" [ -f "${FTP_BASE}/${repo}/os/${arch}/${pkg1}" ] && fail "${repo}/os/${arch}/${pkg1} found" local pkg2="pkg-simple-b-1-1-${arch}.pkg.tar.xz" @@ -263,7 +263,7 @@ testCleanupAnyPackages() { local pkg1='pkg-any-a-1-1-any.pkg.tar.xz' checkRemovedAnyPackage extra 'pkg-any-a' - [ -f "${FTP_BASE}/packages/arch/${arch}/${pkg1}" ] && fail "packages/arch/${arch}/${pkg1} found" + [ -f "${FTP_BASE}/$(get_pkgpool_for_host)/${pkg1}" ] && fail "$(get_pkgpool_for_host)/${pkg1} found" [ -f "${FTP_BASE}/${repo}/os/${arch}/${pkg1}" ] && fail "${repo}/os/${arch}/${pkg1} found" local pkg2="pkg-any-b-1-1-${arch}.pkg.tar.xz" @@ -294,7 +294,7 @@ testCleanupSplitPackages() { for arch in ${arches[@]}; do for pkg in "${pkgdir}/${pkgs[0]}"/*-${arch}.pkg.tar.*; do checkRemovedPackage extra ${pkgs[0]} ${arch} - [ -f "${FTP_BASE}/packages/arch/${arch}/${pkg}" ] && fail "packages/arch/${arch}/${pkg} found" + [ -f "${FTP_BASE}/$(get_pkgpool_for_host)/${pkg}" ] && fail "$(get_pkgpool_for_host)/${pkg} found" [ -f "${FTP_BASE}/${repo}/os/${arch}/${pkg}" ] && fail "${repo}/os/${arch}/${pkg} found" done @@ -329,7 +329,7 @@ testMovePackagesWithoutPool() { for old in 0 2; do for pkg in "${pkgdir}/${pkgs[${old}]}"/*-${arch}.pkg.tar.*; do pkg=$(basename $pkg) - mv -f "${FTP_BASE}/packages/arch/${arch}/${pkg}" "${FTP_BASE}/testing/os/${arch}/${pkg}" + mv -f "${FTP_BASE}/$(get_pkgpool_for_host)/${pkg}" "${FTP_BASE}/testing/os/${arch}/${pkg}" done done done @@ -369,7 +369,7 @@ testMoveAnyPackagesWithoutPool() { # transform a package to old style layout for pkg in "${pkgdir}/${pkgs[0]}"/*-any.pkg.tar.*; do pkg=$(basename $pkg) - mv -f "${FTP_BASE}/packages/arch/any/${pkg}" "${FTP_BASE}/testing/os/any/${pkg}" + mv -f "${FTP_BASE}/$(get_pkgpool_for_host)/${pkg}" "${FTP_BASE}/testing/os/any/${pkg}" for arch in i686 x86_64; do ln -sf "../any/${pkg}" "${FTP_BASE}/testing/os/${arch}/${pkg}" done -- cgit v1.2.2 From e2c005b490df6762e23da3223944151c17d1de80 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Fri, 13 Aug 2010 08:20:27 +0200 Subject: Check permission before any action Added a function to check if user has permission to alter the repos and db files. --- db-functions | 29 ++++++++++++++++++++++++++++- db-move | 5 +++++ db-remove | 5 +++++ db-update | 10 +--------- test/lib/common.inc | 49 +++++++++++++++++++++++++++++++++---------------- 5 files changed, 72 insertions(+), 26 deletions(-) diff --git a/db-functions b/db-functions index c4fc8f3..f22567b 100644 --- a/db-functions +++ b/db-functions @@ -73,7 +73,7 @@ die() { cleanup 1 } -trap abort INT QUIT TERM +trap abort INT QUIT TERM HUP trap cleanup EXIT @@ -274,3 +274,30 @@ pkgver_from_src() { tmp=${tmp%-any} echo $tmp | sed 's|.*-\(.*-.*\)$|\1|g' } + +check_repo_permission() { + local repo=$1 + + local repos="$(get_repos_for_host)" + local found=false + local r + for r in $repos; do + if [ "$r" = "$repo" ]; then + found=true + fi + done + [ $found ] || return 1 + + [ -w "$FTP_BASE/$(get_pkgpool_for_host)" ] || return 1 + + local arch + for arch in ${ARCHES} any; do + local w + local ws=("${FTP_BASE}/${repo}/os/${arch}/"{,${repo}${DBEXT}}) + for w in ws; do + [ -w ] || return 1 + done + done + + return 0 +} diff --git a/db-move b/db-move index d18a4fa..41b360d 100755 --- a/db-move +++ b/db-move @@ -23,6 +23,11 @@ ftppath_to="$FTP_BASE/$repoto/os/" svnrepo_from="$repofrom-$arch" svnrepo_to="$repoto-$arch" +if ! check_repo_permission $repoto || ! check_repo_permission $repofrom; then + echo "Error: You don't have permission to move packages from ${repofrom} to ${repoto}" + exit 1 +fi + repo_lock $repoto $arch || exit 1 repo_lock $repofrom $arch || exit 1 diff --git a/db-remove b/db-remove index 5a55849..1492d18 100755 --- a/db-remove +++ b/db-remove @@ -15,6 +15,11 @@ arch="$3" ftppath="$FTP_BASE/$reponame/os" svnrepo="$reponame-$arch" +if ! check_repo_permission $reponame; then + echo "Error: You don't have permission to remove packages from ${reponam}" + exit 1 +fi + repo_lock $reponame $arch || exit 1 echo -n "Removing $packagebase from $reponame..." diff --git a/db-update b/db-update index 46becac..c049714 100755 --- a/db-update +++ b/db-update @@ -11,15 +11,7 @@ fi reponame="$1" current_arch="" -# ensure we should be playing with this DB on this server -repos="$(get_repos_for_host)" -found=0 -for r in $repos; do - if [ "$r" = "$reponame" ]; then - found=1 - fi -done -if [ $found -ne 1 ]; then +if ! check_repo_permission "$reponame"; then echo "error: you shouldn't be updating $reponame on this server!" exit 1 fi diff --git a/test/lib/common.inc b/test/lib/common.inc index 89155c7..795d01a 100644 --- a/test/lib/common.inc +++ b/test/lib/common.inc @@ -1,3 +1,5 @@ +set -E + . "${curdir}/../db-functions" oneTimeSetUp() { @@ -8,7 +10,9 @@ oneTimeSetUp() { for p in "${pkgdir}"/*; do pushd $p >/dev/null linux32 makepkg -cf --config ${curdir}/lib/makepkg-i686.conf >/dev/null 2>&1 || die 'makepkg failed' - [ -f *-any.pkg.tar.* ] || linux64 makepkg -cf --config ${curdir}/lib/makepkg-x86_64.conf >/dev/null 2>&1 || die 'makepkg failed' + [ -f *-any.pkg.tar.* ] \ + || linux64 makepkg -cf --config ${curdir}/lib/makepkg-x86_64.conf >/dev/null 2>&1 \ + || die 'makepkg failed' popd >/dev/null done echo 'done' @@ -23,7 +27,7 @@ setUp() { local p local pkg - [ -f "${curdir}/../config.local" ] && die "${curdir}/../config.local exists" + #[ -f "${curdir}/../config.local" ] && die "${curdir}/../config.local exists" TMP="$(mktemp -d /dev/shm/$(basename $0).XXXXXXXXXX)" #echo "Using ${TMP}" @@ -82,17 +86,20 @@ checkAnyPackage() { local pkg=$2 local arch - [ -f "${FTP_BASE}/$(get_pkgpool_for_host)/${pkg}" ] || fail "$(get_pkgpool_for_host)/${pkg} not found" + [ -r "${FTP_BASE}/$(get_pkgpool_for_host)/${pkg}" ] || fail "$(get_pkgpool_for_host)/${pkg} not found" for arch in i686 x86_64; do [ -L "${FTP_BASE}/${repo}/os/${arch}/${pkg}" ] || fail "${repo}/os/${arch}/${pkg} not a symlink" - [ "$(readlink -e "${FTP_BASE}/${repo}/os/${arch}/${pkg}")" == "${FTP_BASE}/$(get_pkgpool_for_host)/${pkg}" ] || fail "${repo}/os/${arch}/${pkg} does not link to $(get_pkgpool_for_host)/${pkg}" + [ "$(readlink -e "${FTP_BASE}/${repo}/os/${arch}/${pkg}")" == "${FTP_BASE}/$(get_pkgpool_for_host)/${pkg}" ] \ + || fail "${repo}/os/${arch}/${pkg} does not link to $(get_pkgpool_for_host)/${pkg}" done - [ -f "${STAGING}"/${repo}/${pkg} ] && fail "${repo}/${pkg} found in staging dir" + [ -r "${STAGING}"/${repo}/${pkg} ] && fail "${repo}/${pkg} found in staging dir" - bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" -O | grep -q ${pkg} || fail "${pkg} not in ${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" + ( [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" ] \ + && bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" -O | grep -q ${pkg}) \ + || fail "${pkg} not in ${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" - [ -f "${FTP_BASE}/${repo}/os/any/${pkg}" ] && fail "${repo}/os/any/${pkg} should not exist" + [ -r "${FTP_BASE}/${repo}/os/any/${pkg}" ] && fail "${repo}/os/any/${pkg} should not exist" } checkPackage() { @@ -100,17 +107,21 @@ checkPackage() { local pkg=$2 local arch=$3 - [ -f "${FTP_BASE}/$(get_pkgpool_for_host)/${pkg}" ] || fail "$(get_pkgpool_for_host)/${pkg} not found" + [ -r "${FTP_BASE}/$(get_pkgpool_for_host)/${pkg}" ] || fail "$(get_pkgpool_for_host)/${pkg} not found" [ -L "${FTP_BASE}/${repo}/os/${arch}/${pkg}" ] || fail "${repo}/os/${arch}/${pkg} not a symlink" - [ -f "${STAGING}"/${repo}/${pkg} ] && fail "${repo}/${pkg} found in staging dir" + [ -r "${STAGING}"/${repo}/${pkg} ] && fail "${repo}/${pkg} found in staging dir" - [ "$(readlink -e "${FTP_BASE}/${repo}/os/${arch}/${pkg}")" == "${FTP_BASE}/$(get_pkgpool_for_host)/${pkg}" ] || fail "${repo}/os/${arch}/${pkg} does not link to $(get_pkgpool_for_host)/${pkg}" + [ "$(readlink -e "${FTP_BASE}/${repo}/os/${arch}/${pkg}")" == "${FTP_BASE}/$(get_pkgpool_for_host)/${pkg}" ] \ + || fail "${repo}/os/${arch}/${pkg} does not link to $(get_pkgpool_for_host)/${pkg}" - bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" -O | grep -q ${pkg} || fail "${pkg} not in ${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" + ( [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" ] \ + && bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" -O | grep -q ${pkg}) \ + || fail "${pkg} not in ${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" local pkgbase=$(getpkgbase "${FTP_BASE}/$(get_pkgpool_for_host)/${pkg}") svn up -q "${TMP}/svn-packages-copy/${pkgbase}" - [ -d "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo}-${arch}" ] || fail "svn-packages-copy/${pkgbase}/repos/${repo}-${arch} does not exist" + [ -d "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo}-${arch}" ] \ + || fail "svn-packages-copy/${pkgbase}/repos/${repo}-${arch} does not exist" } checkRemovedPackage() { @@ -118,10 +129,13 @@ checkRemovedPackage() { local pkgbase=$2 local arch=$3 - bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" -O | grep -q ${pkgbase} && fail "${pkgbase} should not be in ${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" + ( [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" ] \ + && bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" -O | grep -q ${pkgbase}) \ + && fail "${pkgbase} should not be in ${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" svn up -q "${TMP}/svn-packages-copy/${pkgbase}" - [ -d "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo}-${arch}" ] && fail "svn-packages-copy/${pkgbase}/repos/${repo}-${arch} should not exist" + [ -d "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo}-${arch}" ] \ + && fail "svn-packages-copy/${pkgbase}/repos/${repo}-${arch} should not exist" } checkRemovedAnyPackage() { @@ -130,9 +144,12 @@ checkRemovedAnyPackage() { local arch for arch in i686 x86_64; do - bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" -O | grep -q ${pkgbase} && fail "${pkgbase} should not be in ${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" + ( [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" ] \ + && bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" -O | grep -q ${pkgbase}) \ + && fail "${pkgbase} should not be in ${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" done svn up -q "${TMP}/svn-packages-copy/${pkgbase}" - [ -d "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo}-any" ] && fail "svn-packages-copy/${pkgbase}/repos/${repo}-any should not exist" + [ -d "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo}-any" ] \ + && fail "svn-packages-copy/${pkgbase}/repos/${repo}-any should not exist" } \ No newline at end of file -- cgit v1.2.2 From a422060414670bb49d2422a38467b73ae01e7ecb Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Fri, 13 Aug 2010 09:47:31 +0200 Subject: Use common functions to print messages, warnings and errors These functions are copied from makepkg --- cron-jobs/create-filelists | 4 +-- cron-jobs/ftpdir-cleanup | 8 ++--- cron-jobs/sourceballs | 2 +- db-functions | 70 +++++++++++++++++++++++++++--------- db-move | 10 +++--- db-remove | 10 +++--- db-update | 48 ++++++++++++------------- misc-scripts/ftpdir-cleanup-repo | 78 +++++++++++++++------------------------- misc-scripts/make-sourceball | 14 ++++---- test/lib/common.inc | 10 +++--- testing2x | 6 ++-- 11 files changed, 134 insertions(+), 126 deletions(-) diff --git a/cron-jobs/create-filelists b/cron-jobs/create-filelists index 8c354fc..9249408 100755 --- a/cron-jobs/create-filelists +++ b/cron-jobs/create-filelists @@ -23,7 +23,7 @@ case "${DBEXT}" in *.gz) TAR_OPT="z" ;; *.bz2) TAR_OPT="j" ;; *.xz) TAR_OPT="J" ;; - *) echo "Unknown compression type for DBEXT=${DBEXT}" && exit 1 ;; + *) die "Unknown compression type for DBEXT=${DBEXT}" ;; esac FILESEXT="${DBEXT//db/files}" @@ -42,7 +42,7 @@ for repo in ${repos[@]}; do mkdir -p "${DBDIR}/${repodir}" bsdtar -xf "${targetdir}/${repodir}/${REPO_DB_FILE}" -C "${DBDIR}/${repodir}" else - echo "Fail! Does the repo $repo with arch $arch even exist?" + error "Fail! Does the repo $repo with arch $arch even exist?" continue fi diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index 7668ab5..3456e9c 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -29,13 +29,11 @@ done popd >/dev/null if [ -n "$to_cleanup" ]; then - echo " The following packages are no longer in any repo" - echo " They will be moved to $CLEANUP_DESTDIR" + msg "The following packages are no longer in any repo" for f in $to_cleanup; do - echo " $(basename "$f")" + msg2 "$(basename "$f")" done - echo "" - mv $to_cleanup "$CLEANUP_DESTDIR" + ${CLEANUP_DRYRUN} || mv $to_cleanup "$CLEANUP_DESTDIR" fi script_unlock diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 78c3959..b82e375 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -22,7 +22,7 @@ for repo in ${repos[@]}; do for arch in ${ARCHES[@]} any; do ftppath="$ftpbase/$repo/os/$arch" if [ ! -d "$ftppath" ]; then - echo "FTP path does not exist: $ftppath" >2 + error "FTP path does not exist: $ftppath" continue fi cd $ftppath diff --git a/db-functions b/db-functions index f22567b..0c96913 100644 --- a/db-functions +++ b/db-functions @@ -16,11 +16,48 @@ restore_umask () { WORKDIR=$(mktemp -d /tmp/$(basename $0).XXXXXXXXXX) LOCKS=() +# check if messages are to be printed using color +unset ALL_OFF BOLD BLUE GREEN RED YELLOW +if [[ -t 2 ]]; then + ALL_OFF="$(tput sgr0)" + BOLD="$(tput bold)" + BLUE="${BOLD}$(tput setaf 4)" + GREEN="${BOLD}$(tput setaf 2)" + RED="${BOLD}$(tput setaf 1)" + YELLOW="${BOLD}$(tput setaf 3)" +fi +readonly ALL_OFF BOLD BLUE GREEN RED YELLOW + +plain() { + local mesg=$1; shift + printf "${BOLD} ${mesg}${ALL_OFF}\n" "$@" +} + +msg() { + local mesg=$1; shift + printf "${GREEN}==>${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" +} + +msg2() { + local mesg=$1; shift + printf "${BLUE} ->${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" +} + +warning() { + local mesg=$1; shift + printf "${YELLOW}==> WARNING:${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2 +} + +error() { + local mesg=$1; shift + printf "${RED}==> ERROR${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2 +} + script_lock() { local LOCKDIR="$TMPDIR/.scriptlock.$(basename $0)" if ! mkdir "$LOCKDIR" >/dev/null 2>&1 ; then local _owner="$(/usr/bin/stat -c %U $LOCKDIR)" - echo "Error: Script $(basename $0) is already locked by $_owner." >&2 + error "Script $(basename $0) is already locked by $_owner." exit 1 else set_umask @@ -31,7 +68,7 @@ script_lock() { script_unlock() { local LOCKDIR="$TMPDIR/.scriptlock.$(basename $0)" if [ ! -d "$LOCKDIR" ]; then - echo "Warning: Script $(basename $0) was not locked!" >&2 + warning "Script $(basename $0) was not locked!" restore_umask return 1 else @@ -51,12 +88,12 @@ cleanup() { repo=${l%.*} arch=${l#*.} if [ -d "$TMPDIR/.repolock.$repo.$arch" ]; then - echo "Removing left over lock from $repo/$arch" >&2 + msg "Removing left over lock from $repo/$arch" repo_unlock $repo $arch fi done if [ -d "$TMPDIR/.scriptlock.$(basename $0)" ]; then - echo "Removing left over lock from $(basename $0)" >&2 + msg "Removing left over lock from $(basename $0)" script_unlock fi rm -rf "$WORKDIR" @@ -64,12 +101,12 @@ cleanup() { } abort() { - echo 'Aborting...' >&2 + msg 'Aborting...' cleanup 0 } die() { - echo "$*" >&2 + error "$*" cleanup 1 } @@ -99,7 +136,8 @@ repo_lock () { while [ $_count -le $_trial ] || $_lockblock ; do if ! mkdir "$LOCKDIR" >/dev/null 2>&1 ; then _owner="$(/usr/bin/stat -c %U $LOCKDIR)" - echo "Warning: Repo $1-$2 is already locked by $_owner. Retrying in $LOCK_DELAY seconds..." >&2 + warning "Repo $1-$2 is already locked by $_owner. " \ + "Retrying in $LOCK_DELAY seconds..." else LOCKS[${#LOCKS[*]}]="$1.$2" set_umask @@ -109,14 +147,14 @@ repo_lock () { let _count=$_count+1 done - echo "Error: Repo $1-$2 is already locked by $_owner. Giving up!" >&2 + error "Repo $1-$2 is already locked by $_owner. Giving up!" return 1 } repo_unlock () { #repo_unlock local LOCKDIR="$TMPDIR/.repolock.$1.$2" if [ ! -d "$LOCKDIR" ]; then - echo "Warning: Repo lock $1-$2 was not locked!" >&2 + warning "Repo lock $1-$2 was not locked!" restore_umask return 1 else @@ -153,7 +191,7 @@ getpkgname() { _name="$(_grep_pkginfo "$1" "^pkgname")" if [ -z "$_name" ]; then - echo "Error: Package '$1' has no pkgname in the PKGINFO. Fail!" >&2 + error "Package '$1' has no pkgname in the PKGINFO. Fail!" exit 1 fi @@ -166,7 +204,7 @@ getpkgver() { _ver="$(_grep_pkginfo "$1" "^pkgver")" if [ -z "$_ver" ]; then - echo "Error: Package '$1' has no pkgver in the PKGINFO. Fail!" >&2 + error "Package '$1' has no pkgver in the PKGINFO. Fail!" exit 1 fi @@ -175,10 +213,10 @@ getpkgver() { getpkgfile() { if [[ ${#} -ne 1 ]]; then - echo 'Error: No canonical package found!' >&2 + error 'No canonical package found!' exit 1 elif [ ! -f "${1}" ]; then - echo "Error: Package ${1} not found!" >&2 + error "Package ${1} not found!" exit 1 fi @@ -188,13 +226,13 @@ getpkgfile() { getpkgfiles() { local f if [ ! -z "$(echo ${@%\.*} | sed "s/ /\n/g" | sort | uniq -D)" ]; then - echo 'Error: Duplicate packages found!'>&2 + error 'Duplicate packages found!' exit 1 fi for f in ${@}; do if [ ! -f "${f}" ]; then - echo "Error: Package ${f} not found!" >&2 + error "Package ${f} not found!" exit 1 fi done @@ -208,7 +246,7 @@ check_pkg_arch () { _arch="$(_grep_pkginfo "$1" "^arch")" if [ -z "$_arch" ]; then - echo "Error: Package '$1' has no arch in the PKGINFO. Fail!" >&2 + error "Package '$1' has no arch in the PKGINFO. Fail!" return 1 fi if [ "$_arch" = "$2" ]; then diff --git a/db-move b/db-move index 41b360d..207baec 100755 --- a/db-move +++ b/db-move @@ -1,7 +1,7 @@ #!/bin/bash if [ $# -ne 4 ]; then - echo "usage: $(basename $0) " + msg "usage: $(basename $0) " exit 1 fi @@ -24,7 +24,7 @@ svnrepo_from="$repofrom-$arch" svnrepo_to="$repoto-$arch" if ! check_repo_permission $repoto || ! check_repo_permission $repofrom; then - echo "Error: You don't have permission to move packages from ${repofrom} to ${repoto}" + error "You don't have permission to move packages from ${repofrom} to ${repoto}" exit 1 fi @@ -41,7 +41,7 @@ if [ -d "$packagebase/repos/$svnrepo_from" ]; then pkgver=$(. "$packagebase/repos/$svnrepo_from/PKGBUILD"; echo ${pkgver}) pkgrel=$(. "$packagebase/repos/$svnrepo_from/PKGBUILD"; echo ${pkgrel}) - echo -n "Moving $packagebase from $repofrom to $repoto..." + msg "Moving $packagebase from $repofrom to $repoto..." if [ -d "$packagebase/repos/$svnrepo_to" ]; then /usr/bin/svn rm --force -q "$packagebase/repos/$svnrepo_to" /usr/bin/svn commit -q -m "$(basename $0): $packagebase removed by $(id -un) for move to $repoto" @@ -71,10 +71,8 @@ if [ -d "$packagebase/repos/$svnrepo_from" ]; then /usr/bin/repo-add -q "$ftppath_to/${tarch}/$repoto$DBEXT" ${pkgfiles} >/dev/null || die "Error in repo-add $pkgfiles" /usr/bin/repo-remove -q "$ftppath_from/${tarch}/$repofrom$DBEXT" ${pkgname[@]} >/dev/null || die "Error in repo-remove ${pkgname[@]}" done - - echo 'done' else - die "Error: $packagebase is not in repo $repofrom" + die "$packagebase is not in repo $repofrom" fi repo_unlock $repoto $arch || exit 1 diff --git a/db-remove b/db-remove index 1492d18..2f05c16 100755 --- a/db-remove +++ b/db-remove @@ -1,7 +1,7 @@ #!/bin/bash if [ $# -ne 3 ]; then - echo "usage: $(basename $0) " + msg "usage: $(basename $0) " exit 1 fi @@ -16,13 +16,13 @@ ftppath="$FTP_BASE/$reponame/os" svnrepo="$reponame-$arch" if ! check_repo_permission $reponame; then - echo "Error: You don't have permission to remove packages from ${reponam}" + error "You don't have permission to remove packages from ${reponam}" exit 1 fi repo_lock $reponame $arch || exit 1 -echo -n "Removing $packagebase from $reponame..." +msg "Removing $packagebase from $reponame..." cd "$WORKDIR" /usr/bin/svn checkout -q -N $SVNREPO checkout >/dev/null cd checkout @@ -33,7 +33,7 @@ if [ -d "$packagebase/repos/$svnrepo" ]; then /usr/bin/svn rm --force -q "$packagebase/repos/$svnrepo" /usr/bin/svn commit -q -m "$(basename $0): $packagebase removed by $(id -un)" else - die "Error: $packagebase not found in $svnrepo" + die "$packagebase not found in $svnrepo" fi cd "$WORKDIR" @@ -53,8 +53,6 @@ for tarch in $arches; do /usr/bin/repo-remove -q "$ftppath/$tarch/$reponame$DBEXT" ${pkgname[@]} >/dev/null done -echo 'done' - repo_unlock $reponame $arch || exit 1 # vim: set ts=4 sw=4 noet ft=sh: diff --git a/db-update b/db-update index c049714..b56a70f 100755 --- a/db-update +++ b/db-update @@ -1,7 +1,7 @@ #!/bin/bash if [ $# -ne 1 ]; then - echo "usage: $(basename $0) " + msg "usage: $(basename $0) " exit 1 fi @@ -12,7 +12,7 @@ reponame="$1" current_arch="" if ! check_repo_permission "$reponame"; then - echo "error: you shouldn't be updating $reponame on this server!" + error "you shouldn't be updating $reponame on this server!" exit 1 fi @@ -21,11 +21,11 @@ ANYPKGS="" stagedir="$STAGING/$reponame" if [ ! -d $stagedir ]; then - echo "error: staging directory missing: $stagedir" >&2 + error "staging directory missing: $stagedir" exit 1 fi -echo -n "Updating $reponame..." +msg "Updating $reponame..." # Remove any package from $stagedir that is already in the FTP repository for current_arch in ${ARCHES[@]} any; do @@ -33,8 +33,8 @@ for current_arch in ${ARCHES[@]} any; do for f in $stagedir/*-$current_arch$PKGEXT; do bf=$(basename $f) if [[ -f $ftppath/$bf ]]; then - echo " WARNING: Package file $bf already exists in FTP repo" - echo " Removing from $stagedir" + warning " Package file $bf already exists in FTP repo" \ + " Removing from $stagedir" /bin/rm $f fi done @@ -59,7 +59,7 @@ if [ -n "$ANYPKGS" ]; then pkgbase="$(getpkgbase $pkg)" svnrepo="$reponame-any" if ! check_pkg_arch "$pkg" "any"; then - echo " ERROR: $pkgfile is not architecture independent!" + error "$pkgfile is not architecture independent!" else /usr/bin/svn up -q $pkgbase if [ -d "$pkgbase/repos/$svnrepo" ]; then @@ -68,10 +68,10 @@ if [ -n "$ANYPKGS" ]; then if echo "$pkgfile" | grep -q "$pkgname-$pkgver-$pkgrel-any"; then to_add_any="$to_add_any $pkg" else - echo " WARNING: $pkgfile does not match PKGBUILD in $svnrepo" + warning "$pkgfile does not match PKGBUILD in $svnrepo" fi else - echo " WARNING: Package $pkgbase not found in $svnrepo" + warning "Package $pkgbase not found in $svnrepo" fi fi done @@ -86,9 +86,9 @@ for current_arch in ${ARCHES[@]}; do poolrel="../../../$(get_pkgpool_for_host)" if [ ! -d "$ftppath" ]; then - echo "FTP path for this repo ($reponame) is missing" - echo " -> $ftppath" - echo "Please contact a system administrator" + error "FTP path for this repo ($reponame) is missing" \ + " -> $ftppath" \ + "Please contact a system administrator" exit 1 fi @@ -120,7 +120,7 @@ for current_arch in ${ARCHES[@]}; do cd "$WORKDIR" /usr/bin/svn checkout -N $SVNREPO checkout >/dev/null cd checkout - + if [ -n "$ADDPKGS" ]; then for pkg in $ADDPKGS; do pkgfile=$(basename $pkg) @@ -128,7 +128,7 @@ for current_arch in ${ARCHES[@]}; do pkgbase="$(getpkgbase $pkg)" if ! check_pkg_arch "$pkg" "$current_arch"; then - echo " ERROR: $pkgfile was built for the wrong architecture" + error "$pkgfile was built for the wrong architecture" else /usr/bin/svn up -q $pkgbase if [ -d "$pkgbase/repos/$svnrepo" ]; then @@ -137,10 +137,10 @@ for current_arch in ${ARCHES[@]}; do if echo "$pkgfile" | grep -q "$pkgname-$pkgver-$pkgrel-$current_arch"; then to_add="$to_add $pkg" else - echo " WARNING: $pkgfile does not match PKGBUILD in $svnrepo" + warning "$pkgfile does not match PKGBUILD in $svnrepo" fi else - echo " WARNING: Package $pkgbase not found in $svnrepo" + warning "Package $pkgbase not found in $svnrepo" fi fi done @@ -156,10 +156,10 @@ for current_arch in ${ARCHES[@]}; do /usr/bin/repo-add -q "$reponame-$current_arch$DBEXT" $pkgs >/dev/null else rm -f "build/$reponame-$current_arch$DBEXT" - echo "Errors found when adding packages" + error "Errors found when adding packages" fi else - echo "No packages to add" + warning "No packages to add" fi # if non empty, move all build dirs @@ -168,11 +168,11 @@ for current_arch in ${ARCHES[@]}; do for f in "$WORKDIR/build/"*-$current_arch$PKGEXT; do /bin/chmod 664 "$f" &>/dev/null if ! /bin/cp "$f" "$poolpath/"; then - die "error: failure while copying files to $poolpath" + die "failure while copying files to $poolpath" fi fname="$(basename $f)" if ! ln -s "$poolrel/$fname" "$ftppath/$fname"; then - die "error: failure symlinking $fname to $ftppath" + die "failure symlinking $fname to $ftppath" fi done fi @@ -181,11 +181,11 @@ for current_arch in ${ARCHES[@]}; do /bin/chmod 664 "$f" &>/dev/null fname="$(basename $f)" if ! /bin/cp "$f" "$poolpath/"; then - die "error: failure while copying files to $poolpath" + die "failure while copying files to $poolpath" fi if ! ln -s "$poolrel/$fname" "$ftppath/$fname"; then - die "error: failure symlinking $fname to $ftppath" + die "failure symlinking $fname to $ftppath" fi done fi @@ -194,7 +194,7 @@ for current_arch in ${ARCHES[@]}; do fi ln -sf "$reponame$DBEXT" "$ftppath/$reponame${DBEXT%.tar.*}" else - echo "Nothing to copy, no work done" + warning "Nothing to copy, no work done" fi if [ -n "$to_add" ]; then @@ -208,6 +208,4 @@ if [ -n "$to_add_any" ]; then /bin/rm $to_add_any fi -echo 'done' - # vim: set ts=4 sw=4 noet ft=sh: diff --git a/misc-scripts/ftpdir-cleanup-repo b/misc-scripts/ftpdir-cleanup-repo index 12f726c..20f297b 100755 --- a/misc-scripts/ftpdir-cleanup-repo +++ b/misc-scripts/ftpdir-cleanup-repo @@ -1,7 +1,7 @@ #!/bin/bash if [ $# -ne 1 ]; then - echo "usage: $(basename $0) " + msg "usage: $(basename $0) " exit 1 fi @@ -13,6 +13,7 @@ reponame=$1 . "$(dirname $0)/../config" clean_pkgs () { + if ! ${CLEANUP_DRYRUN}; then for pkg in "$@"; do if [ -h "$pkg" ]; then rm -f "$pkg" @@ -20,9 +21,10 @@ clean_pkgs () { mv "$pkg" "$CLEANUP_DESTDIR" fi done + fi } -${CLEANUP_DRYRUN} && echo 'dry run mode is active' +${CLEANUP_DRYRUN} && warning 'dry run mode is active' ftppath_base="$FTP_BASE/$reponame/os" @@ -38,26 +40,23 @@ for arch in ${ARCHES[@]}; do EXTRAFILES="" if [ ! -d "$ftppath" ]; then - echo "FTP path '$ftppath' does not exist" + error "FTP path '$ftppath' does not exist" exit 1 fi if ! cd "${CLEANUP_TMPDIR}" ; then - echo "Failed to cd to ${CLEANUP_TMPDIR}" + error "Failed to cd to ${CLEANUP_TMPDIR}" exit 1 fi if [ ! -f "$ftppath/$reponame$DBEXT" ]; then - echo "" - echo "WARNING: The file \"$ftppath/$reponame$DBEXT\" could not be found, skipping." - echo "" + msg "The file \"$ftppath/$reponame$DBEXT\" could not be found, skipping." repo_unlock $reponame $arch continue fi if ! bsdtar xf "$ftppath/$reponame$DBEXT"; then - echo "" - echo "ERROR: Command failed: bsdtar xf \"$ftppath/$reponame$DBEXT\"" + error "Command failed: bsdtar xf \"$ftppath/$reponame$DBEXT\"" exit 1 fi @@ -106,7 +105,7 @@ for arch in ${ARCHES[@]}; do MISSINGFILES="" for mf in $missfiles; do if [ -e "${ftppath_base}/any/${mf}" ]; then - echo "Restoring missing 'any' symlink: ${mf}" + msg "Restoring missing 'any' symlink: ${mf}" ${CLEANUP_DRYRUN} || ln -s "../any/${mf}" "${ftppath}" else MISSINGFILES="${MISSINGFILES} ${mf}" @@ -120,53 +119,36 @@ for arch in ${ARCHES[@]}; do continue fi - echo "Scan complete for $reponame ($arch) at ${ftppath}" - - if [ -n "$DELETEFILES" ]; then - echo " The following files are out of date" - for f in $DELETEFILES; do - echo " $f" - done - echo "" - fi - - if [ -n "$DELETESYMLINKS" ]; then - echo " The following symlinks are out of date" - echo " They will be deleted" - for f in $DELETESYMLINKS; do - echo " $f" - done - echo "" - fi + msg "Scan complete for $reponame ($arch) at ${ftppath}" if [ -n "$MISSINGFILES" ]; then - echo " The following files are missing in the repo" for f in $MISSINGFILES; do - echo " $f" - done - echo "" - fi - - if [ -n "$EXTRAFILES" ]; then - echo " The following files are in the repo but not the db" - for f in $EXTRAFILES; do - echo " $f" + error "$f is missing" done fi if [ -n "${DELETEFILES}" ]; then - ${CLEANUP_DRYRUN} || clean_pkgs ${DELETEFILES} - echo "" + msg "The following files are out of date" + for f in $DELETEFILES; do + msg2 "$f" + done + clean_pkgs ${DELETEFILES} fi if [ -n "${DELETESYMLINKS}" ]; then - ${CLEANUP_DRYRUN} || clean_pkgs ${DELETESYMLINKS} - echo "" + msg "The following symlinks are out of date" + for f in $DELETESYMLINKS; do + msg2 "$f" + done + clean_pkgs ${DELETESYMLINKS} fi if [ -n "${EXTRAFILES}" ]; then - ${CLEANUP_DRYRUN} || clean_pkgs ${EXTRAFILES} - echo "" + msg "The following files are in the repo but not the db" + for f in $EXTRAFILES; do + msg2 "$f" + done + clean_pkgs ${EXTRAFILES} fi done @@ -193,15 +175,13 @@ if [ -d "$ftppath_base/any" ]; then fi if [ -n "$ARCHINDEPFILES" ]; then - echo " The following architecture independent packages" - echo " are not symlinked in the architecture repositories." + msg "The following architecture independent packages are not symlinked in the architecture repositories." for f in $ARCHINDEPFILES; do - echo " $f" + msg2 "$f" done fi if [ -d "$ftppath_base/any" -a -n "${ARCHINDEPFILES}" ]; then cd "$ftppath_base/any" - ${CLEANUP_DRYRUN} || clean_pkgs ${ARCHINDEPFILES} - echo "" + clean_pkgs ${ARCHINDEPFILES} fi diff --git a/misc-scripts/make-sourceball b/misc-scripts/make-sourceball index 8e26eec..5bd155f 100755 --- a/misc-scripts/make-sourceball +++ b/misc-scripts/make-sourceball @@ -1,8 +1,8 @@ #!/bin/bash if [ $# -ne 3 -a $# -ne 4 ]; then - echo "usage: $(basename $0) [-f] " - echo " -f Force building. Skip license checks" + msg "usage: $(basename $0) [-f] " + msg " -f Force building. Skip license checks" exit 1 fi @@ -31,17 +31,17 @@ create_srcpackage() { pkgrel=$(. PKGBUILD; echo ${pkgrel}) if ! [ $FORCE == 1 ] && ! chk_license ${license[@]} ; then #Removed so as not to clutter failed.txt - #echo -e "\t$packagename license (${license[@]}) does not require source tarballs" >&2 + #warning "$packagename license (${license[@]}) does not require source tarballs" cleanup 0 else - echo "Creating source tarball for $packagename-$pkgver-$pkgrel" + msg "Creating source tarball for $packagename-$pkgver-$pkgrel" fi local logfile="$logpath/$packagename" if ! /usr/bin/makepkg --allsource --ignorearch >"$logfile" 2>&1; then popd >/dev/null /bin/gzip -f -9 "$logfile" - die "\tFailed to download source for $packagename-$pkgver-$pkgrel ($reponame-$arch)" + die "Failed to download source for $packagename-$pkgver-$pkgrel ($reponame-$arch)" fi /bin/rm -f "$logfile"{,.gz} @@ -66,13 +66,13 @@ if [[ "$reponame" = "community" || "$reponame" = "community-testing" ]]; then if /usr/bin/svn export -q "$SVNREPOCOMMUNITY/$packagename" $packagename; then create_srcpackage "$packagename/repos/$reponame-$arch" else - die "\tPackage '$packagename' does not exist in repo '$reponame-$arch'" + die "Package '$packagename' does not exist in repo '$reponame-$arch'" fi else if /usr/bin/svn export -q "$SVNREPO/$packagename" $packagename; then create_srcpackage "$packagename/repos/$reponame-$arch" else - die "\tPackage '$packagename' does not exist in repo '$reponame-$arch'" + die "Package '$packagename' does not exist in repo '$reponame-$arch'" fi fi diff --git a/test/lib/common.inc b/test/lib/common.inc index 795d01a..5c76eb5 100644 --- a/test/lib/common.inc +++ b/test/lib/common.inc @@ -6,7 +6,7 @@ oneTimeSetUp() { local p pkgdir="$(mktemp -d /dev/shm/$(basename $0).XXXXXXXXXX)" cp -r ${curdir}/packages/* "${pkgdir}" - echo -n 'Building packages...' + msg 'Building packages...' for p in "${pkgdir}"/*; do pushd $p >/dev/null linux32 makepkg -cf --config ${curdir}/lib/makepkg-i686.conf >/dev/null 2>&1 || die 'makepkg failed' @@ -15,7 +15,6 @@ oneTimeSetUp() { || die 'makepkg failed' popd >/dev/null done - echo 'done' echo } @@ -29,14 +28,14 @@ setUp() { #[ -f "${curdir}/../config.local" ] && die "${curdir}/../config.local exists" TMP="$(mktemp -d /dev/shm/$(basename $0).XXXXXXXXXX)" - #echo "Using ${TMP}" + #msg "Using ${TMP}" mkdir -p "${TMP}/"{ftp,tmp,staging,{package,source}-cleanup,svn-{packages,community}-{copy,repo}} mkdir -p "${TMP}/ftp/"{core,extra,community,testing,community-testing}/os/{i686,any,x86_64} mkdir -p "${TMP}/ftp/pool/"{packages,community} mkdir -p "${TMP}/staging/"{core,extra,community,testing,community-testing} - echo -n 'Creating svn repository...' + msg 'Creating svn repository...' svnadmin create "${TMP}/svn-packages-repo" svnadmin create "${TMP}/svn-community-repo" svn checkout -q "file://${TMP}/svn-packages-repo" "${TMP}/svn-packages-copy" @@ -49,7 +48,6 @@ setUp() { svn add -q "${TMP}/svn-packages-copy"/${pkg} svn commit -q -m"initial commit of ${pkg}" "${TMP}/svn-packages-copy" done - echo 'done' cat < "${curdir}/../config.local" FTP_BASE="${TMP}/ftp" @@ -152,4 +150,4 @@ checkRemovedAnyPackage() { svn up -q "${TMP}/svn-packages-copy/${pkgbase}" [ -d "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo}-any" ] \ && fail "svn-packages-copy/${pkgbase}/repos/${repo}-any should not exist" -} \ No newline at end of file +} diff --git a/testing2x b/testing2x index bf48be6..3626432 100755 --- a/testing2x +++ b/testing2x @@ -22,16 +22,16 @@ for pkg in $*; do if [ -f "${pkg}/repos/testing-${_arch}/PKGBUILD" ]; then for repo in core extra; do if [ -f "${pkg}/repos/${repo}-${_arch}/PKGBUILD" ]; then - echo "===> Moving package '${pkg}': testing-${_arch} -> ${repo}-${_arch}" + msg "Moving package '${pkg}': testing-${_arch} -> ${repo}-${_arch}" $(dirname $0)/db-move "${pkg}" "testing" "${repo}" "${_arch}" moved=1 break fi done if [ ${moved} -eq 0 ]; then - echo "===> Warning: ${pkg} is only in testing-${_arch}, cannot determine where to move it" + warning "${pkg} is only in testing-${_arch}, cannot determine where to move it" fi else - echo "===> Warning: ${pkg} is not in testing-${_arch}" + warning "${pkg} is not in testing-${_arch}" fi done -- cgit v1.2.2 From 53fd8f019acfb8c3fa994d964a92a48660c9f4ab Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Fri, 13 Aug 2010 11:14:02 +0200 Subject: Abort if package already exists in repo Don't try to be smart and remove packages from the staging dir without asking. --- db-update | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/db-update b/db-update index b56a70f..f70bdb9 100755 --- a/db-update +++ b/db-update @@ -27,15 +27,13 @@ fi msg "Updating $reponame..." -# Remove any package from $stagedir that is already in the FTP repository for current_arch in ${ARCHES[@]} any; do ftppath="$FTP_BASE/$reponame/os/$current_arch" for f in $stagedir/*-$current_arch$PKGEXT; do bf=$(basename $f) if [[ -f $ftppath/$bf ]]; then - warning " Package file $bf already exists in FTP repo" \ - " Removing from $stagedir" - /bin/rm $f + error "Package $bf already exists in $ftppath" + exit 1 fi done done -- cgit v1.2.2 From 480544ebe82512afc536a7aa50bb0449446f6a87 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Fri, 13 Aug 2010 11:18:50 +0200 Subject: Remove check which was already covered by check_repo_permission --- db-update | 7 ------- 1 file changed, 7 deletions(-) diff --git a/db-update b/db-update index f70bdb9..e4bf25d 100755 --- a/db-update +++ b/db-update @@ -83,13 +83,6 @@ for current_arch in ${ARCHES[@]}; do # The following is used to create relative symlinks poolrel="../../../$(get_pkgpool_for_host)" - if [ ! -d "$ftppath" ]; then - error "FTP path for this repo ($reponame) is missing" \ - " -> $ftppath" \ - "Please contact a system administrator" - exit 1 - fi - svnrepo="$reponame-$current_arch" repo_lock $reponame $current_arch || continue -- cgit v1.2.2 From 2f7415d9e0383dc9f86defdb9e41e397b0ae4cde Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Fri, 13 Aug 2010 11:30:06 +0200 Subject: removed useless statements --- db-update | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/db-update b/db-update index e4bf25d..3372dfb 100755 --- a/db-update +++ b/db-update @@ -39,15 +39,11 @@ for current_arch in ${ARCHES[@]} any; do done # Process architecture-independent packages first. -if [ -d "$stagedir" ]; then - ANYPKGS="$(getpkgfiles $stagedir/*-any$PKGEXT 2>/dev/null)" -fi +ANYPKGS="$(getpkgfiles $stagedir/*-any$PKGEXT 2>/dev/null)" cd "$WORKDIR" if [ -n "$ANYPKGS" ]; then - pkgtotal=$(echo "$ANYPKGS" | wc -w) - /usr/bin/svn checkout -q -N $SVNREPO checkout >/dev/null cd checkout to_add_any="" @@ -76,13 +72,11 @@ if [ -n "$ANYPKGS" ]; then fi for current_arch in ${ARCHES[@]}; do - ftppath="$FTP_BASE/$reponame/os/$current_arch" ftppath_any="$FTP_BASE/$reponame/os/any" poolpath="$FTP_BASE/$(get_pkgpool_for_host)" # The following is used to create relative symlinks poolrel="../../../$(get_pkgpool_for_host)" - svnrepo="$reponame-$current_arch" repo_lock $reponame $current_arch || continue @@ -106,8 +100,6 @@ for current_arch in ${ARCHES[@]}; do /bin/cp "$ftppath/$reponame$DBEXT" build/$reponame-$current_arch$DBEXT fi - pkgtotal=$(echo "$ADDPKGS $ANYPKGS" | wc -w) - cd "$WORKDIR" /usr/bin/svn checkout -N $SVNREPO checkout >/dev/null cd checkout -- cgit v1.2.2 From 27abfce26dc4f667fb4702db40acf27317ecfd9f Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Fri, 13 Aug 2010 11:38:31 +0200 Subject: Simplify check for existing packages --- db-update | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/db-update b/db-update index 3372dfb..78a08b5 100755 --- a/db-update +++ b/db-update @@ -9,7 +9,6 @@ fi . "$(dirname $0)/config" reponame="$1" -current_arch="" if ! check_repo_permission "$reponame"; then error "you shouldn't be updating $reponame on this server!" @@ -27,15 +26,11 @@ fi msg "Updating $reponame..." -for current_arch in ${ARCHES[@]} any; do - ftppath="$FTP_BASE/$reponame/os/$current_arch" - for f in $stagedir/*-$current_arch$PKGEXT; do - bf=$(basename $f) - if [[ -f $ftppath/$bf ]]; then - error "Package $bf already exists in $ftppath" - exit 1 - fi - done +for f in $stagedir/*$PKGEXT; do + if [ -f "$FTP_BASE/$(get_pkgpool_for_host)/$(basename f)" ]; then + error "Package $(basename f) already exists" + exit 1 + fi done # Process architecture-independent packages first. -- cgit v1.2.2 From cdf17a4c4c83c1363a59fa688654276c3d807eae Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sat, 14 Aug 2010 20:15:50 +0200 Subject: Rewrite of db-update * db-update now updates all repos with packages in its staging dirs * sanity checks are performed before any repo is touched * improved performance * less code; easier to maintain --- db-community | 3 - db-community-testing | 3 - db-core | 3 - db-extra | 3 - db-functions | 50 ++++++++++-- db-testing | 3 - db-update | 227 +++++++++++++-------------------------------------- test/runTest | 26 +++--- 8 files changed, 114 insertions(+), 204 deletions(-) delete mode 100755 db-community delete mode 100755 db-community-testing delete mode 100755 db-core delete mode 100755 db-extra delete mode 100755 db-testing diff --git a/db-community b/db-community deleted file mode 100755 index 44767a5..0000000 --- a/db-community +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -$(dirname $0)/db-update "community" diff --git a/db-community-testing b/db-community-testing deleted file mode 100755 index eb4cf67..0000000 --- a/db-community-testing +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -$(dirname $0)/db-update "community-testing" diff --git a/db-core b/db-core deleted file mode 100755 index eaa5032..0000000 --- a/db-core +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -$(dirname $0)/db-update "core" diff --git a/db-extra b/db-extra deleted file mode 100755 index d14b50e..0000000 --- a/db-extra +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -$(dirname $0)/db-update "extra" diff --git a/db-functions b/db-functions index 0c96913..16b5ad3 100644 --- a/db-functions +++ b/db-functions @@ -211,6 +211,18 @@ getpkgver() { echo "$_ver" } +getpkgarch() { + local _ver + + _ver="$(_grep_pkginfo "$1" "^arch")" + if [ -z "$_ver" ]; then + error "Package '$1' has no arch in the PKGINFO. Fail!" + exit 1 + fi + + echo "$_ver" +} + getpkgfile() { if [[ ${#} -ne 1 ]]; then error 'No canonical package found!' @@ -240,16 +252,40 @@ getpkgfiles() { echo ${@} } -#check_pkg_arch pkgfile arch -check_pkg_arch () { - local _arch - _arch="$(_grep_pkginfo "$1" "^arch")" +check_pkgfile() { + local pkgfile=$1 - if [ -z "$_arch" ]; then - error "Package '$1' has no arch in the PKGINFO. Fail!" + local pkgname="$(getpkgname ${pkgfile})" + [ $? -ge 1 ] && return 1 + local pkgver="$(getpkgver ${pkgfile})" + [ $? -ge 1 ] && return 1 + local pkgarch="$(getpkgarch ${pkgfile})" + [ $? -ge 1 ] && return 1 + + if echo "$(basename ${pkgfile})" | grep -q "${pkgname}-${pkgver}-${pkgarch}"; then + return 0 + else return 1 fi - if [ "$_arch" = "$2" ]; then +} + +check_pkgsvn() { + local pkgfile="${1}" + local pkgbase="$(getpkgbase $pkg)" + [ $? -ge 1 ] && return 1 + local pkgarch="$(getpkgarch ${pkgfile})" + [ $? -ge 1 ] && return 1 + local repo="${2}" + + if [ ! -f "${WORKDIR}/pkgbuilds/${pkgbase}" ]; then + mkdir -p "${WORKDIR}/pkgbuilds" + svn export -q "${SVNREPO}/${pkgbase}/repos/${repo}-${pkgarch}/PKGBUILD" \ + "${WORKDIR}/pkgbuilds/${pkgbase}" >/dev/null + [ $? -ge 1 ] && return 1 + fi + + local pkgver="$(. "${WORKDIR}/pkgbuilds/${pkgbase}"; echo "${pkgver}-${pkgrel}")" + if echo "$(basename ${pkgfile})" | grep -q "${pkgver}-${pkgarch}"; then return 0 else return 1 diff --git a/db-testing b/db-testing deleted file mode 100755 index 579ae37..0000000 --- a/db-testing +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -$(dirname $0)/db-update "testing" diff --git a/db-update b/db-update index 78a08b5..43c44f7 100755 --- a/db-update +++ b/db-update @@ -1,189 +1,78 @@ #!/bin/bash -if [ $# -ne 1 ]; then - msg "usage: $(basename $0) " - exit 1 -fi - . "$(dirname $0)/db-functions" . "$(dirname $0)/config" -reponame="$1" - -if ! check_repo_permission "$reponame"; then - error "you shouldn't be updating $reponame on this server!" +if [ $# -ge 1 ]; then + warning "Calling $(basename $0) with a specific repository is no longer supported" exit 1 fi -ADDPKGS="" -ANYPKGS="" - -stagedir="$STAGING/$reponame" -if [ ! -d $stagedir ]; then - error "staging directory missing: $stagedir" +# Find repos with packages to release +repos=($(find "${STAGING}" -mindepth 1 -type d ! -empty -printf '%f ' 2>/dev/null)) +if [ $? -ge 1 ]; then + error "Could not read ${STAGING}" exit 1 fi -msg "Updating $reponame..." - -for f in $stagedir/*$PKGEXT; do - if [ -f "$FTP_BASE/$(get_pkgpool_for_host)/$(basename f)" ]; then - error "Package $(basename f) already exists" +for repo in ${repos[@]}; do + if ! check_repo_permission "${repo}"; then + error "You don't have permission to update packages in ${repo}" exit 1 fi -done - -# Process architecture-independent packages first. -ANYPKGS="$(getpkgfiles $stagedir/*-any$PKGEXT 2>/dev/null)" - -cd "$WORKDIR" - -if [ -n "$ANYPKGS" ]; then - /usr/bin/svn checkout -q -N $SVNREPO checkout >/dev/null - cd checkout - to_add_any="" - for pkg in $ANYPKGS; do - pkgfile=$(basename $pkg) - pkgname="$(getpkgname $pkg)" - pkgbase="$(getpkgbase $pkg)" - svnrepo="$reponame-any" - if ! check_pkg_arch "$pkg" "any"; then - error "$pkgfile is not architecture independent!" - else - /usr/bin/svn up -q $pkgbase - if [ -d "$pkgbase/repos/$svnrepo" ]; then - pkgver=$(. "$pkgbase/repos/$svnrepo/PKGBUILD"; echo ${pkgver}) - pkgrel=$(. "$pkgbase/repos/$svnrepo/PKGBUILD"; echo ${pkgrel}) - if echo "$pkgfile" | grep -q "$pkgname-$pkgver-$pkgrel-any"; then - to_add_any="$to_add_any $pkg" - else - warning "$pkgfile does not match PKGBUILD in $svnrepo" - fi - else - warning "Package $pkgbase not found in $svnrepo" + pkgs=($(getpkgfiles "${STAGING}/${repo}/"*${PKGEXT})) + if [ $? -eq 0 ]; then + for pkg in ${pkgs[@]}; do + if ! check_pkgfile "${pkg}"; then + error "Package ${repo}/$(basename ${pkg}) is not consistent with its meta data" + exit 1 fi - fi - done -fi - -for current_arch in ${ARCHES[@]}; do - ftppath="$FTP_BASE/$reponame/os/$current_arch" - ftppath_any="$FTP_BASE/$reponame/os/any" - poolpath="$FTP_BASE/$(get_pkgpool_for_host)" - # The following is used to create relative symlinks - poolrel="../../../$(get_pkgpool_for_host)" - svnrepo="$reponame-$current_arch" - - repo_lock $reponame $current_arch || continue - - /bin/mkdir -p "$WORKDIR/build" - cd "$WORKDIR" - - # copy the db file into our working area - if [ -f "$ftppath/$reponame$DBEXT" ]; then - /bin/cp "$ftppath/$reponame$DBEXT" build/$reponame-$current_arch$DBEXT - fi - - to_add="" - if [ -d "$stagedir" ]; then - ADDPKGS="$(getpkgfiles $stagedir/*-${current_arch}$PKGEXT 2>/dev/null)" - fi - - if [ -n "$ADDPKGS" -o -n "$ANYPKGS" ]; then - - if [ -f "$ftppath/$reponame$DBEXT" ]; then - /bin/cp "$ftppath/$reponame$DBEXT" build/$reponame-$current_arch$DBEXT - fi - - cd "$WORKDIR" - /usr/bin/svn checkout -N $SVNREPO checkout >/dev/null - cd checkout - - if [ -n "$ADDPKGS" ]; then - for pkg in $ADDPKGS; do - pkgfile=$(basename $pkg) - pkgname="$(getpkgname $pkg)" - pkgbase="$(getpkgbase $pkg)" - - if ! check_pkg_arch "$pkg" "$current_arch"; then - error "$pkgfile was built for the wrong architecture" - else - /usr/bin/svn up -q $pkgbase - if [ -d "$pkgbase/repos/$svnrepo" ]; then - pkgver=$(. "$pkgbase/repos/$svnrepo/PKGBUILD"; echo ${pkgver}) - pkgrel=$(. "$pkgbase/repos/$svnrepo/PKGBUILD"; echo ${pkgrel}) - if echo "$pkgfile" | grep -q "$pkgname-$pkgver-$pkgrel-$current_arch"; then - to_add="$to_add $pkg" - else - warning "$pkgfile does not match PKGBUILD in $svnrepo" - fi - else - warning "Package $pkgbase not found in $svnrepo" - fi - fi - done - fi - - if [ -n "$to_add" -o -n "$to_add_any" ]; then - cd "$WORKDIR/build/" - for f in $to_add $to_add_any; do /bin/cp "$f" .; done - - pkgs="" - for pkg in $to_add $to_add_any; do pkgs="$pkgs $(basename $pkg)"; done - - /usr/bin/repo-add -q "$reponame-$current_arch$DBEXT" $pkgs >/dev/null - else - rm -f "build/$reponame-$current_arch$DBEXT" - error "Errors found when adding packages" - fi - else - warning "No packages to add" - fi - - # if non empty, move all build dirs - if [ $(/bin/ls "$WORKDIR/build/" 2>/dev/null | wc -l) != 0 ]; then - if [ $(getpkgfiles "$WORKDIR/build/"*-$current_arch$PKGEXT 2>/dev/null | wc -l) != 0 ]; then - for f in "$WORKDIR/build/"*-$current_arch$PKGEXT; do - /bin/chmod 664 "$f" &>/dev/null - if ! /bin/cp "$f" "$poolpath/"; then - die "failure while copying files to $poolpath" - fi - fname="$(basename $f)" - if ! ln -s "$poolrel/$fname" "$ftppath/$fname"; then - die "failure symlinking $fname to $ftppath" - fi - done - fi - if [ $(getpkgfiles "$WORKDIR/build/"*-any$PKGEXT 2>/dev/null | wc -l) != 0 ]; then - for f in "$WORKDIR/build/"*-any$PKGEXT; do - /bin/chmod 664 "$f" &>/dev/null - fname="$(basename $f)" - if ! /bin/cp "$f" "$poolpath/"; then - die "failure while copying files to $poolpath" - fi - - if ! ln -s "$poolrel/$fname" "$ftppath/$fname"; then - die "failure symlinking $fname to $ftppath" - fi - done - fi - if ! /bin/cp "$WORKDIR/build/$reponame-$current_arch$DBEXT" "$ftppath/$reponame$DBEXT"; then - die "failed to move repository $reponame-$current_arch". - fi - ln -sf "$reponame$DBEXT" "$ftppath/$reponame${DBEXT%.tar.*}" + if ! check_pkgsvn "${pkg}" "${repo}"; then + error "Package ${repo}/$(basename ${pkg}) is not consistent with svn repository" + exit 1 + fi + if [ -f "${FTP_BASE}/$(get_pkgpool_for_host)/$(basename ${pkg})" ]; then + error "Package ${repo}/$(basename pkg) already exists in ${FTP_BASE}/$(get_pkgpool_for_host)" + exit 1 + fi + done else - warning "Nothing to copy, no work done" - fi - - if [ -n "$to_add" ]; then - /bin/rm $to_add + exit 1 fi +done - repo_unlock $reponame $current_arch +# TODO: this might lock too much (architectures) +for repo in ${repos[@]}; do + for pkgarch in ${ARCHES[@]}; do + repo_lock ${repo} ${pkgarch} || exit 1 + done done -if [ -n "$to_add_any" ]; then - /bin/rm $to_add_any -fi +for repo in ${repos[@]}; do + any_pkgs=($(getpkgfiles "${STAGING}/${repo}/"*-any${PKGEXT} 2>/dev/null)) + for pkgarch in ${ARCHES[@]}; do + msg "Updating [${repo}] (${pkgarch})..." + add_pkgs=() + arch_pkgs=($(getpkgfiles "${STAGING}/${repo}/"*-${pkgarch}${PKGEXT} 2>/dev/null)) + for pkg in ${arch_pkgs[@]} ${any_pkgs[@]}; do + pkgfile="$(basename ${pkg})" + msg2 "${pkgfile}" + # any packages might have been moved by the previous run + if [ -f "${pkg}" ]; then + mv "${pkg}" "$FTP_BASE/$(get_pkgpool_for_host)" + fi + ln -s "../../../$(get_pkgpool_for_host)/${pkgfile}" "$FTP_BASE/$repo/os/${pkgarch}" + add_pkgs[${#add_pkgs[*]}]=${pkgfile} + done + pushd "$FTP_BASE/$repo/os/${pkgarch}" >/dev/null + /usr/bin/repo-add -q "$repo$DBEXT" ${add_pkgs[@]} >/dev/null \ + || die "Could not add packages" + popd >/dev/null + done +done -# vim: set ts=4 sw=4 noet ft=sh: +for repo in ${repos[@]}; do + for pkgarch in ${ARCHES[@]}; do + repo_unlock ${repo} ${pkgarch} + done +done diff --git a/test/runTest b/test/runTest index 75c7a05..06ca000 100755 --- a/test/runTest +++ b/test/runTest @@ -33,7 +33,7 @@ testAddSimplePackages() { done done - ../db-update extra + ../db-update for pkgbase in ${pkgs[@]}; do for arch in ${arches[@]}; do @@ -50,7 +50,7 @@ testAddAnyPackages() { releasePackage extra ${pkgbase} any done - ../db-update extra + ../db-update for pkgbase in ${pkgs[@]}; do checkAnyPackage extra ${pkgbase}-1-1-any.pkg.tar.xz @@ -70,7 +70,7 @@ testAddSplitPackages() { done done - ../db-update extra + ../db-update for pkgbase in ${pkgs[@]}; do for arch in ${arches[@]}; do @@ -98,7 +98,7 @@ testRemovePackages() { done done - ../db-update extra + ../db-update for pkgbase in ${pkgs[@]}; do for arch in ${arches[@]}; do @@ -121,7 +121,7 @@ testRemoveAnyPackages() { releasePackage extra ${pkgbase} any done - ../db-update extra + ../db-update for pkgbase in ${pkgs[@]}; do ../db-remove ${pkgbase} extra any @@ -149,7 +149,7 @@ testMoveSimplePackages() { done done - ../db-update testing + ../db-update for arch in ${arches[@]}; do ../db-move pkg-simple-a testing extra ${arch} @@ -171,7 +171,7 @@ testMoveAnyPackages() { releasePackage testing ${pkgbase} any done - ../db-update testing + ../db-update ../db-move pkg-any-a testing extra any checkAnyPackage extra pkg-any-a-1-1-any.pkg.tar.xz @@ -192,7 +192,7 @@ testMoveSplitPackages() { done done - ../db-update testing + ../db-update for arch in ${arches[@]}; do ../db-move pkg-split-a testing extra ${arch} @@ -229,7 +229,7 @@ testCleanupSimplePackages() { done done - ../db-update extra + ../db-update for arch in ${arches[@]}; do ../db-remove pkg-simple-a extra ${arch} @@ -257,7 +257,7 @@ testCleanupAnyPackages() { releasePackage extra ${pkgbase} any done - ../db-update extra + ../db-update ../db-remove pkg-any-a extra any ../cron-jobs/ftpdir-cleanup >/dev/null @@ -283,7 +283,7 @@ testCleanupSplitPackages() { done done - ../db-update extra + ../db-update for arch in ${arches[@]}; do ../db-remove ${pkgs[0]} extra ${arch} @@ -322,7 +322,7 @@ testMovePackagesWithoutPool() { done done - ../db-update testing + ../db-update # transform two packages to old style layout for arch in ${arches[@]}; do @@ -364,7 +364,7 @@ testMoveAnyPackagesWithoutPool() { releasePackage testing ${pkgbase} any done - ../db-update testing + ../db-update # transform a package to old style layout for pkg in "${pkgdir}/${pkgs[0]}"/*-any.pkg.tar.*; do -- cgit v1.2.2 From cfa19b31cdeb6698d69ae9a82904c09d3d65a5d7 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sat, 14 Aug 2010 20:20:11 +0200 Subject: Don't hardcode supported architectures in db-move --- db-move | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db-move b/db-move index 207baec..68cfc07 100755 --- a/db-move +++ b/db-move @@ -13,7 +13,7 @@ repofrom="$2" repoto="$3" arch="$4" if [ "${arch}" == 'any' ]; then - tarches=('i686' 'x86_64') + tarches=(${ARCHES[@]}) else tarches=("${arch}") fi -- cgit v1.2.2 From 7dd7a3a980b0d42295650fe5a4cf45b1eb07fd38 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sat, 14 Aug 2010 22:38:52 +0200 Subject: Add common function to check for correct repo and arch --- db-functions | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/db-functions b/db-functions index 16b5ad3..102f2c8 100644 --- a/db-functions +++ b/db-functions @@ -53,6 +53,21 @@ error() { printf "${RED}==> ERROR${ALL_OFF}${BOLD} ${mesg}${ALL_OFF}\n" "$@" >&2 } +## +# usage : in_array( $needle, $haystack ) +# return : 0 - found +# 1 - not found +## +in_array() { + local needle=$1; shift + [[ -z $1 ]] && return 1 # Not Found + local item + for item in "$@"; do + [[ $item = $needle ]] && return 0 # Found + done + return 1 # Not Found +} + script_lock() { local LOCKDIR="$TMPDIR/.scriptlock.$(basename $0)" if ! mkdir "$LOCKDIR" >/dev/null 2>&1 ; then @@ -262,6 +277,8 @@ check_pkgfile() { local pkgarch="$(getpkgarch ${pkgfile})" [ $? -ge 1 ] && return 1 + in_array "${pkgarch}" ${ARCHES[@]} 'any' || return 1 + if echo "$(basename ${pkgfile})" | grep -q "${pkgname}-${pkgver}-${pkgarch}"; then return 0 else @@ -277,6 +294,8 @@ check_pkgsvn() { [ $? -ge 1 ] && return 1 local repo="${2}" + in_array "${repo}" $(get_repos_for_host) || return 1 + if [ ! -f "${WORKDIR}/pkgbuilds/${pkgbase}" ]; then mkdir -p "${WORKDIR}/pkgbuilds" svn export -q "${SVNREPO}/${pkgbase}/repos/${repo}-${pkgarch}/PKGBUILD" \ @@ -352,15 +371,7 @@ pkgver_from_src() { check_repo_permission() { local repo=$1 - local repos="$(get_repos_for_host)" - local found=false - local r - for r in $repos; do - if [ "$r" = "$repo" ]; then - found=true - fi - done - [ $found ] || return 1 + in_array "${repo}" $(get_repos_for_host) || return 1 [ -w "$FTP_BASE/$(get_pkgpool_for_host)" ] || return 1 -- cgit v1.2.2 From 1aec9771b5c2b29ef2427e3aaabd7ef5f58195c5 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sat, 14 Aug 2010 22:40:14 +0200 Subject: Don't use hard coded architectures in db-remove --- db-remove | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/db-remove b/db-remove index 2f05c16..3a4a38e 100755 --- a/db-remove +++ b/db-remove @@ -39,13 +39,13 @@ fi cd "$WORKDIR" if [ "$arch" == "any" ]; then - arches="i686 x86_64" + tarches=(${ARCHES[@]}) else - arches="$arch" + tarches=("$arch") fi # copy the db file into our working area -for tarch in $arches; do +for tarch in ${tarches[@]}; do if [ ! -f "$ftppath/$tarch/$reponame$DBEXT" ]; then die "No database found at '$ftppath/$tarch', nothing more to do" fi -- cgit v1.2.2 From 55aa721771c7486c447b270325c504724622fd00 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sat, 14 Aug 2010 22:41:16 +0200 Subject: Prepare support for multiple packages in db-move --- db-move | 87 +++++++++++++++++++++++++++++++++--------------------------- test/runTest | 10 +++---- 2 files changed, 53 insertions(+), 44 deletions(-) diff --git a/db-move b/db-move index 68cfc07..4c18167 100755 --- a/db-move +++ b/db-move @@ -1,17 +1,18 @@ #!/bin/bash if [ $# -ne 4 ]; then - msg "usage: $(basename $0) " + msg "usage: $(basename $0) ..." exit 1 fi . "$(dirname $0)/db-functions" . "$(dirname $0)/config" -packagebase="$1" -repofrom="$2" -repoto="$3" -arch="$4" +args=(${@}) +repofrom="${args[0]}" +repoto="${args[1]}" +arch="${args[2]}" +# TODO: make db-move arch-independent? if [ "${arch}" == 'any' ]; then tarches=(${ARCHES[@]}) else @@ -27,53 +28,61 @@ if ! check_repo_permission $repoto || ! check_repo_permission $repofrom; then error "You don't have permission to move packages from ${repofrom} to ${repoto}" exit 1 fi +# TODO: add other tests before touch the repos repo_lock $repoto $arch || exit 1 repo_lock $repofrom $arch || exit 1 -cd "$WORKDIR" +pushd "$WORKDIR" >/dev/null /usr/bin/svn checkout -q -N $SVNREPO checkout >/dev/null -cd checkout +pushd checkout >/dev/null -/usr/bin/svn up -q $packagebase -if [ -d "$packagebase/repos/$svnrepo_from" ]; then - pkgname=($(. "$packagebase/repos/$svnrepo_from/PKGBUILD"; echo ${pkgname[@]})) - pkgver=$(. "$packagebase/repos/$svnrepo_from/PKGBUILD"; echo ${pkgver}) - pkgrel=$(. "$packagebase/repos/$svnrepo_from/PKGBUILD"; echo ${pkgrel}) +add_pkgs=() +remove_pkgs=() +for pkgbase in ${args[@]:3}; do + # TODO: optimize + /usr/bin/svn up -q $pkgbase + if [ -d "$pkgbase/repos/$svnrepo_from" ]; then + pkgname=($(. "$pkgbase/repos/$svnrepo_from/PKGBUILD"; echo ${pkgname[@]})) + pkgver=$(. "$pkgbase/repos/$svnrepo_from/PKGBUILD"; echo ${pkgver}) + pkgrel=$(. "$pkgbase/repos/$svnrepo_from/PKGBUILD"; echo ${pkgrel}) - msg "Moving $packagebase from $repofrom to $repoto..." - if [ -d "$packagebase/repos/$svnrepo_to" ]; then - /usr/bin/svn rm --force -q "$packagebase/repos/$svnrepo_to" - /usr/bin/svn commit -q -m "$(basename $0): $packagebase removed by $(id -un) for move to $repoto" - fi + msg "Moving $pkgbase from $repofrom to $repoto..." + if [ -d "$pkgbase/repos/$svnrepo_to" ]; then + /usr/bin/svn rm --force -q "$pkgbase/repos/$svnrepo_to" + /usr/bin/svn commit -q -m "$(basename $0): $pkgbase removed by $(id -un) for move to $repoto" + fi - /usr/bin/svn mv -q -r HEAD "$packagebase/repos/$svnrepo_from" "$packagebase/repos/$svnrepo_to" - /usr/bin/svn commit -q -m "$(basename $0): moved $packagebase from [$repofrom] to [$repoto] ($arch)" + /usr/bin/svn mv -q -r HEAD "$pkgbase/repos/$svnrepo_from" "$pkgbase/repos/$svnrepo_to" + /usr/bin/svn commit -q -m "$(basename $0): moved $pkgbase from [$repofrom] to [$repoto] ($arch)" - pkgfiles='' - for i in ${pkgname[@]}; do - for tarch in ${tarches[@]}; do - pkgpath=$(getpkgfile "$ftppath_from/${tarch}/"$i-$pkgver-$pkgrel-$arch$PKGEXT) - pkgfile=$(basename "${pkgpath}") + for i in ${pkgname[@]}; do + for tarch in ${tarches[@]}; do + pkgpath=$(getpkgfile "$ftppath_from/${tarch}/"$i-$pkgver-$pkgrel-$arch$PKGEXT) + pkgfile=$(basename "${pkgpath}") - # copy package to pool if needed - # TODO: can be removed once every package has been moved to the package pool - if [ ! -f $FTP_BASE/$(get_pkgpool_for_host)/$pkgfile ]; then - cp $pkgpath $FTP_BASE/$(get_pkgpool_for_host) - fi - ln -s "../../../$(get_pkgpool_for_host)/${pkgfile}" $ftppath_to/${tarch}/ + # copy package to pool if needed + # TODO: can be removed once every package has been moved to the package pool + if [ ! -f $FTP_BASE/$(get_pkgpool_for_host)/$pkgfile ]; then + cp $pkgpath $FTP_BASE/$(get_pkgpool_for_host) + fi + ln -s "../../../$(get_pkgpool_for_host)/${pkgfile}" $ftppath_to/${tarch}/ + done + add_pkgs[${#add_pkgs[*]}]="$FTP_BASE/$(get_pkgpool_for_host)/${pkgfile}" + remove_pkgs[${#remove_pkgs[*]}]=${i} done + else + die "$pkgbase is not in repo $repofrom" + fi +done - pkgfiles="${pkgfiles} $FTP_BASE/$(get_pkgpool_for_host)/${pkgfile}" - done +for tarch in ${tarches[@]}; do + /usr/bin/repo-add -q "$ftppath_to/${tarch}/$repoto$DBEXT" ${add_pkgs[@]} >/dev/null || die "Error in repo-add" + /usr/bin/repo-remove -q "$ftppath_from/${tarch}/$repofrom$DBEXT" ${remove_pkgs[@]} >/dev/null || die "Error in repo-remove" +done - for tarch in ${tarches[@]}; do - /usr/bin/repo-add -q "$ftppath_to/${tarch}/$repoto$DBEXT" ${pkgfiles} >/dev/null || die "Error in repo-add $pkgfiles" - /usr/bin/repo-remove -q "$ftppath_from/${tarch}/$repofrom$DBEXT" ${pkgname[@]} >/dev/null || die "Error in repo-remove ${pkgname[@]}" - done -else - die "$packagebase is not in repo $repofrom" -fi +popd >/dev/null +popd >/dev/null repo_unlock $repoto $arch || exit 1 repo_unlock $repofrom $arch || exit 1 diff --git a/test/runTest b/test/runTest index 06ca000..8c1aa04 100755 --- a/test/runTest +++ b/test/runTest @@ -152,7 +152,7 @@ testMoveSimplePackages() { ../db-update for arch in ${arches[@]}; do - ../db-move pkg-simple-a testing extra ${arch} + ../db-move testing extra ${arch} pkg-simple-a done for arch in ${arches[@]}; do @@ -172,7 +172,7 @@ testMoveAnyPackages() { done ../db-update - ../db-move pkg-any-a testing extra any + ../db-move testing extra any pkg-any-a checkAnyPackage extra pkg-any-a-1-1-any.pkg.tar.xz checkRemovedAnyPackage testing pkg-any-a @@ -195,7 +195,7 @@ testMoveSplitPackages() { ../db-update for arch in ${arches[@]}; do - ../db-move pkg-split-a testing extra ${arch} + ../db-move testing extra ${arch} pkg-split-a done for arch in ${arches[@]}; do @@ -338,7 +338,7 @@ testMovePackagesWithoutPool() { for pkgbase in ${pkgs[@]}; do for arch in ${arches[@]}; do - ../db-move ${pkgbase} testing extra ${arch} + ../db-move testing extra ${arch} ${pkgbase} done done @@ -378,7 +378,7 @@ testMoveAnyPackagesWithoutPool() { ../cron-jobs/ftpdir-cleanup >/dev/null for pkgbase in ${pkgs[@]}; do - ../db-move ${pkgbase} testing extra any + ../db-move testing extra any ${pkgbase} done ../cron-jobs/ftpdir-cleanup >/dev/null -- cgit v1.2.2 From e81b73f7cd4d0fc5944c26ccd1ff22e0883c295e Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 15 Aug 2010 18:17:10 +0200 Subject: Use common names for repos --- db-functions | 8 ++++---- db-remove | 2 +- db-update | 4 ++-- testing2community | 11 ----------- testing2community-any | 11 ----------- testing2community64 | 11 ----------- testing2core | 11 ----------- testing2core-any | 11 ----------- testing2core64 | 11 ----------- testing2extra | 11 ----------- testing2extra-any | 11 ----------- testing2extra64 | 11 ----------- testing2x64 | 1 - 13 files changed, 7 insertions(+), 107 deletions(-) delete mode 100755 testing2community delete mode 100755 testing2community-any delete mode 100755 testing2community64 delete mode 100755 testing2core delete mode 100755 testing2core-any delete mode 100755 testing2core64 delete mode 100755 testing2extra delete mode 100755 testing2extra-any delete mode 100755 testing2extra64 delete mode 120000 testing2x64 diff --git a/db-functions b/db-functions index 102f2c8..1f54dd9 100644 --- a/db-functions +++ b/db-functions @@ -103,7 +103,7 @@ cleanup() { repo=${l%.*} arch=${l#*.} if [ -d "$TMPDIR/.repolock.$repo.$arch" ]; then - msg "Removing left over lock from $repo/$arch" + msg "Removing left over lock from [${repo}] (${arch})" repo_unlock $repo $arch fi done @@ -151,7 +151,7 @@ repo_lock () { while [ $_count -le $_trial ] || $_lockblock ; do if ! mkdir "$LOCKDIR" >/dev/null 2>&1 ; then _owner="$(/usr/bin/stat -c %U $LOCKDIR)" - warning "Repo $1-$2 is already locked by $_owner. " \ + warning "Repo [${1}] (${2}) is already locked by $_owner. " \ "Retrying in $LOCK_DELAY seconds..." else LOCKS[${#LOCKS[*]}]="$1.$2" @@ -162,14 +162,14 @@ repo_lock () { let _count=$_count+1 done - error "Repo $1-$2 is already locked by $_owner. Giving up!" + error "Repo [${1}] (${2}) is already locked by $_owner. Giving up!" return 1 } repo_unlock () { #repo_unlock local LOCKDIR="$TMPDIR/.repolock.$1.$2" if [ ! -d "$LOCKDIR" ]; then - warning "Repo lock $1-$2 was not locked!" + warning "Repo lock [${1}] (${2}) was not locked!" restore_umask return 1 else diff --git a/db-remove b/db-remove index 3a4a38e..5afd858 100755 --- a/db-remove +++ b/db-remove @@ -22,7 +22,7 @@ fi repo_lock $reponame $arch || exit 1 -msg "Removing $packagebase from $reponame..." +msg "Removing $packagebase from [$reponame]..." cd "$WORKDIR" /usr/bin/svn checkout -q -N $SVNREPO checkout >/dev/null cd checkout diff --git a/db-update b/db-update index 43c44f7..73897f3 100755 --- a/db-update +++ b/db-update @@ -49,14 +49,14 @@ for repo in ${repos[@]}; do done for repo in ${repos[@]}; do + msg "Updating [${repo}]..." any_pkgs=($(getpkgfiles "${STAGING}/${repo}/"*-any${PKGEXT} 2>/dev/null)) for pkgarch in ${ARCHES[@]}; do - msg "Updating [${repo}] (${pkgarch})..." add_pkgs=() arch_pkgs=($(getpkgfiles "${STAGING}/${repo}/"*-${pkgarch}${PKGEXT} 2>/dev/null)) for pkg in ${arch_pkgs[@]} ${any_pkgs[@]}; do pkgfile="$(basename ${pkg})" - msg2 "${pkgfile}" + msg2 "${pkgfile} (${pkgarch})" # any packages might have been moved by the previous run if [ -f "${pkg}" ]; then mv "${pkg}" "$FTP_BASE/$(get_pkgpool_for_host)" diff --git a/testing2community b/testing2community deleted file mode 100755 index 557b39d..0000000 --- a/testing2community +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -if [ $# -le 0 ]; then - echo "usage: $(basename 0) [ [ Moving package '$pkg'" - $(dirname $0)/db-move "$pkg" "community-testing" "community" "i686" -done diff --git a/testing2community-any b/testing2community-any deleted file mode 100755 index e63a7ec..0000000 --- a/testing2community-any +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -if [ $# -le 0 ]; then - echo "usage: $(basename 0) [ [ Moving package '$pkg'" - $(dirname $0)/db-move "$pkg" "community-testing" "community" "any" -done diff --git a/testing2community64 b/testing2community64 deleted file mode 100755 index 3f88a4d..0000000 --- a/testing2community64 +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -if [ $# -le 0 ]; then - echo "usage: $(basename 0) [ [ Moving package '$pkg'" - $(dirname $0)/db-move "$pkg" "community-testing" "community" "x86_64" -done diff --git a/testing2core b/testing2core deleted file mode 100755 index 3a7acbf..0000000 --- a/testing2core +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -if [ $# -le 0 ]; then - echo "usage: $(basename 0) [ [ Moving package '$pkg'" - $(dirname $0)/db-move "$pkg" "testing" "core" "i686" -done diff --git a/testing2core-any b/testing2core-any deleted file mode 100755 index a0993e8..0000000 --- a/testing2core-any +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -if [ $# -le 0 ]; then - echo "usage: $(basename 0) [ [ Moving package '$pkg'" - $(dirname $0)/db-move "$pkg" "testing" "core" "any" -done diff --git a/testing2core64 b/testing2core64 deleted file mode 100755 index 4d30036..0000000 --- a/testing2core64 +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -if [ $# -le 0 ]; then - echo "usage: $(basename 0) [ [ Moving package '$pkg'" - $(dirname $0)/db-move "$pkg" "testing" "core" "x86_64" -done diff --git a/testing2extra b/testing2extra deleted file mode 100755 index 3be108e..0000000 --- a/testing2extra +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -if [ $# -le 0 ]; then - echo "usage: $(basename 0) [ [ Moving package '$pkg'" - $(dirname $0)/db-move "$pkg" "testing" "extra" "i686" -done diff --git a/testing2extra-any b/testing2extra-any deleted file mode 100755 index 46ab2f0..0000000 --- a/testing2extra-any +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -if [ $# -le 0 ]; then - echo "usage: $(basename 0) [ [ Moving package '$pkg'" - $(dirname $0)/db-move "$pkg" "testing" "extra" "any" -done diff --git a/testing2extra64 b/testing2extra64 deleted file mode 100755 index 418876c..0000000 --- a/testing2extra64 +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -if [ $# -le 0 ]; then - echo "usage: $(basename 0) [ [ Moving package '$pkg'" - $(dirname $0)/db-move "$pkg" "testing" "extra" "x86_64" -done diff --git a/testing2x64 b/testing2x64 deleted file mode 120000 index ff08aa7..0000000 --- a/testing2x64 +++ /dev/null @@ -1 +0,0 @@ -testing2x \ No newline at end of file -- cgit v1.2.2 From 79db58732efbab54911ccbbb8cd876da29ea48c8 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 15 Aug 2010 18:21:33 +0200 Subject: Move packages of all arches within one transaction db-move does no longer need a specific architecture. It will move all architecures of a given package at once. testing2x has been rewritten to respect these changes and testing2x64 is no longer needed. --- db-move | 143 +++++++++++++++++++++++++++++++---------------------------- test/runTest | 21 +++------ testing2x | 67 ++++++++++++++++------------ 3 files changed, 119 insertions(+), 112 deletions(-) diff --git a/db-move b/db-move index 4c18167..ae98958 100755 --- a/db-move +++ b/db-move @@ -1,88 +1,97 @@ #!/bin/bash -if [ $# -ne 4 ]; then - msg "usage: $(basename $0) ..." - exit 1 -fi - . "$(dirname $0)/db-functions" . "$(dirname $0)/config" -args=(${@}) -repofrom="${args[0]}" -repoto="${args[1]}" -arch="${args[2]}" -# TODO: make db-move arch-independent? -if [ "${arch}" == 'any' ]; then - tarches=(${ARCHES[@]}) -else - tarches=("${arch}") +if [ $# -lt 3 ]; then + msg "usage: $(basename $0) ..." + exit 1 fi -ftppath_from="$FTP_BASE/$repofrom/os/" -ftppath_to="$FTP_BASE/$repoto/os/" -svnrepo_from="$repofrom-$arch" -svnrepo_to="$repoto-$arch" +args=(${@}) +repo_from="${args[0]}" +repo_to="${args[1]}" +ftppath_from="${FTP_BASE}/${repo_from}/os/" +ftppath_to="${FTP_BASE}/${repo_to}/os/" -if ! check_repo_permission $repoto || ! check_repo_permission $repofrom; then - error "You don't have permission to move packages from ${repofrom} to ${repoto}" - exit 1 +if ! check_repo_permission $repo_to || ! check_repo_permission $repo_from; then + die "You don't have permission to move packages from ${repo_from} to ${repo_to}" fi -# TODO: add other tests before touch the repos -repo_lock $repoto $arch || exit 1 -repo_lock $repofrom $arch || exit 1 +/usr/bin/svn checkout -q -N "${SVNREPO}" "${WORKDIR}/svn" >/dev/null +for pkgbase in ${args[@]:2}; do + /usr/bin/svn up -q "${WORKDIR}/svn/${pkgbase}" >/dev/null + for pkgarch in ${ARCHES[@]} 'any'; do + svnrepo_from="${WORKDIR}/svn/${pkgbase}/repos/${repo_from}-${pkgarch}" + if [ -r "${svnrepo_from}/PKGBUILD" ]; then + continue 2 + fi + done + die "${pkgbase} not found in ${repo_from}" +done + +# TODO: this might lock too much (architectures) +for pkgarch in ${ARCHES[@]}; do + repo_lock ${repo_to} ${pkgarch} || exit 1 + repo_lock ${repo_from} ${pkgarch} || exit 1 +done -pushd "$WORKDIR" >/dev/null -/usr/bin/svn checkout -q -N $SVNREPO checkout >/dev/null -pushd checkout >/dev/null +msg "Moving packages from [${repo_from}] to [${repo_to}]..." -add_pkgs=() -remove_pkgs=() -for pkgbase in ${args[@]:3}; do - # TODO: optimize - /usr/bin/svn up -q $pkgbase - if [ -d "$pkgbase/repos/$svnrepo_from" ]; then - pkgname=($(. "$pkgbase/repos/$svnrepo_from/PKGBUILD"; echo ${pkgname[@]})) - pkgver=$(. "$pkgbase/repos/$svnrepo_from/PKGBUILD"; echo ${pkgver}) - pkgrel=$(. "$pkgbase/repos/$svnrepo_from/PKGBUILD"; echo ${pkgrel}) +declare -A add_pkgs +declare -A remove_pkgs +for pkgbase in ${args[@]:2}; do + for pkgarch in ${ARCHES[@]} 'any'; do + svnrepo_from="${WORKDIR}/svn/${pkgbase}/repos/${repo_from}-${pkgarch}" + svnrepo_to="${WORKDIR}/svn/${pkgbase}/repos/${repo_to}-${pkgarch}" - msg "Moving $pkgbase from $repofrom to $repoto..." - if [ -d "$pkgbase/repos/$svnrepo_to" ]; then - /usr/bin/svn rm --force -q "$pkgbase/repos/$svnrepo_to" - /usr/bin/svn commit -q -m "$(basename $0): $pkgbase removed by $(id -un) for move to $repoto" - fi + if [ -f "${svnrepo_from}/PKGBUILD" ]; then + if [ "${pkgarch}" == 'any' ]; then + tarches=(${ARCHES[@]}) + else + tarches=("${pkgarch}") + fi + msg2 "${pkgbase} ($(echo ${tarches[@]}))" + pkgnames=($(. "${svnrepo_from}/PKGBUILD"; echo ${pkgname[@]})) + pkgver=$(. "${svnrepo_from}/PKGBUILD"; echo "${pkgver}-${pkgrel}") - /usr/bin/svn mv -q -r HEAD "$pkgbase/repos/$svnrepo_from" "$pkgbase/repos/$svnrepo_to" - /usr/bin/svn commit -q -m "$(basename $0): moved $pkgbase from [$repofrom] to [$repoto] ($arch)" + if [ -d "${svnrepo_to}" ]; then + /usr/bin/svn rm --force -q "${svnrepo_to}" + /usr/bin/svn commit -q "${WORKDIR}/svn/${pkgbase}" -m "$(basename $0): ${pkgbase} removed by $(id -un) for move to [${repo_to}] (${pkgarch})" + fi - for i in ${pkgname[@]}; do - for tarch in ${tarches[@]}; do - pkgpath=$(getpkgfile "$ftppath_from/${tarch}/"$i-$pkgver-$pkgrel-$arch$PKGEXT) - pkgfile=$(basename "${pkgpath}") + /usr/bin/svn mv -q -r HEAD "${svnrepo_from}" "${svnrepo_to}" + /usr/bin/svn commit -q "${WORKDIR}/svn/${pkgbase}" -m "$(basename $0): moved ${pkgbase} from [${repo_from}] to [${repo_to}] (${pkgarch})" - # copy package to pool if needed - # TODO: can be removed once every package has been moved to the package pool - if [ ! -f $FTP_BASE/$(get_pkgpool_for_host)/$pkgfile ]; then - cp $pkgpath $FTP_BASE/$(get_pkgpool_for_host) - fi - ln -s "../../../$(get_pkgpool_for_host)/${pkgfile}" $ftppath_to/${tarch}/ + for pkgname in ${pkgnames[@]}; do + for tarch in ${tarches[@]}; do + pkgpath=$(getpkgfile "${ftppath_from}/${tarch}/"${pkgname}-${pkgver}-${pkgarch}${PKGEXT}) + pkgfile=$(basename "${pkgpath}") + + # copy package to pool if needed + # TODO: can be removed once every package has been moved to the package pool + if [ ! -f ${FTP_BASE}/$(get_pkgpool_for_host)/${pkgfile} ]; then + cp ${pkgpath} ${FTP_BASE}/$(get_pkgpool_for_host) + fi + ln -s "../../../$(get_pkgpool_for_host)/${pkgfile}" ${ftppath_to}/${tarch}/ + add_pkgs[${tarch}]+="${FTP_BASE}/$(get_pkgpool_for_host)/${pkgfile} " + remove_pkgs[${tarch}]+="${pkgname} " + done done - add_pkgs[${#add_pkgs[*]}]="$FTP_BASE/$(get_pkgpool_for_host)/${pkgfile}" - remove_pkgs[${#remove_pkgs[*]}]=${i} - done - else - die "$pkgbase is not in repo $repofrom" - fi + fi + done done -for tarch in ${tarches[@]}; do - /usr/bin/repo-add -q "$ftppath_to/${tarch}/$repoto$DBEXT" ${add_pkgs[@]} >/dev/null || die "Error in repo-add" - /usr/bin/repo-remove -q "$ftppath_from/${tarch}/$repofrom$DBEXT" ${remove_pkgs[@]} >/dev/null || die "Error in repo-remove" +for tarch in ${ARCHES[@]}; do + if [ -n "${add_pkgs[${tarch}]}" ]; then + /usr/bin/repo-add -q "${ftppath_to}/${tarch}/${repo_to}${DBEXT}" ${add_pkgs[${tarch}]} >/dev/null \ + || die "Error in repo-add ${add_pkgs[${tarch}]}" + /usr/bin/repo-remove -q "${ftppath_from}/${tarch}/${repo_from}${DBEXT}" ${remove_pkgs[${tarch}]} >/dev/null \ + || die "Error in repo-remove ${remove_pkgs[${tarch}]}" + fi done -popd >/dev/null -popd >/dev/null - -repo_unlock $repoto $arch || exit 1 -repo_unlock $repofrom $arch || exit 1 +for pkgarch in ${ARCHES[@]}; do + repo_unlock ${repo_from} ${pkgarch} + repo_unlock ${repo_to} ${pkgarch} +done diff --git a/test/runTest b/test/runTest index 8c1aa04..03f94c2 100755 --- a/test/runTest +++ b/test/runTest @@ -151,9 +151,7 @@ testMoveSimplePackages() { ../db-update - for arch in ${arches[@]}; do - ../db-move testing extra ${arch} pkg-simple-a - done + ../db-move testing extra pkg-simple-a for arch in ${arches[@]}; do checkPackage extra pkg-simple-a-1-1-${arch}.pkg.tar.xz ${arch} @@ -172,7 +170,7 @@ testMoveAnyPackages() { done ../db-update - ../db-move testing extra any pkg-any-a + ../db-move testing extra pkg-any-a checkAnyPackage extra pkg-any-a-1-1-any.pkg.tar.xz checkRemovedAnyPackage testing pkg-any-a @@ -193,10 +191,7 @@ testMoveSplitPackages() { done ../db-update - - for arch in ${arches[@]}; do - ../db-move testing extra ${arch} pkg-split-a - done + ../db-move testing extra pkg-split-a for arch in ${arches[@]}; do for pkg in "${pkgdir}/pkg-split-a"/*-${arch}.pkg.tar.*; do @@ -336,11 +331,7 @@ testMovePackagesWithoutPool() { ../cron-jobs/ftpdir-cleanup >/dev/null - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - ../db-move testing extra ${arch} ${pkgbase} - done - done + ../db-move testing extra ${pkgs[@]} ../cron-jobs/ftpdir-cleanup >/dev/null @@ -377,9 +368,7 @@ testMoveAnyPackagesWithoutPool() { ../cron-jobs/ftpdir-cleanup >/dev/null - for pkgbase in ${pkgs[@]}; do - ../db-move testing extra any ${pkgbase} - done + ../db-move testing extra ${pkgs[@]} ../cron-jobs/ftpdir-cleanup >/dev/null diff --git a/testing2x b/testing2x index 3626432..83b12fa 100755 --- a/testing2x +++ b/testing2x @@ -3,35 +3,44 @@ . "$(dirname $0)/db-functions" . "$(dirname $0)/config" -case "$0" in - *64) - _arch="x86_64" - ;; - *) - _arch="i686" - ;; -esac +if [ $# -lt 1 ]; then + msg "usage: $(basename $0) ..." + exit 1 +fi -cd "${WORKDIR}" -/usr/bin/svn checkout -N $SVNREPO checkout -cd checkout -for pkg in $*; do - moved=0 - /usr/bin/svn up -q ${pkg} - if [ -f "${pkg}/repos/testing-${_arch}/PKGBUILD" ]; then - for repo in core extra; do - if [ -f "${pkg}/repos/${repo}-${_arch}/PKGBUILD" ]; then - msg "Moving package '${pkg}': testing-${_arch} -> ${repo}-${_arch}" - $(dirname $0)/db-move "${pkg}" "testing" "${repo}" "${_arch}" - moved=1 - break - fi - done - if [ ${moved} -eq 0 ]; then - warning "${pkg} is only in testing-${_arch}, cannot determine where to move it" - fi - else - warning "${pkg} is not in testing-${_arch}" - fi +declare -A pkgs + +for pkgbase in $*; do + if [ ! -d "${WORKDIR}/${pkgbase}" ]; then + /usr/bin/svn export -q "${SVNREPO}/${pkgbase}/repos" "${WORKDIR}/${pkgbase}" >/dev/null + + found_source=false + for pkgarch in ${ARCHES[@]} 'any'; do + svnrepo_from="${WORKDIR}/${pkgbase}/testing-${pkgarch}" + if [ -r "${svnrepo_from}/PKGBUILD" ]; then + found_source=true + break + fi + done + ${found_source} || die "${pkgbase} not found in [testing]" + found_target=false + for pkgarch in ${ARCHES[@]} 'any'; do + for repo in 'core' 'extra'; do + svnrepo_to="${WORKDIR}/${pkgbase}/${repo}-${pkgarch}" + if [ -r "${svnrepo_to}/PKGBUILD" ]; then + found_target=true + pkgs[${repo}]+="${pkgbase} " + break 2 + fi + done + done + ${found_target} || die "${pkgbase} neither found in [core] nor [extra]" + fi +done + +for repo in 'core' 'extra'; do + if [ -n "${pkgs[${repo}]}" ]; then + "$(dirname $0)/db-move" 'testing' "${repo}" ${pkgs[${repo}]} + fi done -- cgit v1.2.2 From ce604bc3a2569198fa749acca190fef036e059a1 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 15 Aug 2010 19:17:05 +0200 Subject: add test to check updating a package --- test/lib/common.inc | 3 ++- test/runTest | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/test/lib/common.inc b/test/lib/common.inc index 5c76eb5..e55ce38 100644 --- a/test/lib/common.inc +++ b/test/lib/common.inc @@ -75,8 +75,9 @@ releasePackage() { pushd "${TMP}/svn-packages-copy"/${pkgbase}/trunk/ >/dev/null archrelease ${repo}-${arch} >/dev/null 2&>1 + pkgver=$(. PKGBUILD; echo "${pkgver}-${pkgrel}") popd >/dev/null - cp "${pkgdir}/${pkgbase}"/*.pkg.tar.* "${STAGING}"/${repo}/ + cp "${pkgdir}/${pkgbase}"/*-${pkgver}-${arch}.pkg.tar.* "${STAGING}"/${repo}/ } checkAnyPackage() { diff --git a/test/runTest b/test/runTest index 03f94c2..a16165d 100755 --- a/test/runTest +++ b/test/runTest @@ -81,6 +81,24 @@ testAddSplitPackages() { done } +testUpdateAnyPackage() { + releasePackage extra pkg-any-a any + ../db-update + + pushd "${TMP}/svn-packages-copy/pkg-any-a/trunk/" >/dev/null + sed 's/pkgrel=1/pkgrel=2/g' -i PKGBUILD + svn commit -q -m"update pkg to pkgrel=2" >/dev/null + makepkg -cf --config ${curdir}/lib/makepkg-i686.conf >/dev/null 2>&1 + mv pkg-any-a-1-2-any.pkg.tar.xz "${pkgdir}/pkg-any-a/" + popd >/dev/null + + releasePackage extra pkg-any-a any + ../db-update + + checkAnyPackage extra pkg-any-a-1-2-any.pkg.tar.xz any + + rm "${pkgdir}/pkg-any-a/pkg-any-a-1-2-any.pkg.tar.xz" +} # # db-remove -- cgit v1.2.2 From 583863043ad11eb203c479c9d0eea825a29ace39 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 15 Aug 2010 19:21:38 +0200 Subject: add test for testing2x --- test/runTest | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/test/runTest b/test/runTest index a16165d..95d8c6c 100755 --- a/test/runTest +++ b/test/runTest @@ -226,6 +226,32 @@ testMoveSplitPackages() { } +# +# testing2x +# + +testTesting2xAnyPackage() { + releasePackage core pkg-any-a any + ../db-update + + pushd "${TMP}/svn-packages-copy/pkg-any-a/trunk/" >/dev/null + sed 's/pkgrel=1/pkgrel=2/g' -i PKGBUILD + svn commit -q -m"update pkg to pkgrel=2" >/dev/null + makepkg -cf --config ${curdir}/lib/makepkg-i686.conf >/dev/null 2>&1 + mv pkg-any-a-1-2-any.pkg.tar.xz "${pkgdir}/pkg-any-a/" + popd >/dev/null + + releasePackage testing pkg-any-a any + ../db-update + rm "${pkgdir}/pkg-any-a/pkg-any-a-1-2-any.pkg.tar.xz" + + ../testing2x pkg-any-a + + checkAnyPackage core pkg-any-a-1-2-any.pkg.tar.xz any + checkRemovedAnyPackage testing pkg-any-a +} + + # # ftpdir-cleanup # -- cgit v1.2.2 From b5fb2927cde513dc0fd82e4fd4dfe3836cf35728 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 15 Aug 2010 20:55:26 +0200 Subject: Lock repos before checking --- db-functions | 4 ++-- db-move | 25 +++++++++++++++++++------ db-update | 32 ++++++++++++++------------------ testing2x | 15 +++++++++++++++ 4 files changed, 50 insertions(+), 26 deletions(-) diff --git a/db-functions b/db-functions index 1f54dd9..015f254 100644 --- a/db-functions +++ b/db-functions @@ -151,8 +151,8 @@ repo_lock () { while [ $_count -le $_trial ] || $_lockblock ; do if ! mkdir "$LOCKDIR" >/dev/null 2>&1 ; then _owner="$(/usr/bin/stat -c %U $LOCKDIR)" - warning "Repo [${1}] (${2}) is already locked by $_owner. " \ - "Retrying in $LOCK_DELAY seconds..." + warning "Repo [${1}] (${2}) is already locked by $_owner. " + msg2 "Retrying in $LOCK_DELAY seconds..." else LOCKS[${#LOCKS[*]}]="$1.$2" set_umask diff --git a/db-move b/db-move index ae98958..a274cd3 100755 --- a/db-move +++ b/db-move @@ -18,24 +18,37 @@ if ! check_repo_permission $repo_to || ! check_repo_permission $repo_from; then die "You don't have permission to move packages from ${repo_from} to ${repo_to}" fi +# TODO: this might lock too much (architectures) +for pkgarch in ${ARCHES[@]}; do + repo_lock ${repo_to} ${pkgarch} || exit 1 + repo_lock ${repo_from} ${pkgarch} || exit 1 +done + +# check if packages to be moved exist in svn and ftp dir /usr/bin/svn checkout -q -N "${SVNREPO}" "${WORKDIR}/svn" >/dev/null for pkgbase in ${args[@]:2}; do /usr/bin/svn up -q "${WORKDIR}/svn/${pkgbase}" >/dev/null for pkgarch in ${ARCHES[@]} 'any'; do svnrepo_from="${WORKDIR}/svn/${pkgbase}/repos/${repo_from}-${pkgarch}" if [ -r "${svnrepo_from}/PKGBUILD" ]; then + pkgnames=($(. "${svnrepo_from}/PKGBUILD"; echo ${pkgname[@]})) + pkgver=$(. "${svnrepo_from}/PKGBUILD"; echo "${pkgver}-${pkgrel}") + if [ "${pkgarch}" == 'any' ]; then + tarches=(${ARCHES[@]}) + else + tarches=("${pkgarch}") + fi + for pkgname in ${pkgnames[@]}; do + for tarch in ${tarches[@]}; do + getpkgfile "${ftppath_from}/${tarch}/"${pkgname}-${pkgver}-${pkgarch}${PKGEXT} >/dev/null + done + done continue 2 fi done die "${pkgbase} not found in ${repo_from}" done -# TODO: this might lock too much (architectures) -for pkgarch in ${ARCHES[@]}; do - repo_lock ${repo_to} ${pkgarch} || exit 1 - repo_lock ${repo_from} ${pkgarch} || exit 1 -done - msg "Moving packages from [${repo_from}] to [${repo_to}]..." declare -A add_pkgs diff --git a/db-update b/db-update index 73897f3..b11face 100755 --- a/db-update +++ b/db-update @@ -11,43 +11,39 @@ fi # Find repos with packages to release repos=($(find "${STAGING}" -mindepth 1 -type d ! -empty -printf '%f ' 2>/dev/null)) if [ $? -ge 1 ]; then - error "Could not read ${STAGING}" - exit 1 + die "Could not read ${STAGING}" fi +# TODO: this might lock too much (architectures) +for repo in ${repos[@]}; do + for pkgarch in ${ARCHES[@]}; do + repo_lock ${repo} ${pkgarch} || exit 1 + done +done + +# check if packages are valid for repo in ${repos[@]}; do if ! check_repo_permission "${repo}"; then - error "You don't have permission to update packages in ${repo}" - exit 1 + die "You don't have permission to update packages in ${repo}" fi pkgs=($(getpkgfiles "${STAGING}/${repo}/"*${PKGEXT})) if [ $? -eq 0 ]; then for pkg in ${pkgs[@]}; do if ! check_pkgfile "${pkg}"; then - error "Package ${repo}/$(basename ${pkg}) is not consistent with its meta data" - exit 1 + die "Package ${repo}/$(basename ${pkg}) is not consistent with its meta data" fi if ! check_pkgsvn "${pkg}" "${repo}"; then - error "Package ${repo}/$(basename ${pkg}) is not consistent with svn repository" - exit 1 + die "Package ${repo}/$(basename ${pkg}) is not consistent with svn repository" fi if [ -f "${FTP_BASE}/$(get_pkgpool_for_host)/$(basename ${pkg})" ]; then - error "Package ${repo}/$(basename pkg) already exists in ${FTP_BASE}/$(get_pkgpool_for_host)" - exit 1 + die "Package ${repo}/$(basename pkg) already exists in ${FTP_BASE}/$(get_pkgpool_for_host)" fi done else - exit 1 + die "Could not read ${STAGING}" fi done -# TODO: this might lock too much (architectures) -for repo in ${repos[@]}; do - for pkgarch in ${ARCHES[@]}; do - repo_lock ${repo} ${pkgarch} || exit 1 - done -done - for repo in ${repos[@]}; do msg "Updating [${repo}]..." any_pkgs=($(getpkgfiles "${STAGING}/${repo}/"*-any${PKGEXT} 2>/dev/null)) diff --git a/testing2x b/testing2x index 83b12fa..54cae11 100755 --- a/testing2x +++ b/testing2x @@ -8,6 +8,13 @@ if [ $# -lt 1 ]; then exit 1 fi +# Lock everything to reduce possibility of interfering task between the different repo-updates +script_lock +for repo in 'core' 'extra' 'testing'; do + for pkgarch in ${ARCHES[@]}; do + repo_lock ${repo} ${pkgarch} || exit 1 + done +done declare -A pkgs @@ -39,8 +46,16 @@ for pkgbase in $*; do fi done +for pkgarch in ${ARCHES[@]}; do + repo_unlock 'testing' ${pkgarch} +done for repo in 'core' 'extra'; do + for pkgarch in ${ARCHES[@]}; do + repo_unlock ${repo} ${pkgarch} + done if [ -n "${pkgs[${repo}]}" ]; then "$(dirname $0)/db-move" 'testing' "${repo}" ${pkgs[${repo}]} fi done + +script_unlock -- cgit v1.2.2 From 6f6ed4dc9f1bbce3638824def5d07b78aff44cde Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 15 Aug 2010 20:56:12 +0200 Subject: Fix locking in db-remove --- db-remove | 56 +++++++++++++++++++++++++------------------------------- 1 file changed, 25 insertions(+), 31 deletions(-) diff --git a/db-remove b/db-remove index 5afd858..188cc60 100755 --- a/db-remove +++ b/db-remove @@ -1,58 +1,52 @@ #!/bin/bash if [ $# -ne 3 ]; then - msg "usage: $(basename $0) " + msg "usage: $(basename $0) " exit 1 fi . "$(dirname $0)/db-functions" . "$(dirname $0)/config" -packagebase="$1" -reponame="$2" +pkgbase="$1" +repo="$2" arch="$3" -ftppath="$FTP_BASE/$reponame/os" -svnrepo="$reponame-$arch" +ftppath="$FTP_BASE/$repo/os" +svnrepo="$repo-$arch" -if ! check_repo_permission $reponame; then - error "You don't have permission to remove packages from ${reponam}" - exit 1 +if ! check_repo_permission $repo; then + die "You don't have permission to remove packages from ${reponam}" fi -repo_lock $reponame $arch || exit 1 - -msg "Removing $packagebase from [$reponame]..." -cd "$WORKDIR" -/usr/bin/svn checkout -q -N $SVNREPO checkout >/dev/null -cd checkout - -/usr/bin/svn up -q $packagebase -if [ -d "$packagebase/repos/$svnrepo" ]; then - pkgname=($(. "$packagebase/repos/$svnrepo/PKGBUILD"; echo ${pkgname[@]})) - /usr/bin/svn rm --force -q "$packagebase/repos/$svnrepo" - /usr/bin/svn commit -q -m "$(basename $0): $packagebase removed by $(id -un)" +if [ "$arch" == "any" ]; then + tarches=(${ARCHES[@]}) else - die "$packagebase not found in $svnrepo" + tarches=("$arch") fi -cd "$WORKDIR" +for tarch in ${tarches[@]}; do + repo_lock $repo $tarch || exit 1 +done -if [ "$arch" == "any" ]; then - tarches=(${ARCHES[@]}) +msg "Removing $pkgbase from [$repo]..." +/usr/bin/svn checkout -q "${SVNREPO}/${pkgbase}" "${WORKDIR}/svn/${pkgbase}" >/dev/null + +if [ -d "${WORKDIR}/svn/$pkgbase/repos/$svnrepo" ]; then + pkgnames=($(. "${WORKDIR}/svn/$pkgbase/repos/$svnrepo/PKGBUILD"; echo ${pkgname[@]})) + /usr/bin/svn rm --force -q "${WORKDIR}/svn/$pkgbase/repos/$svnrepo" + /usr/bin/svn commit -q "${WORKDIR}/svn/$pkgbase" -m "$(basename $0): $pkgbase removed by $(id -un)" else - tarches=("$arch") + die "$pkgbase not found in $svnrepo" fi + # copy the db file into our working area for tarch in ${tarches[@]}; do - if [ ! -f "$ftppath/$tarch/$reponame$DBEXT" ]; then + if [ ! -f "$ftppath/$tarch/$repo$DBEXT" ]; then die "No database found at '$ftppath/$tarch', nothing more to do" fi - /usr/bin/repo-remove -q "$ftppath/$tarch/$reponame$DBEXT" ${pkgname[@]} >/dev/null + /usr/bin/repo-remove -q "$ftppath/$tarch/$repo$DBEXT" ${pkgnames[@]} >/dev/null + repo_unlock $repo $tarch done - -repo_unlock $reponame $arch || exit 1 - -# vim: set ts=4 sw=4 noet ft=sh: -- cgit v1.2.2 From 4daeb0a71e65618c56d672118675c9efd57c5979 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Tue, 17 Aug 2010 08:00:21 +0200 Subject: Add [staging] repository --- cron-jobs/adjust-permissions | 2 ++ cron-jobs/create-filelists | 2 +- cron-jobs/sourceballs | 2 +- db-functions | 2 +- test/lib/common.inc | 4 ++-- 5 files changed, 7 insertions(+), 5 deletions(-) diff --git a/cron-jobs/adjust-permissions b/cron-jobs/adjust-permissions index c28ca7c..6acfcf2 100755 --- a/cron-jobs/adjust-permissions +++ b/cron-jobs/adjust-permissions @@ -13,6 +13,8 @@ get_dir_owner() { echo "ftp:ftp-extra" ;; testing) echo "ftp:ftp-extra" ;; + staging) + echo "ftp:ftp-extra" ;; community) echo "root:tusers" ;; community-testing) diff --git a/cron-jobs/create-filelists b/cron-jobs/create-filelists index 9249408..fd07617 100755 --- a/cron-jobs/create-filelists +++ b/cron-jobs/create-filelists @@ -5,7 +5,7 @@ reposdir=${FTP_BASE} targetdir=${FTP_BASE} -repos=(core extra testing community community-testing) +repos=(core extra testing community community-testing staging) script_lock diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index b82e375..a6f7a19 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -5,7 +5,7 @@ ftpbase="${FTP_BASE}" srcbase="${FTP_BASE}/sources" -repos=(core extra testing community community-testing) +repos=(core extra testing community community-testing staging) script_lock diff --git a/db-functions b/db-functions index 015f254..a04dbf5 100644 --- a/db-functions +++ b/db-functions @@ -315,7 +315,7 @@ get_repos_for_host() { if [ "$(hostname)" = "sigurd" ]; then echo "community community-testing" else - echo "core extra testing" + echo "core extra testing staging" fi } diff --git a/test/lib/common.inc b/test/lib/common.inc index e55ce38..a79558c 100644 --- a/test/lib/common.inc +++ b/test/lib/common.inc @@ -31,9 +31,9 @@ setUp() { #msg "Using ${TMP}" mkdir -p "${TMP}/"{ftp,tmp,staging,{package,source}-cleanup,svn-{packages,community}-{copy,repo}} - mkdir -p "${TMP}/ftp/"{core,extra,community,testing,community-testing}/os/{i686,any,x86_64} + mkdir -p "${TMP}/ftp/"{core,extra,community,testing,community-testing,staging}/os/{i686,any,x86_64} mkdir -p "${TMP}/ftp/pool/"{packages,community} - mkdir -p "${TMP}/staging/"{core,extra,community,testing,community-testing} + mkdir -p "${TMP}/staging/"{core,extra,community,testing,community-testing,staging} msg 'Creating svn repository...' svnadmin create "${TMP}/svn-packages-repo" -- cgit v1.2.2 From cb39feaff074c5e08ff2dfce8dfa04c545233b7e Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Tue, 17 Aug 2010 16:41:31 +0200 Subject: Don't try to update no package Check if there are packages to update for given architecure. Previously db-update quit when only one arch of a package was available. --- db-update | 10 ++++++---- test/runTest | 6 ++++++ 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/db-update b/db-update index b11face..74abea8 100755 --- a/db-update +++ b/db-update @@ -60,10 +60,12 @@ for repo in ${repos[@]}; do ln -s "../../../$(get_pkgpool_for_host)/${pkgfile}" "$FTP_BASE/$repo/os/${pkgarch}" add_pkgs[${#add_pkgs[*]}]=${pkgfile} done - pushd "$FTP_BASE/$repo/os/${pkgarch}" >/dev/null - /usr/bin/repo-add -q "$repo$DBEXT" ${add_pkgs[@]} >/dev/null \ - || die "Could not add packages" - popd >/dev/null + if [ ${#add_pkgs[@]} -ge 1 ]; then + pushd "$FTP_BASE/$repo/os/${pkgarch}" >/dev/null + /usr/bin/repo-add -q "$repo$DBEXT" ${add_pkgs[@]} >/dev/null \ + || die "Could not add packages" + popd >/dev/null + fi done done diff --git a/test/runTest b/test/runTest index 95d8c6c..72f9b1c 100755 --- a/test/runTest +++ b/test/runTest @@ -42,6 +42,12 @@ testAddSimplePackages() { done } +testAddSingleSimplePackage() { + releasePackage extra 'pkg-simple-a' 'i686' + ../db-update + checkPackage extra 'pkg-simple-a-1-1-i686.pkg.tar.xz' 'i686' +} + testAddAnyPackages() { local pkgs=('pkg-any-a' 'pkg-any-b') local pkgbase -- cgit v1.2.2 From a2b3b2bb2bde44289d41b956dde1f5bca61e581b Mon Sep 17 00:00:00 2001 From: Allan McRae Date: Tue, 17 Aug 2010 22:42:15 +1000 Subject: minor fix to error message Signed-off-by: Allan McRae Signed-off-by: Pierre Schmitz --- db-update | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db-update b/db-update index 74abea8..dd8d1c0 100755 --- a/db-update +++ b/db-update @@ -36,7 +36,7 @@ for repo in ${repos[@]}; do die "Package ${repo}/$(basename ${pkg}) is not consistent with svn repository" fi if [ -f "${FTP_BASE}/$(get_pkgpool_for_host)/$(basename ${pkg})" ]; then - die "Package ${repo}/$(basename pkg) already exists in ${FTP_BASE}/$(get_pkgpool_for_host)" + die "Package ${repo}/$(basename ${pkg}) already exists in ${FTP_BASE}/$(get_pkgpool_for_host)" fi done else -- cgit v1.2.2 From 17c9dab4b420ca55fd4fb3f277805af148159a4d Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Tue, 17 Aug 2010 20:32:47 +0200 Subject: Add additional checks when reading PKGBUILDs If reading from a PKGBUILD fails stop immediatly. Also put out more usefull error messages. --- db-functions | 1 + db-move | 13 +++++++++++-- db-update | 2 +- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/db-functions b/db-functions index a04dbf5..292e3ff 100644 --- a/db-functions +++ b/db-functions @@ -304,6 +304,7 @@ check_pkgsvn() { fi local pkgver="$(. "${WORKDIR}/pkgbuilds/${pkgbase}"; echo "${pkgver}-${pkgrel}")" + [ -z "${pkgver}" ] && return 1 if echo "$(basename ${pkgfile})" | grep -q "${pkgver}-${pkgarch}"; then return 0 else diff --git a/db-move b/db-move index a274cd3..5ffb8bc 100755 --- a/db-move +++ b/db-move @@ -32,12 +32,21 @@ for pkgbase in ${args[@]:2}; do svnrepo_from="${WORKDIR}/svn/${pkgbase}/repos/${repo_from}-${pkgarch}" if [ -r "${svnrepo_from}/PKGBUILD" ]; then pkgnames=($(. "${svnrepo_from}/PKGBUILD"; echo ${pkgname[@]})) + if [ ${#pkgnames[@]} -lt 1 ]; then + die "Could not read pkgname" + fi + pkgver=$(. "${svnrepo_from}/PKGBUILD"; echo "${pkgver}-${pkgrel}") + if [ -z "${pkgver}" ]; then + die "Could not read pkgver" + fi + if [ "${pkgarch}" == 'any' ]; then tarches=(${ARCHES[@]}) else tarches=("${pkgarch}") fi + for pkgname in ${pkgnames[@]}; do for tarch in ${tarches[@]}; do getpkgfile "${ftppath_from}/${tarch}/"${pkgname}-${pkgver}-${pkgarch}${PKGEXT} >/dev/null @@ -98,9 +107,9 @@ done for tarch in ${ARCHES[@]}; do if [ -n "${add_pkgs[${tarch}]}" ]; then /usr/bin/repo-add -q "${ftppath_to}/${tarch}/${repo_to}${DBEXT}" ${add_pkgs[${tarch}]} >/dev/null \ - || die "Error in repo-add ${add_pkgs[${tarch}]}" + || error "repo-add ${tarch}/${repo_to}${DBEXT} ${add_pkgs[${tarch}]}" /usr/bin/repo-remove -q "${ftppath_from}/${tarch}/${repo_from}${DBEXT}" ${remove_pkgs[${tarch}]} >/dev/null \ - || die "Error in repo-remove ${remove_pkgs[${tarch}]}" + || error "repo-remove ${tarch}/${repo_from}${DBEXT} ${remove_pkgs[${tarch}]}" fi done diff --git a/db-update b/db-update index dd8d1c0..68450b4 100755 --- a/db-update +++ b/db-update @@ -63,7 +63,7 @@ for repo in ${repos[@]}; do if [ ${#add_pkgs[@]} -ge 1 ]; then pushd "$FTP_BASE/$repo/os/${pkgarch}" >/dev/null /usr/bin/repo-add -q "$repo$DBEXT" ${add_pkgs[@]} >/dev/null \ - || die "Could not add packages" + || die "repo-add $repo$DBEXT ${add_pkgs[@]}" popd >/dev/null fi done -- cgit v1.2.2 From 8647de4f52e05eadc0a4832813e9a57c20a487dd Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Wed, 18 Aug 2010 11:58:02 +0200 Subject: Fix cleanup of old packages * This should fix the cleanup of packages that were not within the pacakge pool before being updated. * a test case for this was added --- misc-scripts/ftpdir-cleanup-repo | 2 +- test/runTest | 43 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/misc-scripts/ftpdir-cleanup-repo b/misc-scripts/ftpdir-cleanup-repo index 20f297b..bfc971e 100755 --- a/misc-scripts/ftpdir-cleanup-repo +++ b/misc-scripts/ftpdir-cleanup-repo @@ -63,7 +63,7 @@ for arch in ${ARCHES[@]}; do cd "$ftppath" for pkg in $CLEANUP_TMPDIR/*; do - [ -f "${pkg}" ] || continue + [ ! -d "${pkg}" ] && continue filename=$(grep -A1 '^%FILENAME%$' "${pkg}/desc" | tail -n1) if [ ! -e "${filename}" ]; then diff --git a/test/runTest b/test/runTest index 72f9b1c..cfc4ee6 100755 --- a/test/runTest +++ b/test/runTest @@ -395,6 +395,42 @@ testMovePackagesWithoutPool() { done } +testUpdateAnyPackageWithoutPool() { + local pkgname='pkg-any-a' + local pkg1='pkg-any-a-1-1-any.pkg.tar.xz' + local pkg2='pkg-any-a-1-2-any.pkg.tar.xz' + local arch + + + releasePackage extra pkg-any-a any + ../db-update + # transform two packages to old style layout + mv -f "${FTP_BASE}/$(get_pkgpool_for_host)/${pkg1}" "${FTP_BASE}/extra/os/any" + for arch in i686 x86_64; do + ln -sf "../any/${pkg1}" "${FTP_BASE}/extra/os/${arch}" + done + + pushd "${TMP}/svn-packages-copy/${pkgname}/trunk/" >/dev/null + sed 's/pkgrel=1/pkgrel=2/g' -i PKGBUILD + svn commit -q -m"update pkg to pkgrel=2" >/dev/null + makepkg -cf --config ${curdir}/lib/makepkg-i686.conf >/dev/null 2>&1 + mv "${pkg2}" "${pkgdir}/${pkgname}/" + popd >/dev/null + + releasePackage extra ${pkgname} any + ../db-update + rm "${pkgdir}/${pkgname}/${pkg2}" + + ../cron-jobs/ftpdir-cleanup >/dev/null + + checkAnyPackage extra "${pkg2}" + + [ -f "${FTP_BASE}/$(get_pkgpool_for_host)/${pkg1}" ] && fail "$(get_pkgpool_for_host)/${pkg1} found" + for arch in any i686 x86_64; do + [ -f "${FTP_BASE}/extra/os/${arch}/${pkg1}" ] && fail "extra/os/${arch}/${pkg1} found" + done +} + testMoveAnyPackagesWithoutPool() { local pkgs=('pkg-any-a' 'pkg-any-b') local pkgbase @@ -428,6 +464,13 @@ testMoveAnyPackagesWithoutPool() { done checkRemovedAnyPackage testing ${pkgbase} done + + for pkg in "${pkgdir}/${pkgs[0]}"/*-any.pkg.tar.*; do + pkg=$(basename $pkg) + for arch in any i686 x86_64; do + [ -f "${FTP_BASE}/testing/os/${arch}/${pkg}" ] && fail "testing/os/${arch}/${pkg} found" + done + done } . "${curdir}/lib/shunit2" -- cgit v1.2.2 From 683db123710ff9092ae99929fce232620932fdb0 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Wed, 18 Aug 2010 16:20:27 +0200 Subject: Rewrite ftpdir-cleanup * runtime reduced to < 1 second * uses file lists and the comm command * searches for missing packages * removes empty legacy directories like extra/os/any --- cron-jobs/ftpdir-cleanup | 116 +++++++++++++++++++----- misc-scripts/ftpdir-cleanup-repo | 187 --------------------------------------- 2 files changed, 92 insertions(+), 211 deletions(-) delete mode 100755 misc-scripts/ftpdir-cleanup-repo diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index 3456e9c..0a43bd1 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -3,37 +3,105 @@ . "$(dirname $0)/../db-functions" . "$(dirname $0)/../config" -repos="$(get_repos_for_host)" +clean_pkg() { + local pkg + local target + + if ! ${CLEANUP_DRYRUN}; then + for pkg in "$@"; do + if [ -h "$pkg" ]; then + rm -f "$pkg" + else + mv -f "$pkg" "$CLEANUP_DESTDIR" + fi + done + fi +} + +repos=($(get_repos_for_host)) script_lock -#adjust the nice level to run at a lower priority -/usr/bin/renice +10 -p $$ > /dev/null +for repo in ${repos[@]}; do + for arch in ${ARCHES[@]}; do + repo_lock ${repo} ${arch} || exit 1 + done +done + +${CLEANUP_DRYRUN} && warning 'dry run mode is active' -repopaths='' -for repo in $repos; do - $(dirname $0)/../misc-scripts/ftpdir-cleanup-repo $repo - repopaths="${repopaths} ${FTP_BASE}/${repo}/os/" +for repo in ${repos[@]}; do + for arch in ${ARCHES[@]}; do + if [ ! -f "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" ]; then + warning "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT} not found, skipping" + continue + fi + # get a list of actual available package files + find "${FTP_BASE}/${repo}/os/${arch}" -xtype f -name "*${PKGEXT}" -printf '%f\n' | sort > "${WORKDIR}/repo-${repo}-${arch}" + # get a list of package files defined in the repo db + bsdtar -xOf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" | awk '/^%FILENAME%/{getline;print}' | sort > "${WORKDIR}/db-${repo}-${arch}" + + missing_pkgs=($(comm -13 "${WORKDIR}/repo-${repo}-${arch}" "${WORKDIR}/db-${repo}-${arch}")) + if [ ${#missing_pkgs[@]} -ge 1 ]; then + error "Missing packages in [${repo}] (${arch})..." + for missing_pkg in ${missing_pkgs[@]}; do + msg2 "${missing_pkg}" + done + fi + + old_pkgs=($(comm -23 "${WORKDIR}/repo-${repo}-${arch}" "${WORKDIR}/db-${repo}-${arch}")) + if [ ${#old_pkgs[@]} -ge 1 ]; then + msg "Removing old packages from [${repo}] (${arch})..." + for old_pkg in ${old_pkgs[@]}; do + msg2 "${old_pkg}" + clean_pkg "${FTP_BASE}/${repo}/os/${arch}/${old_pkg}" + done + fi + done done -to_cleanup="" -poolpath="$FTP_BASE/$(get_pkgpool_for_host)/" -pushd $poolpath >/dev/null -for pkg in *$PKGEXT; do -[ -f "$pkg" ] || continue # in case we get a file named "*.pkg.tar.gz" -LINKS="$(/usr/bin/find $repopaths -type l -name "$pkg" 2>/dev/null)" -if [ -z "$LINKS" ]; then - to_cleanup="$to_cleanup $poolpath/$pkg" +# get a list of all available packages in the pacakge pool +find "$FTP_BASE/$(get_pkgpool_for_host)" -name "*${PKGEXT}" -printf '%f\n' | sort > "${WORKDIR}/pool" +# create a list of packages in our db +cat "${WORKDIR}/db-"* | sort -u > "${WORKDIR}/db" + +old_pkgs=($(comm -23 "${WORKDIR}/pool" "${WORKDIR}/db")) +if [ ${#old_pkgs[@]} -ge 1 ]; then + msg "Removing old packages from package pool..." + for old_pkg in ${old_pkgs[@]}; do + msg2 "${old_pkg}" + clean_pkg "$FTP_BASE/$(get_pkgpool_for_host)/${old_pkg}" + done fi + +# cleanup of legacy $repo/os/any directories +for repo in ${repos[@]}; do + if [ ! -d "${FTP_BASE}/${repo}/os/any" ]; then + continue + fi + if [ -n "$(find "${FTP_BASE}/${repo}/os/any" -type d -empty)" ]; then + msg "Removing empty legacy directory ${repo}/os/any" + ${CLEANUP_DRYRUN} && rmdir "${FTP_BASE}/${repo}/os/any" + continue + fi + find "${FTP_BASE}/${repo}/os/any" -name "*${PKGEXT}" -printf '%f\n' | sort > "${WORKDIR}/any-${repo}" + cat "${WORKDIR}/db-${repo}-"* | sort -u > "${WORKDIR}/all-${repo}" + + old_pkgs=($(comm -23 "${WORKDIR}/any-${repo}" "${WORKDIR}/all-${repo}")) + if [ ${#old_pkgs[@]} -ge 1 ]; then + msg "Removing old packages from [${repo}] (any)..." + for old_pkg in ${old_pkgs[@]}; do + msg2 "${old_pkg}" + clean_pkg "${FTP_BASE}/${repo}/os/any/${old_pkg}" + done + fi +done + + +for repo in ${repos[@]}; do + for arch in ${ARCHES[@]}; do + repo_unlock ${repo} ${arch} + done done -popd >/dev/null - -if [ -n "$to_cleanup" ]; then - msg "The following packages are no longer in any repo" - for f in $to_cleanup; do - msg2 "$(basename "$f")" - done - ${CLEANUP_DRYRUN} || mv $to_cleanup "$CLEANUP_DESTDIR" -fi script_unlock diff --git a/misc-scripts/ftpdir-cleanup-repo b/misc-scripts/ftpdir-cleanup-repo deleted file mode 100755 index bfc971e..0000000 --- a/misc-scripts/ftpdir-cleanup-repo +++ /dev/null @@ -1,187 +0,0 @@ -#!/bin/bash - -if [ $# -ne 1 ]; then - msg "usage: $(basename $0) " - exit 1 -fi - -reponame=$1 - -############################################################ - -. "$(dirname $0)/../db-functions" -. "$(dirname $0)/../config" - -clean_pkgs () { - if ! ${CLEANUP_DRYRUN}; then - for pkg in "$@"; do - if [ -h "$pkg" ]; then - rm -f "$pkg" - else - mv "$pkg" "$CLEANUP_DESTDIR" - fi - done - fi -} - -${CLEANUP_DRYRUN} && warning 'dry run mode is active' - -ftppath_base="$FTP_BASE/$reponame/os" - -for arch in ${ARCHES[@]}; do - - repo_lock $reponame $arch $LOCK_TIMEOUT || continue - - CLEANUP_TMPDIR=$(mktemp -d ${WORKDIR}/cleanup-XXXXXX) || exit 1 - ftppath="$ftppath_base/$arch" - MISSINGFILES="" - DELETEFILES="" - DELETESYMLINKS="" - EXTRAFILES="" - - if [ ! -d "$ftppath" ]; then - error "FTP path '$ftppath' does not exist" - exit 1 - fi - - if ! cd "${CLEANUP_TMPDIR}" ; then - error "Failed to cd to ${CLEANUP_TMPDIR}" - exit 1 - fi - - if [ ! -f "$ftppath/$reponame$DBEXT" ]; then - msg "The file \"$ftppath/$reponame$DBEXT\" could not be found, skipping." - repo_unlock $reponame $arch - continue - fi - - if ! bsdtar xf "$ftppath/$reponame$DBEXT"; then - error "Command failed: bsdtar xf \"$ftppath/$reponame$DBEXT\"" - exit 1 - fi - - cd "$ftppath" - - for pkg in $CLEANUP_TMPDIR/*; do - [ ! -d "${pkg}" ] && continue - filename=$(grep -A1 '^%FILENAME%$' "${pkg}/desc" | tail -n1) - - if [ ! -e "${filename}" ]; then - MISSINGFILES="${MISSINGFILES} ${filename}" - else - pkgname="$(getpkgname ${filename})" - for otherfile in ${pkgname}-*; do - if [ "${otherfile}" != "${filename}" -a "${pkgname}" = "$(getpkgname ${otherfile})" ]; then - if [ -h "${otherfile}" ]; then - DELETESYMLINKS="${DELETESYMLINKS} ${otherfile}" - else - DELETEFILES="${DELETEFILES} ${otherfile}" - fi - fi - done - fi - done - - for pkg in *$PKGEXT; do - if [ ! -e "$pkg" ]; then - continue - fi - pkgname="$(getpkgname $pkg)" - for p in ${CLEANUP_TMPDIR}/${pkgname}-*; do - [ ! -d "${p}" ] || continue 2 - dbpkgname=$(grep -A1 '^%FILENAME%$' "${p}/desc" 2>/dev/null| tail -n1) - if [ "${dbpkgname}" = "${pkgname}" ]; then - continue 2 - fi - done - EXTRAFILES="$EXTRAFILES $pkg" - done - - rm -rf ${CLEANUP_TMPDIR} - - # Do a quick check to see if a missing ARCHINDEPFILE is in the any dir - # If it is, and the file is MISSING, restore it - missfiles="$MISSINGFILES" - MISSINGFILES="" - for mf in $missfiles; do - if [ -e "${ftppath_base}/any/${mf}" ]; then - msg "Restoring missing 'any' symlink: ${mf}" - ${CLEANUP_DRYRUN} || ln -s "../any/${mf}" "${ftppath}" - else - MISSINGFILES="${MISSINGFILES} ${mf}" - fi - done - - repo_unlock $reponame $arch - - #Make sure we've done *something* before outputting anything - if [ -z "$DELETEFILES$DELETESYMLINKS$MISSINGFILES$EXTRAFILES" ]; then - continue - fi - - msg "Scan complete for $reponame ($arch) at ${ftppath}" - - if [ -n "$MISSINGFILES" ]; then - for f in $MISSINGFILES; do - error "$f is missing" - done - fi - - if [ -n "${DELETEFILES}" ]; then - msg "The following files are out of date" - for f in $DELETEFILES; do - msg2 "$f" - done - clean_pkgs ${DELETEFILES} - fi - - if [ -n "${DELETESYMLINKS}" ]; then - msg "The following symlinks are out of date" - for f in $DELETESYMLINKS; do - msg2 "$f" - done - clean_pkgs ${DELETESYMLINKS} - fi - - if [ -n "${EXTRAFILES}" ]; then - msg "The following files are in the repo but not the db" - for f in $EXTRAFILES; do - msg2 "$f" - done - clean_pkgs ${EXTRAFILES} - fi - -done - -ARCHINDEPFILES="" - -if [ -d "$ftppath_base/any" ]; then - cd "$ftppath_base/any" - for pkg in *$PKGEXT; do - [ -f "$pkg" ] || continue # in case we get a file named "*.pkg.tar.gz" - found=0 - #check for any existing symlinks - for arch in ${ARCHES[@]}; do - if [ -h "$ftppath_base/$arch/$pkg" ]; then - found=1 - break - fi - done - if [ $found -eq 0 ]; then - # We found no symlinks to this, delete it - ARCHINDEPFILES="$ARCHINDEPFILES $pkg" - fi - done -fi - -if [ -n "$ARCHINDEPFILES" ]; then - msg "The following architecture independent packages are not symlinked in the architecture repositories." - for f in $ARCHINDEPFILES; do - msg2 "$f" - done -fi - -if [ -d "$ftppath_base/any" -a -n "${ARCHINDEPFILES}" ]; then - cd "$ftppath_base/any" - clean_pkgs ${ARCHINDEPFILES} -fi -- cgit v1.2.2 From b48b6b62b70fda2cc0318668eeda457c0f8d2123 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Wed, 18 Aug 2010 16:38:21 +0200 Subject: fix typo --- cron-jobs/ftpdir-cleanup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index 0a43bd1..5c8776e 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -81,7 +81,7 @@ for repo in ${repos[@]}; do fi if [ -n "$(find "${FTP_BASE}/${repo}/os/any" -type d -empty)" ]; then msg "Removing empty legacy directory ${repo}/os/any" - ${CLEANUP_DRYRUN} && rmdir "${FTP_BASE}/${repo}/os/any" + ${CLEANUP_DRYRUN} || rmdir "${FTP_BASE}/${repo}/os/any" continue fi find "${FTP_BASE}/${repo}/os/any" -name "*${PKGEXT}" -printf '%f\n' | sort > "${WORKDIR}/any-${repo}" -- cgit v1.2.2 From 86ad38857be6646033cf0b98a6debba3da3d7542 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 22 Aug 2010 21:57:04 +0200 Subject: add config.local to .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index b25c15b..01daf28 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ *~ +/config.local -- cgit v1.2.2 From 9e8897aeadb2aa51b70c5fc0ed1ae281b015d146 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 22 Aug 2010 22:01:09 +0200 Subject: add makepkg.conf for [multilib] --- cron-jobs/adjust-permissions | 2 ++ cron-jobs/create-filelists | 2 +- cron-jobs/sourceballs | 2 +- db-functions | 2 +- test/lib/common.inc | 4 ++-- 5 files changed, 7 insertions(+), 5 deletions(-) diff --git a/cron-jobs/adjust-permissions b/cron-jobs/adjust-permissions index 6acfcf2..2cc8b40 100755 --- a/cron-jobs/adjust-permissions +++ b/cron-jobs/adjust-permissions @@ -19,6 +19,8 @@ get_dir_owner() { echo "root:tusers" ;; community-testing) echo "root:tusers" ;; + multilib) + echo "root:tusers" ;; pool/packages) echo "ftp:ftp-extra" ;; pool/community) diff --git a/cron-jobs/create-filelists b/cron-jobs/create-filelists index fd07617..e280f84 100755 --- a/cron-jobs/create-filelists +++ b/cron-jobs/create-filelists @@ -5,7 +5,7 @@ reposdir=${FTP_BASE} targetdir=${FTP_BASE} -repos=(core extra testing community community-testing staging) +repos=(core extra testing community community-testing staging multilib) script_lock diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index a6f7a19..a482eb7 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -5,7 +5,7 @@ ftpbase="${FTP_BASE}" srcbase="${FTP_BASE}/sources" -repos=(core extra testing community community-testing staging) +repos=(core extra testing community community-testing staging multilib) script_lock diff --git a/db-functions b/db-functions index 292e3ff..9f87298 100644 --- a/db-functions +++ b/db-functions @@ -314,7 +314,7 @@ check_pkgsvn() { get_repos_for_host() { if [ "$(hostname)" = "sigurd" ]; then - echo "community community-testing" + echo "community community-testing multilib" else echo "core extra testing staging" fi diff --git a/test/lib/common.inc b/test/lib/common.inc index a79558c..9017aa5 100644 --- a/test/lib/common.inc +++ b/test/lib/common.inc @@ -31,9 +31,9 @@ setUp() { #msg "Using ${TMP}" mkdir -p "${TMP}/"{ftp,tmp,staging,{package,source}-cleanup,svn-{packages,community}-{copy,repo}} - mkdir -p "${TMP}/ftp/"{core,extra,community,testing,community-testing,staging}/os/{i686,any,x86_64} + mkdir -p "${TMP}/ftp/"{core,extra,community,testing,community-testing,staging,multilib}/os/{i686,any,x86_64} mkdir -p "${TMP}/ftp/pool/"{packages,community} - mkdir -p "${TMP}/staging/"{core,extra,community,testing,community-testing,staging} + mkdir -p "${TMP}/staging/"{core,extra,community,testing,community-testing,staging,multilib} msg 'Creating svn repository...' svnadmin create "${TMP}/svn-packages-repo" -- cgit v1.2.2 From e9948d8392d5ab986541c0d7120f1e5c510261ad Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Tue, 24 Aug 2010 12:27:52 +0200 Subject: db-remove: source functions before using them --- db-remove | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/db-remove b/db-remove index 188cc60..8dabb18 100755 --- a/db-remove +++ b/db-remove @@ -1,13 +1,13 @@ #!/bin/bash +. "$(dirname $0)/db-functions" +. "$(dirname $0)/config" + if [ $# -ne 3 ]; then msg "usage: $(basename $0) " exit 1 fi -. "$(dirname $0)/db-functions" -. "$(dirname $0)/config" - pkgbase="$1" repo="$2" arch="$3" -- cgit v1.2.2 From 476cce1c57a62821a2da8f5481e21e3e9c0d68fd Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Tue, 24 Aug 2010 13:10:58 +0200 Subject: db-update: Make sure there are no links in the staging directory --- db-update | 3 +++ 1 file changed, 3 insertions(+) diff --git a/db-update b/db-update index 68450b4..13c7441 100755 --- a/db-update +++ b/db-update @@ -29,6 +29,9 @@ for repo in ${repos[@]}; do pkgs=($(getpkgfiles "${STAGING}/${repo}/"*${PKGEXT})) if [ $? -eq 0 ]; then for pkg in ${pkgs[@]}; do + if [ -h "${pkg}" ]; then + die "Package ${repo}/$(basename ${pkg}) is a symbolic link" + fi if ! check_pkgfile "${pkg}"; then die "Package ${repo}/$(basename ${pkg}) is not consistent with its meta data" fi -- cgit v1.2.2 From 9f695214d0d63f60d67bb5c1caeae50110c03f7a Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Tue, 24 Aug 2010 20:23:30 +0200 Subject: adjust group for multilib repository --- cron-jobs/adjust-permissions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cron-jobs/adjust-permissions b/cron-jobs/adjust-permissions index 2cc8b40..a68584c 100755 --- a/cron-jobs/adjust-permissions +++ b/cron-jobs/adjust-permissions @@ -20,7 +20,7 @@ get_dir_owner() { community-testing) echo "root:tusers" ;; multilib) - echo "root:tusers" ;; + echo "root:multilib" ;; pool/packages) echo "ftp:ftp-extra" ;; pool/community) -- cgit v1.2.2 From 0e9dc9f69e290fe352de7b2bbd288890a78eced5 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Wed, 25 Aug 2010 11:30:42 -0500 Subject: Add update-web-db cronjob This has been untracked by version control for a while on gerolde in both the cron-jobs directory and in /etc/cron.hourly/. Add it here so we can make changes and know what is going on. This is an improved script over what we currently have. It is one script instead of two, and it does things a little smarter with the logging. Signed-off-by: Dan McGee --- cron-jobs/update-web-db | 47 +++++++++++++++++++++++++++++++++++++++++++ cron-jobs/update-web-files-db | 1 + 2 files changed, 48 insertions(+) create mode 100755 cron-jobs/update-web-db create mode 120000 cron-jobs/update-web-files-db diff --git a/cron-jobs/update-web-db b/cron-jobs/update-web-db new file mode 100755 index 0000000..8631a04 --- /dev/null +++ b/cron-jobs/update-web-db @@ -0,0 +1,47 @@ +#!/bin/bash + +# run at nice 5. it can churn quite a bit of cpu after all. +renice +5 -p $$ > /dev/null + +# setup paths +SPATH="/srv/http/archweb" +ARCHES="i686 x86_64" +# having "more important repos" last should make [core] trickle to the top of +# the updates list each hour rather than being overwhelmed by big [extra] and +# [community] updates +REPOS="community-testing multilib community extra testing core" +LOGOUT="/tmp/archweb_update.log" + +# figure out what operation to perform +cmd="$(basename $0)" +if [[ $cmd != "update-web-db" && $cmd != "update-web-files-db" ]]; then + echo "Invalid command name '$cmd' specified!" + exit 1 +fi + +echo "$cmd: Updating DB at $(date)" >> "${LOGOUT}" + +## do the dirty, then output the result to a file. +cd $SPATH +for arch in ${ARCHES}; do + for repo in ${REPOS}; do + echo "Updating ${repo}-${arch}" + case "$cmd" in + update-web-db) + dbfile="/srv/ftp/${repo}/os/${arch}/${repo}.db.tar.gz" + flags="" ;; + update-web-files-db) + dbfile="/srv/ftp/${repo}/os/${arch}/${repo}.files.tar.gz" + flags="--filesonly" ;; + esac + ./manage.py reporead ${flags} ${arch} ${dbfile} + echo "" + done +done >> "${LOGOUT}" 2>&1 + +echo "" >> "${LOGOUT}" + +# rotate the file if it is getting big (> 10M), overwriting any old backup +if [[ $(stat -c%s update-web-db) -gt 10485760 ]]; then + mv "${LOGOUT}" "${LOGOUT}.old" +fi diff --git a/cron-jobs/update-web-files-db b/cron-jobs/update-web-files-db new file mode 120000 index 0000000..0c2c4fa --- /dev/null +++ b/cron-jobs/update-web-files-db @@ -0,0 +1 @@ +update-web-db \ No newline at end of file -- cgit v1.2.2 From 960917cdb0b23163029dc81e52f295e3281492e1 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Wed, 25 Aug 2010 11:38:10 -0500 Subject: Fix a few issues with update-web-db Signed-off-by: Dan McGee --- cron-jobs/update-web-db | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/cron-jobs/update-web-db b/cron-jobs/update-web-db index 8631a04..4d1519c 100755 --- a/cron-jobs/update-web-db +++ b/cron-jobs/update-web-db @@ -25,7 +25,6 @@ echo "$cmd: Updating DB at $(date)" >> "${LOGOUT}" cd $SPATH for arch in ${ARCHES}; do for repo in ${REPOS}; do - echo "Updating ${repo}-${arch}" case "$cmd" in update-web-db) dbfile="/srv/ftp/${repo}/os/${arch}/${repo}.db.tar.gz" @@ -34,14 +33,17 @@ for arch in ${ARCHES}; do dbfile="/srv/ftp/${repo}/os/${arch}/${repo}.files.tar.gz" flags="--filesonly" ;; esac - ./manage.py reporead ${flags} ${arch} ${dbfile} - echo "" + if [[ -f $dbfile ]]; then + echo "Updating ${repo}-${arch}" + ./manage.py reporead ${flags} ${arch} ${dbfile} + echo "" + fi done -done >> "${LOGOUT}" 2>&1 +done >> "${LOGOUT}" 2>&1 echo "" >> "${LOGOUT}" # rotate the file if it is getting big (> 10M), overwriting any old backup -if [[ $(stat -c%s update-web-db) -gt 10485760 ]]; then +if [[ $(stat -c%s "${LOGOUT}") -gt 10485760 ]]; then mv "${LOGOUT}" "${LOGOUT}.old" fi -- cgit v1.2.2 From 1d6eb15bcc4c355edef1d6c249e9c67bca7d0404 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Wed, 25 Aug 2010 11:39:16 -0500 Subject: update-web-db: reformat to coding style Signed-off-by: Dan McGee --- cron-jobs/update-web-db | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/cron-jobs/update-web-db b/cron-jobs/update-web-db index 4d1519c..3504813 100755 --- a/cron-jobs/update-web-db +++ b/cron-jobs/update-web-db @@ -15,8 +15,8 @@ LOGOUT="/tmp/archweb_update.log" # figure out what operation to perform cmd="$(basename $0)" if [[ $cmd != "update-web-db" && $cmd != "update-web-files-db" ]]; then - echo "Invalid command name '$cmd' specified!" - exit 1 + echo "Invalid command name '$cmd' specified!" + exit 1 fi echo "$cmd: Updating DB at $(date)" >> "${LOGOUT}" @@ -25,19 +25,19 @@ echo "$cmd: Updating DB at $(date)" >> "${LOGOUT}" cd $SPATH for arch in ${ARCHES}; do for repo in ${REPOS}; do - case "$cmd" in - update-web-db) - dbfile="/srv/ftp/${repo}/os/${arch}/${repo}.db.tar.gz" - flags="" ;; - update-web-files-db) - dbfile="/srv/ftp/${repo}/os/${arch}/${repo}.files.tar.gz" - flags="--filesonly" ;; - esac - if [[ -f $dbfile ]]; then - echo "Updating ${repo}-${arch}" - ./manage.py reporead ${flags} ${arch} ${dbfile} - echo "" - fi + case "$cmd" in + update-web-db) + dbfile="/srv/ftp/${repo}/os/${arch}/${repo}.db.tar.gz" + flags="" ;; + update-web-files-db) + dbfile="/srv/ftp/${repo}/os/${arch}/${repo}.files.tar.gz" + flags="--filesonly" ;; + esac + if [[ -f $dbfile ]]; then + echo "Updating ${repo}-${arch}" + ./manage.py reporead ${flags} ${arch} ${dbfile} + echo "" + fi done done >> "${LOGOUT}" 2>&1 @@ -45,5 +45,7 @@ echo "" >> "${LOGOUT}" # rotate the file if it is getting big (> 10M), overwriting any old backup if [[ $(stat -c%s "${LOGOUT}") -gt 10485760 ]]; then - mv "${LOGOUT}" "${LOGOUT}.old" + mv "${LOGOUT}" "${LOGOUT}.old" fi + +# vim: set ts=4 sw=4 et ft=sh: -- cgit v1.2.2 From 3c07e2e3ff951242a5d9fc124c6937e75e961da2 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Wed, 25 Aug 2010 11:39:57 -0500 Subject: update-web-db: flip the arch/repo loops This will really make sure [core] packages flow to the top of the list, regardless of architecture. They should anyway, but anything that was only updated in the first architecture updated could get artificially pushed down before. Signed-off-by: Dan McGee --- cron-jobs/update-web-db | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cron-jobs/update-web-db b/cron-jobs/update-web-db index 3504813..8f91ca2 100755 --- a/cron-jobs/update-web-db +++ b/cron-jobs/update-web-db @@ -23,8 +23,8 @@ echo "$cmd: Updating DB at $(date)" >> "${LOGOUT}" ## do the dirty, then output the result to a file. cd $SPATH -for arch in ${ARCHES}; do - for repo in ${REPOS}; do +for repo in ${REPOS}; do + for arch in ${ARCHES}; do case "$cmd" in update-web-db) dbfile="/srv/ftp/${repo}/os/${arch}/${repo}.db.tar.gz" -- cgit v1.2.2 From 328b1ce478e25902aba5d8f19024dadeaaf3f678 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Wed, 1 Sep 2010 19:34:47 +0200 Subject: Simplify repo configuration * Repositories can now be defined in the config file for each host * added community-staging, gnome-unstable and kde-unstable * Exception is the adjust-permission cron-job; but we might want to use acls in future anyway Signed-off-by: Pierre Schmitz --- config | 8 ++++++++ cron-jobs/adjust-permissions | 6 ++++++ cron-jobs/create-filelists | 3 +-- cron-jobs/integrity-check | 2 +- cron-jobs/sourceballs | 3 +-- db-functions | 12 ++++++------ test/lib/common.inc | 16 +++++++++++++--- 7 files changed, 36 insertions(+), 14 deletions(-) diff --git a/config b/config index f6600a6..f0d6cfc 100644 --- a/config +++ b/config @@ -4,6 +4,14 @@ FTP_BASE="/srv/ftp" SVNREPO="file:///srv/svn-packages" SVNREPOCOMMUNITY="file:///srv/svn-community" +declare -A PKGREPO +PKGREPO['default']='core extra testing staging kde-unstable gnome-unstable' +PKGREPO['sigurd']='community community-testing community-staging multilib' + +declare -A PKGPOOL +PKGPOOL['default']='pool/packages' +PKGPOOL['sigurd']='pool/community' + CLEANUP_DESTDIR="/srv/package-cleanup" CLEANUP_DRYRUN=false SOURCE_CLEANUP_DESTDIR="/srv/source-cleanup" diff --git a/cron-jobs/adjust-permissions b/cron-jobs/adjust-permissions index a68584c..e0b6e4e 100755 --- a/cron-jobs/adjust-permissions +++ b/cron-jobs/adjust-permissions @@ -15,10 +15,16 @@ get_dir_owner() { echo "ftp:ftp-extra" ;; staging) echo "ftp:ftp-extra" ;; + kde-unstable) + echo "ftp:ftp-extra" ;; + gnome-unstable) + echo "ftp:ftp-extra" ;; community) echo "root:tusers" ;; community-testing) echo "root:tusers" ;; + community-staging) + echo "root:tusers" ;; multilib) echo "root:multilib" ;; pool/packages) diff --git a/cron-jobs/create-filelists b/cron-jobs/create-filelists index e280f84..a37080f 100755 --- a/cron-jobs/create-filelists +++ b/cron-jobs/create-filelists @@ -5,7 +5,6 @@ reposdir=${FTP_BASE} targetdir=${FTP_BASE} -repos=(core extra testing community community-testing staging multilib) script_lock @@ -28,7 +27,7 @@ esac FILESEXT="${DBEXT//db/files}" -for repo in ${repos[@]}; do +for repo in ${PKGREPO[@]}; do REPO_DB_FILE="${repo}$DBEXT" FILES_DB_FILE="${repo}$FILESEXT" for arch in ${ARCHES[@]}; do diff --git a/cron-jobs/integrity-check b/cron-jobs/integrity-check index 0b2b9c7..c120cfd 100755 --- a/cron-jobs/integrity-check +++ b/cron-jobs/integrity-check @@ -7,7 +7,7 @@ if [ $# -ne 2 ]; then exit 1 fi -for arch in i686 x86_64; do +for arch in ${ARCHES[@]}; do $basedir/check_archlinux/check_packages.py \ --repos="$1" \ --abs-tree="/srv/abs/rsync/$arch,/srv/abs/rsync/any" --arch="$arch" |\ diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index a482eb7..0cf062f 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -5,7 +5,6 @@ ftpbase="${FTP_BASE}" srcbase="${FTP_BASE}/sources" -repos=(core extra testing community community-testing staging multilib) script_lock @@ -18,7 +17,7 @@ FAILED_PKGS="" [ -e "$srcbase/errors.txt" ] && /bin/mv "$srcbase/errors.txt" "$srcbase/errors.txt.old" echo "Errors occured during run:" > "$srcbase/errors.txt" -for repo in ${repos[@]}; do +for repo in ${PKGREPO[@]}; do for arch in ${ARCHES[@]} any; do ftppath="$ftpbase/$repo/os/$arch" if [ ! -d "$ftppath" ]; then diff --git a/db-functions b/db-functions index 9f87298..28928da 100644 --- a/db-functions +++ b/db-functions @@ -313,18 +313,18 @@ check_pkgsvn() { } get_repos_for_host() { - if [ "$(hostname)" = "sigurd" ]; then - echo "community community-testing multilib" + if [ -n "${PKGREPO[$(hostname -s)]}" ]; then + echo "${PKGREPO[$(hostname -s)]}" else - echo "core extra testing staging" + echo "${PKGREPO['default']}" fi } get_pkgpool_for_host() { - if [ "$(hostname)" = "sigurd" ]; then - echo "pool/community" + if [ -n "${PKGPOOL[$(hostname -s)]}" ]; then + echo "${PKGPOOL[$(hostname -s)]}" else - echo "pool/packages" + echo "${PKGPOOL['default']}" fi } diff --git a/test/lib/common.inc b/test/lib/common.inc index 9017aa5..71dfd3d 100644 --- a/test/lib/common.inc +++ b/test/lib/common.inc @@ -1,5 +1,6 @@ set -E +. "${curdir}/../config" . "${curdir}/../db-functions" oneTimeSetUp() { @@ -25,15 +26,24 @@ oneTimeTearDown() { setUp() { local p local pkg + local r + local a #[ -f "${curdir}/../config.local" ] && die "${curdir}/../config.local exists" TMP="$(mktemp -d /dev/shm/$(basename $0).XXXXXXXXXX)" #msg "Using ${TMP}" mkdir -p "${TMP}/"{ftp,tmp,staging,{package,source}-cleanup,svn-{packages,community}-{copy,repo}} - mkdir -p "${TMP}/ftp/"{core,extra,community,testing,community-testing,staging,multilib}/os/{i686,any,x86_64} - mkdir -p "${TMP}/ftp/pool/"{packages,community} - mkdir -p "${TMP}/staging/"{core,extra,community,testing,community-testing,staging,multilib} + + for r in ${PKGREPO[@]}; do + mkdir -p "${TMP}/staging/${r}" + for a in ${ARCHES[@]} any; do + mkdir -p "${TMP}/ftp/${r}/os/${a}" + done + done + for p in ${PKGPOOL[@]}; do + mkdir -p "${TMP}/ftp/${p}" + done msg 'Creating svn repository...' svnadmin create "${TMP}/svn-packages-repo" -- cgit v1.2.2 From df78f6e877325ddfd9569e9704612ee2509791b1 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Thu, 2 Sep 2010 00:45:38 -0500 Subject: Simplify case statement Don't duplicate the user:group pair all over the place; put all of the associated repos and directories in the same case statement for a given set of credentials. Signed-off-by: Dan McGee Signed-off-by: Pierre Schmitz --- cron-jobs/adjust-permissions | 23 +++++------------------ 1 file changed, 5 insertions(+), 18 deletions(-) diff --git a/cron-jobs/adjust-permissions b/cron-jobs/adjust-permissions index e0b6e4e..6510d79 100755 --- a/cron-jobs/adjust-permissions +++ b/cron-jobs/adjust-permissions @@ -7,30 +7,17 @@ script_lock get_dir_owner() { case $1 in + # on main server, e.g. gerolde core) echo "ftp:ftp-arch" ;; - extra) + extra|testing|staging|kde-unstable|gnome-unstable|pool/packages) echo "ftp:ftp-extra" ;; - testing) - echo "ftp:ftp-extra" ;; - staging) - echo "ftp:ftp-extra" ;; - kde-unstable) - echo "ftp:ftp-extra" ;; - gnome-unstable) - echo "ftp:ftp-extra" ;; - community) - echo "root:tusers" ;; - community-testing) - echo "root:tusers" ;; - community-staging) + + # on community server, e.g. sigurd + community|community-testing|community-staging|pool/community) echo "root:tusers" ;; multilib) echo "root:multilib" ;; - pool/packages) - echo "ftp:ftp-extra" ;; - pool/community) - echo "root:tusers" ;; esac } -- cgit v1.2.2 From 3b903c5c46287cc6f453b3138842715a467c57fb Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Thu, 2 Sep 2010 14:50:04 +0200 Subject: Set correct group after touching the db file * When writing the db file ensure that it has write permission of the group which owns the parent directory. * This should make the adjust-permissions cron job obsolete. --- db-functions | 14 ++++++++++++++ db-move | 3 +++ db-remove | 1 + db-update | 1 + 4 files changed, 19 insertions(+) diff --git a/db-functions b/db-functions index 28928da..7a751bf 100644 --- a/db-functions +++ b/db-functions @@ -387,3 +387,17 @@ check_repo_permission() { return 0 } + +set_repo_permission() { + local repo=$1 + local arch=$2 + local dbfile="${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" + + if [ -w "${dbfile}" ]; then + local group=$(/usr/bin/stat --printf='%G' "$(dirname "${dbfile}")") + chgrp $group "${dbfile}" || error "Could not change group of ${dbfile} to $group" + chmod g+w "${dbfile}" || error "Could not set write permission for group $group to ${dbfile}" + else + error "You don't have permission to change ${dbfile}" + fi +} diff --git a/db-move b/db-move index 5ffb8bc..fe9d2c9 100755 --- a/db-move +++ b/db-move @@ -108,8 +108,11 @@ for tarch in ${ARCHES[@]}; do if [ -n "${add_pkgs[${tarch}]}" ]; then /usr/bin/repo-add -q "${ftppath_to}/${tarch}/${repo_to}${DBEXT}" ${add_pkgs[${tarch}]} >/dev/null \ || error "repo-add ${tarch}/${repo_to}${DBEXT} ${add_pkgs[${tarch}]}" + set_repo_permission "${repo_to}" "${tarch}" + /usr/bin/repo-remove -q "${ftppath_from}/${tarch}/${repo_from}${DBEXT}" ${remove_pkgs[${tarch}]} >/dev/null \ || error "repo-remove ${tarch}/${repo_from}${DBEXT} ${remove_pkgs[${tarch}]}" + set_repo_permission "${repo_from}" "${tarch}" fi done diff --git a/db-remove b/db-remove index 8dabb18..6743eea 100755 --- a/db-remove +++ b/db-remove @@ -48,5 +48,6 @@ for tarch in ${tarches[@]}; do fi /usr/bin/repo-remove -q "$ftppath/$tarch/$repo$DBEXT" ${pkgnames[@]} >/dev/null + set_repo_permission "${repo}" "${tarch}" repo_unlock $repo $tarch done diff --git a/db-update b/db-update index 13c7441..61d1636 100755 --- a/db-update +++ b/db-update @@ -68,6 +68,7 @@ for repo in ${repos[@]}; do /usr/bin/repo-add -q "$repo$DBEXT" ${add_pkgs[@]} >/dev/null \ || die "repo-add $repo$DBEXT ${add_pkgs[@]}" popd >/dev/null + set_repo_permission "${repo}" "${pkgarch}" fi done done -- cgit v1.2.2 From cca13c2b41ba1389ce4187a728e05a376ad3ad0d Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 5 Sep 2010 12:54:37 +0200 Subject: Fix updating of same package into different repositories at aonce See FS#20745 --- db-functions | 8 ++++---- test/runTest | 20 ++++++++++++++++++++ 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/db-functions b/db-functions index 7a751bf..4ac282a 100644 --- a/db-functions +++ b/db-functions @@ -296,14 +296,14 @@ check_pkgsvn() { in_array "${repo}" $(get_repos_for_host) || return 1 - if [ ! -f "${WORKDIR}/pkgbuilds/${pkgbase}" ]; then - mkdir -p "${WORKDIR}/pkgbuilds" + if [ ! -f "${WORKDIR}/pkgbuilds/${repo}/${pkgbase}" ]; then + mkdir -p "${WORKDIR}/pkgbuilds/${repo}" svn export -q "${SVNREPO}/${pkgbase}/repos/${repo}-${pkgarch}/PKGBUILD" \ - "${WORKDIR}/pkgbuilds/${pkgbase}" >/dev/null + "${WORKDIR}/pkgbuilds/${repo}/${pkgbase}" >/dev/null [ $? -ge 1 ] && return 1 fi - local pkgver="$(. "${WORKDIR}/pkgbuilds/${pkgbase}"; echo "${pkgver}-${pkgrel}")" + local pkgver="$(. "${WORKDIR}/pkgbuilds/${repo}/${pkgbase}"; echo "${pkgver}-${pkgrel}")" [ -z "${pkgver}" ] && return 1 if echo "$(basename ${pkgfile})" | grep -q "${pkgver}-${pkgarch}"; then return 0 diff --git a/test/runTest b/test/runTest index cfc4ee6..e254de0 100755 --- a/test/runTest +++ b/test/runTest @@ -106,6 +106,26 @@ testUpdateAnyPackage() { rm "${pkgdir}/pkg-any-a/pkg-any-a-1-2-any.pkg.tar.xz" } +testUpdateAnyPackageInDifferentRepositoriesAtOnce() { + releasePackage extra pkg-any-a any + + pushd "${TMP}/svn-packages-copy/pkg-any-a/trunk/" >/dev/null + sed 's/pkgrel=1/pkgrel=2/g' -i PKGBUILD + svn commit -q -m"update pkg to pkgrel=2" >/dev/null + makepkg -cf --config ${curdir}/lib/makepkg-i686.conf >/dev/null 2>&1 + mv pkg-any-a-1-2-any.pkg.tar.xz "${pkgdir}/pkg-any-a/" + popd >/dev/null + + releasePackage testing pkg-any-a any + + ../db-update + + checkAnyPackage extra pkg-any-a-1-1-any.pkg.tar.xz any + checkAnyPackage testing pkg-any-a-1-2-any.pkg.tar.xz any + + rm "${pkgdir}/pkg-any-a/pkg-any-a-1-2-any.pkg.tar.xz" +} + # # db-remove # -- cgit v1.2.2 From 6f29ee2f02c2a8e2991599ce1d76a59b58a7ee67 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 5 Sep 2010 13:32:27 +0200 Subject: Check if package exists in any other repository on update This also checks if the sam package exists within the old package layout (without package pool) --- db-functions | 25 +++++++++++++++++++++++++ db-update | 4 ++-- test/runTest | 45 ++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 71 insertions(+), 3 deletions(-) diff --git a/db-functions b/db-functions index 4ac282a..4081d54 100644 --- a/db-functions +++ b/db-functions @@ -312,6 +312,31 @@ check_pkgsvn() { fi } +check_pkgrepos() { + local pkgfile=$1 + + local pkgname="$(getpkgname ${pkgfile})" + [ $? -ge 1 ] && return 1 + local pkgver="$(getpkgver ${pkgfile})" + [ $? -ge 1 ] && return 1 + + if [ -f "${FTP_BASE}/$(get_pkgpool_for_host)/${pkgname}-${pkgver}-"*${PKGEXT} ]; then + return 1 + fi + + local repo + local arch + for repo in $(get_repos_for_host); do + for arch in ${ARCHES[@]}; do + if [ -f "${FTP_BASE}/${repo}/os/${arch}/${pkgname}-${pkgver}"*${PKGEXT} ]; then + return 1 + fi + done + done + + return 0 +} + get_repos_for_host() { if [ -n "${PKGREPO[$(hostname -s)]}" ]; then echo "${PKGREPO[$(hostname -s)]}" diff --git a/db-update b/db-update index 61d1636..70f0525 100755 --- a/db-update +++ b/db-update @@ -38,8 +38,8 @@ for repo in ${repos[@]}; do if ! check_pkgsvn "${pkg}" "${repo}"; then die "Package ${repo}/$(basename ${pkg}) is not consistent with svn repository" fi - if [ -f "${FTP_BASE}/$(get_pkgpool_for_host)/$(basename ${pkg})" ]; then - die "Package ${repo}/$(basename ${pkg}) already exists in ${FTP_BASE}/$(get_pkgpool_for_host)" + if ! check_pkgrepos "${pkg}"; then + die "Package ${repo}/$(basename ${pkg}) already exists in another repository" fi done else diff --git a/test/runTest b/test/runTest index e254de0..b07b099 100755 --- a/test/runTest +++ b/test/runTest @@ -106,7 +106,7 @@ testUpdateAnyPackage() { rm "${pkgdir}/pkg-any-a/pkg-any-a-1-2-any.pkg.tar.xz" } -testUpdateAnyPackageInDifferentRepositoriesAtOnce() { +testUpdateAnyPackageToDifferentRepositoriesAtOnce() { releasePackage extra pkg-any-a any pushd "${TMP}/svn-packages-copy/pkg-any-a/trunk/" >/dev/null @@ -126,6 +126,22 @@ testUpdateAnyPackageInDifferentRepositoriesAtOnce() { rm "${pkgdir}/pkg-any-a/pkg-any-a-1-2-any.pkg.tar.xz" } +testUpdateSameAnyPackageToDifferentRepositories() { + releasePackage extra pkg-any-a any + ../db-update + checkAnyPackage extra pkg-any-a-1-1-any.pkg.tar.xz any + + releasePackage testing pkg-any-a any + ../db-update >/dev/null 2>&1 && (fail 'Adding an existing package to another repository should fail'; return 1) + + local arch + for arch in i686 x86_64; do + ( [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" ] \ + && bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" -O | grep -q ${pkgbase}) \ + && fail "${pkgbase} should not be in ${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" + done +} + # # db-remove # @@ -493,4 +509,31 @@ testMoveAnyPackagesWithoutPool() { done } +testUpdateSameAnyPackageToDifferentRepositoriesWithoutPool() { + local pkg + local arch + + releasePackage extra pkg-any-a any + ../db-update + + # transform a package to old style layout + for pkg in "${pkgdir}/pkg-any-a"/*-any.pkg.tar.*; do + pkg=$(basename $pkg) + mv -f "${FTP_BASE}/$(get_pkgpool_for_host)/${pkg}" "${FTP_BASE}/extra/os/any/${pkg}" + for arch in i686 x86_64; do + ln -sf "../any/${pkg}" "${FTP_BASE}/extra/os/${arch}/${pkg}" + done + done + + releasePackage testing pkg-any-a any + ../db-update >/dev/null 2>&1 && (fail 'Adding an existing package to another repository should fail'; return 1) + + for arch in i686 x86_64; do + ( [ -r "${FTP_BASE}/testing/os/${arch}/testing${DBEXT%.tar.*}" ] \ + && bsdtar -xf "${FTP_BASE}/testing/os/${arch}/testing${DBEXT%.tar.*}" -O | grep -q pkg-any-a) \ + && fail "pkg-any-a should not be in testing/os/${arch}/testing${DBEXT%.tar.*}" + done +} + + . "${curdir}/lib/shunit2" -- cgit v1.2.2 From 7db155d48cd5e6120edd0de1264133ddcc5f249f Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sat, 11 Sep 2010 18:31:23 +0200 Subject: adding multilib-testing repository --- config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config b/config index f0d6cfc..a595331 100644 --- a/config +++ b/config @@ -6,7 +6,7 @@ SVNREPOCOMMUNITY="file:///srv/svn-community" declare -A PKGREPO PKGREPO['default']='core extra testing staging kde-unstable gnome-unstable' -PKGREPO['sigurd']='community community-testing community-staging multilib' +PKGREPO['sigurd']='community community-testing community-staging multilib multilib-testing' declare -A PKGPOOL PKGPOOL['default']='pool/packages' -- cgit v1.2.2 From e24a59e12360c4c807dde8c24f26035b4cd24287 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sat, 11 Sep 2010 18:33:21 +0200 Subject: Remove obsolete cron job --- cron-jobs/adjust-permissions | 41 ----------------------------------------- 1 file changed, 41 deletions(-) delete mode 100755 cron-jobs/adjust-permissions diff --git a/cron-jobs/adjust-permissions b/cron-jobs/adjust-permissions deleted file mode 100755 index 6510d79..0000000 --- a/cron-jobs/adjust-permissions +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash - -script_lock - -. "$(dirname $0)/../db-functions" -. "$(dirname $0)/../config" - -get_dir_owner() { - case $1 in - # on main server, e.g. gerolde - core) - echo "ftp:ftp-arch" ;; - extra|testing|staging|kde-unstable|gnome-unstable|pool/packages) - echo "ftp:ftp-extra" ;; - - # on community server, e.g. sigurd - community|community-testing|community-staging|pool/community) - echo "root:tusers" ;; - multilib) - echo "root:multilib" ;; - esac -} - -#adjust the nice level to run at a lower priority -/usr/bin/renice +10 -p $$ > /dev/null - -pushd $FTP_BASE >/dev/null -for d in $(get_repos_for_host); do - owner="$(get_dir_owner $d)" - /bin/chown -R $owner $d/os/{any,i686,x86_64} - /bin/chmod -R g+w $d/os/{any,i686,x86_64} -done -for p in $(get_pkgpool_for_host); do - owner="$(get_dir_owner $p)" - /bin/chown $owner $p - /bin/chmod g+w $p -done -/bin/chmod 555 $FTP_BASE -popd >/dev/null - -script_unlock -- cgit v1.2.2 From 8768380b62525537e55def81c3273eeb4f9a88d3 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sat, 11 Sep 2010 19:54:00 +0200 Subject: Fix check_repo_permission --- db-functions | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/db-functions b/db-functions index 4081d54..18029ee 100644 --- a/db-functions +++ b/db-functions @@ -402,12 +402,10 @@ check_repo_permission() { [ -w "$FTP_BASE/$(get_pkgpool_for_host)" ] || return 1 local arch - for arch in ${ARCHES} any; do - local w - local ws=("${FTP_BASE}/${repo}/os/${arch}/"{,${repo}${DBEXT}}) - for w in ws; do - [ -w ] || return 1 - done + for arch in ${ARCHES}; do + local dir="${FTP_BASE}/${repo}/os/${arch}/" + [ -w "${dir}" ] || return 1 + [ -f "${dir}"${repo}${DBEXT} -a ! -w "${dir}"${repo}${DBEXT} ] && return 1 done return 0 -- cgit v1.2.2 From f121126f8166fb6dc261ea82f2890ba6693d047e Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sat, 11 Sep 2010 22:52:23 +0200 Subject: Use local config instead of guessing by hostname Using the hostname to decide which repos to use is not releiable and hard to test. Instead use config.local to configure these. config files for sigurd and gerolde were added which can be copied or symlinked to config.local on the specific host. --- config | 10 ++-------- config.local.gerolde | 2 ++ config.local.sigurd | 2 ++ cron-jobs/ftpdir-cleanup | 14 ++++++-------- db-functions | 29 ++++++++--------------------- db-move | 8 ++++---- db-update | 4 ++-- test/lib/common.inc | 26 ++++++++++++++------------ test/runTest | 16 ++++++++-------- 9 files changed, 48 insertions(+), 63 deletions(-) create mode 100644 config.local.gerolde create mode 100644 config.local.sigurd diff --git a/config b/config index a595331..89e89fd 100644 --- a/config +++ b/config @@ -1,16 +1,10 @@ - FTP_BASE="/srv/ftp" SVNREPO="file:///srv/svn-packages" SVNREPOCOMMUNITY="file:///srv/svn-community" -declare -A PKGREPO -PKGREPO['default']='core extra testing staging kde-unstable gnome-unstable' -PKGREPO['sigurd']='community community-testing community-staging multilib multilib-testing' - -declare -A PKGPOOL -PKGPOOL['default']='pool/packages' -PKGPOOL['sigurd']='pool/community' +PKGREPOS=() +PKGPOOL='' CLEANUP_DESTDIR="/srv/package-cleanup" CLEANUP_DRYRUN=false diff --git a/config.local.gerolde b/config.local.gerolde new file mode 100644 index 0000000..dd8ba86 --- /dev/null +++ b/config.local.gerolde @@ -0,0 +1,2 @@ +PKGREPOS=('core' 'extra' 'testing' 'staging' 'kde-unstable' 'gnome-unstable') +PKGPOOL='pool/packages' diff --git a/config.local.sigurd b/config.local.sigurd new file mode 100644 index 0000000..c8c3c9f --- /dev/null +++ b/config.local.sigurd @@ -0,0 +1,2 @@ +PKGREPOS=('community' 'community-testing' 'community-staging' 'multilib' 'multilib-testing') +PKGPOOL='pool/community' diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index 5c8776e..914ef8c 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -18,11 +18,9 @@ clean_pkg() { fi } -repos=($(get_repos_for_host)) - script_lock -for repo in ${repos[@]}; do +for repo in ${PKGREPOS[@]}; do for arch in ${ARCHES[@]}; do repo_lock ${repo} ${arch} || exit 1 done @@ -30,7 +28,7 @@ done ${CLEANUP_DRYRUN} && warning 'dry run mode is active' -for repo in ${repos[@]}; do +for repo in ${PKGREPOS[@]}; do for arch in ${ARCHES[@]}; do if [ ! -f "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" ]; then warning "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT} not found, skipping" @@ -61,7 +59,7 @@ for repo in ${repos[@]}; do done # get a list of all available packages in the pacakge pool -find "$FTP_BASE/$(get_pkgpool_for_host)" -name "*${PKGEXT}" -printf '%f\n' | sort > "${WORKDIR}/pool" +find "$FTP_BASE/${PKGPOOL}" -name "*${PKGEXT}" -printf '%f\n' | sort > "${WORKDIR}/pool" # create a list of packages in our db cat "${WORKDIR}/db-"* | sort -u > "${WORKDIR}/db" @@ -70,12 +68,12 @@ if [ ${#old_pkgs[@]} -ge 1 ]; then msg "Removing old packages from package pool..." for old_pkg in ${old_pkgs[@]}; do msg2 "${old_pkg}" - clean_pkg "$FTP_BASE/$(get_pkgpool_for_host)/${old_pkg}" + clean_pkg "$FTP_BASE/${PKGPOOL}/${old_pkg}" done fi # cleanup of legacy $repo/os/any directories -for repo in ${repos[@]}; do +for repo in ${PKGREPOS[@]}; do if [ ! -d "${FTP_BASE}/${repo}/os/any" ]; then continue fi @@ -98,7 +96,7 @@ for repo in ${repos[@]}; do done -for repo in ${repos[@]}; do +for repo in ${PKGREPOS[@]}; do for arch in ${ARCHES[@]}; do repo_unlock ${repo} ${arch} done diff --git a/db-functions b/db-functions index 18029ee..939f2d5 100644 --- a/db-functions +++ b/db-functions @@ -294,7 +294,7 @@ check_pkgsvn() { [ $? -ge 1 ] && return 1 local repo="${2}" - in_array "${repo}" $(get_repos_for_host) || return 1 + in_array "${repo}" ${PKGREPOS[@]} || return 1 if [ ! -f "${WORKDIR}/pkgbuilds/${repo}/${pkgbase}" ]; then mkdir -p "${WORKDIR}/pkgbuilds/${repo}" @@ -320,13 +320,13 @@ check_pkgrepos() { local pkgver="$(getpkgver ${pkgfile})" [ $? -ge 1 ] && return 1 - if [ -f "${FTP_BASE}/$(get_pkgpool_for_host)/${pkgname}-${pkgver}-"*${PKGEXT} ]; then + if [ -f "${FTP_BASE}/${PKGPOOL}/${pkgname}-${pkgver}-"*${PKGEXT} ]; then return 1 fi local repo local arch - for repo in $(get_repos_for_host); do + for repo in ${PKGREPOS[@]}; do for arch in ${ARCHES[@]}; do if [ -f "${FTP_BASE}/${repo}/os/${arch}/${pkgname}-${pkgver}"*${PKGEXT} ]; then return 1 @@ -337,22 +337,6 @@ check_pkgrepos() { return 0 } -get_repos_for_host() { - if [ -n "${PKGREPO[$(hostname -s)]}" ]; then - echo "${PKGREPO[$(hostname -s)]}" - else - echo "${PKGREPO['default']}" - fi -} - -get_pkgpool_for_host() { - if [ -n "${PKGPOOL[$(hostname -s)]}" ]; then - echo "${PKGPOOL[$(hostname -s)]}" - else - echo "${PKGPOOL['default']}" - fi -} - #usage: chk_license ${license[@]}" chk_license() { local l @@ -397,9 +381,12 @@ pkgver_from_src() { check_repo_permission() { local repo=$1 - in_array "${repo}" $(get_repos_for_host) || return 1 + [ ${#PKGREPOS[@]} -eq 0 ] && return 1 + [ -z "${PKGPOOL}" ] && return 1 + + in_array "${repo}" ${PKGREPOS[@]} || return 1 - [ -w "$FTP_BASE/$(get_pkgpool_for_host)" ] || return 1 + [ -w "$FTP_BASE/${PKGPOOL}" ] || return 1 local arch for arch in ${ARCHES}; do diff --git a/db-move b/db-move index fe9d2c9..cdc261c 100755 --- a/db-move +++ b/db-move @@ -92,11 +92,11 @@ for pkgbase in ${args[@]:2}; do # copy package to pool if needed # TODO: can be removed once every package has been moved to the package pool - if [ ! -f ${FTP_BASE}/$(get_pkgpool_for_host)/${pkgfile} ]; then - cp ${pkgpath} ${FTP_BASE}/$(get_pkgpool_for_host) + if [ ! -f ${FTP_BASE}/${PKGPOOL}/${pkgfile} ]; then + cp ${pkgpath} ${FTP_BASE}/${PKGPOOL} fi - ln -s "../../../$(get_pkgpool_for_host)/${pkgfile}" ${ftppath_to}/${tarch}/ - add_pkgs[${tarch}]+="${FTP_BASE}/$(get_pkgpool_for_host)/${pkgfile} " + ln -s "../../../${PKGPOOL}/${pkgfile}" ${ftppath_to}/${tarch}/ + add_pkgs[${tarch}]+="${FTP_BASE}/${PKGPOOL}/${pkgfile} " remove_pkgs[${tarch}]+="${pkgname} " done done diff --git a/db-update b/db-update index 70f0525..49ecbde 100755 --- a/db-update +++ b/db-update @@ -58,9 +58,9 @@ for repo in ${repos[@]}; do msg2 "${pkgfile} (${pkgarch})" # any packages might have been moved by the previous run if [ -f "${pkg}" ]; then - mv "${pkg}" "$FTP_BASE/$(get_pkgpool_for_host)" + mv "${pkg}" "$FTP_BASE/${PKGPOOL}" fi - ln -s "../../../$(get_pkgpool_for_host)/${pkgfile}" "$FTP_BASE/$repo/os/${pkgarch}" + ln -s "../../../${PKGPOOL}/${pkgfile}" "$FTP_BASE/$repo/os/${pkgarch}" add_pkgs[${#add_pkgs[*]}]=${pkgfile} done if [ ${#add_pkgs[@]} -ge 1 ]; then diff --git a/test/lib/common.inc b/test/lib/common.inc index 71dfd3d..a0d864a 100644 --- a/test/lib/common.inc +++ b/test/lib/common.inc @@ -29,13 +29,15 @@ setUp() { local r local a - #[ -f "${curdir}/../config.local" ] && die "${curdir}/../config.local exists" + [ -f "${curdir}/../config.local" ] && die "${curdir}/../config.local exists" TMP="$(mktemp -d /dev/shm/$(basename $0).XXXXXXXXXX)" #msg "Using ${TMP}" - mkdir -p "${TMP}/"{ftp,tmp,staging,{package,source}-cleanup,svn-{packages,community}-{copy,repo}} + PKGREPOS=('core' 'extra' 'testing') + PKGPOOL='pool/packages' + mkdir -p "${TMP}/"{ftp,tmp,staging,{package,source}-cleanup,svn-packages-{copy,repo}} - for r in ${PKGREPO[@]}; do + for r in ${PKGREPOS[@]}; do mkdir -p "${TMP}/staging/${r}" for a in ${ARCHES[@]} any; do mkdir -p "${TMP}/ftp/${r}/os/${a}" @@ -47,9 +49,7 @@ setUp() { msg 'Creating svn repository...' svnadmin create "${TMP}/svn-packages-repo" - svnadmin create "${TMP}/svn-community-repo" svn checkout -q "file://${TMP}/svn-packages-repo" "${TMP}/svn-packages-copy" - svn checkout -q "file://${TMP}/svn-community-repo" "${TMP}/svn-community-copy" for p in "${pkgdir}"/*; do pkg=$(basename $p) @@ -63,6 +63,8 @@ setUp() { FTP_BASE="${TMP}/ftp" SVNREPO="file://${TMP}/svn-packages-repo" SVNREPOCOMMUNITY="file://${TMP}/svn-community-repo" + PKGREPOS=(${PKGREPOS[@]}) + PKGPOOL="${PKGPOOL}" CLEANUP_DESTDIR="${TMP}/package-cleanup" SOURCE_CLEANUP_DESTDIR="${TMP}/source-cleanup" STAGING="${TMP}/staging" @@ -95,12 +97,12 @@ checkAnyPackage() { local pkg=$2 local arch - [ -r "${FTP_BASE}/$(get_pkgpool_for_host)/${pkg}" ] || fail "$(get_pkgpool_for_host)/${pkg} not found" + [ -r "${FTP_BASE}/${PKGPOOL}/${pkg}" ] || fail "${PKGPOOL}/${pkg} not found" for arch in i686 x86_64; do [ -L "${FTP_BASE}/${repo}/os/${arch}/${pkg}" ] || fail "${repo}/os/${arch}/${pkg} not a symlink" - [ "$(readlink -e "${FTP_BASE}/${repo}/os/${arch}/${pkg}")" == "${FTP_BASE}/$(get_pkgpool_for_host)/${pkg}" ] \ - || fail "${repo}/os/${arch}/${pkg} does not link to $(get_pkgpool_for_host)/${pkg}" + [ "$(readlink -e "${FTP_BASE}/${repo}/os/${arch}/${pkg}")" == "${FTP_BASE}/${PKGPOOL}/${pkg}" ] \ + || fail "${repo}/os/${arch}/${pkg} does not link to ${PKGPOOL}/${pkg}" done [ -r "${STAGING}"/${repo}/${pkg} ] && fail "${repo}/${pkg} found in staging dir" @@ -116,18 +118,18 @@ checkPackage() { local pkg=$2 local arch=$3 - [ -r "${FTP_BASE}/$(get_pkgpool_for_host)/${pkg}" ] || fail "$(get_pkgpool_for_host)/${pkg} not found" + [ -r "${FTP_BASE}/${PKGPOOL}/${pkg}" ] || fail "${PKGPOOL}/${pkg} not found" [ -L "${FTP_BASE}/${repo}/os/${arch}/${pkg}" ] || fail "${repo}/os/${arch}/${pkg} not a symlink" [ -r "${STAGING}"/${repo}/${pkg} ] && fail "${repo}/${pkg} found in staging dir" - [ "$(readlink -e "${FTP_BASE}/${repo}/os/${arch}/${pkg}")" == "${FTP_BASE}/$(get_pkgpool_for_host)/${pkg}" ] \ - || fail "${repo}/os/${arch}/${pkg} does not link to $(get_pkgpool_for_host)/${pkg}" + [ "$(readlink -e "${FTP_BASE}/${repo}/os/${arch}/${pkg}")" == "${FTP_BASE}/${PKGPOOL}/${pkg}" ] \ + || fail "${repo}/os/${arch}/${pkg} does not link to ${PKGPOOL}/${pkg}" ( [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" ] \ && bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" -O | grep -q ${pkg}) \ || fail "${pkg} not in ${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" - local pkgbase=$(getpkgbase "${FTP_BASE}/$(get_pkgpool_for_host)/${pkg}") + local pkgbase=$(getpkgbase "${FTP_BASE}/${PKGPOOL}/${pkg}") svn up -q "${TMP}/svn-packages-copy/${pkgbase}" [ -d "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo}-${arch}" ] \ || fail "svn-packages-copy/${pkgbase}/repos/${repo}-${arch} does not exist" diff --git a/test/runTest b/test/runTest index b07b099..4681944 100755 --- a/test/runTest +++ b/test/runTest @@ -321,7 +321,7 @@ testCleanupSimplePackages() { for arch in ${arches[@]}; do local pkg1="pkg-simple-a-1-1-${arch}.pkg.tar.xz" checkRemovedPackage extra 'pkg-simple-a' ${arch} - [ -f "${FTP_BASE}/$(get_pkgpool_for_host)/${pkg1}" ] && fail "$(get_pkgpool_for_host)/${pkg1} found" + [ -f "${FTP_BASE}/${PKGPOOL}/${pkg1}" ] && fail "${PKGPOOL}/${pkg1} found" [ -f "${FTP_BASE}/${repo}/os/${arch}/${pkg1}" ] && fail "${repo}/os/${arch}/${pkg1} found" local pkg2="pkg-simple-b-1-1-${arch}.pkg.tar.xz" @@ -344,7 +344,7 @@ testCleanupAnyPackages() { local pkg1='pkg-any-a-1-1-any.pkg.tar.xz' checkRemovedAnyPackage extra 'pkg-any-a' - [ -f "${FTP_BASE}/$(get_pkgpool_for_host)/${pkg1}" ] && fail "$(get_pkgpool_for_host)/${pkg1} found" + [ -f "${FTP_BASE}/${PKGPOOL}/${pkg1}" ] && fail "${PKGPOOL}/${pkg1} found" [ -f "${FTP_BASE}/${repo}/os/${arch}/${pkg1}" ] && fail "${repo}/os/${arch}/${pkg1} found" local pkg2="pkg-any-b-1-1-${arch}.pkg.tar.xz" @@ -375,7 +375,7 @@ testCleanupSplitPackages() { for arch in ${arches[@]}; do for pkg in "${pkgdir}/${pkgs[0]}"/*-${arch}.pkg.tar.*; do checkRemovedPackage extra ${pkgs[0]} ${arch} - [ -f "${FTP_BASE}/$(get_pkgpool_for_host)/${pkg}" ] && fail "$(get_pkgpool_for_host)/${pkg} found" + [ -f "${FTP_BASE}/${PKGPOOL}/${pkg}" ] && fail "${PKGPOOL}/${pkg} found" [ -f "${FTP_BASE}/${repo}/os/${arch}/${pkg}" ] && fail "${repo}/os/${arch}/${pkg} found" done @@ -410,7 +410,7 @@ testMovePackagesWithoutPool() { for old in 0 2; do for pkg in "${pkgdir}/${pkgs[${old}]}"/*-${arch}.pkg.tar.*; do pkg=$(basename $pkg) - mv -f "${FTP_BASE}/$(get_pkgpool_for_host)/${pkg}" "${FTP_BASE}/testing/os/${arch}/${pkg}" + mv -f "${FTP_BASE}/${PKGPOOL}/${pkg}" "${FTP_BASE}/testing/os/${arch}/${pkg}" done done done @@ -441,7 +441,7 @@ testUpdateAnyPackageWithoutPool() { releasePackage extra pkg-any-a any ../db-update # transform two packages to old style layout - mv -f "${FTP_BASE}/$(get_pkgpool_for_host)/${pkg1}" "${FTP_BASE}/extra/os/any" + mv -f "${FTP_BASE}/${PKGPOOL}/${pkg1}" "${FTP_BASE}/extra/os/any" for arch in i686 x86_64; do ln -sf "../any/${pkg1}" "${FTP_BASE}/extra/os/${arch}" done @@ -461,7 +461,7 @@ testUpdateAnyPackageWithoutPool() { checkAnyPackage extra "${pkg2}" - [ -f "${FTP_BASE}/$(get_pkgpool_for_host)/${pkg1}" ] && fail "$(get_pkgpool_for_host)/${pkg1} found" + [ -f "${FTP_BASE}/${PKGPOOL}/${pkg1}" ] && fail "${PKGPOOL}/${pkg1} found" for arch in any i686 x86_64; do [ -f "${FTP_BASE}/extra/os/${arch}/${pkg1}" ] && fail "extra/os/${arch}/${pkg1} found" done @@ -482,7 +482,7 @@ testMoveAnyPackagesWithoutPool() { # transform a package to old style layout for pkg in "${pkgdir}/${pkgs[0]}"/*-any.pkg.tar.*; do pkg=$(basename $pkg) - mv -f "${FTP_BASE}/$(get_pkgpool_for_host)/${pkg}" "${FTP_BASE}/testing/os/any/${pkg}" + mv -f "${FTP_BASE}/${PKGPOOL}/${pkg}" "${FTP_BASE}/testing/os/any/${pkg}" for arch in i686 x86_64; do ln -sf "../any/${pkg}" "${FTP_BASE}/testing/os/${arch}/${pkg}" done @@ -519,7 +519,7 @@ testUpdateSameAnyPackageToDifferentRepositoriesWithoutPool() { # transform a package to old style layout for pkg in "${pkgdir}/pkg-any-a"/*-any.pkg.tar.*; do pkg=$(basename $pkg) - mv -f "${FTP_BASE}/$(get_pkgpool_for_host)/${pkg}" "${FTP_BASE}/extra/os/any/${pkg}" + mv -f "${FTP_BASE}/${PKGPOOL}/${pkg}" "${FTP_BASE}/extra/os/any/${pkg}" for arch in i686 x86_64; do ln -sf "../any/${pkg}" "${FTP_BASE}/extra/os/${arch}/${pkg}" done -- cgit v1.2.2 From a8832a42d659b58f3494e414e67ea3d7f334c87d Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sat, 11 Sep 2010 23:38:41 +0200 Subject: Fix check_pkgrepos Don't use globing as this will match a package of the same version but another arch. We can be more specific here as there are no longer packages without $arch extension. --- db-functions | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/db-functions b/db-functions index 939f2d5..d9b38e2 100644 --- a/db-functions +++ b/db-functions @@ -319,18 +319,18 @@ check_pkgrepos() { [ $? -ge 1 ] && return 1 local pkgver="$(getpkgver ${pkgfile})" [ $? -ge 1 ] && return 1 + local pkgarch="$(getpkgarch ${pkgfile})" + [ $? -ge 1 ] && return 1 - if [ -f "${FTP_BASE}/${PKGPOOL}/${pkgname}-${pkgver}-"*${PKGEXT} ]; then - return 1 - fi + [ -f "${FTP_BASE}/${PKGPOOL}/${pkgname}-${pkgver}-${pkgarch}"${PKGEXT} ] && return 1 + [ -f "${FTP_BASE}/${PKGPOOL}/$(basename ${pkgfile})" ] && return 1 local repo local arch for repo in ${PKGREPOS[@]}; do for arch in ${ARCHES[@]}; do - if [ -f "${FTP_BASE}/${repo}/os/${arch}/${pkgname}-${pkgver}"*${PKGEXT} ]; then - return 1 - fi + [ -f "${FTP_BASE}/${repo}/os/${arch}/${pkgname}-${pkgver}-${pkgarch}"${PKGEXT} ] && return 1 + [ -f "${FTP_BASE}/${repo}/os/${arch}/$(basename ${pkgfile})" ] && return 1 done done -- cgit v1.2.2 From dcf9b85295b36b3e510870052f3e617df84fce5e Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 12 Sep 2010 12:47:34 +0200 Subject: check_pkgsvn: check if pkgname is in sync with svn In addition to pkgver and pkgrel it is checked if pkgname is defined in the PKGBUILD. --- db-functions | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/db-functions b/db-functions index d9b38e2..f7699ff 100644 --- a/db-functions +++ b/db-functions @@ -288,28 +288,32 @@ check_pkgfile() { check_pkgsvn() { local pkgfile="${1}" - local pkgbase="$(getpkgbase $pkg)" + local _pkgbase="$(getpkgbase ${pkgfile})" [ $? -ge 1 ] && return 1 - local pkgarch="$(getpkgarch ${pkgfile})" + local _pkgname="$(getpkgname ${pkgfile})" + [ $? -ge 1 ] && return 1 + local _pkgver="$(getpkgver ${pkgfile})" + [ $? -ge 1 ] && return 1 + local _pkgarch="$(getpkgarch ${pkgfile})" [ $? -ge 1 ] && return 1 local repo="${2}" in_array "${repo}" ${PKGREPOS[@]} || return 1 - if [ ! -f "${WORKDIR}/pkgbuilds/${repo}/${pkgbase}" ]; then + if [ ! -f "${WORKDIR}/pkgbuilds/${repo}/${_pkgbase}" ]; then mkdir -p "${WORKDIR}/pkgbuilds/${repo}" - svn export -q "${SVNREPO}/${pkgbase}/repos/${repo}-${pkgarch}/PKGBUILD" \ - "${WORKDIR}/pkgbuilds/${repo}/${pkgbase}" >/dev/null + svn export -q "${SVNREPO}/${_pkgbase}/repos/${repo}-${_pkgarch}/PKGBUILD" \ + "${WORKDIR}/pkgbuilds/${repo}/${_pkgbase}" >/dev/null [ $? -ge 1 ] && return 1 fi - local pkgver="$(. "${WORKDIR}/pkgbuilds/${repo}/${pkgbase}"; echo "${pkgver}-${pkgrel}")" - [ -z "${pkgver}" ] && return 1 - if echo "$(basename ${pkgfile})" | grep -q "${pkgver}-${pkgarch}"; then - return 0 - else - return 1 - fi + local svnver="$(. "${WORKDIR}/pkgbuilds/${repo}/${_pkgbase}"; echo "${pkgver}-${pkgrel}")" + [ "${svnver}" == "${_pkgver}" ] || return 1 + + local svnnames=($(. "${WORKDIR}/pkgbuilds/${repo}/${_pkgbase}"; echo ${pkgname[@]})) + in_array "${_pkgname}" ${svnnames[@]} || return 1 + + return 0 } check_pkgrepos() { -- cgit v1.2.2 From e67a4695ee9550dfa7de153c48d3949891acb048 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 12 Sep 2010 14:48:01 +0200 Subject: check_pkgsvn: Don't assume the same PKGBUILD for all architectures --- db-functions | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/db-functions b/db-functions index f7699ff..cf13531 100644 --- a/db-functions +++ b/db-functions @@ -300,17 +300,17 @@ check_pkgsvn() { in_array "${repo}" ${PKGREPOS[@]} || return 1 - if [ ! -f "${WORKDIR}/pkgbuilds/${repo}/${_pkgbase}" ]; then - mkdir -p "${WORKDIR}/pkgbuilds/${repo}" + if [ ! -f "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}" ]; then + mkdir -p "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}" svn export -q "${SVNREPO}/${_pkgbase}/repos/${repo}-${_pkgarch}/PKGBUILD" \ - "${WORKDIR}/pkgbuilds/${repo}/${_pkgbase}" >/dev/null + "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}" >/dev/null [ $? -ge 1 ] && return 1 fi - local svnver="$(. "${WORKDIR}/pkgbuilds/${repo}/${_pkgbase}"; echo "${pkgver}-${pkgrel}")" + local svnver="$(. "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}"; echo "${pkgver}-${pkgrel}")" [ "${svnver}" == "${_pkgver}" ] || return 1 - local svnnames=($(. "${WORKDIR}/pkgbuilds/${repo}/${_pkgbase}"; echo ${pkgname[@]})) + local svnnames=($(. "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}"; echo ${pkgname[@]})) in_array "${_pkgname}" ${svnnames[@]} || return 1 return 0 -- cgit v1.2.2 From 8a5f6e5542590edb94499bd6f36ba3787efbc2ce Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 12 Sep 2010 09:50:44 -0500 Subject: Add multilib-testing to web update script Signed-off-by: Dan McGee --- cron-jobs/update-web-db | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cron-jobs/update-web-db b/cron-jobs/update-web-db index 8f91ca2..8b74a3a 100755 --- a/cron-jobs/update-web-db +++ b/cron-jobs/update-web-db @@ -9,7 +9,7 @@ ARCHES="i686 x86_64" # having "more important repos" last should make [core] trickle to the top of # the updates list each hour rather than being overwhelmed by big [extra] and # [community] updates -REPOS="community-testing multilib community extra testing core" +REPOS="community-testing multilib-testing multilib community extra testing core" LOGOUT="/tmp/archweb_update.log" # figure out what operation to perform -- cgit v1.2.2 From 2c6b7c3ebefa33e865d2293331dcf166cb580bd9 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 12 Sep 2010 19:20:01 +0200 Subject: _grep_pkginfo: use a more specific format --- db-functions | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/db-functions b/db-functions index cf13531..03859fb 100644 --- a/db-functions +++ b/db-functions @@ -183,8 +183,8 @@ repo_unlock () { #repo_unlock _grep_pkginfo() { local _ret - _ret="$(/usr/bin/bsdtar -xOqf "$1" .PKGINFO | /bin/grep -m 1 -E "$2" | /bin/sed 's|\w*\s*=\s*\(.*\)|\1|')" - echo "$_ret" + _ret="$(/usr/bin/bsdtar -xOqf "$1" .PKGINFO | /bin/grep -m 1 "^${2} = ")" + echo "${_ret#${2} = }" } @@ -192,7 +192,7 @@ _grep_pkginfo() { getpkgbase() { local _base - _base="$(_grep_pkginfo "$1" "^pkgbase")" + _base="$(_grep_pkginfo "$1" "pkgbase")" if [ -z "$_base" ]; then getpkgname "$1" fi @@ -204,7 +204,7 @@ getpkgbase() { getpkgname() { local _name - _name="$(_grep_pkginfo "$1" "^pkgname")" + _name="$(_grep_pkginfo "$1" "pkgname")" if [ -z "$_name" ]; then error "Package '$1' has no pkgname in the PKGINFO. Fail!" exit 1 @@ -217,7 +217,7 @@ getpkgname() { getpkgver() { local _ver - _ver="$(_grep_pkginfo "$1" "^pkgver")" + _ver="$(_grep_pkginfo "$1" "pkgver")" if [ -z "$_ver" ]; then error "Package '$1' has no pkgver in the PKGINFO. Fail!" exit 1 @@ -229,7 +229,7 @@ getpkgver() { getpkgarch() { local _ver - _ver="$(_grep_pkginfo "$1" "^arch")" + _ver="$(_grep_pkginfo "$1" "arch")" if [ -z "$_ver" ]; then error "Package '$1' has no arch in the PKGINFO. Fail!" exit 1 -- cgit v1.2.2 From d9efa9b7bbbc854b987f158cd19d5f646594e42b Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Mon, 13 Sep 2010 23:08:20 +0200 Subject: Fix typos --- cron-jobs/create-filelists | 2 +- cron-jobs/sourceballs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cron-jobs/create-filelists b/cron-jobs/create-filelists index a37080f..2526a61 100755 --- a/cron-jobs/create-filelists +++ b/cron-jobs/create-filelists @@ -27,7 +27,7 @@ esac FILESEXT="${DBEXT//db/files}" -for repo in ${PKGREPO[@]}; do +for repo in ${PKGREPOS[@]}; do REPO_DB_FILE="${repo}$DBEXT" FILES_DB_FILE="${repo}$FILESEXT" for arch in ${ARCHES[@]}; do diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 0cf062f..0b01321 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -17,7 +17,7 @@ FAILED_PKGS="" [ -e "$srcbase/errors.txt" ] && /bin/mv "$srcbase/errors.txt" "$srcbase/errors.txt.old" echo "Errors occured during run:" > "$srcbase/errors.txt" -for repo in ${PKGREPO[@]}; do +for repo in ${PKGREPOS[@]}; do for arch in ${ARCHES[@]} any; do ftppath="$ftpbase/$repo/os/$arch" if [ ! -d "$ftppath" ]; then -- cgit v1.2.2 From 44c24c5d8e98eb51c8612dfaaac3ddd2ce0337dd Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Thu, 16 Sep 2010 18:50:53 -0500 Subject: Move output redirection Signed-off-by: Dan McGee --- cron-jobs/update-web-db | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cron-jobs/update-web-db b/cron-jobs/update-web-db index 8b74a3a..275676e 100755 --- a/cron-jobs/update-web-db +++ b/cron-jobs/update-web-db @@ -34,12 +34,12 @@ for repo in ${REPOS}; do flags="--filesonly" ;; esac if [[ -f $dbfile ]]; then - echo "Updating ${repo}-${arch}" - ./manage.py reporead ${flags} ${arch} ${dbfile} + echo "Updating ${repo}-${arch}" >> "${LOGOUT}" + ./manage.py reporead ${flags} ${arch} ${dbfile} >> "${LOGOUT}" 2>&1 echo "" fi done -done >> "${LOGOUT}" 2>&1 +done echo "" >> "${LOGOUT}" -- cgit v1.2.2 From 7711a02ed5ce1be7f5f0cb572a1efc4c29b6dfe0 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Mon, 11 Oct 2010 12:28:52 -0500 Subject: Fix missing output redirection Signed-off-by: Dan McGee --- cron-jobs/update-web-db | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cron-jobs/update-web-db b/cron-jobs/update-web-db index 275676e..3f4a2fc 100755 --- a/cron-jobs/update-web-db +++ b/cron-jobs/update-web-db @@ -36,7 +36,7 @@ for repo in ${REPOS}; do if [[ -f $dbfile ]]; then echo "Updating ${repo}-${arch}" >> "${LOGOUT}" ./manage.py reporead ${flags} ${arch} ${dbfile} >> "${LOGOUT}" 2>&1 - echo "" + echo "" >> "${LOGOUT}" fi done done -- cgit v1.2.2 From a1ba979a0e3c763449d91fc787333f4058e89aa8 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Tue, 12 Oct 2010 11:17:07 +0200 Subject: Dont try to index empty repos --- cron-jobs/create-filelists | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cron-jobs/create-filelists b/cron-jobs/create-filelists index 2526a61..d8a5ba1 100755 --- a/cron-jobs/create-filelists +++ b/cron-jobs/create-filelists @@ -41,7 +41,7 @@ for repo in ${PKGREPOS[@]}; do mkdir -p "${DBDIR}/${repodir}" bsdtar -xf "${targetdir}/${repodir}/${REPO_DB_FILE}" -C "${DBDIR}/${repodir}" else - error "Fail! Does the repo $repo with arch $arch even exist?" + warning "The repo $repo with arch $arch does not exist" continue fi @@ -71,7 +71,7 @@ for repo in ${PKGREPOS[@]}; do done # create new file archive - if [ "$cached" == "no" ]; then + if [ "$cached" == "no" -a -d "${MYTMPDIR}/${repodir}" ]; then # at least one package has changed, so let's rebuild the archive pkgdir="${targetdir}/${repodir}" mkdir -p "$pkgdir" -- cgit v1.2.2 From cb2dcc6ee207e9c5ba4b875d70e387e6347591ed Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sat, 20 Nov 2010 16:11:59 +0100 Subject: Fix sourceballs cron job * add unit test for sourceballs and cleanup * introduce SRCPOOL and LOGDIR variables in config --- config | 2 ++ config.local.gerolde | 1 + config.local.sigurd | 1 + cron-jobs/sourceballs | 21 ++++++-------- misc-scripts/make-sourceball | 13 +++------ misc-scripts/sourceballs-cleanup | 9 +++--- test/lib/common.inc | 7 +++-- test/runTest | 59 ++++++++++++++++++++++++++++++++++++++++ 8 files changed, 84 insertions(+), 29 deletions(-) diff --git a/config b/config index 89e89fd..efc4381 100644 --- a/config +++ b/config @@ -5,6 +5,7 @@ SVNREPOCOMMUNITY="file:///srv/svn-community" PKGREPOS=() PKGPOOL='' +SRCPOOL='' CLEANUP_DESTDIR="/srv/package-cleanup" CLEANUP_DRYRUN=false @@ -15,6 +16,7 @@ LOCK_TIMEOUT=300 STAGING="$HOME/staging" TMPDIR="/srv/tmp" +LOGDIR="/var/log/dbscripts" ARCHES=(i686 x86_64) DBEXT=".db.tar.gz" PKGEXT=".pkg.tar.*" diff --git a/config.local.gerolde b/config.local.gerolde index dd8ba86..03e8a70 100644 --- a/config.local.gerolde +++ b/config.local.gerolde @@ -1,2 +1,3 @@ PKGREPOS=('core' 'extra' 'testing' 'staging' 'kde-unstable' 'gnome-unstable') PKGPOOL='pool/packages' +SRCPOOL='sources/packages' diff --git a/config.local.sigurd b/config.local.sigurd index c8c3c9f..ddb5ba4 100644 --- a/config.local.sigurd +++ b/config.local.sigurd @@ -1,2 +1,3 @@ PKGREPOS=('community' 'community-testing' 'community-staging' 'multilib' 'multilib-testing') PKGPOOL='pool/community' +SRCPOOL='sources/community' diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 0b01321..4fc194f 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -3,9 +3,6 @@ . "$(dirname $0)/../db-functions" . "$(dirname $0)/../config" -ftpbase="${FTP_BASE}" -srcbase="${FTP_BASE}/sources" - script_lock set_umask @@ -14,12 +11,12 @@ dirname="$(/bin/readlink -f $(/usr/bin/dirname $0))" FAILED_PKGS="" -[ -e "$srcbase/errors.txt" ] && /bin/mv "$srcbase/errors.txt" "$srcbase/errors.txt.old" -echo "Errors occured during run:" > "$srcbase/errors.txt" +[ ! -d "${LOGDIR}/sourceballs" ] && mkdir -p "${LOGDIR}/sourceballs" +[ -e "${LOGDIR}/sourceballs/errors.txt" ] && /bin/mv "${LOGDIR}/sourceballs/errors.txt" "${LOGDIR}/sourceballs/errors.txt.old" for repo in ${PKGREPOS[@]}; do - for arch in ${ARCHES[@]} any; do - ftppath="$ftpbase/$repo/os/$arch" + for arch in ${ARCHES[@]}; do + ftppath="${FTP_BASE}/$repo/os/$arch" if [ ! -d "$ftppath" ]; then error "FTP path does not exist: $ftppath" continue @@ -28,7 +25,6 @@ for repo in ${PKGREPOS[@]}; do for pkg in *$PKGEXT; do [ -f "$pkg" ] || continue pkgbase=$(getpkgbase $pkg) - srcpath="$srcbase/" srcpkg="${pkg//$PKGEXT/$SRCEXT}" srcpkg="${srcpkg//-$arch/}" srcpkgname="${srcpkg%-*-*$SRCEXT}" @@ -51,9 +47,9 @@ for repo in ${PKGREPOS[@]}; do force="-f" fi - if [ ! \( -f "$srcpath$srcpkg" -o -f "$srcpath$srcpkgbase" \) ]; then + if [ ! \( -f "${FTP_BASE}/${SRCPOOL}/$srcpkg" -o -f "${FTP_BASE}/${SRCPOOL}/$srcpkgbase" \) ]; then if ! $dirname/../misc-scripts/make-sourceball $force \ - $pkgbase $repo $arch 2>>"$srcbase/errors.txt"; then + $pkgbase $repo $arch 2>>"${LOGDIR}/sourceballs/errors.txt"; then FAILED_PKGS="$FAILED_PKGS $pkgbase" fi fi @@ -62,9 +58,8 @@ for repo in ${PKGREPOS[@]}; do done if [ -n "$FAILED_PKGS" ]; then - [ -e "$srcbase/failed.txt" ] && /bin/mv "$srcbase/failed.txt" "$srcbase/failed.txt.old" - echo "The following package bases failed:" > "$srcbase/failed.txt" - echo -e $FAILED_PKGS | sed "s| |\n|g" | sort -u >> "$srcbase/failed.txt" + [ -e "${LOGDIR}/sourceballs/failed.txt" ] && /bin/mv "${LOGDIR}/sourceballs/failed.txt" "${LOGDIR}/sourceballs/failed.txt.old" + echo -e $FAILED_PKGS | sed "s| |\n|g" | sort -u >> "${LOGDIR}/sourceballs/failed.txt" fi $dirname/../misc-scripts/sourceballs-cleanup diff --git a/misc-scripts/make-sourceball b/misc-scripts/make-sourceball index 5bd155f..221d0ce 100755 --- a/misc-scripts/make-sourceball +++ b/misc-scripts/make-sourceball @@ -19,9 +19,6 @@ packagename="$1" reponame="$2" arch="$3" -srcpath="$FTP_BASE/sources/" -logpath="/var/log/sourceballs/" - script_lock create_srcpackage() { @@ -29,6 +26,7 @@ create_srcpackage() { pushd "$1" >/dev/null pkgver=$(. PKGBUILD; echo ${pkgver}) pkgrel=$(. PKGBUILD; echo ${pkgrel}) + license=($(. PKGBUILD; echo ${license[@]})) if ! [ $FORCE == 1 ] && ! chk_license ${license[@]} ; then #Removed so as not to clutter failed.txt #warning "$packagename license (${license[@]}) does not require source tarballs" @@ -37,7 +35,7 @@ create_srcpackage() { msg "Creating source tarball for $packagename-$pkgver-$pkgrel" fi - local logfile="$logpath/$packagename" + local logfile="${LOGDIR}/sourceballs/$packagename" if ! /usr/bin/makepkg --allsource --ignorearch >"$logfile" 2>&1; then popd >/dev/null /bin/gzip -f -9 "$logfile" @@ -47,10 +45,7 @@ create_srcpackage() { local pkg_file="${packagename}-${pkgver}-${pkgrel}${SRCEXT}" - if [ ! -d "$srcpath" ]; then - mkdir -p "$srcpath" - fi - cp "$pkg_file" "$srcpath" + cp "$pkg_file" "${FTP_BASE}/${SRCPOOL}" popd >/dev/null @@ -59,7 +54,7 @@ create_srcpackage() { } set_umask -/bin/mkdir -p "$logpath" +[ ! -d "${LOGDIR}/sourceballs" ] && mkdir -p "${LOGDIR}/sourceballs" cd "$WORKDIR" if [[ "$reponame" = "community" || "$reponame" = "community-testing" ]]; then diff --git a/misc-scripts/sourceballs-cleanup b/misc-scripts/sourceballs-cleanup index e407989..788cc98 100755 --- a/misc-scripts/sourceballs-cleanup +++ b/misc-scripts/sourceballs-cleanup @@ -3,8 +3,8 @@ . "$(dirname $0)/../db-functions" . "$(dirname $0)/../config" -srcpath="$FTP_BASE/sources/" -logfile="$srcpath/cleanup.txt" +[ ! -d "${LOGDIR}/sourceballs" ] && mkdir -p "${LOGDIR}/sourceballs" +logfile="${LOGDIR}/sourceballs/cleanup.txt" script_lock @@ -20,7 +20,7 @@ remove_old() { cd .. done - for srcpkg in "$srcpath/$packagename-"*; do + for srcpkg in "${FTP_BASE}/${SRCPOOL}/$packagename-"*; do [ -f "$srcpkg" ] || continue if [ "$(pkgname_from_src $srcpkg)" == "$packagename" ]; then skip=0 @@ -47,10 +47,11 @@ remove_old() { set_umask cd "$WORKDIR" + [ -e "$logfile" ] && /bin/mv "$logfile" "$logfile.old" echo "Orphaned sourceballs:" > "$logfile" -for sourceball in "$srcpath"/*$SRCEXT; do +for sourceball in "${FTP_BASE}/${SRCPOOL}"/*$SRCEXT; do packagename=$(basename $sourceball) packagename=${packagename%-*-*$SRCEXT} diff --git a/test/lib/common.inc b/test/lib/common.inc index a0d864a..14dc000 100644 --- a/test/lib/common.inc +++ b/test/lib/common.inc @@ -43,9 +43,9 @@ setUp() { mkdir -p "${TMP}/ftp/${r}/os/${a}" done done - for p in ${PKGPOOL[@]}; do - mkdir -p "${TMP}/ftp/${p}" - done + mkdir -p "${TMP}/ftp/${PKGPOOL}" + mkdir -p "${TMP}/ftp/${SRCPOOL}" + mkdir -p "${TMP}/log" msg 'Creating svn repository...' svnadmin create "${TMP}/svn-packages-repo" @@ -69,6 +69,7 @@ setUp() { SOURCE_CLEANUP_DESTDIR="${TMP}/source-cleanup" STAGING="${TMP}/staging" TMPDIR="${TMP}/tmp" + LOGDIR="${TMP}/log" CLEANUP_DRYRUN=false eot . "${curdir}/../config" diff --git a/test/runTest b/test/runTest index 4681944..d3e2eff 100755 --- a/test/runTest +++ b/test/runTest @@ -535,5 +535,64 @@ testUpdateSameAnyPackageToDifferentRepositoriesWithoutPool() { done } +# +#sourceballs tests +# + +testSourceballs() { + local arches=('i686' 'x86_64') + local pkgs=('pkg-simple-a' 'pkg-simple-b') + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + releasePackage extra ${pkgbase} ${arch} + done + done + ../db-update + + ../cron-jobs/sourceballs + + for log in errors failed; do + if [ -s "${LOGDIR}/sourceballs/${log}.txt" ]; then + fail "${LOGDIR}/sourceballs/${log}.txt includes the following errors:" + cat "${LOGDIR}/sourceballs/${log}.txt" + fi + done + for pkgbase in ${pkgs[@]}; do + [ ! -r ${FTP_BASE}/${SRCPOOL}/${pkgbase}-*${SRCEXT} ] && fail "source package not found!" + done +} + +testSourceballsCleanup() { + local arches=('i686' 'x86_64') + local pkgs=('pkg-simple-a' 'pkg-simple-b') + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + releasePackage extra ${pkgbase} ${arch} + done + done + ../db-update + ../cron-jobs/sourceballs + + for arch in ${arches[@]}; do + ../db-remove pkg-simple-a extra ${arch} + done + ../cron-jobs/ftpdir-cleanup >/dev/null + + ../cron-jobs/sourceballs + for log in errors failed; do + if [ -s "${LOGDIR}/sourceballs/${log}.txt" ]; then + fail "${LOGDIR}/sourceballs/${log}.txt includes the following errors:" + cat "${LOGDIR}/sourceballs/${log}.txt" + fi + done + [ -r ${FTP_BASE}/${SRCPOOL}/pkg-simple-a-*${SRCEXT} ] && fail "source package was not removed!" + [ ! -r ${FTP_BASE}/${SRCPOOL}/pkg-simple-b-*${SRCEXT} ] && fail "source package not found!" +} . "${curdir}/lib/shunit2" -- cgit v1.2.2 From a7591f4be3f9e741f5d1e5aeadd3ab20b497a252 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sat, 20 Nov 2010 20:38:38 +0100 Subject: Simplify sourceballs creation * Read package lists directly from DB file * Make SVNREPO configurable --- config | 5 +---- config.local.gerolde | 1 + config.local.sigurd | 1 + cron-jobs/sourceballs | 46 ++++++++++++++++---------------------------- misc-scripts/make-sourceball | 34 ++++++++++++-------------------- test/lib/common.inc | 1 - 6 files changed, 32 insertions(+), 56 deletions(-) diff --git a/config b/config index efc4381..e149b18 100644 --- a/config +++ b/config @@ -1,8 +1,5 @@ FTP_BASE="/srv/ftp" - -SVNREPO="file:///srv/svn-packages" -SVNREPOCOMMUNITY="file:///srv/svn-community" - +SVNREPO='' PKGREPOS=() PKGPOOL='' SRCPOOL='' diff --git a/config.local.gerolde b/config.local.gerolde index 03e8a70..4501a93 100644 --- a/config.local.gerolde +++ b/config.local.gerolde @@ -1,3 +1,4 @@ PKGREPOS=('core' 'extra' 'testing' 'staging' 'kde-unstable' 'gnome-unstable') PKGPOOL='pool/packages' SRCPOOL='sources/packages' +SVNREPO='file:///srv/svn-packages' diff --git a/config.local.sigurd b/config.local.sigurd index ddb5ba4..f62372c 100644 --- a/config.local.sigurd +++ b/config.local.sigurd @@ -1,3 +1,4 @@ PKGREPOS=('community' 'community-testing' 'community-staging' 'multilib' 'multilib-testing') PKGPOOL='pool/community' SRCPOOL='sources/community' +SVNREPO='file:///srv/svn-community' diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 4fc194f..1add36e 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -9,59 +9,47 @@ set_umask dirname="$(/bin/readlink -f $(/usr/bin/dirname $0))" -FAILED_PKGS="" - [ ! -d "${LOGDIR}/sourceballs" ] && mkdir -p "${LOGDIR}/sourceballs" [ -e "${LOGDIR}/sourceballs/errors.txt" ] && /bin/mv "${LOGDIR}/sourceballs/errors.txt" "${LOGDIR}/sourceballs/errors.txt.old" +[ -e "${LOGDIR}/sourceballs/failed.txt" ] && /bin/mv "${LOGDIR}/sourceballs/failed.txt" "${LOGDIR}/sourceballs/failed.txt.old" for repo in ${PKGREPOS[@]}; do for arch in ${ARCHES[@]}; do ftppath="${FTP_BASE}/$repo/os/$arch" - if [ ! -d "$ftppath" ]; then - error "FTP path does not exist: $ftppath" + dbfile="${ftppath}/${repo}${DBEXT}" + if [ ! -r "${dbfile}" ]; then + warning "DB file does not exist: ${dbfile}" continue fi - cd $ftppath - for pkg in *$PKGEXT; do - [ -f "$pkg" ] || continue - pkgbase=$(getpkgbase $pkg) - srcpkg="${pkg//$PKGEXT/$SRCEXT}" - srcpkg="${srcpkg//-$arch/}" - srcpkgname="${srcpkg%-*-*$SRCEXT}" - srcpkgbase="${srcpkg/$srcpkgname/$pkgbase}" - - #Don't do anything for package in this 'blacklist' - if grep "^$pkgbase\$" "$dirname/sourceballs.skip" >/dev/null 2>&1; then - continue - fi - - #This pkgbase has already failed. No sense in trying it again this run - if echo $FAILED_PKGS | grep "\<$pkgbase\>" >&/dev/null; then + for pkg in $(bsdtar -xOf "${dbfile}" \ + | awk '/^%NAME%/{getline b};/^%BASE%/{getline b};/^%VERSION%/{getline v};/^%ARCH%/{printf "%s/%s\n", b, v}' \ + | sort -u); do + pkgbase=${pkg%/*} + pkgver=${pkg#*/} + srcpkg="${pkgbase}-${pkgver}${SRCEXT}" + + # Don't do anything for package in this 'blacklist' + if grep -q "^${pkgbase}\$" "$dirname/sourceballs.skip"; then continue fi - #Use this file to 'whitelist' or force building some sourceballs, + # Use this file to 'whitelist' or force building some sourceballs, # skipping the license check force="" - if grep "^$pkgbase\$" "$dirname/sourceballs.force" >/dev/null 2>&1; then + if grep -q "^$pkgbase\$" "$dirname/sourceballs.force"; then force="-f" fi - if [ ! \( -f "${FTP_BASE}/${SRCPOOL}/$srcpkg" -o -f "${FTP_BASE}/${SRCPOOL}/$srcpkgbase" \) ]; then + if [ ! -f "${FTP_BASE}/${SRCPOOL}/$srcpkg" ]; then if ! $dirname/../misc-scripts/make-sourceball $force \ $pkgbase $repo $arch 2>>"${LOGDIR}/sourceballs/errors.txt"; then - FAILED_PKGS="$FAILED_PKGS $pkgbase" + echo "$pkgbase" >> "${LOGDIR}/sourceballs/failed.txt" fi fi done done done -if [ -n "$FAILED_PKGS" ]; then - [ -e "${LOGDIR}/sourceballs/failed.txt" ] && /bin/mv "${LOGDIR}/sourceballs/failed.txt" "${LOGDIR}/sourceballs/failed.txt.old" - echo -e $FAILED_PKGS | sed "s| |\n|g" | sort -u >> "${LOGDIR}/sourceballs/failed.txt" -fi - $dirname/../misc-scripts/sourceballs-cleanup script_unlock diff --git a/misc-scripts/make-sourceball b/misc-scripts/make-sourceball index 221d0ce..661f481 100755 --- a/misc-scripts/make-sourceball +++ b/misc-scripts/make-sourceball @@ -24,28 +24,26 @@ script_lock create_srcpackage() { if [ -d "$1" ]; then pushd "$1" >/dev/null - pkgver=$(. PKGBUILD; echo ${pkgver}) - pkgrel=$(. PKGBUILD; echo ${pkgrel}) - license=($(. PKGBUILD; echo ${license[@]})) - if ! [ $FORCE == 1 ] && ! chk_license ${license[@]} ; then - #Removed so as not to clutter failed.txt + pkgver=$(. PKGBUILD; echo ${pkgver}) + pkgrel=$(. PKGBUILD; echo ${pkgrel}) + license=($(. PKGBUILD; echo ${license[@]})) + if ! [ $FORCE == 1 ] && ! chk_license ${license[@]} ; then + # Removed so as not to clutter failed.txt #warning "$packagename license (${license[@]}) does not require source tarballs" cleanup 0 else msg "Creating source tarball for $packagename-$pkgver-$pkgrel" fi - local logfile="${LOGDIR}/sourceballs/$packagename" - if ! /usr/bin/makepkg --allsource --ignorearch >"$logfile" 2>&1; then - popd >/dev/null - /bin/gzip -f -9 "$logfile" + local logfile="${LOGDIR}/sourceballs/$packagename.gz" + if ! /usr/bin/makepkg --allsource --ignorearch 2>&1 | gzip >"$logfile"; then die "Failed to download source for $packagename-$pkgver-$pkgrel ($reponame-$arch)" fi - /bin/rm -f "$logfile"{,.gz} + /bin/rm -f "$logfile" local pkg_file="${packagename}-${pkgver}-${pkgrel}${SRCEXT}" - cp "$pkg_file" "${FTP_BASE}/${SRCPOOL}" + mv "$pkg_file" "${FTP_BASE}/${SRCPOOL}" popd >/dev/null @@ -57,18 +55,10 @@ set_umask [ ! -d "${LOGDIR}/sourceballs" ] && mkdir -p "${LOGDIR}/sourceballs" cd "$WORKDIR" -if [[ "$reponame" = "community" || "$reponame" = "community-testing" ]]; then - if /usr/bin/svn export -q "$SVNREPOCOMMUNITY/$packagename" $packagename; then - create_srcpackage "$packagename/repos/$reponame-$arch" - else - die "Package '$packagename' does not exist in repo '$reponame-$arch'" - fi +if /usr/bin/svn export -q "$SVNREPO/$packagename" $packagename; then + create_srcpackage "$packagename/repos/$reponame-$arch" else - if /usr/bin/svn export -q "$SVNREPO/$packagename" $packagename; then - create_srcpackage "$packagename/repos/$reponame-$arch" - else - die "Package '$packagename' does not exist in repo '$reponame-$arch'" - fi + die "Package '$packagename' does not exist in repo '$reponame-$arch'" fi script_unlock diff --git a/test/lib/common.inc b/test/lib/common.inc index 14dc000..e47ae2d 100644 --- a/test/lib/common.inc +++ b/test/lib/common.inc @@ -62,7 +62,6 @@ setUp() { cat < "${curdir}/../config.local" FTP_BASE="${TMP}/ftp" SVNREPO="file://${TMP}/svn-packages-repo" - SVNREPOCOMMUNITY="file://${TMP}/svn-community-repo" PKGREPOS=(${PKGREPOS[@]}) PKGPOOL="${PKGPOOL}" CLEANUP_DESTDIR="${TMP}/package-cleanup" -- cgit v1.2.2 From 7edb6b55cbacbc32c0234026df38401a60561dc7 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sat, 20 Nov 2010 21:14:42 +0100 Subject: sourceballs: skip in a clean way if source package should not be created --- misc-scripts/make-sourceball | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc-scripts/make-sourceball b/misc-scripts/make-sourceball index 661f481..da51c93 100755 --- a/misc-scripts/make-sourceball +++ b/misc-scripts/make-sourceball @@ -30,7 +30,7 @@ create_srcpackage() { if ! [ $FORCE == 1 ] && ! chk_license ${license[@]} ; then # Removed so as not to clutter failed.txt #warning "$packagename license (${license[@]}) does not require source tarballs" - cleanup 0 + return 0 else msg "Creating source tarball for $packagename-$pkgver-$pkgrel" fi -- cgit v1.2.2 From a2cdd819ed32cd4ccfefab2c2ce0ead118f9772d Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sat, 20 Nov 2010 21:29:47 +0100 Subject: sourceballs: Lock the repo while reading the db file --- cron-jobs/sourceballs | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 1add36e..351b06d 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -21,9 +21,16 @@ for repo in ${PKGREPOS[@]}; do warning "DB file does not exist: ${dbfile}" continue fi - for pkg in $(bsdtar -xOf "${dbfile}" \ - | awk '/^%NAME%/{getline b};/^%BASE%/{getline b};/^%VERSION%/{getline v};/^%ARCH%/{printf "%s/%s\n", b, v}' \ - | sort -u); do + + repo_lock ${repo} ${arch} || exit 1 + # Read packages from db file + # Format is: /- + pkgs=($(bsdtar -xOf "${dbfile}" \ + | awk '/^%NAME%/{getline b};/^%BASE%/{getline b};/^%VERSION%/{getline v};/^%ARCH%/{printf "%s/%s ", b, v}' \ + | sort -u)) + repo_unlock ${repo} ${arch} + + for pkg in ${pkgs[@]}; do pkgbase=${pkg%/*} pkgver=${pkg#*/} srcpkg="${pkgbase}-${pkgver}${SRCEXT}" -- cgit v1.2.2 From de07478719e8462a1f8bb4f61cb83571a37952b8 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 21 Nov 2010 00:01:48 +0100 Subject: sourceballs: add LGPL2.1 --- config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config b/config index e149b18..4b89f2d 100644 --- a/config +++ b/config @@ -20,7 +20,7 @@ PKGEXT=".pkg.tar.*" SRCEXT=".src.tar.gz" # Allowed licenses: get sourceballs only for licenses in this array -ALLOWED_LICENSES=('GPL' 'GPL1' 'GPL2' 'LGPL' 'LGPL1' 'LGPL2') +ALLOWED_LICENSES=('GPL' 'GPL1' 'GPL2' 'LGPL' 'LGPL1' 'LGPL2' 'LGPL2.1') # Override default config with config.local [ -f "$(dirname ${BASH_SOURCE[0]})/config.local" ] && . "$(dirname ${BASH_SOURCE[0]})/config.local" -- cgit v1.2.2 From 6dc2b46cc17fcac93d3ed1758edd53222631e849 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 21 Nov 2010 00:58:33 +0100 Subject: When on sigurd the svn repo is actually called svn-packages --- config.local.sigurd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.local.sigurd b/config.local.sigurd index f62372c..d28aa37 100644 --- a/config.local.sigurd +++ b/config.local.sigurd @@ -1,4 +1,4 @@ PKGREPOS=('community' 'community-testing' 'community-staging' 'multilib' 'multilib-testing') PKGPOOL='pool/community' SRCPOOL='sources/community' -SVNREPO='file:///srv/svn-community' +SVNREPO='file:///srv/svn-packages' -- cgit v1.2.2 From e5539fc65872aa98d510a03d1e20c207ca6ed058 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 21 Nov 2010 10:07:59 +0100 Subject: sourceballs: fix error handling --- misc-scripts/make-sourceball | 29 ++++++++++++++++++++++++----- misc-scripts/sourceballs-cleanup | 12 ++++++++---- 2 files changed, 32 insertions(+), 9 deletions(-) diff --git a/misc-scripts/make-sourceball b/misc-scripts/make-sourceball index da51c93..bdf5f59 100755 --- a/misc-scripts/make-sourceball +++ b/misc-scripts/make-sourceball @@ -25,8 +25,20 @@ create_srcpackage() { if [ -d "$1" ]; then pushd "$1" >/dev/null pkgver=$(. PKGBUILD; echo ${pkgver}) + if [ $? -ne 0 ]; then + error "Failed to read pkgver from $1" + return 1 + fi pkgrel=$(. PKGBUILD; echo ${pkgrel}) + if [ $? -ne 0 ]; then + error "Failed to read pkgrel from $1" + return 1 + fi license=($(. PKGBUILD; echo ${license[@]})) + if [ $? -ne 0 ]; then + error "Failed to read license from $1" + return 1 + fi if ! [ $FORCE == 1 ] && ! chk_license ${license[@]} ; then # Removed so as not to clutter failed.txt #warning "$packagename license (${license[@]}) does not require source tarballs" @@ -35,9 +47,11 @@ create_srcpackage() { msg "Creating source tarball for $packagename-$pkgver-$pkgrel" fi - local logfile="${LOGDIR}/sourceballs/$packagename.gz" - if ! /usr/bin/makepkg --allsource --ignorearch 2>&1 | gzip >"$logfile"; then - die "Failed to download source for $packagename-$pkgver-$pkgrel ($reponame-$arch)" + local logfile="${LOGDIR}/sourceballs/$packagename" + if ! /usr/bin/makepkg --allsource --ignorearch >"$logfile" 2>&1; then + error "Failed to download source for $packagename-$pkgver-$pkgrel ($reponame-$arch)" + popd >/dev/null + return 1 fi /bin/rm -f "$logfile" @@ -55,10 +69,15 @@ set_umask [ ! -d "${LOGDIR}/sourceballs" ] && mkdir -p "${LOGDIR}/sourceballs" cd "$WORKDIR" +failed=0 + if /usr/bin/svn export -q "$SVNREPO/$packagename" $packagename; then - create_srcpackage "$packagename/repos/$reponame-$arch" + create_srcpackage "$packagename/repos/$reponame-$arch" || failed=1 else - die "Package '$packagename' does not exist in repo '$reponame-$arch'" + error "Package '$packagename' does not exist in repo '$reponame-$arch'" + failed=1 fi script_unlock + +exit $failed \ No newline at end of file diff --git a/misc-scripts/sourceballs-cleanup b/misc-scripts/sourceballs-cleanup index 788cc98..fc5f649 100755 --- a/misc-scripts/sourceballs-cleanup +++ b/misc-scripts/sourceballs-cleanup @@ -52,13 +52,17 @@ cd "$WORKDIR" echo "Orphaned sourceballs:" > "$logfile" for sourceball in "${FTP_BASE}/${SRCPOOL}"/*$SRCEXT; do + [ -f "$sourceball" ] || continue packagename=$(basename $sourceball) packagename=${packagename%-*-*$SRCEXT} - if ! /usr/bin/svn export -q --force "$SVNREPO/$packagename" "$packagename" >/dev/null 2>&1 \ - && ! /usr/bin/svn export -q --force "$SVNREPOCOMMUNITY/$packagename" "$packagename" >/dev/null 2>&1 ; then - echo "$packagename : no longer in svn. Removing sourceball." >> "$logfile" - mv $sourceball $SOURCE_CLEANUP_DESTDIR + if ! /usr/bin/svn export -q --force "$SVNREPO/$packagename" "$packagename" >/dev/null 2>&1; then + if [ $? -ne 1 ]; then + echo "$packagename : svn died during export. Skipping sourceball." >> "$logfile" + else + echo "$packagename : no longer in svn. Removing sourceball." >> "$logfile" + mv $sourceball $SOURCE_CLEANUP_DESTDIR + fi elif [ -z "$(ls -A "$packagename/repos")" ]; then echo "$packagename : no longer in repos but trunk is still in svn. Removing sourceball." >> "$logfile" mv $sourceball $SOURCE_CLEANUP_DESTDIR -- cgit v1.2.2 From 42197d1f0f313f3608b8656662347d00885e6b5c Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 21 Nov 2010 11:38:48 +0100 Subject: sourceballs: check if PKGBUILD exists --- misc-scripts/make-sourceball | 4 +++- misc-scripts/sourceballs-cleanup | 10 +++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/misc-scripts/make-sourceball b/misc-scripts/make-sourceball index bdf5f59..66d3dad 100755 --- a/misc-scripts/make-sourceball +++ b/misc-scripts/make-sourceball @@ -22,7 +22,7 @@ arch="$3" script_lock create_srcpackage() { - if [ -d "$1" ]; then + if [ -f "$1/PKGBUILD" ]; then pushd "$1" >/dev/null pkgver=$(. PKGBUILD; echo ${pkgver}) if [ $? -ne 0 ]; then @@ -62,6 +62,8 @@ create_srcpackage() { popd >/dev/null return 0 + else + error "PKGBUILD not found in $1" fi } diff --git a/misc-scripts/sourceballs-cleanup b/misc-scripts/sourceballs-cleanup index fc5f649..f75c688 100755 --- a/misc-scripts/sourceballs-cleanup +++ b/misc-scripts/sourceballs-cleanup @@ -14,9 +14,13 @@ remove_old() { PKGVERS="" for repo in *; do cd "$repo" - pkgver=$(. PKGBUILD; echo ${pkgver}) - pkgrel=$(. PKGBUILD; echo ${pkgrel}) - PKGVERS="$PKGVERS $pkgver-$pkgrel" + if [ -f PKGBUILD ]; then + pkgver=$(. PKGBUILD; echo ${pkgver}) + pkgrel=$(. PKGBUILD; echo ${pkgrel}) + PKGVERS="$PKGVERS $pkgver-$pkgrel" + else + error "PKGBUILD not found in $1/$repo" + fi cd .. done -- cgit v1.2.2 From 948583aebe9e8e5a3daa9131c99dc68724e06b49 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 21 Nov 2010 12:40:29 +0100 Subject: sourceballs: respect the dryrun setting --- misc-scripts/sourceballs-cleanup | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/misc-scripts/sourceballs-cleanup b/misc-scripts/sourceballs-cleanup index f75c688..450d137 100755 --- a/misc-scripts/sourceballs-cleanup +++ b/misc-scripts/sourceballs-cleanup @@ -8,6 +8,8 @@ logfile="${LOGDIR}/sourceballs/cleanup.txt" script_lock +${CLEANUP_DRYRUN} && warning 'dry run mode is active' + remove_old() { if [ -d "$1" ]; then pushd "$1" >/dev/null @@ -36,7 +38,7 @@ remove_old() { fi done if [ $skip -ne 1 ]; then - mv "$srcpkg" $SOURCE_CLEANUP_DESTDIR + ${CLEANUP_DRYRUN} || mv "$srcpkg" $SOURCE_CLEANUP_DESTDIR fi fi done @@ -65,14 +67,14 @@ for sourceball in "${FTP_BASE}/${SRCPOOL}"/*$SRCEXT; do echo "$packagename : svn died during export. Skipping sourceball." >> "$logfile" else echo "$packagename : no longer in svn. Removing sourceball." >> "$logfile" - mv $sourceball $SOURCE_CLEANUP_DESTDIR + ${CLEANUP_DRYRUN} || mv $sourceball $SOURCE_CLEANUP_DESTDIR fi elif [ -z "$(ls -A "$packagename/repos")" ]; then echo "$packagename : no longer in repos but trunk is still in svn. Removing sourceball." >> "$logfile" - mv $sourceball $SOURCE_CLEANUP_DESTDIR + ${CLEANUP_DRYRUN} || mv $sourceball $SOURCE_CLEANUP_DESTDIR elif ! source "$packagename/trunk/PKGBUILD" && chk_license ${license[@]}; then echo "$packagename : source hosting no longer required by license. Removing sourceball." >> "$logfile" - mv $sourceball $SOURCE_CLEANUP_DESTDIR + ${CLEANUP_DRYRUN} || mv $sourceball $SOURCE_CLEANUP_DESTDIR else remove_old "$packagename/repos/" fi -- cgit v1.2.2 From 30a128a864bdbfc294b6ba6a49c9264570bb3c58 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 21 Nov 2010 13:23:09 +0100 Subject: sourceballs: Don't write any log This is consistent with the other dbscripts. The output will be send to the mailinglist. --- config | 1 - cron-jobs/sourceballs | 11 +++-------- misc-scripts/make-sourceball | 15 +++++++-------- misc-scripts/sourceballs-cleanup | 17 +++++------------ test/lib/common.inc | 2 -- test/runTest | 13 ------------- 6 files changed, 15 insertions(+), 44 deletions(-) diff --git a/config b/config index 4b89f2d..01956ba 100644 --- a/config +++ b/config @@ -13,7 +13,6 @@ LOCK_TIMEOUT=300 STAGING="$HOME/staging" TMPDIR="/srv/tmp" -LOGDIR="/var/log/dbscripts" ARCHES=(i686 x86_64) DBEXT=".db.tar.gz" PKGEXT=".pkg.tar.*" diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 351b06d..23e8421 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -9,16 +9,12 @@ set_umask dirname="$(/bin/readlink -f $(/usr/bin/dirname $0))" -[ ! -d "${LOGDIR}/sourceballs" ] && mkdir -p "${LOGDIR}/sourceballs" -[ -e "${LOGDIR}/sourceballs/errors.txt" ] && /bin/mv "${LOGDIR}/sourceballs/errors.txt" "${LOGDIR}/sourceballs/errors.txt.old" -[ -e "${LOGDIR}/sourceballs/failed.txt" ] && /bin/mv "${LOGDIR}/sourceballs/failed.txt" "${LOGDIR}/sourceballs/failed.txt.old" - for repo in ${PKGREPOS[@]}; do for arch in ${ARCHES[@]}; do ftppath="${FTP_BASE}/$repo/os/$arch" dbfile="${ftppath}/${repo}${DBEXT}" if [ ! -r "${dbfile}" ]; then - warning "DB file does not exist: ${dbfile}" + warning "${dbfile} not found, skipping" continue fi @@ -48,9 +44,8 @@ for repo in ${PKGREPOS[@]}; do fi if [ ! -f "${FTP_BASE}/${SRCPOOL}/$srcpkg" ]; then - if ! $dirname/../misc-scripts/make-sourceball $force \ - $pkgbase $repo $arch 2>>"${LOGDIR}/sourceballs/errors.txt"; then - echo "$pkgbase" >> "${LOGDIR}/sourceballs/failed.txt" + if ! $dirname/../misc-scripts/make-sourceball $force $pkgbase $repo $arch; then + error "Failed to download sources for $pkgbase" fi fi done diff --git a/misc-scripts/make-sourceball b/misc-scripts/make-sourceball index 66d3dad..fccc4b5 100755 --- a/misc-scripts/make-sourceball +++ b/misc-scripts/make-sourceball @@ -47,17 +47,17 @@ create_srcpackage() { msg "Creating source tarball for $packagename-$pkgver-$pkgrel" fi - local logfile="${LOGDIR}/sourceballs/$packagename" - if ! /usr/bin/makepkg --allsource --ignorearch >"$logfile" 2>&1; then + /usr/bin/makepkg --allsource --ignorearch > "$WORKDIR/makepkg.log" 2>&1 + if [ $? -ne 0 ]; then error "Failed to download source for $packagename-$pkgver-$pkgrel ($reponame-$arch)" - popd >/dev/null + local line + while read line; do + msg2 "${line}" + done < "$WORKDIR/makepkg.log" return 1 fi - /bin/rm -f "$logfile" - - local pkg_file="${packagename}-${pkgver}-${pkgrel}${SRCEXT}" - mv "$pkg_file" "${FTP_BASE}/${SRCPOOL}" + mv "${packagename}-${pkgver}-${pkgrel}${SRCEXT}" "${FTP_BASE}/${SRCPOOL}" popd >/dev/null @@ -68,7 +68,6 @@ create_srcpackage() { } set_umask -[ ! -d "${LOGDIR}/sourceballs" ] && mkdir -p "${LOGDIR}/sourceballs" cd "$WORKDIR" failed=0 diff --git a/misc-scripts/sourceballs-cleanup b/misc-scripts/sourceballs-cleanup index 450d137..fdb1355 100755 --- a/misc-scripts/sourceballs-cleanup +++ b/misc-scripts/sourceballs-cleanup @@ -3,9 +3,6 @@ . "$(dirname $0)/../db-functions" . "$(dirname $0)/../config" -[ ! -d "${LOGDIR}/sourceballs" ] && mkdir -p "${LOGDIR}/sourceballs" -logfile="${LOGDIR}/sourceballs/cleanup.txt" - script_lock ${CLEANUP_DRYRUN} && warning 'dry run mode is active' @@ -53,10 +50,6 @@ remove_old() { set_umask cd "$WORKDIR" - -[ -e "$logfile" ] && /bin/mv "$logfile" "$logfile.old" -echo "Orphaned sourceballs:" > "$logfile" - for sourceball in "${FTP_BASE}/${SRCPOOL}"/*$SRCEXT; do [ -f "$sourceball" ] || continue packagename=$(basename $sourceball) @@ -64,16 +57,16 @@ for sourceball in "${FTP_BASE}/${SRCPOOL}"/*$SRCEXT; do if ! /usr/bin/svn export -q --force "$SVNREPO/$packagename" "$packagename" >/dev/null 2>&1; then if [ $? -ne 1 ]; then - echo "$packagename : svn died during export. Skipping sourceball." >> "$logfile" + error "$packagename : svn died during export. Skipping sourceball." else - echo "$packagename : no longer in svn. Removing sourceball." >> "$logfile" + msg "$packagename : no longer in svn. Removing sourceball." ${CLEANUP_DRYRUN} || mv $sourceball $SOURCE_CLEANUP_DESTDIR fi elif [ -z "$(ls -A "$packagename/repos")" ]; then - echo "$packagename : no longer in repos but trunk is still in svn. Removing sourceball." >> "$logfile" + warning "$packagename : no longer in repos but trunk is still in svn. Removing sourceball." ${CLEANUP_DRYRUN} || mv $sourceball $SOURCE_CLEANUP_DESTDIR - elif ! source "$packagename/trunk/PKGBUILD" && chk_license ${license[@]}; then - echo "$packagename : source hosting no longer required by license. Removing sourceball." >> "$logfile" + elif ! chk_license $(. "$packagename/trunk/PKGBUILD"; echo ${license[@]}); then + msg "$packagename : source hosting no longer required by license. Removing sourceball." ${CLEANUP_DRYRUN} || mv $sourceball $SOURCE_CLEANUP_DESTDIR else remove_old "$packagename/repos/" diff --git a/test/lib/common.inc b/test/lib/common.inc index e47ae2d..6619c0a 100644 --- a/test/lib/common.inc +++ b/test/lib/common.inc @@ -45,7 +45,6 @@ setUp() { done mkdir -p "${TMP}/ftp/${PKGPOOL}" mkdir -p "${TMP}/ftp/${SRCPOOL}" - mkdir -p "${TMP}/log" msg 'Creating svn repository...' svnadmin create "${TMP}/svn-packages-repo" @@ -68,7 +67,6 @@ setUp() { SOURCE_CLEANUP_DESTDIR="${TMP}/source-cleanup" STAGING="${TMP}/staging" TMPDIR="${TMP}/tmp" - LOGDIR="${TMP}/log" CLEANUP_DRYRUN=false eot . "${curdir}/../config" diff --git a/test/runTest b/test/runTest index d3e2eff..6c60ddc 100755 --- a/test/runTest +++ b/test/runTest @@ -553,13 +553,6 @@ testSourceballs() { ../db-update ../cron-jobs/sourceballs - - for log in errors failed; do - if [ -s "${LOGDIR}/sourceballs/${log}.txt" ]; then - fail "${LOGDIR}/sourceballs/${log}.txt includes the following errors:" - cat "${LOGDIR}/sourceballs/${log}.txt" - fi - done for pkgbase in ${pkgs[@]}; do [ ! -r ${FTP_BASE}/${SRCPOOL}/${pkgbase}-*${SRCEXT} ] && fail "source package not found!" done @@ -585,12 +578,6 @@ testSourceballsCleanup() { ../cron-jobs/ftpdir-cleanup >/dev/null ../cron-jobs/sourceballs - for log in errors failed; do - if [ -s "${LOGDIR}/sourceballs/${log}.txt" ]; then - fail "${LOGDIR}/sourceballs/${log}.txt includes the following errors:" - cat "${LOGDIR}/sourceballs/${log}.txt" - fi - done [ -r ${FTP_BASE}/${SRCPOOL}/pkg-simple-a-*${SRCEXT} ] && fail "source package was not removed!" [ ! -r ${FTP_BASE}/${SRCPOOL}/pkg-simple-b-*${SRCEXT} ] && fail "source package not found!" } -- cgit v1.2.2 From cf642f924a4404f85926619324e59b3c878c1284 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 21 Nov 2010 13:59:30 +0100 Subject: sourceballs: Support any packages --- misc-scripts/make-sourceball | 11 ++++++++--- test/runTest | 15 +++++++++++++++ 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/misc-scripts/make-sourceball b/misc-scripts/make-sourceball index fccc4b5..ce87b31 100755 --- a/misc-scripts/make-sourceball +++ b/misc-scripts/make-sourceball @@ -62,8 +62,6 @@ create_srcpackage() { popd >/dev/null return 0 - else - error "PKGBUILD not found in $1" fi } @@ -73,7 +71,14 @@ cd "$WORKDIR" failed=0 if /usr/bin/svn export -q "$SVNREPO/$packagename" $packagename; then - create_srcpackage "$packagename/repos/$reponame-$arch" || failed=1 + if [ -f "$packagename/repos/$reponame-$arch/PKGBUILD" ]; then + create_srcpackage "$packagename/repos/$reponame-$arch" || failed=1 + elif [ -f "$packagename/repos/$reponame-any/PKGBUILD" ]; then + create_srcpackage "$packagename/repos/$reponame-any" || failed=1 + else + error "PKGBUILD for '$packagename' does not exist in repo '$reponame-$arch' or '$reponame-any'" + failed=1 + fi else error "Package '$packagename' does not exist in repo '$reponame-$arch'" failed=1 diff --git a/test/runTest b/test/runTest index 6c60ddc..fe22ce7 100755 --- a/test/runTest +++ b/test/runTest @@ -558,6 +558,21 @@ testSourceballs() { done } +testAnySourceballs() { + local pkgs=('pkg-any-a' 'pkg-any-b') + local pkgbase + + for pkgbase in ${pkgs[@]}; do + releasePackage extra ${pkgbase} any + done + ../db-update + + ../cron-jobs/sourceballs + for pkgbase in ${pkgs[@]}; do + [ ! -r ${FTP_BASE}/${SRCPOOL}/${pkgbase}-*${SRCEXT} ] && fail "source package not found!" + done +} + testSourceballsCleanup() { local arches=('i686' 'x86_64') local pkgs=('pkg-simple-a' 'pkg-simple-b') -- cgit v1.2.2 From 952f8dc9ddbce506e4943afd8e053feebfdf9313 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 21 Nov 2010 14:00:27 +0100 Subject: sourceballs.skip: nexuiz-data has the same sources as nexuiz --- cron-jobs/sourceballs.skip | 1 + 1 file changed, 1 insertion(+) diff --git a/cron-jobs/sourceballs.skip b/cron-jobs/sourceballs.skip index e69de29..9f11862 100644 --- a/cron-jobs/sourceballs.skip +++ b/cron-jobs/sourceballs.skip @@ -0,0 +1 @@ +nexuiz-data -- cgit v1.2.2 From 44a15304da96944f440d2dce3d21111ba4e89ba4 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 21 Nov 2010 15:24:38 +0100 Subject: sourceballs.skip: torcs-data has the same sources as torcs --- cron-jobs/sourceballs.skip | 1 + 1 file changed, 1 insertion(+) diff --git a/cron-jobs/sourceballs.skip b/cron-jobs/sourceballs.skip index 9f11862..64aa8d0 100644 --- a/cron-jobs/sourceballs.skip +++ b/cron-jobs/sourceballs.skip @@ -1 +1,2 @@ nexuiz-data +torcs-data -- cgit v1.2.2 From d3a7d0e50748b9ac973ce24a2920f923c13694e4 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 21 Nov 2010 15:28:34 +0100 Subject: sourceballs.skip: tremulous-data has the same sources as tremulous --- cron-jobs/sourceballs.skip | 1 + 1 file changed, 1 insertion(+) diff --git a/cron-jobs/sourceballs.skip b/cron-jobs/sourceballs.skip index 64aa8d0..b723e47 100644 --- a/cron-jobs/sourceballs.skip +++ b/cron-jobs/sourceballs.skip @@ -1,2 +1,3 @@ nexuiz-data torcs-data +tremulous-data -- cgit v1.2.2 From 2e09ce0c56d27febc6b351e87ca00617217fb7b4 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 21 Nov 2010 15:31:05 +0100 Subject: sourceballs.skip: ufoai-data has the same sources as ufoai --- cron-jobs/sourceballs.skip | 1 + 1 file changed, 1 insertion(+) diff --git a/cron-jobs/sourceballs.skip b/cron-jobs/sourceballs.skip index b723e47..17f3e78 100644 --- a/cron-jobs/sourceballs.skip +++ b/cron-jobs/sourceballs.skip @@ -1,3 +1,4 @@ nexuiz-data torcs-data tremulous-data +ufoai-data -- cgit v1.2.2 From ef593548773265e0418f4a51a0ea817850dfa0ed Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 21 Nov 2010 15:43:25 +0100 Subject: Blacklist more duplicate data packages --- cron-jobs/sourceballs.skip | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cron-jobs/sourceballs.skip b/cron-jobs/sourceballs.skip index 17f3e78..ed88f85 100644 --- a/cron-jobs/sourceballs.skip +++ b/cron-jobs/sourceballs.skip @@ -2,3 +2,8 @@ nexuiz-data torcs-data tremulous-data ufoai-data +frogatto-data +vdrift-data +naev-data +btanks-data +wesnoth-data -- cgit v1.2.2 From 027e0f547c445cce252c8cccfe3d5342de9bb5ec Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 21 Nov 2010 17:02:09 +0100 Subject: sourceballs: Of course sort does only operate on lines --- cron-jobs/sourceballs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 23e8421..b708009 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -22,7 +22,7 @@ for repo in ${PKGREPOS[@]}; do # Read packages from db file # Format is: /- pkgs=($(bsdtar -xOf "${dbfile}" \ - | awk '/^%NAME%/{getline b};/^%BASE%/{getline b};/^%VERSION%/{getline v};/^%ARCH%/{printf "%s/%s ", b, v}' \ + | awk '/^%NAME%/{getline b};/^%BASE%/{getline b};/^%VERSION%/{getline v};/^%ARCH%/{printf "%s/%s\n", b, v}' \ | sort -u)) repo_unlock ${repo} ${arch} -- cgit v1.2.2 From 073f431b2717fec1a3fad24158ca6e2d885ac4f3 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 21 Nov 2010 17:46:01 +0100 Subject: texlive sources are already on ftp.archlinux.org --- cron-jobs/sourceballs.skip | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/cron-jobs/sourceballs.skip b/cron-jobs/sourceballs.skip index ed88f85..14d6f4b 100644 --- a/cron-jobs/sourceballs.skip +++ b/cron-jobs/sourceballs.skip @@ -7,3 +7,23 @@ vdrift-data naev-data btanks-data wesnoth-data +texlive-bin +texlive-bibtexextra +texlive-core +texlive-fontsextra +texlive-formatsextra +texlive-games +texlive-genericextra +texlive-htmlxml +texlive-humanities +texlive-langcjk +texlive-langcyrillic +texlive-langextra +texlive-langgreek +texlive-latexextra +texlive-music +texlive-pictures +texlive-plainextra +texlive-pstricks +texlive-publishers +texlive-science -- cgit v1.2.2 From cb03c851fe6ce0921ddc3f102ab7f5243612dc8b Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Mon, 22 Nov 2010 12:31:28 +0100 Subject: Set CARCH as needed by some PKGBUILDs PKGBUILDs expect CARCH to be set. This is needed when sourcing them directly. --- db-functions | 3 +++ 1 file changed, 3 insertions(+) diff --git a/db-functions b/db-functions index 03859fb..d2df1a7 100644 --- a/db-functions +++ b/db-functions @@ -1,5 +1,8 @@ #!/bin/bash +# Some PKGBUILDs need CARCH to be set +CARCH="x86_64" + # Useful functions UMASK="" set_umask () { -- cgit v1.2.2 From 5869af885163b763abb204dff2cc847556fb247b Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Mon, 22 Nov 2010 14:35:30 +0100 Subject: sourceballs: add for split packages --- test/runTest | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/test/runTest b/test/runTest index fe22ce7..f438704 100755 --- a/test/runTest +++ b/test/runTest @@ -573,6 +573,27 @@ testAnySourceballs() { done } +testSplitSourceballs() { + local arches=('i686' 'x86_64') + local pkgs=('pkg-split-a' 'pkg-split-b') + local pkg + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + releasePackage extra ${pkgbase} ${arch} + done + done + + ../db-update + + ../cron-jobs/sourceballs + for pkgbase in ${pkgs[@]}; do + [ ! -r ${FTP_BASE}/${SRCPOOL}/${pkgbase}-*${SRCEXT} ] && fail "source package not found!" + done +} + testSourceballsCleanup() { local arches=('i686' 'x86_64') local pkgs=('pkg-simple-a' 'pkg-simple-b') -- cgit v1.2.2 From 53ecffc2f30b2678709105892b60073f7c9c0847 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Mon, 22 Nov 2010 14:36:09 +0100 Subject: Add wrappers for repo-add and repo-remove --- db-remove | 4 +--- db-repo-add | 40 ++++++++++++++++++++++++++++++++++++++++ db-repo-remove | 41 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 82 insertions(+), 3 deletions(-) create mode 100755 db-repo-add create mode 100755 db-repo-remove diff --git a/db-remove b/db-remove index 6743eea..b246ce3 100755 --- a/db-remove +++ b/db-remove @@ -16,7 +16,7 @@ ftppath="$FTP_BASE/$repo/os" svnrepo="$repo-$arch" if ! check_repo_permission $repo; then - die "You don't have permission to remove packages from ${reponam}" + die "You don't have permission to remove packages from ${repo}" fi if [ "$arch" == "any" ]; then @@ -40,8 +40,6 @@ else die "$pkgbase not found in $svnrepo" fi - -# copy the db file into our working area for tarch in ${tarches[@]}; do if [ ! -f "$ftppath/$tarch/$repo$DBEXT" ]; then die "No database found at '$ftppath/$tarch', nothing more to do" diff --git a/db-repo-add b/db-repo-add new file mode 100755 index 0000000..b6dd8b8 --- /dev/null +++ b/db-repo-add @@ -0,0 +1,40 @@ +#!/bin/bash + +. "$(dirname $0)/db-functions" +. "$(dirname $0)/config" + +if [ $# -ne 3 ]; then + msg "usage: $(basename $0) " + exit 1 +fi + +pkgfile="$1" +repo="$2" +arch="$3" + +ftppath="$FTP_BASE/$repo/os" + +if ! check_repo_permission $repo; then + die "You don't have permission to add packages to ${repo}" +fi + +if [ "$arch" == "any" ]; then + tarches=(${ARCHES[@]}) +else + tarches=("$arch") +fi + +for tarch in ${tarches[@]}; do + repo_lock $repo $tarch || exit 1 +done + +msg "Adding $pkgfile to [$repo]..." + +for tarch in ${tarches[@]}; do + if [ ! -f "${pkgfile}" ]; then + die "Package file ${pkgfile} not found" + fi + /usr/bin/repo-add -q "$ftppath/$tarch/$repo$DBEXT" ${pkgfile} >/dev/null + set_repo_permission "${repo}" "${tarch}" + repo_unlock $repo $tarch +done diff --git a/db-repo-remove b/db-repo-remove new file mode 100755 index 0000000..6fa0e05 --- /dev/null +++ b/db-repo-remove @@ -0,0 +1,41 @@ +#!/bin/bash + +. "$(dirname $0)/db-functions" +. "$(dirname $0)/config" + +if [ $# -ne 3 ]; then + msg "usage: $(basename $0) " + exit 1 +fi + +pkgname="$1" +repo="$2" +arch="$3" + +ftppath="$FTP_BASE/$repo/os" + +if ! check_repo_permission $repo; then + die "You don't have permission to remove packages from ${repo}" +fi + +if [ "$arch" == "any" ]; then + tarches=(${ARCHES[@]}) +else + tarches=("$arch") +fi + +for tarch in ${tarches[@]}; do + repo_lock $repo $tarch || exit 1 +done + +msg "Removing $pkgname from [$repo]..." + +for tarch in ${tarches[@]}; do + if [ ! -f "$ftppath/$tarch/$repo$DBEXT" ]; then + die "No database found at '$ftppath/$tarch', nothing more to do" + fi + + /usr/bin/repo-remove -q "$ftppath/$tarch/$repo$DBEXT" ${pkgname} >/dev/null + set_repo_permission "${repo}" "${tarch}" + repo_unlock $repo $tarch +done -- cgit v1.2.2 From e343366efd1762336c4c7250dbbf62b7ea0bf47e Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Mon, 22 Nov 2010 19:55:41 +0100 Subject: sourceballs-cleanup: respect .force and .skip files --- misc-scripts/sourceballs-cleanup | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/misc-scripts/sourceballs-cleanup b/misc-scripts/sourceballs-cleanup index fdb1355..595f501 100755 --- a/misc-scripts/sourceballs-cleanup +++ b/misc-scripts/sourceballs-cleanup @@ -55,6 +55,10 @@ for sourceball in "${FTP_BASE}/${SRCPOOL}"/*$SRCEXT; do packagename=$(basename $sourceball) packagename=${packagename%-*-*$SRCEXT} + if grep -q "^${packagename}\$" "$(dirname $0)/../cron-jobs/sourceballs.skip"; then + msg "$packagename : package found in sourceballs.skip. Removing sourceball." + ${CLEANUP_DRYRUN} || mv $sourceball $SOURCE_CLEANUP_DESTDIR + fi if ! /usr/bin/svn export -q --force "$SVNREPO/$packagename" "$packagename" >/dev/null 2>&1; then if [ $? -ne 1 ]; then error "$packagename : svn died during export. Skipping sourceball." @@ -66,8 +70,10 @@ for sourceball in "${FTP_BASE}/${SRCPOOL}"/*$SRCEXT; do warning "$packagename : no longer in repos but trunk is still in svn. Removing sourceball." ${CLEANUP_DRYRUN} || mv $sourceball $SOURCE_CLEANUP_DESTDIR elif ! chk_license $(. "$packagename/trunk/PKGBUILD"; echo ${license[@]}); then - msg "$packagename : source hosting no longer required by license. Removing sourceball." - ${CLEANUP_DRYRUN} || mv $sourceball $SOURCE_CLEANUP_DESTDIR + if ! grep -q "^${packagename}\$" "$(dirname $0)/../cron-jobs/sourceballs.force"; then + msg "$packagename : source hosting no longer required by license. Removing sourceball." + ${CLEANUP_DRYRUN} || mv $sourceball $SOURCE_CLEANUP_DESTDIR + fi else remove_old "$packagename/repos/" fi -- cgit v1.2.2 From 7da36896293bac0c4632c342d99f9ebae9fd7a29 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Mon, 22 Nov 2010 21:49:40 +0100 Subject: Prefer any packages --- misc-scripts/make-sourceball | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/misc-scripts/make-sourceball b/misc-scripts/make-sourceball index ce87b31..1dbb655 100755 --- a/misc-scripts/make-sourceball +++ b/misc-scripts/make-sourceball @@ -71,10 +71,10 @@ cd "$WORKDIR" failed=0 if /usr/bin/svn export -q "$SVNREPO/$packagename" $packagename; then - if [ -f "$packagename/repos/$reponame-$arch/PKGBUILD" ]; then - create_srcpackage "$packagename/repos/$reponame-$arch" || failed=1 - elif [ -f "$packagename/repos/$reponame-any/PKGBUILD" ]; then + if [ -f "$packagename/repos/$reponame-any/PKGBUILD" ]; then create_srcpackage "$packagename/repos/$reponame-any" || failed=1 + elif [ -f "$packagename/repos/$reponame-$arch/PKGBUILD" ]; then + create_srcpackage "$packagename/repos/$reponame-$arch" || failed=1 else error "PKGBUILD for '$packagename' does not exist in repo '$reponame-$arch' or '$reponame-any'" failed=1 -- cgit v1.2.2 From 98113d63c9438eb60c89989edcb57f8239f99456 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Mon, 22 Nov 2010 22:20:16 +0100 Subject: sourceballs: check if source package was really created --- misc-scripts/make-sourceball | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/misc-scripts/make-sourceball b/misc-scripts/make-sourceball index 1dbb655..9a43376 100755 --- a/misc-scripts/make-sourceball +++ b/misc-scripts/make-sourceball @@ -47,17 +47,25 @@ create_srcpackage() { msg "Creating source tarball for $packagename-$pkgver-$pkgrel" fi + local line /usr/bin/makepkg --allsource --ignorearch > "$WORKDIR/makepkg.log" 2>&1 if [ $? -ne 0 ]; then error "Failed to download source for $packagename-$pkgver-$pkgrel ($reponame-$arch)" - local line while read line; do msg2 "${line}" done < "$WORKDIR/makepkg.log" return 1 fi - mv "${packagename}-${pkgver}-${pkgrel}${SRCEXT}" "${FTP_BASE}/${SRCPOOL}" + if [ -f "${packagename}-${pkgver}-${pkgrel}${SRCEXT}" ]; then + mv "${packagename}-${pkgver}-${pkgrel}${SRCEXT}" "${FTP_BASE}/${SRCPOOL}" + else + error "Source package not found: ${packagename}-${pkgver}-${pkgrel}${SRCEXT}" + while read line; do + msg2 "${line}" + done < "$WORKDIR/makepkg.log" + return 1 + fi popd >/dev/null -- cgit v1.2.2 From 9c624b099eb16e27c80ed8ff5f551ce6e8ad0682 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Mon, 22 Nov 2010 22:21:37 +0100 Subject: check_archlinux: fix for python2 usage --- cron-jobs/check_archlinux/README | 2 +- cron-jobs/check_archlinux/alpm.so | Bin 7238 -> 7166 bytes cron-jobs/check_archlinux/check_packages.py | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cron-jobs/check_archlinux/README b/cron-jobs/check_archlinux/README index f3a1b90..8f178dc 100644 --- a/cron-jobs/check_archlinux/README +++ b/cron-jobs/check_archlinux/README @@ -1,5 +1,5 @@ 1) Build the python module -$ python setup.py build +$ python2 setup.py build 2) copy it back to the current working directory $ cp build/lib.*/alpm.* . diff --git a/cron-jobs/check_archlinux/alpm.so b/cron-jobs/check_archlinux/alpm.so index b87b894..50f0f97 100755 Binary files a/cron-jobs/check_archlinux/alpm.so and b/cron-jobs/check_archlinux/alpm.so differ diff --git a/cron-jobs/check_archlinux/check_packages.py b/cron-jobs/check_archlinux/check_packages.py index e8830e0..465d87f 100755 --- a/cron-jobs/check_archlinux/check_packages.py +++ b/cron-jobs/check_archlinux/check_packages.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python2 # # check_archlinux.py # -- cgit v1.2.2 From 1dca721c5149f69067f38d4a33dabbebef98009b Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Mon, 22 Nov 2010 22:41:21 +0100 Subject: Attempt to fix integrity-check --- cron-jobs/integrity-check | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/cron-jobs/integrity-check b/cron-jobs/integrity-check index c120cfd..0a8650c 100755 --- a/cron-jobs/integrity-check +++ b/cron-jobs/integrity-check @@ -1,15 +1,16 @@ #!/bin/bash -basedir="$(dirname $0)" +. "$(dirname $0)/../db-functions" +. "$(dirname $0)/../config" + if [ $# -ne 2 ]; then - echo "usage: $(basename $0) [,,...] " >&2 - exit 1 + die "usage: $(basename $0) [,,...] " fi for arch in ${ARCHES[@]}; do - $basedir/check_archlinux/check_packages.py \ + $(dirname $0)/check_archlinux/check_packages.py \ --repos="$1" \ --abs-tree="/srv/abs/rsync/$arch,/srv/abs/rsync/any" --arch="$arch" |\ - $basedir/devlist-mailer "Integrity Check $arch: $1" "$2" + $(dirname $0)/devlist-mailer "Integrity Check $arch: $1" "$2" done -- cgit v1.2.2 From 4d16d294f338d5e90a92acb2096bca77e9e924e7 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Tue, 23 Nov 2010 12:04:30 +0100 Subject: Rewrite sourceballs to increase performance and reliability * Decrease file stats as much as possible * Create a list of all packages and meta data only once * Create a list of available source packages only once * Create a list of expected packages only once * Combine all three scripts into one to share data and code * Use as much information from the db files as possible and avoid using svn * Avoid attempting to create the same source package twice Logic works as follows: 1) create a list of all packages 2) Check for each package if we need a src package and create one 3) During this process create a list of all src packages that should be there 4) Diff both lists for the cleanup --- cron-jobs/sourceballs | 144 +++++++++++++++++++++++++++------------ db-functions | 35 +--------- misc-scripts/make-sourceball | 97 -------------------------- misc-scripts/sourceballs-cleanup | 82 ---------------------- 4 files changed, 101 insertions(+), 257 deletions(-) delete mode 100755 misc-scripts/make-sourceball delete mode 100755 misc-scripts/sourceballs-cleanup diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index b708009..3f690e7 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -1,57 +1,111 @@ #!/bin/bash -. "$(dirname $0)/../db-functions" -. "$(dirname $0)/../config" +dirname="$(dirname $0)" +. "${dirname}/../db-functions" +. "${dirname}/../config" script_lock -set_umask +for repo in ${PKGREPOS[@]}; do + for arch in ${ARCHES[@]}; do + repo_lock ${repo} ${arch} || exit 1 + done +done + +# Create a readable file for each repo with the following format +# - [ ] +for repo in ${PKGREPOS[@]}; do + for arch in ${ARCHES[@]}; do + if [ ! -f "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" ]; then + warning "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT} not found, skipping" + continue + fi + bsdtar -xOf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" \ + | awk '/^%NAME%/ { getline b }; + /^%BASE%/ { getline b }; + /^%VERSION%/ { getline v }; + /^%LICENSE%/,/^$/ { + if ( !/^%LICENSE%/ ) { l=l" "$0 } + }; + /^%ARCH%/ { + getline a; + printf "%s %s %s %s\n", b, v, a, l; + l=""; + }' + done | sort -u > "${WORKDIR}/db-${repo}" +done + +for repo in ${PKGREPOS[@]}; do + for arch in ${ARCHES[@]}; do + repo_unlock ${repo} ${arch} + done +done -dirname="$(/bin/readlink -f $(/usr/bin/dirname $0))" +# Create a list of all available source package file names +find "${FTP_BASE}/${SRCPOOL}" -xtype f -name "*${SRCEXT}" -printf '%f\n' | sort -u > "${WORKDIR}/available-src-pkgs" +# Check for all packages if we need to build a source package for repo in ${PKGREPOS[@]}; do - for arch in ${ARCHES[@]}; do - ftppath="${FTP_BASE}/$repo/os/$arch" - dbfile="${ftppath}/${repo}${DBEXT}" - if [ ! -r "${dbfile}" ]; then - warning "${dbfile} not found, skipping" - continue - fi - - repo_lock ${repo} ${arch} || exit 1 - # Read packages from db file - # Format is: /- - pkgs=($(bsdtar -xOf "${dbfile}" \ - | awk '/^%NAME%/{getline b};/^%BASE%/{getline b};/^%VERSION%/{getline v};/^%ARCH%/{printf "%s/%s\n", b, v}' \ - | sort -u)) - repo_unlock ${repo} ${arch} - - for pkg in ${pkgs[@]}; do - pkgbase=${pkg%/*} - pkgver=${pkg#*/} - srcpkg="${pkgbase}-${pkgver}${SRCEXT}" - - # Don't do anything for package in this 'blacklist' - if grep -q "^${pkgbase}\$" "$dirname/sourceballs.skip"; then - continue - fi - - # Use this file to 'whitelist' or force building some sourceballs, - # skipping the license check - force="" - if grep -q "^$pkgbase\$" "$dirname/sourceballs.force"; then - force="-f" - fi - - if [ ! -f "${FTP_BASE}/${SRCPOOL}/$srcpkg" ]; then - if ! $dirname/../misc-scripts/make-sourceball $force $pkgbase $repo $arch; then - error "Failed to download sources for $pkgbase" - fi - fi - done - done + msg "Updating source packages for ${repo}..." + while read line; do + pkginfo=(${line}) + pkgbase=${pkginfo[0]} + pkgver=${pkginfo[1]} + pkgarch=${pkginfo[2]} + pkglicense=(${pkginfo[@]:3}) + + # Should this package be skipped? + if grep -Fqx "${pkgbase}" "${dirname}/sourceballs.skip"; then + continue + fi + # Check if the license or .force file does not enforce creating a source package + if ! (chk_license ${pkglicense[@]} || grep -Fqx "${pkgbase}" "${dirname}/sourceballs.force"); then + continue + fi + # Store the expected file name of the source package + echo "${pkgbase}-${pkgver}${SRCEXT}" >> "${WORKDIR}/expected-src-pkgs" + + # Build the source package if its not already there + if ! grep -Fqx "${pkgbase}-${pkgver}${SRCEXT}" "${WORKDIR}/available-src-pkgs"; then + msg2 "${pkgbase}" + # Looks like a previous arch faild; skip it + if [ -d "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" ]; then + continue + fi + mkdir -p "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}" + svn export -q "${SVNREPO}/${pkgbase}/repos/${repo}-${pkgarch}" \ + "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" >/dev/null + if [ $? -ge 1 ]; then + error "Could not check out ${pkgbase}/repos/${repo}-${pkgarch}" + continue + fi + + pushd "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" >/dev/null + makepkg --nocolor --allsource --ignorearch >/dev/null 2>&1 + if [ $? -eq 0 ] && [ -f "${pkgbase}-${pkgver}${SRCEXT}" ]; then + mv "${pkgbase}-${pkgver}${SRCEXT}" "${FTP_BASE}/${SRCPOOL}" + # Avoid creating the same source package for every arch + echo "${pkgbase}-${pkgver}${SRCEXT}" >> "${WORKDIR}/available-src-pkgs" + else + error "Could not create source package for ${pkgbase}/repos/${repo}-${pkgarch}" + fi + popd >/dev/null + fi + done < "${WORKDIR}/db-${repo}" done -$dirname/../misc-scripts/sourceballs-cleanup +# Cleanup old source packages +cat "${WORKDIR}/expected-src-pkgs" | sort -u > "${WORKDIR}/expected-src-pkgs.sort" +cat "${WORKDIR}/available-src-pkgs" | sort -u > "${WORKDIR}/available-src-pkgs.sort" +old_pkgs=($(comm -23 "${WORKDIR}/available-src-pkgs.sort" "${WORKDIR}/expected-src-pkgs.sort")) + +if [ ${#old_pkgs[@]} -ge 1 ]; then + msg "Removing old source packages..." + ${CLEANUP_DRYRUN} && warning 'dry run mode is active' + for old_pkg in ${old_pkgs[@]}; do + msg2 "${old_pkg}" + ${CLEANUP_DRYRUN} || mv "$FTP_BASE/${SRCPOOL}/${old_pkg}" "${SOURCE_CLEANUP_DESTDIR}" + done +fi script_unlock diff --git a/db-functions b/db-functions index d2df1a7..67a52aa 100644 --- a/db-functions +++ b/db-functions @@ -347,44 +347,13 @@ check_pkgrepos() { #usage: chk_license ${license[@]}" chk_license() { local l - local allowed - for l in "$@"; do - l="$(echo $l | tr '[:upper:]' '[:lower:]')" - for allowed in ${ALLOWED_LICENSES[@]}; do - allowed="$(echo $allowed | tr '[:upper:]' '[:lower:]')" - if [ "$l" = "$allowed" ]; then - return 0 - fi - done + for l in ${@}; do + in_array ${l} ${ALLOWED_LICENSES[@]} && return 0 done return 1 } -pkgname_from_src() { - local tmp - local a - tmp=${1##*/} - tmp=${tmp%$SRCEXT} - for a in ${ARCHES[@]}; do - tmp=${tmp%-$a} - done - tmp=${tmp%-any} - echo ${tmp%-*-*} -} - -pkgver_from_src() { - local tmp - local a - tmp=${1##*/} - tmp=${tmp%$SRCEXT} - for a in ${ARCHES[@]}; do - tmp=${tmp%-$a} - done - tmp=${tmp%-any} - echo $tmp | sed 's|.*-\(.*-.*\)$|\1|g' -} - check_repo_permission() { local repo=$1 diff --git a/misc-scripts/make-sourceball b/misc-scripts/make-sourceball deleted file mode 100755 index 9a43376..0000000 --- a/misc-scripts/make-sourceball +++ /dev/null @@ -1,97 +0,0 @@ -#!/bin/bash - -if [ $# -ne 3 -a $# -ne 4 ]; then - msg "usage: $(basename $0) [-f] " - msg " -f Force building. Skip license checks" - exit 1 -fi - -. "$(dirname $0)/../db-functions" -. "$(dirname $0)/../config" - -FORCE=0 -if [ "$1" = "-f" ]; then - FORCE=1 - shift -fi - -packagename="$1" -reponame="$2" -arch="$3" - -script_lock - -create_srcpackage() { - if [ -f "$1/PKGBUILD" ]; then - pushd "$1" >/dev/null - pkgver=$(. PKGBUILD; echo ${pkgver}) - if [ $? -ne 0 ]; then - error "Failed to read pkgver from $1" - return 1 - fi - pkgrel=$(. PKGBUILD; echo ${pkgrel}) - if [ $? -ne 0 ]; then - error "Failed to read pkgrel from $1" - return 1 - fi - license=($(. PKGBUILD; echo ${license[@]})) - if [ $? -ne 0 ]; then - error "Failed to read license from $1" - return 1 - fi - if ! [ $FORCE == 1 ] && ! chk_license ${license[@]} ; then - # Removed so as not to clutter failed.txt - #warning "$packagename license (${license[@]}) does not require source tarballs" - return 0 - else - msg "Creating source tarball for $packagename-$pkgver-$pkgrel" - fi - - local line - /usr/bin/makepkg --allsource --ignorearch > "$WORKDIR/makepkg.log" 2>&1 - if [ $? -ne 0 ]; then - error "Failed to download source for $packagename-$pkgver-$pkgrel ($reponame-$arch)" - while read line; do - msg2 "${line}" - done < "$WORKDIR/makepkg.log" - return 1 - fi - - if [ -f "${packagename}-${pkgver}-${pkgrel}${SRCEXT}" ]; then - mv "${packagename}-${pkgver}-${pkgrel}${SRCEXT}" "${FTP_BASE}/${SRCPOOL}" - else - error "Source package not found: ${packagename}-${pkgver}-${pkgrel}${SRCEXT}" - while read line; do - msg2 "${line}" - done < "$WORKDIR/makepkg.log" - return 1 - fi - - popd >/dev/null - - return 0 - fi -} - -set_umask -cd "$WORKDIR" - -failed=0 - -if /usr/bin/svn export -q "$SVNREPO/$packagename" $packagename; then - if [ -f "$packagename/repos/$reponame-any/PKGBUILD" ]; then - create_srcpackage "$packagename/repos/$reponame-any" || failed=1 - elif [ -f "$packagename/repos/$reponame-$arch/PKGBUILD" ]; then - create_srcpackage "$packagename/repos/$reponame-$arch" || failed=1 - else - error "PKGBUILD for '$packagename' does not exist in repo '$reponame-$arch' or '$reponame-any'" - failed=1 - fi -else - error "Package '$packagename' does not exist in repo '$reponame-$arch'" - failed=1 -fi - -script_unlock - -exit $failed \ No newline at end of file diff --git a/misc-scripts/sourceballs-cleanup b/misc-scripts/sourceballs-cleanup deleted file mode 100755 index 595f501..0000000 --- a/misc-scripts/sourceballs-cleanup +++ /dev/null @@ -1,82 +0,0 @@ -#!/bin/bash - -. "$(dirname $0)/../db-functions" -. "$(dirname $0)/../config" - -script_lock - -${CLEANUP_DRYRUN} && warning 'dry run mode is active' - -remove_old() { - if [ -d "$1" ]; then - pushd "$1" >/dev/null - PKGVERS="" - for repo in *; do - cd "$repo" - if [ -f PKGBUILD ]; then - pkgver=$(. PKGBUILD; echo ${pkgver}) - pkgrel=$(. PKGBUILD; echo ${pkgrel}) - PKGVERS="$PKGVERS $pkgver-$pkgrel" - else - error "PKGBUILD not found in $1/$repo" - fi - cd .. - done - - for srcpkg in "${FTP_BASE}/${SRCPOOL}/$packagename-"*; do - [ -f "$srcpkg" ] || continue - if [ "$(pkgname_from_src $srcpkg)" == "$packagename" ]; then - skip=0 - pver="$(pkgver_from_src $srcpkg)" - for v in $PKGVERS; do - if [ "$v" = "$pver" ]; then - skip=1 - break - fi - done - if [ $skip -ne 1 ]; then - ${CLEANUP_DRYRUN} || mv "$srcpkg" $SOURCE_CLEANUP_DESTDIR - fi - fi - done - - popd >/dev/null - fi -} - -#adjust the nice level to run at a lower priority -/usr/bin/renice +10 -p $$ > /dev/null - -set_umask -cd "$WORKDIR" - -for sourceball in "${FTP_BASE}/${SRCPOOL}"/*$SRCEXT; do - [ -f "$sourceball" ] || continue - packagename=$(basename $sourceball) - packagename=${packagename%-*-*$SRCEXT} - - if grep -q "^${packagename}\$" "$(dirname $0)/../cron-jobs/sourceballs.skip"; then - msg "$packagename : package found in sourceballs.skip. Removing sourceball." - ${CLEANUP_DRYRUN} || mv $sourceball $SOURCE_CLEANUP_DESTDIR - fi - if ! /usr/bin/svn export -q --force "$SVNREPO/$packagename" "$packagename" >/dev/null 2>&1; then - if [ $? -ne 1 ]; then - error "$packagename : svn died during export. Skipping sourceball." - else - msg "$packagename : no longer in svn. Removing sourceball." - ${CLEANUP_DRYRUN} || mv $sourceball $SOURCE_CLEANUP_DESTDIR - fi - elif [ -z "$(ls -A "$packagename/repos")" ]; then - warning "$packagename : no longer in repos but trunk is still in svn. Removing sourceball." - ${CLEANUP_DRYRUN} || mv $sourceball $SOURCE_CLEANUP_DESTDIR - elif ! chk_license $(. "$packagename/trunk/PKGBUILD"; echo ${license[@]}); then - if ! grep -q "^${packagename}\$" "$(dirname $0)/../cron-jobs/sourceballs.force"; then - msg "$packagename : source hosting no longer required by license. Removing sourceball." - ${CLEANUP_DRYRUN} || mv $sourceball $SOURCE_CLEANUP_DESTDIR - fi - else - remove_old "$packagename/repos/" - fi -done - -script_unlock -- cgit v1.2.2 From 68a93c32579aae3ea47c11dc245d5580a4444274 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Tue, 23 Nov 2010 12:57:52 +0100 Subject: sourceballs: be less verbose --- cron-jobs/sourceballs | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 3f690e7..f226288 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -16,8 +16,8 @@ done # - [ ] for repo in ${PKGREPOS[@]}; do for arch in ${ARCHES[@]}; do + # Repo does not exist; skip it if [ ! -f "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" ]; then - warning "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT} not found, skipping" continue fi bsdtar -xOf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" \ @@ -46,7 +46,8 @@ find "${FTP_BASE}/${SRCPOOL}" -xtype f -name "*${SRCEXT}" -printf '%f\n' | sort # Check for all packages if we need to build a source package for repo in ${PKGREPOS[@]}; do - msg "Updating source packages for ${repo}..." + newpkgs=() + failedpkgs=() while read line; do pkginfo=(${line}) pkgbase=${pkginfo[0]} @@ -67,16 +68,15 @@ for repo in ${PKGREPOS[@]}; do # Build the source package if its not already there if ! grep -Fqx "${pkgbase}-${pkgver}${SRCEXT}" "${WORKDIR}/available-src-pkgs"; then - msg2 "${pkgbase}" # Looks like a previous arch faild; skip it if [ -d "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" ]; then continue fi mkdir -p "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}" svn export -q "${SVNREPO}/${pkgbase}/repos/${repo}-${pkgarch}" \ - "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" >/dev/null + "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" >/dev/null 2>&1 if [ $? -ge 1 ]; then - error "Could not check out ${pkgbase}/repos/${repo}-${pkgarch}" + failedpkgs[${#failedpkgs[*]}]="${pkgbase}" continue fi @@ -87,11 +87,26 @@ for repo in ${PKGREPOS[@]}; do # Avoid creating the same source package for every arch echo "${pkgbase}-${pkgver}${SRCEXT}" >> "${WORKDIR}/available-src-pkgs" else - error "Could not create source package for ${pkgbase}/repos/${repo}-${pkgarch}" + failedpkgs[${#failedpkgs[*]}]="${pkgbase}" fi popd >/dev/null + + newpkgs[${#newpkgs[*]}]="${pkgbase}" fi done < "${WORKDIR}/db-${repo}" + + if [ ${#newpkgs[@]} -ge 1 ]; then + msg "Adding source packages for [${repo}]..." + for new_pkg in ${newpkgs[@]}; do + msg2 "${new_pkg}" + done + fi + if [ ${#failedpkgs[@]} -ge 1 ]; then + msg "Failed to create source packages for [${repo}]..." + for failed_pkg in ${failedpkgs[@]}; do + msg2 "${failed_pkg}" + done + fi done # Cleanup old source packages -- cgit v1.2.2 From f318028961ebf045383bab26305b9232b1d34b94 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Tue, 23 Nov 2010 13:12:16 +0100 Subject: sourceballs: fix output --- cron-jobs/sourceballs | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index f226288..16c41f8 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -68,15 +68,11 @@ for repo in ${PKGREPOS[@]}; do # Build the source package if its not already there if ! grep -Fqx "${pkgbase}-${pkgver}${SRCEXT}" "${WORKDIR}/available-src-pkgs"; then - # Looks like a previous arch faild; skip it - if [ -d "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" ]; then - continue - fi mkdir -p "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}" svn export -q "${SVNREPO}/${pkgbase}/repos/${repo}-${pkgarch}" \ "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" >/dev/null 2>&1 if [ $? -ge 1 ]; then - failedpkgs[${#failedpkgs[*]}]="${pkgbase}" + failedpkgs[${#failedpkgs[*]}]="${pkgbase}-${pkgver}${SRCEXT}" continue fi @@ -86,12 +82,11 @@ for repo in ${PKGREPOS[@]}; do mv "${pkgbase}-${pkgver}${SRCEXT}" "${FTP_BASE}/${SRCPOOL}" # Avoid creating the same source package for every arch echo "${pkgbase}-${pkgver}${SRCEXT}" >> "${WORKDIR}/available-src-pkgs" + newpkgs[${#newpkgs[*]}]="${pkgbase}-${pkgver}${SRCEXT}" else - failedpkgs[${#failedpkgs[*]}]="${pkgbase}" + failedpkgs[${#failedpkgs[*]}]="${pkgbase}-${pkgver}${SRCEXT}" fi popd >/dev/null - - newpkgs[${#newpkgs[*]}]="${pkgbase}" fi done < "${WORKDIR}/db-${repo}" -- cgit v1.2.2 From 79f70daa2a450f24d7ce00957651c2cac89599ec Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Tue, 23 Nov 2010 13:19:59 +0100 Subject: sourceballs: avoid failing twice --- cron-jobs/sourceballs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 16c41f8..9b6a1e4 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -68,6 +68,12 @@ for repo in ${PKGREPOS[@]}; do # Build the source package if its not already there if ! grep -Fqx "${pkgbase}-${pkgver}${SRCEXT}" "${WORKDIR}/available-src-pkgs"; then + # Check if we had failed before + if in_array "${pkgbase}-${pkgver}${SRCEXT}" ${failedpkgs[@]}; then + continue + fi + + # Get the sources from svn mkdir -p "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}" svn export -q "${SVNREPO}/${pkgbase}/repos/${repo}-${pkgarch}" \ "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" >/dev/null 2>&1 @@ -76,6 +82,7 @@ for repo in ${PKGREPOS[@]}; do continue fi + # Build the actual source package pushd "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" >/dev/null makepkg --nocolor --allsource --ignorearch >/dev/null 2>&1 if [ $? -eq 0 ] && [ -f "${pkgbase}-${pkgver}${SRCEXT}" ]; then -- cgit v1.2.2 From 2a4938ecf8e9bddc75cd2533812cb587812eb928 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Tue, 23 Nov 2010 14:44:16 +0100 Subject: sourceballs: Move to our workdir first to fix warning with sudo usage --- cron-jobs/sourceballs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 9b6a1e4..9d52d11 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -1,5 +1,7 @@ #!/bin/bash +pushd "${WORKDIR}" >/dev/null + dirname="$(dirname $0)" . "${dirname}/../db-functions" . "${dirname}/../config" -- cgit v1.2.2 From 3455be4e4a3fa1c70580fd6ccb521f61c98c64c2 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Wed, 24 Nov 2010 07:32:18 +0100 Subject: sourceballs: fix working directory --- cron-jobs/sourceballs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 9d52d11..6a86a0d 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -1,10 +1,9 @@ #!/bin/bash -pushd "${WORKDIR}" >/dev/null - -dirname="$(dirname $0)" +dirname="$(dirname $(readlink -e $0))" . "${dirname}/../db-functions" . "${dirname}/../config" +pushd "${WORKDIR}" >/dev/null script_lock -- cgit v1.2.2 From cadd215e9d726030bd7b86396eda47c2c69c1bbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20B=C3=A9langer?= Date: Wed, 24 Nov 2010 15:13:13 -0500 Subject: Added seperate dryrun options for the packages and sources cleanup scripts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Eric Bélanger Signed-off-by: Pierre Schmitz --- config | 1 + cron-jobs/sourceballs | 4 ++-- test/lib/common.inc | 1 + 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/config b/config index 01956ba..69f40c7 100644 --- a/config +++ b/config @@ -7,6 +7,7 @@ SRCPOOL='' CLEANUP_DESTDIR="/srv/package-cleanup" CLEANUP_DRYRUN=false SOURCE_CLEANUP_DESTDIR="/srv/source-cleanup" +SOURCE_CLEANUP_DRYRUN=false LOCK_DELAY=10 LOCK_TIMEOUT=300 diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 6a86a0d..9ba003a 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -119,10 +119,10 @@ old_pkgs=($(comm -23 "${WORKDIR}/available-src-pkgs.sort" "${WORKDIR}/expected-s if [ ${#old_pkgs[@]} -ge 1 ]; then msg "Removing old source packages..." - ${CLEANUP_DRYRUN} && warning 'dry run mode is active' + ${SOURCE_CLEANUP_DRYRUN} && warning 'dry run mode is active' for old_pkg in ${old_pkgs[@]}; do msg2 "${old_pkg}" - ${CLEANUP_DRYRUN} || mv "$FTP_BASE/${SRCPOOL}/${old_pkg}" "${SOURCE_CLEANUP_DESTDIR}" + ${SOURCE_CLEANUP_DRYRUN} || mv "$FTP_BASE/${SRCPOOL}/${old_pkg}" "${SOURCE_CLEANUP_DESTDIR}" done fi diff --git a/test/lib/common.inc b/test/lib/common.inc index 6619c0a..d7ea3d4 100644 --- a/test/lib/common.inc +++ b/test/lib/common.inc @@ -68,6 +68,7 @@ setUp() { STAGING="${TMP}/staging" TMPDIR="${TMP}/tmp" CLEANUP_DRYRUN=false + SOURCE_CLEANUP_DRYRUN=false eot . "${curdir}/../config" } -- cgit v1.2.2 From ec20965dbd3180a92d2daa69a075284688b1be04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eric=20B=C3=A9langer?= Date: Wed, 24 Nov 2010 15:31:22 -0500 Subject: Make ftpdir-cleanup less verbose MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Eric Bélanger Signed-off-by: Pierre Schmitz --- cron-jobs/ftpdir-cleanup | 1 - 1 file changed, 1 deletion(-) diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index 914ef8c..1df4edc 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -31,7 +31,6 @@ ${CLEANUP_DRYRUN} && warning 'dry run mode is active' for repo in ${PKGREPOS[@]}; do for arch in ${ARCHES[@]}; do if [ ! -f "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" ]; then - warning "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT} not found, skipping" continue fi # get a list of actual available package files -- cgit v1.2.2 From 5f72b8029c07f3357014d8716f18f21ce649c98a Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Thu, 25 Nov 2010 06:55:03 +0100 Subject: Provide more real test packages * Use devtools instead of plain makepkg * This is a preparation for some more advanced test cases. --- .gitignore | 1 + test/lib/common.inc | 39 ++++++++++-- test/lib/makepkg-i686.conf | 115 ------------------------------------ test/lib/makepkg-x86_64.conf | 115 ------------------------------------ test/packages/pkg-any-a/PKGBUILD | 3 +- test/packages/pkg-any-b/PKGBUILD | 3 +- test/packages/pkg-simple-a/Makefile | 1 + test/packages/pkg-simple-a/PKGBUILD | 13 +++- test/packages/pkg-simple-a/test.c | 1 + test/packages/pkg-simple-b/Makefile | 1 + test/packages/pkg-simple-b/PKGBUILD | 13 +++- test/packages/pkg-simple-b/test.c | 1 + test/packages/pkg-split-a/Makefile | 1 + test/packages/pkg-split-a/PKGBUILD | 16 ++++- test/packages/pkg-split-a/test.c | 1 + test/packages/pkg-split-b/Makefile | 1 + test/packages/pkg-split-b/PKGBUILD | 17 +++++- test/packages/pkg-split-b/test.c | 1 + test/runTest | 24 ++++---- test/src/Makefile | 5 ++ test/src/test.c | 7 +++ 21 files changed, 121 insertions(+), 258 deletions(-) delete mode 100644 test/lib/makepkg-i686.conf delete mode 100644 test/lib/makepkg-x86_64.conf create mode 120000 test/packages/pkg-simple-a/Makefile create mode 120000 test/packages/pkg-simple-a/test.c create mode 120000 test/packages/pkg-simple-b/Makefile create mode 120000 test/packages/pkg-simple-b/test.c create mode 120000 test/packages/pkg-split-a/Makefile create mode 120000 test/packages/pkg-split-a/test.c create mode 120000 test/packages/pkg-split-b/Makefile create mode 120000 test/packages/pkg-split-b/test.c create mode 100644 test/src/Makefile create mode 100644 test/src/test.c diff --git a/.gitignore b/.gitignore index 01daf28..de95927 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ *~ /config.local +test/packages/*/*.pkg.tar.* diff --git a/test/lib/common.inc b/test/lib/common.inc index d7ea3d4..34de9eb 100644 --- a/test/lib/common.inc +++ b/test/lib/common.inc @@ -5,15 +5,42 @@ set -E oneTimeSetUp() { local p + local d + local a + local pkgname + local pkgarch + local pkgversion + local build pkgdir="$(mktemp -d /dev/shm/$(basename $0).XXXXXXXXXX)" cp -r ${curdir}/packages/* "${pkgdir}" msg 'Building packages...' - for p in "${pkgdir}"/*; do - pushd $p >/dev/null - linux32 makepkg -cf --config ${curdir}/lib/makepkg-i686.conf >/dev/null 2>&1 || die 'makepkg failed' - [ -f *-any.pkg.tar.* ] \ - || linux64 makepkg -cf --config ${curdir}/lib/makepkg-x86_64.conf >/dev/null 2>&1 \ - || die 'makepkg failed' + for d in "${pkgdir}"/*; do + pushd $d >/dev/null + pkgname=($(. PKGBUILD; echo ${pkgname[@]})) + pkgarch=($(. PKGBUILD; echo ${arch[@]})) + pkgversion=$(. PKGBUILD; echo ${pkgver}-${pkgrel}) + + build=true + for a in ${pkgarch[@]}; do + for p in ${pkgname[@]}; do + [ ! -f ${p}-${pkgversion}-${a}${PKGEXT} ] && build=false + done + done + + if ! ${build}; then + if [ "${pkgarch[0]}" == 'any' ]; then + extra-x86_64-build || die 'extra-x86_64-build failed' + else + for a in ${pkgarch[@]}; do + extra-${a}-build || die "extra-${a}-build failed" + done + fi + for a in ${pkgarch[@]}; do + for p in ${pkgname[@]}; do + cp ${p}-${pkgversion}-${a}${PKGEXT} ${curdir}/packages/$(basename ${d}) + done + done + fi popd >/dev/null done echo diff --git a/test/lib/makepkg-i686.conf b/test/lib/makepkg-i686.conf deleted file mode 100644 index 553f943..0000000 --- a/test/lib/makepkg-i686.conf +++ /dev/null @@ -1,115 +0,0 @@ -# -# /etc/makepkg.conf -# - -######################################################################### -# SOURCE ACQUISITION -######################################################################### -# -#-- The download utilities that makepkg should use to acquire sources -# Format: 'protocol::agent' -DLAGENTS=('ftp::/usr/bin/wget -c --passive-ftp -t 3 --waitretry=3 -O %o %u' - 'http::/usr/bin/wget -c -t 3 --waitretry=3 -O %o %u' - 'https::/usr/bin/wget -c -t 3 --waitretry=3 --no-check-certificate -O %o %u' - 'rsync::/usr/bin/rsync -z %u %o' - 'scp::/usr/bin/scp -C %u %o') - -# Other common tools: -# /usr/bin/snarf -# /usr/bin/lftpget -c -# /usr/bin/curl - -######################################################################### -# ARCHITECTURE, COMPILE FLAGS -######################################################################### -# -CARCH="i686" -CHOST="i686-pc-linux-gnu" - -#-- Exclusive: will only run on i686 -# -march (or -mcpu) builds exclusively for an architecture -# -mtune optimizes for an architecture, but builds for whole processor family -CFLAGS="-march=i686 -mtune=generic -O2 -pipe" -CXXFLAGS="-march=i686 -mtune=generic -O2 -pipe" -LDFLAGS="-Wl,--hash-style=gnu -Wl,--as-needed" -#-- Make Flags: change this for DistCC/SMP systems -#MAKEFLAGS="-j2" - -######################################################################### -# BUILD ENVIRONMENT -######################################################################### -# -# Defaults: BUILDENV=(fakeroot !distcc color !ccache) -# A negated environment option will do the opposite of the comments below. -# -#-- fakeroot: Allow building packages as a non-root user -#-- distcc: Use the Distributed C/C++/ObjC compiler -#-- color: Colorize output messages -#-- ccache: Use ccache to cache compilation -# -BUILDENV=(fakeroot !distcc color !ccache) -# -#-- If using DistCC, your MAKEFLAGS will also need modification. In addition, -#-- specify a space-delimited list of hosts running in the DistCC cluster. -#DISTCC_HOSTS="" - -######################################################################### -# GLOBAL PACKAGE OPTIONS -# These are default values for the options=() settings -######################################################################### -# -# Default: OPTIONS=(strip docs libtool emptydirs zipman purge) -# A negated option will do the opposite of the comments below. -# -#-- strip: Strip symbols from binaries/libraries in STRIP_DIRS -#-- docs: Save doc directories specified by DOC_DIRS -#-- libtool: Leave libtool (.la) files in packages -#-- emptydirs: Leave empty directories in packages -#-- zipman: Compress manual (man and info) pages in MAN_DIRS with gzip -#-- purge: Remove files specified by PURGE_TARGETS -# -OPTIONS=(strip docs libtool emptydirs zipman purge) - -#-- File integrity checks to use. Valid: md5, sha1, sha256, sha384, sha512 -INTEGRITY_CHECK=(md5) -#-- Options to be used when stripping binaries. See `man strip' for details. -STRIP_BINARIES="--strip-all" -#-- Options to be used when stripping shared libraries. See `man strip' for details. -STRIP_SHARED="--strip-unneeded" -#-- Options to be used when stripping static libraries. See `man strip' for details. -STRIP_STATIC="--strip-debug" -#-- Manual (man and info) directories to compress (if zipman is specified) -MAN_DIRS=({usr{,/local}{,/share},opt/*}/{man,info}) -#-- Doc directories to remove (if !docs is specified) -DOC_DIRS=(usr/{,local/}{,share/}{doc,gtk-doc} opt/*/{doc,gtk-doc}) -#-- Directories to be searched for the strip option (if strip is specified) -STRIP_DIRS=(bin lib sbin usr/{bin,lib,sbin,local/{bin,lib,sbin}} opt/*/{bin,lib,sbin}) -#-- Files to be removed from all packages (if purge is specified) -PURGE_TARGETS=(usr/{,share}/info/dir .packlist *.pod) - -######################################################################### -# PACKAGE OUTPUT -######################################################################### -# -# Default: put built package and cached source in build directory -# -#-- Destination: specify a fixed directory where all packages will be placed -#PKGDEST=/home/packages -#-- Source cache: specify a fixed directory where source files will be cached -#SRCDEST=/home/sources -#-- Source packages: specify a fixed directory where all src packages will be placed -#SRCPKGDEST=/home/srcpackages -#-- Packager: name/email of the person or organization building packages -#PACKAGER="John Doe " - -######################################################################### -# EXTENSION DEFAULTS -######################################################################### -# -# WARNING: Do NOT modify these variables unless you know what you are -# doing. -# -PKGEXT='.pkg.tar.xz' -SRCEXT='.src.tar.gz' - -# vim: set ft=sh ts=2 sw=2 et: diff --git a/test/lib/makepkg-x86_64.conf b/test/lib/makepkg-x86_64.conf deleted file mode 100644 index e53375f..0000000 --- a/test/lib/makepkg-x86_64.conf +++ /dev/null @@ -1,115 +0,0 @@ -# -# /etc/makepkg.conf -# - -######################################################################### -# SOURCE ACQUISITION -######################################################################### -# -#-- The download utilities that makepkg should use to acquire sources -# Format: 'protocol::agent' -DLAGENTS=('ftp::/usr/bin/wget -c --passive-ftp -t 3 --waitretry=3 -O %o %u' - 'http::/usr/bin/wget -c -t 3 --waitretry=3 -O %o %u' - 'https::/usr/bin/wget -c -t 3 --waitretry=3 --no-check-certificate -O %o %u' - 'rsync::/usr/bin/rsync -z %u %o' - 'scp::/usr/bin/scp -C %u %o') - -# Other common tools: -# /usr/bin/snarf -# /usr/bin/lftpget -c -# /usr/bin/curl - -######################################################################### -# ARCHITECTURE, COMPILE FLAGS -######################################################################### -# -CARCH="x86_64" -CHOST="x86_64-unknown-linux-gnu" - -#-- Exclusive: will only run on x86_64 -# -march (or -mcpu) builds exclusively for an architecture -# -mtune optimizes for an architecture, but builds for whole processor family -CFLAGS="-march=x86-64 -mtune=generic -O2 -pipe" -CXXFLAGS="-march=x86-64 -mtune=generic -O2 -pipe" -LDFLAGS="-Wl,--hash-style=gnu -Wl,--as-needed" -#-- Make Flags: change this for DistCC/SMP systems -#MAKEFLAGS="-j2" - -######################################################################### -# BUILD ENVIRONMENT -######################################################################### -# -# Defaults: BUILDENV=(fakeroot !distcc color !ccache) -# A negated environment option will do the opposite of the comments below. -# -#-- fakeroot: Allow building packages as a non-root user -#-- distcc: Use the Distributed C/C++/ObjC compiler -#-- color: Colorize output messages -#-- ccache: Use ccache to cache compilation -# -BUILDENV=(fakeroot !distcc color !ccache) -# -#-- If using DistCC, your MAKEFLAGS will also need modification. In addition, -#-- specify a space-delimited list of hosts running in the DistCC cluster. -#DISTCC_HOSTS="" - -######################################################################### -# GLOBAL PACKAGE OPTIONS -# These are default values for the options=() settings -######################################################################### -# -# Default: OPTIONS=(strip docs libtool emptydirs zipman purge) -# A negated option will do the opposite of the comments below. -# -#-- strip: Strip symbols from binaries/libraries in STRIP_DIRS -#-- docs: Save doc directories specified by DOC_DIRS -#-- libtool: Leave libtool (.la) files in packages -#-- emptydirs: Leave empty directories in packages -#-- zipman: Compress manual (man and info) pages in MAN_DIRS with gzip -#-- purge: Remove files specified by PURGE_TARGETS -# -OPTIONS=(strip docs libtool emptydirs zipman purge) - -#-- File integrity checks to use. Valid: md5, sha1, sha256, sha384, sha512 -INTEGRITY_CHECK=(md5) -#-- Options to be used when stripping binaries. See `man strip' for details. -STRIP_BINARIES="--strip-all" -#-- Options to be used when stripping shared libraries. See `man strip' for details. -STRIP_SHARED="--strip-unneeded" -#-- Options to be used when stripping static libraries. See `man strip' for details. -STRIP_STATIC="--strip-debug" -#-- Manual (man and info) directories to compress (if zipman is specified) -MAN_DIRS=({usr{,/local}{,/share},opt/*}/{man,info}) -#-- Doc directories to remove (if !docs is specified) -DOC_DIRS=(usr/{,local/}{,share/}{doc,gtk-doc} opt/*/{doc,gtk-doc}) -#-- Directories to be searched for the strip option (if strip is specified) -STRIP_DIRS=(bin lib sbin usr/{bin,lib,sbin,local/{bin,lib,sbin}} opt/*/{bin,lib,sbin}) -#-- Files to be removed from all packages (if purge is specified) -PURGE_TARGETS=(usr/{,share}/info/dir .packlist *.pod) - -######################################################################### -# PACKAGE OUTPUT -######################################################################### -# -# Default: put built package and cached source in build directory -# -#-- Destination: specify a fixed directory where all packages will be placed -#PKGDEST=/home/packages -#-- Source cache: specify a fixed directory where source files will be cached -#SRCDEST=/home/sources -#-- Source packages: specify a fixed directory where all src packages will be placed -#SRCPKGDEST=/home/srcpackages -#-- Packager: name/email of the person or organization building packages -#PACKAGER="John Doe " - -######################################################################### -# EXTENSION DEFAULTS -######################################################################### -# -# WARNING: Do NOT modify these variables unless you know what you are -# doing. -# -PKGEXT='.pkg.tar.xz' -SRCEXT='.src.tar.gz' - -# vim: set ft=sh ts=2 sw=2 et: diff --git a/test/packages/pkg-any-a/PKGBUILD b/test/packages/pkg-any-a/PKGBUILD index 348bc4b..8749a35 100644 --- a/test/packages/pkg-any-a/PKGBUILD +++ b/test/packages/pkg-any-a/PKGBUILD @@ -7,5 +7,6 @@ url='http://www.archlinux.org/' license=('GPL') package() { - true + install -d -m755 ${pkgdir}/usr/share/${pkgname} + echo 'test' > ${pkgdir}/usr/share/${pkgname}/test } diff --git a/test/packages/pkg-any-b/PKGBUILD b/test/packages/pkg-any-b/PKGBUILD index fd42748..e6a0498 100644 --- a/test/packages/pkg-any-b/PKGBUILD +++ b/test/packages/pkg-any-b/PKGBUILD @@ -7,5 +7,6 @@ url='http://www.archlinux.org/' license=('GPL') package() { - true + install -d -m755 ${pkgdir}/usr/share/${pkgname} + echo 'test' > ${pkgdir}/usr/share/${pkgname}/test } diff --git a/test/packages/pkg-simple-a/Makefile b/test/packages/pkg-simple-a/Makefile new file mode 120000 index 0000000..50be211 --- /dev/null +++ b/test/packages/pkg-simple-a/Makefile @@ -0,0 +1 @@ +../../src/Makefile \ No newline at end of file diff --git a/test/packages/pkg-simple-a/PKGBUILD b/test/packages/pkg-simple-a/PKGBUILD index 78f3c8e..953ecfa 100644 --- a/test/packages/pkg-simple-a/PKGBUILD +++ b/test/packages/pkg-simple-a/PKGBUILD @@ -5,7 +5,18 @@ pkgdesc="A package called ${pkgname}" arch=('i686' 'x86_64') url='http://www.archlinux.org/' license=('GPL') +depends=('glibc') +makedepends=('gcc') +source=('Makefile' 'test.c') +md5sums=('c6cb8dcc86253355fed559416d0c8dcf' + '3c1e4279feb678fd9cabaccdb28e40d0') + +build() { + cd ${srcdir} + make +} package() { - true + cd ${srcdir} + make install DESTDIR=${pkgdir} DESTBIN=${pkgname} } diff --git a/test/packages/pkg-simple-a/test.c b/test/packages/pkg-simple-a/test.c new file mode 120000 index 0000000..ed5b5ac --- /dev/null +++ b/test/packages/pkg-simple-a/test.c @@ -0,0 +1 @@ +../../src/test.c \ No newline at end of file diff --git a/test/packages/pkg-simple-b/Makefile b/test/packages/pkg-simple-b/Makefile new file mode 120000 index 0000000..50be211 --- /dev/null +++ b/test/packages/pkg-simple-b/Makefile @@ -0,0 +1 @@ +../../src/Makefile \ No newline at end of file diff --git a/test/packages/pkg-simple-b/PKGBUILD b/test/packages/pkg-simple-b/PKGBUILD index 1ef298e..95ffd09 100644 --- a/test/packages/pkg-simple-b/PKGBUILD +++ b/test/packages/pkg-simple-b/PKGBUILD @@ -5,7 +5,18 @@ pkgdesc="A package called ${pkgname}" arch=('i686' 'x86_64') url='http://www.archlinux.org/' license=('GPL') +depends=('glibc') +makedepends=('gcc') +source=('Makefile' 'test.c') +md5sums=('c6cb8dcc86253355fed559416d0c8dcf' + '3c1e4279feb678fd9cabaccdb28e40d0') + +build() { + cd ${srcdir} + make +} package() { - true + cd ${srcdir} + make install DESTDIR=${pkgdir} DESTBIN=${pkgname} } diff --git a/test/packages/pkg-simple-b/test.c b/test/packages/pkg-simple-b/test.c new file mode 120000 index 0000000..ed5b5ac --- /dev/null +++ b/test/packages/pkg-simple-b/test.c @@ -0,0 +1 @@ +../../src/test.c \ No newline at end of file diff --git a/test/packages/pkg-split-a/Makefile b/test/packages/pkg-split-a/Makefile new file mode 120000 index 0000000..50be211 --- /dev/null +++ b/test/packages/pkg-split-a/Makefile @@ -0,0 +1 @@ +../../src/Makefile \ No newline at end of file diff --git a/test/packages/pkg-split-a/PKGBUILD b/test/packages/pkg-split-a/PKGBUILD index 6ac77ea..e941976 100644 --- a/test/packages/pkg-split-a/PKGBUILD +++ b/test/packages/pkg-split-a/PKGBUILD @@ -6,11 +6,23 @@ pkgdesc="A split package called ${pkgbase}" arch=('i686' 'x86_64') url='http://www.archlinux.org/' license=('GPL') +depends=('glibc') +makedepends=('gcc') +source=('Makefile' 'test.c') +md5sums=('c6cb8dcc86253355fed559416d0c8dcf' + '3c1e4279feb678fd9cabaccdb28e40d0') + +build() { + cd ${srcdir} + make +} package_pkg-split-a1() { - true + cd ${srcdir} + make install DESTDIR=${pkgdir} DESTBIN=${pkgname[0]} } package_pkg-split-a2() { - true + cd ${srcdir} + make install DESTDIR=${pkgdir} DESTBIN=${pkgname[1]} } diff --git a/test/packages/pkg-split-a/test.c b/test/packages/pkg-split-a/test.c new file mode 120000 index 0000000..ed5b5ac --- /dev/null +++ b/test/packages/pkg-split-a/test.c @@ -0,0 +1 @@ +../../src/test.c \ No newline at end of file diff --git a/test/packages/pkg-split-b/Makefile b/test/packages/pkg-split-b/Makefile new file mode 120000 index 0000000..50be211 --- /dev/null +++ b/test/packages/pkg-split-b/Makefile @@ -0,0 +1 @@ +../../src/Makefile \ No newline at end of file diff --git a/test/packages/pkg-split-b/PKGBUILD b/test/packages/pkg-split-b/PKGBUILD index e1817b7..6ddbc45 100644 --- a/test/packages/pkg-split-b/PKGBUILD +++ b/test/packages/pkg-split-b/PKGBUILD @@ -7,10 +7,23 @@ arch=('i686' 'x86_64') url='http://www.archlinux.org/' license=('GPL') +depends=('glibc') +makedepends=('gcc') +source=('Makefile' 'test.c') +md5sums=('c6cb8dcc86253355fed559416d0c8dcf' + '3c1e4279feb678fd9cabaccdb28e40d0') + +build() { + cd ${srcdir} + make +} + package_pkg-split-b1() { - true + cd ${srcdir} + make install DESTDIR=${pkgdir} DESTBIN=${pkgname[0]} } package_pkg-split-b2() { - true + cd ${srcdir} + make install DESTDIR=${pkgdir} DESTBIN=${pkgname[1]} } diff --git a/test/packages/pkg-split-b/test.c b/test/packages/pkg-split-b/test.c new file mode 120000 index 0000000..ed5b5ac --- /dev/null +++ b/test/packages/pkg-split-b/test.c @@ -0,0 +1 @@ +../../src/test.c \ No newline at end of file diff --git a/test/runTest b/test/runTest index f438704..0fcdf18 100755 --- a/test/runTest +++ b/test/runTest @@ -8,12 +8,8 @@ curdir=$(readlink -e $(dirname $0)) # testPackages() { - local p - for p in "${pkgdir}"/*; do - pushd $p >/dev/null - namcap *.pkg.tar.* || fail 'namcap failed' - popd >/dev/null - done + # TODO: namcap -r depends fails with i686 packages + find "${pkgdir}" -name "*${PKGEXT}" -exec namcap -e depends {} + || fail 'namcap failed' } @@ -94,7 +90,7 @@ testUpdateAnyPackage() { pushd "${TMP}/svn-packages-copy/pkg-any-a/trunk/" >/dev/null sed 's/pkgrel=1/pkgrel=2/g' -i PKGBUILD svn commit -q -m"update pkg to pkgrel=2" >/dev/null - makepkg -cf --config ${curdir}/lib/makepkg-i686.conf >/dev/null 2>&1 + extra-i686-build >/dev/null 2>&1 mv pkg-any-a-1-2-any.pkg.tar.xz "${pkgdir}/pkg-any-a/" popd >/dev/null @@ -103,7 +99,7 @@ testUpdateAnyPackage() { checkAnyPackage extra pkg-any-a-1-2-any.pkg.tar.xz any - rm "${pkgdir}/pkg-any-a/pkg-any-a-1-2-any.pkg.tar.xz" + rm -f "${pkgdir}/pkg-any-a/pkg-any-a-1-2-any.pkg.tar.xz" } testUpdateAnyPackageToDifferentRepositoriesAtOnce() { @@ -112,7 +108,7 @@ testUpdateAnyPackageToDifferentRepositoriesAtOnce() { pushd "${TMP}/svn-packages-copy/pkg-any-a/trunk/" >/dev/null sed 's/pkgrel=1/pkgrel=2/g' -i PKGBUILD svn commit -q -m"update pkg to pkgrel=2" >/dev/null - makepkg -cf --config ${curdir}/lib/makepkg-i686.conf >/dev/null 2>&1 + extra-i686-build >/dev/null 2>&1 mv pkg-any-a-1-2-any.pkg.tar.xz "${pkgdir}/pkg-any-a/" popd >/dev/null @@ -123,7 +119,7 @@ testUpdateAnyPackageToDifferentRepositoriesAtOnce() { checkAnyPackage extra pkg-any-a-1-1-any.pkg.tar.xz any checkAnyPackage testing pkg-any-a-1-2-any.pkg.tar.xz any - rm "${pkgdir}/pkg-any-a/pkg-any-a-1-2-any.pkg.tar.xz" + rm -f "${pkgdir}/pkg-any-a/pkg-any-a-1-2-any.pkg.tar.xz" } testUpdateSameAnyPackageToDifferentRepositories() { @@ -279,13 +275,13 @@ testTesting2xAnyPackage() { pushd "${TMP}/svn-packages-copy/pkg-any-a/trunk/" >/dev/null sed 's/pkgrel=1/pkgrel=2/g' -i PKGBUILD svn commit -q -m"update pkg to pkgrel=2" >/dev/null - makepkg -cf --config ${curdir}/lib/makepkg-i686.conf >/dev/null 2>&1 + extra-i686-build >/dev/null 2>&1 mv pkg-any-a-1-2-any.pkg.tar.xz "${pkgdir}/pkg-any-a/" popd >/dev/null releasePackage testing pkg-any-a any ../db-update - rm "${pkgdir}/pkg-any-a/pkg-any-a-1-2-any.pkg.tar.xz" + rm -f "${pkgdir}/pkg-any-a/pkg-any-a-1-2-any.pkg.tar.xz" ../testing2x pkg-any-a @@ -449,13 +445,13 @@ testUpdateAnyPackageWithoutPool() { pushd "${TMP}/svn-packages-copy/${pkgname}/trunk/" >/dev/null sed 's/pkgrel=1/pkgrel=2/g' -i PKGBUILD svn commit -q -m"update pkg to pkgrel=2" >/dev/null - makepkg -cf --config ${curdir}/lib/makepkg-i686.conf >/dev/null 2>&1 + extra-i686-build >/dev/null 2>&1 mv "${pkg2}" "${pkgdir}/${pkgname}/" popd >/dev/null releasePackage extra ${pkgname} any ../db-update - rm "${pkgdir}/${pkgname}/${pkg2}" + rm -f "${pkgdir}/${pkgname}/${pkg2}" ../cron-jobs/ftpdir-cleanup >/dev/null diff --git a/test/src/Makefile b/test/src/Makefile new file mode 100644 index 0000000..105b730 --- /dev/null +++ b/test/src/Makefile @@ -0,0 +1,5 @@ +all: + gcc $(CFLAGS) -o test test.c + +install: + install -D -m755 test $(DESTDIR)/usr/bin/$(DESTBIN) diff --git a/test/src/test.c b/test/src/test.c new file mode 100644 index 0000000..a661689 --- /dev/null +++ b/test/src/test.c @@ -0,0 +1,7 @@ +#include +#include + +int main(void) { + printf("Arch is the best!\n"); + return EXIT_SUCCESS; +} -- cgit v1.2.2 From 793d78130ba808ee688c8b7139a2bc6180b29534 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Thu, 25 Nov 2010 08:18:26 +0100 Subject: Split tests into several files * tests can be run seperatly * runTest will run all tests that have the x bit set --- test/lib/common.inc | 17 +- test/runTest | 628 +---------------------------------------- test/test.d/db-move.sh | 76 +++++ test/test.d/db-remove.sh | 52 ++++ test/test.d/db-update.sh | 127 +++++++++ test/test.d/ftpdir-cleanup.sh | 93 ++++++ test/test.d/packages.sh | 11 + test/test.d/pool-transition.sh | 152 ++++++++++ test/test.d/sourceballs.sh | 85 ++++++ test/test.d/testing2x.sh | 27 ++ 10 files changed, 644 insertions(+), 624 deletions(-) create mode 100755 test/test.d/db-move.sh create mode 100755 test/test.d/db-remove.sh create mode 100755 test/test.d/db-update.sh create mode 100755 test/test.d/ftpdir-cleanup.sh create mode 100755 test/test.d/packages.sh create mode 100755 test/test.d/pool-transition.sh create mode 100755 test/test.d/sourceballs.sh create mode 100755 test/test.d/testing2x.sh diff --git a/test/lib/common.inc b/test/lib/common.inc index 34de9eb..ba5d685 100644 --- a/test/lib/common.inc +++ b/test/lib/common.inc @@ -1,7 +1,7 @@ set -E -. "${curdir}/../config" -. "${curdir}/../db-functions" +. "$(dirname ${BASH_SOURCE[0]})/../../config" +. "$(dirname ${BASH_SOURCE[0]})/../../db-functions" oneTimeSetUp() { local p @@ -12,7 +12,7 @@ oneTimeSetUp() { local pkgversion local build pkgdir="$(mktemp -d /dev/shm/$(basename $0).XXXXXXXXXX)" - cp -r ${curdir}/packages/* "${pkgdir}" + cp -r $(dirname ${BASH_SOURCE[0]})/../packages/* "${pkgdir}" msg 'Building packages...' for d in "${pkgdir}"/*; do pushd $d >/dev/null @@ -37,13 +37,12 @@ oneTimeSetUp() { fi for a in ${pkgarch[@]}; do for p in ${pkgname[@]}; do - cp ${p}-${pkgversion}-${a}${PKGEXT} ${curdir}/packages/$(basename ${d}) + cp ${p}-${pkgversion}-${a}${PKGEXT} $(dirname ${BASH_SOURCE[0]})/../packages/$(basename ${d}) done done fi popd >/dev/null done - echo } oneTimeTearDown() { @@ -56,7 +55,7 @@ setUp() { local r local a - [ -f "${curdir}/../config.local" ] && die "${curdir}/../config.local exists" + [ -f "$(dirname ${BASH_SOURCE[0]})/../../config.local" ] && die "$(dirname ${BASH_SOURCE[0]})/../../config.local exists" TMP="$(mktemp -d /dev/shm/$(basename $0).XXXXXXXXXX)" #msg "Using ${TMP}" @@ -85,7 +84,7 @@ setUp() { svn commit -q -m"initial commit of ${pkg}" "${TMP}/svn-packages-copy" done - cat < "${curdir}/../config.local" + cat < "$(dirname ${BASH_SOURCE[0]})/../../config.local" FTP_BASE="${TMP}/ftp" SVNREPO="file://${TMP}/svn-packages-repo" PKGREPOS=(${PKGREPOS[@]}) @@ -97,12 +96,12 @@ setUp() { CLEANUP_DRYRUN=false SOURCE_CLEANUP_DRYRUN=false eot - . "${curdir}/../config" + . "$(dirname ${BASH_SOURCE[0]})/../../config" } tearDown() { rm -rf "${TMP}" - rm -f "${curdir}/../config.local" + rm -f "$(dirname ${BASH_SOURCE[0]})/../../config.local" echo } diff --git a/test/runTest b/test/runTest index 0fcdf18..b8713d8 100755 --- a/test/runTest +++ b/test/runTest @@ -1,617 +1,15 @@ #!/bin/bash -curdir=$(readlink -e $(dirname $0)) -. "${curdir}/lib/common.inc" - -# -# self test -# - -testPackages() { - # TODO: namcap -r depends fails with i686 packages - find "${pkgdir}" -name "*${PKGEXT}" -exec namcap -e depends {} + || fail 'namcap failed' -} - - -# -# db-update -# - -testAddSimplePackages() { - local arches=('i686' 'x86_64') - local pkgs=('pkg-simple-a' 'pkg-simple-b') - local pkgbase - local arch - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage extra ${pkgbase} ${arch} - done - done - - ../db-update - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - checkPackage extra ${pkgbase}-1-1-${arch}.pkg.tar.xz ${arch} - done - done -} - -testAddSingleSimplePackage() { - releasePackage extra 'pkg-simple-a' 'i686' - ../db-update - checkPackage extra 'pkg-simple-a-1-1-i686.pkg.tar.xz' 'i686' -} - -testAddAnyPackages() { - local pkgs=('pkg-any-a' 'pkg-any-b') - local pkgbase - - for pkgbase in ${pkgs[@]}; do - releasePackage extra ${pkgbase} any - done - - ../db-update - - for pkgbase in ${pkgs[@]}; do - checkAnyPackage extra ${pkgbase}-1-1-any.pkg.tar.xz - done -} - -testAddSplitPackages() { - local arches=('i686' 'x86_64') - local pkgs=('pkg-split-a' 'pkg-split-b') - local pkg - local pkgbase - local arch - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage extra ${pkgbase} ${arch} - done - done - - ../db-update - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - for pkg in "${pkgdir}/${pkgbase}"/*-${arch}.pkg.tar.*; do - checkPackage extra $(basename ${pkg}) ${arch} - done - done - done -} - -testUpdateAnyPackage() { - releasePackage extra pkg-any-a any - ../db-update - - pushd "${TMP}/svn-packages-copy/pkg-any-a/trunk/" >/dev/null - sed 's/pkgrel=1/pkgrel=2/g' -i PKGBUILD - svn commit -q -m"update pkg to pkgrel=2" >/dev/null - extra-i686-build >/dev/null 2>&1 - mv pkg-any-a-1-2-any.pkg.tar.xz "${pkgdir}/pkg-any-a/" - popd >/dev/null - - releasePackage extra pkg-any-a any - ../db-update - - checkAnyPackage extra pkg-any-a-1-2-any.pkg.tar.xz any - - rm -f "${pkgdir}/pkg-any-a/pkg-any-a-1-2-any.pkg.tar.xz" -} - -testUpdateAnyPackageToDifferentRepositoriesAtOnce() { - releasePackage extra pkg-any-a any - - pushd "${TMP}/svn-packages-copy/pkg-any-a/trunk/" >/dev/null - sed 's/pkgrel=1/pkgrel=2/g' -i PKGBUILD - svn commit -q -m"update pkg to pkgrel=2" >/dev/null - extra-i686-build >/dev/null 2>&1 - mv pkg-any-a-1-2-any.pkg.tar.xz "${pkgdir}/pkg-any-a/" - popd >/dev/null - - releasePackage testing pkg-any-a any - - ../db-update - - checkAnyPackage extra pkg-any-a-1-1-any.pkg.tar.xz any - checkAnyPackage testing pkg-any-a-1-2-any.pkg.tar.xz any - - rm -f "${pkgdir}/pkg-any-a/pkg-any-a-1-2-any.pkg.tar.xz" -} - -testUpdateSameAnyPackageToDifferentRepositories() { - releasePackage extra pkg-any-a any - ../db-update - checkAnyPackage extra pkg-any-a-1-1-any.pkg.tar.xz any - - releasePackage testing pkg-any-a any - ../db-update >/dev/null 2>&1 && (fail 'Adding an existing package to another repository should fail'; return 1) - - local arch - for arch in i686 x86_64; do - ( [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" ] \ - && bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" -O | grep -q ${pkgbase}) \ - && fail "${pkgbase} should not be in ${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" - done -} - -# -# db-remove -# - -testRemovePackages() { - local arches=('i686' 'x86_64') - local pkgs=('pkg-simple-a' 'pkg-simple-b' 'pkg-split-a' 'pkg-split-b') - local pkgbase - local arch - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage extra ${pkgbase} ${arch} - done - done - - ../db-update - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - ../db-remove ${pkgbase} extra ${arch} - done - done - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - checkRemovedPackage extra ${pkgbase} ${arch} - done - done -} - -testRemoveAnyPackages() { - local pkgs=('pkg-any-a' 'pkg-any-b') - local pkgbase - - for pkgbase in ${pkgs[@]}; do - releasePackage extra ${pkgbase} any - done - - ../db-update - - for pkgbase in ${pkgs[@]}; do - ../db-remove ${pkgbase} extra any - done - - for pkgbase in ${pkgs[@]}; do - checkRemovedAnyPackage extra ${pkgbase} - done -} - - -# -# db-move -# - -testMoveSimplePackages() { - local arches=('i686' 'x86_64') - local pkgs=('pkg-simple-a' 'pkg-simple-b') - local pkgbase - local arch - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage testing ${pkgbase} ${arch} - done - done - - ../db-update - - ../db-move testing extra pkg-simple-a - - for arch in ${arches[@]}; do - checkPackage extra pkg-simple-a-1-1-${arch}.pkg.tar.xz ${arch} - checkRemovedPackage testing pkg-simple-a-1-1-${arch}.pkg.tar.xz ${arch} - - checkPackage testing pkg-simple-b-1-1-${arch}.pkg.tar.xz ${arch} - done -} - -testMoveAnyPackages() { - local pkgs=('pkg-any-a' 'pkg-any-b') - local pkgbase - - for pkgbase in ${pkgs[@]}; do - releasePackage testing ${pkgbase} any - done - - ../db-update - ../db-move testing extra pkg-any-a - - checkAnyPackage extra pkg-any-a-1-1-any.pkg.tar.xz - checkRemovedAnyPackage testing pkg-any-a - checkAnyPackage testing pkg-any-b-1-1-any.pkg.tar.xz -} - -testMoveSplitPackages() { - local arches=('i686' 'x86_64') - local pkgs=('pkg-split-a' 'pkg-split-b') - local pkg - local pkgbase - local arch - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage testing ${pkgbase} ${arch} - done - done - - ../db-update - ../db-move testing extra pkg-split-a - - for arch in ${arches[@]}; do - for pkg in "${pkgdir}/pkg-split-a"/*-${arch}.pkg.tar.*; do - checkPackage extra $(basename ${pkg}) ${arch} - done - done - for arch in ${arches[@]}; do - for pkg in "${pkgdir}/pkg-split-b"/*-${arch}.pkg.tar.*; do - checkPackage testing $(basename ${pkg}) ${arch} - done - done - - checkRemovedAnyPackage testing pkg-split-a -} - - -# -# testing2x -# - -testTesting2xAnyPackage() { - releasePackage core pkg-any-a any - ../db-update - - pushd "${TMP}/svn-packages-copy/pkg-any-a/trunk/" >/dev/null - sed 's/pkgrel=1/pkgrel=2/g' -i PKGBUILD - svn commit -q -m"update pkg to pkgrel=2" >/dev/null - extra-i686-build >/dev/null 2>&1 - mv pkg-any-a-1-2-any.pkg.tar.xz "${pkgdir}/pkg-any-a/" - popd >/dev/null - - releasePackage testing pkg-any-a any - ../db-update - rm -f "${pkgdir}/pkg-any-a/pkg-any-a-1-2-any.pkg.tar.xz" - - ../testing2x pkg-any-a - - checkAnyPackage core pkg-any-a-1-2-any.pkg.tar.xz any - checkRemovedAnyPackage testing pkg-any-a -} - - -# -# ftpdir-cleanup -# - -testCleanupSimplePackages() { - local arches=('i686' 'x86_64') - local pkgs=('pkg-simple-a' 'pkg-simple-b') - local pkgbase - local arch - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage extra ${pkgbase} ${arch} - done - done - - ../db-update - - for arch in ${arches[@]}; do - ../db-remove pkg-simple-a extra ${arch} - done - - ../cron-jobs/ftpdir-cleanup >/dev/null - - for arch in ${arches[@]}; do - local pkg1="pkg-simple-a-1-1-${arch}.pkg.tar.xz" - checkRemovedPackage extra 'pkg-simple-a' ${arch} - [ -f "${FTP_BASE}/${PKGPOOL}/${pkg1}" ] && fail "${PKGPOOL}/${pkg1} found" - [ -f "${FTP_BASE}/${repo}/os/${arch}/${pkg1}" ] && fail "${repo}/os/${arch}/${pkg1} found" - - local pkg2="pkg-simple-b-1-1-${arch}.pkg.tar.xz" - checkPackage extra ${pkg2} ${arch} - done -} - -testCleanupAnyPackages() { - local pkgs=('pkg-any-a' 'pkg-any-b') - local pkgbase - local arch='any' - - for pkgbase in ${pkgs[@]}; do - releasePackage extra ${pkgbase} any - done - - ../db-update - ../db-remove pkg-any-a extra any - ../cron-jobs/ftpdir-cleanup >/dev/null - - local pkg1='pkg-any-a-1-1-any.pkg.tar.xz' - checkRemovedAnyPackage extra 'pkg-any-a' - [ -f "${FTP_BASE}/${PKGPOOL}/${pkg1}" ] && fail "${PKGPOOL}/${pkg1} found" - [ -f "${FTP_BASE}/${repo}/os/${arch}/${pkg1}" ] && fail "${repo}/os/${arch}/${pkg1} found" - - local pkg2="pkg-any-b-1-1-${arch}.pkg.tar.xz" - checkAnyPackage extra ${pkg2} -} - -testCleanupSplitPackages() { - local arches=('i686' 'x86_64') - local pkgs=('pkg-split-a' 'pkg-split-b') - local pkg - local pkgbase - local arch - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage extra ${pkgbase} ${arch} - done - done - - ../db-update - - for arch in ${arches[@]}; do - ../db-remove ${pkgs[0]} extra ${arch} - done - - ../cron-jobs/ftpdir-cleanup >/dev/null - - for arch in ${arches[@]}; do - for pkg in "${pkgdir}/${pkgs[0]}"/*-${arch}.pkg.tar.*; do - checkRemovedPackage extra ${pkgs[0]} ${arch} - [ -f "${FTP_BASE}/${PKGPOOL}/${pkg}" ] && fail "${PKGPOOL}/${pkg} found" - [ -f "${FTP_BASE}/${repo}/os/${arch}/${pkg}" ] && fail "${repo}/os/${arch}/${pkg} found" - done - - for pkg in "${pkgdir}/${pkgs[1]}"/*-${arch}.pkg.tar.*; do - checkPackage extra $(basename ${pkg}) ${arch} - done - done -} - -# -# transition tests -# - -testMovePackagesWithoutPool() { - local arches=('i686' 'x86_64') - local pkgs=('pkg-simple-a' 'pkg-simple-b' 'pkg-split-a' 'pkg-split-b') - local pkgbase - local arch - local pkg - local old - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage testing ${pkgbase} ${arch} - done - done - - ../db-update - - # transform two packages to old style layout - for arch in ${arches[@]}; do - for old in 0 2; do - for pkg in "${pkgdir}/${pkgs[${old}]}"/*-${arch}.pkg.tar.*; do - pkg=$(basename $pkg) - mv -f "${FTP_BASE}/${PKGPOOL}/${pkg}" "${FTP_BASE}/testing/os/${arch}/${pkg}" - done - done - done - - ../cron-jobs/ftpdir-cleanup >/dev/null - - ../db-move testing extra ${pkgs[@]} - - ../cron-jobs/ftpdir-cleanup >/dev/null - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - for pkg in "${pkgdir}/${pkgbase}"/*-${arch}.pkg.tar.*; do - checkPackage extra $(basename ${pkg}) ${arch} - done - checkRemovedPackage testing ${pkgbase} ${arch} - done - done -} - -testUpdateAnyPackageWithoutPool() { - local pkgname='pkg-any-a' - local pkg1='pkg-any-a-1-1-any.pkg.tar.xz' - local pkg2='pkg-any-a-1-2-any.pkg.tar.xz' - local arch - - - releasePackage extra pkg-any-a any - ../db-update - # transform two packages to old style layout - mv -f "${FTP_BASE}/${PKGPOOL}/${pkg1}" "${FTP_BASE}/extra/os/any" - for arch in i686 x86_64; do - ln -sf "../any/${pkg1}" "${FTP_BASE}/extra/os/${arch}" - done - - pushd "${TMP}/svn-packages-copy/${pkgname}/trunk/" >/dev/null - sed 's/pkgrel=1/pkgrel=2/g' -i PKGBUILD - svn commit -q -m"update pkg to pkgrel=2" >/dev/null - extra-i686-build >/dev/null 2>&1 - mv "${pkg2}" "${pkgdir}/${pkgname}/" - popd >/dev/null - - releasePackage extra ${pkgname} any - ../db-update - rm -f "${pkgdir}/${pkgname}/${pkg2}" - - ../cron-jobs/ftpdir-cleanup >/dev/null - - checkAnyPackage extra "${pkg2}" - - [ -f "${FTP_BASE}/${PKGPOOL}/${pkg1}" ] && fail "${PKGPOOL}/${pkg1} found" - for arch in any i686 x86_64; do - [ -f "${FTP_BASE}/extra/os/${arch}/${pkg1}" ] && fail "extra/os/${arch}/${pkg1} found" - done -} - -testMoveAnyPackagesWithoutPool() { - local pkgs=('pkg-any-a' 'pkg-any-b') - local pkgbase - local arch - local pkg - - for pkgbase in ${pkgs[@]}; do - releasePackage testing ${pkgbase} any - done - - ../db-update - - # transform a package to old style layout - for pkg in "${pkgdir}/${pkgs[0]}"/*-any.pkg.tar.*; do - pkg=$(basename $pkg) - mv -f "${FTP_BASE}/${PKGPOOL}/${pkg}" "${FTP_BASE}/testing/os/any/${pkg}" - for arch in i686 x86_64; do - ln -sf "../any/${pkg}" "${FTP_BASE}/testing/os/${arch}/${pkg}" - done - done - - ../cron-jobs/ftpdir-cleanup >/dev/null - - ../db-move testing extra ${pkgs[@]} - - ../cron-jobs/ftpdir-cleanup >/dev/null - - for pkgbase in ${pkgs[@]}; do - for pkg in "${pkgdir}/${pkgbase}"/*-any.pkg.tar.*; do - checkAnyPackage extra $(basename ${pkg}) - done - checkRemovedAnyPackage testing ${pkgbase} - done - - for pkg in "${pkgdir}/${pkgs[0]}"/*-any.pkg.tar.*; do - pkg=$(basename $pkg) - for arch in any i686 x86_64; do - [ -f "${FTP_BASE}/testing/os/${arch}/${pkg}" ] && fail "testing/os/${arch}/${pkg} found" - done - done -} - -testUpdateSameAnyPackageToDifferentRepositoriesWithoutPool() { - local pkg - local arch - - releasePackage extra pkg-any-a any - ../db-update - - # transform a package to old style layout - for pkg in "${pkgdir}/pkg-any-a"/*-any.pkg.tar.*; do - pkg=$(basename $pkg) - mv -f "${FTP_BASE}/${PKGPOOL}/${pkg}" "${FTP_BASE}/extra/os/any/${pkg}" - for arch in i686 x86_64; do - ln -sf "../any/${pkg}" "${FTP_BASE}/extra/os/${arch}/${pkg}" - done - done - - releasePackage testing pkg-any-a any - ../db-update >/dev/null 2>&1 && (fail 'Adding an existing package to another repository should fail'; return 1) - - for arch in i686 x86_64; do - ( [ -r "${FTP_BASE}/testing/os/${arch}/testing${DBEXT%.tar.*}" ] \ - && bsdtar -xf "${FTP_BASE}/testing/os/${arch}/testing${DBEXT%.tar.*}" -O | grep -q pkg-any-a) \ - && fail "pkg-any-a should not be in testing/os/${arch}/testing${DBEXT%.tar.*}" - done -} - -# -#sourceballs tests -# - -testSourceballs() { - local arches=('i686' 'x86_64') - local pkgs=('pkg-simple-a' 'pkg-simple-b') - local pkgbase - local arch - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage extra ${pkgbase} ${arch} - done - done - ../db-update - - ../cron-jobs/sourceballs - for pkgbase in ${pkgs[@]}; do - [ ! -r ${FTP_BASE}/${SRCPOOL}/${pkgbase}-*${SRCEXT} ] && fail "source package not found!" - done -} - -testAnySourceballs() { - local pkgs=('pkg-any-a' 'pkg-any-b') - local pkgbase - - for pkgbase in ${pkgs[@]}; do - releasePackage extra ${pkgbase} any - done - ../db-update - - ../cron-jobs/sourceballs - for pkgbase in ${pkgs[@]}; do - [ ! -r ${FTP_BASE}/${SRCPOOL}/${pkgbase}-*${SRCEXT} ] && fail "source package not found!" - done -} - -testSplitSourceballs() { - local arches=('i686' 'x86_64') - local pkgs=('pkg-split-a' 'pkg-split-b') - local pkg - local pkgbase - local arch - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage extra ${pkgbase} ${arch} - done - done - - ../db-update - - ../cron-jobs/sourceballs - for pkgbase in ${pkgs[@]}; do - [ ! -r ${FTP_BASE}/${SRCPOOL}/${pkgbase}-*${SRCEXT} ] && fail "source package not found!" - done -} - -testSourceballsCleanup() { - local arches=('i686' 'x86_64') - local pkgs=('pkg-simple-a' 'pkg-simple-b') - local pkgbase - local arch - - for pkgbase in ${pkgs[@]}; do - for arch in ${arches[@]}; do - releasePackage extra ${pkgbase} ${arch} - done - done - ../db-update - ../cron-jobs/sourceballs - - for arch in ${arches[@]}; do - ../db-remove pkg-simple-a extra ${arch} - done - ../cron-jobs/ftpdir-cleanup >/dev/null - - ../cron-jobs/sourceballs - [ -r ${FTP_BASE}/${SRCPOOL}/pkg-simple-a-*${SRCEXT} ] && fail "source package was not removed!" - [ ! -r ${FTP_BASE}/${SRCPOOL}/pkg-simple-b-*${SRCEXT} ] && fail "source package not found!" -} - -. "${curdir}/lib/shunit2" +. "$(dirname ${BASH_SOURCE[0]})/lib/common.inc" + +for t in "$(dirname ${BASH_SOURCE[0]})/test.d/"*.sh; do + l=$(basename ${t} .sh) + if [ -x ${t} ]; then + msg "Running test '${l}'" + ${t} + [ $? -ne 0 ] && die "Test '${l}' failed" + echo -e "\n\n\n" + else + warning "Skipping test ${l}" + fi +done diff --git a/test/test.d/db-move.sh b/test/test.d/db-move.sh new file mode 100755 index 0000000..57cbf71 --- /dev/null +++ b/test/test.d/db-move.sh @@ -0,0 +1,76 @@ +#!/bin/bash + +curdir=$(readlink -e $(dirname $0)) +. "${curdir}/../lib/common.inc" + +testMoveSimplePackages() { + local arches=('i686' 'x86_64') + local pkgs=('pkg-simple-a' 'pkg-simple-b') + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + releasePackage testing ${pkgbase} ${arch} + done + done + + ../db-update + + ../db-move testing extra pkg-simple-a + + for arch in ${arches[@]}; do + checkPackage extra pkg-simple-a-1-1-${arch}.pkg.tar.xz ${arch} + checkRemovedPackage testing pkg-simple-a-1-1-${arch}.pkg.tar.xz ${arch} + + checkPackage testing pkg-simple-b-1-1-${arch}.pkg.tar.xz ${arch} + done +} + +testMoveAnyPackages() { + local pkgs=('pkg-any-a' 'pkg-any-b') + local pkgbase + + for pkgbase in ${pkgs[@]}; do + releasePackage testing ${pkgbase} any + done + + ../db-update + ../db-move testing extra pkg-any-a + + checkAnyPackage extra pkg-any-a-1-1-any.pkg.tar.xz + checkRemovedAnyPackage testing pkg-any-a + checkAnyPackage testing pkg-any-b-1-1-any.pkg.tar.xz +} + +testMoveSplitPackages() { + local arches=('i686' 'x86_64') + local pkgs=('pkg-split-a' 'pkg-split-b') + local pkg + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + releasePackage testing ${pkgbase} ${arch} + done + done + + ../db-update + ../db-move testing extra pkg-split-a + + for arch in ${arches[@]}; do + for pkg in "${pkgdir}/pkg-split-a"/*-${arch}.pkg.tar.*; do + checkPackage extra $(basename ${pkg}) ${arch} + done + done + for arch in ${arches[@]}; do + for pkg in "${pkgdir}/pkg-split-b"/*-${arch}.pkg.tar.*; do + checkPackage testing $(basename ${pkg}) ${arch} + done + done + + checkRemovedAnyPackage testing pkg-split-a +} + +. "${curdir}/../lib/shunit2" diff --git a/test/test.d/db-remove.sh b/test/test.d/db-remove.sh new file mode 100755 index 0000000..b66466d --- /dev/null +++ b/test/test.d/db-remove.sh @@ -0,0 +1,52 @@ +#!/bin/bash + +curdir=$(readlink -e $(dirname $0)) +. "${curdir}/../lib/common.inc" + +testRemovePackages() { + local arches=('i686' 'x86_64') + local pkgs=('pkg-simple-a' 'pkg-simple-b' 'pkg-split-a' 'pkg-split-b') + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + releasePackage extra ${pkgbase} ${arch} + done + done + + ../db-update + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + ../db-remove ${pkgbase} extra ${arch} + done + done + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + checkRemovedPackage extra ${pkgbase} ${arch} + done + done +} + +testRemoveAnyPackages() { + local pkgs=('pkg-any-a' 'pkg-any-b') + local pkgbase + + for pkgbase in ${pkgs[@]}; do + releasePackage extra ${pkgbase} any + done + + ../db-update + + for pkgbase in ${pkgs[@]}; do + ../db-remove ${pkgbase} extra any + done + + for pkgbase in ${pkgs[@]}; do + checkRemovedAnyPackage extra ${pkgbase} + done +} + +. "${curdir}/../lib/shunit2" diff --git a/test/test.d/db-update.sh b/test/test.d/db-update.sh new file mode 100755 index 0000000..3c41b39 --- /dev/null +++ b/test/test.d/db-update.sh @@ -0,0 +1,127 @@ +#!/bin/bash + +curdir=$(readlink -e $(dirname $0)) +. "${curdir}/../lib/common.inc" + +testAddSimplePackages() { + local arches=('i686' 'x86_64') + local pkgs=('pkg-simple-a' 'pkg-simple-b') + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + releasePackage extra ${pkgbase} ${arch} + done + done + + ../db-update + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + checkPackage extra ${pkgbase}-1-1-${arch}.pkg.tar.xz ${arch} + done + done +} + +testAddSingleSimplePackage() { + releasePackage extra 'pkg-simple-a' 'i686' + ../db-update + checkPackage extra 'pkg-simple-a-1-1-i686.pkg.tar.xz' 'i686' +} + +testAddAnyPackages() { + local pkgs=('pkg-any-a' 'pkg-any-b') + local pkgbase + + for pkgbase in ${pkgs[@]}; do + releasePackage extra ${pkgbase} any + done + + ../db-update + + for pkgbase in ${pkgs[@]}; do + checkAnyPackage extra ${pkgbase}-1-1-any.pkg.tar.xz + done +} + +testAddSplitPackages() { + local arches=('i686' 'x86_64') + local pkgs=('pkg-split-a' 'pkg-split-b') + local pkg + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + releasePackage extra ${pkgbase} ${arch} + done + done + + ../db-update + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + for pkg in "${pkgdir}/${pkgbase}"/*-${arch}.pkg.tar.*; do + checkPackage extra $(basename ${pkg}) ${arch} + done + done + done +} + +testUpdateAnyPackage() { + releasePackage extra pkg-any-a any + ../db-update + + pushd "${TMP}/svn-packages-copy/pkg-any-a/trunk/" >/dev/null + sed 's/pkgrel=1/pkgrel=2/g' -i PKGBUILD + svn commit -q -m"update pkg to pkgrel=2" >/dev/null + extra-i686-build >/dev/null 2>&1 + mv pkg-any-a-1-2-any.pkg.tar.xz "${pkgdir}/pkg-any-a/" + popd >/dev/null + + releasePackage extra pkg-any-a any + ../db-update + + checkAnyPackage extra pkg-any-a-1-2-any.pkg.tar.xz any + + rm -f "${pkgdir}/pkg-any-a/pkg-any-a-1-2-any.pkg.tar.xz" +} + +testUpdateAnyPackageToDifferentRepositoriesAtOnce() { + releasePackage extra pkg-any-a any + + pushd "${TMP}/svn-packages-copy/pkg-any-a/trunk/" >/dev/null + sed 's/pkgrel=1/pkgrel=2/g' -i PKGBUILD + svn commit -q -m"update pkg to pkgrel=2" >/dev/null + extra-i686-build >/dev/null 2>&1 + mv pkg-any-a-1-2-any.pkg.tar.xz "${pkgdir}/pkg-any-a/" + popd >/dev/null + + releasePackage testing pkg-any-a any + + ../db-update + + checkAnyPackage extra pkg-any-a-1-1-any.pkg.tar.xz any + checkAnyPackage testing pkg-any-a-1-2-any.pkg.tar.xz any + + rm -f "${pkgdir}/pkg-any-a/pkg-any-a-1-2-any.pkg.tar.xz" +} + +testUpdateSameAnyPackageToDifferentRepositories() { + releasePackage extra pkg-any-a any + ../db-update + checkAnyPackage extra pkg-any-a-1-1-any.pkg.tar.xz any + + releasePackage testing pkg-any-a any + ../db-update >/dev/null 2>&1 && (fail 'Adding an existing package to another repository should fail'; return 1) + + local arch + for arch in i686 x86_64; do + ( [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" ] \ + && bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" -O | grep -q ${pkgbase}) \ + && fail "${pkgbase} should not be in ${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" + done +} + +. "${curdir}/../lib/shunit2" diff --git a/test/test.d/ftpdir-cleanup.sh b/test/test.d/ftpdir-cleanup.sh new file mode 100755 index 0000000..e9b977b --- /dev/null +++ b/test/test.d/ftpdir-cleanup.sh @@ -0,0 +1,93 @@ +#!/bin/bash + +curdir=$(readlink -e $(dirname $0)) +. "${curdir}/../lib/common.inc" + +testCleanupSimplePackages() { + local arches=('i686' 'x86_64') + local pkgs=('pkg-simple-a' 'pkg-simple-b') + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + releasePackage extra ${pkgbase} ${arch} + done + done + + ../db-update + + for arch in ${arches[@]}; do + ../db-remove pkg-simple-a extra ${arch} + done + + ../cron-jobs/ftpdir-cleanup >/dev/null + + for arch in ${arches[@]}; do + local pkg1="pkg-simple-a-1-1-${arch}.pkg.tar.xz" + checkRemovedPackage extra 'pkg-simple-a' ${arch} + [ -f "${FTP_BASE}/${PKGPOOL}/${pkg1}" ] && fail "${PKGPOOL}/${pkg1} found" + [ -f "${FTP_BASE}/${repo}/os/${arch}/${pkg1}" ] && fail "${repo}/os/${arch}/${pkg1} found" + + local pkg2="pkg-simple-b-1-1-${arch}.pkg.tar.xz" + checkPackage extra ${pkg2} ${arch} + done +} + +testCleanupAnyPackages() { + local pkgs=('pkg-any-a' 'pkg-any-b') + local pkgbase + local arch='any' + + for pkgbase in ${pkgs[@]}; do + releasePackage extra ${pkgbase} any + done + + ../db-update + ../db-remove pkg-any-a extra any + ../cron-jobs/ftpdir-cleanup >/dev/null + + local pkg1='pkg-any-a-1-1-any.pkg.tar.xz' + checkRemovedAnyPackage extra 'pkg-any-a' + [ -f "${FTP_BASE}/${PKGPOOL}/${pkg1}" ] && fail "${PKGPOOL}/${pkg1} found" + [ -f "${FTP_BASE}/${repo}/os/${arch}/${pkg1}" ] && fail "${repo}/os/${arch}/${pkg1} found" + + local pkg2="pkg-any-b-1-1-${arch}.pkg.tar.xz" + checkAnyPackage extra ${pkg2} +} + +testCleanupSplitPackages() { + local arches=('i686' 'x86_64') + local pkgs=('pkg-split-a' 'pkg-split-b') + local pkg + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + releasePackage extra ${pkgbase} ${arch} + done + done + + ../db-update + + for arch in ${arches[@]}; do + ../db-remove ${pkgs[0]} extra ${arch} + done + + ../cron-jobs/ftpdir-cleanup >/dev/null + + for arch in ${arches[@]}; do + for pkg in "${pkgdir}/${pkgs[0]}"/*-${arch}.pkg.tar.*; do + checkRemovedPackage extra ${pkgs[0]} ${arch} + [ -f "${FTP_BASE}/${PKGPOOL}/${pkg}" ] && fail "${PKGPOOL}/${pkg} found" + [ -f "${FTP_BASE}/${repo}/os/${arch}/${pkg}" ] && fail "${repo}/os/${arch}/${pkg} found" + done + + for pkg in "${pkgdir}/${pkgs[1]}"/*-${arch}.pkg.tar.*; do + checkPackage extra $(basename ${pkg}) ${arch} + done + done +} + +. "${curdir}/../lib/shunit2" diff --git a/test/test.d/packages.sh b/test/test.d/packages.sh new file mode 100755 index 0000000..324f73a --- /dev/null +++ b/test/test.d/packages.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +curdir=$(readlink -e $(dirname $0)) +. "${curdir}/../lib/common.inc" + +testPackages() { + # TODO: namcap -r depends fails with i686 packages + find "${pkgdir}" -name "*${PKGEXT}" -exec namcap -e depends {} + || fail 'namcap failed' +} + +. "${curdir}/../lib/shunit2" diff --git a/test/test.d/pool-transition.sh b/test/test.d/pool-transition.sh new file mode 100755 index 0000000..6d82e00 --- /dev/null +++ b/test/test.d/pool-transition.sh @@ -0,0 +1,152 @@ +#!/bin/bash + +curdir=$(readlink -e $(dirname $0)) +. "${curdir}/../lib/common.inc" + +testMovePackagesWithoutPool() { + local arches=('i686' 'x86_64') + local pkgs=('pkg-simple-a' 'pkg-simple-b' 'pkg-split-a' 'pkg-split-b') + local pkgbase + local arch + local pkg + local old + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + releasePackage testing ${pkgbase} ${arch} + done + done + + ../db-update + + # transform two packages to old style layout + for arch in ${arches[@]}; do + for old in 0 2; do + for pkg in "${pkgdir}/${pkgs[${old}]}"/*-${arch}.pkg.tar.*; do + pkg=$(basename $pkg) + mv -f "${FTP_BASE}/${PKGPOOL}/${pkg}" "${FTP_BASE}/testing/os/${arch}/${pkg}" + done + done + done + + ../cron-jobs/ftpdir-cleanup >/dev/null + + ../db-move testing extra ${pkgs[@]} + + ../cron-jobs/ftpdir-cleanup >/dev/null + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + for pkg in "${pkgdir}/${pkgbase}"/*-${arch}.pkg.tar.*; do + checkPackage extra $(basename ${pkg}) ${arch} + done + checkRemovedPackage testing ${pkgbase} ${arch} + done + done +} + +testUpdateAnyPackageWithoutPool() { + local pkgname='pkg-any-a' + local pkg1='pkg-any-a-1-1-any.pkg.tar.xz' + local pkg2='pkg-any-a-1-2-any.pkg.tar.xz' + local arch + + + releasePackage extra pkg-any-a any + ../db-update + # transform two packages to old style layout + mv -f "${FTP_BASE}/${PKGPOOL}/${pkg1}" "${FTP_BASE}/extra/os/any" + for arch in i686 x86_64; do + ln -sf "../any/${pkg1}" "${FTP_BASE}/extra/os/${arch}" + done + + pushd "${TMP}/svn-packages-copy/${pkgname}/trunk/" >/dev/null + sed 's/pkgrel=1/pkgrel=2/g' -i PKGBUILD + svn commit -q -m"update pkg to pkgrel=2" >/dev/null + extra-i686-build >/dev/null 2>&1 + mv "${pkg2}" "${pkgdir}/${pkgname}/" + popd >/dev/null + + releasePackage extra ${pkgname} any + ../db-update + rm -f "${pkgdir}/${pkgname}/${pkg2}" + + ../cron-jobs/ftpdir-cleanup >/dev/null + + checkAnyPackage extra "${pkg2}" + + [ -f "${FTP_BASE}/${PKGPOOL}/${pkg1}" ] && fail "${PKGPOOL}/${pkg1} found" + for arch in any i686 x86_64; do + [ -f "${FTP_BASE}/extra/os/${arch}/${pkg1}" ] && fail "extra/os/${arch}/${pkg1} found" + done +} + +testMoveAnyPackagesWithoutPool() { + local pkgs=('pkg-any-a' 'pkg-any-b') + local pkgbase + local arch + local pkg + + for pkgbase in ${pkgs[@]}; do + releasePackage testing ${pkgbase} any + done + + ../db-update + + # transform a package to old style layout + for pkg in "${pkgdir}/${pkgs[0]}"/*-any.pkg.tar.*; do + pkg=$(basename $pkg) + mv -f "${FTP_BASE}/${PKGPOOL}/${pkg}" "${FTP_BASE}/testing/os/any/${pkg}" + for arch in i686 x86_64; do + ln -sf "../any/${pkg}" "${FTP_BASE}/testing/os/${arch}/${pkg}" + done + done + + ../cron-jobs/ftpdir-cleanup >/dev/null + + ../db-move testing extra ${pkgs[@]} + + ../cron-jobs/ftpdir-cleanup >/dev/null + + for pkgbase in ${pkgs[@]}; do + for pkg in "${pkgdir}/${pkgbase}"/*-any.pkg.tar.*; do + checkAnyPackage extra $(basename ${pkg}) + done + checkRemovedAnyPackage testing ${pkgbase} + done + + for pkg in "${pkgdir}/${pkgs[0]}"/*-any.pkg.tar.*; do + pkg=$(basename $pkg) + for arch in any i686 x86_64; do + [ -f "${FTP_BASE}/testing/os/${arch}/${pkg}" ] && fail "testing/os/${arch}/${pkg} found" + done + done +} + +testUpdateSameAnyPackageToDifferentRepositoriesWithoutPool() { + local pkg + local arch + + releasePackage extra pkg-any-a any + ../db-update + + # transform a package to old style layout + for pkg in "${pkgdir}/pkg-any-a"/*-any.pkg.tar.*; do + pkg=$(basename $pkg) + mv -f "${FTP_BASE}/${PKGPOOL}/${pkg}" "${FTP_BASE}/extra/os/any/${pkg}" + for arch in i686 x86_64; do + ln -sf "../any/${pkg}" "${FTP_BASE}/extra/os/${arch}/${pkg}" + done + done + + releasePackage testing pkg-any-a any + ../db-update >/dev/null 2>&1 && (fail 'Adding an existing package to another repository should fail'; return 1) + + for arch in i686 x86_64; do + ( [ -r "${FTP_BASE}/testing/os/${arch}/testing${DBEXT%.tar.*}" ] \ + && bsdtar -xf "${FTP_BASE}/testing/os/${arch}/testing${DBEXT%.tar.*}" -O | grep -q pkg-any-a) \ + && fail "pkg-any-a should not be in testing/os/${arch}/testing${DBEXT%.tar.*}" + done +} + +. "${curdir}/../lib/shunit2" diff --git a/test/test.d/sourceballs.sh b/test/test.d/sourceballs.sh new file mode 100755 index 0000000..45e9b1d --- /dev/null +++ b/test/test.d/sourceballs.sh @@ -0,0 +1,85 @@ +#!/bin/bash + +curdir=$(readlink -e $(dirname $0)) +. "${curdir}/../lib/common.inc" + +testSourceballs() { + local arches=('i686' 'x86_64') + local pkgs=('pkg-simple-a' 'pkg-simple-b') + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + releasePackage extra ${pkgbase} ${arch} + done + done + ../db-update + + ../cron-jobs/sourceballs + for pkgbase in ${pkgs[@]}; do + [ ! -r ${FTP_BASE}/${SRCPOOL}/${pkgbase}-*${SRCEXT} ] && fail "source package not found!" + done +} + +testAnySourceballs() { + local pkgs=('pkg-any-a' 'pkg-any-b') + local pkgbase + + for pkgbase in ${pkgs[@]}; do + releasePackage extra ${pkgbase} any + done + ../db-update + + ../cron-jobs/sourceballs + for pkgbase in ${pkgs[@]}; do + [ ! -r ${FTP_BASE}/${SRCPOOL}/${pkgbase}-*${SRCEXT} ] && fail "source package not found!" + done +} + +testSplitSourceballs() { + local arches=('i686' 'x86_64') + local pkgs=('pkg-split-a' 'pkg-split-b') + local pkg + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + releasePackage extra ${pkgbase} ${arch} + done + done + + ../db-update + + ../cron-jobs/sourceballs + for pkgbase in ${pkgs[@]}; do + [ ! -r ${FTP_BASE}/${SRCPOOL}/${pkgbase}-*${SRCEXT} ] && fail "source package not found!" + done +} + +testSourceballsCleanup() { + local arches=('i686' 'x86_64') + local pkgs=('pkg-simple-a' 'pkg-simple-b') + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + releasePackage extra ${pkgbase} ${arch} + done + done + ../db-update + ../cron-jobs/sourceballs + + for arch in ${arches[@]}; do + ../db-remove pkg-simple-a extra ${arch} + done + ../cron-jobs/ftpdir-cleanup >/dev/null + + ../cron-jobs/sourceballs + [ -r ${FTP_BASE}/${SRCPOOL}/pkg-simple-a-*${SRCEXT} ] && fail "source package was not removed!" + [ ! -r ${FTP_BASE}/${SRCPOOL}/pkg-simple-b-*${SRCEXT} ] && fail "source package not found!" +} + +. "${curdir}/../lib/shunit2" diff --git a/test/test.d/testing2x.sh b/test/test.d/testing2x.sh new file mode 100755 index 0000000..436716d --- /dev/null +++ b/test/test.d/testing2x.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +curdir=$(readlink -e $(dirname $0)) +. "${curdir}/../lib/common.inc" + +testTesting2xAnyPackage() { + releasePackage core pkg-any-a any + ../db-update + + pushd "${TMP}/svn-packages-copy/pkg-any-a/trunk/" >/dev/null + sed 's/pkgrel=1/pkgrel=2/g' -i PKGBUILD + svn commit -q -m"update pkg to pkgrel=2" >/dev/null + extra-i686-build >/dev/null 2>&1 + mv pkg-any-a-1-2-any.pkg.tar.xz "${pkgdir}/pkg-any-a/" + popd >/dev/null + + releasePackage testing pkg-any-a any + ../db-update + rm -f "${pkgdir}/pkg-any-a/pkg-any-a-1-2-any.pkg.tar.xz" + + ../testing2x pkg-any-a + + checkAnyPackage core pkg-any-a-1-2-any.pkg.tar.xz any + checkRemovedAnyPackage testing pkg-any-a +} + +. "${curdir}/../lib/shunit2" -- cgit v1.2.2 From 447986d17554c183d958cf7de79d1904fc32aaf6 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Thu, 25 Nov 2010 13:28:04 +0100 Subject: Add FILESEXT variable for use in create-filelists --- config | 1 + cron-jobs/create-filelists | 8 +++----- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/config b/config index 69f40c7..ff297ec 100644 --- a/config +++ b/config @@ -16,6 +16,7 @@ STAGING="$HOME/staging" TMPDIR="/srv/tmp" ARCHES=(i686 x86_64) DBEXT=".db.tar.gz" +FILESEXT=".files.tar.gz" PKGEXT=".pkg.tar.*" SRCEXT=".src.tar.gz" diff --git a/cron-jobs/create-filelists b/cron-jobs/create-filelists index d8a5ba1..010194b 100755 --- a/cron-jobs/create-filelists +++ b/cron-jobs/create-filelists @@ -18,15 +18,13 @@ MYTMPDIR="$(mktemp -d ${WORKDIR}/create-filelists.tmpdir.XXXXXX)" || exit 1 #adjust the nice level to run at a lower priority /usr/bin/renice +10 -p $$ > /dev/null -case "${DBEXT}" in +case "${FILESEXT}" in *.gz) TAR_OPT="z" ;; *.bz2) TAR_OPT="j" ;; *.xz) TAR_OPT="J" ;; - *) die "Unknown compression type for DBEXT=${DBEXT}" ;; + *) die "Unknown compression type for FILESEXT=${FILESEXT}" ;; esac -FILESEXT="${DBEXT//db/files}" - for repo in ${PKGREPOS[@]}; do REPO_DB_FILE="${repo}$DBEXT" FILES_DB_FILE="${repo}$FILESEXT" @@ -77,7 +75,7 @@ for repo in ${PKGREPOS[@]}; do mkdir -p "$pkgdir" pushd "${MYTMPDIR}/${repodir}" >/dev/null [ -f "${pkgdir}/${FILES_DB_FILE}" ] && rm "${pkgdir}/${FILES_DB_FILE}" - bsdtar --exclude=*${DBEXT//\.db/} -c${TAR_OPT}f "${pkgdir}/${FILES_DB_FILE}" * + bsdtar --exclude=*${FILESEXT} -c${TAR_OPT}f "${pkgdir}/${FILES_DB_FILE}" * popd >/dev/null fi popd >/dev/null -- cgit v1.2.2 From 72ed0011f7ff967d8a402c84c5ab5783f9db9987 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Thu, 25 Nov 2010 13:31:20 +0100 Subject: sourceballs no longer depends on ftpdir-cleanup being run --- test/test.d/sourceballs.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/test/test.d/sourceballs.sh b/test/test.d/sourceballs.sh index 45e9b1d..8cba017 100755 --- a/test/test.d/sourceballs.sh +++ b/test/test.d/sourceballs.sh @@ -75,7 +75,6 @@ testSourceballsCleanup() { for arch in ${arches[@]}; do ../db-remove pkg-simple-a extra ${arch} done - ../cron-jobs/ftpdir-cleanup >/dev/null ../cron-jobs/sourceballs [ -r ${FTP_BASE}/${SRCPOOL}/pkg-simple-a-*${SRCEXT} ] && fail "source package was not removed!" -- cgit v1.2.2 From e792abefa6eff4cf54616add0b19d731d486045f Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Thu, 25 Nov 2010 16:03:44 +0100 Subject: Rewrote create-filelists * use correct locking of the repos * added test case * removes file lists of deleted packages * add compression independent symlink to files db --- cron-jobs/create-filelists | 147 ++++++++++++++++++++++------------------ test/test.d/create-filelists.sh | 110 ++++++++++++++++++++++++++++++ 2 files changed, 191 insertions(+), 66 deletions(-) create mode 100755 test/test.d/create-filelists.sh diff --git a/cron-jobs/create-filelists b/cron-jobs/create-filelists index 010194b..c3e7d72 100755 --- a/cron-jobs/create-filelists +++ b/cron-jobs/create-filelists @@ -3,20 +3,42 @@ . "$(dirname $0)/../db-functions" . "$(dirname $0)/../config" -reposdir=${FTP_BASE} -targetdir=${FTP_BASE} - script_lock -# location where the package DB is extracted so we know what to include -DBDIR="$(mktemp -d ${WORKDIR}/create-filelists.dbdir.XXXXXX)" || exit 1 -# location where the old files DB is extracted to save us some work -CACHEDIR="$(mktemp -d ${WORKDIR}/create-filelists.cachedir.XXXXXX)" || exit 1 -# location where the new files DB is built up and eventually zipped -MYTMPDIR="$(mktemp -d ${WORKDIR}/create-filelists.tmpdir.XXXXXX)" || exit 1 +for repo in ${PKGREPOS[@]}; do + for arch in ${ARCHES[@]}; do + repo_lock ${repo} ${arch} || exit 1 + done +done #adjust the nice level to run at a lower priority -/usr/bin/renice +10 -p $$ > /dev/null +renice +10 -p $$ > /dev/null + +for repo in ${PKGREPOS[@]}; do + for arch in ${ARCHES[@]}; do + repodb="${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" + filedb="${FTP_BASE}/${repo}/os/${arch}/${repo}${FILESEXT}" + + if [ ! -f "${repodb}" ]; then + continue + fi + # get a list of package files defined in the repo db + mkdir -p "${WORKDIR}/db-dir-${repo}-${arch}" + bsdtar -xf "${repodb}" -C "${WORKDIR}/db-dir-${repo}-${arch}" + find "${WORKDIR}/db-dir-${repo}-${arch}" -name 'desc' \ + -exec awk '/^%FILENAME%/{getline;print}' {} \; | sort > "${WORKDIR}/db-${repo}-${arch}" + + # get a list of package files defined in the files db + mkdir -p "${WORKDIR}/files-current-dir-${repo}-${arch}" + if [ ! -f "${filedb}" ]; then + echo > "${WORKDIR}/files-${repo}-${arch}" + else + bsdtar -xf "${filedb}" -C "${WORKDIR}/files-current-dir-${repo}-${arch}" + find "${WORKDIR}/files-current-dir-${repo}-${arch}" -name 'desc' \ + -exec awk '/^%FILENAME%/{getline;print}' {} \; | sort > "${WORKDIR}/files-${repo}-${arch}" + fi + done +done case "${FILESEXT}" in *.gz) TAR_OPT="z" ;; @@ -26,62 +48,55 @@ case "${FILESEXT}" in esac for repo in ${PKGREPOS[@]}; do - REPO_DB_FILE="${repo}$DBEXT" - FILES_DB_FILE="${repo}$FILESEXT" - for arch in ${ARCHES[@]}; do - pushd "$reposdir" >/dev/null - - repodir="${repo}/os/${arch}" - cached="no" - - # extract package db archive - if [ -f "${targetdir}/${repodir}/${REPO_DB_FILE}" ]; then - mkdir -p "${DBDIR}/${repodir}" - bsdtar -xf "${targetdir}/${repodir}/${REPO_DB_FILE}" -C "${DBDIR}/${repodir}" - else - warning "The repo $repo with arch $arch does not exist" - continue - fi - - # extract old file archive - if [ -f "${targetdir}/${repodir}/${FILES_DB_FILE}" ]; then - mkdir -p "${CACHEDIR}/${repodir}" - bsdtar -xf "${targetdir}/${repodir}/${FILES_DB_FILE}" -C "${CACHEDIR}/${repodir}" - cached="yes" - fi - - # create file lists - for pkg in $(ls ${DBDIR}/${repodir}); do - dbpkgdir="${DBDIR}/${repodir}/${pkg}" - cachepkgdir="${CACHEDIR}/${repodir}/${pkg}" - tmppkgdir="${MYTMPDIR}/${repodir}/${pkg}" - mkdir -p "$tmppkgdir" - ln "${dbpkgdir}/desc" "${tmppkgdir}/desc" - ln "${dbpkgdir}/depends" "${tmppkgdir}/depends" - if [ -f "${cachepkgdir}/files" ]; then - ln "${cachepkgdir}/files" "${tmppkgdir}/files" - else - filename=$(grep -A1 '^%FILENAME%$' "${dbpkgdir}/desc" | tail -n1) - echo '%FILES%' > "${tmppkgdir}/files" - bsdtar --exclude=.* -tf "$repodir/$filename" >> "${tmppkgdir}/files" - cached="no" - fi - done - - # create new file archive - if [ "$cached" == "no" -a -d "${MYTMPDIR}/${repodir}" ]; then - # at least one package has changed, so let's rebuild the archive - pkgdir="${targetdir}/${repodir}" - mkdir -p "$pkgdir" - pushd "${MYTMPDIR}/${repodir}" >/dev/null - [ -f "${pkgdir}/${FILES_DB_FILE}" ] && rm "${pkgdir}/${FILES_DB_FILE}" - bsdtar --exclude=*${FILESEXT} -c${TAR_OPT}f "${pkgdir}/${FILES_DB_FILE}" * - popd >/dev/null - fi - popd >/dev/null - done + for arch in ${ARCHES[@]}; do + filedb="${FTP_BASE}/${repo}/os/${arch}/${repo}${FILESEXT}" + + if [ ! -f "${WORKDIR}/db-${repo}-${arch}" ]; then + # remove any files db that might be in this empty repo + if [ -f "${filedb}" ]; then + rm -f "${filedb}" + fi + continue + fi + + # Check if updating the files db is needed + if ! diff -q "${WORKDIR}/db-${repo}-${arch}" "${WORKDIR}/files-${repo}-${arch}" >/dev/null; then + mkdir -p "${WORKDIR}/files-new-dir-${repo}-${arch}" + + # Include all unchanged file lists + # Note: deleted packages are implicitly excluded + for f in $(comm -12 "${WORKDIR}/db-${repo}-${arch}" "${WORKDIR}/files-${repo}-${arch}"); do + mv "${WORKDIR}/files-current-dir-${repo}-${arch}/${f%*-*${PKGEXT}}" \ + "${WORKDIR}/files-new-dir-${repo}-${arch}" + done + + # Create file lists for new packages + for f in $(comm -23 "${WORKDIR}/db-${repo}-${arch}" "${WORKDIR}/files-${repo}-${arch}"); do + tdir="${WORKDIR}/files-new-dir-${repo}-${arch}/${f%*-*${PKGEXT}}" + mkdir "${tdir}" + echo '%FILES%' > "${tdir}/files" + bsdtar --exclude=.* -tf "${FTP_BASE}/${repo}/os/${arch}/${f}" >> "${tdir}/files" + + # add desc and depends file from db + dbdir="${WORKDIR}/db-dir-${repo}-${arch}/${f%*-*${PKGEXT}}" + mv "${dbdir}/desc" "${tdir}/desc" + mv "${dbdir}/depends" "${tdir}/depends" + done + + # Create the actual file db + pushd "${WORKDIR}/files-new-dir-${repo}-${arch}" >/dev/null + bsdtar -c${TAR_OPT}f "${WORKDIR}/${arch}-${repo}${FILESEXT}" * + popd >/dev/null + mv -f "${WORKDIR}/${arch}-${repo}${FILESEXT}" "${filedb}" + ln -sf "${repo}${FILESEXT}" "${filedb%.tar.*}" + fi + done +done + +for repo in ${PKGREPOS[@]}; do + for arch in ${ARCHES[@]}; do + repo_unlock ${repo} ${arch} + done done script_unlock - -# vim: set ts=4 sw=4 et ft=sh: diff --git a/test/test.d/create-filelists.sh b/test/test.d/create-filelists.sh new file mode 100755 index 0000000..da76710 --- /dev/null +++ b/test/test.d/create-filelists.sh @@ -0,0 +1,110 @@ +#!/bin/bash + +curdir=$(readlink -e $(dirname $0)) +. "${curdir}/../lib/common.inc" + +testCreateSimpleFileLists() { + local arches=('i686' 'x86_64') + local pkgs=('pkg-simple-a' 'pkg-simple-b') + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + releasePackage extra ${pkgbase} ${arch} + done + done + ../db-update + + ../cron-jobs/create-filelists + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + if ! bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra.files.tar.gz" | grep -q "usr/bin/${pkgbase}"; then + fail "usr/bin/${pkgbase} not found in ${arch}/extra.files.tar.gz" + fi + done + done +} + +testCreateAnyFileLists() { + local arches=('i686' 'x86_64') + local pkgs=('pkg-any-a' 'pkg-any-b') + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + releasePackage extra ${pkgbase} any + done + ../db-update + + ../cron-jobs/create-filelists + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + if ! bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra.files.tar.gz" | grep -q "usr/share/${pkgbase}/test"; then + fail "usr/share/${pkgbase}/test not found in ${arch}/extra.files.tar.gz" + fi + done + done +} + +testCreateSplitFileLists() { + local arches=('i686' 'x86_64') + local pkgs=('pkg-split-a' 'pkg-split-b') + local pkg + local pkgbase + local pkgname + local pkgnames + local arch + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + releasePackage extra ${pkgbase} ${arch} + done + done + ../db-update + + ../cron-jobs/create-filelists + for pkgbase in ${pkgs[@]}; do + pkgnames=($(source "${TMP}/svn-packages-copy/${pkgbase}/trunk/PKGBUILD"; echo ${pkgname[@]})) + for pkgname in ${pkgnames[@]}; do + for arch in ${arches[@]}; do + if ! bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra.files.tar.gz" | grep -q "usr/bin/${pkgname}"; then + fail "usr/bin/${pkgname} not found in ${arch}/extra.files.tar.gz" + fi + done + done + done +} + + +testCleanupFileLists() { + local arches=('i686' 'x86_64') + local pkgs=('pkg-simple-a' 'pkg-simple-b') + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + releasePackage extra ${pkgbase} ${arch} + done + done + ../db-update + ../cron-jobs/create-filelists + + for arch in ${arches[@]}; do + ../db-remove pkg-simple-a extra ${arch} + done + ../cron-jobs/create-filelists + + for arch in ${arches[@]}; do + if ! bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra.files.tar.gz" | grep -q "usr/bin/pkg-simple-b"; then + fail "usr/bin/pkg-simple-b not found in ${arch}/extra.files.tar.gz" + fi + if bsdtar -xOf "${FTP_BASE}/extra/os/${arch}/extra.files.tar.gz" | grep -q "usr/bin/pkg-simple-a"; then + fail "usr/bin/pkg-simple-a still found in ${arch}/extra.files.tar.gz" + fi + done + +} + +. "${curdir}/../lib/shunit2" -- cgit v1.2.2 From dea8091103c9683e29477e1828b088e239f414ed Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Thu, 25 Nov 2010 16:06:34 +0100 Subject: sourceballs: be nice --- cron-jobs/sourceballs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 9ba003a..6553327 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -13,6 +13,9 @@ for repo in ${PKGREPOS[@]}; do done done +#adjust the nice level to run at a lower priority +renice +10 -p $$ > /dev/null + # Create a readable file for each repo with the following format # - [ ] for repo in ${PKGREPOS[@]}; do -- cgit v1.2.2 From b9dea6f306df1ffb051f0c7239f416a34f93a71a Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Thu, 25 Nov 2010 18:42:19 +0100 Subject: update-web-db: Lock db and script * usage of db-functions * Lock the db and work on a copy to avoid long lock times * Lock the script itself to avoid multiple calls * Adjust to coding style to other dbscripts --- cron-jobs/update-web-db | 72 +++++++++++++++++++++++++++++++------------------ 1 file changed, 46 insertions(+), 26 deletions(-) diff --git a/cron-jobs/update-web-db b/cron-jobs/update-web-db index 3f4a2fc..8760fc6 100755 --- a/cron-jobs/update-web-db +++ b/cron-jobs/update-web-db @@ -1,51 +1,71 @@ #!/bin/bash -# run at nice 5. it can churn quite a bit of cpu after all. -renice +5 -p $$ > /dev/null +. "$(dirname $0)/../db-functions" +. "$(dirname $0)/../config" # setup paths SPATH="/srv/http/archweb" -ARCHES="i686 x86_64" # having "more important repos" last should make [core] trickle to the top of # the updates list each hour rather than being overwhelmed by big [extra] and # [community] updates -REPOS="community-testing multilib-testing multilib community extra testing core" +REPOS=('community-testing' 'multilib-testing' 'multilib' 'community' 'extra' 'testing' 'core') LOGOUT="/tmp/archweb_update.log" # figure out what operation to perform cmd="$(basename $0)" if [[ $cmd != "update-web-db" && $cmd != "update-web-files-db" ]]; then - echo "Invalid command name '$cmd' specified!" - exit 1 + die "Invalid command name '$cmd' specified!" fi +script_lock + +# run at nice 5. it can churn quite a bit of cpu after all. +renice +5 -p $$ > /dev/null + echo "$cmd: Updating DB at $(date)" >> "${LOGOUT}" -## do the dirty, then output the result to a file. -cd $SPATH -for repo in ${REPOS}; do - for arch in ${ARCHES}; do - case "$cmd" in - update-web-db) - dbfile="/srv/ftp/${repo}/os/${arch}/${repo}.db.tar.gz" - flags="" ;; - update-web-files-db) - dbfile="/srv/ftp/${repo}/os/${arch}/${repo}.files.tar.gz" - flags="--filesonly" ;; - esac - if [[ -f $dbfile ]]; then - echo "Updating ${repo}-${arch}" >> "${LOGOUT}" - ./manage.py reporead ${flags} ${arch} ${dbfile} >> "${LOGOUT}" 2>&1 - echo "" >> "${LOGOUT}" - fi - done +case "$cmd" in + update-web-db) + dbfileext="${DBEXT}" + flags="" + ;; + update-web-files-db) + dbfileext="${FILESEXT}" + flags="--filesonly" + ;; +esac + +# Lock the repos and get a copy of the db files to work on +for repo in ${REPOS[@]}; do + for arch in ${ARCHES[@]}; do + repo_lock ${repo} ${arch} || exit 1 + dbfile="/srv/ftp/${repo}/os/${arch}/${repo}${dbfileext}" + if [ -f "${dbfile}" ]; then + mkdir -p "${WORKDIR}/${repo}/${arch}" + cp "${dbfile}" "${WORKDIR}/${repo}/${arch}/${repo}${dbfileext}" + fi + repo_unlock ${repo} ${arch} + done done +# Run reporead on our db copy +pushd $SPATH >/dev/null +for repo in ${REPOS[@]}; do + for arch in ${ARCHES[@]}; do + dbcopy="${WORKDIR}/${repo}/${arch}/${repo}${dbfileext}" + if [ -f "${dbcopy}" ]; then + echo "Updating ${repo}-${arch}" >> "${LOGOUT}" + ./manage.py reporead ${flags} ${arch} "${dbcopy}" >> "${LOGOUT}" 2>&1 + echo "" >> "${LOGOUT}" + fi + done +done +popd >/dev/null echo "" >> "${LOGOUT}" # rotate the file if it is getting big (> 10M), overwriting any old backup if [[ $(stat -c%s "${LOGOUT}") -gt 10485760 ]]; then - mv "${LOGOUT}" "${LOGOUT}.old" + mv "${LOGOUT}" "${LOGOUT}.old" fi -# vim: set ts=4 sw=4 et ft=sh: +script_unlock -- cgit v1.2.2 From 88a00e08e0394c95f65836be043da75fa2a131f1 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Thu, 25 Nov 2010 19:05:53 +0100 Subject: create-filelists: Improve performance by reading package list from db instead from fs --- cron-jobs/create-filelists | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/cron-jobs/create-filelists b/cron-jobs/create-filelists index c3e7d72..8bcfc0b 100755 --- a/cron-jobs/create-filelists +++ b/cron-jobs/create-filelists @@ -25,17 +25,16 @@ for repo in ${PKGREPOS[@]}; do # get a list of package files defined in the repo db mkdir -p "${WORKDIR}/db-dir-${repo}-${arch}" bsdtar -xf "${repodb}" -C "${WORKDIR}/db-dir-${repo}-${arch}" - find "${WORKDIR}/db-dir-${repo}-${arch}" -name 'desc' \ - -exec awk '/^%FILENAME%/{getline;print}' {} \; | sort > "${WORKDIR}/db-${repo}-${arch}" + # This should actualy be faster than reading all the just extracted files + bsdtar -xOf "${repodb}" | awk '/^%FILENAME%/{getline;print}' | sort > "${WORKDIR}/db-${repo}-${arch}" # get a list of package files defined in the files db mkdir -p "${WORKDIR}/files-current-dir-${repo}-${arch}" if [ ! -f "${filedb}" ]; then - echo > "${WORKDIR}/files-${repo}-${arch}" + touch "${WORKDIR}/files-${repo}-${arch}" else bsdtar -xf "${filedb}" -C "${WORKDIR}/files-current-dir-${repo}-${arch}" - find "${WORKDIR}/files-current-dir-${repo}-${arch}" -name 'desc' \ - -exec awk '/^%FILENAME%/{getline;print}' {} \; | sort > "${WORKDIR}/files-${repo}-${arch}" + bsdtar -xOf "${filedb}" | awk '/^%FILENAME%/{getline;print}' | sort > "${WORKDIR}/files-${repo}-${arch}" fi done done -- cgit v1.2.2 From 69721f4e42fa296de2d5a568d07758888c354328 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sat, 27 Nov 2010 03:08:59 +0100 Subject: Fix test: copy target of symlinks --- test/lib/common.inc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/lib/common.inc b/test/lib/common.inc index ba5d685..208400c 100644 --- a/test/lib/common.inc +++ b/test/lib/common.inc @@ -12,7 +12,7 @@ oneTimeSetUp() { local pkgversion local build pkgdir="$(mktemp -d /dev/shm/$(basename $0).XXXXXXXXXX)" - cp -r $(dirname ${BASH_SOURCE[0]})/../packages/* "${pkgdir}" + cp -Lr $(dirname ${BASH_SOURCE[0]})/../packages/* "${pkgdir}" msg 'Building packages...' for d in "${pkgdir}"/*; do pushd $d >/dev/null @@ -79,7 +79,7 @@ setUp() { for p in "${pkgdir}"/*; do pkg=$(basename $p) mkdir -p "${TMP}/svn-packages-copy/${pkg}"/{trunk,repos} - cp "${p}/PKGBUILD" "${TMP}/svn-packages-copy"/${pkg}/trunk/ + cp "${p}"/* "${TMP}/svn-packages-copy"/${pkg}/trunk/ svn add -q "${TMP}/svn-packages-copy"/${pkg} svn commit -q -m"initial commit of ${pkg}" "${TMP}/svn-packages-copy" done -- cgit v1.2.2 From 6037b3e9aa414d94cd14b8c51b9b04be054c5e04 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Thu, 2 Dec 2010 18:06:45 +0100 Subject: db-update: Fail if a set of split packages is incomplete --- db-functions | 58 +++++++++++++++++++++++++++++++++++++++++++++++- db-update | 3 +++ test/test.d/db-update.sh | 23 +++++++++++++++++++ 3 files changed, 83 insertions(+), 1 deletion(-) diff --git a/db-functions b/db-functions index 67a52aa..cb76e7a 100644 --- a/db-functions +++ b/db-functions @@ -198,9 +198,20 @@ getpkgbase() { _base="$(_grep_pkginfo "$1" "pkgbase")" if [ -z "$_base" ]; then getpkgname "$1" + else + echo "$_base" fi +} - echo "$_base" +issplitpkg() { + local _base + + _base="$(_grep_pkginfo "$1" "pkgbase")" + if [ -z "$_base" ]; then + return 1 + else + return 0 + fi } # Get the package name @@ -319,6 +330,51 @@ check_pkgsvn() { return 0 } +check_splitpkgs() { + local repo="${1}" + shift + local pkgfiles=(${@}) + local pkgfile + local pkgdir + local svnname + + mkdir -p "${WORKDIR}/check_splitpkgs/" + pushd "${WORKDIR}/check_splitpkgs" >/dev/null + + for pkgfile in ${pkgfiles[@]}; do + issplitpkg "${pkgfile}" || continue + local _pkgbase="$(getpkgbase ${pkgfile})" + local _pkgname="$(getpkgname ${pkgfile})" + local _pkgarch="$(getpkgarch ${pkgfile})" + mkdir -p "${repo}/${_pkgarch}/${_pkgbase}" + echo "${_pkgname}" >> "${repo}/${_pkgarch}/${_pkgbase}/staging" + + if [ ! -f "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}" ]; then + mkdir -p "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}" + svn export -q "${SVNREPO}/${_pkgbase}/repos/${repo}-${_pkgarch}/PKGBUILD" \ + "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}" >/dev/null + [ $? -ge 1 ] && return 1 + fi + + local svnnames=($(. "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}"; echo ${pkgname[@]})) + for svnname in ${svnnames[@]}; do + echo "${svnname}" >> "${repo}/${_pkgarch}/${_pkgbase}/svn" + done + done + popd >/dev/null + + for pkgdir in "${WORKDIR}/check_splitpkgs/${repo}"/*/*; do + [ ! -d "${pkgdir}" ] && continue + sort -u "${pkgdir}/staging" -o "${pkgdir}/staging" + sort -u "${pkgdir}/svn" -o "${pkgdir}/svn" + if [ ! -z "$(comm -13 "${pkgdir}/staging" "${pkgdir}/svn")" ]; then + return 1 + fi + done + + return 0 +} + check_pkgrepos() { local pkgfile=$1 diff --git a/db-update b/db-update index 49ecbde..3a49f0a 100755 --- a/db-update +++ b/db-update @@ -42,6 +42,9 @@ for repo in ${repos[@]}; do die "Package ${repo}/$(basename ${pkg}) already exists in another repository" fi done + if ! check_splitpkgs ${repo} ${pkgs[@]}; then + die "Missing split packages for ${repo}" + fi else die "Could not read ${STAGING}" fi diff --git a/test/test.d/db-update.sh b/test/test.d/db-update.sh index 3c41b39..ad0be32 100755 --- a/test/test.d/db-update.sh +++ b/test/test.d/db-update.sh @@ -124,4 +124,27 @@ testUpdateSameAnyPackageToDifferentRepositories() { done } + +testAddIncompleteSplitPackage() { + local arches=('i686' 'x86_64') + local repo='extra' + local pkgbase='pkg-split-a' + local arch + + for arch in ${arches[@]}; do + releasePackage ${repo} ${pkgbase} ${arch} + done + + # remove a split package to make db-update fail + rm "${STAGING}"/extra/${pkgbase}1-* + + ../db-update >/dev/null 2>&1 && fail "db-update should fail when a split package is missing!" + + for arch in ${arches[@]}; do + ( [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" ] \ + && bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" -O | grep -q ${pkgbase}) \ + && fail "${pkgbase} should not be in ${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" + done +} + . "${curdir}/../lib/shunit2" -- cgit v1.2.2 From 639bef5d42d64871bc332c5618e29d457962d5f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Oudompheng?= Date: Sat, 4 Dec 2010 18:09:18 +0100 Subject: Style changes: remove usage of the 'has_key' method. Signed-off-by: Pierre Schmitz --- cron-jobs/check_archlinux/check_packages.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/cron-jobs/check_archlinux/check_packages.py b/cron-jobs/check_archlinux/check_packages.py index 465d87f..3874937 100755 --- a/cron-jobs/check_archlinux/check_packages.py +++ b/cron-jobs/check_archlinux/check_packages.py @@ -86,7 +86,7 @@ def parse_data(repo,data): pkg.name = line pkg.repo = repo dup = None - if packages.has_key(pkg.name): + if pkg.name in packages: dup = packages[pkg.name] else: packages[pkg.name] = pkg @@ -171,11 +171,11 @@ def provcmp(pkg,dep): def verify_dep(dep): dep = splitdep(dep) - if packages.has_key(dep.name): + if dep.name in packages: pkg = packages[dep.name] if depcmp(pkg.name,pkg.version,dep): return [pkg] - if provisions.has_key(dep.name): + if dep.name in provisions: provlist = provisions[dep.name] results = [] for prov in provlist: @@ -212,12 +212,12 @@ def compute_deplist(pkg): stack = [pkg] while stack != []: dep = stack.pop() - if pkgdeps.has_key(dep): + if dep in pkgdeps: for dep2 in pkgdeps[dep]: if dep2 not in list: list.append(dep2) stack.append(dep2) - if makepkgdeps.has_key(dep): + if dep in makepkgdeps: for dep2 in makepkgdeps[dep]: if dep2 not in list: list.append(dep2) @@ -245,7 +245,7 @@ def get_repo_hierarchy(repo): repo_hierarchy = {'core': ['core'], \ 'extra': ['core', 'extra'], \ 'community': ['core', 'extra', 'community']} - if repo_hierarchy.has_key(repo): + if repo in repo_hierarchy: return repo_hierarchy[repo] else: return ['core','extra','community'] @@ -277,10 +277,10 @@ def tarjan(pkg): checked_deps.append(pkg) S.append(pkg) deps = [] - if pkgdeps.has_key(pkg): + if pkg in pkgdeps: deps = pkgdeps[pkg] for dep in deps: - if not pkgindex.has_key(dep): + if dep not in pkgindex: tarjan(dep) pkglowlink[pkg] = min(pkglowlink[pkg],pkglowlink[dep]) elif dep in S: @@ -430,7 +430,7 @@ parse_pkgbuilds(loadrepos,arch) for name,pkg in packages.iteritems(): for prov in pkg.provides: provname=prov.split("=")[0] - if not provisions.has_key(provname): + if provname not in provisions: provisions[provname] = [] provisions[provname].append(pkg) @@ -482,7 +482,7 @@ makedep_hierarchy = check_hierarchy(makedeph) print "==> checking for circular dependencies" # make sure pkgdeps is filled for every package for name,pkg in packages.iteritems(): - if not pkgdeps.has_key(pkg): + if pkg not in pkgdeps: (deps,missdeps,_) = verify_deps(name,pkg.repo,pkg.deps) pkgdeps[pkg] = deps find_scc(repopkgs.values()) @@ -490,7 +490,7 @@ find_scc(repopkgs.values()) print "==> checking for differences between db files and pkgbuilds" for repo in repos: for pkg in dbpkgs[repo]: - if not (pkg in repopkgs.keys() and repopkgs[pkg].repo == repo): + if not (pkg in repopkgs and repopkgs[pkg].repo == repo): dbonly.append("%s/%s" % (repo,pkg)) for name,pkg in repopkgs.iteritems(): if not name in dbpkgs[pkg.repo]: -- cgit v1.2.2 From dd5b341abea441a6a3e729afc6d1231bc0b0abe3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Oudompheng?= Date: Sat, 4 Dec 2010 18:19:45 +0100 Subject: Style changes: add parentheses around print "...". Signed-off-by: Pierre Schmitz --- cron-jobs/check_archlinux/check_packages.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/cron-jobs/check_archlinux/check_packages.py b/cron-jobs/check_archlinux/check_packages.py index 3874937..948a61a 100755 --- a/cron-jobs/check_archlinux/check_packages.py +++ b/cron-jobs/check_archlinux/check_packages.py @@ -421,9 +421,9 @@ for repo in repos: loadrepos = loadrepos | set(get_repo_hierarchy(repo)) print_heading("Integrity Check " + arch + " of " + ",".join(repos)) -print "\nPerforming integrity checks..." +print("\nPerforming integrity checks...") -print "==> parsing pkgbuilds" +print("==> parsing pkgbuilds") parse_pkgbuilds(loadrepos,arch) # fill provisions @@ -439,16 +439,16 @@ for name,pkg in packages.iteritems(): if pkg.repo in repos: repopkgs[name] = pkg -print "==> parsing db files" +print("==> parsing db files") dbpkgs = parse_dbs(repos,arch) -print "==> checking mismatches" +print("==> checking mismatches") for name,pkg in repopkgs.iteritems(): pkgdirname = pkg.path.split("/")[-1] if name != pkgdirname and pkg.base != pkgdirname: mismatches.append(name + " vs. " + pkg.path) -print "==> checking archs" +print("==> checking archs") for name,pkg in repopkgs.iteritems(): archs = verify_archs(name,pkg.repo,pkg.archs) invalid_archs.extend(archs) @@ -461,25 +461,25 @@ for name,pkg in packages.iteritems(): deph,makedeph = [],[] -print "==> checking dependencies" +print("==> checking dependencies") for name,pkg in repopkgs.iteritems(): (deps,missdeps,hierarchy) = verify_deps(name,pkg.repo,pkg.deps) pkgdeps[pkg] = deps missing_deps.extend(missdeps) deph.extend(hierarchy) -print "==> checking makedepends" +print("==> checking makedepends") for name,pkg in repopkgs.iteritems(): (makedeps,missdeps,hierarchy) = verify_deps(name,pkg.repo,pkg.makedeps) makepkgdeps[pkg] = makedeps missing_makedeps.extend(missdeps) makedeph.extend(hierarchy) -print "==> checking hierarchy" +print("==> checking hierarchy") dep_hierarchy = check_hierarchy(deph) makedep_hierarchy = check_hierarchy(makedeph) -print "==> checking for circular dependencies" +print("==> checking for circular dependencies") # make sure pkgdeps is filled for every package for name,pkg in packages.iteritems(): if pkg not in pkgdeps: @@ -487,7 +487,7 @@ for name,pkg in packages.iteritems(): pkgdeps[pkg] = deps find_scc(repopkgs.values()) -print "==> checking for differences between db files and pkgbuilds" +print("==> checking for differences between db files and pkgbuilds") for repo in repos: for pkg in dbpkgs[repo]: if not (pkg in repopkgs and repopkgs[pkg].repo == repo): -- cgit v1.2.2 From d0222cb73c50fd23abd572e080b5b5231cb6d23b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Oudompheng?= Date: Sat, 4 Dec 2010 18:20:39 +0100 Subject: Add support for the [multilib] repo. Signed-off-by: Pierre Schmitz --- cron-jobs/check_archlinux/check_packages.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cron-jobs/check_archlinux/check_packages.py b/cron-jobs/check_archlinux/check_packages.py index 948a61a..86494b7 100755 --- a/cron-jobs/check_archlinux/check_packages.py +++ b/cron-jobs/check_archlinux/check_packages.py @@ -244,7 +244,8 @@ def check_hierarchy(deph): def get_repo_hierarchy(repo): repo_hierarchy = {'core': ['core'], \ 'extra': ['core', 'extra'], \ - 'community': ['core', 'extra', 'community']} + 'community': ['core', 'extra', 'community'], \ + 'multilib': ['core', 'extra', 'community', 'multilib'] } if repo in repo_hierarchy: return repo_hierarchy[repo] else: -- cgit v1.2.2 From bff3dabaf45abbc7d6bed1b3ec1c717ae678c8b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Oudompheng?= Date: Sat, 4 Dec 2010 18:37:17 +0100 Subject: Only emit a warning when a repository does not exist. Signed-off-by: Pierre Schmitz --- cron-jobs/check_archlinux/check_packages.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cron-jobs/check_archlinux/check_packages.py b/cron-jobs/check_archlinux/check_packages.py index 86494b7..bdbbeed 100755 --- a/cron-jobs/check_archlinux/check_packages.py +++ b/cron-jobs/check_archlinux/check_packages.py @@ -403,8 +403,8 @@ for absroot in absroots: for repo in repos: repopath = absroot + "/" + repo if not os.path.isdir(repopath): - print "Error : the repository " + repo + " does not exist in " + absroot - sys.exit() + print("Warning : the repository " + repo + " does not exist in " + absroot) + if not os.path.isdir(repodir): print "Error: the repository directory %s does not exist" % repodir sys.exit() -- cgit v1.2.2 From c9d72df9e6cc51155306b30b2f23b2e4f58fdc05 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Thu, 9 Dec 2010 06:52:34 +0100 Subject: repo_lock: check if repo is locked by repo-add or repo-remove --- db-functions | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/db-functions b/db-functions index cb76e7a..8ac9e63 100644 --- a/db-functions +++ b/db-functions @@ -135,12 +135,19 @@ trap cleanup EXIT #repo_lock [timeout] repo_lock () { local LOCKDIR="$TMPDIR/.repolock.$1.$2" + local LOCKFILE="${FTP_BASE}/${1}/os/${2}/${1}${DBEXT}.lck" local _count local _trial local _timeout local _lockblock local _owner + # This is the lock file used by repo-add and repo-remove + if [ -f "${LOCKFILE}" ]; then + error "Repo [${1}] (${2}) is already locked by repo-{add,remove} process $(cat $LOCKFILE)" + return 1 + fi + if [ $# -eq 2 ]; then _lockblock=true _trial=0 -- cgit v1.2.2 From 10116f6f0d5a0fe36b8582ed23f193683e4e5523 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Thu, 9 Dec 2010 08:28:39 +0100 Subject: Call integrity-check with specific repo list for each architecture --- cron-jobs/integrity-check | 36 ++++++++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/cron-jobs/integrity-check b/cron-jobs/integrity-check index 0a8650c..d4f9694 100755 --- a/cron-jobs/integrity-check +++ b/cron-jobs/integrity-check @@ -1,16 +1,32 @@ #!/bin/bash -. "$(dirname $0)/../db-functions" -. "$(dirname $0)/../config" +dirname="$(dirname $0)" +. "${dirname}/../db-functions" +. "${dirname}/../config" -if [ $# -ne 2 ]; then - die "usage: $(basename $0) [,,...] " +script_lock + +if [ $# -ne 1 ]; then + die "usage: $(basename $0) " fi +mailto=$1 + +check() { + ${dirname}/check_archlinux/check_packages.py \ + --repos="${repos}" \ + --abs-tree="/srv/abs/rsync/${arch},/srv/abs/rsync/any" \ + --repo-dir="${FTP_BASE}" \ + --arch="${arch}" \ + 2>&1 | ${dirname}/devlist-mailer "Integrity Check ${arch}: ${repos}" "${mailto}" +} + +repos='core,extra,community' +arch='i686' +check + +repos='core,extra,community,multilib' +arch='x86_64' +check -for arch in ${ARCHES[@]}; do - $(dirname $0)/check_archlinux/check_packages.py \ - --repos="$1" \ - --abs-tree="/srv/abs/rsync/$arch,/srv/abs/rsync/any" --arch="$arch" |\ - $(dirname $0)/devlist-mailer "Integrity Check $arch: $1" "$2" -done +script_unlock -- cgit v1.2.2 From 76256c33bb6fa06b79bdab6615f2da587cd80f31 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Thu, 9 Dec 2010 08:34:10 +0100 Subject: check_packages.py: remove obsolete klibc workaround --- cron-jobs/check_archlinux/check_packages.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/cron-jobs/check_archlinux/check_packages.py b/cron-jobs/check_archlinux/check_packages.py index bdbbeed..0a2b10e 100755 --- a/cron-jobs/check_archlinux/check_packages.py +++ b/cron-jobs/check_archlinux/check_packages.py @@ -454,12 +454,6 @@ for name,pkg in repopkgs.iteritems(): archs = verify_archs(name,pkg.repo,pkg.archs) invalid_archs.extend(archs) -# ugly hack to strip the weird kblic- deps -for name,pkg in packages.iteritems(): - p = re.compile('klibc-[\w\-]{27}|klibc-\*') - pkg.deps = [dep for dep in pkg.deps if not p.match(dep)] - pkg.makedeps = [dep for dep in pkg.makedeps if not p.match(dep)] - deph,makedeph = [],[] print("==> checking dependencies") -- cgit v1.2.2 From c736d21ae97cbd9e4f6b036029426f77f14fd063 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sat, 18 Dec 2010 01:50:17 +0100 Subject: Touch old packages after moving them to the cleanup dirs This updates the mtime of those files and will help us to remove them later. --- cron-jobs/ftpdir-cleanup | 1 + cron-jobs/sourceballs | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index 1df4edc..fe49706 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -13,6 +13,7 @@ clean_pkg() { rm -f "$pkg" else mv -f "$pkg" "$CLEANUP_DESTDIR" + touch "${CLEANUP_DESTDIR}/$(basename ${pkg})" fi done fi diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 6553327..85221a7 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -125,7 +125,10 @@ if [ ${#old_pkgs[@]} -ge 1 ]; then ${SOURCE_CLEANUP_DRYRUN} && warning 'dry run mode is active' for old_pkg in ${old_pkgs[@]}; do msg2 "${old_pkg}" - ${SOURCE_CLEANUP_DRYRUN} || mv "$FTP_BASE/${SRCPOOL}/${old_pkg}" "${SOURCE_CLEANUP_DESTDIR}" + if ! ${SOURCE_CLEANUP_DRYRUN}; then + mv "$FTP_BASE/${SRCPOOL}/${old_pkg}" "${SOURCE_CLEANUP_DESTDIR}" + touch "${SOURCE_CLEANUP_DESTDIR}/${old_pkg}" + fi done fi -- cgit v1.2.2 From 1f81b42ce2cf0c7517609bd6d5fa941a4bc01233 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sat, 18 Dec 2010 12:41:34 +0100 Subject: Remove old files from the cleanup dirs --- config | 5 +++++ cron-jobs/ftpdir-cleanup | 8 ++++++++ cron-jobs/sourceballs | 9 +++++++++ 3 files changed, 22 insertions(+) diff --git a/config b/config index ff297ec..f1ef098 100644 --- a/config +++ b/config @@ -6,8 +6,13 @@ SRCPOOL='' CLEANUP_DESTDIR="/srv/package-cleanup" CLEANUP_DRYRUN=false +# Time in days to keep moved packages +CLEANUP_KEEP=60 + SOURCE_CLEANUP_DESTDIR="/srv/source-cleanup" SOURCE_CLEANUP_DRYRUN=false +# Time in days to keep moved sourcepackages +SOURCE_CLEANUP_KEEP=14 LOCK_DELAY=10 LOCK_TIMEOUT=300 diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index fe49706..bd2d340 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -95,6 +95,14 @@ for repo in ${PKGREPOS[@]}; do fi done +old_pkgs=($(find ${CLEANUP_DESTDIR} -type f -mtime +${CLEANUP_KEEP} -printf '%f\n')) +if [ ${#old_pkgs[@]} -ge 1 ]; then + msg "Removing old packages from the cleanup directory..." + for old_pkg in ${old_pkgs[@]}; do + msg2 "${old_pkg}" + ${CLEANUP_DRYRUN} || rm -f "${CLEANUP_DESTDIR}/${old_pkg}" + done +fi for repo in ${PKGREPOS[@]}; do for arch in ${ARCHES[@]}; do diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 85221a7..4048e2f 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -132,4 +132,13 @@ if [ ${#old_pkgs[@]} -ge 1 ]; then done fi +old_pkgs=($(find ${SOURCE_CLEANUP_DESTDIR} -type f -mtime +${SOURCE_CLEANUP_KEEP} -printf '%f\n')) +if [ ${#old_pkgs[@]} -ge 1 ]; then + msg "Removing old source packages from the cleanup directory..." + for old_pkg in ${old_pkgs[@]}; do + msg2 "${old_pkg}" + ${SOURCE_CLEANUP_DRYRUN} || rm -f "${SOURCE_CLEANUP_DESTDIR}/${old_pkg}" + done +fi + script_unlock -- cgit v1.2.2 From 7519447709e1ac9a5b5edaa04b32c03ac09ee2ef Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 8 Jan 2011 12:02:11 -0600 Subject: archweb update: source virtualenv if it exists Signed-off-by: Dan McGee --- cron-jobs/update-web-db | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cron-jobs/update-web-db b/cron-jobs/update-web-db index 8760fc6..6ced4c1 100755 --- a/cron-jobs/update-web-db +++ b/cron-jobs/update-web-db @@ -5,6 +5,8 @@ # setup paths SPATH="/srv/http/archweb" +ENVPATH="/srv/http/archweb-env/bin/activate" + # having "more important repos" last should make [core] trickle to the top of # the updates list each hour rather than being overwhelmed by big [extra] and # [community] updates @@ -24,6 +26,11 @@ renice +5 -p $$ > /dev/null echo "$cmd: Updating DB at $(date)" >> "${LOGOUT}" +# source our virtualenv if it exists +if [ -f "$ENVPATH" ]; then + . "$ENVPATH" +fi + case "$cmd" in update-web-db) dbfileext="${DBEXT}" -- cgit v1.2.2 From 4da9bf112b07c94a7fd65a729ce62610ade8e5aa Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sat, 15 Jan 2011 01:12:09 +0100 Subject: Move repo manipulation code into common functions repo-add and repo-remove is now indirectly called by arch_repo_add/remove. This simplifies future extensions like incremental file list creations. See FS#11302 --- db-functions | 28 ++++++++++++++++++++++++++++ db-move | 9 ++------- db-remove | 7 +------ db-repo-add | 3 +-- db-repo-remove | 7 +------ db-update | 6 +----- 6 files changed, 34 insertions(+), 26 deletions(-) diff --git a/db-functions b/db-functions index 8ac9e63..7d431fc 100644 --- a/db-functions +++ b/db-functions @@ -450,3 +450,31 @@ set_repo_permission() { error "You don't have permission to change ${dbfile}" fi } + +arch_repo_add() { + local repo=$1 + local arch=$2 + local pkgs=(${@:3}) + + # package files might be relative to repo dir + pushd "${FTP_BASE}/${repo}/os/${arch}" >/dev/null + /usr/bin/repo-add -q "${repo}${DBEXT}" ${pkgs[@]} >/dev/null \ + || error "repo-add ${repo}${DBEXT} ${pkgs[@]}" + popd >/dev/null + set_repo_permission "${repo}" "${arch}" +} + +arch_repo_remove() { + local repo=$1 + local arch=$2 + local pkgs=(${@:3}) + local dbfile="${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" + + if [ ! -f "${dbfile}" ]; then + error "No database found at '${dbfile}'" + return 1 + fi + /usr/bin/repo-remove -q "${dbfile}" ${pkgs[@]} >/dev/null \ + || error "repo-remove ${dbfile} ${pkgs[@]}" + set_repo_permission "${repo}" "${arch}" +} diff --git a/db-move b/db-move index cdc261c..ae21781 100755 --- a/db-move +++ b/db-move @@ -106,13 +106,8 @@ done for tarch in ${ARCHES[@]}; do if [ -n "${add_pkgs[${tarch}]}" ]; then - /usr/bin/repo-add -q "${ftppath_to}/${tarch}/${repo_to}${DBEXT}" ${add_pkgs[${tarch}]} >/dev/null \ - || error "repo-add ${tarch}/${repo_to}${DBEXT} ${add_pkgs[${tarch}]}" - set_repo_permission "${repo_to}" "${tarch}" - - /usr/bin/repo-remove -q "${ftppath_from}/${tarch}/${repo_from}${DBEXT}" ${remove_pkgs[${tarch}]} >/dev/null \ - || error "repo-remove ${tarch}/${repo_from}${DBEXT} ${remove_pkgs[${tarch}]}" - set_repo_permission "${repo_from}" "${tarch}" + arch_repo_add "${repo_to}" "${tarch}" ${add_pkgs[${tarch}]} + arch_repo_remove "${repo_from}" "${tarch}" ${remove_pkgs[${tarch}]} fi done diff --git a/db-remove b/db-remove index b246ce3..cc65eb9 100755 --- a/db-remove +++ b/db-remove @@ -41,11 +41,6 @@ else fi for tarch in ${tarches[@]}; do - if [ ! -f "$ftppath/$tarch/$repo$DBEXT" ]; then - die "No database found at '$ftppath/$tarch', nothing more to do" - fi - - /usr/bin/repo-remove -q "$ftppath/$tarch/$repo$DBEXT" ${pkgnames[@]} >/dev/null - set_repo_permission "${repo}" "${tarch}" + arch_repo_remove "${repo}" "${tarch}" ${pkgnames[@]} repo_unlock $repo $tarch done diff --git a/db-repo-add b/db-repo-add index b6dd8b8..53cfc84 100755 --- a/db-repo-add +++ b/db-repo-add @@ -34,7 +34,6 @@ for tarch in ${tarches[@]}; do if [ ! -f "${pkgfile}" ]; then die "Package file ${pkgfile} not found" fi - /usr/bin/repo-add -q "$ftppath/$tarch/$repo$DBEXT" ${pkgfile} >/dev/null - set_repo_permission "${repo}" "${tarch}" + arch_repo_add "${repo}" "${tarch}" ${pkgfile} repo_unlock $repo $tarch done diff --git a/db-repo-remove b/db-repo-remove index 6fa0e05..b01910f 100755 --- a/db-repo-remove +++ b/db-repo-remove @@ -31,11 +31,6 @@ done msg "Removing $pkgname from [$repo]..." for tarch in ${tarches[@]}; do - if [ ! -f "$ftppath/$tarch/$repo$DBEXT" ]; then - die "No database found at '$ftppath/$tarch', nothing more to do" - fi - - /usr/bin/repo-remove -q "$ftppath/$tarch/$repo$DBEXT" ${pkgname} >/dev/null - set_repo_permission "${repo}" "${tarch}" + arch_repo_remove "${repo}" "${tarch}" ${pkgname} repo_unlock $repo $tarch done diff --git a/db-update b/db-update index 3a49f0a..5bdce5e 100755 --- a/db-update +++ b/db-update @@ -67,11 +67,7 @@ for repo in ${repos[@]}; do add_pkgs[${#add_pkgs[*]}]=${pkgfile} done if [ ${#add_pkgs[@]} -ge 1 ]; then - pushd "$FTP_BASE/$repo/os/${pkgarch}" >/dev/null - /usr/bin/repo-add -q "$repo$DBEXT" ${add_pkgs[@]} >/dev/null \ - || die "repo-add $repo$DBEXT ${add_pkgs[@]}" - popd >/dev/null - set_repo_permission "${repo}" "${pkgarch}" + arch_repo_add "${repo}" "${pkgarch}" ${add_pkgs[@]} fi done done -- cgit v1.2.2 From 7222a50e44f815fd2ba17a9a94efd3cc9fa4bcd7 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Thu, 27 Jan 2011 13:30:02 +0100 Subject: test: add testUpdateSameAnyPackageToSameRepository --- test/test.d/db-update.sh | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/test/test.d/db-update.sh b/test/test.d/db-update.sh index ad0be32..5a3d01d 100755 --- a/test/test.d/db-update.sh +++ b/test/test.d/db-update.sh @@ -108,6 +108,15 @@ testUpdateAnyPackageToDifferentRepositoriesAtOnce() { rm -f "${pkgdir}/pkg-any-a/pkg-any-a-1-2-any.pkg.tar.xz" } +testUpdateSameAnyPackageToSameRepository() { + releasePackage extra pkg-any-a any + ../db-update + checkAnyPackage extra pkg-any-a-1-1-any.pkg.tar.xz any + + releasePackage extra pkg-any-a any + ../db-update >/dev/null 2>&1 && (fail 'Adding an existing package to the same repository should fail'; return 1) +} + testUpdateSameAnyPackageToDifferentRepositories() { releasePackage extra pkg-any-a any ../db-update @@ -118,9 +127,9 @@ testUpdateSameAnyPackageToDifferentRepositories() { local arch for arch in i686 x86_64; do - ( [ -r "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" ] \ - && bsdtar -xf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" -O | grep -q ${pkgbase}) \ - && fail "${pkgbase} should not be in ${repo}/os/${arch}/${repo}${DBEXT%.tar.*}" + ( [ -r "${FTP_BASE}/testing/os/${arch}/testing${DBEXT%.tar.*}" ] \ + && bsdtar -xf "${FTP_BASE}/testing/os/${arch}/testing${DBEXT%.tar.*}" -O | grep -q ${pkgbase}) \ + && fail "${pkgbase} should not be in testing/os/${arch}/testing${DBEXT%.tar.*}" done } -- cgit v1.2.2 From d9aad61303816aa7a4b1ba950856b30368acb968 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 30 Jan 2011 12:07:30 +0100 Subject: Make sure to only remove package files from the cleanup dirs --- cron-jobs/ftpdir-cleanup | 2 +- cron-jobs/sourceballs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index bd2d340..09e8a49 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -95,7 +95,7 @@ for repo in ${PKGREPOS[@]}; do fi done -old_pkgs=($(find ${CLEANUP_DESTDIR} -type f -mtime +${CLEANUP_KEEP} -printf '%f\n')) +old_pkgs=($(find ${CLEANUP_DESTDIR} -type f -name "*${PKGEXT}" -mtime +${CLEANUP_KEEP} -printf '%f\n')) if [ ${#old_pkgs[@]} -ge 1 ]; then msg "Removing old packages from the cleanup directory..." for old_pkg in ${old_pkgs[@]}; do diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 4048e2f..b55de05 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -132,7 +132,7 @@ if [ ${#old_pkgs[@]} -ge 1 ]; then done fi -old_pkgs=($(find ${SOURCE_CLEANUP_DESTDIR} -type f -mtime +${SOURCE_CLEANUP_KEEP} -printf '%f\n')) +old_pkgs=($(find ${SOURCE_CLEANUP_DESTDIR} -type f -name "*${SRCEXT}" -mtime +${SOURCE_CLEANUP_KEEP} -printf '%f\n')) if [ ${#old_pkgs[@]} -ge 1 ]; then msg "Removing old source packages from the cleanup directory..." for old_pkg in ${old_pkgs[@]}; do -- cgit v1.2.2 From af06266bf8d7eb0377dd6dcfad7bf1b9c1f79825 Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 30 Jan 2011 12:17:42 +0100 Subject: Try to remove a package even if it is no longer in svn --- db-remove | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db-remove b/db-remove index cc65eb9..292af5d 100755 --- a/db-remove +++ b/db-remove @@ -37,7 +37,7 @@ if [ -d "${WORKDIR}/svn/$pkgbase/repos/$svnrepo" ]; then /usr/bin/svn rm --force -q "${WORKDIR}/svn/$pkgbase/repos/$svnrepo" /usr/bin/svn commit -q "${WORKDIR}/svn/$pkgbase" -m "$(basename $0): $pkgbase removed by $(id -un)" else - die "$pkgbase not found in $svnrepo" + warning "$pkgbase not found in $svnrepo" fi for tarch in ${tarches[@]}; do -- cgit v1.2.2 From 9e4f661adf89d150b0891d6edd39e19682d923bf Mon Sep 17 00:00:00 2001 From: Pierre Schmitz Date: Sun, 30 Jan 2011 12:18:56 +0100 Subject: Keeping old packages for a month should be long enough --- config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config b/config index f1ef098..53191e0 100644 --- a/config +++ b/config @@ -7,7 +7,7 @@ SRCPOOL='' CLEANUP_DESTDIR="/srv/package-cleanup" CLEANUP_DRYRUN=false # Time in days to keep moved packages -CLEANUP_KEEP=60 +CLEANUP_KEEP=30 SOURCE_CLEANUP_DESTDIR="/srv/source-cleanup" SOURCE_CLEANUP_DRYRUN=false -- cgit v1.2.2 From df9a29ad9051f89658874959ebbc2f538e565771 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Wed, 16 Feb 2011 19:31:30 -0300 Subject: Applied changes for Parabola --- config | 22 +++++++++++----------- db-functions | 33 ++++++++++++++++++++++++--------- db-update | 9 +++------ 3 files changed, 38 insertions(+), 26 deletions(-) diff --git a/config b/config index 53191e0..e55958d 100644 --- a/config +++ b/config @@ -1,24 +1,24 @@ -FTP_BASE="/srv/ftp" -SVNREPO='' -PKGREPOS=() -PKGPOOL='' -SRCPOOL='' +FTP_BASE="/home/parabolavnx/parabolagnulinux.org/free" +SVNREPO="/home/parabolavnx/parabolagnulinux.org/abslibre" +PKGREPOS=('core' 'extra' 'community' 'libre' 'libre-testing' 'social' 'sugar' 'testing') +PKGPOOL='pool/packages' +SRCPOOL='sources/packages' -CLEANUP_DESTDIR="/srv/package-cleanup" +CLEANUP_DESTDIR="$FTP_BASE/old/packages" CLEANUP_DRYRUN=false # Time in days to keep moved packages CLEANUP_KEEP=30 -SOURCE_CLEANUP_DESTDIR="/srv/source-cleanup" +SOURCE_CLEANUP_DESTDIR="$FTP_BASE/old/sources" SOURCE_CLEANUP_DRYRUN=false # Time in days to keep moved sourcepackages -SOURCE_CLEANUP_KEEP=14 +SOURCE_CLEANUP_KEEP=30 LOCK_DELAY=10 LOCK_TIMEOUT=300 -STAGING="$HOME/staging" -TMPDIR="/srv/tmp" +STAGING="$FTP_BASE/staging" +TMPDIR="$HOME/tmp" ARCHES=(i686 x86_64) DBEXT=".db.tar.gz" FILESEXT=".files.tar.gz" @@ -29,4 +29,4 @@ SRCEXT=".src.tar.gz" ALLOWED_LICENSES=('GPL' 'GPL1' 'GPL2' 'LGPL' 'LGPL1' 'LGPL2' 'LGPL2.1') # Override default config with config.local -[ -f "$(dirname ${BASH_SOURCE[0]})/config.local" ] && . "$(dirname ${BASH_SOURCE[0]})/config.local" +#[ -f "$(dirname ${BASH_SOURCE[0]})/config.local" ] && . "$(dirname ${BASH_SOURCE[0]})/config.local" diff --git a/db-functions b/db-functions index 7d431fc..0553188 100644 --- a/db-functions +++ b/db-functions @@ -74,7 +74,7 @@ in_array() { script_lock() { local LOCKDIR="$TMPDIR/.scriptlock.$(basename $0)" if ! mkdir "$LOCKDIR" >/dev/null 2>&1 ; then - local _owner="$(/usr/bin/stat -c %U $LOCKDIR)" + local _owner="$(stat -c %U $LOCKDIR)" error "Script $(basename $0) is already locked by $_owner." exit 1 else @@ -160,7 +160,7 @@ repo_lock () { _count=0 while [ $_count -le $_trial ] || $_lockblock ; do if ! mkdir "$LOCKDIR" >/dev/null 2>&1 ; then - _owner="$(/usr/bin/stat -c %U $LOCKDIR)" + _owner="$(stat -c %U $LOCKDIR)" warning "Repo [${1}] (${2}) is already locked by $_owner. " msg2 "Retrying in $LOCK_DELAY seconds..." else @@ -193,7 +193,7 @@ repo_unlock () { #repo_unlock _grep_pkginfo() { local _ret - _ret="$(/usr/bin/bsdtar -xOqf "$1" .PKGINFO | /bin/grep -m 1 "^${2} = ")" + _ret="$(bsdtar -xOqf "$1" .PKGINFO | /bin/grep -m 1 "^${2} = ")" echo "${_ret#${2} = }" } @@ -358,9 +358,24 @@ check_splitpkgs() { if [ ! -f "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}" ]; then mkdir -p "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}" - svn export -q "${SVNREPO}/${_pkgbase}/repos/${repo}-${_pkgarch}/PKGBUILD" \ - "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}" >/dev/null - [ $? -ge 1 ] && return 1 + +# Decide whether to look for PKGBUILD on [libre] or [libre-testing] + case $repo in + testing) + altrepo=libre-testing + ;; + *) + altrepo=libre + ;; + esac + + cp -r ${SVNREPO}/$repo/$_pkgbase/PKGBUILD "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}" >/dev/null 2>&1 || \ + cp -r ${SVNREPO}/$altrepo/$_pkgbase/PKGBUILD "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/$_pkgbase">/dev/null 2>&1 + + [[ $? -ge 1 ]] && { + echo "Failed $_pkgbase-$_pkgver-$_pkgarch" + return 1 + } fi local svnnames=($(. "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}"; echo ${pkgname[@]})) @@ -443,7 +458,7 @@ set_repo_permission() { local dbfile="${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" if [ -w "${dbfile}" ]; then - local group=$(/usr/bin/stat --printf='%G' "$(dirname "${dbfile}")") + local group=$(stat --printf='%G' "$(dirname "${dbfile}")") chgrp $group "${dbfile}" || error "Could not change group of ${dbfile} to $group" chmod g+w "${dbfile}" || error "Could not set write permission for group $group to ${dbfile}" else @@ -458,7 +473,7 @@ arch_repo_add() { # package files might be relative to repo dir pushd "${FTP_BASE}/${repo}/os/${arch}" >/dev/null - /usr/bin/repo-add -q "${repo}${DBEXT}" ${pkgs[@]} >/dev/null \ + repo-add -q "${repo}${DBEXT}" ${pkgs[@]} >/dev/null \ || error "repo-add ${repo}${DBEXT} ${pkgs[@]}" popd >/dev/null set_repo_permission "${repo}" "${arch}" @@ -474,7 +489,7 @@ arch_repo_remove() { error "No database found at '${dbfile}'" return 1 fi - /usr/bin/repo-remove -q "${dbfile}" ${pkgs[@]} >/dev/null \ + repo-remove -q "${dbfile}" ${pkgs[@]} >/dev/null \ || error "repo-remove ${dbfile} ${pkgs[@]}" set_repo_permission "${repo}" "${arch}" } diff --git a/db-update b/db-update index 5bdce5e..4740809 100755 --- a/db-update +++ b/db-update @@ -35,12 +35,9 @@ for repo in ${repos[@]}; do if ! check_pkgfile "${pkg}"; then die "Package ${repo}/$(basename ${pkg}) is not consistent with its meta data" fi - if ! check_pkgsvn "${pkg}" "${repo}"; then - die "Package ${repo}/$(basename ${pkg}) is not consistent with svn repository" - fi - if ! check_pkgrepos "${pkg}"; then - die "Package ${repo}/$(basename ${pkg}) already exists in another repository" - fi + #if ! check_pkgrepos "${pkg}"; then + # die "Package ${repo}/$(basename ${pkg}) already exists in another repository" + #fi done if ! check_splitpkgs ${repo} ${pkgs[@]}; then die "Missing split packages for ${repo}" -- cgit v1.2.2 From 11db46275d14c9a9c2e59019fed3de4b8803f9f4 Mon Sep 17 00:00:00 2001 From: Parabola Date: Fri, 18 Feb 2011 15:42:59 -0800 Subject: Parabola specific changes --- config | 24 ++++++++++++------------ db-functions | 33 ++++++++++++++++++++++++--------- db-update | 9 +++------ 3 files changed, 39 insertions(+), 27 deletions(-) diff --git a/config b/config index 53191e0..720d110 100644 --- a/config +++ b/config @@ -1,25 +1,25 @@ -FTP_BASE="/srv/ftp" -SVNREPO='' -PKGREPOS=() -PKGPOOL='' -SRCPOOL='' +FTP_BASE="/home/parabolavnx/parabolagnulinux.org/free" +SVNREPO="/home/parabolavnx/parabolagnulinux.org/abslibre" +PKGREPOS=('core' 'extra' 'community' 'libre' 'libre-testing' 'social' 'sugar' 'testing') +PKGPOOL='pool/packages' +SRCPOOL='sources/packages' -CLEANUP_DESTDIR="/srv/package-cleanup" +CLEANUP_DESTDIR="$FTP_BASE/old/packages" CLEANUP_DRYRUN=false # Time in days to keep moved packages CLEANUP_KEEP=30 -SOURCE_CLEANUP_DESTDIR="/srv/source-cleanup" +SOURCE_CLEANUP_DESTDIR="$FTP_BASE/old/sources" SOURCE_CLEANUP_DRYRUN=false # Time in days to keep moved sourcepackages -SOURCE_CLEANUP_KEEP=14 +SOURCE_CLEANUP_KEEP=30 LOCK_DELAY=10 LOCK_TIMEOUT=300 -STAGING="$HOME/staging" -TMPDIR="/srv/tmp" -ARCHES=(i686 x86_64) +STAGING="$FTP_BASE/staging" +TMPDIR="$HOME/tmp" +ARCHES=(i686 x86_64 mipsel) DBEXT=".db.tar.gz" FILESEXT=".files.tar.gz" PKGEXT=".pkg.tar.*" @@ -29,4 +29,4 @@ SRCEXT=".src.tar.gz" ALLOWED_LICENSES=('GPL' 'GPL1' 'GPL2' 'LGPL' 'LGPL1' 'LGPL2' 'LGPL2.1') # Override default config with config.local -[ -f "$(dirname ${BASH_SOURCE[0]})/config.local" ] && . "$(dirname ${BASH_SOURCE[0]})/config.local" +#[ -f "$(dirname ${BASH_SOURCE[0]})/config.local" ] && . "$(dirname ${BASH_SOURCE[0]})/config.local" diff --git a/db-functions b/db-functions index 7d431fc..0553188 100644 --- a/db-functions +++ b/db-functions @@ -74,7 +74,7 @@ in_array() { script_lock() { local LOCKDIR="$TMPDIR/.scriptlock.$(basename $0)" if ! mkdir "$LOCKDIR" >/dev/null 2>&1 ; then - local _owner="$(/usr/bin/stat -c %U $LOCKDIR)" + local _owner="$(stat -c %U $LOCKDIR)" error "Script $(basename $0) is already locked by $_owner." exit 1 else @@ -160,7 +160,7 @@ repo_lock () { _count=0 while [ $_count -le $_trial ] || $_lockblock ; do if ! mkdir "$LOCKDIR" >/dev/null 2>&1 ; then - _owner="$(/usr/bin/stat -c %U $LOCKDIR)" + _owner="$(stat -c %U $LOCKDIR)" warning "Repo [${1}] (${2}) is already locked by $_owner. " msg2 "Retrying in $LOCK_DELAY seconds..." else @@ -193,7 +193,7 @@ repo_unlock () { #repo_unlock _grep_pkginfo() { local _ret - _ret="$(/usr/bin/bsdtar -xOqf "$1" .PKGINFO | /bin/grep -m 1 "^${2} = ")" + _ret="$(bsdtar -xOqf "$1" .PKGINFO | /bin/grep -m 1 "^${2} = ")" echo "${_ret#${2} = }" } @@ -358,9 +358,24 @@ check_splitpkgs() { if [ ! -f "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}" ]; then mkdir -p "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}" - svn export -q "${SVNREPO}/${_pkgbase}/repos/${repo}-${_pkgarch}/PKGBUILD" \ - "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}" >/dev/null - [ $? -ge 1 ] && return 1 + +# Decide whether to look for PKGBUILD on [libre] or [libre-testing] + case $repo in + testing) + altrepo=libre-testing + ;; + *) + altrepo=libre + ;; + esac + + cp -r ${SVNREPO}/$repo/$_pkgbase/PKGBUILD "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}" >/dev/null 2>&1 || \ + cp -r ${SVNREPO}/$altrepo/$_pkgbase/PKGBUILD "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/$_pkgbase">/dev/null 2>&1 + + [[ $? -ge 1 ]] && { + echo "Failed $_pkgbase-$_pkgver-$_pkgarch" + return 1 + } fi local svnnames=($(. "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}"; echo ${pkgname[@]})) @@ -443,7 +458,7 @@ set_repo_permission() { local dbfile="${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" if [ -w "${dbfile}" ]; then - local group=$(/usr/bin/stat --printf='%G' "$(dirname "${dbfile}")") + local group=$(stat --printf='%G' "$(dirname "${dbfile}")") chgrp $group "${dbfile}" || error "Could not change group of ${dbfile} to $group" chmod g+w "${dbfile}" || error "Could not set write permission for group $group to ${dbfile}" else @@ -458,7 +473,7 @@ arch_repo_add() { # package files might be relative to repo dir pushd "${FTP_BASE}/${repo}/os/${arch}" >/dev/null - /usr/bin/repo-add -q "${repo}${DBEXT}" ${pkgs[@]} >/dev/null \ + repo-add -q "${repo}${DBEXT}" ${pkgs[@]} >/dev/null \ || error "repo-add ${repo}${DBEXT} ${pkgs[@]}" popd >/dev/null set_repo_permission "${repo}" "${arch}" @@ -474,7 +489,7 @@ arch_repo_remove() { error "No database found at '${dbfile}'" return 1 fi - /usr/bin/repo-remove -q "${dbfile}" ${pkgs[@]} >/dev/null \ + repo-remove -q "${dbfile}" ${pkgs[@]} >/dev/null \ || error "repo-remove ${dbfile} ${pkgs[@]}" set_repo_permission "${repo}" "${arch}" } diff --git a/db-update b/db-update index 5bdce5e..4740809 100755 --- a/db-update +++ b/db-update @@ -35,12 +35,9 @@ for repo in ${repos[@]}; do if ! check_pkgfile "${pkg}"; then die "Package ${repo}/$(basename ${pkg}) is not consistent with its meta data" fi - if ! check_pkgsvn "${pkg}" "${repo}"; then - die "Package ${repo}/$(basename ${pkg}) is not consistent with svn repository" - fi - if ! check_pkgrepos "${pkg}"; then - die "Package ${repo}/$(basename ${pkg}) already exists in another repository" - fi + #if ! check_pkgrepos "${pkg}"; then + # die "Package ${repo}/$(basename ${pkg}) already exists in another repository" + #fi done if ! check_splitpkgs ${repo} ${pkgs[@]}; then die "Missing split packages for ${repo}" -- cgit v1.2.2 From 439bd2112c01bc9b185d2552aa0888d7055929e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Fri, 18 Feb 2011 20:43:56 -0300 Subject: First stab at sourceballing everything --- config | 2 +- config.local.gerolde | 4 ---- config.local.sigurd | 4 ---- cron-jobs/sourceballs | 13 ++++++++----- 4 files changed, 9 insertions(+), 14 deletions(-) delete mode 100644 config.local.gerolde delete mode 100644 config.local.sigurd diff --git a/config b/config index e55958d..720d110 100644 --- a/config +++ b/config @@ -19,7 +19,7 @@ LOCK_TIMEOUT=300 STAGING="$FTP_BASE/staging" TMPDIR="$HOME/tmp" -ARCHES=(i686 x86_64) +ARCHES=(i686 x86_64 mipsel) DBEXT=".db.tar.gz" FILESEXT=".files.tar.gz" PKGEXT=".pkg.tar.*" diff --git a/config.local.gerolde b/config.local.gerolde deleted file mode 100644 index 4501a93..0000000 --- a/config.local.gerolde +++ /dev/null @@ -1,4 +0,0 @@ -PKGREPOS=('core' 'extra' 'testing' 'staging' 'kde-unstable' 'gnome-unstable') -PKGPOOL='pool/packages' -SRCPOOL='sources/packages' -SVNREPO='file:///srv/svn-packages' diff --git a/config.local.sigurd b/config.local.sigurd deleted file mode 100644 index d28aa37..0000000 --- a/config.local.sigurd +++ /dev/null @@ -1,4 +0,0 @@ -PKGREPOS=('community' 'community-testing' 'community-staging' 'multilib' 'multilib-testing') -PKGPOOL='pool/community' -SRCPOOL='sources/community' -SVNREPO='file:///srv/svn-packages' diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index b55de05..0df007e 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -63,10 +63,11 @@ for repo in ${PKGREPOS[@]}; do if grep -Fqx "${pkgbase}" "${dirname}/sourceballs.skip"; then continue fi + # Commenting out, we'll sourceball everything # Check if the license or .force file does not enforce creating a source package - if ! (chk_license ${pkglicense[@]} || grep -Fqx "${pkgbase}" "${dirname}/sourceballs.force"); then - continue - fi +# if ! (chk_license ${pkglicense[@]} || grep -Fqx "${pkgbase}" "${dirname}/sourceballs.force"); then +# continue +# fi # Store the expected file name of the source package echo "${pkgbase}-${pkgver}${SRCEXT}" >> "${WORKDIR}/expected-src-pkgs" @@ -79,8 +80,10 @@ for repo in ${PKGREPOS[@]}; do # Get the sources from svn mkdir -p "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}" - svn export -q "${SVNREPO}/${pkgbase}/repos/${repo}-${pkgarch}" \ - "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" >/dev/null 2>&1 + #svn export -q "${SVNREPO}/${pkgbase}/repos/${repo}-${pkgarch}" \ + # "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" >/dev/null 2>&1 + cp -r "${SVNREPO}/libre/${pkgbase}" "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/" >/dev/null 2>&1 || \ + cp -r "${SVNREPO}/libre-testing/${pkgbase}" "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/" >/dev/null 2>&1 if [ $? -ge 1 ]; then failedpkgs[${#failedpkgs[*]}]="${pkgbase}-${pkgver}${SRCEXT}" continue -- cgit v1.2.2 From f8c49cc2915f66b53bd9dd248b41ab6a96c1059a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Sun, 20 Feb 2011 22:09:28 -0300 Subject: Changes to sourceball everything --- config | 1 + cron-jobs/sourceballs | 15 +++++++++------ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/config b/config index 720d110..12bdaaa 100644 --- a/config +++ b/config @@ -1,4 +1,5 @@ FTP_BASE="/home/parabolavnx/parabolagnulinux.org/free" +ARCH_BASE="/home/parabolavnx/parabolagnulinux.org/repo" SVNREPO="/home/parabolavnx/parabolagnulinux.org/abslibre" PKGREPOS=('core' 'extra' 'community' 'libre' 'libre-testing' 'social' 'sugar' 'testing') PKGPOOL='pool/packages' diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 0df007e..5726484 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -21,10 +21,10 @@ renice +10 -p $$ > /dev/null for repo in ${PKGREPOS[@]}; do for arch in ${ARCHES[@]}; do # Repo does not exist; skip it - if [ ! -f "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" ]; then + if [ ! -f "${ARCH_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" ]; then continue fi - bsdtar -xOf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" \ + bsdtar -xOf "${ARCH_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" \ | awk '/^%NAME%/ { getline b }; /^%BASE%/ { getline b }; /^%VERSION%/ { getline v }; @@ -46,7 +46,7 @@ for repo in ${PKGREPOS[@]}; do done # Create a list of all available source package file names -find "${FTP_BASE}/${SRCPOOL}" -xtype f -name "*${SRCEXT}" -printf '%f\n' | sort -u > "${WORKDIR}/available-src-pkgs" +find "${ARCH_BASE}/${SRCPOOL}" -xtype f -name "*${SRCEXT}" -printf '%f\n' | sort -u > "${WORKDIR}/available-src-pkgs" # Check for all packages if we need to build a source package for repo in ${PKGREPOS[@]}; do @@ -59,7 +59,7 @@ for repo in ${PKGREPOS[@]}; do pkgarch=${pkginfo[2]} pkglicense=(${pkginfo[@]:3}) - # Should this package be skipped? + # Should this packages be skipped? if grep -Fqx "${pkgbase}" "${dirname}/sourceballs.skip"; then continue fi @@ -82,6 +82,9 @@ for repo in ${PKGREPOS[@]}; do mkdir -p "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}" #svn export -q "${SVNREPO}/${pkgbase}/repos/${repo}-${pkgarch}" \ # "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" >/dev/null 2>&1 + + # If it's on official repos, nor [libre], nor [libre-testing] + cp -r "${SVNREPO}/$repo/${pkgbase}" "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/" >/dev/null 2>&1 || \ cp -r "${SVNREPO}/libre/${pkgbase}" "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/" >/dev/null 2>&1 || \ cp -r "${SVNREPO}/libre-testing/${pkgbase}" "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/" >/dev/null 2>&1 if [ $? -ge 1 ]; then @@ -93,7 +96,7 @@ for repo in ${PKGREPOS[@]}; do pushd "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" >/dev/null makepkg --nocolor --allsource --ignorearch >/dev/null 2>&1 if [ $? -eq 0 ] && [ -f "${pkgbase}-${pkgver}${SRCEXT}" ]; then - mv "${pkgbase}-${pkgver}${SRCEXT}" "${FTP_BASE}/${SRCPOOL}" + mv "${pkgbase}-${pkgver}${SRCEXT}" "${ARCH_BASE}/${SRCPOOL}" # Avoid creating the same source package for every arch echo "${pkgbase}-${pkgver}${SRCEXT}" >> "${WORKDIR}/available-src-pkgs" newpkgs[${#newpkgs[*]}]="${pkgbase}-${pkgver}${SRCEXT}" @@ -129,7 +132,7 @@ if [ ${#old_pkgs[@]} -ge 1 ]; then for old_pkg in ${old_pkgs[@]}; do msg2 "${old_pkg}" if ! ${SOURCE_CLEANUP_DRYRUN}; then - mv "$FTP_BASE/${SRCPOOL}/${old_pkg}" "${SOURCE_CLEANUP_DESTDIR}" + mv "$ARCH_BASE/${SRCPOOL}/${old_pkg}" "${SOURCE_CLEANUP_DESTDIR}" touch "${SOURCE_CLEANUP_DESTDIR}/${old_pkg}" fi done -- cgit v1.2.2 From 0162a16595059e269ba2d7b47e69212bc2ab1336 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=A1s=20Reynolds?= Date: Sun, 20 Feb 2011 22:09:40 -0300 Subject: Changed looking for a package on abslibre logic --- db-functions | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/db-functions b/db-functions index 0553188..e2731cb 100644 --- a/db-functions +++ b/db-functions @@ -1,5 +1,7 @@ #!/bin/bash +. config + # Some PKGBUILDs need CARCH to be set CARCH="x86_64" @@ -16,7 +18,7 @@ restore_umask () { } # set up general environment -WORKDIR=$(mktemp -d /tmp/$(basename $0).XXXXXXXXXX) +WORKDIR=$(mktemp -d ${TMPDIR}/$(basename $0).XXXXXXXXXX) LOCKS=() # check if messages are to be printed using color @@ -359,18 +361,9 @@ check_splitpkgs() { if [ ! -f "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}" ]; then mkdir -p "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}" -# Decide whether to look for PKGBUILD on [libre] or [libre-testing] - case $repo in - testing) - altrepo=libre-testing - ;; - *) - altrepo=libre - ;; - esac - cp -r ${SVNREPO}/$repo/$_pkgbase/PKGBUILD "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}" >/dev/null 2>&1 || \ - cp -r ${SVNREPO}/$altrepo/$_pkgbase/PKGBUILD "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/$_pkgbase">/dev/null 2>&1 + cp -r ${SVNREPO}/libre/$_pkgbase/PKGBUILD "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}" >/dev/null 2>&1 || \ + cp -r ${SVNREPO}/libre-testing/$_pkgbase/PKGBUILD "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/$_pkgbase">/dev/null 2>&1 [[ $? -ge 1 ]] && { echo "Failed $_pkgbase-$_pkgver-$_pkgarch" -- cgit v1.2.2 From c6516eca2803b780a01a6b2bf3e081c8bd570301 Mon Sep 17 00:00:00 2001 From: Parabola Date: Sun, 20 Feb 2011 17:10:40 -0800 Subject: sourceballs --- config | 1 + cron-jobs/sourceballs | 12 ++++++------ db-functions | 5 ++++- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/config b/config index 720d110..12bdaaa 100644 --- a/config +++ b/config @@ -1,4 +1,5 @@ FTP_BASE="/home/parabolavnx/parabolagnulinux.org/free" +ARCH_BASE="/home/parabolavnx/parabolagnulinux.org/repo" SVNREPO="/home/parabolavnx/parabolagnulinux.org/abslibre" PKGREPOS=('core' 'extra' 'community' 'libre' 'libre-testing' 'social' 'sugar' 'testing') PKGPOOL='pool/packages' diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 0df007e..53d43ce 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -21,10 +21,10 @@ renice +10 -p $$ > /dev/null for repo in ${PKGREPOS[@]}; do for arch in ${ARCHES[@]}; do # Repo does not exist; skip it - if [ ! -f "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" ]; then + if [ ! -f "${ARCH_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" ]; then continue fi - bsdtar -xOf "${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" \ + bsdtar -xOf "${ARCH_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" \ | awk '/^%NAME%/ { getline b }; /^%BASE%/ { getline b }; /^%VERSION%/ { getline v }; @@ -46,7 +46,7 @@ for repo in ${PKGREPOS[@]}; do done # Create a list of all available source package file names -find "${FTP_BASE}/${SRCPOOL}" -xtype f -name "*${SRCEXT}" -printf '%f\n' | sort -u > "${WORKDIR}/available-src-pkgs" +find "${ARCH_BASE}/${SRCPOOL}" -xtype f -name "*${SRCEXT}" -printf '%f\n' | sort -u > "${WORKDIR}/available-src-pkgs" # Check for all packages if we need to build a source package for repo in ${PKGREPOS[@]}; do @@ -91,9 +91,9 @@ for repo in ${PKGREPOS[@]}; do # Build the actual source package pushd "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" >/dev/null - makepkg --nocolor --allsource --ignorearch >/dev/null 2>&1 + makepkg --nocolor --allsource --ignorearch # >/dev/null 2>&1 if [ $? -eq 0 ] && [ -f "${pkgbase}-${pkgver}${SRCEXT}" ]; then - mv "${pkgbase}-${pkgver}${SRCEXT}" "${FTP_BASE}/${SRCPOOL}" + mv "${pkgbase}-${pkgver}${SRCEXT}" "${ARCH_BASE}/${SRCPOOL}" # Avoid creating the same source package for every arch echo "${pkgbase}-${pkgver}${SRCEXT}" >> "${WORKDIR}/available-src-pkgs" newpkgs[${#newpkgs[*]}]="${pkgbase}-${pkgver}${SRCEXT}" @@ -129,7 +129,7 @@ if [ ${#old_pkgs[@]} -ge 1 ]; then for old_pkg in ${old_pkgs[@]}; do msg2 "${old_pkg}" if ! ${SOURCE_CLEANUP_DRYRUN}; then - mv "$FTP_BASE/${SRCPOOL}/${old_pkg}" "${SOURCE_CLEANUP_DESTDIR}" + mv "$ARCH_BASE/${SRCPOOL}/${old_pkg}" "${SOURCE_CLEANUP_DESTDIR}" touch "${SOURCE_CLEANUP_DESTDIR}/${old_pkg}" fi done diff --git a/db-functions b/db-functions index 0553188..e790fb6 100644 --- a/db-functions +++ b/db-functions @@ -16,7 +16,7 @@ restore_umask () { } # set up general environment -WORKDIR=$(mktemp -d /tmp/$(basename $0).XXXXXXXXXX) +WORKDIR=$(mktemp -d /home/parabolavnx/tmp/$(basename $0).XXXXXXXXXX) LOCKS=() # check if messages are to be printed using color @@ -376,6 +376,9 @@ check_splitpkgs() { echo "Failed $_pkgbase-$_pkgver-$_pkgarch" return 1 } + + sleep 20s + fi local svnnames=($(. "${WORKDIR}/pkgbuilds/${repo}-${_pkgarch}/${_pkgbase}"; echo ${pkgname[@]})) -- cgit v1.2.2 From 229a9c504cbd733c93cf91399dc54bedf5160cc5 Mon Sep 17 00:00:00 2001 From: Parabola Date: Sun, 6 Mar 2011 15:22:21 -0800 Subject: Fixes --- config | 8 ++--- cron-jobs/sourceballs2 | 87 ++++++++++++++++++++++++++++++++++++++++++++++++++ db-functions | 4 +-- 3 files changed, 90 insertions(+), 9 deletions(-) create mode 100755 cron-jobs/sourceballs2 diff --git a/config b/config index 12bdaaa..217b627 100644 --- a/config +++ b/config @@ -20,14 +20,10 @@ LOCK_TIMEOUT=300 STAGING="$FTP_BASE/staging" TMPDIR="$HOME/tmp" -ARCHES=(i686 x86_64 mipsel) +ARCHES=(i686 x86_64 mips64el) DBEXT=".db.tar.gz" FILESEXT=".files.tar.gz" PKGEXT=".pkg.tar.*" SRCEXT=".src.tar.gz" -# Allowed licenses: get sourceballs only for licenses in this array -ALLOWED_LICENSES=('GPL' 'GPL1' 'GPL2' 'LGPL' 'LGPL1' 'LGPL2' 'LGPL2.1') - -# Override default config with config.local -#[ -f "$(dirname ${BASH_SOURCE[0]})/config.local" ] && . "$(dirname ${BASH_SOURCE[0]})/config.local" +MAKEPKGCONF="$HOME/etc/makepkg.conf" diff --git a/cron-jobs/sourceballs2 b/cron-jobs/sourceballs2 new file mode 100755 index 0000000..452208e --- /dev/null +++ b/cron-jobs/sourceballs2 @@ -0,0 +1,87 @@ +#!/bin/bash + +dirname="$(dirname $(readlink -e $0))" +. "${dirname}/../db-functions" +. "${dirname}/../config" +. "${MAKEPKGCONF}" + +pushd "${WORKDIR}" >/dev/null + +script_lock + +#adjust the nice level to run at a lower priority +renice +10 -p $$ > /dev/null + +# Create a list of all available source package file names +find "${ARCH_BASE}/${SRCPOOL}" -xtype f -name "*${SRCEXT}" -printf '%f\n' | sort -u > "${WORKDIR}/available-src-pkgs" + +# Steps +# Traverse the ABSLibre +# Makepkg --allsource every package +# Remove the old packages +pushd "${SVNREPO}" >/dev/null + +failedpkgs=() +for repo in ${PKGREPOS[@]}; do + pushd $repo >/dev/null + find . -maxdepth 1 -type d | while read pkg; do + pushd "${SVNREPO}/$repo/$pkg" >/dev/null + + [[ ! -e PKGBUILD ]] && { + warning "$repo/$pkg is not a package" + continue + } + + unset pkgbase pkgname + source PKGBUILD + pkgbase=${pkgbase:-$pkgname} + + echo "${pkgbase}-${pkgver}-${pkgrel}${SRCEXT}" >> "${WORKDIR}/expected-src-pkgs" + + # Skip already sourceballed + [[ -e "${SRCPKGDEST}/${pkgbase}-${pkgver}-${pkgrel}${SRCEXT}" ]] && \ + continue + + makepkg --allsource --ignorearch -c >/dev/null 2>&1 + + [[ $? -ne 0 ]] && \ + failedpkgs[${#failedpkgs[*]}]="${pkgbase}-${pkgver}-${pkgrel}${SRCEXT}" + + done + popd >/dev/null +done + +# Cleanup old source packages +cat "${WORKDIR}/expected-src-pkgs" | sort -u > "${WORKDIR}/expected-src-pkgs.sort" +cat "${WORKDIR}/available-src-pkgs" | sort -u > "${WORKDIR}/available-src-pkgs.sort" +old_pkgs=($(comm -23 "${WORKDIR}/available-src-pkgs.sort" "${WORKDIR}/expected-src-pkgs.sort")) + +if [ ${#old_pkgs[@]} -ge 1 ]; then + msg "Removing old source packages..." + ${SOURCE_CLEANUP_DRYRUN} && warning 'dry run mode is active' + for old_pkg in ${old_pkgs[@]}; do + msg2 "${old_pkg}" + if ! ${SOURCE_CLEANUP_DRYRUN}; then + mv "$ARCH_BASE/${SRCPOOL}/${old_pkg}" "${SOURCE_CLEANUP_DESTDIR}" + touch "${SOURCE_CLEANUP_DESTDIR}/${old_pkg}" + fi + done +fi + +old_pkgs=($(find ${SOURCE_CLEANUP_DESTDIR} -type f -name "*${SRCEXT}" -mtime +${SOURCE_CLEANUP_KEEP} -printf '%f\n')) +if [ ${#old_pkgs[@]} -ge 1 ]; then + msg "Removing old source packages from the cleanup directory..." + for old_pkg in ${old_pkgs[@]}; do + msg2 "${old_pkg}" + ${SOURCE_CLEANUP_DRYRUN} || rm -f "${SOURCE_CLEANUP_DESTDIR}/${old_pkg}" + done +fi + +msg "Failed" +for _fail in ${failedpkgs[@]}; do + msg2 "$_fail" +done + + +script_unlock + diff --git a/db-functions b/db-functions index e2731cb..b900669 100644 --- a/db-functions +++ b/db-functions @@ -1,7 +1,5 @@ #!/bin/bash -. config - # Some PKGBUILDs need CARCH to be set CARCH="x86_64" @@ -18,7 +16,7 @@ restore_umask () { } # set up general environment -WORKDIR=$(mktemp -d ${TMPDIR}/$(basename $0).XXXXXXXXXX) +WORKDIR=$(mktemp -d /tmp/$(basename $0).XXXXXXXXXX) LOCKS=() # check if messages are to be printed using color -- cgit v1.2.2
PackageVersionCollectionDescription