summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorLuke Shumaker <lukeshu@parabola.nu>2018-10-07 19:30:50 -0400
committerLuke Shumaker <lukeshu@parabola.nu>2018-10-07 19:34:50 -0400
commit8c51bc0247f6d69e50e8597aaf31902c4cb3ea71 (patch)
tree3fb8da1c09032b2721ff98934c08f89c005020e4
parent24158f712e9b808366c4f6eeeb125ae3fd1cb942 (diff)
Add from Parabola: db-import
If you're seeing this in `git blame`, it chose to follow the wrong ancestor of a merge commit.
-rw-r--r--README.md21
-rw-r--r--config2
-rwxr-xr-xconfig-list-mirrors-archlinux61
-rwxr-xr-xconfig-list-mirrors-archlinuxarm67
-rwxr-xr-xconfig-list-tags80
-rw-r--r--config.local.archlinux3220
-rw-r--r--config.local.archlinuxarm27
-rw-r--r--config.local.community24
-rw-r--r--config.local.import-ourarches14
-rw-r--r--config.local.packages24
-rw-r--r--config.local.parabola4
-rw-r--r--db-functions17
-rwxr-xr-xdb-import-any68
-rw-r--r--db-import-any.conf4
-rwxr-xr-xdb-import-archlinuxarm-src125
-rw-r--r--db-import-archlinuxarm.conf15
-rwxr-xr-xdb-import-keyring98
-rw-r--r--db-import-keyring.conf7
-rwxr-xr-xdb-import-pkg360
-rwxr-xr-xdb-update19
-rw-r--r--systemd/db-import-any.service10
-rw-r--r--systemd/db-import-any.timer9
-rw-r--r--systemd/db-import-keyring.service11
-rw-r--r--systemd/db-import-keyring.timer9
-rw-r--r--systemd/db-import@.service12
-rw-r--r--systemd/db-import@.timer8
-rw-r--r--systemd/db-import@archlinux32.timer.d/time.conf2
-rw-r--r--systemd/db-import@archlinuxarm.timer.d/time.conf2
-rw-r--r--systemd/db-import@community.timer.d/time.conf2
-rw-r--r--systemd/db-import@packages.timer.d/time.conf2
-rw-r--r--test/cases/common.bats4
-rw-r--r--test/cases/db-import-any.bats41
-rw-r--r--test/cases/db-import-keyring.bats27
-rw-r--r--test/cases/db-import-pkg.bats260
-rwxr-xr-xtest/cases/db-update.bats21
-rw-r--r--test/fixtures/archlinux32-keyring/PKGBUILD13
-rw-r--r--test/fixtures/archlinuxarm-keyring/PKGBUILD13
-rw-r--r--test/fixtures/pkg-any32/PKGBUILD13
-rw-r--r--test/fixtures/pkg-any64/PKGBUILD13
-rw-r--r--test/fixtures/pkg-simple-c/PKGBUILD13
-rw-r--r--test/fixtures/slavery/PKGBUILD13
-rw-r--r--test/lib/common.bash4
-rw-r--r--test/lib/import.bash195
-rwxr-xr-xtest/lib/runserver39
44 files changed, 1768 insertions, 25 deletions
diff --git a/README.md b/README.md
index 7522376..cb485ba 100644
--- a/README.md
+++ b/README.md
@@ -13,6 +13,10 @@ The executables that you (might) care about are:
│   ├── ftpdir-cleanup
│   ├── integrity-check
│   └── sourceballs
+ ├── db-import-archlinuxarm-src [Parabola only]
+ ├── db-import-any [Parabola only]
+ ├── db-import-pkg [Parabola only]
+ ├── db-import-keyring [Parabola only]
├── db-move
├── db-remove
├── db-repo-add
@@ -51,6 +55,23 @@ Both of these programs do the exact same thing. Parabola developers
decided to write their own from scratch, instead of modifying
`ftpdir-cleanup`. They should eventually be merged.
+But, Parabola doesn't just publish our own packages, we also import
+packages from elsewhere:
+
+ - `db-import-pkg`
+ - `db-import-keyring`
+ - `db-import-archlinuxarm-src`
+
+`db-import-pkg` is the main import program, and it handles importing
+most of the binary packages, as well as the source-packages from Arch
+Linux. `db-import-keyring` is a specialized variant of it for
+importing the "archlinuxarm-keyring" and "archlinux32-keyring"
+packages, since those packages need special treatment to end up on the
+nescessary architectures. `db-import-archlinuxarm-src` is supposed to
+import the source-packages from Arch Linux ARM, but it has been
+defunct for some time now and needs updated. There is currently no
+program to import source-packages from Arch Linux 32.
+
Things that haven't been mentioned yet:
- `cron-jobs/devlist-mailer`
diff --git a/config b/config
index f362b09..d27b112 100644
--- a/config
+++ b/config
@@ -3,6 +3,7 @@
FTP_BASE="/srv/repo/main"
PKGREPOS=()
+ARCHES=()
PKGPOOL=''
SRCPOOL=''
@@ -24,7 +25,6 @@ LOCK_DELAY=10
[ -n "${STAGING:-}" ] || STAGING="$HOME/staging/unknown/staging"
[[ $STAGING = /* ]] || STAGING="$PWD/$STAGING"
export TMPDIR="/tmp"
-ARCHES=(x86_64 i686 armv7h)
DBEXT=".db.tar.gz"
FILESEXT=".files.tar.gz"
SRCEXT=".src.tar.gz"
diff --git a/config-list-mirrors-archlinux b/config-list-mirrors-archlinux
new file mode 100755
index 0000000..e59ea2c
--- /dev/null
+++ b/config-list-mirrors-archlinux
@@ -0,0 +1,61 @@
+#!/usr/bin/env ruby
+# 2013, 2016, 2018 Luke Shumaker
+
+require 'json'
+require 'net/http'
+require 'optparse'
+require 'set'
+devnull = open("/dev/null")
+
+requires = Set.new()
+begin
+ OptionParser.new do |parser|
+ parser.on("--require=DIR", "Optional directory that the mirror needs ('archive', 'iso', 'other', or 'sources')") do |dir|
+ requires.add(dir)
+ end
+ parser.on("-h", "--help", "Display this help") do
+ puts parser
+ exit
+ end
+ end.parse!
+rescue OptionParser::ParseError => e
+ STDERR.puts "#{$0}: #{e}"
+ STDERR.puts "Try '#{$0} --help' for more information."
+ exit 2
+end
+unless ARGV.empty?
+ STDERR.puts "#{$0}: extra argument '#{ARGV[0]}'"
+ STDERR.puts "Try '#{$0} --help' for more information."
+ exit 2
+end
+
+data = JSON::parse(Net::HTTP.get(URI("https://www.archlinux.org/mirrors/status/tier/1/json/")))
+
+if data["version"] != 3
+ print "Data format version != 3"
+ exit 1
+end
+
+# Filter out URLs with incomplete information
+urls = data["urls"].select{|a| a.none?{|k,v|v.nil?}}
+
+# Filter based on our criteria
+rsync_urls = urls.select{|a| a["protocol"]=="rsync"}
+if requires.include?("iso")
+ rsync_urls = rsync_urls.select{|a| a["isos"]==true}
+ requires.delete("iso")
+end
+unless requires.empty?
+ listing_procs = {}
+ rsync_urls.each do |a|
+ # trailing slash is important; ensure we have one
+ listing_procs[a["url"]] ||= IO.popen(["rsync", "--no-motd", "--list-only", "--", a["url"]+"/"], :err => devnull)
+ end
+ listings = {}
+ listing_procs.map.each do |url,listing_proc|
+ listings[url] = listing_proc.read.lines.map{|line|line.chomp.sub(/.*\s/,'')}
+ listing_proc.close
+ end
+ rsync_urls = rsync_urls.select{|a|requires.all?{|dir|listings[a["url"]].include?(dir)}}
+end
+puts JSON.dump(rsync_urls)
diff --git a/config-list-mirrors-archlinuxarm b/config-list-mirrors-archlinuxarm
new file mode 100755
index 0000000..63f1814
--- /dev/null
+++ b/config-list-mirrors-archlinuxarm
@@ -0,0 +1,67 @@
+#!/usr/bin/env ruby
+# 2018 Luke Shumaker
+
+require 'json'
+require 'net/http'
+require 'optparse'
+require 'set'
+devnull = open("/dev/null")
+
+begin
+ OptionParser.new do |parser|
+ parser.on("-h", "--help", "Display this help") do
+ puts parser
+ exit
+ end
+ end.parse!
+rescue OptionParser::ParseError => e
+ STDERR.puts "#{$0}: #{e}"
+ STDERR.puts "Try '#{$0} --help' for more information."
+ exit 2
+end
+unless ARGV.empty?
+ STDERR.puts "#{$0}: extra argument '#{ARGV[0]}'"
+ STDERR.puts "Try '#{$0} --help' for more information."
+ exit 2
+end
+
+# Get a listing of hostnames that have HTTP Arch Linux ARM mirrors
+data = JSON::parse(Net::HTTP.post(
+ URI("https://archlinuxarm.org/data/mirrors/list"), "", {
+ 'Referer' => 'https://archlinuxarm.org/about/mirrors',
+ 'X-Requested-With' => 'XMLHttpRequest',
+ }).body)
+
+hosts = data["data"].map{|m|m[2]}
+
+# Get a listing of rsync URLs for each host
+modulelist_procs = {}
+hosts.each do |host|
+ modulelist_procs[host] ||= IO.popen(["timeout", "2s", "rsync", "--password-file", "/dev/null", "#{host}::"], :err => devnull)
+end
+rsync_urls = Set.new()
+modulelist_procs.each do |host,modulelist_proc|
+ modules = modulelist_proc.read.lines.map{|line| line.chomp.sub(/\s.*/, '') }
+ modulelist_proc.close
+ rsync_urls.merge(modules.map{|m| "rsync://#{host}/#{m}/"})
+end
+
+# Get a listing of the top-level files in each
+listing_procs = {}
+rsync_urls.each do |url|
+ listing_procs[url] ||= IO.popen(["rsync", "--password-file", "/dev/null", "--no-motd", "--list-only", "--", url], :err => devnull)
+end
+listings = {}
+listing_procs.map.each do |url,listing_proc|
+ listings[url] = listing_proc.read.lines.map{|line|line.chomp.sub(/.*\s/,'')}
+ listing_proc.close
+end
+
+# Filter for listings that look like Arch Linux ARM mirrors
+requires = ['aarch64', 'arm', 'armv6h', 'armv7h', 'os']
+rsync_urls = rsync_urls.select{|url| requires.all?{|file|listings[url].include?(file)} }
+
+# Print the results
+rsync_urls.sort.each do |url|
+ puts url
+end
diff --git a/config-list-tags b/config-list-tags
new file mode 100755
index 0000000..679c0ea
--- /dev/null
+++ b/config-list-tags
@@ -0,0 +1,80 @@
+#!/bin/bash
+
+set -eE -o pipefail
+shopt -s extglob globstar nullglob
+source "$(librelib messages)"
+setup_traps
+
+# usage: fetch_dbs <from> <into>
+fetch_dbs() {
+ rsync --no-motd -mrtLH --no-p \
+ --include="*/" \
+ --include="*.db" \
+ --exclude="*" \
+ --delete-after \
+ "$1" "$2"
+}
+
+list_found() {
+ local ret=0
+ local dbfile repo re arch tag
+ while read -r -d '' dbfile; do
+ repo=${dbfile##*/}
+ repo=${repo%.db}
+ re=$(repo=$repo arch='(.*)' envsubst '$repo $arch' <<<"$ARCHPATH")
+ arch=$(sed -rn "s,^$re\$,\1,p" <<<"${dbfile%/*}")
+ if [[ -z $arch ]]; then
+ error 'Could not figure out architecture for %q' "$dbfile"
+ ret=1
+ continue
+ fi
+ printf '%s\n' "$repo-$arch"
+ done < <(find "$WORKDIR" -name '*.db' -printf '%P\0') | sort -u
+ return $ret
+}
+
+list_configured() {
+ printf '%s\n' "${ARCHTAGS[@]}" | sort -u
+}
+
+main() {
+ if [[ $# -ne 0 ]] || [[ -z "$DBSCRIPTS_CONFIG" ]] || ! grep -q ARCHMIRROR -- "$DBSCRIPTS_CONFIG"; then
+ msg 'usage: DBSCRIPTS_CONFIG=/path/to/file %s' "${0##*/}"
+ exit $EXIT_INVALIDARGUMENT
+ fi
+
+ local config_file
+ config_file="$(dirname "$(readlink -e "$0")")/config"
+ source "$config_file"
+
+ WORKDIR=$(mktemp -dt "${0##*/}.XXXXXXXXXX")
+ readonly WORKDIR
+ trap "rm -rf -- ${WORKDIR@Q}" EXIT
+
+ fetch_dbs "${ARCHMIRROR}/" "$WORKDIR"
+
+ y="${GREEN}✓${ALL_OFF}"
+ n="${RED}✗${ALL_OFF}"
+ while IFS='' read -r line; do
+ case "$line" in
+ $'\t\t'*)
+ rmt=$y
+ cfg=$y
+ val=${line#$'\t\t'}
+ ;;
+ $'\t'*)
+ rmt=$n
+ cfg=$y
+ val=${line#$'\t'}
+ ;;
+ *)
+ rmt=$y
+ cfg=$n
+ val=$line
+ ;;
+ esac
+ printf '%s:%s:%s\n' "$val" "$rmt" "$cfg"
+ done < <(comm <(list_found) <(list_configured)) | column -t -s:
+}
+
+main "$@"
diff --git a/config.local.archlinux32 b/config.local.archlinux32
new file mode 100644
index 0000000..1a5f18c
--- /dev/null
+++ b/config.local.archlinux32
@@ -0,0 +1,20 @@
+#!/hint/bash
+
+PKGREPOS=(build-support community community-staging community-testing core extra gnome-unstable kde-unstable staging testing)
+ARCHES=('i686')
+PKGPOOL='pool/archlinux32'
+SRCPOOL='sources/archlinux32'
+
+# "tags" are repo-arch pairs
+ARCHTAGS=("${PKGREPOS[@]/%/-i686}") # leaves out *-i486 and releng-x86_64
+ARCHPATH='$arch/$repo'
+ARCHPKGPOOL='pool'
+
+INHERIT=(
+ 'pool/packages'
+ 'pool/community'
+ 'pool/alarm'
+)
+
+# https://git.archlinux32.org/archlinux32/packages/raw/branch/master/core/pacman-mirrorlist/mirrorlist
+ARCHMIRROR='rsync://mirror.archlinux32.org/archlinux32/'
diff --git a/config.local.archlinuxarm b/config.local.archlinuxarm
new file mode 100644
index 0000000..ec1cb72
--- /dev/null
+++ b/config.local.archlinuxarm
@@ -0,0 +1,27 @@
+#!/hint/bash
+
+PKGREPOS=(alarm aur community core extra)
+ARCHES=('armv7h')
+PKGPOOL='pool/alarm'
+SRCPOOL='sources/alarm'
+
+# "tags" are repo-arch pairs
+ARCHTAGS=("${PKGREPOS[@]/%/-armv7h}")
+ARCHPATH='$arch/$repo'
+
+INHERIT=(
+ 'pool/packages'
+ 'pool/community'
+)
+
+# Of the ALARM mirrors that have public rsync:
+#
+# $ ./config-list-mirrors-archlinuxarm
+# rsync://de3.mirror.archlinuxarm.org/archlinux-arm/
+# rsync://dk.mirror.archlinuxarm.org/archlinuxarm/
+# rsync://ru.mirror.archlinuxarm.org/archlinuxarm/
+# rsync://sg.mirror.archlinuxarm.org/archlinuxarm/
+#
+# 'dk' (Denmark) has the best ping time from
+# winston.parabola.nu (Iceland).
+ARCHMIRROR='rsync://dk.mirror.archlinuxarm.org/archlinuxarm/'
diff --git a/config.local.community b/config.local.community
index 5d61b5e..cabfa5b 100644
--- a/config.local.community
+++ b/config.local.community
@@ -1,13 +1,23 @@
#!/hint/bash
PKGREPOS=('community' 'community-testing' 'community-staging' 'multilib' 'multilib-testing' 'multilib-staging')
+ARCHES=('x86_64')
PKGPOOL='pool/community'
SRCPOOL='sources/community'
-SVNREPO='file:///srv/repos/svn-community/svn'
-SVNUSER='svn-community'
-TESTING_REPO='community-testing'
-STABLE_REPOS=('community')
-CLEANUP_DESTDIR="/srv/repos/svn-community/package-cleanup"
-SOURCE_CLEANUP_DESTDIR="/srv/repos/svn-community/source-cleanup"
-TMPDIR="/srv/repos/svn-community/tmp"
+# "tags" are repo-arch pairs
+ARCHTAGS=({community,multilib}{,-testing,-staging}-x86_64)
+ARCHPATH='$repo/os/$arch'
+ARCHPKGPOOL='pool/community'
+ARCHSRCPOOL='sources/community'
+
+# Of the few tier-1 mirrors with the 'sources/' folder:
+#
+# $ ./config-list-mirrors-archlinux --require=sources | jq -r '.[].url'
+# rsync://archlinux.c3sl.ufpr.br/archlinux/
+# rsync://mirrors.kernel.org/archlinux/
+# rsync://ftp.acc.umu.se/mirror/archlinux/
+#
+# ftp.acc.umu.se (Sweden) has (by far) the best ping time from
+# winston.parabola.nu (Iceland).
+ARCHMIRROR='rsync://ftp.acc.umu.se/mirror/archlinux/'
diff --git a/config.local.import-ourarches b/config.local.import-ourarches
new file mode 100644
index 0000000..958ac6d
--- /dev/null
+++ b/config.local.import-ourarches
@@ -0,0 +1,14 @@
+#!/hint/bash
+
+PKGREPOS=(
+ # packages
+ core extra testing staging kde-unstable gnome-unstable
+ # community
+ community{,-testing,-staging}
+ multilib{,-testing,-staging}
+)
+# ARCHES should list arches that ./config.local.parabola has but
+# there's no Arch Linux upstream for.
+ARCHES=(ppc64le)
+PKGPOOL='pool/import-ourarches'
+SRCPOOL='sources/import-ourarches'
diff --git a/config.local.packages b/config.local.packages
index 34aab35..403fc9c 100644
--- a/config.local.packages
+++ b/config.local.packages
@@ -1,13 +1,23 @@
#!/hint/bash
PKGREPOS=('core' 'extra' 'testing' 'staging' 'kde-unstable' 'gnome-unstable')
+ARCHES=('x86_64')
PKGPOOL='pool/packages'
SRCPOOL='sources/packages'
-SVNREPO='file:///srv/repos/svn-packages/svn'
-SVNUSER='svn-packages'
-TESTING_REPO='testing'
-STABLE_REPOS=('core' 'extra')
-CLEANUP_DESTDIR="/srv/repos/svn-packages/package-cleanup"
-SOURCE_CLEANUP_DESTDIR="/srv/repos/svn-packages/source-cleanup"
-TMPDIR="/srv/repos/svn-packages/tmp"
+# "tags" are repo-arch pairs
+ARCHTAGS=("${PKGREPOS[@]/%/-x86_64}")
+ARCHPATH='$repo/os/$arch'
+ARCHPKGPOOL='pool/packages'
+ARCHSRCPOOL='sources/packages'
+
+# Of the few tier-1 mirrors with the 'sources/' folder:
+#
+# $ ./config-list-mirrors-archlinux --require=sources | jq -r '.[].url'
+# rsync://archlinux.c3sl.ufpr.br/archlinux/
+# rsync://mirrors.kernel.org/archlinux/
+# rsync://ftp.acc.umu.se/mirror/archlinux/
+#
+# ftp.acc.umu.se (Sweden) has (by far) the best ping time from
+# winston.parabola.nu (Iceland).
+ARCHMIRROR='rsync://ftp.acc.umu.se/mirror/archlinux/'
diff --git a/config.local.parabola b/config.local.parabola
index 424aff6..441e00a 100644
--- a/config.local.parabola
+++ b/config.local.parabola
@@ -1,9 +1,6 @@
#!/hint/bash
PKGREPOS=(
- 'core' 'testing' 'extra' 'community' 'multilib' 'multilib-testing'
- 'build-support'
-
libre{,-multilib}{,-testing}
pcr{,-multilib}{,-testing}
@@ -16,5 +13,6 @@ PKGREPOS=(
"~$USER"
)
+ARCHES=(x86_64 i686 armv7h ppc64le)
PKGPOOL='pool/parabola'
SRCPOOL='sources/parabola'
diff --git a/db-functions b/db-functions
index 75dedad..0792757 100644
--- a/db-functions
+++ b/db-functions
@@ -34,6 +34,23 @@ mv_acl() {
rm -f "$1"
}
+# just like mv -f, but won't break relative symlinks
+mv_ln() {
+ if [[ -L $1 ]]; then
+ if [[ -d $2 ]]; then
+ set -- "$1" "$2/${1##*/}"
+ fi
+ if [[ ! $1 -ef $2 ]]; then
+ local target
+ target=$(readlink -f -- "$1") || return 1
+ ln -srfT -- "$target" "$2" || return 1
+ fi
+ rm -f -- "$1"
+ else
+ mv -f -- "$1" "$2"
+ fi
+}
+
# set up general environment
WORKDIR=$(mktemp -dt "${0##*/}.XXXXXXXXXX")
LOCKS=()
diff --git a/db-import-any b/db-import-any
new file mode 100755
index 0000000..7827443
--- /dev/null
+++ b/db-import-any
@@ -0,0 +1,68 @@
+#!/bin/bash
+# Releases 'any' packages from Arch arches to our arches
+
+set -eu -o pipefail
+source "$(dirname "$(readlink -e "$0")")/config"
+source "$(dirname "$(readlink -e "$0")")/db-import-any.conf"
+source "$(librelib messages)"
+setup_traps
+
+# usage: expac_file <file.db> <expac_args>
+#
+# Uses the ${WORKDIR} global
+expac_file() {
+ local dbfile=$1
+ local args=("${@:2}")
+
+ local reponame=${dbfile##*/}
+ reponame=${reponame%%.*}
+
+ mkdir -p -- "${WORKDIR}/expac/root"
+ cat >"${WORKDIR}/expac/pacman.conf" <<-EOT
+ [options]
+ RootDir = ${WORKDIR}/expac/root
+ DBPath = ${WORKDIR}/expac/root
+
+ [${reponame}]
+ Server = file://$(realpath --no-symlinks -- "${dbfile%/*}")
+ EOT
+
+ fakeroot pacman --config="${WORKDIR}/expac/pacman.conf" -Syy >/dev/null
+ # expac exits with non-zero on emtpy databases, so ignore errors
+ expac --config="${WORKDIR}/expac/pacman.conf" --sync "${args[@]}" || true
+}
+
+db_list_any_pkgfiles() {
+ local dbfile="$1"
+ expac_file "$dbfile" '%a %f' | awk '$1 == "any" { print $2 }'
+}
+
+main() {
+ WORKDIR=$(mktemp -dt "${0##*/}.XXXXXXXXXX")
+ readonly WORKDIR
+ trap "rm -rf -- ${WORKDIR@Q}" EXIT
+
+ local repo arch
+ for repo in "${PKGREPOS[@]}"; do
+ msg "Processing %s..." "${repo}"
+ mkdir -p -- "${WORKDIR}/staging/${repo}"
+ # Look for arch=(any) packages that exist in
+ # ${BASEARCH} for this repo but is missing from one or
+ # more of ${ARCHES[@]}.
+ db_list_any_pkgfiles "${FTP_BASE}/${repo}/os/${BASEARCH}/${repo}.db" > "${WORKDIR}/base.txt"
+ for arch in "${ARCHES[@]}"; do
+ [[ $arch != "$BASEARCH" ]] || continue
+ db_list_any_pkgfiles "${FTP_BASE}/${repo}/os/${arch}/${repo}.db" > "${WORKDIR}/arch.txt"
+ comm -23 "${WORKDIR}/base.txt" "${WORKDIR}/arch.txt"
+ done \
+ | sort -u \
+ | xargs -d '\n' -r -n1 -- printf '%s/%s\n' "${FTP_BASE}/${repo}/os/${BASEARCH}" \
+ | sed 's/.*/&\n&.sig/' \
+ | xargs -d '\n' -r -- ln -srv -t "${WORKDIR}/staging/${repo}" --
+ done
+
+ msg "Running db-update..."
+ STAGING=${WORKDIR}/staging db-update
+}
+
+main "$@"
diff --git a/db-import-any.conf b/db-import-any.conf
new file mode 100644
index 0000000..65fc58e
--- /dev/null
+++ b/db-import-any.conf
@@ -0,0 +1,4 @@
+#!/hint/bash
+
+# The architecture to compare with
+BASEARCH='x86_64'
diff --git a/db-import-archlinuxarm-src b/db-import-archlinuxarm-src
new file mode 100755
index 0000000..b0c1181
--- /dev/null
+++ b/db-import-archlinuxarm-src
@@ -0,0 +1,125 @@
+#!/bin/bash
+
+set -e
+
+source "$(dirname "$(readlink -e "$0")")/config"
+source "$(dirname "$(readlink -e "$0")")/db-import-archlinuxarm.conf"
+
+# Steps
+# * Sync abs
+# * Download blacklist.txt
+# * Sync abslibre from abs excluding from blacklist
+# * Create repo.abs.tar.gz tarballs
+
+function sync_abs() {
+ # sync from ${REPO}.abs.tar.gz files in ALARM mirrors
+ local TMP_SUBDIR="${tmpdir}/abs-alarm/"
+ rm -rf -- "$TMP_SUBDIR" "$ABS_ROOT"
+ mkdir -- "$TMP_SUBDIR"
+ for ARCH in armv7h; do
+ for REPO in "${ARCHTAGS[@]%-*}"; do
+ rsync ${SYNCARGS} \
+ rsync://${ABS_SERVER}/${ARCH}/${REPO}/${REPO}.abs.tar.gz \
+ "$TMP_SUBDIR" || return $?
+ mkdir -p -- "${ABS_ROOT}/${ARCH}"
+ bsdtar xf "${TMP_SUBDIR}/${REPO}.abs.tar.gz" \
+ -C "${ABS_ROOT}/${ARCH}"
+ done
+ done
+
+ # add changes by ALARM since those aren't present in .abs.tar.gz files
+ if ! git -C "$ALARM_ROOT" pull &>/dev/null; then
+ rm -rf -- "$ALARM_ROOT"
+ if ! git clone "$ALARM_GIT" "$ALARM_ROOT"; then
+ printf "[FAILED]\n"
+ return 1
+ fi
+ fi
+ # FIXME: $ALARM_ROOT doesn't separate packages by architecture
+ for ARCH in armv7h; do
+ for REPO in "${ARCHTAGS[@]%-*}"; do
+ rsync -mrtq --no-p --no-o --no-g --exclude='README' \
+ "${ALARM_ROOT}/${REPO}/" \
+ "${ABS_ROOT}/${ARCH}/${REPO}"
+ done
+ done
+
+ # fix some permissions
+ find "${ABS_ROOT}" -type d -print0 | xargs -0 chmod 755
+ find "${ABS_ROOT}" -type f -print0 | xargs -0 chmod 644
+}
+
+function get_blacklist() {
+ libreblacklist update
+ if ! libreblacklist cat | libreblacklist get-pkg | sort -u | sed "s/^/**\//" > ${BLFILE}; then
+ printf "[FAILED]\n"
+ return 1
+ fi
+
+ # Prevent using an empty blacklist
+ [ $(wc -l ${BLFILE} | cut -d " " -f1) -eq 0 ] && return 1
+
+ printf "[OK]\n"
+}
+
+function sync_abs_libre() {
+
+ # Clone ABSLibre git repo
+ rm -rf -- "$tmpdir/abslibre"
+ git clone "$ABSLIBRE_GIT" "$tmpdir/abslibre"
+
+ # Sync from ABS and then sync from ABSLibre
+ printf ":: Syncing ABSLibre...\t"
+ if ! rsync ${SYNCARGS} --delete-excluded --exclude-from=${BLFILE} ${ABS_ROOT} ${ABSLIBRE_ROOT}; then
+ printf "[FAILED]\n"
+ return 1
+ fi
+ for ARCH in armv7h; do
+ if ! rsync -v -mrtq --no-motd --no-p --no-o --no-g --quiet --exclude=.git/ "$tmpdir/abslibre/" ${ABSLIBRE_ROOT}/${ARCH}/; then
+ printf "[FAILED]\n"
+ return 1
+ fi
+ done
+
+ # fix some permissions
+ find "${ABSLIBRE_ROOT}" -type d -print0 | xargs -0 chmod 755
+ find "${ABSLIBRE_ROOT}" -type f -print0 | xargs -0 chmod 644
+
+ printf "[OK]\n"
+}
+
+# Create .abs.tar.gz tarballs
+create_tarballs() {
+ for repo in ${ABSLIBRE_ROOT}/armv7h/*; do
+ baserepo=${repo##*/}
+ arch=$(basename $(dirname $repo))
+
+ # Remove the old one
+ mkdir -p $FTP_BASE/$baserepo/os/$arch/
+ rm -fv $FTP_BASE/$baserepo/os/$arch/$baserepo.abs.tar.gz
+ # Create a new one joining arch and any
+ # Remove the first part of the path (it could be $repo but any isn't hit)
+ include=($repo/*)
+ if [[ -d ${ABSLIBRE_ROOT}/any/${baserepo}/ ]]; then
+ include+=(${ABSLIBRE_ROOT}/any/${baserepo}/*)
+ fi
+ bsdtar -czf $FTP_BASE/$baserepo/os/$arch/$baserepo.abs.tar.gz \
+ -s ":${ABSLIBRE_ROOT}/[a-z0-9_]\+/[a-z]\+::" \
+ "${include[@]}"
+ done
+}
+
+main() {
+ trap 'rm -rf -- "$tmpdir"' EXIT
+ tmpdir=$(mktemp --tmpdir -d "${0##*/}.XXXXXXXXXX")
+
+ BLFILE=${tmpdir}/blacklist.txt
+ mkdir -p -- "$ABSLIBRE_ROOT" "$ABS_ROOT" "$ALARM_ROOT"
+
+ sync_abs
+ get_blacklist
+ sync_abs_libre
+ create_tarballs
+}
+
+main "$@"
diff --git a/db-import-archlinuxarm.conf b/db-import-archlinuxarm.conf
new file mode 100644
index 0000000..c4b0b1c
--- /dev/null
+++ b/db-import-archlinuxarm.conf
@@ -0,0 +1,15 @@
+#!/hint/bash
+
+# NB: I am unsure if the presence or absence of a trailing slash in
+# the _ROOT variables is significant. -- lukeshu
+SYNCARGS='-mrtv --no-motd --delete-after --no-p --no-o --no-g --quiet --exclude .~tmp~/'
+ABSLIBRE_ROOT=/srv/repo/db-import-archlinuxarm-src/abslibre
+ABSLIBRE_GIT=https://git.parabola.nu/abslibre.git/
+ABS_ROOT='/srv/repo/db-import-archlinuxarm-src/abs/'
+ABS_SERVER="${ARCHMIRROR#rsync://}/"
+ALARM_GIT='https://github.com/archlinuxarm/PKGBUILDs'
+ALARM_ROOT='/srv/repo/db-import-archlinuxarm-src/alarm/'
+
+if [[ -n ${DBIMPORT_CONFIG} ]]; then
+ source "${DBIMPORT_CONFIG}"
+fi
diff --git a/db-import-keyring b/db-import-keyring
new file mode 100755
index 0000000..ef37a5a
--- /dev/null
+++ b/db-import-keyring
@@ -0,0 +1,98 @@
+#!/bin/bash
+# Imports to [libre] specific packages from upstream Arch repos.
+#
+# License: GPLv3
+
+set -eu -o pipefail
+shopt -s extglob globstar nullglob
+source "$(librelib messages)"
+setup_traps
+
+# usage: expac_file <file.db> <expac_args>
+#
+# Uses the ${WORKDIR} global
+expac_file() {
+ local dbfile=$1
+ local args=("${@:2}")
+
+ local reponame=${dbfile##*/}
+ reponame=${reponame%%.*}
+
+ mkdir -p -- "${WORKDIR}/expac/root"
+ cat >"${WORKDIR}/expac/pacman.conf" <<-EOT
+ [options]
+ RootDir = ${WORKDIR}/expac/root
+ DBPath = ${WORKDIR}/expac/root
+
+ [${reponame}]
+ Server = file://$(realpath --no-symlinks -- "${dbfile%/*}")
+ EOT
+
+ fakeroot pacman --config="${WORKDIR}/expac/pacman.conf" -Syy >/dev/null
+ # expac exits with non-zero on emtpy databases, so ignore errors
+ expac --config="${WORKDIR}/expac/pacman.conf" --sync "${args[@]}" || true
+}
+
+# usage: get_repo_rsync_dir <upstream> <tag>
+get_repo_rsync_dir() {
+ if ! grep -q ARCHMIRROR -- "${LOCAL_CONFIG%/*}/config.local.$1"; then
+ msg "db-import-keyring.conf: Doesn't look like an upstream: %s" "$1" >&2
+ exit $EXIT_NOTCONFIGURED
+ fi
+ (
+ source "${LOCAL_CONFIG%/*}/config.local.$1" >&2
+ printf '%s\n' "${ARCHMIRROR}/$(repo=${2%-*} arch=${2##*-} envsubst '$repo $arch' <<<"$ARCHPATH")"
+ )
+}
+
+main() {
+ if [[ $# -ne 0 ]]; then
+ msg 'usage: %s' "${0##*/}"
+ exit $EXIT_INVALIDARGUMENT
+ fi
+
+ local config_dir
+ config_dir="$(dirname "$(readlink -e "$0")")"
+ source "${config_dir}/config" # for FTP_BASE
+ source "${config_dir}/db-import-keyring.conf" # for KEYRINGS
+
+ WORKDIR=$(mktemp -dt "${0##*/}.XXXXXXXXXX")
+ readonly WORKDIR
+ trap "rm -rf -- ${WORKDIR@Q}" EXIT
+
+ ##############################################################
+
+ local fromspec upstream tag pkgname repo repo_rsync_dir from_pkgver to_pkgver filename
+ for fromspec in "${KEYRINGS[@]}"; do
+ msg "Processing %s" "$fromspec"
+ IFS=/ read -r upstream tag pkgname <<<"$fromspec"
+ repo_rsync_dir=$(get_repo_rsync_dir "$upstream" "$tag")
+ repo=${tag%-*}
+
+ # Download the upstream database
+ msg2 'Downloading %s' "${repo_rsync_dir}/${repo}.db"
+ mkdir -p -- "${WORKDIR}/dbs/${upstream}"
+ rsync --no-motd -mrtLH --no-p "${repo_rsync_dir}/${repo}.db" "${WORKDIR}/dbs/${upstream}/${tag}.db"
+
+ # Compare it against the libre database
+ from_pkgver=$(expac_file "${WORKDIR}/dbs/${upstream}/${tag}.db" '%v' "$pkgname")
+ to_pkgver=$(expac_file "${FTP_BASE}/libre/os/x86_64/libre.db" '%v' "$pkgname")
+
+ # And download a new keyring if necessary
+ if [[ "$from_pkgver" == "$to_pkgver" ]]; then
+ msg2 "Up to date: %s == %s" "$from_pkgver" "$to_pkgver"
+ else
+ msg2 "Updating: %s != %s" "$from_pkgver" "$to_pkgver"
+ filename=$(expac_file "${WORKDIR}/dbs/${upstream}/${tag}.db" '%f' "$pkgname")
+ mkdir -p -- "${WORKDIR}/staging/libre/"
+ rsync --no-motd -mrtLH --no-p "${repo_rsync_dir}/${filename}"{,.sig} "${WORKDIR}/staging/libre/"
+ fi
+ done
+
+ ##############################################################
+
+ msg "Running db-update..."
+ STAGING=${WORKDIR}/staging db-update
+}
+
+main "$@"
diff --git a/db-import-keyring.conf b/db-import-keyring.conf
new file mode 100644
index 0000000..8e78ff8
--- /dev/null
+++ b/db-import-keyring.conf
@@ -0,0 +1,7 @@
+#!/hint/bash
+
+KEYRINGS=(
+ # Format is "UPSTREAM/TAG/PKGNAME"
+ archlinux32/core-i686/archlinux32-keyring
+ archlinuxarm/core-armv7h/archlinuxarm-keyring
+)
diff --git a/db-import-pkg b/db-import-pkg
new file mode 100755
index 0000000..bff210c
--- /dev/null
+++ b/db-import-pkg
@@ -0,0 +1,360 @@
+#!/bin/bash
+# Imports packages from upstream Arch repos.
+#
+# License: GPLv3
+
+set -e -o pipefail
+shopt -s extglob globstar nullglob
+source "$(librelib messages)"
+setup_traps
+
+indent() {
+ /usr/lib/libretools/chroot/indent ' | '
+}
+
+# usage: expac_file <file.db> <expac_args>
+#
+# Uses the ${WORKDIR} global
+expac_file() {
+ local dbfile=$1
+ local args=("${@:2}")
+
+ local reponame=${dbfile##*/}
+ reponame=${reponame%%.*}
+
+ mkdir -p -- "${WORKDIR}/expac/root"
+ cat >"${WORKDIR}/expac/pacman.conf" <<-EOT
+ [options]
+ RootDir = ${WORKDIR}/expac/root
+ DBPath = ${WORKDIR}/expac/root
+
+ [${reponame}]
+ Server = file://$(realpath --no-symlinks -- "${dbfile%/*}")
+ EOT
+
+ fakeroot pacman --config="${WORKDIR}/expac/pacman.conf" -Syy >/dev/null
+ # expac exits with non-zero on emtpy databases, so ignore errors
+ expac --config="${WORKDIR}/expac/pacman.conf" --sync "${args[@]}" || true
+}
+
+# usage: fetch_dbs <from> <into>
+#
+# Fetch excluding everything but db files
+# TODO: we could be doing without things other than what is in
+# ${ARCHTAGS[@]}
+fetch_dbs() {
+ local extra=()
+ if [[ $arg_verbose = true ]]; then
+ extra+=(-v)
+ fi
+
+ rsync "${extra[@]}" --no-motd -mrtLH --no-p \
+ --include="*/" \
+ --include="*.db" \
+ --exclude="*" \
+ --delete-after \
+ "$1" "$2"
+}
+
+# usage: get_repo_dir <repo> <arch>
+#
+# Prints repo directory path for the given <repo> <arch> combination,
+# relative to the rsync root.
+get_repo_dir() {
+ repo=$1 arch=$2 envsubst '$repo $arch' <<<"$ARCHPATH"
+}
+
+# usage: db_list_pkgs <path-to-db>
+#
+# Prints a list of packages within a given <path-to-db>, one-per-line,
+# in the format:
+#
+# pkgname [epoch:]pkgver-pkgrel
+db_list_pkgs() {
+ expac_file "$1" '%n %v' | sort -u
+}
+
+# usage: filter_blacklisted <FULL_LIST >FILTERED_LIST
+#
+# Given a list of packages in the format:
+#
+# pkgname [epoch:]pkgver-pkgrel
+#
+# filter out all of the packages named in blacklist.txt.
+filter_blacklisted() {
+ sort -u | join -v1 \
+ - \
+ <(libreblacklist cat | libreblacklist get-pkg | sort -u)
+}
+
+# usage: sync_pool <from> <path-to-whitelist> <into>
+#
+# Sync excluding everything but whitelist
+sync_pool() {
+ local -r _from=$1 _whitelist=$2 _into=$3
+
+ local extra=()
+ if [[ $arg_verbose = true ]]; then
+ extra+=(-v)
+ fi
+
+ mkdir -p -- "$_into"
+ msg2 "Retrieving up to %d files from %s pool" \
+ "$(wc -l < "$_whitelist")" \
+ "$(basename "$_into")"
+
+ # *Don't delete-after*, this is the job of
+ # cleanup scripts. It will remove our packages too
+ rsync "${extra[@]}" --no-motd -rtlH --no-t \
+ --delay-updates \
+ --safe-links \
+ --include-from="$_whitelist" \
+ --exclude="*" \
+ "$_from" \
+ "$_into"
+}
+
+# Main function. Process the databases and get the libre packages
+# Outline:
+# 1. Fetch package info
+# * Get blacklist.txt
+# * Get repo.db from an Arch-like repo
+# 2. Figure out what we want
+# * Generate textfiles describing the current repo state, (using
+# blacklist.txt) the desired repo state, and how to get from one
+# to the other.
+# 3. Fetch the packages we want
+# * Symlink to files with the same name in INHERIT pools
+# * sync_pool to download the others
+# 4. Modify the repos
+# * db-move
+# * db-update
+# * db-remove
+#
+# Files:
+# (misc)
+# - ${WORKDIR}/expac/ : Scratch directory for expac_file()
+# (download)
+# - ${WORKDIR}/rsync/ : Where we download '.db' files to
+# - ${WORKDIR}/staging/${repo} : Where we download packages to
+# - ${WORKDIR}/staging/${SRCPOOL} : Where we download sources to
+# (analysis)
+# - ${FTP_BASE}/${INHERIT} : Where we look for duplicate files
+# - ${FTP_BASE}/.../${repo}.db : Where we generate ${WORKDIR}/old/ from
+# - ${WORKDIR}/old/ : .txt files describing the way the repos are
+# - ${WORKDIR}/new/ : .txt files describing the way we want them to be
+# - ${WORKDIR}/dif/ : .txt files describing how to make it happen
+# - ${WORKDIR}/${tag}.pkg.whitelist : List of package filenames to download
+# - ${WORKDIR}/all.src.whitelist : Glob list of source-package filenames to download
+# (release)
+# - ${WORKDIR}/staging/ : STAGING= directory for db-update
+main() {
+ ##############################################################
+ # 0. Initialization #
+ ##############################################################
+
+ # Run as `V=true db-import-pkg` to get verbose output
+ declare -r arg_verbose="$V"
+
+ # Print usage message
+ if [[ $# -ne 0 ]] || [[ -z "$DBSCRIPTS_CONFIG" ]] || ! grep -q ARCHMIRROR -- "$DBSCRIPTS_CONFIG"; then
+ msg 'usage: [V=true] DBSCRIPTS_CONFIG=/path/to/file %s' "${0##*/}"
+ exit $EXIT_INVALIDARGUMENT
+ fi
+
+ local config_file
+ config_file="$(dirname "$(readlink -e "$0")")/config"
+ source "$config_file"
+
+ local ret=0 varname varref
+ for varname in PKGEXTS FTP_BASE PKGPOOL SRCPOOL; do
+ if [[ -z ${!varname:-} ]] || is_array "$varname"; then
+ print "Configure '%s' as a non-empty string in %q (or %q):" "$varname" "$config_file" "$LOCAL_CONFIG"
+ ret=$EXIT_NOTCONFIGURED
+ fi
+ done
+ for varname in ARCHMIRROR ARCHPATH; do # optional: ARCHPKGPOOL ARCHSRCPOOL
+ if [[ -z ${!varname:-} ]] || is_array "$varname"; then
+ print "Configure '%s' as a non-empty string in DBSCRIPTS_CONFIG=%q (did you set DBSCRIPTS_CONFIG correctly?):" "$varname" "$LOCAL_CONFIG"
+ ret=$EXIT_NOTCONFIGURED
+ fi
+ done
+ for varname in ARCHTAGS; do # optional: INHERIT
+ declare -n varref="$varname"
+ if (( ${#varref[*]} == 0 )) || ! is_array "$varname"; then
+ print "Configure '%s' as a non-empty array in DBSCRIPTS_CONFIG=%q (did you set DBSCRIPTS_CONFIG correctly?):" "$varname" "$LOCAL_CONFIG"
+ ret=$EXIT_NOTCONFIGURED
+ fi
+ done
+
+ WORKDIR=$(mktemp -dt "${0##*/}.XXXXXXXXXX")
+ readonly WORKDIR
+ trap "rm -rf -- ${WORKDIR@Q}" EXIT
+
+ ##############################################################
+ # 1. Fetch package info #
+ ##############################################################
+
+ # Get the blacklisted packages
+ libreblacklist update
+
+ # Sync the repos databases
+ msg 'Downloading .db and .files files to import'
+ mkdir "${WORKDIR}/rsync"
+ fetch_dbs "${ARCHMIRROR}/" "${WORKDIR}/rsync"
+
+ ##############################################################
+ # 2. Figure out what we want #
+ ##############################################################
+
+ mkdir "${WORKDIR}"/{old,new,dif}
+ local _tag _repo _arch db_file
+ for _tag in "${ARCHTAGS[@]}"; do
+ _repo=${_tag%-*}
+ _arch=${_tag##*-}
+ # FIXME: this should use db-functions to lock the
+ # repos while we read them.
+ db_file="${FTP_BASE}/${_repo}/os/${_arch}/${_repo}.db"
+ db_list_pkgs "$db_file" > "${WORKDIR}/old/${_tag}.txt"
+
+ db_file="${WORKDIR}/rsync/$(get_repo_dir "${_repo}" "${_arch}")/${_repo}.db"
+ db_list_pkgs "$db_file" | filter_blacklisted > "${WORKDIR}/new/${_tag}.txt"
+ done
+
+ # We now have ${WORKDIR}/old/ describing the way the repos
+ # are, and ${WORKDIR}/new/ describing the way we want them to
+ # be. We now create ${WORKDIR}/dif/ describing how to get
+ # from point A to point B.
+ cat "${WORKDIR}"/old/*-*.txt | sort -u > "${WORKDIR}/old/all.txt"
+ # db-move <repo-from> <repo-to> <pkgname|pkgbase> ...
+ #
+ # db-move doesn't allow us to limit the operation to a
+ # specific arch, but the DBSCRIPTS_CONFIG will limit what
+ # arches it applies to, and we currently only import 1 arch
+ # from each upstream.
+ local tag_from tag_to
+ for tag_from in "${ARCHTAGS[@]}"; do
+ arch_from=${tag_from##*-}
+ for tag_to in "${ARCHTAGS[@]}"; do
+ arch_to=${tag_to##*-}
+ [[ $tag_from != $tag_to ]] || continue
+ [[ $arch_from == $arch_to ]] || continue
+
+ comm -12 \
+ "${WORKDIR}/old/${tag_from}.txt" \
+ "${WORKDIR}/new/${tag_to}.txt" \
+ >> "${WORKDIR}/dif/move:${tag_from}:${tag_to}.txt"
+ done
+ done
+ # db-update
+ local tag
+ for tag in "${ARCHTAGS[@]}"; do
+ comm -13 \
+ "${WORKDIR}/old/all.txt" \
+ "${WORKDIR}/new/${tag}.txt" \
+ > "${WORKDIR}/dif/update:${tag}.txt"
+ done
+ # db-remove <repo> <arch> <pkgname|pkgbase> ...
+ for tag in "${ARCHTAGS[@]}"; do
+ # pkgnames (that need to leave this tag) AND (haven't
+ # already been removed from this tag by db-move).
+ comm -23 \
+ <(cut -d' ' -f1 -- "${WORKDIR}/old/${tag}.txt" | sort -u) \
+ <(cut -d' ' -f1 -- "${WORKDIR}/new/${tag}.txt" "${WORKDIR}/dif/move:${tag}":*.txt | sort -u) \
+ > "${WORKDIR}/dif/remove:${tag}.txt"
+ done
+
+ ##############################################################
+ # 3. Fetch the packages we want #
+ ##############################################################
+
+ # For some packages, "fetch" means to create a symlink to a
+ # pool we INHERIT from. For others, it means to actually
+ # download it from arg_upstream with rsync.
+ for tag in "${ARCHTAGS[@]}";do
+ msg "Processing %s" "$tag"
+ repo=${tag%-*}
+ arch=${tag##*-}
+ mkdir -p -- "${WORKDIR}/staging/${repo}"
+
+ local pkgname filename pgpsig
+ while read -r pkgname filename pgpsig; do
+ local pool staged=false
+ for pool in "$PKGPOOL" "${INHERIT[@]}"; do
+ filepath=("${FTP_BASE}/${pool}/${filename}")
+ if [[ -f $filepath && ! -h $filepath ]]; then
+ ln -srT -- "$filepath" "${WORKDIR}/staging/${repo}/${filename}"
+ ln -srT -- "$filepath".sig "${WORKDIR}/staging/${repo}/${filename}".sig
+ staged=true
+ break
+ fi
+ done
+ if ! $staged; then
+ printf '%s\n' "$filename"{,.sig} >> "${WORKDIR}/${tag}.pkg.whitelist"
+ printf '%s\n' "${filename%$PKGEXTS}*.src.tar*" >> "${WORKDIR}/all.src.whitelist"
+ fi
+ done < <(
+ mapfile -t pkgnames < <(cut -d' ' -f1 <"${WORKDIR}/dif/update:${tag}.txt")
+ db_file="${WORKDIR}/rsync/$(get_repo_dir "${repo}" "${arch}")/${repo}.db"
+ if (( ${#pkgnames[@]} > 0 )); then
+ expac_file "$db_file" '%n %f %g' "${pkgnames[@]}"
+ fi
+ )
+ if [[ -f "${WORKDIR}/${tag}.pkg.whitelist" ]]; then
+ sync_pool \
+ "${ARCHMIRROR}/${ARCHPKGPOOL:-$(get_repo_dir "${repo}" "${arch}")}/" \
+ "${WORKDIR}/${tag}.pkg.whitelist" \
+ "${WORKDIR}/staging/${repo}/"
+ fi
+ done
+ if [[ -n ${ARCHSRCPOOL:-} && -f "${WORKDIR}/all.src.whitelist" ]]; then
+ sync_pool \
+ "${ARCHMIRROR}/${ARCHSRCPOOL}/" \
+ "${WORKDIR}/all.src.whitelist" \
+ "${WORKDIR}/staging/${SRCPOOL}/"
+ fi
+
+ ##############################################################
+ # 4. Modify the repos #
+ ##############################################################
+
+ msg "Modifying the actual repos"
+
+ # db-move
+ msg2 'Step 1 of 3: db-move'
+ for tag_from in "${ARCHTAGS[@]}"; do
+ repo_from=${tag_from%-*}
+ arch_from=${tag_from##*-}
+ for tag_to in "${ARCHTAGS[@]}"; do
+ repo_to=${tag_to%-*}
+ arch_to=${tag_to##*-}
+ [[ $tag_from != $tag_to ]] || continue
+ [[ $arch_from == $arch_to ]] || continue
+
+ plain '%s -> %s' "$tag_from" "$tag_to"
+ < "${WORKDIR}/dif/move:${tag_from}:${tag_to}.txt" \
+ cut -d' ' -f1 | \
+ sed 's/^/pkgname=/' | \
+ xargs -r -d $'\n' db-move "$repo_from" "$repo_to" |& \
+ indent
+ done
+ done
+ # db-update
+ msg2 'Step 2 of 3: db-update'
+ STAGING=${WORKDIR}/staging db-update |& indent
+ # db-remove
+ msg2 'Step 3 of 3: db-remove'
+ for tag in "${ARCHTAGS[@]}"; do
+ repo=${tag%-*}
+ arch=${tag##*-}
+
+ plain '%s' "$tag"
+ < "${WORKDIR}/dif/remove:${tag}.txt" \
+ cut -d' ' -f1 | \
+ sed 's/^/pkgname=/' | \
+ xargs -r -d $'\n' db-remove "$repo" "$arch" |& \
+ indent
+ done
+}
+
+main "$@"
diff --git a/db-update b/db-update
index bd1e4e4..b6b4d0d 100755
--- a/db-update
+++ b/db-update
@@ -51,6 +51,15 @@ for repo in "${repos[@]}"; do
if (( $? == 0 )); then
for pkg in "${pkgs[@]}"; do
if [[ -h ${pkg} ]]; then
+ # As a special hack for
+ # 1. inheriting arch=(any) packages between db-import
+ # upstreams, and
+ # 2. re-adding previously removed binaries that have not been
+ # cleaned up yet,
+ # allow symlinks that point to packages under ${FTP_BASE}/pool/.
+ if [[ -f ${pkg} && "$(readlink -f -- "$pkg")" = "${FTP_BASE}"/pool/*${PKGEXTS} ]]; then
+ continue
+ fi
die "Package %s is a symbolic link" "$repo/${pkg##*/}"
fi
if ! check_pkgfile "${pkg}"; then
@@ -66,7 +75,11 @@ for repo in "${repos[@]}"; do
die "Package %s already exists in another repository" "$repo/${pkg##*/}"
fi
if ! check_packager "${pkg}"; then
- die "Package %s does not have a valid packager" "$repo/${pkg##*/}"
+ if [[ $DBSCRIPTS_CONFIG = *.archlinux32 ]]; then
+ warning "Package %s does not have a valid packager" "$repo/${pkg##*/}"
+ else
+ die "Package %s does not have a valid packager" "$repo/${pkg##*/}"
+ fi
fi
if ! check_buildinfo "${pkg}"; then
die "Package %s does not have a .BUILDINFO file" "$repo/${pkg##*/}"
@@ -94,12 +107,12 @@ for repo in "${repos[@]}"; do
msg2 '%s (%s)' "$pkgfile" "$pkgarch"
# any packages might have been moved by the previous run
if [[ -f ${pkg} ]]; then
- mv "${pkg}" "$FTP_BASE/${PKGPOOL}"
+ mv_ln "${pkg}" "$FTP_BASE/${PKGPOOL}"
fi
ln -s "../../../${PKGPOOL}/${pkgfile}" "$FTP_BASE/$repo/os/${pkgarch}"
# also move signatures
if [[ -f ${pkg}.sig ]]; then
- mv "${pkg}.sig" "$FTP_BASE/${PKGPOOL}"
+ mv_ln "${pkg}.sig" "$FTP_BASE/${PKGPOOL}"
fi
if [[ -f $FTP_BASE/${PKGPOOL}/${pkgfile}.sig ]]; then
ln -s "../../../${PKGPOOL}/${pkgfile}.sig" "$FTP_BASE/$repo/os/${pkgarch}"
diff --git a/systemd/db-import-any.service b/systemd/db-import-any.service
new file mode 100644
index 0000000..e50510c
--- /dev/null
+++ b/systemd/db-import-any.service
@@ -0,0 +1,10 @@
+[Unit]
+Description=db-import-any
+
+[Service]
+Type=oneshot
+User=repo
+Environment=DBSCRIPTS_CONFIG=/etc/dbscripts/config.local.import-ourarches
+ExecStart=/usr/bin/db-import-any
+
+PrivateTmp=true
diff --git a/systemd/db-import-any.timer b/systemd/db-import-any.timer
new file mode 100644
index 0000000..a14e634
--- /dev/null
+++ b/systemd/db-import-any.timer
@@ -0,0 +1,9 @@
+[Unit]
+Description=Daily db-import-any
+
+[Timer]
+OnCalendar=daily
+Persistent=true
+
+[Install]
+WantedBy=timers.target
diff --git a/systemd/db-import-keyring.service b/systemd/db-import-keyring.service
new file mode 100644
index 0000000..96ec7bd
--- /dev/null
+++ b/systemd/db-import-keyring.service
@@ -0,0 +1,11 @@
+[Unit]
+Description=db-import-keyring
+Wants=network-online.target
+After=network-online.target
+
+[Service]
+Type=oneshot
+User=repo
+ExecStart=/usr/bin/db-import-keyring
+
+PrivateTmp=true
diff --git a/systemd/db-import-keyring.timer b/systemd/db-import-keyring.timer
new file mode 100644
index 0000000..3818733
--- /dev/null
+++ b/systemd/db-import-keyring.timer
@@ -0,0 +1,9 @@
+[Unit]
+Description=Daily db-import-keyring
+
+[Timer]
+OnCalendar=daily
+Persistent=true
+
+[Install]
+WantedBy=timers.target
diff --git a/systemd/db-import@.service b/systemd/db-import@.service
new file mode 100644
index 0000000..1509449
--- /dev/null
+++ b/systemd/db-import@.service
@@ -0,0 +1,12 @@
+[Unit]
+Description=db-import %I
+Wants=network-online.target
+After=network-online.target
+
+[Service]
+Type=oneshot
+User=repo
+Environment=DBSCRIPTS_CONFIG=/etc/dbscripts/config.local.%I
+ExecStart=/usr/bin/db-import-pkg
+
+PrivateTmp=true
diff --git a/systemd/db-import@.timer b/systemd/db-import@.timer
new file mode 100644
index 0000000..77570d6
--- /dev/null
+++ b/systemd/db-import@.timer
@@ -0,0 +1,8 @@
+[Unit]
+Description=Daily db-import %I
+
+[Timer]
+Persistent=true
+
+[Install]
+WantedBy=timers.target
diff --git a/systemd/db-import@archlinux32.timer.d/time.conf b/systemd/db-import@archlinux32.timer.d/time.conf
new file mode 100644
index 0000000..028665c
--- /dev/null
+++ b/systemd/db-import@archlinux32.timer.d/time.conf
@@ -0,0 +1,2 @@
+[Timer]
+OnCalendar=*-*-* 08:13:00
diff --git a/systemd/db-import@archlinuxarm.timer.d/time.conf b/systemd/db-import@archlinuxarm.timer.d/time.conf
new file mode 100644
index 0000000..455b0bc
--- /dev/null
+++ b/systemd/db-import@archlinuxarm.timer.d/time.conf
@@ -0,0 +1,2 @@
+[Timer]
+OnCalendar=*-*-* 16:13:00
diff --git a/systemd/db-import@community.timer.d/time.conf b/systemd/db-import@community.timer.d/time.conf
new file mode 100644
index 0000000..3ec5e02
--- /dev/null
+++ b/systemd/db-import@community.timer.d/time.conf
@@ -0,0 +1,2 @@
+[Timer]
+OnCalendar=*-*-* 04:13:00
diff --git a/systemd/db-import@packages.timer.d/time.conf b/systemd/db-import@packages.timer.d/time.conf
new file mode 100644
index 0000000..2561a96
--- /dev/null
+++ b/systemd/db-import@packages.timer.d/time.conf
@@ -0,0 +1,2 @@
+[Timer]
+OnCalendar=*-*-* 00:13:00
diff --git a/test/cases/common.bats b/test/cases/common.bats
index 8b4c3a0..337df48 100644
--- a/test/cases/common.bats
+++ b/test/cases/common.bats
@@ -1,10 +1,10 @@
load ../lib/common
@test "commands display usage message by default" {
- for cmd in db-move db-remove db-repo-add db-repo-remove; do
+ for cmd in db-move db-remove db-repo-add db-repo-remove db-import-pkg; do
echo Testing $cmd
run $cmd
- (( $status == 1 ))
+ (( $status != 0 ))
[[ $output == *'usage: '* ]]
done
}
diff --git a/test/cases/db-import-any.bats b/test/cases/db-import-any.bats
new file mode 100644
index 0000000..e6fc5c6
--- /dev/null
+++ b/test/cases/db-import-any.bats
@@ -0,0 +1,41 @@
+load ../lib/import
+
+@test "import any on Parabola repos" {
+ # Initial run
+ releasePackage libre 'pkg-any-a'
+ STAGING=$STAGING db-update
+ checkPackage libre pkg-any-a 1-1
+ rmdir "$STAGING"
+
+ # Add an arch
+ echo 'ARCHES+=(ppc64le)' >> "$DBSCRIPTS_CONFIG"
+ ARCHES+=(ppc64le)
+ db-init
+
+ # Adding the arch broke checkPackage
+ if checkPackage libre pkg-any-a 1-1; then return 1; fi
+ # But db-import-any fixes it
+ db-import-any
+ checkPackage libre pkg-any-a 1-1
+}
+
+@test "import any on imported repos" {
+ cat >"${TMP}/config.local.import-ourarches" <<-eot
+ source ${DBSCRIPTS_CONFIG@Q}
+ source ../config.local.import-ourarches
+ PKGREPOS=(core testing)
+ eot
+ DBSCRIPTS_CONFIG="${TMP}/config.local.import-ourarches" db-init
+
+ __releaseImportedPackage pkg-any-a x86_64 "$TMP/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz" "$TMP/rsyncd/archlinux/pool/packages"
+ __releaseImportedPackage pkg-simple-c x86_64 "$TMP/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz" "$TMP/rsyncd/archlinux/pool/packages"
+ DBSCRIPTS_CONFIG="${TMP}/config.local.packages" db-import-pkg
+
+ [[ -z $(bsdtar tf "${TMP}/ftp/core/os/ppc64le/core.db") ]]
+
+ DBSCRIPTS_CONFIG="${TMP}/config.local.import-ourarches" db-import-any
+
+ bsdtar tf "${TMP}/ftp/core/os/ppc64le/core.db" pkg-any-a-1-1/desc
+ if bsdtar tf "${TMP}/ftp/core/os/ppc64le/core.db" pkg-simple-c-1-1; then return 1; fi
+ __isLinkTo "${TMP}/ftp/core/os/ppc64le/pkg-any-a-1-1-any.pkg.tar.xz" "${TMP}/ftp/pool/packages/pkg-any-a-1-1-any.pkg.tar.xz"
+}
diff --git a/test/cases/db-import-keyring.bats b/test/cases/db-import-keyring.bats
new file mode 100644
index 0000000..d78dbb4
--- /dev/null
+++ b/test/cases/db-import-keyring.bats
@@ -0,0 +1,27 @@
+load ../lib/import
+
+@test "import keyrings" {
+ echo PKGPOOL='pool/parabola' >>"${TMP}/config.local"
+ mkdir -p "${TMP}/ftp/pool/parabola"
+
+ __releaseImportedPackage archlinux32-keyring i686 "$TMP/rsyncd/archlinux32/i686/core/core.db.tar.gz" "$TMP/rsyncd/archlinux32/pool"
+ __releaseImportedPackage archlinuxarm-keyring armv7h "$TMP/rsyncd/archlinuxarm/armv7h/core/core.db.tar.gz"
+
+ db-import-keyring
+
+ bsdtar tf "$TMP/ftp/libre/os/x86_64/libre.db.tar.gz" archlinux32-keyring-1-1/desc
+ bsdtar tf "$TMP/ftp/libre/os/i686/libre.db.tar.gz" archlinux32-keyring-1-1/desc
+ bsdtar tf "$TMP/ftp/libre/os/armv7h/libre.db.tar.gz" archlinux32-keyring-1-1/desc
+
+ bsdtar tf "$TMP/ftp/libre/os/x86_64/libre.db.tar.gz" archlinuxarm-keyring-1-1/desc
+ bsdtar tf "$TMP/ftp/libre/os/i686/libre.db.tar.gz" archlinuxarm-keyring-1-1/desc
+ bsdtar tf "$TMP/ftp/libre/os/armv7h/libre.db.tar.gz" archlinuxarm-keyring-1-1/desc
+
+ __isLinkTo "$TMP/ftp/libre/os/x86_64/archlinux32-keyring-1-1-any.pkg.tar.xz" "$TMP/ftp/pool/parabola/archlinux32-keyring-1-1-any.pkg.tar.xz"
+ __isLinkTo "$TMP/ftp/libre/os/i686/archlinux32-keyring-1-1-any.pkg.tar.xz" "$TMP/ftp/pool/parabola/archlinux32-keyring-1-1-any.pkg.tar.xz"
+ __isLinkTo "$TMP/ftp/libre/os/armv7h/archlinux32-keyring-1-1-any.pkg.tar.xz" "$TMP/ftp/pool/parabola/archlinux32-keyring-1-1-any.pkg.tar.xz"
+
+ __isLinkTo "$TMP/ftp/libre/os/x86_64/archlinuxarm-keyring-1-1-any.pkg.tar.xz" "$TMP/ftp/pool/parabola/archlinuxarm-keyring-1-1-any.pkg.tar.xz"
+ __isLinkTo "$TMP/ftp/libre/os/i686/archlinuxarm-keyring-1-1-any.pkg.tar.xz" "$TMP/ftp/pool/parabola/archlinuxarm-keyring-1-1-any.pkg.tar.xz"
+ __isLinkTo "$TMP/ftp/libre/os/armv7h/archlinuxarm-keyring-1-1-any.pkg.tar.xz" "$TMP/ftp/pool/parabola/archlinuxarm-keyring-1-1-any.pkg.tar.xz"
+}
diff --git a/test/cases/db-import-pkg.bats b/test/cases/db-import-pkg.bats
new file mode 100644
index 0000000..f973c84
--- /dev/null
+++ b/test/cases/db-import-pkg.bats
@@ -0,0 +1,260 @@
+load ../lib/import
+
+# Run the command in a new mount namespace with /tmp remounted
+# read-only, but with $TMP (which might be under /tmp) still writable.
+#
+# Arguments are passed as arguments to `sudo`.
+__withRoTmp() {
+ local mount="mount -o bind ${TMP@Q}{,} && mount -o bind,remount,ro /tmp{,}"
+ local env=(
+ "DBSCRIPTS_CONFIG=${DBSCRIPTS_CONFIG}"
+ "XDG_CONFIG_HOME=${XDG_CONFIG_HOME}"
+ )
+ sudo -- unshare -m -- sh -c "${mount} && sudo -u ${USER@Q} ${env[*]@Q} \$@" -- "$@"
+}
+
+__db-import-pkg() {
+ local ret=0
+ # Since common.bash->config.local sets TMPDIR=${TMP}/tmp,
+ # TMPDIR is necessarily != /tmp.
+ # Which means that if we try to write anything directly under /tmp,
+ # then we are erroneously disregarding TMPDIR.
+ # So, make /tmp read-only to make that be an error.
+ __withRoTmp db-import-pkg "$@" || ret=$?
+ # Verify that it cleaned up after itself and TMPDIR is empty
+ find "$TMPDIR" -mindepth 1 | diff - /dev/null
+ return $ret
+}
+
+######################################################################
+
+@test "import no blacklisted packages (x86_64)" {
+ __releaseImportedPackage slavery x86_64 "$TMP/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz" "$TMP/rsyncd/archlinux/pool/packages"
+ __releaseImportedPackage pkg-simple-c x86_64 "$TMP/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz" "$TMP/rsyncd/archlinux/pool/packages"
+
+ DBSCRIPTS_CONFIG="${TMP}/config.local.packages" __db-import-pkg
+
+ bsdtar xfO "$TMP/ftp/core/os/x86_64/core.db.tar.gz" pkg-simple-c-1-1/desc
+ __isLinkTo "$TMP/ftp/core/os/x86_64/pkg-simple-c-1-1-x86_64.pkg.tar.xz" "$TMP/ftp/pool/packages/pkg-simple-c-1-1-x86_64.pkg.tar.xz"
+ __doesNotExist "$TMP"/ftp/{core/os/x86_64,pool/packages,sources/packages}/slavery-*
+}
+
+@test "import no blacklisted packages (i686)" {
+ __releaseImportedPackage slavery i686 "$TMP/rsyncd/archlinux32/i686/core/core.db.tar.gz" "$TMP/rsyncd/archlinux32/pool"
+ __releaseImportedPackage pkg-simple-c i686 "$TMP/rsyncd/archlinux32/i686/core/core.db.tar.gz" "$TMP/rsyncd/archlinux32/pool"
+
+ DBSCRIPTS_CONFIG="${TMP}/config.local.archlinux32" __db-import-pkg
+
+ bsdtar xfO "$TMP/ftp/core/os/i686/core.db.tar.gz" pkg-simple-c-1-1/desc
+ __isLinkTo "$TMP/ftp/core/os/i686/pkg-simple-c-1-1-i686.pkg.tar.xz" "$TMP/ftp/pool/archlinux32/pkg-simple-c-1-1-i686.pkg.tar.xz"
+ __doesNotExist "$TMP"/ftp/{core/os/i686,pool/archlinux32,sources/archlinux32}/slavery-*
+}
+
+@test "import no blacklisted packages (armv7h)" {
+ __releaseImportedPackage slavery armv7h "$TMP/rsyncd/archlinuxarm/armv7h/core/core.db.tar.gz"
+ __releaseImportedPackage pkg-simple-c armv7h "$TMP/rsyncd/archlinuxarm/armv7h/core/core.db.tar.gz"
+
+ DBSCRIPTS_CONFIG="${TMP}/config.local.archlinuxarm" __db-import-pkg
+
+ bsdtar xfO "$TMP/ftp/core/os/armv7h/core.db.tar.gz" pkg-simple-c-1-1/desc
+ __isLinkTo "$TMP/ftp/core/os/armv7h/pkg-simple-c-1-1-armv7h.pkg.tar.xz" "$TMP/ftp/pool/alarm/pkg-simple-c-1-1-armv7h.pkg.tar.xz"
+ __doesNotExist "$TMP"/ftp/{core/os/alarm,pool/alarm,sources/alarm}/slavery-*
+}
+
+@test "import DBs with no blacklisted packages" {
+ __releaseImportedPackage pkg-simple-c x86_64 "$TMP/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz" "$TMP/rsyncd/archlinux/pool/packages"
+
+ DBSCRIPTS_CONFIG="${TMP}/config.local.packages" __db-import-pkg
+
+ bsdtar xfO "$TMP/ftp/core/os/x86_64/core.db.tar.gz" pkg-simple-c-1-1/desc
+ __isLinkTo "$TMP/ftp/core/os/x86_64/pkg-simple-c-1-1-x86_64.pkg.tar.xz" "$TMP/ftp/pool/packages/pkg-simple-c-1-1-x86_64.pkg.tar.xz"
+}
+
+@test "import updated packages" {
+ __releaseImportedPackage slavery x86_64 "$TMP/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz" "$TMP/rsyncd/archlinux/pool/packages"
+ __releaseImportedPackage pkg-simple-c x86_64 "$TMP/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz" "$TMP/rsyncd/archlinux/pool/packages"
+
+ DBSCRIPTS_CONFIG="${TMP}/config.local.packages" __db-import-pkg
+
+ bsdtar xfO "$TMP/ftp/core/os/x86_64/core.db.tar.gz" pkg-simple-c-1-1/desc
+ __isLinkTo "$TMP/ftp/core/os/x86_64/pkg-simple-c-1-1-x86_64.pkg.tar.xz" "$TMP/ftp/pool/packages/pkg-simple-c-1-1-x86_64.pkg.tar.xz"
+
+ __updateImportedPackage pkg-simple-c
+ __releaseImportedPackage pkg-simple-c x86_64 "$TMP/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz" "$TMP/rsyncd/archlinux/pool/packages"
+
+ DBSCRIPTS_CONFIG="${TMP}/config.local.packages" __db-import-pkg
+
+ bsdtar xfO "$TMP/ftp/core/os/x86_64/core.db.tar.gz" pkg-simple-c-1-2/desc
+ __isLinkTo "$TMP/ftp/core/os/x86_64/pkg-simple-c-1-2-x86_64.pkg.tar.xz" "$TMP/ftp/pool/packages/pkg-simple-c-1-2-x86_64.pkg.tar.xz"
+}
+
+@test "import .db files as 0664 (x86_64)" {
+ __releaseImportedPackage slavery x86_64 "$TMP/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz" "$TMP/rsyncd/archlinux/pool/packages"
+ __releaseImportedPackage pkg-simple-c x86_64 "$TMP/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz" "$TMP/rsyncd/archlinux/pool/packages"
+
+ DBSCRIPTS_CONFIG="${TMP}/config.local.packages" __db-import-pkg
+
+ bsdtar xfO "$TMP/ftp/core/os/x86_64/core.db.tar.gz" pkg-simple-c-1-1/desc
+ __isLinkTo "$TMP/ftp/core/os/x86_64/pkg-simple-c-1-1-x86_64.pkg.tar.xz" "$TMP/ftp/pool/packages/pkg-simple-c-1-1-x86_64.pkg.tar.xz"
+ __doesNotExist "$TMP"/ftp/{core/os/x86_64,pool/packages,sources/packages}/slavery-*
+ [[ "$(stat -c '%a' -- "$TMP/ftp/core/os/x86_64/core.db.tar.gz")" = 664 ]]
+}
+
+@test "import .db files as 0664 (i686)" {
+ __releaseImportedPackage slavery i686 "$TMP/rsyncd/archlinux32/i686/core/core.db.tar.gz" "$TMP/rsyncd/archlinux32/pool"
+ __releaseImportedPackage pkg-simple-c i686 "$TMP/rsyncd/archlinux32/i686/core/core.db.tar.gz" "$TMP/rsyncd/archlinux32/pool"
+
+ DBSCRIPTS_CONFIG="${TMP}/config.local.archlinux32" __db-import-pkg
+
+ bsdtar xfO "$TMP/ftp/core/os/i686/core.db.tar.gz" pkg-simple-c-1-1/desc
+ __isLinkTo "$TMP/ftp/core/os/i686/pkg-simple-c-1-1-i686.pkg.tar.xz" "$TMP/ftp/pool/archlinux32/pkg-simple-c-1-1-i686.pkg.tar.xz"
+ __doesNotExist "$TMP"/ftp/{core/os/i686,pool/archlinux32,sources/archlinux32}/slavery-*
+ stat -- "$TMP/ftp/core/os/i686/core.db.tar.gz"
+ [[ "$(stat -c '%a' -- "$TMP/ftp/core/os/i686/core.db.tar.gz")" = 664 ]]
+}
+
+@test "import .db files as 0664 (armv7h)" {
+ __releaseImportedPackage slavery armv7h "$TMP/rsyncd/archlinuxarm/armv7h/core/core.db.tar.gz"
+ __releaseImportedPackage pkg-simple-c armv7h "$TMP/rsyncd/archlinuxarm/armv7h/core/core.db.tar.gz"
+
+ DBSCRIPTS_CONFIG="${TMP}/config.local.archlinuxarm" __db-import-pkg
+
+ bsdtar xfO "$TMP/ftp/core/os/armv7h/core.db.tar.gz" pkg-simple-c-1-1/desc
+ __isLinkTo "$TMP/ftp/core/os/armv7h/pkg-simple-c-1-1-armv7h.pkg.tar.xz" "$TMP/ftp/pool/alarm/pkg-simple-c-1-1-armv7h.pkg.tar.xz"
+ __doesNotExist "$TMP"/ftp/{core/os/armv7h,pool/alarm,sources/alarm}/slavery-*
+ stat -- "$TMP/ftp/core/os/armv7h/core.db.tar.gz"
+ [[ "$(stat -c '%a' -- "$TMP/ftp/core/os/armv7h/core.db.tar.gz")" = 664 ]]
+}
+
+@test "import fully-masked upstream" {
+ __releaseImportedPackage pkg-any-a x86_64 "$TMP/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz" "$TMP/rsyncd/archlinux/pool/packages"
+ __releaseImportedPackage pkg-any-a i686 "$TMP/rsyncd/archlinux32/i686/core/core.db.tar.gz" "$TMP/rsyncd/archlinux32/pool"
+ __releaseImportedPackage pkg-any-a armv7h "$TMP/rsyncd/archlinuxarm/armv7h/core/core.db.tar.gz"
+
+ DBSCRIPTS_CONFIG="${TMP}/config.local.packages" __db-import-pkg
+ DBSCRIPTS_CONFIG="${TMP}/config.local.archlinux32" __db-import-pkg
+ DBSCRIPTS_CONFIG="${TMP}/config.local.archlinuxarm" __db-import-pkg
+
+ bsdtar xfO "$TMP/ftp/core/os/x86_64/core.db.tar.gz" pkg-any-a-1-1/desc
+ bsdtar xfO "$TMP/ftp/core/os/i686/core.db.tar.gz" pkg-any-a-1-1/desc
+ bsdtar xfO "$TMP/ftp/core/os/armv7h/core.db.tar.gz" pkg-any-a-1-1/desc
+
+ __isLinkTo "$TMP/ftp/core/os/x86_64/pkg-any-a-1-1-any.pkg.tar.xz" "$TMP/ftp/pool/packages/pkg-any-a-1-1-any.pkg.tar.xz"
+ __isLinkTo "$TMP/ftp/core/os/i686/pkg-any-a-1-1-any.pkg.tar.xz" "$TMP/ftp/pool/packages/pkg-any-a-1-1-any.pkg.tar.xz"
+ __isLinkTo "$TMP/ftp/core/os/armv7h/pkg-any-a-1-1-any.pkg.tar.xz" "$TMP/ftp/pool/packages/pkg-any-a-1-1-any.pkg.tar.xz"
+}
+
+@test "import removes dropped packages" {
+ __releaseImportedPackage pkg-simple-c x86_64 "$TMP/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz" "$TMP/rsyncd/archlinux/pool/packages"
+
+ DBSCRIPTS_CONFIG="${TMP}/config.local.packages" __db-import-pkg
+
+ __isLinkTo "$TMP/ftp/core/os/x86_64/pkg-simple-c-1-1-x86_64.pkg.tar.xz" "$TMP/ftp/pool/packages/pkg-simple-c-1-1-x86_64.pkg.tar.xz"
+ bsdtar xfO "$TMP/ftp/core/os/x86_64/core.db.tar.gz" pkg-simple-c-1-1/desc
+
+ repo-remove -q "$TMP/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz" pkg-simple-c
+
+ DBSCRIPTS_CONFIG="${TMP}/config.local.packages" __db-import-pkg
+
+ if bsdtar xfO "$TMP/ftp/core/os/x86_64/core.db.tar.gz" pkg-simple-c-1-1/desc; then
+ return 1
+ fi
+}
+
+@test "import removes blacklisted packages" {
+ __releaseImportedPackage pkg-simple-c x86_64 "$TMP/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz" "$TMP/rsyncd/archlinux/pool/packages"
+
+ DBSCRIPTS_CONFIG="${TMP}/config.local.packages" __db-import-pkg
+
+ __isLinkTo "$TMP/ftp/core/os/x86_64/pkg-simple-c-1-1-x86_64.pkg.tar.xz" "$TMP/ftp/pool/packages/pkg-simple-c-1-1-x86_64.pkg.tar.xz"
+ bsdtar xfO "$TMP/ftp/core/os/x86_64/core.db.tar.gz" pkg-simple-c-1-1/desc >/dev/null
+
+ echo 'pkg-simple-c::::' >> "$TMP/httpd/blacklist.txt"
+
+ DBSCRIPTS_CONFIG="${TMP}/config.local.packages" __db-import-pkg
+
+ if bsdtar xfO "$TMP/ftp/core/os/x86_64/core.db.tar.gz" pkg-simple-c-1-1/desc >/dev/null; then
+ return 1
+ fi
+}
+
+@test "import moves packages" {
+ __releaseImportedPackage pkg-simple-c x86_64 "$TMP/rsyncd/archlinux/testing/os/x86_64/testing.db.tar.gz" "$TMP/rsyncd/archlinux/pool/packages"
+
+ DBSCRIPTS_CONFIG="${TMP}/config.local.packages" __db-import-pkg
+
+ __isLinkTo "$TMP/ftp/testing/os/x86_64/pkg-simple-c-1-1-x86_64.pkg.tar.xz" "$TMP/ftp/pool/packages/pkg-simple-c-1-1-x86_64.pkg.tar.xz"
+ bsdtar xfO "$TMP/ftp/testing/os/x86_64/testing.db.tar.gz" pkg-simple-c-1-1/desc >/dev/null
+
+ repo-remove -q "$TMP/rsyncd/archlinux/testing/os/x86_64/testing.db.tar.gz" pkg-simple-c
+ __releaseImportedPackage pkg-simple-c x86_64 "$TMP/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz" "$TMP/rsyncd/archlinux/pool/packages"
+
+ DBSCRIPTS_CONFIG="${TMP}/config.local.packages" __db-import-pkg
+
+ __isLinkTo "$TMP/ftp/core/os/x86_64/pkg-simple-c-1-1-x86_64.pkg.tar.xz" "$TMP/ftp/pool/packages/pkg-simple-c-1-1-x86_64.pkg.tar.xz"
+ bsdtar xfO "$TMP/ftp/core/os/x86_64/core.db.tar.gz" pkg-simple-c-1-1/desc >/dev/null
+ if bsdtar xfO "$TMP/ftp/testing/os/x86_64/testing.db.tar.gz" pkg-simple-c-1-1/desc >/dev/null; then
+ return 1
+ fi
+}
+
+@test "import arch=any packages with sub-pkgrel" {
+ # This is modeled after the situation with 'asp' and 'asp32'
+
+ __releaseImportedPackage pkg-any64 x86_64 "$TMP/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz" "$TMP/rsyncd/archlinux/pool/packages"
+ DBSCRIPTS_CONFIG="${TMP}/config.local.packages" __db-import-pkg
+ __isLinkTo "$TMP/ftp/core/os/x86_64/pkg-any-2-1-any.pkg.tar.xz" "$TMP/ftp/pool/packages/pkg-any-2-1-any.pkg.tar.xz"
+ bsdtar xfO "$TMP/ftp/core/os/x86_64/core.db.tar.gz" pkg-any-2-1/desc
+
+ __releaseImportedPackage pkg-any32 i686 "$TMP/rsyncd/archlinux32/i686/core/core.db.tar.gz" "$TMP/rsyncd/archlinux32/pool"
+ DBSCRIPTS_CONFIG="${TMP}/config.local.archlinux32" __db-import-pkg
+ __isLinkTo "$TMP/ftp/core/os/i686/pkg-any-1-1.2-any.pkg.tar.xz" "$TMP/ftp/pool/archlinux32/pkg-any-1-1.2-any.pkg.tar.xz"
+ bsdtar xfO "$TMP/ftp/core/os/i686/core.db.tar.gz" pkg-any-1-1.2/desc
+}
+
+@test "import respects INHERIT precedence" {
+ __releaseImportedPackage pkg-any-a armv7h "$TMP/rsyncd/archlinuxarm/armv7h/core/core.db.tar.gz"
+ DBSCRIPTS_CONFIG="${TMP}/config.local.archlinux32" __db-import-pkg
+
+ __releaseImportedPackage pkg-any-a x86_64 "$TMP/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz" "$TMP/rsyncd/archlinux/pool/packages"
+ DBSCRIPTS_CONFIG="${TMP}/config.local.packages" __db-import-pkg
+ DBSCRIPTS_CONFIG="${TMP}/config.local.archlinuxarm" __db-import-pkg
+
+ bsdtar xfO "$TMP/ftp/core/os/x86_64/core.db.tar.gz" pkg-any-a-1-1/desc
+ bsdtar xfO "$TMP/ftp/core/os/armv7h/core.db.tar.gz" pkg-any-a-1-1/desc
+
+ __isLinkTo "$TMP/ftp/core/os/x86_64/pkg-any-a-1-1-any.pkg.tar.xz" "$TMP/ftp/pool/packages/pkg-any-a-1-1-any.pkg.tar.xz"
+ __isLinkTo "$TMP/ftp/core/os/armv7h/pkg-any-a-1-1-any.pkg.tar.xz" "$TMP/ftp/pool/packages/pkg-any-a-1-1-any.pkg.tar.xz"
+}
+
+@test "import doesn't backdate packages" {
+ __releaseImportedPackage pkg-simple-c x86_64 "$TMP/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz" "$TMP/rsyncd/archlinux/pool/packages"
+
+ touch "$TMP/stamp"
+ DBSCRIPTS_CONFIG="${TMP}/config.local.packages" __db-import-pkg
+
+ bsdtar xfO "$TMP/ftp/core/os/x86_64/core.db.tar.gz" pkg-simple-c-1-1/desc
+ __isLinkTo "$TMP/ftp/core/os/x86_64/pkg-simple-c-1-1-x86_64.pkg.tar.xz" "$TMP/ftp/pool/packages/pkg-simple-c-1-1-x86_64.pkg.tar.xz"
+ [[ "$TMP/ftp/pool/packages/pkg-simple-c-1-1-x86_64.pkg.tar.xz" -nt "$TMP/stamp" ]]
+ [[ "$TMP/ftp/lastupdate" -nt "$TMP/stamp" ]]
+}
+
+@test "import previously removed packages" {
+ # Import the package once
+ __releaseImportedPackage pkg-simple-c x86_64 "$TMP/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz" "$TMP/rsyncd/archlinux/pool/packages"
+ DBSCRIPTS_CONFIG="${TMP}/config.local.packages" __db-import-pkg
+ bsdtar xfO "$TMP/ftp/core/os/x86_64/core.db.tar.gz" pkg-simple-c-1-1/desc
+ __isLinkTo "$TMP/ftp/core/os/x86_64/pkg-simple-c-1-1-x86_64.pkg.tar.xz" "$TMP/ftp/pool/packages/pkg-simple-c-1-1-x86_64.pkg.tar.xz"
+
+ # Remove it
+ mv -fT -- "$TMP/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz"{,.old}
+ :> "$TMP/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz"
+ DBSCRIPTS_CONFIG="${TMP}/config.local.packages" __db-import-pkg
+ if bsdtar xfO "$TMP/ftp/core/os/x86_64/core.db.tar.gz" pkg-simple-c-1-1/desc; then return 1; fi
+
+ # And add it again
+ mv -fT -- "$TMP/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz"{.old,}
+ DBSCRIPTS_CONFIG="${TMP}/config.local.packages" __db-import-pkg
+ bsdtar xfO "$TMP/ftp/core/os/x86_64/core.db.tar.gz" pkg-simple-c-1-1/desc
+ __isLinkTo "$TMP/ftp/core/os/x86_64/pkg-simple-c-1-1-x86_64.pkg.tar.xz" "$TMP/ftp/pool/packages/pkg-simple-c-1-1-x86_64.pkg.tar.xz"
+}
diff --git a/test/cases/db-update.bats b/test/cases/db-update.bats
index e805f99..0ac4853 100755
--- a/test/cases/db-update.bats
+++ b/test/cases/db-update.bats
@@ -239,3 +239,24 @@ load ../lib/common
[ "$status" -ne 0 ]
checkRemovedPackageDB extra "pkg-simple-a"
}
+
+@test "re-release to new architectures" {
+ # Initial run
+ releasePackage extra 'pkg-any-a'
+ db-update
+ checkPackage extra pkg-any-a 1-1
+ rmdir "$STAGING"
+
+ # Add an arch
+ echo 'ARCHES+=(ppc64le)' >> "$DBSCRIPTS_CONFIG"
+ ARCHES+=(ppc64le)
+ mkdir -p "${TMP}/ftp/extra/os/ppc64le"
+ if checkPackage extra pkg-any-a 1-1; then return 1; fi
+
+ # Re-run
+ mkdir -p "${STAGING}/extra"
+ ln -s -t "${STAGING}/extra" -- "${TMP}/ftp/pool/packages/pkg-any-a-1-1-any.pkg.tar.xz"{,.sig}
+ db-update
+ checkPackage extra pkg-any-a 1-1
+ rmdir "$STAGING"
+}
diff --git a/test/fixtures/archlinux32-keyring/PKGBUILD b/test/fixtures/archlinux32-keyring/PKGBUILD
new file mode 100644
index 0000000..a045b0b
--- /dev/null
+++ b/test/fixtures/archlinux32-keyring/PKGBUILD
@@ -0,0 +1,13 @@
+pkgname=archlinux32-keyring
+pkgver=1
+pkgrel=1
+pkgdesc="A package called ${pkgname}"
+arch=('any')
+url='http://www.archlinux.org/'
+license=('GPL')
+options=(!strip)
+
+package() {
+ install -d "${pkgdir}"/usr/share/${pkgname}
+ :>"${pkgdir}"/usr/share/${pkgname}/test
+}
diff --git a/test/fixtures/archlinuxarm-keyring/PKGBUILD b/test/fixtures/archlinuxarm-keyring/PKGBUILD
new file mode 100644
index 0000000..bb2bc9d
--- /dev/null
+++ b/test/fixtures/archlinuxarm-keyring/PKGBUILD
@@ -0,0 +1,13 @@
+pkgname=archlinuxarm-keyring
+pkgver=1
+pkgrel=1
+pkgdesc="A package called ${pkgname}"
+arch=('any')
+url='http://www.archlinux.org/'
+license=('GPL')
+options=(!strip)
+
+package() {
+ install -d "${pkgdir}"/usr/share/${pkgname}
+ :>"${pkgdir}"/usr/share/${pkgname}/test
+}
diff --git a/test/fixtures/pkg-any32/PKGBUILD b/test/fixtures/pkg-any32/PKGBUILD
new file mode 100644
index 0000000..e0421da
--- /dev/null
+++ b/test/fixtures/pkg-any32/PKGBUILD
@@ -0,0 +1,13 @@
+pkgname=pkg-any
+pkgver=1
+pkgrel=1.2
+pkgdesc="A package called ${pkgname} (32)"
+arch=('any')
+url='http://www.archlinux.org/'
+license=('GPL')
+options=(!strip)
+
+package() {
+ install -d -m755 "${pkgdir}"/usr/share/${pkgname}
+ echo 'test' > "${pkgdir}"/usr/share/${pkgname}/test
+}
diff --git a/test/fixtures/pkg-any64/PKGBUILD b/test/fixtures/pkg-any64/PKGBUILD
new file mode 100644
index 0000000..1290ebd
--- /dev/null
+++ b/test/fixtures/pkg-any64/PKGBUILD
@@ -0,0 +1,13 @@
+pkgname=pkg-any
+pkgver=2
+pkgrel=1
+pkgdesc="A package called ${pkgname} (64)"
+arch=('any')
+url='http://www.archlinux.org/'
+license=('GPL')
+options=(!strip)
+
+package() {
+ install -d -m755 "${pkgdir}"/usr/share/${pkgname}
+ echo 'test' > "${pkgdir}"/usr/share/${pkgname}/test
+}
diff --git a/test/fixtures/pkg-simple-c/PKGBUILD b/test/fixtures/pkg-simple-c/PKGBUILD
new file mode 100644
index 0000000..1e8361d
--- /dev/null
+++ b/test/fixtures/pkg-simple-c/PKGBUILD
@@ -0,0 +1,13 @@
+pkgname=pkg-simple-c
+pkgver=1
+pkgrel=1
+pkgdesc="A package called ${pkgname}"
+arch=('i686' 'x86_64' 'armv7h')
+url='http://www.archlinux.org/'
+license=('GPL' 'LGPL')
+depends=('glibc')
+options=(!strip)
+
+package() {
+ install -D -m755 /bin/true "${pkgdir}"/usr/bin/${pkgname}
+}
diff --git a/test/fixtures/slavery/PKGBUILD b/test/fixtures/slavery/PKGBUILD
new file mode 100644
index 0000000..93cc079
--- /dev/null
+++ b/test/fixtures/slavery/PKGBUILD
@@ -0,0 +1,13 @@
+pkgname=slavery
+pkgver=1
+pkgrel=1
+pkgdesc="A package called ${pkgname}"
+arch=('any')
+url='http://www.archlinux.org/'
+license=('GPL')
+options=(!strip)
+
+package() {
+ install -d -m755 "${pkgdir}"/usr/share/${pkgname}
+ echo 'test' > "${pkgdir}"/usr/share/${pkgname}/test
+}
diff --git a/test/lib/common.bash b/test/lib/common.bash
index 6e9394c..764def5 100644
--- a/test/lib/common.bash
+++ b/test/lib/common.bash
@@ -77,7 +77,8 @@ setup() {
export DBSCRIPTS_CONFIG=${TMP}/config.local
cat <<eot > "${DBSCRIPTS_CONFIG}"
FTP_BASE="${TMP}/ftp"
- PKGREPOS=('core' 'extra' 'testing')
+ PKGREPOS=('core' 'extra' 'testing' 'libre')
+ ARCHES=(x86_64 i686 armv7h)
PKGPOOL='pool/packages'
SRCPOOL='sources/packages'
TESTING_REPO='testing'
@@ -86,7 +87,6 @@ setup() {
SOURCE_CLEANUP_DESTDIR="${TMP}/source-cleanup"
STAGING="${TMP}/staging"
TMPDIR="${TMP}/tmp"
- ARCHES=(x86_64 i686)
CLEANUP_DRYRUN=false
SOURCE_CLEANUP_DRYRUN=false
eot
diff --git a/test/lib/import.bash b/test/lib/import.bash
new file mode 100644
index 0000000..84d8688
--- /dev/null
+++ b/test/lib/import.bash
@@ -0,0 +1,195 @@
+#!/hint/bash
+. "lib/common.bash"
+
+httpserver() {
+ read -r verb path version || exit
+ if [[ $verb != GET ]]; then
+ printf "HTTP/1.1 405 Method Not Allowed\\r\\nContent-Type: text/plain\\r\\nContent-length: 0\\r\\n\\r\\n"
+ exit
+ fi
+ path="$(cd / && realpath -ms "$path")"
+ if ! [[ -f "$1/$path" ]]; then
+ printf "HTTP/1.1 404 Not found\\r\\nContent-Type: text/plain\\r\\nContent-length: 0\\r\\n\\r\\n"
+ exit
+ fi
+ printf "HTTP/1.1 200 OK\\r\\nContent-Type: text/plain\\r\\nContent-length: %d\\r\\n\\r\\n" "$(stat -L -c %s "$1/$path")"
+ cat "$1/$path"
+}
+
+eval "__common_$(declare -f setup)"
+setup() {
+ __common_setup
+ local repo arch
+
+ # Set up rsync server
+ cat <<-eot >"${TMP}/rsyncd.conf"
+ use chroot = no
+ [rsyncd]
+ path = ${TMP}/rsyncd
+ eot
+ local rsyncport
+ rsyncport=$(./lib/runserver "$TMP/rsyncd.pid" \
+ rsync --daemon --config "${TMP}/rsyncd.conf")
+
+ # Set up rsync contents
+ for repo in core testing; do
+ mkdir -p -- "${TMP}/rsyncd/archlinux/${repo}/os/x86_64"
+ touch -- "${TMP}/rsyncd/archlinux/${repo}/os/x86_64/${repo}.db.tar.gz"
+ ln -sT -- "${repo}.db.tar.gz" "${TMP}/rsyncd/archlinux/${repo}/os/x86_64/${repo}.db"
+ done
+ mkdir -p -- "${TMP}/rsyncd/archlinux"/{pool,sources}/{packages,community}
+ date +%s > "${TMP}/rsyncd/archlinux/lastupdate"
+ date +%s > "${TMP}/rsyncd/archlinux/lastsync"
+
+ for repo in core testing community community-testing; do
+ mkdir -p -- "${TMP}/rsyncd/archlinux32/i686/${repo}"
+ touch -- "${TMP}/rsyncd/archlinux32/i686/${repo}/${repo}.db.tar.gz"
+ ln -sT -- "${repo}.db.tar.gz" "${TMP}/rsyncd/archlinux32/i686/${repo}/${repo}.db"
+ done
+ mkdir -p -- "${TMP}/rsyncd/archlinux32/pool"
+ date +%s > "${TMP}/rsyncd/archlinux32/lastupdate"
+ date +%s > "${TMP}/rsyncd/archlinux32/lastsync"
+
+ for repo in core testing community community-testing; do
+ mkdir -p -- "${TMP}/rsyncd/archlinuxarm/armv7h/${repo}"
+ touch -- "${TMP}/rsyncd/archlinuxarm/armv7h/${repo}/${repo}.db.tar.gz"
+ ln -sT -- "${repo}.db.tar.gz" "${TMP}/rsyncd/archlinuxarm/armv7h/${repo}/${repo}.db"
+ done
+ date +%s > "${TMP}/rsyncd/archlinuxarm/lastupdate"
+ date +%s > "${TMP}/rsyncd/archlinuxarm/lastsync"
+
+ # Configure db-import to use that rsyncd server
+ sed -i /STAGING=/d "${TMP}/config.local"
+ cat <<-eot >"${TMP}/config.local.packages"
+ source ${DBSCRIPTS_CONFIG@Q}
+ source ../config.local.packages
+ ARCHTAGS=('core-x86_64' 'testing-x86_64')
+ ARCHMIRROR=rsync://localhost:${rsyncport@Q}/rsyncd/archlinux/
+ eot
+ cat <<-eot >"${TMP}/config.local.community"
+ source ${DBSCRIPTS_CONFIG@Q}
+ source ../config.local.community
+ ARCHTAGS=('community-x86_64' 'community-testing-x86_64')
+ ARCHMIRROR=rsync://localhost:${rsyncport@Q}/rsyncd/archlinux/
+ eot
+ cat <<-eot >"${TMP}/config.local.archlinux32"
+ source ${DBSCRIPTS_CONFIG@Q}
+ source ../config.local.archlinux32
+ ARCHTAGS=('core-i686' 'testing-i686' 'community-i686' 'community-testing-i686')
+ ARCHMIRROR=rsync://localhost:${rsyncport@Q}/rsyncd/archlinux32/
+ eot
+ cat <<-eot >"${TMP}/config.local.archlinuxarm"
+ source ${DBSCRIPTS_CONFIG@Q}
+ source ../config.local.archlinuxarm
+ ARCHTAGS=('core-armv7h' 'testing-armv7h' 'community-armv7h' 'community-testing-armv7h')
+ ARCHMIRROR=rsync://localhost:${rsyncport@Q}/rsyncd/archlinuxarm/
+ eot
+
+ # Set up HTTP server
+ local httpport
+ httpport=$(./lib/runserver "$TMP/httpd.pid" \
+ bash -c "$(declare -f httpserver); httpserver \"\$@\"" -- "$TMP/httpd")
+
+ # Set up HTTP contents
+ mkdir -- "$TMP/httpd"
+ cat <<-eot >"$TMP/httpd/blacklist.txt"
+ slavery:freedom:fsf:slavekit:Obviously
+ eot
+
+ # Configure db-import to use that HTTP server
+ export XDG_CONFIG_HOME="${TMP}/home/.config"
+ mkdir -p "$XDG_CONFIG_HOME"/libretools
+ cat <<-eot >"$XDG_CONFIG_HOME"/libretools/libretools.conf
+ BLACKLIST=http://localhost:${httpport@Q}/blacklist.txt
+ eot
+
+ # Set up repo contents
+ for arch in x86_64 i686 armv7h; do
+ for repo in core testing community community-testing libre; do
+ mkdir -p -- "${TMP}/ftp/${repo}/os/${arch}"
+ touch -- "${TMP}/ftp/${repo}/os/${arch}/${repo}.db.tar.gz"
+ ln -sT -- "${repo}.db.tar.gz" "${TMP}/ftp/${repo}/os/${arch}/${repo}.db"
+ done
+ done
+ mkdir -p -- "${TMP}/ftp"/{pool,sources}/{packages,community,archlinux32,alarm}
+ date +%s > "${TMP}/ftp/lastupdate"
+ date +%s > "${TMP}/ftp/lastsync"
+}
+eval "__common_$(declare -f teardown)"
+teardown() {
+ xargs -a "${TMP}/httpd.pid" kill --
+ xargs -a "${TMP}/rsyncd.pid" kill --
+ __common_teardown
+}
+
+######################################################################
+
+# __releaseImportedPackage PKGBASE ARCH DBFILE [POOLDIR]
+#
+# This is different from common.bash:releasePackage because
+# - it doesn't mess with VCS
+# - it adds the package to the .db file
+__releaseImportedPackage() {
+ local pkgbase=$1
+ local arch=$2
+ local dbfile=$3
+ local pooldir=$4
+ local repodir="${dbfile%/*}"
+ local dir restore pkgfiles pkgfile pkgs
+
+ dir="$TMP/import-build/$pkgbase"
+ if ! [[ -d "$dir" ]]; then
+ mkdir -p -- "$dir"
+ cp -t "$dir" -- "fixtures/${pkgbase}"/*
+ fi
+ pushd "$dir"
+ __buildPackage
+ restore="$(shopt -p nullglob || true)"
+ shopt -s nullglob
+ pkgfiles=(*-{"$arch",any}"$PKGEXT"{,.sig})
+ $restore
+ popd
+
+ mkdir -p "$repodir"
+ if [[ -z $pooldir ]]; then
+ mv -t "$repodir" -- "${pkgfiles[@]/#/"$dir/"}"
+ else
+ mkdir -p "$pooldir"
+ mv -t "$pooldir" -- "${pkgfiles[@]/#/"$dir/"}"
+ ln -sr -t "$repodir" -- "${pkgfiles[@]/#/"$pooldir/"}"
+ fi
+
+ pushd "$repodir"
+ pkgs=()
+ for pkgfile in "${pkgfiles[@]}"; do
+ if [[ "$pkgfile" = *.sig ]]; then
+ continue
+ fi
+ pkgs+=("$pkgfile")
+ done
+ repo-add -q "${dbfile##*/}" "${pkgs[@]}"
+ popd
+}
+
+__updateImportedPackage() {
+ pushd "$TMP/import-build/$1"
+ local pkgrel
+ pkgrel=$(. PKGBUILD; expr ${pkgrel} + 1)
+ sed "s/pkgrel=.*/pkgrel=${pkgrel}/" -i PKGBUILD
+ popd
+}
+
+__isLinkTo() {
+ [[ -L $1 ]]
+ [[ $1 -ef $2 ]]
+}
+
+__doesNotExist() {
+ local file
+ for file in "$@"; do
+ if stat "$file" 2>/dev/null; then
+ echo "TEST ERROR: File shouldn't exist, but does: $file"
+ return 1
+ fi
+ done
+}
diff --git a/test/lib/runserver b/test/lib/runserver
new file mode 100755
index 0000000..7705e40
--- /dev/null
+++ b/test/lib/runserver
@@ -0,0 +1,39 @@
+#!/usr/bin/env perl
+use strict;
+use warnings;
+
+use IO::Socket::INET;
+use Fcntl;
+
+my $socket = IO::Socket::INET->new(
+ LocalAddr => "localhost:0",
+ Listen => 1
+);
+
+sysopen(my $fh, $ARGV[0], O_WRONLY|O_CREAT|O_EXCL, 0666) or die "open: $ARGV[0]: $!";
+my $pid = fork();
+if ($pid > 0) {
+ print $fh $pid;
+ print $socket->sockport()."\n";
+ exit;
+}
+close($fh);
+open(STDIN, '</dev/null');
+open(STDOUT, '</dev/null');
+
+my @cmd = @ARGV;
+shift @cmd;
+
+while (1) {
+ my $conn = $socket->accept();
+ my $worker = fork();
+ if ($worker == 0) {
+ open(STDIN, '<&', $conn);
+ open(STDOUT, '>&', $conn);
+ close($conn);
+ close($socket);
+ exec @cmd;
+ exit
+ }
+ close($conn);
+}