diff options
author | Luke Shumaker <lukeshu@parabola.nu> | 2018-10-07 19:30:50 -0400 |
---|---|---|
committer | Luke Shumaker <lukeshu@parabola.nu> | 2018-10-07 19:34:50 -0400 |
commit | 8c51bc0247f6d69e50e8597aaf31902c4cb3ea71 (patch) | |
tree | 3fb8da1c09032b2721ff98934c08f89c005020e4 | |
parent | 24158f712e9b808366c4f6eeeb125ae3fd1cb942 (diff) |
Add from Parabola: db-import
If you're seeing this in `git blame`, it chose to follow the wrong
ancestor of a merge commit.
44 files changed, 1768 insertions, 25 deletions
@@ -13,6 +13,10 @@ The executables that you (might) care about are: │ ├── ftpdir-cleanup │ ├── integrity-check │ └── sourceballs + ├── db-import-archlinuxarm-src [Parabola only] + ├── db-import-any [Parabola only] + ├── db-import-pkg [Parabola only] + ├── db-import-keyring [Parabola only] ├── db-move ├── db-remove ├── db-repo-add @@ -51,6 +55,23 @@ Both of these programs do the exact same thing. Parabola developers decided to write their own from scratch, instead of modifying `ftpdir-cleanup`. They should eventually be merged. +But, Parabola doesn't just publish our own packages, we also import +packages from elsewhere: + + - `db-import-pkg` + - `db-import-keyring` + - `db-import-archlinuxarm-src` + +`db-import-pkg` is the main import program, and it handles importing +most of the binary packages, as well as the source-packages from Arch +Linux. `db-import-keyring` is a specialized variant of it for +importing the "archlinuxarm-keyring" and "archlinux32-keyring" +packages, since those packages need special treatment to end up on the +nescessary architectures. `db-import-archlinuxarm-src` is supposed to +import the source-packages from Arch Linux ARM, but it has been +defunct for some time now and needs updated. There is currently no +program to import source-packages from Arch Linux 32. + Things that haven't been mentioned yet: - `cron-jobs/devlist-mailer` @@ -3,6 +3,7 @@ FTP_BASE="/srv/repo/main" PKGREPOS=() +ARCHES=() PKGPOOL='' SRCPOOL='' @@ -24,7 +25,6 @@ LOCK_DELAY=10 [ -n "${STAGING:-}" ] || STAGING="$HOME/staging/unknown/staging" [[ $STAGING = /* ]] || STAGING="$PWD/$STAGING" export TMPDIR="/tmp" -ARCHES=(x86_64 i686 armv7h) DBEXT=".db.tar.gz" FILESEXT=".files.tar.gz" SRCEXT=".src.tar.gz" diff --git a/config-list-mirrors-archlinux b/config-list-mirrors-archlinux new file mode 100755 index 0000000..e59ea2c --- /dev/null +++ b/config-list-mirrors-archlinux @@ -0,0 +1,61 @@ +#!/usr/bin/env ruby +# 2013, 2016, 2018 Luke Shumaker + +require 'json' +require 'net/http' +require 'optparse' +require 'set' +devnull = open("/dev/null") + +requires = Set.new() +begin + OptionParser.new do |parser| + parser.on("--require=DIR", "Optional directory that the mirror needs ('archive', 'iso', 'other', or 'sources')") do |dir| + requires.add(dir) + end + parser.on("-h", "--help", "Display this help") do + puts parser + exit + end + end.parse! +rescue OptionParser::ParseError => e + STDERR.puts "#{$0}: #{e}" + STDERR.puts "Try '#{$0} --help' for more information." + exit 2 +end +unless ARGV.empty? + STDERR.puts "#{$0}: extra argument '#{ARGV[0]}'" + STDERR.puts "Try '#{$0} --help' for more information." + exit 2 +end + +data = JSON::parse(Net::HTTP.get(URI("https://www.archlinux.org/mirrors/status/tier/1/json/"))) + +if data["version"] != 3 + print "Data format version != 3" + exit 1 +end + +# Filter out URLs with incomplete information +urls = data["urls"].select{|a| a.none?{|k,v|v.nil?}} + +# Filter based on our criteria +rsync_urls = urls.select{|a| a["protocol"]=="rsync"} +if requires.include?("iso") + rsync_urls = rsync_urls.select{|a| a["isos"]==true} + requires.delete("iso") +end +unless requires.empty? + listing_procs = {} + rsync_urls.each do |a| + # trailing slash is important; ensure we have one + listing_procs[a["url"]] ||= IO.popen(["rsync", "--no-motd", "--list-only", "--", a["url"]+"/"], :err => devnull) + end + listings = {} + listing_procs.map.each do |url,listing_proc| + listings[url] = listing_proc.read.lines.map{|line|line.chomp.sub(/.*\s/,'')} + listing_proc.close + end + rsync_urls = rsync_urls.select{|a|requires.all?{|dir|listings[a["url"]].include?(dir)}} +end +puts JSON.dump(rsync_urls) diff --git a/config-list-mirrors-archlinuxarm b/config-list-mirrors-archlinuxarm new file mode 100755 index 0000000..63f1814 --- /dev/null +++ b/config-list-mirrors-archlinuxarm @@ -0,0 +1,67 @@ +#!/usr/bin/env ruby +# 2018 Luke Shumaker + +require 'json' +require 'net/http' +require 'optparse' +require 'set' +devnull = open("/dev/null") + +begin + OptionParser.new do |parser| + parser.on("-h", "--help", "Display this help") do + puts parser + exit + end + end.parse! +rescue OptionParser::ParseError => e + STDERR.puts "#{$0}: #{e}" + STDERR.puts "Try '#{$0} --help' for more information." + exit 2 +end +unless ARGV.empty? + STDERR.puts "#{$0}: extra argument '#{ARGV[0]}'" + STDERR.puts "Try '#{$0} --help' for more information." + exit 2 +end + +# Get a listing of hostnames that have HTTP Arch Linux ARM mirrors +data = JSON::parse(Net::HTTP.post( + URI("https://archlinuxarm.org/data/mirrors/list"), "", { + 'Referer' => 'https://archlinuxarm.org/about/mirrors', + 'X-Requested-With' => 'XMLHttpRequest', + }).body) + +hosts = data["data"].map{|m|m[2]} + +# Get a listing of rsync URLs for each host +modulelist_procs = {} +hosts.each do |host| + modulelist_procs[host] ||= IO.popen(["timeout", "2s", "rsync", "--password-file", "/dev/null", "#{host}::"], :err => devnull) +end +rsync_urls = Set.new() +modulelist_procs.each do |host,modulelist_proc| + modules = modulelist_proc.read.lines.map{|line| line.chomp.sub(/\s.*/, '') } + modulelist_proc.close + rsync_urls.merge(modules.map{|m| "rsync://#{host}/#{m}/"}) +end + +# Get a listing of the top-level files in each +listing_procs = {} +rsync_urls.each do |url| + listing_procs[url] ||= IO.popen(["rsync", "--password-file", "/dev/null", "--no-motd", "--list-only", "--", url], :err => devnull) +end +listings = {} +listing_procs.map.each do |url,listing_proc| + listings[url] = listing_proc.read.lines.map{|line|line.chomp.sub(/.*\s/,'')} + listing_proc.close +end + +# Filter for listings that look like Arch Linux ARM mirrors +requires = ['aarch64', 'arm', 'armv6h', 'armv7h', 'os'] +rsync_urls = rsync_urls.select{|url| requires.all?{|file|listings[url].include?(file)} } + +# Print the results +rsync_urls.sort.each do |url| + puts url +end diff --git a/config-list-tags b/config-list-tags new file mode 100755 index 0000000..679c0ea --- /dev/null +++ b/config-list-tags @@ -0,0 +1,80 @@ +#!/bin/bash + +set -eE -o pipefail +shopt -s extglob globstar nullglob +source "$(librelib messages)" +setup_traps + +# usage: fetch_dbs <from> <into> +fetch_dbs() { + rsync --no-motd -mrtLH --no-p \ + --include="*/" \ + --include="*.db" \ + --exclude="*" \ + --delete-after \ + "$1" "$2" +} + +list_found() { + local ret=0 + local dbfile repo re arch tag + while read -r -d '' dbfile; do + repo=${dbfile##*/} + repo=${repo%.db} + re=$(repo=$repo arch='(.*)' envsubst '$repo $arch' <<<"$ARCHPATH") + arch=$(sed -rn "s,^$re\$,\1,p" <<<"${dbfile%/*}") + if [[ -z $arch ]]; then + error 'Could not figure out architecture for %q' "$dbfile" + ret=1 + continue + fi + printf '%s\n' "$repo-$arch" + done < <(find "$WORKDIR" -name '*.db' -printf '%P\0') | sort -u + return $ret +} + +list_configured() { + printf '%s\n' "${ARCHTAGS[@]}" | sort -u +} + +main() { + if [[ $# -ne 0 ]] || [[ -z "$DBSCRIPTS_CONFIG" ]] || ! grep -q ARCHMIRROR -- "$DBSCRIPTS_CONFIG"; then + msg 'usage: DBSCRIPTS_CONFIG=/path/to/file %s' "${0##*/}" + exit $EXIT_INVALIDARGUMENT + fi + + local config_file + config_file="$(dirname "$(readlink -e "$0")")/config" + source "$config_file" + + WORKDIR=$(mktemp -dt "${0##*/}.XXXXXXXXXX") + readonly WORKDIR + trap "rm -rf -- ${WORKDIR@Q}" EXIT + + fetch_dbs "${ARCHMIRROR}/" "$WORKDIR" + + y="${GREEN}✓${ALL_OFF}" + n="${RED}✗${ALL_OFF}" + while IFS='' read -r line; do + case "$line" in + $'\t\t'*) + rmt=$y + cfg=$y + val=${line#$'\t\t'} + ;; + $'\t'*) + rmt=$n + cfg=$y + val=${line#$'\t'} + ;; + *) + rmt=$y + cfg=$n + val=$line + ;; + esac + printf '%s:%s:%s\n' "$val" "$rmt" "$cfg" + done < <(comm <(list_found) <(list_configured)) | column -t -s: +} + +main "$@" diff --git a/config.local.archlinux32 b/config.local.archlinux32 new file mode 100644 index 0000000..1a5f18c --- /dev/null +++ b/config.local.archlinux32 @@ -0,0 +1,20 @@ +#!/hint/bash + +PKGREPOS=(build-support community community-staging community-testing core extra gnome-unstable kde-unstable staging testing) +ARCHES=('i686') +PKGPOOL='pool/archlinux32' +SRCPOOL='sources/archlinux32' + +# "tags" are repo-arch pairs +ARCHTAGS=("${PKGREPOS[@]/%/-i686}") # leaves out *-i486 and releng-x86_64 +ARCHPATH='$arch/$repo' +ARCHPKGPOOL='pool' + +INHERIT=( + 'pool/packages' + 'pool/community' + 'pool/alarm' +) + +# https://git.archlinux32.org/archlinux32/packages/raw/branch/master/core/pacman-mirrorlist/mirrorlist +ARCHMIRROR='rsync://mirror.archlinux32.org/archlinux32/' diff --git a/config.local.archlinuxarm b/config.local.archlinuxarm new file mode 100644 index 0000000..ec1cb72 --- /dev/null +++ b/config.local.archlinuxarm @@ -0,0 +1,27 @@ +#!/hint/bash + +PKGREPOS=(alarm aur community core extra) +ARCHES=('armv7h') +PKGPOOL='pool/alarm' +SRCPOOL='sources/alarm' + +# "tags" are repo-arch pairs +ARCHTAGS=("${PKGREPOS[@]/%/-armv7h}") +ARCHPATH='$arch/$repo' + +INHERIT=( + 'pool/packages' + 'pool/community' +) + +# Of the ALARM mirrors that have public rsync: +# +# $ ./config-list-mirrors-archlinuxarm +# rsync://de3.mirror.archlinuxarm.org/archlinux-arm/ +# rsync://dk.mirror.archlinuxarm.org/archlinuxarm/ +# rsync://ru.mirror.archlinuxarm.org/archlinuxarm/ +# rsync://sg.mirror.archlinuxarm.org/archlinuxarm/ +# +# 'dk' (Denmark) has the best ping time from +# winston.parabola.nu (Iceland). +ARCHMIRROR='rsync://dk.mirror.archlinuxarm.org/archlinuxarm/' diff --git a/config.local.community b/config.local.community index 5d61b5e..cabfa5b 100644 --- a/config.local.community +++ b/config.local.community @@ -1,13 +1,23 @@ #!/hint/bash PKGREPOS=('community' 'community-testing' 'community-staging' 'multilib' 'multilib-testing' 'multilib-staging') +ARCHES=('x86_64') PKGPOOL='pool/community' SRCPOOL='sources/community' -SVNREPO='file:///srv/repos/svn-community/svn' -SVNUSER='svn-community' -TESTING_REPO='community-testing' -STABLE_REPOS=('community') -CLEANUP_DESTDIR="/srv/repos/svn-community/package-cleanup" -SOURCE_CLEANUP_DESTDIR="/srv/repos/svn-community/source-cleanup" -TMPDIR="/srv/repos/svn-community/tmp" +# "tags" are repo-arch pairs +ARCHTAGS=({community,multilib}{,-testing,-staging}-x86_64) +ARCHPATH='$repo/os/$arch' +ARCHPKGPOOL='pool/community' +ARCHSRCPOOL='sources/community' + +# Of the few tier-1 mirrors with the 'sources/' folder: +# +# $ ./config-list-mirrors-archlinux --require=sources | jq -r '.[].url' +# rsync://archlinux.c3sl.ufpr.br/archlinux/ +# rsync://mirrors.kernel.org/archlinux/ +# rsync://ftp.acc.umu.se/mirror/archlinux/ +# +# ftp.acc.umu.se (Sweden) has (by far) the best ping time from +# winston.parabola.nu (Iceland). +ARCHMIRROR='rsync://ftp.acc.umu.se/mirror/archlinux/' diff --git a/config.local.import-ourarches b/config.local.import-ourarches new file mode 100644 index 0000000..958ac6d --- /dev/null +++ b/config.local.import-ourarches @@ -0,0 +1,14 @@ +#!/hint/bash + +PKGREPOS=( + # packages + core extra testing staging kde-unstable gnome-unstable + # community + community{,-testing,-staging} + multilib{,-testing,-staging} +) +# ARCHES should list arches that ./config.local.parabola has but +# there's no Arch Linux upstream for. +ARCHES=(ppc64le) +PKGPOOL='pool/import-ourarches' +SRCPOOL='sources/import-ourarches' diff --git a/config.local.packages b/config.local.packages index 34aab35..403fc9c 100644 --- a/config.local.packages +++ b/config.local.packages @@ -1,13 +1,23 @@ #!/hint/bash PKGREPOS=('core' 'extra' 'testing' 'staging' 'kde-unstable' 'gnome-unstable') +ARCHES=('x86_64') PKGPOOL='pool/packages' SRCPOOL='sources/packages' -SVNREPO='file:///srv/repos/svn-packages/svn' -SVNUSER='svn-packages' -TESTING_REPO='testing' -STABLE_REPOS=('core' 'extra') -CLEANUP_DESTDIR="/srv/repos/svn-packages/package-cleanup" -SOURCE_CLEANUP_DESTDIR="/srv/repos/svn-packages/source-cleanup" -TMPDIR="/srv/repos/svn-packages/tmp" +# "tags" are repo-arch pairs +ARCHTAGS=("${PKGREPOS[@]/%/-x86_64}") +ARCHPATH='$repo/os/$arch' +ARCHPKGPOOL='pool/packages' +ARCHSRCPOOL='sources/packages' + +# Of the few tier-1 mirrors with the 'sources/' folder: +# +# $ ./config-list-mirrors-archlinux --require=sources | jq -r '.[].url' +# rsync://archlinux.c3sl.ufpr.br/archlinux/ +# rsync://mirrors.kernel.org/archlinux/ +# rsync://ftp.acc.umu.se/mirror/archlinux/ +# +# ftp.acc.umu.se (Sweden) has (by far) the best ping time from +# winston.parabola.nu (Iceland). +ARCHMIRROR='rsync://ftp.acc.umu.se/mirror/archlinux/' diff --git a/config.local.parabola b/config.local.parabola index 424aff6..441e00a 100644 --- a/config.local.parabola +++ b/config.local.parabola @@ -1,9 +1,6 @@ #!/hint/bash PKGREPOS=( - 'core' 'testing' 'extra' 'community' 'multilib' 'multilib-testing' - 'build-support' - libre{,-multilib}{,-testing} pcr{,-multilib}{,-testing} @@ -16,5 +13,6 @@ PKGREPOS=( "~$USER" ) +ARCHES=(x86_64 i686 armv7h ppc64le) PKGPOOL='pool/parabola' SRCPOOL='sources/parabola' diff --git a/db-functions b/db-functions index 75dedad..0792757 100644 --- a/db-functions +++ b/db-functions @@ -34,6 +34,23 @@ mv_acl() { rm -f "$1" } +# just like mv -f, but won't break relative symlinks +mv_ln() { + if [[ -L $1 ]]; then + if [[ -d $2 ]]; then + set -- "$1" "$2/${1##*/}" + fi + if [[ ! $1 -ef $2 ]]; then + local target + target=$(readlink -f -- "$1") || return 1 + ln -srfT -- "$target" "$2" || return 1 + fi + rm -f -- "$1" + else + mv -f -- "$1" "$2" + fi +} + # set up general environment WORKDIR=$(mktemp -dt "${0##*/}.XXXXXXXXXX") LOCKS=() diff --git a/db-import-any b/db-import-any new file mode 100755 index 0000000..7827443 --- /dev/null +++ b/db-import-any @@ -0,0 +1,68 @@ +#!/bin/bash +# Releases 'any' packages from Arch arches to our arches + +set -eu -o pipefail +source "$(dirname "$(readlink -e "$0")")/config" +source "$(dirname "$(readlink -e "$0")")/db-import-any.conf" +source "$(librelib messages)" +setup_traps + +# usage: expac_file <file.db> <expac_args> +# +# Uses the ${WORKDIR} global +expac_file() { + local dbfile=$1 + local args=("${@:2}") + + local reponame=${dbfile##*/} + reponame=${reponame%%.*} + + mkdir -p -- "${WORKDIR}/expac/root" + cat >"${WORKDIR}/expac/pacman.conf" <<-EOT + [options] + RootDir = ${WORKDIR}/expac/root + DBPath = ${WORKDIR}/expac/root + + [${reponame}] + Server = file://$(realpath --no-symlinks -- "${dbfile%/*}") + EOT + + fakeroot pacman --config="${WORKDIR}/expac/pacman.conf" -Syy >/dev/null + # expac exits with non-zero on emtpy databases, so ignore errors + expac --config="${WORKDIR}/expac/pacman.conf" --sync "${args[@]}" || true +} + +db_list_any_pkgfiles() { + local dbfile="$1" + expac_file "$dbfile" '%a %f' | awk '$1 == "any" { print $2 }' +} + +main() { + WORKDIR=$(mktemp -dt "${0##*/}.XXXXXXXXXX") + readonly WORKDIR + trap "rm -rf -- ${WORKDIR@Q}" EXIT + + local repo arch + for repo in "${PKGREPOS[@]}"; do + msg "Processing %s..." "${repo}" + mkdir -p -- "${WORKDIR}/staging/${repo}" + # Look for arch=(any) packages that exist in + # ${BASEARCH} for this repo but is missing from one or + # more of ${ARCHES[@]}. + db_list_any_pkgfiles "${FTP_BASE}/${repo}/os/${BASEARCH}/${repo}.db" > "${WORKDIR}/base.txt" + for arch in "${ARCHES[@]}"; do + [[ $arch != "$BASEARCH" ]] || continue + db_list_any_pkgfiles "${FTP_BASE}/${repo}/os/${arch}/${repo}.db" > "${WORKDIR}/arch.txt" + comm -23 "${WORKDIR}/base.txt" "${WORKDIR}/arch.txt" + done \ + | sort -u \ + | xargs -d '\n' -r -n1 -- printf '%s/%s\n' "${FTP_BASE}/${repo}/os/${BASEARCH}" \ + | sed 's/.*/&\n&.sig/' \ + | xargs -d '\n' -r -- ln -srv -t "${WORKDIR}/staging/${repo}" -- + done + + msg "Running db-update..." + STAGING=${WORKDIR}/staging db-update +} + +main "$@" diff --git a/db-import-any.conf b/db-import-any.conf new file mode 100644 index 0000000..65fc58e --- /dev/null +++ b/db-import-any.conf @@ -0,0 +1,4 @@ +#!/hint/bash + +# The architecture to compare with +BASEARCH='x86_64' diff --git a/db-import-archlinuxarm-src b/db-import-archlinuxarm-src new file mode 100755 index 0000000..b0c1181 --- /dev/null +++ b/db-import-archlinuxarm-src @@ -0,0 +1,125 @@ +#!/bin/bash + +set -e + +source "$(dirname "$(readlink -e "$0")")/config" +source "$(dirname "$(readlink -e "$0")")/db-import-archlinuxarm.conf" + +# Steps +# * Sync abs +# * Download blacklist.txt +# * Sync abslibre from abs excluding from blacklist +# * Create repo.abs.tar.gz tarballs + +function sync_abs() { + # sync from ${REPO}.abs.tar.gz files in ALARM mirrors + local TMP_SUBDIR="${tmpdir}/abs-alarm/" + rm -rf -- "$TMP_SUBDIR" "$ABS_ROOT" + mkdir -- "$TMP_SUBDIR" + for ARCH in armv7h; do + for REPO in "${ARCHTAGS[@]%-*}"; do + rsync ${SYNCARGS} \ + rsync://${ABS_SERVER}/${ARCH}/${REPO}/${REPO}.abs.tar.gz \ + "$TMP_SUBDIR" || return $? + mkdir -p -- "${ABS_ROOT}/${ARCH}" + bsdtar xf "${TMP_SUBDIR}/${REPO}.abs.tar.gz" \ + -C "${ABS_ROOT}/${ARCH}" + done + done + + # add changes by ALARM since those aren't present in .abs.tar.gz files + if ! git -C "$ALARM_ROOT" pull &>/dev/null; then + rm -rf -- "$ALARM_ROOT" + if ! git clone "$ALARM_GIT" "$ALARM_ROOT"; then + printf "[FAILED]\n" + return 1 + fi + fi + # FIXME: $ALARM_ROOT doesn't separate packages by architecture + for ARCH in armv7h; do + for REPO in "${ARCHTAGS[@]%-*}"; do + rsync -mrtq --no-p --no-o --no-g --exclude='README' \ + "${ALARM_ROOT}/${REPO}/" \ + "${ABS_ROOT}/${ARCH}/${REPO}" + done + done + + # fix some permissions + find "${ABS_ROOT}" -type d -print0 | xargs -0 chmod 755 + find "${ABS_ROOT}" -type f -print0 | xargs -0 chmod 644 +} + +function get_blacklist() { + libreblacklist update + if ! libreblacklist cat | libreblacklist get-pkg | sort -u | sed "s/^/**\//" > ${BLFILE}; then + printf "[FAILED]\n" + return 1 + fi + + # Prevent using an empty blacklist + [ $(wc -l ${BLFILE} | cut -d " " -f1) -eq 0 ] && return 1 + + printf "[OK]\n" +} + +function sync_abs_libre() { + + # Clone ABSLibre git repo + rm -rf -- "$tmpdir/abslibre" + git clone "$ABSLIBRE_GIT" "$tmpdir/abslibre" + + # Sync from ABS and then sync from ABSLibre + printf ":: Syncing ABSLibre...\t" + if ! rsync ${SYNCARGS} --delete-excluded --exclude-from=${BLFILE} ${ABS_ROOT} ${ABSLIBRE_ROOT}; then + printf "[FAILED]\n" + return 1 + fi + for ARCH in armv7h; do + if ! rsync -v -mrtq --no-motd --no-p --no-o --no-g --quiet --exclude=.git/ "$tmpdir/abslibre/" ${ABSLIBRE_ROOT}/${ARCH}/; then + printf "[FAILED]\n" + return 1 + fi + done + + # fix some permissions + find "${ABSLIBRE_ROOT}" -type d -print0 | xargs -0 chmod 755 + find "${ABSLIBRE_ROOT}" -type f -print0 | xargs -0 chmod 644 + + printf "[OK]\n" +} + +# Create .abs.tar.gz tarballs +create_tarballs() { + for repo in ${ABSLIBRE_ROOT}/armv7h/*; do + baserepo=${repo##*/} + arch=$(basename $(dirname $repo)) + + # Remove the old one + mkdir -p $FTP_BASE/$baserepo/os/$arch/ + rm -fv $FTP_BASE/$baserepo/os/$arch/$baserepo.abs.tar.gz + # Create a new one joining arch and any + # Remove the first part of the path (it could be $repo but any isn't hit) + include=($repo/*) + if [[ -d ${ABSLIBRE_ROOT}/any/${baserepo}/ ]]; then + include+=(${ABSLIBRE_ROOT}/any/${baserepo}/*) + fi + bsdtar -czf $FTP_BASE/$baserepo/os/$arch/$baserepo.abs.tar.gz \ + -s ":${ABSLIBRE_ROOT}/[a-z0-9_]\+/[a-z]\+::" \ + "${include[@]}" + done +} + +main() { + trap 'rm -rf -- "$tmpdir"' EXIT + tmpdir=$(mktemp --tmpdir -d "${0##*/}.XXXXXXXXXX") + + BLFILE=${tmpdir}/blacklist.txt + mkdir -p -- "$ABSLIBRE_ROOT" "$ABS_ROOT" "$ALARM_ROOT" + + sync_abs + get_blacklist + sync_abs_libre + create_tarballs +} + +main "$@" diff --git a/db-import-archlinuxarm.conf b/db-import-archlinuxarm.conf new file mode 100644 index 0000000..c4b0b1c --- /dev/null +++ b/db-import-archlinuxarm.conf @@ -0,0 +1,15 @@ +#!/hint/bash + +# NB: I am unsure if the presence or absence of a trailing slash in +# the _ROOT variables is significant. -- lukeshu +SYNCARGS='-mrtv --no-motd --delete-after --no-p --no-o --no-g --quiet --exclude .~tmp~/' +ABSLIBRE_ROOT=/srv/repo/db-import-archlinuxarm-src/abslibre +ABSLIBRE_GIT=https://git.parabola.nu/abslibre.git/ +ABS_ROOT='/srv/repo/db-import-archlinuxarm-src/abs/' +ABS_SERVER="${ARCHMIRROR#rsync://}/" +ALARM_GIT='https://github.com/archlinuxarm/PKGBUILDs' +ALARM_ROOT='/srv/repo/db-import-archlinuxarm-src/alarm/' + +if [[ -n ${DBIMPORT_CONFIG} ]]; then + source "${DBIMPORT_CONFIG}" +fi diff --git a/db-import-keyring b/db-import-keyring new file mode 100755 index 0000000..ef37a5a --- /dev/null +++ b/db-import-keyring @@ -0,0 +1,98 @@ +#!/bin/bash +# Imports to [libre] specific packages from upstream Arch repos. +# +# License: GPLv3 + +set -eu -o pipefail +shopt -s extglob globstar nullglob +source "$(librelib messages)" +setup_traps + +# usage: expac_file <file.db> <expac_args> +# +# Uses the ${WORKDIR} global +expac_file() { + local dbfile=$1 + local args=("${@:2}") + + local reponame=${dbfile##*/} + reponame=${reponame%%.*} + + mkdir -p -- "${WORKDIR}/expac/root" + cat >"${WORKDIR}/expac/pacman.conf" <<-EOT + [options] + RootDir = ${WORKDIR}/expac/root + DBPath = ${WORKDIR}/expac/root + + [${reponame}] + Server = file://$(realpath --no-symlinks -- "${dbfile%/*}") + EOT + + fakeroot pacman --config="${WORKDIR}/expac/pacman.conf" -Syy >/dev/null + # expac exits with non-zero on emtpy databases, so ignore errors + expac --config="${WORKDIR}/expac/pacman.conf" --sync "${args[@]}" || true +} + +# usage: get_repo_rsync_dir <upstream> <tag> +get_repo_rsync_dir() { + if ! grep -q ARCHMIRROR -- "${LOCAL_CONFIG%/*}/config.local.$1"; then + msg "db-import-keyring.conf: Doesn't look like an upstream: %s" "$1" >&2 + exit $EXIT_NOTCONFIGURED + fi + ( + source "${LOCAL_CONFIG%/*}/config.local.$1" >&2 + printf '%s\n' "${ARCHMIRROR}/$(repo=${2%-*} arch=${2##*-} envsubst '$repo $arch' <<<"$ARCHPATH")" + ) +} + +main() { + if [[ $# -ne 0 ]]; then + msg 'usage: %s' "${0##*/}" + exit $EXIT_INVALIDARGUMENT + fi + + local config_dir + config_dir="$(dirname "$(readlink -e "$0")")" + source "${config_dir}/config" # for FTP_BASE + source "${config_dir}/db-import-keyring.conf" # for KEYRINGS + + WORKDIR=$(mktemp -dt "${0##*/}.XXXXXXXXXX") + readonly WORKDIR + trap "rm -rf -- ${WORKDIR@Q}" EXIT + + ############################################################## + + local fromspec upstream tag pkgname repo repo_rsync_dir from_pkgver to_pkgver filename + for fromspec in "${KEYRINGS[@]}"; do + msg "Processing %s" "$fromspec" + IFS=/ read -r upstream tag pkgname <<<"$fromspec" + repo_rsync_dir=$(get_repo_rsync_dir "$upstream" "$tag") + repo=${tag%-*} + + # Download the upstream database + msg2 'Downloading %s' "${repo_rsync_dir}/${repo}.db" + mkdir -p -- "${WORKDIR}/dbs/${upstream}" + rsync --no-motd -mrtLH --no-p "${repo_rsync_dir}/${repo}.db" "${WORKDIR}/dbs/${upstream}/${tag}.db" + + # Compare it against the libre database + from_pkgver=$(expac_file "${WORKDIR}/dbs/${upstream}/${tag}.db" '%v' "$pkgname") + to_pkgver=$(expac_file "${FTP_BASE}/libre/os/x86_64/libre.db" '%v' "$pkgname") + + # And download a new keyring if necessary + if [[ "$from_pkgver" == "$to_pkgver" ]]; then + msg2 "Up to date: %s == %s" "$from_pkgver" "$to_pkgver" + else + msg2 "Updating: %s != %s" "$from_pkgver" "$to_pkgver" + filename=$(expac_file "${WORKDIR}/dbs/${upstream}/${tag}.db" '%f' "$pkgname") + mkdir -p -- "${WORKDIR}/staging/libre/" + rsync --no-motd -mrtLH --no-p "${repo_rsync_dir}/${filename}"{,.sig} "${WORKDIR}/staging/libre/" + fi + done + + ############################################################## + + msg "Running db-update..." + STAGING=${WORKDIR}/staging db-update +} + +main "$@" diff --git a/db-import-keyring.conf b/db-import-keyring.conf new file mode 100644 index 0000000..8e78ff8 --- /dev/null +++ b/db-import-keyring.conf @@ -0,0 +1,7 @@ +#!/hint/bash + +KEYRINGS=( + # Format is "UPSTREAM/TAG/PKGNAME" + archlinux32/core-i686/archlinux32-keyring + archlinuxarm/core-armv7h/archlinuxarm-keyring +) diff --git a/db-import-pkg b/db-import-pkg new file mode 100755 index 0000000..bff210c --- /dev/null +++ b/db-import-pkg @@ -0,0 +1,360 @@ +#!/bin/bash +# Imports packages from upstream Arch repos. +# +# License: GPLv3 + +set -e -o pipefail +shopt -s extglob globstar nullglob +source "$(librelib messages)" +setup_traps + +indent() { + /usr/lib/libretools/chroot/indent ' | ' +} + +# usage: expac_file <file.db> <expac_args> +# +# Uses the ${WORKDIR} global +expac_file() { + local dbfile=$1 + local args=("${@:2}") + + local reponame=${dbfile##*/} + reponame=${reponame%%.*} + + mkdir -p -- "${WORKDIR}/expac/root" + cat >"${WORKDIR}/expac/pacman.conf" <<-EOT + [options] + RootDir = ${WORKDIR}/expac/root + DBPath = ${WORKDIR}/expac/root + + [${reponame}] + Server = file://$(realpath --no-symlinks -- "${dbfile%/*}") + EOT + + fakeroot pacman --config="${WORKDIR}/expac/pacman.conf" -Syy >/dev/null + # expac exits with non-zero on emtpy databases, so ignore errors + expac --config="${WORKDIR}/expac/pacman.conf" --sync "${args[@]}" || true +} + +# usage: fetch_dbs <from> <into> +# +# Fetch excluding everything but db files +# TODO: we could be doing without things other than what is in +# ${ARCHTAGS[@]} +fetch_dbs() { + local extra=() + if [[ $arg_verbose = true ]]; then + extra+=(-v) + fi + + rsync "${extra[@]}" --no-motd -mrtLH --no-p \ + --include="*/" \ + --include="*.db" \ + --exclude="*" \ + --delete-after \ + "$1" "$2" +} + +# usage: get_repo_dir <repo> <arch> +# +# Prints repo directory path for the given <repo> <arch> combination, +# relative to the rsync root. +get_repo_dir() { + repo=$1 arch=$2 envsubst '$repo $arch' <<<"$ARCHPATH" +} + +# usage: db_list_pkgs <path-to-db> +# +# Prints a list of packages within a given <path-to-db>, one-per-line, +# in the format: +# +# pkgname [epoch:]pkgver-pkgrel +db_list_pkgs() { + expac_file "$1" '%n %v' | sort -u +} + +# usage: filter_blacklisted <FULL_LIST >FILTERED_LIST +# +# Given a list of packages in the format: +# +# pkgname [epoch:]pkgver-pkgrel +# +# filter out all of the packages named in blacklist.txt. +filter_blacklisted() { + sort -u | join -v1 \ + - \ + <(libreblacklist cat | libreblacklist get-pkg | sort -u) +} + +# usage: sync_pool <from> <path-to-whitelist> <into> +# +# Sync excluding everything but whitelist +sync_pool() { + local -r _from=$1 _whitelist=$2 _into=$3 + + local extra=() + if [[ $arg_verbose = true ]]; then + extra+=(-v) + fi + + mkdir -p -- "$_into" + msg2 "Retrieving up to %d files from %s pool" \ + "$(wc -l < "$_whitelist")" \ + "$(basename "$_into")" + + # *Don't delete-after*, this is the job of + # cleanup scripts. It will remove our packages too + rsync "${extra[@]}" --no-motd -rtlH --no-t \ + --delay-updates \ + --safe-links \ + --include-from="$_whitelist" \ + --exclude="*" \ + "$_from" \ + "$_into" +} + +# Main function. Process the databases and get the libre packages +# Outline: +# 1. Fetch package info +# * Get blacklist.txt +# * Get repo.db from an Arch-like repo +# 2. Figure out what we want +# * Generate textfiles describing the current repo state, (using +# blacklist.txt) the desired repo state, and how to get from one +# to the other. +# 3. Fetch the packages we want +# * Symlink to files with the same name in INHERIT pools +# * sync_pool to download the others +# 4. Modify the repos +# * db-move +# * db-update +# * db-remove +# +# Files: +# (misc) +# - ${WORKDIR}/expac/ : Scratch directory for expac_file() +# (download) +# - ${WORKDIR}/rsync/ : Where we download '.db' files to +# - ${WORKDIR}/staging/${repo} : Where we download packages to +# - ${WORKDIR}/staging/${SRCPOOL} : Where we download sources to +# (analysis) +# - ${FTP_BASE}/${INHERIT} : Where we look for duplicate files +# - ${FTP_BASE}/.../${repo}.db : Where we generate ${WORKDIR}/old/ from +# - ${WORKDIR}/old/ : .txt files describing the way the repos are +# - ${WORKDIR}/new/ : .txt files describing the way we want them to be +# - ${WORKDIR}/dif/ : .txt files describing how to make it happen +# - ${WORKDIR}/${tag}.pkg.whitelist : List of package filenames to download +# - ${WORKDIR}/all.src.whitelist : Glob list of source-package filenames to download +# (release) +# - ${WORKDIR}/staging/ : STAGING= directory for db-update +main() { + ############################################################## + # 0. Initialization # + ############################################################## + + # Run as `V=true db-import-pkg` to get verbose output + declare -r arg_verbose="$V" + + # Print usage message + if [[ $# -ne 0 ]] || [[ -z "$DBSCRIPTS_CONFIG" ]] || ! grep -q ARCHMIRROR -- "$DBSCRIPTS_CONFIG"; then + msg 'usage: [V=true] DBSCRIPTS_CONFIG=/path/to/file %s' "${0##*/}" + exit $EXIT_INVALIDARGUMENT + fi + + local config_file + config_file="$(dirname "$(readlink -e "$0")")/config" + source "$config_file" + + local ret=0 varname varref + for varname in PKGEXTS FTP_BASE PKGPOOL SRCPOOL; do + if [[ -z ${!varname:-} ]] || is_array "$varname"; then + print "Configure '%s' as a non-empty string in %q (or %q):" "$varname" "$config_file" "$LOCAL_CONFIG" + ret=$EXIT_NOTCONFIGURED + fi + done + for varname in ARCHMIRROR ARCHPATH; do # optional: ARCHPKGPOOL ARCHSRCPOOL + if [[ -z ${!varname:-} ]] || is_array "$varname"; then + print "Configure '%s' as a non-empty string in DBSCRIPTS_CONFIG=%q (did you set DBSCRIPTS_CONFIG correctly?):" "$varname" "$LOCAL_CONFIG" + ret=$EXIT_NOTCONFIGURED + fi + done + for varname in ARCHTAGS; do # optional: INHERIT + declare -n varref="$varname" + if (( ${#varref[*]} == 0 )) || ! is_array "$varname"; then + print "Configure '%s' as a non-empty array in DBSCRIPTS_CONFIG=%q (did you set DBSCRIPTS_CONFIG correctly?):" "$varname" "$LOCAL_CONFIG" + ret=$EXIT_NOTCONFIGURED + fi + done + + WORKDIR=$(mktemp -dt "${0##*/}.XXXXXXXXXX") + readonly WORKDIR + trap "rm -rf -- ${WORKDIR@Q}" EXIT + + ############################################################## + # 1. Fetch package info # + ############################################################## + + # Get the blacklisted packages + libreblacklist update + + # Sync the repos databases + msg 'Downloading .db and .files files to import' + mkdir "${WORKDIR}/rsync" + fetch_dbs "${ARCHMIRROR}/" "${WORKDIR}/rsync" + + ############################################################## + # 2. Figure out what we want # + ############################################################## + + mkdir "${WORKDIR}"/{old,new,dif} + local _tag _repo _arch db_file + for _tag in "${ARCHTAGS[@]}"; do + _repo=${_tag%-*} + _arch=${_tag##*-} + # FIXME: this should use db-functions to lock the + # repos while we read them. + db_file="${FTP_BASE}/${_repo}/os/${_arch}/${_repo}.db" + db_list_pkgs "$db_file" > "${WORKDIR}/old/${_tag}.txt" + + db_file="${WORKDIR}/rsync/$(get_repo_dir "${_repo}" "${_arch}")/${_repo}.db" + db_list_pkgs "$db_file" | filter_blacklisted > "${WORKDIR}/new/${_tag}.txt" + done + + # We now have ${WORKDIR}/old/ describing the way the repos + # are, and ${WORKDIR}/new/ describing the way we want them to + # be. We now create ${WORKDIR}/dif/ describing how to get + # from point A to point B. + cat "${WORKDIR}"/old/*-*.txt | sort -u > "${WORKDIR}/old/all.txt" + # db-move <repo-from> <repo-to> <pkgname|pkgbase> ... + # + # db-move doesn't allow us to limit the operation to a + # specific arch, but the DBSCRIPTS_CONFIG will limit what + # arches it applies to, and we currently only import 1 arch + # from each upstream. + local tag_from tag_to + for tag_from in "${ARCHTAGS[@]}"; do + arch_from=${tag_from##*-} + for tag_to in "${ARCHTAGS[@]}"; do + arch_to=${tag_to##*-} + [[ $tag_from != $tag_to ]] || continue + [[ $arch_from == $arch_to ]] || continue + + comm -12 \ + "${WORKDIR}/old/${tag_from}.txt" \ + "${WORKDIR}/new/${tag_to}.txt" \ + >> "${WORKDIR}/dif/move:${tag_from}:${tag_to}.txt" + done + done + # db-update + local tag + for tag in "${ARCHTAGS[@]}"; do + comm -13 \ + "${WORKDIR}/old/all.txt" \ + "${WORKDIR}/new/${tag}.txt" \ + > "${WORKDIR}/dif/update:${tag}.txt" + done + # db-remove <repo> <arch> <pkgname|pkgbase> ... + for tag in "${ARCHTAGS[@]}"; do + # pkgnames (that need to leave this tag) AND (haven't + # already been removed from this tag by db-move). + comm -23 \ + <(cut -d' ' -f1 -- "${WORKDIR}/old/${tag}.txt" | sort -u) \ + <(cut -d' ' -f1 -- "${WORKDIR}/new/${tag}.txt" "${WORKDIR}/dif/move:${tag}":*.txt | sort -u) \ + > "${WORKDIR}/dif/remove:${tag}.txt" + done + + ############################################################## + # 3. Fetch the packages we want # + ############################################################## + + # For some packages, "fetch" means to create a symlink to a + # pool we INHERIT from. For others, it means to actually + # download it from arg_upstream with rsync. + for tag in "${ARCHTAGS[@]}";do + msg "Processing %s" "$tag" + repo=${tag%-*} + arch=${tag##*-} + mkdir -p -- "${WORKDIR}/staging/${repo}" + + local pkgname filename pgpsig + while read -r pkgname filename pgpsig; do + local pool staged=false + for pool in "$PKGPOOL" "${INHERIT[@]}"; do + filepath=("${FTP_BASE}/${pool}/${filename}") + if [[ -f $filepath && ! -h $filepath ]]; then + ln -srT -- "$filepath" "${WORKDIR}/staging/${repo}/${filename}" + ln -srT -- "$filepath".sig "${WORKDIR}/staging/${repo}/${filename}".sig + staged=true + break + fi + done + if ! $staged; then + printf '%s\n' "$filename"{,.sig} >> "${WORKDIR}/${tag}.pkg.whitelist" + printf '%s\n' "${filename%$PKGEXTS}*.src.tar*" >> "${WORKDIR}/all.src.whitelist" + fi + done < <( + mapfile -t pkgnames < <(cut -d' ' -f1 <"${WORKDIR}/dif/update:${tag}.txt") + db_file="${WORKDIR}/rsync/$(get_repo_dir "${repo}" "${arch}")/${repo}.db" + if (( ${#pkgnames[@]} > 0 )); then + expac_file "$db_file" '%n %f %g' "${pkgnames[@]}" + fi + ) + if [[ -f "${WORKDIR}/${tag}.pkg.whitelist" ]]; then + sync_pool \ + "${ARCHMIRROR}/${ARCHPKGPOOL:-$(get_repo_dir "${repo}" "${arch}")}/" \ + "${WORKDIR}/${tag}.pkg.whitelist" \ + "${WORKDIR}/staging/${repo}/" + fi + done + if [[ -n ${ARCHSRCPOOL:-} && -f "${WORKDIR}/all.src.whitelist" ]]; then + sync_pool \ + "${ARCHMIRROR}/${ARCHSRCPOOL}/" \ + "${WORKDIR}/all.src.whitelist" \ + "${WORKDIR}/staging/${SRCPOOL}/" + fi + + ############################################################## + # 4. Modify the repos # + ############################################################## + + msg "Modifying the actual repos" + + # db-move + msg2 'Step 1 of 3: db-move' + for tag_from in "${ARCHTAGS[@]}"; do + repo_from=${tag_from%-*} + arch_from=${tag_from##*-} + for tag_to in "${ARCHTAGS[@]}"; do + repo_to=${tag_to%-*} + arch_to=${tag_to##*-} + [[ $tag_from != $tag_to ]] || continue + [[ $arch_from == $arch_to ]] || continue + + plain '%s -> %s' "$tag_from" "$tag_to" + < "${WORKDIR}/dif/move:${tag_from}:${tag_to}.txt" \ + cut -d' ' -f1 | \ + sed 's/^/pkgname=/' | \ + xargs -r -d $'\n' db-move "$repo_from" "$repo_to" |& \ + indent + done + done + # db-update + msg2 'Step 2 of 3: db-update' + STAGING=${WORKDIR}/staging db-update |& indent + # db-remove + msg2 'Step 3 of 3: db-remove' + for tag in "${ARCHTAGS[@]}"; do + repo=${tag%-*} + arch=${tag##*-} + + plain '%s' "$tag" + < "${WORKDIR}/dif/remove:${tag}.txt" \ + cut -d' ' -f1 | \ + sed 's/^/pkgname=/' | \ + xargs -r -d $'\n' db-remove "$repo" "$arch" |& \ + indent + done +} + +main "$@" @@ -51,6 +51,15 @@ for repo in "${repos[@]}"; do if (( $? == 0 )); then for pkg in "${pkgs[@]}"; do if [[ -h ${pkg} ]]; then + # As a special hack for + # 1. inheriting arch=(any) packages between db-import + # upstreams, and + # 2. re-adding previously removed binaries that have not been + # cleaned up yet, + # allow symlinks that point to packages under ${FTP_BASE}/pool/. + if [[ -f ${pkg} && "$(readlink -f -- "$pkg")" = "${FTP_BASE}"/pool/*${PKGEXTS} ]]; then + continue + fi die "Package %s is a symbolic link" "$repo/${pkg##*/}" fi if ! check_pkgfile "${pkg}"; then @@ -66,7 +75,11 @@ for repo in "${repos[@]}"; do die "Package %s already exists in another repository" "$repo/${pkg##*/}" fi if ! check_packager "${pkg}"; then - die "Package %s does not have a valid packager" "$repo/${pkg##*/}" + if [[ $DBSCRIPTS_CONFIG = *.archlinux32 ]]; then + warning "Package %s does not have a valid packager" "$repo/${pkg##*/}" + else + die "Package %s does not have a valid packager" "$repo/${pkg##*/}" + fi fi if ! check_buildinfo "${pkg}"; then die "Package %s does not have a .BUILDINFO file" "$repo/${pkg##*/}" @@ -94,12 +107,12 @@ for repo in "${repos[@]}"; do msg2 '%s (%s)' "$pkgfile" "$pkgarch" # any packages might have been moved by the previous run if [[ -f ${pkg} ]]; then - mv "${pkg}" "$FTP_BASE/${PKGPOOL}" + mv_ln "${pkg}" "$FTP_BASE/${PKGPOOL}" fi ln -s "../../../${PKGPOOL}/${pkgfile}" "$FTP_BASE/$repo/os/${pkgarch}" # also move signatures if [[ -f ${pkg}.sig ]]; then - mv "${pkg}.sig" "$FTP_BASE/${PKGPOOL}" + mv_ln "${pkg}.sig" "$FTP_BASE/${PKGPOOL}" fi if [[ -f $FTP_BASE/${PKGPOOL}/${pkgfile}.sig ]]; then ln -s "../../../${PKGPOOL}/${pkgfile}.sig" "$FTP_BASE/$repo/os/${pkgarch}" diff --git a/systemd/db-import-any.service b/systemd/db-import-any.service new file mode 100644 index 0000000..e50510c --- /dev/null +++ b/systemd/db-import-any.service @@ -0,0 +1,10 @@ +[Unit] +Description=db-import-any + +[Service] +Type=oneshot +User=repo +Environment=DBSCRIPTS_CONFIG=/etc/dbscripts/config.local.import-ourarches +ExecStart=/usr/bin/db-import-any + +PrivateTmp=true diff --git a/systemd/db-import-any.timer b/systemd/db-import-any.timer new file mode 100644 index 0000000..a14e634 --- /dev/null +++ b/systemd/db-import-any.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Daily db-import-any + +[Timer] +OnCalendar=daily +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/systemd/db-import-keyring.service b/systemd/db-import-keyring.service new file mode 100644 index 0000000..96ec7bd --- /dev/null +++ b/systemd/db-import-keyring.service @@ -0,0 +1,11 @@ +[Unit] +Description=db-import-keyring +Wants=network-online.target +After=network-online.target + +[Service] +Type=oneshot +User=repo +ExecStart=/usr/bin/db-import-keyring + +PrivateTmp=true diff --git a/systemd/db-import-keyring.timer b/systemd/db-import-keyring.timer new file mode 100644 index 0000000..3818733 --- /dev/null +++ b/systemd/db-import-keyring.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Daily db-import-keyring + +[Timer] +OnCalendar=daily +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/systemd/db-import@.service b/systemd/db-import@.service new file mode 100644 index 0000000..1509449 --- /dev/null +++ b/systemd/db-import@.service @@ -0,0 +1,12 @@ +[Unit] +Description=db-import %I +Wants=network-online.target +After=network-online.target + +[Service] +Type=oneshot +User=repo +Environment=DBSCRIPTS_CONFIG=/etc/dbscripts/config.local.%I +ExecStart=/usr/bin/db-import-pkg + +PrivateTmp=true diff --git a/systemd/db-import@.timer b/systemd/db-import@.timer new file mode 100644 index 0000000..77570d6 --- /dev/null +++ b/systemd/db-import@.timer @@ -0,0 +1,8 @@ +[Unit] +Description=Daily db-import %I + +[Timer] +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/systemd/db-import@archlinux32.timer.d/time.conf b/systemd/db-import@archlinux32.timer.d/time.conf new file mode 100644 index 0000000..028665c --- /dev/null +++ b/systemd/db-import@archlinux32.timer.d/time.conf @@ -0,0 +1,2 @@ +[Timer] +OnCalendar=*-*-* 08:13:00 diff --git a/systemd/db-import@archlinuxarm.timer.d/time.conf b/systemd/db-import@archlinuxarm.timer.d/time.conf new file mode 100644 index 0000000..455b0bc --- /dev/null +++ b/systemd/db-import@archlinuxarm.timer.d/time.conf @@ -0,0 +1,2 @@ +[Timer] +OnCalendar=*-*-* 16:13:00 diff --git a/systemd/db-import@community.timer.d/time.conf b/systemd/db-import@community.timer.d/time.conf new file mode 100644 index 0000000..3ec5e02 --- /dev/null +++ b/systemd/db-import@community.timer.d/time.conf @@ -0,0 +1,2 @@ +[Timer] +OnCalendar=*-*-* 04:13:00 diff --git a/systemd/db-import@packages.timer.d/time.conf b/systemd/db-import@packages.timer.d/time.conf new file mode 100644 index 0000000..2561a96 --- /dev/null +++ b/systemd/db-import@packages.timer.d/time.conf @@ -0,0 +1,2 @@ +[Timer] +OnCalendar=*-*-* 00:13:00 diff --git a/test/cases/common.bats b/test/cases/common.bats index 8b4c3a0..337df48 100644 --- a/test/cases/common.bats +++ b/test/cases/common.bats @@ -1,10 +1,10 @@ load ../lib/common @test "commands display usage message by default" { - for cmd in db-move db-remove db-repo-add db-repo-remove; do + for cmd in db-move db-remove db-repo-add db-repo-remove db-import-pkg; do echo Testing $cmd run $cmd - (( $status == 1 )) + (( $status != 0 )) [[ $output == *'usage: '* ]] done } diff --git a/test/cases/db-import-any.bats b/test/cases/db-import-any.bats new file mode 100644 index 0000000..e6fc5c6 --- /dev/null +++ b/test/cases/db-import-any.bats @@ -0,0 +1,41 @@ +load ../lib/import + +@test "import any on Parabola repos" { + # Initial run + releasePackage libre 'pkg-any-a' + STAGING=$STAGING db-update + checkPackage libre pkg-any-a 1-1 + rmdir "$STAGING" + + # Add an arch + echo 'ARCHES+=(ppc64le)' >> "$DBSCRIPTS_CONFIG" + ARCHES+=(ppc64le) + db-init + + # Adding the arch broke checkPackage + if checkPackage libre pkg-any-a 1-1; then return 1; fi + # But db-import-any fixes it + db-import-any + checkPackage libre pkg-any-a 1-1 +} + +@test "import any on imported repos" { + cat >"${TMP}/config.local.import-ourarches" <<-eot + source ${DBSCRIPTS_CONFIG@Q} + source ../config.local.import-ourarches + PKGREPOS=(core testing) + eot + DBSCRIPTS_CONFIG="${TMP}/config.local.import-ourarches" db-init + + __releaseImportedPackage pkg-any-a x86_64 "$TMP/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz" "$TMP/rsyncd/archlinux/pool/packages" + __releaseImportedPackage pkg-simple-c x86_64 "$TMP/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz" "$TMP/rsyncd/archlinux/pool/packages" + DBSCRIPTS_CONFIG="${TMP}/config.local.packages" db-import-pkg + + [[ -z $(bsdtar tf "${TMP}/ftp/core/os/ppc64le/core.db") ]] + + DBSCRIPTS_CONFIG="${TMP}/config.local.import-ourarches" db-import-any + + bsdtar tf "${TMP}/ftp/core/os/ppc64le/core.db" pkg-any-a-1-1/desc + if bsdtar tf "${TMP}/ftp/core/os/ppc64le/core.db" pkg-simple-c-1-1; then return 1; fi + __isLinkTo "${TMP}/ftp/core/os/ppc64le/pkg-any-a-1-1-any.pkg.tar.xz" "${TMP}/ftp/pool/packages/pkg-any-a-1-1-any.pkg.tar.xz" +} diff --git a/test/cases/db-import-keyring.bats b/test/cases/db-import-keyring.bats new file mode 100644 index 0000000..d78dbb4 --- /dev/null +++ b/test/cases/db-import-keyring.bats @@ -0,0 +1,27 @@ +load ../lib/import + +@test "import keyrings" { + echo PKGPOOL='pool/parabola' >>"${TMP}/config.local" + mkdir -p "${TMP}/ftp/pool/parabola" + + __releaseImportedPackage archlinux32-keyring i686 "$TMP/rsyncd/archlinux32/i686/core/core.db.tar.gz" "$TMP/rsyncd/archlinux32/pool" + __releaseImportedPackage archlinuxarm-keyring armv7h "$TMP/rsyncd/archlinuxarm/armv7h/core/core.db.tar.gz" + + db-import-keyring + + bsdtar tf "$TMP/ftp/libre/os/x86_64/libre.db.tar.gz" archlinux32-keyring-1-1/desc + bsdtar tf "$TMP/ftp/libre/os/i686/libre.db.tar.gz" archlinux32-keyring-1-1/desc + bsdtar tf "$TMP/ftp/libre/os/armv7h/libre.db.tar.gz" archlinux32-keyring-1-1/desc + + bsdtar tf "$TMP/ftp/libre/os/x86_64/libre.db.tar.gz" archlinuxarm-keyring-1-1/desc + bsdtar tf "$TMP/ftp/libre/os/i686/libre.db.tar.gz" archlinuxarm-keyring-1-1/desc + bsdtar tf "$TMP/ftp/libre/os/armv7h/libre.db.tar.gz" archlinuxarm-keyring-1-1/desc + + __isLinkTo "$TMP/ftp/libre/os/x86_64/archlinux32-keyring-1-1-any.pkg.tar.xz" "$TMP/ftp/pool/parabola/archlinux32-keyring-1-1-any.pkg.tar.xz" + __isLinkTo "$TMP/ftp/libre/os/i686/archlinux32-keyring-1-1-any.pkg.tar.xz" "$TMP/ftp/pool/parabola/archlinux32-keyring-1-1-any.pkg.tar.xz" + __isLinkTo "$TMP/ftp/libre/os/armv7h/archlinux32-keyring-1-1-any.pkg.tar.xz" "$TMP/ftp/pool/parabola/archlinux32-keyring-1-1-any.pkg.tar.xz" + + __isLinkTo "$TMP/ftp/libre/os/x86_64/archlinuxarm-keyring-1-1-any.pkg.tar.xz" "$TMP/ftp/pool/parabola/archlinuxarm-keyring-1-1-any.pkg.tar.xz" + __isLinkTo "$TMP/ftp/libre/os/i686/archlinuxarm-keyring-1-1-any.pkg.tar.xz" "$TMP/ftp/pool/parabola/archlinuxarm-keyring-1-1-any.pkg.tar.xz" + __isLinkTo "$TMP/ftp/libre/os/armv7h/archlinuxarm-keyring-1-1-any.pkg.tar.xz" "$TMP/ftp/pool/parabola/archlinuxarm-keyring-1-1-any.pkg.tar.xz" +} diff --git a/test/cases/db-import-pkg.bats b/test/cases/db-import-pkg.bats new file mode 100644 index 0000000..f973c84 --- /dev/null +++ b/test/cases/db-import-pkg.bats @@ -0,0 +1,260 @@ +load ../lib/import + +# Run the command in a new mount namespace with /tmp remounted +# read-only, but with $TMP (which might be under /tmp) still writable. +# +# Arguments are passed as arguments to `sudo`. +__withRoTmp() { + local mount="mount -o bind ${TMP@Q}{,} && mount -o bind,remount,ro /tmp{,}" + local env=( + "DBSCRIPTS_CONFIG=${DBSCRIPTS_CONFIG}" + "XDG_CONFIG_HOME=${XDG_CONFIG_HOME}" + ) + sudo -- unshare -m -- sh -c "${mount} && sudo -u ${USER@Q} ${env[*]@Q} \$@" -- "$@" +} + +__db-import-pkg() { + local ret=0 + # Since common.bash->config.local sets TMPDIR=${TMP}/tmp, + # TMPDIR is necessarily != /tmp. + # Which means that if we try to write anything directly under /tmp, + # then we are erroneously disregarding TMPDIR. + # So, make /tmp read-only to make that be an error. + __withRoTmp db-import-pkg "$@" || ret=$? + # Verify that it cleaned up after itself and TMPDIR is empty + find "$TMPDIR" -mindepth 1 | diff - /dev/null + return $ret +} + +###################################################################### + +@test "import no blacklisted packages (x86_64)" { + __releaseImportedPackage slavery x86_64 "$TMP/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz" "$TMP/rsyncd/archlinux/pool/packages" + __releaseImportedPackage pkg-simple-c x86_64 "$TMP/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz" "$TMP/rsyncd/archlinux/pool/packages" + + DBSCRIPTS_CONFIG="${TMP}/config.local.packages" __db-import-pkg + + bsdtar xfO "$TMP/ftp/core/os/x86_64/core.db.tar.gz" pkg-simple-c-1-1/desc + __isLinkTo "$TMP/ftp/core/os/x86_64/pkg-simple-c-1-1-x86_64.pkg.tar.xz" "$TMP/ftp/pool/packages/pkg-simple-c-1-1-x86_64.pkg.tar.xz" + __doesNotExist "$TMP"/ftp/{core/os/x86_64,pool/packages,sources/packages}/slavery-* +} + +@test "import no blacklisted packages (i686)" { + __releaseImportedPackage slavery i686 "$TMP/rsyncd/archlinux32/i686/core/core.db.tar.gz" "$TMP/rsyncd/archlinux32/pool" + __releaseImportedPackage pkg-simple-c i686 "$TMP/rsyncd/archlinux32/i686/core/core.db.tar.gz" "$TMP/rsyncd/archlinux32/pool" + + DBSCRIPTS_CONFIG="${TMP}/config.local.archlinux32" __db-import-pkg + + bsdtar xfO "$TMP/ftp/core/os/i686/core.db.tar.gz" pkg-simple-c-1-1/desc + __isLinkTo "$TMP/ftp/core/os/i686/pkg-simple-c-1-1-i686.pkg.tar.xz" "$TMP/ftp/pool/archlinux32/pkg-simple-c-1-1-i686.pkg.tar.xz" + __doesNotExist "$TMP"/ftp/{core/os/i686,pool/archlinux32,sources/archlinux32}/slavery-* +} + +@test "import no blacklisted packages (armv7h)" { + __releaseImportedPackage slavery armv7h "$TMP/rsyncd/archlinuxarm/armv7h/core/core.db.tar.gz" + __releaseImportedPackage pkg-simple-c armv7h "$TMP/rsyncd/archlinuxarm/armv7h/core/core.db.tar.gz" + + DBSCRIPTS_CONFIG="${TMP}/config.local.archlinuxarm" __db-import-pkg + + bsdtar xfO "$TMP/ftp/core/os/armv7h/core.db.tar.gz" pkg-simple-c-1-1/desc + __isLinkTo "$TMP/ftp/core/os/armv7h/pkg-simple-c-1-1-armv7h.pkg.tar.xz" "$TMP/ftp/pool/alarm/pkg-simple-c-1-1-armv7h.pkg.tar.xz" + __doesNotExist "$TMP"/ftp/{core/os/alarm,pool/alarm,sources/alarm}/slavery-* +} + +@test "import DBs with no blacklisted packages" { + __releaseImportedPackage pkg-simple-c x86_64 "$TMP/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz" "$TMP/rsyncd/archlinux/pool/packages" + + DBSCRIPTS_CONFIG="${TMP}/config.local.packages" __db-import-pkg + + bsdtar xfO "$TMP/ftp/core/os/x86_64/core.db.tar.gz" pkg-simple-c-1-1/desc + __isLinkTo "$TMP/ftp/core/os/x86_64/pkg-simple-c-1-1-x86_64.pkg.tar.xz" "$TMP/ftp/pool/packages/pkg-simple-c-1-1-x86_64.pkg.tar.xz" +} + +@test "import updated packages" { + __releaseImportedPackage slavery x86_64 "$TMP/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz" "$TMP/rsyncd/archlinux/pool/packages" + __releaseImportedPackage pkg-simple-c x86_64 "$TMP/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz" "$TMP/rsyncd/archlinux/pool/packages" + + DBSCRIPTS_CONFIG="${TMP}/config.local.packages" __db-import-pkg + + bsdtar xfO "$TMP/ftp/core/os/x86_64/core.db.tar.gz" pkg-simple-c-1-1/desc + __isLinkTo "$TMP/ftp/core/os/x86_64/pkg-simple-c-1-1-x86_64.pkg.tar.xz" "$TMP/ftp/pool/packages/pkg-simple-c-1-1-x86_64.pkg.tar.xz" + + __updateImportedPackage pkg-simple-c + __releaseImportedPackage pkg-simple-c x86_64 "$TMP/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz" "$TMP/rsyncd/archlinux/pool/packages" + + DBSCRIPTS_CONFIG="${TMP}/config.local.packages" __db-import-pkg + + bsdtar xfO "$TMP/ftp/core/os/x86_64/core.db.tar.gz" pkg-simple-c-1-2/desc + __isLinkTo "$TMP/ftp/core/os/x86_64/pkg-simple-c-1-2-x86_64.pkg.tar.xz" "$TMP/ftp/pool/packages/pkg-simple-c-1-2-x86_64.pkg.tar.xz" +} + +@test "import .db files as 0664 (x86_64)" { + __releaseImportedPackage slavery x86_64 "$TMP/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz" "$TMP/rsyncd/archlinux/pool/packages" + __releaseImportedPackage pkg-simple-c x86_64 "$TMP/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz" "$TMP/rsyncd/archlinux/pool/packages" + + DBSCRIPTS_CONFIG="${TMP}/config.local.packages" __db-import-pkg + + bsdtar xfO "$TMP/ftp/core/os/x86_64/core.db.tar.gz" pkg-simple-c-1-1/desc + __isLinkTo "$TMP/ftp/core/os/x86_64/pkg-simple-c-1-1-x86_64.pkg.tar.xz" "$TMP/ftp/pool/packages/pkg-simple-c-1-1-x86_64.pkg.tar.xz" + __doesNotExist "$TMP"/ftp/{core/os/x86_64,pool/packages,sources/packages}/slavery-* + [[ "$(stat -c '%a' -- "$TMP/ftp/core/os/x86_64/core.db.tar.gz")" = 664 ]] +} + +@test "import .db files as 0664 (i686)" { + __releaseImportedPackage slavery i686 "$TMP/rsyncd/archlinux32/i686/core/core.db.tar.gz" "$TMP/rsyncd/archlinux32/pool" + __releaseImportedPackage pkg-simple-c i686 "$TMP/rsyncd/archlinux32/i686/core/core.db.tar.gz" "$TMP/rsyncd/archlinux32/pool" + + DBSCRIPTS_CONFIG="${TMP}/config.local.archlinux32" __db-import-pkg + + bsdtar xfO "$TMP/ftp/core/os/i686/core.db.tar.gz" pkg-simple-c-1-1/desc + __isLinkTo "$TMP/ftp/core/os/i686/pkg-simple-c-1-1-i686.pkg.tar.xz" "$TMP/ftp/pool/archlinux32/pkg-simple-c-1-1-i686.pkg.tar.xz" + __doesNotExist "$TMP"/ftp/{core/os/i686,pool/archlinux32,sources/archlinux32}/slavery-* + stat -- "$TMP/ftp/core/os/i686/core.db.tar.gz" + [[ "$(stat -c '%a' -- "$TMP/ftp/core/os/i686/core.db.tar.gz")" = 664 ]] +} + +@test "import .db files as 0664 (armv7h)" { + __releaseImportedPackage slavery armv7h "$TMP/rsyncd/archlinuxarm/armv7h/core/core.db.tar.gz" + __releaseImportedPackage pkg-simple-c armv7h "$TMP/rsyncd/archlinuxarm/armv7h/core/core.db.tar.gz" + + DBSCRIPTS_CONFIG="${TMP}/config.local.archlinuxarm" __db-import-pkg + + bsdtar xfO "$TMP/ftp/core/os/armv7h/core.db.tar.gz" pkg-simple-c-1-1/desc + __isLinkTo "$TMP/ftp/core/os/armv7h/pkg-simple-c-1-1-armv7h.pkg.tar.xz" "$TMP/ftp/pool/alarm/pkg-simple-c-1-1-armv7h.pkg.tar.xz" + __doesNotExist "$TMP"/ftp/{core/os/armv7h,pool/alarm,sources/alarm}/slavery-* + stat -- "$TMP/ftp/core/os/armv7h/core.db.tar.gz" + [[ "$(stat -c '%a' -- "$TMP/ftp/core/os/armv7h/core.db.tar.gz")" = 664 ]] +} + +@test "import fully-masked upstream" { + __releaseImportedPackage pkg-any-a x86_64 "$TMP/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz" "$TMP/rsyncd/archlinux/pool/packages" + __releaseImportedPackage pkg-any-a i686 "$TMP/rsyncd/archlinux32/i686/core/core.db.tar.gz" "$TMP/rsyncd/archlinux32/pool" + __releaseImportedPackage pkg-any-a armv7h "$TMP/rsyncd/archlinuxarm/armv7h/core/core.db.tar.gz" + + DBSCRIPTS_CONFIG="${TMP}/config.local.packages" __db-import-pkg + DBSCRIPTS_CONFIG="${TMP}/config.local.archlinux32" __db-import-pkg + DBSCRIPTS_CONFIG="${TMP}/config.local.archlinuxarm" __db-import-pkg + + bsdtar xfO "$TMP/ftp/core/os/x86_64/core.db.tar.gz" pkg-any-a-1-1/desc + bsdtar xfO "$TMP/ftp/core/os/i686/core.db.tar.gz" pkg-any-a-1-1/desc + bsdtar xfO "$TMP/ftp/core/os/armv7h/core.db.tar.gz" pkg-any-a-1-1/desc + + __isLinkTo "$TMP/ftp/core/os/x86_64/pkg-any-a-1-1-any.pkg.tar.xz" "$TMP/ftp/pool/packages/pkg-any-a-1-1-any.pkg.tar.xz" + __isLinkTo "$TMP/ftp/core/os/i686/pkg-any-a-1-1-any.pkg.tar.xz" "$TMP/ftp/pool/packages/pkg-any-a-1-1-any.pkg.tar.xz" + __isLinkTo "$TMP/ftp/core/os/armv7h/pkg-any-a-1-1-any.pkg.tar.xz" "$TMP/ftp/pool/packages/pkg-any-a-1-1-any.pkg.tar.xz" +} + +@test "import removes dropped packages" { + __releaseImportedPackage pkg-simple-c x86_64 "$TMP/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz" "$TMP/rsyncd/archlinux/pool/packages" + + DBSCRIPTS_CONFIG="${TMP}/config.local.packages" __db-import-pkg + + __isLinkTo "$TMP/ftp/core/os/x86_64/pkg-simple-c-1-1-x86_64.pkg.tar.xz" "$TMP/ftp/pool/packages/pkg-simple-c-1-1-x86_64.pkg.tar.xz" + bsdtar xfO "$TMP/ftp/core/os/x86_64/core.db.tar.gz" pkg-simple-c-1-1/desc + + repo-remove -q "$TMP/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz" pkg-simple-c + + DBSCRIPTS_CONFIG="${TMP}/config.local.packages" __db-import-pkg + + if bsdtar xfO "$TMP/ftp/core/os/x86_64/core.db.tar.gz" pkg-simple-c-1-1/desc; then + return 1 + fi +} + +@test "import removes blacklisted packages" { + __releaseImportedPackage pkg-simple-c x86_64 "$TMP/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz" "$TMP/rsyncd/archlinux/pool/packages" + + DBSCRIPTS_CONFIG="${TMP}/config.local.packages" __db-import-pkg + + __isLinkTo "$TMP/ftp/core/os/x86_64/pkg-simple-c-1-1-x86_64.pkg.tar.xz" "$TMP/ftp/pool/packages/pkg-simple-c-1-1-x86_64.pkg.tar.xz" + bsdtar xfO "$TMP/ftp/core/os/x86_64/core.db.tar.gz" pkg-simple-c-1-1/desc >/dev/null + + echo 'pkg-simple-c::::' >> "$TMP/httpd/blacklist.txt" + + DBSCRIPTS_CONFIG="${TMP}/config.local.packages" __db-import-pkg + + if bsdtar xfO "$TMP/ftp/core/os/x86_64/core.db.tar.gz" pkg-simple-c-1-1/desc >/dev/null; then + return 1 + fi +} + +@test "import moves packages" { + __releaseImportedPackage pkg-simple-c x86_64 "$TMP/rsyncd/archlinux/testing/os/x86_64/testing.db.tar.gz" "$TMP/rsyncd/archlinux/pool/packages" + + DBSCRIPTS_CONFIG="${TMP}/config.local.packages" __db-import-pkg + + __isLinkTo "$TMP/ftp/testing/os/x86_64/pkg-simple-c-1-1-x86_64.pkg.tar.xz" "$TMP/ftp/pool/packages/pkg-simple-c-1-1-x86_64.pkg.tar.xz" + bsdtar xfO "$TMP/ftp/testing/os/x86_64/testing.db.tar.gz" pkg-simple-c-1-1/desc >/dev/null + + repo-remove -q "$TMP/rsyncd/archlinux/testing/os/x86_64/testing.db.tar.gz" pkg-simple-c + __releaseImportedPackage pkg-simple-c x86_64 "$TMP/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz" "$TMP/rsyncd/archlinux/pool/packages" + + DBSCRIPTS_CONFIG="${TMP}/config.local.packages" __db-import-pkg + + __isLinkTo "$TMP/ftp/core/os/x86_64/pkg-simple-c-1-1-x86_64.pkg.tar.xz" "$TMP/ftp/pool/packages/pkg-simple-c-1-1-x86_64.pkg.tar.xz" + bsdtar xfO "$TMP/ftp/core/os/x86_64/core.db.tar.gz" pkg-simple-c-1-1/desc >/dev/null + if bsdtar xfO "$TMP/ftp/testing/os/x86_64/testing.db.tar.gz" pkg-simple-c-1-1/desc >/dev/null; then + return 1 + fi +} + +@test "import arch=any packages with sub-pkgrel" { + # This is modeled after the situation with 'asp' and 'asp32' + + __releaseImportedPackage pkg-any64 x86_64 "$TMP/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz" "$TMP/rsyncd/archlinux/pool/packages" + DBSCRIPTS_CONFIG="${TMP}/config.local.packages" __db-import-pkg + __isLinkTo "$TMP/ftp/core/os/x86_64/pkg-any-2-1-any.pkg.tar.xz" "$TMP/ftp/pool/packages/pkg-any-2-1-any.pkg.tar.xz" + bsdtar xfO "$TMP/ftp/core/os/x86_64/core.db.tar.gz" pkg-any-2-1/desc + + __releaseImportedPackage pkg-any32 i686 "$TMP/rsyncd/archlinux32/i686/core/core.db.tar.gz" "$TMP/rsyncd/archlinux32/pool" + DBSCRIPTS_CONFIG="${TMP}/config.local.archlinux32" __db-import-pkg + __isLinkTo "$TMP/ftp/core/os/i686/pkg-any-1-1.2-any.pkg.tar.xz" "$TMP/ftp/pool/archlinux32/pkg-any-1-1.2-any.pkg.tar.xz" + bsdtar xfO "$TMP/ftp/core/os/i686/core.db.tar.gz" pkg-any-1-1.2/desc +} + +@test "import respects INHERIT precedence" { + __releaseImportedPackage pkg-any-a armv7h "$TMP/rsyncd/archlinuxarm/armv7h/core/core.db.tar.gz" + DBSCRIPTS_CONFIG="${TMP}/config.local.archlinux32" __db-import-pkg + + __releaseImportedPackage pkg-any-a x86_64 "$TMP/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz" "$TMP/rsyncd/archlinux/pool/packages" + DBSCRIPTS_CONFIG="${TMP}/config.local.packages" __db-import-pkg + DBSCRIPTS_CONFIG="${TMP}/config.local.archlinuxarm" __db-import-pkg + + bsdtar xfO "$TMP/ftp/core/os/x86_64/core.db.tar.gz" pkg-any-a-1-1/desc + bsdtar xfO "$TMP/ftp/core/os/armv7h/core.db.tar.gz" pkg-any-a-1-1/desc + + __isLinkTo "$TMP/ftp/core/os/x86_64/pkg-any-a-1-1-any.pkg.tar.xz" "$TMP/ftp/pool/packages/pkg-any-a-1-1-any.pkg.tar.xz" + __isLinkTo "$TMP/ftp/core/os/armv7h/pkg-any-a-1-1-any.pkg.tar.xz" "$TMP/ftp/pool/packages/pkg-any-a-1-1-any.pkg.tar.xz" +} + +@test "import doesn't backdate packages" { + __releaseImportedPackage pkg-simple-c x86_64 "$TMP/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz" "$TMP/rsyncd/archlinux/pool/packages" + + touch "$TMP/stamp" + DBSCRIPTS_CONFIG="${TMP}/config.local.packages" __db-import-pkg + + bsdtar xfO "$TMP/ftp/core/os/x86_64/core.db.tar.gz" pkg-simple-c-1-1/desc + __isLinkTo "$TMP/ftp/core/os/x86_64/pkg-simple-c-1-1-x86_64.pkg.tar.xz" "$TMP/ftp/pool/packages/pkg-simple-c-1-1-x86_64.pkg.tar.xz" + [[ "$TMP/ftp/pool/packages/pkg-simple-c-1-1-x86_64.pkg.tar.xz" -nt "$TMP/stamp" ]] + [[ "$TMP/ftp/lastupdate" -nt "$TMP/stamp" ]] +} + +@test "import previously removed packages" { + # Import the package once + __releaseImportedPackage pkg-simple-c x86_64 "$TMP/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz" "$TMP/rsyncd/archlinux/pool/packages" + DBSCRIPTS_CONFIG="${TMP}/config.local.packages" __db-import-pkg + bsdtar xfO "$TMP/ftp/core/os/x86_64/core.db.tar.gz" pkg-simple-c-1-1/desc + __isLinkTo "$TMP/ftp/core/os/x86_64/pkg-simple-c-1-1-x86_64.pkg.tar.xz" "$TMP/ftp/pool/packages/pkg-simple-c-1-1-x86_64.pkg.tar.xz" + + # Remove it + mv -fT -- "$TMP/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz"{,.old} + :> "$TMP/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz" + DBSCRIPTS_CONFIG="${TMP}/config.local.packages" __db-import-pkg + if bsdtar xfO "$TMP/ftp/core/os/x86_64/core.db.tar.gz" pkg-simple-c-1-1/desc; then return 1; fi + + # And add it again + mv -fT -- "$TMP/rsyncd/archlinux/core/os/x86_64/core.db.tar.gz"{.old,} + DBSCRIPTS_CONFIG="${TMP}/config.local.packages" __db-import-pkg + bsdtar xfO "$TMP/ftp/core/os/x86_64/core.db.tar.gz" pkg-simple-c-1-1/desc + __isLinkTo "$TMP/ftp/core/os/x86_64/pkg-simple-c-1-1-x86_64.pkg.tar.xz" "$TMP/ftp/pool/packages/pkg-simple-c-1-1-x86_64.pkg.tar.xz" +} diff --git a/test/cases/db-update.bats b/test/cases/db-update.bats index e805f99..0ac4853 100755 --- a/test/cases/db-update.bats +++ b/test/cases/db-update.bats @@ -239,3 +239,24 @@ load ../lib/common [ "$status" -ne 0 ] checkRemovedPackageDB extra "pkg-simple-a" } + +@test "re-release to new architectures" { + # Initial run + releasePackage extra 'pkg-any-a' + db-update + checkPackage extra pkg-any-a 1-1 + rmdir "$STAGING" + + # Add an arch + echo 'ARCHES+=(ppc64le)' >> "$DBSCRIPTS_CONFIG" + ARCHES+=(ppc64le) + mkdir -p "${TMP}/ftp/extra/os/ppc64le" + if checkPackage extra pkg-any-a 1-1; then return 1; fi + + # Re-run + mkdir -p "${STAGING}/extra" + ln -s -t "${STAGING}/extra" -- "${TMP}/ftp/pool/packages/pkg-any-a-1-1-any.pkg.tar.xz"{,.sig} + db-update + checkPackage extra pkg-any-a 1-1 + rmdir "$STAGING" +} diff --git a/test/fixtures/archlinux32-keyring/PKGBUILD b/test/fixtures/archlinux32-keyring/PKGBUILD new file mode 100644 index 0000000..a045b0b --- /dev/null +++ b/test/fixtures/archlinux32-keyring/PKGBUILD @@ -0,0 +1,13 @@ +pkgname=archlinux32-keyring +pkgver=1 +pkgrel=1 +pkgdesc="A package called ${pkgname}" +arch=('any') +url='http://www.archlinux.org/' +license=('GPL') +options=(!strip) + +package() { + install -d "${pkgdir}"/usr/share/${pkgname} + :>"${pkgdir}"/usr/share/${pkgname}/test +} diff --git a/test/fixtures/archlinuxarm-keyring/PKGBUILD b/test/fixtures/archlinuxarm-keyring/PKGBUILD new file mode 100644 index 0000000..bb2bc9d --- /dev/null +++ b/test/fixtures/archlinuxarm-keyring/PKGBUILD @@ -0,0 +1,13 @@ +pkgname=archlinuxarm-keyring +pkgver=1 +pkgrel=1 +pkgdesc="A package called ${pkgname}" +arch=('any') +url='http://www.archlinux.org/' +license=('GPL') +options=(!strip) + +package() { + install -d "${pkgdir}"/usr/share/${pkgname} + :>"${pkgdir}"/usr/share/${pkgname}/test +} diff --git a/test/fixtures/pkg-any32/PKGBUILD b/test/fixtures/pkg-any32/PKGBUILD new file mode 100644 index 0000000..e0421da --- /dev/null +++ b/test/fixtures/pkg-any32/PKGBUILD @@ -0,0 +1,13 @@ +pkgname=pkg-any +pkgver=1 +pkgrel=1.2 +pkgdesc="A package called ${pkgname} (32)" +arch=('any') +url='http://www.archlinux.org/' +license=('GPL') +options=(!strip) + +package() { + install -d -m755 "${pkgdir}"/usr/share/${pkgname} + echo 'test' > "${pkgdir}"/usr/share/${pkgname}/test +} diff --git a/test/fixtures/pkg-any64/PKGBUILD b/test/fixtures/pkg-any64/PKGBUILD new file mode 100644 index 0000000..1290ebd --- /dev/null +++ b/test/fixtures/pkg-any64/PKGBUILD @@ -0,0 +1,13 @@ +pkgname=pkg-any +pkgver=2 +pkgrel=1 +pkgdesc="A package called ${pkgname} (64)" +arch=('any') +url='http://www.archlinux.org/' +license=('GPL') +options=(!strip) + +package() { + install -d -m755 "${pkgdir}"/usr/share/${pkgname} + echo 'test' > "${pkgdir}"/usr/share/${pkgname}/test +} diff --git a/test/fixtures/pkg-simple-c/PKGBUILD b/test/fixtures/pkg-simple-c/PKGBUILD new file mode 100644 index 0000000..1e8361d --- /dev/null +++ b/test/fixtures/pkg-simple-c/PKGBUILD @@ -0,0 +1,13 @@ +pkgname=pkg-simple-c +pkgver=1 +pkgrel=1 +pkgdesc="A package called ${pkgname}" +arch=('i686' 'x86_64' 'armv7h') +url='http://www.archlinux.org/' +license=('GPL' 'LGPL') +depends=('glibc') +options=(!strip) + +package() { + install -D -m755 /bin/true "${pkgdir}"/usr/bin/${pkgname} +} diff --git a/test/fixtures/slavery/PKGBUILD b/test/fixtures/slavery/PKGBUILD new file mode 100644 index 0000000..93cc079 --- /dev/null +++ b/test/fixtures/slavery/PKGBUILD @@ -0,0 +1,13 @@ +pkgname=slavery +pkgver=1 +pkgrel=1 +pkgdesc="A package called ${pkgname}" +arch=('any') +url='http://www.archlinux.org/' +license=('GPL') +options=(!strip) + +package() { + install -d -m755 "${pkgdir}"/usr/share/${pkgname} + echo 'test' > "${pkgdir}"/usr/share/${pkgname}/test +} diff --git a/test/lib/common.bash b/test/lib/common.bash index 6e9394c..764def5 100644 --- a/test/lib/common.bash +++ b/test/lib/common.bash @@ -77,7 +77,8 @@ setup() { export DBSCRIPTS_CONFIG=${TMP}/config.local cat <<eot > "${DBSCRIPTS_CONFIG}" FTP_BASE="${TMP}/ftp" - PKGREPOS=('core' 'extra' 'testing') + PKGREPOS=('core' 'extra' 'testing' 'libre') + ARCHES=(x86_64 i686 armv7h) PKGPOOL='pool/packages' SRCPOOL='sources/packages' TESTING_REPO='testing' @@ -86,7 +87,6 @@ setup() { SOURCE_CLEANUP_DESTDIR="${TMP}/source-cleanup" STAGING="${TMP}/staging" TMPDIR="${TMP}/tmp" - ARCHES=(x86_64 i686) CLEANUP_DRYRUN=false SOURCE_CLEANUP_DRYRUN=false eot diff --git a/test/lib/import.bash b/test/lib/import.bash new file mode 100644 index 0000000..84d8688 --- /dev/null +++ b/test/lib/import.bash @@ -0,0 +1,195 @@ +#!/hint/bash +. "lib/common.bash" + +httpserver() { + read -r verb path version || exit + if [[ $verb != GET ]]; then + printf "HTTP/1.1 405 Method Not Allowed\\r\\nContent-Type: text/plain\\r\\nContent-length: 0\\r\\n\\r\\n" + exit + fi + path="$(cd / && realpath -ms "$path")" + if ! [[ -f "$1/$path" ]]; then + printf "HTTP/1.1 404 Not found\\r\\nContent-Type: text/plain\\r\\nContent-length: 0\\r\\n\\r\\n" + exit + fi + printf "HTTP/1.1 200 OK\\r\\nContent-Type: text/plain\\r\\nContent-length: %d\\r\\n\\r\\n" "$(stat -L -c %s "$1/$path")" + cat "$1/$path" +} + +eval "__common_$(declare -f setup)" +setup() { + __common_setup + local repo arch + + # Set up rsync server + cat <<-eot >"${TMP}/rsyncd.conf" + use chroot = no + [rsyncd] + path = ${TMP}/rsyncd + eot + local rsyncport + rsyncport=$(./lib/runserver "$TMP/rsyncd.pid" \ + rsync --daemon --config "${TMP}/rsyncd.conf") + + # Set up rsync contents + for repo in core testing; do + mkdir -p -- "${TMP}/rsyncd/archlinux/${repo}/os/x86_64" + touch -- "${TMP}/rsyncd/archlinux/${repo}/os/x86_64/${repo}.db.tar.gz" + ln -sT -- "${repo}.db.tar.gz" "${TMP}/rsyncd/archlinux/${repo}/os/x86_64/${repo}.db" + done + mkdir -p -- "${TMP}/rsyncd/archlinux"/{pool,sources}/{packages,community} + date +%s > "${TMP}/rsyncd/archlinux/lastupdate" + date +%s > "${TMP}/rsyncd/archlinux/lastsync" + + for repo in core testing community community-testing; do + mkdir -p -- "${TMP}/rsyncd/archlinux32/i686/${repo}" + touch -- "${TMP}/rsyncd/archlinux32/i686/${repo}/${repo}.db.tar.gz" + ln -sT -- "${repo}.db.tar.gz" "${TMP}/rsyncd/archlinux32/i686/${repo}/${repo}.db" + done + mkdir -p -- "${TMP}/rsyncd/archlinux32/pool" + date +%s > "${TMP}/rsyncd/archlinux32/lastupdate" + date +%s > "${TMP}/rsyncd/archlinux32/lastsync" + + for repo in core testing community community-testing; do + mkdir -p -- "${TMP}/rsyncd/archlinuxarm/armv7h/${repo}" + touch -- "${TMP}/rsyncd/archlinuxarm/armv7h/${repo}/${repo}.db.tar.gz" + ln -sT -- "${repo}.db.tar.gz" "${TMP}/rsyncd/archlinuxarm/armv7h/${repo}/${repo}.db" + done + date +%s > "${TMP}/rsyncd/archlinuxarm/lastupdate" + date +%s > "${TMP}/rsyncd/archlinuxarm/lastsync" + + # Configure db-import to use that rsyncd server + sed -i /STAGING=/d "${TMP}/config.local" + cat <<-eot >"${TMP}/config.local.packages" + source ${DBSCRIPTS_CONFIG@Q} + source ../config.local.packages + ARCHTAGS=('core-x86_64' 'testing-x86_64') + ARCHMIRROR=rsync://localhost:${rsyncport@Q}/rsyncd/archlinux/ + eot + cat <<-eot >"${TMP}/config.local.community" + source ${DBSCRIPTS_CONFIG@Q} + source ../config.local.community + ARCHTAGS=('community-x86_64' 'community-testing-x86_64') + ARCHMIRROR=rsync://localhost:${rsyncport@Q}/rsyncd/archlinux/ + eot + cat <<-eot >"${TMP}/config.local.archlinux32" + source ${DBSCRIPTS_CONFIG@Q} + source ../config.local.archlinux32 + ARCHTAGS=('core-i686' 'testing-i686' 'community-i686' 'community-testing-i686') + ARCHMIRROR=rsync://localhost:${rsyncport@Q}/rsyncd/archlinux32/ + eot + cat <<-eot >"${TMP}/config.local.archlinuxarm" + source ${DBSCRIPTS_CONFIG@Q} + source ../config.local.archlinuxarm + ARCHTAGS=('core-armv7h' 'testing-armv7h' 'community-armv7h' 'community-testing-armv7h') + ARCHMIRROR=rsync://localhost:${rsyncport@Q}/rsyncd/archlinuxarm/ + eot + + # Set up HTTP server + local httpport + httpport=$(./lib/runserver "$TMP/httpd.pid" \ + bash -c "$(declare -f httpserver); httpserver \"\$@\"" -- "$TMP/httpd") + + # Set up HTTP contents + mkdir -- "$TMP/httpd" + cat <<-eot >"$TMP/httpd/blacklist.txt" + slavery:freedom:fsf:slavekit:Obviously + eot + + # Configure db-import to use that HTTP server + export XDG_CONFIG_HOME="${TMP}/home/.config" + mkdir -p "$XDG_CONFIG_HOME"/libretools + cat <<-eot >"$XDG_CONFIG_HOME"/libretools/libretools.conf + BLACKLIST=http://localhost:${httpport@Q}/blacklist.txt + eot + + # Set up repo contents + for arch in x86_64 i686 armv7h; do + for repo in core testing community community-testing libre; do + mkdir -p -- "${TMP}/ftp/${repo}/os/${arch}" + touch -- "${TMP}/ftp/${repo}/os/${arch}/${repo}.db.tar.gz" + ln -sT -- "${repo}.db.tar.gz" "${TMP}/ftp/${repo}/os/${arch}/${repo}.db" + done + done + mkdir -p -- "${TMP}/ftp"/{pool,sources}/{packages,community,archlinux32,alarm} + date +%s > "${TMP}/ftp/lastupdate" + date +%s > "${TMP}/ftp/lastsync" +} +eval "__common_$(declare -f teardown)" +teardown() { + xargs -a "${TMP}/httpd.pid" kill -- + xargs -a "${TMP}/rsyncd.pid" kill -- + __common_teardown +} + +###################################################################### + +# __releaseImportedPackage PKGBASE ARCH DBFILE [POOLDIR] +# +# This is different from common.bash:releasePackage because +# - it doesn't mess with VCS +# - it adds the package to the .db file +__releaseImportedPackage() { + local pkgbase=$1 + local arch=$2 + local dbfile=$3 + local pooldir=$4 + local repodir="${dbfile%/*}" + local dir restore pkgfiles pkgfile pkgs + + dir="$TMP/import-build/$pkgbase" + if ! [[ -d "$dir" ]]; then + mkdir -p -- "$dir" + cp -t "$dir" -- "fixtures/${pkgbase}"/* + fi + pushd "$dir" + __buildPackage + restore="$(shopt -p nullglob || true)" + shopt -s nullglob + pkgfiles=(*-{"$arch",any}"$PKGEXT"{,.sig}) + $restore + popd + + mkdir -p "$repodir" + if [[ -z $pooldir ]]; then + mv -t "$repodir" -- "${pkgfiles[@]/#/"$dir/"}" + else + mkdir -p "$pooldir" + mv -t "$pooldir" -- "${pkgfiles[@]/#/"$dir/"}" + ln -sr -t "$repodir" -- "${pkgfiles[@]/#/"$pooldir/"}" + fi + + pushd "$repodir" + pkgs=() + for pkgfile in "${pkgfiles[@]}"; do + if [[ "$pkgfile" = *.sig ]]; then + continue + fi + pkgs+=("$pkgfile") + done + repo-add -q "${dbfile##*/}" "${pkgs[@]}" + popd +} + +__updateImportedPackage() { + pushd "$TMP/import-build/$1" + local pkgrel + pkgrel=$(. PKGBUILD; expr ${pkgrel} + 1) + sed "s/pkgrel=.*/pkgrel=${pkgrel}/" -i PKGBUILD + popd +} + +__isLinkTo() { + [[ -L $1 ]] + [[ $1 -ef $2 ]] +} + +__doesNotExist() { + local file + for file in "$@"; do + if stat "$file" 2>/dev/null; then + echo "TEST ERROR: File shouldn't exist, but does: $file" + return 1 + fi + done +} diff --git a/test/lib/runserver b/test/lib/runserver new file mode 100755 index 0000000..7705e40 --- /dev/null +++ b/test/lib/runserver @@ -0,0 +1,39 @@ +#!/usr/bin/env perl +use strict; +use warnings; + +use IO::Socket::INET; +use Fcntl; + +my $socket = IO::Socket::INET->new( + LocalAddr => "localhost:0", + Listen => 1 +); + +sysopen(my $fh, $ARGV[0], O_WRONLY|O_CREAT|O_EXCL, 0666) or die "open: $ARGV[0]: $!"; +my $pid = fork(); +if ($pid > 0) { + print $fh $pid; + print $socket->sockport()."\n"; + exit; +} +close($fh); +open(STDIN, '</dev/null'); +open(STDOUT, '</dev/null'); + +my @cmd = @ARGV; +shift @cmd; + +while (1) { + my $conn = $socket->accept(); + my $worker = fork(); + if ($worker == 0) { + open(STDIN, '<&', $conn); + open(STDOUT, '>&', $conn); + close($conn); + close($socket); + exec @cmd; + exit + } + close($conn); +} |