From 310f65479d6d139045a4aea7bc1d5fea3ccd12bb Mon Sep 17 00:00:00 2001 From: Luke Shumaker Date: Fri, 22 Jun 2018 12:47:57 -0400 Subject: Add from Parabola: torrents If you're seeing this in `git blame`, it chose to follow the wrong ancestor of a merge commit. --- README.md | 2 + cron-jobs/db-cleanup | 2 +- cron-jobs/make_repo_torrents | 89 ++++++++++++++++++++++++++++++++++++++ db-cleanup.conf | 9 +++- systemd/make_repo_torrents.service | 7 +++ systemd/make_repo_torrents.timer | 9 ++++ test/cases/make_repo_torrents.bats | 69 +++++++++++++++++++++++++++++ torrent.conf | 17 ++++++++ 8 files changed, 201 insertions(+), 3 deletions(-) create mode 100755 cron-jobs/make_repo_torrents create mode 100644 systemd/make_repo_torrents.service create mode 100644 systemd/make_repo_torrents.timer create mode 100644 test/cases/make_repo_torrents.bats create mode 100644 torrent.conf diff --git a/README.md b/README.md index cb485ba..935b94e 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,7 @@ The executables that you (might) care about are: │   ├── devlist-mailer │   ├── ftpdir-cleanup │   ├── integrity-check + │   ├── make_repo_torrents [Parabola only] │   └── sourceballs ├── db-import-archlinuxarm-src [Parabola only] ├── db-import-any [Parabola only] @@ -75,6 +76,7 @@ program to import source-packages from Arch Linux 32. Things that haven't been mentioned yet: - `cron-jobs/devlist-mailer` + - `cron-jobs/make_repo_torrents` - `cron-jobs/sourceballs` ## Testing * Install the `base-devel` package group, as well as the `bash-bats`, `kcov`, `librelib`, and `subversion` packages. diff --git a/cron-jobs/db-cleanup b/cron-jobs/db-cleanup index 04a559a..c391088 100755 --- a/cron-jobs/db-cleanup +++ b/cron-jobs/db-cleanup @@ -30,7 +30,7 @@ done | cut -d'/' -f1 | sort -u | sed "s|$|*|" > "$filter" msg "Removing old files:" -for POOL in "${PKGPOOLS[@]}" "${SRCPOOLS[@]}"; do +for POOL in "${PKGPOOLS[@]}" "${SRCPOOLS[@]}" "${TORRENTPOOLS[@]}"; do msg2 '%s' "${POOL}" rsync "${EXTRAFLAGS[@]}" -va --delete-excluded \ diff --git a/cron-jobs/make_repo_torrents b/cron-jobs/make_repo_torrents new file mode 100755 index 0000000..1bf66af --- /dev/null +++ b/cron-jobs/make_repo_torrents @@ -0,0 +1,89 @@ +#!/bin/bash +# Copyright (C) 2014, 2017 Joseph Graham +# Copyright (C) 2018 Luke Shumaker +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# This script finds any updated packages and calls +# `make_indivudual_torrent' for each of them. Run every hour from cron. + +# pacman doesn't support multiple different packages of the same name, +# so it's OK to just stuff all the torrents into a single directory. + +set -eE +shopt -s extglob globstar nullglob +. "$(dirname "$(readlink -e "$0")")/../config" +. "$(dirname "$(readlink -e "$0")")/../torrent.conf" +. "$(librelib messages)" + +mkdir -p -- "${FTP_BASE}/${TORRENTPOOL}" + +IFS=, # for ${TRACKERS[*]} and ${WEBSEEDS[*]} + +pool_broken=0 +maintain_pool() { + local pkgfile="$1" + if (( pool_broken )); then + return 0 + fi + local link="${FTP_BASE}/${TORRENTPOOL}/${pkgfile##*/}.torrent" + if [[ -L $link && -f $link ]]; then + return 0 + fi + pkgfile=$(readlink -f -- "$pkgfile") + if ! ln -sfrT -- "${pkgfile}.torrent" "$link"; then + pool_broken=1 + fi +} + +make_torrent() { + local pkgfile="$1" + local target + + if [[ -f ${pkgfile}.torrent ]]; then + maintain_pool "$pkgfile" + return 0 + fi + if ! [[ -f $pkgfile ]]; then + die 'Is not a file: %s' "$pkgfile" + fi + + if [[ -L "$pkgfile" ]]; then + target=$(readlink -- "$pkgfile") + make_torrent "${pkgfile%/*}/${target}" + ln -sT -- "${target}.torrent" "${pkgfile}.torrent" + else + # Create the .torrent + if [[ -f "${FTP_BASE}/${TORRENTPOOL}/${pkgfile##*/}.torrent" && + ! -L "${FTP_BASE}/${TORRENTPOOL}/${pkgfile##*/}.torrent" ]]; then + ln -T -- "${FTP_BASE}/${TORRENTPOOL}/${pkgfile##*/}.torrent" "${pkgfile}.torrent" + maintain_pool "$pkgfile" + else + msg2 'Creating %s' "${pkgfile}.torrent" + mktorrent \ + --announce="${TRACKERS[*]}" \ + --web-seed="${WEBSEEDS[*]/%/"${pkgfile#"${FTP_BASE}"}"}" \ + --output="${pkgfile}.torrent" \ + -- "${pkgfile}" >/dev/null + maintain_pool "$pkgfile" + fi + fi +} + +msg 'Creating .torrent files...' +for pkgfile in "$FTP_BASE"/*/os/*/*${PKGEXTS}; do + make_torrent "$pkgfile" +done +msg 'Done' +exit $pool_broken diff --git a/db-cleanup.conf b/db-cleanup.conf index 12d65f9..9944438 100644 --- a/db-cleanup.conf +++ b/db-cleanup.conf @@ -1,6 +1,6 @@ #!/hint/bash -# Both PKGPOOLS and SRCPOOLS are relative to `config:FTP_BASE`. +# All `*POOLS` variables are relative to `config:FTP_BASE`. # Directories where packages are shared between repos PKGPOOLS=( @@ -11,7 +11,7 @@ PKGPOOLS=( pool/community # Arch Linux (community) ) -# Directories where sources are stored +# Directories where source-packages are stored SRCPOOLS=( sources/parabola # Parabola GNU/Linux-libre sources/alarm # Arch Linux ARM @@ -19,3 +19,8 @@ SRCPOOLS=( sources/packages # Arch Linux (project) sources/community # Arch Linux (community) ) + +# Directories where torrents are stored +TORRENTPOOLS=( + torrents +) diff --git a/systemd/make_repo_torrents.service b/systemd/make_repo_torrents.service new file mode 100644 index 0000000..fc084ee --- /dev/null +++ b/systemd/make_repo_torrents.service @@ -0,0 +1,7 @@ +[Unit] +Description=Create pacman2pacman torrent files + +[Service] +Type=oneshot +User=repo +ExecStart=/opt/dbscripts/cron-jobs/make_repo_torrents diff --git a/systemd/make_repo_torrents.timer b/systemd/make_repo_torrents.timer new file mode 100644 index 0000000..9d02cb4 --- /dev/null +++ b/systemd/make_repo_torrents.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Periodic creation of pacman2pacman torrent files + +[Timer] +OnCalendar=hourly +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/test/cases/make_repo_torrents.bats b/test/cases/make_repo_torrents.bats new file mode 100644 index 0000000..4d37ded --- /dev/null +++ b/test/cases/make_repo_torrents.bats @@ -0,0 +1,69 @@ +load ../lib/common + +@test "make torrents" { + releasePackage core 'pkg-any-a' + releasePackage extra 'pkg-any-b' + db-update + + make_repo_torrents + + [[ -f "${FTP_BASE}/torrents/pkg-any-a-1-1-any${PKGEXT}.torrent" ]] + [[ -L "${FTP_BASE}/torrents/pkg-any-a-1-1-any${PKGEXT}.torrent" ]] + [[ -f "${FTP_BASE}/core/os/i686/pkg-any-a-1-1-any${PKGEXT}.torrent" ]] + + [[ -f "${FTP_BASE}/torrents/pkg-any-b-1-1-any${PKGEXT}.torrent" ]] + [[ -L "${FTP_BASE}/torrents/pkg-any-b-1-1-any${PKGEXT}.torrent" ]] + [[ -f "${FTP_BASE}/extra/os/i686/pkg-any-b-1-1-any${PKGEXT}.torrent" ]] +} + +@test "make torrents migrates torrents" { + releasePackage core 'pkg-any-a' + releasePackage extra 'pkg-any-b' + db-update + + # Emulate old make_repo_torrents + mkdir -p -- "${FTP_BASE}/torrents" + echo mktorrent > "${FTP_BASE}/torrents/pkg-any-a-1-1-any${PKGEXT}.torrent" + echo mktorrent > "${FTP_BASE}/torrents/pkg-any-b-1-1-any${PKGEXT}.torrent" + + make_repo_torrents + + [[ -f "${FTP_BASE}/torrents/pkg-any-a-1-1-any${PKGEXT}.torrent" ]] + [[ -L "${FTP_BASE}/torrents/pkg-any-a-1-1-any${PKGEXT}.torrent" ]] + [[ -f "${FTP_BASE}/core/os/i686/pkg-any-a-1-1-any${PKGEXT}.torrent" ]] + [[ "$(cat "${FTP_BASE}/torrents/pkg-any-a-1-1-any${PKGEXT}.torrent")" = mktorrent ]] + + [[ -f "${FTP_BASE}/torrents/pkg-any-b-1-1-any${PKGEXT}.torrent" ]] + [[ -L "${FTP_BASE}/torrents/pkg-any-b-1-1-any${PKGEXT}.torrent" ]] + [[ -f "${FTP_BASE}/extra/os/i686/pkg-any-b-1-1-any${PKGEXT}.torrent" ]] + [[ "$(cat "${FTP_BASE}/torrents/pkg-any-b-1-1-any${PKGEXT}.torrent")" = mktorrent ]] +} + +@test "make torrents with insufficient permissions fails" { + releasePackage core 'pkg-any-a' + releasePackage extra 'pkg-any-b' + db-update + + chmod -w "${FTP_BASE}/core/os/i686" + if make_repo_torrents; then return 1; fi + chmod +w "${FTP_BASE}/core/os/i686" +} + +@test "make torrents maintains the pool after failure" { + releasePackage core 'pkg-any-a' + db-update + + mkdir "${FTP_BASE}/torrents" + chmod -w "${FTP_BASE}/torrents" + if make_repo_torrents; then return 1; fi + chmod +w "${FTP_BASE}/torrents" + + [[ ! -f "${FTP_BASE}/torrents/pkg-any-a-1-1-any${PKGEXT}.torrent" ]] + [[ -f "${FTP_BASE}/core/os/i686/pkg-any-a-1-1-any${PKGEXT}.torrent" ]] + + make_repo_torrents + + [[ -f "${FTP_BASE}/torrents/pkg-any-a-1-1-any${PKGEXT}.torrent" ]] + [[ -L "${FTP_BASE}/torrents/pkg-any-a-1-1-any${PKGEXT}.torrent" ]] + [[ -f "${FTP_BASE}/core/os/i686/pkg-any-a-1-1-any${PKGEXT}.torrent" ]] +} diff --git a/torrent.conf b/torrent.conf new file mode 100644 index 0000000..1a64b1e --- /dev/null +++ b/torrent.conf @@ -0,0 +1,17 @@ +#!/hint/bash + +TRACKERS=( + http://taskenizer.crabdance.com:6969/announce # run by Xylon + http://tracker.opentrackr.org:1337/announce # from https://github.com/ngosang/trackerslist + udp://tracker.pirateparty.gr:6969/announce # from https://github.com/ngosang/trackerslist +) + +# Mirrors to put as webseeds. Which mirrors we use for webseeds +# doesn't really matter since it's re-written on the client machine by +# pacman2pacman so it won't normally be used anyway. +WEBSEEDS=( + 'https://repomirror.parabola.nu/' +) + +# Where under $FTP_BASE/ to put the torrents +TORRENTPOOL=torrents -- cgit v1.2.2