diff options
author | Luke Shumaker <lukeshu@parabola.nu> | 2018-05-04 22:44:16 -0400 |
---|---|---|
committer | Luke Shumaker <lukeshu@parabola.nu> | 2018-05-05 10:07:52 -0400 |
commit | 8393137cd49de14f0f74caee8e93fa43ca881e2b (patch) | |
tree | 8be24428b26d119369d9f2644b3bdabd360eb293 | |
parent | e46378dc0dbe9ece0149e16752d9a2e93539085d (diff) |
db-import-pkg: Don't do any work directly in $FTP_BASE/$repo or $WORKDIR/rsyncparabola/20180505
Currently, it
- rsync's/creates (depending on $UPSTREAM) symlinks in
"${FTP_BASE}/${repo}/os/${arch}" directly
- runs repo-add on DBs in ${WORKDIR}/rsync/$(get_repo_dir "$repo" "$arch")
Instead, do both of these things in $WORKDIR/staging-rsync/$tag. There's
no reason to re-use the initial /rsync/ dir (we don't re-use any files in
it--we rm them), and the sync_repo's --delete-excluded means that we
temporarily delete the .db files on the live repo.
Also, to do this efficiently, we need to create hardlinks between package
files in $FTP_BASE and $WORKDIR, so fiddle with tmp directories in the
config.
This also has the effect that imported repos no longer have '.db.old'
files, which is fine because they were never Parabola's old version of the
file, they were the verbatim .old file from where we're importing from.
-rw-r--r-- | config | 4 | ||||
-rwxr-xr-x | db-import-pkg | 24 | ||||
-rw-r--r-- | systemd/db-import@.service | 2 |
3 files changed, 18 insertions, 12 deletions
@@ -22,7 +22,9 @@ SOURCE_CLEANUP_KEEP=30 LOCK_DELAY=10 [ -n "${STAGING:-}" ] || STAGING="$HOME/staging/unknown/staging" -TMPDIR="/tmp" +#TMPDIR="/tmp" +TMPDIR=/srv/repo/tmp +mkdir -p "$TMPDIR" ARCHES=(x86_64 i686 armv7h) DBEXT=".db.tar.gz" FILESEXT=".files.tar.gz" diff --git a/db-import-pkg b/db-import-pkg index 5b702e7..3591419 100755 --- a/db-import-pkg +++ b/db-import-pkg @@ -125,8 +125,10 @@ sync_repo() { make_repo_symlinks() { local -r pool=$1 whitelist=$2 repo=$3 arch=$4 + local repodir="${WORKDIR}/staging-rsync/${repo}-${arch}" + msg2 "Putting symlinks in ${repo}/os/${arch}" - mkdir -p -- "${FTP_BASE}/${repo}/os/${arch}" + mkdir -p -- "${repodir}" local pkgfile while read pkgfile; do @@ -148,7 +150,7 @@ make_repo_symlinks() { "${pkgfile%-*}" continue fi - local symlink="${FTP_BASE}/${repo}/os/${arch}/${path##*/}" + local symlink="${repodir}/${path##*/}" ln -sfv "../../../pool/${path##*/pool/}" "$symlink" ln -sfv "../../../pool/${path##*/pool/}.sig" "${symlink}.sig" local -a new_whitelist+=($symlink) @@ -158,7 +160,7 @@ make_repo_symlinks() { # usage: make_repo_dbs <repo> <arch> make_repo_dbs() { - local -r from=${WORKDIR}/rsync/$(get_repo_dir "$1" "$2") + local -r from=${WORKDIR}/staging-rsync/${1}-${2} local -r into=${FTP_BASE}/${1}/os/${2}/ local -r db_file=${from}/${1}${DBEXT} local -r files_file=${from}/${1}${FILESEXT} @@ -166,9 +168,7 @@ make_repo_dbs() { # create fresh databases to reflect actual `any.pkg.tar.xz` packages. # this also avoids corrupt upstream metadata (ALARM) - msg2 "Adding whitelisted packages to clean %s and %s ..." \ - "${db_file##*/}" "${files_file##*/}" - rm "$db_file" "$files_file" + msg2 "Adding whitelisted packages to clean DBs ..." local pkgfiles=() case "$UPSTREAM" in packages|community) @@ -194,7 +194,8 @@ make_repo_dbs() { msg2 "Updating %s-%s databases" "$2" "$1" rsync "${extra[@]}" --no-motd -rtlpH \ --delay-updates \ - --safe-links \ + --delete-after \ + --links \ "$from/" "$into" } @@ -310,6 +311,8 @@ main() { # So, just nuke the current repos and entirely re-create # everything from /new/. + mkdir "${WORKDIR}/staging-rsync" + local whitelists=() local reponame for _tag in "${ARCHTAGS[@]}"; do @@ -331,10 +334,13 @@ main() { # later sync_pool() all packages whitelists+=("${WORKDIR}/${reponame}-${_arch}.whitelist") # Get repo packages (symlinks) + mkdir "${WORKDIR}/staging-rsync/${_tag}" + ln -t "${WORKDIR}/staging-rsync/${_tag}" \ + "${FTP_BASE}/${reponame}/os/${_arch}"/* sync_repo \ - "${ARCHMIRROR_fullmodule}/${reponame}/os/${_arch}/" \ + "${ARCHMIRROR_fullmodule}/$(get_repo_dir "${reponame}" "${_arch}")/" \ "${WORKDIR}/${reponame}-${_arch}.whitelist" \ - "${FTP_BASE}/${reponame}/os/${_arch}/" + "${WORKDIR}/staging-rsync/${_tag}/" ;; archlinux32|archlinuxarm) # Upstream doesn't use an $ARCHPKGPOOL diff --git a/systemd/db-import@.service b/systemd/db-import@.service index 884d4af..f8d0679 100644 --- a/systemd/db-import@.service +++ b/systemd/db-import@.service @@ -8,5 +8,3 @@ Type=oneshot User=repo Environment=DBSCRIPTS_CONFIG=/etc/dbscripts/config.local.%I ExecStart=/usr/bin/db-import-pkg %I - -PrivateTmp=true |