diff options
-rwxr-xr-x | db-import-pkg | 297 |
1 files changed, 76 insertions, 221 deletions
diff --git a/db-import-pkg b/db-import-pkg index c1dffe0..6dff6bf 100755 --- a/db-import-pkg +++ b/db-import-pkg @@ -83,29 +83,6 @@ filter_blacklisted() { <(libreblacklist cat | libreblacklist get-pkg | sort -u) } -# usage: filter_duplicates <FULL_LIST >FILTERED_LIST -# -# Given a list of packages in the format: -# -# pkgname [epoch:]pkgver-pkgrel -# -# filter out arch=(any) packages present elsewhere, as it confuses -# parabolaweb, librechroot, and who-knows-what-else. This only -# filters exact pkgname/epoch/pkgver/pkgrel matches. -filter_duplicates() { - sort -u | comm -23 - <( - for pool in "${INHERIT[@]}"; do - for f in "${FTP_BASE}/${pool}"/*-any${PKGEXTS}; do - f=${f##*/} - f=${f%-any$PKGEXTS} - pkgname=${f%-*-*} - fullpkgver=${f#"${pkgname}-"} - printf '%s %s\n' "$pkgname" "$fullpkgver" - done - done | sort -u - ) -} - # usage: sync_pool <from> <path-to-whitelist> <into> # # Sync excluding everything but whitelist @@ -118,7 +95,7 @@ sync_pool() { fi mkdir -p -- "$_into" - msg2 "Retrieving %d packages from %s pool" \ + msg "Retrieving up to %d files from %s pool" \ "$(wc -l < "$_whitelist")" \ "$(basename "$_into")" @@ -133,97 +110,6 @@ sync_pool() { "$_into" } -# usage: poolify <arch> -# -# Given a list of packages in the format: -# -# pkgname [epoch:]pkgver-pkgrel -# -# Resolve each to a file in `${FTP_BASE}/pool/`. The output is -# relative to `${FTP_BASE}/pool/`. That is, something along the lines -# of: -# -# poolname/pkgname-[epoch:]pkgver-pkgrel-arch.pkg.tar.xz -# archlinux32/zip-3.0-7-i686.pkg.tar.xz -# packages/rhino-1.7.7.1-1-any.pkg.tar.xz -poolify() { - local -r arch=$1 - - local pkgname fullpkgver - local restore paths path - while read -r pkgname fullpkgver; do - paths=() - for pool in "${INHERIT[@]}" "$PKGPOOL"; do - paths+=( - "${FTP_BASE}/${pool}/${pkgname}-${fullpkgver}-any"${PKGEXTS} - "${FTP_BASE}/${pool}/${pkgname}-${fullpkgver}-${arch}"${PKGEXTS} - ) - done - path="${paths[0]:-}" - if ! [[ -f "$path" && -f "${path}.sig" ]]; then - error "No file was found for %q=%q, aborting" "$pkgname" "$fullpkgver" - exit 1 - fi - printf '%s\n' "${path#"${FTP_BASE}/pool/"}" - done -} - -# usage: make_repo_symlinks TAG <POOLFILELIST -# -# Uses the ${WORKDIR} global -make_repo_symlinks() { - local -r tag=$1 - - local -r repo=${tag%-*} - local -r arch=${tag##*-} - local -r repodir=${WORKDIR}/staging-rsync/${repo}/os/${arch} - - msg2 "Putting symlinks in %s" "${repo}/os/${arch}" - mkdir -p -- "${repodir}" - - local poolfile - while read -r poolfile; do - ln -sfvT "../../../pool/${poolfile##*/pool/}" "${repodir}/${poolfile##*/}" - ln -sfvT "../../../pool/${poolfile##*/pool/}.sig" "${repodir}/${poolfile##*/}.sig" - done -} - -# usage: make_repo_dbs <repo> <arch> -# -# Uses the ${WORKDIR} global -make_repo_dbs() { - local -r from=${WORKDIR}/staging-rsync/${1}/os/${2} - local -r into=${FTP_BASE}/${1}/os/${2}/ - local -r db_file=${from}/${1}${DBEXT} - local -r files_file=${from}/${1}${FILESEXT} - - # create fresh databases to reflect actual `any.pkg.tar.xz` packages. - # this also avoids corrupt upstream metadata (ALARM) - msg2 "Adding whitelisted packages to clean DBs ..." - - pushd "${from}" - local -r UMASK=$(umask) - umask 002 - repo-add "${db_file##*/}" *${PKGEXTS} - umask "$UMASK" >/dev/null - popd >/dev/null - - mkdir -p -- "$into" - # This bit is based on db-functions:set_repo_permission() - local -r group=$(/usr/bin/stat --printf='%G' "${into}") - chgrp "$group" "${db_file}" - chgrp "$group" "${files_file}" - chmod g+w "${db_file}" - chmod g+w "${files_file}" - - msg2 "Updating %s-%s databases" "$2" "$1" - rsync "${extra[@]}" --no-motd -rtlpH --no-t \ - --delay-updates \ - --delete-after \ - --links \ - "$from/" "$into" -} - # Main function. Process the databases and get the libre packages # Outline: # 1. Fetch package info @@ -234,32 +120,30 @@ make_repo_dbs() { # blacklist.txt) the desired repo state, and how to get from one # to the other. # 3. Fetch the packages we want -# * Create sync whitelist (based on package blacklist) -# * Call sync_pool to fetch packages and signatures -# 4. Put the packages in the repos -# * Create new repo.db with them (repo-add) -# * rsync scratch directory => repos +# * Symlink to files with the same name in INHERIT pools +# * sync_pool to download the others +# 4. Modify the repos +# * db-move +# * db-update +# * db-remove # # Files: # (misc) # - ${WORKDIR}/expac/ : Scratch directory for expac_file() # (download) # - ${WORKDIR}/rsync/ : Where we download '.db' files to -# - ${FTP_BASE}/${PKGPOOL} : Where we download packages to -# - ${FTP_BASE}/${SRCPOOL} : Where we download sources to +# - ${WORKDIR}/staging/${repo} : Where we download packages to +# - ${WORKDIR}/staging/${SRCPOOL} : Where we download sources to # (analysis) # - ${FTP_BASE}/${INHERIT} : Where we look for duplicate files # - ${FTP_BASE}/.../${repo}.db : Where we generate ${WORKDIR}/old/ from # - ${WORKDIR}/old/ : .txt files describing the way the repos are # - ${WORKDIR}/new/ : .txt files describing the way we want them to be # - ${WORKDIR}/dif/ : .txt files describing how to make it happen -# - ${WORKDIR}/all.whitelist : Glob list of (source-)package filenames to download -# - ${WORKDIR}/${tag}.whitelist : Glob list of (source-)package filenames to download -# - ${WORKDIR}/${tag}.pool : See poolify() +# - ${WORKDIR}/${repo}.pkg.whitelist : List of package filenames to download +# - ${WORKDIR}/all.src.whitelist : Glob list of source-package filenames to download # (release) -# - ${WORKDIR}/staging-rsync/ : .db files and symlinks we'll rsync in to ${FTP_BASE} -# - ${FTP_BASE}/ : Where we rsync ${WORKDIR}/staging-rsync/ to -# - ${FTP_BASE}/lastupdate : Timestamp file of last repo change +# - ${WORKDIR}/staging/ : STAGING= directory for db-update main() { ############################################################## # 0. Initialization # @@ -279,7 +163,7 @@ main() { source "$config_file" local ret=0 varname varref - for varname in PKGEXTS DBEXT FILESEXT FTP_BASE PKGPOOL SRCPOOL; do + for varname in PKGEXTS FTP_BASE PKGPOOL SRCPOOL; do if [[ -z ${!varname:-} ]] || is_array "$varname"; then print "Configure '%s' as a non-empty string in %q (or %q):" "$varname" "$config_file" "$LOCAL_CONFIG" ret=$EXIT_NOTCONFIGURED @@ -299,15 +183,6 @@ main() { fi done - if [[ -n ${ARCHSRCPOOL:-} && -z ${ARCHPKGPOOL:-} ]]; then - print '%s requires that %s is also set' ARCH{SRC,PKG}POOL - ret=$EXIT_NOTCONFIGURED - fi - - if (( ret != 0 )); then - exit $ret - fi - WORKDIR=$(mktemp -dt "${0##*/}.XXXXXXXXXX") readonly WORKDIR trap "rm -rf -- ${WORKDIR@Q}" EXIT @@ -388,98 +263,78 @@ main() { # 3. Fetch the packages we want # ############################################################## - # OK, now we have ${WORKDIR}/old/ describing the way the repos - # are, ${WORKDIR}/new/ describing the way we want them to be, - # and ${WORKDIR}/dif/ describing how to get from `old` to - # `new`. We should (TODO) now use db-move, db-update, and - # db-remove to apply that diff. - # - # But, - # - db-move is broken - # - The code that populates /dif/ isn't finished - # So, just nuke the current repos and entirely re-create - # everything from /new/. - - local whitelists=() - for _tag in "${ARCHTAGS[@]}"; do - msg "Processing %s" "$_tag" - _repo=${_tag%-*} - _arch=${_tag##*-} - # Create a whitelist, add * wildcard to end. - # - # FIXME: due to lack of -arch suffix, the pool sync - # retrieves every arch even if we aren't syncing them. - # - # IMPORTANT: the . in the sed command is needed - # because an empty whitelist would consist of a single - # * allowing any package to pass through. - filter_duplicates \ - <"${WORKDIR}/new/${_tag}.txt" \ - | sed -e 's/ /-/' -e 's|.$|&*|g' \ - > "${WORKDIR}/${_tag}.whitelist" - if [[ -n ${ARCHPKGPOOL:-} ]]; then - # Append to whitelists array so that we can - # later sync_pool() all packages - whitelists+=("${WORKDIR}/${_tag}.whitelist") - else - # Upstream doesn't use an $ARCHPKGPOOL - sync_pool \ - "${ARCHMIRROR}/$(get_repo_dir "${_repo}" "${_arch}")/" \ - "${WORKDIR}/${_tag}.whitelist" \ - "${FTP_BASE}/${PKGPOOL}/" - poolify "${_arch}" \ - <"${WORKDIR}/new/${_tag}.txt" \ - >"${WORKDIR}/${_tag}.pool" - make_repo_symlinks "$_tag" \ - <"${WORKDIR}/${_tag}.pool" - fi + # For some packages, "fetch" means to create a symlink to a + # pool we INHERIT from. For others, it means to actually + # download it from arg_upstream with rsync. + for tag in "${ARCHTAGS[@]}";do + msg "Processing %s" "$tag" + repo=${tag%-*} + arch=${tag##*-} + mkdir -p -- "${WORKDIR}/staging/${repo}" + + local pkgname filename pgpsig + while read -r pkgname filename pgpsig; do + local pool staged=false + for pool in "${INHERIT[@]}"; do + filepath=("${FTP_BASE}/${pool}/${filename}") + if [[ -f $filepath && ! -h $filepath ]]; then + ln -srT -- "$filepath" "${WORKDIR}/staging/${repo}/${filename}" + ln -srT -- "$filepath".sig "${WORKDIR}/staging/${repo}/${filename}".sig + staged=true + fi + done + if ! $staged; then + printf '%s\n' "$filename"{,.sig} >> "${WORKDIR}/${repo}.pkg.whitelist" + printf '%s\n' "${filename%$PKGEXTS}*.src.tar*" >> "${WORKDIR}/all.src.whitelist" + fi + done < <( + mapfile -t pkgnames < <(cut -d' ' -f1 <"${WORKDIR}/dif/update:${tag}.txt") + db_file="${WORKDIR}/rsync/$(get_repo_dir "${repo}" "${arch}")/${repo}.db" + expac_file "$db_file" '%n %f %g' "${pkgnames[@]}" + ) done - - if (( ${#whitelists[@]} > 0 )); then - # Concatenate all whitelists, check for single *s just in case - cat "${whitelists[@]}" | grep -v "^\*$" | - sort -u > "${WORKDIR}/all.whitelist" - # FIXME: make_whitelist() wildcards should be narrowed - # down to respect the architecture of the tag - - msg "Syncing package pool" + for whitelist in "${WORKDIR}"/*.pkg.whitelist; do sync_pool \ - "${ARCHMIRROR}/${ARCHPKGPOOL}/" \ - "${WORKDIR}/all.whitelist" \ - "${FTP_BASE}/${PKGPOOL}/" - for _tag in "${ARCHTAGS[@]}"; do - _repo=${_tag%-*} - _arch=${_tag##*-} - poolify "${_arch}" "${PKGPOOL}" \ - <"${WORKDIR}/new/${_tag}.txt" \ - >"${WORKDIR}/${_tag}.pool" - make_repo_symlinks "$_tag" \ - <"${WORKDIR}/${_tag}.pool" - done - - if [[ -n ${ARCHSRCPOOL:-} ]]; then - msg "Syncing source pool" - sync_pool \ - "${ARCHMIRROR}/${ARCHSRCPOOL}/" \ - "${WORKDIR}/all.whitelist" \ - "${FTP_BASE}/${SRCPOOL}/" - fi + "${ARCHMIRROR}/${ARCHPKGPOOL:-$(get_repo_dir "${_repo}" "${_arch}")}/" \ + "$whitelist" \ + "${WORKDIR}/staging/${repo}/" + done + if [[ -n ${ARCHSRCPOOL:-} ]]; then + sync_pool \ + "${ARCHMIRROR}/${ARCHSRCPOOL}/" \ + "${WORKDIR}/all.src.whitelist" \ + "${WORKDIR}/staging/${SRCPOOL}/" fi ############################################################## - # 4. Put the packages in the repos # + # 4. Modify the repos # ############################################################## - msg "Putting databases back in place" + msg "Modifying the actual repos" - # FIXME: all repo DBs should be replaced at once (per architecture) - ln -srT "$FTP_BASE/pool" "${WORKDIR}/staging-rsync/pool" - for _tag in "${ARCHTAGS[@]}"; do - _repo=${_tag%-*} - _arch=${_tag##*-} - make_repo_dbs "$_repo" "$_arch" + # db-move + for tag_from in "${ARCHTAGS[@]}"; do + repo_from=${tag_from%-*} + arch_from=${tag_from##*-} + for tag_to in "${ARCHTAGS[@]}"; do + repo_to=${tag_to%-*} + arch_to=${tag_to##*-} + [[ $tag_from != $tag_to ]] || continue + [[ $arch_from == $arch_to ]] || continue + + < "${WORKDIR}/diff/move:${tag_from}:${tag_to}.txt" \ + cut -d' ' -f1 | \ + xargs -r0 -d $'\n' db-move "$repo_from" "$repo_to" + done + done + # db-update + STAGING=${WORKDIR}/staging db-update + # db-remove + for tag in "${ARCHTAGS[@]}"; do + < "${WORKDIR}/diff/remove:${tag}.txt" \ + cut -d' ' -f1 | \ + xargs -r0 -d $'\n' db-move "$repo_from" "$repo_to" done - date +%s > "${FTP_BASE}/lastupdate" } main "$@" |