diff options
author | David P <megver83@parabola.nu> | 2021-10-19 19:40:52 -0300 |
---|---|---|
committer | David P <megver83@parabola.nu> | 2021-10-20 20:16:19 -0300 |
commit | c451b72c089484e689387f04bc45c71a92392de4 (patch) | |
tree | 1316e100defb5a99cb07c6a261b949532d40e656 | |
parent | e3eeb82dc48d248e18ff240e778d77b765b336ce (diff) |
find-deprecated-pkgs: major upgrade, rename to find-deprecated
ChangeLog:
* split custimizable find-deprecated-pkgs variables into configuration file
* update copyright years
* create a mkmirrorlist() function to better handle mirrors
* check arguments with $# instead of ${#@}
* declare local variables in functions instead of unsetting them at the end
* rename check() to main()
* create mkpkglist_from_mirror() function
* remove mkpkglist()
* code optimizations
* fail if connection to specific mirror fails
* remove $netcheck
Signed-off-by: David P <megver83@parabola.nu>
-rw-r--r-- | .gitignore | 3 | ||||
-rwxr-xr-x | find-deprecated | 284 | ||||
-rwxr-xr-x | find-deprecated-pkgs | 283 | ||||
-rw-r--r-- | find-deprecated.conf | 40 |
4 files changed, 326 insertions, 284 deletions
@@ -5,7 +5,8 @@ !check.sh !COPYING !find-replacements -!find-deprecated-pkgs +!find-deprecated +!find-deprecated.conf !README !SYNTAX !your-freedom_emu-blacklist.txt diff --git a/find-deprecated b/find-deprecated new file mode 100755 index 0000000..ce2124c --- /dev/null +++ b/find-deprecated @@ -0,0 +1,284 @@ +#!/bin/bash +# Copyright (C) 2018-2021 David P. <megver83@parabola.nu> +# Find deprecated Arch packages that are still in our blacklists. +# shellcheck disable=SC2154 + +usage(){ + # NOTE: the package architecture is always specified with 'parabola', and + # in 'arch' only when there's an uneeded replacement. This is because your-freedom + # is for any architecture and no matter if you use Parabola x86_64 or i686, it + # will always conflict packages that are even for armv7h only (and vice versa). + # The 'parabola' argument compares with our repositories and Arch's + # since in blacklists like your-privacy and your-initfreedom we also block + # some packages from [pcr] (like jitsi) and [libre] (like icedove), and + # probably also from their -multilib, -testing and -multilib-testing derivatives. + + cat <<EOM +Usage: ${0##*/} [arch|parabola] +Check if there are inexistent packages in the blacklists. + +Arguments: + arch Compares ArchLinux{32,ARM} blacklists with their + repositories. + parabola Compares Arch & Parabola blacklists with Parabola's + and Arch's repos. + +Configuration is read from a file called find-deprecated.conf in the same directory +as this script. + +To easily remove a package from a blacklist, you can use the following sed expresion: + + sed '/^pkgname:/d' -i blacklist.txt +EOM +} + +err(){ + printf '%s==> Error:%s %s\n' \ + "$(tput bold;tput setaf 1)" \ + "$(tput sgr0)" \ + "$1" + exit 1 +} + +msg(){ + printf '%s==>%s %s\n' \ + "$(tput bold;tput setaf 2)" \ + "$(tput sgr0)" \ + "$1" +} + +submsg(){ + printf ' %s->%s %s\n' \ + "$(tput bold;tput setaf 4)" \ + "$(tput sgr0)" \ + "$1" +} + +add(){ + list="$(( list + 1 ))" +} + +compare_pkgs(){ + # $1 is the pkgname and replacement (used when checking Arch pkgs) + # $2 is the packages file list + local to_be_removed + local is_not_for + local to_be_deleted + local isnt_for + local arches + local arch + + # First check if the pkg is available for + # a specific architecture, if not, check + # if it's for 'any' + local package="${1%%:*}" + local replacement + replacement="$(echo "$1" | cut -d ":" -f2)" + + for arch in x86_64 i686 armv7h; do + grep $arch$ "$2" | awk '{print $1}' | grep -xw ^"$package" &> /dev/null || \ + grep any$ "$2" | awk '{print $1}' | grep -xw ^"$package" &> /dev/null || \ + if [ "$2" = "$arch_pkgs" ]; then + get_libre_pkgs + # Check if the package has a replacement, and + # if such replacement is available for the same + # architectures + if ! [[ $replacement = "" ]]; then + # If this works, it means the pkg doesn't exist for $arch in Arch, + # but we have the [libre] replacement which should be deprecated. + grep $arch$ "$libre_pkgs" | awk '{print $1}' | grep -xw ^"$replacement" &> /dev/null || \ + grep any$ "$libre_pkgs" | awk '{print $1}' | grep -xw ^"$replacement" &> /dev/null + + case $? in + 0) to_be_removed+=("$arch") + ;; + *) is_not_for+=("$arch") + ;; + esac + else + # However if this fails, it means the pkg doesn't have a replacement + # for $arch, or it simply doesn't have a replacement. + is_not_for+=("$arch") + fi + elif [ "$2" = "$parabola_pkgs" ]; then + if [[ -n $replacement ]]; then + # Look for the replacement + grep $arch$ "$parabola_pkgs" | awk '{print $1}' | grep -xw ^"$replacement" &> /dev/null || \ + grep any$ "$parabola_pkgs" | awk '{print $1}' | grep -xw ^"$replacement" &> /dev/null + + case $? in + 0) to_be_deleted+=("$arch") + ;; + *) isnt_for+=("$arch") + ;; + esac + else + isnt_for+=("$arch") + fi + fi + done + + # Arch + if [[ "${is_not_for[*]}" = "x86_64 i686 armv7h" ]]; then + submsg "$package was not found" + add + elif [[ "${to_be_removed[*]}" = "x86_64 i686 armv7h" ]]; then + submsg "$package was not found, but we've [libre] replacements which should be removed" + add + elif [[ -n "${is_not_for[*]}" ]] || [[ -n "${to_be_removed[*]}" ]]; then + for arch in "${to_be_removed[@]}"; do + # If the replacement is available for the three arch'es supported + # by Parabola, then shut up + for a in x86_64 i686 armv7h; do + grep -xw "^$replacement $a$" "$libre_pkgs" &> /dev/null && arches+=("$a") + done + if ! [[ "${arches[*]}" = "x86_64 i686 armv7h" ]]; then + submsg "$package ($arch) was not found, but we've a [libre] replacement which should be removed" + add + fi + unset arches + done + fi + + # Parabola + if [[ "${isnt_for[*]}" = "x86_64 i686 armv7h" ]]; then + submsg "$package was not found" + add + elif [[ -n "${isnt_for[*]}" ]]; then + for arch in "${to_be_deleted[@]}"; do + submsg "$package ($arch) was not found, but we've $replacement as replacement and should be removed" + add + done + fi +} + +mkpkglist_from_mirror(){ + # $1 is the temporary package list file + local _mirror + local arch + + # Clean ${1}.1 just in case... + test -e "${1}.1" && echo > "${1}.1" + for _mirror in "${mirrors[@]}"; do + curl -sLf "$_mirror" >> "${1}.1" || err "Connection failed for $_mirror" + done + + # Create the parsed package list + grep '".*.pkg.tar.xz"\|".*.pkg.tar.zst"' "${1}.1" | sed "$sedexp" > "$1" + rm "${1}.1" + + # Separate packages by architecture + for arch in $supported_architectures; do + grep "$arch$" "$1" | for f in $(</dev/stdin); do + echo "${f%-*-*-*} $arch" >> "$1-$arch" + done + done + cat "$1"-{x86_64,i686,armv7h,any} | sort -u > "$1" + rm "$1"-{x86_64,i686,armv7h,any} +} + +check_packages(){ + local pkg_list + local pkgs + + # shellcheck disable=SC2068 + pkgs="$(grep -hv ^# $@ | awk '{print $1}')" + pkg_list="$(mktemp)" + eval "${distro}_pkgs=$pkg_list" # required for compare_pkgs + + msg "Comparing blacklists with ${distro^} packages, this might take a while..." + case $distro in + parabola) + add_parabola_mirrors + mkpkglist_from_mirror "$parabola_pkgs" + ;; + arch) + mkpkglist_from_mirror "$arch_pkgs" + ;; + esac + + # Here the magic begins + for p in $pkgs; do + compare_pkgs "$p" "$pkg_list" + done + rm "$pkg_list" +} + +main(){ + mkmirrorlist + local distro="$1" + + case $distro in + parabola) + check_packages "$blacklists_parabola" + ;; + arch) + check_packages "$blacklists" + rm "$libre_pkgs" + ;; + *) + err "$1 is not a valid argument" + ;; + esac + + if ! [[ $list -gt 0 ]]; then + submsg 'No packages to show' + fi + msg 'done' + + unset list +} + +# We'll use this when we check Arch's pkgs only +get_libre_pkgs(){ + local mirrors + if ! [[ -e $libre_pkgs ]]; then + libre_pkgs=$(mktemp) + for r in $repos_libre; do + mirrors+=("$mirror_parabola/$r/os/x86_64/") + mirrors+=("$mirror_parabola/$r/os/i686/") + mirrors+=("$mirror_parabola/$r/os/armv7h/") + done + mkpkglist_from_mirror "$libre_pkgs" + fi +} + +# Parabola mirrors, used to check [libre] and [pcr] packages +add_parabola_mirrors(){ + for r in $repos_parabola; do + mirrors+=("$mirror_parabola/$r/os/x86_64/") + mirrors+=("$mirror_parabola/$r/os/i686/") + mirrors+=("$mirror_parabola/$r/os/armv7h/") + done +} + +# Create mirrors lists +mkmirrorlist(){ + unset mirrors + for r in $repos_x86_64; do + mirrors+=("$mirror_x86_64/$r/os/x86_64/") + done + for r in $repos_i686; do + mirrors+=("$mirror_i686/i686/$r/") + done + for r in $repos_armv7h; do + mirrors+=("$mirror_armv7h/armv7h/$r/") + done +} + +conf="$(dirname "$(readlink -f "$0")")/find-deprecated.conf" +if test -e "$conf"; then + # shellcheck disable=SC1090 + . "$conf" +else + usage + err "Configuration file not found: $conf" +fi + +if [ "$#" -eq 0 ]; then + usage +else + for arg in "$@"; do + main "$arg" + done +fi diff --git a/find-deprecated-pkgs b/find-deprecated-pkgs deleted file mode 100755 index 0020a22..0000000 --- a/find-deprecated-pkgs +++ /dev/null @@ -1,283 +0,0 @@ -#!/bin/bash -# Copyright (C) 2018 David P. <megver83@parabola.nu> -# Find deprecated Arch packages that are still in our blacklists. - -# Repositories for specific arch'es. -# Common repos are in 'repos' variable. -repos="core community extra" -repos_x86_64="$repos testing multilib multilib-testing" -repos_i686="$repos testing build-support" -repos_armv7h="$repos alarm" # although we don't sync [alarm] we blacklist its packages to conflict with your-freedom - -# Parabola repos for checking blacklists like your-privacy, -# which sometimes block [libre] and [pcr] packages. -# [nonprism] and [nonsystemd] are not listened because -# we do not blacklists those packages anywhere -repos_parabola="pcr pcr-multilib pcr-multilib-testing pcr-testing libre libre-multilib libre-multilib-testing libre-testing" - -# Set the blacklist files. The packages here always have a [libre] replacement if there's one. -# aur-blacklist.txt is not here since we want to check Arch's official repos -blacklists="blacklist.txt" - -# List here files that may also blacklist Parabola packages. -# These are intended to be used with $repos_parabola -blacklists_parabola="your-initfreedom-blacklist.txt your-privacy-blacklist.txt your-freedom_emu-blacklist.txt" - -# Mirrors. Please choose HTTPS over HTTP when possible -# x86_64 mirrors -for r in $repos_x86_64; do - mirrors+=("https://mirrors.edge.kernel.org/archlinux/$r/os/x86_64/") -done -# i686 mirrors -for r in $repos_i686; do - mirrors+=("https://mirror.archlinux32.org/i686/$r/") -done -# armv7h mirrors -for r in $repos_armv7h; do - mirrors+=("https://fl.us.mirror.archlinuxarm.org/armv7h/$r/") -done - -add_parabola_mirrors(){ - # Parabola mirrors, used to check [libre] and [pcr] packages - for r in $repos_parabola; do - mirrors+=("https://mirror.grapentin.org/parabola/$r/os/x86_64/") - mirrors+=("https://mirror.grapentin.org/parabola/$r/os/i686/") - mirrors+=("https://mirror.grapentin.org/parabola/$r/os/armv7h/") - done -} - -# Sed expresion tested for Nginx, change if needed -sedexp='s/.*href="//;s/\.pkg.tar.xz.*//;s/\.pkg.tar.zst.*//' -# extra sedexps for special characters/symbols -sedexp+=';s/%2B/+/g;s/%3A/:/g;s/%40/@/g' - -usage(){ -cat <<EOM -Usage: ${0##*/} [arch|parabola] -Check if there are inexistent packages in the blacklists. - -Arguments: - arch Compares ArchLinux{32,ARM} blacklists - with their repositories - parabola Compares Arch/Parabola blacklists (e.g. - your-privacy) with Parabola's and Arch's - repos - -The 'parabola' argument compares with our repositories and Arch's -since in blacklists like your-privacy and your-initfreedom we also block -some packages from [pcr] (like jitsi) and [libre] (like icedove), and -probably also from their -multilib, -testing and -multilib-testing derivatives. - -Note that the package architecture is always specified with 'parabola', and -in 'arch' only when there's an uneeded replacement. This is because your-freedom -is for any architecture and no matter if you use Parabola x86_64 or i686, it -will always conflict packages that are even for armv7h only (and vice versa). - -To easily remove a line, you can use the following sed expresion: - - sed '/^pkgname:/d' -i blacklist_file.txt -EOM -} - -mkpkglist(){ - parabola_pkgs=$(mktemp) - arch_pkgs=$(mktemp) - case $1 in - parabola) add_parabola_mirrors - curl -s "${mirrors[@]}" | grep '".*.pkg.tar.xz"\|".*.pkg.tar.zst"' | sed $sedexp > "$parabola_pkgs" - - # Separate packages by architecture - for arch in x86_64 i686 armv7h any; do - grep $arch$ "$parabola_pkgs" | for f in $(</dev/stdin); do - echo "${f%-*-*-*} $arch" >> "$parabola_pkgs"-$arch - done - done - cat "$parabola_pkgs"-{x86_64,i686,armv7h,any} | sort -u > "$parabola_pkgs" - rm "$parabola_pkgs"-{x86_64,i686,armv7h,any} - ;; - arch) curl -s "${mirrors[@]}" | grep '".*.pkg.tar.xz"\|".*.pkg.tar.zst"' | sed $sedexp > "$arch_pkgs" - - # Separate packages by architecture - for arch in x86_64 i686 armv7h any; do - grep $arch$ "$arch_pkgs" | for f in $(</dev/stdin); do - echo "${f%-*-*-*} $arch" >> "$arch_pkgs"-$arch - done - done - cat "$arch_pkgs"-{x86_64,i686,armv7h,any} | sort -u > "$arch_pkgs" - rm "$arch_pkgs"-{x86_64,i686,armv7h,any} - ;; - esac -} - -err(){ - printf '%s==> Error:%s %s\n' \ - "$(tput bold;tput setaf 1)" \ - "$(tput sgr0)" \ - "$1" - exit 1 -} - -msg(){ - printf '%s==>%s %s\n' \ - "$(tput bold;tput setaf 2)" \ - "$(tput sgr0)" \ - "$1" -} - -submsg(){ - printf ' %s->%s %s\n' \ - "$(tput bold;tput setaf 4)" \ - "$(tput sgr0)" \ - "$1" -} - -get_libre_pkgs(){ - # We'll use this when we check Arch's pkgs only - if ! [[ -e $libre_pkgs ]]; then - libre_pkgs=$(mktemp) - for r in libre libre-multilib libre-testing libre-multilib-testing; do - mirrors_parabola+=("https://mirror.grapentin.org/parabola/$r/os/x86_64/") - mirrors_parabola+=("https://mirror.grapentin.org/parabola/$r/os/i686/") - mirrors_parabola+=("https://mirror.grapentin.org/parabola/$r/os/armv7h/") - done - curl -s "${mirrors_parabola[@]}" | grep '".*.pkg.tar.xz"\|".*.pkg.tar.zst"' | sed $sedexp > "$libre_pkgs" - for arch in x86_64 i686 armv7h any; do - grep $arch$ "$libre_pkgs" | for f in $(</dev/stdin); do - echo "${f%-*-*-*} $arch" >> "$libre_pkgs"-$arch - done - done - cat "$libre_pkgs"-{x86_64,i686,armv7h,any} | sort -u > "$libre_pkgs" - fi -} - -add(){ - list="$(( list + 1 ))" -} - -compare_pkgs(){ - # $1 is the pkgname and replacement (used when checking Arch pkgs) - # $2 is the packages file list - - # First check if the pkg is available for - # a specific architecture, if not, check - # if it's for 'any' - package="${1%%:*}" - replacement="$(cut -d ":" -f2 <(echo "$1"))" - - for arch in x86_64 i686 armv7h; do - grep $arch$ "$2" | awk '{print $1}' | grep -xw ^"$package" &> /dev/null || \ - grep any$ "$2" | awk '{print $1}' | grep -xw ^"$package" &> /dev/null || \ - if [ "$2" = "$arch_pkgs" ]; then - get_libre_pkgs - # Check if the package has a replacement, and - # if such replacement is available for the same - # architectures - if ! [[ $replacement = "" ]]; then - # If this works, it means the pkg doesn't exist for $arch in Arch, - # but we have the [libre] replacement which should be deprecated. - grep $arch$ "$libre_pkgs" | awk '{print $1}' | grep -xw ^"$replacement" &> /dev/null || \ - grep any$ "$libre_pkgs" | awk '{print $1}' | grep -xw ^"$replacement" &> /dev/null - - case $? in - 0) to_be_removed+=("$arch") - ;; - *) is_not_for+=("$arch") - ;; - esac - else - # However if this fails, it means the pkg doesn't have a replacement - # for $arch, or it simply doesn't have a replacement. - is_not_for+=("$arch") - fi - elif [ "$2" = "$parabola_pkgs" ]; then - if [[ -n $replacement ]]; then - # Look for the replacement - grep $arch$ "$parabola_pkgs" | awk '{print $1}' | grep -xw ^"$replacement" &> /dev/null || \ - grep any$ "$parabola_pkgs" | awk '{print $1}' | grep -xw ^"$replacement" &> /dev/null - - case $? in - 0) to_be_deleted+=("$arch") - ;; - *) isnt_for+=("$arch") - ;; - esac - else - isnt_for+=("$arch") - fi - fi - done - - # Arch - if [[ "${is_not_for[*]}" = "x86_64 i686 armv7h" ]]; then - submsg "$package was not found" - add - elif [[ "${to_be_removed[*]}" = "x86_64 i686 armv7h" ]]; then - submsg "$package was not found, but we've [libre] replacements which should be removed" - add - elif [[ -n "${is_not_for[*]}" ]] || [[ -n "${to_be_removed[*]}" ]]; then - for arch in "${to_be_removed[@]}"; do - # If the replacement is available for the three arch'es supported - # by Parabola, then shut up - for a in x86_64 i686 armv7h; do - grep -xw "^$replacement $a$" "$libre_pkgs" &> /dev/null && arches+=("$a") - done - if ! [[ "${arches[*]}" = "x86_64 i686 armv7h" ]]; then - submsg "$package ($arch) was not found, but we've a [libre] replacement which should be removed" - add - fi - unset arches - done - fi - - # Parabola - if [[ "${isnt_for[*]}" = "x86_64 i686 armv7h" ]]; then - submsg "$package was not found" - add - elif [[ -n "${isnt_for[*]}" ]]; then - for arch in "${to_be_deleted[@]}"; do - submsg "$package ($arch) was not found, but we've $replacement as replacement and should be removed" - add - done - fi - - # Unset the variables so they can be re-used - unset to_be_deleted to_be_removed is_not_for isnt_for arch package replacement -} - -check(){ - mkpkglist "$1" - case $1 in - parabola) msg 'Comparing blacklists with Parabola packages ...' - pkgs=$(for bl in $blacklists_parabola; do grep -v ^# "$bl" | awk '{print $1}'; done) - for p in $pkgs; do - compare_pkgs "$p" "$parabola_pkgs" - done - rm -f "$parabola_pkgs" - ;; - arch) msg 'Comparing blacklists with Arch packages ...' - pkgs=$(for bl in $blacklists; do grep -v ^# $bl | awk '{print $1}'; done) - for p in $pkgs; do - compare_pkgs "$p" "$arch_pkgs" - done - rm -f "$arch_pkgs" "$libre_pkgs" - ;; - *) err "$1 is not a valid argument" - ;; - esac - if ! [[ $list -gt 0 ]]; then - submsg 'No packages to show' - fi - msg 'done' - unset list -} - -if [[ ${#@} -gt 0 ]]; then - netcheck="$(curl -sL networkcheck.kde.org)" - if [[ "$netcheck" = OK ]]; then - for arg in "$@"; do check "$arg"; done - else - err 'You must have internet connection to run this program' - fi -else - usage -fi diff --git a/find-deprecated.conf b/find-deprecated.conf new file mode 100644 index 0000000..f80c22b --- /dev/null +++ b/find-deprecated.conf @@ -0,0 +1,40 @@ +# Repositories for specific arch'es. +# Common repos are in 'repos' variable. +repos="core community extra" +repos_x86_64="$repos testing multilib multilib-testing" +repos_i686="$repos testing build-support" +repos_armv7h="$repos alarm" # we don't sync [alarm] but blacklist its packages to conflict with your-freedom + +# Parabola repos for checking blacklists like your-privacy, which sometimes block [libre] and [pcr] packages. +# [nonprism] and [nonsystemd] are not listened because we do not blacklists those packages anywhere +repos_libre="libre libre-multilib libre-testing libre-multilib-testing" +repos_parabola="$repos_libre pcr pcr-multilib pcr-multilib-testing pcr-testing" + +# Arch blacklist files, checked when running with the 'arch' argument. +# The packages here always have a [libre] replacement if there's one. +# aur-blacklist.txt is not here since we want to check Arch's official repos. +blacklists="blacklist.txt" + +# Parabola blacklist files, checked when running with the 'parabola' argument. +# List here files that may also blacklist Parabola packages. These are intended to be used with repos_parabola +blacklists_parabola="your-initfreedom-blacklist.txt your-privacy-blacklist.txt your-freedom_emu-blacklist.txt" + +# Mirrors. Choose HTTPS when possible. +# NOTE: Nginx servers are preferred. Otherwise update sedexp variable. +# Arch Linux Official mirror +mirror_x86_64="https://mirrors.edge.kernel.org/archlinux" +# Arch Linux 32 mirror +mirror_i686="https://mirror.archlinux32.org" +# Arch Linux ARM mirror +mirror_armv7h="https://fl.us.mirror.archlinuxarm.org" +# Parabola GNU/Linux-libre mirror +mirror_parabola="https://mirror.grapentin.org/parabola" + +# Supported Parabola architectures +supported_architectures="x86_64 i686 armv7h any" + +# Sed expresions to parse packages from HTTP servers. +# Tested for Nginx, change if needed. +sedexp='s/.*href="//;s/\.pkg.tar.xz.*//;s/\.pkg.tar.zst.*//;' +# Extra sedexps for special characters/symbols. +sedexp+='s/%2B/+/g;s/%3A/:/g;s/%40/@/g;' |