Add from Parabola: db-check-*

If you're seeing this in `git blame`, it chose to follow the wrong
ancestor of a merge commit.

5 files changed, 441 insertions, 0 deletions

diff --git a/db-check-nonfree b/db-check-nonfree
new file mode 100755
index 0000000..7cbeb8f
--- /dev/null
+++ b/db-check-nonfree
@@ -0,0 +1,47 @@
+#!/bin/bash
+
+. "$(dirname "$(readlink -e "$0")")/config"
+. "$(dirname "$(readlink -e "$0")")/db-functions"
+
+if [ $# -ge 1 ]; then
+	error "Calling %s with a specific repository is not supported" "${0##*/}"
+	exit 1
+fi
+
+# TODO: this might lock too much (architectures)
+for repo in "${repos[@]}"; do
+	for pkgarch in "${ARCHES[@]}"; do
+		repo_lock "${repo}" "${pkgarch}" || exit 1
+	done
+done
+
+msg "Check nonfree in repo:"
+libreblacklist update
+nonfree=($(libreblacklist cat | libreblacklist get-pkg | sort -u))
+for repo in "${ARCHREPOS[@]}"; do
+	for pkgarch in "${ARCHES[@]}"; do
+		msg2 "%s %s" "$repo" "$pkgarch"
+		if [ ! -f "${FTP_BASE}/${repo}/os/${pkgarch}/${repo}${DBEXT}" ]; then
+			continue
+		fi
+		unset dbpkgs
+		unset cleanpkgs
+		cleanpkgs=()
+		dbpkgs=($(bsdtar -xOf "${FTP_BASE}/${repo}/os/${pkgarch}/${repo}${DBEXT}" | awk '/^%NAME%/{getline;print}' | sort -u ))
+		for pkgname in "${dbpkgs[@]}"; do
+			if in_array "${pkgname}" "${nonfree[@]}"; then
+				cleanpkgs+=("${pkgname}")
+			fi
+		done
+		if [ ${#cleanpkgs[@]} -ge 1 ]; then
+			msg2 "Nonfree: %s" "${cleanpkgs[*]}"
+			arch_repo_remove "${repo}" "${pkgarch}" "${cleanpkgs[@]}"
+		fi
+	done
+done
+
+for repo in "${repos[@]}"; do
+	for pkgarch in "${ARCHES[@]}"; do
+		repo_unlock "${repo}" "${pkgarch}"
+	done
+done
diff --git a/db-check-package-libraries b/db-check-package-libraries
new file mode 100755
index 0000000..e24b58e
--- /dev/null
+++ b/db-check-package-libraries
@@ -0,0 +1,203 @@
+#!/usr/bin/env python3
+# Copyright (C) 2012  Michał Masłowski  <mtjm@mtjm.eu>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+"""
+Check which libraries are provided or required by a package, store
+this in a database, update and list broken packages.
+
+Dependencies:
+
+- Python 3.2 or later with SQLite 3 support
+
+- ``bsdtar``
+
+- ``readelf``
+"""
+
+
+import os.path
+import re
+import sqlite3
+import subprocess
+import tempfile
+
+
+#: Regexp matching an interesting dynamic entry.
+_DYNAMIC = re.compile(r"^\s*[0-9a-fx]+"
+                      "\s*\((NEEDED|SONAME)\)[^:]*:\s*\[(.+)\]$")
+
+
+def make_db(path):
+    """Make a new, empty, library database at *path*."""
+    con = sqlite3.connect(path)
+    con.executescript("""
+create table provided(
+  library varchar not null,
+  package varchar not null
+);
+create table used(
+  library varchar not null,
+  package varchar not null
+);
+""")
+    con.close()
+
+
+def begin(database):
+    """Connect to *database* and start a transaction."""
+    con = sqlite3.connect(database)
+    con.execute("begin exclusive")
+    return con
+
+
+def add_provided(con, package, libraries):
+    """Write that *package* provides *libraries*."""
+    for library in libraries:
+        con.execute("insert into provided (package, library) values (?,?)",
+                    (package, library))
+
+
+def add_used(con, package, libraries):
+    """Write that *package* uses *libraries*."""
+    for library in libraries:
+        con.execute("insert into used (package, library) values (?,?)",
+                    (package, library))
+
+
+def remove_package(con, package):
+    """Remove all entries for a package."""
+    con.execute("delete from provided where package=?", (package,))
+    con.execute("delete from used where package=?", (package,))
+
+
+def add_package(con, package):
+    """Add entries from a named *package*."""
+    # Extract to a temporary directory.  This could be done more
+    # efficiently, since there is no need to store more than one file
+    # at once.
+    print("adding package:", package)
+    with tempfile.TemporaryDirectory(None, "db-check-package-libraries."+os.path.basename(package)+".") as temp:
+        subprocess.Popen(("bsdtar", "xf", package, "-C", temp)).communicate()
+        subprocess.Popen(('find', temp,
+                          '-type', 'd',
+                          '(', '-not', '-readable', '-o', '-not', '-executable', ')',
+                          '-exec', 'chmod', '755', '--', '{}', ';')).communicate()
+        subprocess.Popen(('find', temp,
+                          '-type', 'f',
+                          '-not', '-readable',
+                          '-exec', 'chmod', '644', '--', '{}', ';')).communicate()
+        with open(os.path.join(temp, ".PKGINFO")) as pkginfo:
+            for line in pkginfo:
+                if line.startswith("pkgname ="):
+                    pkgname = line[len("pkgname ="):].strip()
+                    break
+        # Don't list previously removed libraries.
+        remove_package(con, pkgname)
+        provided = set()
+        used = set()
+        # Search for ELFs.
+        for dirname, dirnames, filenames in os.walk(temp):
+            assert dirnames is not None  # unused, avoid pylint warning
+            for file_name in filenames:
+                path = os.path.join(dirname, file_name)
+                if os.path.islink(path) or not os.path.isfile(path):
+                    continue
+                with open(path, "rb") as file_object:
+                    if file_object.read(4) != b"\177ELF":
+                        continue
+                readelf = subprocess.Popen(("readelf", "-d", path),
+                                           stdout=subprocess.PIPE)
+                for line in readelf.communicate()[0].split(b"\n"):
+                    match = _DYNAMIC.match(line.decode("ascii"))
+                    if match:
+                        if match.group(1) == "SONAME":
+                            provided.add(match.group(2))
+                        elif match.group(1) == "NEEDED":
+                            used.add(match.group(2))
+                        else:
+                            raise AssertionError("unknown entry type "
+                                                 + match.group(1))
+        add_provided(con, pkgname, provided)
+        add_used(con, pkgname, used)
+
+
+def init(arguments):
+    """Initialize."""
+    make_db(arguments.database)
+
+
+def add(arguments):
+    """Add packages."""
+    con = begin(arguments.database)
+    for package in arguments.packages:
+        add_package(con, package)
+    con.commit()
+    con.close()
+
+
+def remove(arguments):
+    """Remove packages."""
+    con = begin(arguments.database)
+    for package in arguments.packages:
+        remove_package(con, package)
+    con.commit()
+    con.close()
+
+
+def check(arguments):
+    """List broken packages."""
+    con = begin(arguments.database)
+    available = set(row[0] for row
+                    in con.execute("select library from provided"))
+    for package, library in con.execute("select package, library from used"):
+        if library not in available:
+            print(package, "needs", library)
+    con.close()
+
+
+def main():
+    """Get arguments and run the command."""
+    from argparse import ArgumentParser
+    parser = ArgumentParser(prog="db-check-package-libraries",
+                            description="Check packages for "
+                            "provided/needed libraries")
+    parser.add_argument("-d", "--database", type=str,
+                        help="Database file to use",
+                        default="package-libraries.sqlite")
+    subparsers = parser.add_subparsers()
+    subparser = subparsers.add_parser(name="init",
+                                      help="initialize the database")
+    subparser.set_defaults(command=init)
+    subparser = subparsers.add_parser(name="add",
+                                      help="add packages to database")
+    subparser.add_argument("packages", nargs="+", type=str,
+                           help="package files to add")
+    subparser.set_defaults(command=add)
+    subparser = subparsers.add_parser(name="remove",
+                                      help="remove packages from database")
+    subparser.add_argument("packages", nargs="+", type=str,
+                           help="package names to remove")
+    subparser.set_defaults(command=remove)
+    subparser = subparsers.add_parser(name="check",
+                                      help="list broken packages")
+    subparser.set_defaults(command=check)
+    arguments = parser.parse_args()
+    arguments.command(arguments)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/db-check-repo-sanity b/db-check-repo-sanity
new file mode 100755
index 0000000..239f042
--- /dev/null
+++ b/db-check-repo-sanity
@@ -0,0 +1,57 @@
+#!/bin/bash
+# Solves issue165... on the old roundup install.  From the database
+# backups, the title was "Older/deprecated packages never leave the
+# repo", I don't know how the body of the issue is stored in the DB,
+# but the title says enough, I think.
+
+. "$(dirname "$(readlink -e "$0")")/../config"
+. "$(dirname "$(readlink -e "$0")")/../db-functions"
+
+# Traverse all repos
+for _repo in "${PKGREPOS[@]}"; do
+	msg "Cleaning up [%s]" "${_repo}"
+
+	# Find all pkgnames on this repo's abs
+	on_abs=($(
+			find "${SVNREPO}/${_repo}" -name PKGBUILD | \
+				while read pkgbuild; do
+				source "${pkgbuild}" >/dev/null 2>&1
+				# cleanup to save memory
+				unset build package source md5sums pkgdesc pkgver pkgrel epoch \
+					url license arch depends makedepends optdepends options \
+					>/dev/null 2>&1
+
+				# also cleanup package functions
+				for _pkg in "${pkgname[@]}"; do
+					unset "package_${pkg}" >/dev/null 2>&1
+				done
+
+				# this fills the on_abs array
+				echo "${pkgname[@]}"
+			done
+			))
+
+	# quit if abs is empty
+	if [ ${#on_abs[*]} -eq 0 ]; then
+		warning "[%s]'s ABS tree is empty, skipping" "${_repo}"
+		break
+	fi
+
+	# Find all pkgnames on repos
+	on_repo=($(
+			find "${FTP_BASE}/${_repo}" -name "*.pkg.tar.?z" \
+				-printf "%f\n" | sed "s/^\(.\+\)-[^-]\+-[^-]\+-[^-]\+$/\1/"
+			))
+
+	# Compares them, whatever is on repos but not on abs should be removed
+	remove=($(comm -13 \
+	               <(printf '%s\n' "${on_abs[@]}"  | sort -u) \
+	               <(printf '%s\n' "${on_repo[@]}" | sort -u) ))
+
+	# Remove them from databases, ftpdir-cleanup will take care of the rest
+	find "${FTP_BASE}/${_repo}" -name "*.db.tar.?z" \
+		-exec repo-remove {} "${remove[@]}" \; >/dev/null 2>&1
+
+	msg2 "Removed the following packages:"
+	plain '%s' "${remove[@]}"
+done
diff --git a/db-check-unsigned-packages b/db-check-unsigned-packages
new file mode 100755
index 0000000..0fc053b
--- /dev/null
+++ b/db-check-unsigned-packages
@@ -0,0 +1,38 @@
+#!/bin/bash
+# Copyright (C) 2012  Michał Masłowski  <mtjm@mtjm.eu>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+set -e
+
+# Output a list of repo/package-name-and-version pairs representing
+# unsigned packages available for architecture $1 and specified for
+# architecture $2 (usually $1 or any, default is to list all).
+
+. "$(dirname "$(readlink -e "$0")")/config"
+. "$(dirname "$(readlink -e "$0")")/db-functions"
+
+if [ $# -lt 1 ]; then
+	msg "usage: %s <architecture>" "${0##*/}"
+	exit 1
+fi
+
+arch=$1
+shift
+
+for repo in "${PKGREPOS[@]}"
+do
+	db="${FTP_BASE}/${repo}/os/${arch}/${repo}.db"
+	[ -f "$db" ] && "$(dirname "$(readlink -e "$0")")/db-check-unsigned-packages.py" "$repo" "$@" < "$db"
+done
diff --git a/db-check-unsigned-packages.py b/db-check-unsigned-packages.py
new file mode 100755
index 0000000..80cff51
--- /dev/null
+++ b/db-check-unsigned-packages.py
@@ -0,0 +1,96 @@
+#!/usr/bin/env python3
+# Copyright (C) 2012  Michał Masłowski  <mtjm@mtjm.eu>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+"""
+Output a list of repo/package-name-and-version pairs representing
+unsigned packages in the database at standard input of repo named in
+the first argument and specified for architectures listed in the
+following arguments (usually the one of the database or any, default
+is to list all).
+
+If the --keyset argument is passed, print the key fingerprint of every
+signed package.
+"""
+
+
+import base64
+import subprocess
+import sys
+import tarfile
+
+
+def main():
+    """Do the job."""
+    check_keys = False
+    if "--keyset" in  sys.argv:
+        sys.argv.remove("--keyset")
+        check_keys = True
+    repo = sys.argv[1]
+    pkgarches = frozenset(name.encode("utf-8") for name in sys.argv[2:])
+    packages = []
+    keys = []
+    with tarfile.open(fileobj=sys.stdin.buffer) as archive:
+        for entry in archive:
+            if entry.name.endswith("/desc"):
+                content = archive.extractfile(entry)
+                skip = False
+                is_arch = False
+                key = None
+                for line in content:
+                    if is_arch:
+                        is_arch = False
+                        if pkgarches and line.strip() not in pkgarches:
+                            skip = True  # different architecture
+                            break
+                    if line == b"%PGPSIG%\n":
+                        skip = True  # signed
+                        key = b""
+                        if check_keys:
+                            continue
+                        else:
+                            break
+                    if line == b"%ARCH%\n":
+                        is_arch = True
+                        continue
+                    if key is not None:
+                        if line.strip():
+                            key += line.strip()
+                        else:
+                            break
+                if check_keys and key:
+                    key_binary = base64.b64decode(key)
+                    keys.append(key_binary)
+                    packages.append(repo + "/" + entry.name[:-5])
+                if skip:
+                    continue
+                print(repo + "/" + entry.name[:-5])
+    if check_keys and keys:
+        # We have collected all signed package names in packages and
+        # all keys in keys.  Let's now ask gpg to list all signatures
+        # and find which keys made them.
+        packets = subprocess.check_output(("gpg", "--list-packets"),
+                                          input=b"".join(keys))
+        i = 0
+        for line in packets.decode("latin1").split("\n"):
+            if line.startswith(":signature packet:"):
+                keyid = line[line.index("keyid ") + len("keyid "):]
+                print(packages[i], keyid)
+                i += 1
+
+
+if __name__ == "__main__":
+    main()