apport/bin/crash-digger

#!/usr/bin/python3

# Copyright (C) 2007 - 2011 Canonical Ltd.
# Author: Martin Pitt <martin.pitt@ubuntu.com>
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 2 of the License, or (at your
# option) any later version.  See http://www.gnu.org/copyleft/gpl.html for
# the full text of the license.

"""Do duplicate check and retrace crashes in the crash database."""

# pylint: disable=invalid-name
# pylint: enable=invalid-name

import argparse
import errno
import os
import shutil
import subprocess
import sys
import zlib

import apport
from apport.crashdb import get_crashdb


# pylint: disable-next=missing-class-docstring
class CrashDigger:  # pylint: disable=too-many-instance-attributes
    def __init__(
        self,
        config_dir,
        auth_file,
        cache_dir,
        sandbox_dir,
        apport_retrace,
        verbose=False,
        dup_db=None,
        dupcheck_mode=False,
        publish_dir=None,
        crash_db=None,
        gdb_sandbox=False,
    ):  # pylint: disable=too-many-arguments
        """Initialize pools."""

        self.retrace_pool = set()
        self.dupcheck_pool = set()
        self.config_dir = config_dir
        self.cache_dir = cache_dir
        self.sandbox_dir = sandbox_dir
        self.verbose = verbose
        self.auth_file = auth_file
        self.dup_db = dup_db
        self.dupcheck_mode = dupcheck_mode
        self.gdb_sandbox = gdb_sandbox
        try:
            self.crashdb = get_crashdb(auth_file, name=crash_db)
        except KeyError:
            apport.error("Crash database %s does not exist", crash_db)
            sys.exit(1)
        self.lp = False
        try:
            if self.crashdb.launchpad:
                self.lp = True
        except AttributeError:
            pass
        self.apport_retrace = apport_retrace
        self.publish_dir = publish_dir
        if config_dir:
            self.releases = os.listdir(config_dir)
            self.releases.sort()
            apport.log(f"Available releases: {str(self.releases)}", True)
        else:
            self.releases = None

        if self.dup_db:
            self.crashdb.init_duplicate_db(self.dup_db)
            # this verified DB integrity; make a backup now
            shutil.copy2(self.dup_db, f"{self.dup_db}.backup")

    def fill_pool(self):
        """Query crash db for new IDs to process."""

        if self.dupcheck_mode:
            self.dupcheck_pool.update(self.crashdb.get_dup_unchecked())
            apport.log(
                f"fill_pool: dup check pool now: {str(self.dupcheck_pool)}", True
            )
        else:
            self.retrace_pool.update(self.crashdb.get_unretraced())
            apport.log(f"fill_pool: retrace pool now: {str(self.retrace_pool)}", True)

    def retrace_next(self):
        """Grab an ID from the retrace pool and retrace it."""

        crash_id = self.retrace_pool.pop()
        apport.log(
            f"retracing {'LP: ' if self.lp else ''}#{crash_id}"
            f" (left in pool: {len(self.retrace_pool)})",
            True,
        )

        try:
            rel = self.crashdb.get_distro_release(crash_id)
        except ValueError:
            apport.log("could not determine release -- no DistroRelease field?", True)
            self.crashdb.mark_retraced(crash_id)
            return
        if rel not in self.releases:
            apport.log(
                f"crash is release {rel} which does not have a config"
                f" available, skipping",
                True,
            )
            return

        argv = [
            self.apport_retrace,
            "-S",
            self.config_dir,
            "--auth",
            self.auth_file,
            "--timestamps",
        ]
        if self.cache_dir:
            argv += ["--cache", self.cache_dir]
        if self.sandbox_dir:
            argv += ["--sandbox-dir", self.sandbox_dir]
        if self.dup_db:
            argv += ["--duplicate-db", self.dup_db]
        if self.gdb_sandbox:
            argv += ["--gdb-sandbox"]
        if self.verbose:
            argv.append("-v")
        argv.append(str(crash_id))

        result = subprocess.call(argv, stdout=sys.stdout, stderr=subprocess.STDOUT)
        if result != 0:
            apport.log(
                f"retracing {'LP: ' if self.lp else ''}#{crash_id} failed"
                f" with status: {result}",
                True,
            )
            if result == 99:
                self.retrace_pool = set()
                apport.log("transient error reported; halting", True)
                return

        self.crashdb.mark_retraced(crash_id)

    def dupcheck_next(self):
        """Grab an ID from the dupcheck pool and process it."""

        crash_id = self.dupcheck_pool.pop()
        apport.log(
            f"checking {'LP: ' if self.lp else ''}#{crash_id} for duplicate"
            f" (left in pool: {len(self.dupcheck_pool)})",
            True,
        )

        try:
            report = self.crashdb.download(crash_id)
        except (
            MemoryError,
            TypeError,
            ValueError,
            OSError,
            AssertionError,
            zlib.error,
        ) as error:
            if str(error) == "bug description must contain standard apport format data":
                apport.log(f"Cannot download report: {str(error)}", True)
                apport.error("Cannot download report %s: %s", crash_id, str(error))
                return
            apport.log(f"Cannot download report: {str(error)}", True)
            apport.error("Cannot download report %i: %s", crash_id, str(error))
            return

        res = self.crashdb.check_duplicate(crash_id, report)
        if res:
            if res[1] is None:
                version = "not fixed yet"
            elif res[1] == "":
                version = "fixed in latest version"
            else:
                version = f"fixed in version {res[1]}"
            apport.log(f"Report is a duplicate of #{res[0]} ({version})", True)
        else:
            apport.log("Duplicate check negative", True)

    def run(self):
        """Process the work pools until they are empty."""

        self.fill_pool()
        while self.dupcheck_pool:
            self.dupcheck_next()
        while self.retrace_pool:
            self.retrace_next()

        if self.publish_dir:
            self.crashdb.duplicate_db_publish(self.publish_dir)


def parse_options():
    """Parse command line options and return arguments."""

    parser = argparse.ArgumentParser()
    parser.add_argument(
        "-c",
        "--config-dir",
        metavar="DIR",
        help="Packaging system configuration base directory.",
    )
    parser.add_argument(
        "--sandbox-dir",
        metavar="DIR",
        help="Directory for unpacked packages. Future runs will assume that"
        " any already downloaded package is also extracted to this sandbox.",
    )
    parser.add_argument(
        "--gdb-sandbox",
        dest="gdb_sandbox",
        action="store_true",
        help="Use a temporary sandbox for installing the version of GDB (and"
        " its dependencies) from the same release as the report.",
    )
    parser.add_argument(
        "-C",
        "--cache",
        metavar="DIR",
        help="Cache directory for packages downloaded in the sandbox",
    )
    parser.add_argument(
        "-a",
        "--auth",
        dest="auth_file",
        help="Path to a file with the crash database authentication information.",
    )
    parser.add_argument(
        "-l",
        "--lock",
        dest="lockfile",
        help="Lock file; will be created and removed on successful exit, and "
        "program immediately aborts if it already exists",
    )
    parser.add_argument(
        "-d",
        "--duplicate-db",
        dest="dup_db",
        metavar="PATH",
        help="Path to the duplicate sqlite database (default: disabled)",
    )
    parser.add_argument(
        "--crash-db",
        metavar="NAME",
        help='Use a different crash database than the "default"'
        " in /etc/apport/crashdb.conf",
    )
    parser.add_argument(
        "-D",
        "--dupcheck",
        dest="dupcheck_mode",
        action="store_true",
        help="Only check duplicates for architecture independent crashes"
        " (like Python exceptions)",
    )
    parser.add_argument(
        "-v",
        "--verbose",
        action="store_true",
        help="Verbose operation (also passed to apport-retrace)",
    )
    parser.add_argument(
        "--apport-retrace",
        metavar="PATH",
        help="Path to apport-retrace script"
        " (default: directory of crash-digger or $PATH)",
    )
    parser.add_argument(
        "--publish-db",
        metavar="DIR",
        help="After processing all reports, publish duplicate database"
        " to given directory",
    )

    args = parser.parse_args()

    if not args.config_dir and not args.dupcheck_mode:
        apport.fatal("Error: --config-dir or --dupcheck needs to be given")
    if not args.auth_file:
        apport.fatal("Error: -a/--auth needs to be given")

    return args


# pylint: disable-next=missing-function-docstring
def main():
    opts = parse_options()

    # support running from tree, then fall back to $PATH
    if not opts.apport_retrace:
        opts.apport_retrace = os.path.join(
            os.path.dirname(sys.argv[0]), "apport-retrace"
        )
        if not os.access(opts.apport_retrace, os.X_OK):
            opts.apport_retrace = "apport-retrace"

    if opts.lockfile:
        try:
            lock_file = os.open(
                opts.lockfile, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o666
            )
            os.write(lock_file, (f"{os.getpid()}\n").encode())
            os.close(lock_file)
        except OSError as error:
            if error.errno == errno.EEXIST:
                sys.exit(0)
            else:
                raise

    try:
        CrashDigger(
            opts.config_dir,
            opts.auth_file,
            opts.cache,
            opts.sandbox_dir,
            opts.apport_retrace,
            opts.verbose,
            opts.dup_db,
            opts.dupcheck_mode,
            opts.publish_db,
            opts.crash_db,
            opts.gdb_sandbox,
        ).run()
    except SystemExit as error:
        if error.code == 99:
            pass  # fall through lock cleanup
        else:
            raise

    if opts.lockfile:
        os.unlink(opts.lockfile)


if __name__ == "__main__":
    main()