import os
import sys
import time
import copy
import inspect
import contextlib
import collections
import logging
import http.client
import json
import urllib.request
import urllib.parse
import urllib.error
import re
import getpass

import keyring

import debian.debian_support

import mini_buildd.misc
import mini_buildd.net

LOG = logging.getLogger(__name__)

#: On error, API commands fail with some HTTP error. For further perusal, more information should
#: be available via special HTTP headers starting with this prefix.
HTTP_MESSAGE_PREFIX = "X-Mini-Buildd-Message"

#: Monkey-patch python's maxheader limitation. The default (100) is sometimes too low.
http.client._MAXHEADERS = 5000  # pylint: disable=protected-access


class Argument():
    def __init__(self, id_list, doc="Undocumented", default=None):
        """
        :param id_list: List like '['--with-rollbacks', '-R']' for option or '['distributions']' for positionals; 1st entry always denotes the id.

        >>> Argument(["--long-id", "-s"]).identity
        'long_id'
        >>> Argument(["posi-tional"]).identity
        'posi_tional'
        """
        self.id_list = id_list
        self.doc = doc
        # default, value: Always the str representation (as given on the command line)
        self.default = default
        self.raw_value = default
        self.given = False

        # identity: 1st of id_list with leading '--' removed and hyphens turned to underscores
        self.identity = id_list[0][2 if id_list[0].startswith("--") else 0:].replace("-", "_")

        # kvsargs for argparse
        self.argparse_kvsargs = {}
        self.argparse_kvsargs["help"] = doc
        if default is not None:
            self.argparse_kvsargs["default"] = default

    def _r2v(self):
        """Raw to value. Pre: self.raw_value is not None."""
        return self.raw_value

    @classmethod
    def _v2r(cls, value):
        """Value to raw."""
        return str(value)

    def set(self, value):
        if isinstance(value, str):
            self.raw_value = value
        else:
            self.raw_value = self._v2r(value)
        self.given = True

    @property
    def value(self):
        """Get value, including convenience transformations."""
        return self._r2v() if self.raw_value is not None else None

    # do we really need that?
    def false2none(self):
        return self.raw_value if self.raw_value else None


class StringArgument(Argument):
    TYPE = "string"

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.argparse_kvsargs["action"] = "store"


class URLArgument(StringArgument):
    TYPE = "url"


class TextArgument(StringArgument):
    TYPE = "text"


class IntArgument(StringArgument):
    TYPE = "int"

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.argparse_kvsargs["type"] = int

    def _r2v(self):
        return int(self.raw_value)


class BoolArgument(Argument):
    TYPE = "bool"

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.argparse_kvsargs["action"] = "store_true"

    def _r2v(self):
        return self.raw_value in ("True", "true", "1")


class SelectArgument(Argument):
    TYPE = "select"

    def __init__(self, *args, choices=None, **kwargs):
        super().__init__(*args, **kwargs)
        self.choices = [] if choices is None else choices
        if choices:
            self.argparse_kvsargs["choices"] = choices


class MultiSelectArgument(StringArgument):
    TYPE = "multiselect"

    def __init__(self, *args, separator=",", choices=None, **kwargs):
        super().__init__(*args, **kwargs)
        self.choices = [] if choices is None else choices
        self.separator = separator

    def _r2v(self):
        return self.raw_value.split(self.separator) if self.raw_value else []

    def _v2r(self, value):
        return self.separator.join(value)


class Command():
    # Auth
    NONE = 0
    LOGIN = 1
    STAFF = 2
    ADMIN = 3
    AUTH = NONE
    AUTH_STRINGS = {NONE: "anonymous",
                    LOGIN: "any user login",
                    STAFF: "staff user login",
                    ADMIN: "super user login"}

    CONFIRM = False
    NEEDS_RUNNING_DAEMON = False
    ARGUMENTS = []

    PLAIN = "__plain__"

    @classmethod
    def isgroup(cls):
        return cls.__name__.startswith("Group")

    @classmethod
    def name(cls):
        return cls.__name__.lower()

    @classmethod
    def doc(cls):
        return inspect.getdoc(cls)

    @classmethod
    def arg_identities(cls):
        return [a.identity for a in cls.ARGUMENTS]

    @classmethod
    def auth_err(cls, user):
        """
        Check if django user is authorized to call command.

        Empty string means user is authorized.
        """
        def chk_login():
            return user.is_authenticated and user.is_active

        if user is None:
            return f"API: '{cls.name()}': Internal Error: No user information available"
        if (cls.AUTH == cls.LOGIN) and not chk_login():
            return f"API: '{cls.name()}': Please login to run this command"
        if (cls.AUTH == cls.STAFF) and not (chk_login() and user.is_staff):
            return f"API: '{cls.name()}': Please login as 'staff' user to run this command"
        if (cls.AUTH == cls.ADMIN) and not (chk_login() and user.is_superuser):
            return f"API: '{cls.name()}': Please login as superuser ('admin') to run this command"
        return ""  # Auth OK

    def __init__(self, given_args, daemon=None, request=None, msglog=LOG):
        self.args = {}
        for arg in self.ARGUMENTS:
            self.args[arg.identity] = copy.copy(arg)

        self.daemon = daemon
        self.request = request
        self.msglog = msglog
        self.result = {}  # json result

        self.update(given_args)

    @classmethod
    def json_pretty(cls, result):
        return json.dumps(result, indent=2)

    @classmethod
    def plain(cls, result):
        return result.get(cls.PLAIN, cls.json_pretty(result))

    def update(self, given_args):
        def _get(key):
            try:
                # django request.GET args. Only consider non-empty values.
                return ",".join([v for v in given_args.getlist(key) if v])
            except BaseException:
                # dict arg (like from get_default_args)
                return given_args[key]

        for argument in self.args.values():
            if argument.identity in given_args:
                argument.set(_get(argument.identity))

        self._update()

    def _update(self):
        pass

    def run(self):
        # Sanity checks
        for argument in self.args.values():
            if argument.raw_value is None:
                raise Exception(f"Missing required argument '{argument.identity}'")

        # Run
        self._run()

    def _run(self):
        raise Exception(f"No _run() function defined for: {self.name()}")

    def has_flag(self, flag):
        return self.args.get(flag, "False") == "True"


def _pimpdoc(result_doc=None):
    def pimp(cls):
        indent = "    "  # Be sure to keep the same indent like a top level class doc (4 chars) for all lines, so inspect.getdoc() will properly unindent.
        if result_doc:
            cls.__doc__ += "\n" + "\n".join([f"{indent}{line}" for line in result_doc]) + "\n"
        if not cls.isgroup():
            cls.__doc__ += f"\n{indent}Authorization: {cls.AUTH_STRINGS[cls.AUTH]}\n"
        return cls
    return pimp


class GroupDaemon(Command):
    """
    Daemon commands.
    """

    _STATUS_RESULT_DOC = (
        "JSON Result (dict):",
        "  identity    : string: instance identity",
        "  version     : string: mini-buildd version",
        "  http        : string: HTTP URL",
        "  ftp         : string: FTP URL",
        "  running     : bool: Whether instance is accepting incoming",
        "  load        : float: machine load (0 =< load <= 1+)",
        "  chroots     : dict: {codename: [arch, ..]}: Active chroots",
        "  repositories: dict: {identity: [codename, ..]}: Active repositories",
        "  remotes     : list: Active or auto-reactivate remotes",
        "  packaging   : list: Active (source) packages being processed",
        "  building    : list: Active (binary) package builds")

    def _status_result(self):
        chroots = {}
        for c in self.daemon.get_active_chroots():
            chroots.setdefault(c.source.codename, [])
            chroots[c.source.codename].append(c.architecture.name)

        repositories = {}
        for r in self.daemon.get_active_repositories():
            repositories[r.identity] = {}
            repositories[r.identity]["layout"] = {s.suite.name: {"uploadable": s.uploadable,
                                                                 "experimental": s.experimental,
                                                                 "migrates_to": s.migrates_to.suite.name if s.migrates_to else None} for s in r.layout.suiteoption_set.all()}
            repositories[r.identity]["codenames"] = [d.base_source.codename for d in r.distributions.all()]

        return {
            "identity": self.daemon.model.identity,
            "version": mini_buildd.__version__,
            "http": mini_buildd.config.primary_http_endpoint().geturl(),
            "ftp": self.daemon.model.mbd_get_ftp_endpoint().geturl(),
            "running": self.daemon.is_running(),
            "load": self.daemon.build_queue.load,
            "chroots": chroots,
            "repositories": repositories,
            "remotes": [r.mbd_url() for r in self.daemon.get_active_or_auto_reactivate_remotes()],
            "packaging": [f"{p}" for p in list(self.daemon.packages.values())],
            "building": [f"{b}" for b in list(self.daemon.builds.values())]}

    def _upload_template_package(self, template_package, dist):
        """Portext macro. Used for keyringpackages and testpackages."""
        with contextlib.closing(template_package) as package:
            dsc_url = "file://" + package.dsc  # pilint: disable=no-member; see https://github.com/PyCQA/pylint/issues/1437
            info = f"Port for {dist}: {os.path.basename(dsc_url)}"
            try:
                self.msglog.info(f"Requesting: {info}")
                return self.daemon.portext(dsc_url, dist)
            except BaseException as e:
                mini_buildd.config.log_exception(self.msglog, f"FAILED: {info}", e)


@_pimpdoc(GroupDaemon._STATUS_RESULT_DOC)
class Status(GroupDaemon):
    """
    Show the status of the mini-buildd instance.
    """

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)

    def _run(self):
        self.result = self._status_result()

    @classmethod
    def has_chroot(cls, result, codename, arch):
        return codename in result["chroots"] and arch in result["chroots"][codename]

    def __test_msglog(self):
        self.msglog.debug("DEBUG USER MESSAGE")
        self.msglog.info("INFO USER MESSAGE")
        self.msglog.warning("WARN USER MESSAGE")
        self.msglog.error("ERROR USER MESSAGE")
        self.msglog.critical("CRITICAL USER MESSAGE")


@_pimpdoc(GroupDaemon._STATUS_RESULT_DOC)
class Start(GroupDaemon):
    """
    Start the Daemon (engine).
    """

    AUTH = Command.ADMIN
    ARGUMENTS = [BoolArgument(["--force-check", "-C"], default=False, doc="run checks on instances even if already checked.")]

    def _run(self):
        if not self.daemon.start(force_check=self.has_flag("force_check"), msglog=self.msglog):
            raise Exception("Could not start Daemon (check logs and messages).")
        self.result = self._status_result()


@_pimpdoc(GroupDaemon._STATUS_RESULT_DOC)
class Stop(GroupDaemon):
    """
    Stop the Daemon (engine).
    """

    AUTH = Command.ADMIN
    ARGUMENTS = []

    def _run(self):
        if not self.daemon.stop(msglog=self.msglog):
            raise Exception("Could not stop Daemon (check logs and messages).")
        self.result = self._status_result()


@_pimpdoc()
class GetUploaders(GroupDaemon):
    """
    Get all GPG keys allowed to upload to repositories.

    JSON Result (dict): {repository: {'allow_unauthenticated_uploads': True|False, 'uploaders': {long_key_id: {key info...}), ..}}}
    """

    AUTH = Command.ADMIN
    NEEDS_RUNNING_DAEMON = True
    ARGUMENTS = [SelectArgument(["--repository", "-R"], default=".*", doc="repository name regex.")]

    def _update(self):
        if self.daemon:
            self.args["repository"].choices = [r.identity for r in self.daemon.get_active_repositories()]

    def _run(self):
        for r in self.daemon.get_active_repositories().filter(identity__regex=rf"^{self.args['repository'].value}$"):
            self.result[r.identity] = {"allow_unauthenticated_uploads": r.allow_unauthenticated_uploads}
            self.result[r.identity]["uploaders"] = self.daemon.keyrings.get_uploaders()[r.identity].get_pub_keys()


@_pimpdoc()
class Meta(GroupDaemon):
    """Call arbitrary meta functions for models; usually for internal use only."""

    AUTH = Command.ADMIN
    ARGUMENTS = [StringArgument(["model"], doc="Model path, for example 'source.Archive'"),
                 StringArgument(["function"], doc="Meta function to call, for example 'add_local'")]

    def _run(self):
        self.daemon.meta(self.args["model"].value, self.args["function"].value, msglog=self.msglog)


@_pimpdoc()
class AutoSetup(GroupDaemon):
    """
    Auto setup / bootstrap.
    """

    AUTH = Command.ADMIN
    CONFIRM = True
    ARGUMENTS = [
        MultiSelectArgument(["--vendors", "-V"], default="debian", choices=["debian", "ubuntu"], doc="comma-separated list of vendors to auto-setup for."),
        MultiSelectArgument(["--repositories", "-R"], default="test", choices=["test", "debdev"], doc="comma-separated list of repositories to auto-setup for."),
        SelectArgument(["--chroot-backend", "-C"], default="Dir", choices=["Dir", "File", "LVM", "LoopLVM", "BtrfsSnapshot"], doc="chroot backend to use, or empty string to not create chroots.")
    ]

    def _run(self):
        self.daemon.stop()

        # Daemon
        self.daemon.meta("daemon.Daemon", "pca_all", msglog=self.msglog)

        # Sources
        self.daemon.meta("source.Archive", "add_local", msglog=self.msglog)
        for v in self.args["vendors"].value:
            self.daemon.meta("source.Archive", f"add_{v}", msglog=self.msglog)
            self.daemon.meta("source.Source", f"add_{v}", msglog=self.msglog)
        self.daemon.meta("source.PrioritySource", "add_extras", msglog=self.msglog)
        self.daemon.meta("source.Source", "pca_all", msglog=self.msglog)

        # Repositories
        self.daemon.meta("repository.Layout", "create_defaults", msglog=self.msglog)
        self.daemon.meta("repository.Distribution", "add_base_sources", msglog=self.msglog)
        for r in self.args["repositories"].value:
            self.daemon.meta("repository.Repository", f"add_{r}", msglog=self.msglog)
        self.daemon.meta("repository.Repository", "pca_all", msglog=self.msglog)

        # Chroots
        if self.args["chroot_backend"].value:
            cb_class = f"chroot.{self.args['chroot_backend'].value}Chroot"
            self.daemon.meta(cb_class, "add_base_sources", msglog=self.msglog)
            self.daemon.meta(cb_class, "pca_all", msglog=self.msglog)

        self.daemon.start()


@_pimpdoc()
class KeyringPackages(GroupDaemon):
    """
    Build keyring packages for all active repositories.
    """

    AUTH = Command.ADMIN
    NEEDS_RUNNING_DAEMON = True
    CONFIRM = True
    ARGUMENTS = [
        MultiSelectArgument(["--distributions", "-D"], doc="comma-separated list of distributions to act on (defaults to all 'build_keyring_package distributions')."),
        BoolArgument(["--no-migration", "-N"], default=False, doc="don't migrate packages."),
    ]

    def _update(self):
        if self.daemon:
            # Possible choices
            self.args["distributions"].choices = []
            for r in self.daemon.get_active_repositories():
                self.args["distributions"].choices += r.mbd_distribution_strings(build_keyring_package=True)

            # Reasonable default
            if not self.args["distributions"].given:
                self.args["distributions"].set(self.args["distributions"].choices)

    def _run(self):
        uploaded = set()
        for d in self.args["distributions"].value:
            repository, distribution, suite, _rollback = self.daemon.parse_distribution(d)
            if not suite.build_keyring_package:
                raise Exception(f"Port failed: Keyring package to non-keyring suite requested (see 'build_keyring_package' flag): '{d}'")

            uploaded.add(self._upload_template_package(mini_buildd.daemon.KeyringPackage(self.daemon.model), d))

        built = set()
        for dist, package, version in uploaded:
            pkg_info = f"{package}-{version} in {dist}..."

            tries, max_tries, sleep = 0, 50, 15
            while tries <= max_tries:
                if repository.mbd_package_find(package, dist, version):
                    built.add((dist, package, version))
                    break
                self.msglog.info(f"Waiting for {pkg_info} ({tries}/{max_tries})")
                tries += 1
                time.sleep(sleep)

        if uploaded != built:
            self.msglog.warning(f"Timed out waiting for these packages (skipping migrate): {uploaded - built}")

        if not self.args["no_migration"].value:
            for dist, package, version in built:
                repository, distribution, suite, _rollback = self.daemon.parse_distribution(dist)
                self.msglog.info(f"Migrating {pkg_info}...")
                repository.mbd_package_migrate(package, distribution, suite, full=True, version=version, msglog=self.msglog)


@_pimpdoc()
class TestPackages(GroupDaemon):
    """
    Build internal test packages.

    Per default, we build all test packages for all active
    distributions ending on 'experimental'.
    """

    AUTH = Command.ADMIN
    NEEDS_RUNNING_DAEMON = True
    CONFIRM = True
    ARGUMENTS = [
        MultiSelectArgument(["--packages", "-P"],
                            default="mbd-test-archall,mbd-test-cpp,mbd-test-ftbfs",
                            choices=["mbd-test-archall", "mbd-test-cpp", "mbd-test-ftbfs"],
                            doc="what test packages to use."),
        MultiSelectArgument(["--distributions", "-D"], doc="comma-separated list of distributions to upload to (defaults to all distributions ending in 'experimental')."),
    ]

    def _update(self):
        if self.daemon:
            # Possible choices
            self.args["distributions"].choices = []
            for r in self.daemon.get_active_repositories():
                self.args["distributions"].choices += r.mbd_distribution_strings(uploadable=True)

            # Reasonable default
            # Default layout has two (snapshot and experimental) suites flagged as experimental.
            # So we go here for the string "experimental" (not the flag) to avoid double testing in the standard case.
            if not self.args["distributions"].given:
                self.args["distributions"].set([d for d in self.args["distributions"].choices if d.endswith("experimental")])

    def _run(self):
        for d in self.args["distributions"].value:
            for p in self.args["packages"].value:
                self._upload_template_package(mini_buildd.daemon.TestPackage(p), d)


class GroupConfig(Command):
    """Configuration convenience commands."""


@_pimpdoc()
class GetKey(GroupConfig):
    """
    Get GnuPG public key.

    JSON Result (dict):
      __plain__: string: ASCII-armored key.
    """

    def _run(self):
        self.result[self.PLAIN] = self.daemon.model.mbd_get_pub_key()


@_pimpdoc()
class GetDputConf(GroupConfig):
    """
    Get recommended dput config snippet.

    Usually, this is for integration in your personal ~/.dput.cf.

    JSON Result (dict):
      __plain__: string: dput config snippet.
    """

    def _run(self):
        self.result[self.PLAIN] = self.daemon.model.mbd_get_dput_conf()


@_pimpdoc()
class GetSourcesList(GroupConfig):
    """
    Get sources.list (apt lines).

    Usually, this output is put to a file like '/etc/sources.list.d/mini-buildd-xyz.list'.

    JSON Result (dict):
      __plain__: string: apt lines
    """

    ARGUMENTS = [
        SelectArgument(["codename"], doc="codename (base distribution) to get apt lines for"),
        SelectArgument(["--repository", "-R"], default=".*", doc="repository name regex."),
        SelectArgument(["--suite", "-S"], default=".*", doc="suite name regex."),
        BoolArgument(["--with-deb-src", "-s"], default=False, doc="also list deb-src apt lines."),
        BoolArgument(["--with-extra-sources", "-x"], default=False, doc="also list extra sources needed.")
    ]

    def _update(self):
        if self.daemon:
            self.args["codename"].choices = self.daemon.get_active_codenames()
            self.args["repository"].choices = [r.identity for r in self.daemon.get_active_repositories()]
            self.args["suite"].choices = [s.name for s in self.daemon.get_suites()]

    def _run(self):
        self.result[self.PLAIN] = self.daemon.mbd_get_sources_list(self.args["codename"].value,
                                                                   self.args["repository"].value,
                                                                   self.args["suite"].value,
                                                                   ["deb ", "deb-src "] if self.args["with_deb_src"].value else ["deb "],
                                                                   self.args["with_extra_sources"].value)


@_pimpdoc()
class LogCat(GroupConfig):
    """
    Cat last n lines of the mini-buildd's log.

    JSON Result (dict):
      __plain__: string: last n loglines
    """

    AUTH = Command.STAFF
    ARGUMENTS = [
        IntArgument(["--lines", "-n"], default=500, doc="cat (approx.) the last N lines")
    ]

    def _run(self):
        self.result[self.PLAIN] = self.daemon.logcat(lines=self.args["lines"].value)


class GroupPackage(Command):
    """Package management commands."""

    # Used in: migrate, remove, port
    _ARG_VERSION = StringArgument(["--version", "-V"], default="", doc="""
limit command to that version. Use it for the rare case of
multiple version of the same package in one distribution (in
different components), or just as safeguard
""")

    # Used in: port, portext
    _ARG_OPTIONS = StringArgument(["--options", "-O"],
                                        default="ignore-lintian=true",
                                        doc="""list of upload options, separated by '|' (see user manual for complete docs). Some useful examples:

ignore-lintian=true:
  Ignore lintian failures (install anyway).
run-lintian=false:
  Avoid lintian run in the 1st place.
internal-apt-priority=500:
  Install newer versions from our repo (even if deps don't require it).
auto-ports=buster-test-unstable
  List of distributions (comma-separated) to automatically run ports for after successful install.
""")


@_pimpdoc()
class List(GroupPackage):
    """
    List packages matching a shell-like glob pattern; matches both source and binary package names.

    JSON Result (dict): {repository: [{package_info}...]}
    """

    AUTH = Command.LOGIN
    ARGUMENTS = [
        SelectArgument(["pattern"], doc="limit packages by name (glob pattern)"),
        BoolArgument(["--with-rollbacks", "-r"], default=False, doc="also list packages on rollback distributions"),
        SelectArgument(["--distribution", "-D"], default="", doc="limit distributions by name (regex)"),
        SelectArgument(["--type", "-T"], default="", choices=["dsc", "deb", "udeb"], doc="package type: dsc, deb or udeb (like reprepo --type)")
    ]

    def _update(self):
        if self.daemon:
            self.args["distribution"].choices = []
            for r in self.daemon.get_active_repositories():
                self.args["distribution"].choices += r.mbd_distribution_strings()
            self.args["pattern"].choices = self.daemon.get_last_packages()

    def _run(self):
        # Save all results of all repos in a top-level dict (don't add repos with empty results).
        for r in self.daemon.get_active_repositories():
            r_result = r.mbd_package_list(self.args["pattern"].value,
                                          typ=self.args["type"].false2none(),
                                          with_rollbacks=self.args["with_rollbacks"].value,
                                          dist_regex=self.args["distribution"].value)
            if r_result:
                self.result[r.identity] = r_result


@_pimpdoc()
class Show(GroupPackage):
    """
    Show a source package.

    JSON Result (dict): {repository: [(codename, {package_info}),...]}
    """

    ARGUMENTS = [SelectArgument(["package"], doc="source package name")]

    def _update(self):
        if self.daemon:
            self.args["package"].choices = self.daemon.get_last_packages()

    def _run(self):
        # Save all results of all repos in a top-level dict (don't add repos with empty results).
        for r in self.daemon.get_active_repositories():
            r_result = r.mbd_package_show(self.args["package"].value)
            if r_result:
                self.result[r.identity] = r_result


@_pimpdoc()
class Find(GroupPackage):
    """
    Find a source package.

    JSON Result (dict): {repository: [{package_info},...]}
    """

    ARGUMENTS = [
        SelectArgument(["package"], doc="source package name"),
        MultiSelectArgument(["--distributions", "-D"], default="", doc="limit to this comma-separated list of distributions"),
        StringArgument(["--version", "-V"], default="", doc="limit to this version")
    ]

    def _run(self):
        found = False
        for r in self.daemon.get_active_repositories():
            show = r.mbd_reprepro().show(self.args["package"].value)
            self.result[r.identity] = [s for s in show if
                                       (not self.args["version"].value or s["sourceversion"] == self.args["version"].value) and
                                       (not self.args["distributions"].value or s["distribution"] in self.args["distributions"].value)]
            found = found or self.result[r.identity]

        if not found:
            raise Exception(f"{self.args['package'].value} (version={self.args['version'].value or 'any'}) not found in {self.args['distributions'].value or 'any distribution'}")


@_pimpdoc()
class Migrate(GroupPackage):
    """
    Migrate a source package (along with all binary packages).

    JSON Result (dict):
      __plain__: string: textual migration log.
    """

    AUTH = Command.STAFF
    CONFIRM = True
    ARGUMENTS = [
        SelectArgument(["package"], doc="source package name"),
        SelectArgument(["distribution"], doc="distribution to migrate from (if this is a '-rollbackN' distribution, this will perform a rollback restore)"),
        BoolArgument(["--full", "-F"], default=False, doc="migrate all 'migrates_to' suites up (f.e. unstable->testing->stable)."),
        GroupPackage._ARG_VERSION
    ]

    def _update(self):
        if self.daemon:
            self.args["package"].choices = self.daemon.get_last_packages()
            self.args["distribution"].choices = []
            for r in self.daemon.get_active_repositories():
                self.args["distribution"].choices += r.mbd_distribution_strings(migrates_to__isnull=False)

    def _run(self):
        repository, distribution, suite, rollback = self.daemon.parse_distribution(self.args["distribution"].value)
        self.result[self.PLAIN] = repository.mbd_package_migrate(self.args["package"].value,
                                                                 distribution,
                                                                 suite,
                                                                 full=self.args["full"].value,
                                                                 rollback=rollback,
                                                                 version=self.args["version"].false2none(),
                                                                 msglog=self.msglog)


@_pimpdoc()
class Remove(GroupPackage):
    """
    Remove a source package (along with all binary packages).

    JSON Result (dict):
      __plain__: string: textual removal log.
    """

    AUTH = Command.ADMIN
    CONFIRM = True
    ARGUMENTS = [
        SelectArgument(["package"], doc="source package name"),
        SelectArgument(["distribution"], doc="distribution to remove from"),
        GroupPackage._ARG_VERSION
    ]

    def _update(self):
        if self.daemon:
            self.args["package"].choices = self.daemon.get_last_packages()
            self.args["distribution"].choices = []
            for r in self.daemon.get_active_repositories():
                self.args["distribution"].choices += r.mbd_distribution_strings()

    def _run(self):
        repository, distribution, suite, rollback = self.daemon.parse_distribution(self.args["distribution"].value)
        self.result[self.PLAIN] = repository.mbd_package_remove(self.args["package"].value,
                                                                distribution,
                                                                suite,
                                                                rollback=rollback,
                                                                version=self.args["version"].false2none(),
                                                                msglog=self.msglog)


@_pimpdoc()
class Port(GroupPackage):
    """
    Port an internal package.

    An internal 'port' is a no-changes (i.e., only the changelog
    will be adapted) rebuild of the given locally-installed
    package.

    JSON Result (dict):
      requested: list: Info string about each port that has been triggered
    """

    AUTH = Command.STAFF
    NEEDS_RUNNING_DAEMON = True
    CONFIRM = True
    ARGUMENTS = [
        SelectArgument(["package"], doc="source package name"),
        SelectArgument(["from_distribution"], doc="distribution to port from"),
        MultiSelectArgument(["to_distributions"], doc="comma-separated list of distributions to port to (when this equals the from-distribution, a rebuild will be done)"),
        GroupPackage._ARG_VERSION,
        GroupPackage._ARG_OPTIONS]

    def _update(self):
        if self.daemon:
            self.args["package"].choices = self.daemon.get_last_packages()
            self.args["from_distribution"].choices = []
            for r in self.daemon.get_active_repositories():
                self.args["from_distribution"].choices += r.mbd_distribution_strings()
            if self.args["from_distribution"].value:
                repository, _distribution, suite, _rollback_no = self.daemon.parse_distribution(self.args["from_distribution"].value)
                self.args["to_distributions"].choices = repository.mbd_distribution_strings(uploadable=True, experimental=suite.experimental)
            else:
                for r in self.daemon.get_active_repositories():
                    self.args["to_distributions"].choices += r.mbd_distribution_strings(uploadable=True)

    def _run(self):
        self.result["requested"] = []

        # Parse and pre-check all dists
        for to_distribution in self.args["to_distributions"].value:
            info = f"Port {self.args['package'].value}/{self.args['from_distribution'].value} -> {to_distribution}"
            self.msglog.info(f"Trying: {info}")
            self.daemon.port(self.args["package"].value,
                             self.args["from_distribution"].value,
                             to_distribution,
                             version=self.args["version"].false2none(),
                             options=self.args["options"].value.split("|"))
            self.msglog.info(f"Requested: {info}")
            self.result["requested"].append(info)


@_pimpdoc()
class PortExt(GroupPackage):
    """
    Port an external package.

    An external 'port' is a no-changes (i.e., only the changelog
    will be adapted) rebuild of any given source package.

    JSON Result (dict):
      requested: list: Info string about each port that has been triggered.
    """

    AUTH = Command.STAFF
    NEEDS_RUNNING_DAEMON = True
    CONFIRM = True
    ARGUMENTS = [
        URLArgument(["dsc"], doc="URL of any Debian source package (dsc) to port"),
        MultiSelectArgument(["distributions"], doc="comma-separated list of distributions to port to"),
        GroupPackage._ARG_OPTIONS
    ]

    def _update(self):
        if self.daemon:
            self.args["distributions"].choices = []
            for r in self.daemon.get_active_repositories():
                self.args["distributions"].choices += r.mbd_distribution_strings(uploadable=True)

    def _run(self):
        self.result["requested"] = []

        # Parse and pre-check all dists
        for d in self.args["distributions"].value:
            info = f"External port {self.args['dsc'].value} -> {d}"
            self.msglog.info(f"Trying: {info}")
            self.daemon.portext(self.args["dsc"].value, d, options=self.args["options"].value.split("|"))
            self.msglog.info(f"Requested: {info}")
            self.result["requested"].append(info)


@_pimpdoc()
class Retry(GroupPackage):
    """
    Retry a previously failed package.

    JSON Result (dict):
      requested: string: Info string about each retry that has been triggered.
    """

    AUTH = Command.STAFF
    NEEDS_RUNNING_DAEMON = True
    CONFIRM = True
    ARGUMENTS = [
        SelectArgument(["package"], doc="source package name"),
        SelectArgument(["version"], doc="source package's version"),
        SelectArgument(["--repository", "-R"], default="*", doc="Repository name -- use only in case of multiple matches.")
    ]

    def _update(self):
        if self.daemon:
            self.args["repository"].choices = [r.identity for r in self.daemon.get_active_repositories()]
            self.args["package"].choices = self.daemon.get_last_packages()
            if self.args["package"].value:
                self.args["version"].choices = self.daemon.get_last_versions(self.args["package"].value)

    def _run(self):
        pkg_log = mini_buildd.misc.PkgLog(self.args["repository"].value, False, self.args["package"].value, self.args["version"].value)
        if not pkg_log.changes:
            raise Exception("No matching changes found for your retry query.")
        self.daemon.incoming_queue.put(pkg_log.changes)
        self.msglog.info(f"Retrying: {os.path.basename(pkg_log.changes)}")

        self.result["requested"] = os.path.basename(os.path.basename(pkg_log.changes))


class GroupUser(Command):
    """User management commands."""


@_pimpdoc()
class SetUserKey(GroupUser):
    """
    Set a user's GnuPG public key.
    """

    AUTH = Command.LOGIN
    CONFIRM = True
    ARGUMENTS = [
        TextArgument(["key"], doc="GnuPG public key; multiline inputs will be handled as ascii armored full key, one-liners as key ids")
    ]

    def _run(self):
        uploader = self.request.user.uploader
        uploader.Admin.mbd_remove(self.request, uploader)
        key = self.args["key"].value

        if "\n" in key:
            self.msglog.info("Using given key argument as full ascii-armored GPG key")
            uploader.key_id = ""
            uploader.key = key
        else:
            self.msglog.info("Using given key argument as key ID")
            uploader.key_id = key
            uploader.key = ""

        uploader.Admin.mbd_prepare(self.request, uploader)
        uploader.Admin.mbd_check(self.request, uploader)
        self.msglog.info(f"Uploader profile changed: {uploader}")
        self.msglog.warning("Your uploader profile must be (re-)activated by the mini-buildd staff before you can actually use it.")


@_pimpdoc()
class Subscription(GroupUser):
    """
    Manage subscriptions to package notifications.

    A package subscription is a tuple 'PACKAGE:DISTRIBUTION',
    where both PACKAGE or DISTRIBUTION may be empty to denote
    all resp. items.

    JSON Result (dict):
      list: list: (action=list only) Human readable list of subscriptions.
      add: list: (action=add only) Human readable list of added subscriptions.
      remove: list: (action=lremoved only) Human readable list of removed subscriptions.
    """

    AUTH = Command.LOGIN
    ARGUMENTS = [
        SelectArgument(["action"], doc="action to run", choices=["list", "add", "remove"]),
        SelectArgument(["subscription"], doc="subscription pattern")
    ]

    def _update(self):
        if self.daemon and self.args["subscription"].value:
            self.args["subscription"].choices = [self.args["subscription"].value]
            package, _sep, _distribution = self.args["subscription"].value.partition(":")
            for r in self.daemon.get_active_repositories():
                self.args["subscription"].choices += [f"{package}:{d}" for d in r.mbd_distribution_strings()]

    def _run(self):
        package, _sep, distribution = self.args["subscription"].value.partition(":")

        def _filter():
            for s in self.daemon.get_subscription_objects().filter(subscriber=self.request.user):
                if package in ("", s.package) and distribution in ("", s.distribution):
                    yield s

        def _delete(subscription):
            result = f"{subscription}"
            subscription.delete()
            return result

        if self.args["action"].value == "list":
            self.result["list"] = [f"{subscription}." for subscription in _filter()]

        elif self.args["action"].value == "add":
            subscription, created = self.daemon.get_subscription_objects().get_or_create(subscriber=self.request.user,
                                                                                         package=package,
                                                                                         distribution=distribution)
            self.result["add"] = f"{'Added' if created else 'Exists'}: {subscription}."

        elif self.args["action"].value == "remove":
            self.result["remove"] = ["Removed: {s}.".format(s=_delete(subscription)) for subscription in _filter()]

        else:
            raise Exception(f"Unknown action '{self.args['action'].value}': Use one of 'list', 'add' or 'remove'.")


class Commands(collections.OrderedDict):
    """Automatically collect all commands defined in this module, and make them accessible."""

    def __init__(self):
        super().__init__({c.name(): c for c in sys.modules[__name__].__dict__.values() if inspect.isclass(c) and issubclass(c, Command) and c != Command})
        self.defaults = {cmd: cls({}) for cmd, cls in self.items()}


COMMANDS = Commands()


class Client():
    def input(self, prompt):
        return input(f"{self.endpoint}: {prompt}") if self.interactive else None

    def getpass(self, prompt):
        return getpass.getpass(f"{self.endpoint}: {prompt}") if self.interactive else None

    def __init__(self, endpoint, auto_confirm=False, auto_save_passwords=False, interactive=sys.stdout.isatty() and sys.stdin.isatty()):
        self.endpoint = mini_buildd.net.ClientEndpoint(endpoint, protocol="http")
        self.keyring = keyring.get_keyring()
        self.interactive = interactive
        self.auto_confirm = auto_confirm
        self.auto_save_passwords = auto_save_passwords

        # Setup login/logout handling. Auto-login now if the endpoint was given w/ user
        if self.endpoint.user:
            self.login(self.endpoint.user)

        LOG.info(f"{self.endpoint}: Client initialized: Keyring backend: {self.keyring.__class__.__qualname__}, "
                 f"auto_confirm={self.auto_confirm}, auto_save_passwords={self.auto_save_passwords}, interactive={self.interactive}")

        # Extra: status caching
        self._status = None

    def login(self, user):
        # Retrieve password
        password = self.keyring.get_password(self.endpoint.geturl(), user)
        if password:
            LOG.info(f"{self.endpoint}: Password retrieved for '{user}'")
            new_password = False
        else:
            password = self.getpass(f"Password for '{user}': ")
            new_password = True

        # Login
        self.endpoint.login(user, password)
        LOG.info(f"{self.endpoint}: User '{user}' logged in")

        # Logged in: Optionally save credentials
        if new_password and (self.auto_save_passwords or self.input(f"Save password for '{user}' to {self.keyring.__class__.__qualname__}: (Y)es, (N)o? ").upper() == "Y"):
            self.keyring.set_password(self.endpoint.geturl(), user, password)
        password = ""  # Maybe safer

        return self

    def try_login(self, user):
        try:
            self.login(user)
        except Exception as e:
            LOG.error(f"{self.endpoint}: Login with user '{user}' failed: {e}")

    def call(self, command, args=None):
        result = None
        confirm_next_call = False
        while result is None:
            try:
                http_args = {"command": command}
                http_args.update({"confirm": command} if (self.auto_confirm or confirm_next_call) else {})
                confirm_next_call = False
                http_args.update(args or {})

                url = self.endpoint.geturl(path=mini_buildd.config.PATH_API, query=http_args)
                LOG.info(f"{self.endpoint}: Calling API URL: {url}")
                result = json.load(self.endpoint.urlopen(url))
            except urllib.error.HTTPError as e:
                # Default to HTTP exception as error message; overwrite with the last daemon message.
                # There should always be exactly one daemon message in the HTTP header.
                http_msg = f"API call failed with HTTP Status {e.getcode()}: {e}"
                daemon_msgs = [f"Server says: {mini_buildd.misc.b642u(msg)}" for msg in [v for k, v in list(e.headers.items()) if k.startswith(mini_buildd.api.HTTP_MESSAGE_PREFIX)]]
                error_msg = daemon_msgs[-1] or http_msg

                if e.getcode() == 401 and self.interactive:
                    # note: Would be so much nicer w/ assign. expr. (pep-0572), but we need to be compatible with buster/3.7
                    def _get_action():
                        return self.input(f"{error_msg}\nRetry options: Change (u)ser [{self.endpoint.logged_in}], (c)onfirm next [{confirm_next_call}] or (C)onfirm all calls [{self.auto_confirm}]? ")
                    action = _get_action()
                    while action:
                        if action == "u":
                            self.try_login(self.input(f"Log in as [{self.endpoint.user}]: ") or self.endpoint.user)
                        elif action == "c":
                            confirm_next_call = True
                        elif action == "C":
                            self.auto_confirm = True
                        action = _get_action()
                else:
                    raise Exception(error_msg)

        return result

    # Extra functionality
    @property
    def identity(self):
        """Get Archive's Identity."""
        return self.status["identity"]

    @property
    def status(self):
        if self._status is None:
            self._status = self.call("status")
        return self._status

    def get_codenames(self, repo):
        return self.status["repositories"][repo]["codenames"]

    def get_package_versions(self, src_package, dist_regex=".*"):
        """Produce a dict with all (except rollback) available versions of this package (key=distribution, value=info dict: version, dsc_url, log_url, changes_url*)."""
        def _base_url(url):
            url_split = urllib.parse.urlsplit(url)
            return f"{url_split.scheme}://{url_split.hostname}:{url_split.port}"

        show = self.call("show", {"package": src_package})
        result = {}
        for repository in show.items():
            for versions in repository[1]:
                for version in versions[1]:
                    dist = version["distribution"]
                    vers = version["sourceversion"]
                    # Note: 'vers' may be empty when only rollbacks exist
                    if vers and re.match(dist_regex, dist):
                        info = {}

                        repository = mini_buildd.misc.Distribution(dist).repository

                        info["version"] = vers
                        info["dsc_url"] = version["dsc_path"]
                        base_url = _base_url(info["dsc_url"])
                        info["log_url"] = f"{base_url}/mini_buildd/log/{repository}/{src_package}/{vers}/"
                        # Note: Path may also be "/source all/", not "/source/" (on non-source-only uploads?)
                        info["changes_url"] = f"{base_url}/log/{repository}/{src_package}/{vers}/source/{src_package}_{vers}_source.changes"
                        result[dist] = info

        return result

    def wait_for_package(self, distribution, src_package, version=None, or_greater=False,  # pylint: disable=inconsistent-return-statements
                         max_tries=-1, sleep=60, initial_sleep=0):
        """Block until a specific package is in repository."""
        item = f"\"{src_package}_{version}\" in \"{distribution}\""

        def _sleep(secs):
            LOG.info(f"{self.endpoint}: Waiting for {item}: Idling {secs} seconds (Ctrl-C to abort)...")
            time.sleep(secs)

        tries = 0
        _sleep(initial_sleep)
        while max_tries < 0 or tries < max_tries:
            pkg_info = self.get_package_versions(src_package, distribution).get(distribution, {})
            actual_version = pkg_info.get("version", None)
            LOG.info(f"{self.endpoint}: Actual version for {item}: {actual_version}")

            # pylint: disable=too-many-boolean-expressions
            if (version is None and actual_version) or \
               (version is not None and (actual_version == version or or_greater and debian.debian_support.Version(actual_version) >= debian.debian_support.Version(version))):
                LOG.info(f"{self.endpoint}: Match found: {item}.")
                return pkg_info
            _sleep(sleep)
            tries += 1
        raise Exception(f"Could not find {item} within {initial_sleep + tries * sleep} seconds.")

    def has_package(self, distribution, src_package, version=None, or_greater=False):
        """Check if a specific package is in repository."""
        try:
            return self.wait_for_package(distribution, src_package, version, or_greater=or_greater,
                                         max_tries=1, sleep=0, initial_sleep=0)
        except:  # noqa (pep8 E722)  # pylint: disable=bare-except
            pass

    def bulk_migrate(self, packages, repositories=None, codenames=None, suites=None):
        """Bulk-migrate a package over repositories, base distributions and suites."""
        status = self.call("status")

        if repositories is None:
            repositories = list(status["repositories"]["codenames"].keys())
        if suites is None:
            suites = ["unstable", "testing"]

        for package in packages:
            for repository in repositories:
                iter_codenames = codenames
                if iter_codenames is None:
                    iter_codenames = list(status["repositories"]["codenames"][repository])
                for codename in iter_codenames:
                    for suite in suites:
                        dist = f"{codename}-{repository}-{suite}"
                        try:
                            self.call("migrate", {"package": package, "distribution": dist})
                        except:  # noqa (pep8 E722)  # pylint: disable=bare-except
                            pass
