#!/usr/bin/env python3

from __future__ import print_function
from collections import defaultdict
from datetime import datetime
from functools import cmp_to_key
import sys
sys.path.append('/usr/share/botch')
from util import write_plain, read_yaml_file, cmp, parse_dose_yaml
from util import read_tag_file
import os


def print_package(arg, bin2src, source=None, color=True):
    pkgname, version = arg
    sn, _ = src_of_pkg(arg, bin2src)
    if source:
        highlight = (sn == source[0])
    else:
        highlight = False
    if pkgname == "":
        pkgstring = "(*)"
        bugs = False
    elif pkgname.startswith('src:'):
        pkgname = pkgname[4:]
        bugs = srcpkgbugs.get(pkgname)
        if version:
            pkgstring = '<a title="%s" style="">src:%s</a>' % (
                version, pkgname)
        else:
            pkgstring = 'src:%s' % pkgname
    else:
        arch, pkgname = pkgname.split(':', 1)
        bugs = binpkgbugs.get(pkgname)
        if bin2src.get((pkgname, 'all', version)):
            arch = 'all'
        if version:
            pkgstring = '<a title="%s">%s:%s</a>' % (version, pkgname, arch)
        else:
            pkgstring = '%s:%s' % (pkgname, arch)
    if highlight:
        pkgstring = '<span style="color:#090;font-weight:bold">'+pkgstring
        pkgstring += '</span>'
    if bugs:
        bugstring = " ".join('<a href="http://bugs.debian.org/%s">%s</a>' % (b,
                                                                             b)
                             for b in bugs)
        if color:
            pkgstring = '<span style="color:#f00">%s</span> (%s)' % (pkgstring,
                                                                     bugstring)
        else:
            pkgstring = '%s (%s)' % (pkgstring, bugstring)
    if sn:
        wnpp = wnpp_stats.get(sn)
        if wnpp:
            t, i = wnpp
            pkgstring = '%s (%s: <a href="http://bugs.debian.org/%s">#%s</a>)' \
                        % (pkgstring, t, i, i)

    return pkgstring


def by_srcname(a, b):
    num_src = cmp(len(b[1]), len(a[1]))
    if num_src:
        return num_src
    else:
        return cmp(min(a[1]), min(b[1]))


def by_affected_sources_missing(a, b):
    a1 = sum([len(v) for v in a[1].values()])
    b1 = sum([len(v) for v in b[1].values()])
    num_src = cmp(b1, a1)
    if num_src:
        return num_src
    else:
        return cmp(a[0], b[0])


def by_affected_sources_conflict(a, b):
    a1 = sum([len(v) for v in a[1].values()])
    b1 = sum([len(v) for v in b[1].values()])
    num_src = cmp(b1, a1)
    if num_src:
        return num_src
    else:
        return cmp(a[0][0], b[0][0])


def print_header(outfile, description, btsuser, btstag):
    print("""<html>
    <head>
    <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
    <style>
    table, th, td
    {
        border: 1px solid black;
    }
    </style>
    </head>
    <body>
    """, file=outfile)

    print(description, file=outfile)

    if btsuser and btstag:
        print("<p>Bugs are associated with packages on this page if they " +
              "carry the usertag \"%s\" of the user " % btstag +
              "\"%s\".</p>" % btsuser, file=outfile)
        print("<p>You can get an overview of all bugs tagged like that in " +
              "the <a href=\"https://bugs.debian.org/cgi-bin/pkgreport.cgi?t" +
              "ag=%s;users=%s\"" % (btstag, btsuser) +
              ">Debian bts</a></p>", file=outfile)


def print_missing(outfile, missing, bin2src, source=None):
    print("<h2>missing</h2>", file=outfile)
    print("<p>The packages in the third column cannot satisfy their " +
          "(possibly transitive) dependencies because the " +
          "binary package in the last column. This is mostly because the " +
          "binary package in the last column is Multi-Arch:no. Some of " +
          "these packages need to be Multi-Arch:foreign instead. In some " +
          "other cases, Build-Depends can be annotated with :native. The " +
          "depchains column shows the dependency chain from the packages in " +
          "the third column to the binary package in the last column. The " +
          "\"(*)\" placeholder in the depchains column represents any " +
          "package in the third column.</p>",
          file=outfile)
    print("<p>The output is first grouped by the shared unsatisfied " +
          "dependency (last column) and then by shared dependency chain " +
          "(fourth column). The groups are sorted by the number of " +
          "packages missing the dependency in the last column. Within each " +
          "group, the output is sorted by the number of packages " +
          "sharing the same dependency chain.</p>", file=outfile)
    print("<table><tr><th># of packages per missing</th><th># of packages per"
          " depchain</th><th>packages with missing (possibly transitive) depe"
          "ndencies</th><th>Depchains</th>" +
          "<th>Unsatisfied dependency</th></tr>", file=outfile)

    for pkg, v in sorted(list(missing.items()),
                         key=cmp_to_key(by_affected_sources_missing)):
        rows = len(v)
        for c in list(v.keys()):
            if c:
                rows += len(c) - 1
        print('<tr><td rowspan="%d">%d</td>' % (rows, sum([len(s)
                                                           for s
                                                           in v.values()])),
              file=outfile)
        first = True
        for chain, v in sorted(list(v.items()), key=cmp_to_key(by_srcname)):
            if chain is None:
                maxnumchains = 1
            else:
                maxnumchains = len(chain)
            if chain:
                c = " → ".join([print_package(t, bin2src, source)
                                 for t in chain[0]])
            else:
                c = ""
            if not first:
                print("<tr>", file=outfile)
            print('<td rowspan="%d">%d</td>' % (maxnumchains, len(v)) +
                  '<td rowspan="%d">%s</td><td>%s</td>'
                  % (maxnumchains, " ".join([print_package(t, bin2src, source)
                                             for t in sorted(v)]), c),
                  file=outfile)
            if first:
                print('<td rowspan="%d">%s</td></tr>'
                      % (rows, print_package(pkg, bin2src, source)),
                      file=outfile)
            else:
                print('</tr>', file=outfile)
            if chain:
                for i in range(1, len(chain)):
                    print('<tr><td>%s</td></tr>' %
                          (" → ".join([print_package(t, bin2src, source)
                                        for t in chain[i]])),
                          file=outfile)
            first = False
    print("</table>", file=outfile)


def print_conflict(outfile, conflict, bin2src, source=None):
    print("<h2>conflict</h2>", file=outfile)
    print("<p>The packages in the third column cannot satisfy their " +
          "dependencies because the two binary packages in the " +
          "last column cannot be installed together. The two " +
          "depchain columns show the paths from the packages in the " +
          "third column to each of the conflicting binary packages in the " +
          "last column, respectively. Sometimes, multiple " +
          "dependency chains exist.<p>", file=outfile)
    print("<p>The output is first grouped by the shared conflicting " +
          "dependency (last column) and then by the shared dependency " +
          "chains (fourth and fifth column). The groups are sorted by the " +
          "number of packages sharing the conflict in the last " +
          "column. Within each group, the output is sorted by the number of " +
          "packages sharing the same dependency chains.</p>",
          file=outfile)
    print(
        "<table><tr><th># of packages per conflict</th><th># of packages per d"
        "epchain</th><th>packages with (possibly transitive) conflicting depen"
        "dencies</th><th>Depchain " +
        "1</th><th>Depchain2</th><th>Conflict</th></tr>", file=outfile)

    for (pkg1, pkg2), v in sorted(list(conflict.items()),
                                  key=cmp_to_key(
                                      by_affected_sources_conflict)):
        rows = len(v)
        for c1, c2 in list(v.keys()):
            if c1 is not None:
                c1 = len(c1)
            else:
                c1 = 0
            if c2 is not None:
                c2 = len(c2)
            else:
                c2 = 0
            if max(c1, c2) > 1:
                rows += max(c1, c2) - 1
        print('<tr><td rowspan="%d">%d</td>'
              % (rows, sum([len(s) for s in v.values()])),
              file=outfile)
        first = True
        for (chain1, chain2), v in sorted(list(v.items()),
                                          key=cmp_to_key(by_srcname)):
            if chain1 is None:
                maxnumchains = len(chain2)
            elif chain2 is None:
                maxnumchains = len(chain1)
            else:
                maxnumchains = max(len(chain1), len(chain2))
            if chain1:
                c1 = " → ".join([print_package(t, bin2src, source)
                                  for t in chain1[0]])
            else:
                c1 = ""
            if chain2:
                c2 = " → ".join([print_package(t, bin2src, source)
                                  for t in chain2[0]])
            else:
                c2 = ""
            if not first:
                print("<tr>", file=outfile)
            print('<td rowspan="%d">%d</td>' % (maxnumchains, len(v)) +
                  '<td rowspan="%d">%s</td><td>%s</td><td>%s</td>'
                  % (maxnumchains, " ".join([print_package(t, bin2src, source)
                                             for t in sorted(v)]), c1, c2),
                  file=outfile)
            if first:
                print('<td rowspan="%d">%s ↔ %s</td></tr>' %
                      (rows, print_package(pkg1, bin2src, source),
                       print_package(pkg2, bin2src, source)),
                      file=outfile)
            else:
                print('</tr>', file=outfile)
            for i in range(1, maxnumchains):
                if chain1 is not None and len(chain1) >= i + 1:
                    c1 = " → ".join([print_package(t, bin2src, source)
                                      for t in chain1[i]])
                else:
                    c1 = ""
                if chain2 is not None and len(chain2) >= i + 1:
                    c2 = " → ".join([print_package(t, bin2src, source)
                                      for t in chain2[i]])
                else:
                    c2 = ""
                print('<tr><td>%s</td><td>%s</td></tr>' %
                      (c1, c2), file=outfile)
            first = False
    print("</table>", file=outfile)


def print_footer(outfile, srcs=[], srcsdir=""):
    if srcs:
        print("<h2>Affected source packages:</h2>", file=outfile)
        for src in sorted([s for s in srcs if s is not None]):
            print("<a href=\"%s/%s.html\">%s</a>" % (srcsdir, src, src),
                  file=outfile)

    url = \
        "https://gitlab.mister-muffin.de/debian-bootstrap/bootstrap_debian_net"
    footer = """
    <hr />
    <p>The JSON data used to generate these pages was computed using botch, the
    bootstrap/build ordering tool chain. The source code of botch can be
    redistributed under the terms of the LGPL3+ with an OCaml linking
    exception. The source code can be retrieved from <a
    href="https://gitlab.mister-muffin.de/debian-bootstrap/botch">
    https://gitlab.mister-muffin.de/debian-bootstrap/botch</a></p>

    <p>The html pages were generated by code which can be retrieved from <a
    href="%s"> %s</a> and which can be redistributed under the terms of the
    AGPL3+</p>

    <p>For questions and bugreports please contact j [dot] schauer [at] email
    [dot] de.</p>
    """ % (url, url)

    print("<p>generated: %s</p>" % (datetime.now().isoformat()), file=outfile)
    print("%s</body></html>" % footer, file=outfile)


def src_of_pkg(t, bin2src):
    n, v = t
    if n.startswith("src:"):
        return n[4:], v
    if n == "":
        return (None, None)
    a, n = n.split(":", 1)
    res = bin2src.get((n, a, v))
    if res:
        return res
    res = bin2src.get((n, 'all', v))
    if res:
        return res
    # this can happen for a dependency on a package that is not in the archive
    print("cannot find source for %s %s" % (n, v), file=sys.stderr)
    return (None, None)


def print_top10summary(outfile, missing, conflict, bin2src, source=None):
    if not missing and not conflict:
        return

    print("<h2>Top 10 summary</h2>", file=outfile)

    print("<p>The following is a summary of the full \"missing\" and " +
          "\"conflict\" tables below. It only shows the first and last " +
          "columns of the full tables and only displays the top 10 rows.</p>",
          file=outfile)

    if missing:
        print("<h3>Missing</h3>", file=outfile)

        print("<table><tr><th># of packages per missing</th><th>Unsatisfied " +
              "dependency</th></tr>", file=outfile)
        for pkg, v in sorted(list(missing.items()),
                             key=cmp_to_key(by_affected_sources_missing))[:10]:
            rows = len(v)
            for c in list(v.keys()):
                if c:
                    rows += len(c) - 1
            print('<tr><td>%d</td><td>%s</td></tr>' % (sum([len(s)
                                                            for s
                                                            in v.values()]),
                                                       print_package(pkg,
                                                                     bin2src,
                                                                     source)),
                  file=outfile)
        print("</table>", file=outfile)

    if conflict:
        print("<h3>Conflict</h3>", file=outfile)
        print("<table><tr><th># of packages per conflict</th>" +
              "<th>Conflict</th></tr>", file=outfile)

        for (pkg1, pkg2), v in sorted(list(conflict.items()),
                                      key=cmp_to_key(
                                          by_affected_sources_conflict))[:10]:
            print('<tr><td>%d</td><td>%s ↔ %s</td></tr>'
                  % (sum([len(s) for s in v.values()]),
                      print_package(pkg1, bin2src, source),
                      print_package(pkg2, bin2src, source)),
                  file=outfile)
        print("</table>", file=outfile)


def create_srcpkg_pages(missing, conflict, bin2src, srcsdir, description,
                        btsuser, btstag):
    srcsdict = defaultdict(lambda: defaultdict(lambda:
                                               defaultdict(lambda:
                                                           defaultdict(dict))))

    # now go through all missing and conflict and copy them to the source
    # packages that associate with them
    for pkg, v in missing.items():
        sn, sv = src_of_pkg(pkg, bin2src)
        srcsdict[sn][sv]['missing'][pkg] = v
        for chain, v in v.items():
            for c in chain:
                for p in c:
                    sn, sv = src_of_pkg(p, bin2src)
                    if not sn:
                        continue
                    srcsdict[sn][sv]['missing'][pkg][chain] = v
            for p in v:
                sn, sv = src_of_pkg(p, bin2src)
                srcsdict[sn][sv]['missing'][pkg][chain] = v

    for (pkg1, pkg2), v in conflict.items():
        sn, sv = src_of_pkg(pkg1, bin2src)
        srcsdict[sn][sv]['conflict'][(pkg1, pkg2)] = v
        sn, sv = src_of_pkg(pkg2, bin2src)
        srcsdict[sn][sv]['conflict'][(pkg1, pkg2)] = v
        for (chain1, chain2), v in v.items():
            for c in chain1:
                for p in c:
                    sn, sv = src_of_pkg(p, bin2src)
                    if not sn:
                        continue
                    srcsdict[sn][sv]['conflict'][(pkg1, pkg2)][(chain1,
                                                                chain2)] = v
            for c in chain2:
                for p in c:
                    sn, sv = src_of_pkg(p, bin2src)
                    if not sn:
                        continue
                    srcsdict[sn][sv]['conflict'][(pkg1, pkg2)][(chain1,
                                                                chain2)] = v
            for p in v:
                sn, sv = src_of_pkg(p, bin2src)
                srcsdict[sn][sv]['conflict'][(pkg1, pkg2)][(chain1,
                                                            chain2)] = v

    if not os.path.isdir(srcsdir):
        os.mkdir(srcsdir)
    for srcpkg, versions in srcsdict.items():
        if not srcpkg:
            continue
        fname = os.path.join(srcsdir, srcpkg)+".html"
        with open(fname, "w") as srcpkgpage:
            print_header(srcpkgpage, description, btsuser, btstag)
            for ver in versions.keys():
                print("<h1>%s</h1>" % print_package(("src:"+srcpkg, ver),
                                                    bin2src,
                                                    color=False),
                      file=srcpkgpage)
                print_top10summary(srcpkgpage, versions[ver].get('missing'),
                                   versions[ver].get('conflict'), bin2src,
                                   (srcpkg, ver))
                if versions[ver].get('missing'):
                    print_missing(srcpkgpage, versions[ver]['missing'],
                                  bin2src, (srcpkg, ver))
                if versions[ver].get('conflict'):
                    print_conflict(srcpkgpage, versions[ver]['conflict'],
                                   bin2src, (srcpkg, ver))
            print_footer(srcpkgpage)

    return srcsdict.keys()


def dose2html(yamlin, outfile, description="", srcpkgbugs=set(),
              binpkgbugs=set(), btsuser=None, btstag=None, srcsdir=None,
              packages=[], verbose=False):
    bin2src = dict()

    for pkg in packages:
        name = pkg['Package']
        arch = pkg['Architecture']
        ver = pkg['Version']
        src = pkg.get('Source')
        if src:
            if ' ' in src:
                srcname, srcver = src.split(' ', 1)
                srcver = srcver[1:-1]
            else:
                srcname = src
                srcver = ver
        else:
            srcname = name
            srcver = ver
        bin2src[(name, arch, ver)] = (srcname, srcver)
        bin2src[(name, arch, None)] = (srcname, None)
        provides = pkg.get('Provides')
        if not provides:
            continue
        for pname in provides.split(','):
            pname = pname.strip()
            bin2src[(pname, arch, ver)] = (srcname, srcver)
            bin2src[(pname, arch, None)] = (srcname, None)

    missing, conflict, _ = parse_dose_yaml(yamlin)

    print_header(outfile, description, btsuser, btstag)
    print_top10summary(outfile, missing, conflict, bin2src)
    print_missing(outfile, missing, bin2src)
    print_conflict(outfile, conflict, bin2src)

    if srcsdir and packages:
        srcpkgs = create_srcpkg_pages(missing, conflict, bin2src, srcsdir,
                                      description, btsuser, btstag)
        print_footer(outfile, srcpkgs, srcsdir)
    else:
        print_footer(outfile)

if __name__ == '__main__':
    import argparse
    parser = argparse.ArgumentParser(
        description='given a buildcheck result, create a html overview')
    parser.add_argument('yamlin', type=read_yaml_file,
                        help='input in yaml format (dose3 output)')
    parser.add_argument(
        'htmlout', type=write_plain, help='output in html format')
    parser.add_argument('--desc', default="", help='descriptive HTML snippet')
    parser.add_argument('--btsuser',
                        help='bts user to associate packages with')
    parser.add_argument('--btstag',
                        help='bts tag to associate packages with')
    parser.add_argument('--srcsdir', help='output directory for ' +
                        'individual source package overviews')
    parser.add_argument('--wnpp', action='store_true',
                        help='retrieve and print wnpp status for packages')
    parser.add_argument('--packages', type=read_tag_file,
                        help='Packages file to create a mapping from binary ' +
                             'to source packages')
    parser.add_argument(
        '-v', '--verbose', action='store_true', help='be verbose')
    args = parser.parse_args()
    srcpkgbugs = defaultdict(list)
    binpkgbugs = defaultdict(list)

    if args.btsuser and args.btstag:
        # this only corks with python2 because SOAPpy is not available for
        # python3
        # import SOAPpy

        # server = SOAPpy.SOAPProxy('http://bugs.debian.org/cgi-bin/soap.cgi',
        #                           'Debbugs/SOAP')
        # bugnrs = server.get_usertag('debian-cross@lists.debian.org',
        #                             'cross-satisfiability')
        # buginfos = server.get_status(bugnrs['cross-satisfiability'])

        # for bug in buginfos['item']:
        #    k = bug['key']
        #    v = bug['value']
        #    if v['package']:
        #        binpkgbugs[v['package']].append(k)
        #    else:
        #        for s in v['source'].split(','):
        #            s = s.strip()
        #            srcpkgbugs[s].append(k)
        import subprocess
        res = subprocess.check_output(
            "bts select users:%s " % args.btsuser
            + "tag:%s | bts status " % args.btstag
            + "fields:source,package,bug_num file:-", shell=True)
        for b in res.decode().split("\n\n"):
            if not b:
                continue
            d = {k: v for k, v in [l.split("\t")
                                   for l in b.split("\n") if l.strip()]}
            binpkgbugs[d['package']].append(d['bug_num'])
            for s in d['source'].split(','):
                s = s.strip()
                srcpkgbugs[s].append(d['bug_num'])

    wnpp_stats = dict()
    if args.wnpp and args.packages:
        import urllib.request
        url = 'https://qa.debian.org/data/bts/wnpp_rm'
        with urllib.request.urlopen(url) as f:
            # the following snippet is from
            # distro_tracker/vendor/debian/tracker_tasks.py
            for line in f:
                line = line.decode().strip()
                try:
                    package_name, wnpp_type, bug_id = \
                        line.split('|')[0].split()
                    bug_id = int(bug_id)
                except:
                    # Badly formatted
                    continue
                # Strip the colon from the end of the package name
                package_name = package_name[:-1]

                wnpp_stats[package_name] = (wnpp_type, bug_id)

    with args.htmlout as f:
        dose2html(args.yamlin, f, args.desc, srcpkgbugs, binpkgbugs,
                  args.btsuser, args.btstag, args.srcsdir, args.packages,
                  args.verbose)
