tests-scan

#!/usr/bin/env python3

# This file is part of Cockpit.
#
# Copyright (C) 2015 Red Hat, Inc.
#
# Cockpit is free software; you can redistribute it and/or modify it
# under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation; either version 2.1 of the License, or
# (at your option) any later version.
#
# Cockpit is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with Cockpit; If not, see <http://www.gnu.org/licenses/>.

import argparse
import json
import pipes
import sys
import time
import logging
import itertools

sys.dont_write_bytecode = True
logging.basicConfig(level=logging.INFO)

from task import github, redhat_network, labels_of_pull, distributed_queue, testmap

no_amqp = False
try:
    import pika
except ImportError:
    no_amqp = True


def build_policy(repo, context):
    policy = testmap.tests_for_project(repo)
    if context:
        short_contexts = []
        for c in context:
            short_contexts.append(c.split("@")[0])
        new_policy = {}
        for (branch, contexts) in policy.items():
            branch_context = []
            for c in short_contexts:
                if c in contexts:
                    branch_context.append(c)
            if branch_context:
                new_policy[branch] = branch_context
        policy = new_policy
    return policy


def main():
    parser = argparse.ArgumentParser(description='Bot: scan and update status of pull requests on GitHub')
    parser.add_argument('-v', '--human-readable', action="store_true", default=False,
                        help='Display human readable output rather than tasks')
    parser.add_argument('-d', '--dry', action="store_true", default=False,
                        help='Don''t actually change anything on GitHub')
    parser.add_argument('--repo', default=None,
                        help='Repository to scan and checkout.')
    parser.add_argument('-c', '--context', action="append", default=[],
                        help='Test contexts to use.')
    parser.add_argument('-p', '--pull-number', default=None,
                        help='Single pull request to scan for tasks')
    parser.add_argument('--pull-data', default=None,
                        help='pull_request event GitHub JSON data to evaluate; mutualy exclusive with -p and -s')
    parser.add_argument('-s', '--sha', default=None,
                        help='SHA beloging to pull request to scan for tasks')
    parser.add_argument('--amqp', default=None,
                        help='The host:port of the AMQP server to publish to (format host:port)')

    opts = parser.parse_args()
    if opts.amqp and no_amqp:
        logging.error("AMQP host:port specified but python-amqp not available")
        return 1
    if opts.pull_data and (opts.pull_number or opts.sha):
        parser.error("--pull-data and --pull-number/--sha are mutually exclusive")

    api = github.GitHub(repo=opts.repo)

    # HACK: The `repo` option is used throughout the code, for example repo from
    # opts is needed in `tests_invoke`, `tests_human`, `queue_test` etc.
    # Would be better to use api.repo everywhere
    opts.repo = api.repo

    try:
        results = scan_for_pull_tasks(api, opts.context, opts, opts.repo)
    except RuntimeError as ex:
        logging.error("tests-scan: " + str(ex))
        return 1

    for result in results:
        if result:
            sys.stdout.write(result + "\n")

    return 0


# Prepare a human readable output
def tests_human(priority, name, number, revision, ref, context, base, original_base, repo, bots_ref):
    if not priority:
        return
    try:
        priority = int(priority)
    except (ValueError, TypeError):
        pass
    return "{name:11} {context:25} {revision:10} {priority:2}{repo}{bots_ref}{branches}".format(
        priority=priority,
        revision=revision[0:7],
        context=context,
        name=name,
        repo=repo and "  (%s)" % repo or "",
        bots_ref=bots_ref and (" [bots@%s]" % bots_ref) or "",
        branches=base != original_base and ("   {%s...%s}" % (original_base, base)) or ""
    )


def is_internal_context(context):
    for pattern in ["rhel", "edge", "vmware", "openstack"]:
        if pattern in context:
            return True
    return False


# Prepare a test invocation command
def tests_invoke(priority, name, number, revision, ref, context, base, original_base, repo, bots_ref, options):
    if not options.amqp and not redhat_network() and is_internal_context(context):
        return ''

    try:
        priority = int(priority)
    except (ValueError, TypeError):
        priority = 0
    if priority <= 0:
        return
    current = time.strftime('%Y%m%d-%H%M%M')

    checkout = "PRIORITY={priority:04d} ./make-checkout --verbose"
    cmd = "TEST_PROJECT={repo} TEST_NAME={name}-{current} TEST_REVISION={revision} " \
          "../tests-invoke --pull-number={pull_number} "
    if base != ref:
        checkout += " --rebase={base}"

    checkout += " --repo={repo}"

    # The repo of this test differs from the PR's repo
    if options.repo != repo:
        cmd = "GITHUB_BASE={github_base} " + cmd

    # Let tests-invoke know that we are triggering different branch - it needs to post correct status
    if base != original_base:
        cmd = "TEST_BRANCH={base} " + cmd

    cmd += " {context}"
    if bots_ref:
        # we are checking the external repo on a cockpit PR, so stay on the project's master
        checkout += " {ref} && "
        cmd = "COCKPIT_BOTS_REF={bots_ref} " + cmd
    else:
        # we are testing the repo itself, checkout revision from the PR
        checkout += " {ref} {revision} && "

    return (checkout + "cd make-checkout-workdir && " + cmd + " ; cd ..").format(
        priority=priority,
        name=pipes.quote(name),
        revision=pipes.quote(revision),
        base=pipes.quote(str(base)),
        ref=pipes.quote(ref),
        bots_ref=pipes.quote(bots_ref),
        context=pipes.quote(context),
        current=current,
        pull_number=number,
        repo=pipes.quote(repo),
        github_base=pipes.quote(options.repo),
    )


def queue_test(priority, name, number, revision, ref, context, base, original_base, repo, bots_ref, channel, options):
    command = tests_invoke(priority, name, number, revision, ref, context, base, original_base, repo, bots_ref, options)
    if command:
        if priority > distributed_queue.MAX_PRIORITY:
            priority = distributed_queue.MAX_PRIORITY

        body = {
            "command": command,
            "type": "test",
            "sha": revision,
            "ref": ref,
            "name": name,
        }
        queue = 'rhel' if is_internal_context(context) else 'public'
        channel.basic_publish('', queue, json.dumps(body), properties=pika.BasicProperties(priority=priority))
        logging.info("Published '{0}' on '{1}' with command: '{2}'".format(name, revision, command))


def prioritize(status, title, labels, priority, context, number):
    state = status.get("state", None)
    update = {"state": "pending"}

    # This commit definitively succeeded or failed
    if state in ["success", "failure"]:
        logging.info("Skipping '{0}' on #{1} because it has already finished".format(context, number))
        priority = 0
        update = None

    # This test errored, we try again but low priority
    elif state in ["error"]:
        priority -= 2

    elif state in ["pending"]:
        logging.info("Not updating status for '{0}' on #{1} because it is pending".format(context, number))
        update = None

    # Ignore context when the PR has [no-test] in the title or as label, unless
    # the context was directly triggered
    if (('no-test' in labels or '[no-test]' in title) and status.get("description", "") != github.NOT_TESTED_DIRECT):
        logging.info("Skipping '{0}' on #{1} because it is no-test".format(context, number))
        priority = 0
        update = None

    if priority > 0:
        if "priority" in labels:
            priority += 2
        if "blocked" in labels:
            priority -= 1

        # Pull requests where the title starts with WIP get penalized
        if title.startswith("WIP") or "needswork" in labels:
            priority -= 1

        # Is testing already in progress?
        description = status.get("description", "")
        if description.startswith(github.TESTING):
            logging.info("Skipping '{0}' on #{1} because it is already running".format(context, number))
            priority = description
            update = None

    if update:
        if priority <= 0:
            logging.info("Not updating status for '{0}' on #{1} because of low priority".format(context, number))
            update = None
        else:
            update["description"] = github.NOT_TESTED

    return [priority, update]


def dict_is_subset(full, check):
    for (key, value) in check.items():
        if key not in full or full[key] != value:
            return False
    return True


def update_status(api, revision, context, last, changes):
    if changes:
        changes["context"] = context
    if changes and not dict_is_subset(last, changes):
        response = api.post("statuses/" + revision, changes, accept=[422])  # 422 Unprocessable Entity
        errors = response.get("errors", None)
        if not errors:
            return True
        for error in response.get("errors", []):
            sys.stderr.write("{0}: {1}\n".format(revision, error.get('message', json.dumps(error))))
            sys.stderr.write(json.dumps(changes))
        return False
    return True


def cockpit_tasks(api, update, contexts, repo, pull_data, pull_number, sha, amqp):
    results = []
    pulls = []
    branch_contexts = testmap.tests_for_project(repo).values()
    repo_contexts = set(itertools.chain(*branch_contexts))

    if pull_data:
        pulls.append(json.loads(pull_data)['pull_request'])
    elif pull_number:
        pull = api.get("pulls/{0}".format(pull_number))
        if pull:
            pulls.append(pull)
        else:
            logging.error("Can't find pull request {0}".format(pull_number))
            return 1
    else:
        pulls = api.pulls()

    whitelist = api.whitelist()
    for pull in pulls:
        title = pull["title"]
        number = pull["number"]
        revision = pull["head"]["sha"]
        statuses = api.statuses(revision)
        login = pull["head"]["user"]["login"]
        base = pull["base"]["ref"]  # The branch this pull request targets

        logging.info("Processing #{0} titled '{1}' on revision {2}".format(number, title, revision))

        # If sha is present only scan PR with selected sha
        if sha and revision != sha and not revision.startswith(sha):
            continue

        labels = labels_of_pull(pull)

        baseline = distributed_queue.BASELINE_PRIORITY
        # amqp automatically prioritizes on age
        if not amqp:
            # modify the baseline slightly to favor older pull requests, so that we don't
            # end up with a bunch of half tested pull requests
            baseline += 1.0 - (min(100000, float(number)) / 100000)

        def get_externals():
            result = []
            for proj_repo in testmap.projects():
                if proj_repo == "cockpit-project/bots":
                    continue
                for context in testmap.tests_for_project(proj_repo).get("master", []):
                    result.append(context + "@" + proj_repo)
            return result

        def is_valid_context(context):
            (os_scenario, _, repo_branch) = context.partition("@")

            # If contexts were specified on commandline, only those are valid
            # Of course if you specify invalid context in `tests-scan` it will be happy about it
            # But this can be used when want to try new context which does not yet exists in map
            if contexts:
                for c in contexts:
                    c = c.split("@")[0]
                    if c == os_scenario:
                        return True
                return False

            # If repo in context, consider only contexts from the given repo
            if repo_branch:
                repo_branch = "/".join(repo_branch.split("/")[:2])
                repo_cs = testmap.tests_for_project(repo_branch).values()
                return os_scenario in set(itertools.chain(*repo_cs))

            # Valid contexts are the ones that exist in the current repo
            return os_scenario in repo_contexts

        # Create list of statuses to process
        todos = {}
        for status in statuses:  # Firstly add all valid contexts that already exist in github
            if is_valid_context(status):
                todos[status] = statuses[status]
        if not statuses:  # If none defined in github add basic set of contexts
            for context in build_policy(repo, contexts).get(base, []):
                todos[context] = {}

            # test external projects on bots PRs
            if repo == "cockpit-project/bots":
                for context in get_externals():
                    if context not in todos:
                        todos[context] = {}

        # there are 3 different HEADs
        # ref:    the PR that we are testing
        # base:   the target branch of that PR
        # branch: the branch of the external project that we are testing
        #         against this PR (only applies to cockpit-project/bots PRs)
        for context in todos:
            # Get correct project and branch. Ones from test name have priority
            project = repo
            branch = base
            (os_scenario, _, repo_branch) = context.partition("@")
            repo_branch = repo_branch.split("/")
            if len(repo_branch) == 2:
                project = "/".join(repo_branch)
                branch = "master"
            elif len(repo_branch) == 3:
                project = "/".join(repo_branch[:2])
                branch = repo_branch[2]

            ref = "pull/%d/head" % number

            # For unmarked and untested status, user must be in whitelist
            # Not this only applies to this specific commit. A new status
            # will apply if the user pushes a new commit.
            status = todos[context]
            if login not in whitelist and status.get("description", github.NO_TESTING) == github.NO_TESTING:
                priority = github.NO_TESTING
                changes = {"description": github.NO_TESTING, "context": context, "state": "pending"}
            else:
                (priority, changes) = prioritize(status, title, labels, baseline, context, number)
            if not update or update_status(api, revision, context, status, changes):
                checkout_ref = ref
                if project != repo:
                    checkout_ref = "master"
                if base != branch:
                    checkout_ref = branch

                if repo == "cockpit-project/bots":
                    # bots own test doesn't need bots/ setup as there is a permanent symlink to itself there
                    # otherwise if we're testing an external project (repo != project) then checkout bots from the PR
                    bots_ref = None if repo == project else ref
                else:
                    # test one external project against another non-bots project, doesn't happen (yet)
                    bots_ref = "master"

                results.append((priority,
                                "pull-%d" % number,
                                number,
                                revision,
                                checkout_ref,
                                os_scenario,
                                branch,
                                base,
                                project,
                                bots_ref
                                ))

    return results


def scan_for_pull_tasks(api, contexts, opts, repo):
    results = cockpit_tasks(api, not opts.dry, contexts, repo, opts.pull_data, opts.pull_number, opts.sha, opts.amqp)

    if opts.human_readable:
        results.sort(reverse=True, key=lambda x: str(x))
        return list([tests_human(*x) for x in results])
    if not opts.amqp:
        return list([tests_invoke(*x, options=opts) for x in results])
    with distributed_queue.DistributedQueue(opts.amqp, ['rhel', 'public']) as q:
        return list([queue_test(*x, channel=q.channel, options=opts) for x in results])


if __name__ == '__main__':
    sys.exit(main())