lldbsuite/test/decorators.py

from __future__ import print_function
from __future__ import absolute_import

# System modules
from distutils.version import LooseVersion, StrictVersion
from functools import wraps
import itertools
import os
import re
import sys

# Third-party modules
import six
import unittest2

# LLDB modules
import use_lldb_suite

import lldb
from . import configuration
from . import test_categories
from .result_formatter import EventBuilder
from lldbsuite.support import funcutils
from lldbsuite.test import lldbplatform
from lldbsuite.test import lldbplatformutil

class DecorateMode:
    Skip, Xfail = range(2)


# You can use no_match to reverse the test of the conditional that is used to match keyword
# arguments in the skip / xfail decorators.  If oslist=["windows", "linux"] skips windows
# and linux, oslist=no_match(["windows", "linux"]) skips *unless* windows or linux.
class no_match:
    def __init__(self, item):
        self.item = item

def _check_expected_version(comparison, expected, actual):
    def fn_leq(x,y): return x <= y
    def fn_less(x,y): return x < y
    def fn_geq(x,y): return x >= y
    def fn_greater(x,y): return x > y
    def fn_eq(x,y): return x == y
    def fn_neq(x,y): return x != y

    op_lookup = {
        "==": fn_eq,
        "=": fn_eq,
        "!=": fn_neq,
        "<>": fn_neq,
        ">": fn_greater,
        "<": fn_less,
        ">=": fn_geq,
        "<=": fn_leq
        }
    expected_str = '.'.join([str(x) for x in expected])
    actual_str = '.'.join([str(x) for x in actual])

    return op_lookup[comparison](LooseVersion(actual_str), LooseVersion(expected_str))

def _match_decorator_property(expected, actual):
    if actual is None or expected is None:
        return True

    if isinstance(expected, no_match):
        return not _match_decorator_property(expected.item, actual)
    elif isinstance(expected, (re._pattern_type,)+six.string_types):
        return re.search(expected, actual) is not None
    elif hasattr(expected, "__iter__"):
        return any([x is not None and _match_decorator_property(x, actual) for x in expected])
    else:
        return expected == actual

def expectedFailure(expected_fn, bugnumber=None):
    def expectedFailure_impl(func):
        if isinstance(func, type) and issubclass(func, unittest2.TestCase):
            raise Exception("Decorator can only be used to decorate a test method")
        @wraps(func)
        def wrapper(*args, **kwargs):
            from unittest2 import case
            self = args[0]
            if funcutils.requires_self(expected_fn):
                xfail_reason = expected_fn(self)
            else:
                xfail_reason = expected_fn()
            if xfail_reason is not None:
                if configuration.results_formatter_object is not None:
                    # Mark this test as expected to fail.
                    configuration.results_formatter_object.handle_event(
                        EventBuilder.event_for_mark_test_expected_failure(self))
                xfail_func = unittest2.expectedFailure(func)
                xfail_func(*args, **kwargs)
            else:
                func(*args, **kwargs)
        return wrapper
    # Some decorators can be called both with no arguments (e.g. @expectedFailureWindows)
    # or with arguments (e.g. @expectedFailureWindows(compilers=['gcc'])).  When called
    # the first way, the first argument will be the actual function because decorators are
    # weird like that.  So this is basically a check that says "which syntax was the original
    # function decorated with?"
    if six.callable(bugnumber):
        return expectedFailure_impl(bugnumber)
    else:
        return expectedFailure_impl

def skipTestIfFn(expected_fn, bugnumber=None):
    def skipTestIfFn_impl(func):
        if isinstance(func, type) and issubclass(func, unittest2.TestCase):
            raise Exception("@skipTestIfFn can only be used to decorate a test method")

        @wraps(func)
        def wrapper(*args, **kwargs):
            from unittest2 import case
            self = args[0]
            if funcutils.requires_self(expected_fn):
                reason = expected_fn(self)
            else:
                reason = expected_fn()

            if reason is not None:
               self.skipTest(reason)
            else:
                func(*args, **kwargs)
        return wrapper

    # Some decorators can be called both with no arguments (e.g. @expectedFailureWindows)
    # or with arguments (e.g. @expectedFailureWindows(compilers=['gcc'])).  When called
    # the first way, the first argument will be the actual function because decorators are
    # weird like that.  So this is basically a check that says "how was the decorator used"
    if six.callable(bugnumber):
        return skipTestIfFn_impl(bugnumber)
    else:
        return skipTestIfFn_impl

def _decorateTest(mode,
                 bugnumber=None, oslist=None, hostoslist=None,
                 compiler=None, compiler_version=None,
                 archs=None, triple=None,
                 debug_info=None,
                 swig_version=None, py_version=None,
                 remote=None):
    def fn(self):
        skip_for_os = _match_decorator_property(lldbplatform.translate(oslist), self.getPlatform())
        skip_for_hostos = _match_decorator_property(lldbplatform.translate(hostoslist), lldbplatformutil.getHostPlatform())
        skip_for_compiler = _match_decorator_property(compiler, self.getCompiler()) and self.expectedCompilerVersion(compiler_version)
        skip_for_arch = _match_decorator_property(archs, self.getArchitecture())
        skip_for_debug_info = _match_decorator_property(debug_info, self.debug_info)
        skip_for_triple = _match_decorator_property(triple, lldb.DBG.GetSelectedPlatform().GetTriple())
        skip_for_remote = _match_decorator_property(remote, lldb.remote_platform is not None)

        skip_for_swig_version = (swig_version is None) or (not hasattr(lldb, 'swig_version')) or (_check_expected_version(swig_version[0], swig_version[1], lldb.swig_version))
        skip_for_py_version = (py_version is None) or _check_expected_version(py_version[0], py_version[1], sys.version_info)

        # For the test to be skipped, all specified (e.g. not None) parameters must be True.
        # An unspecified parameter means "any", so those are marked skip by default.  And we skip
        # the final test if all conditions are True.
        conditions = [(oslist, skip_for_os, "target o/s"),
                      (hostoslist, skip_for_hostos, "host o/s"),
                      (compiler, skip_for_compiler, "compiler or version"),
                      (archs, skip_for_arch, "architecture"),
                      (debug_info, skip_for_debug_info, "debug info format"),
                      (triple, skip_for_triple, "target triple"),
                      (swig_version, skip_for_swig_version, "swig version"),
                      (py_version, skip_for_py_version, "python version"),
                      (remote, skip_for_remote, "platform locality (remote/local)")]
        reasons = []
        final_skip_result = True
        for this_condition in conditions:
            final_skip_result = final_skip_result and this_condition[1]
            if this_condition[0] is not None and this_condition[1]:
                reasons.append(this_condition[2])
        reason_str = None
        if final_skip_result:
            mode_str = {DecorateMode.Skip : "skipping", DecorateMode.Xfail : "xfailing"}[mode]
            if len(reasons) > 0:
                reason_str = ",".join(reasons)
                reason_str = "{} due to the following parameter(s): {}".format(mode_str, reason_str)
            else:
                reason_str = "{} unconditionally"
            if bugnumber is not None and not six.callable(bugnumber):
                reason_str = reason_str + " [" + str(bugnumber) + "]"
        return reason_str

    if mode == DecorateMode.Skip:
        return skipTestIfFn(fn, bugnumber)
    elif mode == DecorateMode.Xfail:
        return expectedFailure(fn, bugnumber)
    else:
        return None

# provide a function to xfail on defined oslist, compiler version, and archs
# if none is specified for any argument, that argument won't be checked and thus means for all
# for example,
# @expectedFailureAll, xfail for all platform/compiler/arch,
# @expectedFailureAll(compiler='gcc'), xfail for gcc on all platform/architecture
# @expectedFailureAll(bugnumber, ["linux"], "gcc", ['>=', '4.9'], ['i386']), xfail for gcc>=4.9 on linux with i386
def expectedFailureAll(bugnumber=None,
                       oslist=None, hostoslist=None,
                       compiler=None, compiler_version=None,
                       archs=None, triple=None,
                       debug_info=None,
                       swig_version=None, py_version=None,
                       remote=None):
    return _decorateTest(DecorateMode.Xfail,
                        bugnumber=bugnumber,
                        oslist=oslist, hostoslist=hostoslist,
                        compiler=compiler, compiler_version=compiler_version,
                        archs=archs, triple=triple,
                        debug_info=debug_info,
                        swig_version=swig_version, py_version=py_version,
                        remote=remote)


# provide a function to skip on defined oslist, compiler version, and archs
# if none is specified for any argument, that argument won't be checked and thus means for all
# for example,
# @skipIf, skip for all platform/compiler/arch,
# @skipIf(compiler='gcc'), skip for gcc on all platform/architecture
# @skipIf(bugnumber, ["linux"], "gcc", ['>=', '4.9'], ['i386']), skip for gcc>=4.9 on linux with i386
def skipIf(bugnumber=None,
           oslist=None, hostoslist=None,
           compiler=None, compiler_version=None,
           archs=None, triple=None,
           debug_info=None,
           swig_version=None, py_version=None,
           remote=None):
    return _decorateTest(DecorateMode.Skip,
                        bugnumber=bugnumber,
                        oslist=oslist, hostoslist=hostoslist,
                        compiler=compiler, compiler_version=compiler_version,
                        archs=archs, triple=triple,
                        debug_info=debug_info,
                        swig_version=swig_version, py_version=py_version,
                        remote=remote)

def _skip_for_android(reason, api_levels, archs):
    def impl(obj):
        result = lldbplatformutil.match_android_device(obj.getArchitecture(), valid_archs=archs, valid_api_levels=api_levels)
        return reason if result else None
    return impl

def add_test_categories(cat):
    """Add test categories to a TestCase method"""
    cat = test_categories.validate(cat, True)
    def impl(func):
        if isinstance(func, type) and issubclass(func, unittest2.TestCase):
            raise Exception("@add_test_categories can only be used to decorate a test method")
        if hasattr(func, "categories"):
            cat.extend(func.categories)
        func.categories = cat
        return func

    return impl

def benchmarks_test(func):
    """Decorate the item as a benchmarks test."""
    def should_skip_benchmarks_test():
        return "benchmarks test"

    # Mark this function as such to separate them from the regular tests.
    result = skipTestIfFn(should_skip_benchmarks_test)(func)
    result.__benchmarks_test__ = True
    return result

def no_debug_info_test(func):
    """Decorate the item as a test what don't use any debug info. If this annotation is specified
       then the test runner won't generate a separate test for each debug info format. """
    if isinstance(func, type) and issubclass(func, unittest2.TestCase):
        raise Exception("@no_debug_info_test can only be used to decorate a test method")
    @wraps(func)
    def wrapper(self, *args, **kwargs):
        return func(self, *args, **kwargs)

    # Mark this function as such to separate them from the regular tests.
    wrapper.__no_debug_info_test__ = True
    return wrapper

def debugserver_test(func):
    """Decorate the item as a debugserver test."""
    def should_skip_debugserver_test():
        return "debugserver tests" if configuration.dont_do_debugserver_test else None
    return skipTestIfFn(should_skip_debugserver_test)(func)

def llgs_test(func):
    """Decorate the item as a lldb-server test."""
    def should_skip_llgs_tests():
        return "llgs tests" if configuration.dont_do_llgs_test else None
    return skipTestIfFn(should_skip_llgs_tests)(func)

def not_remote_testsuite_ready(func):
    """Decorate the item as a test which is not ready yet for remote testsuite."""
    def is_remote():
        return "Not ready for remote testsuite" if lldb.remote_platform else None
    return skipTestIfFn(is_remote)(func)

def expectedFailureOS(oslist, bugnumber=None, compilers=None, debug_info=None, archs=None):
    return expectedFailureAll(oslist=oslist, bugnumber=bugnumber, compiler=compilers, archs=archs, debug_info=debug_info)

def expectedFailureDarwin(bugnumber=None, compilers=None, debug_info=None):
    # For legacy reasons, we support both "darwin" and "macosx" as OS X triples.
    return expectedFailureOS(lldbplatform.darwin_all, bugnumber, compilers, debug_info=debug_info)

def expectedFailureAndroid(bugnumber=None, api_levels=None, archs=None):
    """ Mark a test as xfail for Android.

    Arguments:
        bugnumber - The LLVM pr associated with the problem.
        api_levels - A sequence of numbers specifying the Android API levels
            for which a test is expected to fail. None means all API level.
        arch - A sequence of architecture names specifying the architectures
            for which a test is expected to fail. None means all architectures.
    """
    return expectedFailure(_skip_for_android("xfailing on android", api_levels, archs), bugnumber)

# Flakey tests get two chances to run. If they fail the first time round, the result formatter
# makes sure it is run one more time.
def expectedFlakey(expected_fn, bugnumber=None):
    def expectedFailure_impl(func):
        @wraps(func)
        def wrapper(*args, **kwargs):
            self = args[0]
            if expected_fn(self):
                # Send event marking test as explicitly eligible for rerunning.
                if configuration.results_formatter_object is not None:
                    # Mark this test as rerunnable.
                    configuration.results_formatter_object.handle_event(
                        EventBuilder.event_for_mark_test_rerun_eligible(self))
            func(*args, **kwargs)
        return wrapper
    # Some decorators can be called both with no arguments (e.g. @expectedFailureWindows)
    # or with arguments (e.g. @expectedFailureWindows(compilers=['gcc'])).  When called
    # the first way, the first argument will be the actual function because decorators are
    # weird like that.  So this is basically a check that says "which syntax was the original
    # function decorated with?"
    if six.callable(bugnumber):
        return expectedFailure_impl(bugnumber)
    else:
        return expectedFailure_impl

def expectedFlakeyDwarf(bugnumber=None):
    def fn(self):
        return self.debug_info == "dwarf"
    return expectedFlakey(fn, bugnumber)

def expectedFlakeyDsym(bugnumber=None):
    def fn(self):
        return self.debug_info == "dwarf"
    return expectedFlakey(fn, bugnumber)

def expectedFlakeyOS(oslist, bugnumber=None, compilers=None):
    def fn(self):
        return (self.getPlatform() in oslist and
                self.expectedCompiler(compilers))
    return expectedFlakey(fn, bugnumber)

def expectedFlakeyDarwin(bugnumber=None, compilers=None):
    # For legacy reasons, we support both "darwin" and "macosx" as OS X triples.
    return expectedFlakeyOS(lldbplatformutil.getDarwinOSTriples(), bugnumber, compilers)

def expectedFlakeyFreeBSD(bugnumber=None, compilers=None):
    return expectedFlakeyOS(['freebsd'], bugnumber, compilers)

def expectedFlakeyLinux(bugnumber=None, compilers=None):
    return expectedFlakeyOS(['linux'], bugnumber, compilers)

def expectedFlakeyNetBSD(bugnumber=None, compilers=None):
    return expectedFlakeyOS(['netbsd'], bugnumber, compilers)

def expectedFlakeyCompiler(compiler, compiler_version=None, bugnumber=None):
    if compiler_version is None:
        compiler_version=['=', None]
    def fn(self):
        return compiler in self.getCompiler() and self.expectedCompilerVersion(compiler_version)
    return expectedFlakey(fn, bugnumber)

# @expectedFlakeyClang('bugnumber', ['<=', '3.4'])
def expectedFlakeyClang(bugnumber=None, compiler_version=None):
    return expectedFlakeyCompiler('clang', compiler_version, bugnumber)

# @expectedFlakeyGcc('bugnumber', ['<=', '3.4'])
def expectedFlakeyGcc(bugnumber=None, compiler_version=None):
    return expectedFlakeyCompiler('gcc', compiler_version, bugnumber)

def expectedFlakeyAndroid(bugnumber=None, api_levels=None, archs=None):
    return expectedFlakey(_skip_for_android("flakey on android", api_levels, archs), bugnumber)

def skipIfRemote(func):
    """Decorate the item to skip tests if testing remotely."""
    def is_remote():
        return "skip on remote platform" if lldb.remote_platform else None
    return skipTestIfFn(is_remote)(func)

def skipIfRemoteDueToDeadlock(func):
    """Decorate the item to skip tests if testing remotely due to the test deadlocking."""
    def is_remote():
        return "skip on remote platform (deadlocks)" if lldb.remote_platform else None
    return skipTestIfFn(is_remote)(func)

def skipIfNoSBHeaders(func):
    """Decorate the item to mark tests that should be skipped when LLDB is built with no SB API headers."""
    def are_sb_headers_missing():
        if lldbplatformutil.getHostPlatform() == 'darwin':
            header = os.path.join(os.environ["LLDB_LIB_DIR"], 'LLDB.framework', 'Versions','Current','Headers','LLDB.h')
        else:
            header = os.path.join(os.environ["LLDB_SRC"], "include", "lldb", "API", "LLDB.h")
        if not os.path.exists(header):
            return "skip because LLDB.h header not found"
        return None

    return skipTestIfFn(are_sb_headers_missing)(func)

def skipIfiOSSimulator(func):
    """Decorate the item to skip tests that should be skipped on the iOS Simulator."""
    def is_ios_simulator():
        return "skip on the iOS Simulator" if configuration.lldb_platform_name == 'ios-simulator' else None
    return skipTestIfFn(is_ios_simulator)(func)

def skipIfFreeBSD(func):
    """Decorate the item to skip tests that should be skipped on FreeBSD."""
    return skipIfPlatform(["freebsd"])(func)

def skipIfNetBSD(func):
    """Decorate the item to skip tests that should be skipped on NetBSD."""
    return skipIfPlatform(["netbsd"])(func)

def skipIfDarwin(func):
    """Decorate the item to skip tests that should be skipped on Darwin."""
    return skipIfPlatform(lldbplatform.translate(lldbplatform.darwin_all))(func)

def skipIfLinux(func):
    """Decorate the item to skip tests that should be skipped on Linux."""
    return skipIfPlatform(["linux"])(func)

def skipIfWindows(func):
    """Decorate the item to skip tests that should be skipped on Windows."""
    return skipIfPlatform(["windows"])(func)

def skipUnlessWindows(func):
    """Decorate the item to skip tests that should be skipped on any non-Windows platform."""
    return skipUnlessPlatform(["windows"])(func)

def skipUnlessDarwin(func):
    """Decorate the item to skip tests that should be skipped on any non Darwin platform."""
    return skipUnlessPlatform(lldbplatformutil.getDarwinOSTriples())(func)

def skipUnlessGoInstalled(func):
    """Decorate the item to skip tests when no Go compiler is available."""
    def is_go_missing(self):
        compiler = self.getGoCompilerVersion()
        if not compiler:
            return "skipping because go compiler not found"
        match_version = re.search(r"(\d+\.\d+(\.\d+)?)", compiler)
        if not match_version:
            # Couldn't determine version.
            return "skipping because go version could not be parsed out of {}".format(compiler)
        else:
            min_strict_version = StrictVersion("1.4.0")
            compiler_strict_version = StrictVersion(match_version.group(1))
            if compiler_strict_version < min_strict_version:
                return "skipping because available version ({}) does not meet minimum required version ({})".format(
                    compiler_strict_version, min_strict_version)
        return None
    return skipTestIfFn(is_go_missing)(func)

def skipIfHostIncompatibleWithRemote(func):
    """Decorate the item to skip tests if binaries built on this host are incompatible."""
    def is_host_incompatible_with_remote(self):
        host_arch = self.getLldbArchitecture()
        host_platform = lldbplatformutil.getHostPlatform()
        target_arch = self.getArchitecture()
        target_platform = 'darwin' if self.platformIsDarwin() else self.getPlatform()
        if not (target_arch == 'x86_64' and host_arch == 'i386') and host_arch != target_arch:
            return "skipping because target %s is not compatible with host architecture %s" % (target_arch, host_arch)
        elif target_platform != host_platform:
            return "skipping because target is %s but host is %s" % (target_platform, host_platform)
        return None
    return skipTestIfFn(is_host_incompatible_with_remote)(func)

def skipIfPlatform(oslist):
    """Decorate the item to skip tests if running on one of the listed platforms."""
    # This decorator cannot be ported to `skipIf` yet because it is used on entire
    # classes, which `skipIf` explicitly forbids.
    return unittest2.skipIf(lldbplatformutil.getPlatform() in oslist,
                            "skip on %s" % (", ".join(oslist)))

def skipUnlessPlatform(oslist):
    """Decorate the item to skip tests unless running on one of the listed platforms."""
    # This decorator cannot be ported to `skipIf` yet because it is used on entire
    # classes, which `skipIf` explicitly forbids.
    return unittest2.skipUnless(lldbplatformutil.getPlatform() in oslist,
                                "requires on of %s" % (", ".join(oslist)))

def skipIfTargetAndroid(api_levels=None, archs=None):
    """Decorator to skip tests when the target is Android.

    Arguments:
        api_levels - The API levels for which the test should be skipped. If
            it is None, then the test will be skipped for all API levels.
        arch - A sequence of architecture names specifying the architectures
            for which a test is skipped. None means all architectures.
    """
    return skipTestIfFn(_skip_for_android("skipping for android", api_levels, archs))

def skipUnlessCompilerRt(func):
    """Decorate the item to skip tests if testing remotely."""
    def is_compiler_rt_missing():
        compilerRtPath = os.path.join(os.path.dirname(__file__), "..", "..", "..", "..", "llvm","projects","compiler-rt")
        return "compiler-rt not found" if not os.path.exists(compilerRtPath) else None
    return skipTestIfFn(is_compiler_rt_missing)(func)