1"""
2A simple testing framework for lldb using python's unit testing framework.
3
4Tests for lldb are written as python scripts which take advantage of the script
5bridging provided by LLDB.framework to interact with lldb core.
6
7A specific naming pattern is followed by the .py script to be recognized as
8a module which implements a test scenario, namely, Test*.py.
9
10To specify the directories where "Test*.py" python test scripts are located,
11you need to pass in a list of directory names.  By default, the current
12working directory is searched if nothing is specified on the command line.
13
14Type:
15
16./dotest.py -h
17
18for available options.
19"""
20
21from __future__ import absolute_import
22from __future__ import print_function
23
24# System modules
25import atexit
26import datetime
27import errno
28import logging
29import os
30import platform
31import re
32import signal
33import subprocess
34import sys
35import tempfile
36
37# Third-party modules
38import six
39import unittest2
40
41# LLDB Modules
42import lldbsuite
43from . import configuration
44from . import dotest_args
45from . import lldbtest_config
46from . import test_categories
47from . import test_result
48from ..support import seven
49
50
51def is_exe(fpath):
52    """Returns true if fpath is an executable."""
53    if fpath == None:
54        return False
55    return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
56
57
58def which(program):
59    """Returns the full path to a program; None otherwise."""
60    fpath, _ = os.path.split(program)
61    if fpath:
62        if is_exe(program):
63            return program
64    else:
65        for path in os.environ["PATH"].split(os.pathsep):
66            exe_file = os.path.join(path, program)
67            if is_exe(exe_file):
68                return exe_file
69    return None
70
71
72def usage(parser):
73    parser.print_help()
74    if configuration.verbose > 0:
75        print("""
76Examples:
77
78This is an example of using the -f option to pinpoint to a specific test class
79and test method to be run:
80
81$ ./dotest.py -f ClassTypesTestCase.test_with_dsym_and_run_command
82----------------------------------------------------------------------
83Collected 1 test
84
85test_with_dsym_and_run_command (TestClassTypes.ClassTypesTestCase)
86Test 'frame variable this' when stopped on a class constructor. ... ok
87
88----------------------------------------------------------------------
89Ran 1 test in 1.396s
90
91OK
92
93And this is an example of using the -p option to run a single file (the filename
94matches the pattern 'ObjC' and it happens to be 'TestObjCMethods.py'):
95
96$ ./dotest.py -v -p ObjC
97----------------------------------------------------------------------
98Collected 4 tests
99
100test_break_with_dsym (TestObjCMethods.FoundationTestCase)
101Test setting objc breakpoints using '_regexp-break' and 'breakpoint set'. ... ok
102test_break_with_dwarf (TestObjCMethods.FoundationTestCase)
103Test setting objc breakpoints using '_regexp-break' and 'breakpoint set'. ... ok
104test_data_type_and_expr_with_dsym (TestObjCMethods.FoundationTestCase)
105Lookup objective-c data types and evaluate expressions. ... ok
106test_data_type_and_expr_with_dwarf (TestObjCMethods.FoundationTestCase)
107Lookup objective-c data types and evaluate expressions. ... ok
108
109----------------------------------------------------------------------
110Ran 4 tests in 16.661s
111
112OK
113
114Running of this script also sets up the LLDB_TEST environment variable so that
115individual test cases can locate their supporting files correctly.  The script
116tries to set up Python's search paths for modules by looking at the build tree
117relative to this script.  See also the '-i' option in the following example.
118
119Finally, this is an example of using the lldb.py module distributed/installed by
120Xcode4 to run against the tests under the 'forward' directory, and with the '-w'
121option to add some delay between two tests.  It uses ARCH=x86_64 to specify that
122as the architecture and CC=clang to specify the compiler used for the test run:
123
124$ PYTHONPATH=/Xcode4/Library/PrivateFrameworks/LLDB.framework/Versions/A/Resources/Python ARCH=x86_64 CC=clang ./dotest.py -v -w -i forward
125
126Session logs for test failures/errors will go into directory '2010-11-11-13_56_16'
127----------------------------------------------------------------------
128Collected 2 tests
129
130test_with_dsym_and_run_command (TestForwardDeclaration.ForwardDeclarationTestCase)
131Display *bar_ptr when stopped on a function with forward declaration of struct bar. ... ok
132test_with_dwarf_and_run_command (TestForwardDeclaration.ForwardDeclarationTestCase)
133Display *bar_ptr when stopped on a function with forward declaration of struct bar. ... ok
134
135----------------------------------------------------------------------
136Ran 2 tests in 5.659s
137
138OK
139
140The 'Session ...' verbiage is recently introduced (see also the '-s' option) to
141notify the directory containing the session logs for test failures or errors.
142In case there is any test failure/error, a similar message is appended at the
143end of the stderr output for your convenience.
144
145ENABLING LOGS FROM TESTS
146
147Option 1:
148
149Writing logs into different files per test case::
150
151$ ./dotest.py --channel "lldb all"
152
153$ ./dotest.py --channel "lldb all" --channel "gdb-remote packets"
154
155These log files are written to:
156
157<session-dir>/<test-id>-host.log (logs from lldb host process)
158<session-dir>/<test-id>-server.log (logs from debugserver/lldb-server)
159<session-dir>/<test-id>-<test-result>.log (console logs)
160
161By default, logs from successful runs are deleted.  Use the --log-success flag
162to create reference logs for debugging.
163
164$ ./dotest.py --log-success
165
166""")
167    sys.exit(0)
168
169
170def parseExclusion(exclusion_file):
171    """Parse an exclusion file, of the following format, where
172       'skip files', 'skip methods', 'xfail files', and 'xfail methods'
173       are the possible list heading values:
174
175       skip files
176       <file name>
177       <file name>
178
179       xfail methods
180       <method name>
181    """
182    excl_type = None
183
184    with open(exclusion_file) as f:
185        for line in f:
186            line = line.strip()
187            if not excl_type:
188                excl_type = line
189                continue
190
191            if not line:
192                excl_type = None
193            elif excl_type == 'skip':
194                if not configuration.skip_tests:
195                    configuration.skip_tests = []
196                configuration.skip_tests.append(line)
197            elif excl_type == 'xfail':
198                if not configuration.xfail_tests:
199                    configuration.xfail_tests = []
200                configuration.xfail_tests.append(line)
201
202
203def parseOptionsAndInitTestdirs():
204    """Initialize the list of directories containing our unittest scripts.
205
206    '-h/--help as the first option prints out usage info and exit the program.
207    """
208
209    do_help = False
210
211    platform_system = platform.system()
212    platform_machine = platform.machine()
213
214    try:
215        parser = dotest_args.create_parser()
216        args = parser.parse_args()
217    except:
218        raise
219
220    if args.unset_env_varnames:
221        for env_var in args.unset_env_varnames:
222            if env_var in os.environ:
223                # From Python Doc: When unsetenv() is supported, deletion of items in os.environ
224                # is automatically translated into a corresponding call to
225                # unsetenv().
226                del os.environ[env_var]
227                # os.unsetenv(env_var)
228
229    if args.set_env_vars:
230        for env_var in args.set_env_vars:
231            parts = env_var.split('=', 1)
232            if len(parts) == 1:
233                os.environ[parts[0]] = ""
234            else:
235                os.environ[parts[0]] = parts[1]
236
237    if args.set_inferior_env_vars:
238        lldbtest_config.inferior_env = ' '.join(args.set_inferior_env_vars)
239
240    if args.h:
241        do_help = True
242
243    if args.compiler:
244        configuration.compiler = os.path.abspath(args.compiler)
245        if not is_exe(configuration.compiler):
246            configuration.compiler = which(args.compiler)
247        if not is_exe(configuration.compiler):
248            logging.error(
249                    '%s is not a valid compiler executable; aborting...',
250                    args.compiler)
251            sys.exit(-1)
252    else:
253        # Use a compiler appropriate appropriate for the Apple SDK if one was
254        # specified
255        if platform_system == 'Darwin' and args.apple_sdk:
256            configuration.compiler = seven.get_command_output(
257                'xcrun -sdk "%s" -find clang 2> /dev/null' %
258                (args.apple_sdk))
259        else:
260            # 'clang' on ubuntu 14.04 is 3.4 so we try clang-3.5 first
261            candidateCompilers = ['clang-3.5', 'clang', 'gcc']
262            for candidate in candidateCompilers:
263                if which(candidate):
264                    configuration.compiler = candidate
265                    break
266
267    if args.dsymutil:
268        configuration.dsymutil = args.dsymutil
269    elif platform_system == 'Darwin':
270        configuration.dsymutil = seven.get_command_output(
271            'xcrun -find -toolchain default dsymutil')
272
273
274    # The lldb-dotest script produced by the CMake build passes in a path to a
275    # working FileCheck and yaml2obj binary. So does one specific Xcode
276    # project target. However, when invoking dotest.py directly, a valid
277    # --filecheck and --yaml2obj option needs to be given.
278    if args.filecheck:
279        configuration.filecheck = os.path.abspath(args.filecheck)
280
281    if args.yaml2obj:
282        configuration.yaml2obj = os.path.abspath(args.yaml2obj)
283
284    if not configuration.get_filecheck_path():
285        logging.warning('No valid FileCheck executable; some tests may fail...')
286        logging.warning('(Double-check the --filecheck argument to dotest.py)')
287
288    if args.channels:
289        lldbtest_config.channels = args.channels
290
291    if args.log_success:
292        lldbtest_config.log_success = args.log_success
293
294    if args.out_of_tree_debugserver:
295        lldbtest_config.out_of_tree_debugserver = args.out_of_tree_debugserver
296
297    # Set SDKROOT if we are using an Apple SDK
298    if platform_system == 'Darwin' and args.apple_sdk:
299        configuration.sdkroot = seven.get_command_output(
300            'xcrun --sdk "%s" --show-sdk-path 2> /dev/null' %
301            (args.apple_sdk))
302
303    if args.arch:
304        configuration.arch = args.arch
305    else:
306        configuration.arch = platform_machine
307
308    if args.categories_list:
309        configuration.categories_list = set(
310            test_categories.validate(
311                args.categories_list, False))
312        configuration.use_categories = True
313    else:
314        configuration.categories_list = []
315
316    if args.skip_categories:
317        configuration.skip_categories += test_categories.validate(
318            args.skip_categories, False)
319
320    if args.xfail_categories:
321        configuration.xfail_categories += test_categories.validate(
322            args.xfail_categories, False)
323
324    if args.E:
325        os.environ['CFLAGS_EXTRAS'] = args.E
326
327    if args.dwarf_version:
328        configuration.dwarf_version = args.dwarf_version
329        # We cannot modify CFLAGS_EXTRAS because they're used in test cases
330        # that explicitly require no debug info.
331        os.environ['CFLAGS'] = '-gdwarf-{}'.format(configuration.dwarf_version)
332
333    if args.settings:
334        for setting in args.settings:
335            if not len(setting) == 1 or not setting[0].count('='):
336                logging.error('"%s" is not a setting in the form "key=value"',
337                              setting[0])
338                sys.exit(-1)
339            setting_list = setting[0].split('=', 1)
340            configuration.settings.append((setting_list[0], setting_list[1]))
341
342    if args.d:
343        sys.stdout.write(
344            "Suspending the process %d to wait for debugger to attach...\n" %
345            os.getpid())
346        sys.stdout.flush()
347        os.kill(os.getpid(), signal.SIGSTOP)
348
349    if args.f:
350        if any([x.startswith('-') for x in args.f]):
351            usage(parser)
352        configuration.filters.extend(args.f)
353
354    if args.framework:
355        configuration.lldb_framework_path = args.framework
356
357    if args.executable:
358        # lldb executable is passed explicitly
359        lldbtest_config.lldbExec = os.path.realpath(args.executable)
360        if not is_exe(lldbtest_config.lldbExec):
361            lldbtest_config.lldbExec = which(args.executable)
362        if not is_exe(lldbtest_config.lldbExec):
363            logging.error(
364                    '%s is not a valid executable to test; aborting...',
365                    args.executable)
366            sys.exit(-1)
367
368    if args.server and args.out_of_tree_debugserver:
369        logging.warning('Both --server and --out-of-tree-debugserver are set')
370
371    if args.server and not args.out_of_tree_debugserver:
372        os.environ['LLDB_DEBUGSERVER_PATH'] = args.server
373
374    if args.excluded:
375        for excl_file in args.excluded:
376            parseExclusion(excl_file)
377
378    if args.p:
379        if args.p.startswith('-'):
380            usage(parser)
381        configuration.regexp = args.p
382
383    if args.s:
384        configuration.sdir_name = args.s
385    else:
386        timestamp_started = datetime.datetime.now().strftime("%Y-%m-%d-%H_%M_%S")
387        configuration.sdir_name = os.path.join(os.getcwd(), timestamp_started)
388
389    configuration.session_file_format = args.session_file_format
390
391    if args.t:
392        os.environ['LLDB_COMMAND_TRACE'] = 'YES'
393
394    if args.v:
395        configuration.verbose = 2
396
397    # argparse makes sure we have a number
398    if args.sharp:
399        configuration.count = args.sharp
400
401    if sys.platform.startswith('win32'):
402        os.environ['LLDB_DISABLE_CRASH_DIALOG'] = str(
403            args.disable_crash_dialog)
404        os.environ['LLDB_LAUNCH_INFERIORS_WITHOUT_CONSOLE'] = str(True)
405
406    if do_help:
407        usage(parser)
408
409    # Reproducer arguments
410    if args.capture_path and args.replay_path:
411        logging.error('Cannot specify both a capture and a replay path.')
412        sys.exit(-1)
413
414    if args.capture_path:
415        configuration.capture_path = args.capture_path
416
417    if args.replay_path:
418        configuration.replay_path = args.replay_path
419    if args.lldb_platform_name:
420        configuration.lldb_platform_name = args.lldb_platform_name
421    if args.lldb_platform_url:
422        configuration.lldb_platform_url = args.lldb_platform_url
423    if args.lldb_platform_working_dir:
424        configuration.lldb_platform_working_dir = args.lldb_platform_working_dir
425    if platform_system == 'Darwin'  and args.apple_sdk:
426        configuration.apple_sdk = args.apple_sdk
427    if args.test_build_dir:
428        configuration.test_build_dir = args.test_build_dir
429    if args.lldb_module_cache_dir:
430        configuration.lldb_module_cache_dir = args.lldb_module_cache_dir
431    else:
432        configuration.lldb_module_cache_dir = os.path.join(
433            configuration.test_build_dir, 'module-cache-lldb')
434    if args.clang_module_cache_dir:
435        configuration.clang_module_cache_dir = args.clang_module_cache_dir
436    else:
437        configuration.clang_module_cache_dir = os.path.join(
438            configuration.test_build_dir, 'module-cache-clang')
439
440    if args.lldb_libs_dir:
441        configuration.lldb_libs_dir = args.lldb_libs_dir
442
443    if args.enabled_plugins:
444        configuration.enabled_plugins = args.enabled_plugins
445
446    # Gather all the dirs passed on the command line.
447    if len(args.args) > 0:
448        configuration.testdirs = [os.path.realpath(os.path.abspath(x)) for x in args.args]
449
450    lldbtest_config.codesign_identity = args.codesign_identity
451
452def setupSysPath():
453    """
454    Add LLDB.framework/Resources/Python to the search paths for modules.
455    As a side effect, we also discover the 'lldb' executable and export it here.
456    """
457
458    # Get the directory containing the current script.
459    if "DOTEST_PROFILE" in os.environ and "DOTEST_SCRIPT_DIR" in os.environ:
460        scriptPath = os.environ["DOTEST_SCRIPT_DIR"]
461    else:
462        scriptPath = os.path.dirname(os.path.abspath(__file__))
463    if not scriptPath.endswith('test'):
464        print("This script expects to reside in lldb's test directory.")
465        sys.exit(-1)
466
467    os.environ["LLDB_TEST"] = scriptPath
468
469    # Set up the root build directory.
470    if not configuration.test_build_dir:
471        raise Exception("test_build_dir is not set")
472    configuration.test_build_dir = os.path.abspath(configuration.test_build_dir)
473
474    # Set up the LLDB_SRC environment variable, so that the tests can locate
475    # the LLDB source code.
476    os.environ["LLDB_SRC"] = lldbsuite.lldb_root
477
478    pluginPath = os.path.join(scriptPath, 'plugins')
479    toolsLLDBVSCode = os.path.join(scriptPath, 'tools', 'lldb-vscode')
480    toolsLLDBServerPath = os.path.join(scriptPath, 'tools', 'lldb-server')
481
482    # Insert script dir, plugin dir and lldb-server dir to the sys.path.
483    sys.path.insert(0, pluginPath)
484    # Adding test/tools/lldb-vscode to the path makes it easy to
485    # "import lldb_vscode_testcase" from the VSCode tests
486    sys.path.insert(0, toolsLLDBVSCode)
487    # Adding test/tools/lldb-server to the path makes it easy
488    sys.path.insert(0, toolsLLDBServerPath)
489    # to "import lldbgdbserverutils" from the lldb-server tests
490
491    # This is the root of the lldb git/svn checkout
492    # When this changes over to a package instead of a standalone script, this
493    # will be `lldbsuite.lldb_root`
494    lldbRootDirectory = lldbsuite.lldb_root
495
496    # Some of the tests can invoke the 'lldb' command directly.
497    # We'll try to locate the appropriate executable right here.
498
499    # The lldb executable can be set from the command line
500    # if it's not set, we try to find it now
501    # first, we try the environment
502    if not lldbtest_config.lldbExec:
503        # First, you can define an environment variable LLDB_EXEC specifying the
504        # full pathname of the lldb executable.
505        if "LLDB_EXEC" in os.environ:
506            lldbtest_config.lldbExec = os.environ["LLDB_EXEC"]
507
508    if not lldbtest_config.lldbExec:
509        # Last, check the path
510        lldbtest_config.lldbExec = which('lldb')
511
512    if lldbtest_config.lldbExec and not is_exe(lldbtest_config.lldbExec):
513        print(
514            "'{}' is not a path to a valid executable".format(
515                lldbtest_config.lldbExec))
516        lldbtest_config.lldbExec = None
517
518    if not lldbtest_config.lldbExec:
519        print("The 'lldb' executable cannot be located.  Some of the tests may not be run as a result.")
520        sys.exit(-1)
521
522    # confusingly, this is the "bin" directory
523    lldbLibDir = os.path.dirname(lldbtest_config.lldbExec)
524    os.environ["LLDB_LIB_DIR"] = lldbLibDir
525    lldbImpLibDir = configuration.lldb_libs_dir
526    os.environ["LLDB_IMPLIB_DIR"] = lldbImpLibDir
527    print("LLDB library dir:", os.environ["LLDB_LIB_DIR"])
528    print("LLDB import library dir:", os.environ["LLDB_IMPLIB_DIR"])
529    os.system('%s -v' % lldbtest_config.lldbExec)
530
531    lldbDir = os.path.dirname(lldbtest_config.lldbExec)
532
533    lldbVSCodeExec = os.path.join(lldbDir, "lldb-vscode")
534    if is_exe(lldbVSCodeExec):
535        os.environ["LLDBVSCODE_EXEC"] = lldbVSCodeExec
536    else:
537        if not configuration.shouldSkipBecauseOfCategories(["lldb-vscode"]):
538            print(
539                "The 'lldb-vscode' executable cannot be located.  The lldb-vscode tests can not be run as a result.")
540            configuration.skip_categories.append("lldb-vscode")
541
542    lldbPythonDir = None  # The directory that contains 'lldb/__init__.py'
543    if not configuration.lldb_framework_path and os.path.exists(os.path.join(lldbLibDir, "LLDB.framework")):
544        configuration.lldb_framework_path = os.path.join(lldbLibDir, "LLDB.framework")
545    if configuration.lldb_framework_path:
546        lldbtest_config.lldb_framework_path = configuration.lldb_framework_path
547        candidatePath = os.path.join(
548            configuration.lldb_framework_path, 'Resources', 'Python')
549        if os.path.isfile(os.path.join(candidatePath, 'lldb/__init__.py')):
550            lldbPythonDir = candidatePath
551        if not lldbPythonDir:
552            print(
553                'Resources/Python/lldb/__init__.py was not found in ' +
554                configuration.lldb_framework_path)
555            sys.exit(-1)
556    else:
557        # If our lldb supports the -P option, use it to find the python path:
558        init_in_python_dir = os.path.join('lldb', '__init__.py')
559
560        lldb_dash_p_result = subprocess.check_output(
561            [lldbtest_config.lldbExec, "-P"], stderr=subprocess.STDOUT, universal_newlines=True)
562
563        if lldb_dash_p_result and not lldb_dash_p_result.startswith(
564                ("<", "lldb: invalid option:")) and not lldb_dash_p_result.startswith("Traceback"):
565            lines = lldb_dash_p_result.splitlines()
566
567            # Workaround for readline vs libedit issue on FreeBSD.  If stdout
568            # is not a terminal Python executes
569            #     rl_variable_bind ("enable-meta-key", "off");
570            # This produces a warning with FreeBSD's libedit because the
571            # enable-meta-key variable is unknown.  Not an issue on Apple
572            # because cpython commit f0ab6f9f0603 added a #ifndef __APPLE__
573            # around the call.  See http://bugs.python.org/issue19884 for more
574            # information.  For now we just discard the warning output.
575            if len(lines) >= 1 and lines[0].startswith(
576                    "bind: Invalid command"):
577                lines.pop(0)
578
579            # Taking the last line because lldb outputs
580            # 'Cannot read termcap database;\nusing dumb terminal settings.\n'
581            # before the path
582            if len(lines) >= 1 and os.path.isfile(
583                    os.path.join(lines[-1], init_in_python_dir)):
584                lldbPythonDir = lines[-1]
585                if "freebsd" in sys.platform or "linux" in sys.platform:
586                    os.environ['LLDB_LIB_DIR'] = os.path.join(
587                        lldbPythonDir, '..', '..')
588
589        if not lldbPythonDir:
590            print(
591                "Unable to load lldb extension module.  Possible reasons for this include:")
592            print("  1) LLDB was built with LLDB_ENABLE_PYTHON=0")
593            print(
594                "  2) PYTHONPATH and PYTHONHOME are not set correctly.  PYTHONHOME should refer to")
595            print(
596                "     the version of Python that LLDB built and linked against, and PYTHONPATH")
597            print(
598                "     should contain the Lib directory for the same python distro, as well as the")
599            print("     location of LLDB\'s site-packages folder.")
600            print(
601                "  3) A different version of Python than that which was built against is exported in")
602            print("     the system\'s PATH environment variable, causing conflicts.")
603            print(
604                "  4) The executable '%s' could not be found.  Please check " %
605                lldbtest_config.lldbExec)
606            print("     that it exists and is executable.")
607
608    if lldbPythonDir:
609        lldbPythonDir = os.path.normpath(lldbPythonDir)
610        # Some of the code that uses this path assumes it hasn't resolved the Versions... link.
611        # If the path we've constructed looks like that, then we'll strip out
612        # the Versions/A part.
613        (before, frameWithVersion, after) = lldbPythonDir.rpartition(
614            "LLDB.framework/Versions/A")
615        if frameWithVersion != "":
616            lldbPythonDir = before + "LLDB.framework" + after
617
618        lldbPythonDir = os.path.abspath(lldbPythonDir)
619
620        # If tests need to find LLDB_FRAMEWORK, now they can do it
621        os.environ["LLDB_FRAMEWORK"] = os.path.dirname(
622            os.path.dirname(lldbPythonDir))
623
624        # This is to locate the lldb.py module.  Insert it right after
625        # sys.path[0].
626        sys.path[1:1] = [lldbPythonDir]
627
628
629def visit_file(dir, name):
630    # Try to match the regexp pattern, if specified.
631    if configuration.regexp:
632        if not re.search(configuration.regexp, name):
633            # We didn't match the regex, we're done.
634            return
635
636    if configuration.skip_tests:
637        for file_regexp in configuration.skip_tests:
638            if re.search(file_regexp, name):
639                return
640
641    # We found a match for our test.  Add it to the suite.
642
643    # Update the sys.path first.
644    if not sys.path.count(dir):
645        sys.path.insert(0, dir)
646    base = os.path.splitext(name)[0]
647
648    # Thoroughly check the filterspec against the base module and admit
649    # the (base, filterspec) combination only when it makes sense.
650
651    def check(obj, parts):
652        for part in parts:
653            try:
654                parent, obj = obj, getattr(obj, part)
655            except AttributeError:
656                # The filterspec has failed.
657                return False
658        return True
659
660    module = __import__(base)
661
662    def iter_filters():
663        for filterspec in configuration.filters:
664            parts = filterspec.split('.')
665            if check(module, parts):
666                yield filterspec
667            elif parts[0] == base and len(parts) > 1 and check(module, parts[1:]):
668                yield '.'.join(parts[1:])
669            else:
670                for key,value in module.__dict__.items():
671                    if check(value, parts):
672                        yield key + '.' + filterspec
673
674    filtered = False
675    for filterspec in iter_filters():
676        filtered = True
677        print("adding filter spec %s to module %s" % (filterspec, repr(module)))
678        tests = unittest2.defaultTestLoader.loadTestsFromName(filterspec, module)
679        configuration.suite.addTests(tests)
680
681    # Forgo this module if the (base, filterspec) combo is invalid
682    if configuration.filters and not filtered:
683        return
684
685    if not filtered:
686        # Add the entire file's worth of tests since we're not filtered.
687        # Also the fail-over case when the filterspec branch
688        # (base, filterspec) combo doesn't make sense.
689        configuration.suite.addTests(
690            unittest2.defaultTestLoader.loadTestsFromName(base))
691
692
693def visit(prefix, dir, names):
694    """Visitor function for os.path.walk(path, visit, arg)."""
695
696    dir_components = set(dir.split(os.sep))
697    excluded_components = set(['.svn', '.git'])
698    if dir_components.intersection(excluded_components):
699        return
700
701    # Gather all the Python test file names that follow the Test*.py pattern.
702    python_test_files = [
703        name
704        for name in names
705        if name.endswith('.py') and name.startswith(prefix)]
706
707    # Visit all the python test files.
708    for name in python_test_files:
709        # Ensure we error out if we have multiple tests with the same
710        # base name.
711        # Future improvement: find all the places where we work with base
712        # names and convert to full paths.  We have directory structure
713        # to disambiguate these, so we shouldn't need this constraint.
714        if name in configuration.all_tests:
715            raise Exception("Found multiple tests with the name %s" % name)
716        configuration.all_tests.add(name)
717
718        # Run the relevant tests in the python file.
719        visit_file(dir, name)
720
721
722# ======================================== #
723#                                          #
724# Execution of the test driver starts here #
725#                                          #
726# ======================================== #
727
728
729def checkDsymForUUIDIsNotOn():
730    cmd = ["defaults", "read", "com.apple.DebugSymbols"]
731    process = subprocess.Popen(
732        cmd,
733        stdout=subprocess.PIPE,
734        stderr=subprocess.STDOUT)
735    cmd_output = process.stdout.read()
736    output_str = cmd_output.decode("utf-8")
737    if "DBGFileMappedPaths = " in output_str:
738        print("%s =>" % ' '.join(cmd))
739        print(output_str)
740        print(
741            "Disable automatic lookup and caching of dSYMs before running the test suite!")
742        print("Exiting...")
743        sys.exit(0)
744
745
746def exitTestSuite(exitCode=None):
747    # lldb.py does SBDebugger.Initialize().
748    # Call SBDebugger.Terminate() on exit.
749    import lldb
750    lldb.SBDebugger.Terminate()
751    if exitCode:
752        sys.exit(exitCode)
753
754
755def getVersionForSDK(sdk):
756    sdk = str.lower(sdk)
757    full_path = seven.get_command_output('xcrun -sdk %s --show-sdk-path' % sdk)
758    basename = os.path.basename(full_path)
759    basename = os.path.splitext(basename)[0]
760    basename = str.lower(basename)
761    ver = basename.replace(sdk, '')
762    return ver
763
764
765def checkCompiler():
766    # Add some intervention here to sanity check that the compiler requested is sane.
767    # If found not to be an executable program, we abort.
768    c = configuration.compiler
769    if which(c):
770        return
771
772    if not sys.platform.startswith("darwin"):
773        raise Exception(c + " is not a valid compiler")
774
775    pipe = subprocess.Popen(
776        ['xcrun', '-find', c], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
777    cmd_output = pipe.stdout.read()
778    if not cmd_output or "not found" in cmd_output:
779        raise Exception(c + " is not a valid compiler")
780
781    configuration.compiler = cmd_output.split('\n')[0]
782    print("'xcrun -find %s' returning %s" % (c, configuration.compiler))
783
784def canRunLibcxxTests():
785    from lldbsuite.test import lldbplatformutil
786
787    platform = lldbplatformutil.getPlatform()
788
789    if lldbplatformutil.target_is_android() or lldbplatformutil.platformIsDarwin():
790        return True, "libc++ always present"
791
792    if platform == "linux":
793        if os.path.isdir("/usr/include/c++/v1"):
794            return True, "Headers found, let's hope they work"
795        with tempfile.NamedTemporaryFile() as f:
796            cmd = [configuration.compiler, "-xc++", "-stdlib=libc++", "-o", f.name, "-"]
797            p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
798            _, stderr = p.communicate("#include <algorithm>\nint main() {}")
799            if not p.returncode:
800                return True, "Compiling with -stdlib=libc++ works"
801            return False, "Compiling with -stdlib=libc++ fails with the error: %s" % stderr
802
803    return False, "Don't know how to build with libc++ on %s" % platform
804
805def checkLibcxxSupport():
806    result, reason = canRunLibcxxTests()
807    if result:
808        return # libc++ supported
809    if "libc++" in configuration.categories_list:
810        return # libc++ category explicitly requested, let it run.
811    print("Libc++ tests will not be run because: " + reason)
812    configuration.skip_categories.append("libc++")
813
814def canRunLibstdcxxTests():
815    from lldbsuite.test import lldbplatformutil
816
817    platform = lldbplatformutil.getPlatform()
818    if lldbplatformutil.target_is_android():
819        platform = "android"
820    if platform == "linux":
821        return True, "libstdcxx always present"
822    return False, "Don't know how to build with libstdcxx on %s" % platform
823
824def checkLibstdcxxSupport():
825    result, reason = canRunLibstdcxxTests()
826    if result:
827        return # libstdcxx supported
828    if "libstdcxx" in configuration.categories_list:
829        return # libstdcxx category explicitly requested, let it run.
830    print("libstdcxx tests will not be run because: " + reason)
831    configuration.skip_categories.append("libstdcxx")
832
833def canRunWatchpointTests():
834    from lldbsuite.test import lldbplatformutil
835
836    platform = lldbplatformutil.getPlatform()
837    if platform == "netbsd":
838        if os.geteuid() == 0:
839            return True, "root can always write dbregs"
840        try:
841            output = subprocess.check_output(["/sbin/sysctl", "-n",
842              "security.models.extensions.user_set_dbregs"]).decode().strip()
843            if output == "1":
844                return True, "security.models.extensions.user_set_dbregs enabled"
845        except subprocess.CalledProcessError:
846            pass
847        return False, "security.models.extensions.user_set_dbregs disabled"
848    return True, "watchpoint support available"
849
850def checkWatchpointSupport():
851    result, reason = canRunWatchpointTests()
852    if result:
853        return # watchpoints supported
854    if "watchpoint" in configuration.categories_list:
855        return # watchpoint category explicitly requested, let it run.
856    print("watchpoint tests will not be run because: " + reason)
857    configuration.skip_categories.append("watchpoint")
858
859def checkDebugInfoSupport():
860    import lldb
861
862    platform = lldb.selected_platform.GetTriple().split('-')[2]
863    compiler = configuration.compiler
864    skipped = []
865    for cat in test_categories.debug_info_categories:
866        if cat in configuration.categories_list:
867            continue # Category explicitly requested, let it run.
868        if test_categories.is_supported_on_platform(cat, platform, compiler):
869            continue
870        configuration.skip_categories.append(cat)
871        skipped.append(cat)
872    if skipped:
873        print("Skipping following debug info categories:", skipped)
874
875def run_suite():
876    # On MacOS X, check to make sure that domain for com.apple.DebugSymbols defaults
877    # does not exist before proceeding to running the test suite.
878    if sys.platform.startswith("darwin"):
879        checkDsymForUUIDIsNotOn()
880
881    # Start the actions by first parsing the options while setting up the test
882    # directories, followed by setting up the search paths for lldb utilities;
883    # then, we walk the directory trees and collect the tests into our test suite.
884    #
885    parseOptionsAndInitTestdirs()
886
887    setupSysPath()
888
889    import lldbconfig
890    if configuration.capture_path or configuration.replay_path:
891        lldbconfig.INITIALIZE = False
892    import lldb
893
894    if configuration.capture_path:
895        lldb.SBReproducer.Capture(configuration.capture_path)
896        lldb.SBReproducer.SetAutoGenerate(True)
897    elif configuration.replay_path:
898        lldb.SBReproducer.PassiveReplay(configuration.replay_path)
899
900    if not lldbconfig.INITIALIZE:
901        lldb.SBDebugger.Initialize()
902
903    # Use host platform by default.
904    lldb.selected_platform = lldb.SBPlatform.GetHostPlatform()
905
906    # Now we can also import lldbutil
907    from lldbsuite.test import lldbutil
908
909    if configuration.lldb_platform_name:
910        print("Setting up remote platform '%s'" %
911              (configuration.lldb_platform_name))
912        lldb.remote_platform = lldb.SBPlatform(
913            configuration.lldb_platform_name)
914        if not lldb.remote_platform.IsValid():
915            print(
916                "error: unable to create the LLDB platform named '%s'." %
917                (configuration.lldb_platform_name))
918            exitTestSuite(1)
919        if configuration.lldb_platform_url:
920            # We must connect to a remote platform if a LLDB platform URL was
921            # specified
922            print(
923                "Connecting to remote platform '%s' at '%s'..." %
924                (configuration.lldb_platform_name, configuration.lldb_platform_url))
925            platform_connect_options = lldb.SBPlatformConnectOptions(
926                configuration.lldb_platform_url)
927            err = lldb.remote_platform.ConnectRemote(platform_connect_options)
928            if err.Success():
929                print("Connected.")
930            else:
931                print("error: failed to connect to remote platform using URL '%s': %s" % (
932                    configuration.lldb_platform_url, err))
933                exitTestSuite(1)
934        else:
935            configuration.lldb_platform_url = None
936
937    if configuration.lldb_platform_working_dir:
938        print("Setting remote platform working directory to '%s'..." %
939              (configuration.lldb_platform_working_dir))
940        error = lldb.remote_platform.MakeDirectory(
941            configuration.lldb_platform_working_dir, 448)  # 448 = 0o700
942        if error.Fail():
943            raise Exception("making remote directory '%s': %s" % (
944                configuration.lldb_platform_working_dir, error))
945
946        if not lldb.remote_platform.SetWorkingDirectory(
947                configuration.lldb_platform_working_dir):
948            raise Exception("failed to set working directory '%s'" % configuration.lldb_platform_working_dir)
949        lldb.selected_platform = lldb.remote_platform
950    else:
951        lldb.remote_platform = None
952        configuration.lldb_platform_working_dir = None
953        configuration.lldb_platform_url = None
954
955    # Set up the working directory.
956    # Note that it's not dotest's job to clean this directory.
957    lldbutil.mkdir_p(configuration.test_build_dir)
958
959    target_platform = lldb.selected_platform.GetTriple().split('-')[2]
960
961    checkLibcxxSupport()
962    checkLibstdcxxSupport()
963    checkWatchpointSupport()
964    checkDebugInfoSupport()
965
966    # Don't do debugserver tests on anything except OS X.
967    configuration.dont_do_debugserver_test = (
968            "linux" in target_platform or
969            "freebsd" in target_platform or
970            "netbsd" in target_platform or
971            "windows" in target_platform)
972
973    # Don't do lldb-server (llgs) tests on anything except Linux and Windows.
974    configuration.dont_do_llgs_test = not (
975            "linux" in target_platform or
976            "netbsd" in target_platform or
977            "windows" in target_platform)
978
979    for testdir in configuration.testdirs:
980        for (dirpath, dirnames, filenames) in os.walk(testdir):
981            visit('Test', dirpath, filenames)
982
983    #
984    # Now that we have loaded all the test cases, run the whole test suite.
985    #
986
987    # Install the control-c handler.
988    unittest2.signals.installHandler()
989
990    lldbutil.mkdir_p(configuration.sdir_name)
991    os.environ["LLDB_SESSION_DIRNAME"] = configuration.sdir_name
992
993    sys.stderr.write(
994        "\nSession logs for test failures/errors/unexpected successes"
995        " will go into directory '%s'\n" %
996        configuration.sdir_name)
997
998    #
999    # Invoke the default TextTestRunner to run the test suite
1000    #
1001    checkCompiler()
1002
1003    if configuration.verbose:
1004        print("compiler=%s" % configuration.compiler)
1005
1006    # Iterating over all possible architecture and compiler combinations.
1007    configString = "arch=%s compiler=%s" % (configuration.arch,
1008                                            configuration.compiler)
1009
1010    # Output the configuration.
1011    if configuration.verbose:
1012        sys.stderr.write("\nConfiguration: " + configString + "\n")
1013
1014    # First, write out the number of collected test cases.
1015    if configuration.verbose:
1016        sys.stderr.write(configuration.separator + "\n")
1017        sys.stderr.write(
1018            "Collected %d test%s\n\n" %
1019            (configuration.suite.countTestCases(),
1020             configuration.suite.countTestCases() != 1 and "s" or ""))
1021
1022    # Invoke the test runner.
1023    if configuration.count == 1:
1024        result = unittest2.TextTestRunner(
1025            stream=sys.stderr,
1026            verbosity=configuration.verbose,
1027            resultclass=test_result.LLDBTestResult).run(
1028            configuration.suite)
1029    else:
1030        # We are invoking the same test suite more than once.  In this case,
1031        # mark __ignore_singleton__ flag as True so the signleton pattern is
1032        # not enforced.
1033        test_result.LLDBTestResult.__ignore_singleton__ = True
1034        for i in range(configuration.count):
1035
1036            result = unittest2.TextTestRunner(
1037                stream=sys.stderr,
1038                verbosity=configuration.verbose,
1039                resultclass=test_result.LLDBTestResult).run(
1040                configuration.suite)
1041
1042    configuration.failed = not result.wasSuccessful()
1043
1044    if configuration.sdir_has_content and configuration.verbose:
1045        sys.stderr.write(
1046            "Session logs for test failures/errors/unexpected successes"
1047            " can be found in directory '%s'\n" %
1048            configuration.sdir_name)
1049
1050    if configuration.use_categories and len(
1051            configuration.failures_per_category) > 0:
1052        sys.stderr.write("Failures per category:\n")
1053        for category in configuration.failures_per_category:
1054            sys.stderr.write(
1055                "%s - %d\n" %
1056                (category, configuration.failures_per_category[category]))
1057
1058    # Exiting.
1059    exitTestSuite(configuration.failed)
1060
1061if __name__ == "__main__":
1062    print(
1063        __file__ +
1064        " is for use as a module only.  It should not be run as a standalone script.")
1065    sys.exit(-1)
1066