1"""
2A simple testing framework for lldb using python's unit testing framework.
3
4Tests for lldb are written as python scripts which take advantage of the script
5bridging provided by LLDB.framework to interact with lldb core.
6
7A specific naming pattern is followed by the .py script to be recognized as
8a module which implements a test scenario, namely, Test*.py.
9
10To specify the directories where "Test*.py" python test scripts are located,
11you need to pass in a list of directory names.  By default, the current
12working directory is searched if nothing is specified on the command line.
13
14Type:
15
16./dotest.py -h
17
18for available options.
19"""
20
21from __future__ import absolute_import
22from __future__ import print_function
23
24# System modules
25import atexit
26import datetime
27import errno
28import logging
29import os
30import platform
31import re
32import signal
33import subprocess
34import sys
35
36# Third-party modules
37import six
38import unittest2
39
40# LLDB Modules
41import lldbsuite
42from . import configuration
43from . import dotest_args
44from . import lldbtest_config
45from . import test_categories
46from lldbsuite.test_event import formatter
47from . import test_result
48from lldbsuite.test_event.event_builder import EventBuilder
49from ..support import seven
50
51def get_dotest_invocation():
52    return ' '.join(sys.argv)
53
54
55def is_exe(fpath):
56    """Returns true if fpath is an executable."""
57    if fpath == None:
58        return False
59    return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
60
61
62def which(program):
63    """Returns the full path to a program; None otherwise."""
64    fpath, _ = os.path.split(program)
65    if fpath:
66        if is_exe(program):
67            return program
68    else:
69        for path in os.environ["PATH"].split(os.pathsep):
70            exe_file = os.path.join(path, program)
71            if is_exe(exe_file):
72                return exe_file
73    return None
74
75
76def usage(parser):
77    parser.print_help()
78    if configuration.verbose > 0:
79        print("""
80Examples:
81
82This is an example of using the -f option to pinpoint to a specific test class
83and test method to be run:
84
85$ ./dotest.py -f ClassTypesTestCase.test_with_dsym_and_run_command
86----------------------------------------------------------------------
87Collected 1 test
88
89test_with_dsym_and_run_command (TestClassTypes.ClassTypesTestCase)
90Test 'frame variable this' when stopped on a class constructor. ... ok
91
92----------------------------------------------------------------------
93Ran 1 test in 1.396s
94
95OK
96
97And this is an example of using the -p option to run a single file (the filename
98matches the pattern 'ObjC' and it happens to be 'TestObjCMethods.py'):
99
100$ ./dotest.py -v -p ObjC
101----------------------------------------------------------------------
102Collected 4 tests
103
104test_break_with_dsym (TestObjCMethods.FoundationTestCase)
105Test setting objc breakpoints using '_regexp-break' and 'breakpoint set'. ... ok
106test_break_with_dwarf (TestObjCMethods.FoundationTestCase)
107Test setting objc breakpoints using '_regexp-break' and 'breakpoint set'. ... ok
108test_data_type_and_expr_with_dsym (TestObjCMethods.FoundationTestCase)
109Lookup objective-c data types and evaluate expressions. ... ok
110test_data_type_and_expr_with_dwarf (TestObjCMethods.FoundationTestCase)
111Lookup objective-c data types and evaluate expressions. ... ok
112
113----------------------------------------------------------------------
114Ran 4 tests in 16.661s
115
116OK
117
118Running of this script also sets up the LLDB_TEST environment variable so that
119individual test cases can locate their supporting files correctly.  The script
120tries to set up Python's search paths for modules by looking at the build tree
121relative to this script.  See also the '-i' option in the following example.
122
123Finally, this is an example of using the lldb.py module distributed/installed by
124Xcode4 to run against the tests under the 'forward' directory, and with the '-w'
125option to add some delay between two tests.  It uses ARCH=x86_64 to specify that
126as the architecture and CC=clang to specify the compiler used for the test run:
127
128$ PYTHONPATH=/Xcode4/Library/PrivateFrameworks/LLDB.framework/Versions/A/Resources/Python ARCH=x86_64 CC=clang ./dotest.py -v -w -i forward
129
130Session logs for test failures/errors will go into directory '2010-11-11-13_56_16'
131----------------------------------------------------------------------
132Collected 2 tests
133
134test_with_dsym_and_run_command (TestForwardDeclaration.ForwardDeclarationTestCase)
135Display *bar_ptr when stopped on a function with forward declaration of struct bar. ... ok
136test_with_dwarf_and_run_command (TestForwardDeclaration.ForwardDeclarationTestCase)
137Display *bar_ptr when stopped on a function with forward declaration of struct bar. ... ok
138
139----------------------------------------------------------------------
140Ran 2 tests in 5.659s
141
142OK
143
144The 'Session ...' verbiage is recently introduced (see also the '-s' option) to
145notify the directory containing the session logs for test failures or errors.
146In case there is any test failure/error, a similar message is appended at the
147end of the stderr output for your convenience.
148
149ENABLING LOGS FROM TESTS
150
151Option 1:
152
153Writing logs into different files per test case::
154
155$ ./dotest.py --channel "lldb all"
156
157$ ./dotest.py --channel "lldb all" --channel "gdb-remote packets"
158
159These log files are written to:
160
161<session-dir>/<test-id>-host.log (logs from lldb host process)
162<session-dir>/<test-id>-server.log (logs from debugserver/lldb-server)
163<session-dir>/<test-id>-<test-result>.log (console logs)
164
165By default, logs from successful runs are deleted.  Use the --log-success flag
166to create reference logs for debugging.
167
168$ ./dotest.py --log-success
169
170""")
171    sys.exit(0)
172
173
174def parseExclusion(exclusion_file):
175    """Parse an exclusion file, of the following format, where
176       'skip files', 'skip methods', 'xfail files', and 'xfail methods'
177       are the possible list heading values:
178
179       skip files
180       <file name>
181       <file name>
182
183       xfail methods
184       <method name>
185    """
186    excl_type = None
187
188    with open(exclusion_file) as f:
189        for line in f:
190            line = line.strip()
191            if not excl_type:
192                excl_type = line
193                continue
194
195            if not line:
196                excl_type = None
197            elif excl_type == 'skip':
198                if not configuration.skip_tests:
199                    configuration.skip_tests = []
200                configuration.skip_tests.append(line)
201            elif excl_type == 'xfail':
202                if not configuration.xfail_tests:
203                    configuration.xfail_tests = []
204                configuration.xfail_tests.append(line)
205
206
207def parseOptionsAndInitTestdirs():
208    """Initialize the list of directories containing our unittest scripts.
209
210    '-h/--help as the first option prints out usage info and exit the program.
211    """
212
213    do_help = False
214
215    platform_system = platform.system()
216    platform_machine = platform.machine()
217
218    try:
219        parser = dotest_args.create_parser()
220        args = parser.parse_args()
221    except:
222        print(get_dotest_invocation())
223        raise
224
225    if args.unset_env_varnames:
226        for env_var in args.unset_env_varnames:
227            if env_var in os.environ:
228                # From Python Doc: When unsetenv() is supported, deletion of items in os.environ
229                # is automatically translated into a corresponding call to
230                # unsetenv().
231                del os.environ[env_var]
232                # os.unsetenv(env_var)
233
234    if args.set_env_vars:
235        for env_var in args.set_env_vars:
236            parts = env_var.split('=', 1)
237            if len(parts) == 1:
238                os.environ[parts[0]] = ""
239            else:
240                os.environ[parts[0]] = parts[1]
241
242    if args.set_inferior_env_vars:
243        lldbtest_config.inferior_env = ' '.join(args.set_inferior_env_vars)
244
245    # Only print the args if being verbose.
246    if args.v:
247        print(get_dotest_invocation())
248
249    if args.h:
250        do_help = True
251
252    if args.compiler:
253        configuration.compiler = os.path.realpath(args.compiler)
254        if not is_exe(configuration.compiler):
255            configuration.compiler = which(args.compiler)
256        if not is_exe(configuration.compiler):
257            logging.error(
258                    '%s is not a valid compiler executable; aborting...',
259                    args.compiler)
260            sys.exit(-1)
261    else:
262        # Use a compiler appropriate appropriate for the Apple SDK if one was
263        # specified
264        if platform_system == 'Darwin' and args.apple_sdk:
265            configuration.compiler = seven.get_command_output(
266                'xcrun -sdk "%s" -find clang 2> /dev/null' %
267                (args.apple_sdk))
268        else:
269            # 'clang' on ubuntu 14.04 is 3.4 so we try clang-3.5 first
270            candidateCompilers = ['clang-3.5', 'clang', 'gcc']
271            for candidate in candidateCompilers:
272                if which(candidate):
273                    configuration.compiler = candidate
274                    break
275
276    if args.dsymutil:
277        os.environ['DSYMUTIL'] = args.dsymutil
278    elif platform_system == 'Darwin':
279        os.environ['DSYMUTIL'] = seven.get_command_output(
280            'xcrun -find -toolchain default dsymutil')
281
282    if args.filecheck:
283        # The lldb-dotest script produced by the CMake build passes in a path
284        # to a working FileCheck binary. So does one specific Xcode project
285        # target. However, when invoking dotest.py directly, a valid --filecheck
286        # option needs to be given.
287        configuration.filecheck = os.path.abspath(args.filecheck)
288
289    if not configuration.get_filecheck_path():
290        logging.warning('No valid FileCheck executable; some tests may fail...')
291        logging.warning('(Double-check the --filecheck argument to dotest.py)')
292
293    if args.channels:
294        lldbtest_config.channels = args.channels
295
296    if args.log_success:
297        lldbtest_config.log_success = args.log_success
298
299    if args.out_of_tree_debugserver:
300        lldbtest_config.out_of_tree_debugserver = args.out_of_tree_debugserver
301
302    # Set SDKROOT if we are using an Apple SDK
303    if platform_system == 'Darwin' and args.apple_sdk:
304        os.environ['SDKROOT'] = seven.get_command_output(
305            'xcrun --sdk "%s" --show-sdk-path 2> /dev/null' %
306            (args.apple_sdk))
307
308    if args.arch:
309        configuration.arch = args.arch
310        if configuration.arch.startswith(
311                'arm') and platform_system == 'Darwin' and not args.apple_sdk:
312            os.environ['SDKROOT'] = seven.get_command_output(
313                'xcrun --sdk iphoneos.internal --show-sdk-path 2> /dev/null')
314            if not os.path.exists(os.environ['SDKROOT']):
315                os.environ['SDKROOT'] = seven.get_command_output(
316                    'xcrun --sdk iphoneos --show-sdk-path 2> /dev/null')
317    else:
318        configuration.arch = platform_machine
319
320    if args.categoriesList:
321        configuration.categoriesList = set(
322            test_categories.validate(
323                args.categoriesList, False))
324        configuration.useCategories = True
325    else:
326        configuration.categoriesList = []
327
328    if args.skipCategories:
329        configuration.skipCategories += test_categories.validate(
330            args.skipCategories, False)
331
332    if args.E:
333        os.environ['CFLAGS_EXTRAS'] = args.E
334
335    if args.dwarf_version:
336        configuration.dwarf_version = args.dwarf_version
337        # We cannot modify CFLAGS_EXTRAS because they're used in test cases
338        # that explicitly require no debug info.
339        os.environ['CFLAGS'] = '-gdwarf-{}'.format(configuration.dwarf_version)
340
341    if args.d:
342        sys.stdout.write(
343            "Suspending the process %d to wait for debugger to attach...\n" %
344            os.getpid())
345        sys.stdout.flush()
346        os.kill(os.getpid(), signal.SIGSTOP)
347
348    if args.f:
349        if any([x.startswith('-') for x in args.f]):
350            usage(parser)
351        configuration.filters.extend(args.f)
352
353    if args.framework:
354        configuration.lldbFrameworkPath = args.framework
355
356    if args.executable:
357        # lldb executable is passed explicitly
358        lldbtest_config.lldbExec = os.path.realpath(args.executable)
359        if not is_exe(lldbtest_config.lldbExec):
360            lldbtest_config.lldbExec = which(args.executable)
361        if not is_exe(lldbtest_config.lldbExec):
362            logging.error(
363                    '%s is not a valid executable to test; aborting...',
364                    args.executable)
365            sys.exit(-1)
366
367    if args.server:
368        os.environ['LLDB_DEBUGSERVER_PATH'] = args.server
369
370    if args.excluded:
371        for excl_file in args.excluded:
372            parseExclusion(excl_file)
373
374    if args.p:
375        if args.p.startswith('-'):
376            usage(parser)
377        configuration.regexp = args.p
378
379    if args.s:
380        configuration.sdir_name = args.s
381    else:
382        timestamp_started = datetime.datetime.now().strftime("%Y-%m-%d-%H_%M_%S")
383        configuration.sdir_name = os.path.join(os.getcwd(), timestamp_started)
384
385    configuration.session_file_format = args.session_file_format
386
387    if args.t:
388        os.environ['LLDB_COMMAND_TRACE'] = 'YES'
389
390    if args.v:
391        configuration.verbose = 2
392
393    # argparse makes sure we have a number
394    if args.sharp:
395        configuration.count = args.sharp
396
397    if sys.platform.startswith('win32'):
398        os.environ['LLDB_DISABLE_CRASH_DIALOG'] = str(
399            args.disable_crash_dialog)
400        os.environ['LLDB_LAUNCH_INFERIORS_WITHOUT_CONSOLE'] = str(True)
401
402    if do_help:
403        usage(parser)
404
405    if args.results_file:
406        configuration.results_filename = args.results_file
407
408    if args.results_formatter:
409        configuration.results_formatter_name = args.results_formatter
410    if args.results_formatter_options:
411        configuration.results_formatter_options = args.results_formatter_options
412
413    # Default to using the BasicResultsFormatter if no formatter is specified.
414    if configuration.results_formatter_name is None:
415        configuration.results_formatter_name = (
416            "lldbsuite.test_event.formatter.results_formatter.ResultsFormatter")
417
418    # rerun-related arguments
419    configuration.rerun_all_issues = args.rerun_all_issues
420
421    if args.lldb_platform_name:
422        configuration.lldb_platform_name = args.lldb_platform_name
423    if args.lldb_platform_url:
424        configuration.lldb_platform_url = args.lldb_platform_url
425    if args.lldb_platform_working_dir:
426        configuration.lldb_platform_working_dir = args.lldb_platform_working_dir
427    if args.test_build_dir:
428        configuration.test_build_dir = args.test_build_dir
429    if args.lldb_module_cache_dir:
430        configuration.lldb_module_cache_dir = args.lldb_module_cache_dir
431    else:
432        configuration.lldb_module_cache_dir = os.path.join(
433            configuration.test_build_dir, 'module-cache-lldb')
434    if args.clang_module_cache_dir:
435        configuration.clang_module_cache_dir = args.clang_module_cache_dir
436    else:
437        configuration.clang_module_cache_dir = os.path.join(
438            configuration.test_build_dir, 'module-cache-clang')
439
440    os.environ['CLANG_MODULE_CACHE_DIR'] = configuration.clang_module_cache_dir
441
442    # Gather all the dirs passed on the command line.
443    if len(args.args) > 0:
444        configuration.testdirs = [os.path.realpath(os.path.abspath(x)) for x in args.args]
445
446    lldbtest_config.codesign_identity = args.codesign_identity
447
448
449def setupTestResults():
450    """Sets up test results-related objects based on arg settings."""
451    # Setup the results formatter configuration.
452    formatter_config = formatter.FormatterConfig()
453    formatter_config.filename = configuration.results_filename
454    formatter_config.formatter_name = configuration.results_formatter_name
455    formatter_config.formatter_options = (
456        configuration.results_formatter_options)
457
458    # Create the results formatter.
459    formatter_spec = formatter.create_results_formatter(
460        formatter_config)
461    if formatter_spec is not None and formatter_spec.formatter is not None:
462        configuration.results_formatter_object = formatter_spec.formatter
463
464        # Send an initialize message to the formatter.
465        initialize_event = EventBuilder.bare_event("initialize")
466        initialize_event["worker_count"] = 1
467
468        formatter_spec.formatter.handle_event(initialize_event)
469
470        # Make sure we clean up the formatter on shutdown.
471        if formatter_spec.cleanup_func is not None:
472            atexit.register(formatter_spec.cleanup_func)
473
474
475def setupSysPath():
476    """
477    Add LLDB.framework/Resources/Python to the search paths for modules.
478    As a side effect, we also discover the 'lldb' executable and export it here.
479    """
480
481    # Get the directory containing the current script.
482    if "DOTEST_PROFILE" in os.environ and "DOTEST_SCRIPT_DIR" in os.environ:
483        scriptPath = os.environ["DOTEST_SCRIPT_DIR"]
484    else:
485        scriptPath = os.path.dirname(os.path.realpath(__file__))
486    if not scriptPath.endswith('test'):
487        print("This script expects to reside in lldb's test directory.")
488        sys.exit(-1)
489
490    os.environ["LLDB_TEST"] = scriptPath
491
492    # Set up the root build directory.
493    builddir = configuration.test_build_dir
494    if not configuration.test_build_dir:
495        raise Exception("test_build_dir is not set")
496    os.environ["LLDB_BUILD"] = os.path.abspath(configuration.test_build_dir)
497
498    # Set up the LLDB_SRC environment variable, so that the tests can locate
499    # the LLDB source code.
500    os.environ["LLDB_SRC"] = lldbsuite.lldb_root
501
502    pluginPath = os.path.join(scriptPath, 'plugins')
503    toolsLLDBVSCode = os.path.join(scriptPath, 'tools', 'lldb-vscode')
504    toolsLLDBServerPath = os.path.join(scriptPath, 'tools', 'lldb-server')
505
506    # Insert script dir, plugin dir and lldb-server dir to the sys.path.
507    sys.path.insert(0, pluginPath)
508    # Adding test/tools/lldb-vscode to the path makes it easy to
509    # "import lldb_vscode_testcase" from the VSCode tests
510    sys.path.insert(0, toolsLLDBVSCode)
511    # Adding test/tools/lldb-server to the path makes it easy
512    sys.path.insert(0, toolsLLDBServerPath)
513    # to "import lldbgdbserverutils" from the lldb-server tests
514
515    # This is the root of the lldb git/svn checkout
516    # When this changes over to a package instead of a standalone script, this
517    # will be `lldbsuite.lldb_root`
518    lldbRootDirectory = lldbsuite.lldb_root
519
520    # Some of the tests can invoke the 'lldb' command directly.
521    # We'll try to locate the appropriate executable right here.
522
523    # The lldb executable can be set from the command line
524    # if it's not set, we try to find it now
525    # first, we try the environment
526    if not lldbtest_config.lldbExec:
527        # First, you can define an environment variable LLDB_EXEC specifying the
528        # full pathname of the lldb executable.
529        if "LLDB_EXEC" in os.environ:
530            lldbtest_config.lldbExec = os.environ["LLDB_EXEC"]
531
532    if not lldbtest_config.lldbExec:
533        # Last, check the path
534        lldbtest_config.lldbExec = which('lldb')
535
536    if lldbtest_config.lldbExec and not is_exe(lldbtest_config.lldbExec):
537        print(
538            "'{}' is not a path to a valid executable".format(
539                lldbtest_config.lldbExec))
540        lldbtest_config.lldbExec = None
541
542    if not lldbtest_config.lldbExec:
543        print("The 'lldb' executable cannot be located.  Some of the tests may not be run as a result.")
544        sys.exit(-1)
545
546    # confusingly, this is the "bin" directory
547    lldbLibDir = os.path.dirname(lldbtest_config.lldbExec)
548    os.environ["LLDB_LIB_DIR"] = lldbLibDir
549    lldbImpLibDir = os.path.join(
550        lldbLibDir,
551        '..',
552        'lib') if sys.platform.startswith('win32') else lldbLibDir
553    os.environ["LLDB_IMPLIB_DIR"] = lldbImpLibDir
554    print("LLDB library dir:", os.environ["LLDB_LIB_DIR"])
555    print("LLDB import library dir:", os.environ["LLDB_IMPLIB_DIR"])
556    os.system('%s -v' % lldbtest_config.lldbExec)
557
558    lldbDir = os.path.dirname(lldbtest_config.lldbExec)
559
560    lldbVSCodeExec = os.path.join(lldbDir, "lldb-vscode")
561    if is_exe(lldbVSCodeExec):
562        os.environ["LLDBVSCODE_EXEC"] = lldbVSCodeExec
563    else:
564        if not configuration.shouldSkipBecauseOfCategories(["lldb-vscode"]):
565            print(
566                "The 'lldb-vscode' executable cannot be located.  The lldb-vscode tests can not be run as a result.")
567            configuration.skipCategories.append("lldb-vscode")
568
569    lldbPythonDir = None  # The directory that contains 'lldb/__init__.py'
570    if not configuration.lldbFrameworkPath and os.path.exists(os.path.join(lldbLibDir, "LLDB.framework")):
571        configuration.lldbFrameworkPath = os.path.join(lldbLibDir, "LLDB.framework")
572    if configuration.lldbFrameworkPath:
573        lldbtest_config.lldbFrameworkPath = configuration.lldbFrameworkPath
574        candidatePath = os.path.join(
575            configuration.lldbFrameworkPath, 'Resources', 'Python')
576        if os.path.isfile(os.path.join(candidatePath, 'lldb/__init__.py')):
577            lldbPythonDir = candidatePath
578        if not lldbPythonDir:
579            print(
580                'Resources/Python/lldb/__init__.py was not found in ' +
581                configuration.lldbFrameworkPath)
582            sys.exit(-1)
583    else:
584        # If our lldb supports the -P option, use it to find the python path:
585        init_in_python_dir = os.path.join('lldb', '__init__.py')
586
587        lldb_dash_p_result = subprocess.check_output(
588            [lldbtest_config.lldbExec, "-P"], stderr=subprocess.STDOUT, universal_newlines=True)
589
590        if lldb_dash_p_result and not lldb_dash_p_result.startswith(
591                ("<", "lldb: invalid option:")) and not lldb_dash_p_result.startswith("Traceback"):
592            lines = lldb_dash_p_result.splitlines()
593
594            # Workaround for readline vs libedit issue on FreeBSD.  If stdout
595            # is not a terminal Python executes
596            #     rl_variable_bind ("enable-meta-key", "off");
597            # This produces a warning with FreeBSD's libedit because the
598            # enable-meta-key variable is unknown.  Not an issue on Apple
599            # because cpython commit f0ab6f9f0603 added a #ifndef __APPLE__
600            # around the call.  See http://bugs.python.org/issue19884 for more
601            # information.  For now we just discard the warning output.
602            if len(lines) >= 1 and lines[0].startswith(
603                    "bind: Invalid command"):
604                lines.pop(0)
605
606            # Taking the last line because lldb outputs
607            # 'Cannot read termcap database;\nusing dumb terminal settings.\n'
608            # before the path
609            if len(lines) >= 1 and os.path.isfile(
610                    os.path.join(lines[-1], init_in_python_dir)):
611                lldbPythonDir = lines[-1]
612                if "freebsd" in sys.platform or "linux" in sys.platform:
613                    os.environ['LLDB_LIB_DIR'] = os.path.join(
614                        lldbPythonDir, '..', '..')
615
616        if not lldbPythonDir:
617            print(
618                "Unable to load lldb extension module.  Possible reasons for this include:")
619            print("  1) LLDB was built with LLDB_DISABLE_PYTHON=1")
620            print(
621                "  2) PYTHONPATH and PYTHONHOME are not set correctly.  PYTHONHOME should refer to")
622            print(
623                "     the version of Python that LLDB built and linked against, and PYTHONPATH")
624            print(
625                "     should contain the Lib directory for the same python distro, as well as the")
626            print("     location of LLDB\'s site-packages folder.")
627            print(
628                "  3) A different version of Python than that which was built against is exported in")
629            print("     the system\'s PATH environment variable, causing conflicts.")
630            print(
631                "  4) The executable '%s' could not be found.  Please check " %
632                lldbtest_config.lldbExec)
633            print("     that it exists and is executable.")
634
635    if lldbPythonDir:
636        lldbPythonDir = os.path.normpath(lldbPythonDir)
637        # Some of the code that uses this path assumes it hasn't resolved the Versions... link.
638        # If the path we've constructed looks like that, then we'll strip out
639        # the Versions/A part.
640        (before, frameWithVersion, after) = lldbPythonDir.rpartition(
641            "LLDB.framework/Versions/A")
642        if frameWithVersion != "":
643            lldbPythonDir = before + "LLDB.framework" + after
644
645        lldbPythonDir = os.path.abspath(lldbPythonDir)
646
647        # If tests need to find LLDB_FRAMEWORK, now they can do it
648        os.environ["LLDB_FRAMEWORK"] = os.path.dirname(
649            os.path.dirname(lldbPythonDir))
650
651        # This is to locate the lldb.py module.  Insert it right after
652        # sys.path[0].
653        sys.path[1:1] = [lldbPythonDir]
654
655
656def visit_file(dir, name):
657    # Try to match the regexp pattern, if specified.
658    if configuration.regexp:
659        if not re.search(configuration.regexp, name):
660            # We didn't match the regex, we're done.
661            return
662
663    if configuration.skip_tests:
664        for file_regexp in configuration.skip_tests:
665            if re.search(file_regexp, name):
666                return
667
668    # We found a match for our test.  Add it to the suite.
669
670    # Update the sys.path first.
671    if not sys.path.count(dir):
672        sys.path.insert(0, dir)
673    base = os.path.splitext(name)[0]
674
675    # Thoroughly check the filterspec against the base module and admit
676    # the (base, filterspec) combination only when it makes sense.
677
678    def check(obj, parts):
679        for part in parts:
680            try:
681                parent, obj = obj, getattr(obj, part)
682            except AttributeError:
683                # The filterspec has failed.
684                return False
685        return True
686
687    module = __import__(base)
688
689    def iter_filters():
690        for filterspec in configuration.filters:
691            parts = filterspec.split('.')
692            if check(module, parts):
693                yield filterspec
694            elif parts[0] == base and len(parts) > 1 and check(module, parts[1:]):
695                yield '.'.join(parts[1:])
696            else:
697                for key,value in module.__dict__.items():
698                    if check(value, parts):
699                        yield key + '.' + filterspec
700
701    filtered = False
702    for filterspec in iter_filters():
703        filtered = True
704        print("adding filter spec %s to module %s" % (filterspec, repr(module)))
705        tests = unittest2.defaultTestLoader.loadTestsFromName(filterspec, module)
706        configuration.suite.addTests(tests)
707
708    # Forgo this module if the (base, filterspec) combo is invalid
709    if configuration.filters and not filtered:
710        return
711
712    if not filtered:
713        # Add the entire file's worth of tests since we're not filtered.
714        # Also the fail-over case when the filterspec branch
715        # (base, filterspec) combo doesn't make sense.
716        configuration.suite.addTests(
717            unittest2.defaultTestLoader.loadTestsFromName(base))
718
719
720def visit(prefix, dir, names):
721    """Visitor function for os.path.walk(path, visit, arg)."""
722
723    dir_components = set(dir.split(os.sep))
724    excluded_components = set(['.svn', '.git'])
725    if dir_components.intersection(excluded_components):
726        return
727
728    # Gather all the Python test file names that follow the Test*.py pattern.
729    python_test_files = [
730        name
731        for name in names
732        if name.endswith('.py') and name.startswith(prefix)]
733
734    # Visit all the python test files.
735    for name in python_test_files:
736        try:
737            # Ensure we error out if we have multiple tests with the same
738            # base name.
739            # Future improvement: find all the places where we work with base
740            # names and convert to full paths.  We have directory structure
741            # to disambiguate these, so we shouldn't need this constraint.
742            if name in configuration.all_tests:
743                raise Exception("Found multiple tests with the name %s" % name)
744            configuration.all_tests.add(name)
745
746            # Run the relevant tests in the python file.
747            visit_file(dir, name)
748        except Exception as ex:
749            # Convert this exception to a test event error for the file.
750            test_filename = os.path.abspath(os.path.join(dir, name))
751            if configuration.results_formatter_object is not None:
752                # Grab the backtrace for the exception.
753                import traceback
754                backtrace = traceback.format_exc()
755
756                # Generate the test event.
757                configuration.results_formatter_object.handle_event(
758                    EventBuilder.event_for_job_test_add_error(
759                        test_filename, ex, backtrace))
760            raise
761
762
763def disabledynamics():
764    import lldb
765    ci = lldb.DBG.GetCommandInterpreter()
766    res = lldb.SBCommandReturnObject()
767    ci.HandleCommand(
768        "setting set target.prefer-dynamic-value no-dynamic-values",
769        res,
770        False)
771    if not res.Succeeded():
772        raise Exception('disabling dynamic type support failed')
773
774
775# ======================================== #
776#                                          #
777# Execution of the test driver starts here #
778#                                          #
779# ======================================== #
780
781
782def checkDsymForUUIDIsNotOn():
783    cmd = ["defaults", "read", "com.apple.DebugSymbols"]
784    process = subprocess.Popen(
785        cmd,
786        stdout=subprocess.PIPE,
787        stderr=subprocess.STDOUT)
788    cmd_output = process.stdout.read()
789    output_str = cmd_output.decode("utf-8")
790    if "DBGFileMappedPaths = " in output_str:
791        print("%s =>" % ' '.join(cmd))
792        print(output_str)
793        print(
794            "Disable automatic lookup and caching of dSYMs before running the test suite!")
795        print("Exiting...")
796        sys.exit(0)
797
798
799def exitTestSuite(exitCode=None):
800    import lldb
801    lldb.SBDebugger.Terminate()
802    if exitCode:
803        sys.exit(exitCode)
804
805
806def getVersionForSDK(sdk):
807    sdk = str.lower(sdk)
808    full_path = seven.get_command_output('xcrun -sdk %s --show-sdk-path' % sdk)
809    basename = os.path.basename(full_path)
810    basename = os.path.splitext(basename)[0]
811    basename = str.lower(basename)
812    ver = basename.replace(sdk, '')
813    return ver
814
815
816def setDefaultTripleForPlatform():
817    if configuration.lldb_platform_name == 'ios-simulator':
818        triple_str = 'x86_64-apple-ios%s' % (
819            getVersionForSDK('iphonesimulator'))
820        os.environ['TRIPLE'] = triple_str
821        return {'TRIPLE': triple_str}
822    return {}
823
824
825def checkCompiler():
826    # Add some intervention here to sanity check that the compiler requested is sane.
827    # If found not to be an executable program, we abort.
828    c = configuration.compiler
829    if which(c):
830        return
831
832    if not sys.platform.startswith("darwin"):
833        raise Exception(c + " is not a valid compiler")
834
835    pipe = subprocess.Popen(
836        ['xcrun', '-find', c], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
837    cmd_output = pipe.stdout.read()
838    if not cmd_output or "not found" in cmd_output:
839        raise Exception(c + " is not a valid compiler")
840
841    configuration.compiler = cmd_output.split('\n')[0]
842    print("'xcrun -find %s' returning %s" % (c, configuration.compiler))
843
844def canRunLibcxxTests():
845    from lldbsuite.test import lldbplatformutil
846
847    platform = lldbplatformutil.getPlatform()
848
849    if lldbplatformutil.target_is_android() or lldbplatformutil.platformIsDarwin():
850        return True, "libc++ always present"
851
852    if platform == "linux":
853        if not os.path.isdir("/usr/include/c++/v1"):
854            return False, "Unable to find libc++ installation"
855        return True, "Headers found, let's hope they work"
856
857    return False, "Don't know how to build with libc++ on %s" % platform
858
859def checkLibcxxSupport():
860    result, reason = canRunLibcxxTests()
861    if result:
862        return # libc++ supported
863    if "libc++" in configuration.categoriesList:
864        return # libc++ category explicitly requested, let it run.
865    print("Libc++ tests will not be run because: " + reason)
866    configuration.skipCategories.append("libc++")
867
868def canRunLibstdcxxTests():
869    from lldbsuite.test import lldbplatformutil
870
871    platform = lldbplatformutil.getPlatform()
872    if lldbplatformutil.target_is_android():
873        platform = "android"
874    if platform == "linux":
875        return True, "libstdcxx always present"
876    return False, "Don't know how to build with libstdcxx on %s" % platform
877
878def checkLibstdcxxSupport():
879    result, reason = canRunLibstdcxxTests()
880    if result:
881        return # libstdcxx supported
882    if "libstdcxx" in configuration.categoriesList:
883        return # libstdcxx category explicitly requested, let it run.
884    print("libstdcxx tests will not be run because: " + reason)
885    configuration.skipCategories.append("libstdcxx")
886
887def canRunWatchpointTests():
888    from lldbsuite.test import lldbplatformutil
889
890    platform = lldbplatformutil.getPlatform()
891    if platform == "netbsd":
892        if os.geteuid() == 0:
893            return True, "root can always write dbregs"
894        try:
895            output = subprocess.check_output(["/sbin/sysctl", "-n",
896              "security.models.extensions.user_set_dbregs"]).decode().strip()
897            if output == "1":
898                return True, "security.models.extensions.user_set_dbregs enabled"
899        except subprocess.CalledProcessError:
900            pass
901        return False, "security.models.extensions.user_set_dbregs disabled"
902    return True, "watchpoint support available"
903
904def checkWatchpointSupport():
905    result, reason = canRunWatchpointTests()
906    if result:
907        return # watchpoints supported
908    if "watchpoint" in configuration.categoriesList:
909        return # watchpoint category explicitly requested, let it run.
910    print("watchpoint tests will not be run because: " + reason)
911    configuration.skipCategories.append("watchpoint")
912
913def checkDebugInfoSupport():
914    import lldb
915
916    platform = lldb.DBG.GetSelectedPlatform().GetTriple().split('-')[2]
917    compiler = configuration.compiler
918    skipped = []
919    for cat in test_categories.debug_info_categories:
920        if cat in configuration.categoriesList:
921            continue # Category explicitly requested, let it run.
922        if test_categories.is_supported_on_platform(cat, platform, compiler):
923            continue
924        configuration.skipCategories.append(cat)
925        skipped.append(cat)
926    if skipped:
927        print("Skipping following debug info categories:", skipped)
928
929def run_suite():
930    # On MacOS X, check to make sure that domain for com.apple.DebugSymbols defaults
931    # does not exist before proceeding to running the test suite.
932    if sys.platform.startswith("darwin"):
933        checkDsymForUUIDIsNotOn()
934
935    # Start the actions by first parsing the options while setting up the test
936    # directories, followed by setting up the search paths for lldb utilities;
937    # then, we walk the directory trees and collect the tests into our test suite.
938    #
939    parseOptionsAndInitTestdirs()
940
941    # Setup test results (test results formatter and output handling).
942    setupTestResults()
943
944    setupSysPath()
945
946
947    # For the time being, let's bracket the test runner within the
948    # lldb.SBDebugger.Initialize()/Terminate() pair.
949    import lldb
950
951    # Now we can also import lldbutil
952    from lldbsuite.test import lldbutil
953
954    # Create a singleton SBDebugger in the lldb namespace.
955    lldb.DBG = lldb.SBDebugger.Create()
956
957    if configuration.lldb_platform_name:
958        print("Setting up remote platform '%s'" %
959              (configuration.lldb_platform_name))
960        lldb.remote_platform = lldb.SBPlatform(
961            configuration.lldb_platform_name)
962        if not lldb.remote_platform.IsValid():
963            print(
964                "error: unable to create the LLDB platform named '%s'." %
965                (configuration.lldb_platform_name))
966            exitTestSuite(1)
967        if configuration.lldb_platform_url:
968            # We must connect to a remote platform if a LLDB platform URL was
969            # specified
970            print(
971                "Connecting to remote platform '%s' at '%s'..." %
972                (configuration.lldb_platform_name, configuration.lldb_platform_url))
973            platform_connect_options = lldb.SBPlatformConnectOptions(
974                configuration.lldb_platform_url)
975            err = lldb.remote_platform.ConnectRemote(platform_connect_options)
976            if err.Success():
977                print("Connected.")
978            else:
979                print("error: failed to connect to remote platform using URL '%s': %s" % (
980                    configuration.lldb_platform_url, err))
981                exitTestSuite(1)
982        else:
983            configuration.lldb_platform_url = None
984
985    platform_changes = setDefaultTripleForPlatform()
986    first = True
987    for key in platform_changes:
988        if first:
989            print("Environment variables setup for platform support:")
990            first = False
991        print("%s = %s" % (key, platform_changes[key]))
992
993    if configuration.lldb_platform_working_dir:
994        print("Setting remote platform working directory to '%s'..." %
995              (configuration.lldb_platform_working_dir))
996        error = lldb.remote_platform.MakeDirectory(
997            configuration.lldb_platform_working_dir, 448)  # 448 = 0o700
998        if error.Fail():
999            raise Exception("making remote directory '%s': %s" % (
1000                configuration.lldb_platform_working_dir, error))
1001
1002        if not lldb.remote_platform.SetWorkingDirectory(
1003                configuration.lldb_platform_working_dir):
1004            raise Exception("failed to set working directory '%s'" % configuration.lldb_platform_working_dir)
1005        lldb.DBG.SetSelectedPlatform(lldb.remote_platform)
1006    else:
1007        lldb.remote_platform = None
1008        configuration.lldb_platform_working_dir = None
1009        configuration.lldb_platform_url = None
1010
1011    # Set up the working directory.
1012    # Note that it's not dotest's job to clean this directory.
1013    build_dir = configuration.test_build_dir
1014    lldbutil.mkdir_p(build_dir)
1015
1016    target_platform = lldb.DBG.GetSelectedPlatform().GetTriple().split('-')[2]
1017
1018    checkLibcxxSupport()
1019    checkLibstdcxxSupport()
1020    checkWatchpointSupport()
1021    checkDebugInfoSupport()
1022
1023    # Don't do debugserver tests on anything except OS X.
1024    configuration.dont_do_debugserver_test = "linux" in target_platform or "freebsd" in target_platform or "windows" in target_platform
1025
1026    # Don't do lldb-server (llgs) tests on anything except Linux and Windows.
1027    configuration.dont_do_llgs_test = not ("linux" in target_platform) and not ("windows" in target_platform)
1028
1029    # Collect tests from the specified testing directories. If a test
1030    # subdirectory filter is explicitly specified, limit the search to that
1031    # subdirectory.
1032    exclusive_test_subdir = configuration.get_absolute_path_to_exclusive_test_subdir()
1033    if exclusive_test_subdir:
1034        dirs_to_search = [exclusive_test_subdir]
1035    else:
1036        dirs_to_search = configuration.testdirs
1037    for testdir in dirs_to_search:
1038        for (dirpath, dirnames, filenames) in os.walk(testdir):
1039            visit('Test', dirpath, filenames)
1040
1041    #
1042    # Now that we have loaded all the test cases, run the whole test suite.
1043    #
1044
1045    # Disable default dynamic types for testing purposes
1046    disabledynamics()
1047
1048    # Install the control-c handler.
1049    unittest2.signals.installHandler()
1050
1051    lldbutil.mkdir_p(configuration.sdir_name)
1052    os.environ["LLDB_SESSION_DIRNAME"] = configuration.sdir_name
1053
1054    sys.stderr.write(
1055        "\nSession logs for test failures/errors/unexpected successes"
1056        " will go into directory '%s'\n" %
1057        configuration.sdir_name)
1058    sys.stderr.write("Command invoked: %s\n" % get_dotest_invocation())
1059
1060    #
1061    # Invoke the default TextTestRunner to run the test suite
1062    #
1063    checkCompiler()
1064
1065    if configuration.verbose:
1066        print("compiler=%s" % configuration.compiler)
1067
1068    # Iterating over all possible architecture and compiler combinations.
1069    os.environ["ARCH"] = configuration.arch
1070    os.environ["CC"] = configuration.compiler
1071    configString = "arch=%s compiler=%s" % (configuration.arch,
1072                                            configuration.compiler)
1073
1074    # Output the configuration.
1075    if configuration.verbose:
1076        sys.stderr.write("\nConfiguration: " + configString + "\n")
1077
1078    # First, write out the number of collected test cases.
1079    if configuration.verbose:
1080        sys.stderr.write(configuration.separator + "\n")
1081        sys.stderr.write(
1082            "Collected %d test%s\n\n" %
1083            (configuration.suite.countTestCases(),
1084             configuration.suite.countTestCases() != 1 and "s" or ""))
1085
1086    # Invoke the test runner.
1087    if configuration.count == 1:
1088        result = unittest2.TextTestRunner(
1089            stream=sys.stderr,
1090            verbosity=configuration.verbose,
1091            resultclass=test_result.LLDBTestResult).run(
1092            configuration.suite)
1093    else:
1094        # We are invoking the same test suite more than once.  In this case,
1095        # mark __ignore_singleton__ flag as True so the signleton pattern is
1096        # not enforced.
1097        test_result.LLDBTestResult.__ignore_singleton__ = True
1098        for i in range(configuration.count):
1099
1100            result = unittest2.TextTestRunner(
1101                stream=sys.stderr,
1102                verbosity=configuration.verbose,
1103                resultclass=test_result.LLDBTestResult).run(
1104                configuration.suite)
1105
1106    configuration.failed = not result.wasSuccessful()
1107
1108    if configuration.sdir_has_content and configuration.verbose:
1109        sys.stderr.write(
1110            "Session logs for test failures/errors/unexpected successes"
1111            " can be found in directory '%s'\n" %
1112            configuration.sdir_name)
1113
1114    if configuration.useCategories and len(
1115            configuration.failuresPerCategory) > 0:
1116        sys.stderr.write("Failures per category:\n")
1117        for category in configuration.failuresPerCategory:
1118            sys.stderr.write(
1119                "%s - %d\n" %
1120                (category, configuration.failuresPerCategory[category]))
1121
1122    # Exiting.
1123    exitTestSuite(configuration.failed)
1124
1125if __name__ == "__main__":
1126    print(
1127        __file__ +
1128        " is for use as a module only.  It should not be run as a standalone script.")
1129    sys.exit(-1)
1130