1""" 2A simple testing framework for lldb using python's unit testing framework. 3 4Tests for lldb are written as python scripts which take advantage of the script 5bridging provided by LLDB.framework to interact with lldb core. 6 7A specific naming pattern is followed by the .py script to be recognized as 8a module which implements a test scenario, namely, Test*.py. 9 10To specify the directories where "Test*.py" python test scripts are located, 11you need to pass in a list of directory names. By default, the current 12working directory is searched if nothing is specified on the command line. 13 14Type: 15 16./dotest.py -h 17 18for available options. 19""" 20 21from __future__ import absolute_import 22from __future__ import print_function 23 24# System modules 25import atexit 26import os 27import errno 28import platform 29import re 30import signal 31import socket 32import subprocess 33import sys 34 35# Third-party modules 36import six 37import unittest2 38 39# LLDB Modules 40import lldbsuite 41from . import configuration 42from . import dotest_args 43from . import lldbtest_config 44from . import test_categories 45from lldbsuite.test_event import formatter 46from . import test_result 47from lldbsuite.test_event.event_builder import EventBuilder 48from ..support import seven 49 50 51def is_exe(fpath): 52 """Returns true if fpath is an executable.""" 53 return os.path.isfile(fpath) and os.access(fpath, os.X_OK) 54 55 56def which(program): 57 """Returns the full path to a program; None otherwise.""" 58 fpath, fname = os.path.split(program) 59 if fpath: 60 if is_exe(program): 61 return program 62 else: 63 for path in os.environ["PATH"].split(os.pathsep): 64 exe_file = os.path.join(path, program) 65 if is_exe(exe_file): 66 return exe_file 67 return None 68 69 70class _WritelnDecorator(object): 71 """Used to decorate file-like objects with a handy 'writeln' method""" 72 73 def __init__(self, stream): 74 self.stream = stream 75 76 def __getattr__(self, attr): 77 if attr in ('stream', '__getstate__'): 78 raise AttributeError(attr) 79 return getattr(self.stream, attr) 80 81 def writeln(self, arg=None): 82 if arg: 83 self.write(arg) 84 self.write('\n') # text-mode streams translate to \r\n if needed 85 86# 87# Global variables: 88# 89 90 91def usage(parser): 92 parser.print_help() 93 if configuration.verbose > 0: 94 print(""" 95Examples: 96 97This is an example of using the -f option to pinpoint to a specific test class 98and test method to be run: 99 100$ ./dotest.py -f ClassTypesTestCase.test_with_dsym_and_run_command 101---------------------------------------------------------------------- 102Collected 1 test 103 104test_with_dsym_and_run_command (TestClassTypes.ClassTypesTestCase) 105Test 'frame variable this' when stopped on a class constructor. ... ok 106 107---------------------------------------------------------------------- 108Ran 1 test in 1.396s 109 110OK 111 112And this is an example of using the -p option to run a single file (the filename 113matches the pattern 'ObjC' and it happens to be 'TestObjCMethods.py'): 114 115$ ./dotest.py -v -p ObjC 116---------------------------------------------------------------------- 117Collected 4 tests 118 119test_break_with_dsym (TestObjCMethods.FoundationTestCase) 120Test setting objc breakpoints using '_regexp-break' and 'breakpoint set'. ... ok 121test_break_with_dwarf (TestObjCMethods.FoundationTestCase) 122Test setting objc breakpoints using '_regexp-break' and 'breakpoint set'. ... ok 123test_data_type_and_expr_with_dsym (TestObjCMethods.FoundationTestCase) 124Lookup objective-c data types and evaluate expressions. ... ok 125test_data_type_and_expr_with_dwarf (TestObjCMethods.FoundationTestCase) 126Lookup objective-c data types and evaluate expressions. ... ok 127 128---------------------------------------------------------------------- 129Ran 4 tests in 16.661s 130 131OK 132 133Running of this script also sets up the LLDB_TEST environment variable so that 134individual test cases can locate their supporting files correctly. The script 135tries to set up Python's search paths for modules by looking at the build tree 136relative to this script. See also the '-i' option in the following example. 137 138Finally, this is an example of using the lldb.py module distributed/installed by 139Xcode4 to run against the tests under the 'forward' directory, and with the '-w' 140option to add some delay between two tests. It uses ARCH=x86_64 to specify that 141as the architecture and CC=clang to specify the compiler used for the test run: 142 143$ PYTHONPATH=/Xcode4/Library/PrivateFrameworks/LLDB.framework/Versions/A/Resources/Python ARCH=x86_64 CC=clang ./dotest.py -v -w -i forward 144 145Session logs for test failures/errors will go into directory '2010-11-11-13_56_16' 146---------------------------------------------------------------------- 147Collected 2 tests 148 149test_with_dsym_and_run_command (TestForwardDeclaration.ForwardDeclarationTestCase) 150Display *bar_ptr when stopped on a function with forward declaration of struct bar. ... ok 151test_with_dwarf_and_run_command (TestForwardDeclaration.ForwardDeclarationTestCase) 152Display *bar_ptr when stopped on a function with forward declaration of struct bar. ... ok 153 154---------------------------------------------------------------------- 155Ran 2 tests in 5.659s 156 157OK 158 159The 'Session ...' verbiage is recently introduced (see also the '-s' option) to 160notify the directory containing the session logs for test failures or errors. 161In case there is any test failure/error, a similar message is appended at the 162end of the stderr output for your convenience. 163 164ENABLING LOGS FROM TESTS 165 166Option 1: 167 168Writing logs into different files per test case:: 169 170This option is particularly useful when multiple dotest instances are created 171by dosep.py 172 173$ ./dotest.py --channel "lldb all" 174 175$ ./dotest.py --channel "lldb all" --channel "gdb-remote packets" 176 177These log files are written to: 178 179<session-dir>/<test-id>-host.log (logs from lldb host process) 180<session-dir>/<test-id>-server.log (logs from debugserver/lldb-server) 181<session-dir>/<test-id>-<test-result>.log (console logs) 182 183By default, logs from successful runs are deleted. Use the --log-success flag 184to create reference logs for debugging. 185 186$ ./dotest.py --log-success 187 188Option 2: (DEPRECATED) 189 190The following options can only enable logs from the host lldb process. 191Only categories from the "lldb" or "gdb-remote" channels can be enabled 192They also do not automatically enable logs in locally running debug servers. 193Also, logs from all test case are written into each log file 194 195o LLDB_LOG: if defined, specifies the log file pathname for the 'lldb' subsystem 196 with a default option of 'event process' if LLDB_LOG_OPTION is not defined. 197 198o GDB_REMOTE_LOG: if defined, specifies the log file pathname for the 199 'process.gdb-remote' subsystem with a default option of 'packets' if 200 GDB_REMOTE_LOG_OPTION is not defined. 201 202""") 203 sys.exit(0) 204 205 206def parseExclusion(exclusion_file): 207 """Parse an exclusion file, of the following format, where 208 'skip files', 'skip methods', 'xfail files', and 'xfail methods' 209 are the possible list heading values: 210 211 skip files 212 <file name> 213 <file name> 214 215 xfail methods 216 <method name> 217 """ 218 excl_type = None 219 220 with open(exclusion_file) as f: 221 for line in f: 222 line = line.strip() 223 if not excl_type: 224 excl_type = line 225 continue 226 227 if not line: 228 excl_type = None 229 elif excl_type == 'skip': 230 if not configuration.skip_tests: 231 configuration.skip_tests = [] 232 configuration.skip_tests.append(line) 233 elif excl_type == 'xfail': 234 if not configuration.xfail_tests: 235 configuration.xfail_tests = [] 236 configuration.xfail_tests.append(line) 237 238 239def parseOptionsAndInitTestdirs(): 240 """Initialize the list of directories containing our unittest scripts. 241 242 '-h/--help as the first option prints out usage info and exit the program. 243 """ 244 245 do_help = False 246 247 platform_system = platform.system() 248 platform_machine = platform.machine() 249 250 parser = dotest_args.create_parser() 251 args = dotest_args.parse_args(parser, sys.argv[1:]) 252 253 if args.unset_env_varnames: 254 for env_var in args.unset_env_varnames: 255 if env_var in os.environ: 256 # From Python Doc: When unsetenv() is supported, deletion of items in os.environ 257 # is automatically translated into a corresponding call to 258 # unsetenv(). 259 del os.environ[env_var] 260 # os.unsetenv(env_var) 261 262 if args.set_env_vars: 263 for env_var in args.set_env_vars: 264 parts = env_var.split('=', 1) 265 if len(parts) == 1: 266 os.environ[parts[0]] = "" 267 else: 268 os.environ[parts[0]] = parts[1] 269 270 # only print the args if being verbose (and parsable is off) 271 if args.v and not args.q: 272 print(sys.argv) 273 274 if args.h: 275 do_help = True 276 277 if args.compilers: 278 configuration.compilers = args.compilers 279 else: 280 # Use a compiler appropriate appropriate for the Apple SDK if one was 281 # specified 282 if platform_system == 'Darwin' and args.apple_sdk: 283 configuration.compilers = [ 284 seven.get_command_output( 285 'xcrun -sdk "%s" -find clang 2> /dev/null' % 286 (args.apple_sdk))] 287 else: 288 # 'clang' on ubuntu 14.04 is 3.4 so we try clang-3.5 first 289 candidateCompilers = ['clang-3.5', 'clang', 'gcc'] 290 for candidate in candidateCompilers: 291 if which(candidate): 292 configuration.compilers = [candidate] 293 break 294 295 if args.channels: 296 lldbtest_config.channels = args.channels 297 298 if args.log_success: 299 lldbtest_config.log_success = args.log_success 300 301 # Set SDKROOT if we are using an Apple SDK 302 if platform_system == 'Darwin' and args.apple_sdk: 303 os.environ['SDKROOT'] = seven.get_command_output( 304 'xcrun --sdk "%s" --show-sdk-path 2> /dev/null' % 305 (args.apple_sdk)) 306 307 if args.archs: 308 configuration.archs = args.archs 309 for arch in configuration.archs: 310 if arch.startswith( 311 'arm') and platform_system == 'Darwin' and not args.apple_sdk: 312 os.environ['SDKROOT'] = seven.get_command_output( 313 'xcrun --sdk iphoneos.internal --show-sdk-path 2> /dev/null') 314 if not os.path.exists(os.environ['SDKROOT']): 315 os.environ['SDKROOT'] = seven.get_command_output( 316 'xcrun --sdk iphoneos --show-sdk-path 2> /dev/null') 317 else: 318 configuration.archs = [platform_machine] 319 320 if args.categoriesList: 321 configuration.categoriesList = set( 322 test_categories.validate( 323 args.categoriesList, False)) 324 configuration.useCategories = True 325 else: 326 configuration.categoriesList = [] 327 328 if args.skipCategories: 329 configuration.skipCategories = test_categories.validate( 330 args.skipCategories, False) 331 332 if args.E: 333 cflags_extras = args.E 334 os.environ['CFLAGS_EXTRAS'] = cflags_extras 335 336 if args.d: 337 sys.stdout.write( 338 "Suspending the process %d to wait for debugger to attach...\n" % 339 os.getpid()) 340 sys.stdout.flush() 341 os.kill(os.getpid(), signal.SIGSTOP) 342 343 if args.f: 344 if any([x.startswith('-') for x in args.f]): 345 usage(parser) 346 configuration.filters.extend(args.f) 347 # Shut off multiprocessing mode when additional filters are specified. 348 # The rational is that the user is probably going after a very specific 349 # test and doesn't need a bunch of parallel test runners all looking for 350 # it in a frenzy. Also, '-v' now spits out all test run output even 351 # on success, so the standard recipe for redoing a failing test (with -v 352 # and a -f to filter to the specific test) now causes all test scanning 353 # (in parallel) to print results for do-nothing runs in a very distracting 354 # manner. If we really need filtered parallel runs in the future, consider 355 # adding a --no-output-on-success that prevents -v from setting 356 # output-on-success. 357 configuration.no_multiprocess_test_runner = True 358 359 if args.l: 360 configuration.skip_long_running_test = False 361 362 if args.framework: 363 configuration.lldbFrameworkPath = args.framework 364 365 if args.executable: 366 lldbtest_config.lldbExec = os.path.realpath(args.executable) 367 368 if args.excluded: 369 for excl_file in args.excluded: 370 parseExclusion(excl_file) 371 372 if args.p: 373 if args.p.startswith('-'): 374 usage(parser) 375 configuration.regexp = args.p 376 377 if args.q: 378 configuration.parsable = True 379 380 if args.s: 381 if args.s.startswith('-'): 382 usage(parser) 383 configuration.sdir_name = args.s 384 configuration.session_file_format = args.session_file_format 385 386 if args.t: 387 os.environ['LLDB_COMMAND_TRACE'] = 'YES' 388 389 if args.v: 390 configuration.verbose = 2 391 392 # argparse makes sure we have a number 393 if args.sharp: 394 configuration.count = args.sharp 395 396 if sys.platform.startswith('win32'): 397 os.environ['LLDB_DISABLE_CRASH_DIALOG'] = str( 398 args.disable_crash_dialog) 399 os.environ['LLDB_LAUNCH_INFERIORS_WITHOUT_CONSOLE'] = str(True) 400 401 if do_help: 402 usage(parser) 403 404 if args.no_multiprocess: 405 configuration.no_multiprocess_test_runner = True 406 407 if args.inferior: 408 configuration.is_inferior_test_runner = True 409 410 if args.num_threads: 411 configuration.num_threads = args.num_threads 412 413 if args.test_subdir: 414 configuration.multiprocess_test_subdir = args.test_subdir 415 416 if args.test_runner_name: 417 configuration.test_runner_name = args.test_runner_name 418 419 # Capture test results-related args. 420 if args.curses and not args.inferior: 421 # Act as if the following args were set. 422 args.results_formatter = "lldbsuite.test_event.formatter.curses.Curses" 423 args.results_file = "stdout" 424 425 if args.results_file: 426 configuration.results_filename = args.results_file 427 428 if args.results_port: 429 configuration.results_port = args.results_port 430 431 if args.results_file and args.results_port: 432 sys.stderr.write( 433 "only one of --results-file and --results-port should " 434 "be specified\n") 435 usage(args) 436 437 if args.results_formatter: 438 configuration.results_formatter_name = args.results_formatter 439 if args.results_formatter_options: 440 configuration.results_formatter_options = args.results_formatter_options 441 442 # Default to using the BasicResultsFormatter if no formatter is specified 443 # and we're not a test inferior. 444 if not args.inferior and configuration.results_formatter_name is None: 445 configuration.results_formatter_name = ( 446 "lldbsuite.test_event.formatter.results_formatter.ResultsFormatter") 447 448 # rerun-related arguments 449 configuration.rerun_all_issues = args.rerun_all_issues 450 configuration.rerun_max_file_threshold = args.rerun_max_file_threshold 451 452 if args.lldb_platform_name: 453 configuration.lldb_platform_name = args.lldb_platform_name 454 if args.lldb_platform_url: 455 configuration.lldb_platform_url = args.lldb_platform_url 456 if args.lldb_platform_working_dir: 457 configuration.lldb_platform_working_dir = args.lldb_platform_working_dir 458 459 if args.event_add_entries and len(args.event_add_entries) > 0: 460 entries = {} 461 # Parse out key=val pairs, separated by comma 462 for keyval in args.event_add_entries.split(","): 463 key_val_entry = keyval.split("=") 464 if len(key_val_entry) == 2: 465 (key, val) = key_val_entry 466 val_parts = val.split(':') 467 if len(val_parts) > 1: 468 (val, val_type) = val_parts 469 if val_type == 'int': 470 val = int(val) 471 entries[key] = val 472 # Tell the event builder to create all events with these 473 # key/val pairs in them. 474 if len(entries) > 0: 475 EventBuilder.add_entries_to_all_events(entries) 476 477 # Gather all the dirs passed on the command line. 478 if len(args.args) > 0: 479 configuration.testdirs = list( 480 map(lambda x: os.path.realpath(os.path.abspath(x)), args.args)) 481 # Shut off multiprocessing mode when test directories are specified. 482 configuration.no_multiprocess_test_runner = True 483 484 lldbtest_config.codesign_identity = args.codesign_identity 485 486 #print("testdirs:", testdirs) 487 488 489def getXcodeOutputPaths(lldbRootDirectory): 490 result = [] 491 492 # These are for xcode build directories. 493 xcode3_build_dir = ['build'] 494 xcode4_build_dir = ['build', 'lldb', 'Build', 'Products'] 495 496 configurations = [ 497 ['Debug'], 498 ['DebugClang'], 499 ['Release'], 500 ['BuildAndIntegration']] 501 xcode_build_dirs = [xcode3_build_dir, xcode4_build_dir] 502 for configuration in configurations: 503 for xcode_build_dir in xcode_build_dirs: 504 outputPath = os.path.join( 505 lldbRootDirectory, *(xcode_build_dir + configuration)) 506 result.append(outputPath) 507 508 return result 509 510 511def createSocketToLocalPort(port): 512 def socket_closer(s): 513 """Close down an opened socket properly.""" 514 s.shutdown(socket.SHUT_RDWR) 515 s.close() 516 517 sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 518 sock.connect(("localhost", port)) 519 return (sock, lambda: socket_closer(sock)) 520 521 522def setupTestResults(): 523 """Sets up test results-related objects based on arg settings.""" 524 # Setup the results formatter configuration. 525 formatter_config = formatter.FormatterConfig() 526 formatter_config.filename = configuration.results_filename 527 formatter_config.formatter_name = configuration.results_formatter_name 528 formatter_config.formatter_options = ( 529 configuration.results_formatter_options) 530 formatter_config.port = configuration.results_port 531 532 # Create the results formatter. 533 formatter_spec = formatter.create_results_formatter( 534 formatter_config) 535 if formatter_spec is not None and formatter_spec.formatter is not None: 536 configuration.results_formatter_object = formatter_spec.formatter 537 538 # Send an initialize message to the formatter. 539 initialize_event = EventBuilder.bare_event("initialize") 540 if isMultiprocessTestRunner(): 541 if (configuration.test_runner_name is not None and 542 configuration.test_runner_name == "serial"): 543 # Only one worker queue here. 544 worker_count = 1 545 else: 546 # Workers will be the number of threads specified. 547 worker_count = configuration.num_threads 548 else: 549 worker_count = 1 550 initialize_event["worker_count"] = worker_count 551 552 formatter_spec.formatter.handle_event(initialize_event) 553 554 # Make sure we clean up the formatter on shutdown. 555 if formatter_spec.cleanup_func is not None: 556 atexit.register(formatter_spec.cleanup_func) 557 558 559def getOutputPaths(lldbRootDirectory): 560 """ 561 Returns typical build output paths for the lldb executable 562 563 lldbDirectory - path to the root of the lldb svn/git repo 564 """ 565 result = [] 566 567 if sys.platform == 'darwin': 568 result.extend(getXcodeOutputPaths(lldbRootDirectory)) 569 570 # cmake builds? look for build or build/host folder next to llvm directory 571 # lldb is located in llvm/tools/lldb so we need to go up three levels 572 llvmParentDir = os.path.abspath( 573 os.path.join( 574 lldbRootDirectory, 575 os.pardir, 576 os.pardir, 577 os.pardir)) 578 result.append(os.path.join(llvmParentDir, 'build', 'bin')) 579 result.append(os.path.join(llvmParentDir, 'build', 'host', 'bin')) 580 581 # some cmake developers keep their build directory beside their lldb 582 # directory 583 lldbParentDir = os.path.abspath(os.path.join(lldbRootDirectory, os.pardir)) 584 result.append(os.path.join(lldbParentDir, 'build', 'bin')) 585 result.append(os.path.join(lldbParentDir, 'build', 'host', 'bin')) 586 587 return result 588 589 590def setupSysPath(): 591 """ 592 Add LLDB.framework/Resources/Python to the search paths for modules. 593 As a side effect, we also discover the 'lldb' executable and export it here. 594 """ 595 596 # Get the directory containing the current script. 597 if "DOTEST_PROFILE" in os.environ and "DOTEST_SCRIPT_DIR" in os.environ: 598 scriptPath = os.environ["DOTEST_SCRIPT_DIR"] 599 else: 600 scriptPath = os.path.dirname(os.path.realpath(__file__)) 601 if not scriptPath.endswith('test'): 602 print("This script expects to reside in lldb's test directory.") 603 sys.exit(-1) 604 605 os.environ["LLDB_TEST"] = scriptPath 606 607 # Set up the LLDB_SRC environment variable, so that the tests can locate 608 # the LLDB source code. 609 os.environ["LLDB_SRC"] = lldbsuite.lldb_root 610 611 pluginPath = os.path.join(scriptPath, 'plugins') 612 toolsLLDBMIPath = os.path.join(scriptPath, 'tools', 'lldb-mi') 613 toolsLLDBServerPath = os.path.join(scriptPath, 'tools', 'lldb-server') 614 615 # Insert script dir, plugin dir, lldb-mi dir and lldb-server dir to the 616 # sys.path. 617 sys.path.insert(0, pluginPath) 618 # Adding test/tools/lldb-mi to the path makes it easy 619 sys.path.insert(0, toolsLLDBMIPath) 620 # to "import lldbmi_testcase" from the MI tests 621 # Adding test/tools/lldb-server to the path makes it easy 622 sys.path.insert(0, toolsLLDBServerPath) 623 # to "import lldbgdbserverutils" from the lldb-server tests 624 625 # This is the root of the lldb git/svn checkout 626 # When this changes over to a package instead of a standalone script, this 627 # will be `lldbsuite.lldb_root` 628 lldbRootDirectory = lldbsuite.lldb_root 629 630 # Some of the tests can invoke the 'lldb' command directly. 631 # We'll try to locate the appropriate executable right here. 632 633 # The lldb executable can be set from the command line 634 # if it's not set, we try to find it now 635 # first, we try the environment 636 if not lldbtest_config.lldbExec: 637 # First, you can define an environment variable LLDB_EXEC specifying the 638 # full pathname of the lldb executable. 639 if "LLDB_EXEC" in os.environ: 640 lldbtest_config.lldbExec = os.environ["LLDB_EXEC"] 641 642 if not lldbtest_config.lldbExec: 643 outputPaths = getOutputPaths(lldbRootDirectory) 644 for outputPath in outputPaths: 645 candidatePath = os.path.join(outputPath, 'lldb') 646 if is_exe(candidatePath): 647 lldbtest_config.lldbExec = candidatePath 648 break 649 650 if not lldbtest_config.lldbExec: 651 # Last, check the path 652 lldbtest_config.lldbExec = which('lldb') 653 654 if lldbtest_config.lldbExec and not is_exe(lldbtest_config.lldbExec): 655 print( 656 "'{}' is not a path to a valid executable".format( 657 lldbtest_config.lldbExec)) 658 lldbtest_config.lldbExec = None 659 660 if not lldbtest_config.lldbExec: 661 print("The 'lldb' executable cannot be located. Some of the tests may not be run as a result.") 662 sys.exit(-1) 663 664 # confusingly, this is the "bin" directory 665 lldbLibDir = os.path.dirname(lldbtest_config.lldbExec) 666 os.environ["LLDB_LIB_DIR"] = lldbLibDir 667 lldbImpLibDir = os.path.join( 668 lldbLibDir, 669 '..', 670 'lib') if sys.platform.startswith('win32') else lldbLibDir 671 os.environ["LLDB_IMPLIB_DIR"] = lldbImpLibDir 672 print("LLDB library dir:", os.environ["LLDB_LIB_DIR"]) 673 print("LLDB import library dir:", os.environ["LLDB_IMPLIB_DIR"]) 674 os.system('%s -v' % lldbtest_config.lldbExec) 675 676 # Assume lldb-mi is in same place as lldb 677 # If not found, disable the lldb-mi tests 678 # TODO: Append .exe on Windows 679 # - this will be in a separate commit in case the mi tests fail horribly 680 lldbDir = os.path.dirname(lldbtest_config.lldbExec) 681 lldbMiExec = os.path.join(lldbDir, "lldb-mi") 682 if is_exe(lldbMiExec): 683 os.environ["LLDBMI_EXEC"] = lldbMiExec 684 else: 685 if not configuration.shouldSkipBecauseOfCategories(["lldb-mi"]): 686 print( 687 "The 'lldb-mi' executable cannot be located. The lldb-mi tests can not be run as a result.") 688 configuration.skipCategories.append("lldb-mi") 689 690 lldbPythonDir = None # The directory that contains 'lldb/__init__.py' 691 if configuration.lldbFrameworkPath: 692 candidatePath = os.path.join( 693 configuration.lldbFrameworkPath, 'Resources', 'Python') 694 if os.path.isfile(os.path.join(candidatePath, 'lldb/__init__.py')): 695 lldbPythonDir = candidatePath 696 if not lldbPythonDir: 697 print( 698 'Resources/Python/lldb/__init__.py was not found in ' + 699 configuration.lldbFrameworkPath) 700 sys.exit(-1) 701 else: 702 # If our lldb supports the -P option, use it to find the python path: 703 init_in_python_dir = os.path.join('lldb', '__init__.py') 704 705 lldb_dash_p_result = subprocess.check_output( 706 [lldbtest_config.lldbExec, "-P"], stderr=subprocess.STDOUT, universal_newlines=True) 707 708 if lldb_dash_p_result and not lldb_dash_p_result.startswith( 709 ("<", "lldb: invalid option:")) and not lldb_dash_p_result.startswith("Traceback"): 710 lines = lldb_dash_p_result.splitlines() 711 712 # Workaround for readline vs libedit issue on FreeBSD. If stdout 713 # is not a terminal Python executes 714 # rl_variable_bind ("enable-meta-key", "off"); 715 # This produces a warning with FreeBSD's libedit because the 716 # enable-meta-key variable is unknown. Not an issue on Apple 717 # because cpython commit f0ab6f9f0603 added a #ifndef __APPLE__ 718 # around the call. See http://bugs.python.org/issue19884 for more 719 # information. For now we just discard the warning output. 720 if len(lines) >= 1 and lines[0].startswith( 721 "bind: Invalid command"): 722 lines.pop(0) 723 724 # Taking the last line because lldb outputs 725 # 'Cannot read termcap database;\nusing dumb terminal settings.\n' 726 # before the path 727 if len(lines) >= 1 and os.path.isfile( 728 os.path.join(lines[-1], init_in_python_dir)): 729 lldbPythonDir = lines[-1] 730 if "freebsd" in sys.platform or "linux" in sys.platform: 731 os.environ['LLDB_LIB_DIR'] = os.path.join( 732 lldbPythonDir, '..', '..') 733 734 if not lldbPythonDir: 735 if platform.system() == "Darwin": 736 python_resource_dir = ['LLDB.framework', 'Resources', 'Python'] 737 outputPaths = getXcodeOutputPaths(lldbRootDirectory) 738 for outputPath in outputPaths: 739 candidatePath = os.path.join( 740 outputPath, *python_resource_dir) 741 if os.path.isfile( 742 os.path.join( 743 candidatePath, 744 init_in_python_dir)): 745 lldbPythonDir = candidatePath 746 break 747 748 if not lldbPythonDir: 749 print("lldb.py is not found, some tests may fail.") 750 else: 751 print( 752 "Unable to load lldb extension module. Possible reasons for this include:") 753 print(" 1) LLDB was built with LLDB_DISABLE_PYTHON=1") 754 print( 755 " 2) PYTHONPATH and PYTHONHOME are not set correctly. PYTHONHOME should refer to") 756 print( 757 " the version of Python that LLDB built and linked against, and PYTHONPATH") 758 print( 759 " should contain the Lib directory for the same python distro, as well as the") 760 print(" location of LLDB\'s site-packages folder.") 761 print( 762 " 3) A different version of Python than that which was built against is exported in") 763 print(" the system\'s PATH environment variable, causing conflicts.") 764 print( 765 " 4) The executable '%s' could not be found. Please check " % 766 lldbtest_config.lldbExec) 767 print(" that it exists and is executable.") 768 769 if lldbPythonDir: 770 lldbPythonDir = os.path.normpath(lldbPythonDir) 771 # Some of the code that uses this path assumes it hasn't resolved the Versions... link. 772 # If the path we've constructed looks like that, then we'll strip out 773 # the Versions/A part. 774 (before, frameWithVersion, after) = lldbPythonDir.rpartition( 775 "LLDB.framework/Versions/A") 776 if frameWithVersion != "": 777 lldbPythonDir = before + "LLDB.framework" + after 778 779 lldbPythonDir = os.path.abspath(lldbPythonDir) 780 781 # If tests need to find LLDB_FRAMEWORK, now they can do it 782 os.environ["LLDB_FRAMEWORK"] = os.path.dirname( 783 os.path.dirname(lldbPythonDir)) 784 785 # This is to locate the lldb.py module. Insert it right after 786 # sys.path[0]. 787 sys.path[1:1] = [lldbPythonDir] 788 789 790def visit_file(dir, name): 791 # Try to match the regexp pattern, if specified. 792 if configuration.regexp: 793 if not re.search(configuration.regexp, name): 794 # We didn't match the regex, we're done. 795 return 796 797 if configuration.skip_tests: 798 for file_regexp in configuration.skip_tests: 799 if re.search(file_regexp, name): 800 return 801 802 # We found a match for our test. Add it to the suite. 803 804 # Update the sys.path first. 805 if not sys.path.count(dir): 806 sys.path.insert(0, dir) 807 base = os.path.splitext(name)[0] 808 809 # Thoroughly check the filterspec against the base module and admit 810 # the (base, filterspec) combination only when it makes sense. 811 filterspec = None 812 for filterspec in configuration.filters: 813 # Optimistically set the flag to True. 814 filtered = True 815 module = __import__(base) 816 parts = filterspec.split('.') 817 obj = module 818 for part in parts: 819 try: 820 parent, obj = obj, getattr(obj, part) 821 except AttributeError: 822 # The filterspec has failed. 823 filtered = False 824 break 825 826 # If filtered, we have a good filterspec. Add it. 827 if filtered: 828 # print("adding filter spec %s to module %s" % (filterspec, module)) 829 configuration.suite.addTests( 830 unittest2.defaultTestLoader.loadTestsFromName( 831 filterspec, module)) 832 continue 833 834 # Forgo this module if the (base, filterspec) combo is invalid 835 if configuration.filters and not filtered: 836 return 837 838 if not filterspec or not filtered: 839 # Add the entire file's worth of tests since we're not filtered. 840 # Also the fail-over case when the filterspec branch 841 # (base, filterspec) combo doesn't make sense. 842 configuration.suite.addTests( 843 unittest2.defaultTestLoader.loadTestsFromName(base)) 844 845 846def visit(prefix, dir, names): 847 """Visitor function for os.path.walk(path, visit, arg).""" 848 849 dir_components = set(dir.split(os.sep)) 850 excluded_components = set(['.svn', '.git']) 851 if dir_components.intersection(excluded_components): 852 return 853 854 # Gather all the Python test file names that follow the Test*.py pattern. 855 python_test_files = [ 856 name 857 for name in names 858 if name.endswith('.py') and name.startswith(prefix)] 859 860 # Visit all the python test files. 861 for name in python_test_files: 862 try: 863 # Ensure we error out if we have multiple tests with the same 864 # base name. 865 # Future improvement: find all the places where we work with base 866 # names and convert to full paths. We have directory structure 867 # to disambiguate these, so we shouldn't need this constraint. 868 if name in configuration.all_tests: 869 raise Exception("Found multiple tests with the name %s" % name) 870 configuration.all_tests.add(name) 871 872 # Run the relevant tests in the python file. 873 visit_file(dir, name) 874 except Exception as ex: 875 # Convert this exception to a test event error for the file. 876 test_filename = os.path.abspath(os.path.join(dir, name)) 877 if configuration.results_formatter_object is not None: 878 # Grab the backtrace for the exception. 879 import traceback 880 backtrace = traceback.format_exc() 881 882 # Generate the test event. 883 configuration.results_formatter_object.handle_event( 884 EventBuilder.event_for_job_test_add_error( 885 test_filename, ex, backtrace)) 886 raise 887 888 889def disabledynamics(): 890 import lldb 891 ci = lldb.DBG.GetCommandInterpreter() 892 res = lldb.SBCommandReturnObject() 893 ci.HandleCommand( 894 "setting set target.prefer-dynamic-value no-dynamic-values", 895 res, 896 False) 897 if not res.Succeeded(): 898 raise Exception('disabling dynamic type support failed') 899 900 901def lldbLoggings(): 902 import lldb 903 """Check and do lldb loggings if necessary.""" 904 905 # Turn on logging for debugging purposes if ${LLDB_LOG} environment variable is 906 # defined. Use ${LLDB_LOG} to specify the log file. 907 ci = lldb.DBG.GetCommandInterpreter() 908 res = lldb.SBCommandReturnObject() 909 if ("LLDB_LOG" in os.environ): 910 open(os.environ["LLDB_LOG"], 'w').close() 911 if ("LLDB_LOG_OPTION" in os.environ): 912 lldb_log_option = os.environ["LLDB_LOG_OPTION"] 913 else: 914 lldb_log_option = "event process expr state api" 915 ci.HandleCommand( 916 "log enable -n -f " + 917 os.environ["LLDB_LOG"] + 918 " lldb " + 919 lldb_log_option, 920 res) 921 if not res.Succeeded(): 922 raise Exception('log enable failed (check LLDB_LOG env variable)') 923 924 if ("LLDB_LINUX_LOG" in os.environ): 925 open(os.environ["LLDB_LINUX_LOG"], 'w').close() 926 if ("LLDB_LINUX_LOG_OPTION" in os.environ): 927 lldb_log_option = os.environ["LLDB_LINUX_LOG_OPTION"] 928 else: 929 lldb_log_option = "event process expr state api" 930 ci.HandleCommand( 931 "log enable -n -f " + 932 os.environ["LLDB_LINUX_LOG"] + 933 " linux " + 934 lldb_log_option, 935 res) 936 if not res.Succeeded(): 937 raise Exception( 938 'log enable failed (check LLDB_LINUX_LOG env variable)') 939 940 # Ditto for gdb-remote logging if ${GDB_REMOTE_LOG} environment variable is defined. 941 # Use ${GDB_REMOTE_LOG} to specify the log file. 942 if ("GDB_REMOTE_LOG" in os.environ): 943 if ("GDB_REMOTE_LOG_OPTION" in os.environ): 944 gdb_remote_log_option = os.environ["GDB_REMOTE_LOG_OPTION"] 945 else: 946 gdb_remote_log_option = "packets process" 947 ci.HandleCommand( 948 "log enable -n -f " + os.environ["GDB_REMOTE_LOG"] + " gdb-remote " 949 + gdb_remote_log_option, 950 res) 951 if not res.Succeeded(): 952 raise Exception( 953 'log enable failed (check GDB_REMOTE_LOG env variable)') 954 955 956def getMyCommandLine(): 957 return ' '.join(sys.argv) 958 959# ======================================== # 960# # 961# Execution of the test driver starts here # 962# # 963# ======================================== # 964 965 966def checkDsymForUUIDIsNotOn(): 967 cmd = ["defaults", "read", "com.apple.DebugSymbols"] 968 pipe = subprocess.Popen( 969 cmd, 970 stdout=subprocess.PIPE, 971 stderr=subprocess.STDOUT) 972 cmd_output = pipe.stdout.read() 973 if cmd_output and "DBGFileMappedPaths = " in cmd_output: 974 print("%s =>" % ' '.join(cmd)) 975 print(cmd_output) 976 print( 977 "Disable automatic lookup and caching of dSYMs before running the test suite!") 978 print("Exiting...") 979 sys.exit(0) 980 981 982def exitTestSuite(exitCode=None): 983 import lldb 984 lldb.SBDebugger.Terminate() 985 if exitCode: 986 sys.exit(exitCode) 987 988 989def isMultiprocessTestRunner(): 990 # We're not multiprocess when we're either explicitly 991 # the inferior (as specified by the multiprocess test 992 # runner) OR we've been told to skip using the multiprocess 993 # test runner 994 return not ( 995 configuration.is_inferior_test_runner or configuration.no_multiprocess_test_runner) 996 997 998def getVersionForSDK(sdk): 999 sdk = str.lower(sdk) 1000 full_path = seven.get_command_output('xcrun -sdk %s --show-sdk-path' % sdk) 1001 basename = os.path.basename(full_path) 1002 basename = os.path.splitext(basename)[0] 1003 basename = str.lower(basename) 1004 ver = basename.replace(sdk, '') 1005 return ver 1006 1007 1008def getPathForSDK(sdk): 1009 sdk = str.lower(sdk) 1010 full_path = seven.get_command_output('xcrun -sdk %s --show-sdk-path' % sdk) 1011 if os.path.exists(full_path): 1012 return full_path 1013 return None 1014 1015 1016def setDefaultTripleForPlatform(): 1017 if configuration.lldb_platform_name == 'ios-simulator': 1018 triple_str = 'x86_64-apple-ios%s' % ( 1019 getVersionForSDK('iphonesimulator')) 1020 os.environ['TRIPLE'] = triple_str 1021 return {'TRIPLE': triple_str} 1022 return {} 1023 1024 1025def run_suite(): 1026 # On MacOS X, check to make sure that domain for com.apple.DebugSymbols defaults 1027 # does not exist before proceeding to running the test suite. 1028 if sys.platform.startswith("darwin"): 1029 checkDsymForUUIDIsNotOn() 1030 1031 # 1032 # Start the actions by first parsing the options while setting up the test 1033 # directories, followed by setting up the search paths for lldb utilities; 1034 # then, we walk the directory trees and collect the tests into our test suite. 1035 # 1036 parseOptionsAndInitTestdirs() 1037 1038 # Setup test results (test results formatter and output handling). 1039 setupTestResults() 1040 1041 # If we are running as the multiprocess test runner, kick off the 1042 # multiprocess test runner here. 1043 if isMultiprocessTestRunner(): 1044 from . import dosep 1045 dosep.main( 1046 configuration.num_threads, 1047 configuration.multiprocess_test_subdir, 1048 configuration.test_runner_name, 1049 configuration.results_formatter_object) 1050 raise Exception("should never get here") 1051 elif configuration.is_inferior_test_runner: 1052 # Shut off Ctrl-C processing in inferiors. The parallel 1053 # test runner handles this more holistically. 1054 signal.signal(signal.SIGINT, signal.SIG_IGN) 1055 1056 setupSysPath() 1057 configuration.setupCrashInfoHook() 1058 1059 # 1060 # If '-l' is specified, do not skip the long running tests. 1061 if not configuration.skip_long_running_test: 1062 os.environ["LLDB_SKIP_LONG_RUNNING_TEST"] = "NO" 1063 1064 # For the time being, let's bracket the test runner within the 1065 # lldb.SBDebugger.Initialize()/Terminate() pair. 1066 import lldb 1067 1068 # Create a singleton SBDebugger in the lldb namespace. 1069 lldb.DBG = lldb.SBDebugger.Create() 1070 1071 if configuration.lldb_platform_name: 1072 print("Setting up remote platform '%s'" % 1073 (configuration.lldb_platform_name)) 1074 lldb.remote_platform = lldb.SBPlatform( 1075 configuration.lldb_platform_name) 1076 if not lldb.remote_platform.IsValid(): 1077 print( 1078 "error: unable to create the LLDB platform named '%s'." % 1079 (configuration.lldb_platform_name)) 1080 exitTestSuite(1) 1081 if configuration.lldb_platform_url: 1082 # We must connect to a remote platform if a LLDB platform URL was 1083 # specified 1084 print( 1085 "Connecting to remote platform '%s' at '%s'..." % 1086 (configuration.lldb_platform_name, configuration.lldb_platform_url)) 1087 platform_connect_options = lldb.SBPlatformConnectOptions( 1088 configuration.lldb_platform_url) 1089 err = lldb.remote_platform.ConnectRemote(platform_connect_options) 1090 if err.Success(): 1091 print("Connected.") 1092 else: 1093 print("error: failed to connect to remote platform using URL '%s': %s" % ( 1094 configuration.lldb_platform_url, err)) 1095 exitTestSuite(1) 1096 else: 1097 configuration.lldb_platform_url = None 1098 1099 platform_changes = setDefaultTripleForPlatform() 1100 first = True 1101 for key in platform_changes: 1102 if first: 1103 print("Environment variables setup for platform support:") 1104 first = False 1105 print("%s = %s" % (key, platform_changes[key])) 1106 1107 if configuration.lldb_platform_working_dir: 1108 print("Setting remote platform working directory to '%s'..." % 1109 (configuration.lldb_platform_working_dir)) 1110 lldb.remote_platform.SetWorkingDirectory( 1111 configuration.lldb_platform_working_dir) 1112 lldb.DBG.SetSelectedPlatform(lldb.remote_platform) 1113 else: 1114 lldb.remote_platform = None 1115 configuration.lldb_platform_working_dir = None 1116 configuration.lldb_platform_url = None 1117 1118 target_platform = lldb.DBG.GetSelectedPlatform().GetTriple().split('-')[2] 1119 1120 # Don't do debugserver tests on everything except OS X. 1121 configuration.dont_do_debugserver_test = "linux" in target_platform or "freebsd" in target_platform or "windows" in target_platform 1122 1123 # Don't do lldb-server (llgs) tests on anything except Linux. 1124 configuration.dont_do_llgs_test = not ("linux" in target_platform) 1125 1126 # 1127 # Walk through the testdirs while collecting tests. 1128 # 1129 for testdir in configuration.testdirs: 1130 for (dirpath, dirnames, filenames) in os.walk(testdir): 1131 visit('Test', dirpath, filenames) 1132 1133 # 1134 # Now that we have loaded all the test cases, run the whole test suite. 1135 # 1136 1137 # Turn on lldb loggings if necessary. 1138 lldbLoggings() 1139 1140 # Disable default dynamic types for testing purposes 1141 disabledynamics() 1142 1143 # Install the control-c handler. 1144 unittest2.signals.installHandler() 1145 1146 # If sdir_name is not specified through the '-s sdir_name' option, get a 1147 # timestamp string and export it as LLDB_SESSION_DIR environment var. This will 1148 # be used when/if we want to dump the session info of individual test cases 1149 # later on. 1150 # 1151 # See also TestBase.dumpSessionInfo() in lldbtest.py. 1152 import datetime 1153 # The windows platforms don't like ':' in the pathname. 1154 timestamp_started = datetime.datetime.now().strftime("%Y-%m-%d-%H_%M_%S") 1155 if not configuration.sdir_name: 1156 configuration.sdir_name = timestamp_started 1157 os.environ["LLDB_SESSION_DIRNAME"] = os.path.join( 1158 os.getcwd(), configuration.sdir_name) 1159 1160 sys.stderr.write( 1161 "\nSession logs for test failures/errors/unexpected successes" 1162 " will go into directory '%s'\n" % 1163 configuration.sdir_name) 1164 sys.stderr.write("Command invoked: %s\n" % getMyCommandLine()) 1165 1166 if not os.path.isdir(configuration.sdir_name): 1167 try: 1168 os.mkdir(configuration.sdir_name) 1169 except OSError as exception: 1170 if exception.errno != errno.EEXIST: 1171 raise 1172 1173 # 1174 # Invoke the default TextTestRunner to run the test suite, possibly iterating 1175 # over different configurations. 1176 # 1177 1178 iterArchs = False 1179 iterCompilers = False 1180 1181 if isinstance(configuration.archs, list) and len(configuration.archs) >= 1: 1182 iterArchs = True 1183 1184 # 1185 # Add some intervention here to sanity check that the compilers requested are sane. 1186 # If found not to be an executable program, the invalid one is dropped 1187 # from the list. 1188 for i in range(len(configuration.compilers)): 1189 c = configuration.compilers[i] 1190 if which(c): 1191 continue 1192 else: 1193 if sys.platform.startswith("darwin"): 1194 pipe = subprocess.Popen( 1195 ['xcrun', '-find', c], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) 1196 cmd_output = pipe.stdout.read() 1197 if cmd_output: 1198 if "not found" in cmd_output: 1199 print("dropping %s from the compilers used" % c) 1200 configuration.compilers.remove(i) 1201 else: 1202 configuration.compilers[i] = cmd_output.split('\n')[0] 1203 print( 1204 "'xcrun -find %s' returning %s" % 1205 (c, configuration.compilers[i])) 1206 1207 if not configuration.parsable: 1208 print("compilers=%s" % str(configuration.compilers)) 1209 1210 if not configuration.compilers or len(configuration.compilers) == 0: 1211 print("No eligible compiler found, exiting.") 1212 exitTestSuite(1) 1213 1214 if isinstance( 1215 configuration.compilers, 1216 list) and len( 1217 configuration.compilers) >= 1: 1218 iterCompilers = True 1219 1220 # If we iterate on archs or compilers, there is a chance we want to split 1221 # stderr/stdout. 1222 if iterArchs or iterCompilers: 1223 old_stderr = sys.stderr 1224 old_stdout = sys.stdout 1225 new_stderr = None 1226 new_stdout = None 1227 1228 # Iterating over all possible architecture and compiler combinations. 1229 for ia in range(len(configuration.archs) if iterArchs else 1): 1230 archConfig = "" 1231 if iterArchs: 1232 os.environ["ARCH"] = configuration.archs[ia] 1233 archConfig = "arch=%s" % configuration.archs[ia] 1234 for ic in range(len(configuration.compilers) if iterCompilers else 1): 1235 if iterCompilers: 1236 os.environ["CC"] = configuration.compilers[ic] 1237 configString = "%s compiler=%s" % ( 1238 archConfig, configuration.compilers[ic]) 1239 else: 1240 configString = archConfig 1241 1242 if iterArchs or iterCompilers: 1243 # Translate ' ' to '-' for pathname component. 1244 if six.PY2: 1245 import string 1246 tbl = string.maketrans(' ', '-') 1247 else: 1248 tbl = str.maketrans(' ', '-') 1249 configPostfix = configString.translate(tbl) 1250 1251 # Output the configuration. 1252 if not configuration.parsable: 1253 sys.stderr.write("\nConfiguration: " + configString + "\n") 1254 1255 #print("sys.stderr name is", sys.stderr.name) 1256 #print("sys.stdout name is", sys.stdout.name) 1257 1258 # First, write out the number of collected test cases. 1259 if not configuration.parsable: 1260 sys.stderr.write(configuration.separator + "\n") 1261 sys.stderr.write( 1262 "Collected %d test%s\n\n" % 1263 (configuration.suite.countTestCases(), 1264 configuration.suite.countTestCases() != 1 and "s" or "")) 1265 1266 if configuration.parsable: 1267 v = 0 1268 else: 1269 v = configuration.verbose 1270 1271 # Invoke the test runner. 1272 if configuration.count == 1: 1273 result = unittest2.TextTestRunner( 1274 stream=sys.stderr, 1275 verbosity=v, 1276 resultclass=test_result.LLDBTestResult).run( 1277 configuration.suite) 1278 else: 1279 # We are invoking the same test suite more than once. In this case, 1280 # mark __ignore_singleton__ flag as True so the signleton pattern is 1281 # not enforced. 1282 test_result.LLDBTestResult.__ignore_singleton__ = True 1283 for i in range(configuration.count): 1284 1285 result = unittest2.TextTestRunner( 1286 stream=sys.stderr, 1287 verbosity=v, 1288 resultclass=test_result.LLDBTestResult).run( 1289 configuration.suite) 1290 1291 configuration.failed = configuration.failed or not result.wasSuccessful() 1292 1293 if configuration.sdir_has_content and not configuration.parsable: 1294 sys.stderr.write( 1295 "Session logs for test failures/errors/unexpected successes" 1296 " can be found in directory '%s'\n" % 1297 configuration.sdir_name) 1298 1299 if configuration.useCategories and len( 1300 configuration.failuresPerCategory) > 0: 1301 sys.stderr.write("Failures per category:\n") 1302 for category in configuration.failuresPerCategory: 1303 sys.stderr.write( 1304 "%s - %d\n" % 1305 (category, configuration.failuresPerCategory[category])) 1306 1307 # Terminate the test suite if ${LLDB_TESTSUITE_FORCE_FINISH} is defined. 1308 # This should not be necessary now. 1309 if ("LLDB_TESTSUITE_FORCE_FINISH" in os.environ): 1310 print("Terminating Test suite...") 1311 subprocess.Popen(["/bin/sh", "-c", "kill %s; exit 0" % (os.getpid())]) 1312 1313 # Exiting. 1314 exitTestSuite(configuration.failed) 1315 1316if __name__ == "__main__": 1317 print( 1318 __file__ + 1319 " is for use as a module only. It should not be run as a standalone script.") 1320 sys.exit(-1) 1321