1#!/usr/bin/env python 2 3""" 4A simple testing framework for lldb using python's unit testing framework. 5 6Tests for lldb are written as python scripts which take advantage of the script 7bridging provided by LLDB.framework to interact with lldb core. 8 9A specific naming pattern is followed by the .py script to be recognized as 10a module which implements a test scenario, namely, Test*.py. 11 12To specify the directories where "Test*.py" python test scripts are located, 13you need to pass in a list of directory names. By default, the current 14working directory is searched if nothing is specified on the command line. 15 16Type: 17 18./dotest.py -h 19 20for available options. 21""" 22 23from __future__ import print_function 24 25import use_lldb_suite 26 27import lldbsuite 28 29import atexit 30import commands 31import importlib 32import os 33import dotest_args 34import errno 35import platform 36import progress 37import signal 38import socket 39import subprocess 40import sys 41import test_results 42from test_results import EventBuilder 43import inspect 44import unittest2 45import lldbtest_config 46import test_categories 47 48import six 49 50def is_exe(fpath): 51 """Returns true if fpath is an executable.""" 52 return os.path.isfile(fpath) and os.access(fpath, os.X_OK) 53 54def which(program): 55 """Returns the full path to a program; None otherwise.""" 56 fpath, fname = os.path.split(program) 57 if fpath: 58 if is_exe(program): 59 return program 60 else: 61 for path in os.environ["PATH"].split(os.pathsep): 62 exe_file = os.path.join(path, program) 63 if is_exe(exe_file): 64 return exe_file 65 return None 66 67class _WritelnDecorator(object): 68 """Used to decorate file-like objects with a handy 'writeln' method""" 69 def __init__(self,stream): 70 self.stream = stream 71 72 def __getattr__(self, attr): 73 if attr in ('stream', '__getstate__'): 74 raise AttributeError(attr) 75 return getattr(self.stream,attr) 76 77 def writeln(self, arg=None): 78 if arg: 79 self.write(arg) 80 self.write('\n') # text-mode streams translate to \r\n if needed 81 82# 83# Global variables: 84# 85 86# The test suite. 87suite = unittest2.TestSuite() 88 89# By default, benchmarks tests are not run. 90just_do_benchmarks_test = False 91 92dont_do_dsym_test = False 93dont_do_dwarf_test = False 94dont_do_dwo_test = False 95 96# The blacklist is optional (-b blacklistFile) and allows a central place to skip 97# testclass's and/or testclass.testmethod's. 98blacklist = None 99 100# The dictionary as a result of sourcing blacklistFile. 101blacklistConfig = {} 102 103# The list of categories we said we care about 104categoriesList = None 105# set to true if we are going to use categories for cherry-picking test cases 106useCategories = False 107# Categories we want to skip 108skipCategories = [] 109# use this to track per-category failures 110failuresPerCategory = {} 111 112# The path to LLDB.framework is optional. 113lldbFrameworkPath = None 114 115# The config file is optional. 116configFile = None 117 118# Test suite repeat count. Can be overwritten with '-# count'. 119count = 1 120 121# The dictionary as a result of sourcing configFile. 122config = {} 123# The pre_flight and post_flight functions come from reading a config file. 124pre_flight = None 125post_flight = None 126# So do the lldbtest_remote_sandbox and lldbtest_remote_shell_template variables. 127lldbtest_remote_sandbox = None 128lldbtest_remote_shell_template = None 129 130# The 'archs' and 'compilers' can be specified via either command line or configFile, 131# with the command line overriding the configFile. The corresponding options can be 132# specified more than once. For example, "-A x86_64 -A i386" => archs=['x86_64', 'i386'] 133# and "-C gcc -C clang" => compilers=['gcc', 'clang']. 134archs = None # Must be initialized after option parsing 135compilers = None # Must be initialized after option parsing 136 137# The arch might dictate some specific CFLAGS to be passed to the toolchain to build 138# the inferior programs. The global variable cflags_extras provides a hook to do 139# just that. 140cflags_extras = '' 141 142# Dump the Python sys.path variable. Use '-D' to dump sys.path. 143dumpSysPath = False 144 145# Full path of the benchmark executable, as specified by the '-e' option. 146bmExecutable = None 147# The breakpoint specification of bmExecutable, as specified by the '-x' option. 148bmBreakpointSpec = None 149# The benchmark iteration count, as specified by the '-y' option. 150bmIterationCount = -1 151 152# By default, don't exclude any directories. Use '-X' to add one excluded directory. 153excluded = set(['.svn', '.git']) 154 155# By default, failfast is False. Use '-F' to overwrite it. 156failfast = False 157 158# The filters (testclass.testmethod) used to admit tests into our test suite. 159filters = [] 160 161# The runhooks is a list of lldb commands specifically for the debugger. 162# Use '-k' to specify a runhook. 163runHooks = [] 164 165# If '-g' is specified, the filterspec is not exclusive. If a test module does 166# not contain testclass.testmethod which matches the filterspec, the whole test 167# module is still admitted into our test suite. fs4all flag defaults to True. 168fs4all = True 169 170# Ignore the build search path relative to this script to locate the lldb.py module. 171ignore = False 172 173# By default, we do not skip build and cleanup. Use '-S' option to override. 174skip_build_and_cleanup = False 175 176# By default, we skip long running test case. Use '-l' option to override. 177skip_long_running_test = True 178 179# By default, we print the build dir, lldb version, and svn info. Use '-n' option to 180# turn it off. 181noHeaders = False 182 183# Parsable mode silences headers, and any other output this script might generate, and instead 184# prints machine-readable output similar to what clang tests produce. 185parsable = False 186 187# The regular expression pattern to match against eligible filenames as our test cases. 188regexp = None 189 190# By default, tests are executed in place and cleanups are performed afterwards. 191# Use '-r dir' option to relocate the tests and their intermediate files to a 192# different directory and to forgo any cleanups. The directory specified must 193# not exist yet. 194rdir = None 195 196# By default, recorded session info for errored/failed test are dumped into its 197# own file under a session directory named after the timestamp of the test suite 198# run. Use '-s session-dir-name' to specify a specific dir name. 199sdir_name = None 200 201# Set this flag if there is any session info dumped during the test run. 202sdir_has_content = False 203 204# svn_info stores the output from 'svn info lldb.base.dir'. 205svn_info = '' 206 207# svn_silent means do not try to obtain svn status 208svn_silent = True 209 210# Default verbosity is 0. 211verbose = 1 212 213# Set to True only if verbose is 0 and LLDB trace mode is off. 214progress_bar = False 215 216# By default, search from the script directory. 217# We can't use sys.path[0] to determine the script directory 218# because it doesn't work under a debugger 219testdirs = [ os.path.dirname(os.path.realpath(__file__)) ] 220 221# Separator string. 222separator = '-' * 70 223 224failed = False 225 226# LLDB Remote platform setting 227lldb_platform_name = None 228lldb_platform_url = None 229lldb_platform_working_dir = None 230 231# Parallel execution settings 232is_inferior_test_runner = False 233multiprocess_test_subdir = None 234num_threads = None 235output_on_success = False 236no_multiprocess_test_runner = False 237test_runner_name = None 238 239# Test results handling globals 240results_filename = None 241results_port = None 242results_file_object = None 243results_formatter_name = None 244results_formatter_object = None 245results_formatter_options = None 246 247# The names of all tests. Used to assert we don't have two tests with the same base name. 248all_tests = set() 249 250def usage(parser): 251 parser.print_help() 252 if verbose > 0: 253 print(""" 254Examples: 255 256This is an example of using the -f option to pinpoint to a specific test class 257and test method to be run: 258 259$ ./dotest.py -f ClassTypesTestCase.test_with_dsym_and_run_command 260---------------------------------------------------------------------- 261Collected 1 test 262 263test_with_dsym_and_run_command (TestClassTypes.ClassTypesTestCase) 264Test 'frame variable this' when stopped on a class constructor. ... ok 265 266---------------------------------------------------------------------- 267Ran 1 test in 1.396s 268 269OK 270 271And this is an example of using the -p option to run a single file (the filename 272matches the pattern 'ObjC' and it happens to be 'TestObjCMethods.py'): 273 274$ ./dotest.py -v -p ObjC 275---------------------------------------------------------------------- 276Collected 4 tests 277 278test_break_with_dsym (TestObjCMethods.FoundationTestCase) 279Test setting objc breakpoints using '_regexp-break' and 'breakpoint set'. ... ok 280test_break_with_dwarf (TestObjCMethods.FoundationTestCase) 281Test setting objc breakpoints using '_regexp-break' and 'breakpoint set'. ... ok 282test_data_type_and_expr_with_dsym (TestObjCMethods.FoundationTestCase) 283Lookup objective-c data types and evaluate expressions. ... ok 284test_data_type_and_expr_with_dwarf (TestObjCMethods.FoundationTestCase) 285Lookup objective-c data types and evaluate expressions. ... ok 286 287---------------------------------------------------------------------- 288Ran 4 tests in 16.661s 289 290OK 291 292Running of this script also sets up the LLDB_TEST environment variable so that 293individual test cases can locate their supporting files correctly. The script 294tries to set up Python's search paths for modules by looking at the build tree 295relative to this script. See also the '-i' option in the following example. 296 297Finally, this is an example of using the lldb.py module distributed/installed by 298Xcode4 to run against the tests under the 'forward' directory, and with the '-w' 299option to add some delay between two tests. It uses ARCH=x86_64 to specify that 300as the architecture and CC=clang to specify the compiler used for the test run: 301 302$ PYTHONPATH=/Xcode4/Library/PrivateFrameworks/LLDB.framework/Versions/A/Resources/Python ARCH=x86_64 CC=clang ./dotest.py -v -w -i forward 303 304Session logs for test failures/errors will go into directory '2010-11-11-13_56_16' 305---------------------------------------------------------------------- 306Collected 2 tests 307 308test_with_dsym_and_run_command (TestForwardDeclaration.ForwardDeclarationTestCase) 309Display *bar_ptr when stopped on a function with forward declaration of struct bar. ... ok 310test_with_dwarf_and_run_command (TestForwardDeclaration.ForwardDeclarationTestCase) 311Display *bar_ptr when stopped on a function with forward declaration of struct bar. ... ok 312 313---------------------------------------------------------------------- 314Ran 2 tests in 5.659s 315 316OK 317 318The 'Session ...' verbiage is recently introduced (see also the '-s' option) to 319notify the directory containing the session logs for test failures or errors. 320In case there is any test failure/error, a similar message is appended at the 321end of the stderr output for your convenience. 322 323ENABLING LOGS FROM TESTS 324 325Option 1: 326 327Writing logs into different files per test case:: 328 329This option is particularly useful when multiple dotest instances are created 330by dosep.py 331 332$ ./dotest.py --channel "lldb all" 333 334$ ./dotest.py --channel "lldb all" --channel "gdb-remote packets" 335 336These log files are written to: 337 338<session-dir>/<test-id>-host.log (logs from lldb host process) 339<session-dir>/<test-id>-server.log (logs from debugserver/lldb-server) 340<session-dir>/<test-id>-<test-result>.log (console logs) 341 342By default, logs from successful runs are deleted. Use the --log-success flag 343to create reference logs for debugging. 344 345$ ./dotest.py --log-success 346 347Option 2: (DEPRECATED) 348 349The following options can only enable logs from the host lldb process. 350Only categories from the "lldb" or "gdb-remote" channels can be enabled 351They also do not automatically enable logs in locally running debug servers. 352Also, logs from all test case are written into each log file 353 354o LLDB_LOG: if defined, specifies the log file pathname for the 'lldb' subsystem 355 with a default option of 'event process' if LLDB_LOG_OPTION is not defined. 356 357o GDB_REMOTE_LOG: if defined, specifies the log file pathname for the 358 'process.gdb-remote' subsystem with a default option of 'packets' if 359 GDB_REMOTE_LOG_OPTION is not defined. 360 361""") 362 sys.exit(0) 363 364 365def setCrashInfoHook_Mac(text): 366 import crashinfo 367 crashinfo.setCrashReporterDescription(text) 368 369# implement this in some suitable way for your platform, and then bind it 370# to setCrashInfoHook 371def setCrashInfoHook_NonMac(text): 372 pass 373 374setCrashInfoHook = None 375 376def deleteCrashInfoDylib(dylib_path): 377 try: 378 # Need to modify this to handle multiple tests running at the same time. If we move this 379 # to the test's real dir, all should be we run sequentially within a test directory. 380 # os.remove(dylib_path) 381 None 382 finally: 383 pass 384 385def setupCrashInfoHook(): 386 global setCrashInfoHook 387 setCrashInfoHook = setCrashInfoHook_NonMac # safe default 388 if platform.system() == "Darwin": 389 import lock 390 test_dir = os.environ['LLDB_TEST'] 391 if not test_dir or not os.path.exists(test_dir): 392 return 393 dylib_lock = os.path.join(test_dir,"crashinfo.lock") 394 dylib_src = os.path.join(test_dir,"crashinfo.c") 395 dylib_dst = os.path.join(test_dir,"crashinfo.so") 396 try: 397 compile_lock = lock.Lock(dylib_lock) 398 compile_lock.acquire() 399 if not os.path.isfile(dylib_dst) or os.path.getmtime(dylib_dst) < os.path.getmtime(dylib_src): 400 # we need to compile 401 cmd = "SDKROOT= xcrun clang %s -o %s -framework Python -Xlinker -dylib -iframework /System/Library/Frameworks/ -Xlinker -F /System/Library/Frameworks/" % (dylib_src,dylib_dst) 402 if subprocess.call(cmd,shell=True) != 0 or not os.path.isfile(dylib_dst): 403 raise Exception('command failed: "{}"'.format(cmd)) 404 finally: 405 compile_lock.release() 406 del compile_lock 407 408 setCrashInfoHook = setCrashInfoHook_Mac 409 410 else: 411 pass 412 413def shouldSkipBecauseOfCategories(test_categories): 414 global useCategories, categoriesList, skipCategories 415 416 if useCategories: 417 if len(test_categories) == 0 or len(categoriesList & set(test_categories)) == 0: 418 return True 419 420 for category in skipCategories: 421 if category in test_categories: 422 return True 423 424 return False 425 426def parseOptionsAndInitTestdirs(): 427 """Initialize the list of directories containing our unittest scripts. 428 429 '-h/--help as the first option prints out usage info and exit the program. 430 """ 431 432 global just_do_benchmarks_test 433 global dont_do_dsym_test 434 global dont_do_dwarf_test 435 global dont_do_dwo_test 436 global blacklist 437 global blacklistConfig 438 global categoriesList 439 global validCategories 440 global useCategories 441 global skipCategories 442 global lldbFrameworkPath 443 global configFile 444 global archs 445 global compilers 446 global count 447 global dumpSysPath 448 global bmExecutable 449 global bmBreakpointSpec 450 global bmIterationCount 451 global failfast 452 global filters 453 global fs4all 454 global ignore 455 global progress_bar 456 global runHooks 457 global skip_build_and_cleanup 458 global skip_long_running_test 459 global noHeaders 460 global parsable 461 global regexp 462 global rdir 463 global sdir_name 464 global svn_silent 465 global verbose 466 global testdirs 467 global lldb_platform_name 468 global lldb_platform_url 469 global lldb_platform_working_dir 470 global setCrashInfoHook 471 global is_inferior_test_runner 472 global multiprocess_test_subdir 473 global num_threads 474 global output_on_success 475 global no_multiprocess_test_runner 476 global test_runner_name 477 global results_filename 478 global results_formatter_name 479 global results_formatter_options 480 global results_port 481 482 do_help = False 483 484 platform_system = platform.system() 485 platform_machine = platform.machine() 486 487 parser = dotest_args.create_parser() 488 args = dotest_args.parse_args(parser, sys.argv[1:]) 489 490 if args.unset_env_varnames: 491 for env_var in args.unset_env_varnames: 492 if env_var in os.environ: 493 # From Python Doc: When unsetenv() is supported, deletion of items in os.environ 494 # is automatically translated into a corresponding call to unsetenv(). 495 del os.environ[env_var] 496 #os.unsetenv(env_var) 497 498 if args.set_env_vars: 499 for env_var in args.set_env_vars: 500 parts = env_var.split('=', 1) 501 if len(parts) == 1: 502 os.environ[parts[0]] = "" 503 else: 504 os.environ[parts[0]] = parts[1] 505 506 # only print the args if being verbose (and parsable is off) 507 if args.v and not args.q: 508 print(sys.argv) 509 510 if args.h: 511 do_help = True 512 513 if args.compilers: 514 compilers = args.compilers 515 else: 516 # Use a compiler appropriate appropriate for the Apple SDK if one was specified 517 if platform_system == 'Darwin' and args.apple_sdk: 518 compilers = [commands.getoutput('xcrun -sdk "%s" -find clang 2> /dev/null' % (args.apple_sdk))] 519 else: 520 # 'clang' on ubuntu 14.04 is 3.4 so we try clang-3.5 first 521 candidateCompilers = ['clang-3.5', 'clang', 'gcc'] 522 for candidate in candidateCompilers: 523 if which(candidate): 524 compilers = [candidate] 525 break 526 527 if args.channels: 528 lldbtest_config.channels = args.channels 529 530 if args.log_success: 531 lldbtest_config.log_success = args.log_success 532 533 # Set SDKROOT if we are using an Apple SDK 534 if platform_system == 'Darwin' and args.apple_sdk: 535 os.environ['SDKROOT'] = commands.getoutput('xcrun --sdk "%s" --show-sdk-path 2> /dev/null' % (args.apple_sdk)) 536 537 if args.archs: 538 archs = args.archs 539 for arch in archs: 540 if arch.startswith('arm') and platform_system == 'Darwin' and not args.apple_sdk: 541 os.environ['SDKROOT'] = commands.getoutput('xcrun --sdk iphoneos.internal --show-sdk-path 2> /dev/null') 542 if not os.path.exists(os.environ['SDKROOT']): 543 os.environ['SDKROOT'] = commands.getoutput('xcrun --sdk iphoneos --show-sdk-path 2> /dev/null') 544 else: 545 archs = [platform_machine] 546 547 if args.categoriesList: 548 categoriesList = set(test_categories.validate(args.categoriesList, False)) 549 useCategories = True 550 else: 551 categoriesList = [] 552 553 if args.skipCategories: 554 skipCategories = test_categories.validate(args.skipCategories, False) 555 556 if args.D: 557 dumpSysPath = True 558 559 if args.E: 560 cflags_extras = args.E 561 os.environ['CFLAGS_EXTRAS'] = cflags_extras 562 563 # argparse makes sure we have correct options 564 if args.N == 'dwarf': 565 dont_do_dwarf_test = True 566 elif args.N == 'dwo': 567 dont_do_dwo_test = True 568 elif args.N == 'dsym': 569 dont_do_dsym_test = True 570 571 if args.a or args.plus_a: 572 print("Options '-a' and '+a' have been deprecated. Please use the test category\n" 573 "functionality (-G pyapi, --skip-category pyapi) instead.") 574 sys.exit(1) 575 576 if args.m or args.plus_m: 577 print("Options '-m' and '+m' have been deprecated. Please use the test category\n" 578 "functionality (-G lldb-mi, --skip-category lldb-mi) instead.") 579 sys.exit(1) 580 581 if args.plus_b: 582 just_do_benchmarks_test = True 583 584 if args.b: 585 if args.b.startswith('-'): 586 usage(parser) 587 blacklistFile = args.b 588 if not os.path.isfile(blacklistFile): 589 print('Blacklist file:', blacklistFile, 'does not exist!') 590 usage(parser) 591 # Now read the blacklist contents and assign it to blacklist. 592 execfile(blacklistFile, globals(), blacklistConfig) 593 blacklist = blacklistConfig.get('blacklist') 594 595 if args.c: 596 if args.c.startswith('-'): 597 usage(parser) 598 configFile = args.c 599 if not os.path.isfile(configFile): 600 print('Config file:', configFile, 'does not exist!') 601 usage(parser) 602 603 if args.d: 604 sys.stdout.write("Suspending the process %d to wait for debugger to attach...\n" % os.getpid()) 605 sys.stdout.flush() 606 os.kill(os.getpid(), signal.SIGSTOP) 607 608 if args.e: 609 if args.e.startswith('-'): 610 usage(parser) 611 bmExecutable = args.e 612 if not is_exe(bmExecutable): 613 usage(parser) 614 615 if args.F: 616 failfast = True 617 618 if args.f: 619 if any([x.startswith('-') for x in args.f]): 620 usage(parser) 621 filters.extend(args.f) 622 # Shut off multiprocessing mode when additional filters are specified. 623 # The rational is that the user is probably going after a very specific 624 # test and doesn't need a bunch of parallel test runners all looking for 625 # it in a frenzy. Also, '-v' now spits out all test run output even 626 # on success, so the standard recipe for redoing a failing test (with -v 627 # and a -f to filter to the specific test) now causes all test scanning 628 # (in parallel) to print results for do-nothing runs in a very distracting 629 # manner. If we really need filtered parallel runs in the future, consider 630 # adding a --no-output-on-success that prevents -v from setting 631 # output-on-success. 632 no_multiprocess_test_runner = True 633 634 if args.g: 635 fs4all = False 636 637 if args.i: 638 ignore = True 639 640 if args.k: 641 runHooks.extend(args.k) 642 643 if args.l: 644 skip_long_running_test = False 645 646 if args.framework: 647 lldbFrameworkPath = args.framework 648 649 if args.executable: 650 lldbtest_config.lldbExec = args.executable 651 652 if args.libcxx: 653 os.environ["LIBCXX_PATH"] = args.libcxx 654 655 if args.n: 656 noHeaders = True 657 658 if args.p: 659 if args.p.startswith('-'): 660 usage(parser) 661 regexp = args.p 662 663 if args.q: 664 noHeaders = True 665 parsable = True 666 667 if args.P and not args.v: 668 progress_bar = True 669 verbose = 0 670 671 if args.R: 672 if args.R.startswith('-'): 673 usage(parser) 674 rdir = os.path.abspath(args.R) 675 if os.path.exists(rdir): 676 import shutil 677 print('Removing tree:', rdir) 678 shutil.rmtree(rdir) 679 680 if args.r: 681 if args.r.startswith('-'): 682 usage(parser) 683 rdir = os.path.abspath(args.r) 684 if os.path.exists(rdir): 685 print('Relocated directory:', rdir, 'must not exist!') 686 usage(parser) 687 688 if args.S: 689 skip_build_and_cleanup = True 690 691 if args.s: 692 if args.s.startswith('-'): 693 usage(parser) 694 sdir_name = args.s 695 696 if args.t: 697 os.environ['LLDB_COMMAND_TRACE'] = 'YES' 698 699 if args.T: 700 svn_silent = False 701 702 if args.v: 703 verbose = 2 704 705 if args.w: 706 os.environ['LLDB_WAIT_BETWEEN_TEST_CASES'] = 'YES' 707 708 if args.X: 709 if args.X.startswith('-'): 710 usage(parser) 711 excluded.add(args.X) 712 713 if args.x: 714 if args.x.startswith('-'): 715 usage(parser) 716 bmBreakpointSpec = args.x 717 718 # argparse makes sure we have a number 719 if args.y: 720 bmIterationCount = args.y 721 722 # argparse makes sure we have a number 723 if args.sharp: 724 count = args.sharp 725 726 if sys.platform.startswith('win32'): 727 os.environ['LLDB_DISABLE_CRASH_DIALOG'] = str(args.disable_crash_dialog) 728 os.environ['LLDB_LAUNCH_INFERIORS_WITHOUT_CONSOLE'] = str(args.hide_inferior_console) 729 730 if do_help == True: 731 usage(parser) 732 733 if args.no_multiprocess: 734 no_multiprocess_test_runner = True 735 736 if args.inferior: 737 is_inferior_test_runner = True 738 739 # Turn on output_on_sucess if either explicitly added or -v specified. 740 if args.output_on_success or args.v: 741 output_on_success = True 742 743 if args.num_threads: 744 num_threads = args.num_threads 745 746 if args.test_subdir: 747 multiprocess_test_subdir = args.test_subdir 748 749 if args.test_runner_name: 750 test_runner_name = args.test_runner_name 751 752 # Capture test results-related args. 753 if args.results_file: 754 results_filename = args.results_file 755 756 if args.results_port: 757 results_port = args.results_port 758 759 if args.results_file and args.results_port: 760 sys.stderr.write( 761 "only one of --results-file and --results-port should " 762 "be specified\n") 763 usage(args) 764 765 if args.results_formatter: 766 results_formatter_name = args.results_formatter 767 if args.results_formatter_options: 768 results_formatter_options = args.results_formatter_options 769 770 if args.lldb_platform_name: 771 lldb_platform_name = args.lldb_platform_name 772 if args.lldb_platform_url: 773 lldb_platform_url = args.lldb_platform_url 774 if args.lldb_platform_working_dir: 775 lldb_platform_working_dir = args.lldb_platform_working_dir 776 777 if args.event_add_entries and len(args.event_add_entries) > 0: 778 entries = {} 779 # Parse out key=val pairs, separated by comma 780 for keyval in args.event_add_entries.split(","): 781 key_val_entry = keyval.split("=") 782 if len(key_val_entry) == 2: 783 (key, val) = key_val_entry 784 val_parts = val.split(':') 785 if len(val_parts) > 1: 786 (val, val_type) = val_parts 787 if val_type == 'int': 788 val = int(val) 789 entries[key] = val 790 # Tell the event builder to create all events with these 791 # key/val pairs in them. 792 if len(entries) > 0: 793 test_results.EventBuilder.add_entries_to_all_events(entries) 794 795 # Gather all the dirs passed on the command line. 796 if len(args.args) > 0: 797 testdirs = list(map(os.path.abspath, args.args)) 798 # Shut off multiprocessing mode when test directories are specified. 799 no_multiprocess_test_runner = True 800 801 # If '-r dir' is specified, the tests should be run under the relocated 802 # directory. Let's copy the testdirs over. 803 if rdir: 804 from shutil import copytree, ignore_patterns 805 806 tmpdirs = [] 807 orig_testdirs = testdirs[:] 808 for srcdir in testdirs: 809 # For example, /Volumes/data/lldb/svn/ToT/test/functionalities/watchpoint/hello_watchpoint 810 # shall be split into ['/Volumes/data/lldb/svn/ToT/', 'functionalities/watchpoint/hello_watchpoint']. 811 # Utilize the relative path to the 'test' directory to make our destination dir path. 812 if ("test" + os.sep) in srcdir: 813 to_split_on = "test" + os.sep 814 else: 815 to_split_on = "test" 816 dstdir = os.path.join(rdir, srcdir.split(to_split_on)[1]) 817 dstdir = dstdir.rstrip(os.sep) 818 # Don't copy the *.pyc and .svn stuffs. 819 copytree(srcdir, dstdir, ignore=ignore_patterns('*.pyc', '.svn')) 820 tmpdirs.append(dstdir) 821 822 # This will be our modified testdirs. 823 testdirs = tmpdirs 824 825 # With '-r dir' specified, there's no cleanup of intermediate test files. 826 os.environ["LLDB_DO_CLEANUP"] = 'NO' 827 828 # If the original testdirs is ['test'], the make directory has already been copied 829 # recursively and is contained within the rdir/test dir. For anything 830 # else, we would need to copy over the make directory and its contents, 831 # so that, os.listdir(rdir) looks like, for example: 832 # 833 # array_types conditional_break make 834 # 835 # where the make directory contains the Makefile.rules file. 836 if len(testdirs) != 1 or os.path.basename(orig_testdirs[0]) != 'test': 837 scriptdir = os.path.dirname(__file__) 838 # Don't copy the .svn stuffs. 839 copytree(os.path.join(scriptdir, 'make'), os.path.join(rdir, 'make'), 840 ignore=ignore_patterns('.svn')) 841 842 #print("testdirs:", testdirs) 843 844 # Source the configFile if specified. 845 # The side effect, if any, will be felt from this point on. An example 846 # config file may be these simple two lines: 847 # 848 # sys.stderr = open("/tmp/lldbtest-stderr", "w") 849 # sys.stdout = open("/tmp/lldbtest-stdout", "w") 850 # 851 # which will reassign the two file objects to sys.stderr and sys.stdout, 852 # respectively. 853 # 854 # See also lldb-trunk/examples/test/usage-config. 855 global config, pre_flight, post_flight, lldbtest_remote_sandbox, lldbtest_remote_shell_template 856 if configFile: 857 # Pass config (a dictionary) as the locals namespace for side-effect. 858 execfile(configFile, globals(), config) 859 #print("config:", config) 860 if "pre_flight" in config: 861 pre_flight = config["pre_flight"] 862 if not six.callable(pre_flight): 863 print("fatal error: pre_flight is not callable, exiting.") 864 sys.exit(1) 865 if "post_flight" in config: 866 post_flight = config["post_flight"] 867 if not six.callable(post_flight): 868 print("fatal error: post_flight is not callable, exiting.") 869 sys.exit(1) 870 if "lldbtest_remote_sandbox" in config: 871 lldbtest_remote_sandbox = config["lldbtest_remote_sandbox"] 872 if "lldbtest_remote_shell_template" in config: 873 lldbtest_remote_shell_template = config["lldbtest_remote_shell_template"] 874 #print("sys.stderr:", sys.stderr) 875 #print("sys.stdout:", sys.stdout) 876 877def getXcodeOutputPaths(lldbRootDirectory): 878 result = [] 879 880 # These are for xcode build directories. 881 xcode3_build_dir = ['build'] 882 xcode4_build_dir = ['build', 'lldb', 'Build', 'Products'] 883 884 configurations = [['Debug'], ['DebugClang'], ['Release'], ['BuildAndIntegration']] 885 xcode_build_dirs = [xcode3_build_dir, xcode4_build_dir] 886 for configuration in configurations: 887 for xcode_build_dir in xcode_build_dirs: 888 outputPath = os.path.join(lldbRootDirectory, *(xcode_build_dir+configuration) ) 889 result.append(outputPath) 890 891 return result 892 893 894def createSocketToLocalPort(port): 895 def socket_closer(s): 896 """Close down an opened socket properly.""" 897 s.shutdown(socket.SHUT_RDWR) 898 s.close() 899 900 sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 901 sock.connect(("localhost", port)) 902 return (sock, lambda: socket_closer(sock)) 903 904 905def setupTestResults(): 906 """Sets up test results-related objects based on arg settings.""" 907 global results_filename 908 global results_file_object 909 global results_formatter_name 910 global results_formatter_object 911 global results_formatter_options 912 global results_port 913 914 default_formatter_name = None 915 cleanup_func = None 916 917 if results_filename: 918 # Open the results file for writing. 919 if results_filename == 'stdout': 920 results_file_object = sys.stdout 921 cleanup_func = None 922 elif results_filename == 'stderr': 923 results_file_object = sys.stderr 924 cleanup_func = None 925 else: 926 results_file_object = open(results_filename, "w") 927 cleanup_func = results_file_object.close 928 default_formatter_name = "test_results.XunitFormatter" 929 elif results_port: 930 # Connect to the specified localhost port. 931 results_file_object, cleanup_func = createSocketToLocalPort( 932 results_port) 933 default_formatter_name = "test_results.RawPickledFormatter" 934 935 if results_file_object: 936 # We care about the formatter. Choose user-specified or, if 937 # none specified, use the default for the output type. 938 if results_formatter_name: 939 formatter_name = results_formatter_name 940 else: 941 formatter_name = default_formatter_name 942 943 # Create an instance of the class. First figure out the package/module. 944 components = formatter_name.split(".") 945 module = importlib.import_module(".".join(components[:-1])) 946 947 # Create the class name we need to load. 948 clazz = getattr(module, components[-1]) 949 950 # Handle formatter options for the results formatter class. 951 formatter_arg_parser = clazz.arg_parser() 952 if results_formatter_options and len(results_formatter_options) > 0: 953 command_line_options = results_formatter_options 954 else: 955 command_line_options = [] 956 957 formatter_options = formatter_arg_parser.parse_args( 958 command_line_options) 959 960 # Create the TestResultsFormatter given the processed options. 961 results_formatter_object = clazz(results_file_object, formatter_options) 962 963 # Start the results formatter session - we'll only have one 964 # during a given dotest process invocation. 965 initialize_event = EventBuilder.bare_event("initialize") 966 if isMultiprocessTestRunner(): 967 if test_runner_name is not None and test_runner_name == "serial": 968 # Only one worker queue here. 969 worker_count = 1 970 else: 971 # Workers will be the number of threads specified. 972 worker_count = num_threads 973 else: 974 worker_count = 1 975 initialize_event["worker_count"] = worker_count 976 977 results_formatter_object.handle_event(initialize_event) 978 979 def shutdown_formatter(): 980 # Tell the formatter to write out anything it may have 981 # been saving until the very end (e.g. xUnit results 982 # can't complete its output until this point). 983 results_formatter_object.send_terminate_as_needed() 984 985 # And now close out the output file-like object. 986 if cleanup_func is not None: 987 cleanup_func() 988 989 atexit.register(shutdown_formatter) 990 991 992def getOutputPaths(lldbRootDirectory): 993 """ 994 Returns typical build output paths for the lldb executable 995 996 lldbDirectory - path to the root of the lldb svn/git repo 997 """ 998 result = [] 999 1000 if sys.platform == 'darwin': 1001 result.extend(getXcodeOutputPaths(lldbRootDirectory)) 1002 1003 # cmake builds? look for build or build/host folder next to llvm directory 1004 # lldb is located in llvm/tools/lldb so we need to go up three levels 1005 llvmParentDir = os.path.abspath(os.path.join(lldbRootDirectory, os.pardir, os.pardir, os.pardir)) 1006 result.append(os.path.join(llvmParentDir, 'build', 'bin')) 1007 result.append(os.path.join(llvmParentDir, 'build', 'host', 'bin')) 1008 1009 # some cmake developers keep their build directory beside their lldb directory 1010 lldbParentDir = os.path.abspath(os.path.join(lldbRootDirectory, os.pardir)) 1011 result.append(os.path.join(lldbParentDir, 'build', 'bin')) 1012 result.append(os.path.join(lldbParentDir, 'build', 'host', 'bin')) 1013 1014 return result 1015 1016def setupSysPath(): 1017 """ 1018 Add LLDB.framework/Resources/Python to the search paths for modules. 1019 As a side effect, we also discover the 'lldb' executable and export it here. 1020 """ 1021 1022 global rdir 1023 global testdirs 1024 global dumpSysPath 1025 global noHeaders 1026 global svn_info 1027 global svn_silent 1028 global lldbFrameworkPath 1029 1030 # Get the directory containing the current script. 1031 if "DOTEST_PROFILE" in os.environ and "DOTEST_SCRIPT_DIR" in os.environ: 1032 scriptPath = os.environ["DOTEST_SCRIPT_DIR"] 1033 else: 1034 scriptPath = os.path.dirname(os.path.realpath(__file__)) 1035 if not scriptPath.endswith('test'): 1036 print("This script expects to reside in lldb's test directory.") 1037 sys.exit(-1) 1038 1039 if rdir: 1040 # Set up the LLDB_TEST environment variable appropriately, so that the 1041 # individual tests can be located relatively. 1042 # 1043 # See also lldbtest.TestBase.setUpClass(cls). 1044 if len(testdirs) == 1 and os.path.basename(testdirs[0]) == 'test': 1045 os.environ["LLDB_TEST"] = os.path.join(rdir, 'test') 1046 else: 1047 os.environ["LLDB_TEST"] = rdir 1048 else: 1049 os.environ["LLDB_TEST"] = scriptPath 1050 1051 # Set up the LLDB_SRC environment variable, so that the tests can locate 1052 # the LLDB source code. 1053 os.environ["LLDB_SRC"] = lldbsuite.lldb_root 1054 1055 pluginPath = os.path.join(scriptPath, 'plugins') 1056 toolsLLDBMIPath = os.path.join(scriptPath, 'tools', 'lldb-mi') 1057 toolsLLDBServerPath = os.path.join(scriptPath, 'tools', 'lldb-server') 1058 1059 # Insert script dir, plugin dir, lldb-mi dir and lldb-server dir to the sys.path. 1060 sys.path.insert(0, scriptPath) 1061 sys.path.insert(0, pluginPath) 1062 sys.path.insert(0, toolsLLDBMIPath) # Adding test/tools/lldb-mi to the path makes it easy 1063 # to "import lldbmi_testcase" from the MI tests 1064 sys.path.insert(0, toolsLLDBServerPath) # Adding test/tools/lldb-server to the path makes it easy 1065 # to "import lldbgdbserverutils" from the lldb-server tests 1066 1067 # This is the root of the lldb git/svn checkout 1068 # When this changes over to a package instead of a standalone script, this 1069 # will be `lldbsuite.lldb_root` 1070 lldbRootDirectory = lldbsuite.lldb_root 1071 1072 # Some of the tests can invoke the 'lldb' command directly. 1073 # We'll try to locate the appropriate executable right here. 1074 1075 # The lldb executable can be set from the command line 1076 # if it's not set, we try to find it now 1077 # first, we try the environment 1078 if not lldbtest_config.lldbExec: 1079 # First, you can define an environment variable LLDB_EXEC specifying the 1080 # full pathname of the lldb executable. 1081 if "LLDB_EXEC" in os.environ: 1082 lldbtest_config.lldbExec = os.environ["LLDB_EXEC"] 1083 1084 if not lldbtest_config.lldbExec: 1085 outputPaths = getOutputPaths(lldbRootDirectory) 1086 for outputPath in outputPaths: 1087 candidatePath = os.path.join(outputPath, 'lldb') 1088 if is_exe(candidatePath): 1089 lldbtest_config.lldbExec = candidatePath 1090 break 1091 1092 if not lldbtest_config.lldbExec: 1093 # Last, check the path 1094 lldbtest_config.lldbExec = which('lldb') 1095 1096 if lldbtest_config.lldbExec and not is_exe(lldbtest_config.lldbExec): 1097 print("'{}' is not a path to a valid executable".format(lldbtest_config.lldbExec)) 1098 lldbtest_config.lldbExec = None 1099 1100 if not lldbtest_config.lldbExec: 1101 print("The 'lldb' executable cannot be located. Some of the tests may not be run as a result.") 1102 sys.exit(-1) 1103 1104 lldbLibDir = os.path.dirname(lldbtest_config.lldbExec) # confusingly, this is the "bin" directory 1105 os.environ["LLDB_LIB_DIR"] = lldbLibDir 1106 lldbImpLibDir = os.path.join(lldbLibDir, '..', 'lib') if sys.platform.startswith('win32') else lldbLibDir 1107 os.environ["LLDB_IMPLIB_DIR"] = lldbImpLibDir 1108 if not noHeaders: 1109 print("LLDB library dir:", os.environ["LLDB_LIB_DIR"]) 1110 print("LLDB import library dir:", os.environ["LLDB_IMPLIB_DIR"]) 1111 os.system('%s -v' % lldbtest_config.lldbExec) 1112 1113 # Assume lldb-mi is in same place as lldb 1114 # If not found, disable the lldb-mi tests 1115 lldbMiExec = None 1116 if lldbtest_config.lldbExec and is_exe(lldbtest_config.lldbExec + "-mi"): 1117 lldbMiExec = lldbtest_config.lldbExec + "-mi" 1118 if not lldbMiExec: 1119 if not shouldSkipBecauseOfCategories(["lldb-mi"]): 1120 print("The 'lldb-mi' executable cannot be located. The lldb-mi tests can not be run as a result.") 1121 skipCategories.append("lldb-mi") 1122 else: 1123 os.environ["LLDBMI_EXEC"] = lldbMiExec 1124 1125 # Skip printing svn/git information when running in parsable (lit-test compatibility) mode 1126 if not svn_silent and not parsable: 1127 if os.path.isdir(os.path.join(lldbRootDirectory, '.svn')) and which("svn") is not None: 1128 pipe = subprocess.Popen([which("svn"), "info", lldbRootDirectory], stdout = subprocess.PIPE) 1129 svn_info = pipe.stdout.read() 1130 elif os.path.isdir(os.path.join(lldbRootDirectory, '.git')) and which("git") is not None: 1131 pipe = subprocess.Popen([which("git"), "svn", "info", lldbRootDirectory], stdout = subprocess.PIPE) 1132 svn_info = pipe.stdout.read() 1133 if not noHeaders: 1134 print(svn_info) 1135 1136 global ignore 1137 1138 lldbPythonDir = None # The directory that contains 'lldb/__init__.py' 1139 if lldbFrameworkPath: 1140 candidatePath = os.path.join(lldbFrameworkPath, 'Resources', 'Python') 1141 if os.path.isfile(os.path.join(candidatePath, 'lldb/__init__.py')): 1142 lldbPythonDir = candidatePath 1143 if not lldbPythonDir: 1144 print('Resources/Python/lldb/__init__.py was not found in ' + lldbFrameworkPath) 1145 sys.exit(-1) 1146 else: 1147 # The '-i' option is used to skip looking for lldb.py in the build tree. 1148 if ignore: 1149 return 1150 1151 # If our lldb supports the -P option, use it to find the python path: 1152 init_in_python_dir = os.path.join('lldb', '__init__.py') 1153 1154 lldb_dash_p_result = subprocess.check_output([lldbtest_config.lldbExec, "-P"], stderr=subprocess.STDOUT) 1155 1156 if lldb_dash_p_result and not lldb_dash_p_result.startswith(("<", "lldb: invalid option:")) \ 1157 and not lldb_dash_p_result.startswith("Traceback"): 1158 lines = lldb_dash_p_result.splitlines() 1159 1160 # Workaround for readline vs libedit issue on FreeBSD. If stdout 1161 # is not a terminal Python executes 1162 # rl_variable_bind ("enable-meta-key", "off"); 1163 # This produces a warning with FreeBSD's libedit because the 1164 # enable-meta-key variable is unknown. Not an issue on Apple 1165 # because cpython commit f0ab6f9f0603 added a #ifndef __APPLE__ 1166 # around the call. See http://bugs.python.org/issue19884 for more 1167 # information. For now we just discard the warning output. 1168 if len(lines) >= 1 and lines[0].startswith("bind: Invalid command"): 1169 lines.pop(0) 1170 1171 # Taking the last line because lldb outputs 1172 # 'Cannot read termcap database;\nusing dumb terminal settings.\n' 1173 # before the path 1174 if len(lines) >= 1 and os.path.isfile(os.path.join(lines[-1], init_in_python_dir)): 1175 lldbPythonDir = lines[-1] 1176 if "freebsd" in sys.platform or "linux" in sys.platform: 1177 os.environ['LLDB_LIB_DIR'] = os.path.join(lldbPythonDir, '..', '..') 1178 1179 if not lldbPythonDir: 1180 if platform.system() == "Darwin": 1181 python_resource_dir = ['LLDB.framework', 'Resources', 'Python'] 1182 outputPaths = getXcodeOutputPaths() 1183 for outputPath in outputPaths: 1184 candidatePath = os.path.join(outputPath, python_resource_dir) 1185 if os.path.isfile(os.path.join(candidatePath, init_in_python_dir)): 1186 lldbPythonDir = candidatePath 1187 break 1188 1189 if not lldbPythonDir: 1190 print('This script requires lldb.py to be in either ' + dbgPath + ',', end=' ') 1191 print(relPath + ', or ' + baiPath + '. Some tests might fail.') 1192 else: 1193 print("Unable to load lldb extension module. Possible reasons for this include:") 1194 print(" 1) LLDB was built with LLDB_DISABLE_PYTHON=1") 1195 print(" 2) PYTHONPATH and PYTHONHOME are not set correctly. PYTHONHOME should refer to") 1196 print(" the version of Python that LLDB built and linked against, and PYTHONPATH") 1197 print(" should contain the Lib directory for the same python distro, as well as the") 1198 print(" location of LLDB\'s site-packages folder.") 1199 print(" 3) A different version of Python than that which was built against is exported in") 1200 print(" the system\'s PATH environment variable, causing conflicts.") 1201 print(" 4) The executable '%s' could not be found. Please check " % lldbExecutable) 1202 print(" that it exists and is executable.") 1203 1204 if lldbPythonDir: 1205 lldbPythonDir = os.path.normpath(lldbPythonDir) 1206 # Some of the code that uses this path assumes it hasn't resolved the Versions... link. 1207 # If the path we've constructed looks like that, then we'll strip out the Versions/A part. 1208 (before, frameWithVersion, after) = lldbPythonDir.rpartition("LLDB.framework/Versions/A") 1209 if frameWithVersion != "" : 1210 lldbPythonDir = before + "LLDB.framework" + after 1211 1212 lldbPythonDir = os.path.abspath(lldbPythonDir) 1213 1214 # If tests need to find LLDB_FRAMEWORK, now they can do it 1215 os.environ["LLDB_FRAMEWORK"] = os.path.dirname(os.path.dirname(lldbPythonDir)) 1216 1217 # This is to locate the lldb.py module. Insert it right after sys.path[0]. 1218 sys.path[1:1] = [lldbPythonDir] 1219 if dumpSysPath: 1220 print("sys.path:", sys.path) 1221 1222def visit(prefix, dir, names): 1223 """Visitor function for os.path.walk(path, visit, arg).""" 1224 1225 global suite 1226 global regexp 1227 global filters 1228 global fs4all 1229 global excluded 1230 global all_tests 1231 1232 if set(dir.split(os.sep)).intersection(excluded): 1233 #print("Detected an excluded dir component: %s" % dir) 1234 return 1235 1236 for name in names: 1237 if os.path.isdir(os.path.join(dir, name)): 1238 continue 1239 1240 if '.py' == os.path.splitext(name)[1] and name.startswith(prefix): 1241 1242 if name in all_tests: 1243 raise Exception("Found multiple tests with the name %s" % name) 1244 all_tests.add(name) 1245 1246 # Try to match the regexp pattern, if specified. 1247 if regexp: 1248 import re 1249 if re.search(regexp, name): 1250 #print("Filename: '%s' matches pattern: '%s'" % (name, regexp)) 1251 pass 1252 else: 1253 #print("Filename: '%s' does not match pattern: '%s'" % (name, regexp)) 1254 continue 1255 1256 # We found a match for our test. Add it to the suite. 1257 1258 # Update the sys.path first. 1259 if not sys.path.count(dir): 1260 sys.path.insert(0, dir) 1261 base = os.path.splitext(name)[0] 1262 1263 # Thoroughly check the filterspec against the base module and admit 1264 # the (base, filterspec) combination only when it makes sense. 1265 filterspec = None 1266 for filterspec in filters: 1267 # Optimistically set the flag to True. 1268 filtered = True 1269 module = __import__(base) 1270 parts = filterspec.split('.') 1271 obj = module 1272 for part in parts: 1273 try: 1274 parent, obj = obj, getattr(obj, part) 1275 except AttributeError: 1276 # The filterspec has failed. 1277 filtered = False 1278 break 1279 1280 # If filtered, we have a good filterspec. Add it. 1281 if filtered: 1282 #print("adding filter spec %s to module %s" % (filterspec, module)) 1283 suite.addTests( 1284 unittest2.defaultTestLoader.loadTestsFromName(filterspec, module)) 1285 continue 1286 1287 # Forgo this module if the (base, filterspec) combo is invalid 1288 # and no '-g' option is specified 1289 if filters and fs4all and not filtered: 1290 continue 1291 1292 # Add either the filtered test case(s) (which is done before) or the entire test class. 1293 if not filterspec or not filtered: 1294 # A simple case of just the module name. Also the failover case 1295 # from the filterspec branch when the (base, filterspec) combo 1296 # doesn't make sense. 1297 suite.addTests(unittest2.defaultTestLoader.loadTestsFromName(base)) 1298 1299 1300def disabledynamics(): 1301 import lldb 1302 ci = lldb.DBG.GetCommandInterpreter() 1303 res = lldb.SBCommandReturnObject() 1304 ci.HandleCommand("setting set target.prefer-dynamic-value no-dynamic-values", res, False) 1305 if not res.Succeeded(): 1306 raise Exception('disabling dynamic type support failed') 1307 1308def lldbLoggings(): 1309 import lldb 1310 """Check and do lldb loggings if necessary.""" 1311 1312 # Turn on logging for debugging purposes if ${LLDB_LOG} environment variable is 1313 # defined. Use ${LLDB_LOG} to specify the log file. 1314 ci = lldb.DBG.GetCommandInterpreter() 1315 res = lldb.SBCommandReturnObject() 1316 if ("LLDB_LOG" in os.environ): 1317 open(os.environ["LLDB_LOG"], 'w').close() 1318 if ("LLDB_LOG_OPTION" in os.environ): 1319 lldb_log_option = os.environ["LLDB_LOG_OPTION"] 1320 else: 1321 lldb_log_option = "event process expr state api" 1322 ci.HandleCommand( 1323 "log enable -n -f " + os.environ["LLDB_LOG"] + " lldb " + lldb_log_option, 1324 res) 1325 if not res.Succeeded(): 1326 raise Exception('log enable failed (check LLDB_LOG env variable)') 1327 1328 if ("LLDB_LINUX_LOG" in os.environ): 1329 open(os.environ["LLDB_LINUX_LOG"], 'w').close() 1330 if ("LLDB_LINUX_LOG_OPTION" in os.environ): 1331 lldb_log_option = os.environ["LLDB_LINUX_LOG_OPTION"] 1332 else: 1333 lldb_log_option = "event process expr state api" 1334 ci.HandleCommand( 1335 "log enable -n -f " + os.environ["LLDB_LINUX_LOG"] + " linux " + lldb_log_option, 1336 res) 1337 if not res.Succeeded(): 1338 raise Exception('log enable failed (check LLDB_LINUX_LOG env variable)') 1339 1340 # Ditto for gdb-remote logging if ${GDB_REMOTE_LOG} environment variable is defined. 1341 # Use ${GDB_REMOTE_LOG} to specify the log file. 1342 if ("GDB_REMOTE_LOG" in os.environ): 1343 if ("GDB_REMOTE_LOG_OPTION" in os.environ): 1344 gdb_remote_log_option = os.environ["GDB_REMOTE_LOG_OPTION"] 1345 else: 1346 gdb_remote_log_option = "packets process" 1347 ci.HandleCommand( 1348 "log enable -n -f " + os.environ["GDB_REMOTE_LOG"] + " gdb-remote " 1349 + gdb_remote_log_option, 1350 res) 1351 if not res.Succeeded(): 1352 raise Exception('log enable failed (check GDB_REMOTE_LOG env variable)') 1353 1354def getMyCommandLine(): 1355 return ' '.join(sys.argv) 1356 1357# ======================================== # 1358# # 1359# Execution of the test driver starts here # 1360# # 1361# ======================================== # 1362 1363def checkDsymForUUIDIsNotOn(): 1364 cmd = ["defaults", "read", "com.apple.DebugSymbols"] 1365 pipe = subprocess.Popen(cmd, stdout = subprocess.PIPE, stderr = subprocess.STDOUT) 1366 cmd_output = pipe.stdout.read() 1367 if cmd_output and "DBGFileMappedPaths = " in cmd_output: 1368 print("%s =>" % ' '.join(cmd)) 1369 print(cmd_output) 1370 print("Disable automatic lookup and caching of dSYMs before running the test suite!") 1371 print("Exiting...") 1372 sys.exit(0) 1373 1374def exitTestSuite(exitCode = None): 1375 import lldb 1376 lldb.SBDebugger.Terminate() 1377 if exitCode: 1378 sys.exit(exitCode) 1379 1380 1381def isMultiprocessTestRunner(): 1382 # We're not multiprocess when we're either explicitly 1383 # the inferior (as specified by the multiprocess test 1384 # runner) OR we've been told to skip using the multiprocess 1385 # test runner 1386 return not (is_inferior_test_runner or no_multiprocess_test_runner) 1387 1388def run_suite(): 1389 global just_do_benchmarks_test 1390 global dont_do_dsym_test 1391 global dont_do_dwarf_test 1392 global dont_do_dwo_test 1393 global blacklist 1394 global blacklistConfig 1395 global categoriesList 1396 global validCategories 1397 global useCategories 1398 global skipCategories 1399 global lldbFrameworkPath 1400 global configFile 1401 global archs 1402 global compilers 1403 global count 1404 global dumpSysPath 1405 global bmExecutable 1406 global bmBreakpointSpec 1407 global bmIterationCount 1408 global failed 1409 global failfast 1410 global filters 1411 global fs4all 1412 global ignore 1413 global progress_bar 1414 global runHooks 1415 global skip_build_and_cleanup 1416 global skip_long_running_test 1417 global noHeaders 1418 global parsable 1419 global regexp 1420 global rdir 1421 global sdir_name 1422 global svn_silent 1423 global verbose 1424 global testdirs 1425 global lldb_platform_name 1426 global lldb_platform_url 1427 global lldb_platform_working_dir 1428 global setCrashInfoHook 1429 global is_inferior_test_runner 1430 global multiprocess_test_subdir 1431 global num_threads 1432 global output_on_success 1433 global no_multiprocess_test_runner 1434 global test_runner_name 1435 global results_filename 1436 global results_formatter_name 1437 global results_formatter_options 1438 global results_port 1439 1440 # On MacOS X, check to make sure that domain for com.apple.DebugSymbols defaults 1441 # does not exist before proceeding to running the test suite. 1442 if sys.platform.startswith("darwin"): 1443 checkDsymForUUIDIsNotOn() 1444 1445 # 1446 # Start the actions by first parsing the options while setting up the test 1447 # directories, followed by setting up the search paths for lldb utilities; 1448 # then, we walk the directory trees and collect the tests into our test suite. 1449 # 1450 parseOptionsAndInitTestdirs() 1451 1452 # Setup test results (test results formatter and output handling). 1453 setupTestResults() 1454 1455 # If we are running as the multiprocess test runner, kick off the 1456 # multiprocess test runner here. 1457 if isMultiprocessTestRunner(): 1458 import dosep 1459 dosep.main(output_on_success, num_threads, multiprocess_test_subdir, 1460 test_runner_name, results_formatter_object) 1461 raise Exception("should never get here") 1462 elif is_inferior_test_runner: 1463 # Shut off Ctrl-C processing in inferiors. The parallel 1464 # test runner handles this more holistically. 1465 signal.signal(signal.SIGINT, signal.SIG_IGN) 1466 1467 setupSysPath() 1468 setupCrashInfoHook() 1469 1470 # 1471 # If '-l' is specified, do not skip the long running tests. 1472 if not skip_long_running_test: 1473 os.environ["LLDB_SKIP_LONG_RUNNING_TEST"] = "NO" 1474 1475 # For the time being, let's bracket the test runner within the 1476 # lldb.SBDebugger.Initialize()/Terminate() pair. 1477 import lldb 1478 1479 # Create a singleton SBDebugger in the lldb namespace. 1480 lldb.DBG = lldb.SBDebugger.Create() 1481 1482 if lldb_platform_name: 1483 print("Setting up remote platform '%s'" % (lldb_platform_name)) 1484 lldb.remote_platform = lldb.SBPlatform(lldb_platform_name) 1485 if not lldb.remote_platform.IsValid(): 1486 print("error: unable to create the LLDB platform named '%s'." % (lldb_platform_name)) 1487 exitTestSuite(1) 1488 if lldb_platform_url: 1489 # We must connect to a remote platform if a LLDB platform URL was specified 1490 print("Connecting to remote platform '%s' at '%s'..." % (lldb_platform_name, lldb_platform_url)) 1491 lldb.platform_url = lldb_platform_url 1492 platform_connect_options = lldb.SBPlatformConnectOptions(lldb_platform_url) 1493 err = lldb.remote_platform.ConnectRemote(platform_connect_options) 1494 if err.Success(): 1495 print("Connected.") 1496 else: 1497 print("error: failed to connect to remote platform using URL '%s': %s" % (lldb_platform_url, err)) 1498 exitTestSuite(1) 1499 else: 1500 lldb.platform_url = None 1501 1502 if lldb_platform_working_dir: 1503 print("Setting remote platform working directory to '%s'..." % (lldb_platform_working_dir)) 1504 lldb.remote_platform.SetWorkingDirectory(lldb_platform_working_dir) 1505 1506 lldb.remote_platform_working_dir = lldb_platform_working_dir 1507 lldb.DBG.SetSelectedPlatform(lldb.remote_platform) 1508 else: 1509 lldb.remote_platform = None 1510 lldb.remote_platform_working_dir = None 1511 lldb.platform_url = None 1512 1513 target_platform = lldb.DBG.GetSelectedPlatform().GetTriple().split('-')[2] 1514 1515 # By default, both dsym, dwarf and dwo tests are performed. 1516 # Use @dsym_test, @dwarf_test or @dwo_test decorators, defined in lldbtest.py, to mark a test as 1517 # a dsym, dwarf or dwo test. Use '-N dsym', '-N dwarf' or '-N dwo' to exclude dsym, dwarf or 1518 # dwo tests from running. 1519 dont_do_dsym_test = dont_do_dsym_test or any(platform in target_platform for platform in ["linux", "freebsd", "windows"]) 1520 dont_do_dwo_test = dont_do_dwo_test or any(platform in target_platform for platform in ["darwin", "macosx", "ios"]) 1521 1522 # Don't do debugserver tests on everything except OS X. 1523 dont_do_debugserver_test = "linux" in target_platform or "freebsd" in target_platform or "windows" in target_platform 1524 1525 # Don't do lldb-server (llgs) tests on anything except Linux. 1526 dont_do_llgs_test = not ("linux" in target_platform) 1527 1528 # 1529 # Walk through the testdirs while collecting tests. 1530 # 1531 for testdir in testdirs: 1532 os.path.walk(testdir, visit, 'Test') 1533 1534 # 1535 # Now that we have loaded all the test cases, run the whole test suite. 1536 # 1537 1538 # Put the blacklist in the lldb namespace, to be used by lldb.TestBase. 1539 lldb.blacklist = blacklist 1540 1541 # The pre_flight and post_flight come from reading a config file. 1542 lldb.pre_flight = pre_flight 1543 lldb.post_flight = post_flight 1544 def getsource_if_available(obj): 1545 """ 1546 Return the text of the source code for an object if available. Otherwise, 1547 a print representation is returned. 1548 """ 1549 import inspect 1550 try: 1551 return inspect.getsource(obj) 1552 except: 1553 return repr(obj) 1554 1555 if not noHeaders: 1556 print("lldb.pre_flight:", getsource_if_available(lldb.pre_flight)) 1557 print("lldb.post_flight:", getsource_if_available(lldb.post_flight)) 1558 1559 # If either pre_flight or post_flight is defined, set lldb.test_remote to True. 1560 if lldb.pre_flight or lldb.post_flight: 1561 lldb.test_remote = True 1562 else: 1563 lldb.test_remote = False 1564 1565 # So do the lldbtest_remote_sandbox and lldbtest_remote_shell_template variables. 1566 lldb.lldbtest_remote_sandbox = lldbtest_remote_sandbox 1567 lldb.lldbtest_remote_sandboxed_executable = None 1568 lldb.lldbtest_remote_shell_template = lldbtest_remote_shell_template 1569 1570 # Put all these test decorators in the lldb namespace. 1571 lldb.just_do_benchmarks_test = just_do_benchmarks_test 1572 lldb.dont_do_dsym_test = dont_do_dsym_test 1573 lldb.dont_do_dwarf_test = dont_do_dwarf_test 1574 lldb.dont_do_dwo_test = dont_do_dwo_test 1575 lldb.dont_do_debugserver_test = dont_do_debugserver_test 1576 lldb.dont_do_llgs_test = dont_do_llgs_test 1577 1578 # Do we need to skip build and cleanup? 1579 lldb.skip_build_and_cleanup = skip_build_and_cleanup 1580 1581 # Put bmExecutable, bmBreakpointSpec, and bmIterationCount into the lldb namespace, too. 1582 lldb.bmExecutable = bmExecutable 1583 lldb.bmBreakpointSpec = bmBreakpointSpec 1584 lldb.bmIterationCount = bmIterationCount 1585 1586 # And don't forget the runHooks! 1587 lldb.runHooks = runHooks 1588 1589 # Turn on lldb loggings if necessary. 1590 lldbLoggings() 1591 1592 # Disable default dynamic types for testing purposes 1593 disabledynamics() 1594 1595 # Install the control-c handler. 1596 unittest2.signals.installHandler() 1597 1598 # If sdir_name is not specified through the '-s sdir_name' option, get a 1599 # timestamp string and export it as LLDB_SESSION_DIR environment var. This will 1600 # be used when/if we want to dump the session info of individual test cases 1601 # later on. 1602 # 1603 # See also TestBase.dumpSessionInfo() in lldbtest.py. 1604 import datetime 1605 # The windows platforms don't like ':' in the pathname. 1606 timestamp_started = datetime.datetime.now().strftime("%Y-%m-%d-%H_%M_%S") 1607 if not sdir_name: 1608 sdir_name = timestamp_started 1609 os.environ["LLDB_SESSION_DIRNAME"] = os.path.join(os.getcwd(), sdir_name) 1610 1611 if not noHeaders: 1612 sys.stderr.write("\nSession logs for test failures/errors/unexpected successes" 1613 " will go into directory '%s'\n" % sdir_name) 1614 sys.stderr.write("Command invoked: %s\n" % getMyCommandLine()) 1615 1616 if not os.path.isdir(sdir_name): 1617 try: 1618 os.mkdir(sdir_name) 1619 except OSError as exception: 1620 if exception.errno != errno.EEXIST: 1621 raise 1622 where_to_save_session = os.getcwd() 1623 fname = os.path.join(sdir_name, "TestStarted-%d" % os.getpid()) 1624 with open(fname, "w") as f: 1625 print("Test started at: %s\n" % timestamp_started, file=f) 1626 print(svn_info, file=f) 1627 print("Command invoked: %s\n" % getMyCommandLine(), file=f) 1628 1629 # 1630 # Invoke the default TextTestRunner to run the test suite, possibly iterating 1631 # over different configurations. 1632 # 1633 1634 iterArchs = False 1635 iterCompilers = False 1636 1637 if not archs and "archs" in config: 1638 archs = config["archs"] 1639 1640 if isinstance(archs, list) and len(archs) >= 1: 1641 iterArchs = True 1642 1643 if not compilers and "compilers" in config: 1644 compilers = config["compilers"] 1645 1646 # 1647 # Add some intervention here to sanity check that the compilers requested are sane. 1648 # If found not to be an executable program, the invalid one is dropped from the list. 1649 for i in range(len(compilers)): 1650 c = compilers[i] 1651 if which(c): 1652 continue 1653 else: 1654 if sys.platform.startswith("darwin"): 1655 pipe = subprocess.Popen(['xcrun', '-find', c], stdout = subprocess.PIPE, stderr = subprocess.STDOUT) 1656 cmd_output = pipe.stdout.read() 1657 if cmd_output: 1658 if "not found" in cmd_output: 1659 print("dropping %s from the compilers used" % c) 1660 compilers.remove(i) 1661 else: 1662 compilers[i] = cmd_output.split('\n')[0] 1663 print("'xcrun -find %s' returning %s" % (c, compilers[i])) 1664 1665 if not parsable: 1666 print("compilers=%s" % str(compilers)) 1667 1668 if not compilers or len(compilers) == 0: 1669 print("No eligible compiler found, exiting.") 1670 exitTestSuite(1) 1671 1672 if isinstance(compilers, list) and len(compilers) >= 1: 1673 iterCompilers = True 1674 1675 # Make a shallow copy of sys.path, we need to manipulate the search paths later. 1676 # This is only necessary if we are relocated and with different configurations. 1677 if rdir: 1678 old_sys_path = sys.path[:] 1679 # If we iterate on archs or compilers, there is a chance we want to split stderr/stdout. 1680 if iterArchs or iterCompilers: 1681 old_stderr = sys.stderr 1682 old_stdout = sys.stdout 1683 new_stderr = None 1684 new_stdout = None 1685 1686 # Iterating over all possible architecture and compiler combinations. 1687 for ia in range(len(archs) if iterArchs else 1): 1688 archConfig = "" 1689 if iterArchs: 1690 os.environ["ARCH"] = archs[ia] 1691 archConfig = "arch=%s" % archs[ia] 1692 for ic in range(len(compilers) if iterCompilers else 1): 1693 if iterCompilers: 1694 os.environ["CC"] = compilers[ic] 1695 configString = "%s compiler=%s" % (archConfig, compilers[ic]) 1696 else: 1697 configString = archConfig 1698 1699 if iterArchs or iterCompilers: 1700 # Translate ' ' to '-' for pathname component. 1701 from string import maketrans 1702 tbl = maketrans(' ', '-') 1703 configPostfix = configString.translate(tbl) 1704 1705 # Check whether we need to split stderr/stdout into configuration 1706 # specific files. 1707 if old_stderr.name != '<stderr>' and config.get('split_stderr'): 1708 if new_stderr: 1709 new_stderr.close() 1710 new_stderr = open("%s.%s" % (old_stderr.name, configPostfix), "w") 1711 sys.stderr = new_stderr 1712 if old_stdout.name != '<stdout>' and config.get('split_stdout'): 1713 if new_stdout: 1714 new_stdout.close() 1715 new_stdout = open("%s.%s" % (old_stdout.name, configPostfix), "w") 1716 sys.stdout = new_stdout 1717 1718 # If we specified a relocated directory to run the test suite, do 1719 # the extra housekeeping to copy the testdirs to a configStringified 1720 # directory and to update sys.path before invoking the test runner. 1721 # The purpose is to separate the configuration-specific directories 1722 # from each other. 1723 if rdir: 1724 from shutil import copytree, rmtree, ignore_patterns 1725 1726 newrdir = "%s.%s" % (rdir, configPostfix) 1727 1728 # Copy the tree to a new directory with postfix name configPostfix. 1729 if os.path.exists(newrdir): 1730 rmtree(newrdir) 1731 copytree(rdir, newrdir, ignore=ignore_patterns('*.pyc', '*.o', '*.d')) 1732 1733 # Update the LLDB_TEST environment variable to reflect new top 1734 # level test directory. 1735 # 1736 # See also lldbtest.TestBase.setUpClass(cls). 1737 if len(testdirs) == 1 and os.path.basename(testdirs[0]) == 'test': 1738 os.environ["LLDB_TEST"] = os.path.join(newrdir, 'test') 1739 else: 1740 os.environ["LLDB_TEST"] = newrdir 1741 1742 # And update the Python search paths for modules. 1743 sys.path = [x.replace(rdir, newrdir, 1) for x in old_sys_path] 1744 1745 # Output the configuration. 1746 if not parsable: 1747 sys.stderr.write("\nConfiguration: " + configString + "\n") 1748 1749 #print("sys.stderr name is", sys.stderr.name) 1750 #print("sys.stdout name is", sys.stdout.name) 1751 1752 # First, write out the number of collected test cases. 1753 if not parsable: 1754 sys.stderr.write(separator + "\n") 1755 sys.stderr.write("Collected %d test%s\n\n" 1756 % (suite.countTestCases(), 1757 suite.countTestCases() != 1 and "s" or "")) 1758 1759 class LLDBTestResult(unittest2.TextTestResult): 1760 """ 1761 Enforce a singleton pattern to allow introspection of test progress. 1762 1763 Overwrite addError(), addFailure(), and addExpectedFailure() methods 1764 to enable each test instance to track its failure/error status. It 1765 is used in the LLDB test framework to emit detailed trace messages 1766 to a log file for easier human inspection of test failures/errors. 1767 """ 1768 __singleton__ = None 1769 __ignore_singleton__ = False 1770 1771 @staticmethod 1772 def getTerminalSize(): 1773 import os 1774 env = os.environ 1775 def ioctl_GWINSZ(fd): 1776 try: 1777 import fcntl, termios, struct, os 1778 cr = struct.unpack('hh', fcntl.ioctl(fd, termios.TIOCGWINSZ, 1779 '1234')) 1780 except: 1781 return 1782 return cr 1783 cr = ioctl_GWINSZ(0) or ioctl_GWINSZ(1) or ioctl_GWINSZ(2) 1784 if not cr: 1785 try: 1786 fd = os.open(os.ctermid(), os.O_RDONLY) 1787 cr = ioctl_GWINSZ(fd) 1788 os.close(fd) 1789 except: 1790 pass 1791 if not cr: 1792 cr = (env.get('LINES', 25), env.get('COLUMNS', 80)) 1793 return int(cr[1]), int(cr[0]) 1794 1795 def __init__(self, *args): 1796 if not LLDBTestResult.__ignore_singleton__ and LLDBTestResult.__singleton__: 1797 raise Exception("LLDBTestResult instantiated more than once") 1798 super(LLDBTestResult, self).__init__(*args) 1799 LLDBTestResult.__singleton__ = self 1800 # Now put this singleton into the lldb module namespace. 1801 lldb.test_result = self 1802 # Computes the format string for displaying the counter. 1803 global suite 1804 counterWidth = len(str(suite.countTestCases())) 1805 self.fmt = "%" + str(counterWidth) + "d: " 1806 self.indentation = ' ' * (counterWidth + 2) 1807 # This counts from 1 .. suite.countTestCases(). 1808 self.counter = 0 1809 (width, height) = LLDBTestResult.getTerminalSize() 1810 self.progressbar = None 1811 global progress_bar 1812 if width > 10 and not parsable and progress_bar: 1813 try: 1814 self.progressbar = progress.ProgressWithEvents(stdout=self.stream,start=0,end=suite.countTestCases(),width=width-10) 1815 except: 1816 self.progressbar = None 1817 self.results_formatter = results_formatter_object 1818 1819 def _config_string(self, test): 1820 compiler = getattr(test, "getCompiler", None) 1821 arch = getattr(test, "getArchitecture", None) 1822 return "%s-%s" % (compiler() if compiler else "", arch() if arch else "") 1823 1824 def _exc_info_to_string(self, err, test): 1825 """Overrides superclass TestResult's method in order to append 1826 our test config info string to the exception info string.""" 1827 if hasattr(test, "getArchitecture") and hasattr(test, "getCompiler"): 1828 return '%sConfig=%s-%s' % (super(LLDBTestResult, self)._exc_info_to_string(err, test), 1829 test.getArchitecture(), 1830 test.getCompiler()) 1831 else: 1832 return super(LLDBTestResult, self)._exc_info_to_string(err, test) 1833 1834 def getDescription(self, test): 1835 doc_first_line = test.shortDescription() 1836 if self.descriptions and doc_first_line: 1837 return '\n'.join((str(test), self.indentation + doc_first_line)) 1838 else: 1839 return str(test) 1840 1841 def getCategoriesForTest(self,test): 1842 if hasattr(test,"_testMethodName"): 1843 test_method = getattr(test,"_testMethodName") 1844 test_method = getattr(test,test_method) 1845 else: 1846 test_method = None 1847 if test_method != None and hasattr(test_method,"getCategories"): 1848 test_categories = test_method.getCategories(test) 1849 elif hasattr(test,"getCategories"): 1850 test_categories = test.getCategories() 1851 elif inspect.ismethod(test) and test.__self__ != None and hasattr(test.__self__,"getCategories"): 1852 test_categories = test.__self__.getCategories() 1853 else: 1854 test_categories = [] 1855 if test_categories == None: 1856 test_categories = [] 1857 return test_categories 1858 1859 def hardMarkAsSkipped(self,test): 1860 getattr(test, test._testMethodName).__func__.__unittest_skip__ = True 1861 getattr(test, test._testMethodName).__func__.__unittest_skip_why__ = "test case does not fall in any category of interest for this run" 1862 test.__class__.__unittest_skip__ = True 1863 test.__class__.__unittest_skip_why__ = "test case does not fall in any category of interest for this run" 1864 1865 def startTest(self, test): 1866 if shouldSkipBecauseOfCategories(self.getCategoriesForTest(test)): 1867 self.hardMarkAsSkipped(test) 1868 global setCrashInfoHook 1869 setCrashInfoHook("%s at %s" % (str(test),inspect.getfile(test.__class__))) 1870 self.counter += 1 1871 #if self.counter == 4: 1872 # import crashinfo 1873 # crashinfo.testCrashReporterDescription(None) 1874 test.test_number = self.counter 1875 if self.showAll: 1876 self.stream.write(self.fmt % self.counter) 1877 super(LLDBTestResult, self).startTest(test) 1878 if self.results_formatter: 1879 self.results_formatter.handle_event( 1880 EventBuilder.event_for_start(test)) 1881 1882 def addSuccess(self, test): 1883 global parsable 1884 super(LLDBTestResult, self).addSuccess(test) 1885 if parsable: 1886 self.stream.write("PASS: LLDB (%s) :: %s\n" % (self._config_string(test), str(test))) 1887 if self.results_formatter: 1888 self.results_formatter.handle_event( 1889 EventBuilder.event_for_success(test)) 1890 1891 def addError(self, test, err): 1892 global sdir_has_content 1893 global parsable 1894 sdir_has_content = True 1895 super(LLDBTestResult, self).addError(test, err) 1896 method = getattr(test, "markError", None) 1897 if method: 1898 method() 1899 if parsable: 1900 self.stream.write("FAIL: LLDB (%s) :: %s\n" % (self._config_string(test), str(test))) 1901 if self.results_formatter: 1902 self.results_formatter.handle_event( 1903 EventBuilder.event_for_error(test, err)) 1904 1905 def addCleanupError(self, test, err): 1906 global sdir_has_content 1907 global parsable 1908 sdir_has_content = True 1909 super(LLDBTestResult, self).addCleanupError(test, err) 1910 method = getattr(test, "markCleanupError", None) 1911 if method: 1912 method() 1913 if parsable: 1914 self.stream.write("CLEANUP ERROR: LLDB (%s) :: %s\n" % (self._config_string(test), str(test))) 1915 if self.results_formatter: 1916 self.results_formatter.handle_event( 1917 EventBuilder.event_for_cleanup_error( 1918 test, err)) 1919 1920 def addFailure(self, test, err): 1921 global sdir_has_content 1922 global failuresPerCategory 1923 global parsable 1924 sdir_has_content = True 1925 super(LLDBTestResult, self).addFailure(test, err) 1926 method = getattr(test, "markFailure", None) 1927 if method: 1928 method() 1929 if parsable: 1930 self.stream.write("FAIL: LLDB (%s) :: %s\n" % (self._config_string(test), str(test))) 1931 if useCategories: 1932 test_categories = self.getCategoriesForTest(test) 1933 for category in test_categories: 1934 if category in failuresPerCategory: 1935 failuresPerCategory[category] = failuresPerCategory[category] + 1 1936 else: 1937 failuresPerCategory[category] = 1 1938 if self.results_formatter: 1939 self.results_formatter.handle_event( 1940 EventBuilder.event_for_failure(test, err)) 1941 1942 1943 def addExpectedFailure(self, test, err, bugnumber): 1944 global sdir_has_content 1945 global parsable 1946 sdir_has_content = True 1947 super(LLDBTestResult, self).addExpectedFailure(test, err, bugnumber) 1948 method = getattr(test, "markExpectedFailure", None) 1949 if method: 1950 method(err, bugnumber) 1951 if parsable: 1952 self.stream.write("XFAIL: LLDB (%s) :: %s\n" % (self._config_string(test), str(test))) 1953 if self.results_formatter: 1954 self.results_formatter.handle_event( 1955 EventBuilder.event_for_expected_failure( 1956 test, err, bugnumber)) 1957 1958 def addSkip(self, test, reason): 1959 global sdir_has_content 1960 global parsable 1961 sdir_has_content = True 1962 super(LLDBTestResult, self).addSkip(test, reason) 1963 method = getattr(test, "markSkippedTest", None) 1964 if method: 1965 method() 1966 if parsable: 1967 self.stream.write("UNSUPPORTED: LLDB (%s) :: %s (%s) \n" % (self._config_string(test), str(test), reason)) 1968 if self.results_formatter: 1969 self.results_formatter.handle_event( 1970 EventBuilder.event_for_skip(test, reason)) 1971 1972 def addUnexpectedSuccess(self, test, bugnumber): 1973 global sdir_has_content 1974 global parsable 1975 sdir_has_content = True 1976 super(LLDBTestResult, self).addUnexpectedSuccess(test, bugnumber) 1977 method = getattr(test, "markUnexpectedSuccess", None) 1978 if method: 1979 method(bugnumber) 1980 if parsable: 1981 self.stream.write("XPASS: LLDB (%s) :: %s\n" % (self._config_string(test), str(test))) 1982 if self.results_formatter: 1983 self.results_formatter.handle_event( 1984 EventBuilder.event_for_unexpected_success( 1985 test, bugnumber)) 1986 1987 1988 if parsable: 1989 v = 0 1990 elif progress_bar: 1991 v = 1 1992 else: 1993 v = verbose 1994 1995 # Invoke the test runner. 1996 if count == 1: 1997 result = unittest2.TextTestRunner(stream=sys.stderr, 1998 verbosity=v, 1999 failfast=failfast, 2000 resultclass=LLDBTestResult).run(suite) 2001 else: 2002 # We are invoking the same test suite more than once. In this case, 2003 # mark __ignore_singleton__ flag as True so the signleton pattern is 2004 # not enforced. 2005 LLDBTestResult.__ignore_singleton__ = True 2006 for i in range(count): 2007 2008 result = unittest2.TextTestRunner(stream=sys.stderr, 2009 verbosity=v, 2010 failfast=failfast, 2011 resultclass=LLDBTestResult).run(suite) 2012 2013 failed = failed or not result.wasSuccessful() 2014 2015 if sdir_has_content and not parsable: 2016 sys.stderr.write("Session logs for test failures/errors/unexpected successes" 2017 " can be found in directory '%s'\n" % sdir_name) 2018 2019 if useCategories and len(failuresPerCategory) > 0: 2020 sys.stderr.write("Failures per category:\n") 2021 for category in failuresPerCategory: 2022 sys.stderr.write("%s - %d\n" % (category,failuresPerCategory[category])) 2023 2024 os.chdir(where_to_save_session) 2025 fname = os.path.join(sdir_name, "TestFinished-%d" % os.getpid()) 2026 with open(fname, "w") as f: 2027 print("Test finished at: %s\n" % datetime.datetime.now().strftime("%Y-%m-%d-%H_%M_%S"), file=f) 2028 2029 # Terminate the test suite if ${LLDB_TESTSUITE_FORCE_FINISH} is defined. 2030 # This should not be necessary now. 2031 if ("LLDB_TESTSUITE_FORCE_FINISH" in os.environ): 2032 print("Terminating Test suite...") 2033 subprocess.Popen(["/bin/sh", "-c", "kill %s; exit 0" % (os.getpid())]) 2034 2035 # Exiting. 2036 exitTestSuite(failed) 2037 2038if __name__ == "__main__": 2039 run_suite()