1#!/usr/bin/env bash 2# This script is executed by Sandcastle 3# to determine next steps to run 4 5# Usage: 6# EMAIL=<email> ONCALL=<email> TRIGGER=<trigger> SUBSCRIBER=<email> rocks_ci.py <test-name> 7# 8# Input Value 9# ------------------------------------------------------------------------- 10# EMAIL Email address to report on trigger conditions 11# ONCALL Email address to raise a task on failure 12# TRIGGER Trigger conditions for email. Valid values are fail, warn, all 13# SUBSCRIBER Email addresss to add as subscriber for task 14# 15 16# 17# Report configuration 18# 19REPORT_EMAIL= 20if [ ! -z $EMAIL ]; then 21 if [ -z $TRIGGER ]; then 22 TRIGGER="fail" 23 fi 24 25 REPORT_EMAIL=" 26 { 27 'type':'email', 28 'triggers': [ '$TRIGGER' ], 29 'emails':['$EMAIL'] 30 }," 31fi 32 33CREATE_TASK= 34if [ ! -z $ONCALL ]; then 35 CREATE_TASK=" 36 { 37 'type':'task', 38 'triggers':[ 'fail' ], 39 'priority':0, 40 'subscribers':[ '$SUBSCRIBER' ], 41 'tags':[ 'rocksdb', 'ci' ], 42 }," 43fi 44 45# For now, create the tasks using only the dedicated task creation tool. 46CREATE_TASK= 47 48REPORT= 49if [[ ! -z $REPORT_EMAIL || ! -z $CREATE_TASK ]]; then 50 REPORT="'report': [ 51 $REPORT_EMAIL 52 $CREATE_TASK 53 ]" 54fi 55 56# 57# Helper variables 58# 59CLEANUP_ENV=" 60{ 61 'name':'Cleanup environment', 62 'shell':'rm -rf /dev/shm/rocksdb && mkdir /dev/shm/rocksdb && (chmod +t /dev/shm || true) && make clean', 63 'user':'root' 64}" 65 66UPLOAD_DB_DIR=" 67{ 68 'name':'Upload database directory', 69 'shell':'tar -cvzf rocksdb_db.tar.gz /dev/shm/rocksdb/', 70 'user':'root', 71 'cleanup':true, 72 'provide_artifacts': [ 73 { 74 'name':'rocksdb_db_dir', 75 'paths': ['rocksdb_db.tar.gz'], 76 'bundle': false, 77 }, 78 ], 79}" 80 81# We will eventually set the RATIO to 1, but we want do this 82# in steps. RATIO=$(nproc) will make it work as J=1 83if [ -z $RATIO ]; then 84 RATIO=$(nproc) 85fi 86 87if [ -z $PARALLEL_J ]; then 88 PARALLEL_J="J=$(expr $(nproc) / ${RATIO})" 89fi 90 91if [ -z $PARALLEL_j ]; then 92 PARALLEL_j="-j$(expr $(nproc) / ${RATIO})" 93fi 94 95PARALLELISM="$PARALLEL_J $PARALLEL_j" 96 97DEBUG="OPT=-g" 98SHM="TEST_TMPDIR=/dev/shm/rocksdb" 99NON_SHM="TMPD=/tmp/rocksdb_test_tmp" 100GCC_481="ROCKSDB_FBCODE_BUILD_WITH_481=1" 101ASAN="COMPILE_WITH_ASAN=1" 102CLANG="USE_CLANG=1" 103# in gcc-5 there are known problems with TSAN like https://gcc.gnu.org/bugzilla/show_bug.cgi?id=71090. 104# using platform007 gives us gcc-8 or higher which has that bug fixed. 105TSAN="ROCKSDB_FBCODE_BUILD_WITH_PLATFORM007=1 COMPILE_WITH_TSAN=1" 106UBSAN="COMPILE_WITH_UBSAN=1" 107TSAN_CRASH='CRASH_TEST_EXT_ARGS="--compression_type=zstd --log2_keys_per_lock=22"' 108NON_TSAN_CRASH="CRASH_TEST_EXT_ARGS=--compression_type=zstd" 109DISABLE_JEMALLOC="DISABLE_JEMALLOC=1" 110HTTP_PROXY="https_proxy=http://fwdproxy.29.prn1:8080 http_proxy=http://fwdproxy.29.prn1:8080 ftp_proxy=http://fwdproxy.29.prn1:8080" 111SETUP_JAVA_ENV="export $HTTP_PROXY; export JAVA_HOME=/usr/local/jdk-8u60-64/; export PATH=\$JAVA_HOME/bin:\$PATH" 112PARSER="'parser':'python build_tools/error_filter.py $1'" 113 114CONTRUN_NAME="ROCKSDB_CONTRUN_NAME" 115 116# This code is getting called under various scenarios. What we care about is to 117# understand when it's called from nightly contruns because in that case we'll 118# create tasks for any failures. To follow the existing pattern, we'll check 119# the value of $ONCALL. If it's a diff then just call `false` to make sure 120# that errors will be properly propagated to the caller. 121if [ ! -z $ONCALL ]; then 122 TASK_CREATION_TOOL="/usr/local/bin/mysql_mtr_filter --rocksdb --oncall $ONCALL" 123else 124 TASK_CREATION_TOOL="false" 125fi 126 127# 128# A mechanism to disable tests temporarily 129# 130DISABLE_COMMANDS="[ 131 { 132 'name':'Disable test', 133 'oncall':'$ONCALL', 134 'steps': [ 135 { 136 'name':'Job disabled. Please contact test owner', 137 'shell':'exit 1', 138 'user':'root' 139 }, 140 ], 141 } 142]" 143 144# 145# RocksDB unit test 146# 147UNIT_TEST_COMMANDS="[ 148 { 149 'name':'Rocksdb Unit Test', 150 'oncall':'$ONCALL', 151 'executeLocal': 'true', 152 'steps': [ 153 $CLEANUP_ENV, 154 { 155 'name':'Build and test RocksDB debug version', 156 'shell':'$SHM $DEBUG make $PARALLELISM check || $CONTRUN_NAME=check $TASK_CREATION_TOOL', 157 'user':'root', 158 $PARSER 159 }, 160 ], 161 $REPORT 162 } 163]" 164 165# 166# RocksDB unit test not under /dev/shm 167# 168UNIT_TEST_NON_SHM_COMMANDS="[ 169 { 170 'name':'Rocksdb Unit Test', 171 'oncall':'$ONCALL', 172 'executeLocal': 'true', 173 'timeout': 86400, 174 'steps': [ 175 $CLEANUP_ENV, 176 { 177 'name':'Build and test RocksDB debug version', 178 'timeout': 86400, 179 'shell':'$NON_SHM $DEBUG make $PARALLELISM check || $CONTRUN_NAME=non_shm_check $TASK_CREATION_TOOL', 180 'user':'root', 181 $PARSER 182 }, 183 ], 184 $REPORT 185 } 186]" 187 188# 189# RocksDB release build and unit tests 190# 191RELEASE_BUILD_COMMANDS="[ 192 { 193 'name':'Rocksdb Release Build', 194 'oncall':'$ONCALL', 195 'executeLocal': 'true', 196 'steps': [ 197 $CLEANUP_ENV, 198 { 199 'name':'Build RocksDB release', 200 'shell':'make $PARALLEL_j release || $CONTRUN_NAME=release $TASK_CREATION_TOOL', 201 'user':'root', 202 $PARSER 203 }, 204 ], 205 $REPORT 206 } 207]" 208 209# 210# RocksDB unit test on gcc-4.8.1 211# 212UNIT_TEST_COMMANDS_481="[ 213 { 214 'name':'Rocksdb Unit Test on GCC 4.8.1', 215 'oncall':'$ONCALL', 216 'executeLocal': 'true', 217 'steps': [ 218 $CLEANUP_ENV, 219 { 220 'name':'Build and test RocksDB debug version', 221 'shell':'$SHM $GCC_481 $DEBUG make $PARALLELISM check || $CONTRUN_NAME=unit_gcc_481_check $TASK_CREATION_TOOL', 222 'user':'root', 223 $PARSER 224 }, 225 ], 226 $REPORT 227 } 228]" 229 230# 231# RocksDB release build and unit tests 232# 233RELEASE_BUILD_COMMANDS_481="[ 234 { 235 'name':'Rocksdb Release on GCC 4.8.1', 236 'oncall':'$ONCALL', 237 'executeLocal': 'true', 238 'steps': [ 239 $CLEANUP_ENV, 240 { 241 'name':'Build RocksDB release on GCC 4.8.1', 242 'shell':'$GCC_481 make $PARALLEL_j release || $CONTRUN_NAME=release_gcc481 $TASK_CREATION_TOOL', 243 'user':'root', 244 $PARSER 245 }, 246 ], 247 $REPORT 248 } 249]" 250 251# 252# RocksDB unit test with CLANG 253# 254CLANG_UNIT_TEST_COMMANDS="[ 255 { 256 'name':'Rocksdb Unit Test', 257 'oncall':'$ONCALL', 258 'executeLocal': 'true', 259 'steps': [ 260 $CLEANUP_ENV, 261 { 262 'name':'Build and test RocksDB debug', 263 'shell':'$CLANG $SHM $DEBUG make $PARALLELISM check || $CONTRUN_NAME=clang_check $TASK_CREATION_TOOL', 264 'user':'root', 265 $PARSER 266 }, 267 ], 268 $REPORT 269 } 270]" 271 272# 273# RocksDB release build with CLANG 274# 275CLANG_RELEASE_BUILD_COMMANDS="[ 276 { 277 'name':'Rocksdb CLANG Release Build', 278 'oncall':'$ONCALL', 279 'executeLocal': 'true', 280 'steps': [ 281 $CLEANUP_ENV, 282 { 283 'name':'Build RocksDB release', 284 'shell':'$CLANG make $PARALLEL_j release|| $CONTRUN_NAME=clang_release $TASK_CREATION_TOOL', 285 'user':'root', 286 $PARSER 287 }, 288 ], 289 $REPORT 290 } 291]" 292 293# 294# RocksDB analyze 295# 296CLANG_ANALYZE_COMMANDS="[ 297 { 298 'name':'Rocksdb analyze', 299 'oncall':'$ONCALL', 300 'executeLocal': 'true', 301 'steps': [ 302 $CLEANUP_ENV, 303 { 304 'name':'RocksDB build and analyze', 305 'shell':'$CLANG $SHM $DEBUG make $PARALLEL_j analyze || $CONTRUN_NAME=clang_analyze $TASK_CREATION_TOOL', 306 'user':'root', 307 $PARSER 308 }, 309 ], 310 $REPORT 311 } 312]" 313 314# 315# RocksDB code coverage 316# 317CODE_COV_COMMANDS="[ 318 { 319 'name':'Rocksdb Unit Test Code Coverage', 320 'oncall':'$ONCALL', 321 'executeLocal': 'true', 322 'steps': [ 323 $CLEANUP_ENV, 324 { 325 'name':'Build, test and collect code coverage info', 326 'shell':'$SHM $DEBUG make $PARALLELISM coverage || $CONTRUN_NAME=coverage $TASK_CREATION_TOOL', 327 'user':'root', 328 $PARSER 329 }, 330 ], 331 $REPORT 332 } 333]" 334 335# 336# RocksDB unity 337# 338UNITY_COMMANDS="[ 339 { 340 'name':'Rocksdb Unity', 341 'oncall':'$ONCALL', 342 'executeLocal': 'true', 343 'steps': [ 344 $CLEANUP_ENV, 345 { 346 'name':'Build, test unity test', 347 'shell':'$SHM $DEBUG V=1 make J=1 unity_test || $CONTRUN_NAME=unity_test $TASK_CREATION_TOOL', 348 'user':'root', 349 $PARSER 350 }, 351 ], 352 $REPORT 353 } 354]" 355 356# 357# Build RocksDB lite 358# 359LITE_BUILD_COMMANDS="[ 360 { 361 'name':'Rocksdb Lite build', 362 'oncall':'$ONCALL', 363 'executeLocal': 'true', 364 'steps': [ 365 $CLEANUP_ENV, 366 { 367 'name':'Build RocksDB debug version', 368 'shell':'make J=1 LITE=1 all check || $CONTRUN_NAME=lite $TASK_CREATION_TOOL', 369 'user':'root', 370 $PARSER 371 }, 372 ], 373 $REPORT 374 } 375]" 376 377# 378# Report RocksDB lite binary size to scuba 379REPORT_LITE_BINARY_SIZE_COMMANDS="[ 380 { 381 'name':'Rocksdb Lite Binary Size', 382 'oncall':'$ONCALL', 383 'executeLocal': 'true', 384 'steps': [ 385 $CLEANUP_ENV, 386 { 387 'name':'Report RocksDB Lite binary size to scuba', 388 'shell':'tools/report_lite_binary_size.sh', 389 'user':'root', 390 }, 391 ], 392]" 393 394# 395# RocksDB stress/crash test 396# 397STRESS_CRASH_TEST_COMMANDS="[ 398 { 399 'name':'Rocksdb Stress and Crash Test', 400 'oncall':'$ONCALL', 401 'executeLocal': 'true', 402 'timeout': 86400, 403 'steps': [ 404 $CLEANUP_ENV, 405 { 406 'name':'Build and run RocksDB debug stress tests', 407 'shell':'$SHM $DEBUG $NON_TSAN_CRASH make J=1 db_stress || $CONTRUN_NAME=db_stress $TASK_CREATION_TOOL', 408 'user':'root', 409 $PARSER 410 }, 411 { 412 'name':'Build and run RocksDB debug crash tests', 413 'timeout': 86400, 414 'shell':'$SHM $DEBUG $NON_TSAN_CRASH make J=1 crash_test || $CONTRUN_NAME=crash_test $TASK_CREATION_TOOL', 415 'user':'root', 416 $PARSER 417 }, 418 $UPLOAD_DB_DIR, 419 ], 420 $REPORT 421 } 422]" 423 424# 425# RocksDB stress/crash test with atomic flush 426# 427STRESS_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS="[ 428 { 429 'name':'Rocksdb Stress and Crash Test with atomic flush', 430 'oncall':'$ONCALL', 431 'executeLocal': 'true', 432 'timeout': 86400, 433 'steps': [ 434 $CLEANUP_ENV, 435 { 436 'name':'Build and run RocksDB debug stress tests', 437 'shell':'$SHM $DEBUG $NON_TSAN_CRASH make J=1 db_stress || $CONTRUN_NAME=db_stress $TASK_CREATION_TOOL', 438 'user':'root', 439 $PARSER 440 }, 441 { 442 'name':'Build and run RocksDB debug crash tests with atomic flush', 443 'timeout': 86400, 444 'shell':'$SHM $DEBUG $NON_TSAN_CRASH make J=1 crash_test_with_atomic_flush || $CONTRUN_NAME=crash_test_with_atomic_flush $TASK_CREATION_TOOL', 445 'user':'root', 446 $PARSER 447 }, 448 $UPLOAD_DB_DIR, 449 ], 450 $REPORT 451 } 452]" 453 454# 455# RocksDB stress/crash test with txn 456# 457STRESS_CRASH_TEST_WITH_TXN_COMMANDS="[ 458 { 459 'name':'Rocksdb Stress and Crash Test with txn', 460 'oncall':'$ONCALL', 461 'executeLocal': 'true', 462 'timeout': 86400, 463 'steps': [ 464 $CLEANUP_ENV, 465 { 466 'name':'Build and run RocksDB debug stress tests', 467 'shell':'$SHM $DEBUG $NON_TSAN_CRASH make J=1 db_stress || $CONTRUN_NAME=db_stress $TASK_CREATION_TOOL', 468 'user':'root', 469 $PARSER 470 }, 471 { 472 'name':'Build and run RocksDB debug crash tests with txn', 473 'timeout': 86400, 474 'shell':'$SHM $DEBUG $NON_TSAN_CRASH make J=1 crash_test_with_txn || $CONTRUN_NAME=crash_test_with_txn $TASK_CREATION_TOOL', 475 'user':'root', 476 $PARSER 477 }, 478 $UPLOAD_DB_DIR, 479 ], 480 $REPORT 481 } 482]" 483 484# RocksDB write stress test. 485# We run on disk device on purpose (i.e. no $SHM) 486# because we want to add some randomness to fsync commands 487WRITE_STRESS_COMMANDS="[ 488 { 489 'name':'Rocksdb Write Stress Test', 490 'oncall':'$ONCALL', 491 'executeLocal': 'true', 492 'steps': [ 493 $CLEANUP_ENV, 494 { 495 'name':'Build and run RocksDB write stress tests', 496 'shell':'make write_stress && python tools/write_stress_runner.py --runtime_sec=3600 --db=/tmp/rocksdb_write_stress || $CONTRUN_NAME=write_stress $TASK_CREATION_TOOL', 497 'user':'root', 498 $PARSER 499 } 500 ], 501 'artifacts': [{'name': 'database', 'paths': ['/tmp/rocksdb_write_stress']}], 502 $REPORT 503 } 504]" 505 506 507# 508# RocksDB test under address sanitizer 509# 510ASAN_TEST_COMMANDS="[ 511 { 512 'name':'Rocksdb Unit Test under ASAN', 513 'oncall':'$ONCALL', 514 'executeLocal': 'true', 515 'steps': [ 516 $CLEANUP_ENV, 517 { 518 'name':'Test RocksDB debug under ASAN', 519'shell':'set -o pipefail && ($SHM $ASAN $DEBUG make $PARALLELISM asan_check || $CONTRUN_NAME=asan_check $TASK_CREATION_TOOL) |& /usr/facebook/ops/scripts/asan_symbolize.py -d', 520 'user':'root', 521 $PARSER 522 } 523 ], 524 $REPORT 525 } 526]" 527 528# 529# RocksDB crash testing under address sanitizer 530# 531ASAN_CRASH_TEST_COMMANDS="[ 532 { 533 'name':'Rocksdb crash test under ASAN', 534 'oncall':'$ONCALL', 535 'executeLocal': 'true', 536 'timeout': 86400, 537 'steps': [ 538 $CLEANUP_ENV, 539 { 540 'name':'Build and run RocksDB debug asan_crash_test', 541 'timeout': 86400, 542 'shell':'$SHM $DEBUG $NON_TSAN_CRASH make J=1 asan_crash_test || $CONTRUN_NAME=asan_crash_test $TASK_CREATION_TOOL', 543 'user':'root', 544 $PARSER 545 }, 546 $UPLOAD_DB_DIR, 547 ], 548 $REPORT 549 } 550]" 551 552# 553# RocksDB crash testing with atomic flush under address sanitizer 554# 555ASAN_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS="[ 556 { 557 'name':'Rocksdb crash test with atomic flush under ASAN', 558 'oncall':'$ONCALL', 559 'executeLocal': 'true', 560 'timeout': 86400, 561 'steps': [ 562 $CLEANUP_ENV, 563 { 564 'name':'Build and run RocksDB debug asan_crash_test_with_atomic_flush', 565 'timeout': 86400, 566 'shell':'$SHM $DEBUG $NON_TSAN_CRASH make J=1 asan_crash_test_with_atomic_flush || $CONTRUN_NAME=asan_crash_test_with_atomic_flush $TASK_CREATION_TOOL', 567 'user':'root', 568 $PARSER 569 }, 570 $UPLOAD_DB_DIR, 571 ], 572 $REPORT 573 } 574]" 575 576# 577# RocksDB crash testing with txn under address sanitizer 578# 579ASAN_CRASH_TEST_WITH_TXN_COMMANDS="[ 580 { 581 'name':'Rocksdb crash test with txn under ASAN', 582 'oncall':'$ONCALL', 583 'executeLocal': 'true', 584 'timeout': 86400, 585 'steps': [ 586 $CLEANUP_ENV, 587 { 588 'name':'Build and run RocksDB debug asan_crash_test_with_txn', 589 'timeout': 86400, 590 'shell':'$SHM $DEBUG $NON_TSAN_CRASH make J=1 asan_crash_test_with_txn || $CONTRUN_NAME=asan_crash_test_with_txn $TASK_CREATION_TOOL', 591 'user':'root', 592 $PARSER 593 }, 594 $UPLOAD_DB_DIR, 595 ], 596 $REPORT 597 } 598]" 599 600# 601# RocksDB test under undefined behavior sanitizer 602# 603UBSAN_TEST_COMMANDS="[ 604 { 605 'name':'Rocksdb Unit Test under UBSAN', 606 'oncall':'$ONCALL', 607 'executeLocal': 'true', 608 'steps': [ 609 $CLEANUP_ENV, 610 { 611 'name':'Test RocksDB debug under UBSAN', 612 'shell':'set -o pipefail && $SHM $UBSAN $CLANG $DEBUG make $PARALLELISM ubsan_check || $CONTRUN_NAME=ubsan_check $TASK_CREATION_TOOL', 613 'user':'root', 614 $PARSER 615 } 616 ], 617 $REPORT 618 } 619]" 620 621# 622# RocksDB crash testing under udnefined behavior sanitizer 623# 624UBSAN_CRASH_TEST_COMMANDS="[ 625 { 626 'name':'Rocksdb crash test under UBSAN', 627 'oncall':'$ONCALL', 628 'executeLocal': 'true', 629 'timeout': 86400, 630 'steps': [ 631 $CLEANUP_ENV, 632 { 633 'name':'Build and run RocksDB debug ubsan_crash_test', 634 'timeout': 86400, 635 'shell':'$SHM $DEBUG $NON_TSAN_CRASH $CLANG make J=1 ubsan_crash_test || $CONTRUN_NAME=ubsan_crash_test $TASK_CREATION_TOOL', 636 'user':'root', 637 $PARSER 638 }, 639 $UPLOAD_DB_DIR, 640 ], 641 $REPORT 642 } 643]" 644 645# 646# RocksDB crash testing with atomic flush under undefined behavior sanitizer 647# 648UBSAN_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS="[ 649 { 650 'name':'Rocksdb crash test with atomic flush under UBSAN', 651 'oncall':'$ONCALL', 652 'executeLocal': 'true', 653 'timeout': 86400, 654 'steps': [ 655 $CLEANUP_ENV, 656 { 657 'name':'Build and run RocksDB debug ubsan_crash_test_with_atomic_flush', 658 'timeout': 86400, 659 'shell':'$SHM $DEBUG $NON_TSAN_CRASH $CLANG make J=1 ubsan_crash_test_with_atomic_flush || $CONTRUN_NAME=ubsan_crash_test_with_atomic_flush $TASK_CREATION_TOOL', 660 'user':'root', 661 $PARSER 662 }, 663 $UPLOAD_DB_DIR, 664 ], 665 $REPORT 666 } 667]" 668 669# 670# RocksDB crash testing with txn under undefined behavior sanitizer 671# 672UBSAN_CRASH_TEST_WITH_TXN_COMMANDS="[ 673 { 674 'name':'Rocksdb crash test with txn under UBSAN', 675 'oncall':'$ONCALL', 676 'executeLocal': 'true', 677 'timeout': 86400, 678 'steps': [ 679 $CLEANUP_ENV, 680 { 681 'name':'Build and run RocksDB debug ubsan_crash_test_with_txn', 682 'timeout': 86400, 683 'shell':'$SHM $DEBUG $NON_TSAN_CRASH $CLANG make J=1 ubsan_crash_test_with_txn || $CONTRUN_NAME=ubsan_crash_test_with_txn $TASK_CREATION_TOOL', 684 'user':'root', 685 $PARSER 686 }, 687 $UPLOAD_DB_DIR, 688 ], 689 $REPORT 690 } 691]" 692 693# 694# RocksDB unit test under valgrind 695# 696VALGRIND_TEST_COMMANDS="[ 697 { 698 'name':'Rocksdb Unit Test under valgrind', 699 'oncall':'$ONCALL', 700 'executeLocal': 'true', 701 'timeout': 86400, 702 'steps': [ 703 $CLEANUP_ENV, 704 { 705 'name':'Run RocksDB debug unit tests', 706 'timeout': 86400, 707 'shell':'$SHM $DEBUG make $PARALLELISM valgrind_test || $CONTRUN_NAME=valgrind_check $TASK_CREATION_TOOL', 708 'user':'root', 709 $PARSER 710 }, 711 ], 712 $REPORT 713 } 714]" 715 716# 717# RocksDB test under TSAN 718# 719TSAN_UNIT_TEST_COMMANDS="[ 720 { 721 'name':'Rocksdb Unit Test under TSAN', 722 'oncall':'$ONCALL', 723 'executeLocal': 'true', 724 'timeout': 86400, 725 'steps': [ 726 $CLEANUP_ENV, 727 { 728 'name':'Run RocksDB debug unit test', 729 'timeout': 86400, 730 'shell':'set -o pipefail && $SHM $DEBUG $TSAN make $PARALLELISM check || $CONTRUN_NAME=tsan_check $TASK_CREATION_TOOL', 731 'user':'root', 732 $PARSER 733 }, 734 ], 735 $REPORT 736 } 737]" 738 739# 740# RocksDB crash test under TSAN 741# 742TSAN_CRASH_TEST_COMMANDS="[ 743 { 744 'name':'Rocksdb Crash Test under TSAN', 745 'oncall':'$ONCALL', 746 'executeLocal': 'true', 747 'timeout': 86400, 748 'steps': [ 749 $CLEANUP_ENV, 750 { 751 'name':'Compile and run', 752 'timeout': 86400, 753 'shell':'set -o pipefail && $SHM $DEBUG $TSAN $TSAN_CRASH CRASH_TEST_KILL_ODD=1887 make J=1 crash_test || $CONTRUN_NAME=tsan_crash_test $TASK_CREATION_TOOL', 754 'user':'root', 755 $PARSER 756 }, 757 $UPLOAD_DB_DIR, 758 ], 759 $REPORT 760 } 761]" 762 763# 764# RocksDB crash test with atomic flush under TSAN 765# 766TSAN_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS="[ 767 { 768 'name':'Rocksdb Crash Test with atomic flush under TSAN', 769 'oncall':'$ONCALL', 770 'executeLocal': 'true', 771 'timeout': 86400, 772 'steps': [ 773 $CLEANUP_ENV, 774 { 775 'name':'Compile and run', 776 'timeout': 86400, 777 'shell':'set -o pipefail && $SHM $DEBUG $TSAN $TSAN_CRASH CRASH_TEST_KILL_ODD=1887 make J=1 crash_test_with_atomic_flush || $CONTRUN_NAME=tsan_crash_test_with_atomic_flush $TASK_CREATION_TOOL', 778 'user':'root', 779 $PARSER 780 }, 781 $UPLOAD_DB_DIR, 782 ], 783 $REPORT 784 } 785]" 786 787# 788# RocksDB crash test with txn under TSAN 789# 790TSAN_CRASH_TEST_WITH_TXN_COMMANDS="[ 791 { 792 'name':'Rocksdb Crash Test with txn under TSAN', 793 'oncall':'$ONCALL', 794 'executeLocal': 'true', 795 'timeout': 86400, 796 'steps': [ 797 $CLEANUP_ENV, 798 { 799 'name':'Compile and run', 800 'timeout': 86400, 801 'shell':'set -o pipefail && $SHM $DEBUG $TSAN $TSAN_CRASH CRASH_TEST_KILL_ODD=1887 make J=1 crash_test_with_txn || $CONTRUN_NAME=tsan_crash_test_with_txn $TASK_CREATION_TOOL', 802 'user':'root', 803 $PARSER 804 }, 805 $UPLOAD_DB_DIR, 806 ], 807 $REPORT 808 } 809]" 810 811# 812# RocksDB format compatible 813# 814 815run_format_compatible() 816{ 817 export TEST_TMPDIR=/dev/shm/rocksdb 818 rm -rf /dev/shm/rocksdb 819 mkdir /dev/shm/rocksdb 820 821 tools/check_format_compatible.sh 822} 823 824FORMAT_COMPATIBLE_COMMANDS="[ 825 { 826 'name':'Rocksdb Format Compatible tests', 827 'oncall':'$ONCALL', 828 'executeLocal': 'true', 829 'steps': [ 830 $CLEANUP_ENV, 831 { 832 'name':'Run RocksDB debug unit test', 833 'shell':'build_tools/rocksdb-lego-determinator run_format_compatible || $CONTRUN_NAME=run_format_compatible $TASK_CREATION_TOOL', 834 'user':'root', 835 $PARSER 836 }, 837 ], 838 $REPORT 839 } 840]" 841 842# 843# RocksDB no compression 844# 845run_no_compression() 846{ 847 export TEST_TMPDIR=/dev/shm/rocksdb 848 rm -rf /dev/shm/rocksdb 849 mkdir /dev/shm/rocksdb 850 make clean 851 cat build_tools/fbcode_config.sh | grep -iv dzstd | grep -iv dzlib | grep -iv dlz4 | grep -iv dsnappy | grep -iv dbzip2 > .tmp.fbcode_config.sh 852 mv .tmp.fbcode_config.sh build_tools/fbcode_config.sh 853 cat Makefile | grep -v tools/ldb_test.py > .tmp.Makefile 854 mv .tmp.Makefile Makefile 855 make $DEBUG J=1 check 856} 857 858NO_COMPRESSION_COMMANDS="[ 859 { 860 'name':'Rocksdb No Compression tests', 861 'oncall':'$ONCALL', 862 'executeLocal': 'true', 863 'steps': [ 864 $CLEANUP_ENV, 865 { 866 'name':'Run RocksDB debug unit test', 867 'shell':'build_tools/rocksdb-lego-determinator run_no_compression || $CONTRUN_NAME=run_no_compression $TASK_CREATION_TOOL', 868 'user':'root', 869 $PARSER 870 }, 871 ], 872 $REPORT 873 } 874]" 875 876# 877# RocksDB regression 878# 879run_regression() 880{ 881 time -v bash -vx ./build_tools/regression_build_test.sh $(mktemp -d $WORKSPACE/leveldb.XXXX) $(mktemp leveldb_test_stats.XXXX) 882 883 # ======= report size to ODS ======== 884 885 # parameters: $1 -- key, $2 -- value 886 function send_size_to_ods { 887 curl --silent "https://www.intern.facebook.com/intern/agent/ods_set.php?entity=rocksdb_build&key=rocksdb.build_size.$1&value=$2" \ 888 --connect-timeout 60 889 } 890 891 # === normal build === 892 make clean 893 make -j$(nproc) static_lib 894 send_size_to_ods static_lib $(stat --printf="%s" librocksdb.a) 895 strip librocksdb.a 896 send_size_to_ods static_lib_stripped $(stat --printf="%s" librocksdb.a) 897 898 make -j$(nproc) shared_lib 899 send_size_to_ods shared_lib $(stat --printf="%s" `readlink -f librocksdb.so`) 900 strip `readlink -f librocksdb.so` 901 send_size_to_ods shared_lib_stripped $(stat --printf="%s" `readlink -f librocksdb.so`) 902 903 # === lite build === 904 make clean 905 make LITE=1 -j$(nproc) static_lib 906 send_size_to_ods static_lib_lite $(stat --printf="%s" librocksdb.a) 907 strip librocksdb.a 908 send_size_to_ods static_lib_lite_stripped $(stat --printf="%s" librocksdb.a) 909 910 make LITE=1 -j$(nproc) shared_lib 911 send_size_to_ods shared_lib_lite $(stat --printf="%s" `readlink -f librocksdb.so`) 912 strip `readlink -f librocksdb.so` 913 send_size_to_ods shared_lib_lite_stripped $(stat --printf="%s" `readlink -f librocksdb.so`) 914} 915 916REGRESSION_COMMANDS="[ 917 { 918 'name':'Rocksdb regression commands', 919 'oncall':'$ONCALL', 920 'steps': [ 921 $CLEANUP_ENV, 922 { 923 'name':'Make and run script', 924 'shell':'build_tools/rocksdb-lego-determinator run_regression || $CONTRUN_NAME=run_regression $TASK_CREATION_TOOL', 925 'user':'root', 926 $PARSER 927 }, 928 ], 929 $REPORT 930 } 931]" 932 933# 934# RocksDB Java build 935# 936JAVA_BUILD_TEST_COMMANDS="[ 937 { 938 'name':'Rocksdb Java Build', 939 'oncall':'$ONCALL', 940 'executeLocal': 'true', 941 'steps': [ 942 $CLEANUP_ENV, 943 { 944 'name':'Build RocksDB for Java', 945 'shell':'$SETUP_JAVA_ENV; $SHM make rocksdbjava || $CONTRUN_NAME=rocksdbjava $TASK_CREATION_TOOL', 946 'user':'root', 947 $PARSER 948 }, 949 ], 950 $REPORT 951 } 952]" 953 954 955case $1 in 956 unit) 957 echo $UNIT_TEST_COMMANDS 958 ;; 959 unit_non_shm) 960 echo $UNIT_TEST_NON_SHM_COMMANDS 961 ;; 962 release) 963 echo $RELEASE_BUILD_COMMANDS 964 ;; 965 unit_481) 966 echo $UNIT_TEST_COMMANDS_481 967 ;; 968 release_481) 969 echo $RELEASE_BUILD_COMMANDS_481 970 ;; 971 clang_unit) 972 echo $CLANG_UNIT_TEST_COMMANDS 973 ;; 974 clang_release) 975 echo $CLANG_RELEASE_BUILD_COMMANDS 976 ;; 977 clang_analyze) 978 echo $CLANG_ANALYZE_COMMANDS 979 ;; 980 code_cov) 981 echo $CODE_COV_COMMANDS 982 ;; 983 unity) 984 echo $UNITY_COMMANDS 985 ;; 986 lite) 987 echo $LITE_BUILD_COMMANDS 988 ;; 989 report_lite_binary_size) 990 echo $REPORT_LITE_BINARY_SIZE_COMMANDS 991 ;; 992 stress_crash) 993 echo $STRESS_CRASH_TEST_COMMANDS 994 ;; 995 stress_crash_with_atomic_flush) 996 echo $STRESS_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS 997 ;; 998 stress_crash_with_txn) 999 echo $STRESS_CRASH_TEST_WITH_TXN_COMMANDS 1000 ;; 1001 write_stress) 1002 echo $WRITE_STRESS_COMMANDS 1003 ;; 1004 asan) 1005 echo $ASAN_TEST_COMMANDS 1006 ;; 1007 asan_crash) 1008 echo $ASAN_CRASH_TEST_COMMANDS 1009 ;; 1010 asan_crash_with_atomic_flush) 1011 echo $ASAN_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS 1012 ;; 1013 asan_crash_with_txn) 1014 echo $ASAN_CRASH_TEST_WITH_TXN_COMMANDS 1015 ;; 1016 ubsan) 1017 echo $UBSAN_TEST_COMMANDS 1018 ;; 1019 ubsan_crash) 1020 echo $UBSAN_CRASH_TEST_COMMANDS 1021 ;; 1022 ubsan_crash_with_atomic_flush) 1023 echo $UBSAN_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS 1024 ;; 1025 ubsan_crash_with_txn) 1026 echo $UBSAN_CRASH_TEST_WITH_TXN_COMMANDS 1027 ;; 1028 valgrind) 1029 echo $VALGRIND_TEST_COMMANDS 1030 ;; 1031 tsan) 1032 echo $TSAN_UNIT_TEST_COMMANDS 1033 ;; 1034 tsan_crash) 1035 echo $TSAN_CRASH_TEST_COMMANDS 1036 ;; 1037 tsan_crash_with_atomic_flush) 1038 echo $TSAN_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS 1039 ;; 1040 tsan_crash_with_txn) 1041 echo $TSAN_CRASH_TEST_WITH_TXN_COMMANDS 1042 ;; 1043 format_compatible) 1044 echo $FORMAT_COMPATIBLE_COMMANDS 1045 ;; 1046 run_format_compatible) 1047 run_format_compatible 1048 ;; 1049 no_compression) 1050 echo $NO_COMPRESSION_COMMANDS 1051 ;; 1052 run_no_compression) 1053 run_no_compression 1054 ;; 1055 regression) 1056 echo $REGRESSION_COMMANDS 1057 ;; 1058 run_regression) 1059 run_regression 1060 ;; 1061 java_build) 1062 echo $JAVA_BUILD_TEST_COMMANDS 1063 ;; 1064 *) 1065 echo "Invalid determinator command" 1066 exit 1 1067 ;; 1068esac 1069