1#!/usr/bin/env bash 2# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. 3# 4# A shell script to load some pre generated data file to a DB using ldb tool 5# ./ldb needs to be avaible to be executed. 6# 7# Usage: <SCRIPT> [checkout] 8# `checkout` can be a tag, commit or branch name. Will build using it and check DBs generated by all previous branches (or tags for very old versions without branch) can be opened by it. 9# Return value 0 means all regression tests pass. 1 if not pass. 10 11scriptpath=`dirname $BASH_SOURCE` 12test_dir=${TEST_TMPDIR:-"/tmp"}"/format_compatible_check" 13script_copy_dir=$test_dir"/script_copy" 14input_data_path=$test_dir"/test_data_input/" 15 16mkdir $test_dir || true 17mkdir $input_data_path || true 18rm -rf $script_copy_dir 19cp $scriptpath $script_copy_dir -rf 20 21# Generate random files. 22for i in {1..6} 23do 24 input_data[$i]=$input_data_path/data$i 25 echo == Generating random input file ${input_data[$i]} 26 python - <<EOF 27import random 28random.seed($i) 29symbols=['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9'] 30with open('${input_data[$i]}', 'w') as f: 31 for i in range(1,1024): 32 k = "" 33 for j in range(1, random.randint(1,32)): 34 k=k + symbols[random.randint(0, len(symbols) - 1)] 35 vb = "" 36 for j in range(1, random.randint(0,128)): 37 vb = vb + symbols[random.randint(0, len(symbols) - 1)] 38 v = "" 39 for j in range(1, random.randint(1, 5)): 40 v = v + vb 41 print >> f, k + " ==> " + v 42EOF 43done 44 45# Generate file(s) with sorted keys. 46sorted_input_data=$input_data_path/sorted_data 47echo == Generating file with sorted keys ${sorted_input_data} 48python - <<EOF 49with open('${sorted_input_data}', 'w') as f: 50 for i in range(0,10): 51 k = str(i) 52 v = "value" + k 53 print >> f, k + " ==> " + v 54EOF 55 56declare -a backward_compatible_checkout_objs=("2.2.fb.branch" "2.3.fb.branch" "2.4.fb.branch" "2.5.fb.branch" "2.6.fb.branch" "2.7.fb.branch" "2.8.1.fb" "3.0.fb.branch" "3.1.fb" "3.2.fb" "3.3.fb" "3.4.fb" "3.5.fb" "3.6.fb" "3.7.fb" "3.8.fb" "3.9.fb" "4.2.fb" "4.3.fb" "4.4.fb" "4.5.fb" "4.6.fb" "4.7.fb" "4.8.fb" "4.9.fb" "4.10.fb" "4.11.fb" "4.12.fb" "4.13.fb" "5.0.fb" "5.1.fb" "5.2.fb" "5.3.fb" "5.4.fb" "5.5.fb" "5.6.fb" "5.7.fb" "5.8.fb" "5.9.fb" "5.10.fb" "5.11.fb" "5.12.fb" "5.13.fb" "5.14.fb" "5.15.fb") 57declare -a forward_compatible_checkout_objs=() # N/A at the moment 58declare -a forward_compatible_with_options_checkout_objs=("5.16.fb" "5.17.fb" "5.18.fb" "6.0.fb" "6.1.fb" "6.2.fb" "6.3.fb" "6.4.fb" "6.5.fb" "6.6.fb" "6.7.fb" "6.8.fb") 59declare -a checkout_objs=(${backward_compatible_checkout_objs[@]} ${forward_compatible_checkout_objs[@]} ${forward_compatible_with_options_checkout_objs[@]}) 60declare -a extern_sst_ingestion_compatible_checkout_objs=("5.16.fb" "5.17.fb" "5.18.fb" "6.0.fb" "6.1.fb" "6.2.fb" "6.3.fb" "6.4.fb" "6.5.fb" "6.6.fb" "6.7.fb" "6.8.fb") 61 62generate_db() 63{ 64 set +e 65 $script_copy_dir/generate_random_db.sh $1 $2 66 if [ $? -ne 0 ]; then 67 echo ==== Error loading data from $2 to $1 ==== 68 exit 1 69 fi 70 set -e 71} 72 73compare_db() 74{ 75 set +e 76 $script_copy_dir/verify_random_db.sh $1 $2 $3 $4 $5 77 if [ $? -ne 0 ]; then 78 echo ==== Read different content from $1 and $2 or error happened. ==== 79 exit 1 80 fi 81 set -e 82} 83 84write_external_sst() 85{ 86 set +e 87 $script_copy_dir/write_external_sst.sh $1 $2 $3 88 if [ $? -ne 0 ]; then 89 echo ==== Error writing external SST file using data from $1 to $3 ==== 90 exit 1 91 fi 92 set -e 93} 94 95ingest_external_sst() 96{ 97 set +e 98 $script_copy_dir/ingest_external_sst.sh $1 $2 99 if [ $? -ne 0 ]; then 100 echo ==== Error ingesting external SST in $2 to DB at $1 ==== 101 exit 1 102 fi 103 set -e 104} 105 106# Sandcastle sets us up with a remote that is just another directory on the same 107# machine and doesn't have our branches. Need to fetch them so checkout works. 108# Remote add may fail if added previously (we don't cleanup). 109git remote add github_origin "https://github.com/facebook/rocksdb.git" 110set -e 111https_proxy="fwdproxy:8080" git fetch github_origin 112 113# Compatibility test for external SST file ingestion 114for checkout_obj in "${extern_sst_ingestion_compatible_checkout_objs[@]}" 115do 116 echo == Generating DB with extern SST file in "$checkout_obj" ... 117 https_proxy="fwdproxy:8080" git checkout github_origin/$checkout_obj -b $checkout_obj 118 make clean 119 make ldb -j32 120 write_external_sst $input_data_path $test_dir/$checkout_obj $test_dir/$checkout_obj 121 ingest_external_sst $test_dir/$checkout_obj $test_dir/$checkout_obj 122done 123 124checkout_flag=${1:-"master"} 125 126echo == Building $checkout_flag debug 127https_proxy="fwdproxy:8080" git checkout github_origin/$checkout_flag -b tmp-$checkout_flag 128make clean 129make ldb -j32 130compare_base_db_dir=$test_dir"/base_db_dir" 131write_external_sst $input_data_path $compare_base_db_dir $compare_base_db_dir 132ingest_external_sst $compare_base_db_dir $compare_base_db_dir 133 134for checkout_obj in "${extern_sst_ingestion_compatible_checkout_objs[@]}" 135do 136 echo == Build "$checkout_obj" and try to open DB generated using $checkout_flag 137 git checkout $checkout_obj 138 make clean 139 make ldb -j32 140 compare_db $test_dir/$checkout_obj $compare_base_db_dir db_dump.txt 1 1 141 git checkout tmp-$checkout_flag 142 # Clean up 143 git branch -D $checkout_obj 144done 145 146echo == Finish compatibility test for SST ingestion. 147 148for checkout_obj in "${checkout_objs[@]}" 149do 150 echo == Generating DB from "$checkout_obj" ... 151 https_proxy="fwdproxy:8080" git checkout github_origin/$checkout_obj -b $checkout_obj 152 make clean 153 make ldb -j32 154 generate_db $input_data_path $test_dir/$checkout_obj 155done 156 157checkout_flag=${1:-"master"} 158 159echo == Building $checkout_flag debug 160git checkout tmp-$checkout_flag 161make clean 162make ldb -j32 163compare_base_db_dir=$test_dir"/base_db_dir" 164echo == Generate compare base DB to $compare_base_db_dir 165generate_db $input_data_path $compare_base_db_dir 166 167for checkout_obj in "${checkout_objs[@]}" 168do 169 echo == Opening DB from "$checkout_obj" using debug build of $checkout_flag ... 170 compare_db $test_dir/$checkout_obj $compare_base_db_dir db_dump.txt 1 0 171done 172 173for checkout_obj in "${forward_compatible_checkout_objs[@]}" 174do 175 echo == Build "$checkout_obj" and try to open DB generated using $checkout_flag... 176 git checkout $checkout_obj 177 make clean 178 make ldb -j32 179 compare_db $test_dir/$checkout_obj $compare_base_db_dir forward_${checkout_obj}_dump.txt 0 180done 181 182for checkout_obj in "${forward_compatible_with_options_checkout_objs[@]}" 183do 184 echo == Build "$checkout_obj" and try to open DB generated using $checkout_flag with its options... 185 git checkout $checkout_obj 186 make clean 187 make ldb -j32 188 compare_db $test_dir/$checkout_obj $compare_base_db_dir forward_${checkout_obj}_dump.txt 1 1 189done 190 191echo ==== Compatibility Test PASSED ==== 192