1# 2# CDDL HEADER START 3# 4# The contents of this file are subject to the terms of the 5# Common Development and Distribution License (the "License"). 6# You may not use this file except in compliance with the License. 7# 8# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9# or http://www.opensolaris.org/os/licensing. 10# See the License for the specific language governing permissions 11# and limitations under the License. 12# 13# When distributing Covered Code, include this CDDL HEADER in each 14# file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15# If applicable, add the following below this CDDL HEADER, with the 16# fields enclosed by brackets "[]" replaced with your own identifying 17# information: Portions Copyright [yyyy] [name of copyright owner] 18# 19# CDDL HEADER END 20# 21 22# 23# Copyright 2009 Sun Microsystems, Inc. All rights reserved. 24# Use is subject to license terms. 25# 26 27# 28# Copyright (c) 2013, 2016 by Delphix. All rights reserved. 29# 30 31. $STF_SUITE/include/libtest.shlib 32. $STF_SUITE/tests/functional/redundancy/redundancy.cfg 33 34function cleanup 35{ 36 if poolexists $TESTPOOL; then 37 destroy_pool $TESTPOOL 38 fi 39 typeset dir 40 for dir in $TESTDIR $BASEDIR; do 41 if [[ -d $dir ]]; then 42 log_must rm -rf $dir 43 fi 44 done 45} 46 47# 48# Get random number between min and max number. 49# 50# $1 Minimal value 51# $2 Maximal value 52# 53function random 54{ 55 typeset -i min=$1 56 typeset -i max=$2 57 typeset -i value 58 59 while true; do 60 ((value = RANDOM % (max + 1))) 61 if ((value >= min)); then 62 break 63 fi 64 done 65 66 echo $value 67} 68 69# 70# Get the number of checksum errors for the pool. 71# 72# $1 Pool 73# 74function cksum_pool 75{ 76 typeset -i cksum=$(zpool status $1 | awk ' 77 !NF { isvdev = 0 } 78 isvdev { errors += $NF } 79 /CKSUM$/ { isvdev = 1 } 80 END { print errors } 81 ') 82 83 echo $cksum 84} 85 86# 87# Record the directories construction and checksum all the files which reside 88# within the specified pool 89# 90# $1 The specified pool 91# $2 The file which save the record. 92# 93function record_data 94{ 95 typeset pool=$1 96 typeset recordfile=$2 97 98 [[ -z $pool ]] && log_fail "No specified pool." 99 [[ -f $recordfile ]] && log_must rm -f $recordfile 100 101 sync_pool $pool 102 typeset mntpnt 103 mntpnt=$(get_prop mountpoint $pool) 104 log_must eval "du -a $mntpnt > $recordfile 2>&1" 105 # 106 # When the data was damaged, checksum is failing and return 1 107 # So, will not use log_must 108 # 109 find $mntpnt -type f -exec cksum {} + >> $recordfile 2>&1 110} 111 112# 113# Create test pool and fill with files and directories. 114# 115# $1 pool name 116# $2 pool type 117# $3 virtual devices number 118# 119function setup_test_env 120{ 121 typeset pool=$1 122 typeset keyword=$2 123 typeset -i vdev_cnt=$3 124 typeset vdevs 125 126 typeset -i i=0 127 while (( i < vdev_cnt )); do 128 vdevs="$vdevs $BASEDIR/vdev$i" 129 ((i += 1)) 130 done 131 132 if [[ ! -d $BASEDIR ]]; then 133 log_must mkdir $BASEDIR 134 fi 135 136 if poolexists $pool ; then 137 destroy_pool $pool 138 fi 139 140 log_must truncate -s $MINVDEVSIZE $vdevs 141 142 log_must zpool create -f -m $TESTDIR $pool $keyword $vdevs 143 144 log_note "Filling up the filesystem ..." 145 typeset -i ret=0 146 typeset -i i=0 147 typeset file=$TESTDIR/file 148 typeset -i limit 149 (( limit = $(get_prop available $pool) / 4 )) 150 151 while true ; do 152 [[ $(get_prop available $pool) -lt $limit ]] && break 153 file_write -o create -f $file.$i -b $BLOCKSZ -c $NUM_WRITES 154 ret=$? 155 (( $ret != 0 )) && break 156 (( i = i + 1 )) 157 done 158 159 record_data $TESTPOOL $PRE_RECORD_FILE 160} 161 162function refill_test_env 163{ 164 log_note "Re-filling the filesystem ..." 165 typeset -i ret=0 166 typeset -i i=0 167 typeset mntpnt 168 mntpnt=$(get_prop mountpoint $pool) 169 typeset file=$mntpnt/file 170 while [[ -e $file.$i ]]; do 171 log_must rm -f $file.$i 172 file_write -o create -f $file.$i -b $BLOCKSZ -c $NUM_WRITES 173 ret=$? 174 (( $ret != 0 )) && break 175 (( i = i + 1 )) 176 done 177 178 record_data $TESTPOOL $PRE_RECORD_FILE 179} 180 181# 182# Check pool status is healthy 183# 184# $1 pool 185# 186function is_healthy 187{ 188 typeset pool=$1 189 190 typeset healthy_output="pool '$pool' is healthy" 191 typeset real_output=$(zpool status -x $pool) 192 193 if [[ "$real_output" == "$healthy_output" ]]; then 194 return 0 195 else 196 typeset -i ret 197 zpool status -x $pool | grep "state:" | \ 198 grep "FAULTED" >/dev/null 2>&1 199 ret=$? 200 (( $ret == 0 )) && return 1 201 typeset l_scan 202 typeset errnum 203 l_scan=$(zpool status -x $pool | grep "scan:") 204 l_scan=${l_scan##*"with"} 205 errnum=$(echo $l_scan | awk '{print $1}') 206 207 return $errnum 208 fi 209} 210 211# 212# Check pool data is valid 213# 214# $1 pool 215# 216function is_data_valid 217{ 218 typeset pool=$1 219 220 record_data $pool $PST_RECORD_FILE 221 if ! diff $PRE_RECORD_FILE $PST_RECORD_FILE > /dev/null 2>&1; then 222 return 1 223 fi 224 225 return 0 226} 227 228# 229# Get the specified count devices name 230# 231# $1 pool name 232# $2 devices count 233# 234function get_vdevs #pool cnt 235{ 236 typeset pool=$1 237 typeset -i cnt=$2 238 239 typeset all_devs=$(zpool iostat -v $pool | awk '{print $1}'| \ 240 egrep -v "^pool$|^capacity$|^mirror$|^raidz1$|^raidz2$|---" | \ 241 egrep -v "/old$|^$pool$") 242 typeset -i i=0 243 typeset vdevs 244 while ((i < cnt)); do 245 typeset dev=$(echo $all_devs | awk '{print $1}') 246 eval all_devs=\${all_devs##*$dev} 247 248 vdevs="$dev $vdevs" 249 ((i += 1)) 250 done 251 252 echo "$vdevs" 253} 254 255# 256# Create and replace the same name virtual device files 257# 258# $1 pool name 259# $2-n virtual device files 260# 261function replace_missing_devs 262{ 263 typeset pool=$1 264 shift 265 266 typeset vdev 267 for vdev in $@; do 268 log_must gnudd if=/dev/zero of=$vdev \ 269 bs=1024k count=$(($MINDEVSIZE / (1024 * 1024))) \ 270 oflag=fdatasync 271 log_must zpool replace -wf $pool $vdev $vdev 272 done 273} 274 275# 276# Damage the pool's virtual device files. 277# 278# $1 pool name 279# $2 Failing devices count 280# $3 damage vdevs method, if not null, we keep 281# the label for the vdevs 282# 283function damage_devs 284{ 285 typeset pool=$1 286 typeset -i cnt=$2 287 typeset label="$3" 288 typeset vdevs 289 typeset -i bs_count=$((64 * 1024)) 290 291 vdevs=$(get_vdevs $pool $cnt) 292 typeset dev 293 if [[ -n $label ]]; then 294 for dev in $vdevs; do 295 dd if=/dev/zero of=$dev seek=512 bs=1024 \ 296 count=$bs_count conv=notrunc >/dev/null 2>&1 297 done 298 else 299 for dev in $vdevs; do 300 dd if=/dev/zero of=$dev bs=1024 count=$bs_count \ 301 conv=notrunc >/dev/null 2>&1 302 done 303 fi 304 305 sync_pool $pool 306} 307 308# 309# Clear errors in the pool caused by data corruptions 310# 311# $1 pool name 312# 313function clear_errors 314{ 315 typeset pool=$1 316 317 log_must zpool clear $pool 318 319 if ! is_healthy $pool ; then 320 log_note "$pool should be healthy." 321 return 1 322 fi 323 if ! is_data_valid $pool ; then 324 log_note "Data should be valid in $pool." 325 return 1 326 fi 327 328 return 0 329} 330 331# 332# Remove the specified pool's virtual device files 333# 334# $1 Pool name 335# $2 Missing devices count 336# 337function remove_devs 338{ 339 typeset pool=$1 340 typeset -i cnt=$2 341 typeset vdevs 342 343 vdevs=$(get_vdevs $pool $cnt) 344 log_must rm -f $vdevs 345 346 sync_pool $pool 347} 348 349# 350# Recover the bad or missing device files in the pool 351# 352# $1 Pool name 353# $2 Missing devices count 354# 355function recover_bad_missing_devs 356{ 357 typeset pool=$1 358 typeset -i cnt=$2 359 typeset vdevs 360 361 vdevs=$(get_vdevs $pool $cnt) 362 replace_missing_devs $pool $vdevs 363 364 if ! is_healthy $pool ; then 365 log_note "$pool should be healthy." 366 return 1 367 fi 368 if ! is_data_valid $pool ; then 369 log_note "Data should be valid in $pool." 370 return 1 371 fi 372 373 return 0 374} 375