1#! /bin/bash 2 3############################ Common Utilities ################################ 4FL_PCAP=1 5FL_DPDK=2 6 7# vercomp returns 0 (=), 1 (>), or 2 (<) 8vercomp () { 9 if [[ $1 == $2 ]] 10 then 11 return 0 12 fi 13 local IFS=. 14 local i ver1=($1) ver2=($2) 15 # fill empty fields in ver1 with zeros 16 for ((i=${#ver1[@]}; i<${#ver2[@]}; i++)) 17 do 18 ver1[i]=0 19 done 20 for ((i=0; i<${#ver1[@]}; i++)) 21 do 22 if [[ -z ${ver2[i]} ]] 23 then 24 # fill empty fields in ver2 with zeros 25 ver2[i]=0 26 fi 27 if ((10#${ver1[i]} > 10#${ver2[i]})) 28 then 29 return 1 30 fi 31 if ((10#${ver1[i]} < 10#${ver2[i]})) 32 then 33 return 2 34 fi 35 done 36 return 0 37} 38 39# valid_ip_addr returns 1 for valid ip address 40valid_ip_addr() 41{ 42 local ip=$1 43 local stat=1 44 45 if [[ $ip =~ ^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ ]]; then 46 OIFS=$IFS 47 IFS='.' 48 ip=($ip) 49 IFS=$OIFS 50 [[ ${ip[0]} -le 255 && ${ip[1]} -le 255 \ 51 && ${ip[2]} -le 255 && ${ip[3]} -le 255 ]] 52 stat=$? 53 fi 54 return $stat 55} 56 57# Create a mos.conf file 58create_config() 59{ 60 cd $CUR_DIR 61 mkdir -p $2 62 CONF_FILE=$2/mos.conf 63 64 echo 65 echo "----------------------------------------" 66 echo "Creating mos.conf file in $2" 67 echo "----------------------------------------" 68 69 if [ -f $CONF_FILE ]; then 70 echo 71 echo "The config file $2/mos.conf already exists." 72 echo "Skip writing a new config file.." 73 return 74 fi 75 76 # Get the number of CPUs and translate it into CPU mask 77 let "z=2**$(nproc)-1" 78 CPU_MASK=$(printf '0x%04X\n' $z) 79 80 # Get # of CPU sockets 81 let "sockets=$(cat /proc/cpuinfo | grep "physical id" | awk '{print $4}' | sort -ur | head -n 1) + 1" 82 83 # Get the number of memory channels 84 command -v dmidecode >/dev/null && 85 let "NUM_MEMCH=($(sudo dmidecode -t 17 | grep -c 'Size:')-$(sudo dmidecode -t 17 | grep -c 'Size: No Module'))/$sockets" || 86 { echo "dmidecode command not found. setting num_mem_ch = 4"; let "NUM_MEMCH=4"; } 87 88 # Get the number of interfaces 89 counter=0 90 if [[ $1 == $FL_DPDK ]]; then 91 cd /sys/module/igb_uio/drivers/pci:igb_uio/ 92 DEV_NAME="dpdk" 93 elif [[ $1 != $FL_PCAP ]]; then 94 echo "invalid function call" 95 exit 1 96 fi 97 for i in * 98 do 99 if [[ $i == *":"* ]] 100 then 101 let "counter=$counter + 1" 102 fi 103 done 104 cd $CUR_DIR 105 106 # Write up a new configuration file 107 if [[ $1 == $FL_PCAP ]]; then 108 cat .standalone-template.conf > $CONF_FILE 109 DEVLIST=`ifconfig -s | grep -Eo '^[^ ]+' | tail -n+2` 110 for dev in $DEVLIST; do 111 printf "Found $dev. Press y to select [y/N]: " 112 read option 113 if [[ "$option" == y* ]]; then 114 DEVICE="\n\t\t$dev" 115 DEVICEMASK="__coremask__devicemask" 116 FMT=$(printf 's/__devicemask/%s %s/g' $DEVICE $DEVICEMASK) 117 sed -i -e "$FMT" $CONF_FILE 118 119 DEVICE="$dev""__devicelist" 120 FMT=$(printf 's/__devicelist/ %s/g' $DEVICE) 121 sed -i -e "$FMT" $CONF_FILE 122 fi 123 done 124 sed -i -e 's/__devicemask//g' $CONF_FILE 125 sed -i -e 's/__devicelist//g' $CONF_FILE 126 sed -i -e 's/__coremask/0x0001/g' $CONF_FILE 127 sed -i -e 's/__num_memch//g' $CONF_FILE 128 sed -i -e 's/__forward/1/g' $CONF_FILE 129 130 else 131 if [ "$3" = "epserver" ] || [ "$3" = "epwget" ] ; then 132 cat .end-template.conf > $CONF_FILE 133 else 134 cat .standalone-template.conf > $CONF_FILE 135 fi 136 start=0 137 while [ $start -lt $counter ] 138 do 139 DEVICE="\n\t\t$DEV_NAME$(($start))" 140 DEVICEMASK="__coremask__devicemask" 141 FMT=$(printf 's/__devicemask/%s %s/g' $DEVICE $DEVICEMASK) 142 sed -i -e "$FMT" $CONF_FILE 143 144 DEVICE="$DEV_NAME$(($start))__devicelist" 145 FMT=$(printf 's/__devicelist/ %s/g' $DEVICE) 146 sed -i -e "$FMT" $CONF_FILE 147 148 let "start=$start + 1" 149 done 150 sed -i -e 's/__devicemask//g' $CONF_FILE 151 sed -i -e 's/__devicelist//g' $CONF_FILE 152 sed -i -e 's/__coremask/'$CPU_MASK'/g' $CONF_FILE 153 sed -i -e 's/__forward/1/g' $CONF_FILE 154 sed -i -e 's/__app/'$3'/g' $CONF_FILE 155 156 if [ "$3" = "nat" ] ; then 157 sed -i -e 's/\# tcp_tw_interval = 30/tcp_tw_interval = 30/g' $CONF_FILE 158 fi 159 160 if [[ $1 == $FL_DPDK ]]; then 161 FMT=$(printf 's/__num_memch/%snb_mem_channels = %d%s/g' '# number of memory channels per socket [mandatory for DPDK]\n\t' $NUM_MEMCH '\n') 162 sed -i -e "$FMT" $CONF_FILE 163 else 164 echo "invalid function call" 165 exit 1 166 fi 167 fi 168 echo 169 cat $CONF_FILE 170} 171 172create_makefile() 173{ 174 cd $CUR_DIR/samples 175 for d in * ; do 176 if [ -f $CUR_DIR/samples/$d/Makefile.in ]; then 177 cp $CUR_DIR/samples/$d/Makefile.in $CUR_DIR/samples/$d/Makefile 178 if [[ $1 == $FL_DPDK ]]; then 179 FMT=$(printf 's/__IO_LIB_ARGS/LIBS += -m64 -g -pthread -lrt -march=native -Wl,-export-dynamic -L..\/..\/drivers\/dpdk\/lib -Wl,-lnuma -Wl,-lmtcp -Wl,-lpthread -Wl,-lrt -Wl,-ldl -Wl,$(shell cat ..\/..\/drivers\/dpdk\/lib\/ldflags.txt)/g') 180 elif [[ $1 == $FL_PCAP ]]; then 181 FMT=$(printf 's/__IO_LIB_ARGS/GCC_OPT += -D__thread="" -DBE_RESILIENT_TO_PACKET_DROP\\nINC += -DENABLE_PCAP\\nLIBS += -lpcap/g') 182 fi 183 sed -i -e "$FMT" $CUR_DIR/samples/$d/Makefile 184 fi 185 done 186} 187 188# Setup MOS library 189setup_mos_library() { 190 echo "Start building up the MOS library" 191 cd $CUR_DIR 192 193 if [[ $1 == $FL_DPDK ]]; then 194 mkdir -p $DPDK_DIR 195 rmdir $DPDK_DIR/include $DPDK_DIR/lib 2> /dev/null 196 rm $DPDK_DIR/include $DPDK_DIR/lib 2> /dev/null 197 ln -s $CUR_DIR/$RTE_SDK/$RTE_TARGET/include $DPDK_DIR/include 198 ln -s $CUR_DIR/$RTE_SDK/$RTE_TARGET/lib $DPDK_DIR/lib 199 cd $CUR_DIR/scripts/ 200 ./configure --enable-dpdk 201 elif [[ $1 == $FL_PCAP ]]; then 202 cd $CUR_DIR/scripts/ 203 ./configure --enable-pcap 204 fi 205 cd $CUR_DIR/core/src 206 make clean;make 207 cd $CUR_DIR 208 create_makefile $1 209 echo 210 echo "----------------------------------------------------------" 211 echo "Done with MOS library setup" 212 echo "----------------------------------------------------------" 213 echo 214} 215 216############################################################################# 217 218############################### DPDK Utilities ############################## 219# Creates hugepage filesystem. 220create_mnt_huge() 221{ 222 echo "Creating /mnt/huge and mounting as hugetlbfs" 223 sudo mkdir -p /mnt/huge 224 225 grep -s '/mnt/huge' /proc/mounts > /dev/null 226 if [ $? -ne 0 ] ; then 227 sudo mount -t hugetlbfs nodev /mnt/huge 228 fi 229} 230 231# Removes hugepage filesystem. 232remove_mnt_huge() 233{ 234 echo "Unmounting /mnt/huge and removing directory" 235 grep -s '/mnt/huge' /proc/mounts > /dev/null 236 if [ $? -eq 0 ] ; then 237 sudo umount /mnt/huge 238 fi 239 240 if [ -d /mnt/huge ] ; then 241 sudo rm -R /mnt/huge 242 fi 243} 244 245# Unloads igb_uio.ko. 246remove_igb_uio_module() 247{ 248 echo "Unloading any existing DPDK UIO module" 249 /sbin/lsmod | grep -s igb_uio > /dev/null 250 if [ $? -eq 0 ] ; then 251 sudo /sbin/rmmod igb_uio 252 fi 253} 254 255# Loads new igb_uio.ko (and uio module if needed). 256load_igb_uio_module() 257{ 258 if [ ! -f $RTE_SDK/$RTE_TARGET/kmod/igb_uio.ko ];then 259 echo "## ERROR: Target does not have the DPDK UIO Kernel Module." 260 echo " To fix, please try to rebuild target." 261 return 262 fi 263 264 remove_igb_uio_module 265 266 /sbin/lsmod | grep -s uio > /dev/null 267 if [ $? -ne 0 ] ; then 268 modinfo uio > /dev/null 269 if [ $? -eq 0 ]; then 270 echo "Loading uio module" 271 sudo /sbin/modprobe uio 272 fi 273 fi 274 275 # UIO may be compiled into kernel, so it may not be an error if it can't 276 # be loaded. 277 echo "Loading DPDK UIO module" 278 sudo /sbin/insmod $RTE_SDK/$RTE_TARGET/kmod/igb_uio.ko 279 if [ $? -ne 0 ] ; then 280 echo "## ERROR: Could not load kmod/igb_uio.ko." 281 quit 282 fi 283} 284 285# Removes all reserved hugepages. 286clear_numa_pages() 287{ 288 echo > .echo_tmp 289 for d in /sys/devices/system/node/node? ; do 290 echo "echo 0 > $d/hugepages/hugepages-2048kB/nr_hugepages" >> .echo_tmp 291 done 292 echo "Removing currently reserved hugepages" 293 sudo sh .echo_tmp 294 rm -f .echo_tmp 295 296 remove_mnt_huge 297} 298# Removes all reserved hugepages. 299clear_non_numa_pages() 300{ 301 echo > .echo_tmp 302 echo "echo 0 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages" > .echo_tmp 303 echo "Removing currently reserved hugepages" 304 sudo sh .echo_tmp 305 rm -f .echo_tmp 306 remove_mnt_huge 307} 308 309# 310# Creates hugepages. 311# 312set_non_numa_pages_default() 313{ 314 clear_non_numa_pages 315 echo > .echo_tmp 316 echo "Number of pages : 1024" 317 318 Pages=1024 319 echo "echo $Pages > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages" >> .echo_tmp 320 321 echo "Reserving hugepages" 322 sudo sh .echo_tmp 323 rm -f .echo_tmp 324 325 create_mnt_huge 326} 327 328# 329# Creates hugepages. 330# 331set_non_numa_pages() 332{ 333 334 clear_non_numa_pages 335 336 echo "" 337 echo " Input the number of 2MB pages" 338 echo " Example: to have 128MB of hugepages available, enter '64' to" 339 echo " reserve 64 * 2MB pages" 340 echo -n "Number of pages: " 341 read Pages 342 343 echo "echo $Pages > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages" > .echo_tmp 344 345 echo "Reserving hugepages" 346 sudo sh .echo_tmp 347 rm -f .echo_tmp 348 349 create_mnt_huge 350 351} 352 353# Creates hugepages on specific NUMA nodes. 354set_numa_pages_default() 355{ 356 clear_numa_pages 357 echo > .echo_tmp 358 for d in /sys/devices/system/node/node? ; do 359 node=$(basename $d) 360 echo "Number of pages for $node: 1024" 361 Pages=1024 362 echo "echo $Pages > $d/hugepages/hugepages-2048kB/nr_hugepages" >> .echo_tmp 363 done 364 echo "Reserving hugepages" 365 sudo sh .echo_tmp 366 rm -f .echo_tmp 367 368 create_mnt_huge 369} 370 371# Creates hugepages on specific NUMA nodes. 372set_numa_pages() 373{ 374 clear_numa_pages 375 echo "" 376 echo " Input the number of 2MB pages for each node" 377 echo " Example: to have 128MB of hugepages available per node," 378 echo " enter '64' to reserve 64 * 2MB pages on each node" 379 380 echo > .echo_tmp 381 for d in /sys/devices/system/node/node? ; do 382 node=$(basename $d) 383 echo -n "Number of pages for $node: " 384 read Pages 385 echo "echo $Pages > $d/hugepages/hugepages-2048kB/nr_hugepages" >> .echo_tmp 386 done 387 echo "Reserving hugepages" 388 sudo sh .echo_tmp 389 rm -f .echo_tmp 390 391 create_mnt_huge 392} 393 394# Uses dpdk_nic_bind.py to move devices to work with igb_uio 395bind_nics_to_igb_uio() 396{ 397 if /sbin/lsmod | grep -q igb_uio ; then 398 ${RTE_SDK}/tools/dpdk_nic_bind.py --status 399 echo "" 400 echo "Enter PCI address of device(s) to bind to IGB UIO driver (e.g., \"04:00.0 04:00.1\")." 401 echo -n "> " 402 read PCI_PATH 403 404 405 sudo ${RTE_SDK}/tools/dpdk_nic_bind.py -b igb_uio $PCI_PATH && echo "OK" 406 else 407 echo "# Please load the 'igb_uio' kernel module before querying or " 408 echo "# adjusting NIC device bindings" 409 fi 410} 411 412# 413# Uses dpdk_nic_bind.py to move devices to work with kernel drivers again 414# 415unbind_nics() 416{ 417 DEVLIST=`ifconfig -s | grep -Eo '^[^ ]+' | tail -n+2` 418 for dev in $DEVLIST 419 do 420 echo $dev 421 if [[ "$dev" == dpdk* ]]; then 422 sudo ifconfig $dev down 423 fi 424 done 425 426 ${RTE_SDK}/tools/dpdk_nic_bind.py --status 427 echo "" 428 echo -n "Enter PCI address of device to unbind: " 429 read PCI_PATH 430 echo "" 431 echo -n "Enter name of kernel driver to bind the device to: " 432 read DRV 433 sudo ${RTE_SDK}/tools/dpdk_nic_bind.py -b $DRV $PCI_PATH && echo "OK" 434} 435 436# Brings up the interface of DPDK devices up 437setup_iface_dpdk() 438{ 439 # Create & configure /dev/dpdk-iface 440 sudo rm -rf /dev/dpdk-iface 441 sudo mknod /dev/dpdk-iface c 1110 0 442 sudo chmod 666 /dev/dpdk-iface 443 444 # First check whether igb_uio module is already loaded 445 MODULE="igb_uio" 446 447 if sudo lsmod | grep "$MODULE" &> /dev/null ; then 448 echo "$MODULE is loaded!" 449 else 450 echo "$MODULE is not loaded!" 451 exit 1 452 fi 453 454 # Next check how many devices are there in the system 455 counter=0 456 cd /sys/module/igb_uio/drivers/pci:igb_uio/ 457 for i in * 458 do 459 if [[ $i == *":"* ]] 460 then 461 let "counter=$counter + 1" 462 fi 463 done 464 cd $CUR_DIR 465 466 iter=0 467 # Configure each device (single-process version) 468 while [ $iter -lt $counter ] 469 do 470 while [ 1 ]; do 471 echo 472 echo "[dpdk$(($iter))] enter IP address (e.g., 10.0.$iter.9)" 473 echo -n "> " 474 read ip_addr 475 valid_ip_addr $ip_addr 476 if [ $? -eq 0 ]; then 477 break 478 fi 479 echo "invalid IP address!" # continue 480 done 481 echo "sudo /sbin/ifconfig dpdk$(($iter)) $ip_addr netmask 255.255.255.0 up" 482 sudo /sbin/ifconfig dpdk$(($iter)) $ip_addr netmask 255.255.255.0 up 483 let "iter=$iter + 1" 484 done 485} 486############################################################################# 487 488############################### DPDK ######################################## 489step0_func_dpdk() 490{ 491 if [ -d "/sys/devices/system/node" ]; then 492 set_numa_pages_default 493 else 494 set_non_numa_pages_default 495 fi 496 497 step2_func_dpdk 498 step3_func_dpdk 499 step4_func_dpdk 500} 501step1_func_dpdk() 502{ 503 if [ -d "/sys/devices/system/node" ]; then 504 set_numa_pages 505 else 506 set_non_numa_pages 507 fi 508} 509step2_func_dpdk() 510{ 511 load_igb_uio_module 512 bind_nics_to_igb_uio 513} 514step3_func_dpdk() 515{ 516 setup_iface_dpdk $last_octet 517} 518step4_func_dpdk() 519{ 520 cd $CUR_DIR/samples 521 for d in * ; do 522 create_config $FL_DPDK "samples/$d/config" $d 523 done 524 echo 525 echo "------------------------------------------------" 526 echo "Done with configuration file setup." 527 echo "Use the arp command to add static ARP entries" 528 echo "------------------------------------------------" 529} 530step5_func_dpdk() 531{ 532 unbind_nics 533} 534step6_func_dpdk() 535{ 536 echo 537 exit 1 538} 539############################################################################# 540 541############################### PCAP ######################################## 542step0_func_pcap() 543{ 544 cd $CUR_DIR/samples 545 for d in * ; do 546 if [ -d $CUR_DIR/samples/$d/config ]; then 547 create_config $FL_PCAP "samples/$d/config" $d 548 fi 549 done 550 echo 551 echo "------------------------------------------------" 552 echo "Done with configuration file setup." 553 echo "Use the arp command to add static ARP entries" 554 echo "------------------------------------------------" 555 exit 1 556} 557############################################################################# 558 559############################# Main Script ################################### 560export CUR_DIR=$PWD 561kerver=$(uname -r) 562 563if [ "$1" == "--compile-dpdk" ]; then 564 vercomp $kerver "2.6.33" 565 # if kernel version < 2.6.33 566 if [ "$?" == "2" ]; then 567 echo "[note] current kerner version ("$kerver") does not support DPDK" 568 exit 1 569 fi 570 # Build and install DPDK library 571 export DPDK_DIR="drivers/dpdk" 572 export RTE_SDK="drivers/dpdk-2.2.0" 573 export RTE_TARGET="x86_64-native-linuxapp-gcc" 574 export DESTDIR="." 575 echo 576 echo "Selected DPDK library to be used for MOS" 577 echo "----------------------------------------------------------" 578 echo " RTE_SDK exported as $RTE_SDK" 579 echo " RTE_TARGET exported as $RTE_TARGET" 580 echo "----------------------------------------------------------" 581 echo 582 echo -n "Press enter to continue ..."; read 583 cd $RTE_SDK 584 make install T=$RTE_TARGET 585 echo 586 echo -n "Done with DPDK setup. Press enter to start MOS setup ..."; read 587 # Build and compile MOS library 588 setup_mos_library $FL_DPDK 589 590elif [ "$1" == "--compile-pcap" ]; then 591 setup_mos_library $FL_PCAP 592 593elif [ "$1" == "--run-dpdk" ]; then 594 export DPDK_DIR="drivers/dpdk" 595 export RTE_SDK="drivers/dpdk-2.2.0" 596 export RTE_TARGET="x86_64-native-linuxapp-gcc" 597 while [ 1 ]; do 598 clear 599 echo 600 echo "----------------------------------------------------------" 601 echo " Full setup (from start)" 602 echo "----------------------------------------------------------" 603 echo "[0] Full setup for running mOS with DPDK" 604 echo 605 echo "----------------------------------------------------------" 606 echo " Step-by-step setup for running mOS with DPDK" 607 echo "----------------------------------------------------------" 608 echo "[1] Setup hugepage mappings" 609 echo "[2] Load and bind Ethernet devices to IGB_UIO module" 610 echo "[3] Bring the interfaces up (DPDK devices)" 611 echo "[4] Create new MOS configuration files for sample apps" 612 echo "[5] Unbind dpdk-registered NICs" 613 echo 614 echo "[6] Exit script" 615 echo 616 echo -n "Option: " 617 read entry 618 if [ "$entry" ]; then 619 step"$entry"_func_dpdk 620 echo 621 echo -n "Press enter to continue ..."; read 622 fi 623 done 624 625elif [ "$1" == "--run-pcap" ]; then 626 while [ 1 ]; do 627 clear 628 echo 629 echo "----------------------------------------------------------" 630 echo " Full setup (from start)" 631 echo "----------------------------------------------------------" 632 step0_func_pcap 633 echo 634 echo -n "Press enter to continue ..."; read 635 done 636 637elif [ "$1" == "--cleanup" ]; then 638 find | grep mos.conf | xargs rm -f 639 find | grep log__* | xargs rm -f 640 641 for d in $CUR_DIR/samples/* ; do 642 cd $d 643 make clean > /dev/null 2> /dev/null 644 cd .. 645 done 646 cd .. 647 find ./samples/ -name 'Makefile' | xargs rm -f 648 rm -f scripts/config.log scripts/config.status 649else 650 echo "[error] please specify one of the following options" 651 echo "--compile-dpdk : compile and build mOS library with dpdk" 652 echo "--compile-pcap : compile and build mOS library with pcap" 653 echo "--run-dpdk : setup environments and configurations for running applications with dpdk" 654 echo "--run-pcap : setup environments and configurations for running applications with pcap" 655 echo "--cleanup : delete all binaries, config and Makefiles" 656fi 657