scripts/configure-compute-node-UBUNTU14.04.sh

   1 #!/bin/bash
   2
   3 ##
   4 # Copyright 2015 Telefónica Investigación y Desarrollo, S.A.U.
   5 # This file is part of openvim
   6 # All Rights Reserved.
   7 #
   8 # Licensed under the Apache License, Version 2.0 (the "License"); you may
   9 # not use this file except in compliance with the License. You may obtain
  10 # a copy of the License at
  11 #
  12 #         http://www.apache.org/licenses/LICENSE-2.0
  13 #
  14 # Unless required by applicable law or agreed to in writing, software
  15 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  16 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  17 # License for the specific language governing permissions and limitations
  18 # under the License.
  19 #
  20 # For those usages not covered by the Apache License, Version 2.0 please
  21 # contact with: nfvlabs@tid.es
  22 ##
  23
  24 # Authors: Antonio Lopez, Pablo Montes, Alfonso Tierno
  25 # June 2015
  26
  27 # Personalize RHEL7.1 on compute nodes
  28 # Prepared to work with the following network card drivers:
  29 #       tg3, igb drivers for management interfaces
  30 #       ixgbe (Intel Niantic) and i40e (Intel Fortville) drivers for data plane interfaces
  31
  32 # To download:
  33 # wget https://raw.githubusercontent.com/nfvlabs/openmano/master/scripts/configure-compute-node-RHEL7.1.sh
  34 # To execute:
  35 # chmod +x ./configure-compute-node-RHEL7.1.sh
  36 # sudo ./configure-compute-node-RHEL7.1.sh <user> <iface>
  37
  38 # Assumptions:
  39 # All virtualization options activated on BIOS (vt-d, vt-x, SR-IOV, no power savings...)
  40 # RHEL7.1 installed without /home partition and with the following packages selection:
  41 # @base, @core, @development, @network-file-system-client, @virtualization-hypervisor, @virtualization-platform, @virtualization-tools
  42
  43
  44 function usage(){
  45     echo -e "Usage: sudo $0 [-y] <user-name>  [ <iface-name>  [<ip-address>|dhcp] ]"
  46     echo -e "  Configure compute host for VIM usage. (version 0.4). Params:"
  47     echo -e "     -y  do not prompt for confirmation. If a new user is created, the user name is set as password"
  48     echo -e "     <user-name> Create if not exist and configure this user for openvim to connect"
  49     echo -e "     <iface-name> if suplied creates bridge interfaces on this interface, needed for openvim"
  50     echo -e "     ip or dhcp if suplied, configure the interface with this ip address (/24) or 'dhcp' "
  51 }
  52
  53
  54 #1 CHECK input parameters
  55 #1.1 root privileges
  56 [ "$USER" != "root" ] && echo "Needed root privileges" && usage && exit -1
  57
  58 #1.2 input parameters
  59 FORCE=""
  60 while getopts "y" o; do
  61     case "${o}" in
  62         y)
  63             FORCE="yes"
  64             ;;
  65         *)
  66             usage
  67             exit -1
  68             ;;
  69     esac
  70 done
  71 shift $((OPTIND-1))
  72
  73
  74
  75 if [ $# -lt 1 ]
  76 then
  77   usage
  78   exit
  79 fi
  80
  81
  82 user_name=$1
  83 interface=$2
  84 ip_iface=$3
  85
  86 if [ -n "$interface" ] && ! ifconfig $interface &> /dev/null
  87 then
  88   echo "Error: interface '$interface' is not present in the system"
  89   usage
  90   exit 1
  91 fi
  92
  93 echo '
  94 #################################################################
  95 #####       INSTALL NEEDED PACKETS                          #####
  96 #################################################################'
  97
  98 # Required packages
  99 apt-get -y update
 100 #apt-get -y install grub-common screen virt-manager ethtool build-essential x11-common x11-utils x11-apps libguestfs-tools hwloc libguestfs-tools numactl vlan nfs-common nfs-kernel-server
 101 apt-get -y install grub-common screen virt-manager ethtool build-essential x11-common x11-utils libguestfs-tools hwloc libguestfs-tools numactl vlan nfs-common nfs-kernel-server
 102
 103 echo "Remove unneeded packages....."
 104 apt-get -y autoremove
 105 # Selinux management
 106 #yum install -y policycoreutils-python
 107
 108
 109
 110 echo '
 111 #################################################################
 112 #####       INSTALL USER                                    #####
 113 #################################################################'
 114
 115 # Add required groups
 116 groupadd -f admin
 117 groupadd -f libvirt   #for other operating systems may be libvirtd
 118
 119 # Adds user, default password same as name
 120 if grep -q "^${user_name}:" /etc/passwd
 121 then
 122   #user exist, add to group
 123   echo "adding user ${user_name} to groups libvirt,admin"
 124   usermod -a -G libvirt,admin -g admin $user_name
 125 else
 126   #create user if it does not exist
 127   [ -z "$FORCE" ] && read -p "user '${user_name}' does not exist, create (Y/n)" kk
 128   if ! [ -z "$kk" -o "$kk"="y" -o "$kk"="Y" ]
 129   then
 130     exit
 131   fi
 132   echo "creating and configuring user ${user_name}"
 133   useradd -m -G libvirt,admin -g admin $user_name
 134   #Password
 135   if [ -z "$FORCE" ]
 136   then
 137      echo "Provide a password for $user_name"
 138      passwd $user_name
 139   else
 140      echo -e "$user_name\n$user_name" | passwd --stdin $user_name
 141   fi
 142 fi
 143
 144 # Allow admin users to access without password
 145 if ! grep -q "#openmano" /etc/sudoers
 146 then
 147     cat >> /home/${user_name}/script_visudo.sh << EOL
 148 #!/bin/bash
 149 cat \$1 | awk '(\$0~"requiretty"){print "#"\$0}(\$0!~"requiretty"){print \$0}' > tmp
 150 cat tmp > \$1
 151 rm tmp
 152 echo "" >> \$1
 153 echo "#openmano allow to group admin to grant root privileges without password" >> \$1
 154 echo "%admin ALL=(ALL) NOPASSWD: ALL" >> \$1
 155 EOL
 156     chmod +x /home/${user_name}/script_visudo.sh
 157     echo "allowing admin user to get root privileges withut password"
 158     export EDITOR=/home/${user_name}/script_visudo.sh && sudo -E visudo
 159     rm -f /home/${user_name}/script_visudo.sh
 160 fi
 161
 162
 163 echo '
 164 #################################################################
 165 #####       INSTALL HUGEPAGES ISOLCPUS GRUB                 #####
 166 #################################################################'
 167
 168 # Huge pages 1G auto mount
 169 mkdir -p /mnt/huge
 170 if ! grep -q "Huge pages" /etc/fstab
 171 then
 172   echo "" >> /etc/fstab
 173   echo "# Huge pages" >> /etc/fstab
 174   echo "nodev /mnt/huge hugetlbfs pagesize=1GB 0 0" >> /etc/fstab
 175   echo "" >> /etc/fstab
 176 fi
 177
 178 # Grub virtualization options:
 179
 180 # Get isolcpus
 181 isolcpus=`gawk 'BEGIN{pre=-2;}
 182   ($1=="processor"){pro=$3;}
 183   ($1=="core" && $4!=0){
 184      if (pre+1==pro){endrange="-" pro}
 185      else{cpus=cpus endrange sep pro; sep=","; endrange="";};
 186      pre=pro;}
 187   END{printf("%s",cpus endrange);}' /proc/cpuinfo`
 188
 189
 190 echo "CPUS: $isolcpus"
 191
 192 # Huge pages reservation file: reserving all memory apart from 4GB per NUMA node
 193 # Get the number of hugepages: all memory but 8GB reserved for the OS
 194 #totalmem=`dmidecode --type 17|grep Size |grep MB |gawk '{suma+=$2} END {print suma/1024}'`
 195 #hugepages=$(($totalmem-8))
 196
 197 if ! [ -f /usr/lib/systemd/hugetlb-reserve-pages ]
 198 then
 199   cat > /usr/lib/systemd/hugetlb-reserve-pages << EOL
 200 #!/bin/bash
 201 nodes_path=/sys/devices/system/node/
 202 if [ ! -d \$nodes_path ]; then
 203         echo "ERROR: \$nodes_path does not exist"
 204         exit 1
 205 fi
 206
 207 reserve_pages()
 208 {
 209         echo \$1 > \$nodes_path/\$2/hugepages/hugepages-1048576kB/nr_hugepages
 210 }
 211
 212 # This example reserves all available memory apart from 4 GB for linux
 213 # using 1GB size. You can modify it to your needs or comment the lines
 214 # to avoid reserve memory in a numa node
 215 EOL
 216   for f in /sys/devices/system/node/node?/meminfo
 217   do
 218     node=`head -n1 $f | gawk '($5=="kB"){print $2}'`
 219     memory=`head -n1 $f | gawk '($5=="kB"){print $4}'`
 220     memory=$((memory+1048576-1))   #memory must be ceiled
 221     memory=$((memory/1048576))   #from `kB to GB
 222     #if memory
 223     [ $memory -gt 4 ] && echo "reserve_pages $((memory-4)) node$node" >> /usr/lib/systemd/hugetlb-reserve-pages
 224   done
 225
 226   # Run the following commands to enable huge pages early boot reservation:
 227   chmod +x /usr/lib/systemd/hugetlb-reserve-pages
 228   systemctl enable hugetlb-gigantic-pages
 229 fi
 230
 231 # Prepares the text to add at the end of the grub line, including blacklisting ixgbevf driver in the host
 232 memtotal=`grep MemTotal /proc/meminfo | awk '{ print $2 }' `
 233 hpages=$(( ($memtotal/(1024*1024))-8 ))
 234
 235 memtotal=$((memtotal+1048576-1))   #memory must be ceiled
 236 memtotal=$((memtotal/1048576))   #from `kB to GBa
 237 hpages=$((memtotal-8))
 238 [[ $hpages -lt 0 ]] $$ hpages=0
 239
 240
 241 echo "------> memtotal: $memtotal"
 242
 243 textokernel="intel_iommu=on default_hugepagesz=1G hugepagesz=1G hugepages=$hpages isolcpus=$isolcpus modprobe.blacklist=ixgbevf modprobe.blacklist=i40evf"
 244
 245 echo "Text to kernel: $textokernel"
 246
 247
 248 # Add text to the kernel line
 249 if ! grep -q "intel_iommu=on default_hugepagesz=1G hugepagesz=1G" /etc/default/grub
 250 then
 251   echo ">>>>>>>  adding cmdline ${textokernel}"
 252   sed -i "/^GRUB_CMDLINE_LINUX_DEFAULT=/s/\"\$/${textokernel}\"/" /etc/default/grub
 253   # grub2 upgrade
 254   #grub2-mkconfig -o /boot/grub2/grub.cfg
 255   update-grub
 256 fi
 257
 258 echo '
 259 #################################################################
 260 #####       OTHER CONFIGURATION                             #####
 261 #################################################################'
 262
 263 # Links the OpenMANO required folder /opt/VNF/images to /var/lib/libvirt/images. The OS installation
 264 # should have only a / partition with all possible space available
 265
 266 echo " link /opt/VNF/images to /var/lib/libvirt/images"
 267 if [ "$user_name" != "" ]
 268 then
 269   #mkdir -p /home/${user_name}/VNF_images
 270   #chown -R ${user_name}:admin /home/${user_name}/VNF_images
 271   #chmod go+x $HOME
 272
 273   # The orchestator needs to link the images folder
 274   rm -f /opt/VNF/images
 275   mkdir -p /opt/VNF/
 276   ln -s /var/lib/libvirt/images /opt/VNF/images
 277   chown -R ${user_name}:admin /opt/VNF
 278   chown -R root:admin /var/lib/libvirt/images
 279   chmod g+rwx /var/lib/libvirt/images
 280
 281   # Selinux management
 282   #echo "configure  Selinux management"
 283   #semanage fcontext -a -t virt_image_t "/home/${user_name}/VNF_images(/.*)?"
 284   #cat /etc/selinux/targeted/contexts/files/file_contexts.local |grep virt_image
 285   #restorecon -R -v /home/${user_name}/VNF_images
 286 else
 287   mkdir -p /opt/VNF/images
 288   chmod o+rx /opt/VNF/images
 289 fi
 290
 291 echo "creating local information /opt/VNF/images/hostinfo.yaml"
 292 echo "#By default openvim assumes control plane interface naming as em1,em2,em3,em4 " > /opt/VNF/images/hostinfo.yaml
 293 echo "#and bridge ifaces as virbrMan1, virbrMan2, ..." >> /opt/VNF/images/hostinfo.yaml
 294 echo "#if compute node contain a different name it must be indicated in this file" >> /opt/VNF/images/hostinfo.yaml
 295 echo "#with the format extandard-name: compute-name" >> /opt/VNF/images/hostinfo.yaml
 296 if [ "$interface" != "" -a "$interface" != "em1" ]
 297 then
 298   echo "iface_names:"  >> /opt/VNF/images/hostinfo.yaml
 299   echo "  em1: ${interface}" >> /opt/VNF/images/hostinfo.yaml
 300 fi
 301 chmod o+r /opt/VNF/images/hostinfo.yaml
 302
 303 # deactivate memory overcommit
 304 #echo "deactivate memory overcommit"
 305 #service ksmtuned stop
 306 #service ksm stop
 307 #chkconfig ksmtuned off
 308 #chkconfig ksm off
 309
 310
 311 # Libvirt options (uncomment the following)
 312 echo "configure Libvirt options"
 313 sed -i 's/#unix_sock_group = "libvirt"/unix_sock_group = "libvirt"/' /etc/libvirt/libvirtd.conf
 314 sed -i 's/#unix_sock_rw_perms = "0770"/unix_sock_rw_perms = "0770"/' /etc/libvirt/libvirtd.conf
 315 sed -i 's/#unix_sock_dir = "\/var\/run\/libvirt"/unix_sock_dir = "\/var\/run\/libvirt"/' /etc/libvirt/libvirtd.conf
 316 sed -i 's/#auth_unix_rw = "none"/auth_unix_rw = "none"/' /etc/libvirt/libvirtd.conf
 317
 318
 319 echo '
 320 #################################################################
 321 #####       NETWORK CONFIGURATION                           #####
 322 #################################################################'
 323 # Network config (if the second parameter is net)
 324 echo "Interface ==> $interface"
 325 if [ -n "$interface" ]
 326 then
 327
 328
 329   # For management and data interfaces
 330   rm -f /etc/udev/rules.d/pci_config.rules # it will be created to define VFs
 331
 332
 333   # Set ONBOOT=on and MTU=9000 on the interface used for the bridges
 334   echo "configuring iface $interface"
 335
 336 #MTU for interfaces and bridges
 337 MTU=9000
 338
 339 cp /etc/network/interfaces interfaces.tmp
 340
 341
 342   #Create infrastructure bridge, normally used for connecting to compute nodes, openflow controller, ...
 343
 344
 345     #Create VLAN for infrastructure bridge
 346
 347  echo "
 348 ######### CUTLINE #########
 349
 350 auto ${interface}
 351 iface ${interface} inet static
 352         mtu $MTU
 353
 354 auto ${interface}.1001
 355 iface ${interface}.1001 inet static
 356         mtu $MTU
 357 " >> interfaces.tmp
 358
 359  echo "ifconfig ${interface} mtu $MTU
 360  ifconfig ${interface} up
 361 " > mtu.tmp
 362
 363
 364   #Create bridge interfaces
 365   echo "Creating bridge ifaces: "
 366   for ((i=1;i<=20;i++))
 367   do
 368     i2digits=$i
 369     [ $i -lt 10 ] && i2digits="0$i"
 370     echo "    virbrMan$i  vlan 20$i2digits"
 371
 372     j=$i
 373
 374  echo "
 375 auto ${interface}.20$i2digits
 376 iface ${interface}.20$i2digits inet static
 377         mtu $MTU
 378
 379 auto virbrMan$j
 380 iface virbrMan$j inet static
 381         bridge_ports ${interface}.20$i2digits
 382         mtu $MTU
 383 " >> interfaces.tmp
 384
 385  echo "ifconfig ${interface}.20$i2digits mtu $MTU
 386 ifconfig virbrMan$j mtu $MTU
 387 ifconfig virbrMan$j up
 388 " >> mtu.tmp
 389
 390   done
 391
 392  echo "
 393 auto em2.1001
 394 iface em2.1001 inet static
 395
 396 auto virbrInf
 397 iface virbrInf inet static
 398         bridge_ports em2.1001
 399 " >> interfaces.tmp
 400
 401  echo "ifconfig em2.1001 mtu $MTU
 402 ifconfig virbrInf mtu $MTU
 403 ifconfig virbrInf up
 404 " >> mtu.tmp
 405
 406 if ! grep -q "#### CUTLINE ####" /etc/network/interfaces
 407 then
 408         echo "====== Copying interfaces.tmp to /etc/network/interfaces"
 409         cp interfaces.tmp /etc/network/interfaces
 410 fi
 411
 412
 413   #popd
 414 fi
 415
 416
 417 # Activate 8 Virtual Functions per PF on Niantic cards (ixgbe driver)
 418 if [[ `lsmod | cut -d" " -f1 | grep "ixgbe" | grep -v vf` ]]
 419 then
 420         if ! grep -q "ixgbe" /etc/modprobe.d/ixgbe.conf
 421         then
 422         echo "options ixgbe max_vfs=8" >> /etc/modprobe.d/ixgbe.conf
 423         fi
 424
 425 fi
 426
 427 # Set dataplane MTU
 428
 429 echo "sleep 10" >> mtu.tmp
 430
 431 interfaces=`ifconfig -a | grep ^p | cut -d " " -f 1`
 432 for ph in $interfaces
 433 do
 434         echo "ifconfig $ph mtu $MTU" >> mtu.tmp
 435         echo "ifconfig $ph up" >> mtu.tmp
 436 done
 437
 438
 439
 440 cp mtu.tmp /etc/setmtu.sh
 441 chmod +x /etc/setmtu.sh
 442
 443 # To define 8 VFs per PF we do it on rc.local, because the driver needs to be unloaded and loaded again
 444 #if ! grep -q "NFV" /etc/rc.local
 445 #then
 446   echo "#!/bin/sh -e
 447 " > /etc/rc.local
 448   echo "# NFV" >> /etc/rc.local
 449   echo "modprobe -r ixgbe" >> /etc/rc.local
 450   echo "modprobe ixgbe max_vfs=8" >> /etc/rc.local
 451   echo "/etc/setmtu.sh" >> /etc/rc.local
 452   echo "
 453 exit 0" >> /etc/rc.local
 454   echo "" >> /etc/rc.local
 455
 456   chmod +x /etc/rc.d/rc.local
 457
 458 #fi
 459
 460 chmod a+rwx /var/lib/libvirt/images
 461 mkdir /usr/libexec/
 462 pushd /usr/libexec/
 463 ln -s /usr/bin/qemu-system-x86_64 qemu-kvm
 464 popd
 465
 466 #Deactivating apparmor while looking for a better solution
 467 /etc/init.d/apparmor stop
 468 update-rc.d -f apparmor remove
 469
 470 echo
 471 echo "Do not forget to create a shared (NFS, Samba, ...) where original virtual machine images are allocated"
 472 echo
 473 echo "Do not forget to copy the public ssh key into /home/${user_name}/.ssh/authorized_keys for authomatic login from openvim controller"
 474 echo
 475
 476 echo "Reboot the system to make the changes effective"
 477
 478