scripts/configure-compute-node-RHEL7.1.sh

   1 #!/bin/bash
   2
   3 ##
   4 # Copyright 2015 Telefónica Investigación y Desarrollo, S.A.U.
   5 # This file is part of openvim
   6 # All Rights Reserved.
   7 #
   8 # Licensed under the Apache License, Version 2.0 (the "License"); you may
   9 # not use this file except in compliance with the License. You may obtain
  10 # a copy of the License at
  11 #
  12 #         http://www.apache.org/licenses/LICENSE-2.0
  13 #
  14 # Unless required by applicable law or agreed to in writing, software
  15 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  16 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  17 # License for the specific language governing permissions and limitations
  18 # under the License.
  19 #
  20 # For those usages not covered by the Apache License, Version 2.0 please
  21 # contact with: nfvlabs@tid.es
  22 ##
  23
  24 # Authors: Antonio Lopez, Pablo Montes, Alfonso Tierno
  25 # July 2015
  26
  27 # Personalize RHEL7.1 on compute nodes
  28 # Prepared to work with the following network card drivers:
  29 #       tg3, igb drivers for management interfaces
  30 #       ixgbe (Intel Niantic) and i40e (Intel Fortville) drivers for data plane interfaces
  31
  32 # To download:
  33 # wget https://raw.githubusercontent.com/nfvlabs/openmano/master/scripts/configure-compute-node-RHEL7.1.sh
  34 # To execute:
  35 # chmod +x ./configure-compute-node-RHEL7.1.sh
  36 # sudo ./configure-compute-node-RHEL7.1.sh <user> <iface>
  37
  38 # Assumptions:
  39 # All virtualization options activated on BIOS (vt-d, vt-x, SR-IOV, no power savings...)
  40 # RHEL7.1 installed without /home partition and with the following packages selection:
  41 # @base, @core, @development, @network-file-system-client, @virtualization-hypervisor, @virtualization-platform, @virtualization-tools
  42
  43
  44 function usage(){
  45     echo -e "Usage: sudo $0 [-y] <user-name>  [ <iface-name>  [<ip-address>|dhcp] ]"
  46     echo -e "  Configure compute host for VIM usage. (version 0.4). Params:"
  47     echo -e "     -y  do not prompt for confirmation. If a new user is created, the user name is set as password"
  48     echo -e "     <user-name> Create if not exist and configure this user for openvim to connect"
  49     echo -e "     <iface-name> if suplied creates bridge interfaces on this interface, needed for openvim"
  50     echo -e "     ip or dhcp if suplied, configure the interface with this ip address (/24) or 'dhcp' "
  51 }
  52
  53
  54 #1 CHECK input parameters
  55 #1.1 root privileges
  56 [ "$USER" != "root" ] && echo "Needed root privileges" && usage && exit -1
  57
  58 #1.2 input parameters
  59 FORCE=""
  60 while getopts "y" o; do
  61     case "${o}" in
  62         y)
  63             FORCE="yes"
  64             ;;
  65         *)
  66             usage
  67             exit -1
  68             ;;
  69     esac
  70 done
  71 shift $((OPTIND-1))
  72
  73
  74 if [ $# -lt 1 ]
  75 then
  76   usage
  77   exit
  78 fi
  79
  80 user_name=$1
  81 interface=$2
  82 ip_iface=$3
  83
  84 if [ -n "$interface" ] && ! ifconfig $interface &> /dev/null
  85 then
  86   echo "Error: interface '$interface' is not present in the system"
  87   usage
  88   exit 1
  89 fi
  90
  91 echo '
  92 #################################################################
  93 #####       INSTALL NEEDED PACKETS                          #####
  94 #################################################################'
  95
  96 # Required packages
  97 yum repolist
  98 yum check-update
  99 yum update -y
 100 yum install -y screen virt-manager ethtool gcc gcc-c++ xorg-x11-xauth xorg-x11-xinit xorg-x11-deprecated-libs libXtst guestfish hwloc libhugetlbfs-utils libguestfs-tools numactl
 101 # Selinux management
 102 yum install -y policycoreutils-python
 103
 104 echo '
 105 #################################################################
 106 #####       INSTALL USER                                    #####
 107 #################################################################'
 108
 109 # Add required groups
 110 groupadd -f nfvgroup
 111 groupadd -f libvirt   #for other operating systems may be libvirtd
 112
 113 # Adds user, default password same as name
 114 if grep -q "^${user_name}:" /etc/passwd
 115 then
 116   #user exist, add to group
 117   echo "adding user ${user_name} to groups libvirt,nfvgroup"
 118   usermod -a -G libvirt,nfvgroup -g nfvgroup $user_name
 119 else
 120   #create user if it does not exist
 121   [ -z "$FORCE" ] && read -p "user '${user_name}' does not exist, create (Y/n)" kk
 122   if ! [ -z "$kk" -o "$kk"="y" -o "$kk"="Y" ]
 123   then
 124     exit
 125   fi
 126   echo "creating and configuring user ${user_name}"
 127   useradd -m -G libvirt,nfvgroup -g nfvgroup $user_name
 128   #Password
 129   if [ -z "$FORCE" ]
 130   then
 131      echo "Provide a password for $user_name"
 132      passwd $user_name
 133   else
 134      echo -e "$user_name\n$user_name" | passwd --stdin $user_name
 135   fi
 136 fi
 137
 138 #Setting default libvirt URI for the user
 139 echo "Setting default libvirt URI for the user"
 140 echo "if test -x `which virsh`; then" >> /home/${user_name}/.bash_profile
 141 echo "  export LIBVIRT_DEFAULT_URI=qemu:///system" >> /home/${user_name}/.bash_profile
 142 echo "fi" >> /home/${user_name}/.bash_profile
 143
 144 echo '
 145 #################################################################
 146 #####       INSTALL HUGEPAGES ISOLCPUS GRUB                 #####
 147 #################################################################'
 148
 149 # Huge pages 1G auto mount
 150 mkdir -p /mnt/huge
 151 if ! grep -q "Huge pages" /etc/fstab
 152 then
 153   echo "" >> /etc/fstab
 154   echo "# Huge pages" >> /etc/fstab
 155   echo "nodev /mnt/huge hugetlbfs pagesize=1GB 0 0" >> /etc/fstab
 156   echo "" >> /etc/fstab
 157 fi
 158
 159 # Huge pages reservation service
 160 if ! [ -f /usr/lib/systemd/system/hugetlb-gigantic-pages.service ]
 161 then
 162   echo "configuring huge pages service"
 163   cat > /usr/lib/systemd/system/hugetlb-gigantic-pages.service << EOL
 164 [Unit]
 165 Description=HugeTLB Gigantic Pages Reservation
 166 DefaultDependencies=no
 167 Before=dev-hugepages.mount
 168 ConditionPathExists=/sys/devices/system/node
 169 ConditionKernelCommandLine=hugepagesz=1G
 170
 171 [Service]
 172 Type=oneshot
 173 RemainAfterExit=yes
 174 ExecStart=/usr/lib/systemd/hugetlb-reserve-pages
 175
 176 [Install]
 177 WantedBy=sysinit.target
 178 EOL
 179 fi
 180 # Grub virtualization options:
 181
 182 # Get isolcpus
 183 isolcpus=`gawk 'BEGIN{pre=-2;}
 184   ($1=="processor"){pro=$3;}
 185   ($1=="core" && $4!=0){
 186      if (pre+1==pro){endrange="-" pro}
 187      else{cpus=cpus endrange sep pro; sep=","; endrange="";};
 188      pre=pro;}
 189   END{printf("%s",cpus endrange);}' /proc/cpuinfo`
 190
 191
 192 # Huge pages reservation file: reserving all memory apart from 4GB per NUMA node
 193 # Get the number of hugepages: all memory but 8GB reserved for the OS
 194 #totalmem=`dmidecode --type 17|grep Size |grep MB |gawk '{suma+=$2} END {print suma/1024}'`
 195 #hugepages=$(($totalmem-8))
 196
 197 if ! [ -f /usr/lib/systemd/hugetlb-reserve-pages ]
 198 then
 199   cat > /usr/lib/systemd/hugetlb-reserve-pages << EOL
 200 #!/bin/bash
 201 nodes_path=/sys/devices/system/node/
 202 if [ ! -d \$nodes_path ]; then
 203         echo "ERROR: \$nodes_path does not exist"
 204         exit 1
 205 fi
 206
 207 reserve_pages()
 208 {
 209         echo \$1 > \$nodes_path/\$2/hugepages/hugepages-1048576kB/nr_hugepages
 210 }
 211
 212 # This example reserves all available memory apart from 4 GB for linux
 213 # using 1GB size. You can modify it to your needs or comment the lines
 214 # to avoid reserve memory in a numa node
 215 EOL
 216   for f in /sys/devices/system/node/node?/meminfo
 217   do
 218     node=`head -n1 $f | gawk '($5=="kB"){print $2}'`
 219     memory=`head -n1 $f | gawk '($5=="kB"){print $4}'`
 220     memory=$((memory+1048576-1))   #memory must be ceiled
 221     memory=$((memory/1048576))   #from `kB to GB
 222     #if memory
 223     [ $memory -gt 4 ] && echo "reserve_pages $((memory-4)) node$node" >> /usr/lib/systemd/hugetlb-reserve-pages
 224   done
 225
 226   # Run the following commands to enable huge pages early boot reservation:
 227   chmod +x /usr/lib/systemd/hugetlb-reserve-pages
 228   systemctl enable hugetlb-gigantic-pages
 229 fi
 230
 231 # Prepares the text to add at the end of the grub line, including blacklisting ixgbevf driver in the host
 232 textokernel="intel_iommu=on default_hugepagesz=1G hugepagesz=1G isolcpus=$isolcpus modprobe.blacklist=ixgbevf modprobe.blacklist=i40evf"
 233
 234 # Add text to the kernel line
 235 if ! grep -q "intel_iommu=on default_hugepagesz=1G hugepagesz=1G" /etc/default/grub
 236 then
 237   echo "adding cmdline ${textokernel}"
 238   sed -i "/^GRUB_CMDLINE_LINUX=/s/\"\$/ ${textokernel}\"/" /etc/default/grub
 239   # grub2 upgrade
 240   grub2-mkconfig -o /boot/grub2/grub.cfg
 241 fi
 242
 243 echo '
 244 #################################################################
 245 #####       OTHER CONFIGURATION                             #####
 246 #################################################################'
 247
 248 # Disable requiretty
 249 if ! grep -q "#openmano" /etc/sudoers
 250 then
 251     cat >> /home/${user_name}/script_visudo.sh << EOL
 252 #!/bin/bash
 253 cat \$1 | awk '(\$0~"requiretty"){print "#"\$0}(\$0!~"requiretty"){print \$0}' > tmp
 254 cat tmp > \$1
 255 rm tmp
 256 EOL
 257     chmod +x /home/${user_name}/script_visudo.sh
 258     echo "Disabling requitetty"
 259     export EDITOR=/home/${user_name}/script_visudo.sh && sudo -E visudo
 260     rm -f /home/${user_name}/script_visudo.sh
 261 fi
 262
 263 #Configure polkint to run virsh as a normal user
 264 echo "Configuring polkint to run virsh as a normal user"
 265 cat >> /etc/polkit-1/localauthority/50-local.d/50-org.libvirt-access.pkla  << EOL
 266 [libvirt Admin Access]
 267 Identity=unix-group:libvirt
 268 Action=org.libvirt.unix.manage
 269 ResultAny=yes
 270 ResultInactive=yes
 271 ResultActive=yes
 272 EOL
 273
 274 # Links the OpenMANO required folder /opt/VNF/images to /var/lib/libvirt/images. The OS installation
 275 # should have only a / partition with all possible space available
 276
 277 echo " link /opt/VNF/images to /var/lib/libvirt/images"
 278 if [ "$user_name" != "" ]
 279 then
 280   #mkdir -p /home/${user_name}/VNF_images
 281   #chown -R ${user_name}:nfvgroup /home/${user_name}/VNF_images
 282   #chmod go+x $HOME
 283
 284   # The orchestator needs to link the images folder
 285   rm -f /opt/VNF/images
 286   mkdir -p /opt/VNF/
 287   ln -s /var/lib/libvirt/images /opt/VNF/images
 288   chown -R ${user_name}:nfvgroup /opt/VNF
 289   chown -R root:nfvgroup /var/lib/libvirt/images
 290   chmod g+rwx /var/lib/libvirt/images
 291
 292   # Selinux management
 293   #echo "configure  Selinux management"
 294   #semanage fcontext -a -t virt_image_t "/home/${user_name}/VNF_images(/.*)?"
 295   #cat /etc/selinux/targeted/contexts/files/file_contexts.local |grep virt_image
 296   #restorecon -R -v /home/${user_name}/VNF_images
 297 else
 298   mkdir -p /opt/VNF/images
 299   chmod o+rx /opt/VNF/images
 300 fi
 301
 302 echo "creating local information /opt/VNF/images/hostinfo.yaml"
 303 echo "#By default openvim assumes control plane interface naming as em1,em2,em3,em4 " > /opt/VNF/images/hostinfo.yaml
 304 echo "#and bridge ifaces as virbrMan1, virbrMan2, ..." >> /opt/VNF/images/hostinfo.yaml
 305 echo "#if compute node contain a different name it must be indicated in this file" >> /opt/VNF/images/hostinfo.yaml
 306 echo "#with the format extandard-name: compute-name" >> /opt/VNF/images/hostinfo.yaml
 307 if [ "$interface" != "" -a "$interface" != "em1" ]
 308 then
 309   echo "iface_names:"  >> /opt/VNF/images/hostinfo.yaml
 310   echo "  em1: ${interface}" >> /opt/VNF/images/hostinfo.yaml
 311 fi
 312 chmod o+r /opt/VNF/images/hostinfo.yaml
 313
 314 # deactivate memory overcommit
 315 echo "deactivate memory overcommit"
 316 service ksmtuned stop
 317 service ksm stop
 318 chkconfig ksmtuned off
 319 chkconfig ksm off
 320
 321
 322 # Libvirt options (uncomment the following)
 323 echo "configure Libvirt options"
 324 sed -i 's/#unix_sock_group = "libvirt"/unix_sock_group = "libvirt"/' /etc/libvirt/libvirtd.conf
 325 sed -i 's/#unix_sock_rw_perms = "0770"/unix_sock_rw_perms = "0770"/' /etc/libvirt/libvirtd.conf
 326 sed -i 's/#unix_sock_dir = "\/var\/run\/libvirt"/unix_sock_dir = "\/var\/run\/libvirt"/' /etc/libvirt/libvirtd.conf
 327 sed -i 's/#auth_unix_rw = "none"/auth_unix_rw = "none"/' /etc/libvirt/libvirtd.conf
 328
 329 #creating the polkit grant access for libvirt user.
 330 #This does not work !!!! so commented. No way to get running without uncomented the auth_unix_rw = "none" line
 331 #
 332 #cat > /etc/polkit-1/localauthority/50-local.d/50-org.example-libvirt-remote-access.pkla << EOL
 333 #[libvirt Management Access]
 334 # Identity=unix-user:n2;unix-user:kk
 335 # Action=org.libvirt.unix.manage
 336 # ResultAny=yes
 337 # ResultInactive=yes
 338 # ResultActive=yes
 339 #EOL
 340
 341 # Configuration change of qemu for the numatune bug issue
 342 # RHEL7.1: for this version should not be necesary - to revise
 343 #if ! grep -q "cgroup_controllers = [ \"cpu\", \"devices\", \"memory\", \"blkio\", \"cpuacct\" ]" /etc/libvirt/qemu.conf
 344 #then
 345 #cat /etc/libvirt/qemu.conf | awk '{print $0}($0~"#cgroup_controllers"){print "cgroup_controllers = [ \"cpu\", \"devices\", \"memory\", \"blkio\", \"cpuacct\" ]"}' > tmp
 346 #mv tmp /etc/libvirt/qemu.conf
 347 #fi
 348
 349 echo '
 350 #################################################################
 351 #####       NETWORK CONFIGURATION                           #####
 352 #################################################################'
 353 # Network config (if the second parameter is net)
 354 if [ -n "$interface" ]
 355 then
 356
 357   # Deactivate network manager
 358   systemctl stop NetworkManager
 359   systemctl disable NetworkManager
 360
 361   # For management and data interfaces
 362   rm -f /etc/udev/rules.d/pci_config.rules # it will be created to define VFs
 363
 364   pushd /etc/sysconfig/network-scripts/
 365
 366   # Set ONBOOT=on and MTU=9000 on the interface used for the bridges
 367   echo "configuring iface $interface"
 368   cat ifcfg-$interface | grep -e HWADDR -e UUID > $interface.tmp
 369   echo "TYPE=Ethernet
 370 NAME=$interface
 371 DEVICE=$interface
 372 TYPE=Ethernet
 373 ONBOOT=yes
 374 NM_CONTROLLED=no
 375 MTU=9000
 376 BOOTPROTO=none
 377 IPV6INIT=no" >> $interface.tmp
 378     mv $interface.tmp  ifcfg-$interface
 379
 380   # Management interfaces
 381 #  integrated_interfaces=""
 382 #  nb_ifaces=0
 383 #  for iface in `ifconfig -a | grep ":\ " | cut -f 1 -d":"| grep -v "_" | grep -v "\." | grep -v "lo" | sort`
 384 #  do
 385 #    driver=`ethtool -i $iface| awk '($0~"driver"){print $2}'`
 386 #    if [ $driver != "ixgbe" ] && [ $driver != "bridge" ]
 387 #    then
 388 #      integrated_interfaces="$integrated_interfaces $iface"
 389 #      nb_ifaces=$((nb_ifaces+1))
 390 #      eval iface${nb_ifaces}=$iface
 391 #    fi
 392 #  done
 393
 394   #Create infrastructure bridge, normally used for connecting to compute nodes, openflow controller, ...
 395   echo "DEVICE=virbrInf
 396 TYPE=Bridge
 397 ONBOOT=yes
 398 DELAY=0
 399 NM_CONTROLLED=no
 400 USERCTL=no" > ifcfg-virbrInf
 401
 402     #Create VLAN for infrastructure bridge
 403     echo "DEVICE=${interface}.1001
 404 ONBOOT=yes
 405 NM_CONTROLLED=no
 406 USERCTL=no
 407 VLAN=yes
 408 BOOTPROTO=none
 409 BRIDGE=virbrInf" > ifcfg-${interface}.1001
 410
 411
 412   #Create bridge interfaces
 413   echo "Creating bridge ifaces: "
 414   for ((i=1;i<=20;i++))
 415   do
 416     i2digits=$i
 417     [ $i -lt 10 ] && i2digits="0$i"
 418     echo "    virbrMan$i  vlan 20$i2digits"
 419     echo "DEVICE=virbrMan$i
 420 TYPE=Bridge
 421 ONBOOT=yes
 422 DELAY=0
 423 NM_CONTROLLED=no
 424 USERCTL=no" > ifcfg-virbrMan$i
 425
 426 #Without IP:
 427 #BOOTPROTO=static
 428 #IPADDR=10.10.10.$((i+209))
 429 #NETMASK=255.255.255.0" > ifcfg-virbrMan$i
 430
 431     # create the required interfaces to connect the bridges
 432     echo "DEVICE=${interface}.20$i2digits
 433 ONBOOT=yes
 434 NM_CONTROLLED=no
 435 USERCTL=no
 436 VLAN=yes
 437 BOOTPROTO=none
 438 BRIDGE=virbrMan$i" > ifcfg-${interface}.20$i2digits
 439   done
 440
 441   if [ -n "$ip_iface" ]
 442   then
 443     echo "configuring iface $iface interface with ip $ip_iface"
 444     # Network interfaces
 445     # 1Gbps interfaces are configured with ONBOOT=yes and static IP address
 446     cat ifcfg-$iface | grep -e HWADDR -e UUID > $iface.tmp
 447     echo "TYPE=Ethernet
 448 NAME=$iface
 449 DEVICE=$iface
 450 TYPE=Ethernet
 451 ONBOOT=yes
 452 NM_CONTROLLED=no
 453 IPV6INIT=no" >> $iface.tmp
 454     [ $ip_iface = "dhcp" ] && echo -e "BOOTPROTO=dhcp\nDHCP_HOSTNAME=$HOSTNAME" >> $iface.tmp
 455     [ $ip_iface != "dhcp" ] && echo -e "BOOTPROTO=static\nIPADDR=${ip_iface}\nNETMASK=255.255.255.0" >> $iface.tmp
 456     mv $iface.tmp  ifcfg-$iface
 457   fi
 458
 459   for iface in `ifconfig -a | grep ": " | cut -f 1 -d":" | grep -v -e "_" -e "\." -e "lo" -e "virbr" -e "tap"`
 460   do
 461     # 10/40 Gbps interfaces
 462     # Intel X520 cards: driver ixgbe
 463     # Intel XL710 Fortville cards: driver i40e
 464     driver=`ethtool -i $iface| awk '($0~"driver"){print $2}'`
 465     if [ "$driver" = "i40e" -o "$driver" = "ixgbe" ]
 466     then
 467       echo "configuring dataplane iface $iface"
 468
 469       # Create 8 SR-IOV per PF by udev rules only for Fortville cards (i40e driver)
 470       if [ "$driver" = "i40e" ]
 471       then
 472         pci=`ethtool -i $iface | awk '($0~"bus-info"){print $2}'`
 473         echo "ACTION==\"add\", KERNEL==\"$pci\", SUBSYSTEM==\"pci\", RUN+=\"/usr/bin/bash -c 'echo 8 > /sys/bus/pci/devices/$pci/sriov_numvfs'\"" >> /etc/udev/rules.d/pci_config.rules
 474       fi
 475
 476       # Configure PF to boot automatically and to have a big MTU
 477       # 10Gbps interfaces are configured with ONBOOT=yes and  MTU=2000
 478       cat ifcfg-$iface | grep -e HWADDR -e UUID > $iface.tmp
 479       echo "TYPE=Ethernet
 480 NAME=$iface
 481 DEVICE=$iface
 482 ONBOOT=yes
 483 MTU=9000
 484 NM_CONTROLLED=no
 485 IPV6INIT=no
 486 BOOTPROTO=none" >> $iface.tmp
 487       mv $iface.tmp ifcfg-$iface
 488     fi
 489   done
 490   popd
 491 fi
 492
 493
 494 # Activate 8 Virtual Functions per PF on Niantic cards (ixgbe driver)
 495 if [[ `lsmod | cut -d" " -f1 | grep "ixgbe" | grep -v vf` ]]
 496 then
 497         if ! grep -q "ixgbe" /etc/modprobe.d/ixgbe.conf
 498         then
 499         echo "options ixgbe max_vfs=8" >> /etc/modprobe.d/ixgbe.conf
 500         fi
 501
 502 fi
 503
 504 # Executes dracut to load drivers on boot
 505 echo "Regenerating initramfs"
 506 dracut --force
 507
 508 # To define 8 VFs per PF we do it on rc.local, because the driver needs to be unloaded and loaded again
 509 #if ! grep -q "NFV" /etc/rc.local
 510 #then
 511 #  echo "" >> /etc/rc.local
 512 #  echo "# NFV" >> /etc/rc.local
 513 #  echo "modprobe -r ixgbe" >> /etc/rc.local
 514 #  echo "modprobe ixgbe max_vfs=8" >> /etc/rc.local
 515 #  echo "" >> /etc/rc.local
 516
 517 #  chmod +x /etc/rc.d/rc.local
 518
 519 #fi
 520
 521 echo
 522 echo "Do not forget to create a shared (NFS, Samba, ...) where original virtual machine images are allocated"
 523 echo
 524 echo "Do not forget to copy the public ssh key into /home/${user_name}/.ssh/authorized_keys for authomatic login from openvim controller"
 525 echo
 526
 527 echo "Reboot the system to make the changes effective"
 528