scripts/configure-compute-node-UBUNTU16.0.4.sh

   1 #!/bin/bash
   2
   3 ##
   4 # Copyright 2015 Telefónica Investigación y Desarrollo, S.A.U.
   5 # This file is part of openvim
   6 # All Rights Reserved.
   7 #
   8 # Licensed under the Apache License, Version 2.0 (the "License"); you may
   9 # not use this file except in compliance with the License. You may obtain
  10 # a copy of the License at
  11 #
  12 #         http://www.apache.org/licenses/LICENSE-2.0
  13 #
  14 # Unless required by applicable law or agreed to in writing, software
  15 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  16 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  17 # License for the specific language governing permissions and limitations
  18 # under the License.
  19 #
  20 # For those usages not covered by the Apache License, Version 2.0 please
  21 # contact with: nfvlabs@tid.es
  22 ##
  23
  24 # Authors: Antonio Lopez, Pablo Montes, Alfonso Tierno, Leonardo Mirabal
  25 # June 2015
  26
  27 # Personalize RHEL7.1 on compute nodes
  28 # Prepared to work with the following network card drivers:
  29 #       tg3, igb drivers for management interfaces
  30 #       ixgbe (Intel Niantic) and i40e (Intel Fortville) drivers for data plane interfaces
  31
  32 # To download:
  33 # wget https://raw.githubusercontent.com/nfvlabs/openmano/master/scripts/configure-compute-node-UBUNTU14.04.sh
  34 # To execute:
  35 # chmod +x ./configure-compute-node-UBUNTU14.04.sh
  36 # sudo ./configure-compute-node-RHEL7.1.sh <user> <iface>
  37
  38 # Assumptions:
  39 # All virtualization options activated on BIOS (vt-d, vt-x, SR-IOV, no power savings...)
  40 # RHEL7.1 installed without /home partition and with the following packages selection:
  41 # @base, @core, @development, @network-file-system-client, @virtualization-hypervisor, @virtualization-platform, @virtualization-tools
  42
  43
  44
  45 function usage(){
  46     echo -e "Usage: sudo $0 [-f] <user-name> [<iface-for-overlay-bridges>]"
  47     echo -e "  Configure compute host for VIM usage. (version 0.4). Params:"
  48     echo -e "     -f  do not prompt for confirmation. If a new user is created, the user name is set as password"
  49     echo -e "     <user-name> Create if not exist and configure this user for openvim to connect"
  50     echo -e "     [<iface-for-overlay-bridges>] Only needed for old openvim versions. Iface to create pre-provioned bridges for network conectivity"
  51
  52 }
  53
  54
  55 #1 CHECK input parameters
  56 #1.1 root privileges
  57 [ "$USER" != "root" ] && echo "Needed root privileges" && usage && exit -1
  58
  59 #1.2 input parameters
  60 FORCE=""
  61 while getopts "f" o; do
  62     case "${o}" in
  63         f)
  64             FORCE="yes"
  65             ;;
  66         *)
  67             usage
  68             exit -1
  69             ;;
  70     esac
  71 done
  72 shift $((OPTIND-1))
  73
  74
  75
  76 if [ $# -lt 1 ]
  77 then
  78   usage
  79   exit
  80 fi
  81
  82
  83 user_name=$1
  84 interface=$2
  85
  86
  87 if [ -z $interface ]  && ! ifconfig $interface &> /dev/null
  88 then
  89   echo "Error: interface '$interface' is not present in the system"
  90   usage
  91   exit 1
  92 fi
  93
  94 echo '
  95 #################################################################
  96 #####       INSTALL NEEDED PACKETS                          #####
  97 #################################################################'
  98
  99 # Required packages
 100 apt-get -y  update
 101 apt-get -y  install grub-common screen virt-manager ethtool build-essential x11-common x11-utils \
 102             libguestfs-tools hwloc libguestfs-tools numactl vlan nfs-common nfs-kernel-server openvswitch-switch
 103 echo "Remove unneeded packages....."
 104 apt-get -y autoremove
 105 # Selinux management
 106 #yum install -y policycoreutils-python
 107
 108 echo '
 109 #################################################################
 110 #####       INSTALL USER                                    #####
 111 #################################################################'
 112
 113 # Add required groups
 114 groupadd -f admin
 115 groupadd -f libvirt   #for other operating systems may be libvirtd
 116
 117 # Adds user, default password same as name
 118 if grep -q "^${user_name}:" /etc/passwd
 119 then
 120   #user exist, add to group
 121   echo "adding user ${user_name} to groups libvirt,admin"
 122   usermod -a -G libvirt,admin -g admin $user_name
 123 else
 124   #create user if it does not exist
 125   [ -z "$FORCE" ] && read -p "user '${user_name}' does not exist, create (Y/n)" kk
 126   if ! [ -z "$kk" -o "$kk"="y" -o "$kk"="Y" ]
 127   then
 128     exit
 129   fi
 130   echo "creating and configuring user {user_name}"
 131   useradd -m -G libvirt,admin -g admin $user_name
 132   #Password
 133   if [ -z "$FORCE" ]
 134   then
 135      echo "Provide a password for $user_name"
 136      passwd $user_name
 137   else
 138      echo -e "$user_name\n$user_name" | passwd --stdin $user_name
 139   fi
 140 fi
 141
 142 # Allow admin users to access without password
 143 if ! grep -q "#openmano" /etc/sudoers
 144 then
 145     cat >> /home/${user_name}/script_visudo.sh << EOL
 146 #!/bin/bash
 147 echo "#openmano allow to group admin to grant root privileges without password" >> \$1
 148 echo "${user_name} ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
 149 EOL
 150     chmod +x /home/${user_name}/script_visudo.sh
 151     echo "allowing admin user to get root privileges withut password"
 152     export EDITOR=/home/${user_name}/script_visudo.sh && sudo -E visudo
 153     rm -f /home/${user_name}/script_visudo.sh
 154 fi
 155
 156 echo '
 157 #################################################################
 158 #####       INSTALL HUGEPAGES ISOLCPUS GRUB                 #####
 159 #################################################################'
 160
 161 # Huge pages 1G auto mount
 162 mkdir -p /mnt/huge
 163 if ! grep -q "Huge pages" /etc/fstab
 164 then
 165   echo "" >> /etc/fstab
 166   echo "# Huge pages" >> /etc/fstab
 167   echo "nodev /mnt/huge hugetlbfs pagesize=1GB 0 0" >> /etc/fstab
 168   echo "" >> /etc/fstab
 169 fi
 170
 171 # Huge pages reservation service
 172 if ! [ -f /lib/systemd/system/hugetlb-gigantic-pages.service ]
 173 then
 174   echo "configuring huge pages service"
 175   echo "[Unit]
 176 Description=HugeTLB Gigantic Pages Reservation
 177 DefaultDependencies=no
 178 Before=dev-hugepages.mount
 179 ConditionPathExists=/sys/devices/system/node
 180 ConditionKernelCommandLine=hugepagesz=1G
 181
 182 [Service]
 183 Type=oneshot
 184 RemainAfterExit=yes
 185 ExecStart=/usr/lib/systemd/hugetlb-reserve-pages
 186
 187 [Install]
 188 WantedBy=sysinit.target" >> /lib/systemd/system/hugetlb-gigantic-pages.service
 189
 190     #  cat > /usr/lib/systemd/system/hugetlb-gigantic-pages.service << EOL
 191     #[Unit]
 192     #Description=HugeTLB Gigantic Pages Reservation
 193     #DefaultDependencies=no
 194     #Before=dev-hugepages.mount
 195     #ConditionPathExists=/sys/devices/system/node
 196     #ConditionKernelCommandLine=hugepagesz=1G
 197    #
 198     #[Service]
 199     #Type=oneshot
 200     #RemainAfterExit=yes
 201     #ExecStart=/usr/lib/systemd/hugetlb-reserve-pages
 202     #
 203     #[Install]
 204     #WantedBy=sysinit.target
 205     #EOL
 206 fi
 207
 208
 209 # Get isolcpus
 210 isolcpus=`gawk 'BEGIN{pre=-2;}
 211   ($1=="processor"){pro=$3;}
 212   ($1=="core" && $4!=0){
 213      if (pre+1==pro){endrange="-" pro}
 214      else{cpus=cpus endrange sep pro; sep=","; endrange="";};
 215      pre=pro;}
 216   END{printf("%s",cpus endrange);}' /proc/cpuinfo`
 217
 218
 219 echo "CPUS: $isolcpus"
 220
 221 # Huge pages reservation file: reserving all memory apart from 4GB per NUMA node
 222 # Get the number of hugepages: all memory but 8GB reserved for the OS
 223 #totalmem=`dmidecode --type 17|grep Size |grep MB |gawk '{suma+=$2} END {print suma/1024}'`
 224 #hugepages=$(($totalmem-8))
 225
 226 if ! [ -f /usr/lib/systemd/hugetlb-reserve-pages ]
 227 then
 228   cat > /usr/lib/systemd/hugetlb-reserve-pages << EOL
 229 #!/bin/bash
 230 nodes_path=/sys/devices/system/node/
 231 if [ ! -d \$nodes_path ]; then
 232         echo "ERROR: \$nodes_path does not exist"
 233         exit 1
 234 fi
 235
 236 reserve_pages()
 237 {
 238         echo \$1 > \$nodes_path/\$2/hugepages/hugepages-1048576kB/nr_hugepages
 239 }
 240
 241 # This example reserves all available memory apart from 4 GB for linux
 242 # using 1GB size. You can modify it to your needs or comment the lines
 243 # to avoid reserve memory in a numa node
 244 EOL
 245   for f in /sys/devices/system/node/node?/meminfo
 246   do
 247     node=`head -n1 $f | gawk '($5=="kB"){print $2}'`
 248     memory=`head -n1 $f | gawk '($5=="kB"){print $4}'`
 249     memory=$((memory+1048576-1))   #memory must be ceiled
 250     memory=$((memory/1048576))   #from `kB to GB
 251     #if memory
 252     [ $memory -gt 4 ] && echo "reserve_pages $((memory-4)) node$node" >> /usr/lib/systemd/hugetlb-reserve-pages
 253   done
 254
 255   # Run the following commands to enable huge pages early boot reservation:
 256   chmod +x /usr/lib/systemd/hugetlb-reserve-pages
 257   systemctl enable hugetlb-gigantic-pages
 258 fi
 259
 260 # Prepares the text to add at the end of the grub line, including blacklisting ixgbevf driver in the host
 261 memtotal=`grep MemTotal /proc/meminfo | awk '{ print $2 }' `
 262 hpages=$(( ($memtotal/(1024*1024))-8 ))
 263
 264 memtotal=$((memtotal+1048576-1))   #memory must be ceiled
 265 memtotal=$((memtotal/1048576))   #from `kB to GBa
 266 hpages=$((memtotal-8))
 267 [[ $hpages -lt 0 ]] && hpages=0
 268
 269
 270 echo "------> memtotal: $memtotal"
 271
 272 textokernel=" intel_iommu=on default_hugepagesz=1G hugepagesz=1G hugepages=$hpages isolcpus=$isolcpus modprobe.blacklist=ixgbevf modprobe.blacklist=i40evf"
 273
 274 echo "Text to kernel: $textokernel"
 275
 276
 277 # Add text to the kernel line
 278 if ! grep -q " intel_iommu=on default_hugepagesz=1G hugepagesz=1G" /etc/default/grub
 279 then
 280   echo ">>>>>>>  adding cmdline ${textokernel}"
 281   sed -i "/^GRUB_CMDLINE_LINUX_DEFAULT=/s/\"\$/${textokernel}\"/" /etc/default/grub
 282   sed -i "/^GRUB_CMDLINE_LINUX=/s/\"\$/${textokernel}\"/" /etc/default/grub
 283
 284   update-grub
 285   # BIOS based systems
 286   grub-mkconfig -o /boot/efi/EFI/ubuntu/grub.cfg
 287   # UEFI based systems
 288   grub-mkconfig -o /boot/grub/grub.cfg
 289 fi
 290
 291 echo '
 292 #################################################################
 293 #####       OTHER CONFIGURATION                             #####
 294 #################################################################'
 295
 296 # Links the OpenMANO required folder /opt/VNF/images to /var/lib/libvirt/images. The OS installation
 297 # should have only a / partition with all possible space available
 298
 299 echo " link /opt/VNF/images to /var/lib/libvirt/images"
 300 if [ "$user_name" != "" ]
 301 then
 302   #mkdir -p /home/${user_name}/VNF_images
 303   #chown -R ${user_name}:admin /home/${user_name}/VNF_images
 304   #chmod go+x $HOME
 305
 306   # The orchestator needs to link the images folder
 307   rm -f /opt/VNF/images
 308   mkdir -p /opt/VNF/
 309   ln -s /var/lib/libvirt/images /opt/VNF/images
 310   chown -R ${user_name}:admin /opt/VNF
 311   chown -R root:admin /var/lib/libvirt/images
 312   chmod g+rwx /var/lib/libvirt/images
 313
 314   # Selinux management
 315   #echo "configure  Selinux management"
 316   #semanage fcontext -a -t virt_image_t "/home/${user_name}/VNF_images(/.*)?"
 317   #cat /etc/selinux/targeted/contexts/files/file_contexts.local |grep virt_image
 318   #restorecon -R -v /home/${user_name}/VNF_images
 319 else
 320   mkdir -p /opt/VNF/images
 321   chmod o+rx /opt/VNF/images
 322 fi
 323
 324 echo "creating local information /opt/VNF/images/hostinfo.yaml"
 325 echo "#By default openvim assumes control plane interface naming as em1,em2,em3,em4 " > /opt/VNF/images/hostinfo.yaml
 326 echo "#and bridge ifaces as virbrMan1, virbrMan2, ..." >> /opt/VNF/images/hostinfo.yaml
 327 echo "#if compute node contain a different name it must be indicated in this file" >> /opt/VNF/images/hostinfo.yaml
 328 echo "#with the format extandard-name: compute-name" >> /opt/VNF/images/hostinfo.yaml
 329 if [ "$interface" != "" -a "$interface" != "em1" ]
 330 then
 331   echo "iface_names:"  >> /opt/VNF/images/hostinfo.yaml
 332   echo "  em1: ${interface}" >> /opt/VNF/images/hostinfo.yaml
 333 fi
 334 chmod o+r /opt/VNF/images/hostinfo.yaml
 335
 336 # deactivate memory overcommit
 337 #echo "deactivate memory overcommit"
 338 #service ksmtuned stop
 339 #service ksm stop
 340 #chkconfig ksmtuned off
 341 #chkconfig ksm off
 342
 343
 344 # Libvirt options (uncomment the following)
 345 echo "configure Libvirt options"
 346 sed -i 's/#unix_sock_group = "libvirt"/unix_sock_group = "libvirt"/' /etc/libvirt/libvirtd.conf
 347 sed -i 's/#unix_sock_rw_perms = "0770"/unix_sock_rw_perms = "0770"/' /etc/libvirt/libvirtd.conf
 348 sed -i 's/#unix_sock_dir = "\/var\/run\/libvirt"/unix_sock_dir = "\/var\/run\/libvirt"/' /etc/libvirt/libvirtd.conf
 349 sed -i 's/#auth_unix_rw = "none"/auth_unix_rw = "none"/' /etc/libvirt/libvirtd.conf
 350
 351
 352 echo '
 353 #################################################################
 354 #####       NETWORK CONFIGURATION                           #####
 355 #################################################################'
 356 # Network config (if the second parameter is net)
 357 echo "Interface ==> $interface"
 358 if [ -n "$interface" ]
 359 then
 360
 361     # For management and data interfaces
 362     rm -f /etc/udev/rules.d/pci_config.rules # it will be created to define VFs
 363
 364
 365     # Set ONBOOT=on and MTU=9000 on the interface used for the bridges
 366     echo "configuring iface $interface"
 367
 368     #MTU for interfaces and bridges
 369     MTU=9000
 370
 371     cp /etc/network/interfaces interfaces.tmp
 372
 373
 374     #Create infrastructure bridge, normally used for connecting to compute nodes, openflow controller, ...
 375     #Create VLAN for infrastructure bridge
 376
 377 echo "
 378 ######### CUTLINE #########
 379
 380 auto ${interface}
 381 iface ${interface} inet manual
 382 post-up ip link set dev ${interface} mtu ${MTU}
 383
 384 auto ${interface}.1001
 385 iface ${interface}.1001 inet manual
 386 vlan-raw-device ${interface}
 387 post-up ip link set mtu $MTU dev ${interface}.1001
 388 " >> interfaces.tmp
 389
 390         # echo "ifconfig ${interface} mtu $MTU
 391         # ifconfig ${interface} up
 392         #" > mtu.tmp
 393
 394
 395     #Create bridge interfaces
 396     echo "Creating bridge ifaces: "
 397     for ((i=1;i<=20;i++))
 398     do
 399         i2digits=$i
 400         [ $i -lt 10 ] && i2digits="0$i"
 401         echo "    virbrMan$i  vlan 20$i2digits"
 402
 403         j=$i
 404
 405 echo "
 406 auto ${interface}.20$i2digits
 407 iface ${interface}.20$i2digits inet manual
 408 vlan-raw-device ${interface}
 409 post-up ip link set mtu $MTU dev ${interface}.20$i2digits
 410
 411 auto virbrMan$j
 412 iface virbrMan$j inet manual
 413 bridge_ports ${interface}.20$i2digits
 414 post-up ip link set dev virbrMan$j && ip link set mtu $MTU dev virbrMan$j
 415 " >> interfaces.tmp
 416
 417     done
 418
 419     # echo "ifconfig em2.1001 mtu $MTU
 420     #ifconfig virbrInf mtu $MTU
 421     #ifconfig virbrInf up
 422     #" >> mtu.tmp
 423
 424     if ! grep -q "#### CUTLINE ####" /etc/network/interfaces
 425     then
 426         echo "====== Copying interfaces.tmp to /etc/network/interfaces"
 427         cp interfaces.tmp /etc/network/interfaces
 428     fi
 429
 430
 431   #popd
 432 fi
 433
 434 ################### Activate 8 Virtual Functions per PF on Niantic cards (ixgbe driver)
 435 if [[ `lsmod | cut -d" " -f1 | grep "ixgbe" | grep -v vf` ]]
 436     then
 437         if ! grep -q "ixgbe" /etc/modprobe.d/ixgbe.conf
 438         then
 439         echo "options ixgbe max_vfs=8" >> /etc/modprobe.d/ixgbe.conf
 440         fi
 441
 442     fi
 443
 444 echo "#!/bin/bash" > /etc/activate-vfs.sh
 445 chmod +x /etc/activate-vfs.sh
 446 for iface in `ip -o link show | awk -F': ' '{print $2}'  | grep -v  -e "\." -e "lo" -e "virbr" -e "tap"`
 447 do
 448 # 10/40 Gbps interfaces
 449 # Intel X520 cards: driver ixgbe
 450 # Intel XL710 Fortville cards: driver i40e
 451 driver=`ethtool -i $iface| awk '($0~"driver"){print $2}'`
 452 if [ "$driver" = "i40e" -o "$driver" = "ixgbe" ]
 453 then
 454   echo "configuring dataplane iface $iface"
 455
 456   # Create 8 SR-IOV per PF by udev rules only for Fortville cards (i40e driver)
 457   if [ "$driver" = "i40e" ]
 458   then
 459   pci=`ethtool -i $iface | awk '($0~"bus-info"){print $2}'`
 460    echo "echo 8 > /sys/bus/pci/devices/$pci/sriov_numvfs" >> /etc/activate-vfs.sh
 461   fi
 462 fi
 463 done
 464
 465 # Set dataplane MTU
 466
 467 #echo "sleep 10" >> mtu.tmp
 468
 469 #interfaces=`ifconfig -a | grep ^p | cut -d " " -f 1`
 470 #for ph in $interfaces
 471 #do
 472 #        echo "ifconfig $ph mtu $MTU" >> mtu.tmp
 473 #        echo "ifconfig $ph up" >> mtu.tmp
 474 #done
 475
 476
 477
 478 #cp mtu.tmp /etc/setmtu.sh
 479 #chmod +x /etc/setmtu.sh
 480
 481 # To define 8 VFs per PF we do it on rc.local, because the driver needs to be unloaded and loaded again
 482 #if ! grep -q "NFV" /etc/rc.local
 483 #then
 484   echo "#!/bin/sh -e
 485 " > /etc/rc.local
 486   echo "# NFV" >> /etc/rc.local
 487   echo "modprobe -r ixgbe" >> /etc/rc.local
 488   echo "modprobe ixgbe max_vfs=8" >> /etc/rc.local
 489   echo "/etc/activate-vfs.sh" >> /etc/rc.local
 490   #echo "/etc/setmtu.sh" >> /etc/rc.local
 491   echo "
 492 exit 0" >> /etc/rc.local
 493   echo "" >> /etc/rc.local
 494
 495   chmod +x /etc/rc.local
 496   chmod +x /etc/activate-vfs.sh
 497
 498 #fi
 499
 500 chmod a+rwx /var/lib/libvirt/images
 501 mkdir -p /usr/libexec/
 502 pushd /usr/libexec/
 503 ln -s /usr/bin/qemu-system-x86_64 qemu-kvm
 504 popd
 505
 506 #Deactivating apparmor while looking for a better solution
 507 /etc/init.d/apparmor stop
 508 update-rc.d -f apparmor remove
 509
 510 echo
 511 echo "Do not forget to create a shared (NFS, Samba, ...) where original virtual machine images are allocated"
 512 echo
 513 echo "Do not forget to copy the public ssh key into /home/${user_name}/.ssh/authorized_keys for authomatic login from openvim controller"
 514 echo
 515
 516 echo "Reboot the system to make the changes effective"
 517