| #!/bin/bash |
| |
| ## |
| # Copyright 2015 Telefónica Investigación y Desarrollo, S.A.U. |
| # This file is part of openmano |
| # All Rights Reserved. |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); you may |
| # not use this file except in compliance with the License. You may obtain |
| # a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
| # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
| # License for the specific language governing permissions and limitations |
| # under the License. |
| # |
| # For those usages not covered by the Apache License, Version 2.0 please |
| # contact with: nfvlabs@tid.es |
| ## |
| |
| # Authors: Antonio Lopez, Pablo Montes, Alfonso Tierno |
| # 2016 March 18 |
| # Modified to run on grub2 and efi boot |
| |
| # Personalize RHEL7.2 on compute nodes |
| # Prepared to work with the following network card drivers: |
| # tg3, igb drivers for management interfaces |
| # ixgbe (Intel Niantic) and i40e (Intel Fortville) drivers for data plane interfaces |
| |
| # To download: |
| # wget https://raw.githubusercontent.com/nfvlabs/openmano/master/scripts/configure-compute-node-RHEL7.1.sh |
| # To execute: |
| # chmod +x ./configure-compute-node-RHEL7.1.sh |
| # sudo ./configure-compute-node-RHEL7.1.sh <user> <iface> |
| |
| # Assumptions: |
| # All virtualization options activated on BIOS (vt-d, vt-x, SR-IOV, no power savings...) |
| # RHEL7.2 installed without /home partition and with the following packages selection: |
| # @base, @core, @development, @network-file-system-client, @virtualization-hypervisor, @virtualization-platform, @virtualization-tools |
| |
| # 2016 Aug 17 Antonio López |
| # Changed virbrInf to virbrVIM, to reflect that this bridge is used to communicate with the VIM (OpenVIM) |
| # Changed the vlan tag used by virbrVIM from 2000 to 1100 |
| |
| function usage(){ |
| echo -e "Usage: sudo $0 [-y] <user-name> [ <iface-name> [<ip-address>|dhcp] ]" |
| echo -e " Configure compute host for VIM usage. (version 0.4). Params:" |
| echo -e " -y do not prompt for confirmation. If a new user is created, the user name is set as password" |
| echo -e " <user-name> Create if not exist and configure this user for openvim to connect" |
| echo -e " <iface-name> if suplied creates bridge interfaces on this interface, needed for openvim" |
| echo -e " ip or dhcp if suplied, configure the interface with this ip address (/24) or 'dhcp' " |
| } |
| |
| |
| #1 CHECK input parameters |
| #1.1 root privileges |
| [ "$USER" != "root" ] && echo "Needed root privileges" && usage && exit -1 |
| |
| #1.2 input parameters |
| FORCE="" |
| while getopts "y" o; do |
| case "${o}" in |
| y) |
| FORCE="yes" |
| ;; |
| *) |
| usage |
| exit -1 |
| ;; |
| esac |
| done |
| shift $((OPTIND-1)) |
| |
| |
| if [ $# -lt 1 ] |
| then |
| usage |
| exit |
| fi |
| |
| user_name=$1 |
| interface=$2 |
| ip_iface=$3 |
| |
| if [ -n "$interface" ] && ! ifconfig $interface &> /dev/null |
| then |
| echo "Error: interface '$interface' is not present in the system" |
| usage |
| exit 1 |
| fi |
| |
| echo ' |
| ################################################################# |
| ##### INSTALL NEEDED PACKETS ##### |
| #################################################################' |
| |
| # Required packages |
| yum repolist |
| yum check-update |
| yum update -y |
| yum install -y screen virt-manager ethtool gcc gcc-c++ xorg-x11-xauth xorg-x11-xinit xorg-x11-deprecated-libs libXtst guestfish hwloc libhugetlbfs-utils libguestfs-tools numactl |
| # Selinux management |
| yum install -y policycoreutils-python |
| |
| echo ' |
| ################################################################# |
| ##### INSTALL USER ##### |
| #################################################################' |
| |
| # Add required groups |
| groupadd -f nfvgroup |
| groupadd -f libvirt #for other operating systems may be libvirtd |
| |
| # Adds user, default password same as name |
| if grep -q "^${user_name}:" /etc/passwd |
| then |
| #user exist, add to group |
| echo "adding user ${user_name} to groups libvirt,nfvgroup" |
| usermod -a -G libvirt,nfvgroup -g nfvgroup $user_name |
| else |
| #create user if it does not exist |
| [ -z "$FORCE" ] && read -p "user '${user_name}' does not exist, create (Y/n)" kk |
| if ! [ -z "$kk" -o "$kk"="y" -o "$kk"="Y" ] |
| then |
| exit |
| fi |
| echo "creating and configuring user ${user_name}" |
| useradd -m -G libvirt,nfvgroup -g nfvgroup $user_name |
| #Password |
| if [ -z "$FORCE" ] |
| then |
| echo "Provide a password for $user_name" |
| passwd $user_name |
| else |
| echo -e "$user_name\n$user_name" | passwd --stdin $user_name |
| fi |
| fi |
| |
| #Setting default libvirt URI for the user |
| echo "Setting default libvirt URI for the user" |
| echo "if test -x `which virsh`; then" >> /home/${user_name}/.bash_profile |
| echo " export LIBVIRT_DEFAULT_URI=qemu:///system" >> /home/${user_name}/.bash_profile |
| echo "fi" >> /home/${user_name}/.bash_profile |
| |
| echo ' |
| ################################################################# |
| ##### INSTALL HUGEPAGES ISOLCPUS GRUB ##### |
| #################################################################' |
| |
| # Huge pages 1G auto mount |
| mkdir -p /mnt/huge |
| if ! grep -q "Huge pages" /etc/fstab |
| then |
| echo "" >> /etc/fstab |
| echo "# Huge pages" >> /etc/fstab |
| echo "nodev /mnt/huge hugetlbfs pagesize=1GB 0 0" >> /etc/fstab |
| echo "" >> /etc/fstab |
| fi |
| |
| # Huge pages reservation service |
| if ! [ -f /usr/lib/systemd/system/hugetlb-gigantic-pages.service ] |
| then |
| echo "configuring huge pages service" |
| cat > /usr/lib/systemd/system/hugetlb-gigantic-pages.service << EOL |
| [Unit] |
| Description=HugeTLB Gigantic Pages Reservation |
| DefaultDependencies=no |
| Before=dev-hugepages.mount |
| ConditionPathExists=/sys/devices/system/node |
| ConditionKernelCommandLine=hugepagesz=1G |
| |
| [Service] |
| Type=oneshot |
| RemainAfterExit=yes |
| ExecStart=/usr/lib/systemd/hugetlb-reserve-pages |
| |
| [Install] |
| WantedBy=sysinit.target |
| EOL |
| fi |
| # Grub virtualization options: |
| |
| # Get isolcpus |
| isolcpus=`gawk 'BEGIN{pre=-2;} |
| ($1=="processor"){pro=$3;} |
| ($1=="core" && $4!=0){ |
| if (pre+1==pro){endrange="-" pro} |
| else{cpus=cpus endrange sep pro; sep=","; endrange="";}; |
| pre=pro;} |
| END{printf("%s",cpus endrange);}' /proc/cpuinfo` |
| |
| |
| # Huge pages reservation file: reserving all memory apart from 4GB per NUMA node |
| # Get the number of hugepages: all memory but 8GB reserved for the OS |
| #totalmem=`dmidecode --type 17|grep Size |grep MB |gawk '{suma+=$2} END {print suma/1024}'` |
| #hugepages=$(($totalmem-8)) |
| |
| if ! [ -f /usr/lib/systemd/hugetlb-reserve-pages ] |
| then |
| cat > /usr/lib/systemd/hugetlb-reserve-pages << EOL |
| #!/bin/bash |
| nodes_path=/sys/devices/system/node/ |
| if [ ! -d \$nodes_path ]; then |
| echo "ERROR: \$nodes_path does not exist" |
| exit 1 |
| fi |
| |
| reserve_pages() |
| { |
| echo \$1 > \$nodes_path/\$2/hugepages/hugepages-1048576kB/nr_hugepages |
| } |
| |
| # This example reserves all available memory apart from 4 GB for linux |
| # using 1GB size. You can modify it to your needs or comment the lines |
| # to avoid reserve memory in a numa node |
| EOL |
| for f in /sys/devices/system/node/node?/meminfo |
| do |
| node=`head -n1 $f | gawk '($5=="kB"){print $2}'` |
| memory=`head -n1 $f | gawk '($5=="kB"){print $4}'` |
| memory=$((memory+1048576-1)) #memory must be ceiled |
| memory=$((memory/1048576)) #from `kB to GB |
| #if memory |
| [ $memory -gt 4 ] && echo "reserve_pages $((memory-4)) node$node" >> /usr/lib/systemd/hugetlb-reserve-pages |
| done |
| |
| # Run the following commands to enable huge pages early boot reservation: |
| chmod +x /usr/lib/systemd/hugetlb-reserve-pages |
| systemctl enable hugetlb-gigantic-pages |
| fi |
| |
| # Prepares the text to add at the end of the grub line, including blacklisting ixgbevf driver in the host |
| |
| textokernel="intel_iommu=on default_hugepagesz=1G hugepagesz=1G isolcpus=$isolcpus modprobe.blacklist=ixgbevf modprobe.blacklist=i40evf" |
| |
| # Add text to the kernel line |
| if ! grep -q "intel_iommu=on default_hugepagesz=1G hugepagesz=1G" /etc/default/grub |
| then |
| echo "adding cmdline ${textokernel}" |
| sed -i "/^GRUB_CMDLINE_LINUX=/s/\"\$/ ${textokernel}\"/" /etc/default/grub |
| |
| # grub2 upgrade |
| |
| # BIOS based systems |
| grub2-mkconfig -o /boot/grub2/grub.cfg |
| |
| # UEFI based systems |
| grub2-mkconfig -o /boot/efi/EFI/redhat/grub.cfg |
| |
| fi |
| |
| echo ' |
| ################################################################# |
| ##### OTHER CONFIGURATION ##### |
| #################################################################' |
| |
| # Disable requiretty |
| if ! grep -q "#openmano" /etc/sudoers |
| then |
| cat >> /home/${user_name}/script_visudo.sh << EOL |
| #!/bin/bash |
| cat \$1 | awk '(\$0~"requiretty"){print "#"\$0}(\$0!~"requiretty"){print \$0}' > tmp |
| cat tmp > \$1 |
| rm tmp |
| EOL |
| chmod +x /home/${user_name}/script_visudo.sh |
| echo "Disabling requitetty" |
| export EDITOR=/home/${user_name}/script_visudo.sh && sudo -E visudo |
| rm -f /home/${user_name}/script_visudo.sh |
| fi |
| |
| #Configure polkint to run virsh as a normal user |
| echo "Configuring polkint to run virsh as a normal user" |
| cat >> /etc/polkit-1/localauthority/50-local.d/50-org.libvirt-access.pkla << EOL |
| [libvirt Admin Access] |
| Identity=unix-group:libvirt |
| Action=org.libvirt.unix.manage |
| ResultAny=yes |
| ResultInactive=yes |
| ResultActive=yes |
| EOL |
| |
| # Links the OpenMANO required folder /opt/VNF/images to /var/lib/libvirt/images. The OS installation |
| # should have only a / partition with all possible space available |
| |
| echo " link /opt/VNF/images to /var/lib/libvirt/images" |
| if [ "$user_name" != "" ] |
| then |
| #mkdir -p /home/${user_name}/VNF_images |
| #chown -R ${user_name}:nfvgroup /home/${user_name}/VNF_images |
| #chmod go+x $HOME |
| |
| # The orchestator needs to link the images folder |
| rm -f /opt/VNF/images |
| mkdir -p /opt/VNF/ |
| ln -s /var/lib/libvirt/images /opt/VNF/images |
| chown -R ${user_name}:nfvgroup /opt/VNF |
| chown -R root:nfvgroup /var/lib/libvirt/images |
| chmod g+rwx /var/lib/libvirt/images |
| |
| # Selinux management |
| #echo "configure Selinux management" |
| #semanage fcontext -a -t virt_image_t "/home/${user_name}/VNF_images(/.*)?" |
| #cat /etc/selinux/targeted/contexts/files/file_contexts.local |grep virt_image |
| #restorecon -R -v /home/${user_name}/VNF_images |
| else |
| mkdir -p /opt/VNF/images |
| chmod o+rx /opt/VNF/images |
| fi |
| |
| echo "creating local information /opt/VNF/images/hostinfo.yaml" |
| echo "#By default openvim assumes control plane interface naming as em1,em2,em3,em4 " > /opt/VNF/images/hostinfo.yaml |
| echo "#and bridge ifaces as virbrMan1, virbrMan2, ..." >> /opt/VNF/images/hostinfo.yaml |
| echo "#if compute node contain a different name it must be indicated in this file" >> /opt/VNF/images/hostinfo.yaml |
| echo "#with the format extandard-name: compute-name" >> /opt/VNF/images/hostinfo.yaml |
| if [ "$interface" != "" -a "$interface" != "em1" ] |
| then |
| echo "iface_names:" >> /opt/VNF/images/hostinfo.yaml |
| echo " em1: ${interface}" >> /opt/VNF/images/hostinfo.yaml |
| fi |
| chmod o+r /opt/VNF/images/hostinfo.yaml |
| |
| # deactivate memory overcommit |
| echo "deactivate memory overcommit" |
| service ksmtuned stop |
| service ksm stop |
| chkconfig ksmtuned off |
| chkconfig ksm off |
| |
| |
| # Libvirt options (uncomment the following) |
| echo "configure Libvirt options" |
| sed -i 's/#unix_sock_group = "libvirt"/unix_sock_group = "libvirt"/' /etc/libvirt/libvirtd.conf |
| sed -i 's/#unix_sock_rw_perms = "0770"/unix_sock_rw_perms = "0770"/' /etc/libvirt/libvirtd.conf |
| sed -i 's/#unix_sock_dir = "\/var\/run\/libvirt"/unix_sock_dir = "\/var\/run\/libvirt"/' /etc/libvirt/libvirtd.conf |
| sed -i 's/#auth_unix_rw = "none"/auth_unix_rw = "none"/' /etc/libvirt/libvirtd.conf |
| |
| #creating the polkit grant access for libvirt user. |
| #This does not work !!!! so commented. No way to get running without uncomented the auth_unix_rw = "none" line |
| # |
| #cat > /etc/polkit-1/localauthority/50-local.d/50-org.example-libvirt-remote-access.pkla << EOL |
| #[libvirt Management Access] |
| # Identity=unix-user:n2;unix-user:kk |
| # Action=org.libvirt.unix.manage |
| # ResultAny=yes |
| # ResultInactive=yes |
| # ResultActive=yes |
| #EOL |
| |
| # Configuration change of qemu for the numatune bug issue |
| # RHEL7.1: for this version should not be necesary - to revise |
| #if ! grep -q "cgroup_controllers = [ \"cpu\", \"devices\", \"memory\", \"blkio\", \"cpuacct\" ]" /etc/libvirt/qemu.conf |
| #then |
| #cat /etc/libvirt/qemu.conf | awk '{print $0}($0~"#cgroup_controllers"){print "cgroup_controllers = [ \"cpu\", \"devices\", \"memory\", \"blkio\", \"cpuacct\" ]"}' > tmp |
| #mv tmp /etc/libvirt/qemu.conf |
| #fi |
| |
| echo ' |
| ################################################################# |
| ##### NETWORK CONFIGURATION ##### |
| #################################################################' |
| # Network config (if the second parameter is net) |
| if [ -n "$interface" ] |
| then |
| |
| # Deactivate network manager |
| systemctl stop NetworkManager |
| systemctl disable NetworkManager |
| |
| # For management and data interfaces |
| #rm -f /etc/udev/rules.d/pci_config.rules # it will be created to define VFs |
| |
| pushd /etc/sysconfig/network-scripts/ |
| |
| # Set ONBOOT=on and MTU=9000 on the interface used for the bridges |
| echo "configuring iface $interface" |
| cat ifcfg-$interface | grep -e HWADDR -e UUID > $interface.tmp |
| echo "TYPE=Ethernet |
| NAME=$interface |
| DEVICE=$interface |
| TYPE=Ethernet |
| ONBOOT=yes |
| NM_CONTROLLED=no |
| MTU=9000 |
| BOOTPROTO=none |
| IPV6INIT=no" >> $interface.tmp |
| mv $interface.tmp ifcfg-$interface |
| |
| # Management interfaces |
| # integrated_interfaces="" |
| # nb_ifaces=0 |
| # for iface in `ifconfig -a | grep ":\ " | cut -f 1 -d":"| grep -v "_" | grep -v "\." | grep -v "lo" | sort` |
| # do |
| # driver=`ethtool -i $iface| awk '($0~"driver"){print $2}'` |
| # if [ $driver != "ixgbe" ] && [ $driver != "bridge" ] |
| # then |
| # integrated_interfaces="$integrated_interfaces $iface" |
| # nb_ifaces=$((nb_ifaces+1)) |
| # eval iface${nb_ifaces}=$iface |
| # fi |
| # done |
| |
| #Create infrastructure bridge, normally used for connecting to compute nodes, openflow controller, ... |
| echo "DEVICE=virbrVIM |
| NAME=virbrVIM |
| TYPE=Bridge |
| ONBOOT=yes |
| DELAY=0 |
| NM_CONTROLLED=no |
| MTU=9000 |
| USERCTL=no" > ifcfg-virbrVIM |
| [[ $ip_iface != "dhcp" ]] && [[ $ip_iface != "" ]] && echo -e "BOOTPROTO=static\nIPADDR=${ip_iface}\nNETMASK=255.255.255.0" >> ifcfg-virbrVIM |
| |
| #Create VLAN for infrastructure bridge |
| echo "DEVICE=${interface}.1100 |
| NAME=${interface}.1100 |
| ONBOOT=yes |
| NM_CONTROLLED=no |
| USERCTL=no |
| VLAN=yes |
| MTU=9000 |
| BOOTPROTO=none |
| BRIDGE=virbrVIM" > ifcfg-${interface}.1100 |
| |
| |
| #Create bridge interfaces |
| echo "Creating bridge ifaces: " |
| for ((i=1;i<=20;i++)) |
| do |
| i2digits=$i |
| [ $i -lt 10 ] && i2digits="0$i" |
| echo " virbrMan$i vlan 20$i2digits" |
| echo "DEVICE=virbrMan$i |
| NAME=virbrMan$i |
| TYPE=Bridge |
| ONBOOT=yes |
| DELAY=0 |
| NM_CONTROLLED=no |
| MTU=9000 |
| USERCTL=no" > ifcfg-virbrMan$i |
| |
| #Without IP: |
| #BOOTPROTO=static |
| #IPADDR=10.10.10.$((i+209)) |
| #NETMASK=255.255.255.0" > ifcfg-virbrMan$i |
| |
| # create the required interfaces to connect the bridges |
| echo "DEVICE=${interface}.20$i2digits |
| NAME=${interface}.20$i2digits |
| ONBOOT=yes |
| NM_CONTROLLED=no |
| USERCTL=no |
| VLAN=yes |
| BOOTPROTO=none |
| MTU=9000 |
| BRIDGE=virbrMan$i" > ifcfg-${interface}.20$i2digits |
| done |
| |
| iface=$interface |
| if [ -n "$ip_iface" ] |
| then |
| echo "configuring iface $iface interface with ip $ip_iface" |
| # Network interfaces |
| # 1Gbps interfaces are configured with ONBOOT=yes and static IP address |
| cat ifcfg-$iface | grep -e HWADDR -e UUID > $iface.tmp |
| echo "TYPE=Ethernet |
| NAME=$iface |
| DEVICE=$iface |
| TYPE=Ethernet |
| ONBOOT=yes |
| NM_CONTROLLED=no |
| MTU=9000 |
| IPV6INIT=no" >> $iface.tmp |
| [ $ip_iface = "dhcp" ] && echo -e "BOOTPROTO=dhcp\nDHCP_HOSTNAME=$HOSTNAME" >> $iface.tmp |
| [ $ip_iface != "dhcp" ] && echo -e "BOOTPROTO=static\nIPADDR=${ip_iface}\nNETMASK=255.255.255.0" >> $iface.tmp |
| mv $iface.tmp ifcfg-$iface |
| fi |
| # Script to create vfs |
| echo "#!/bin/bash" > /root/activate-vfs.sh |
| chmod +x /root/activate-vfs.sh |
| for iface in `ifconfig -a | grep ": " | cut -f 1 -d":" | grep -v -e "_" -e "\." -e "lo" -e "virbr" -e "tap"` |
| do |
| # 10/40 Gbps interfaces |
| # Intel X520 cards: driver ixgbe |
| # Intel XL710 Fortville cards: driver i40e |
| driver=`ethtool -i $iface| awk '($0~"driver"){print $2}'` |
| if [ "$driver" = "i40e" -o "$driver" = "ixgbe" ] |
| then |
| echo "configuring dataplane iface $iface" |
| |
| # Create 8 SR-IOV per PF by udev rules only for Fortville cards (i40e driver) |
| if [ "$driver" = "i40e" ] |
| then |
| pci=`ethtool -i $iface | awk '($0~"bus-info"){print $2}'` |
| echo "echo 8 > /sys/bus/pci/devices/$pci/sriov_numvfs" >> /root/activate-vfs.sh |
| fi |
| |
| # Configure PF to boot automatically and to have a big MTU |
| # 10Gbps interfaces are configured with ONBOOT=yes and MTU=2000 |
| cat ifcfg-$iface | grep -e HWADDR -e UUID > $iface.tmp |
| echo "TYPE=Ethernet |
| NAME=$iface |
| DEVICE=$iface |
| ONBOOT=yes |
| MTU=9000 |
| NM_CONTROLLED=no |
| IPV6INIT=no |
| BOOTPROTO=none" >> $iface.tmp |
| mv $iface.tmp ifcfg-$iface |
| fi |
| done |
| popd |
| fi |
| # add entry in rc.local for activate-vfs |
| grep -q 'touch /var/lock/subsys/local' '/etc/rc.d/rc.local' |
| if [[ $? == 0 ]] |
| then |
| echo "/root/activate-vfs.sh" >> /etc/rc.local |
| fi |
| |
| |
| |
| # Activate 8 Virtual Functions per PF on Niantic cards (ixgbe driver) |
| if [[ `lsmod | cut -d" " -f1 | grep "ixgbe" | grep -v vf` ]] |
| then |
| if ! grep -q "ixgbe" /etc/modprobe.d/ixgbe.conf |
| then |
| echo "options ixgbe max_vfs=8" >> /etc/modprobe.d/ixgbe.conf |
| fi |
| |
| fi |
| |
| # Executes dracut to load drivers on boot |
| echo "Regenerating initramfs" |
| dracut --force |
| |
| # To define 8 VFs per PF we do it on rc.local, because the driver needs to be unloaded and loaded again |
| #if ! grep -q "NFV" /etc/rc.local |
| #then |
| # echo "" >> /etc/rc.local |
| # echo "# NFV" >> /etc/rc.local |
| # echo "modprobe -r ixgbe" >> /etc/rc.local |
| # echo "modprobe ixgbe max_vfs=8" >> /etc/rc.local |
| # echo "" >> /etc/rc.local |
| |
| # chmod +x /etc/rc.d/rc.local |
| |
| #fi |
| |
| echo |
| echo "Do not forget to create a shared (NFS, Samba, ...) where original virtual machine images are allocated" |
| echo |
| echo "Do not forget to copy the public ssh key of openvim user into /home/${user_name}/.ssh/authorized_keys for authomatic login from openvim controller" |
| echo |
| |
| echo "Reboot the system to make the changes effective" |
| |