4 # Copyright 2015 Telefónica Investigación y Desarrollo, S.A.U.
5 # This file is part of openvim
8 # Licensed under the Apache License, Version 2.0 (the "License"); you may
9 # not use this file except in compliance with the License. You may obtain
10 # a copy of the License at
12 # http://www.apache.org/licenses/LICENSE-2.0
14 # Unless required by applicable law or agreed to in writing, software
15 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
16 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
17 # License for the specific language governing permissions and limitations
20 # For those usages not covered by the Apache License, Version 2.0 please
21 # contact with: nfvlabs@tid.es
24 # Authors: Antonio Lopez, Pablo Montes, Alfonso Tierno
26 # Modified to run on grub2 and efi boot
28 # Personalize RHEL7.2 on compute nodes
29 # Prepared to work with the following network card drivers:
30 # tg3, igb drivers for management interfaces
31 # ixgbe (Intel Niantic) and i40e (Intel Fortville) drivers for data plane interfaces
34 # wget https://raw.githubusercontent.com/nfvlabs/openmano/master/scripts/configure-compute-node-RHEL7.1.sh
36 # chmod +x ./configure-compute-node-RHEL7.1.sh
37 # sudo ./configure-compute-node-RHEL7.1.sh <user> <iface>
40 # All virtualization options activated on BIOS (vt-d, vt-x, SR-IOV, no power savings...)
41 # RHEL7.2 installed without /home partition and with the following packages selection:
42 # @base, @core, @development, @network-file-system-client, @virtualization-hypervisor, @virtualization-platform, @virtualization-tools
44 # 2016 Aug 17 Antonio López
45 # Changed virbrInf to virbrVIM, to reflect that this bridge is used to communicate with the VIM (OpenVIM)
46 # Changed the vlan tag used by virbrVIM from 2000 to 1100
49 echo -e "Usage: sudo $0 [-f] <user-name> [ <iface-for-overlay-bridges> [<ip-address>|dhcp] ]"
50 echo -e " Configure compute host for VIM usage. (version 0.4). Params:"
51 echo -e " -f do not prompt for confirmation. If a new user is created, the user name is set as password"
52 echo -e " <user-name> Create if not exist and configure this user for openvim to connect"
53 echo -e " <iface-for-overlay-bridges> if suplied creates bridge interfaces on this interface, needed for older openvim versions"
54 echo -e " ip or dhcp if suplied, configure the interface with this ip address (/24) or 'dhcp' "
58 function _install_openvswitch
(){
60 echo "Installing openvswitch"
61 curl
-O http
://openvswitch.org
/releases
/openvswitch-2.5
.1.
tar.gz
62 mkdir
-p ~
/rpmbuild
/SOURCES
63 cp openvswitch-2.5
.1.
tar.gz ~
/rpmbuild
/SOURCES
/
64 tar -zxvf openvswitch-2.5
.1.
tar.gz
65 cp -r openvswitch-2.5
.1 ~
/rpmbuild
/SOURCES
/
66 rpmbuild
-bb --without check ~
/rpmbuild
/SOURCES
/openvswitch-2.5
.1/rhel
/openvswitch.spec
67 yum
-y localinstall
/root
/rpmbuild
/RPMS
/x86_64
/openvswitch-2.5
.1-1.x86_64.rpm
68 systemctl start openvswitch.service
71 #1 CHECK input parameters
73 [ "$USER" != "root" ] && echo "Needed root privileges" && usage
&& exit -1
77 while getopts "f" o
; do
102 if [ -n "$interface" ] && ! ifconfig
$interface &> /dev
/null
104 echo "Error: interface '$interface' is not present in the system"
111 #################################################################
112 ##### INSTALL NEEDED PACKETS #####
113 #################################################################'
119 yum
install -y screen virt-manager ethtool gcc gcc-c
++ xorg-x11-xauth xorg-x11-xinit xorg-x11-deprecated-libs libXtst \
120 guestfish hwloc libhugetlbfs-utils libguestfs-tools numactl
123 # gcc make python-devel openssl-devel kernel-devel graphviz \ kernel-debug-devel autoconf automake rpm-build redhat-rpm-config \ libtool
126 yum
install -y policycoreutils-python
129 #################################################################
130 ##### INSTALL USER #####
131 #################################################################'
133 # Add required groups
135 groupadd
-f libvirt
#for other operating systems may be libvirtd
137 # Adds user, default password same as name
138 if grep -q "^${user_name}:" /etc
/passwd
140 #user exist, add to group
141 echo "adding user ${user_name} to groups libvirt,nfvgroup"
142 usermod
-a -G libvirt
,nfvgroup
-g nfvgroup
$user_name
144 #create user if it does not exist
145 [ -z "$FORCE" ] && read -p "user '${user_name}' does not exist, create (Y/n)" kk
146 if ! [ -z "$kk" -o "$kk"="y" -o "$kk"="Y" ]
150 echo "creating and configuring user ${user_name}"
151 useradd
-m -G libvirt
,nfvgroup
-g nfvgroup
$user_name
155 echo "Provide a password for $user_name"
158 echo -e "$user_name\n$user_name" | passwd
--stdin $user_name
162 #Setting default libvirt URI for the user
163 echo "Setting default libvirt URI for the user"
164 echo "if test -x `which virsh`; then" >> /home
/${user_name}/.bash_profile
165 echo " export LIBVIRT_DEFAULT_URI=qemu:///system" >> /home
/${user_name}/.bash_profile
166 echo "fi" >> /home
/${user_name}/.bash_profile
169 #################################################################
170 ##### INSTALL HUGEPAGES ISOLCPUS GRUB #####
171 #################################################################'
173 # Huge pages 1G auto mount
175 if ! grep -q "Huge pages" /etc
/fstab
177 echo "" >> /etc
/fstab
178 echo "# Huge pages" >> /etc
/fstab
179 echo "nodev /mnt/huge hugetlbfs pagesize=1GB 0 0" >> /etc
/fstab
180 echo "" >> /etc
/fstab
183 # Huge pages reservation service
184 if ! [ -f /usr
/lib
/systemd
/system
/hugetlb-gigantic-pages.service
]
186 echo "configuring huge pages service"
187 cat > /usr
/lib
/systemd
/system
/hugetlb-gigantic-pages.service
<< EOL
189 Description=HugeTLB Gigantic Pages Reservation
190 DefaultDependencies=no
191 Before=dev-hugepages.mount
192 ConditionPathExists=/sys/devices/system/node
193 ConditionKernelCommandLine=hugepagesz=1G
198 ExecStart=/usr/lib/systemd/hugetlb-reserve-pages
201 WantedBy=sysinit.target
204 # Grub virtualization options:
207 isolcpus
=`gawk 'BEGIN{pre=-2;}
208 ($1=="processor"){pro=$3;}
209 ($1=="core" && $4!=0){
210 if (pre+1==pro){endrange="-" pro}
211 else{cpus=cpus endrange sep pro; sep=","; endrange="";};
213 END{printf("%s",cpus endrange);}' /proc/cpuinfo`
216 # Huge pages reservation file: reserving all memory apart from 4GB per NUMA node
217 # Get the number of hugepages: all memory but 8GB reserved for the OS
218 #totalmem=`dmidecode --type 17|grep Size |grep MB |gawk '{suma+=$2} END {print suma/1024}'`
219 #hugepages=$(($totalmem-8))
221 if ! [ -f /usr
/lib
/systemd
/hugetlb-reserve-pages
]
223 cat > /usr
/lib
/systemd
/hugetlb-reserve-pages
<< EOL
225 nodes_path=/sys/devices/system/node/
226 if [ ! -d \$nodes_path ]; then
227 echo "ERROR: \$nodes_path does not exist"
233 echo \$1 > \$nodes_path/\$2/hugepages/hugepages-1048576kB/nr_hugepages
236 # This example reserves all available memory apart from 4 GB for linux
237 # using 1GB size. You can modify it to your needs or comment the lines
238 # to avoid reserve memory in a numa node
240 for f
in /sys
/devices
/system
/node
/node?
/meminfo
242 node
=`head -n1 $f | gawk '($5=="kB"){print $2}'`
243 memory
=`head -n1 $f | gawk '($5=="kB"){print $4}'`
244 memory
=$
((memory
+1048576-1)) #memory must be ceiled
245 memory
=$
((memory
/1048576)) #from `kB to GB
247 [ $memory -gt 4 ] && echo "reserve_pages $((memory-4)) node$node" >> /usr
/lib
/systemd
/hugetlb-reserve-pages
250 # Run the following commands to enable huge pages early boot reservation:
251 chmod +x
/usr
/lib
/systemd
/hugetlb-reserve-pages
252 systemctl
enable hugetlb-gigantic-pages
255 # Prepares the text to add at the end of the grub line, including blacklisting ixgbevf driver in the host
257 textokernel
="intel_iommu=on default_hugepagesz=1G hugepagesz=1G isolcpus=$isolcpus modprobe.blacklist=ixgbevf modprobe.blacklist=i40evf"
259 # Add text to the kernel line
260 if ! grep -q "intel_iommu=on default_hugepagesz=1G hugepagesz=1G" /etc
/default
/grub
262 echo "adding cmdline ${textokernel}"
263 sed -i "/^GRUB_CMDLINE_LINUX=/s/\"\$/ ${textokernel}\"/" /etc
/default
/grub
268 grub2-mkconfig
-o /boot
/grub
2/grub.cfg
271 grub2-mkconfig
-o /boot
/efi
/EFI
/redhat
/grub.cfg
276 #################################################################
277 ##### OTHER CONFIGURATION #####
278 #################################################################'
281 if ! grep -q "#openmano" /etc
/sudoers
283 cat >> /home
/${user_name}/script_visudo.sh
<< EOL
285 cat \$1 | awk '(\$0~"requiretty"){print "#"\$0}(\$0!~"requiretty"){print \$0}' > tmp
289 chmod +x
/home
/${user_name}/script_visudo.sh
290 echo "Disabling requitetty"
291 export EDITOR
=/home
/${user_name}/script_visudo.sh
&& sudo
-E visudo
292 rm -f /home
/${user_name}/script_visudo.sh
295 #Configure polkint to run virsh as a normal user
296 echo "Configuring polkint to run virsh as a normal user"
297 cat >> /etc
/polkit-
1/localauthority
/50-local.d
/50-org.libvirt-access.pkla
<< EOL
298 [libvirt Admin Access]
299 Identity=unix-group:libvirt
300 Action=org.libvirt.unix.manage
306 # Links the OpenMANO required folder /opt/VNF/images to /var/lib/libvirt/images. The OS installation
307 # should have only a / partition with all possible space available
309 echo " link /opt/VNF/images to /var/lib/libvirt/images"
310 if [ "$user_name" != "" ]
312 #mkdir -p /home/${user_name}/VNF_images
313 #chown -R ${user_name}:nfvgroup /home/${user_name}/VNF_images
316 # The orchestator needs to link the images folder
317 rm -f /opt
/VNF
/images
319 ln -s /var
/lib
/libvirt
/images
/opt
/VNF
/images
320 chown
-R ${user_name}:nfvgroup
/opt
/VNF
321 chown
-R root
:nfvgroup
/var
/lib
/libvirt
/images
322 chmod g
+rwx
/var
/lib
/libvirt
/images
325 #echo "configure Selinux management"
326 #semanage fcontext -a -t virt_image_t "/home/${user_name}/VNF_images(/.*)?"
327 #cat /etc/selinux/targeted/contexts/files/file_contexts.local |grep virt_image
328 #restorecon -R -v /home/${user_name}/VNF_images
330 mkdir
-p /opt
/VNF
/images
331 chmod o
+rx
/opt
/VNF
/images
334 echo "creating local information /opt/VNF/images/hostinfo.yaml"
335 echo "#By default openvim assumes control plane interface naming as em1,em2,em3,em4 " > /opt
/VNF
/images
/hostinfo.yaml
336 echo "#and bridge ifaces as virbrMan1, virbrMan2, ..." >> /opt
/VNF
/images
/hostinfo.yaml
337 echo "#if compute node contain a different name it must be indicated in this file" >> /opt
/VNF
/images
/hostinfo.yaml
338 echo "#with the format extandard-name: compute-name" >> /opt
/VNF
/images
/hostinfo.yaml
339 if [ "$interface" != "" -a "$interface" != "em1" ]
341 echo "iface_names:" >> /opt
/VNF
/images
/hostinfo.yaml
342 echo " em1: ${interface}" >> /opt
/VNF
/images
/hostinfo.yaml
344 chmod o
+r
/opt
/VNF
/images
/hostinfo.yaml
346 # deactivate memory overcommit
347 echo "deactivate memory overcommit"
348 service ksmtuned stop
350 chkconfig ksmtuned off
354 # Libvirt options (uncomment the following)
355 echo "configure Libvirt options"
356 sed -i 's/#unix_sock_group = "libvirt"/unix_sock_group = "libvirt"/' /etc
/libvirt
/libvirtd.conf
357 sed -i 's/#unix_sock_rw_perms = "0770"/unix_sock_rw_perms = "0770"/' /etc
/libvirt
/libvirtd.conf
358 sed -i 's/#unix_sock_dir = "\/var\/run\/libvirt"/unix_sock_dir = "\/var\/run\/libvirt"/' /etc
/libvirt
/libvirtd.conf
359 sed -i 's/#auth_unix_rw = "none"/auth_unix_rw = "none"/' /etc
/libvirt
/libvirtd.conf
361 #creating the polkit grant access for libvirt user.
362 #This does not work !!!! so commented. No way to get running without uncomented the auth_unix_rw = "none" line
364 #cat > /etc/polkit-1/localauthority/50-local.d/50-org.example-libvirt-remote-access.pkla << EOL
365 #[libvirt Management Access]
366 # Identity=unix-user:n2;unix-user:kk
367 # Action=org.libvirt.unix.manage
373 # Configuration change of qemu for the numatune bug issue
374 # RHEL7.1: for this version should not be necesary - to revise
375 #if ! grep -q "cgroup_controllers = [ \"cpu\", \"devices\", \"memory\", \"blkio\", \"cpuacct\" ]" /etc/libvirt/qemu.conf
377 #cat /etc/libvirt/qemu.conf | awk '{print $0}($0~"#cgroup_controllers"){print "cgroup_controllers = [ \"cpu\", \"devices\", \"memory\", \"blkio\", \"cpuacct\" ]"}' > tmp
378 #mv tmp /etc/libvirt/qemu.conf
382 #################################################################
383 ##### NETWORK CONFIGURATION #####
384 #################################################################'
385 # Network config (if the second parameter is net)
386 if [ -n "$interface" ]
389 # Deactivate network manager
390 systemctl stop NetworkManager
391 systemctl disable NetworkManager
393 # For management and data interfaces
394 #rm -f /etc/udev/rules.d/pci_config.rules # it will be created to define VFs
396 pushd /etc
/sysconfig
/network-scripts
/
398 # Set ONBOOT=on and MTU=9000 on the interface used for the bridges
399 echo "configuring iface $interface"
400 cat ifcfg-
$interface |
grep -e HWADDR
-e UUID
> $interface.tmp
409 IPV6INIT=no" >> $interface.tmp
410 mv $interface.tmp ifcfg-
$interface
414 # Management interfaces
415 # integrated_interfaces=""
417 # for iface in `ifconfig -a | grep ":\ " | cut -f 1 -d":"| grep -v "_" | grep -v "\." | grep -v "lo" | sort`
419 # driver=`ethtool -i $iface| awk '($0~"driver"){print $2}'`
420 # if [ $driver != "ixgbe" ] && [ $driver != "bridge" ]
422 # integrated_interfaces="$integrated_interfaces $iface"
423 # nb_ifaces=$((nb_ifaces+1))
424 # eval iface${nb_ifaces}=$iface
428 #Create infrastructure bridge, normally used for connecting to compute nodes, openflow controller, ...
429 echo "DEVICE=virbrVIM
436 USERCTL=no" > ifcfg-virbrVIM
437 [[ $ip_iface != "dhcp" ]] && [[ $ip_iface != "" ]] && echo -e "BOOTPROTO=static\nIPADDR=${ip_iface}\nNETMASK=255.255.255.0" >> ifcfg-virbrVIM
439 #Create VLAN for infrastructure bridge
440 echo "DEVICE=${interface}.1100
441 NAME=${interface}.1100
448 BRIDGE=virbrVIM" > ifcfg-
${interface}.1100
451 #Create bridge interfaces
452 echo "Creating bridge ifaces: "
453 for ((i
=1;i
<=20;i
++))
456 [ $i -lt 10 ] && i2digits
="0$i"
457 echo " virbrMan$i vlan 20$i2digits"
458 echo "DEVICE=virbrMan$i
465 USERCTL=no" > ifcfg-virbrMan
$i
469 #IPADDR=10.10.10.$((i+209))
470 #NETMASK=255.255.255.0" > ifcfg-virbrMan$i
472 # create the required interfaces to connect the bridges
473 echo "DEVICE=${interface}.20$i2digits
474 NAME=${interface}.20$i2digits
481 BRIDGE=virbrMan$i" > ifcfg-
${interface}.20$i2digits
485 if [ -n "$ip_iface" ]
487 echo "configuring iface $iface interface with ip $ip_iface"
489 # 1Gbps interfaces are configured with ONBOOT=yes and static IP address
490 cat ifcfg-
$iface |
grep -e HWADDR
-e UUID
> $iface.tmp
498 IPV6INIT=no" >> $iface.tmp
499 [ $ip_iface = "dhcp" ] && echo -e "BOOTPROTO=dhcp\nDHCP_HOSTNAME=$HOSTNAME" >> $iface.tmp
500 [ $ip_iface != "dhcp" ] && echo -e "BOOTPROTO=static\nIPADDR=${ip_iface}\nNETMASK=255.255.255.0" >> $iface.tmp
501 mv $iface.tmp ifcfg-
$iface
503 # Script to create vfs
504 echo "#!/bin/bash" > /root
/activate-vfs.sh
505 chmod +x
/root
/activate-vfs.sh
506 for iface
in `ifconfig -a | grep ": " | cut -f 1 -d":" | grep -v -e "_" -e "\." -e "lo" -e "virbr" -e "tap"`
508 # 10/40 Gbps interfaces
509 # Intel X520 cards: driver ixgbe
510 # Intel XL710 Fortville cards: driver i40e
511 driver
=`ethtool -i $iface| awk '($0~"driver"){print $2}'`
512 if [ "$driver" = "i40e" -o "$driver" = "ixgbe" ]
514 echo "configuring dataplane iface $iface"
516 # Create 8 SR-IOV per PF by udev rules only for Fortville cards (i40e driver)
517 if [ "$driver" = "i40e" ]
519 pci
=`ethtool -i $iface | awk '($0~"bus-info"){print $2}'`
520 echo "echo 8 > /sys/bus/pci/devices/$pci/sriov_numvfs" >> /root
/activate-vfs.sh
523 # Configure PF to boot automatically and to have a big MTU
524 # 10Gbps interfaces are configured with ONBOOT=yes and MTU=2000
525 cat ifcfg-
$iface |
grep -e HWADDR
-e UUID
> $iface.tmp
533 BOOTPROTO=none" >> $iface.tmp
534 mv $iface.tmp ifcfg-
$iface
539 # add entry in rc.local for activate-vfs
540 grep -q 'touch /var/lock/subsys/local' '/etc/rc.d/rc.local'
543 echo "/root/activate-vfs.sh" >> /etc
/rc.
local
548 # Activate 8 Virtual Functions per PF on Niantic cards (ixgbe driver)
549 if [[ `lsmod | cut -d" " -f1 | grep "ixgbe" | grep -v vf` ]]
551 if ! grep -q "ixgbe" /etc
/modprobe.d
/ixgbe.conf
553 echo "options ixgbe max_vfs=8" >> /etc
/modprobe.d
/ixgbe.conf
558 # Executes dracut to load drivers on boot
559 echo "Regenerating initramfs"
562 # To define 8 VFs per PF we do it on rc.local, because the driver needs to be unloaded and loaded again
563 #if ! grep -q "NFV" /etc/rc.local
565 # echo "" >> /etc/rc.local
566 # echo "# NFV" >> /etc/rc.local
567 # echo "modprobe -r ixgbe" >> /etc/rc.local
568 # echo "modprobe ixgbe max_vfs=8" >> /etc/rc.local
569 # echo "" >> /etc/rc.local
571 # chmod +x /etc/rc.d/rc.local
577 echo "Do not forget to create a shared (NFS, Samba, ...) where original virtual machine images are allocated"
579 echo "Do not forget to copy the public ssh key of openvim user into /home/${user_name}/.ssh/authorized_keys for authomatic login from openvim controller"
582 echo "Reboot the system to make the changes effective"