Initial openvim v0.4.6 upload
[osm/openvim.git] / scripts / configure-compute-node-RHEL7.2.sh
1 #!/bin/bash
2
3 ##
4 # Copyright 2015 Telefónica Investigación y Desarrollo, S.A.U.
5 # This file is part of openmano
6 # All Rights Reserved.
7 #
8 # Licensed under the Apache License, Version 2.0 (the "License"); you may
9 # not use this file except in compliance with the License. You may obtain
10 # a copy of the License at
11 #
12 # http://www.apache.org/licenses/LICENSE-2.0
13 #
14 # Unless required by applicable law or agreed to in writing, software
15 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
16 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
17 # License for the specific language governing permissions and limitations
18 # under the License.
19 #
20 # For those usages not covered by the Apache License, Version 2.0 please
21 # contact with: nfvlabs@tid.es
22 ##
23
24 # Authors: Antonio Lopez, Pablo Montes, Alfonso Tierno
25 # 2016 March 18
26 # Modified to run on grub2 and efi boot
27
28 # Personalize RHEL7.2 on compute nodes
29 # Prepared to work with the following network card drivers:
30 # tg3, igb drivers for management interfaces
31 # ixgbe (Intel Niantic) and i40e (Intel Fortville) drivers for data plane interfaces
32
33 # To download:
34 # wget https://raw.githubusercontent.com/nfvlabs/openmano/master/scripts/configure-compute-node-RHEL7.1.sh
35 # To execute:
36 # chmod +x ./configure-compute-node-RHEL7.1.sh
37 # sudo ./configure-compute-node-RHEL7.1.sh <user> <iface>
38
39 # Assumptions:
40 # All virtualization options activated on BIOS (vt-d, vt-x, SR-IOV, no power savings...)
41 # RHEL7.2 installed without /home partition and with the following packages selection:
42 # @base, @core, @development, @network-file-system-client, @virtualization-hypervisor, @virtualization-platform, @virtualization-tools
43
44 # 2016 Aug 17 Antonio López
45 # Changed virbrInf to virbrVIM, to reflect that this bridge is used to communicate with the VIM (OpenVIM)
46 # Changed the vlan tag used by virbrVIM from 2000 to 1100
47
48 function usage(){
49 echo -e "Usage: sudo $0 [-y] <user-name> [ <iface-name> [<ip-address>|dhcp] ]"
50 echo -e " Configure compute host for VIM usage. (version 0.4). Params:"
51 echo -e " -y do not prompt for confirmation. If a new user is created, the user name is set as password"
52 echo -e " <user-name> Create if not exist and configure this user for openvim to connect"
53 echo -e " <iface-name> if suplied creates bridge interfaces on this interface, needed for openvim"
54 echo -e " ip or dhcp if suplied, configure the interface with this ip address (/24) or 'dhcp' "
55 }
56
57
58 #1 CHECK input parameters
59 #1.1 root privileges
60 [ "$USER" != "root" ] && echo "Needed root privileges" && usage && exit -1
61
62 #1.2 input parameters
63 FORCE=""
64 while getopts "y" o; do
65 case "${o}" in
66 y)
67 FORCE="yes"
68 ;;
69 *)
70 usage
71 exit -1
72 ;;
73 esac
74 done
75 shift $((OPTIND-1))
76
77
78 if [ $# -lt 1 ]
79 then
80 usage
81 exit
82 fi
83
84 user_name=$1
85 interface=$2
86 ip_iface=$3
87
88 if [ -n "$interface" ] && ! ifconfig $interface &> /dev/null
89 then
90 echo "Error: interface '$interface' is not present in the system"
91 usage
92 exit 1
93 fi
94
95 echo '
96 #################################################################
97 ##### INSTALL NEEDED PACKETS #####
98 #################################################################'
99
100 # Required packages
101 yum repolist
102 yum check-update
103 yum update -y
104 yum install -y screen virt-manager ethtool gcc gcc-c++ xorg-x11-xauth xorg-x11-xinit xorg-x11-deprecated-libs libXtst guestfish hwloc libhugetlbfs-utils libguestfs-tools numactl
105 # Selinux management
106 yum install -y policycoreutils-python
107
108 echo '
109 #################################################################
110 ##### INSTALL USER #####
111 #################################################################'
112
113 # Add required groups
114 groupadd -f nfvgroup
115 groupadd -f libvirt #for other operating systems may be libvirtd
116
117 # Adds user, default password same as name
118 if grep -q "^${user_name}:" /etc/passwd
119 then
120 #user exist, add to group
121 echo "adding user ${user_name} to groups libvirt,nfvgroup"
122 usermod -a -G libvirt,nfvgroup -g nfvgroup $user_name
123 else
124 #create user if it does not exist
125 [ -z "$FORCE" ] && read -p "user '${user_name}' does not exist, create (Y/n)" kk
126 if ! [ -z "$kk" -o "$kk"="y" -o "$kk"="Y" ]
127 then
128 exit
129 fi
130 echo "creating and configuring user ${user_name}"
131 useradd -m -G libvirt,nfvgroup -g nfvgroup $user_name
132 #Password
133 if [ -z "$FORCE" ]
134 then
135 echo "Provide a password for $user_name"
136 passwd $user_name
137 else
138 echo -e "$user_name\n$user_name" | passwd --stdin $user_name
139 fi
140 fi
141
142 #Setting default libvirt URI for the user
143 echo "Setting default libvirt URI for the user"
144 echo "if test -x `which virsh`; then" >> /home/${user_name}/.bash_profile
145 echo " export LIBVIRT_DEFAULT_URI=qemu:///system" >> /home/${user_name}/.bash_profile
146 echo "fi" >> /home/${user_name}/.bash_profile
147
148 echo '
149 #################################################################
150 ##### INSTALL HUGEPAGES ISOLCPUS GRUB #####
151 #################################################################'
152
153 # Huge pages 1G auto mount
154 mkdir -p /mnt/huge
155 if ! grep -q "Huge pages" /etc/fstab
156 then
157 echo "" >> /etc/fstab
158 echo "# Huge pages" >> /etc/fstab
159 echo "nodev /mnt/huge hugetlbfs pagesize=1GB 0 0" >> /etc/fstab
160 echo "" >> /etc/fstab
161 fi
162
163 # Huge pages reservation service
164 if ! [ -f /usr/lib/systemd/system/hugetlb-gigantic-pages.service ]
165 then
166 echo "configuring huge pages service"
167 cat > /usr/lib/systemd/system/hugetlb-gigantic-pages.service << EOL
168 [Unit]
169 Description=HugeTLB Gigantic Pages Reservation
170 DefaultDependencies=no
171 Before=dev-hugepages.mount
172 ConditionPathExists=/sys/devices/system/node
173 ConditionKernelCommandLine=hugepagesz=1G
174
175 [Service]
176 Type=oneshot
177 RemainAfterExit=yes
178 ExecStart=/usr/lib/systemd/hugetlb-reserve-pages
179
180 [Install]
181 WantedBy=sysinit.target
182 EOL
183 fi
184 # Grub virtualization options:
185
186 # Get isolcpus
187 isolcpus=`gawk 'BEGIN{pre=-2;}
188 ($1=="processor"){pro=$3;}
189 ($1=="core" && $4!=0){
190 if (pre+1==pro){endrange="-" pro}
191 else{cpus=cpus endrange sep pro; sep=","; endrange="";};
192 pre=pro;}
193 END{printf("%s",cpus endrange);}' /proc/cpuinfo`
194
195
196 # Huge pages reservation file: reserving all memory apart from 4GB per NUMA node
197 # Get the number of hugepages: all memory but 8GB reserved for the OS
198 #totalmem=`dmidecode --type 17|grep Size |grep MB |gawk '{suma+=$2} END {print suma/1024}'`
199 #hugepages=$(($totalmem-8))
200
201 if ! [ -f /usr/lib/systemd/hugetlb-reserve-pages ]
202 then
203 cat > /usr/lib/systemd/hugetlb-reserve-pages << EOL
204 #!/bin/bash
205 nodes_path=/sys/devices/system/node/
206 if [ ! -d \$nodes_path ]; then
207 echo "ERROR: \$nodes_path does not exist"
208 exit 1
209 fi
210
211 reserve_pages()
212 {
213 echo \$1 > \$nodes_path/\$2/hugepages/hugepages-1048576kB/nr_hugepages
214 }
215
216 # This example reserves all available memory apart from 4 GB for linux
217 # using 1GB size. You can modify it to your needs or comment the lines
218 # to avoid reserve memory in a numa node
219 EOL
220 for f in /sys/devices/system/node/node?/meminfo
221 do
222 node=`head -n1 $f | gawk '($5=="kB"){print $2}'`
223 memory=`head -n1 $f | gawk '($5=="kB"){print $4}'`
224 memory=$((memory+1048576-1)) #memory must be ceiled
225 memory=$((memory/1048576)) #from `kB to GB
226 #if memory
227 [ $memory -gt 4 ] && echo "reserve_pages $((memory-4)) node$node" >> /usr/lib/systemd/hugetlb-reserve-pages
228 done
229
230 # Run the following commands to enable huge pages early boot reservation:
231 chmod +x /usr/lib/systemd/hugetlb-reserve-pages
232 systemctl enable hugetlb-gigantic-pages
233 fi
234
235 # Prepares the text to add at the end of the grub line, including blacklisting ixgbevf driver in the host
236
237 textokernel="intel_iommu=on default_hugepagesz=1G hugepagesz=1G isolcpus=$isolcpus modprobe.blacklist=ixgbevf modprobe.blacklist=i40evf"
238
239 # Add text to the kernel line
240 if ! grep -q "intel_iommu=on default_hugepagesz=1G hugepagesz=1G" /etc/default/grub
241 then
242 echo "adding cmdline ${textokernel}"
243 sed -i "/^GRUB_CMDLINE_LINUX=/s/\"\$/ ${textokernel}\"/" /etc/default/grub
244
245 # grub2 upgrade
246
247 # BIOS based systems
248 grub2-mkconfig -o /boot/grub2/grub.cfg
249
250 # UEFI based systems
251 grub2-mkconfig -o /boot/efi/EFI/redhat/grub.cfg
252
253 fi
254
255 echo '
256 #################################################################
257 ##### OTHER CONFIGURATION #####
258 #################################################################'
259
260 # Disable requiretty
261 if ! grep -q "#openmano" /etc/sudoers
262 then
263 cat >> /home/${user_name}/script_visudo.sh << EOL
264 #!/bin/bash
265 cat \$1 | awk '(\$0~"requiretty"){print "#"\$0}(\$0!~"requiretty"){print \$0}' > tmp
266 cat tmp > \$1
267 rm tmp
268 EOL
269 chmod +x /home/${user_name}/script_visudo.sh
270 echo "Disabling requitetty"
271 export EDITOR=/home/${user_name}/script_visudo.sh && sudo -E visudo
272 rm -f /home/${user_name}/script_visudo.sh
273 fi
274
275 #Configure polkint to run virsh as a normal user
276 echo "Configuring polkint to run virsh as a normal user"
277 cat >> /etc/polkit-1/localauthority/50-local.d/50-org.libvirt-access.pkla << EOL
278 [libvirt Admin Access]
279 Identity=unix-group:libvirt
280 Action=org.libvirt.unix.manage
281 ResultAny=yes
282 ResultInactive=yes
283 ResultActive=yes
284 EOL
285
286 # Links the OpenMANO required folder /opt/VNF/images to /var/lib/libvirt/images. The OS installation
287 # should have only a / partition with all possible space available
288
289 echo " link /opt/VNF/images to /var/lib/libvirt/images"
290 if [ "$user_name" != "" ]
291 then
292 #mkdir -p /home/${user_name}/VNF_images
293 #chown -R ${user_name}:nfvgroup /home/${user_name}/VNF_images
294 #chmod go+x $HOME
295
296 # The orchestator needs to link the images folder
297 rm -f /opt/VNF/images
298 mkdir -p /opt/VNF/
299 ln -s /var/lib/libvirt/images /opt/VNF/images
300 chown -R ${user_name}:nfvgroup /opt/VNF
301 chown -R root:nfvgroup /var/lib/libvirt/images
302 chmod g+rwx /var/lib/libvirt/images
303
304 # Selinux management
305 #echo "configure Selinux management"
306 #semanage fcontext -a -t virt_image_t "/home/${user_name}/VNF_images(/.*)?"
307 #cat /etc/selinux/targeted/contexts/files/file_contexts.local |grep virt_image
308 #restorecon -R -v /home/${user_name}/VNF_images
309 else
310 mkdir -p /opt/VNF/images
311 chmod o+rx /opt/VNF/images
312 fi
313
314 echo "creating local information /opt/VNF/images/hostinfo.yaml"
315 echo "#By default openvim assumes control plane interface naming as em1,em2,em3,em4 " > /opt/VNF/images/hostinfo.yaml
316 echo "#and bridge ifaces as virbrMan1, virbrMan2, ..." >> /opt/VNF/images/hostinfo.yaml
317 echo "#if compute node contain a different name it must be indicated in this file" >> /opt/VNF/images/hostinfo.yaml
318 echo "#with the format extandard-name: compute-name" >> /opt/VNF/images/hostinfo.yaml
319 if [ "$interface" != "" -a "$interface" != "em1" ]
320 then
321 echo "iface_names:" >> /opt/VNF/images/hostinfo.yaml
322 echo " em1: ${interface}" >> /opt/VNF/images/hostinfo.yaml
323 fi
324 chmod o+r /opt/VNF/images/hostinfo.yaml
325
326 # deactivate memory overcommit
327 echo "deactivate memory overcommit"
328 service ksmtuned stop
329 service ksm stop
330 chkconfig ksmtuned off
331 chkconfig ksm off
332
333
334 # Libvirt options (uncomment the following)
335 echo "configure Libvirt options"
336 sed -i 's/#unix_sock_group = "libvirt"/unix_sock_group = "libvirt"/' /etc/libvirt/libvirtd.conf
337 sed -i 's/#unix_sock_rw_perms = "0770"/unix_sock_rw_perms = "0770"/' /etc/libvirt/libvirtd.conf
338 sed -i 's/#unix_sock_dir = "\/var\/run\/libvirt"/unix_sock_dir = "\/var\/run\/libvirt"/' /etc/libvirt/libvirtd.conf
339 sed -i 's/#auth_unix_rw = "none"/auth_unix_rw = "none"/' /etc/libvirt/libvirtd.conf
340
341 #creating the polkit grant access for libvirt user.
342 #This does not work !!!! so commented. No way to get running without uncomented the auth_unix_rw = "none" line
343 #
344 #cat > /etc/polkit-1/localauthority/50-local.d/50-org.example-libvirt-remote-access.pkla << EOL
345 #[libvirt Management Access]
346 # Identity=unix-user:n2;unix-user:kk
347 # Action=org.libvirt.unix.manage
348 # ResultAny=yes
349 # ResultInactive=yes
350 # ResultActive=yes
351 #EOL
352
353 # Configuration change of qemu for the numatune bug issue
354 # RHEL7.1: for this version should not be necesary - to revise
355 #if ! grep -q "cgroup_controllers = [ \"cpu\", \"devices\", \"memory\", \"blkio\", \"cpuacct\" ]" /etc/libvirt/qemu.conf
356 #then
357 #cat /etc/libvirt/qemu.conf | awk '{print $0}($0~"#cgroup_controllers"){print "cgroup_controllers = [ \"cpu\", \"devices\", \"memory\", \"blkio\", \"cpuacct\" ]"}' > tmp
358 #mv tmp /etc/libvirt/qemu.conf
359 #fi
360
361 echo '
362 #################################################################
363 ##### NETWORK CONFIGURATION #####
364 #################################################################'
365 # Network config (if the second parameter is net)
366 if [ -n "$interface" ]
367 then
368
369 # Deactivate network manager
370 systemctl stop NetworkManager
371 systemctl disable NetworkManager
372
373 # For management and data interfaces
374 #rm -f /etc/udev/rules.d/pci_config.rules # it will be created to define VFs
375
376 pushd /etc/sysconfig/network-scripts/
377
378 # Set ONBOOT=on and MTU=9000 on the interface used for the bridges
379 echo "configuring iface $interface"
380 cat ifcfg-$interface | grep -e HWADDR -e UUID > $interface.tmp
381 echo "TYPE=Ethernet
382 NAME=$interface
383 DEVICE=$interface
384 TYPE=Ethernet
385 ONBOOT=yes
386 NM_CONTROLLED=no
387 MTU=9000
388 BOOTPROTO=none
389 IPV6INIT=no" >> $interface.tmp
390 mv $interface.tmp ifcfg-$interface
391
392 # Management interfaces
393 # integrated_interfaces=""
394 # nb_ifaces=0
395 # for iface in `ifconfig -a | grep ":\ " | cut -f 1 -d":"| grep -v "_" | grep -v "\." | grep -v "lo" | sort`
396 # do
397 # driver=`ethtool -i $iface| awk '($0~"driver"){print $2}'`
398 # if [ $driver != "ixgbe" ] && [ $driver != "bridge" ]
399 # then
400 # integrated_interfaces="$integrated_interfaces $iface"
401 # nb_ifaces=$((nb_ifaces+1))
402 # eval iface${nb_ifaces}=$iface
403 # fi
404 # done
405
406 #Create infrastructure bridge, normally used for connecting to compute nodes, openflow controller, ...
407 echo "DEVICE=virbrVIM
408 NAME=virbrVIM
409 TYPE=Bridge
410 ONBOOT=yes
411 DELAY=0
412 NM_CONTROLLED=no
413 MTU=9000
414 USERCTL=no" > ifcfg-virbrVIM
415 [[ $ip_iface != "dhcp" ]] && [[ $ip_iface != "" ]] && echo -e "BOOTPROTO=static\nIPADDR=${ip_iface}\nNETMASK=255.255.255.0" >> ifcfg-virbrVIM
416
417 #Create VLAN for infrastructure bridge
418 echo "DEVICE=${interface}.1100
419 NAME=${interface}.1100
420 ONBOOT=yes
421 NM_CONTROLLED=no
422 USERCTL=no
423 VLAN=yes
424 MTU=9000
425 BOOTPROTO=none
426 BRIDGE=virbrVIM" > ifcfg-${interface}.1100
427
428
429 #Create bridge interfaces
430 echo "Creating bridge ifaces: "
431 for ((i=1;i<=20;i++))
432 do
433 i2digits=$i
434 [ $i -lt 10 ] && i2digits="0$i"
435 echo " virbrMan$i vlan 20$i2digits"
436 echo "DEVICE=virbrMan$i
437 NAME=virbrMan$i
438 TYPE=Bridge
439 ONBOOT=yes
440 DELAY=0
441 NM_CONTROLLED=no
442 MTU=9000
443 USERCTL=no" > ifcfg-virbrMan$i
444
445 #Without IP:
446 #BOOTPROTO=static
447 #IPADDR=10.10.10.$((i+209))
448 #NETMASK=255.255.255.0" > ifcfg-virbrMan$i
449
450 # create the required interfaces to connect the bridges
451 echo "DEVICE=${interface}.20$i2digits
452 NAME=${interface}.20$i2digits
453 ONBOOT=yes
454 NM_CONTROLLED=no
455 USERCTL=no
456 VLAN=yes
457 BOOTPROTO=none
458 MTU=9000
459 BRIDGE=virbrMan$i" > ifcfg-${interface}.20$i2digits
460 done
461
462 iface=$interface
463 if [ -n "$ip_iface" ]
464 then
465 echo "configuring iface $iface interface with ip $ip_iface"
466 # Network interfaces
467 # 1Gbps interfaces are configured with ONBOOT=yes and static IP address
468 cat ifcfg-$iface | grep -e HWADDR -e UUID > $iface.tmp
469 echo "TYPE=Ethernet
470 NAME=$iface
471 DEVICE=$iface
472 TYPE=Ethernet
473 ONBOOT=yes
474 NM_CONTROLLED=no
475 MTU=9000
476 IPV6INIT=no" >> $iface.tmp
477 [ $ip_iface = "dhcp" ] && echo -e "BOOTPROTO=dhcp\nDHCP_HOSTNAME=$HOSTNAME" >> $iface.tmp
478 [ $ip_iface != "dhcp" ] && echo -e "BOOTPROTO=static\nIPADDR=${ip_iface}\nNETMASK=255.255.255.0" >> $iface.tmp
479 mv $iface.tmp ifcfg-$iface
480 fi
481 # Script to create vfs
482 echo "#!/bin/bash" > /root/activate-vfs.sh
483 chmod +x /root/activate-vfs.sh
484 for iface in `ifconfig -a | grep ": " | cut -f 1 -d":" | grep -v -e "_" -e "\." -e "lo" -e "virbr" -e "tap"`
485 do
486 # 10/40 Gbps interfaces
487 # Intel X520 cards: driver ixgbe
488 # Intel XL710 Fortville cards: driver i40e
489 driver=`ethtool -i $iface| awk '($0~"driver"){print $2}'`
490 if [ "$driver" = "i40e" -o "$driver" = "ixgbe" ]
491 then
492 echo "configuring dataplane iface $iface"
493
494 # Create 8 SR-IOV per PF by udev rules only for Fortville cards (i40e driver)
495 if [ "$driver" = "i40e" ]
496 then
497 pci=`ethtool -i $iface | awk '($0~"bus-info"){print $2}'`
498 echo "echo 8 > /sys/bus/pci/devices/$pci/sriov_numvfs" >> /root/activate-vfs.sh
499 fi
500
501 # Configure PF to boot automatically and to have a big MTU
502 # 10Gbps interfaces are configured with ONBOOT=yes and MTU=2000
503 cat ifcfg-$iface | grep -e HWADDR -e UUID > $iface.tmp
504 echo "TYPE=Ethernet
505 NAME=$iface
506 DEVICE=$iface
507 ONBOOT=yes
508 MTU=9000
509 NM_CONTROLLED=no
510 IPV6INIT=no
511 BOOTPROTO=none" >> $iface.tmp
512 mv $iface.tmp ifcfg-$iface
513 fi
514 done
515 popd
516 fi
517 # add entry in rc.local for activate-vfs
518 grep -q 'touch /var/lock/subsys/local' '/etc/rc.d/rc.local'
519 if [[ $? == 0 ]]
520 then
521 echo "/root/activate-vfs.sh" >> /etc/rc.local
522 fi
523
524
525
526 # Activate 8 Virtual Functions per PF on Niantic cards (ixgbe driver)
527 if [[ `lsmod | cut -d" " -f1 | grep "ixgbe" | grep -v vf` ]]
528 then
529 if ! grep -q "ixgbe" /etc/modprobe.d/ixgbe.conf
530 then
531 echo "options ixgbe max_vfs=8" >> /etc/modprobe.d/ixgbe.conf
532 fi
533
534 fi
535
536 # Executes dracut to load drivers on boot
537 echo "Regenerating initramfs"
538 dracut --force
539
540 # To define 8 VFs per PF we do it on rc.local, because the driver needs to be unloaded and loaded again
541 #if ! grep -q "NFV" /etc/rc.local
542 #then
543 # echo "" >> /etc/rc.local
544 # echo "# NFV" >> /etc/rc.local
545 # echo "modprobe -r ixgbe" >> /etc/rc.local
546 # echo "modprobe ixgbe max_vfs=8" >> /etc/rc.local
547 # echo "" >> /etc/rc.local
548
549 # chmod +x /etc/rc.d/rc.local
550
551 #fi
552
553 echo
554 echo "Do not forget to create a shared (NFS, Samba, ...) where original virtual machine images are allocated"
555 echo
556 echo "Do not forget to copy the public ssh key of openvim user into /home/${user_name}/.ssh/authorized_keys for authomatic login from openvim controller"
557 echo
558
559 echo "Reboot the system to make the changes effective"
560