4 # Copyright 2015 Telefónica Investigación y Desarrollo, S.A.U.
5 # This file is part of openvim
8 # Licensed under the Apache License, Version 2.0 (the "License"); you may
9 # not use this file except in compliance with the License. You may obtain
10 # a copy of the License at
12 # http://www.apache.org/licenses/LICENSE-2.0
14 # Unless required by applicable law or agreed to in writing, software
15 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
16 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
17 # License for the specific language governing permissions and limitations
20 # For those usages not covered by the Apache License, Version 2.0 please
21 # contact with: nfvlabs@tid.es
24 # Authors: Antonio Lopez, Pablo Montes, Alfonso Tierno
27 # Personalize RHEL7.1 on compute nodes
28 # Prepared to work with the following network card drivers:
29 # tg3, igb drivers for management interfaces
30 # ixgbe (Intel Niantic) and i40e (Intel Fortville) drivers for data plane interfaces
33 # wget https://raw.githubusercontent.com/nfvlabs/openmano/master/scripts/configure-compute-node-RHEL7.1.sh
35 # chmod +x ./configure-compute-node-RHEL7.1.sh
36 # sudo ./configure-compute-node-RHEL7.1.sh <user> <iface>
39 # All virtualization options activated on BIOS (vt-d, vt-x, SR-IOV, no power savings...)
40 # RHEL7.1 installed without /home partition and with the following packages selection:
41 # @base, @core, @development, @network-file-system-client, @virtualization-hypervisor, @virtualization-platform, @virtualization-tools
45 echo -e "Usage: sudo $0 [-y] <user-name> [ <iface-name> [<ip-address>|dhcp] ]"
46 echo -e " Configure compute host for VIM usage. (version 0.4). Params:"
47 echo -e " -y do not prompt for confirmation. If a new user is created, the user name is set as password"
48 echo -e " <user-name> Create if not exist and configure this user for openvim to connect"
49 echo -e " <iface-name> if suplied creates bridge interfaces on this interface, needed for openvim"
50 echo -e " ip or dhcp if suplied, configure the interface with this ip address (/24) or 'dhcp' "
54 #1 CHECK input parameters
56 [ "$USER" != "root" ] && echo "Needed root privileges" && usage
&& exit -1
60 while getopts "y" o
; do
86 if [ -n "$interface" ] && ! ifconfig
$interface &> /dev
/null
88 echo "Error: interface '$interface' is not present in the system"
94 #################################################################
95 ##### INSTALL NEEDED PACKETS #####
96 #################################################################'
100 #apt-get -y install grub-common screen virt-manager ethtool build-essential x11-common x11-utils x11-apps libguestfs-tools hwloc libguestfs-tools numactl vlan nfs-common nfs-kernel-server
101 apt-get
-y install grub-common screen virt-manager ethtool build-essential x11-common x11-utils libguestfs-tools hwloc libguestfs-tools numactl vlan nfs-common nfs-kernel-server
103 echo "Remove unneeded packages....."
104 apt-get
-y autoremove
106 #yum install -y policycoreutils-python
111 #################################################################
112 ##### INSTALL USER #####
113 #################################################################'
115 # Add required groups
117 groupadd
-f libvirt
#for other operating systems may be libvirtd
119 # Adds user, default password same as name
120 if grep -q "^${user_name}:" /etc
/passwd
122 #user exist, add to group
123 echo "adding user ${user_name} to groups libvirt,admin"
124 usermod
-a -G libvirt
,admin
-g admin
$user_name
126 #create user if it does not exist
127 [ -z "$FORCE" ] && read -p "user '${user_name}' does not exist, create (Y/n)" kk
128 if ! [ -z "$kk" -o "$kk"="y" -o "$kk"="Y" ]
132 echo "creating and configuring user ${user_name}"
133 useradd
-m -G libvirt
,admin
-g admin
$user_name
137 echo "Provide a password for $user_name"
140 echo -e "$user_name\n$user_name" | passwd
--stdin $user_name
144 # Allow admin users to access without password
145 if ! grep -q "#openmano" /etc
/sudoers
147 cat >> /home
/${user_name}/script_visudo.sh
<< EOL
149 cat \$1 | awk '(\$0~"requiretty"){print "#"\$0}(\$0!~"requiretty"){print \$0}' > tmp
153 echo "#openmano allow to group admin to grant root privileges without password" >> \$1
154 echo "%admin ALL=(ALL) NOPASSWD: ALL" >> \$1
156 chmod +x
/home
/${user_name}/script_visudo.sh
157 echo "allowing admin user to get root privileges withut password"
158 export EDITOR
=/home
/${user_name}/script_visudo.sh
&& sudo
-E visudo
159 rm -f /home
/${user_name}/script_visudo.sh
164 #################################################################
165 ##### INSTALL HUGEPAGES ISOLCPUS GRUB #####
166 #################################################################'
168 # Huge pages 1G auto mount
170 if ! grep -q "Huge pages" /etc
/fstab
172 echo "" >> /etc
/fstab
173 echo "# Huge pages" >> /etc
/fstab
174 echo "nodev /mnt/huge hugetlbfs pagesize=1GB 0 0" >> /etc
/fstab
175 echo "" >> /etc
/fstab
178 # Grub virtualization options:
181 isolcpus
=`gawk 'BEGIN{pre=-2;}
182 ($1=="processor"){pro=$3;}
183 ($1=="core" && $4!=0){
184 if (pre+1==pro){endrange="-" pro}
185 else{cpus=cpus endrange sep pro; sep=","; endrange="";};
187 END{printf("%s",cpus endrange);}' /proc/cpuinfo`
190 echo "CPUS: $isolcpus"
192 # Huge pages reservation file: reserving all memory apart from 4GB per NUMA node
193 # Get the number of hugepages: all memory but 8GB reserved for the OS
194 #totalmem=`dmidecode --type 17|grep Size |grep MB |gawk '{suma+=$2} END {print suma/1024}'`
195 #hugepages=$(($totalmem-8))
197 if ! [ -f /usr
/lib
/systemd
/hugetlb-reserve-pages
]
199 cat > /usr
/lib
/systemd
/hugetlb-reserve-pages
<< EOL
201 nodes_path=/sys/devices/system/node/
202 if [ ! -d \$nodes_path ]; then
203 echo "ERROR: \$nodes_path does not exist"
209 echo \$1 > \$nodes_path/\$2/hugepages/hugepages-1048576kB/nr_hugepages
212 # This example reserves all available memory apart from 4 GB for linux
213 # using 1GB size. You can modify it to your needs or comment the lines
214 # to avoid reserve memory in a numa node
216 for f
in /sys
/devices
/system
/node
/node?
/meminfo
218 node
=`head -n1 $f | gawk '($5=="kB"){print $2}'`
219 memory
=`head -n1 $f | gawk '($5=="kB"){print $4}'`
220 memory
=$
((memory
+1048576-1)) #memory must be ceiled
221 memory
=$
((memory
/1048576)) #from `kB to GB
223 [ $memory -gt 4 ] && echo "reserve_pages $((memory-4)) node$node" >> /usr
/lib
/systemd
/hugetlb-reserve-pages
226 # Run the following commands to enable huge pages early boot reservation:
227 chmod +x
/usr
/lib
/systemd
/hugetlb-reserve-pages
228 systemctl
enable hugetlb-gigantic-pages
231 # Prepares the text to add at the end of the grub line, including blacklisting ixgbevf driver in the host
232 memtotal
=`grep MemTotal /proc/meminfo | awk '{ print $2 }' `
233 hpages
=$
(( ($memtotal/(1024*1024))-8 ))
235 memtotal
=$
((memtotal
+1048576-1)) #memory must be ceiled
236 memtotal
=$
((memtotal
/1048576)) #from `kB to GBa
237 hpages
=$
((memtotal-8
))
238 [[ $hpages -lt 0 ]] $$ hpages
=0
241 echo "------> memtotal: $memtotal"
243 textokernel
="intel_iommu=on default_hugepagesz=1G hugepagesz=1G hugepages=$hpages isolcpus=$isolcpus modprobe.blacklist=ixgbevf modprobe.blacklist=i40evf"
245 echo "Text to kernel: $textokernel"
248 # Add text to the kernel line
249 if ! grep -q "intel_iommu=on default_hugepagesz=1G hugepagesz=1G" /etc
/default
/grub
251 echo ">>>>>>> adding cmdline ${textokernel}"
252 sed -i "/^GRUB_CMDLINE_LINUX_DEFAULT=/s/\"\$/${textokernel}\"/" /etc
/default
/grub
254 #grub2-mkconfig -o /boot/grub2/grub.cfg
259 #################################################################
260 ##### OTHER CONFIGURATION #####
261 #################################################################'
263 # Links the OpenMANO required folder /opt/VNF/images to /var/lib/libvirt/images. The OS installation
264 # should have only a / partition with all possible space available
266 echo " link /opt/VNF/images to /var/lib/libvirt/images"
267 if [ "$user_name" != "" ]
269 #mkdir -p /home/${user_name}/VNF_images
270 #chown -R ${user_name}:admin /home/${user_name}/VNF_images
273 # The orchestator needs to link the images folder
274 rm -f /opt
/VNF
/images
276 ln -s /var
/lib
/libvirt
/images
/opt
/VNF
/images
277 chown
-R ${user_name}:admin
/opt
/VNF
278 chown
-R root
:admin
/var
/lib
/libvirt
/images
279 chmod g
+rwx
/var
/lib
/libvirt
/images
282 #echo "configure Selinux management"
283 #semanage fcontext -a -t virt_image_t "/home/${user_name}/VNF_images(/.*)?"
284 #cat /etc/selinux/targeted/contexts/files/file_contexts.local |grep virt_image
285 #restorecon -R -v /home/${user_name}/VNF_images
287 mkdir
-p /opt
/VNF
/images
288 chmod o
+rx
/opt
/VNF
/images
291 echo "creating local information /opt/VNF/images/hostinfo.yaml"
292 echo "#By default openvim assumes control plane interface naming as em1,em2,em3,em4 " > /opt
/VNF
/images
/hostinfo.yaml
293 echo "#and bridge ifaces as virbrMan1, virbrMan2, ..." >> /opt
/VNF
/images
/hostinfo.yaml
294 echo "#if compute node contain a different name it must be indicated in this file" >> /opt
/VNF
/images
/hostinfo.yaml
295 echo "#with the format extandard-name: compute-name" >> /opt
/VNF
/images
/hostinfo.yaml
296 if [ "$interface" != "" -a "$interface" != "em1" ]
298 echo "iface_names:" >> /opt
/VNF
/images
/hostinfo.yaml
299 echo " em1: ${interface}" >> /opt
/VNF
/images
/hostinfo.yaml
301 chmod o
+r
/opt
/VNF
/images
/hostinfo.yaml
303 # deactivate memory overcommit
304 #echo "deactivate memory overcommit"
305 #service ksmtuned stop
307 #chkconfig ksmtuned off
311 # Libvirt options (uncomment the following)
312 echo "configure Libvirt options"
313 sed -i 's/#unix_sock_group = "libvirt"/unix_sock_group = "libvirt"/' /etc
/libvirt
/libvirtd.conf
314 sed -i 's/#unix_sock_rw_perms = "0770"/unix_sock_rw_perms = "0770"/' /etc
/libvirt
/libvirtd.conf
315 sed -i 's/#unix_sock_dir = "\/var\/run\/libvirt"/unix_sock_dir = "\/var\/run\/libvirt"/' /etc
/libvirt
/libvirtd.conf
316 sed -i 's/#auth_unix_rw = "none"/auth_unix_rw = "none"/' /etc
/libvirt
/libvirtd.conf
320 #################################################################
321 ##### NETWORK CONFIGURATION #####
322 #################################################################'
323 # Network config (if the second parameter is net)
324 echo "Interface ==> $interface"
325 if [ -n "$interface" ]
329 # For management and data interfaces
330 rm -f /etc
/udev
/rules.d
/pci_config.rules
# it will be created to define VFs
333 # Set ONBOOT=on and MTU=9000 on the interface used for the bridges
334 echo "configuring iface $interface"
336 #MTU for interfaces and bridges
339 cp /etc
/network
/interfaces interfaces.tmp
342 #Create infrastructure bridge, normally used for connecting to compute nodes, openflow controller, ...
345 #Create VLAN for infrastructure bridge
348 ######### CUTLINE #########
351 iface ${interface} inet static
354 auto ${interface}.1001
355 iface ${interface}.1001 inet static
359 echo "ifconfig ${interface} mtu $MTU
360 ifconfig ${interface} up
364 #Create bridge interfaces
365 echo "Creating bridge ifaces: "
366 for ((i
=1;i
<=20;i
++))
369 [ $i -lt 10 ] && i2digits
="0$i"
370 echo " virbrMan$i vlan 20$i2digits"
375 auto ${interface}.20$i2digits
376 iface ${interface}.20$i2digits inet static
380 iface virbrMan$j inet static
381 bridge_ports ${interface}.20$i2digits
385 echo "ifconfig ${interface}.20$i2digits mtu $MTU
386 ifconfig virbrMan$j mtu $MTU
387 ifconfig virbrMan$j up
394 iface em2.1001 inet static
397 iface virbrInf inet static
398 bridge_ports em2.1001
401 echo "ifconfig em2.1001 mtu $MTU
402 ifconfig virbrInf mtu $MTU
406 if ! grep -q "#### CUTLINE ####" /etc
/network
/interfaces
408 echo "====== Copying interfaces.tmp to /etc/network/interfaces"
409 cp interfaces.tmp
/etc
/network
/interfaces
417 # Activate 8 Virtual Functions per PF on Niantic cards (ixgbe driver)
418 if [[ `lsmod | cut -d" " -f1 | grep "ixgbe" | grep -v vf` ]]
420 if ! grep -q "ixgbe" /etc
/modprobe.d
/ixgbe.conf
422 echo "options ixgbe max_vfs=8" >> /etc
/modprobe.d
/ixgbe.conf
429 echo "sleep 10" >> mtu.tmp
431 interfaces
=`ifconfig -a | grep ^p | cut -d " " -f 1`
432 for ph
in $interfaces
434 echo "ifconfig $ph mtu $MTU" >> mtu.tmp
435 echo "ifconfig $ph up" >> mtu.tmp
440 cp mtu.tmp
/etc
/setmtu.sh
441 chmod +x
/etc
/setmtu.sh
443 # To define 8 VFs per PF we do it on rc.local, because the driver needs to be unloaded and loaded again
444 #if ! grep -q "NFV" /etc/rc.local
448 echo "# NFV" >> /etc
/rc.
local
449 echo "modprobe -r ixgbe" >> /etc
/rc.
local
450 echo "modprobe ixgbe max_vfs=8" >> /etc
/rc.
local
451 echo "/etc/setmtu.sh" >> /etc
/rc.
local
453 exit 0" >> /etc
/rc.
local
454 echo "" >> /etc
/rc.
local
456 chmod +x
/etc
/rc.d
/rc.
local
460 chmod a
+rwx
/var
/lib
/libvirt
/images
463 ln -s /usr
/bin
/qemu-system-x86_64 qemu-kvm
466 #Deactivating apparmor while looking for a better solution
467 /etc
/init.d
/apparmor stop
468 update-rc.d
-f apparmor remove
471 echo "Do not forget to create a shared (NFS, Samba, ...) where original virtual machine images are allocated"
473 echo "Do not forget to copy the public ssh key into /home/${user_name}/.ssh/authorized_keys for authomatic login from openvim controller"
476 echo "Reboot the system to make the changes effective"