4 # Copyright 2015 Telefónica Investigación y Desarrollo, S.A.U.
5 # This file is part of openvim
8 # Licensed under the Apache License, Version 2.0 (the "License"); you may
9 # not use this file except in compliance with the License. You may obtain
10 # a copy of the License at
12 # http://www.apache.org/licenses/LICENSE-2.0
14 # Unless required by applicable law or agreed to in writing, software
15 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
16 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
17 # License for the specific language governing permissions and limitations
20 # For those usages not covered by the Apache License, Version 2.0 please
21 # contact with: nfvlabs@tid.es
24 # Authors: Antonio Lopez, Pablo Montes, Alfonso Tierno, Leonardo Mirabal
27 # Personalize RHEL7.1 on compute nodes
28 # Prepared to work with the following network card drivers:
29 # tg3, igb drivers for management interfaces
30 # ixgbe (Intel Niantic) and i40e (Intel Fortville) drivers for data plane interfaces
33 # wget https://raw.githubusercontent.com/nfvlabs/openmano/master/scripts/configure-compute-node-UBUNTU14.04.sh
35 # chmod +x ./configure-compute-node-UBUNTU14.04.sh
36 # sudo ./configure-compute-node-RHEL7.1.sh <user> <iface>
39 # All virtualization options activated on BIOS (vt-d, vt-x, SR-IOV, no power savings...)
40 # RHEL7.1 installed without /home partition and with the following packages selection:
41 # @base, @core, @development, @network-file-system-client, @virtualization-hypervisor, @virtualization-platform, @virtualization-tools
46 echo -e "Usage: sudo $0 [-f] <user-name> [<iface-for-overlay-bridges>]"
47 echo -e " Configure compute host for VIM usage. (version 0.4). Params:"
48 echo -e " -f do not prompt for confirmation. If a new user is created, the user name is set as password"
49 echo -e " <user-name> Create if not exist and configure this user for openvim to connect"
50 echo -e " [<iface-for-overlay-bridges>] Only needed for old openvim versions. Iface to create pre-provioned bridges for network conectivity"
55 #1 CHECK input parameters
57 [ "$USER" != "root" ] && echo "Needed root privileges" && usage
&& exit -1
61 while getopts "f" o
; do
87 if [ -z $interface ] && ! ifconfig
$interface &> /dev
/null
89 echo "Error: interface '$interface' is not present in the system"
95 #################################################################
96 ##### INSTALL NEEDED PACKETS #####
97 #################################################################'
101 apt-get
-y install grub-common screen virt-manager ethtool build-essential x11-common x11-utils \
102 libguestfs-tools hwloc libguestfs-tools numactl vlan nfs-common nfs-kernel-server openvswitch-switch
103 echo "Remove unneeded packages....."
104 apt-get
-y autoremove
106 #yum install -y policycoreutils-python
109 #################################################################
110 ##### INSTALL USER #####
111 #################################################################'
113 # Add required groups
115 groupadd
-f libvirt
#for other operating systems may be libvirtd
117 # Adds user, default password same as name
118 if grep -q "^${user_name}:" /etc
/passwd
120 #user exist, add to group
121 echo "adding user ${user_name} to groups libvirt,admin"
122 usermod
-a -G libvirt
,admin
-g admin
$user_name
124 #create user if it does not exist
125 [ -z "$FORCE" ] && read -p "user '${user_name}' does not exist, create (Y/n)" kk
126 if ! [ -z "$kk" -o "$kk"="y" -o "$kk"="Y" ]
130 echo "creating and configuring user {user_name}"
131 useradd
-m -G libvirt
,admin
-g admin
$user_name
135 echo "Provide a password for $user_name"
138 echo -e "$user_name\n$user_name" | passwd
--stdin $user_name
142 # Allow admin users to access without password
143 if ! grep -q "#openmano" /etc
/sudoers
145 cat >> /home
/${user_name}/script_visudo.sh
<< EOL
147 echo "#openmano allow to group admin to grant root privileges without password" >> \$1
148 echo "${user_name} ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
150 chmod +x
/home
/${user_name}/script_visudo.sh
151 echo "allowing admin user to get root privileges withut password"
152 export EDITOR
=/home
/${user_name}/script_visudo.sh
&& sudo
-E visudo
153 rm -f /home
/${user_name}/script_visudo.sh
157 #################################################################
158 ##### INSTALL HUGEPAGES ISOLCPUS GRUB #####
159 #################################################################'
161 # Huge pages 1G auto mount
163 if ! grep -q "Huge pages" /etc
/fstab
165 echo "" >> /etc
/fstab
166 echo "# Huge pages" >> /etc
/fstab
167 echo "nodev /mnt/huge hugetlbfs pagesize=1GB 0 0" >> /etc
/fstab
168 echo "" >> /etc
/fstab
171 # Huge pages reservation service
172 if ! [ -f /lib
/systemd
/system
/hugetlb-gigantic-pages.service
]
174 echo "configuring huge pages service"
176 Description=HugeTLB Gigantic Pages Reservation
177 DefaultDependencies=no
178 Before=dev-hugepages.mount
179 ConditionPathExists=/sys/devices/system/node
180 ConditionKernelCommandLine=hugepagesz=1G
185 ExecStart=/usr/lib/systemd/hugetlb-reserve-pages
188 WantedBy=sysinit.target" >> /lib
/systemd
/system
/hugetlb-gigantic-pages.service
190 # cat > /usr/lib/systemd/system/hugetlb-gigantic-pages.service << EOL
192 #Description=HugeTLB Gigantic Pages Reservation
193 #DefaultDependencies=no
194 #Before=dev-hugepages.mount
195 #ConditionPathExists=/sys/devices/system/node
196 #ConditionKernelCommandLine=hugepagesz=1G
201 #ExecStart=/usr/lib/systemd/hugetlb-reserve-pages
204 #WantedBy=sysinit.target
210 isolcpus
=`gawk 'BEGIN{pre=-2;}
211 ($1=="processor"){pro=$3;}
212 ($1=="core" && $4!=0){
213 if (pre+1==pro){endrange="-" pro}
214 else{cpus=cpus endrange sep pro; sep=","; endrange="";};
216 END{printf("%s",cpus endrange);}' /proc/cpuinfo`
219 echo "CPUS: $isolcpus"
221 # Huge pages reservation file: reserving all memory apart from 4GB per NUMA node
222 # Get the number of hugepages: all memory but 8GB reserved for the OS
223 #totalmem=`dmidecode --type 17|grep Size |grep MB |gawk '{suma+=$2} END {print suma/1024}'`
224 #hugepages=$(($totalmem-8))
226 if ! [ -f /usr
/lib
/systemd
/hugetlb-reserve-pages
]
228 cat > /usr
/lib
/systemd
/hugetlb-reserve-pages
<< EOL
230 nodes_path=/sys/devices/system/node/
231 if [ ! -d \$nodes_path ]; then
232 echo "ERROR: \$nodes_path does not exist"
238 echo \$1 > \$nodes_path/\$2/hugepages/hugepages-1048576kB/nr_hugepages
241 # This example reserves all available memory apart from 4 GB for linux
242 # using 1GB size. You can modify it to your needs or comment the lines
243 # to avoid reserve memory in a numa node
245 for f
in /sys
/devices
/system
/node
/node?
/meminfo
247 node
=`head -n1 $f | gawk '($5=="kB"){print $2}'`
248 memory
=`head -n1 $f | gawk '($5=="kB"){print $4}'`
249 memory
=$
((memory
+1048576-1)) #memory must be ceiled
250 memory
=$
((memory
/1048576)) #from `kB to GB
252 [ $memory -gt 4 ] && echo "reserve_pages $((memory-4)) node$node" >> /usr
/lib
/systemd
/hugetlb-reserve-pages
255 # Run the following commands to enable huge pages early boot reservation:
256 chmod +x
/usr
/lib
/systemd
/hugetlb-reserve-pages
257 systemctl
enable hugetlb-gigantic-pages
260 # Prepares the text to add at the end of the grub line, including blacklisting ixgbevf driver in the host
261 memtotal
=`grep MemTotal /proc/meminfo | awk '{ print $2 }' `
262 hpages
=$
(( ($memtotal/(1024*1024))-8 ))
264 memtotal
=$
((memtotal
+1048576-1)) #memory must be ceiled
265 memtotal
=$
((memtotal
/1048576)) #from `kB to GBa
266 hpages
=$
((memtotal-8
))
267 [[ $hpages -lt 0 ]] && hpages
=0
270 echo "------> memtotal: $memtotal"
272 textokernel
=" intel_iommu=on default_hugepagesz=1G hugepagesz=1G hugepages=$hpages isolcpus=$isolcpus modprobe.blacklist=ixgbevf modprobe.blacklist=i40evf"
274 echo "Text to kernel: $textokernel"
277 # Add text to the kernel line
278 if ! grep -q " intel_iommu=on default_hugepagesz=1G hugepagesz=1G" /etc
/default
/grub
280 echo ">>>>>>> adding cmdline ${textokernel}"
281 sed -i "/^GRUB_CMDLINE_LINUX_DEFAULT=/s/\"\$/${textokernel}\"/" /etc
/default
/grub
282 sed -i "/^GRUB_CMDLINE_LINUX=/s/\"\$/${textokernel}\"/" /etc
/default
/grub
286 grub-mkconfig
-o /boot
/efi
/EFI
/ubuntu
/grub.cfg
288 grub-mkconfig
-o /boot
/grub
/grub.cfg
292 #################################################################
293 ##### OTHER CONFIGURATION #####
294 #################################################################'
296 # Links the OpenMANO required folder /opt/VNF/images to /var/lib/libvirt/images. The OS installation
297 # should have only a / partition with all possible space available
299 echo " link /opt/VNF/images to /var/lib/libvirt/images"
300 if [ "$user_name" != "" ]
302 #mkdir -p /home/${user_name}/VNF_images
303 #chown -R ${user_name}:admin /home/${user_name}/VNF_images
306 # The orchestator needs to link the images folder
307 rm -f /opt
/VNF
/images
309 ln -s /var
/lib
/libvirt
/images
/opt
/VNF
/images
310 chown
-R ${user_name}:admin
/opt
/VNF
311 chown
-R root
:admin
/var
/lib
/libvirt
/images
312 chmod g
+rwx
/var
/lib
/libvirt
/images
315 #echo "configure Selinux management"
316 #semanage fcontext -a -t virt_image_t "/home/${user_name}/VNF_images(/.*)?"
317 #cat /etc/selinux/targeted/contexts/files/file_contexts.local |grep virt_image
318 #restorecon -R -v /home/${user_name}/VNF_images
320 mkdir
-p /opt
/VNF
/images
321 chmod o
+rx
/opt
/VNF
/images
324 echo "creating local information /opt/VNF/images/hostinfo.yaml"
325 echo "#By default openvim assumes control plane interface naming as em1,em2,em3,em4 " > /opt
/VNF
/images
/hostinfo.yaml
326 echo "#and bridge ifaces as virbrMan1, virbrMan2, ..." >> /opt
/VNF
/images
/hostinfo.yaml
327 echo "#if compute node contain a different name it must be indicated in this file" >> /opt
/VNF
/images
/hostinfo.yaml
328 echo "#with the format extandard-name: compute-name" >> /opt
/VNF
/images
/hostinfo.yaml
329 if [ "$interface" != "" -a "$interface" != "em1" ]
331 echo "iface_names:" >> /opt
/VNF
/images
/hostinfo.yaml
332 echo " em1: ${interface}" >> /opt
/VNF
/images
/hostinfo.yaml
334 chmod o
+r
/opt
/VNF
/images
/hostinfo.yaml
336 # deactivate memory overcommit
337 #echo "deactivate memory overcommit"
338 #service ksmtuned stop
340 #chkconfig ksmtuned off
344 # Libvirt options (uncomment the following)
345 echo "configure Libvirt options"
346 sed -i 's/#unix_sock_group = "libvirt"/unix_sock_group = "libvirt"/' /etc
/libvirt
/libvirtd.conf
347 sed -i 's/#unix_sock_rw_perms = "0770"/unix_sock_rw_perms = "0770"/' /etc
/libvirt
/libvirtd.conf
348 sed -i 's/#unix_sock_dir = "\/var\/run\/libvirt"/unix_sock_dir = "\/var\/run\/libvirt"/' /etc
/libvirt
/libvirtd.conf
349 sed -i 's/#auth_unix_rw = "none"/auth_unix_rw = "none"/' /etc
/libvirt
/libvirtd.conf
353 #################################################################
354 ##### NETWORK CONFIGURATION #####
355 #################################################################'
356 # Network config (if the second parameter is net)
357 echo "Interface ==> $interface"
358 if [ -n "$interface" ]
361 # For management and data interfaces
362 rm -f /etc
/udev
/rules.d
/pci_config.rules
# it will be created to define VFs
365 # Set ONBOOT=on and MTU=9000 on the interface used for the bridges
366 echo "configuring iface $interface"
368 #MTU for interfaces and bridges
371 cp /etc
/network
/interfaces interfaces.tmp
374 #Create infrastructure bridge, normally used for connecting to compute nodes, openflow controller, ...
375 #Create VLAN for infrastructure bridge
378 ######### CUTLINE #########
381 iface ${interface} inet manual
382 post-up ip link set dev ${interface} mtu ${MTU}
384 auto ${interface}.1001
385 iface ${interface}.1001 inet manual
386 vlan-raw-device ${interface}
387 post-up ip link set mtu $MTU dev ${interface}.1001
390 # echo "ifconfig ${interface} mtu $MTU
391 # ifconfig ${interface} up
395 #Create bridge interfaces
396 echo "Creating bridge ifaces: "
397 for ((i
=1;i
<=20;i
++))
400 [ $i -lt 10 ] && i2digits
="0$i"
401 echo " virbrMan$i vlan 20$i2digits"
406 auto ${interface}.20$i2digits
407 iface ${interface}.20$i2digits inet manual
408 vlan-raw-device ${interface}
409 post-up ip link set mtu $MTU dev ${interface}.20$i2digits
412 iface virbrMan$j inet manual
413 bridge_ports ${interface}.20$i2digits
414 post-up ip link set dev virbrMan$j && ip link set mtu $MTU dev virbrMan$j
419 # echo "ifconfig em2.1001 mtu $MTU
420 #ifconfig virbrInf mtu $MTU
421 #ifconfig virbrInf up
424 if ! grep -q "#### CUTLINE ####" /etc
/network
/interfaces
426 echo "====== Copying interfaces.tmp to /etc/network/interfaces"
427 cp interfaces.tmp
/etc
/network
/interfaces
434 ################### Activate 8 Virtual Functions per PF on Niantic cards (ixgbe driver)
435 if [[ `lsmod | cut -d" " -f1 | grep "ixgbe" | grep -v vf` ]]
437 if ! grep -q "ixgbe" /etc
/modprobe.d
/ixgbe.conf
439 echo "options ixgbe max_vfs=8" >> /etc
/modprobe.d
/ixgbe.conf
444 echo "#!/bin/bash" > /etc
/activate-vfs.sh
445 chmod +x
/etc
/activate-vfs.sh
446 for iface
in `ip -o link show | awk -F': ' '{print $2}' | grep -v -e "\." -e "lo" -e "virbr" -e "tap"`
448 # 10/40 Gbps interfaces
449 # Intel X520 cards: driver ixgbe
450 # Intel XL710 Fortville cards: driver i40e
451 driver
=`ethtool -i $iface| awk '($0~"driver"){print $2}'`
452 if [ "$driver" = "i40e" -o "$driver" = "ixgbe" ]
454 echo "configuring dataplane iface $iface"
456 # Create 8 SR-IOV per PF by udev rules only for Fortville cards (i40e driver)
457 if [ "$driver" = "i40e" ]
459 pci
=`ethtool -i $iface | awk '($0~"bus-info"){print $2}'`
460 echo "echo 8 > /sys/bus/pci/devices/$pci/sriov_numvfs" >> /etc
/activate-vfs.sh
467 #echo "sleep 10" >> mtu.tmp
469 #interfaces=`ifconfig -a | grep ^p | cut -d " " -f 1`
470 #for ph in $interfaces
472 # echo "ifconfig $ph mtu $MTU" >> mtu.tmp
473 # echo "ifconfig $ph up" >> mtu.tmp
478 #cp mtu.tmp /etc/setmtu.sh
479 #chmod +x /etc/setmtu.sh
481 # To define 8 VFs per PF we do it on rc.local, because the driver needs to be unloaded and loaded again
482 #if ! grep -q "NFV" /etc/rc.local
486 echo "# NFV" >> /etc
/rc.
local
487 echo "modprobe -r ixgbe" >> /etc
/rc.
local
488 echo "modprobe ixgbe max_vfs=8" >> /etc
/rc.
local
489 echo "/etc/activate-vfs.sh" >> /etc
/rc.
local
490 #echo "/etc/setmtu.sh" >> /etc/rc.local
492 exit 0" >> /etc
/rc.
local
493 echo "" >> /etc
/rc.
local
495 chmod +x
/etc
/rc.
local
496 chmod +x
/etc
/activate-vfs.sh
500 chmod a
+rwx
/var
/lib
/libvirt
/images
501 mkdir
-p /usr
/libexec
/
503 ln -s /usr
/bin
/qemu-system-x86_64 qemu-kvm
506 #Deactivating apparmor while looking for a better solution
507 /etc
/init.d
/apparmor stop
508 update-rc.d
-f apparmor remove
511 echo "Do not forget to create a shared (NFS, Samba, ...) where original virtual machine images are allocated"
513 echo "Do not forget to copy the public ssh key into /home/${user_name}/.ssh/authorized_keys for authomatic login from openvim controller"
516 echo "Reboot the system to make the changes effective"