4 # Copyright 2015 Telefónica Investigación y Desarrollo, S.A.U.
5 # This file is part of openvim
8 # Licensed under the Apache License, Version 2.0 (the "License"); you may
9 # not use this file except in compliance with the License. You may obtain
10 # a copy of the License at
12 # http://www.apache.org/licenses/LICENSE-2.0
14 # Unless required by applicable law or agreed to in writing, software
15 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
16 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
17 # License for the specific language governing permissions and limitations
20 # For those usages not covered by the Apache License, Version 2.0 please
21 # contact with: nfvlabs@tid.es
24 # Authors: Antonio Lopez, Pablo Montes, Alfonso Tierno
27 # Personalize RHEL7.1 on compute nodes
28 # Prepared to work with the following network card drivers:
29 # tg3, igb drivers for management interfaces
30 # ixgbe (Intel Niantic) and i40e (Intel Fortville) drivers for data plane interfaces
33 # wget https://raw.githubusercontent.com/nfvlabs/openmano/master/scripts/configure-compute-node-UBUNTU14.04.sh
35 # chmod +x ./configure-compute-node-UBUNTU14.04.sh
36 # sudo ./configure-compute-node-RHEL7.1.sh <user> <iface>
39 # All virtualization options activated on BIOS (vt-d, vt-x, SR-IOV, no power savings...)
40 # RHEL7.1 installed without /home partition and with the following packages selection:
41 # @base, @core, @development, @network-file-system-client, @virtualization-hypervisor, @virtualization-platform, @virtualization-tools
45 echo -e "Usage: sudo $0 [-y] <user-name> [ <iface-name> [<ip-address>|dhcp] ]"
46 echo -e " Configure compute host for VIM usage. (version 0.4). Params:"
47 echo -e " -y do not prompt for confirmation. If a new user is created, the user name is set as password"
48 echo -e " <user-name> Create if not exist and configure this user for openvim to connect"
49 echo -e " <iface-name> if suplied creates bridge interfaces on this interface, needed for openvim"
50 echo -e " ip or dhcp if suplied, configure the interface with this ip address (/24) or 'dhcp' "
54 #1 CHECK input parameters
56 [ "$USER" != "root" ] && echo "Needed root privileges" && usage
&& exit -1
60 while getopts "y" o
; do
86 if [ -n "$interface" ] && ! ifconfig
$interface &> /dev
/null
88 echo "Error: interface '$interface' is not present in the system"
94 #################################################################
95 ##### INSTALL NEEDED PACKETS #####
96 #################################################################'
100 #apt-get -y install grub-common screen virt-manager ethtool build-essential x11-common x11-utils x11-apps libguestfs-tools hwloc libguestfs-tools numactl vlan nfs-common nfs-kernel-server
101 apt-get
-y install grub-common screen virt-manager ethtool build-essential x11-common x11-utils libguestfs-tools hwloc libguestfs-tools numactl vlan nfs-common nfs-kernel-server
103 echo "Remove unneeded packages....."
104 apt-get
-y autoremove
106 #yum install -y policycoreutils-python
111 #################################################################
112 ##### INSTALL USER #####
113 #################################################################'
115 # Add required groups
117 groupadd
-f libvirt
#for other operating systems may be libvirtd
119 # Adds user, default password same as name
120 if grep -q "^${user_name}:" /etc
/passwd
122 #user exist, add to group
123 echo "adding user ${user_name} to groups libvirt,admin"
124 usermod
-a -G libvirt
,admin
-g admin
$user_name
126 #create user if it does not exist
127 [ -z "$FORCE" ] && read -p "user '${user_name}' does not exist, create (Y/n)" kk
128 if ! [ -z "$kk" -o "$kk"="y" -o "$kk"="Y" ]
132 echo "creating and configuring user {user_name}"
133 useradd
-m -G libvirt
,admin
-g admin
$user_name
137 echo "Provide a password for $user_name"
140 echo -e "$user_name\n$user_name" | passwd
--stdin $user_name
144 ## Allow admin users to access without password
145 #if ! grep -q "#openmano" /etc/sudoers
147 # cat >> /home/${user_name}/script_visudo.sh << EOL
149 #cat \$1 | awk '(\$0~"requiretty"){print "#"\$0}(\$0!~"requiretty"){print \$0}' > tmp
153 #echo "#openmano allow to group admin to grant root privileges without password" >> \$1
154 #echo "%admin ALL=(ALL) NOPASSWD: ALL" >> \$1
156 # chmod +x /home/${user_name}/script_visudo.sh
157 # echo "allowing admin user to get root privileges withut password"
158 # export EDITOR=/home/${user_name}/script_visudo.sh && sudo -E visudo
159 # rm -f /home/${user_name}/script_visudo.sh
164 #################################################################
165 ##### INSTALL HUGEPAGES ISOLCPUS GRUB #####
166 #################################################################'
168 # Huge pages 1G auto mount
170 if ! grep -q "Huge pages" /etc
/fstab
172 echo "" >> /etc
/fstab
173 echo "# Huge pages" >> /etc
/fstab
174 echo "nodev /mnt/huge hugetlbfs pagesize=1GB 0 0" >> /etc
/fstab
175 echo "" >> /etc
/fstab
178 # Huge pages reservation service
179 if ! [ -f /lib
/systemd
/system
/hugetlb-gigantic-pages.service
]
181 echo "configuring huge pages service"
183 Description=HugeTLB Gigantic Pages Reservation
184 DefaultDependencies=no
185 Before=dev-hugepages.mount
186 ConditionPathExists=/sys/devices/system/node
187 ConditionKernelCommandLine=hugepagesz=1G
192 ExecStart=/usr/lib/systemd/hugetlb-reserve-pages
195 WantedBy=sysinit.target" >> /lib
/systemd
/system
/hugetlb-gigantic-pages.service
197 # cat > /usr/lib/systemd/system/hugetlb-gigantic-pages.service << EOL
199 #Description=HugeTLB Gigantic Pages Reservation
200 #DefaultDependencies=no
201 #Before=dev-hugepages.mount
202 #ConditionPathExists=/sys/devices/system/node
203 #ConditionKernelCommandLine=hugepagesz=1G
208 #ExecStart=/usr/lib/systemd/hugetlb-reserve-pages
211 #WantedBy=sysinit.target
217 isolcpus
=`gawk 'BEGIN{pre=-2;}
218 ($1=="processor"){pro=$3;}
219 ($1=="core" && $4!=0){
220 if (pre+1==pro){endrange="-" pro}
221 else{cpus=cpus endrange sep pro; sep=","; endrange="";};
223 END{printf("%s",cpus endrange);}' /proc/cpuinfo`
226 echo "CPUS: $isolcpus"
228 # Huge pages reservation file: reserving all memory apart from 4GB per NUMA node
229 # Get the number of hugepages: all memory but 8GB reserved for the OS
230 #totalmem=`dmidecode --type 17|grep Size |grep MB |gawk '{suma+=$2} END {print suma/1024}'`
231 #hugepages=$(($totalmem-8))
233 if ! [ -f /usr
/lib
/systemd
/hugetlb-reserve-pages
]
235 cat > /usr
/lib
/systemd
/hugetlb-reserve-pages
<< EOL
237 nodes_path=/sys/devices/system/node/
238 if [ ! -d \$nodes_path ]; then
239 echo "ERROR: \$nodes_path does not exist"
245 echo \$1 > \$nodes_path/\$2/hugepages/hugepages-1048576kB/nr_hugepages
248 # This example reserves all available memory apart from 4 GB for linux
249 # using 1GB size. You can modify it to your needs or comment the lines
250 # to avoid reserve memory in a numa node
252 for f
in /sys
/devices
/system
/node
/node?
/meminfo
254 node
=`head -n1 $f | gawk '($5=="kB"){print $2}'`
255 memory
=`head -n1 $f | gawk '($5=="kB"){print $4}'`
256 memory
=$
((memory
+1048576-1)) #memory must be ceiled
257 memory
=$
((memory
/1048576)) #from `kB to GB
259 [ $memory -gt 4 ] && echo "reserve_pages $((memory-4)) node$node" >> /usr
/lib
/systemd
/hugetlb-reserve-pages
262 # Run the following commands to enable huge pages early boot reservation:
263 chmod +x
/usr
/lib
/systemd
/hugetlb-reserve-pages
264 systemctl
enable hugetlb-gigantic-pages
267 # Prepares the text to add at the end of the grub line, including blacklisting ixgbevf driver in the host
268 memtotal
=`grep MemTotal /proc/meminfo | awk '{ print $2 }' `
269 hpages
=$
(( ($memtotal/(1024*1024))-8 ))
271 memtotal
=$
((memtotal
+1048576-1)) #memory must be ceiled
272 memtotal
=$
((memtotal
/1048576)) #from `kB to GBa
273 hpages
=$
((memtotal-8
))
274 [[ $hpages -lt 0 ]] && hpages
=0
277 echo "------> memtotal: $memtotal"
279 textokernel
=" intel_iommu=on default_hugepagesz=1G hugepagesz=1G hugepages=$hpages isolcpus=$isolcpus modprobe.blacklist=ixgbevf modprobe.blacklist=i40evf"
281 echo "Text to kernel: $textokernel"
284 # Add text to the kernel line
285 if ! grep -q " intel_iommu=on default_hugepagesz=1G hugepagesz=1G" /etc
/default
/grub
287 echo ">>>>>>> adding cmdline ${textokernel}"
288 sed -i "/^GRUB_CMDLINE_LINUX_DEFAULT=/s/\"\$/${textokernel}\"/" /etc
/default
/grub
289 sed -i "/^GRUB_CMDLINE_LINUX=/s/\"\$/${textokernel}\"/" /etc
/default
/grub
293 grub-mkconfig
-o /boot
/efi
/EFI
/ubuntu
/grub.cfg
295 grub-mkconfig
-o /boot
/grub
/grub.cfg
299 #################################################################
300 ##### OTHER CONFIGURATION #####
301 #################################################################'
303 # Links the OpenMANO required folder /opt/VNF/images to /var/lib/libvirt/images. The OS installation
304 # should have only a / partition with all possible space available
306 echo " link /opt/VNF/images to /var/lib/libvirt/images"
307 if [ "$user_name" != "" ]
309 #mkdir -p /home/${user_name}/VNF_images
310 #chown -R ${user_name}:admin /home/${user_name}/VNF_images
313 # The orchestator needs to link the images folder
314 rm -f /opt
/VNF
/images
316 ln -s /var
/lib
/libvirt
/images
/opt
/VNF
/images
317 chown
-R ${user_name}:admin
/opt
/VNF
318 chown
-R root
:admin
/var
/lib
/libvirt
/images
319 chmod g
+rwx
/var
/lib
/libvirt
/images
322 #echo "configure Selinux management"
323 #semanage fcontext -a -t virt_image_t "/home/${user_name}/VNF_images(/.*)?"
324 #cat /etc/selinux/targeted/contexts/files/file_contexts.local |grep virt_image
325 #restorecon -R -v /home/${user_name}/VNF_images
327 mkdir
-p /opt
/VNF
/images
328 chmod o
+rx
/opt
/VNF
/images
331 echo "creating local information /opt/VNF/images/hostinfo.yaml"
332 echo "#By default openvim assumes control plane interface naming as em1,em2,em3,em4 " > /opt
/VNF
/images
/hostinfo.yaml
333 echo "#and bridge ifaces as virbrMan1, virbrMan2, ..." >> /opt
/VNF
/images
/hostinfo.yaml
334 echo "#if compute node contain a different name it must be indicated in this file" >> /opt
/VNF
/images
/hostinfo.yaml
335 echo "#with the format extandard-name: compute-name" >> /opt
/VNF
/images
/hostinfo.yaml
336 if [ "$interface" != "" -a "$interface" != "em1" ]
338 echo "iface_names:" >> /opt
/VNF
/images
/hostinfo.yaml
339 echo " em1: ${interface}" >> /opt
/VNF
/images
/hostinfo.yaml
341 chmod o
+r
/opt
/VNF
/images
/hostinfo.yaml
343 # deactivate memory overcommit
344 #echo "deactivate memory overcommit"
345 #service ksmtuned stop
347 #chkconfig ksmtuned off
351 # Libvirt options (uncomment the following)
352 echo "configure Libvirt options"
353 sed -i 's/#unix_sock_group = "libvirt"/unix_sock_group = "libvirt"/' /etc
/libvirt
/libvirtd.conf
354 sed -i 's/#unix_sock_rw_perms = "0770"/unix_sock_rw_perms = "0770"/' /etc
/libvirt
/libvirtd.conf
355 sed -i 's/#unix_sock_dir = "\/var\/run\/libvirt"/unix_sock_dir = "\/var\/run\/libvirt"/' /etc
/libvirt
/libvirtd.conf
356 sed -i 's/#auth_unix_rw = "none"/auth_unix_rw = "none"/' /etc
/libvirt
/libvirtd.conf
360 #################################################################
361 ##### NETWORK CONFIGURATION #####
362 #################################################################'
363 # Network config (if the second parameter is net)
364 echo "Interface ==> $interface"
365 if [ -n "$interface" ]
369 # For management and data interfaces
370 rm -f /etc
/udev
/rules.d
/pci_config.rules
# it will be created to define VFs
373 # Set ONBOOT=on and MTU=9000 on the interface used for the bridges
374 echo "configuring iface $interface"
376 #MTU for interfaces and bridges
379 cp /etc
/network
/interfaces interfaces.tmp
382 #Create infrastructure bridge, normally used for connecting to compute nodes, openflow controller, ...
385 #Create VLAN for infrastructure bridge
388 ######### CUTLINE #########
391 iface ${interface} inet manual
392 post-up ip link set dev ${interface} mtu ${MTU}
394 auto ${interface}.1001
395 iface ${interface}.1001 inet manual
396 vlan-raw-device ${interface}
397 post-up ip link set mtu $MTU dev ${interface}.1001
400 # echo "ifconfig ${interface} mtu $MTU
401 # ifconfig ${interface} up
405 #Create bridge interfaces
406 echo "Creating bridge ifaces: "
407 for ((i
=1;i
<=20;i
++))
410 [ $i -lt 10 ] && i2digits
="0$i"
411 echo " virbrMan$i vlan 20$i2digits"
416 auto ${interface}.20$i2digits
417 iface ${interface}.20$i2digits inet manual
418 post-up ip link set mtu $MTU dev ${interface}.20$i2digits
421 iface virbrMan$j inet manual
422 bridge_ports ${interface}.20$i2digits
423 post-up ip link set dev virbrMan$j && ip link set mtu 9000 dev virbrMan$j
426 # echo "ifconfig ${interface}.20$i2digits mtu $MTU
427 #ifconfig virbrMan$j mtu $MTU
428 #ifconfig virbrMan$j up
435 iface em2.1001 inet manual
436 post-up ip link set dev em2.1001 mtu 9000
439 iface virbrInf inet manual
440 bridge_ports em2.1001
441 post-up ip link set dev virbrInf && ip link set mtu 9000 dev virbrInf
444 # echo "ifconfig em2.1001 mtu $MTU
445 #ifconfig virbrInf mtu $MTU
446 #ifconfig virbrInf up
449 if ! grep -q "#### CUTLINE ####" /etc
/network
/interfaces
451 echo "====== Copying interfaces.tmp to /etc/network/interfaces"
452 cp interfaces.tmp
/etc
/network
/interfaces
460 # Activate 8 Virtual Functions per PF on Niantic cards (ixgbe driver)
461 if [[ `lsmod | cut -d" " -f1 | grep "ixgbe" | grep -v vf` ]]
463 if ! grep -q "ixgbe" /etc
/modprobe.d
/ixgbe.conf
465 echo "options ixgbe max_vfs=8" >> /etc
/modprobe.d
/ixgbe.conf
470 echo "#!/bin/bash" > /etc
/activate-vfs.sh
471 chmod +x
/etc
/activate-vfs.sh
472 for iface
in `ip -o link show | awk -F': ' '{print $2}' | grep -v -e "\." -e "lo" -e "virbr" -e "tap"`
474 # 10/40 Gbps interfaces
475 # Intel X520 cards: driver ixgbe
476 # Intel XL710 Fortville cards: driver i40e
477 driver
=`ethtool -i $iface| awk '($0~"driver"){print $2}'`
478 if [ "$driver" = "i40e" -o "$driver" = "ixgbe" ]
480 echo "configuring dataplane iface $iface"
482 # Create 8 SR-IOV per PF by udev rules only for Fortville cards (i40e driver)
483 if [ "$driver" = "i40e" ]
485 pci
=`ethtool -i $iface | awk '($0~"bus-info"){print $2}'`
486 echo "echo 8 > /sys/bus/pci/devices/$pci/sriov_numvfs" >> /etc
/activate-vfs.sh
493 #echo "sleep 10" >> mtu.tmp
495 #interfaces=`ifconfig -a | grep ^p | cut -d " " -f 1`
496 #for ph in $interfaces
498 # echo "ifconfig $ph mtu $MTU" >> mtu.tmp
499 # echo "ifconfig $ph up" >> mtu.tmp
504 #cp mtu.tmp /etc/setmtu.sh
505 #chmod +x /etc/setmtu.sh
507 # To define 8 VFs per PF we do it on rc.local, because the driver needs to be unloaded and loaded again
508 #if ! grep -q "NFV" /etc/rc.local
512 echo "# NFV" >> /etc
/rc.
local
513 echo "modprobe -r ixgbe" >> /etc
/rc.
local
514 echo "modprobe ixgbe max_vfs=8" >> /etc
/rc.
local
515 echo "/etc/activate-vfs.sh" >> /etc
/rc.
local
516 #echo "/etc/setmtu.sh" >> /etc/rc.local
518 exit 0" >> /etc
/rc.
local
519 echo "" >> /etc
/rc.
local
521 chmod +x
/etc
/rc.
local
522 chmod +x
/etc
/activate-vfs.sh
526 chmod a
+rwx
/var
/lib
/libvirt
/images
529 ln -s /usr
/bin
/qemu-system-x86_64 qemu-kvm
532 #Deactivating apparmor while looking for a better solution
533 /etc
/init.d
/apparmor stop
534 update-rc.d
-f apparmor remove
537 echo "Do not forget to create a shared (NFS, Samba, ...) where original virtual machine images are allocated"
539 echo "Do not forget to copy the public ssh key into /home/${user_name}/.ssh/authorized_keys for authomatic login from openvim controller"
542 echo "Reboot the system to make the changes effective"