host_thread.py

   1 # -*- coding: utf-8 -*-
   2
   3 ##
   4 # Copyright 2015 Telefónica Investigación y Desarrollo, S.A.U.
   5 # This file is part of openvim
   6 # All Rights Reserved.
   7 #
   8 # Licensed under the Apache License, Version 2.0 (the "License"); you may
   9 # not use this file except in compliance with the License. You may obtain
  10 # a copy of the License at
  11 #
  12 #         http://www.apache.org/licenses/LICENSE-2.0
  13 #
  14 # Unless required by applicable law or agreed to in writing, software
  15 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  16 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  17 # License for the specific language governing permissions and limitations
  18 # under the License.
  19 #
  20 # For those usages not covered by the Apache License, Version 2.0 please
  21 # contact with: nfvlabs@tid.es
  22 ##
  23
  24 '''
  25 This is thread that interact with the host and the libvirt to manage VM
  26 One thread will be launched per host
  27 '''
  28 __author__ = "Pablo Montes, Alfonso Tierno, Leonardo Mirabal"
  29 __date__ = "$10-jul-2014 12:07:15$"
  30
  31 import json
  32 import yaml
  33 import threading
  34 import time
  35 import Queue
  36 import paramiko
  37 from jsonschema import validate as js_v, exceptions as js_e
  38 #import libvirt
  39 import imp
  40 from vim_schema import localinfo_schema, hostinfo_schema
  41 import random
  42 import os
  43
  44
  45 # from logging import Logger
  46 # import auxiliary_functions as af
  47
  48 # TODO: insert a logging system
  49
  50
  51 # global lvirt_module
  52 # lvirt_module=None  #libvirt module is charged only if not in test mode
  53
  54 class host_thread(threading.Thread):
  55     lvirt_module = None
  56
  57     def __init__(self, name, host, user, db, db_lock, test, image_path, host_id, version, develop_mode,
  58                  develop_bridge_iface):
  59         '''Init a thread.
  60         Arguments:
  61             'id' number of thead
  62             'name' name of thread
  63             'host','user':  host ip or name to manage and user
  64             'db', 'db_lock': database class and lock to use it in exclusion
  65         '''
  66         threading.Thread.__init__(self)
  67         self.name = name
  68         self.host = host
  69         self.user = user
  70         self.db = db
  71         self.db_lock = db_lock
  72         self.test = test
  73
  74         if not test and not host_thread.lvirt_module:
  75             try:
  76                 module_info = imp.find_module("libvirt")
  77                 host_thread.lvirt_module = imp.load_module("libvirt", *module_info)
  78             except (IOError, ImportError) as e:
  79                 raise ImportError("Cannot import python-libvirt. Openvim not properly installed" +str(e))
  80
  81
  82         self.develop_mode = develop_mode
  83         self.develop_bridge_iface = develop_bridge_iface
  84         self.image_path = image_path
  85         self.host_id = host_id
  86         self.version = version
  87
  88         self.xml_level = 0
  89         #self.pending ={}
  90
  91         self.server_status = {} #dictionary with pairs server_uuid:server_status
  92         self.pending_terminate_server =[] #list  with pairs (time,server_uuid) time to send a terminate for a server being destroyed
  93         self.next_update_server_status = 0 #time when must be check servers status
  94
  95         self.hostinfo = None
  96
  97         self.queueLock = threading.Lock()
  98         self.taskQueue = Queue.Queue(2000)
  99         self.ssh_conn = None
 100
 101     def ssh_connect(self):
 102         try:
 103             #Connect SSH
 104             self.ssh_conn = paramiko.SSHClient()
 105             self.ssh_conn.set_missing_host_key_policy(paramiko.AutoAddPolicy())
 106             self.ssh_conn.load_system_host_keys()
 107             self.ssh_conn.connect(self.host, username=self.user, timeout=10) #, None)
 108         except paramiko.ssh_exception.SSHException as e:
 109             text = e.args[0]
 110             print self.name, ": ssh_connect ssh Exception:", text
 111
 112     def load_localinfo(self):
 113         if not self.test:
 114             try:
 115                 #Connect SSH
 116                 self.ssh_connect()
 117
 118                 command = 'mkdir -p ' +  self.image_path
 119                 #print self.name, ': command:', command
 120                 (_, stdout, stderr) = self.ssh_conn.exec_command(command)
 121                 content = stderr.read()
 122                 if len(content) > 0:
 123                     print self.name, ': command:', command, "stderr:", content
 124
 125                 command = 'cat ' +  self.image_path + '/.openvim.yaml'
 126                 #print self.name, ': command:', command
 127                 (_, stdout, stderr) = self.ssh_conn.exec_command(command)
 128                 content = stdout.read()
 129                 if len(content) == 0:
 130                     print self.name, ': command:', command, "stderr:", stderr.read()
 131                     raise paramiko.ssh_exception.SSHException("Error empty file ")
 132                 self.localinfo = yaml.load(content)
 133                 js_v(self.localinfo, localinfo_schema)
 134                 self.localinfo_dirty=False
 135                 if 'server_files' not in self.localinfo:
 136                     self.localinfo['server_files'] = {}
 137                 print self.name, ': localinfo load from host'
 138                 return
 139
 140             except paramiko.ssh_exception.SSHException as e:
 141                 text = e.args[0]
 142                 print self.name, ": load_localinfo ssh Exception:", text
 143             except host_thread.lvirt_module.libvirtError as e:
 144                 text = e.get_error_message()
 145                 print self.name, ": load_localinfo libvirt Exception:", text
 146             except yaml.YAMLError as exc:
 147                 text = ""
 148                 if hasattr(exc, 'problem_mark'):
 149                     mark = exc.problem_mark
 150                     text = " at position: (%s:%s)" % (mark.line+1, mark.column+1)
 151                 print self.name, ": load_localinfo yaml format Exception", text
 152             except js_e.ValidationError as e:
 153                 text = ""
 154                 if len(e.path)>0: text=" at '" + ":".join(map(str, e.path))+"'"
 155                 print self.name, ": load_localinfo format Exception:", text, e.message
 156             except Exception as e:
 157                 text = str(e)
 158                 print self.name, ": load_localinfo Exception:", text
 159
 160         #not loaded, insert a default data and force saving by activating dirty flag
 161         self.localinfo = {'files':{}, 'server_files':{} }
 162         #self.localinfo_dirty=True
 163         self.localinfo_dirty=False
 164
 165     def load_hostinfo(self):
 166         if self.test:
 167             return;
 168         try:
 169             #Connect SSH
 170             self.ssh_connect()
 171
 172
 173             command = 'cat ' +  self.image_path + '/hostinfo.yaml'
 174             #print self.name, ': command:', command
 175             (_, stdout, stderr) = self.ssh_conn.exec_command(command)
 176             content = stdout.read()
 177             if len(content) == 0:
 178                 print self.name, ': command:', command, "stderr:", stderr.read()
 179                 raise paramiko.ssh_exception.SSHException("Error empty file ")
 180             self.hostinfo = yaml.load(content)
 181             js_v(self.hostinfo, hostinfo_schema)
 182             print self.name, ': hostlinfo load from host', self.hostinfo
 183             return
 184
 185         except paramiko.ssh_exception.SSHException as e:
 186             text = e.args[0]
 187             print self.name, ": load_hostinfo ssh Exception:", text
 188         except host_thread.lvirt_module.libvirtError as e:
 189             text = e.get_error_message()
 190             print self.name, ": load_hostinfo libvirt Exception:", text
 191         except yaml.YAMLError as exc:
 192             text = ""
 193             if hasattr(exc, 'problem_mark'):
 194                 mark = exc.problem_mark
 195                 text = " at position: (%s:%s)" % (mark.line+1, mark.column+1)
 196             print self.name, ": load_hostinfo yaml format Exception", text
 197         except js_e.ValidationError as e:
 198             text = ""
 199             if len(e.path)>0: text=" at '" + ":".join(map(str, e.path))+"'"
 200             print self.name, ": load_hostinfo format Exception:", text, e.message
 201         except Exception as e:
 202             text = str(e)
 203             print self.name, ": load_hostinfo Exception:", text
 204
 205         #not loaded, insert a default data
 206         self.hostinfo = None
 207
 208     def save_localinfo(self, tries=3):
 209         if self.test:
 210             self.localinfo_dirty = False
 211             return
 212
 213         while tries>=0:
 214             tries-=1
 215
 216             try:
 217                 command = 'cat > ' +  self.image_path + '/.openvim.yaml'
 218                 print self.name, ': command:', command
 219                 (stdin, _, _) = self.ssh_conn.exec_command(command)
 220                 yaml.safe_dump(self.localinfo, stdin, explicit_start=True, indent=4, default_flow_style=False, tags=False, encoding='utf-8', allow_unicode=True)
 221                 self.localinfo_dirty = False
 222                 break #while tries
 223
 224             except paramiko.ssh_exception.SSHException as e:
 225                 text = e.args[0]
 226                 print self.name, ": save_localinfo ssh Exception:", text
 227                 if "SSH session not active" in text:
 228                     self.ssh_connect()
 229             except host_thread.lvirt_module.libvirtError as e:
 230                 text = e.get_error_message()
 231                 print self.name, ": save_localinfo libvirt Exception:", text
 232             except yaml.YAMLError as exc:
 233                 text = ""
 234                 if hasattr(exc, 'problem_mark'):
 235                     mark = exc.problem_mark
 236                     text = " at position: (%s:%s)" % (mark.line+1, mark.column+1)
 237                 print self.name, ": save_localinfo yaml format Exception", text
 238             except Exception as e:
 239                 text = str(e)
 240                 print self.name, ": save_localinfo Exception:", text
 241
 242     def load_servers_from_db(self):
 243         self.db_lock.acquire()
 244         r,c = self.db.get_table(SELECT=('uuid','status', 'image_id'), FROM='instances', WHERE={'host_id': self.host_id})
 245         self.db_lock.release()
 246
 247         self.server_status = {}
 248         if r<0:
 249             print self.name, ": Error getting data from database:", c
 250             return
 251         for server in c:
 252             self.server_status[ server['uuid'] ] = server['status']
 253
 254             #convert from old version to new one
 255             if 'inc_files' in self.localinfo and server['uuid'] in self.localinfo['inc_files']:
 256                 server_files_dict = {'source file': self.localinfo['inc_files'][ server['uuid'] ] [0],  'file format':'raw' }
 257                 if server_files_dict['source file'][-5:] == 'qcow2':
 258                     server_files_dict['file format'] = 'qcow2'
 259
 260                 self.localinfo['server_files'][ server['uuid'] ] = { server['image_id'] : server_files_dict }
 261         if 'inc_files' in self.localinfo:
 262             del self.localinfo['inc_files']
 263             self.localinfo_dirty = True
 264
 265     def delete_unused_files(self):
 266         '''Compares self.localinfo['server_files'] content with real servers running self.server_status obtained from database
 267         Deletes unused entries at self.loacalinfo and the corresponding local files.
 268         The only reason for this mismatch is the manual deletion of instances (VM) at database
 269         '''
 270         if self.test:
 271             return
 272         for uuid,images in self.localinfo['server_files'].items():
 273             if uuid not in self.server_status:
 274                 for localfile in images.values():
 275                     try:
 276                         print self.name, ": deleting file '%s' of unused server '%s'" %(localfile['source file'], uuid)
 277                         self.delete_file(localfile['source file'])
 278                     except paramiko.ssh_exception.SSHException as e:
 279                         print self.name, ": Exception deleting file '%s': %s" %(localfile['source file'], str(e))
 280                 del self.localinfo['server_files'][uuid]
 281                 self.localinfo_dirty = True
 282
 283     def insert_task(self, task, *aditional):
 284         try:
 285             self.queueLock.acquire()
 286             task = self.taskQueue.put( (task,) + aditional, timeout=5)
 287             self.queueLock.release()
 288             return 1, None
 289         except Queue.Full:
 290             return -1, "timeout inserting a task over host " + self.name
 291
 292     def run(self):
 293         while True:
 294             self.load_localinfo()
 295             self.load_hostinfo()
 296             self.load_servers_from_db()
 297             self.delete_unused_files()
 298             while True:
 299                 self.queueLock.acquire()
 300                 if not self.taskQueue.empty():
 301                     task = self.taskQueue.get()
 302                 else:
 303                     task = None
 304                 self.queueLock.release()
 305
 306                 if task is None:
 307                     now=time.time()
 308                     if self.localinfo_dirty:
 309                         self.save_localinfo()
 310                     elif self.next_update_server_status < now:
 311                         self.update_servers_status()
 312                         self.next_update_server_status = now + 5
 313                     elif len(self.pending_terminate_server)>0 and self.pending_terminate_server[0][0]<now:
 314                         self.server_forceoff()
 315                     else:
 316                         time.sleep(1)
 317                     continue
 318
 319                 if task[0] == 'instance':
 320                     print self.name, ": processing task instance", task[1]['action']
 321                     retry=0
 322                     while retry <2:
 323                         retry += 1
 324                         r=self.action_on_server(task[1], retry==2)
 325                         if r>=0:
 326                             break
 327                 elif task[0] == 'image':
 328                     pass
 329                 elif task[0] == 'exit':
 330                     print self.name, ": processing task exit"
 331                     self.terminate()
 332                     return 0
 333                 elif task[0] == 'reload':
 334                     print self.name, ": processing task reload terminating and relaunching"
 335                     self.terminate()
 336                     break
 337                 elif task[0] == 'edit-iface':
 338                     print self.name, ": processing task edit-iface port=%s, old_net=%s, new_net=%s" % (task[1], task[2], task[3])
 339                     self.edit_iface(task[1], task[2], task[3])
 340                 elif task[0] == 'restore-iface':
 341                     print self.name, ": processing task restore-iface %s mac=%s" % (task[1], task[2])
 342                     self.restore_iface(task[1], task[2])
 343                 elif task[0] == 'new-ovsbridge':
 344                     print self.name, ": Creating compute OVS bridge"
 345                     self.create_ovs_bridge()
 346                     break
 347                 elif task[0] == 'new-vxlan':
 348                     print self.name, ": Creating vxlan tunnel=" + task[1] + ", remote ip=" + task[2]
 349                     self.create_ovs_vxlan_tunnel(task[1], task[2])
 350                     break
 351                 elif task[0] == 'del-ovsbridge':
 352                     print self.name, ": Deleting OVS bridge"
 353                     self.delete_ovs_bridge()
 354                     break
 355                 elif task[0] == 'del-vxlan':
 356                     print self.name, ": Deleting vxlan " + task[1] + " tunnel"
 357                     self.delete_ovs_vxlan_tunnel(task[1])
 358                     break
 359                 elif task[0] == 'create-ovs-bridge-port':
 360                     print self.name, ": Adding port ovim-" + task[1] + " to OVS bridge"
 361                     self.create_ovs_bridge_port(task[1])
 362                 elif task[0] == 'del-ovs-port':
 363                     self.delete_bridge_port_attached_to_ovs(task[1], task[2])
 364                 else:
 365                     print self.name, ": unknown task", task
 366
 367     def server_forceoff(self, wait_until_finished=False):
 368         while len(self.pending_terminate_server)>0:
 369             now = time.time()
 370             if self.pending_terminate_server[0][0]>now:
 371                 if wait_until_finished:
 372                     time.sleep(1)
 373                     continue
 374                 else:
 375                     return
 376             req={'uuid':self.pending_terminate_server[0][1],
 377                 'action':{'terminate':'force'},
 378                 'status': None
 379             }
 380             self.action_on_server(req)
 381             self.pending_terminate_server.pop(0)
 382
 383     def terminate(self):
 384         try:
 385             self.server_forceoff(True)
 386             if self.localinfo_dirty:
 387                 self.save_localinfo()
 388             if not self.test:
 389                 self.ssh_conn.close()
 390         except Exception as e:
 391             text = str(e)
 392             print self.name, ": terminate Exception:", text
 393         print self.name, ": exit from host_thread"
 394
 395     def get_local_iface_name(self, generic_name):
 396         if self.hostinfo != None and "iface_names" in self.hostinfo and generic_name in self.hostinfo["iface_names"]:
 397             return self.hostinfo["iface_names"][generic_name]
 398         return generic_name
 399
 400     def create_xml_server(self, server, dev_list, server_metadata={}):
 401         """Function that implements the generation of the VM XML definition.
 402         Additional devices are in dev_list list
 403         The main disk is upon dev_list[0]"""
 404
 405     #get if operating system is Windows
 406         windows_os = False
 407         os_type = server_metadata.get('os_type', None)
 408         if os_type == None and 'metadata' in dev_list[0]:
 409             os_type = dev_list[0]['metadata'].get('os_type', None)
 410         if os_type != None and os_type.lower() == "windows":
 411             windows_os = True
 412     #get type of hard disk bus
 413         bus_ide = True if windows_os else False
 414         bus = server_metadata.get('bus', None)
 415         if bus == None and 'metadata' in dev_list[0]:
 416             bus = dev_list[0]['metadata'].get('bus', None)
 417         if bus != None:
 418             bus_ide = True if bus=='ide' else False
 419
 420         self.xml_level = 0
 421
 422         text = "<domain type='kvm'>"
 423     #get topology
 424         topo = server_metadata.get('topology', None)
 425         if topo == None and 'metadata' in dev_list[0]:
 426             topo = dev_list[0]['metadata'].get('topology', None)
 427     #name
 428         name = server.get('name','') + "_" + server['uuid']
 429         name = name[:58]  #qemu impose a length  limit of 59 chars or not start. Using 58
 430         text += self.inc_tab() + "<name>" + name+ "</name>"
 431     #uuid
 432         text += self.tab() + "<uuid>" + server['uuid'] + "</uuid>"
 433
 434         numa={}
 435         if 'extended' in server and server['extended']!=None and 'numas' in server['extended']:
 436             numa = server['extended']['numas'][0]
 437     #memory
 438         use_huge = False
 439         memory = int(numa.get('memory',0))*1024*1024 #in KiB
 440         if memory==0:
 441             memory = int(server['ram'])*1024;
 442         else:
 443             if not self.develop_mode:
 444                 use_huge = True
 445         if memory==0:
 446             return -1, 'No memory assigned to instance'
 447         memory = str(memory)
 448         text += self.tab() + "<memory unit='KiB'>" +memory+"</memory>"
 449         text += self.tab() + "<currentMemory unit='KiB'>" +memory+ "</currentMemory>"
 450         if use_huge:
 451             text += self.tab()+'<memoryBacking>'+ \
 452                 self.inc_tab() + '<hugepages/>'+ \
 453                 self.dec_tab()+ '</memoryBacking>'
 454
 455     #cpu
 456         use_cpu_pinning=False
 457         vcpus = int(server.get("vcpus",0))
 458         cpu_pinning = []
 459         if 'cores-source' in numa:
 460             use_cpu_pinning=True
 461             for index in range(0, len(numa['cores-source'])):
 462                 cpu_pinning.append( [ numa['cores-id'][index], numa['cores-source'][index] ] )
 463                 vcpus += 1
 464         if 'threads-source' in numa:
 465             use_cpu_pinning=True
 466             for index in range(0, len(numa['threads-source'])):
 467                 cpu_pinning.append( [ numa['threads-id'][index], numa['threads-source'][index] ] )
 468                 vcpus += 1
 469         if 'paired-threads-source' in numa:
 470             use_cpu_pinning=True
 471             for index in range(0, len(numa['paired-threads-source'])):
 472                 cpu_pinning.append( [numa['paired-threads-id'][index][0], numa['paired-threads-source'][index][0] ] )
 473                 cpu_pinning.append( [numa['paired-threads-id'][index][1], numa['paired-threads-source'][index][1] ] )
 474                 vcpus += 2
 475
 476         if use_cpu_pinning and not self.develop_mode:
 477             text += self.tab()+"<vcpu placement='static'>" +str(len(cpu_pinning)) +"</vcpu>" + \
 478                 self.tab()+'<cputune>'
 479             self.xml_level += 1
 480             for i in range(0, len(cpu_pinning)):
 481                 text += self.tab() + "<vcpupin vcpu='" +str(cpu_pinning[i][0])+ "' cpuset='" +str(cpu_pinning[i][1]) +"'/>"
 482             text += self.dec_tab()+'</cputune>'+ \
 483                 self.tab() + '<numatune>' +\
 484                 self.inc_tab() + "<memory mode='strict' nodeset='" +str(numa['source'])+ "'/>" +\
 485                 self.dec_tab() + '</numatune>'
 486         else:
 487             if vcpus==0:
 488                 return -1, "Instance without number of cpus"
 489             text += self.tab()+"<vcpu>" + str(vcpus)  + "</vcpu>"
 490
 491     #boot
 492         boot_cdrom = False
 493         for dev in dev_list:
 494             if dev['type']=='cdrom' :
 495                 boot_cdrom = True
 496                 break
 497         text += self.tab()+ '<os>' + \
 498             self.inc_tab() + "<type arch='x86_64' machine='pc'>hvm</type>"
 499         if boot_cdrom:
 500             text +=  self.tab() + "<boot dev='cdrom'/>"
 501         text +=  self.tab() + "<boot dev='hd'/>" + \
 502             self.dec_tab()+'</os>'
 503     #features
 504         text += self.tab()+'<features>'+\
 505             self.inc_tab()+'<acpi/>' +\
 506             self.tab()+'<apic/>' +\
 507             self.tab()+'<pae/>'+ \
 508             self.dec_tab() +'</features>'
 509         if windows_os or topo=="oneSocket":
 510             text += self.tab() + "<cpu mode='host-model'> <topology sockets='1' cores='%d' threads='1' /> </cpu>"% vcpus
 511         else:
 512             text += self.tab() + "<cpu mode='host-model'></cpu>"
 513         text += self.tab() + "<clock offset='utc'/>" +\
 514             self.tab() + "<on_poweroff>preserve</on_poweroff>" + \
 515             self.tab() + "<on_reboot>restart</on_reboot>" + \
 516             self.tab() + "<on_crash>restart</on_crash>"
 517         text += self.tab() + "<devices>" + \
 518             self.inc_tab() + "<emulator>/usr/libexec/qemu-kvm</emulator>" + \
 519             self.tab() + "<serial type='pty'>" +\
 520             self.inc_tab() + "<target port='0'/>" + \
 521             self.dec_tab() + "</serial>" +\
 522             self.tab() + "<console type='pty'>" + \
 523             self.inc_tab()+ "<target type='serial' port='0'/>" + \
 524             self.dec_tab()+'</console>'
 525         if windows_os:
 526             text += self.tab() + "<controller type='usb' index='0'/>" + \
 527                 self.tab() + "<controller type='ide' index='0'/>" + \
 528                 self.tab() + "<input type='mouse' bus='ps2'/>" + \
 529                 self.tab() + "<sound model='ich6'/>" + \
 530                 self.tab() + "<video>" + \
 531                 self.inc_tab() + "<model type='cirrus' vram='9216' heads='1'/>" + \
 532                 self.dec_tab() + "</video>" + \
 533                 self.tab() + "<memballoon model='virtio'/>" + \
 534                 self.tab() + "<input type='tablet' bus='usb'/>" #TODO revisar
 535
 536 #>             self.tab()+'<alias name=\'hostdev0\'/>\n' +\
 537 #>             self.dec_tab()+'</hostdev>\n' +\
 538 #>             self.tab()+'<input type=\'tablet\' bus=\'usb\'/>\n'
 539         if windows_os:
 540             text += self.tab() + "<graphics type='vnc' port='-1' autoport='yes'/>"
 541         else:
 542             #If image contains 'GRAPH' include graphics
 543             #if 'GRAPH' in image:
 544             text += self.tab() + "<graphics type='vnc' port='-1' autoport='yes' listen='0.0.0.0'>" +\
 545                 self.inc_tab() + "<listen type='address' address='0.0.0.0'/>" +\
 546                 self.dec_tab() + "</graphics>"
 547
 548         vd_index = 'a'
 549         for dev in dev_list:
 550             bus_ide_dev = bus_ide
 551             if dev['type']=='cdrom' or dev['type']=='disk':
 552                 if dev['type']=='cdrom':
 553                     bus_ide_dev = True
 554                 text += self.tab() + "<disk type='file' device='"+dev['type']+"'>"
 555                 if 'file format' in dev:
 556                     text += self.inc_tab() + "<driver name='qemu' type='"  +dev['file format']+ "' cache='writethrough'/>"
 557                 if 'source file' in dev:
 558                     text += self.tab() + "<source file='" +dev['source file']+ "'/>"
 559                 #elif v['type'] == 'block':
 560                 #    text += self.tab() + "<source dev='" + v['source'] + "'/>"
 561                 #else:
 562                 #    return -1, 'Unknown disk type ' + v['type']
 563                 vpci = dev.get('vpci',None)
 564                 if vpci == None:
 565                     vpci = dev['metadata'].get('vpci',None)
 566                 text += self.pci2xml(vpci)
 567
 568                 if bus_ide_dev:
 569                     text += self.tab() + "<target dev='hd" +vd_index+ "' bus='ide'/>"   #TODO allows several type of disks
 570                 else:
 571                     text += self.tab() + "<target dev='vd" +vd_index+ "' bus='virtio'/>"
 572                 text += self.dec_tab() + '</disk>'
 573                 vd_index = chr(ord(vd_index)+1)
 574             elif dev['type']=='xml':
 575                 dev_text = dev['xml']
 576                 if 'vpci' in dev:
 577                     dev_text = dev_text.replace('__vpci__', dev['vpci'])
 578                 if 'source file' in dev:
 579                     dev_text = dev_text.replace('__file__', dev['source file'])
 580                 if 'file format' in dev:
 581                     dev_text = dev_text.replace('__format__', dev['source file'])
 582                 if '__dev__' in dev_text:
 583                     dev_text = dev_text.replace('__dev__', vd_index)
 584                     vd_index = chr(ord(vd_index)+1)
 585                 text += dev_text
 586             else:
 587                 return -1, 'Unknown device type ' + dev['type']
 588
 589         net_nb=0
 590         bridge_interfaces = server.get('networks', [])
 591         for v in bridge_interfaces:
 592             #Get the brifge name
 593             self.db_lock.acquire()
 594             result, content = self.db.get_table(FROM='nets', SELECT=('provider',),WHERE={'uuid':v['net_id']} )
 595             self.db_lock.release()
 596             if result <= 0:
 597                 print "create_xml_server ERROR getting nets",result, content
 598                 return -1, content
 599             #ALF: Allow by the moment the 'default' bridge net because is confortable for provide internet to VM
 600             #I know it is not secure
 601             #for v in sorted(desc['network interfaces'].itervalues()):
 602             model = v.get("model", None)
 603             if content[0]['provider']=='default':
 604                 text += self.tab() + "<interface type='network'>" + \
 605                     self.inc_tab() + "<source network='" +content[0]['provider']+ "'/>"
 606             elif content[0]['provider'][0:7]=='macvtap':
 607                 text += self.tab()+"<interface type='direct'>" + \
 608                     self.inc_tab() + "<source dev='" + self.get_local_iface_name(content[0]['provider'][8:]) + "' mode='bridge'/>" + \
 609                     self.tab() + "<target dev='macvtap0'/>"
 610                 if windows_os:
 611                     text += self.tab() + "<alias name='net" + str(net_nb) + "'/>"
 612                 elif model==None:
 613                     model = "virtio"
 614             elif content[0]['provider'][0:6]=='bridge':
 615                 text += self.tab() + "<interface type='bridge'>" +  \
 616                     self.inc_tab()+"<source bridge='" +self.get_local_iface_name(content[0]['provider'][7:])+ "'/>"
 617                 if windows_os:
 618                     text += self.tab() + "<target dev='vnet" + str(net_nb)+ "'/>" +\
 619                         self.tab() + "<alias name='net" + str(net_nb)+ "'/>"
 620                 elif model==None:
 621                     model = "virtio"
 622             elif content[0]['provider'][0:3] == "OVS":
 623                 vlan = content[0]['provider'].replace('OVS:', '')
 624                 text += self.tab() + "<interface type='bridge'>" + \
 625                         self.inc_tab() + "<source bridge='ovim-" + vlan + "'/>"
 626             else:
 627                 return -1, 'Unknown Bridge net provider ' + content[0]['provider']
 628             if model!=None:
 629                 text += self.tab() + "<model type='" +model+ "'/>"
 630             if v.get('mac_address', None) != None:
 631                 text+= self.tab() +"<mac address='" +v['mac_address']+ "'/>"
 632             text += self.pci2xml(v.get('vpci',None))
 633             text += self.dec_tab()+'</interface>'
 634
 635             net_nb += 1
 636
 637         interfaces = numa.get('interfaces', [])
 638
 639         net_nb=0
 640         for v in interfaces:
 641             if self.develop_mode: #map these interfaces to bridges
 642                 text += self.tab() + "<interface type='bridge'>" +  \
 643                     self.inc_tab()+"<source bridge='" +self.develop_bridge_iface+ "'/>"
 644                 if windows_os:
 645                     text += self.tab() + "<target dev='vnet" + str(net_nb)+ "'/>" +\
 646                         self.tab() + "<alias name='net" + str(net_nb)+ "'/>"
 647                 else:
 648                     text += self.tab() + "<model type='e1000'/>" #e1000 is more probable to be supported than 'virtio'
 649                 if v.get('mac_address', None) != None:
 650                     text+= self.tab() +"<mac address='" +v['mac_address']+ "'/>"
 651                 text += self.pci2xml(v.get('vpci',None))
 652                 text += self.dec_tab()+'</interface>'
 653                 continue
 654
 655             if v['dedicated'] == 'yes':  #passthrought
 656                 text += self.tab() + "<hostdev mode='subsystem' type='pci' managed='yes'>" + \
 657                     self.inc_tab() + "<source>"
 658                 self.inc_tab()
 659                 text += self.pci2xml(v['source'])
 660                 text += self.dec_tab()+'</source>'
 661                 text += self.pci2xml(v.get('vpci',None))
 662                 if windows_os:
 663                     text += self.tab() + "<alias name='hostdev" + str(net_nb) + "'/>"
 664                 text += self.dec_tab()+'</hostdev>'
 665                 net_nb += 1
 666             else:        #sriov_interfaces
 667                 #skip not connected interfaces
 668                 if v.get("net_id") == None:
 669                     continue
 670                 text += self.tab() + "<interface type='hostdev' managed='yes'>"
 671                 self.inc_tab()
 672                 if v.get('mac_address', None) != None:
 673                     text+= self.tab() + "<mac address='" +v['mac_address']+ "'/>"
 674                 text+= self.tab()+'<source>'
 675                 self.inc_tab()
 676                 text += self.pci2xml(v['source'])
 677                 text += self.dec_tab()+'</source>'
 678                 if v.get('vlan',None) != None:
 679                     text += self.tab() + "<vlan>   <tag id='" + str(v['vlan']) + "'/>   </vlan>"
 680                 text += self.pci2xml(v.get('vpci',None))
 681                 if windows_os:
 682                     text += self.tab() + "<alias name='hostdev" + str(net_nb) + "'/>"
 683                 text += self.dec_tab()+'</interface>'
 684
 685
 686         text += self.dec_tab()+'</devices>'+\
 687         self.dec_tab()+'</domain>'
 688         return 0, text
 689
 690     def pci2xml(self, pci):
 691         '''from a pci format text XXXX:XX:XX.X generates the xml content of <address>
 692         alows an empty pci text'''
 693         if pci is None:
 694             return ""
 695         first_part = pci.split(':')
 696         second_part = first_part[2].split('.')
 697         return self.tab() + "<address type='pci' domain='0x" + first_part[0] + \
 698                     "' bus='0x" + first_part[1] + "' slot='0x" + second_part[0] + \
 699                     "' function='0x" + second_part[1] + "'/>"
 700
 701     def tab(self):
 702         """Return indentation according to xml_level"""
 703         return "\n" + ('  '*self.xml_level)
 704
 705     def inc_tab(self):
 706         """Increment and return indentation according to xml_level"""
 707         self.xml_level += 1
 708         return self.tab()
 709
 710     def dec_tab(self):
 711         """Decrement and return indentation according to xml_level"""
 712         self.xml_level -= 1
 713         return self.tab()
 714
 715     def create_ovs_bridge(self):
 716         """
 717         Create a bridge in compute OVS to allocate VMs
 718         :return: True if success
 719         """
 720         if self.test:
 721             return
 722         command = 'sudo ovs-vsctl --may-exist add-br br-int -- set Bridge br-int stp_enable=true'
 723         print self.name, ': command:', command
 724         (_, stdout, _) = self.ssh_conn.exec_command(command)
 725         content = stdout.read()
 726         if len(content) == 0:
 727             return True
 728         else:
 729             return False
 730
 731     def delete_port_to_ovs_bridge(self, vlan, net_uuid):
 732         """
 733         Delete linux bridge port attched to a OVS bridge, if port is not free the port is not removed
 734         :param vlan: vlan port id
 735         :param net_uuid: network id
 736         :return:
 737         """
 738
 739         if self.test:
 740             return
 741
 742         port_name = 'ovim-' + vlan
 743         command = 'sudo ovs-vsctl del-port br-int ' + port_name
 744         print self.name, ': command:', command
 745         (_, stdout, _) = self.ssh_conn.exec_command(command)
 746         content = stdout.read()
 747         if len(content) == 0:
 748             return True
 749         else:
 750             return False
 751
 752     def delete_dhcp_server(self, vlan, net_uuid, dhcp_path):
 753         """
 754         Delete dhcp server process lining in namespace
 755         :param vlan: segmentation id
 756         :param net_uuid: network uuid
 757         :param dhcp_path: conf fiel path that live in namespace side
 758         :return:
 759         """
 760         if self.test:
 761             return
 762         if not self.is_dhcp_port_free(vlan, net_uuid):
 763             return True
 764
 765         net_namespace = 'ovim-' + vlan
 766         dhcp_path = os.path.join(dhcp_path, net_namespace)
 767         pid_file = os.path.join(dhcp_path, 'dnsmasq.pid')
 768
 769         command = 'sudo ip netns exec ' + net_namespace + ' cat ' + pid_file
 770         print self.name, ': command:', command
 771         (_, stdout, _) = self.ssh_conn.exec_command(command)
 772         content = stdout.read()
 773
 774         command = 'sudo ip netns exec ' + net_namespace + ' kill -9 ' + content
 775         print self.name, ': command:', command
 776         (_, stdout, _) = self.ssh_conn.exec_command(command)
 777         content = stdout.read()
 778
 779         # if len(content) == 0:
 780         #     return True
 781         # else:
 782         #     return False
 783
 784     def is_dhcp_port_free(self, host_id, net_uuid):
 785         """
 786         Check if any port attached to the a net in a vxlan mesh across computes nodes
 787         :param host_id: host id
 788         :param net_uuid: network id
 789         :return: True if is not free
 790         """
 791         self.db_lock.acquire()
 792         result, content = self.db.get_table(
 793             FROM='ports',
 794             WHERE={'p.type': 'instance:ovs', 'p.net_id': net_uuid}
 795         )
 796         self.db_lock.release()
 797
 798         if len(content) > 0:
 799             return False
 800         else:
 801             return True
 802
 803     def is_port_free(self, host_id, net_uuid):
 804         """
 805         Check if there not ovs ports of a network in a compute host.
 806         :param host_id:  host id
 807         :param net_uuid: network id
 808         :return: True if is not free
 809         """
 810
 811         self.db_lock.acquire()
 812         result, content = self.db.get_table(
 813             FROM='ports as p join instances as i on p.instance_id=i.uuid',
 814             WHERE={"i.host_id": self.host_id, 'p.type': 'instance:ovs', 'p.net_id': net_uuid}
 815         )
 816         self.db_lock.release()
 817
 818         if len(content) > 0:
 819             return False
 820         else:
 821             return True
 822
 823     def add_port_to_ovs_bridge(self, vlan):
 824         """
 825         Add a bridge linux as a port to a OVS bridge and set a vlan for an specific linux bridge
 826         :param vlan: vlan port id
 827         :return: True if success
 828         """
 829
 830         if self.test:
 831             return
 832
 833         port_name = 'ovim-' + vlan
 834         command = 'sudo ovs-vsctl add-port br-int ' + port_name + ' tag=' + vlan
 835         print self.name, ': command:', command
 836         (_, stdout, _) = self.ssh_conn.exec_command(command)
 837         content = stdout.read()
 838         if len(content) == 0:
 839             return True
 840         else:
 841             return False
 842
 843     def delete_dhcp_port(self, vlan, net_uuid):
 844         """
 845         Delete from an existing OVS bridge a linux bridge port attached and the linux bridge itself.
 846         :param vlan: segmentation id
 847         :param net_uuid: network id
 848         :return: True if success
 849         """
 850
 851         if self.test:
 852             return
 853
 854         if not self.is_dhcp_port_free(vlan, net_uuid):
 855             return True
 856         self.delete_dhcp_interfaces(vlan)
 857         return True
 858
 859     def delete_bridge_port_attached_to_ovs(self, vlan, net_uuid):
 860         """
 861         Delete from an existing OVS bridge a linux bridge port attached and the linux bridge itself.
 862         :param vlan:
 863         :param net_uuid:
 864         :return: True if success
 865         """
 866         if self.test:
 867             return
 868
 869         if not self.is_port_free(vlan, net_uuid):
 870             return True
 871         self.delete_port_to_ovs_bridge(vlan, net_uuid)
 872         self.delete_linux_bridge(vlan)
 873         return True
 874
 875     def delete_linux_bridge(self, vlan):
 876         """
 877         Delete a linux bridge in a scpecific compute.
 878         :param vlan: vlan port id
 879         :return: True if success
 880         """
 881
 882         if self.test:
 883             return
 884
 885         port_name = 'ovim-' + vlan
 886         command = 'sudo ip link set dev veth0-' + vlan + ' down'
 887         print self.name, ': command:', command
 888         (_, stdout, _) = self.ssh_conn.exec_command(command)
 889         content = stdout.read()
 890         #
 891         # if len(content) != 0:
 892         #     return False
 893
 894         command = 'sudo ifconfig ' + port_name + ' down &&  sudo brctl delbr ' + port_name
 895         print self.name, ': command:', command
 896         (_, stdout, _) = self.ssh_conn.exec_command(command)
 897         content = stdout.read()
 898         if len(content) == 0:
 899             return True
 900         else:
 901             return False
 902
 903     def create_ovs_bridge_port(self, vlan):
 904         """
 905         Generate a linux bridge and attache the port to a OVS bridge
 906         :param vlan: vlan port id
 907         :return:
 908         """
 909         if self.test:
 910             return
 911         self.create_linux_bridge(vlan)
 912         self.add_port_to_ovs_bridge(vlan)
 913
 914     def create_linux_bridge(self, vlan):
 915         """
 916         Create a linux bridge with STP active
 917         :param vlan: netowrk vlan id
 918         :return:
 919         """
 920
 921         if self.test:
 922             return
 923
 924         port_name = 'ovim-' + vlan
 925         command = 'sudo brctl show | grep ' + port_name
 926         print self.name, ': command:', command
 927         (_, stdout, _) = self.ssh_conn.exec_command(command)
 928         content = stdout.read()
 929
 930         # if exist nothing to create
 931         # if len(content) == 0:
 932         #     return False
 933
 934         command = 'sudo brctl addbr ' + port_name
 935         print self.name, ': command:', command
 936         (_, stdout, _) = self.ssh_conn.exec_command(command)
 937         content = stdout.read()
 938
 939         # if len(content) == 0:
 940         #     return True
 941         # else:
 942         #     return False
 943
 944         command = 'sudo brctl stp ' + port_name + ' on'
 945         print self.name, ': command:', command
 946         (_, stdout, _) = self.ssh_conn.exec_command(command)
 947         content = stdout.read()
 948
 949         # if len(content) == 0:
 950         #     return True
 951         # else:
 952         #     return False
 953         command = 'sudo ip link set dev ' + port_name + ' up'
 954         print self.name, ': command:', command
 955         (_, stdout, _) = self.ssh_conn.exec_command(command)
 956         content = stdout.read()
 957
 958         if len(content) == 0:
 959             return True
 960         else:
 961             return False
 962
 963     def set_mac_dhcp_server(self, ip, mac, vlan, netmask, dhcp_path):
 964         """
 965         Write into dhcp conf file a rule to assigned a fixed ip given to an specific MAC address
 966         :param ip: IP address asigned to a VM
 967         :param mac: VM vnic mac to be macthed with the IP received
 968         :param vlan: Segmentation id
 969         :param netmask: netmask value
 970         :param path: dhcp conf file path that live in namespace side
 971         :return: True if success
 972         """
 973
 974         if self.test:
 975             return
 976
 977         net_namespace = 'ovim-' + vlan
 978         dhcp_path = os.path.join(dhcp_path, net_namespace)
 979         dhcp_hostsdir = os.path.join(dhcp_path, net_namespace)
 980
 981         if not ip:
 982             return False
 983
 984         ip_data = mac.upper() + ',' + ip
 985
 986         command = 'sudo  ip netns exec ' + net_namespace + ' touch ' + dhcp_hostsdir
 987         print self.name, ': command:', command
 988         (_, stdout, _) = self.ssh_conn.exec_command(command)
 989         content = stdout.read()
 990
 991         command = 'sudo  ip netns exec ' + net_namespace + ' sudo bash -ec "echo ' + ip_data + ' >> ' + dhcp_hostsdir + '"'
 992
 993         print self.name, ': command:', command
 994         (_, stdout, _) = self.ssh_conn.exec_command(command)
 995         content = stdout.read()
 996
 997         if len(content) == 0:
 998             return True
 999         else:
1000             return False
1001
1002     def delete_mac_dhcp_server(self, ip, mac, vlan, dhcp_path):
1003         """
1004         Delete into dhcp conf file the ip  assigned to a specific MAC address
1005
1006         :param ip: IP address asigned to a VM
1007         :param mac:  VM vnic mac to be macthed with the IP received
1008         :param vlan:  Segmentation id
1009         :param dhcp_path: dhcp conf file path that live in namespace side
1010         :return:
1011         """
1012
1013         if self.test:
1014             return
1015
1016         net_namespace = 'ovim-' + vlan
1017         dhcp_path = os.path.join(dhcp_path, net_namespace)
1018         dhcp_hostsdir = os.path.join(dhcp_path, net_namespace)
1019
1020         if not ip:
1021             return False
1022
1023         ip_data = mac.upper() + ',' + ip
1024
1025         command = 'sudo  ip netns exec ' + net_namespace + ' sudo sed -i \'/' + ip_data + '/d\' ' + dhcp_hostsdir
1026         print self.name, ': command:', command
1027         (_, stdout, _) = self.ssh_conn.exec_command(command)
1028         content = stdout.read()
1029
1030         if len(content) == 0:
1031             return True
1032         else:
1033             return False
1034
1035     def launch_dhcp_server(self, vlan, ip_range, netmask, dhcp_path):
1036         """
1037         Generate a linux bridge and attache the port to a OVS bridge
1038         :param self:
1039         :param vlan: Segmentation id
1040         :param ip_range: IP dhcp range
1041         :param netmask: network netmask
1042         :param dhcp_path: dhcp conf file path that live in namespace side
1043         :return: True if success
1044         """
1045
1046         if self.test:
1047             return
1048
1049         interface = 'tap-' + vlan
1050         net_namespace = 'ovim-' + vlan
1051         dhcp_path = os.path.join(dhcp_path, net_namespace)
1052         leases_path = os.path.join(dhcp_path, "dnsmasq.leases")
1053         pid_file = os.path.join(dhcp_path, 'dnsmasq.pid')
1054
1055         dhcp_range = ip_range[0] + ',' + ip_range[1] + ',' + netmask
1056
1057         command = 'sudo ip netns exec ' + net_namespace + ' mkdir -p ' + dhcp_path
1058         print self.name, ': command:', command
1059         (_, stdout, _) = self.ssh_conn.exec_command(command)
1060         content = stdout.read()
1061
1062         pid_path = os.path.join(dhcp_path, 'dnsmasq.pid')
1063         command = 'sudo  ip netns exec ' + net_namespace + ' cat ' + pid_path
1064         print self.name, ': command:', command
1065         (_, stdout, _) = self.ssh_conn.exec_command(command)
1066         content = stdout.read()
1067         # check if pid is runing
1068         pid_status_path = content
1069         if content:
1070             command = "ps aux | awk '{print $2 }' | grep " + pid_status_path
1071             print self.name, ': command:', command
1072             (_, stdout, _) = self.ssh_conn.exec_command(command)
1073             content = stdout.read()
1074         if not content:
1075             command = 'sudo  ip netns exec ' + net_namespace + ' /usr/sbin/dnsmasq --strict-order --except-interface=lo ' \
1076               '--interface=' + interface + ' --bind-interfaces --dhcp-hostsdir=' + dhcp_path + \
1077               ' --dhcp-range ' + dhcp_range + ' --pid-file=' + pid_file + ' --dhcp-leasefile=' + leases_path + '  --listen-address ' + ip_range[0]
1078
1079         print self.name, ': command:', command
1080         (_, stdout, _) = self.ssh_conn.exec_command(command)
1081         content = stdout.readline()
1082
1083         if len(content) == 0:
1084             return True
1085         else:
1086             return False
1087
1088     def delete_dhcp_interfaces(self, vlan):
1089         """
1090         Create a linux bridge with STP active
1091         :param vlan: netowrk vlan id
1092         :return:
1093         """
1094
1095         if self.test:
1096             return
1097
1098         net_namespace = 'ovim-' + vlan
1099         command = 'sudo ovs-vsctl del-port br-int ovs-tap-' + vlan
1100         print self.name, ': command:', command
1101         (_, stdout, _) = self.ssh_conn.exec_command(command)
1102         content = stdout.read()
1103
1104         command = 'sudo ip netns exec ' + net_namespace + ' ip link set dev tap-' + vlan + ' down'
1105         print self.name, ': command:', command
1106         (_, stdout, _) = self.ssh_conn.exec_command(command)
1107         content = stdout.read()
1108
1109         command = 'sudo ip link set dev ovs-tap-' + vlan + ' down'
1110         print self.name, ': command:', command
1111         (_, stdout, _) = self.ssh_conn.exec_command(command)
1112         content = stdout.read()
1113
1114     def create_dhcp_interfaces(self, vlan, ip, netmask):
1115         """
1116         Create a linux bridge with STP active
1117         :param vlan: segmentation id
1118         :param ip: Ip included in the dhcp range for the tap interface living in namesapce side
1119         :param netmask: dhcp net CIDR
1120         :return: True if success
1121         """
1122
1123         if self.test:
1124             return
1125
1126         net_namespace = 'ovim-' + vlan
1127         namespace_interface = 'tap-' + vlan
1128
1129         command = 'sudo ip netns add ' + net_namespace
1130         print self.name, ': command:', command
1131         (_, stdout, _) = self.ssh_conn.exec_command(command)
1132         content = stdout.read()
1133
1134         command = 'sudo ip link add tap-' + vlan + ' type veth peer name ovs-tap-' + vlan
1135         print self.name, ': command:', command
1136         (_, stdout, _) = self.ssh_conn.exec_command(command)
1137         content = stdout.read()
1138
1139         command = 'sudo ovs-vsctl add-port br-int ovs-tap-' + vlan + ' tag=' + vlan
1140         print self.name, ': command:', command
1141         (_, stdout, _) = self.ssh_conn.exec_command(command)
1142         content = stdout.read()
1143
1144         command = 'sudo ip link set tap-' + vlan + ' netns ' + net_namespace
1145         print self.name, ': command:', command
1146         (_, stdout, _) = self.ssh_conn.exec_command(command)
1147         content = stdout.read()
1148
1149         command = 'sudo ip netns exec ' + net_namespace + ' ip link set dev tap-' + vlan + ' up'
1150         print self.name, ': command:', command
1151         (_, stdout, _) = self.ssh_conn.exec_command(command)
1152         content = stdout.read()
1153
1154         command = 'sudo ip link set dev ovs-tap-' + vlan + ' up'
1155         print self.name, ': command:', command
1156         (_, stdout, _) = self.ssh_conn.exec_command(command)
1157         content = stdout.read()
1158
1159         command = 'sudo  ip netns exec ' + net_namespace + ' ' + ' ifconfig  ' + namespace_interface \
1160                   + ' ' + ip + ' netmask ' + netmask
1161         print self.name, ': command:', command
1162         (_, stdout, _) = self.ssh_conn.exec_command(command)
1163         content = stdout.read()
1164
1165         if len(content) == 0:
1166             return True
1167         else:
1168             return False
1169
1170     def create_ovs_vxlan_tunnel(self, vxlan_interface, remote_ip):
1171         """
1172         Create a vlxn tunnel between to computes with an OVS installed. STP is also active at port level
1173         :param vxlan_interface: vlxan inteface name.
1174         :param remote_ip: tunnel endpoint remote compute ip.
1175         :return:
1176         """
1177         if self.test:
1178             return
1179         command = 'sudo ovs-vsctl add-port br-int ' + vxlan_interface + \
1180                   ' -- set Interface ' + vxlan_interface + '  type=vxlan options:remote_ip=' + remote_ip + \
1181                   ' -- set Port ' + vxlan_interface + ' other_config:stp-path-cost=10'
1182         print self.name, ': command:', command
1183         (_, stdout, _) = self.ssh_conn.exec_command(command)
1184         content = stdout.read()
1185         print content
1186         if len(content) == 0:
1187             return True
1188         else:
1189             return False
1190
1191     def delete_ovs_vxlan_tunnel(self, vxlan_interface):
1192         """
1193         Delete a vlxan tunnel  port from a OVS brdige.
1194         :param vxlan_interface: vlxan name to be delete it.
1195         :return: True if success.
1196         """
1197         if self.test:
1198             return
1199         command = 'sudo ovs-vsctl del-port br-int ' + vxlan_interface
1200         print self.name, ': command:', command
1201         (_, stdout, _) = self.ssh_conn.exec_command(command)
1202         content = stdout.read()
1203         print content
1204         if len(content) == 0:
1205             return True
1206         else:
1207             return False
1208
1209     def delete_ovs_bridge(self):
1210         """
1211         Delete a OVS bridge from  a compute.
1212         :return: True if success
1213         """
1214         if self.test:
1215             return
1216         command = 'sudo ovs-vsctl del-br br-int'
1217         print self.name, ': command:', command
1218         (_, stdout, _) = self.ssh_conn.exec_command(command)
1219         content = stdout.read()
1220         if len(content) == 0:
1221             return True
1222         else:
1223             return False
1224
1225     def get_file_info(self, path):
1226         command = 'ls -lL --time-style=+%Y-%m-%dT%H:%M:%S ' + path
1227         print self.name, ': command:', command
1228         (_, stdout, _) = self.ssh_conn.exec_command(command)
1229         content = stdout.read()
1230         if len(content) == 0:
1231             return None # file does not exist
1232         else:
1233             return content.split(" ") #(permission, 1, owner, group, size, date, file)
1234
1235     def qemu_get_info(self, path):
1236         command = 'qemu-img info ' + path
1237         print self.name, ': command:', command
1238         (_, stdout, stderr) = self.ssh_conn.exec_command(command)
1239         content = stdout.read()
1240         if len(content) == 0:
1241             error = stderr.read()
1242             print self.name, ": get_qemu_info error ", error
1243             raise paramiko.ssh_exception.SSHException("Error getting qemu_info: " + error)
1244         else:
1245             try:
1246                 return yaml.load(content)
1247             except yaml.YAMLError as exc:
1248                 text = ""
1249                 if hasattr(exc, 'problem_mark'):
1250                     mark = exc.problem_mark
1251                     text = " at position: (%s:%s)" % (mark.line+1, mark.column+1)
1252                 print self.name, ": get_qemu_info yaml format Exception", text
1253                 raise paramiko.ssh_exception.SSHException("Error getting qemu_info yaml format" + text)
1254
1255     def qemu_change_backing(self, inc_file, new_backing_file):
1256         command = 'qemu-img rebase -u -b ' + new_backing_file + ' ' + inc_file
1257         print self.name, ': command:', command
1258         (_, _, stderr) = self.ssh_conn.exec_command(command)
1259         content = stderr.read()
1260         if len(content) == 0:
1261             return 0
1262         else:
1263             print self.name, ": qemu_change_backing error: ", content
1264             return -1
1265
1266     def get_notused_filename(self, proposed_name, suffix=''):
1267         '''Look for a non existing file_name in the host
1268             proposed_name: proposed file name, includes path
1269             suffix: suffix to be added to the name, before the extention
1270         '''
1271         extension = proposed_name.rfind(".")
1272         slash = proposed_name.rfind("/")
1273         if extension < 0 or extension < slash: # no extension
1274             extension = len(proposed_name)
1275         target_name = proposed_name[:extension] + suffix + proposed_name[extension:]
1276         info = self.get_file_info(target_name)
1277         if info is None:
1278             return target_name
1279
1280         index=0
1281         while info is not None:
1282             target_name = proposed_name[:extension] + suffix +  "-" + str(index) + proposed_name[extension:]
1283             index+=1
1284             info = self.get_file_info(target_name)
1285         return target_name
1286
1287     def get_notused_path(self, proposed_path, suffix=''):
1288         '''Look for a non existing path at database for images
1289             proposed_path: proposed file name, includes path
1290             suffix: suffix to be added to the name, before the extention
1291         '''
1292         extension = proposed_path.rfind(".")
1293         if extension < 0:
1294             extension = len(proposed_path)
1295         if suffix != None:
1296             target_path = proposed_path[:extension] + suffix + proposed_path[extension:]
1297         index=0
1298         while True:
1299             r,_=self.db.get_table(FROM="images",WHERE={"path":target_path})
1300             if r<=0:
1301                 return target_path
1302             target_path = proposed_path[:extension] + suffix +  "-" + str(index) + proposed_path[extension:]
1303             index+=1
1304
1305
1306     def delete_file(self, file_name):
1307         command = 'rm -f '+file_name
1308         print self.name, ': command:', command
1309         (_, _, stderr) = self.ssh_conn.exec_command(command)
1310         error_msg = stderr.read()
1311         if len(error_msg) > 0:
1312             raise paramiko.ssh_exception.SSHException("Error deleting file: " + error_msg)
1313
1314     def copy_file(self, source, destination, perserve_time=True):
1315         if source[0:4]=="http":
1316             command = "wget --no-verbose -O '{dst}' '{src}' 2>'{dst_result}' || cat '{dst_result}' >&2 && rm '{dst_result}'".format(
1317                 dst=destination, src=source, dst_result=destination + ".result" )
1318         else:
1319             command = 'cp --no-preserve=mode'
1320             if perserve_time:
1321                 command += ' --preserve=timestamps'
1322             command +=  " '{}' '{}'".format(source, destination)
1323         print self.name, ': command:', command
1324         (_, _, stderr) = self.ssh_conn.exec_command(command)
1325         error_msg = stderr.read()
1326         if len(error_msg) > 0:
1327             raise paramiko.ssh_exception.SSHException("Error copying image to local host: " + error_msg)
1328
1329     def copy_remote_file(self, remote_file, use_incremental):
1330         ''' Copy a file from the repository to local folder and recursively
1331             copy the backing files in case the remote file is incremental
1332             Read and/or modified self.localinfo['files'] that contain the
1333             unmodified copies of images in the local path
1334             params:
1335                 remote_file: path of remote file
1336                 use_incremental: None (leave the decision to this function), True, False
1337             return:
1338                 local_file: name of local file
1339                 qemu_info: dict with quemu information of local file
1340                 use_incremental_out: True, False; same as use_incremental, but if None a decision is taken
1341         '''
1342
1343         use_incremental_out = use_incremental
1344         new_backing_file = None
1345         local_file = None
1346         file_from_local = True
1347
1348         #in case incremental use is not decided, take the decision depending on the image
1349         #avoid the use of incremental if this image is already incremental
1350         if remote_file[0:4] == "http":
1351             file_from_local = False
1352         if file_from_local:
1353             qemu_remote_info = self.qemu_get_info(remote_file)
1354         if use_incremental_out==None:
1355             use_incremental_out = not ( file_from_local and 'backing file' in qemu_remote_info)
1356         #copy recursivelly the backing files
1357         if  file_from_local and 'backing file' in qemu_remote_info:
1358             new_backing_file, _, _ = self.copy_remote_file(qemu_remote_info['backing file'], True)
1359
1360         #check if remote file is present locally
1361         if use_incremental_out and remote_file in self.localinfo['files']:
1362             local_file = self.localinfo['files'][remote_file]
1363             local_file_info =  self.get_file_info(local_file)
1364             if file_from_local:
1365                 remote_file_info = self.get_file_info(remote_file)
1366             if local_file_info == None:
1367                 local_file = None
1368             elif file_from_local and (local_file_info[4]!=remote_file_info[4] or local_file_info[5]!=remote_file_info[5]):
1369                 #local copy of file not valid because date or size are different.
1370                 #TODO DELETE local file if this file is not used by any active virtual machine
1371                 try:
1372                     self.delete_file(local_file)
1373                     del self.localinfo['files'][remote_file]
1374                 except Exception:
1375                     pass
1376                 local_file = None
1377             else: #check that the local file has the same backing file, or there are not backing at all
1378                 qemu_info = self.qemu_get_info(local_file)
1379                 if new_backing_file != qemu_info.get('backing file'):
1380                     local_file = None
1381
1382
1383         if local_file == None: #copy the file
1384             img_name= remote_file.split('/') [-1]
1385             img_local = self.image_path + '/' + img_name
1386             local_file = self.get_notused_filename(img_local)
1387             self.copy_file(remote_file, local_file, use_incremental_out)
1388
1389             if use_incremental_out:
1390                 self.localinfo['files'][remote_file] = local_file
1391             if new_backing_file:
1392                 self.qemu_change_backing(local_file, new_backing_file)
1393             qemu_info = self.qemu_get_info(local_file)
1394
1395         return local_file, qemu_info, use_incremental_out
1396
1397     def launch_server(self, conn, server, rebuild=False, domain=None):
1398         if self.test:
1399             time.sleep(random.randint(20,150)) #sleep random timeto be make it a bit more real
1400             return 0, 'Success'
1401
1402         server_id = server['uuid']
1403         paused = server.get('paused','no')
1404         try:
1405             if domain!=None and rebuild==False:
1406                 domain.resume()
1407                 #self.server_status[server_id] = 'ACTIVE'
1408                 return 0, 'Success'
1409
1410             self.db_lock.acquire()
1411             result, server_data = self.db.get_instance(server_id)
1412             self.db_lock.release()
1413             if result <= 0:
1414                 print self.name, ": launch_server ERROR getting server from DB",result, server_data
1415                 return result, server_data
1416
1417         #0: get image metadata
1418             server_metadata = server.get('metadata', {})
1419             use_incremental = None
1420
1421             if "use_incremental" in server_metadata:
1422                 use_incremental = False if server_metadata["use_incremental"]=="no" else True
1423
1424             server_host_files = self.localinfo['server_files'].get( server['uuid'], {})
1425             if rebuild:
1426                 #delete previous incremental files
1427                 for file_ in server_host_files.values():
1428                     self.delete_file(file_['source file'] )
1429                 server_host_files={}
1430
1431         #1: obtain aditional devices (disks)
1432             #Put as first device the main disk
1433             devices = [  {"type":"disk", "image_id":server['image_id'], "vpci":server_metadata.get('vpci', None) } ]
1434             if 'extended' in server_data and server_data['extended']!=None and "devices" in server_data['extended']:
1435                 devices += server_data['extended']['devices']
1436
1437             for dev in devices:
1438                 if dev['image_id'] == None:
1439                     continue
1440
1441                 self.db_lock.acquire()
1442                 result, content = self.db.get_table(FROM='images', SELECT=('path', 'metadata'),
1443                                                     WHERE={'uuid': dev['image_id']})
1444                 self.db_lock.release()
1445                 if result <= 0:
1446                     error_text = "ERROR", result, content, "when getting image", dev['image_id']
1447                     print self.name, ": launch_server", error_text
1448                     return -1, error_text
1449                 if content[0]['metadata'] is not None:
1450                     dev['metadata'] = json.loads(content[0]['metadata'])
1451                 else:
1452                     dev['metadata'] = {}
1453
1454                 if dev['image_id'] in server_host_files:
1455                     dev['source file'] = server_host_files[ dev['image_id'] ] ['source file'] #local path
1456                     dev['file format'] = server_host_files[ dev['image_id'] ] ['file format'] # raw or qcow2
1457                     continue
1458
1459             #2: copy image to host
1460                 remote_file = content[0]['path']
1461                 use_incremental_image = use_incremental
1462                 if dev['metadata'].get("use_incremental") == "no":
1463                     use_incremental_image = False
1464                 local_file, qemu_info, use_incremental_image = self.copy_remote_file(remote_file, use_incremental_image)
1465
1466                 #create incremental image
1467                 if use_incremental_image:
1468                     local_file_inc = self.get_notused_filename(local_file, '.inc')
1469                     command = 'qemu-img create -f qcow2 '+local_file_inc+ ' -o backing_file='+ local_file
1470                     print 'command:', command
1471                     (_, _, stderr) = self.ssh_conn.exec_command(command)
1472                     error_msg = stderr.read()
1473                     if len(error_msg) > 0:
1474                         raise paramiko.ssh_exception.SSHException("Error creating incremental file: " + error_msg)
1475                     local_file = local_file_inc
1476                     qemu_info = {'file format':'qcow2'}
1477
1478                 server_host_files[ dev['image_id'] ] = {'source file': local_file, 'file format': qemu_info['file format']}
1479
1480                 dev['source file'] = local_file
1481                 dev['file format'] = qemu_info['file format']
1482
1483             self.localinfo['server_files'][ server['uuid'] ] = server_host_files
1484             self.localinfo_dirty = True
1485
1486         #3 Create XML
1487             result, xml = self.create_xml_server(server_data, devices, server_metadata)  #local_file
1488             if result <0:
1489                 print self.name, ": create xml server error:", xml
1490                 return -2, xml
1491             print self.name, ": create xml:", xml
1492             atribute = host_thread.lvirt_module.VIR_DOMAIN_START_PAUSED if paused == "yes" else 0
1493         #4 Start the domain
1494             if not rebuild: #ensures that any pending destroying server is done
1495                 self.server_forceoff(True)
1496             #print self.name, ": launching instance" #, xml
1497             conn.createXML(xml, atribute)
1498             #self.server_status[server_id] = 'PAUSED' if paused == "yes" else 'ACTIVE'
1499
1500             return 0, 'Success'
1501
1502         except paramiko.ssh_exception.SSHException as e:
1503             text = e.args[0]
1504             print self.name, ": launch_server(%s) ssh Exception: %s" %(server_id, text)
1505             if "SSH session not active" in text:
1506                 self.ssh_connect()
1507         except host_thread.lvirt_module.libvirtError as e:
1508             text = e.get_error_message()
1509             print self.name, ": launch_server(%s) libvirt Exception: %s"  %(server_id, text)
1510         except Exception as e:
1511             text = str(e)
1512             print self.name, ": launch_server(%s) Exception: %s"  %(server_id, text)
1513         return -1, text
1514
1515     def update_servers_status(self):
1516                             # # virDomainState
1517                             # VIR_DOMAIN_NOSTATE = 0
1518                             # VIR_DOMAIN_RUNNING = 1
1519                             # VIR_DOMAIN_BLOCKED = 2
1520                             # VIR_DOMAIN_PAUSED = 3
1521                             # VIR_DOMAIN_SHUTDOWN = 4
1522                             # VIR_DOMAIN_SHUTOFF = 5
1523                             # VIR_DOMAIN_CRASHED = 6
1524                             # VIR_DOMAIN_PMSUSPENDED = 7   #TODO suspended
1525
1526         if self.test or len(self.server_status)==0:
1527             return
1528
1529         try:
1530             conn = host_thread.lvirt_module.open("qemu+ssh://"+self.user+"@"+self.host+"/system")
1531             domains=  conn.listAllDomains()
1532             domain_dict={}
1533             for domain in domains:
1534                 uuid = domain.UUIDString() ;
1535                 libvirt_status = domain.state()
1536                 #print libvirt_status
1537                 if libvirt_status[0] == host_thread.lvirt_module.VIR_DOMAIN_RUNNING or libvirt_status[0] == host_thread.lvirt_module.VIR_DOMAIN_SHUTDOWN:
1538                     new_status = "ACTIVE"
1539                 elif libvirt_status[0] == host_thread.lvirt_module.VIR_DOMAIN_PAUSED:
1540                     new_status = "PAUSED"
1541                 elif libvirt_status[0] == host_thread.lvirt_module.VIR_DOMAIN_SHUTOFF:
1542                     new_status = "INACTIVE"
1543                 elif libvirt_status[0] == host_thread.lvirt_module.VIR_DOMAIN_CRASHED:
1544                     new_status = "ERROR"
1545                 else:
1546                     new_status = None
1547                 domain_dict[uuid] = new_status
1548             conn.close()
1549         except host_thread.lvirt_module.libvirtError as e:
1550             print self.name, ": get_state() Exception '", e.get_error_message()
1551             return
1552
1553         for server_id, current_status in self.server_status.iteritems():
1554             new_status = None
1555             if server_id in domain_dict:
1556                 new_status = domain_dict[server_id]
1557             else:
1558                 new_status = "INACTIVE"
1559
1560             if new_status == None or new_status == current_status:
1561                 continue
1562             if new_status == 'INACTIVE' and current_status == 'ERROR':
1563                 continue #keep ERROR status, because obviously this machine is not running
1564             #change status
1565             print self.name, ": server ", server_id, "status change from ", current_status, "to", new_status
1566             STATUS={'progress':100, 'status':new_status}
1567             if new_status == 'ERROR':
1568                 STATUS['last_error'] = 'machine has crashed'
1569             self.db_lock.acquire()
1570             r,_ = self.db.update_rows('instances', STATUS, {'uuid':server_id}, log=False)
1571             self.db_lock.release()
1572             if r>=0:
1573                 self.server_status[server_id] = new_status
1574
1575     def action_on_server(self, req, last_retry=True):
1576         '''Perform an action on a req
1577         Attributes:
1578             req: dictionary that contain:
1579                 server properties: 'uuid','name','tenant_id','status'
1580                 action: 'action'
1581                 host properties: 'user', 'ip_name'
1582         return (error, text)
1583              0: No error. VM is updated to new state,
1584             -1: Invalid action, as trying to pause a PAUSED VM
1585             -2: Error accessing host
1586             -3: VM nor present
1587             -4: Error at DB access
1588             -5: Error while trying to perform action. VM is updated to ERROR
1589         '''
1590         server_id = req['uuid']
1591         conn = None
1592         new_status = None
1593         old_status = req['status']
1594         last_error = None
1595
1596         if self.test:
1597             if 'terminate' in req['action']:
1598                 new_status = 'deleted'
1599             elif 'shutoff' in req['action'] or 'shutdown' in req['action'] or 'forceOff' in req['action']:
1600                 if req['status']!='ERROR':
1601                     time.sleep(5)
1602                     new_status = 'INACTIVE'
1603             elif 'start' in req['action']  and req['status']!='ERROR':      new_status = 'ACTIVE'
1604             elif 'resume' in req['action'] and req['status']!='ERROR' and req['status']!='INACTIVE' :     new_status = 'ACTIVE'
1605             elif 'pause' in req['action']  and req['status']!='ERROR':      new_status = 'PAUSED'
1606             elif 'reboot' in req['action'] and req['status']!='ERROR':     new_status = 'ACTIVE'
1607             elif 'rebuild' in req['action']:
1608                 time.sleep(random.randint(20,150))
1609                 new_status = 'ACTIVE'
1610             elif 'createImage' in req['action']:
1611                 time.sleep(5)
1612                 self.create_image(None, req)
1613         else:
1614             try:
1615                 conn = host_thread.lvirt_module.open("qemu+ssh://"+self.user+"@"+self.host+"/system")
1616                 try:
1617                     dom = conn.lookupByUUIDString(server_id)
1618                 except host_thread.lvirt_module.libvirtError as e:
1619                     text = e.get_error_message()
1620                     if 'LookupByUUIDString' in text or 'Domain not found' in text or 'No existe un dominio coincidente' in text:
1621                         dom = None
1622                     else:
1623                         print self.name, ": action_on_server(",server_id,") libvirt exception:", text
1624                         raise e
1625
1626                 if 'forceOff' in req['action']:
1627                     if dom == None:
1628                         print self.name, ": action_on_server(",server_id,") domain not running"
1629                     else:
1630                         try:
1631                             print self.name, ": sending DESTROY to server", server_id
1632                             dom.destroy()
1633                         except Exception as e:
1634                             if "domain is not running" not in e.get_error_message():
1635                                 print self.name, ": action_on_server(",server_id,") Exception while sending force off:", e.get_error_message()
1636                                 last_error =  'action_on_server Exception while destroy: ' + e.get_error_message()
1637                                 new_status = 'ERROR'
1638
1639                 elif 'terminate' in req['action']:
1640                     if dom == None:
1641                         print self.name, ": action_on_server(",server_id,") domain not running"
1642                         new_status = 'deleted'
1643                     else:
1644                         try:
1645                             if req['action']['terminate'] == 'force':
1646                                 print self.name, ": sending DESTROY to server", server_id
1647                                 dom.destroy()
1648                                 new_status = 'deleted'
1649                             else:
1650                                 print self.name, ": sending SHUTDOWN to server", server_id
1651                                 dom.shutdown()
1652                                 self.pending_terminate_server.append( (time.time()+10,server_id) )
1653                         except Exception as e:
1654                             print self.name, ": action_on_server(",server_id,") Exception while destroy:", e.get_error_message()
1655                             last_error =  'action_on_server Exception while destroy: ' + e.get_error_message()
1656                             new_status = 'ERROR'
1657                             if "domain is not running" in e.get_error_message():
1658                                 try:
1659                                     dom.undefine()
1660                                     new_status = 'deleted'
1661                                 except Exception:
1662                                     print self.name, ": action_on_server(",server_id,") Exception while undefine:", e.get_error_message()
1663                                     last_error =  'action_on_server Exception2 while undefine:', e.get_error_message()
1664                             #Exception: 'virDomainDetachDevice() failed'
1665                     if new_status=='deleted':
1666                         if server_id in self.server_status:
1667                             del self.server_status[server_id]
1668                         if req['uuid'] in self.localinfo['server_files']:
1669                             for file_ in self.localinfo['server_files'][ req['uuid'] ].values():
1670                                 try:
1671                                     self.delete_file(file_['source file'])
1672                                 except Exception:
1673                                     pass
1674                             del self.localinfo['server_files'][ req['uuid'] ]
1675                             self.localinfo_dirty = True
1676
1677                 elif 'shutoff' in req['action'] or 'shutdown' in req['action']:
1678                     try:
1679                         if dom == None:
1680                             print self.name, ": action_on_server(",server_id,") domain not running"
1681                         else:
1682                             dom.shutdown()
1683 #                        new_status = 'INACTIVE'
1684                         #TODO: check status for changing at database
1685                     except Exception as e:
1686                         new_status = 'ERROR'
1687                         print self.name, ": action_on_server(",server_id,") Exception while shutdown:", e.get_error_message()
1688                         last_error =  'action_on_server Exception while shutdown: ' + e.get_error_message()
1689
1690                 elif 'rebuild' in req['action']:
1691                     if dom != None:
1692                         dom.destroy()
1693                     r = self.launch_server(conn, req, True, None)
1694                     if r[0] <0:
1695                         new_status = 'ERROR'
1696                         last_error = r[1]
1697                     else:
1698                         new_status = 'ACTIVE'
1699                 elif 'start' in req['action']:
1700                     # The instance is only create in DB but not yet at libvirt domain, needs to be create
1701                     rebuild = True if req['action']['start'] == 'rebuild'  else False
1702                     r = self.launch_server(conn, req, rebuild, dom)
1703                     if r[0] <0:
1704                         new_status = 'ERROR'
1705                         last_error = r[1]
1706                     else:
1707                         new_status = 'ACTIVE'
1708
1709                 elif 'resume' in req['action']:
1710                     try:
1711                         if dom == None:
1712                             pass
1713                         else:
1714                             dom.resume()
1715 #                            new_status = 'ACTIVE'
1716                     except Exception as e:
1717                         print self.name, ": action_on_server(",server_id,") Exception while resume:", e.get_error_message()
1718
1719                 elif 'pause' in req['action']:
1720                     try:
1721                         if dom == None:
1722                             pass
1723                         else:
1724                             dom.suspend()
1725 #                            new_status = 'PAUSED'
1726                     except Exception as e:
1727                         print self.name, ": action_on_server(",server_id,") Exception while pause:", e.get_error_message()
1728
1729                 elif 'reboot' in req['action']:
1730                     try:
1731                         if dom == None:
1732                             pass
1733                         else:
1734                             dom.reboot()
1735                         print self.name, ": action_on_server(",server_id,") reboot:"
1736                         #new_status = 'ACTIVE'
1737                     except Exception as e:
1738                         print self.name, ": action_on_server(",server_id,") Exception while reboot:", e.get_error_message()
1739                 elif 'createImage' in req['action']:
1740                     self.create_image(dom, req)
1741
1742
1743                 conn.close()
1744             except host_thread.lvirt_module.libvirtError as e:
1745                 if conn is not None: conn.close()
1746                 text = e.get_error_message()
1747                 new_status = "ERROR"
1748                 last_error = text
1749                 print self.name, ": action_on_server(",server_id,") Exception '", text
1750                 if 'LookupByUUIDString' in text or 'Domain not found' in text or 'No existe un dominio coincidente' in text:
1751                     print self.name, ": action_on_server(",server_id,") Exception removed from host"
1752         #end of if self.test
1753         if new_status ==  None:
1754             return 1
1755
1756         print self.name, ": action_on_server(",server_id,") new status", new_status, last_error
1757         UPDATE = {'progress':100, 'status':new_status}
1758
1759         if new_status=='ERROR':
1760             if not last_retry:  #if there will be another retry do not update database
1761                 return -1
1762             elif 'terminate' in req['action']:
1763                 #PUT a log in the database
1764                 print self.name, ": PANIC deleting server", server_id, last_error
1765                 self.db_lock.acquire()
1766                 self.db.new_row('logs',
1767                             {'uuid':server_id, 'tenant_id':req['tenant_id'], 'related':'instances','level':'panic',
1768                              'description':'PANIC deleting server from host '+self.name+': '+last_error}
1769                         )
1770                 self.db_lock.release()
1771                 if server_id in self.server_status:
1772                     del self.server_status[server_id]
1773                 return -1
1774             else:
1775                 UPDATE['last_error'] = last_error
1776         if new_status != 'deleted' and (new_status != old_status or new_status == 'ERROR') :
1777             self.db_lock.acquire()
1778             self.db.update_rows('instances', UPDATE, {'uuid':server_id}, log=True)
1779             self.server_status[server_id] = new_status
1780             self.db_lock.release()
1781         if new_status == 'ERROR':
1782             return -1
1783         return 1
1784
1785
1786     def restore_iface(self, name, mac, lib_conn=None):
1787         ''' make an ifdown, ifup to restore default parameter of na interface
1788             Params:
1789                 mac: mac address of the interface
1790                 lib_conn: connection to the libvirt, if None a new connection is created
1791             Return 0,None if ok, -1,text if fails
1792         '''
1793         conn=None
1794         ret = 0
1795         error_text=None
1796         if self.test:
1797             print self.name, ": restore_iface '%s' %s" % (name, mac)
1798             return 0, None
1799         try:
1800             if not lib_conn:
1801                 conn = host_thread.lvirt_module.open("qemu+ssh://"+self.user+"@"+self.host+"/system")
1802             else:
1803                 conn = lib_conn
1804
1805             #wait to the pending VM deletion
1806             #TODO.Revise  self.server_forceoff(True)
1807
1808             iface = conn.interfaceLookupByMACString(mac)
1809             iface.destroy()
1810             iface.create()
1811             print self.name, ": restore_iface '%s' %s" % (name, mac)
1812         except host_thread.lvirt_module.libvirtError as e:
1813             error_text = e.get_error_message()
1814             print self.name, ": restore_iface '%s' '%s' libvirt exception: %s" %(name, mac, error_text)
1815             ret=-1
1816         finally:
1817             if lib_conn is None and conn is not None:
1818                 conn.close()
1819         return ret, error_text
1820
1821
1822     def create_image(self,dom, req):
1823         if self.test:
1824             if 'path' in req['action']['createImage']:
1825                 file_dst = req['action']['createImage']['path']
1826             else:
1827                 createImage=req['action']['createImage']
1828                 img_name= createImage['source']['path']
1829                 index=img_name.rfind('/')
1830                 file_dst = self.get_notused_path(img_name[:index+1] + createImage['name'] + '.qcow2')
1831             image_status='ACTIVE'
1832         else:
1833             for retry in (0,1):
1834                 try:
1835                     server_id = req['uuid']
1836                     createImage=req['action']['createImage']
1837                     file_orig = self.localinfo['server_files'][server_id] [ createImage['source']['image_id'] ] ['source file']
1838                     if 'path' in req['action']['createImage']:
1839                         file_dst = req['action']['createImage']['path']
1840                     else:
1841                         img_name= createImage['source']['path']
1842                         index=img_name.rfind('/')
1843                         file_dst = self.get_notused_filename(img_name[:index+1] + createImage['name'] + '.qcow2')
1844
1845                     self.copy_file(file_orig, file_dst)
1846                     qemu_info = self.qemu_get_info(file_orig)
1847                     if 'backing file' in qemu_info:
1848                         for k,v in self.localinfo['files'].items():
1849                             if v==qemu_info['backing file']:
1850                                 self.qemu_change_backing(file_dst, k)
1851                                 break
1852                     image_status='ACTIVE'
1853                     break
1854                 except paramiko.ssh_exception.SSHException as e:
1855                     image_status='ERROR'
1856                     error_text = e.args[0]
1857                     print self.name, "': create_image(",server_id,") ssh Exception:", error_text
1858                     if "SSH session not active" in error_text and retry==0:
1859                         self.ssh_connect()
1860                 except Exception as e:
1861                     image_status='ERROR'
1862                     error_text = str(e)
1863                     print self.name, "': create_image(",server_id,") Exception:", error_text
1864
1865                 #TODO insert a last_error at database
1866         self.db_lock.acquire()
1867         self.db.update_rows('images', {'status':image_status, 'progress': 100, 'path':file_dst},
1868                 {'uuid':req['new_image']['uuid']}, log=True)
1869         self.db_lock.release()
1870
1871     def edit_iface(self, port_id, old_net, new_net):
1872         #This action imply remove and insert interface to put proper parameters
1873         if self.test:
1874             time.sleep(1)
1875         else:
1876         #get iface details
1877             self.db_lock.acquire()
1878             r,c = self.db.get_table(FROM='ports as p join resources_port as rp on p.uuid=rp.port_id',
1879                                     WHERE={'port_id': port_id})
1880             self.db_lock.release()
1881             if r<0:
1882                 print self.name, ": edit_iface(",port_id,") DDBB error:", c
1883                 return
1884             elif r==0:
1885                 print self.name, ": edit_iface(",port_id,") por not found"
1886                 return
1887             port=c[0]
1888             if port["model"]!="VF":
1889                 print self.name, ": edit_iface(",port_id,") ERROR model must be VF"
1890                 return
1891             #create xml detach file
1892             xml=[]
1893             self.xml_level = 2
1894             xml.append("<interface type='hostdev' managed='yes'>")
1895             xml.append("  <mac address='" +port['mac']+ "'/>")
1896             xml.append("  <source>"+ self.pci2xml(port['pci'])+"\n  </source>")
1897             xml.append('</interface>')
1898
1899
1900             try:
1901                 conn=None
1902                 conn = host_thread.lvirt_module.open("qemu+ssh://"+self.user+"@"+self.host+"/system")
1903                 dom = conn.lookupByUUIDString(port["instance_id"])
1904                 if old_net:
1905                     text="\n".join(xml)
1906                     print self.name, ": edit_iface detaching SRIOV interface", text
1907                     dom.detachDeviceFlags(text, flags=host_thread.lvirt_module.VIR_DOMAIN_AFFECT_LIVE)
1908                 if new_net:
1909                     xml[-1] ="  <vlan>   <tag id='" + str(port['vlan']) + "'/>   </vlan>"
1910                     self.xml_level = 1
1911                     xml.append(self.pci2xml(port.get('vpci',None)) )
1912                     xml.append('</interface>')
1913                     text="\n".join(xml)
1914                     print self.name, ": edit_iface attaching SRIOV interface", text
1915                     dom.attachDeviceFlags(text, flags=host_thread.lvirt_module.VIR_DOMAIN_AFFECT_LIVE)
1916
1917             except host_thread.lvirt_module.libvirtError as e:
1918                 text = e.get_error_message()
1919                 print self.name, ": edit_iface(",port["instance_id"],") libvirt exception:", text
1920
1921             finally:
1922                 if conn is not None: conn.close()
1923
1924
1925 def create_server(server, db, db_lock, only_of_ports):
1926     #print "server"
1927     #print "server"
1928     #print server
1929     #print "server"
1930     #print "server"
1931     #try:
1932 #            host_id = server.get('host_id', None)
1933     extended = server.get('extended', None)
1934
1935 #             print '----------------------'
1936 #             print json.dumps(extended, indent=4)
1937
1938     requirements={}
1939     requirements['numa']={'memory':0, 'proc_req_type': 'threads', 'proc_req_nb':0, 'port_list':[], 'sriov_list':[]}
1940     requirements['ram'] = server['flavor'].get('ram', 0)
1941     if requirements['ram']== None:
1942         requirements['ram'] = 0
1943     requirements['vcpus'] = server['flavor'].get('vcpus', 0)
1944     if requirements['vcpus']== None:
1945         requirements['vcpus'] = 0
1946     #If extended is not defined get requirements from flavor
1947     if extended is None:
1948         #If extended is defined in flavor convert to dictionary and use it
1949         if 'extended' in server['flavor'] and  server['flavor']['extended'] != None:
1950             json_acceptable_string = server['flavor']['extended'].replace("'", "\"")
1951             extended = json.loads(json_acceptable_string)
1952         else:
1953             extended = None
1954     #print json.dumps(extended, indent=4)
1955
1956     #For simplicity only one numa VM are supported in the initial implementation
1957     if extended != None:
1958         numas = extended.get('numas', [])
1959         if len(numas)>1:
1960             return (-2, "Multi-NUMA VMs are not supported yet")
1961         #elif len(numas)<1:
1962         #    return (-1, "At least one numa must be specified")
1963
1964         #a for loop is used in order to be ready to multi-NUMA VMs
1965         request = []
1966         for numa in numas:
1967             numa_req = {}
1968             numa_req['memory'] = numa.get('memory', 0)
1969             if 'cores' in numa:
1970                 numa_req['proc_req_nb'] = numa['cores']                     #number of cores or threads to be reserved
1971                 numa_req['proc_req_type'] = 'cores'                         #indicates whether cores or threads must be reserved
1972                 numa_req['proc_req_list'] = numa.get('cores-id', None)      #list of ids to be assigned to the cores or threads
1973             elif 'paired-threads' in numa:
1974                 numa_req['proc_req_nb'] = numa['paired-threads']
1975                 numa_req['proc_req_type'] = 'paired-threads'
1976                 numa_req['proc_req_list'] = numa.get('paired-threads-id', None)
1977             elif 'threads' in numa:
1978                 numa_req['proc_req_nb'] = numa['threads']
1979                 numa_req['proc_req_type'] = 'threads'
1980                 numa_req['proc_req_list'] = numa.get('threads-id', None)
1981             else:
1982                 numa_req['proc_req_nb'] = 0 # by default
1983                 numa_req['proc_req_type'] = 'threads'
1984
1985
1986
1987             #Generate a list of sriov and another for physical interfaces
1988             interfaces = numa.get('interfaces', [])
1989             sriov_list = []
1990             port_list = []
1991             for iface in interfaces:
1992                 iface['bandwidth'] = int(iface['bandwidth'])
1993                 if iface['dedicated'][:3]=='yes':
1994                     port_list.append(iface)
1995                 else:
1996                     sriov_list.append(iface)
1997
1998             #Save lists ordered from more restrictive to less bw requirements
1999             numa_req['sriov_list'] = sorted(sriov_list, key=lambda k: k['bandwidth'], reverse=True)
2000             numa_req['port_list'] = sorted(port_list, key=lambda k: k['bandwidth'], reverse=True)
2001
2002
2003             request.append(numa_req)
2004
2005     #                 print "----------\n"+json.dumps(request[0], indent=4)
2006     #                 print '----------\n\n'
2007
2008         #Search in db for an appropriate numa for each requested numa
2009         #at the moment multi-NUMA VMs are not supported
2010         if len(request)>0:
2011             requirements['numa'].update(request[0])
2012     if requirements['numa']['memory']>0:
2013         requirements['ram']=0  #By the moment I make incompatible ask for both Huge and non huge pages memory
2014     elif requirements['ram']==0:
2015         return (-1, "Memory information not set neither at extended field not at ram")
2016     if requirements['numa']['proc_req_nb']>0:
2017         requirements['vcpus']=0 #By the moment I make incompatible ask for both Isolated and non isolated cpus
2018     elif requirements['vcpus']==0:
2019         return (-1, "Processor information not set neither at extended field not at vcpus")
2020
2021
2022     db_lock.acquire()
2023     result, content = db.get_numas(requirements, server.get('host_id', None), only_of_ports)
2024     db_lock.release()
2025
2026     if result == -1:
2027         return (-1, content)
2028
2029     numa_id = content['numa_id']
2030     host_id = content['host_id']
2031
2032     #obtain threads_id and calculate pinning
2033     cpu_pinning = []
2034     reserved_threads=[]
2035     if requirements['numa']['proc_req_nb']>0:
2036         db_lock.acquire()
2037         result, content = db.get_table(FROM='resources_core',
2038                                        SELECT=('id','core_id','thread_id'),
2039                                        WHERE={'numa_id':numa_id,'instance_id': None, 'status':'ok'} )
2040         db_lock.release()
2041         if result <= 0:
2042             print content
2043             return -1, content
2044
2045         #convert rows to a dictionary indexed by core_id
2046         cores_dict = {}
2047         for row in content:
2048             if not row['core_id'] in cores_dict:
2049                 cores_dict[row['core_id']] = []
2050             cores_dict[row['core_id']].append([row['thread_id'],row['id']])
2051
2052         #In case full cores are requested
2053         paired = 'N'
2054         if requirements['numa']['proc_req_type'] == 'cores':
2055             #Get/create the list of the vcpu_ids
2056             vcpu_id_list = requirements['numa']['proc_req_list']
2057             if vcpu_id_list == None:
2058                 vcpu_id_list = range(0,int(requirements['numa']['proc_req_nb']))
2059
2060             for threads in cores_dict.itervalues():
2061                 #we need full cores
2062                 if len(threads) != 2:
2063                     continue
2064
2065                 #set pinning for the first thread
2066                 cpu_pinning.append( [ vcpu_id_list.pop(0), threads[0][0], threads[0][1] ] )
2067
2068                 #reserve so it is not used the second thread
2069                 reserved_threads.append(threads[1][1])
2070
2071                 if len(vcpu_id_list) == 0:
2072                     break
2073
2074         #In case paired threads are requested
2075         elif requirements['numa']['proc_req_type'] == 'paired-threads':
2076             paired = 'Y'
2077             #Get/create the list of the vcpu_ids
2078             if requirements['numa']['proc_req_list'] != None:
2079                 vcpu_id_list = []
2080                 for pair in requirements['numa']['proc_req_list']:
2081                     if len(pair)!=2:
2082                         return -1, "Field paired-threads-id not properly specified"
2083                         return
2084                     vcpu_id_list.append(pair[0])
2085                     vcpu_id_list.append(pair[1])
2086             else:
2087                 vcpu_id_list = range(0,2*int(requirements['numa']['proc_req_nb']))
2088
2089             for threads in cores_dict.itervalues():
2090                 #we need full cores
2091                 if len(threads) != 2:
2092                     continue
2093                 #set pinning for the first thread
2094                 cpu_pinning.append([vcpu_id_list.pop(0), threads[0][0], threads[0][1]])
2095
2096                 #set pinning for the second thread
2097                 cpu_pinning.append([vcpu_id_list.pop(0), threads[1][0], threads[1][1]])
2098
2099                 if len(vcpu_id_list) == 0:
2100                     break
2101
2102         #In case normal threads are requested
2103         elif requirements['numa']['proc_req_type'] == 'threads':
2104             #Get/create the list of the vcpu_ids
2105             vcpu_id_list = requirements['numa']['proc_req_list']
2106             if vcpu_id_list == None:
2107                 vcpu_id_list = range(0,int(requirements['numa']['proc_req_nb']))
2108
2109             for threads_index in sorted(cores_dict, key=lambda k: len(cores_dict[k])):
2110                 threads = cores_dict[threads_index]
2111                 #set pinning for the first thread
2112                 cpu_pinning.append([vcpu_id_list.pop(0), threads[0][0], threads[0][1]])
2113
2114                 #if exists, set pinning for the second thread
2115                 if len(threads) == 2 and len(vcpu_id_list) != 0:
2116                     cpu_pinning.append([vcpu_id_list.pop(0), threads[1][0], threads[1][1]])
2117
2118                 if len(vcpu_id_list) == 0:
2119                     break
2120
2121         #Get the source pci addresses for the selected numa
2122         used_sriov_ports = []
2123         for port in requirements['numa']['sriov_list']:
2124             db_lock.acquire()
2125             result, content = db.get_table(FROM='resources_port', SELECT=('id', 'pci', 'mac'),WHERE={'numa_id':numa_id,'root_id': port['port_id'], 'port_id': None, 'Mbps_used': 0} )
2126             db_lock.release()
2127             if result <= 0:
2128                 print content
2129                 return -1, content
2130             for row in content:
2131                 if row['id'] in used_sriov_ports or row['id']==port['port_id']:
2132                     continue
2133                 port['pci'] = row['pci']
2134                 if 'mac_address' not in port:
2135                     port['mac_address'] = row['mac']
2136                 del port['mac']
2137                 port['port_id']=row['id']
2138                 port['Mbps_used'] = port['bandwidth']
2139                 used_sriov_ports.append(row['id'])
2140                 break
2141
2142         for port in requirements['numa']['port_list']:
2143             port['Mbps_used'] = None
2144             if port['dedicated'] != "yes:sriov":
2145                 port['mac_address'] = port['mac']
2146                 del port['mac']
2147                 continue
2148             db_lock.acquire()
2149             result, content = db.get_table(FROM='resources_port', SELECT=('id', 'pci', 'mac', 'Mbps'),WHERE={'numa_id':numa_id,'root_id': port['port_id'], 'port_id': None, 'Mbps_used': 0} )
2150             db_lock.release()
2151             if result <= 0:
2152                 print content
2153                 return -1, content
2154             port['Mbps_used'] = content[0]['Mbps']
2155             for row in content:
2156                 if row['id'] in used_sriov_ports or row['id']==port['port_id']:
2157                     continue
2158                 port['pci'] = row['pci']
2159                 if 'mac_address' not in port:
2160                     port['mac_address'] = row['mac']  # mac cannot be set to passthrough ports
2161                 del port['mac']
2162                 port['port_id']=row['id']
2163                 used_sriov_ports.append(row['id'])
2164                 break
2165
2166     #             print '2. Physical ports assignation:'+json.dumps(requirements['port_list'], indent=4)
2167     #             print '2. SR-IOV assignation:'+json.dumps(requirements['sriov_list'], indent=4)
2168
2169     server['host_id'] = host_id
2170
2171
2172     #Generate dictionary for saving in db the instance resources
2173     resources = {}
2174     resources['bridged-ifaces'] = []
2175
2176     numa_dict = {}
2177     numa_dict['interfaces'] = []
2178
2179     numa_dict['interfaces'] += requirements['numa']['port_list']
2180     numa_dict['interfaces'] += requirements['numa']['sriov_list']
2181
2182     #Check bridge information
2183     unified_dataplane_iface=[]
2184     unified_dataplane_iface += requirements['numa']['port_list']
2185     unified_dataplane_iface += requirements['numa']['sriov_list']
2186
2187     for control_iface in server.get('networks', []):
2188         control_iface['net_id']=control_iface.pop('uuid')
2189         #Get the brifge name
2190         db_lock.acquire()
2191         result, content = db.get_table(FROM='nets',
2192                                        SELECT=('name', 'type', 'vlan', 'provider', 'enable_dhcp',
2193                                                  'dhcp_first_ip', 'dhcp_last_ip', 'cidr'),
2194                                        WHERE={'uuid': control_iface['net_id']})
2195         db_lock.release()
2196         if result < 0:
2197             pass
2198         elif result==0:
2199             return -1, "Error at field netwoks: Not found any network wit uuid %s" % control_iface['net_id']
2200         else:
2201             network=content[0]
2202             if control_iface.get("type", 'virtual') == 'virtual':
2203                 if network['type']!='bridge_data' and network['type']!='bridge_man':
2204                     return -1, "Error at field netwoks: network uuid %s for control interface is not of type bridge_man or bridge_data" % control_iface['net_id']
2205                 resources['bridged-ifaces'].append(control_iface)
2206                 if network.get("provider") and network["provider"][0:3] == "OVS":
2207                     control_iface["type"] = "instance:ovs"
2208                 else:
2209                     control_iface["type"] = "instance:bridge"
2210                 if network.get("vlan"):
2211                     control_iface["vlan"] = network["vlan"]
2212
2213                 if network.get("enable_dhcp") == 'true':
2214                     control_iface["enable_dhcp"] = network.get("enable_dhcp")
2215                     control_iface["dhcp_first_ip"] = network["dhcp_first_ip"]
2216                     control_iface["dhcp_last_ip"] = network["dhcp_last_ip"]
2217                     control_iface["cidr"] = network["cidr"]
2218             else:
2219                 if network['type']!='data' and network['type']!='ptp':
2220                     return -1, "Error at field netwoks: network uuid %s for dataplane interface is not of type data or ptp" % control_iface['net_id']
2221                 #dataplane interface, look for it in the numa tree and asign this network
2222                 iface_found=False
2223                 for dataplane_iface in numa_dict['interfaces']:
2224                     if dataplane_iface['name'] == control_iface.get("name"):
2225                         if (dataplane_iface['dedicated'] == "yes" and control_iface["type"] != "PF") or \
2226                             (dataplane_iface['dedicated'] == "no" and control_iface["type"] != "VF") or \
2227                             (dataplane_iface['dedicated'] == "yes:sriov" and control_iface["type"] != "VFnotShared") :
2228                                 return -1, "Error at field netwoks: mismatch at interface '%s' from flavor 'dedicated=%s' and networks 'type=%s'" % \
2229                                     (control_iface.get("name"), dataplane_iface['dedicated'], control_iface["type"])
2230                         dataplane_iface['uuid'] = control_iface['net_id']
2231                         if dataplane_iface['dedicated'] == "no":
2232                             dataplane_iface['vlan'] = network['vlan']
2233                         if dataplane_iface['dedicated'] != "yes" and control_iface.get("mac_address"):
2234                             dataplane_iface['mac_address'] = control_iface.get("mac_address")
2235                         if control_iface.get("vpci"):
2236                             dataplane_iface['vpci'] = control_iface.get("vpci")
2237                         iface_found=True
2238                         break
2239                 if not iface_found:
2240                     return -1, "Error at field netwoks: interface name %s from network not found at flavor" % control_iface.get("name")
2241
2242     resources['host_id'] = host_id
2243     resources['image_id'] = server['image_id']
2244     resources['flavor_id'] = server['flavor_id']
2245     resources['tenant_id'] = server['tenant_id']
2246     resources['ram'] = requirements['ram']
2247     resources['vcpus'] = requirements['vcpus']
2248     resources['status'] = 'CREATING'
2249
2250     if 'description' in server: resources['description'] = server['description']
2251     if 'name' in server: resources['name'] = server['name']
2252
2253     resources['extended'] = {}                          #optional
2254     resources['extended']['numas'] = []
2255     numa_dict['numa_id'] = numa_id
2256     numa_dict['memory'] = requirements['numa']['memory']
2257     numa_dict['cores'] = []
2258
2259     for core in cpu_pinning:
2260         numa_dict['cores'].append({'id': core[2], 'vthread': core[0], 'paired': paired})
2261     for core in reserved_threads:
2262         numa_dict['cores'].append({'id': core})
2263     resources['extended']['numas'].append(numa_dict)
2264     if extended!=None and 'devices' in extended:   #TODO allow extra devices without numa
2265         resources['extended']['devices'] = extended['devices']
2266
2267
2268     print '===================================={'
2269     print json.dumps(resources, indent=4)
2270     print '====================================}'
2271
2272     return 0, resources
2273