X-Git-Url: https://osm.etsi.org/gitweb/?a=blobdiff_plain;f=osm_openvim%2Fhost_thread.py;h=00d253e02408f3c93c6989c327ec8c9c13a65252;hb=caeb224da9840534e46ae0e8f5e194575521c668;hp=d8bca2e2e1ca13d091a34566d29610672f520cb8;hpb=9f6571090b203922cabb0382226be0fa48d6e046;p=osm%2Fopenvim.git diff --git a/osm_openvim/host_thread.py b/osm_openvim/host_thread.py index d8bca2e..00d253e 100644 --- a/osm_openvim/host_thread.py +++ b/osm_openvim/host_thread.py @@ -34,26 +34,21 @@ import threading import time import Queue import paramiko -from jsonschema import validate as js_v, exceptions as js_e -#import libvirt +# import subprocess +# import libvirt import imp -from vim_schema import localinfo_schema, hostinfo_schema import random import os - -#TODO: insert a logging system - -# from logging import Logger -# import auxiliary_functions as af - -# TODO: insert a logging system +import logging +from jsonschema import validate as js_v, exceptions as js_e +from vim_schema import localinfo_schema, hostinfo_schema class host_thread(threading.Thread): lvirt_module = None def __init__(self, name, host, user, db, db_lock, test, image_path, host_id, version, develop_mode, - develop_bridge_iface): + develop_bridge_iface, password=None, keyfile = None, logger_name=None, debug=None): '''Init a thread. Arguments: 'id' number of thead @@ -68,6 +63,9 @@ class host_thread(threading.Thread): self.db = db self.db_lock = db_lock self.test = test + self.password = password + self.keyfile = keyfile + self.localinfo_dirty = False if not test and not host_thread.lvirt_module: try: @@ -75,11 +73,19 @@ class host_thread(threading.Thread): host_thread.lvirt_module = imp.load_module("libvirt", *module_info) except (IOError, ImportError) as e: raise ImportError("Cannot import python-libvirt. Openvim not properly installed" +str(e)) + if logger_name: + self.logger_name = logger_name + else: + self.logger_name = "openvim.host."+name + self.logger = logging.getLogger(self.logger_name) + if debug: + self.logger.setLevel(getattr(logging, debug)) self.develop_mode = develop_mode self.develop_bridge_iface = develop_bridge_iface self.image_path = image_path + self.empty_image_path = image_path self.host_id = host_id self.version = version @@ -95,65 +101,70 @@ class host_thread(threading.Thread): self.queueLock = threading.Lock() self.taskQueue = Queue.Queue(2000) self.ssh_conn = None + self.lvirt_conn_uri = "qemu+ssh://{user}@{host}/system?no_tty=1&no_verify=1".format( + user=self.user, host=self.host) + if keyfile: + self.lvirt_conn_uri += "&keyfile=" + keyfile def ssh_connect(self): try: - #Connect SSH + # Connect SSH self.ssh_conn = paramiko.SSHClient() self.ssh_conn.set_missing_host_key_policy(paramiko.AutoAddPolicy()) self.ssh_conn.load_system_host_keys() - self.ssh_conn.connect(self.host, username=self.user, timeout=10) #, None) + self.ssh_conn.connect(self.host, username=self.user, password=self.password, key_filename=self.keyfile, + timeout=10) #, None) except paramiko.ssh_exception.SSHException as e: text = e.args[0] - print self.name, ": ssh_connect ssh Exception:", text - + self.logger.error("ssh_connect ssh Exception: " + text) + def load_localinfo(self): if not self.test: try: - #Connect SSH + # Connect SSH self.ssh_connect() - + command = 'mkdir -p ' + self.image_path - #print self.name, ': command:', command + # print self.name, ': command:', command (_, stdout, stderr) = self.ssh_conn.exec_command(command) content = stderr.read() if len(content) > 0: - print self.name, ': command:', command, "stderr:", content + self.logger.error("command: '%s' stderr: '%s'", command, content) command = 'cat ' + self.image_path + '/.openvim.yaml' - #print self.name, ': command:', command + # print self.name, ': command:', command (_, stdout, stderr) = self.ssh_conn.exec_command(command) content = stdout.read() if len(content) == 0: - print self.name, ': command:', command, "stderr:", stderr.read() - raise paramiko.ssh_exception.SSHException("Error empty file ") + self.logger.error("command: '%s' stderr='%s'", command, stderr.read()) + raise paramiko.ssh_exception.SSHException("Error empty file, command: '{}'".format(command)) self.localinfo = yaml.load(content) js_v(self.localinfo, localinfo_schema) - self.localinfo_dirty=False + self.localinfo_dirty = False if 'server_files' not in self.localinfo: self.localinfo['server_files'] = {} - print self.name, ': localinfo load from host' + self.logger.debug("localinfo load from host") return - + except paramiko.ssh_exception.SSHException as e: text = e.args[0] - print self.name, ": load_localinfo ssh Exception:", text + self.logger.error("load_localinfo ssh Exception: " + text) except host_thread.lvirt_module.libvirtError as e: text = e.get_error_message() - print self.name, ": load_localinfo libvirt Exception:", text + self.logger.error("load_localinfo libvirt Exception: " + text) except yaml.YAMLError as exc: text = "" if hasattr(exc, 'problem_mark'): mark = exc.problem_mark text = " at position: (%s:%s)" % (mark.line+1, mark.column+1) - print self.name, ": load_localinfo yaml format Exception", text + self.logger.error("load_localinfo yaml format Exception " + text) except js_e.ValidationError as e: text = "" if len(e.path)>0: text=" at '" + ":".join(map(str, e.path))+"'" - print self.name, ": load_localinfo format Exception:", text, e.message + self.logger.error("load_localinfo format Exception: %s %s", text, str(e)) except Exception as e: text = str(e) - print self.name, ": load_localinfo Exception:", text + self.logger.error("load_localinfo Exception: " + text) #not loaded, insert a default data and force saving by activating dirty flag self.localinfo = {'files':{}, 'server_files':{} } @@ -173,32 +184,32 @@ class host_thread(threading.Thread): (_, stdout, stderr) = self.ssh_conn.exec_command(command) content = stdout.read() if len(content) == 0: - print self.name, ': command:', command, "stderr:", stderr.read() + self.logger.error("command: '%s' stderr: '%s'", command, stderr.read()) raise paramiko.ssh_exception.SSHException("Error empty file ") self.hostinfo = yaml.load(content) js_v(self.hostinfo, hostinfo_schema) - print self.name, ': hostlinfo load from host', self.hostinfo + self.logger.debug("hostlinfo load from host " + str(self.hostinfo)) return except paramiko.ssh_exception.SSHException as e: text = e.args[0] - print self.name, ": load_hostinfo ssh Exception:", text + self.logger.error("load_hostinfo ssh Exception: " + text) except host_thread.lvirt_module.libvirtError as e: text = e.get_error_message() - print self.name, ": load_hostinfo libvirt Exception:", text + self.logger.error("load_hostinfo libvirt Exception: " + text) except yaml.YAMLError as exc: text = "" if hasattr(exc, 'problem_mark'): mark = exc.problem_mark text = " at position: (%s:%s)" % (mark.line+1, mark.column+1) - print self.name, ": load_hostinfo yaml format Exception", text + self.logger.error("load_hostinfo yaml format Exception " + text) except js_e.ValidationError as e: text = "" if len(e.path)>0: text=" at '" + ":".join(map(str, e.path))+"'" - print self.name, ": load_hostinfo format Exception:", text, e.message + self.logger.error("load_hostinfo format Exception: %s %s", text, e.message) except Exception as e: text = str(e) - print self.name, ": load_hostinfo Exception:", text + self.logger.error("load_hostinfo Exception: " + text) #not loaded, insert a default data self.hostinfo = None @@ -212,30 +223,31 @@ class host_thread(threading.Thread): tries-=1 try: - command = 'cat > ' + self.image_path + '/.openvim.yaml' - print self.name, ': command:', command + command = 'cat > ' + self.image_path + '/.openvim.yaml' + self.logger.debug("command:" + command) (stdin, _, _) = self.ssh_conn.exec_command(command) yaml.safe_dump(self.localinfo, stdin, explicit_start=True, indent=4, default_flow_style=False, tags=False, encoding='utf-8', allow_unicode=True) + self.localinfo_dirty = False break #while tries - + except paramiko.ssh_exception.SSHException as e: text = e.args[0] - print self.name, ": save_localinfo ssh Exception:", text + self.logger.error("save_localinfo ssh Exception: " + text) if "SSH session not active" in text: self.ssh_connect() except host_thread.lvirt_module.libvirtError as e: text = e.get_error_message() - print self.name, ": save_localinfo libvirt Exception:", text + self.logger.error("save_localinfo libvirt Exception: " + text) except yaml.YAMLError as exc: text = "" if hasattr(exc, 'problem_mark'): mark = exc.problem_mark text = " at position: (%s:%s)" % (mark.line+1, mark.column+1) - print self.name, ": save_localinfo yaml format Exception", text + self.logger.error("save_localinfo yaml format Exception " + text) except Exception as e: text = str(e) - print self.name, ": save_localinfo Exception:", text + self.logger.error("save_localinfo Exception: " + text) def load_servers_from_db(self): self.db_lock.acquire() @@ -244,7 +256,7 @@ class host_thread(threading.Thread): self.server_status = {} if r<0: - print self.name, ": Error getting data from database:", c + self.logger.error("Error getting data from database: " + c) return for server in c: self.server_status[ server['uuid'] ] = server['status'] @@ -271,10 +283,10 @@ class host_thread(threading.Thread): if uuid not in self.server_status: for localfile in images.values(): try: - print self.name, ": deleting file '%s' of unused server '%s'" %(localfile['source file'], uuid) + self.logger.debug("deleting file '%s' of unused server '%s'", localfile['source file'], uuid) self.delete_file(localfile['source file']) except paramiko.ssh_exception.SSHException as e: - print self.name, ": Exception deleting file '%s': %s" %(localfile['source file'], str(e)) + self.logger.error("Exception deleting file '%s': %s", localfile['source file'], str(e)) del self.localinfo['server_files'][uuid] self.localinfo_dirty = True @@ -294,71 +306,76 @@ class host_thread(threading.Thread): self.load_servers_from_db() self.delete_unused_files() while True: - self.queueLock.acquire() - if not self.taskQueue.empty(): - task = self.taskQueue.get() - else: - task = None - self.queueLock.release() - - if task is None: - now=time.time() - if self.localinfo_dirty: - self.save_localinfo() - elif self.next_update_server_status < now: - self.update_servers_status() - self.next_update_server_status = now + 5 - elif len(self.pending_terminate_server)>0 and self.pending_terminate_server[0][0]=0: - break - elif task[0] == 'image': - pass - elif task[0] == 'exit': - print self.name, ": processing task exit" - self.terminate() - return 0 - elif task[0] == 'reload': - print self.name, ": processing task reload terminating and relaunching" - self.terminate() - break - elif task[0] == 'edit-iface': - print self.name, ": processing task edit-iface port=%s, old_net=%s, new_net=%s" % (task[1], task[2], task[3]) - self.edit_iface(task[1], task[2], task[3]) - elif task[0] == 'restore-iface': - print self.name, ": processing task restore-iface %s mac=%s" % (task[1], task[2]) - self.restore_iface(task[1], task[2]) - elif task[0] == 'new-ovsbridge': - print self.name, ": Creating compute OVS bridge" - self.create_ovs_bridge() - elif task[0] == 'new-vxlan': - print self.name, ": Creating vxlan tunnel=" + task[1] + ", remote ip=" + task[2] - self.create_ovs_vxlan_tunnel(task[1], task[2]) - elif task[0] == 'del-ovsbridge': - print self.name, ": Deleting OVS bridge" - self.delete_ovs_bridge() - elif task[0] == 'del-vxlan': - print self.name, ": Deleting vxlan " + task[1] + " tunnel" - self.delete_ovs_vxlan_tunnel(task[1]) - elif task[0] == 'create-ovs-bridge-port': - print self.name, ": Adding port ovim-" + task[1] + " to OVS bridge" - self.create_ovs_bridge_port(task[1]) - elif task[0] == 'del-ovs-port': - print self.name, ": Delete bridge attached to ovs port vlan {} net {}".format(task[1], task[2]) - self.delete_bridge_port_attached_to_ovs(task[1], task[2]) - else: - print self.name, ": unknown task", task - + task = None + self.queueLock.release() + + if task is None: + now=time.time() + if self.localinfo_dirty: + self.save_localinfo() + elif self.next_update_server_status < now: + self.update_servers_status() + self.next_update_server_status = now + 5 + elif len(self.pending_terminate_server)>0 and self.pending_terminate_server[0][0]= 0: + break + elif task[0] == 'image': + pass + elif task[0] == 'exit': + self.logger.debug("processing task exit") + self.terminate() + return 0 + elif task[0] == 'reload': + self.logger.debug("processing task reload terminating and relaunching") + self.terminate() + break + elif task[0] == 'edit-iface': + self.logger.debug("processing task edit-iface port={}, old_net={}, new_net={}".format( + task[1], task[2], task[3])) + self.edit_iface(task[1], task[2], task[3]) + elif task[0] == 'restore-iface': + self.logger.debug("processing task restore-iface={} mac={}".format(task[1], task[2])) + self.restore_iface(task[1], task[2]) + elif task[0] == 'new-ovsbridge': + self.logger.debug("Creating compute OVS bridge") + self.create_ovs_bridge() + elif task[0] == 'new-vxlan': + self.logger.debug("Creating vxlan tunnel='{}', remote ip='{}'".format(task[1], task[2])) + self.create_ovs_vxlan_tunnel(task[1], task[2]) + elif task[0] == 'del-ovsbridge': + self.logger.debug("Deleting OVS bridge") + self.delete_ovs_bridge() + elif task[0] == 'del-vxlan': + self.logger.debug("Deleting vxlan {} tunnel".format(task[1])) + self.delete_ovs_vxlan_tunnel(task[1]) + elif task[0] == 'create-ovs-bridge-port': + self.logger.debug("Adding port ovim-{} to OVS bridge".format(task[1])) + self.create_ovs_bridge_port(task[1]) + elif task[0] == 'del-ovs-port': + self.logger.debug("Delete bridge attached to ovs port vlan {} net {}".format(task[1], task[2])) + self.delete_bridge_port_attached_to_ovs(task[1], task[2]) + else: + self.logger.debug("unknown task " + str(task)) + + except Exception as e: + self.logger.critical("Unexpected exception at run: " + str(e), exc_info=True) + def server_forceoff(self, wait_until_finished=False): while len(self.pending_terminate_server)>0: now = time.time() @@ -384,8 +401,8 @@ class host_thread(threading.Thread): self.ssh_conn.close() except Exception as e: text = str(e) - print self.name, ": terminate Exception:", text - print self.name, ": exit from host_thread" + self.logger.error("terminate Exception: " + text) + self.logger.debug("exit from host_thread") def get_local_iface_name(self, generic_name): if self.hostinfo != None and "iface_names" in self.hostinfo and generic_name in self.hostinfo["iface_names"]: @@ -420,8 +437,7 @@ class host_thread(threading.Thread): if topo == None and 'metadata' in dev_list[0]: topo = dev_list[0]['metadata'].get('topology', None) #name - name = server.get('name','') + "_" + server['uuid'] - name = name[:58] #qemu impose a length limit of 59 chars or not start. Using 58 + name = server.get('name', '')[:28] + "_" + server['uuid'][:28] #qemu impose a length limit of 59 chars or not start. Using 58 text += self.inc_tab() + "" + name+ "" #uuid text += self.tab() + "" + server['uuid'] + "" @@ -504,7 +520,7 @@ class host_thread(threading.Thread): if topo == "oneSocket:hyperthreading": if vcpus % 2 != 0: return -1, 'Cannot expose hyperthreading with an odd number of vcpus' - text += self.tab() + " " % vcpus/2 + text += self.tab() + " " % (vcpus/2) elif windows_os or topo == "oneSocket": text += self.tab() + " " % vcpus else: @@ -560,7 +576,7 @@ class host_thread(threading.Thread): #else: # return -1, 'Unknown disk type ' + v['type'] vpci = dev.get('vpci',None) - if vpci == None: + if vpci == None and 'metadata' in dev: vpci = dev['metadata'].get('vpci',None) text += self.pci2xml(vpci) @@ -593,7 +609,7 @@ class host_thread(threading.Thread): result, content = self.db.get_table(FROM='nets', SELECT=('provider',),WHERE={'uuid':v['net_id']} ) self.db_lock.release() if result <= 0: - print "create_xml_server ERROR getting nets",result, content + self.logger.error("create_xml_server ERROR %d getting nets %s", result, content) return -1, content #ALF: Allow by the moment the 'default' bridge net because is confortable for provide internet to VM #I know it is not secure @@ -621,7 +637,7 @@ class host_thread(threading.Thread): elif content[0]['provider'][0:3] == "OVS": vlan = content[0]['provider'].replace('OVS:', '') text += self.tab() + "" + \ - self.inc_tab() + "" + self.inc_tab() + "" else: return -1, 'Unknown Bridge net provider ' + content[0]['provider'] if model!=None: @@ -717,14 +733,20 @@ class host_thread(threading.Thread): :return: True if success """ if self.test: - return - command = 'sudo ovs-vsctl --may-exist add-br br-int -- set Bridge br-int stp_enable=true' - print self.name, ': command:', command - (_, stdout, _) = self.ssh_conn.exec_command(command) - content = stdout.read() - if len(content) == 0: return True - else: + try: + command = 'sudo ovs-vsctl --may-exist add-br br-int -- set Bridge br-int stp_enable=true' + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + content = stdout.read() + if len(content) == 0: + return True + else: + return False + except paramiko.ssh_exception.SSHException as e: + self.logger.error("create_ovs_bridge ssh Exception: " + str(e)) + if "SSH session not active" in str(e): + self.ssh_connect() return False def delete_port_to_ovs_bridge(self, vlan, net_uuid): @@ -736,16 +758,21 @@ class host_thread(threading.Thread): """ if self.test: - return - - port_name = 'ovim-' + vlan - command = 'sudo ovs-vsctl del-port br-int ' + port_name - print self.name, ': command:', command - (_, stdout, _) = self.ssh_conn.exec_command(command) - content = stdout.read() - if len(content) == 0: return True - else: + try: + port_name = 'ovim-' + str(vlan) + command = 'sudo ovs-vsctl del-port br-int ' + port_name + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + content = stdout.read() + if len(content) == 0: + return True + else: + return False + except paramiko.ssh_exception.SSHException as e: + self.logger.error("delete_port_to_ovs_bridge ssh Exception: " + str(e)) + if "SSH session not active" in str(e): + self.ssh_connect() return False def delete_dhcp_server(self, vlan, net_uuid, dhcp_path): @@ -757,28 +784,33 @@ class host_thread(threading.Thread): :return: """ if self.test: - return + return True if not self.is_dhcp_port_free(vlan, net_uuid): return True + try: + net_namespace = 'ovim-' + str(vlan) + dhcp_path = os.path.join(dhcp_path, net_namespace) + pid_file = os.path.join(dhcp_path, 'dnsmasq.pid') - net_namespace = 'ovim-' + vlan - dhcp_path = os.path.join(dhcp_path, net_namespace) - pid_file = os.path.join(dhcp_path, 'dnsmasq.pid') - - command = 'sudo ip netns exec ' + net_namespace + ' cat ' + pid_file - print self.name, ': command:', command - (_, stdout, _) = self.ssh_conn.exec_command(command) - content = stdout.read() + command = 'sudo ip netns exec ' + net_namespace + ' cat ' + pid_file + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + content = stdout.read() - command = 'sudo ip netns exec ' + net_namespace + ' kill -9 ' + content - print self.name, ': command:', command - (_, stdout, _) = self.ssh_conn.exec_command(command) - content = stdout.read() + command = 'sudo ip netns exec ' + net_namespace + ' kill -9 ' + content + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + content = stdout.read() - # if len(content) == 0: - # return True - # else: - # return False + # if len(content) == 0: + # return True + # else: + # return False + except paramiko.ssh_exception.SSHException as e: + self.logger.error("delete_dhcp_server ssh Exception: " + str(e)) + if "SSH session not active" in str(e): + self.ssh_connect() + return False def is_dhcp_port_free(self, host_id, net_uuid): """ @@ -790,7 +822,7 @@ class host_thread(threading.Thread): self.db_lock.acquire() result, content = self.db.get_table( FROM='ports', - WHERE={'p.type': 'instance:ovs', 'p.net_id': net_uuid} + WHERE={'type': 'instance:ovs', 'net_id': net_uuid} ) self.db_lock.release() @@ -827,16 +859,21 @@ class host_thread(threading.Thread): """ if self.test: - return - - port_name = 'ovim-' + vlan - command = 'sudo ovs-vsctl add-port br-int ' + port_name + ' tag=' + vlan - print self.name, ': command:', command - (_, stdout, _) = self.ssh_conn.exec_command(command) - content = stdout.read() - if len(content) == 0: return True - else: + try: + port_name = 'ovim-' + str(vlan) + command = 'sudo ovs-vsctl add-port br-int ' + port_name + ' tag=' + str(vlan) + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + content = stdout.read() + if len(content) == 0: + return True + else: + return False + except paramiko.ssh_exception.SSHException as e: + self.logger.error("add_port_to_ovs_bridge ssh Exception: " + str(e)) + if "SSH session not active" in str(e): + self.ssh_connect() return False def delete_dhcp_port(self, vlan, net_uuid): @@ -848,7 +885,7 @@ class host_thread(threading.Thread): """ if self.test: - return + return True if not self.is_dhcp_port_free(vlan, net_uuid): return True @@ -879,24 +916,28 @@ class host_thread(threading.Thread): """ if self.test: - return - - port_name = 'ovim-' + vlan - command = 'sudo ip link set dev veth0-' + vlan + ' down' - print self.name, ': command:', command - (_, stdout, _) = self.ssh_conn.exec_command(command) - content = stdout.read() - # - # if len(content) != 0: - # return False - - command = 'sudo ifconfig ' + port_name + ' down && sudo brctl delbr ' + port_name - print self.name, ': command:', command - (_, stdout, _) = self.ssh_conn.exec_command(command) - content = stdout.read() - if len(content) == 0: return True - else: + try: + port_name = 'ovim-' + str(vlan) + command = 'sudo ip link set dev veth0-' + str(vlan) + ' down' + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + # content = stdout.read() + # + # if len(content) != 0: + # return False + command = 'sudo ifconfig ' + port_name + ' down && sudo brctl delbr ' + port_name + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + content = stdout.read() + if len(content) == 0: + return True + else: + return False + except paramiko.ssh_exception.SSHException as e: + self.logger.error("delete_linux_bridge ssh Exception: " + str(e)) + if "SSH session not active" in str(e): + self.ssh_connect() return False def create_ovs_bridge_port(self, vlan): @@ -918,45 +959,50 @@ class host_thread(threading.Thread): """ if self.test: - return - - port_name = 'ovim-' + vlan - command = 'sudo brctl show | grep ' + port_name - print self.name, ': command:', command - (_, stdout, _) = self.ssh_conn.exec_command(command) - content = stdout.read() + return True + try: + port_name = 'ovim-' + str(vlan) + command = 'sudo brctl show | grep ' + port_name + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + content = stdout.read() - # if exist nothing to create - # if len(content) == 0: - # return False + # if exist nothing to create + # if len(content) == 0: + # return False - command = 'sudo brctl addbr ' + port_name - print self.name, ': command:', command - (_, stdout, _) = self.ssh_conn.exec_command(command) - content = stdout.read() + command = 'sudo brctl addbr ' + port_name + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + content = stdout.read() - # if len(content) == 0: - # return True - # else: - # return False + # if len(content) == 0: + # return True + # else: + # return False - command = 'sudo brctl stp ' + port_name + ' on' - print self.name, ': command:', command - (_, stdout, _) = self.ssh_conn.exec_command(command) - content = stdout.read() + command = 'sudo brctl stp ' + port_name + ' on' + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + content = stdout.read() - # if len(content) == 0: - # return True - # else: - # return False - command = 'sudo ip link set dev ' + port_name + ' up' - print self.name, ': command:', command - (_, stdout, _) = self.ssh_conn.exec_command(command) - content = stdout.read() + # if len(content) == 0: + # return True + # else: + # return False + command = 'sudo ip link set dev ' + port_name + ' up' + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + content = stdout.read() - if len(content) == 0: - return True - else: + if len(content) == 0: + return True + else: + return False + except paramiko.ssh_exception.SSHException as e: + self.logger.error("create_linux_bridge ssh Exception: " + str(e)) + if "SSH session not active" in str(e): + self.ssh_connect() return False def set_mac_dhcp_server(self, ip, mac, vlan, netmask, dhcp_path): @@ -971,31 +1017,36 @@ class host_thread(threading.Thread): """ if self.test: - return + return True - net_namespace = 'ovim-' + vlan + net_namespace = 'ovim-' + str(vlan) dhcp_path = os.path.join(dhcp_path, net_namespace) dhcp_hostsdir = os.path.join(dhcp_path, net_namespace) if not ip: return False + try: + ip_data = mac.upper() + ',' + ip - ip_data = mac.upper() + ',' + ip - - command = 'sudo ip netns exec ' + net_namespace + ' touch ' + dhcp_hostsdir - print self.name, ': command:', command - (_, stdout, _) = self.ssh_conn.exec_command(command) - content = stdout.read() + command = 'sudo ip netns exec ' + net_namespace + ' touch ' + dhcp_hostsdir + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + content = stdout.read() - command = 'sudo ip netns exec ' + net_namespace + ' sudo bash -ec "echo ' + ip_data + ' >> ' + dhcp_hostsdir + '"' + command = 'sudo ip netns exec ' + net_namespace + ' sudo bash -ec "echo ' + ip_data + ' >> ' + dhcp_hostsdir + '"' - print self.name, ': command:', command - (_, stdout, _) = self.ssh_conn.exec_command(command) - content = stdout.read() + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + content = stdout.read() - if len(content) == 0: - return True - else: + if len(content) == 0: + return True + else: + return False + except paramiko.ssh_exception.SSHException as e: + self.logger.error("set_mac_dhcp_server ssh Exception: " + str(e)) + if "SSH session not active" in str(e): + self.ssh_connect() return False def delete_mac_dhcp_server(self, ip, mac, vlan, dhcp_path): @@ -1010,25 +1061,31 @@ class host_thread(threading.Thread): """ if self.test: - return + return False + try: + net_namespace = 'ovim-' + str(vlan) + dhcp_path = os.path.join(dhcp_path, net_namespace) + dhcp_hostsdir = os.path.join(dhcp_path, net_namespace) - net_namespace = 'ovim-' + vlan - dhcp_path = os.path.join(dhcp_path, net_namespace) - dhcp_hostsdir = os.path.join(dhcp_path, net_namespace) + if not ip: + return False - if not ip: - return False + ip_data = mac.upper() + ',' + ip - ip_data = mac.upper() + ',' + ip + command = 'sudo ip netns exec ' + net_namespace + ' sudo sed -i \'/' + ip_data + '/d\' ' + dhcp_hostsdir + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + content = stdout.read() - command = 'sudo ip netns exec ' + net_namespace + ' sudo sed -i \'/' + ip_data + '/d\' ' + dhcp_hostsdir - print self.name, ': command:', command - (_, stdout, _) = self.ssh_conn.exec_command(command) - content = stdout.read() + if len(content) == 0: + return True + else: + return False - if len(content) == 0: - return True - else: + except paramiko.ssh_exception.SSHException as e: + self.logger.error("set_mac_dhcp_server ssh Exception: " + str(e)) + if "SSH session not active" in str(e): + self.ssh_connect() return False def launch_dhcp_server(self, vlan, ip_range, netmask, dhcp_path, gateway): @@ -1044,46 +1101,51 @@ class host_thread(threading.Thread): """ if self.test: - return - - interface = 'tap-' + vlan - net_namespace = 'ovim-' + vlan - dhcp_path = os.path.join(dhcp_path, net_namespace) - leases_path = os.path.join(dhcp_path, "dnsmasq.leases") - pid_file = os.path.join(dhcp_path, 'dnsmasq.pid') + return True + try: + interface = 'tap-' + str(vlan) + net_namespace = 'ovim-' + str(vlan) + dhcp_path = os.path.join(dhcp_path, net_namespace) + leases_path = os.path.join(dhcp_path, "dnsmasq.leases") + pid_file = os.path.join(dhcp_path, 'dnsmasq.pid') - dhcp_range = ip_range[0] + ',' + ip_range[1] + ',' + netmask + dhcp_range = ip_range[0] + ',' + ip_range[1] + ',' + netmask - command = 'sudo ip netns exec ' + net_namespace + ' mkdir -p ' + dhcp_path - print self.name, ': command:', command - (_, stdout, _) = self.ssh_conn.exec_command(command) - content = stdout.read() + command = 'sudo ip netns exec ' + net_namespace + ' mkdir -p ' + dhcp_path + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + content = stdout.read() - pid_path = os.path.join(dhcp_path, 'dnsmasq.pid') - command = 'sudo ip netns exec ' + net_namespace + ' cat ' + pid_path - print self.name, ': command:', command - (_, stdout, _) = self.ssh_conn.exec_command(command) - content = stdout.read() - # check if pid is runing - pid_status_path = content - if content: - command = "ps aux | awk '{print $2 }' | grep " + pid_status_path - print self.name, ': command:', command + pid_path = os.path.join(dhcp_path, 'dnsmasq.pid') + command = 'sudo ip netns exec ' + net_namespace + ' cat ' + pid_path + self.logger.debug("command: " + command) (_, stdout, _) = self.ssh_conn.exec_command(command) content = stdout.read() - if not content: - command = 'sudo ip netns exec ' + net_namespace + ' /usr/sbin/dnsmasq --strict-order --except-interface=lo ' \ - '--interface=' + interface + ' --bind-interfaces --dhcp-hostsdir=' + dhcp_path + \ - ' --dhcp-range ' + dhcp_range + ' --pid-file=' + pid_file + ' --dhcp-leasefile=' + leases_path + \ - ' --listen-address ' + gateway + # check if pid is runing + pid_status_path = content + if content: + command = "ps aux | awk '{print $2 }' | grep " + pid_status_path + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + content = stdout.read() + if not content: + command = 'sudo ip netns exec ' + net_namespace + ' /usr/sbin/dnsmasq --strict-order --except-interface=lo ' \ + '--interface=' + interface + ' --bind-interfaces --dhcp-hostsdir=' + dhcp_path + \ + ' --dhcp-range ' + dhcp_range + ' --pid-file=' + pid_file + ' --dhcp-leasefile=' + leases_path + \ + ' --listen-address ' + gateway - print self.name, ': command:', command - (_, stdout, _) = self.ssh_conn.exec_command(command) - content = stdout.readline() + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + content = stdout.readline() - if len(content) == 0: - return True - else: + if len(content) == 0: + return True + else: + return False + except paramiko.ssh_exception.SSHException as e: + self.logger.error("launch_dhcp_server ssh Exception: " + str(e)) + if "SSH session not active" in str(e): + self.ssh_connect() return False def delete_dhcp_interfaces(self, vlan): @@ -1094,80 +1156,96 @@ class host_thread(threading.Thread): """ if self.test: - return - - net_namespace = 'ovim-' + vlan - command = 'sudo ovs-vsctl del-port br-int ovs-tap-' + vlan - print self.name, ': command:', command - (_, stdout, _) = self.ssh_conn.exec_command(command) - content = stdout.read() + return True + try: + net_namespace = 'ovim-' + str(vlan) + command = 'sudo ovs-vsctl del-port br-int ovs-tap-' + str(vlan) + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + content = stdout.read() - command = 'sudo ip netns exec ' + net_namespace + ' ip link set dev tap-' + vlan + ' down' - print self.name, ': command:', command - (_, stdout, _) = self.ssh_conn.exec_command(command) - content = stdout.read() + command = 'sudo ip netns exec ' + net_namespace + ' ip link set dev tap-' + str(vlan) + ' down' + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + content = stdout.read() - command = 'sudo ip link set dev ovs-tap-' + vlan + ' down' - print self.name, ': command:', command - (_, stdout, _) = self.ssh_conn.exec_command(command) - content = stdout.read() + command = 'sudo ip link set dev ovs-tap-' + str(vlan) + ' down' + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + content = stdout.read() + except paramiko.ssh_exception.SSHException as e: + self.logger.error("delete_dhcp_interfaces ssh Exception: " + str(e)) + if "SSH session not active" in str(e): + self.ssh_connect() + return False - def create_dhcp_interfaces(self, vlan, ip, netmask): + def create_dhcp_interfaces(self, vlan, ip_listen_address, netmask): """ Create a linux bridge with STP active :param vlan: segmentation id - :param ip: Ip included in the dhcp range for the tap interface living in namesapce side + :param ip_listen_address: Listen Ip address for the dhcp service, the tap interface living in namesapce side :param netmask: dhcp net CIDR :return: True if success """ if self.test: - return + return True + try: + net_namespace = 'ovim-' + str(vlan) + namespace_interface = 'tap-' + str(vlan) - net_namespace = 'ovim-' + vlan - namespace_interface = 'tap-' + vlan + command = 'sudo ip netns add ' + net_namespace + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + content = stdout.read() - command = 'sudo ip netns add ' + net_namespace - print self.name, ': command:', command - (_, stdout, _) = self.ssh_conn.exec_command(command) - content = stdout.read() + command = 'sudo ip link add tap-' + str(vlan) + ' type veth peer name ovs-tap-' + str(vlan) + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + content = stdout.read() - command = 'sudo ip link add tap-' + vlan + ' type veth peer name ovs-tap-' + vlan - print self.name, ': command:', command - (_, stdout, _) = self.ssh_conn.exec_command(command) - content = stdout.read() + command = 'sudo ovs-vsctl add-port br-int ovs-tap-' + str(vlan) + ' tag=' + str(vlan) + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + content = stdout.read() - command = 'sudo ovs-vsctl add-port br-int ovs-tap-' + vlan + ' tag=' + vlan - print self.name, ': command:', command - (_, stdout, _) = self.ssh_conn.exec_command(command) - content = stdout.read() + command = 'sudo ip link set tap-' + str(vlan) + ' netns ' + net_namespace + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + content = stdout.read() - command = 'sudo ip link set tap-' + vlan + ' netns ' + net_namespace - print self.name, ': command:', command - (_, stdout, _) = self.ssh_conn.exec_command(command) - content = stdout.read() + command = 'sudo ip netns exec ' + net_namespace + ' ip link set dev tap-' + str(vlan) + ' up' + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + content = stdout.read() - command = 'sudo ip netns exec ' + net_namespace + ' ip link set dev tap-' + vlan + ' up' - print self.name, ': command:', command - (_, stdout, _) = self.ssh_conn.exec_command(command) - content = stdout.read() + command = 'sudo ip link set dev ovs-tap-' + str(vlan) + ' up' + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + content = stdout.read() - command = 'sudo ip link set dev ovs-tap-' + vlan + ' up' - print self.name, ': command:', command - (_, stdout, _) = self.ssh_conn.exec_command(command) - content = stdout.read() + command = 'sudo ip netns exec ' + net_namespace + ' ip link set dev lo up' + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + content = stdout.read() - command = 'sudo ip netns exec ' + net_namespace + ' ' + ' ifconfig ' + namespace_interface \ - + ' ' + ip + ' netmask ' + netmask - print self.name, ': command:', command - (_, stdout, _) = self.ssh_conn.exec_command(command) - content = stdout.read() + command = 'sudo ip netns exec ' + net_namespace + ' ' + ' ifconfig ' + namespace_interface \ + + ' ' + ip_listen_address + ' netmask ' + netmask + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + content = stdout.read() - if len(content) == 0: - return True - else: + if len(content) == 0: + return True + else: + return False + except paramiko.ssh_exception.SSHException as e: + self.logger.error("create_dhcp_interfaces ssh Exception: " + str(e)) + if "SSH session not active" in str(e): + self.ssh_connect() return False + def create_ovs_vxlan_tunnel(self, vxlan_interface, remote_ip): """ Create a vlxn tunnel between to computes with an OVS installed. STP is also active at port level @@ -1176,17 +1254,23 @@ class host_thread(threading.Thread): :return: """ if self.test: - return - command = 'sudo ovs-vsctl add-port br-int ' + vxlan_interface + \ - ' -- set Interface ' + vxlan_interface + ' type=vxlan options:remote_ip=' + remote_ip + \ - ' -- set Port ' + vxlan_interface + ' other_config:stp-path-cost=10' - print self.name, ': command:', command - (_, stdout, _) = self.ssh_conn.exec_command(command) - content = stdout.read() - print content - if len(content) == 0: return True - else: + try: + command = 'sudo ovs-vsctl add-port br-int ' + vxlan_interface + \ + ' -- set Interface ' + vxlan_interface + ' type=vxlan options:remote_ip=' + remote_ip + \ + ' -- set Port ' + vxlan_interface + ' other_config:stp-path-cost=10' + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + content = stdout.read() + # print content + if len(content) == 0: + return True + else: + return False + except paramiko.ssh_exception.SSHException as e: + self.logger.error("create_ovs_vxlan_tunnel ssh Exception: " + str(e)) + if "SSH session not active" in str(e): + self.ssh_connect() return False def delete_ovs_vxlan_tunnel(self, vxlan_interface): @@ -1196,15 +1280,21 @@ class host_thread(threading.Thread): :return: True if success. """ if self.test: - return - command = 'sudo ovs-vsctl del-port br-int ' + vxlan_interface - print self.name, ': command:', command - (_, stdout, _) = self.ssh_conn.exec_command(command) - content = stdout.read() - print content - if len(content) == 0: return True - else: + try: + command = 'sudo ovs-vsctl del-port br-int ' + vxlan_interface + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + content = stdout.read() + # print content + if len(content) == 0: + return True + else: + return False + except paramiko.ssh_exception.SSHException as e: + self.logger.error("delete_ovs_vxlan_tunnel ssh Exception: " + str(e)) + if "SSH session not active" in str(e): + self.ssh_connect() return False def delete_ovs_bridge(self): @@ -1213,34 +1303,40 @@ class host_thread(threading.Thread): :return: True if success """ if self.test: - return - command = 'sudo ovs-vsctl del-br br-int' - print self.name, ': command:', command - (_, stdout, _) = self.ssh_conn.exec_command(command) - content = stdout.read() - if len(content) == 0: return True - else: + try: + command = 'sudo ovs-vsctl del-br br-int' + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + content = stdout.read() + if len(content) == 0: + return True + else: + return False + except paramiko.ssh_exception.SSHException as e: + self.logger.error("delete_ovs_bridge ssh Exception: " + str(e)) + if "SSH session not active" in str(e): + self.ssh_connect() return False def get_file_info(self, path): command = 'ls -lL --time-style=+%Y-%m-%dT%H:%M:%S ' + path - print self.name, ': command:', command + self.logger.debug("command: " + command) (_, stdout, _) = self.ssh_conn.exec_command(command) content = stdout.read() if len(content) == 0: - return None # file does not exist + return None # file does not exist else: - return content.split(" ") #(permission, 1, owner, group, size, date, file) + return content.split(" ") # (permission, 1, owner, group, size, date, file) def qemu_get_info(self, path): command = 'qemu-img info ' + path - print self.name, ': command:', command + self.logger.debug("command: " + command) (_, stdout, stderr) = self.ssh_conn.exec_command(command) content = stdout.read() if len(content) == 0: error = stderr.read() - print self.name, ": get_qemu_info error ", error + self.logger.error("get_qemu_info error " + error) raise paramiko.ssh_exception.SSHException("Error getting qemu_info: " + error) else: try: @@ -1250,18 +1346,36 @@ class host_thread(threading.Thread): if hasattr(exc, 'problem_mark'): mark = exc.problem_mark text = " at position: (%s:%s)" % (mark.line+1, mark.column+1) - print self.name, ": get_qemu_info yaml format Exception", text + self.logger.error("get_qemu_info yaml format Exception " + text) raise paramiko.ssh_exception.SSHException("Error getting qemu_info yaml format" + text) def qemu_change_backing(self, inc_file, new_backing_file): command = 'qemu-img rebase -u -b ' + new_backing_file + ' ' + inc_file - print self.name, ': command:', command + self.logger.debug("command: " + command) + (_, _, stderr) = self.ssh_conn.exec_command(command) + content = stderr.read() + if len(content) == 0: + return 0 + else: + self.logger.error("qemu_change_backing error: " + content) + return -1 + + def qemu_create_empty_disk(self, dev): + + if not dev and 'source' not in dev and 'file format' not in dev and 'image_size' not in dev: + self.logger.error("qemu_create_empty_disk error: missing image parameter") + return -1 + + empty_disk_path = dev['source file'] + + command = 'qemu-img create -f qcow2 ' + empty_disk_path + ' ' + str(dev['image_size']) + 'G' + self.logger.debug("command: " + command) (_, _, stderr) = self.ssh_conn.exec_command(command) content = stderr.read() if len(content) == 0: return 0 else: - print self.name, ": qemu_change_backing error: ", content + self.logger.error("qemu_create_empty_disk error: " + content) return -1 def get_notused_filename(self, proposed_name, suffix=''): @@ -1306,7 +1420,7 @@ class host_thread(threading.Thread): def delete_file(self, file_name): command = 'rm -f '+file_name - print self.name, ': command:', command + self.logger.debug("command: " + command) (_, _, stderr) = self.ssh_conn.exec_command(command) error_msg = stderr.read() if len(error_msg) > 0: @@ -1321,7 +1435,7 @@ class host_thread(threading.Thread): if perserve_time: command += ' --preserve=timestamps' command += " '{}' '{}'".format(source, destination) - print self.name, ': command:', command + self.logger.debug("command: " + command) (_, _, stderr) = self.ssh_conn.exec_command(command) error_msg = stderr.read() if len(error_msg) > 0: @@ -1412,7 +1526,7 @@ class host_thread(threading.Thread): result, server_data = self.db.get_instance(server_id) self.db_lock.release() if result <= 0: - print self.name, ": launch_server ERROR getting server from DB",result, server_data + self.logger.error("launch_server ERROR getting server from DB %d %s", result, server_data) return result, server_data #0: get image metadata @@ -1420,7 +1534,7 @@ class host_thread(threading.Thread): use_incremental = None if "use_incremental" in server_metadata: - use_incremental = False if server_metadata["use_incremental"]=="no" else True + use_incremental = False if server_metadata["use_incremental"] == "no" else True server_host_files = self.localinfo['server_files'].get( server['uuid'], {}) if rebuild: @@ -1434,31 +1548,45 @@ class host_thread(threading.Thread): devices = [ {"type":"disk", "image_id":server['image_id'], "vpci":server_metadata.get('vpci', None) } ] if 'extended' in server_data and server_data['extended']!=None and "devices" in server_data['extended']: devices += server_data['extended']['devices'] - + empty_path = None for dev in devices: - if dev['image_id'] == None: + image_id = dev.get('image_id') + if not image_id: + import uuid + uuid_empty = str(uuid.uuid4()) + empty_path = self.empty_image_path + uuid_empty + '.qcow2' # local path for empty disk + + dev['source file'] = empty_path + dev['file format'] = 'qcow2' + self.qemu_create_empty_disk(dev) + server_host_files[uuid_empty] = {'source file': empty_path, + 'file format': dev['file format']} + continue - - self.db_lock.acquire() - result, content = self.db.get_table(FROM='images', SELECT=('path', 'metadata'), - WHERE={'uuid': dev['image_id']}) - self.db_lock.release() - if result <= 0: - error_text = "ERROR", result, content, "when getting image", dev['image_id'] - print self.name, ": launch_server", error_text - return -1, error_text - if content[0]['metadata'] is not None: - dev['metadata'] = json.loads(content[0]['metadata']) else: - dev['metadata'] = {} - - if dev['image_id'] in server_host_files: - dev['source file'] = server_host_files[ dev['image_id'] ] ['source file'] #local path - dev['file format'] = server_host_files[ dev['image_id'] ] ['file format'] # raw or qcow2 - continue + self.db_lock.acquire() + result, content = self.db.get_table(FROM='images', SELECT=('path', 'metadata'), + WHERE={'uuid': image_id}) + self.db_lock.release() + if result <= 0: + error_text = "ERROR", result, content, "when getting image", dev['image_id'] + self.logger.error("launch_server " + error_text) + return -1, error_text + if content[0]['metadata'] is not None: + dev['metadata'] = json.loads(content[0]['metadata']) + else: + dev['metadata'] = {} + + if image_id in server_host_files: + dev['source file'] = server_host_files[image_id]['source file'] #local path + dev['file format'] = server_host_files[image_id]['file format'] # raw or qcow2 + continue #2: copy image to host - remote_file = content[0]['path'] + if image_id: + remote_file = content[0]['path'] + else: + remote_file = empty_path use_incremental_image = use_incremental if dev['metadata'].get("use_incremental") == "no": use_incremental_image = False @@ -1468,7 +1596,7 @@ class host_thread(threading.Thread): if use_incremental_image: local_file_inc = self.get_notused_filename(local_file, '.inc') command = 'qemu-img create -f qcow2 '+local_file_inc+ ' -o backing_file='+ local_file - print 'command:', command + self.logger.debug("command: " + command) (_, _, stderr) = self.ssh_conn.exec_command(command) error_msg = stderr.read() if len(error_msg) > 0: @@ -1487,14 +1615,14 @@ class host_thread(threading.Thread): #3 Create XML result, xml = self.create_xml_server(server_data, devices, server_metadata) #local_file if result <0: - print self.name, ": create xml server error:", xml + self.logger.error("create xml server error: " + xml) return -2, xml - print self.name, ": create xml:", xml + self.logger.debug("create xml: " + xml) atribute = host_thread.lvirt_module.VIR_DOMAIN_START_PAUSED if paused == "yes" else 0 #4 Start the domain if not rebuild: #ensures that any pending destroying server is done self.server_forceoff(True) - #print self.name, ": launching instance" #, xml + #self.logger.debug("launching instance " + xml) conn.createXML(xml, atribute) #self.server_status[server_id] = 'PAUSED' if paused == "yes" else 'ACTIVE' @@ -1502,15 +1630,15 @@ class host_thread(threading.Thread): except paramiko.ssh_exception.SSHException as e: text = e.args[0] - print self.name, ": launch_server(%s) ssh Exception: %s" %(server_id, text) + self.logger.error("launch_server id='%s' ssh Exception: %s", server_id, text) if "SSH session not active" in text: self.ssh_connect() except host_thread.lvirt_module.libvirtError as e: text = e.get_error_message() - print self.name, ": launch_server(%s) libvirt Exception: %s" %(server_id, text) + self.logger.error("launch_server id='%s' libvirt Exception: %s", server_id, text) except Exception as e: text = str(e) - print self.name, ": launch_server(%s) Exception: %s" %(server_id, text) + self.logger.error("launch_server id='%s' Exception: %s", server_id, text) return -1, text def update_servers_status(self): @@ -1523,12 +1651,12 @@ class host_thread(threading.Thread): # VIR_DOMAIN_SHUTOFF = 5 # VIR_DOMAIN_CRASHED = 6 # VIR_DOMAIN_PMSUSPENDED = 7 #TODO suspended - + if self.test or len(self.server_status)==0: - return - + return + try: - conn = host_thread.lvirt_module.open("qemu+ssh://"+self.user+"@"+self.host+"/system") + conn = host_thread.lvirt_module.open(self.lvirt_conn_uri) domains= conn.listAllDomains() domain_dict={} for domain in domains: @@ -1548,7 +1676,7 @@ class host_thread(threading.Thread): domain_dict[uuid] = new_status conn.close() except host_thread.lvirt_module.libvirtError as e: - print self.name, ": get_state() Exception '", e.get_error_message() + self.logger.error("get_state() Exception " + e.get_error_message()) return for server_id, current_status in self.server_status.iteritems(): @@ -1563,7 +1691,7 @@ class host_thread(threading.Thread): if new_status == 'INACTIVE' and current_status == 'ERROR': continue #keep ERROR status, because obviously this machine is not running #change status - print self.name, ": server ", server_id, "status change from ", current_status, "to", new_status + self.logger.debug("server id='%s' status change from '%s' to '%s'", server_id, current_status, new_status) STATUS={'progress':100, 'status':new_status} if new_status == 'ERROR': STATUS['last_error'] = 'machine has crashed' @@ -1601,10 +1729,14 @@ class host_thread(threading.Thread): if req['status']!='ERROR': time.sleep(5) new_status = 'INACTIVE' - elif 'start' in req['action'] and req['status']!='ERROR': new_status = 'ACTIVE' - elif 'resume' in req['action'] and req['status']!='ERROR' and req['status']!='INACTIVE' : new_status = 'ACTIVE' - elif 'pause' in req['action'] and req['status']!='ERROR': new_status = 'PAUSED' - elif 'reboot' in req['action'] and req['status']!='ERROR': new_status = 'ACTIVE' + elif 'start' in req['action'] and req['status']!='ERROR': + new_status = 'ACTIVE' + elif 'resume' in req['action'] and req['status']!='ERROR' and req['status']!='INACTIVE': + new_status = 'ACTIVE' + elif 'pause' in req['action'] and req['status']!='ERROR': + new_status = 'PAUSED' + elif 'reboot' in req['action'] and req['status']!='ERROR': + new_status = 'ACTIVE' elif 'rebuild' in req['action']: time.sleep(random.randint(20,150)) new_status = 'ACTIVE' @@ -1613,7 +1745,7 @@ class host_thread(threading.Thread): self.create_image(None, req) else: try: - conn = host_thread.lvirt_module.open("qemu+ssh://"+self.user+"@"+self.host+"/system") + conn = host_thread.lvirt_module.open(self.lvirt_conn_uri) try: dom = conn.lookupByUUIDString(server_id) except host_thread.lvirt_module.libvirtError as e: @@ -1621,38 +1753,40 @@ class host_thread(threading.Thread): if 'LookupByUUIDString' in text or 'Domain not found' in text or 'No existe un dominio coincidente' in text: dom = None else: - print self.name, ": action_on_server(",server_id,") libvirt exception:", text + self.logger.error("action_on_server id='%s' libvirt exception: %s", server_id, text) raise e if 'forceOff' in req['action']: if dom == None: - print self.name, ": action_on_server(",server_id,") domain not running" + self.logger.debug("action_on_server id='%s' domain not running", server_id) else: try: - print self.name, ": sending DESTROY to server", server_id + self.logger.debug("sending DESTROY to server id='%s'", server_id) dom.destroy() except Exception as e: if "domain is not running" not in e.get_error_message(): - print self.name, ": action_on_server(",server_id,") Exception while sending force off:", e.get_error_message() + self.logger.error("action_on_server id='%s' Exception while sending force off: %s", + server_id, e.get_error_message()) last_error = 'action_on_server Exception while destroy: ' + e.get_error_message() new_status = 'ERROR' elif 'terminate' in req['action']: if dom == None: - print self.name, ": action_on_server(",server_id,") domain not running" + self.logger.debug("action_on_server id='%s' domain not running", server_id) new_status = 'deleted' else: try: if req['action']['terminate'] == 'force': - print self.name, ": sending DESTROY to server", server_id + self.logger.debug("sending DESTROY to server id='%s'", server_id) dom.destroy() new_status = 'deleted' else: - print self.name, ": sending SHUTDOWN to server", server_id + self.logger.debug("sending SHUTDOWN to server id='%s'", server_id) dom.shutdown() self.pending_terminate_server.append( (time.time()+10,server_id) ) except Exception as e: - print self.name, ": action_on_server(",server_id,") Exception while destroy:", e.get_error_message() + self.logger.error("action_on_server id='%s' Exception while destroy: %s", + server_id, e.get_error_message()) last_error = 'action_on_server Exception while destroy: ' + e.get_error_message() new_status = 'ERROR' if "domain is not running" in e.get_error_message(): @@ -1660,7 +1794,8 @@ class host_thread(threading.Thread): dom.undefine() new_status = 'deleted' except Exception: - print self.name, ": action_on_server(",server_id,") Exception while undefine:", e.get_error_message() + self.logger.error("action_on_server id='%s' Exception while undefine: %s", + server_id, e.get_error_message()) last_error = 'action_on_server Exception2 while undefine:', e.get_error_message() #Exception: 'virDomainDetachDevice() failed' if new_status=='deleted': @@ -1678,14 +1813,15 @@ class host_thread(threading.Thread): elif 'shutoff' in req['action'] or 'shutdown' in req['action']: try: if dom == None: - print self.name, ": action_on_server(",server_id,") domain not running" + self.logger.debug("action_on_server id='%s' domain not running", server_id) else: dom.shutdown() # new_status = 'INACTIVE' #TODO: check status for changing at database except Exception as e: new_status = 'ERROR' - print self.name, ": action_on_server(",server_id,") Exception while shutdown:", e.get_error_message() + self.logger.error("action_on_server id='%s' Exception while shutdown: %s", + server_id, e.get_error_message()) last_error = 'action_on_server Exception while shutdown: ' + e.get_error_message() elif 'rebuild' in req['action']: @@ -1715,7 +1851,8 @@ class host_thread(threading.Thread): dom.resume() # new_status = 'ACTIVE' except Exception as e: - print self.name, ": action_on_server(",server_id,") Exception while resume:", e.get_error_message() + self.logger.error("action_on_server id='%s' Exception while resume: %s", + server_id, e.get_error_message()) elif 'pause' in req['action']: try: @@ -1725,7 +1862,8 @@ class host_thread(threading.Thread): dom.suspend() # new_status = 'PAUSED' except Exception as e: - print self.name, ": action_on_server(",server_id,") Exception while pause:", e.get_error_message() + self.logger.error("action_on_server id='%s' Exception while pause: %s", + server_id, e.get_error_message()) elif 'reboot' in req['action']: try: @@ -1733,10 +1871,11 @@ class host_thread(threading.Thread): pass else: dom.reboot() - print self.name, ": action_on_server(",server_id,") reboot:" + self.logger.debug("action_on_server id='%s' reboot:", server_id) #new_status = 'ACTIVE' except Exception as e: - print self.name, ": action_on_server(",server_id,") Exception while reboot:", e.get_error_message() + self.logger.error("action_on_server id='%s' Exception while reboot: %s", + server_id, e.get_error_message()) elif 'createImage' in req['action']: self.create_image(dom, req) @@ -1747,14 +1886,15 @@ class host_thread(threading.Thread): text = e.get_error_message() new_status = "ERROR" last_error = text - print self.name, ": action_on_server(",server_id,") Exception '", text if 'LookupByUUIDString' in text or 'Domain not found' in text or 'No existe un dominio coincidente' in text: - print self.name, ": action_on_server(",server_id,") Exception removed from host" + self.logger.debug("action_on_server id='%s' Exception removed from host", server_id) + else: + self.logger.error("action_on_server id='%s' Exception %s", server_id, text) #end of if self.test if new_status == None: return 1 - print self.name, ": action_on_server(",server_id,") new status", new_status, last_error + self.logger.debug("action_on_server id='%s' new status=%s %s",server_id, new_status, last_error) UPDATE = {'progress':100, 'status':new_status} if new_status=='ERROR': @@ -1762,7 +1902,7 @@ class host_thread(threading.Thread): return -1 elif 'terminate' in req['action']: #PUT a log in the database - print self.name, ": PANIC deleting server", server_id, last_error + self.logger.error("PANIC deleting server id='%s' %s", server_id, last_error) self.db_lock.acquire() self.db.new_row('logs', {'uuid':server_id, 'tenant_id':req['tenant_id'], 'related':'instances','level':'panic', @@ -1795,11 +1935,11 @@ class host_thread(threading.Thread): ret = 0 error_text=None if self.test: - print self.name, ": restore_iface '%s' %s" % (name, mac) + self.logger.debug("restore_iface '%s' %s", name, mac) return 0, None try: if not lib_conn: - conn = host_thread.lvirt_module.open("qemu+ssh://"+self.user+"@"+self.host+"/system") + conn = host_thread.lvirt_module.open(self.lvirt_conn_uri) else: conn = lib_conn @@ -1807,12 +1947,13 @@ class host_thread(threading.Thread): #TODO.Revise self.server_forceoff(True) iface = conn.interfaceLookupByMACString(mac) - iface.destroy() + if iface.isActive(): + iface.destroy() iface.create() - print self.name, ": restore_iface '%s' %s" % (name, mac) + self.logger.debug("restore_iface '%s' %s", name, mac) except host_thread.lvirt_module.libvirtError as e: error_text = e.get_error_message() - print self.name, ": restore_iface '%s' '%s' libvirt exception: %s" %(name, mac, error_text) + self.logger.error("restore_iface '%s' '%s' libvirt exception: %s", name, mac, error_text) ret=-1 finally: if lib_conn is None and conn is not None: @@ -1855,14 +1996,14 @@ class host_thread(threading.Thread): except paramiko.ssh_exception.SSHException as e: image_status='ERROR' error_text = e.args[0] - print self.name, "': create_image(",server_id,") ssh Exception:", error_text + self.logger.error("create_image id='%s' ssh Exception: %s", server_id, error_text) if "SSH session not active" in error_text and retry==0: self.ssh_connect() except Exception as e: image_status='ERROR' error_text = str(e) - print self.name, "': create_image(",server_id,") Exception:", error_text - + self.logger.error("create_image id='%s' Exception: %s", server_id, error_text) + #TODO insert a last_error at database self.db_lock.acquire() self.db.update_rows('images', {'status':image_status, 'progress': 100, 'path':file_dst}, @@ -1880,14 +2021,14 @@ class host_thread(threading.Thread): WHERE={'port_id': port_id}) self.db_lock.release() if r<0: - print self.name, ": edit_iface(",port_id,") DDBB error:", c + self.logger.error("edit_iface %s DDBB error: %s", port_id, c) return elif r==0: - print self.name, ": edit_iface(",port_id,") por not found" + self.logger.error("edit_iface %s port not found", port_id) return port=c[0] if port["model"]!="VF": - print self.name, ": edit_iface(",port_id,") ERROR model must be VF" + self.logger.error("edit_iface %s ERROR model must be VF", port_id) return #create xml detach file xml=[] @@ -1895,16 +2036,16 @@ class host_thread(threading.Thread): xml.append("") xml.append(" ") xml.append(" "+ self.pci2xml(port['pci'])+"\n ") - xml.append('') + xml.append('') try: conn=None - conn = host_thread.lvirt_module.open("qemu+ssh://"+self.user+"@"+self.host+"/system") + conn = host_thread.lvirt_module.open(self.lvirt_conn_uri) dom = conn.lookupByUUIDString(port["instance_id"]) if old_net: text="\n".join(xml) - print self.name, ": edit_iface detaching SRIOV interface", text + self.logger.debug("edit_iface detaching SRIOV interface " + text) dom.detachDeviceFlags(text, flags=host_thread.lvirt_module.VIR_DOMAIN_AFFECT_LIVE) if new_net: xml[-1] =" " @@ -1912,30 +2053,19 @@ class host_thread(threading.Thread): xml.append(self.pci2xml(port.get('vpci',None)) ) xml.append('') text="\n".join(xml) - print self.name, ": edit_iface attaching SRIOV interface", text + self.logger.debug("edit_iface attaching SRIOV interface " + text) dom.attachDeviceFlags(text, flags=host_thread.lvirt_module.VIR_DOMAIN_AFFECT_LIVE) except host_thread.lvirt_module.libvirtError as e: text = e.get_error_message() - print self.name, ": edit_iface(",port["instance_id"],") libvirt exception:", text + self.logger.error("edit_iface %s libvirt exception: %s", port["instance_id"], text) finally: if conn is not None: conn.close() def create_server(server, db, db_lock, only_of_ports): - #print "server" - #print "server" - #print server - #print "server" - #print "server" - #try: -# host_id = server.get('host_id', None) extended = server.get('extended', None) - -# print '----------------------' -# print json.dumps(extended, indent=4) - requirements={} requirements['numa']={'memory':0, 'proc_req_type': 'threads', 'proc_req_nb':0, 'port_list':[], 'sriov_list':[]} requirements['ram'] = server['flavor'].get('ram', 0) @@ -2040,7 +2170,7 @@ def create_server(server, db, db_lock, only_of_ports): WHERE={'numa_id':numa_id,'instance_id': None, 'status':'ok'} ) db_lock.release() if result <= 0: - print content + #print content return -1, content #convert rows to a dictionary indexed by core_id @@ -2126,7 +2256,7 @@ def create_server(server, db, db_lock, only_of_ports): result, content = db.get_table(FROM='resources_port', SELECT=('id', 'pci', 'mac'),WHERE={'numa_id':numa_id,'root_id': port['port_id'], 'port_id': None, 'Mbps_used': 0} ) db_lock.release() if result <= 0: - print content + #print content return -1, content for row in content: if row['id'] in used_sriov_ports or row['id']==port['port_id']: @@ -2150,7 +2280,7 @@ def create_server(server, db, db_lock, only_of_ports): result, content = db.get_table(FROM='resources_port', SELECT=('id', 'pci', 'mac', 'Mbps'),WHERE={'numa_id':numa_id,'root_id': port['port_id'], 'port_id': None, 'Mbps_used': 0} ) db_lock.release() if result <= 0: - print content + #print content return -1, content port['Mbps_used'] = content[0]['Mbps'] for row in content: @@ -2168,7 +2298,6 @@ def create_server(server, db, db_lock, only_of_ports): # print '2. SR-IOV assignation:'+json.dumps(requirements['sriov_list'], indent=4) server['host_id'] = host_id - #Generate dictionary for saving in db the instance resources resources = {} @@ -2266,9 +2395,9 @@ def create_server(server, db, db_lock, only_of_ports): resources['extended']['devices'] = extended['devices'] - print '===================================={' - print json.dumps(resources, indent=4) - print '====================================}' + # '===================================={' + #print json.dumps(resources, indent=4) + #print '====================================}' return 0, resources