X-Git-Url: https://osm.etsi.org/gitweb/?p=osm%2Fopenvim.git;a=blobdiff_plain;f=osm_openvim%2Fhost_thread.py;h=6c3c7e28308e68adcd3847d7a8a902a16528863d;hp=d8bca2e2e1ca13d091a34566d29610672f520cb8;hb=c67abe2ff35fcdd5b85db97915a7833ab31beb56;hpb=9f6571090b203922cabb0382226be0fa48d6e046 diff --git a/osm_openvim/host_thread.py b/osm_openvim/host_thread.py index d8bca2e..6c3c7e2 100644 --- a/osm_openvim/host_thread.py +++ b/osm_openvim/host_thread.py @@ -34,33 +34,30 @@ import threading import time import Queue import paramiko -from jsonschema import validate as js_v, exceptions as js_e -#import libvirt +import subprocess +# import libvirt import imp -from vim_schema import localinfo_schema, hostinfo_schema import random import os +import logging +from jsonschema import validate as js_v, exceptions as js_e +from vim_schema import localinfo_schema, hostinfo_schema -#TODO: insert a logging system - -# from logging import Logger -# import auxiliary_functions as af - -# TODO: insert a logging system - +class RunCommandException(Exception): + pass class host_thread(threading.Thread): lvirt_module = None def __init__(self, name, host, user, db, db_lock, test, image_path, host_id, version, develop_mode, - develop_bridge_iface): - '''Init a thread. - Arguments: - 'id' number of thead - 'name' name of thread - 'host','user': host ip or name to manage and user - 'db', 'db_lock': database class and lock to use it in exclusion - ''' + develop_bridge_iface, password=None, keyfile = None, logger_name=None, debug=None): + """Init a thread to communicate with compute node or ovs_controller. + :param host_id: host identity + :param name: name of the thread + :param host: host ip or name to manage and user + :param user, password, keyfile: user and credentials to connect to host + :param db, db_lock': database class and lock to use it in exclusion + """ threading.Thread.__init__(self) self.name = name self.host = host @@ -68,6 +65,9 @@ class host_thread(threading.Thread): self.db = db self.db_lock = db_lock self.test = test + self.password = password + self.keyfile = keyfile + self.localinfo_dirty = False if not test and not host_thread.lvirt_module: try: @@ -75,16 +75,24 @@ class host_thread(threading.Thread): host_thread.lvirt_module = imp.load_module("libvirt", *module_info) except (IOError, ImportError) as e: raise ImportError("Cannot import python-libvirt. Openvim not properly installed" +str(e)) + if logger_name: + self.logger_name = logger_name + else: + self.logger_name = "openvim.host."+name + self.logger = logging.getLogger(self.logger_name) + if debug: + self.logger.setLevel(getattr(logging, debug)) self.develop_mode = develop_mode self.develop_bridge_iface = develop_bridge_iface self.image_path = image_path + self.empty_image_path = image_path self.host_id = host_id self.version = version self.xml_level = 0 - #self.pending ={} + # self.pending ={} self.server_status = {} #dictionary with pairs server_uuid:server_status self.pending_terminate_server =[] #list with pairs (time,server_uuid) time to send a terminate for a server being destroyed @@ -95,110 +103,178 @@ class host_thread(threading.Thread): self.queueLock = threading.Lock() self.taskQueue = Queue.Queue(2000) self.ssh_conn = None + self.run_command_session = None + self.error = None + self.localhost = True if host == 'localhost' else False + self.lvirt_conn_uri = "qemu+ssh://{user}@{host}/system?no_tty=1&no_verify=1".format( + user=self.user, host=self.host) + if keyfile: + self.lvirt_conn_uri += "&keyfile=" + keyfile + self.remote_ip = None + self.local_ip = None + + def run_command(self, command, keep_session=False): + """Run a command passed as a str on a localhost or at remote machine. + :param command: text with the command to execute. + :param keep_session: if True it returns a for sending input with '.write("text\n")'. + A command with keep_session=True MUST be followed by a command with keep_session=False in order to + close the session and get the output + :return: the output of the command if 'keep_session=False' or the object if 'keep_session=True' + :raises: RunCommandException if command fails + """ + if self.run_command_session and keep_session: + raise RunCommandException("Internal error. A command with keep_session=True must be followed by another " + "command with keep_session=False to close session") + try: + if self.localhost: + if self.run_command_session: + p = self.run_command_session + self.run_command_session = None + (output, outerror) = p.communicate() + returncode = p.returncode + p.stdin.close() + elif keep_session: + p = subprocess.Popen(('bash', "-c", command), stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + self.run_command_session = p + return p.stdin + else: + output = subprocess.check_output(('bash', "-c", command)) + returncode = 0 + else: + if self.run_command_session: + (i, o, e) = self.run_command_session + self.run_command_session = None + i.channel.shutdown_write() + else: + if not self.ssh_conn: + self.ssh_connect() + (i, o, e) = self.ssh_conn.exec_command(command, timeout=10) + if keep_session: + self.run_command_session = (i, o, e) + return i + returncode = o.channel.recv_exit_status() + output = o.read() + outerror = e.read() + if returncode != 0: + text = "run_command='{}' Error='{}'".format(command, outerror) + self.logger.error(text) + raise RunCommandException(text) + + self.logger.debug("run_command='{}' result='{}'".format(command, output)) + return output + + except RunCommandException: + raise + except subprocess.CalledProcessError as e: + text = "run_command Exception '{}' '{}'".format(str(e), e.output) + except (paramiko.ssh_exception.SSHException, Exception) as e: + text = "run_command='{}' Exception='{}'".format(command, str(e)) + self.ssh_conn = None + self.run_command_session = None + raise RunCommandException(text) def ssh_connect(self): try: - #Connect SSH + # Connect SSH self.ssh_conn = paramiko.SSHClient() self.ssh_conn.set_missing_host_key_policy(paramiko.AutoAddPolicy()) self.ssh_conn.load_system_host_keys() - self.ssh_conn.connect(self.host, username=self.user, timeout=10) #, None) - except paramiko.ssh_exception.SSHException as e: - text = e.args[0] - print self.name, ": ssh_connect ssh Exception:", text - - def load_localinfo(self): + self.ssh_conn.connect(self.host, username=self.user, password=self.password, key_filename=self.keyfile, + timeout=10) # auth_timeout=10) + self.remote_ip = self.ssh_conn.get_transport().sock.getpeername()[0] + self.local_ip = self.ssh_conn.get_transport().sock.getsockname()[0] + except (paramiko.ssh_exception.SSHException, Exception) as e: + text = 'ssh connect Exception: {}'.format(e) + self.ssh_conn = None + self.error = text + raise + + def check_connectivity(self): if not self.test: + # TODO change to run_command try: - #Connect SSH - self.ssh_connect() - - command = 'mkdir -p ' + self.image_path - #print self.name, ': command:', command - (_, stdout, stderr) = self.ssh_conn.exec_command(command) + if not self.ssh_conn: + self.ssh_connect() + + command = 'sudo brctl show' + (_, stdout, stderr) = self.ssh_conn.exec_command(command, timeout=10) content = stderr.read() if len(content) > 0: - print self.name, ': command:', command, "stderr:", content + self.connectivity = False + self.logger.error("ssh conection error") + except paramiko.ssh_exception.SSHException as e: + text = e.args[0] + self.connectivity = False + self.logger.error("ssh_connect ssh Exception: " + text) + raise paramiko.ssh_exception.SSHException("ssh error conection") + except Exception as e: + self.connectivity = False + raise paramiko.ssh_exception.SSHException("ssh error conection") - command = 'cat ' + self.image_path + '/.openvim.yaml' - #print self.name, ': command:', command - (_, stdout, stderr) = self.ssh_conn.exec_command(command) - content = stdout.read() - if len(content) == 0: - print self.name, ': command:', command, "stderr:", stderr.read() - raise paramiko.ssh_exception.SSHException("Error empty file ") - self.localinfo = yaml.load(content) + def load_localinfo(self): + if not self.test: + try: + self.run_command('sudo mkdir -p ' + self.image_path) + result = self.run_command('cat {}/.openvim.yaml'.format(self.image_path)) + self.localinfo = yaml.load(result) js_v(self.localinfo, localinfo_schema) - self.localinfo_dirty=False + self.localinfo_dirty = False if 'server_files' not in self.localinfo: self.localinfo['server_files'] = {} - print self.name, ': localinfo load from host' + self.logger.debug("localinfo loaded from host") return - - except paramiko.ssh_exception.SSHException as e: - text = e.args[0] - print self.name, ": load_localinfo ssh Exception:", text + except RunCommandException as e: + self.logger.error("load_localinfo Exception: " + str(e)) except host_thread.lvirt_module.libvirtError as e: text = e.get_error_message() - print self.name, ": load_localinfo libvirt Exception:", text + self.logger.error("load_localinfo libvirt Exception: " + text) except yaml.YAMLError as exc: text = "" if hasattr(exc, 'problem_mark'): mark = exc.problem_mark text = " at position: (%s:%s)" % (mark.line+1, mark.column+1) - print self.name, ": load_localinfo yaml format Exception", text + self.logger.error("load_localinfo yaml format Exception " + text) except js_e.ValidationError as e: text = "" if len(e.path)>0: text=" at '" + ":".join(map(str, e.path))+"'" - print self.name, ": load_localinfo format Exception:", text, e.message + self.logger.error("load_localinfo format Exception: %s %s", text, str(e)) except Exception as e: text = str(e) - print self.name, ": load_localinfo Exception:", text + self.logger.error("load_localinfo Exception: " + text) - #not loaded, insert a default data and force saving by activating dirty flag + # not loaded, insert a default data and force saving by activating dirty flag self.localinfo = {'files':{}, 'server_files':{} } - #self.localinfo_dirty=True + # self.localinfo_dirty=True self.localinfo_dirty=False def load_hostinfo(self): if self.test: - return; + return try: - #Connect SSH - self.ssh_connect() - - - command = 'cat ' + self.image_path + '/hostinfo.yaml' - #print self.name, ': command:', command - (_, stdout, stderr) = self.ssh_conn.exec_command(command) - content = stdout.read() - if len(content) == 0: - print self.name, ': command:', command, "stderr:", stderr.read() - raise paramiko.ssh_exception.SSHException("Error empty file ") - self.hostinfo = yaml.load(content) + result = self.run_command('cat {}/hostinfo.yaml'.format(self.image_path)) + self.hostinfo = yaml.load(result) js_v(self.hostinfo, hostinfo_schema) - print self.name, ': hostlinfo load from host', self.hostinfo + self.logger.debug("hostinfo load from host " + str(self.hostinfo)) return - - except paramiko.ssh_exception.SSHException as e: - text = e.args[0] - print self.name, ": load_hostinfo ssh Exception:", text + except RunCommandException as e: + self.logger.error("load_hostinfo ssh Exception: " + str(e)) except host_thread.lvirt_module.libvirtError as e: text = e.get_error_message() - print self.name, ": load_hostinfo libvirt Exception:", text + self.logger.error("load_hostinfo libvirt Exception: " + text) except yaml.YAMLError as exc: text = "" if hasattr(exc, 'problem_mark'): mark = exc.problem_mark text = " at position: (%s:%s)" % (mark.line+1, mark.column+1) - print self.name, ": load_hostinfo yaml format Exception", text + self.logger.error("load_hostinfo yaml format Exception " + text) except js_e.ValidationError as e: text = "" if len(e.path)>0: text=" at '" + ":".join(map(str, e.path))+"'" - print self.name, ": load_hostinfo format Exception:", text, e.message + self.logger.error("load_hostinfo format Exception: %s %s", text, str(e)) except Exception as e: text = str(e) - print self.name, ": load_hostinfo Exception:", text + self.logger.error("load_hostinfo Exception: " + text) #not loaded, insert a default data self.hostinfo = None @@ -212,30 +288,29 @@ class host_thread(threading.Thread): tries-=1 try: - command = 'cat > ' + self.image_path + '/.openvim.yaml' - print self.name, ': command:', command - (stdin, _, _) = self.ssh_conn.exec_command(command) - yaml.safe_dump(self.localinfo, stdin, explicit_start=True, indent=4, default_flow_style=False, tags=False, encoding='utf-8', allow_unicode=True) + command = 'cat > {}/.openvim.yaml'.format(self.image_path) + in_stream = self.run_command(command, keep_session=True) + yaml.safe_dump(self.localinfo, in_stream, explicit_start=True, indent=4, default_flow_style=False, + tags=False, encoding='utf-8', allow_unicode=True) + result = self.run_command(command, keep_session=False) # to end session + self.localinfo_dirty = False break #while tries - - except paramiko.ssh_exception.SSHException as e: - text = e.args[0] - print self.name, ": save_localinfo ssh Exception:", text - if "SSH session not active" in text: - self.ssh_connect() + + except RunCommandException as e: + self.logger.error("save_localinfo ssh Exception: " + str(e)) except host_thread.lvirt_module.libvirtError as e: text = e.get_error_message() - print self.name, ": save_localinfo libvirt Exception:", text + self.logger.error("save_localinfo libvirt Exception: " + text) except yaml.YAMLError as exc: text = "" if hasattr(exc, 'problem_mark'): mark = exc.problem_mark text = " at position: (%s:%s)" % (mark.line+1, mark.column+1) - print self.name, ": save_localinfo yaml format Exception", text + self.logger.error("save_localinfo yaml format Exception " + text) except Exception as e: text = str(e) - print self.name, ": save_localinfo Exception:", text + self.logger.error("save_localinfo Exception: " + text) def load_servers_from_db(self): self.db_lock.acquire() @@ -244,7 +319,7 @@ class host_thread(threading.Thread): self.server_status = {} if r<0: - print self.name, ": Error getting data from database:", c + self.logger.error("Error getting data from database: " + c) return for server in c: self.server_status[ server['uuid'] ] = server['status'] @@ -271,10 +346,10 @@ class host_thread(threading.Thread): if uuid not in self.server_status: for localfile in images.values(): try: - print self.name, ": deleting file '%s' of unused server '%s'" %(localfile['source file'], uuid) + self.logger.debug("deleting file '%s' of unused server '%s'", localfile['source file'], uuid) self.delete_file(localfile['source file']) - except paramiko.ssh_exception.SSHException as e: - print self.name, ": Exception deleting file '%s': %s" %(localfile['source file'], str(e)) + except RunCommandException as e: + self.logger.error("Exception deleting file '%s': %s", localfile['source file'], str(e)) del self.localinfo['server_files'][uuid] self.localinfo_dirty = True @@ -294,71 +369,76 @@ class host_thread(threading.Thread): self.load_servers_from_db() self.delete_unused_files() while True: - self.queueLock.acquire() - if not self.taskQueue.empty(): - task = self.taskQueue.get() - else: - task = None - self.queueLock.release() - - if task is None: - now=time.time() - if self.localinfo_dirty: - self.save_localinfo() - elif self.next_update_server_status < now: - self.update_servers_status() - self.next_update_server_status = now + 5 - elif len(self.pending_terminate_server)>0 and self.pending_terminate_server[0][0]=0: - break - elif task[0] == 'image': - pass - elif task[0] == 'exit': - print self.name, ": processing task exit" - self.terminate() - return 0 - elif task[0] == 'reload': - print self.name, ": processing task reload terminating and relaunching" - self.terminate() - break - elif task[0] == 'edit-iface': - print self.name, ": processing task edit-iface port=%s, old_net=%s, new_net=%s" % (task[1], task[2], task[3]) - self.edit_iface(task[1], task[2], task[3]) - elif task[0] == 'restore-iface': - print self.name, ": processing task restore-iface %s mac=%s" % (task[1], task[2]) - self.restore_iface(task[1], task[2]) - elif task[0] == 'new-ovsbridge': - print self.name, ": Creating compute OVS bridge" - self.create_ovs_bridge() - elif task[0] == 'new-vxlan': - print self.name, ": Creating vxlan tunnel=" + task[1] + ", remote ip=" + task[2] - self.create_ovs_vxlan_tunnel(task[1], task[2]) - elif task[0] == 'del-ovsbridge': - print self.name, ": Deleting OVS bridge" - self.delete_ovs_bridge() - elif task[0] == 'del-vxlan': - print self.name, ": Deleting vxlan " + task[1] + " tunnel" - self.delete_ovs_vxlan_tunnel(task[1]) - elif task[0] == 'create-ovs-bridge-port': - print self.name, ": Adding port ovim-" + task[1] + " to OVS bridge" - self.create_ovs_bridge_port(task[1]) - elif task[0] == 'del-ovs-port': - print self.name, ": Delete bridge attached to ovs port vlan {} net {}".format(task[1], task[2]) - self.delete_bridge_port_attached_to_ovs(task[1], task[2]) - else: - print self.name, ": unknown task", task - + task = None + self.queueLock.release() + + if task is None: + now=time.time() + if self.localinfo_dirty: + self.save_localinfo() + elif self.next_update_server_status < now: + self.update_servers_status() + self.next_update_server_status = now + 5 + elif len(self.pending_terminate_server)>0 and self.pending_terminate_server[0][0]= 0: + break + elif task[0] == 'image': + pass + elif task[0] == 'exit': + self.logger.debug("processing task exit") + self.terminate() + return 0 + elif task[0] == 'reload': + self.logger.debug("processing task reload terminating and relaunching") + self.terminate() + break + elif task[0] == 'edit-iface': + self.logger.debug("processing task edit-iface port={}, old_net={}, new_net={}".format( + task[1], task[2], task[3])) + self.edit_iface(task[1], task[2], task[3]) + elif task[0] == 'restore-iface': + self.logger.debug("processing task restore-iface={} mac={}".format(task[1], task[2])) + self.restore_iface(task[1], task[2]) + elif task[0] == 'new-ovsbridge': + self.logger.debug("Creating compute OVS bridge") + self.create_ovs_bridge() + elif task[0] == 'new-vxlan': + self.logger.debug("Creating vxlan tunnel='{}', remote ip='{}'".format(task[1], task[2])) + self.create_ovs_vxlan_tunnel(task[1], task[2]) + elif task[0] == 'del-ovsbridge': + self.logger.debug("Deleting OVS bridge") + self.delete_ovs_bridge() + elif task[0] == 'del-vxlan': + self.logger.debug("Deleting vxlan {} tunnel".format(task[1])) + self.delete_ovs_vxlan_tunnel(task[1]) + elif task[0] == 'create-ovs-bridge-port': + self.logger.debug("Adding port ovim-{} to OVS bridge".format(task[1])) + self.create_ovs_bridge_port(task[1]) + elif task[0] == 'del-ovs-port': + self.logger.debug("Delete bridge attached to ovs port vlan {} net {}".format(task[1], task[2])) + self.delete_bridge_port_attached_to_ovs(task[1], task[2]) + else: + self.logger.debug("unknown task " + str(task)) + + except Exception as e: + self.logger.critical("Unexpected exception at run: " + str(e), exc_info=True) + def server_forceoff(self, wait_until_finished=False): while len(self.pending_terminate_server)>0: now = time.time() @@ -384,8 +464,8 @@ class host_thread(threading.Thread): self.ssh_conn.close() except Exception as e: text = str(e) - print self.name, ": terminate Exception:", text - print self.name, ": exit from host_thread" + self.logger.error("terminate Exception: " + text) + self.logger.debug("exit from host_thread") def get_local_iface_name(self, generic_name): if self.hostinfo != None and "iface_names" in self.hostinfo and generic_name in self.hostinfo["iface_names"]: @@ -420,8 +500,7 @@ class host_thread(threading.Thread): if topo == None and 'metadata' in dev_list[0]: topo = dev_list[0]['metadata'].get('topology', None) #name - name = server.get('name','') + "_" + server['uuid'] - name = name[:58] #qemu impose a length limit of 59 chars or not start. Using 58 + name = server.get('name', '')[:28] + "_" + server['uuid'][:28] #qemu impose a length limit of 59 chars or not start. Using 58 text += self.inc_tab() + "" + name+ "" #uuid text += self.tab() + "" + server['uuid'] + "" @@ -504,7 +583,7 @@ class host_thread(threading.Thread): if topo == "oneSocket:hyperthreading": if vcpus % 2 != 0: return -1, 'Cannot expose hyperthreading with an odd number of vcpus' - text += self.tab() + " " % vcpus/2 + text += self.tab() + " " % (vcpus/2) elif windows_os or topo == "oneSocket": text += self.tab() + " " % vcpus else: @@ -560,7 +639,7 @@ class host_thread(threading.Thread): #else: # return -1, 'Unknown disk type ' + v['type'] vpci = dev.get('vpci',None) - if vpci == None: + if vpci == None and 'metadata' in dev: vpci = dev['metadata'].get('vpci',None) text += self.pci2xml(vpci) @@ -593,7 +672,7 @@ class host_thread(threading.Thread): result, content = self.db.get_table(FROM='nets', SELECT=('provider',),WHERE={'uuid':v['net_id']} ) self.db_lock.release() if result <= 0: - print "create_xml_server ERROR getting nets",result, content + self.logger.error("create_xml_server ERROR %d getting nets %s", result, content) return -1, content #ALF: Allow by the moment the 'default' bridge net because is confortable for provide internet to VM #I know it is not secure @@ -621,7 +700,7 @@ class host_thread(threading.Thread): elif content[0]['provider'][0:3] == "OVS": vlan = content[0]['provider'].replace('OVS:', '') text += self.tab() + "" + \ - self.inc_tab() + "" + self.inc_tab() + "" else: return -1, 'Unknown Bridge net provider ' + content[0]['provider'] if model!=None: @@ -716,15 +795,18 @@ class host_thread(threading.Thread): Create a bridge in compute OVS to allocate VMs :return: True if success """ - if self.test: - return - command = 'sudo ovs-vsctl --may-exist add-br br-int -- set Bridge br-int stp_enable=true' - print self.name, ': command:', command - (_, stdout, _) = self.ssh_conn.exec_command(command) - content = stdout.read() - if len(content) == 0: + if self.test or not self.connectivity: return True - else: + + + try: + self.run_command('sudo', 'ovs-vsctl', '--may-exist', 'add-br', 'br-int', '--', 'set', 'Bridge', 'br-int', + 'stp_enable=true') + return True + except RunCommandException as e: + self.logger.error("create_ovs_bridge ssh Exception: " + str(e)) + if "SSH session not active" in str(e): + self.ssh_connect() return False def delete_port_to_ovs_bridge(self, vlan, net_uuid): @@ -735,17 +817,15 @@ class host_thread(threading.Thread): :return: """ - if self.test: - return - - port_name = 'ovim-' + vlan - command = 'sudo ovs-vsctl del-port br-int ' + port_name - print self.name, ': command:', command - (_, stdout, _) = self.ssh_conn.exec_command(command) - content = stdout.read() - if len(content) == 0: + if self.test or not self.connectivity: return True - else: + try: + port_name = 'ovim-' + str(vlan) + command = 'sudo ovs-vsctl del-port br-int ' + port_name + self.run_command(command) + return True + except RunCommandException as e: + self.logger.error("delete_port_to_ovs_bridge ssh Exception: " + str(e)) return False def delete_dhcp_server(self, vlan, net_uuid, dhcp_path): @@ -756,29 +836,34 @@ class host_thread(threading.Thread): :param dhcp_path: conf fiel path that live in namespace side :return: """ - if self.test: - return + if self.test or not self.connectivity: + return True if not self.is_dhcp_port_free(vlan, net_uuid): return True + try: + dhcp_namespace = str(vlan) + '-dnsmasq' + dhcp_path = os.path.join(dhcp_path, dhcp_namespace) + pid_file = os.path.join(dhcp_path, 'dnsmasq.pid') - net_namespace = 'ovim-' + vlan - dhcp_path = os.path.join(dhcp_path, net_namespace) - pid_file = os.path.join(dhcp_path, 'dnsmasq.pid') - - command = 'sudo ip netns exec ' + net_namespace + ' cat ' + pid_file - print self.name, ': command:', command - (_, stdout, _) = self.ssh_conn.exec_command(command) - content = stdout.read() + command = 'sudo ip netns exec ' + dhcp_namespace + ' cat ' + pid_file + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + content = stdout.read() - command = 'sudo ip netns exec ' + net_namespace + ' kill -9 ' + content - print self.name, ': command:', command - (_, stdout, _) = self.ssh_conn.exec_command(command) - content = stdout.read() + command = 'sudo ip netns exec ' + dhcp_namespace + ' kill -9 ' + content + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + content = stdout.read() - # if len(content) == 0: - # return True - # else: - # return False + # if len(content) == 0: + # return True + # else: + # return False + except paramiko.ssh_exception.SSHException as e: + self.logger.error("delete_dhcp_server ssh Exception: " + str(e)) + if "SSH session not active" in str(e): + self.ssh_connect() + return False def is_dhcp_port_free(self, host_id, net_uuid): """ @@ -790,7 +875,7 @@ class host_thread(threading.Thread): self.db_lock.acquire() result, content = self.db.get_table( FROM='ports', - WHERE={'p.type': 'instance:ovs', 'p.net_id': net_uuid} + WHERE={'type': 'instance:ovs', 'net_id': net_uuid} ) self.db_lock.release() @@ -827,16 +912,21 @@ class host_thread(threading.Thread): """ if self.test: - return - - port_name = 'ovim-' + vlan - command = 'sudo ovs-vsctl add-port br-int ' + port_name + ' tag=' + vlan - print self.name, ': command:', command - (_, stdout, _) = self.ssh_conn.exec_command(command) - content = stdout.read() - if len(content) == 0: return True - else: + try: + port_name = 'ovim-' + str(vlan) + command = 'sudo ovs-vsctl add-port br-int ' + port_name + ' tag=' + str(vlan) + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + content = stdout.read() + if len(content) == 0: + return True + else: + return False + except paramiko.ssh_exception.SSHException as e: + self.logger.error("add_port_to_ovs_bridge ssh Exception: " + str(e)) + if "SSH session not active" in str(e): + self.ssh_connect() return False def delete_dhcp_port(self, vlan, net_uuid): @@ -848,7 +938,7 @@ class host_thread(threading.Thread): """ if self.test: - return + return True if not self.is_dhcp_port_free(vlan, net_uuid): return True @@ -879,24 +969,81 @@ class host_thread(threading.Thread): """ if self.test: - return + return True + try: + port_name = 'ovim-' + str(vlan) + command = 'sudo ip link set dev ovim-' + str(vlan) + ' down' + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + # content = stdout.read() + # + # if len(content) != 0: + # return False + command = 'sudo ifconfig ' + port_name + ' down && sudo brctl delbr ' + port_name + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + content = stdout.read() + if len(content) == 0: + return True + else: + return False + except paramiko.ssh_exception.SSHException as e: + self.logger.error("delete_linux_bridge ssh Exception: " + str(e)) + if "SSH session not active" in str(e): + self.ssh_connect() + return False - port_name = 'ovim-' + vlan - command = 'sudo ip link set dev veth0-' + vlan + ' down' - print self.name, ': command:', command - (_, stdout, _) = self.ssh_conn.exec_command(command) - content = stdout.read() - # - # if len(content) != 0: - # return False + def remove_link_bridge_to_ovs(self, vlan, link): + """ + Delete a linux provider net connection to tenatn net + :param vlan: vlan port id + :param link: link name + :return: True if success + """ - command = 'sudo ifconfig ' + port_name + ' down && sudo brctl delbr ' + port_name - print self.name, ': command:', command - (_, stdout, _) = self.ssh_conn.exec_command(command) - content = stdout.read() - if len(content) == 0: + if self.test: return True - else: + try: + br_tap_name = str(vlan) + '-vethBO' + br_ovs_name = str(vlan) + '-vethOB' + + # Delete ovs veth pair + command = 'sudo ip link set dev {} down'.format(br_ovs_name) + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + content = stdout.read() + + command = 'sudo ovs-vsctl del-port br-int {}'.format(br_ovs_name) + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + content = stdout.read() + + # Delete br veth pair + command = 'sudo ip link set dev {} down'.format(br_tap_name) + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + content = stdout.read() + + # Delete br veth interface form bridge + command = 'sudo brctl delif {} {}'.format(link, br_tap_name) + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + content = stdout.read() + + # Delete br veth pair + command = 'sudo ip link set dev {} down'.format(link) + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + content = stdout.read() + + if len(content) == 0: + return True + else: + return False + except paramiko.ssh_exception.SSHException as e: + self.logger.error("delete_linux_bridge ssh Exception: " + str(e)) + if "SSH session not active" in str(e): + self.ssh_connect() return False def create_ovs_bridge_port(self, vlan): @@ -918,48 +1065,53 @@ class host_thread(threading.Thread): """ if self.test: - return - - port_name = 'ovim-' + vlan - command = 'sudo brctl show | grep ' + port_name - print self.name, ': command:', command - (_, stdout, _) = self.ssh_conn.exec_command(command) - content = stdout.read() + return True + try: + port_name = 'ovim-' + str(vlan) + command = 'sudo brctl show | grep ' + port_name + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + content = stdout.read() - # if exist nothing to create - # if len(content) == 0: - # return False + # if exist nothing to create + # if len(content) == 0: + # return False - command = 'sudo brctl addbr ' + port_name - print self.name, ': command:', command - (_, stdout, _) = self.ssh_conn.exec_command(command) - content = stdout.read() + command = 'sudo brctl addbr ' + port_name + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + content = stdout.read() - # if len(content) == 0: - # return True - # else: - # return False + # if len(content) == 0: + # return True + # else: + # return False - command = 'sudo brctl stp ' + port_name + ' on' - print self.name, ': command:', command - (_, stdout, _) = self.ssh_conn.exec_command(command) - content = stdout.read() + command = 'sudo brctl stp ' + port_name + ' on' + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + content = stdout.read() - # if len(content) == 0: - # return True - # else: - # return False - command = 'sudo ip link set dev ' + port_name + ' up' - print self.name, ': command:', command - (_, stdout, _) = self.ssh_conn.exec_command(command) - content = stdout.read() + # if len(content) == 0: + # return True + # else: + # return False + command = 'sudo ip link set dev ' + port_name + ' up' + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + content = stdout.read() - if len(content) == 0: - return True - else: + if len(content) == 0: + return True + else: + return False + except paramiko.ssh_exception.SSHException as e: + self.logger.error("create_linux_bridge ssh Exception: " + str(e)) + if "SSH session not active" in str(e): + self.ssh_connect() return False - def set_mac_dhcp_server(self, ip, mac, vlan, netmask, dhcp_path): + def set_mac_dhcp_server(self, ip, mac, vlan, netmask, first_ip, dhcp_path): """ Write into dhcp conf file a rule to assigned a fixed ip given to an specific MAC address :param ip: IP address asigned to a VM @@ -971,31 +1123,58 @@ class host_thread(threading.Thread): """ if self.test: - return + return True - net_namespace = 'ovim-' + vlan - dhcp_path = os.path.join(dhcp_path, net_namespace) - dhcp_hostsdir = os.path.join(dhcp_path, net_namespace) + dhcp_namespace = str(vlan) + '-dnsmasq' + dhcp_path = os.path.join(dhcp_path, dhcp_namespace) + dhcp_hostsdir = os.path.join(dhcp_path, dhcp_namespace) if not ip: return False + try: - ip_data = mac.upper() + ',' + ip + ns_interface = str(vlan) + '-vethDO' + command = 'sudo ip netns exec ' + dhcp_namespace + ' cat /sys/class/net/{}/address'.format(ns_interface) + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + iface_listen_mac = stdout.read() - command = 'sudo ip netns exec ' + net_namespace + ' touch ' + dhcp_hostsdir - print self.name, ': command:', command - (_, stdout, _) = self.ssh_conn.exec_command(command) - content = stdout.read() + if iface_listen_mac > 0: + command = 'sudo ip netns exec ' + dhcp_namespace + ' cat {} | grep {}'.format(dhcp_hostsdir, dhcp_hostsdir) + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + content = stdout.read() + if content > 0: + ip_data = iface_listen_mac.upper().replace('\n', '') + ',' + first_ip + dhcp_hostsdir = os.path.join(dhcp_path, dhcp_namespace) - command = 'sudo ip netns exec ' + net_namespace + ' sudo bash -ec "echo ' + ip_data + ' >> ' + dhcp_hostsdir + '"' + command = 'sudo ip netns exec ' + dhcp_namespace + ' sudo bash -ec "echo ' + ip_data + ' >> ' + dhcp_hostsdir + '"' + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + content = stdout.read() - print self.name, ': command:', command - (_, stdout, _) = self.ssh_conn.exec_command(command) - content = stdout.read() - if len(content) == 0: - return True - else: + ip_data = mac.upper() + ',' + ip + + command = 'sudo ip netns exec ' + dhcp_namespace + ' touch ' + dhcp_hostsdir + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + content = stdout.read() + + command = 'sudo ip netns exec ' + dhcp_namespace + ' sudo bash -ec "echo ' + ip_data + ' >> ' + dhcp_hostsdir + '"' + + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + content = stdout.read() + + if len(content) == 0: + return True + else: + return False + except paramiko.ssh_exception.SSHException as e: + self.logger.error("set_mac_dhcp_server ssh Exception: " + str(e)) + if "SSH session not active" in str(e): + self.ssh_connect() return False def delete_mac_dhcp_server(self, ip, mac, vlan, dhcp_path): @@ -1010,28 +1189,34 @@ class host_thread(threading.Thread): """ if self.test: - return + return False + try: + dhcp_namespace = str(vlan) + '-dnsmasq' + dhcp_path = os.path.join(dhcp_path, dhcp_namespace) + dhcp_hostsdir = os.path.join(dhcp_path, dhcp_namespace) - net_namespace = 'ovim-' + vlan - dhcp_path = os.path.join(dhcp_path, net_namespace) - dhcp_hostsdir = os.path.join(dhcp_path, net_namespace) + if not ip: + return False - if not ip: - return False + ip_data = mac.upper() + ',' + ip - ip_data = mac.upper() + ',' + ip + command = 'sudo ip netns exec ' + dhcp_namespace + ' sudo sed -i \'/' + ip_data + '/d\' ' + dhcp_hostsdir + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + content = stdout.read() - command = 'sudo ip netns exec ' + net_namespace + ' sudo sed -i \'/' + ip_data + '/d\' ' + dhcp_hostsdir - print self.name, ': command:', command - (_, stdout, _) = self.ssh_conn.exec_command(command) - content = stdout.read() + if len(content) == 0: + return True + else: + return False - if len(content) == 0: - return True - else: + except paramiko.ssh_exception.SSHException as e: + self.logger.error("set_mac_dhcp_server ssh Exception: " + str(e)) + if "SSH session not active" in str(e): + self.ssh_connect() return False - def launch_dhcp_server(self, vlan, ip_range, netmask, dhcp_path, gateway): + def launch_dhcp_server(self, vlan, ip_range, netmask, dhcp_path, gateway, dns_list=None, routes=None): """ Generate a linux bridge and attache the port to a OVS bridge :param self: @@ -1040,50 +1225,82 @@ class host_thread(threading.Thread): :param netmask: network netmask :param dhcp_path: dhcp conf file path that live in namespace side :param gateway: Gateway address for dhcp net + :param dns_list: dns list for dhcp server + :param routes: routes list for dhcp server :return: True if success """ if self.test: - return + return True + try: + ns_interface = str(vlan) + '-vethDO' + dhcp_namespace = str(vlan) + '-dnsmasq' + dhcp_path = os.path.join(dhcp_path, dhcp_namespace, '') + leases_path = os.path.join(dhcp_path, "dnsmasq.leases") + pid_file = os.path.join(dhcp_path, 'dnsmasq.pid') - interface = 'tap-' + vlan - net_namespace = 'ovim-' + vlan - dhcp_path = os.path.join(dhcp_path, net_namespace) - leases_path = os.path.join(dhcp_path, "dnsmasq.leases") - pid_file = os.path.join(dhcp_path, 'dnsmasq.pid') - dhcp_range = ip_range[0] + ',' + ip_range[1] + ',' + netmask + dhcp_range = ip_range[0] + ',' + ip_range[1] + ',' + netmask - command = 'sudo ip netns exec ' + net_namespace + ' mkdir -p ' + dhcp_path - print self.name, ': command:', command - (_, stdout, _) = self.ssh_conn.exec_command(command) - content = stdout.read() + command = 'sudo ip netns exec ' + dhcp_namespace + ' mkdir -p ' + dhcp_path + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + content = stdout.read() - pid_path = os.path.join(dhcp_path, 'dnsmasq.pid') - command = 'sudo ip netns exec ' + net_namespace + ' cat ' + pid_path - print self.name, ': command:', command - (_, stdout, _) = self.ssh_conn.exec_command(command) - content = stdout.read() - # check if pid is runing - pid_status_path = content - if content: - command = "ps aux | awk '{print $2 }' | grep " + pid_status_path - print self.name, ': command:', command + pid_path = os.path.join(dhcp_path, 'dnsmasq.pid') + command = 'sudo ip netns exec ' + dhcp_namespace + ' cat ' + pid_path + self.logger.debug("command: " + command) (_, stdout, _) = self.ssh_conn.exec_command(command) content = stdout.read() - if not content: - command = 'sudo ip netns exec ' + net_namespace + ' /usr/sbin/dnsmasq --strict-order --except-interface=lo ' \ - '--interface=' + interface + ' --bind-interfaces --dhcp-hostsdir=' + dhcp_path + \ - ' --dhcp-range ' + dhcp_range + ' --pid-file=' + pid_file + ' --dhcp-leasefile=' + leases_path + \ - ' --listen-address ' + gateway - print self.name, ': command:', command - (_, stdout, _) = self.ssh_conn.exec_command(command) - content = stdout.readline() + # check if pid is runing + pid_status_path = content + if content: + command = "ps aux | awk '{print $2 }' | grep " + pid_status_path + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + content = stdout.read() - if len(content) == 0: - return True - else: + gateway_option = ' --dhcp-option=3,' + gateway + + dhcp_route_option = '' + if routes: + dhcp_route_option = ' --dhcp-option=121' + for key, value in routes.iteritems(): + if 'default' == key: + gateway_option = ' --dhcp-option=3,' + value + else: + dhcp_route_option += ',' + key + ',' + value + dns_data = '' + if dns_list: + dns_data = ' --dhcp-option=6' + for dns in dns_list: + dns_data += ',' + dns + + if not content: + command = 'sudo ip netns exec ' + dhcp_namespace + ' /usr/sbin/dnsmasq --strict-order --except-interface=lo ' \ + '--interface=' + ns_interface + \ + ' --bind-interfaces --dhcp-hostsdir=' + dhcp_path + \ + ' --dhcp-range ' + dhcp_range + \ + ' --pid-file=' + pid_file + \ + ' --dhcp-leasefile=' + leases_path + \ + ' --listen-address ' + ip_range[0] + \ + gateway_option + \ + dhcp_route_option + \ + dns_data + + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + content = stdout.readline() + + if len(content) == 0: + return True + else: + return False + except paramiko.ssh_exception.SSHException as e: + self.logger.error("launch_dhcp_server ssh Exception: " + str(e)) + if "SSH session not active" in str(e): + self.ssh_connect() return False def delete_dhcp_interfaces(self, vlan): @@ -1094,78 +1311,368 @@ class host_thread(threading.Thread): """ if self.test: - return + return True + try: + br_veth_name = str(vlan) + '-vethDO' + ovs_veth_name = str(vlan) + '-vethOD' + dhcp_namespace = str(vlan) + '-dnsmasq' + + command = 'sudo ovs-vsctl del-port br-int ' + ovs_veth_name + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + content = stdout.read() + + command = 'sudo ip netns exec ' + dhcp_namespace + ' ip link set dev ' + br_veth_name + ' down' + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + content = stdout.read() + + command = 'sudo ip link set dev ' + dhcp_namespace + ' down' + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + content = stdout.read() + + command = 'sudo brctl delbr ' + dhcp_namespace + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + content = stdout.read() + + command = 'sudo ip netns del ' + dhcp_namespace + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + content = stdout.read() + + except paramiko.ssh_exception.SSHException as e: + self.logger.error("delete_dhcp_interfaces ssh Exception: " + str(e)) + if "SSH session not active" in str(e): + self.ssh_connect() + return False + + def create_dhcp_interfaces(self, vlan, ip_listen_address, netmask): + """ + Create a linux bridge with STP active + :param vlan: segmentation id + :param ip_listen_address: Listen Ip address for the dhcp service, the tap interface living in namesapce side + :param netmask: dhcp net CIDR + :return: True if success + """ + + if self.test: + return True + try: + ovs_veth_name = str(vlan) + '-vethOD' + ns_veth = str(vlan) + '-vethDO' + dhcp_namespace = str(vlan) + '-dnsmasq' + + command = 'sudo ip netns add ' + dhcp_namespace + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + content = stdout.read() + + command = 'sudo ip link add ' + ns_veth + ' type veth peer name ' + ovs_veth_name + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + content = stdout.read() + + command = 'sudo ip link set ' + ns_veth + ' netns ' + dhcp_namespace + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + content = stdout.read() + + command = 'sudo ip netns exec ' + dhcp_namespace + ' ip link set dev ' + ns_veth + ' up' + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + content = stdout.read() + + command = 'sudo ovs-vsctl add-port br-int ' + ovs_veth_name + ' tag=' + str(vlan) + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + content = stdout.read() + + command = 'sudo ip link set dev ' + ovs_veth_name + ' up' + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + content = stdout.read() + + command = 'sudo ip netns exec ' + dhcp_namespace + ' ip link set dev lo up' + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + content = stdout.read() + + command = 'sudo ip netns exec ' + dhcp_namespace + ' ' + ' ifconfig ' + ns_veth \ + + ' ' + ip_listen_address + ' netmask ' + netmask + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + content = stdout.read() + if len(content) == 0: + return True + else: + return False + except paramiko.ssh_exception.SSHException as e: + self.logger.error("create_dhcp_interfaces ssh Exception: " + str(e)) + if "SSH session not active" in str(e): + self.ssh_connect() + return False - net_namespace = 'ovim-' + vlan - command = 'sudo ovs-vsctl del-port br-int ovs-tap-' + vlan - print self.name, ': command:', command + def delete_qrouter_connection(self, vlan, link): + """ + Delete qrouter Namesapce with all veth interfaces need it + :param vlan: + :param link: + :return: + """ + + ns_qouter = str(vlan) + '-qrouter' + qrouter_ovs_veth = str(vlan) + '-vethOQ' + qrouter_ns_veth = str(vlan) + '-vethQO' + + qrouter_br_veth = str(vlan) + '-vethBQ' + qrouter_ns_router_veth = str(vlan) + '-vethQB' + + # delete ovs veth to ovs br-int + command = 'sudo ovs-vsctl del-port br-int {}'.format(qrouter_ovs_veth) + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + content = stdout.read() + + # down ns veth + command = 'sudo ip netns exec {} ip link set dev {} down'.format(ns_qouter, qrouter_ns_veth) + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + content = stdout.read() + + # down ovs veth interface + command = 'sudo ip link set dev {} down'.format(qrouter_br_veth) + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + content = stdout.read() + + # down br veth interface + command = 'sudo ip link set dev {} down'.format(qrouter_ovs_veth) + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + content = stdout.read() + + # down br veth interface + command = 'sudo ip link set dev {} down'.format(qrouter_ns_router_veth) + self.logger.debug("command: " + command) (_, stdout, _) = self.ssh_conn.exec_command(command) content = stdout.read() - command = 'sudo ip netns exec ' + net_namespace + ' ip link set dev tap-' + vlan + ' down' - print self.name, ': command:', command + # down br veth interface + command = 'sudo brctl delif {} {}'.format(link, qrouter_br_veth) + self.logger.debug("command: " + command) (_, stdout, _) = self.ssh_conn.exec_command(command) content = stdout.read() - command = 'sudo ip link set dev ovs-tap-' + vlan + ' down' - print self.name, ': command:', command + + # delete NS + command = 'sudo ip netns del ' + ns_qouter + self.logger.debug("command: " + command) (_, stdout, _) = self.ssh_conn.exec_command(command) content = stdout.read() - def create_dhcp_interfaces(self, vlan, ip, netmask): + def create_qrouter_ovs_connection(self, vlan, gateway, dhcp_cidr): """ - Create a linux bridge with STP active - :param vlan: segmentation id - :param ip: Ip included in the dhcp range for the tap interface living in namesapce side - :param netmask: dhcp net CIDR - :return: True if success + Create qrouter Namesapce with all veth interfaces need it between NS and OVS + :param vlan: + :param gateway: + :return: """ - if self.test: - return + ns_qouter = str(vlan) + '-qrouter' + qrouter_ovs_veth = str(vlan) + '-vethOQ' + qrouter_ns_veth = str(vlan) + '-vethQO' - net_namespace = 'ovim-' + vlan - namespace_interface = 'tap-' + vlan + # Create NS + command = 'sudo ip netns add ' + ns_qouter + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + content = stdout.read() - command = 'sudo ip netns add ' + net_namespace - print self.name, ': command:', command + # Create pait veth + command = 'sudo ip link add {} type veth peer name {}'.format(qrouter_ns_veth, qrouter_ovs_veth) + self.logger.debug("command: " + command) (_, stdout, _) = self.ssh_conn.exec_command(command) content = stdout.read() - command = 'sudo ip link add tap-' + vlan + ' type veth peer name ovs-tap-' + vlan - print self.name, ': command:', command + # up ovs veth interface + command = 'sudo ip link set dev {} up'.format(qrouter_ovs_veth) + self.logger.debug("command: " + command) (_, stdout, _) = self.ssh_conn.exec_command(command) content = stdout.read() - command = 'sudo ovs-vsctl add-port br-int ovs-tap-' + vlan + ' tag=' + vlan - print self.name, ': command:', command + # add ovs veth to ovs br-int + command = 'sudo ovs-vsctl add-port br-int {} tag={}'.format(qrouter_ovs_veth, vlan) + self.logger.debug("command: " + command) (_, stdout, _) = self.ssh_conn.exec_command(command) content = stdout.read() - command = 'sudo ip link set tap-' + vlan + ' netns ' + net_namespace - print self.name, ': command:', command + # add veth to ns + command = 'sudo ip link set {} netns {}'.format(qrouter_ns_veth, ns_qouter) + self.logger.debug("command: " + command) (_, stdout, _) = self.ssh_conn.exec_command(command) content = stdout.read() - command = 'sudo ip netns exec ' + net_namespace + ' ip link set dev tap-' + vlan + ' up' - print self.name, ': command:', command + # up ns loopback + command = 'sudo ip netns exec {} ip link set dev lo up'.format(ns_qouter) + self.logger.debug("command: " + command) (_, stdout, _) = self.ssh_conn.exec_command(command) content = stdout.read() - command = 'sudo ip link set dev ovs-tap-' + vlan + ' up' - print self.name, ': command:', command + # up ns veth + command = 'sudo ip netns exec {} ip link set dev {} up'.format(ns_qouter, qrouter_ns_veth) + self.logger.debug("command: " + command) (_, stdout, _) = self.ssh_conn.exec_command(command) content = stdout.read() - command = 'sudo ip netns exec ' + net_namespace + ' ' + ' ifconfig ' + namespace_interface \ - + ' ' + ip + ' netmask ' + netmask - print self.name, ': command:', command + from netaddr import IPNetwork + ip_tools = IPNetwork(dhcp_cidr) + cidr_len = ip_tools.prefixlen + + # set gw to ns veth + command = 'sudo ip netns exec {} ip address add {}/{} dev {}'.format(ns_qouter, gateway, cidr_len, qrouter_ns_veth) + self.logger.debug("command: " + command) (_, stdout, _) = self.ssh_conn.exec_command(command) content = stdout.read() - if len(content) == 0: - return True + def add_ns_routes(self, vlan, routes): + + for key, value in routes.iteritems(): + ns_qouter = str(vlan) + '-qrouter' + qrouter_ns_router_veth = str(vlan) + '-vethQB' + # up ns veth + if key == 'default': + command = 'sudo ip netns exec {} ip route add {} via {} '.format(ns_qouter, key, value) + else: + command = 'sudo ip netns exec {} ip route add {} via {} dev {}'.format(ns_qouter, key, value, + qrouter_ns_router_veth) + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + content = stdout.read() + + def create_qrouter_br_connection(self, vlan, cidr, link): + """ + Create veth interfaces between user bridge (link) and OVS + :param vlan: + :param link: + :return: + """ + + ns_qouter = str(vlan) + '-qrouter' + qrouter_ns_router_veth = str(vlan) + '-vethQB' + qrouter_br_veth = str(vlan) + '-vethBQ' + + # Create pait veth + command = 'sudo ip link add {} type veth peer name {}'.format(qrouter_br_veth, qrouter_ns_router_veth) + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + content = stdout.read() + + # up ovs veth interface + command = 'sudo ip link set dev {} up'.format(qrouter_br_veth) + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + content = stdout.read() + + # add veth to ns + command = 'sudo ip link set {} netns {}'.format(qrouter_ns_router_veth, ns_qouter) + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + content = stdout.read() + + # up ns veth + command = 'sudo ip netns exec {} ip link set dev {} up'.format(ns_qouter, qrouter_ns_router_veth) + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + content = stdout.read() + + command = 'sudo ip netns exec {} ip address add {} dev {}'.format(ns_qouter, link['nat'], qrouter_ns_router_veth) + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + content = stdout.read() + + command = 'sudo brctl show | grep {}'.format(link['iface']) + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + content = stdout.read() + + if content > '': + # up ns veth + command = 'sudo brctl addif {} {}'.format(link['iface'], qrouter_br_veth) + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + content = stdout.read() + + # up ns veth + command = 'sudo ip netns exec {} iptables -t nat -A POSTROUTING -o {} -s {} -d {} -j MASQUERADE' \ + .format(ns_qouter, qrouter_ns_router_veth, link['nat'], cidr) + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + content = stdout.read() + + else: + self.logger.error('Bridge {} given by user not exist'.format(qrouter_br_veth)) + + + + def create_link_bridge_to_ovs(self, vlan, link): + """ + Create interfaces to connect a linux bridge with tenant net + :param vlan: segmentation id + :return: True if success + """ + if self.test: + return True + try: + + br_tap_name = str(vlan) + '-vethBO' + br_ovs_name = str(vlan) + '-vethOB' + + # is a bridge or a interface + command = 'sudo brctl show | grep {}'.format(link) + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + content = stdout.read() + + if content > '': + command = 'sudo ip link add {} type veth peer name {}'.format(br_tap_name, br_ovs_name) + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + content = stdout.read() + + command = 'sudo ip link set dev {} up'.format(br_tap_name) + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + content = stdout.read() + + command = 'sudo ip link set dev {} up'.format(br_ovs_name) + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + content = stdout.read() + + command = 'sudo ovs-vsctl add-port br-int {} tag={}'.format(br_ovs_name, str(vlan)) + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + content = stdout.read() + + command = 'sudo brctl addif ' + link + ' {}'.format(br_tap_name) + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + content = stdout.read() + + if len(content) == 0: + return True + else: + return False + else: + self.logger.error('Link is not present, please check {}'.format(link)) + return False + except paramiko.ssh_exception.SSHException as e: + self.logger.error("create_dhcp_interfaces ssh Exception: " + str(e)) + if "SSH session not active" in str(e): + self.ssh_connect() return False def create_ovs_vxlan_tunnel(self, vxlan_interface, remote_ip): @@ -1175,18 +1682,28 @@ class host_thread(threading.Thread): :param remote_ip: tunnel endpoint remote compute ip. :return: """ - if self.test: - return - command = 'sudo ovs-vsctl add-port br-int ' + vxlan_interface + \ - ' -- set Interface ' + vxlan_interface + ' type=vxlan options:remote_ip=' + remote_ip + \ - ' -- set Port ' + vxlan_interface + ' other_config:stp-path-cost=10' - print self.name, ': command:', command - (_, stdout, _) = self.ssh_conn.exec_command(command) - content = stdout.read() - print content - if len(content) == 0: + if self.test or not self.connectivity: return True - else: + if remote_ip == 'localhost': + if self.localhost: + return # TODO: Cannot create a vxlan between localhost and localhost + remote_ip = self.local_ip + try: + command = 'sudo ovs-vsctl add-port br-int ' + vxlan_interface + \ + ' -- set Interface ' + vxlan_interface + ' type=vxlan options:remote_ip=' + remote_ip + \ + ' -- set Port ' + vxlan_interface + ' other_config:stp-path-cost=10' + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + content = stdout.read() + # print content + if len(content) == 0: + return True + else: + return False + except paramiko.ssh_exception.SSHException as e: + self.logger.error("create_ovs_vxlan_tunnel ssh Exception: " + str(e)) + if "SSH session not active" in str(e): + self.ssh_connect() return False def delete_ovs_vxlan_tunnel(self, vxlan_interface): @@ -1195,16 +1712,22 @@ class host_thread(threading.Thread): :param vxlan_interface: vlxan name to be delete it. :return: True if success. """ - if self.test: - return - command = 'sudo ovs-vsctl del-port br-int ' + vxlan_interface - print self.name, ': command:', command - (_, stdout, _) = self.ssh_conn.exec_command(command) - content = stdout.read() - print content - if len(content) == 0: + if self.test or not self.connectivity: return True - else: + try: + command = 'sudo ovs-vsctl del-port br-int ' + vxlan_interface + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + content = stdout.read() + # print content + if len(content) == 0: + return True + else: + return False + except paramiko.ssh_exception.SSHException as e: + self.logger.error("delete_ovs_vxlan_tunnel ssh Exception: " + str(e)) + if "SSH session not active" in str(e): + self.ssh_connect() return False def delete_ovs_bridge(self): @@ -1212,56 +1735,67 @@ class host_thread(threading.Thread): Delete a OVS bridge from a compute. :return: True if success """ - if self.test: - return - command = 'sudo ovs-vsctl del-br br-int' - print self.name, ': command:', command - (_, stdout, _) = self.ssh_conn.exec_command(command) - content = stdout.read() - if len(content) == 0: + if self.test or not self.connectivity: return True - else: + try: + command = 'sudo ovs-vsctl del-br br-int' + self.logger.debug("command: " + command) + (_, stdout, _) = self.ssh_conn.exec_command(command) + content = stdout.read() + if len(content) == 0: + return True + else: + return False + except paramiko.ssh_exception.SSHException as e: + self.logger.error("delete_ovs_bridge ssh Exception: " + str(e)) + if "SSH session not active" in str(e): + self.ssh_connect() return False def get_file_info(self, path): command = 'ls -lL --time-style=+%Y-%m-%dT%H:%M:%S ' + path - print self.name, ': command:', command - (_, stdout, _) = self.ssh_conn.exec_command(command) - content = stdout.read() - if len(content) == 0: - return None # file does not exist - else: - return content.split(" ") #(permission, 1, owner, group, size, date, file) + try: + content = self.run_command(command) + return content.split(" ") # (permission, 1, owner, group, size, date, file) + except RunCommandException as e: + return None # file does not exist def qemu_get_info(self, path): command = 'qemu-img info ' + path - print self.name, ': command:', command - (_, stdout, stderr) = self.ssh_conn.exec_command(command) - content = stdout.read() - if len(content) == 0: - error = stderr.read() - print self.name, ": get_qemu_info error ", error - raise paramiko.ssh_exception.SSHException("Error getting qemu_info: " + error) - else: - try: - return yaml.load(content) - except yaml.YAMLError as exc: - text = "" - if hasattr(exc, 'problem_mark'): - mark = exc.problem_mark - text = " at position: (%s:%s)" % (mark.line+1, mark.column+1) - print self.name, ": get_qemu_info yaml format Exception", text - raise paramiko.ssh_exception.SSHException("Error getting qemu_info yaml format" + text) + content = self.run_command(command) + try: + return yaml.load(content) + except yaml.YAMLError as exc: + text = "" + if hasattr(exc, 'problem_mark'): + mark = exc.problem_mark + text = " at position: (%s:%s)" % (mark.line+1, mark.column+1) + self.logger.error("get_qemu_info yaml format Exception " + text) + raise RunCommandException("Error getting qemu_info yaml format" + text) def qemu_change_backing(self, inc_file, new_backing_file): - command = 'qemu-img rebase -u -b ' + new_backing_file + ' ' + inc_file - print self.name, ': command:', command - (_, _, stderr) = self.ssh_conn.exec_command(command) - content = stderr.read() - if len(content) == 0: + command = 'qemu-img rebase -u -b {} {}'.format(new_backing_file, inc_file) + try: + self.run_command(command) return 0 - else: - print self.name, ": qemu_change_backing error: ", content + except RunCommandException as e: + self.logger.error("qemu_change_backing error: " + str(e)) + return -1 + + def qemu_create_empty_disk(self, dev): + + if not dev and 'source' not in dev and 'file format' not in dev and 'image_size' not in dev: + self.logger.error("qemu_create_empty_disk error: missing image parameter") + return -1 + + empty_disk_path = dev['source file'] + + command = 'qemu-img create -f qcow2 {} {}G'.format(empty_disk_path, dev['image_size']) + try: + self.run_command(command) + return 0 + except RunCommandException as e: + self.logger.error("qemu_create_empty_disk error: " + str(e)) return -1 def get_notused_filename(self, proposed_name, suffix=''): @@ -1305,12 +1839,8 @@ class host_thread(threading.Thread): def delete_file(self, file_name): - command = 'rm -f '+file_name - print self.name, ': command:', command - (_, _, stderr) = self.ssh_conn.exec_command(command) - error_msg = stderr.read() - if len(error_msg) > 0: - raise paramiko.ssh_exception.SSHException("Error deleting file: " + error_msg) + command = 'rm -f ' + file_name + self.run_command(command) def copy_file(self, source, destination, perserve_time=True): if source[0:4]=="http": @@ -1320,12 +1850,8 @@ class host_thread(threading.Thread): command = 'cp --no-preserve=mode' if perserve_time: command += ' --preserve=timestamps' - command += " '{}' '{}'".format(source, destination) - print self.name, ': command:', command - (_, _, stderr) = self.ssh_conn.exec_command(command) - error_msg = stderr.read() - if len(error_msg) > 0: - raise paramiko.ssh_exception.SSHException("Error copying image to local host: " + error_msg) + command += " '{}' '{}'".format(source, destination) + self.run_command(command) def copy_remote_file(self, remote_file, use_incremental): ''' Copy a file from the repository to local folder and recursively @@ -1412,7 +1938,7 @@ class host_thread(threading.Thread): result, server_data = self.db.get_instance(server_id) self.db_lock.release() if result <= 0: - print self.name, ": launch_server ERROR getting server from DB",result, server_data + self.logger.error("launch_server ERROR getting server from DB %d %s", result, server_data) return result, server_data #0: get image metadata @@ -1420,7 +1946,7 @@ class host_thread(threading.Thread): use_incremental = None if "use_incremental" in server_metadata: - use_incremental = False if server_metadata["use_incremental"]=="no" else True + use_incremental = False if server_metadata["use_incremental"] == "no" else True server_host_files = self.localinfo['server_files'].get( server['uuid'], {}) if rebuild: @@ -1434,31 +1960,45 @@ class host_thread(threading.Thread): devices = [ {"type":"disk", "image_id":server['image_id'], "vpci":server_metadata.get('vpci', None) } ] if 'extended' in server_data and server_data['extended']!=None and "devices" in server_data['extended']: devices += server_data['extended']['devices'] - + empty_path = None for dev in devices: - if dev['image_id'] == None: + image_id = dev.get('image_id') + if not image_id: + import uuid + uuid_empty = str(uuid.uuid4()) + empty_path = self.empty_image_path + uuid_empty + '.qcow2' # local path for empty disk + + dev['source file'] = empty_path + dev['file format'] = 'qcow2' + self.qemu_create_empty_disk(dev) + server_host_files[uuid_empty] = {'source file': empty_path, + 'file format': dev['file format']} + continue - - self.db_lock.acquire() - result, content = self.db.get_table(FROM='images', SELECT=('path', 'metadata'), - WHERE={'uuid': dev['image_id']}) - self.db_lock.release() - if result <= 0: - error_text = "ERROR", result, content, "when getting image", dev['image_id'] - print self.name, ": launch_server", error_text - return -1, error_text - if content[0]['metadata'] is not None: - dev['metadata'] = json.loads(content[0]['metadata']) else: - dev['metadata'] = {} - - if dev['image_id'] in server_host_files: - dev['source file'] = server_host_files[ dev['image_id'] ] ['source file'] #local path - dev['file format'] = server_host_files[ dev['image_id'] ] ['file format'] # raw or qcow2 - continue + self.db_lock.acquire() + result, content = self.db.get_table(FROM='images', SELECT=('path', 'metadata'), + WHERE={'uuid': image_id}) + self.db_lock.release() + if result <= 0: + error_text = "ERROR", result, content, "when getting image", dev['image_id'] + self.logger.error("launch_server " + error_text) + return -1, error_text + if content[0]['metadata'] is not None: + dev['metadata'] = json.loads(content[0]['metadata']) + else: + dev['metadata'] = {} + + if image_id in server_host_files: + dev['source file'] = server_host_files[image_id]['source file'] #local path + dev['file format'] = server_host_files[image_id]['file format'] # raw or qcow2 + continue #2: copy image to host - remote_file = content[0]['path'] + if image_id: + remote_file = content[0]['path'] + else: + remote_file = empty_path use_incremental_image = use_incremental if dev['metadata'].get("use_incremental") == "no": use_incremental_image = False @@ -1467,14 +2007,10 @@ class host_thread(threading.Thread): #create incremental image if use_incremental_image: local_file_inc = self.get_notused_filename(local_file, '.inc') - command = 'qemu-img create -f qcow2 '+local_file_inc+ ' -o backing_file='+ local_file - print 'command:', command - (_, _, stderr) = self.ssh_conn.exec_command(command) - error_msg = stderr.read() - if len(error_msg) > 0: - raise paramiko.ssh_exception.SSHException("Error creating incremental file: " + error_msg) + command = 'qemu-img create -f qcow2 {} -o backing_file={}'.format(local_file_inc, local_file) + self.run_command(command) local_file = local_file_inc - qemu_info = {'file format':'qcow2'} + qemu_info = {'file format': 'qcow2'} server_host_files[ dev['image_id'] ] = {'source file': local_file, 'file format': qemu_info['file format']} @@ -1487,14 +2023,14 @@ class host_thread(threading.Thread): #3 Create XML result, xml = self.create_xml_server(server_data, devices, server_metadata) #local_file if result <0: - print self.name, ": create xml server error:", xml + self.logger.error("create xml server error: " + xml) return -2, xml - print self.name, ": create xml:", xml + self.logger.debug("create xml: " + xml) atribute = host_thread.lvirt_module.VIR_DOMAIN_START_PAUSED if paused == "yes" else 0 #4 Start the domain if not rebuild: #ensures that any pending destroying server is done self.server_forceoff(True) - #print self.name, ": launching instance" #, xml + #self.logger.debug("launching instance " + xml) conn.createXML(xml, atribute) #self.server_status[server_id] = 'PAUSED' if paused == "yes" else 'ACTIVE' @@ -1502,15 +2038,15 @@ class host_thread(threading.Thread): except paramiko.ssh_exception.SSHException as e: text = e.args[0] - print self.name, ": launch_server(%s) ssh Exception: %s" %(server_id, text) + self.logger.error("launch_server id='%s' ssh Exception: %s", server_id, text) if "SSH session not active" in text: self.ssh_connect() except host_thread.lvirt_module.libvirtError as e: text = e.get_error_message() - print self.name, ": launch_server(%s) libvirt Exception: %s" %(server_id, text) + self.logger.error("launch_server id='%s' libvirt Exception: %s", server_id, text) except Exception as e: text = str(e) - print self.name, ": launch_server(%s) Exception: %s" %(server_id, text) + self.logger.error("launch_server id='%s' Exception: %s", server_id, text) return -1, text def update_servers_status(self): @@ -1523,12 +2059,12 @@ class host_thread(threading.Thread): # VIR_DOMAIN_SHUTOFF = 5 # VIR_DOMAIN_CRASHED = 6 # VIR_DOMAIN_PMSUSPENDED = 7 #TODO suspended - + if self.test or len(self.server_status)==0: - return - + return + try: - conn = host_thread.lvirt_module.open("qemu+ssh://"+self.user+"@"+self.host+"/system") + conn = host_thread.lvirt_module.open(self.lvirt_conn_uri) domains= conn.listAllDomains() domain_dict={} for domain in domains: @@ -1548,7 +2084,7 @@ class host_thread(threading.Thread): domain_dict[uuid] = new_status conn.close() except host_thread.lvirt_module.libvirtError as e: - print self.name, ": get_state() Exception '", e.get_error_message() + self.logger.error("get_state() Exception " + e.get_error_message()) return for server_id, current_status in self.server_status.iteritems(): @@ -1563,7 +2099,7 @@ class host_thread(threading.Thread): if new_status == 'INACTIVE' and current_status == 'ERROR': continue #keep ERROR status, because obviously this machine is not running #change status - print self.name, ": server ", server_id, "status change from ", current_status, "to", new_status + self.logger.debug("server id='%s' status change from '%s' to '%s'", server_id, current_status, new_status) STATUS={'progress':100, 'status':new_status} if new_status == 'ERROR': STATUS['last_error'] = 'machine has crashed' @@ -1601,10 +2137,14 @@ class host_thread(threading.Thread): if req['status']!='ERROR': time.sleep(5) new_status = 'INACTIVE' - elif 'start' in req['action'] and req['status']!='ERROR': new_status = 'ACTIVE' - elif 'resume' in req['action'] and req['status']!='ERROR' and req['status']!='INACTIVE' : new_status = 'ACTIVE' - elif 'pause' in req['action'] and req['status']!='ERROR': new_status = 'PAUSED' - elif 'reboot' in req['action'] and req['status']!='ERROR': new_status = 'ACTIVE' + elif 'start' in req['action'] and req['status']!='ERROR': + new_status = 'ACTIVE' + elif 'resume' in req['action'] and req['status']!='ERROR' and req['status']!='INACTIVE': + new_status = 'ACTIVE' + elif 'pause' in req['action'] and req['status']!='ERROR': + new_status = 'PAUSED' + elif 'reboot' in req['action'] and req['status']!='ERROR': + new_status = 'ACTIVE' elif 'rebuild' in req['action']: time.sleep(random.randint(20,150)) new_status = 'ACTIVE' @@ -1613,7 +2153,7 @@ class host_thread(threading.Thread): self.create_image(None, req) else: try: - conn = host_thread.lvirt_module.open("qemu+ssh://"+self.user+"@"+self.host+"/system") + conn = host_thread.lvirt_module.open(self.lvirt_conn_uri) try: dom = conn.lookupByUUIDString(server_id) except host_thread.lvirt_module.libvirtError as e: @@ -1621,38 +2161,40 @@ class host_thread(threading.Thread): if 'LookupByUUIDString' in text or 'Domain not found' in text or 'No existe un dominio coincidente' in text: dom = None else: - print self.name, ": action_on_server(",server_id,") libvirt exception:", text + self.logger.error("action_on_server id='%s' libvirt exception: %s", server_id, text) raise e if 'forceOff' in req['action']: if dom == None: - print self.name, ": action_on_server(",server_id,") domain not running" + self.logger.debug("action_on_server id='%s' domain not running", server_id) else: try: - print self.name, ": sending DESTROY to server", server_id + self.logger.debug("sending DESTROY to server id='%s'", server_id) dom.destroy() except Exception as e: if "domain is not running" not in e.get_error_message(): - print self.name, ": action_on_server(",server_id,") Exception while sending force off:", e.get_error_message() + self.logger.error("action_on_server id='%s' Exception while sending force off: %s", + server_id, e.get_error_message()) last_error = 'action_on_server Exception while destroy: ' + e.get_error_message() new_status = 'ERROR' elif 'terminate' in req['action']: if dom == None: - print self.name, ": action_on_server(",server_id,") domain not running" + self.logger.debug("action_on_server id='%s' domain not running", server_id) new_status = 'deleted' else: try: if req['action']['terminate'] == 'force': - print self.name, ": sending DESTROY to server", server_id + self.logger.debug("sending DESTROY to server id='%s'", server_id) dom.destroy() new_status = 'deleted' else: - print self.name, ": sending SHUTDOWN to server", server_id + self.logger.debug("sending SHUTDOWN to server id='%s'", server_id) dom.shutdown() self.pending_terminate_server.append( (time.time()+10,server_id) ) except Exception as e: - print self.name, ": action_on_server(",server_id,") Exception while destroy:", e.get_error_message() + self.logger.error("action_on_server id='%s' Exception while destroy: %s", + server_id, e.get_error_message()) last_error = 'action_on_server Exception while destroy: ' + e.get_error_message() new_status = 'ERROR' if "domain is not running" in e.get_error_message(): @@ -1660,7 +2202,8 @@ class host_thread(threading.Thread): dom.undefine() new_status = 'deleted' except Exception: - print self.name, ": action_on_server(",server_id,") Exception while undefine:", e.get_error_message() + self.logger.error("action_on_server id='%s' Exception while undefine: %s", + server_id, e.get_error_message()) last_error = 'action_on_server Exception2 while undefine:', e.get_error_message() #Exception: 'virDomainDetachDevice() failed' if new_status=='deleted': @@ -1678,14 +2221,15 @@ class host_thread(threading.Thread): elif 'shutoff' in req['action'] or 'shutdown' in req['action']: try: if dom == None: - print self.name, ": action_on_server(",server_id,") domain not running" + self.logger.debug("action_on_server id='%s' domain not running", server_id) else: dom.shutdown() # new_status = 'INACTIVE' #TODO: check status for changing at database except Exception as e: new_status = 'ERROR' - print self.name, ": action_on_server(",server_id,") Exception while shutdown:", e.get_error_message() + self.logger.error("action_on_server id='%s' Exception while shutdown: %s", + server_id, e.get_error_message()) last_error = 'action_on_server Exception while shutdown: ' + e.get_error_message() elif 'rebuild' in req['action']: @@ -1715,7 +2259,8 @@ class host_thread(threading.Thread): dom.resume() # new_status = 'ACTIVE' except Exception as e: - print self.name, ": action_on_server(",server_id,") Exception while resume:", e.get_error_message() + self.logger.error("action_on_server id='%s' Exception while resume: %s", + server_id, e.get_error_message()) elif 'pause' in req['action']: try: @@ -1725,7 +2270,8 @@ class host_thread(threading.Thread): dom.suspend() # new_status = 'PAUSED' except Exception as e: - print self.name, ": action_on_server(",server_id,") Exception while pause:", e.get_error_message() + self.logger.error("action_on_server id='%s' Exception while pause: %s", + server_id, e.get_error_message()) elif 'reboot' in req['action']: try: @@ -1733,10 +2279,11 @@ class host_thread(threading.Thread): pass else: dom.reboot() - print self.name, ": action_on_server(",server_id,") reboot:" + self.logger.debug("action_on_server id='%s' reboot:", server_id) #new_status = 'ACTIVE' except Exception as e: - print self.name, ": action_on_server(",server_id,") Exception while reboot:", e.get_error_message() + self.logger.error("action_on_server id='%s' Exception while reboot: %s", + server_id, e.get_error_message()) elif 'createImage' in req['action']: self.create_image(dom, req) @@ -1747,14 +2294,15 @@ class host_thread(threading.Thread): text = e.get_error_message() new_status = "ERROR" last_error = text - print self.name, ": action_on_server(",server_id,") Exception '", text if 'LookupByUUIDString' in text or 'Domain not found' in text or 'No existe un dominio coincidente' in text: - print self.name, ": action_on_server(",server_id,") Exception removed from host" + self.logger.debug("action_on_server id='%s' Exception removed from host", server_id) + else: + self.logger.error("action_on_server id='%s' Exception %s", server_id, text) #end of if self.test if new_status == None: return 1 - print self.name, ": action_on_server(",server_id,") new status", new_status, last_error + self.logger.debug("action_on_server id='%s' new status=%s %s",server_id, new_status, last_error) UPDATE = {'progress':100, 'status':new_status} if new_status=='ERROR': @@ -1762,7 +2310,7 @@ class host_thread(threading.Thread): return -1 elif 'terminate' in req['action']: #PUT a log in the database - print self.name, ": PANIC deleting server", server_id, last_error + self.logger.error("PANIC deleting server id='%s' %s", server_id, last_error) self.db_lock.acquire() self.db.new_row('logs', {'uuid':server_id, 'tenant_id':req['tenant_id'], 'related':'instances','level':'panic', @@ -1795,11 +2343,11 @@ class host_thread(threading.Thread): ret = 0 error_text=None if self.test: - print self.name, ": restore_iface '%s' %s" % (name, mac) + self.logger.debug("restore_iface '%s' %s", name, mac) return 0, None try: if not lib_conn: - conn = host_thread.lvirt_module.open("qemu+ssh://"+self.user+"@"+self.host+"/system") + conn = host_thread.lvirt_module.open(self.lvirt_conn_uri) else: conn = lib_conn @@ -1807,12 +2355,13 @@ class host_thread(threading.Thread): #TODO.Revise self.server_forceoff(True) iface = conn.interfaceLookupByMACString(mac) - iface.destroy() + if iface.isActive(): + iface.destroy() iface.create() - print self.name, ": restore_iface '%s' %s" % (name, mac) + self.logger.debug("restore_iface '%s' %s", name, mac) except host_thread.lvirt_module.libvirtError as e: error_text = e.get_error_message() - print self.name, ": restore_iface '%s' '%s' libvirt exception: %s" %(name, mac, error_text) + self.logger.error("restore_iface '%s' '%s' libvirt exception: %s", name, mac, error_text) ret=-1 finally: if lib_conn is None and conn is not None: @@ -1855,14 +2404,14 @@ class host_thread(threading.Thread): except paramiko.ssh_exception.SSHException as e: image_status='ERROR' error_text = e.args[0] - print self.name, "': create_image(",server_id,") ssh Exception:", error_text + self.logger.error("create_image id='%s' ssh Exception: %s", server_id, error_text) if "SSH session not active" in error_text and retry==0: self.ssh_connect() except Exception as e: image_status='ERROR' error_text = str(e) - print self.name, "': create_image(",server_id,") Exception:", error_text - + self.logger.error("create_image id='%s' Exception: %s", server_id, error_text) + #TODO insert a last_error at database self.db_lock.acquire() self.db.update_rows('images', {'status':image_status, 'progress': 100, 'path':file_dst}, @@ -1880,14 +2429,14 @@ class host_thread(threading.Thread): WHERE={'port_id': port_id}) self.db_lock.release() if r<0: - print self.name, ": edit_iface(",port_id,") DDBB error:", c + self.logger.error("edit_iface %s DDBB error: %s", port_id, c) return elif r==0: - print self.name, ": edit_iface(",port_id,") por not found" + self.logger.error("edit_iface %s port not found", port_id) return port=c[0] if port["model"]!="VF": - print self.name, ": edit_iface(",port_id,") ERROR model must be VF" + self.logger.error("edit_iface %s ERROR model must be VF", port_id) return #create xml detach file xml=[] @@ -1895,16 +2444,16 @@ class host_thread(threading.Thread): xml.append("") xml.append(" ") xml.append(" "+ self.pci2xml(port['pci'])+"\n ") - xml.append('') + xml.append('') try: conn=None - conn = host_thread.lvirt_module.open("qemu+ssh://"+self.user+"@"+self.host+"/system") + conn = host_thread.lvirt_module.open(self.lvirt_conn_uri) dom = conn.lookupByUUIDString(port["instance_id"]) if old_net: text="\n".join(xml) - print self.name, ": edit_iface detaching SRIOV interface", text + self.logger.debug("edit_iface detaching SRIOV interface " + text) dom.detachDeviceFlags(text, flags=host_thread.lvirt_module.VIR_DOMAIN_AFFECT_LIVE) if new_net: xml[-1] =" " @@ -1912,30 +2461,19 @@ class host_thread(threading.Thread): xml.append(self.pci2xml(port.get('vpci',None)) ) xml.append('') text="\n".join(xml) - print self.name, ": edit_iface attaching SRIOV interface", text + self.logger.debug("edit_iface attaching SRIOV interface " + text) dom.attachDeviceFlags(text, flags=host_thread.lvirt_module.VIR_DOMAIN_AFFECT_LIVE) except host_thread.lvirt_module.libvirtError as e: text = e.get_error_message() - print self.name, ": edit_iface(",port["instance_id"],") libvirt exception:", text + self.logger.error("edit_iface %s libvirt exception: %s", port["instance_id"], text) finally: if conn is not None: conn.close() def create_server(server, db, db_lock, only_of_ports): - #print "server" - #print "server" - #print server - #print "server" - #print "server" - #try: -# host_id = server.get('host_id', None) extended = server.get('extended', None) - -# print '----------------------' -# print json.dumps(extended, indent=4) - requirements={} requirements['numa']={'memory':0, 'proc_req_type': 'threads', 'proc_req_nb':0, 'port_list':[], 'sriov_list':[]} requirements['ram'] = server['flavor'].get('ram', 0) @@ -2040,7 +2578,7 @@ def create_server(server, db, db_lock, only_of_ports): WHERE={'numa_id':numa_id,'instance_id': None, 'status':'ok'} ) db_lock.release() if result <= 0: - print content + #print content return -1, content #convert rows to a dictionary indexed by core_id @@ -2126,7 +2664,7 @@ def create_server(server, db, db_lock, only_of_ports): result, content = db.get_table(FROM='resources_port', SELECT=('id', 'pci', 'mac'),WHERE={'numa_id':numa_id,'root_id': port['port_id'], 'port_id': None, 'Mbps_used': 0} ) db_lock.release() if result <= 0: - print content + #print content return -1, content for row in content: if row['id'] in used_sriov_ports or row['id']==port['port_id']: @@ -2150,7 +2688,7 @@ def create_server(server, db, db_lock, only_of_ports): result, content = db.get_table(FROM='resources_port', SELECT=('id', 'pci', 'mac', 'Mbps'),WHERE={'numa_id':numa_id,'root_id': port['port_id'], 'port_id': None, 'Mbps_used': 0} ) db_lock.release() if result <= 0: - print content + #print content return -1, content port['Mbps_used'] = content[0]['Mbps'] for row in content: @@ -2168,7 +2706,6 @@ def create_server(server, db, db_lock, only_of_ports): # print '2. SR-IOV assignation:'+json.dumps(requirements['sriov_list'], indent=4) server['host_id'] = host_id - #Generate dictionary for saving in db the instance resources resources = {} @@ -2190,8 +2727,8 @@ def create_server(server, db, db_lock, only_of_ports): #Get the brifge name db_lock.acquire() result, content = db.get_table(FROM='nets', - SELECT=('name', 'type', 'vlan', 'provider', 'enable_dhcp', - 'dhcp_first_ip', 'dhcp_last_ip', 'cidr'), + SELECT=('name', 'type', 'vlan', 'provider', 'enable_dhcp','dhcp_first_ip', + 'dhcp_last_ip', 'cidr', 'gateway_ip', 'dns', 'links', 'routes'), WHERE={'uuid': control_iface['net_id']}) db_lock.release() if result < 0: @@ -2216,6 +2753,13 @@ def create_server(server, db, db_lock, only_of_ports): control_iface["dhcp_first_ip"] = network["dhcp_first_ip"] control_iface["dhcp_last_ip"] = network["dhcp_last_ip"] control_iface["cidr"] = network["cidr"] + + if network.get("dns"): + control_iface["dns"] = yaml.safe_load(network.get("dns")) + if network.get("links"): + control_iface["links"] = yaml.safe_load(network.get("links")) + if network.get("routes"): + control_iface["routes"] = yaml.safe_load(network.get("routes")) else: if network['type']!='data' and network['type']!='ptp': return -1, "Error at field netwoks: network uuid %s for dataplane interface is not of type data or ptp" % control_iface['net_id'] @@ -2266,9 +2810,9 @@ def create_server(server, db, db_lock, only_of_ports): resources['extended']['devices'] = extended['devices'] - print '===================================={' - print json.dumps(resources, indent=4) - print '====================================}' + # '===================================={' + #print json.dumps(resources, indent=4) + #print '====================================}' return 0, resources