X-Git-Url: https://osm.etsi.org/gitweb/?a=blobdiff_plain;f=vim_db.py;h=c34160dddb8316dda68c99d908c842b55c713f8c;hb=refs%2Fchanges%2F17%2F1317%2F3;hp=11a7aa12e417f73dccc5dbfb5efbeb551f9e7aa9;hpb=69a5739aef6bdc5e0ebc12ebfe6a4775e8211304;p=osm%2Fopenvim.git diff --git a/vim_db.py b/vim_db.py index 11a7aa1..c34160d 100644 --- a/vim_db.py +++ b/vim_db.py @@ -37,6 +37,7 @@ import uuid as myUuid import auxiliary_functions as af import json import logging +from netaddr import IPNetwork, IPSet, IPRange, all_matching_cidrs HTTP_Bad_Request = 400 HTTP_Unauthorized = 401 @@ -49,7 +50,7 @@ HTTP_Internal_Server_Error = 500 class vim_db(): - def __init__(self, vlan_range, debug="ERROR"): + def __init__(self, vlan_range, logger_name= None, debug=None): '''vlan_range must be a tuple (vlan_ini, vlan_end) with available vlan values for networks every dataplane network contain a unique value, regardless of it is used or not ''' @@ -58,8 +59,13 @@ class vim_db(): self.net_vlan_usedlist = None self.net_vlan_lastused = self.net_vlan_range[0] -1 self.debug=debug - self.logger = logging.getLogger('vim.db') - self.logger.setLevel( getattr(logging, debug) ) + if logger_name: + self.logger_name = logger_name + else: + self.logger_name = 'openvim.db' + self.logger = logging.getLogger(self.logger_name) + if debug: + self.logger.setLevel( getattr(logging, debug) ) def connect(self, host=None, user=None, passwd=None, database=None): @@ -152,7 +158,7 @@ class vim_db(): return -HTTP_Conflict, "Value %s already in use for %s" % (e.args[1][de+15:fk], e.args[1][fk+7:]) if uk>=0: if wc>=0: - return -HTTP_Bad_Request, "Field %s can not be used for filtering" % e.args[1][uk+14:wc] + return -HTTP_Bad_Request, "Field %s cannot be used for filtering" % e.args[1][uk+14:wc] if fl>=0: return -HTTP_Bad_Request, "Field %s does not exist" % e.args[1][uk+14:wc] return -HTTP_Internal_Server_Error, "Database internal Error %d: %s" % (e.args[0], e.args[1]) @@ -173,7 +179,7 @@ class vim_db(): def __get_used_net_vlan(self): #get used from database if needed try: - cmd = "SELECT vlan FROM nets WHERE vlan>='%s' and (type='ptp' or type='data') ORDER BY vlan LIMIT 25" % self.net_vlan_lastused + cmd = "SELECT vlan FROM nets WHERE vlan>='%s' ORDER BY vlan LIMIT 25" % self.net_vlan_lastused with self.con: self.cur = self.con.cursor() self.logger.debug(cmd) @@ -218,10 +224,14 @@ class vim_db(): 'WHERE_OR': dict of key:values, translated to key=value OR ... (Optional) 'WHERE_AND_OR: str 'AND' or 'OR'(by default) mark the priority to 'WHERE AND (WHERE_OR)' or (WHERE) OR WHERE_OR' (Optional) 'LIMIT': limit of number of rows (Optional) + 'DISTINCT': make a select distinct to remove repeated elements Return: a list with dictionarys at each row ''' #print sql_dict - select_= "SELECT " + ("*" if 'SELECT' not in sql_dict else ",".join(map(str,sql_dict['SELECT'])) ) + select_ = "SELECT " + if sql_dict.get("DISTINCT"): + select_ += "DISTINCT " + select_ += ("*" if not sql_dict.get('SELECT') else ",".join(map(str,sql_dict['SELECT'])) ) #print 'select_', select_ from_ = "FROM " + str(sql_dict['FROM']) #print 'from_', from_ @@ -254,7 +264,7 @@ class vim_db(): else: where_ = "" #print 'where_', where_ - limit_ = "LIMIT " + str(sql_dict['LIMIT']) if 'LIMIT' in sql_dict else "" + limit_ = "LIMIT " + str(sql_dict['LIMIT']) if sql_dict.get("LIMIT") else "" #print 'limit_', limit_ cmd = " ".join( (select_, from_, where_, limit_) ) for retry_ in range(0,2): @@ -1031,15 +1041,16 @@ class vim_db(): with self.con: self.cur = self.con.cursor(mdb.cursors.DictCursor) #get INSTANCE - cmd = "SELECT uuid, name, description, progress, host_id, flavor_id, image_id, status, last_error, tenant_id, ram, vcpus, created_at \ - FROM instances WHERE uuid = '" + str(instance_id) +"'" + cmd = "SELECT uuid, name, description, progress, host_id, flavor_id, image_id, status, last_error, "\ + "tenant_id, ram, vcpus, created_at FROM instances WHERE uuid='{}'".format(instance_id) self.logger.debug(cmd) self.cur.execute(cmd) if self.cur.rowcount == 0 : return 0, "instance '" + str(instance_id) +"'not found." instance = self.cur.fetchone() #get networks - cmd = "SELECT uuid as iface_id, net_id, mac as mac_address, ip_address, name, Mbps as bandwidth, vpci, model \ - FROM ports WHERE type = 'instance:bridge' AND instance_id = '" + instance_id + "'" + cmd = "SELECT uuid as iface_id, net_id, mac as mac_address, ip_address, name, Mbps as bandwidth, "\ + "vpci, model FROM ports WHERE (type='instance:bridge' or type='instance:ovs') AND "\ + "instance_id= '{}'".format(instance_id) self.logger.debug(cmd) self.cur.execute(cmd) if self.cur.rowcount > 0 : @@ -1109,8 +1120,9 @@ class vim_db(): numa_dict['threads-source'] = thread_source #get dedicated ports and SRIOV - cmd = "SELECT port_id as iface_id, p.vlan as vlan, p.mac as mac_address, net_id, if(model='PF','yes',if(model='VF','no','yes:sriov')) as dedicated,\ - rp.Mbps as bandwidth, name, vpci, pci as source \ + cmd = "SELECT port_id as iface_id, p.vlan as vlan, p.mac as mac_address, net_id, if(model='PF',\ + 'yes',if(model='VF','no','yes:sriov')) as dedicated, rp.Mbps as bandwidth, name, vpci, \ + pci as source \ FROM resources_port as rp join ports as p on port_id=uuid WHERE p.instance_id = '%s' AND numa_id = '%s' and p.type='instance:data'" % (instance_id, numa_id) self.logger.debug(cmd) self.cur.execute(cmd) @@ -1169,7 +1181,7 @@ class vim_db(): self.cur = self.con.cursor() match_found = False if len(valid_hosts)<=0: - error_text = 'No room at data center. Can not find a host with %s MB memory and %s cpus available' % (str(requirements['ram']), str(requirements['vcpus'])) + error_text = 'No room at data center. Cannot find a host with %s MB memory and %s cpus available' % (str(requirements['ram']), str(requirements['vcpus'])) #self.logger.debug(error_text) return -1, error_text @@ -1183,7 +1195,7 @@ class vim_db(): self.cur.close() self.cur = self.con.cursor() if len(valid_for_memory)<=0: - error_text = 'No room at data center. Can not find a host with %s GB Hugepages memory available' % str(requirements['numa']['memory']) + error_text = 'No room at data center. Cannot find a host with %s GB Hugepages memory available' % str(requirements['numa']['memory']) #self.logger.debug(error_text) return -1, error_text @@ -1203,7 +1215,7 @@ class vim_db(): self.cur.close() self.cur = self.con.cursor() if len(valid_for_processor)<=0: - error_text = 'No room at data center. Can not find a host with %s %s available' % (str(requirements['numa']['proc_req_nb']),cpu_requirement_text) + error_text = 'No room at data center. Cannot find a host with %s %s available' % (str(requirements['numa']['proc_req_nb']),cpu_requirement_text) #self.logger.debug(error_text) return -1, error_text @@ -1230,7 +1242,7 @@ class vim_db(): else: valid_numas.append(m_numa['numa_id']) if len(valid_numas)<=0: - error_text = 'No room at data center. Can not find a host with %s MB hugepages memory and %s %s available in the same numa' %\ + error_text = 'No room at data center. Cannot find a host with %s MB hugepages memory and %s %s available in the same numa' %\ (requirements['numa']['memory'], str(requirements['numa']['proc_req_nb']),cpu_requirement_text) #self.logger.debug(error_text) return -1, error_text @@ -1343,7 +1355,7 @@ class vim_db(): break if not match_found: - error_text = 'No room at data center. Can not find a host with the required hugepages, vcpus and interfaces' + error_text = 'No room at data center. Cannot find a host with the required hugepages, vcpus and interfaces' #self.logger.debug(error_text) return -1, error_text @@ -1391,15 +1403,28 @@ class vim_db(): #insert resources nb_bridge_ifaces = nb_cores = nb_ifaces = nb_numas = 0 #insert bridged_ifaces + for iface in bridgedifaces: #generate and insert a iface uuid + if 'enable_dhcp' in iface and iface['enable_dhcp']: + dhcp_first_ip = iface["dhcp_first_ip"] + del iface["dhcp_first_ip"] + dhcp_last_ip = iface["dhcp_last_ip"] + del iface["dhcp_last_ip"] + dhcp_cidr = iface["cidr"] + del iface["cidr"] + del iface["enable_dhcp"] + used_dhcp_ips = self._get_dhcp_ip_used_list(iface["net_id"]) + iface["ip_address"] = self.get_free_ip_from_range(dhcp_first_ip, dhcp_last_ip, + dhcp_cidr, used_dhcp_ips) + iface['uuid'] = str(myUuid.uuid1()) # create_uuid cmd = "INSERT INTO uuids (uuid, root_uuid, used_at) VALUES ('%s','%s', 'ports')" % (iface['uuid'], uuid) self.logger.debug(cmd) self.cur.execute(cmd) #insert iface iface['instance_id'] = uuid - iface['type'] = 'instance:bridge' + # iface['type'] = 'instance:bridge' if 'name' not in iface: iface['name']="br"+str(nb_bridge_ifaces) iface['Mbps']=iface.pop('bandwidth', None) if 'mac_address' not in iface: @@ -1499,7 +1524,59 @@ class vim_db(): r,c = self.format_error(e, "new_instance", cmd) if r!=-HTTP_Request_Timeout or retry_==1: return r,c - def delete_instance(self, instance_id, tenant_id, net_list, ports_to_free, logcause="requested by http"): + def get_free_ip_from_range(self, first_ip, last_ip, cidr, ip_used_list): + """ + Calculate a free IP from a range given + :param first_ip: First dhcp ip range + :param last_ip: Last dhcp ip range + :param cidr: net cidr + :param ip_used_list: contain all used ips to avoid ip collisions + :return: + """ + + ip_tools = IPNetwork(cidr) + cidr_len = ip_tools.prefixlen + ips = IPNetwork(first_ip + '/' + str(cidr_len)) + ip_used_list.append(str(ips[0])) # first ip + ip_used_list.append(str(ips[1])) # gw ip + ip_used_list.append(str(ips[-1])) # broadcast ip + for vm_ip in ips: + if str(vm_ip) not in ip_used_list: + return vm_ip + + return None + + def _get_dhcp_ip_used_list(self, net_id): + """ + REtreive from DB all ips already used by the dhcp server for a given net + :param net_id: + :return: + """ + WHERE={'type': 'instance:ovs', 'net_id': net_id} + for retry_ in range(0, 2): + cmd = "" + self.cur = self.con.cursor(mdb.cursors.DictCursor) + select_ = "SELECT uuid, ip_address FROM ports " + + if WHERE is None or len(WHERE) == 0: + where_ = "" + else: + where_ = "WHERE " + " AND ".join( + map(lambda x: str(x) + (" is Null" if WHERE[x] is None else "='" + str(WHERE[x]) + "'"), + WHERE.keys())) + limit_ = "LIMIT 100" + cmd = " ".join((select_, where_, limit_)) + self.logger.debug(cmd) + self.cur.execute(cmd) + ports = self.cur.fetchall() + ip_address_list = [] + for port in ports: + ip_address_list.append(port['ip_address']) + + return ip_address_list + + + def delete_instance(self, instance_id, tenant_id, net_dataplane_list, ports_to_free, net_ovs_list, logcause="requested by http"): for retry_ in range(0,2): cmd="" try: @@ -1519,7 +1596,14 @@ class vim_db(): self.cur.execute(cmd) net_list__ = self.cur.fetchall() for net in net_list__: - net_list.append(net[0]) + net_dataplane_list.append(net[0]) + + # get ovs manangement nets + cmd = "SELECT DISTINCT net_id, vlan, ip_address, mac FROM ports WHERE instance_id='{}' AND net_id is not Null AND "\ + "type='instance:ovs'".format(instance_id) + self.logger.debug(cmd) + self.cur.execute(cmd) + net_ovs_list += self.cur.fetchall() #get dataplane interfaces releases by this VM; both PF and VF with no other VF cmd="SELECT source_name, mac FROM (SELECT root_id, count(instance_id) as used FROM resources_port WHERE instance_id='%s' GROUP BY root_id ) AS A" % instance_id \ @@ -1630,9 +1714,9 @@ class vim_db(): if net['tenant_id']==tenant_id and net['shared']=='false': return -1, "needed admin privileges to attach to the net %s" % net_id #check types - if (net['type'] in ('p2p','data') and 'port_type' == 'instance:bridge') or \ - (net['type'] in ('bridge_data','bridge_man') and 'port_type' != 'instance:bridge') : - return -1, "can not attach a port of type %s into a net of type %s" % (port_type, net['type']) + if (net['type'] in ('ptp','data') and port_type not in ('instance:data','external')) or \ + (net['type'] in ('bridge_data','bridge_man') and port_type not in ('instance:bridge', 'instance:ovs')): + return -1, "Cannot attach a port of type %s into a net of type %s" % (port_type, net['type']) if net['type'] == 'ptp': #look how many nb_ports, data = self.get_ports( {'net_id':net_id} )