| #!/usr/bin/env python |
| # -*- coding: utf-8 -*- |
| |
| ## |
| # Copyright 2015 Telefonica Investigacion y Desarrollo, S.A.U. |
| # This file is part of openvim |
| # All Rights Reserved. |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); you may |
| # not use this file except in compliance with the License. You may obtain |
| # a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
| # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
| # License for the specific language governing permissions and limitations |
| # under the License. |
| # |
| # For those usages not covered by the Apache License, Version 2.0 please |
| # contact with: nfvlabs@tid.es |
| ## |
| |
| """ |
| This module interact with the openvim database, |
| It implements general table management and transactional writes, that is, or all is changed or nothing. |
| It is threading safe using a Lock |
| """ |
| |
| import MySQLdb as mdb |
| import uuid as myUuid |
| import auxiliary_functions as af |
| import json |
| import logging |
| from netaddr import IPNetwork, IPAddress |
| from threading import Lock |
| |
| __author__ = "Alfonso Tierno" |
| __date__ = "$10-jul-2014 12:07:15$" |
| |
| HTTP_Bad_Request = 400 |
| HTTP_Unauthorized = 401 |
| HTTP_Not_Found = 404 |
| HTTP_Method_Not_Allowed = 405 |
| HTTP_Request_Timeout = 408 |
| HTTP_Conflict = 409 |
| HTTP_Service_Unavailable = 503 |
| HTTP_Internal_Server_Error = 500 |
| |
| |
| class vim_db(): |
| def __init__(self, vlan_range, logger_name=None, debug=None, lock=None): |
| """vlan_range must be a tuple (vlan_ini, vlan_end) with available vlan values for networks |
| every dataplane network contain a unique value, regardless of it is used or not |
| """ |
| # initialization |
| self.net_vlan_range = vlan_range |
| self.vlan_config = {} |
| self.host = None |
| self.user = None |
| self.passwd = None |
| self.database = None |
| self.con = None |
| self.cur = None |
| self.debug = debug |
| self.lock = lock or Lock() |
| if logger_name: |
| self.logger_name = logger_name |
| else: |
| self.logger_name = 'openvim.db' |
| self.logger = logging.getLogger(self.logger_name) |
| if debug: |
| self.logger.setLevel(getattr(logging, debug)) |
| |
| def connect(self, host=None, user=None, passwd=None, database=None): |
| """Connect to the concrete data base. |
| The first time a valid host, user, passwd and database must be provided, |
| Following calls can skip this parameters |
| """ |
| try: |
| with self.lock: |
| if host: |
| self.host = host |
| self.con = None |
| if user: |
| self.user = user |
| self.con = None |
| if passwd: |
| self.passwd = passwd |
| self.con = None |
| if database: |
| self.database = database |
| self.con = None |
| if self.con: |
| # try to connect |
| try: |
| with self.con: |
| self.cur = self.con.cursor() |
| cmd = "SELECT version_int,version,openvim_ver FROM schema_version" |
| self.logger.debug("method connect: " + cmd) |
| self.cur.execute(cmd) |
| return 0 |
| except Exception: |
| pass |
| self.con = mdb.connect(self.host, self.user, self.passwd, self.database) |
| self.logger.debug("connected to DB %s at %s@%s", self.database, self.user, self.host) |
| return 0 |
| except mdb.Error as e: |
| self.logger.error("Cannot connect to DB %s at %s@%s Error %d: %s", self.database, self.user, self.host, |
| e.args[0], e.args[1]) |
| return -1 |
| |
| def get_db_version(self): |
| """ Obtain the database schema version. |
| Return: (negative, text) if error or version 0.0 where schema_version table is missing |
| (version_int, version_text) if ok |
| """ |
| cmd = "SELECT version_int,version,openvim_ver FROM schema_version" |
| for retry_ in range(0, 2): |
| try: |
| with self.lock, self.con: |
| self.cur = self.con.cursor() |
| self.logger.debug(cmd) |
| self.cur.execute(cmd) |
| rows = self.cur.fetchall() |
| highest_version_int = 0 |
| highest_version = "" |
| # print rows |
| for row in rows: # look for the latest version |
| if row[0] > highest_version_int: |
| highest_version_int, highest_version = row[0:2] |
| return highest_version_int, highest_version |
| except (mdb.Error, AttributeError) as e: |
| self.logger.error("get_db_version DB Exception %d: %s. Command %s", e.args[0], e.args[1], cmd) |
| r, c = self.format_error(e) |
| if r != -HTTP_Request_Timeout or retry_ == 1: |
| return r, c |
| |
| def disconnect(self): |
| """disconnect from the data base""" |
| try: |
| with self.lock: |
| if not self.con: |
| self.con.close() |
| self.con = None |
| except mdb.Error as e: |
| self.logger.error("while disconnecting from DB: Error: %s", str(e)) |
| return -1 |
| except AttributeError as e: # self.con not defined |
| if e[0][-5:] == "'con'": |
| return -1, "Database internal error, no connection." |
| else: |
| raise |
| |
| def format_error(self, e, func, cmd, command=None, extra=None): |
| """Creates a text error base on the produced exception |
| Params: |
| e: mdb exception |
| func: name of the function that makes the call, for logging purposes |
| cmd: database command that produce the exception |
| command: if the intention is update or delete |
| extra: extra information to add to some commands |
| Return |
| HTTP error in negative, formatted error text |
| """ |
| |
| self.logger.error("%s DB Exception %s. Command %s", func, str(e), cmd) |
| if type(e[0]) is str: |
| if e[0][-5:] == "'con'": |
| return -HTTP_Internal_Server_Error, "DB Exception, no connection." |
| else: |
| return -HTTP_Internal_Server_Error, e.args[1] |
| if e.args[0] == 2006 or e.args[0] == 2013: |
| # MySQL server has gone away (((or))) Exception 2013: Lost connection to MySQL server during query |
| # reconnect |
| self.connect() |
| return -HTTP_Request_Timeout, "Database reconnection. Try Again" |
| fk = e.args[1].find("foreign key constraint fails") |
| if fk >= 0: |
| if command == "update": |
| return -HTTP_Bad_Request, "tenant_id %s not found." % extra |
| elif command == "delete": |
| return -HTTP_Bad_Request, "Resource is not free. There are %s that prevent its deletion." % extra |
| de = e.args[1].find("Duplicate entry") |
| fk = e.args[1].find("for key") |
| uk = e.args[1].find("Unknown column") |
| wc = e.args[1].find("in 'where clause'") |
| fl = e.args[1].find("in 'field list'") |
| # print de, fk, uk, wc,fl |
| if de >= 0: |
| if fk >= 0: # error 1062 |
| return -HTTP_Conflict, "Value %s already in use for %s" % (e.args[1][de + 15:fk], e.args[1][fk + 7:]) |
| if uk >= 0: |
| if wc >= 0: |
| return -HTTP_Bad_Request, "Field %s cannot be used for filtering" % e.args[1][uk + 14:wc] |
| if fl >= 0: |
| return -HTTP_Bad_Request, "Field %s does not exist" % e.args[1][uk + 14:wc] |
| return -HTTP_Internal_Server_Error, "Database internal Error %d: %s" % (e.args[0], e.args[1]) |
| |
| @staticmethod |
| def __data2db_format(data): |
| """convert data to database format. If data is None it return the 'Null' text, |
| otherwise it return the text surrounded by quotes ensuring internal quotes are escaped""" |
| if data is None: |
| return 'Null' |
| out = str(data) |
| if "'" not in out: |
| return "'" + out + "'" |
| elif '"' not in out: |
| return '"' + out + '"' |
| else: |
| return json.dumps(out) |
| |
| def __get_used_net_vlan(self, region=None): |
| # get used from database if needed |
| vlan_region = self.vlan_config[region] |
| try: |
| cmd = "SELECT vlan FROM nets WHERE vlan>='{}' and region{} ORDER BY vlan LIMIT 25".format( |
| vlan_region["lastused"], "='" + region + "'" if region else " is NULL") |
| with self.con: |
| self.cur = self.con.cursor() |
| self.logger.debug(cmd) |
| self.cur.execute(cmd) |
| vlan_tuple = self.cur.fetchall() |
| # convert a tuple of tuples in a list of numbers |
| vlan_region["usedlist"] = [] |
| for k in vlan_tuple: |
| vlan_region["usedlist"].append(k[0]) |
| except (mdb.Error, AttributeError) as e: |
| return self.format_error(e, "get_free_net_vlan", cmd) |
| |
| def get_free_net_vlan(self, region=None): |
| """obtain a vlan not used in any net""" |
| with self.lock: |
| if region not in self.vlan_config: |
| self.vlan_config[region] = { |
| "usedlist": None, |
| "lastused": self.net_vlan_range[0] - 1 |
| } |
| vlan_region = self.vlan_config[region] |
| |
| while True: |
| self.logger.debug("get_free_net_vlan() region[%s]=%s, net_vlan_range:%s-%s", str(region), |
| str(vlan_region), str(self.net_vlan_range[0]), str(self.net_vlan_range[1])) |
| vlan_region["lastused"] += 1 |
| if vlan_region["lastused"] == self.net_vlan_range[1]: |
| # start from the begining |
| vlan_region["lastused"] = self.net_vlan_range[0] |
| vlan_region["usedlist"] = None |
| if vlan_region["usedlist"] is None or \ |
| (len(vlan_region["usedlist"]) == 25 and vlan_region["lastused"] >= vlan_region["usedlist"][-1]): |
| self.__get_used_net_vlan(region) |
| self.logger.debug("new net_vlan_usedlist %s", str(vlan_region["usedlist"])) |
| if vlan_region["lastused"] in vlan_region["usedlist"]: |
| continue |
| else: |
| return vlan_region["lastused"] |
| |
| def get_table(self, **sql_dict): |
| """ Obtain rows from a table. |
| Atribure sql_dir: dictionary with the following key: value |
| 'SELECT': [list of fields to retrieve] (by default all) |
| 'FROM': string of table name (Mandatory) |
| 'WHERE': dict of key:values, translated to key=value AND ... (Optional) |
| 'WHERE_NOT': dict of key:values, translated to key!=value AND ... (Optional) |
| 'WHERE_OR': dict of key:values, translated to key=value OR ... (Optional) |
| 'WHERE_AND_OR: str 'AND' or 'OR'(by default) mark the priority to 'WHERE AND (WHERE_OR)' or (WHERE) OR |
| WHERE_OR' (Optional) |
| 'LIMIT': limit of number of rows (Optional) |
| 'DISTINCT': make a select distinct to remove repeated elements |
| Return: a list with dictionarys at each row |
| """ |
| # print sql_dict |
| select_ = "SELECT " |
| if sql_dict.get("DISTINCT"): |
| select_ += "DISTINCT " |
| select_ += ("*" if not sql_dict.get('SELECT') else ",".join(map(str, sql_dict['SELECT']))) |
| # print 'select_', select_ |
| from_ = "FROM " + str(sql_dict['FROM']) |
| # print 'from_', from_ |
| |
| where_and = None |
| where_or = None |
| w = sql_dict.get('WHERE') |
| if w: |
| where_and = " AND ".join(map(lambda x: str(x) + (" is Null" if w[x] is None else "='" + str(w[x]) + "'"), |
| w.keys())) |
| w = sql_dict.get('WHERE_LIKE') # Unikernels extension -START- |
| if w: |
| where_and_like = " AND ".join( |
| map(lambda x: str(x) + (" is Null" if w[x] is None else " LIKE '" + str(w[x]) + "'"), w.keys())) |
| if where_and: |
| where_and += " AND " + where_and_like |
| else: |
| where_and = where_and_like # Unikernels extension -END- |
| w = sql_dict.get('WHERE_NOT') |
| if w: |
| where_and_not = " AND ".join( |
| map(lambda x: str(x) + (" is not Null" if w[x] is None else "!='" + str(w[x]) + "'"), w.keys())) |
| if where_and: |
| where_and += " AND " + where_and_not |
| else: |
| where_and = where_and_not |
| w = sql_dict.get('WHERE_OR') |
| if w: |
| where_or = " OR ".join(map(lambda x: str(x) + (" is Null" if w[x] is None else "='" + str(w[x]) + "'"), |
| w.keys())) |
| |
| if where_and and where_or: |
| if sql_dict.get("WHERE_AND_OR") == "AND": |
| where_ = "WHERE " + where_and + " AND (" + where_or + ")" |
| else: |
| where_ = "WHERE (" + where_and + ") OR " + where_or |
| elif where_and and where_or is None: |
| where_ = "WHERE " + where_and |
| elif where_and is None and where_or: |
| where_ = "WHERE " + where_or |
| else: |
| where_ = "" |
| # print 'where_', where_ |
| limit_ = "LIMIT " + str(sql_dict['LIMIT']) if sql_dict.get("LIMIT") else "" |
| # print 'limit_', limit_ |
| cmd = " ".join((select_, from_, where_, limit_)) |
| for retry_ in range(0, 2): |
| try: |
| with self.lock, self.con: |
| self.cur = self.con.cursor(mdb.cursors.DictCursor) |
| self.logger.debug(cmd) |
| self.cur.execute(cmd) |
| rows = self.cur.fetchall() |
| return self.cur.rowcount, rows |
| except (mdb.Error, AttributeError) as e: |
| r, c = self.format_error(e, "get_table", cmd) |
| if r != -HTTP_Request_Timeout or retry_ == 1: |
| return r, c |
| |
| def new_tenant(self, tenant_dict): |
| """ Add one row into a table. |
| Attribure |
| tenant_dict: dictionary with the key: value to insert |
| It checks presence of uuid and add one automatically otherwise |
| Return: (result, uuid) where result can be 0 if error, or 1 if ok |
| """ |
| for retry_ in range(0, 2): |
| cmd = "" |
| inserted = -1 |
| try: |
| # create uuid if not provided |
| if 'uuid' not in tenant_dict: |
| uuid = tenant_dict['uuid'] = str(myUuid.uuid1()) # create_uuid |
| else: |
| uuid = str(tenant_dict['uuid']) |
| # obtain tenant_id for logs |
| tenant_id = uuid |
| with self.lock, self.con: |
| self.cur = self.con.cursor() |
| # inserting new uuid |
| cmd = "INSERT INTO uuids (uuid, used_at) VALUES ('%s','tenants')" % uuid |
| self.logger.debug(cmd) |
| self.cur.execute(cmd) |
| # insert tenant |
| cmd = "INSERT INTO tenants (" + \ |
| ",".join(map(str, tenant_dict.keys())) + ") VALUES(" + \ |
| ",".join( |
| map(lambda x: "Null" if x is None else "'" + str(x) + "'", tenant_dict.values())) + ")" |
| self.logger.debug(cmd) |
| self.cur.execute(cmd) |
| inserted = self.cur.rowcount |
| # #inserting new log |
| # del tenant_dict['uuid'] # not interested for the log |
| # cmd = "INSERT INTO logs (related,level,tenant_id,uuid,description) VALUES |
| # ('tenants','debug','%s','%s',\"new tenant %s\")" % (uuid, tenant_id, str(tenant_dict)) |
| # self.logger.debug(cmd) |
| # self.cur.execute(cmd) |
| # commit transaction |
| self.cur.close() |
| if inserted == 0: |
| return 0, uuid |
| with self.lock, self.con: |
| self.cur = self.con.cursor() |
| # adding public flavors |
| cmd = "INSERT INTO tenants_flavors(flavor_id,tenant_id) SELECT uuid as flavor_id,'" + tenant_id + \ |
| "' FROM flavors WHERE public = 'yes'" |
| self.logger.debug(cmd) |
| self.cur.execute(cmd) |
| self.logger.debug("attached public flavors: %s", str(self.cur.rowcount)) |
| # rows = self.cur.fetchall() |
| # for row in rows: |
| # cmd = "INSERT INTO tenants_flavors(flavor_id,tenant_id) VALUES('%s','%s')"%(row[0], tenant_id) |
| # self.cur.execute(cmd ) |
| # adding public images |
| cmd = "INSERT INTO tenants_images(image_id,tenant_id) SELECT uuid as image_id,'" + tenant_id + \ |
| "' FROM images WHERE public = 'yes'" |
| self.logger.debug(cmd) |
| self.cur.execute(cmd) |
| self.logger.debug("attached public images: %s", str(self.cur.rowcount)) |
| return 1, uuid |
| except (mdb.Error, AttributeError) as e: |
| if inserted == 1: |
| self.logger.warning("new_tenant DB Exception %d: %s. Command %s", e.args[0], e.args[1], cmd) |
| return 1, uuid |
| else: |
| r, c = self.format_error(e, "new_tenant", cmd) |
| if r != -HTTP_Request_Timeout or retry_ == 1: |
| return r, c |
| |
| def new_row(self, table, INSERT, add_uuid=False, log=False): |
| """ Add one row into a table. |
| Atribure |
| INSERT: dictionary with the key: value to insert |
| table: table where to insert |
| add_uuid: if True, it will crated an uuid key entry at INSERT if not provided |
| It checks presence of uuid and add one automatically otherwise |
| Return: (result, uuid) where result can be 0 if error, or 1 if ok |
| """ |
| for retry_ in range(0, 2): |
| cmd = "" |
| try: |
| if add_uuid: |
| # create uuid if not provided |
| if 'uuid' not in INSERT: |
| uuid = INSERT['uuid'] = str(myUuid.uuid1()) # create_uuid |
| else: |
| uuid = str(INSERT['uuid']) |
| else: |
| uuid = None |
| with self.lock, self.con: |
| self.cur = self.con.cursor() |
| if add_uuid: |
| # inserting new uuid |
| cmd = "INSERT INTO uuids (uuid, used_at) VALUES ('%s','%s')" % (uuid, table) |
| self.logger.debug(cmd) |
| self.cur.execute(cmd) |
| # insertion |
| cmd = "INSERT INTO " + table + " (" + \ |
| ",".join(map(str, INSERT.keys())) + ") VALUES(" + \ |
| ",".join(map(lambda x: 'Null' if x is None else "'" + str(x) + "'", INSERT.values())) + ")" |
| self.logger.debug(cmd) |
| self.cur.execute(cmd) |
| nb_rows = self.cur.rowcount |
| # inserting new log |
| # if nb_rows > 0 and log: |
| # if add_uuid: del INSERT['uuid'] |
| # #obtain tenant_id for logs |
| # if 'tenant_id' in INSERT: |
| # tenant_id = INSERT['tenant_id'] |
| # del INSERT['tenant_id'] |
| # elif table == 'tenants': |
| # tenant_id = uuid |
| # else: |
| # tenant_id = None |
| # if uuid is None: uuid_k = uuid_v = "" |
| # else: uuid_k=",uuid"; uuid_v=",'" + str(uuid) + "'" |
| # if tenant_id is None: tenant_k = tenant_v = "" |
| # else: tenant_k=",tenant_id"; tenant_v=",'" + str(tenant_id) + "'" |
| # cmd = "INSERT INTO logs (related,level%s%s,description) VALUES |
| # ('%s','debug'%s%s,\"new %s %s\")" \ |
| # % (uuid_k, tenant_k, table, uuid_v, tenant_v, table[:-1], str(INSERT)) |
| # self.logger.debug(cmd) |
| # self.cur.execute(cmd) |
| return nb_rows, uuid |
| |
| except (mdb.Error, AttributeError) as e: |
| r, c = self.format_error(e, "new_row", cmd) |
| if r != -HTTP_Request_Timeout or retry_ == 1: |
| return r, c |
| |
| @staticmethod |
| def __remove_quotes(data): |
| """remove single quotes ' of any string content of data dictionary""" |
| for k, v in data.items(): |
| if type(v) == str: |
| if "'" in v: |
| data[k] = data[k].replace("'", "_") |
| |
| def _update_rows_internal(self, table, UPDATE, WHERE=None): |
| cmd = "UPDATE " + table + " SET " + \ |
| ",".join(map(lambda x: str(x) + '=' + self.__data2db_format(UPDATE[x]), UPDATE.keys())) |
| if WHERE: |
| cmd += " WHERE " + " and ".join( |
| map(lambda x: str(x) + (' is Null' if WHERE[x] is None else "='" + str(WHERE[x]) + "'"), WHERE.keys())) |
| self.logger.debug(cmd) |
| self.cur.execute(cmd) |
| nb_rows = self.cur.rowcount |
| return nb_rows, None |
| |
| def update_rows(self, table, UPDATE, WHERE=None, log=False): |
| """ Update one or several rows into a table. |
| Atributes |
| UPDATE: dictionary with the key-new_value pairs to change |
| table: table to be modified |
| WHERE: dictionary to filter target rows, key-value |
| log: if true, a log entry is added at logs table |
| Return: (result, None) where result indicates the number of updated files |
| """ |
| for retry_ in range(0, 2): |
| cmd = "" |
| try: |
| # gettting uuid |
| uuid = None |
| if WHERE: |
| uuid = WHERE.get('uuid') |
| |
| with self.lock, self.con: |
| self.cur = self.con.cursor() |
| cmd = "UPDATE " + table + " SET " + \ |
| ",".join(map(lambda x: str(x) + '=' + self.__data2db_format(UPDATE[x]), UPDATE.keys())) |
| if WHERE: |
| cmd += " WHERE " + " and ".join( |
| map(lambda x: str(x) + (' is Null' if WHERE[x] is None else "='" + str(WHERE[x]) + "'"), |
| WHERE.keys())) |
| self.logger.debug(cmd) |
| self.cur.execute(cmd) |
| nb_rows = self.cur.rowcount |
| # if nb_rows > 0 and log: |
| # #inserting new log |
| # if uuid is None: uuid_k = uuid_v = "" |
| # else: uuid_k=",uuid"; uuid_v=",'" + str(uuid) + "'" |
| # cmd = "INSERT INTO logs (related,level%s,description) |
| # VALUES ('%s','debug'%s,\"updating %d entry %s\")" \ |
| # % (uuid_k, table, uuid_v, nb_rows, (str(UPDATE)).replace('"','-') ) |
| # self.logger.debug(cmd) |
| # self.cur.execute(cmd) |
| return nb_rows, uuid |
| except (mdb.Error, AttributeError) as e: |
| r, c = self.format_error(e, "update_rows", cmd) |
| if r != -HTTP_Request_Timeout or retry_ == 1: |
| return r, c |
| |
| def get_host(self, host_id): |
| if af.check_valid_uuid(host_id): |
| where_filter = "uuid='" + host_id + "'" |
| else: |
| where_filter = "name='" + host_id + "'" |
| for retry_ in range(0, 2): |
| cmd = "" |
| try: |
| with self.lock, self.con: |
| self.cur = self.con.cursor(mdb.cursors.DictCursor) |
| # get HOST |
| cmd = "SELECT uuid, user, password, keyfile, name, ip_name, description, hypervisors, " \ |
| "ranking, admin_state_up, DATE_FORMAT(created_at,'%Y-%m-%dT%H:%i:%s') as created_at " \ |
| "FROM hosts WHERE " + where_filter # Unikernels extension |
| self.logger.debug(cmd) |
| self.cur.execute(cmd) |
| if self.cur.rowcount == 0: |
| return 0, "host '" + str(host_id) + "'not found." |
| elif self.cur.rowcount > 1: |
| return 0, "host '" + str(host_id) + "' matches more than one result." |
| host = self.cur.fetchone() |
| host_id = host['uuid'] |
| if host.get("password"): |
| host["password"] = "*****" |
| # get numa |
| cmd = "SELECT id, numa_socket, hugepages, memory, admin_state_up FROM numas " \ |
| "WHERE host_id = '" + str(host_id) + "'" |
| self.logger.debug(cmd) |
| self.cur.execute(cmd) |
| host['numas'] = self.cur.fetchall() |
| for numa in host['numas']: |
| # print "SELECT core_id, instance_id, status, thread_id, v_thread_id FROM resources_core |
| # WHERE numa_id = '" + str(numa['id']) + "'" |
| # get cores |
| cmd = "SELECT core_id, instance_id, status, thread_id, v_thread_id FROM resources_core " \ |
| "WHERE numa_id = '" + str(numa['id']) + "'" |
| self.logger.debug(cmd) |
| self.cur.execute(cmd) |
| numa['cores'] = self.cur.fetchall() |
| for core in numa['cores']: |
| if core['instance_id']is None: |
| del core['instance_id'], core['v_thread_id'] |
| if core['status'] == 'ok': |
| del core['status'] |
| # get used memory |
| cmd = "SELECT sum(consumed) as hugepages_consumed FROM resources_mem " \ |
| "WHERE numa_id = '" + str(numa['id']) + "' GROUP BY numa_id" |
| self.logger.debug(cmd) |
| self.cur.execute(cmd) |
| used = self.cur.fetchone() |
| used_ = int(used['hugepages_consumed']) if used is not None else 0 |
| numa['hugepages_consumed'] = used_ |
| # get ports |
| # cmd = "CALL GetPortsFromNuma(%s)'" % str(numa['id']) |
| # self.cur.callproc('GetPortsFromNuma', (numa['id'],) ) |
| # every time a Procedure is launched you need to close and open the cursor |
| # under Error 2014: Commands out of sync; you can't run this command now |
| # self.cur.close() |
| # self.cur = self.con.cursor(mdb.cursors.DictCursor) |
| cmd = "SELECT Mbps, pci, status, Mbps_used, instance_id, if(id=root_id,'PF','VF') as type_, " \ |
| "switch_port, switch_dpid, switch_mac, mac, source_name " \ |
| "FROM resources_port WHERE numa_id={} ORDER BY root_id, type_ DESC".format(numa['id']) |
| self.logger.debug(cmd) |
| self.cur.execute(cmd) |
| ifaces = self.cur.fetchall() |
| # The SQL query will ensure to have SRIOV interfaces from a port first |
| sriovs = [] |
| Mpbs_consumed = 0 |
| numa['interfaces'] = [] |
| for iface in ifaces: |
| if not iface["instance_id"]: |
| del iface["instance_id"] |
| if iface['status'] == 'ok': |
| del iface['status'] |
| Mpbs_consumed += int(iface["Mbps_used"]) |
| del iface["Mbps_used"] |
| if iface["type_"] == 'PF': |
| if not iface["switch_dpid"]: |
| del iface["switch_dpid"] |
| if not iface["switch_port"]: |
| del iface["switch_port"] |
| if not iface["switch_mac"]: |
| del iface["switch_mac"] |
| if sriovs: |
| iface["sriovs"] = sriovs |
| if Mpbs_consumed: |
| iface["Mpbs_consumed"] = Mpbs_consumed |
| del iface["type_"] |
| numa['interfaces'].append(iface) |
| sriovs = [] |
| Mpbs_consumed = 0 |
| else: # VF, SRIOV |
| del iface["switch_port"] |
| del iface["switch_dpid"] |
| del iface["switch_mac"] |
| del iface["type_"] |
| del iface["Mbps"] |
| sriovs.append(iface) |
| |
| # delete internal field |
| del numa['id'] |
| return 1, host |
| except (mdb.Error, AttributeError) as e: |
| r, c = self.format_error(e, "get_host", cmd) |
| if r != -HTTP_Request_Timeout or retry_ == 1: |
| return r, c |
| |
| def new_uuid(self): |
| max_retries = 10 |
| while max_retries > 0: |
| uuid = str(myUuid.uuid1()) |
| if self.check_uuid(uuid)[0] == 0: |
| return uuid |
| max_retries -= 1 |
| return uuid |
| |
| def check_uuid(self, uuid): |
| """check in the database if this uuid is already present""" |
| try: |
| cmd = "SELECT * FROM uuids where uuid='" + str(uuid) + "'" |
| with self.lock, self.con: |
| self.cur = self.con.cursor(mdb.cursors.DictCursor) |
| self.logger.debug(cmd) |
| self.cur.execute(cmd) |
| rows = self.cur.fetchall() |
| return self.cur.rowcount, rows |
| except (mdb.Error, AttributeError) as e: |
| return self.format_error(e, "check_uuid", cmd) |
| |
| def __get_next_ids(self): |
| """get next auto increment index of all table in the database""" |
| self.cur.execute("SELECT table_name,AUTO_INCREMENT FROM information_schema.tables " |
| "WHERE AUTO_INCREMENT IS NOT NULL AND table_schema = DATABASE()") |
| rows = self.cur.fetchall() |
| return self.cur.rowcount, dict(rows) |
| |
| def edit_host(self, host_id, host_dict): |
| # get next port index |
| for retry_ in range(0, 2): |
| cmd = "" |
| try: |
| with self.lock, self.con: |
| self.cur = self.con.cursor() |
| |
| # update table host |
| numa_list = host_dict.pop('numas', ()) |
| if host_dict: |
| self._update_rows_internal("hosts", host_dict, {"uuid": host_id}) |
| |
| where = {"host_id": host_id} |
| for numa_dict in numa_list: |
| where["numa_socket"] = str(numa_dict.pop('numa_socket')) |
| interface_list = numa_dict.pop('interfaces', ()) |
| if numa_dict: |
| self._update_rows_internal("numas", numa_dict, where) |
| for interface in interface_list: |
| source_name = str(interface.pop("source_name")) |
| if interface: |
| # get interface id from resources_port |
| cmd = "SELECT rp.id as id " \ |
| "FROM resources_port as rp join numas as n on n.id=rp.numa_id join " \ |
| "hosts as h on h.uuid=n.host_id " + \ |
| "WHERE host_id='{}' and rp.source_name='{}'".format(host_id, source_name) |
| self.logger.debug(cmd) |
| self.cur.execute(cmd) |
| row = self.cur.fetchone() |
| if self.cur.rowcount <= 0: |
| return -HTTP_Bad_Request, "Interface source_name='{}s' from numa_socket='{}' " \ |
| "not found".format(source_name, where["numa_socket"]) |
| interface_id = row[0] |
| self._update_rows_internal("resources_port", interface, {"root_id": interface_id}) |
| return self.get_host(host_id) |
| except (mdb.Error, AttributeError) as e: |
| r, c = self.format_error(e, "edit_host", cmd) |
| if r != -HTTP_Request_Timeout or retry_ == 1: |
| return r, c |
| |
| def new_host(self, host_dict): |
| # get next port index |
| for retry_ in range(0, 2): |
| cmd = "" |
| try: |
| with self.lock, self.con: |
| self.cur = self.con.cursor() |
| |
| result, next_ids = self.__get_next_ids() |
| # print "next_ids: " + str(next_ids) |
| if result <= 0: |
| return result, "Internal DataBase error getting next id of tables" |
| |
| # create uuid if not provided |
| if 'uuid' not in host_dict: |
| uuid = host_dict['uuid'] = str(myUuid.uuid1()) # create_uuid |
| else: # check uuid is valid |
| uuid = str(host_dict['uuid']) |
| # result, data = self.check_uuid(uuid) |
| # if (result == 1): |
| # return -1, "UUID '%s' already in use" % uuid |
| |
| # inserting new uuid |
| cmd = "INSERT INTO uuids (uuid, used_at) VALUES ('%s','hosts')" % uuid |
| self.logger.debug(cmd) |
| result = self.cur.execute(cmd) |
| |
| # insert in table host |
| numa_list = host_dict.pop('numas', []) |
| # get nonhupages and nonisolated cpus |
| host_dict['RAM'] = 0 |
| host_dict['cpus'] = 0 |
| for numa in numa_list: |
| mem_numa = numa.get('memory', 0) - numa.get('hugepages', 0) |
| if mem_numa > 0: |
| host_dict['RAM'] += mem_numa |
| for core in numa.get("cores", []): |
| if "status" in core and core["status"] == "noteligible": |
| host_dict['cpus'] += 1 |
| host_dict['RAM'] *= 1024 # from GB to MB |
| |
| keys = ",".join(host_dict.keys()) |
| values = ",".join(map(lambda x: "Null" if x is None else "'" + str(x) + "'", host_dict.values())) |
| cmd = "INSERT INTO hosts (" + keys + ") VALUES (" + values + ")" |
| self.logger.debug(cmd) |
| result = self.cur.execute(cmd) |
| # if result != 1: return -1, "Database Error while inserting at hosts table" |
| |
| # insert numas |
| nb_numas = nb_cores = nb_ifaces = 0 |
| for numa_dict in numa_list: |
| nb_numas += 1 |
| interface_list = numa_dict.pop('interfaces', []) |
| core_list = numa_dict.pop('cores', []) |
| numa_dict['id'] = next_ids['numas'] |
| next_ids['numas'] += 1 |
| numa_dict['host_id'] = uuid |
| keys = ",".join(numa_dict.keys()) |
| values = ",".join( |
| map(lambda x: "Null" if x is None else "'" + str(x) + "'", numa_dict.values())) |
| cmd = "INSERT INTO numas (" + keys + ") VALUES (" + values + ")" |
| self.logger.debug(cmd) |
| self.cur.execute(cmd) |
| |
| # insert cores |
| for core_dict in core_list: |
| nb_cores += 1 |
| core_dict['numa_id'] = numa_dict['id'] |
| keys = ",".join(core_dict.keys()) |
| values = ",".join(map(lambda x: "Null" if x is None else "'" + str(x) + "'", |
| core_dict.values())) |
| cmd = "INSERT INTO resources_core (" + keys + ") VALUES (" + values + ")" |
| self.logger.debug(cmd) |
| self.cur.execute(cmd) |
| |
| # insert ports |
| for port_dict in interface_list: |
| nb_ifaces += 1 |
| sriov_list = port_dict.pop('sriovs', []) |
| port_dict['numa_id'] = numa_dict['id'] |
| port_dict['id'] = port_dict['root_id'] = next_ids['resources_port'] |
| next_ids['resources_port'] += 1 |
| switch_port = port_dict.get('switch_port', None) |
| switch_dpid = port_dict.get('switch_dpid', None) |
| keys = ",".join(port_dict.keys()) |
| values = ",".join(map(lambda x: "Null" if x is None else "'" + str(x) + "'", |
| port_dict.values())) |
| cmd = "INSERT INTO resources_port (" + keys + ") VALUES (" + values + ")" |
| self.logger.debug(cmd) |
| self.cur.execute(cmd) |
| |
| # insert sriovs into port table |
| for sriov_dict in sriov_list: |
| sriov_dict['switch_port'] = switch_port |
| sriov_dict['switch_dpid'] = switch_dpid |
| sriov_dict['numa_id'] = port_dict['numa_id'] |
| sriov_dict['Mbps'] = port_dict['Mbps'] |
| sriov_dict['root_id'] = port_dict['id'] |
| sriov_dict['id'] = next_ids['resources_port'] |
| if "vlan" in sriov_dict: |
| del sriov_dict["vlan"] |
| next_ids['resources_port'] += 1 |
| keys = ",".join(sriov_dict.keys()) |
| values = ",".join(map(lambda x: "Null" if x is None else "'" + str(x) + "'", |
| sriov_dict.values())) |
| cmd = "INSERT INTO resources_port (" + keys + ") VALUES (" + values + ")" |
| self.logger.debug(cmd) |
| self.cur.execute(cmd) |
| |
| # inserting new log |
| # cmd = "INSERT INTO logs (related,level,uuid,description) VALUES ('hosts','debug','%s','new host: |
| # %d numas, %d theads, %d ifaces')" % (uuid, nb_numas, nb_cores, nb_ifaces) |
| # self.logger.debug(cmd) |
| # result = self.cur.execute(cmd) |
| |
| # inseted ok |
| with self.lock, self.con: |
| self.cur = self.con.cursor() |
| self.logger.debug("callproc('UpdateSwitchPort', () )") |
| self.cur.callproc('UpdateSwitchPort', ()) |
| |
| self.logger.debug("getting host '%s'", str(host_dict['uuid'])) |
| return self.get_host(host_dict['uuid']) |
| except (mdb.Error, AttributeError) as e: |
| r, c = self.format_error(e, "new_host", cmd) |
| if r != -HTTP_Request_Timeout or retry_ == 1: |
| return r, c |
| |
| def new_flavor(self, flavor_dict, tenant_id): |
| """Add new flavor into the database. Create uuid if not provided |
| Atributes |
| flavor_dict: flavor dictionary with the key: value to insert. Must be valid flavors columns |
| tenant_id: if not 'any', it matches this flavor/tenant inserting at tenants_flavors table |
| Return: (result, data) where result can be |
| negative: error at inserting. data contain text |
| 1, inserted, data contain inserted uuid flavor |
| """ |
| for retry_ in range(0, 2): |
| cmd = "" |
| try: |
| with self.lock, self.con: |
| self.cur = self.con.cursor() |
| |
| # create uuid if not provided |
| if 'uuid' not in flavor_dict: |
| uuid = flavor_dict['uuid'] = str(myUuid.uuid1()) # create_uuid |
| else: # check uuid is valid |
| uuid = str(flavor_dict['uuid']) |
| # result, data = self.check_uuid(uuid) |
| # if (result == 1): |
| # return -1, "UUID '%s' already in use" % uuid |
| |
| # inserting new uuid |
| cmd = "INSERT INTO uuids (uuid, used_at) VALUES ('%s','flavors')" % uuid |
| self.logger.debug(cmd) |
| self.cur.execute(cmd) |
| |
| # insert in table flavor |
| keys = ",".join(flavor_dict.keys()) |
| values = ",".join(map(lambda x: "Null" if x is None else "'" + str(x) + "'", flavor_dict.values())) |
| cmd = "INSERT INTO flavors (" + keys + ") VALUES (" + values + ")" |
| self.logger.debug(cmd) |
| self.cur.execute(cmd) |
| # if result != 1: return -1, "Database Error while inserting at flavors table" |
| |
| # insert tenants_flavors |
| if tenant_id != 'any': |
| cmd = "INSERT INTO tenants_flavors (tenant_id,flavor_id) VALUES ('%s','%s')" % (tenant_id, uuid) |
| self.logger.debug(cmd) |
| self.cur.execute(cmd) |
| |
| # inserting new log |
| # del flavor_dict['uuid'] |
| # if 'extended' in flavor_dict: del flavor_dict['extended'] #remove two many information |
| # cmd = "INSERT INTO logs (related,level,uuid, tenant_id, description) VALUES |
| # ('flavors','debug','%s','%s',\"new flavor: %s\")" \ |
| # % (uuid, tenant_id, str(flavor_dict)) |
| # self.logger.debug(cmd) |
| # self.cur.execute(cmd) |
| |
| # inseted ok |
| return 1, uuid |
| except (mdb.Error, AttributeError) as e: |
| r, c = self.format_error(e, "new_flavor", cmd, "update", tenant_id) |
| if r != -HTTP_Request_Timeout or retry_ == 1: |
| return r, c |
| |
| def new_image(self, image_dict, tenant_id): |
| """Add new image into the database. Create uuid if not provided |
| Atributes |
| image_dict: image dictionary with the key: value to insert. Must be valid images columns |
| tenant_id: if not 'any', it matches this image/tenant inserting at tenants_images table |
| Return: (result, data) where result can be |
| negative: error at inserting. data contain text |
| 1, inserted, data contain inserted uuid image |
| """ |
| for retry_ in range(0, 2): |
| cmd = "" |
| try: |
| with self.lock, self.con: |
| self.cur = self.con.cursor() |
| |
| # create uuid if not provided |
| if 'uuid' not in image_dict: |
| uuid = image_dict['uuid'] = str(myUuid.uuid1()) # create_uuid |
| else: # check uuid is valid |
| uuid = str(image_dict['uuid']) |
| # result, data = self.check_uuid(uuid) |
| # if (result == 1): |
| # return -1, "UUID '%s' already in use" % uuid |
| |
| # inserting new uuid |
| cmd = "INSERT INTO uuids (uuid, used_at) VALUES ('%s','images')" % uuid |
| self.logger.debug(cmd) |
| self.cur.execute(cmd) |
| |
| # insert in table image |
| keys = ",".join(image_dict.keys()) |
| values = ",".join(map(lambda x: "Null" if x is None else "'" + str(x) + "'", image_dict.values())) |
| cmd = "INSERT INTO images (" + keys + ") VALUES (" + values + ")" |
| self.logger.debug(cmd) |
| self.cur.execute(cmd) |
| # if result != 1: return -1, "Database Error while inserting at images table" |
| |
| # insert tenants_images |
| if tenant_id != 'any': |
| cmd = "INSERT INTO tenants_images (tenant_id,image_id) VALUES ('%s','%s')" % (tenant_id, uuid) |
| self.logger.debug(cmd) |
| self.cur.execute(cmd) |
| |
| # #inserting new log |
| # cmd = "INSERT INTO logs (related,level,uuid, tenant_id, description) VALUES |
| # ('images','debug','%s','%s',\"new image: %s path: %s\")" % |
| # (uuid, tenant_id, image_dict['name'], image_dict['path']) |
| # self.logger.debug(cmd) |
| # self.cur.execute(cmd) |
| |
| # inseted ok |
| return 1, uuid |
| except (mdb.Error, AttributeError) as e: |
| r, c = self.format_error(e, "new_image", cmd, "update", tenant_id) |
| if r != -HTTP_Request_Timeout or retry_ == 1: |
| return r, c |
| |
| def delete_image_flavor(self, item_type, item_id, tenant_id): |
| """deletes an image or flavor from database |
| item_type must be a 'image' or 'flavor' |
| item_id is the uuid |
| tenant_id is the asociated tenant, can be 'any' with means all |
| If tenan_id is not any, it deletes from tenants_images/flavors, |
| which means this image/flavor is used by this tenant, and if success, |
| it tries to delete from images/flavors in case this is not public, |
| that only will success if image is private and not used by other tenants |
| If tenant_id is any, it tries to delete from both tables at the same transaction |
| so that image/flavor is completely deleted from all tenants or nothing |
| """ |
| for retry_ in range(0, 2): |
| deleted = -1 |
| deleted_item = -1 |
| result = (-HTTP_Internal_Server_Error, "internal error") |
| cmd = "" |
| try: |
| with self.lock, self.con: |
| self.cur = self.con.cursor() |
| cmd = "DELETE FROM tenants_%ss WHERE %s_id = '%s'" % (item_type, item_type, item_id) |
| if tenant_id != 'any': |
| cmd += " AND tenant_id = '%s'" % tenant_id |
| self.logger.debug(cmd) |
| self.cur.execute(cmd) |
| deleted = self.cur.rowcount |
| if tenant_id == 'any': # delete from images/flavors in the SAME transaction |
| cmd = "DELETE FROM %ss WHERE uuid = '%s'" % (item_type, item_id) |
| self.logger.debug(cmd) |
| self.cur.execute(cmd) |
| deleted = self.cur.rowcount |
| if deleted >= 1: |
| # delete uuid |
| cmd = "DELETE FROM uuids WHERE uuid = '%s'" % item_id |
| self.logger.debug(cmd) |
| self.cur.execute(cmd) |
| # #inserting new log |
| # cmd = "INSERT INTO logs (related,level,uuid,tenant_id,description) \ |
| # VALUES ('%ss','debug','%s','%s','delete %s completely')" % \ |
| # (item_type, item_id, tenant_id, item_type) |
| # self.logger.debug(cmd) |
| # self.cur.execute(cmd) |
| return deleted, "%s '%s' completely deleted" % (item_type, item_id) |
| return 0, "%s '%s' not found" % (item_type, item_id) |
| |
| if deleted == 1: |
| # #inserting new log |
| # cmd = "INSERT INTO logs (related,level,uuid,tenant_id,description) \ |
| # VALUES ('%ss','debug','%s','%s','delete %s reference for this tenant')" % \ |
| # (item_type, item_id, tenant_id, item_type) |
| # self.logger.debug(cmd) |
| # self.cur.execute(cmd) |
| |
| # commit transaction |
| self.cur.close() |
| # if tenant!=any delete from images/flavors in OTHER transaction. |
| # If fails is because dependencies so that not return error |
| if deleted == 1: |
| with self.lock, self.con: |
| self.cur = self.con.cursor() |
| |
| # delete image/flavor if not public |
| cmd = "DELETE FROM %ss WHERE uuid = '%s' AND public = 'no'" % (item_type, item_id) |
| self.logger.debug(cmd) |
| self.cur.execute(cmd) |
| deleted_item = self.cur.rowcount |
| if deleted_item == 1: |
| # delete uuid |
| cmd = "DELETE FROM uuids WHERE uuid = '%s'" % item_id |
| self.logger.debug(cmd) |
| self.cur.execute(cmd) |
| # #inserting new log |
| # cmd = "INSERT INTO logs (related,level,uuid,tenant_id,description) \ |
| # VALUES ('%ss','debug','%s','%s','delete %s completely')" % \ |
| # (item_type, item_id, tenant_id, item_type) |
| # self.logger.debug(cmd) |
| # self.cur.execute(cmd) |
| except (mdb.Error, AttributeError) as e: |
| # print "delete_%s DB Exception %d: %s" % (item_type, e.args[0], e.args[1]) |
| if deleted < 0: |
| result = self.format_error(e, "delete_" + item_type, cmd, "delete", "servers") |
| finally: |
| if deleted == 1: |
| return 1, "{} '{}' from tenant '{}' {}deleted".format(item_type, item_id, tenant_id, |
| "completely " if deleted_item == 1 else "") |
| elif deleted == 0: |
| return 0, "{} '{}' from tenant '{}' not found".format(item_type, item_id, tenant_id) |
| else: |
| if result[0] != -HTTP_Request_Timeout or retry_ == 1: |
| return result |
| |
| def delete_row(self, table, uuid): |
| for retry_ in range(0, 2): |
| cmd = "" |
| try: |
| with self.lock, self.con: |
| # delete host |
| self.cur = self.con.cursor() |
| cmd = "DELETE FROM %s WHERE uuid = '%s'" % (table, uuid) |
| self.logger.debug(cmd) |
| self.cur.execute(cmd) |
| deleted = self.cur.rowcount |
| if deleted == 1: |
| # delete uuid |
| # if table == 'tenants': |
| # tenant_str = uuid |
| # else: |
| # tenant_str = 'Null' |
| self.cur = self.con.cursor() |
| cmd = "DELETE FROM uuids WHERE uuid = '%s'" % uuid |
| self.logger.debug(cmd) |
| self.cur.execute(cmd) |
| # #inserting new log |
| # cmd = "INSERT INTO logs (related,level,uuid,tenant_id,description) VALUES |
| # ('%s','debug','%s','%s','delete %s')" % (table, uuid, tenant_str, table[:-1]) |
| # self.logger.debug(cmd) |
| # self.cur.execute(cmd) |
| return deleted, table[:-1] + " '%s' %s" % (uuid, "deleted" if deleted == 1 else "not found") |
| except (mdb.Error, AttributeError) as e: |
| r, c = self.format_error(e, "delete_row", cmd, "delete", |
| 'instances' if table in ('hosts', 'tenants') else 'dependencies') |
| if r != -HTTP_Request_Timeout or retry_ == 1: |
| return r, c |
| |
| def delete_row_by_key(self, table, key, value): |
| for retry_ in range(0, 2): |
| cmd = "" |
| try: |
| with self.lock, self.con: |
| # delete host |
| self.cur = self.con.cursor() |
| cmd = "DELETE FROM %s" % (table) |
| if key: |
| if value: |
| cmd += " WHERE %s = '%s'" % (key, value) |
| else: |
| cmd += " WHERE %s is null" % (key) |
| else: # delete all |
| pass |
| self.logger.debug(cmd) |
| self.cur.execute(cmd) |
| deleted = self.cur.rowcount |
| if deleted < 1: |
| return -1, 'Not found' |
| # delete uuid |
| return 0, deleted |
| except (mdb.Error, AttributeError) as e: |
| r, c = self.format_error(e, "delete_row_by_key", cmd, "delete", |
| 'instances' if table in ('hosts', 'tenants') else 'dependencies') |
| if r != -HTTP_Request_Timeout or retry_ == 1: |
| return r, c |
| |
| def delete_row_by_dict(self, **sql_dict): |
| """ Deletes rows from a table. |
| Attribute sql_dir: dictionary with the following key: value |
| 'FROM': string of table name (Mandatory) |
| 'WHERE': dict of key:values, translated to key=value AND ... (Optional) |
| 'WHERE_NOT': dict of key:values, translated to key<>value AND ... (Optional) |
| 'WHERE_NOTNULL': (list or tuple of items that must not be null in a where ... (Optional) |
| 'LIMIT': limit of number of rows (Optional) |
| Return: the (number of items deleted, descriptive test) if ok; (negative, descriptive text) if error |
| """ |
| # print sql_dict |
| from_ = "FROM " + str(sql_dict['FROM']) |
| # print 'from_', from_ |
| if 'WHERE' in sql_dict and len(sql_dict['WHERE']) > 0: |
| w = sql_dict['WHERE'] |
| where_ = "WHERE " + " AND ".join(map(lambda x: str(x) + (" is Null" if w[x] is None else "='" + str(w[x]) + |
| "'"), w.keys())) |
| else: |
| where_ = "" |
| if 'WHERE_NOT' in sql_dict and len(sql_dict['WHERE_NOT']) > 0: |
| w = sql_dict['WHERE_NOT'] |
| where_2 = " AND ".join(map(lambda x: str(x) + (" is not Null" if w[x] is None else "<>'" + str(w[x]) + "'"), |
| w.keys())) |
| if len(where_) == 0: |
| where_ = "WHERE " + where_2 |
| else: |
| where_ = where_ + " AND " + where_2 |
| if 'WHERE_NOTNULL' in sql_dict and len(sql_dict['WHERE_NOTNULL']) > 0: |
| w = sql_dict['WHERE_NOTNULL'] |
| where_2 = " AND ".join(map(lambda x: str(x) + " is not Null", w)) |
| if len(where_) == 0: |
| where_ = "WHERE " + where_2 |
| else: |
| where_ = where_ + " AND " + where_2 |
| # print 'where_', where_ |
| limit_ = "LIMIT " + str(sql_dict['LIMIT']) if 'LIMIT' in sql_dict else "" |
| # print 'limit_', limit_ |
| cmd = " ".join(("DELETE", from_, where_, limit_)) |
| self.logger.debug(cmd) |
| for retry_ in range(0, 2): |
| try: |
| with self.lock, self.con: |
| # delete host |
| self.cur = self.con.cursor() |
| self.cur.execute(cmd) |
| deleted = self.cur.rowcount |
| return deleted, "%d deleted from %s" % (deleted, sql_dict['FROM'][:-1]) |
| except (mdb.Error, AttributeError) as e: |
| r, c = self.format_error(e, "delete_row_by_dict", cmd, "delete", 'dependencies') |
| if r != -HTTP_Request_Timeout or retry_ == 1: |
| return r, c |
| |
| def get_instance(self, instance_id): |
| for retry_ in range(0, 2): |
| cmd = "" |
| try: |
| with self.lock, self.con: |
| self.cur = self.con.cursor(mdb.cursors.DictCursor) |
| # get INSTANCE |
| cmd = "SELECT uuid, name, description, progress, host_id, flavor_id, image_id, status, " \ |
| "hypervisor, os_image_type, last_error, tenant_id, ram, vcpus, created_at " \ |
| "FROM instances WHERE uuid='{}'".format(instance_id) # Unikernels extension |
| self.logger.debug(cmd) |
| self.cur.execute(cmd) |
| if self.cur.rowcount == 0: |
| return 0, "instance '" + str(instance_id) + "'not found." |
| instance = self.cur.fetchone() |
| # get networks |
| cmd = "SELECT uuid as iface_id, net_id, mac as mac_address, ip_address, name, Mbps as bandwidth, " \ |
| "vpci, model FROM ports WHERE (type='instance:bridge' or type='instance:ovs') AND " \ |
| "instance_id= '{}'".format(instance_id) |
| self.logger.debug(cmd) |
| self.cur.execute(cmd) |
| if self.cur.rowcount > 0: |
| instance['networks'] = self.cur.fetchall() |
| |
| # get extended |
| extended = {} |
| # get devices |
| cmd = "SELECT type,vpci,image_id,xml,dev,image_size FROM instance_devices " \ |
| "WHERE instance_id = '%s' " % str(instance_id) |
| self.logger.debug(cmd) |
| self.cur.execute(cmd) |
| if self.cur.rowcount > 0: |
| extended['devices'] = self.cur.fetchall() |
| # get numas |
| numas = [] |
| cmd = "SELECT id, numa_socket as source FROM numas WHERE host_id = '{}'".format(instance['host_id']) |
| self.logger.debug(cmd) |
| self.cur.execute(cmd) |
| host_numas = self.cur.fetchall() |
| # print 'host_numas', host_numas |
| for k in host_numas: |
| numa_id = str(k['id']) |
| numa_dict = {} |
| # get memory |
| cmd = "SELECT consumed FROM resources_mem WHERE instance_id = '{}' AND numa_id = '{}'".foramt( |
| instance_id, numa_id) |
| self.logger.debug(cmd) |
| self.cur.execute(cmd) |
| if self.cur.rowcount > 0: |
| mem_dict = self.cur.fetchone() |
| numa_dict['memory'] = mem_dict['consumed'] |
| # get full cores |
| cursor2 = self.con.cursor() |
| cmd = "SELECT core_id, paired, MIN(v_thread_id) as v1, MAX(v_thread_id) as v2, " \ |
| "COUNT(instance_id) as nb, MIN(thread_id) as t1, MAX(thread_id) as t2 " \ |
| "FROM resources_core " \ |
| "WHERE instance_id = '{}' AND numa_id = '{}' GROUP BY core_id,paired".format(instance_id, |
| numa_id) |
| self.logger.debug(cmd) |
| cursor2.execute(cmd) |
| core_list = [] |
| core_source = [] |
| paired_list = [] |
| paired_source = [] |
| thread_list = [] |
| thread_source = [] |
| if cursor2.rowcount > 0: |
| cores = cursor2.fetchall() |
| for core in cores: |
| if core[4] == 2: # number of used threads from core |
| if core[3] == core[2]: # only one thread asigned to VM, so completely core |
| core_list.append(core[2]) |
| core_source.append(core[5]) |
| elif core[1] == 'Y': |
| paired_list.append(core[2:4]) |
| paired_source.append(core[5:7]) |
| else: |
| thread_list.extend(core[2:4]) |
| thread_source.extend(core[5:7]) |
| |
| else: |
| thread_list.append(core[2]) |
| thread_source.append(core[5]) |
| if len(core_list) > 0: |
| numa_dict['cores'] = len(core_list) |
| numa_dict['cores-id'] = core_list |
| numa_dict['cores-source'] = core_source |
| if len(paired_list) > 0: |
| numa_dict['paired-threads'] = len(paired_list) |
| numa_dict['paired-threads-id'] = paired_list |
| numa_dict['paired-threads-source'] = paired_source |
| if len(thread_list) > 0: |
| numa_dict['threads'] = len(thread_list) |
| numa_dict['threads-id'] = thread_list |
| numa_dict['threads-source'] = thread_source |
| |
| # get dedicated ports and SRIOV |
| cmd = "SELECT port_id as iface_id, p.vlan as vlan, p.mac as mac_address, net_id, " \ |
| "if(model='PF','yes',if(model='VF','no','yes:sriov')) as dedicated, p.Mbps as bandwidth" \ |
| ", name, vpci, pci as source " \ |
| "FROM resources_port as rp join ports as p on port_id=uuid " \ |
| "WHERE p.instance_id = '{}' AND numa_id = '{}' and " \ |
| "p.type='instance:data'".format(instance_id, numa_id) |
| self.logger.debug(cmd) |
| self.cur.execute(cmd) |
| if self.cur.rowcount > 0: |
| numa_dict['interfaces'] = self.cur.fetchall() |
| # print 'interfaces', numa_dict |
| |
| if len(numa_dict) > 0: |
| numa_dict['source'] = k['source'] # numa socket |
| numas.append(numa_dict) |
| |
| if len(numas) > 0: |
| extended['numas'] = numas |
| if len(extended) > 0: |
| instance['extended'] = extended |
| af.DeleteNone(instance) |
| return 1, instance |
| except (mdb.Error, AttributeError) as e: |
| r, c = self.format_error(e, "get_instance", cmd) |
| if r != -HTTP_Request_Timeout or retry_ == 1: |
| return r, c |
| |
| def get_numas(self, requirements, prefered_host_id=None, only_of_ports=True): |
| """Obtain a valid NUMA/HOST for deployment a VM |
| requirements: contain requirement regarding: |
| requirements['ram']: Non huge page memory in MB; 0 to skip |
| requirements['vcpus']: Non isolated cpus; 0 to skip |
| requirements['numa']: Requiremets to be fixed in ONE Numa node |
| requirements['numa']['memory']: Huge page memory in GB at ; 0 for any |
| requirements['numa']['proc_req_type']: Type of processor, cores or threads |
| requirements['numa']['proc_req_nb']: Number of isolated cpus |
| requirements['numa']['port_list']: Physical NIC ports list ; [] for any |
| requirements['numa']['sriov_list']: Virtual function NIC ports list ; [] for any |
| prefered_host_id: if not None return this host if it match |
| only_of_ports: if True only those ports conected to the openflow (of) are valid, |
| that is, with switch_port information filled; if False, all NIC ports are valid. |
| Return a valid numa and host |
| """ |
| |
| for retry_ in range(0, 2): |
| cmd = "" |
| try: |
| with self.lock, self.con: |
| # #Find numas of prefered host |
| # prefered_numas = () |
| # if prefered_host_id is not None: |
| # self.cur = self.con.cursor() |
| # self.cur.execute("SELECT id FROM numas WHERE host_id='%s'" + prefered_host_id) |
| # prefered_numas = self.cur.fetchall() |
| # self.cur.close() |
| |
| # Find valid host for the ram and vcpus |
| self.cur = self.con.cursor(mdb.cursors.DictCursor) |
| cmd = "CALL GetHostByMemCpu(%s, %s)" % (str(requirements['ram']), str(requirements['vcpus'])) |
| self.logger.debug(cmd) |
| self.cur.callproc('GetHostByMemCpu', (str(requirements['ram']), str(requirements['vcpus']))) |
| valid_hosts = self.cur.fetchall() |
| self.cur.close() |
| self.cur = self.con.cursor() |
| match_found = False |
| if len(valid_hosts) <= 0: |
| error_text = 'No room at data center. Cannot find a host with %s MB memory and %s cpus ' \ |
| 'available' % (str(requirements['ram']), str(requirements['vcpus'])) |
| # self.logger.debug(error_text) |
| return -1, error_text |
| |
| if 'hypervisor' not in requirements: # Unikernels extension -END- |
| requirements['hypervisor'] = "kvm" |
| for valid_host in valid_hosts: |
| if 'hypervisors' not in valid_host: |
| valid_host['hypervisors'] = "kvm" |
| |
| valid_hosts = tuple(valid_host for valid_host in valid_hosts if |
| requirements['hypervisor'] in valid_host['hypervisors'].split(",")) |
| |
| if len(valid_hosts) <= 0: |
| error_text = 'No room at data center. Cannot find a host with %s hypervisor or not have ' \ |
| 'enough resources available' % (str(requirements['hypervisor'])) |
| # self.logger.debug(error_text) |
| return -1, error_text # Unikernels extension -END- |
| |
| # elif req_numa is not None: |
| # Find valid numa nodes for memory requirements |
| self.cur = self.con.cursor(mdb.cursors.DictCursor) |
| cmd = "CALL GetNumaByMemory(%s)" % str(requirements['numa']['memory']) |
| self.logger.debug(cmd) |
| self.cur.callproc('GetNumaByMemory', (requirements['numa']['memory'],)) |
| valid_for_memory = self.cur.fetchall() |
| self.cur.close() |
| self.cur = self.con.cursor() |
| if len(valid_for_memory) <= 0: |
| error_text = 'No room at data center. Cannot find a host with %s GB Hugepages memory' \ |
| ' available' % str(requirements['numa']['memory']) |
| # self.logger.debug(error_text) |
| return -1, error_text |
| |
| # Find valid numa nodes for processor requirements |
| self.cur = self.con.cursor(mdb.cursors.DictCursor) |
| if requirements['numa']['proc_req_type'] == 'threads': |
| cpu_requirement_text = 'cpu-threads' |
| cmd = "CALL GetNumaByThread(%s)" % str(requirements['numa']['proc_req_nb']) |
| self.logger.debug(cmd) |
| self.cur.callproc('GetNumaByThread', (requirements['numa']['proc_req_nb'],)) |
| else: |
| cpu_requirement_text = 'cpu-cores' |
| cmd = "CALL GetNumaByCore(%s)" % str(requirements['numa']['proc_req_nb']) |
| self.logger.debug(cmd) |
| self.cur.callproc('GetNumaByCore', (requirements['numa']['proc_req_nb'],)) |
| valid_for_processor = self.cur.fetchall() |
| self.cur.close() |
| self.cur = self.con.cursor() |
| if len(valid_for_processor) <= 0: |
| error_text = 'No room at data center. Cannot find a host with %s %s available' % ( |
| str(requirements['numa']['proc_req_nb']), cpu_requirement_text) |
| # self.logger.debug(error_text) |
| return -1, error_text |
| |
| # Find the numa nodes that comply for memory and processor requirements |
| # sorting from less to more memory capacity |
| valid_numas = [] |
| for m_numa in valid_for_memory: |
| numa_valid_for_processor = False |
| for p_numa in valid_for_processor: |
| if m_numa['numa_id'] == p_numa['numa_id']: |
| numa_valid_for_processor = True |
| break |
| numa_valid_for_host = False |
| prefered_numa = False |
| for p_host in valid_hosts: |
| if m_numa['host_id'] == p_host['uuid']: |
| numa_valid_for_host = True |
| if p_host['uuid'] == prefered_host_id: |
| prefered_numa = True |
| break |
| if numa_valid_for_host and numa_valid_for_processor: |
| if prefered_numa: |
| valid_numas.insert(0, m_numa['numa_id']) |
| else: |
| valid_numas.append(m_numa['numa_id']) |
| if len(valid_numas) <= 0: |
| error_text = "No room at data center. Cannot find a host with {} MB hugepages memory and {} " \ |
| "{} available in the same numa".format(requirements['numa']['memory'], |
| requirements['numa']['proc_req_nb'], |
| cpu_requirement_text) |
| # self.logger.debug(error_text) |
| return -1, error_text |
| |
| # print 'Valid numas list: '+str(valid_numas) |
| |
| # Find valid numa nodes for interfaces requirements |
| # For each valid numa we will obtain the number of available ports and check if these are valid |
| match_found = False |
| for numa_id in valid_numas: |
| # print 'Checking '+str(numa_id) |
| match_found = False |
| self.cur = self.con.cursor(mdb.cursors.DictCursor) |
| if only_of_ports: |
| cmd = "CALL GetAvailablePorts(%s)" % str(numa_id) |
| self.logger.debug(cmd) |
| self.cur.callproc('GetAvailablePorts', (numa_id,)) |
| else: |
| cmd = "CALL GetAllAvailablePorts(%s)" % str(numa_id) |
| self.logger.debug(cmd) |
| self.cur.callproc('GetAllAvailablePorts', (numa_id,)) |
| available_ports = self.cur.fetchall() |
| self.cur.close() |
| self.cur = self.con.cursor() |
| |
| # Set/reset reservations |
| for port in available_ports: |
| port['Mbps_reserved'] = 0 |
| port['SRIOV_reserved'] = 0 |
| |
| # Try to allocate physical ports |
| physical_ports_found = True |
| for iface in requirements['numa']['port_list']: |
| # print '\t\tchecking iface: '+str(iface) |
| portFound = False |
| for port in available_ports: |
| # print '\t\t\tfor port: '+str(port) |
| # If the port is not empty continue |
| if port['Mbps_free'] != port['Mbps'] or port['Mbps_reserved'] != 0: |
| # print '\t\t\t\t Not empty port' |
| continue |
| # If the port speed is not enough continue |
| if port['Mbps'] < iface['bandwidth']: |
| # print '\t\t\t\t Not enough speed' |
| continue |
| |
| # Otherwise this is a valid port |
| port['Mbps_reserved'] = port['Mbps'] |
| port['SRIOV_reserved'] = 0 |
| iface['port_id'] = port['port_id'] |
| iface['vlan'] = None |
| iface['mac'] = port['mac'] |
| iface['switch_port'] = port['switch_port'] |
| # print '\t\t\t\t Dedicated port found '+str(port['port_id']) |
| portFound = True |
| break |
| |
| # if all ports have been checked and no match has been found |
| # this is not a valid numa |
| if not portFound: |
| # print '\t\t\t\t\tAll ports have been checked and no match has been found for |
| # numa '+str(numa_id)+'\n\n' |
| physical_ports_found = False |
| break |
| |
| # if there is no match continue checking the following numa |
| if not physical_ports_found: |
| continue |
| |
| # Try to allocate SR-IOVs |
| sriov_ports_found = True |
| for iface in requirements['numa']['sriov_list']: |
| # print '\t\tchecking iface: '+str(iface) |
| portFound = False |
| for port in available_ports: |
| # print '\t\t\tfor port: '+str(port) |
| # If there are not available SR-IOVs continue |
| if port['availableSRIOV'] - port['SRIOV_reserved'] <= 0: |
| # print '\t\t\t\t Not enough SR-IOV' |
| continue |
| # If the port free speed is not enough continue |
| if port['Mbps_free'] - port['Mbps_reserved'] < iface['bandwidth']: |
| # print '\t\t\t\t Not enough speed' |
| continue |
| |
| # Otherwise this is a valid port |
| port['Mbps_reserved'] += iface['bandwidth'] |
| port['SRIOV_reserved'] += 1 |
| # print '\t\t\t\t SR-IOV found '+str(port['port_id']) |
| iface['port_id'] = port['port_id'] |
| iface['vlan'] = None |
| iface['mac'] = port['mac'] |
| iface['switch_port'] = port['switch_port'] |
| portFound = True |
| break |
| |
| # if all ports have been checked and no match has been found |
| # this is not a valid numa |
| if not portFound: |
| # print '\t\t\t\t\tAll ports have been checked and no match has been found for numa |
| # '+str(numa_id)+'\n\n' |
| sriov_ports_found = False |
| break |
| |
| # if there is no match continue checking the following numa |
| if not sriov_ports_found: |
| continue |
| |
| if sriov_ports_found and physical_ports_found: |
| match_found = True |
| break |
| |
| if not match_found: |
| error_text = 'No room at data center. Cannot find a host with the required hugepages, vcpus ' \ |
| 'and interfaces' |
| # self.logger.debug(error_text) |
| return -1, error_text |
| |
| # self.logger.debug('Full match found in numa %s', str(numa_id)) |
| |
| for numa in valid_for_processor: |
| if numa_id == numa['numa_id']: |
| host_id = numa['host_id'] |
| break |
| return 0, {'numa_id': numa_id, 'host_id': host_id} |
| except (mdb.Error, AttributeError) as e: |
| r, c = self.format_error(e, "get_numas", cmd) |
| if r != -HTTP_Request_Timeout or retry_ == 1: |
| return r, c |
| |
| def new_instance(self, instance_dict, nets, ports_to_free): |
| for retry_ in range(0, 2): |
| cmd = "" |
| try: |
| with self.lock, self.con: |
| self.cur = self.con.cursor() |
| |
| # create uuid if not provided |
| if 'uuid' not in instance_dict: |
| uuid = instance_dict['uuid'] = str(myUuid.uuid1()) # create_uuid |
| else: # check uuid is valid |
| uuid = str(instance_dict['uuid']) |
| |
| # inserting new uuid |
| cmd = "INSERT INTO uuids (uuid, root_uuid, used_at) VALUES ('%s','%s', 'instances')" % (uuid, uuid) |
| self.logger.debug(cmd) |
| self.cur.execute(cmd) |
| |
| # insert in table instance |
| extended = instance_dict.pop('extended', None) |
| bridgedifaces = instance_dict.pop('bridged-ifaces', ()) |
| |
| keys = ",".join(instance_dict.keys()) |
| values = ",".join( |
| map(lambda x: "Null" if x is None else "'" + str(x) + "'", instance_dict.values())) |
| cmd = "INSERT INTO instances (" + keys + ") VALUES (" + values + ")" |
| self.logger.debug(cmd) |
| self.cur.execute(cmd) |
| # if result != 1: return -1, "Database Error while inserting at instances table" |
| |
| # insert resources |
| nb_bridge_ifaces = nb_cores = nb_ifaces = nb_numas = 0 |
| # insert bridged_ifaces |
| |
| for iface in bridgedifaces: |
| # generate and insert a iface uuid |
| if 'enable_dhcp' in iface and iface['enable_dhcp']: |
| dhcp_first_ip = iface["dhcp_first_ip"] |
| del iface["dhcp_first_ip"] |
| dhcp_last_ip = iface["dhcp_last_ip"] |
| del iface["dhcp_last_ip"] |
| dhcp_cidr = iface["cidr"] |
| del iface["cidr"] |
| del iface["enable_dhcp"] |
| used_dhcp_ips = self._get_dhcp_ip_used_list(iface["net_id"]) |
| if iface.get("ip_address"): |
| if iface["ip_address"] in used_dhcp_ips: |
| iface["ip_address"] = None |
| else: |
| iface["ip_address"] = self.get_free_ip_from_range(dhcp_first_ip, dhcp_last_ip, |
| dhcp_cidr, used_dhcp_ips) |
| if 'links' in iface: |
| del iface['links'] |
| if 'dns' in iface: |
| del iface['dns'] |
| if 'routes' in iface: |
| del iface['routes'] |
| |
| iface['uuid'] = str(myUuid.uuid1()) # create_uuid |
| cmd = "INSERT INTO uuids (uuid, root_uuid, used_at) VALUES ('%s','%s', 'ports')" % ( |
| iface['uuid'], uuid) |
| self.logger.debug(cmd) |
| self.cur.execute(cmd) |
| # insert iface |
| iface['instance_id'] = uuid |
| # iface['type'] = 'instance:bridge' |
| if 'name' not in iface: |
| iface['name'] = "br" + str(nb_bridge_ifaces) |
| iface['Mbps'] = iface.pop('bandwidth', None) |
| if 'mac_address' not in iface: |
| iface['mac'] = af.gen_random_mac() |
| else: |
| iface['mac'] = iface['mac_address'] |
| del iface['mac_address'] |
| |
| # iface['mac']=iface.pop('mac_address', None) #for leaving mac generation to libvirt |
| keys = ",".join(iface.keys()) |
| values = ",".join(map(lambda x: "Null" if x is None else "'" + str(x) + "'", iface.values())) |
| cmd = "INSERT INTO ports (" + keys + ") VALUES (" + values + ")" |
| self.logger.debug(cmd) |
| self.cur.execute(cmd) |
| nb_bridge_ifaces += 1 |
| |
| if extended is not None: |
| if 'numas' not in extended or extended['numas'] is None: |
| extended['numas'] = () |
| for numa in extended['numas']: |
| nb_numas += 1 |
| # cores |
| if 'cores' not in numa or numa['cores'] is None: |
| numa['cores'] = () |
| for core in numa['cores']: |
| nb_cores += 1 |
| cmd = "UPDATE resources_core SET instance_id='%s'%s%s WHERE id='%s'" \ |
| % (uuid, |
| (",v_thread_id='" + str(core['vthread']) + "'") if 'vthread' in core else '', |
| (",paired='" + core['paired'] + "'") if 'paired' in core else '', core['id']) |
| self.logger.debug(cmd) |
| self.cur.execute(cmd) |
| # interfaces |
| if 'interfaces' not in numa or numa['interfaces'] is None: |
| numa['interfaces'] = () |
| for iface in numa['interfaces']: |
| # generate and insert an uuid; iface[id]=iface_uuid; iface[uuid]= net_id |
| iface['id'] = str(myUuid.uuid1()) # create_uuid |
| cmd = "INSERT INTO uuids (uuid, root_uuid, used_at) VALUES " \ |
| "('{}','{}', 'ports')".format(iface['id'], uuid) |
| self.logger.debug(cmd) |
| self.cur.execute(cmd) |
| nb_ifaces += 1 |
| mbps_ = ("'" + str(iface['Mbps_used']) + "'") if 'Mbps_used' in iface and \ |
| iface['Mbps_used'] is not None \ |
| else "Mbps" |
| if iface["dedicated"] == "yes": |
| iface_model = "PF" |
| elif iface["dedicated"] == "yes:sriov": |
| iface_model = "VFnotShared" |
| elif iface["dedicated"] == "no": |
| iface_model = "VF" |
| # else error |
| INSERT = (iface['mac_address'], iface['switch_port'], iface.get('vlan'), |
| 'instance:data', iface['Mbps_used'], iface['id'], uuid, |
| instance_dict['tenant_id'], iface.get('name'), iface.get('vpci'), |
| iface.get('uuid'), iface_model) |
| cmd = "INSERT INTO ports (mac,switch_port,vlan,type,Mbps,uuid,instance_id,tenant_id," \ |
| "name,vpci,net_id, model) VALUES (" + \ |
| ",".join(map(lambda x: 'Null' if x is None else "'" + str(x) + "'", INSERT)) + ")" |
| self.logger.debug(cmd) |
| self.cur.execute(cmd) |
| if 'uuid' in iface: |
| nets.append(iface['uuid']) |
| |
| # discover if this port is not used by anyone |
| cmd = "SELECT source_name, mac " \ |
| "FROM ( SELECT root_id, count(instance_id) as used FROM resources_port" \ |
| " WHERE root_id=(SELECT root_id from resources_port WHERE id='%s')" \ |
| " GROUP BY root_id ) AS A JOIN resources_port as B ON " \ |
| "A.root_id=B.id AND A.used=0" % iface['port_id'] |
| self.logger.debug(cmd) |
| self.cur.execute(cmd) |
| ports_to_free += self.cur.fetchall() |
| |
| cmd = "UPDATE resources_port SET instance_id='%s', port_id='%s',Mbps_used=%s " \ |
| "WHERE id='%s'" % (uuid, iface['id'], mbps_, iface['port_id']) |
| # if Mbps_used not suply, set the same value of 'Mpbs', that is the total |
| self.logger.debug(cmd) |
| self.cur.execute(cmd) |
| # memory |
| if 'memory' in numa and numa['memory'] is not None and numa['memory'] > 0: |
| cmd = "INSERT INTO resources_mem (numa_id, instance_id, consumed) VALUES " \ |
| "('%s','%s','%s')" % (numa['numa_id'], uuid, numa['memory']) |
| self.logger.debug(cmd) |
| self.cur.execute(cmd) |
| if 'devices' not in extended or extended['devices'] is None: |
| extended['devices'] = () |
| for device in extended['devices']: |
| if 'vpci' in device: |
| vpci = "'" + device['vpci'] + "'" |
| else: |
| vpci = 'Null' |
| if 'image_id' in device: |
| image_id = "'" + device['image_id'] + "'" |
| else: |
| image_id = 'Null' |
| if 'xml' in device: |
| xml = "'" + device['xml'] + "'" |
| else: |
| xml = 'Null' |
| if 'dev' in device: |
| dev = "'" + device['dev'] + "'" |
| else: |
| dev = 'Null' |
| if 'image_size' in device: |
| size = device['image_size'] |
| else: |
| size = 0 |
| cmd = "INSERT INTO instance_devices (type,instance_id,image_id,vpci,xml,dev,image_size) " \ |
| "VALUES ('%s','%s', %s, %s, %s, %s, %s)" % \ |
| (device['type'], uuid, image_id, vpci, xml, dev, str(size)) |
| self.logger.debug(cmd) |
| self.cur.execute(cmd) |
| # #inserting new log |
| # cmd = "INSERT INTO logs (related,level,uuid,description) VALUES ('instances','debug','%s', |
| # 'new instance: %d numas, %d theads, %d ifaces %d bridge_ifaces')" % |
| # (uuid, nb_numas, nb_cores, nb_ifaces, nb_bridge_ifaces) |
| # self.logger.debug(cmd) |
| # self.cur.execute(cmd) |
| # |
| # inseted ok |
| return 1, uuid |
| except (mdb.Error, AttributeError) as e: |
| r, c = self.format_error(e, "new_instance", cmd) |
| if r != -HTTP_Request_Timeout or retry_ == 1: |
| return r, c |
| |
| @staticmethod |
| def get_free_ip_from_range(first_ip, last_ip, cidr, ip_used_list): |
| """ |
| Calculate a free IP from a range given |
| :param first_ip: First dhcp ip range |
| :param last_ip: Last dhcp ip range |
| :param cidr: net cidr |
| :param ip_used_list: contain all used ips to avoid ip collisions |
| :return: |
| """ |
| ip_tools = IPNetwork(cidr) |
| cidr_len = ip_tools.prefixlen |
| ips = IPNetwork(first_ip + '/' + str(cidr_len)) |
| |
| ip_used_list.append(str(ips[1])) # gw ip |
| ip_used_list.append(str(ips[-1])) # broadcast ip |
| ip_used_list.append(first_ip) |
| |
| for vm_ip in ips: |
| if str(vm_ip) not in ip_used_list and IPAddress(first_ip) <= IPAddress(vm_ip) <= IPAddress(last_ip): |
| return vm_ip |
| |
| return None |
| |
| def _get_dhcp_ip_used_list(self, net_id): |
| """ |
| REtreive from DB all ips already used by the dhcp server for a given net |
| :param net_id: |
| :return: |
| """ |
| WHERE = {'type': 'instance:ovs', 'net_id': net_id} |
| for retry_ in range(0, 2): |
| cmd = "" |
| self.cur = self.con.cursor(mdb.cursors.DictCursor) |
| select_ = "SELECT uuid, ip_address FROM ports " |
| |
| if WHERE is None or len(WHERE) == 0: |
| where_ = "" |
| else: |
| where_ = "WHERE " + " AND ".join( |
| map(lambda x: str(x) + (" is Null" if WHERE[x] is None else "='" + str(WHERE[x]) + "'"), |
| WHERE.keys())) |
| limit_ = "LIMIT 100" |
| cmd = " ".join((select_, where_, limit_)) |
| self.logger.debug(cmd) |
| self.cur.execute(cmd) |
| ports = self.cur.fetchall() |
| ip_address_list = [] |
| for port in ports: |
| ip_address_list.append(port['ip_address']) |
| |
| return ip_address_list |
| |
| def delete_instance(self, instance_id, tenant_id, net_dataplane_list, ports_to_free, net_ovs_list, |
| logcause="requested by http"): |
| for retry_ in range(0, 2): |
| cmd = "" |
| try: |
| with self.lock, self.con: |
| self.cur = self.con.cursor() |
| # get INSTANCE |
| cmd = "SELECT uuid FROM instances WHERE uuid='%s' AND tenant_id='%s'" % (instance_id, tenant_id) |
| self.logger.debug(cmd) |
| self.cur.execute(cmd) |
| if self.cur.rowcount == 0: |
| return 0, "instance %s not found in tenant %s" % (instance_id, tenant_id) |
| |
| # delete bridged ifaces, instace_devices, resources_mem; done by database: it is automatic by |
| # Database; FOREIGN KEY DELETE CASCADE |
| |
| # get nets afected |
| cmd = "SELECT DISTINCT net_id from ports WHERE instance_id = '%s' AND net_id is not Null AND " \ |
| "type='instance:data'" % instance_id |
| self.logger.debug(cmd) |
| self.cur.execute(cmd) |
| net_list__ = self.cur.fetchall() |
| for net in net_list__: |
| net_dataplane_list.append(net[0]) |
| |
| # get ovs manangement nets |
| cmd = "SELECT DISTINCT net_id, vlan, ip_address, mac FROM ports WHERE instance_id='{}' AND " \ |
| "net_id is not Null AND type='instance:ovs'".format(instance_id) |
| self.logger.debug(cmd) |
| self.cur.execute(cmd) |
| net_ovs_list += self.cur.fetchall() |
| |
| # get dataplane interfaces releases by this VM; both PF and VF with no other VF |
| cmd = "SELECT source_name, mac FROM (SELECT root_id, count(instance_id) as used " \ |
| "FROM resources_port WHERE instance_id='%s' GROUP BY root_id ) AS A" % instance_id \ |
| + " JOIN (SELECT root_id, count(instance_id) as used FROM resources_port GROUP BY root_id) " \ |
| "AS B ON A.root_id=B.root_id AND A.used=B.used JOIN resources_port as C ON A.root_id=C.id" |
| # cmd = "SELECT DISTINCT root_id FROM resources_port WHERE instance_id = '%s'" % instance_id |
| self.logger.debug(cmd) |
| self.cur.execute(cmd) |
| ports_to_free += self.cur.fetchall() |
| |
| # update resources port |
| cmd = "UPDATE resources_port SET instance_id=Null, port_id=Null, Mbps_used='0' " \ |
| "WHERE instance_id = '%s'" % instance_id |
| self.logger.debug(cmd) |
| self.cur.execute(cmd) |
| |
| # #filter dataplane ports used by this VM that now are free |
| # for port in ports_list__: |
| # cmd = "SELECT mac, count(instance_id) FROM resources_port WHERE root_id = '%s'" % port[0] |
| # self.logger.debug(cmd) |
| # self.cur.execute(cmd) |
| # mac_list__ = self.cur.fetchone() |
| # if mac_list__ and mac_list__[1]==0: |
| # ports_to_free.append(mac_list__[0]) |
| |
| # update resources core |
| cmd = "UPDATE resources_core SET instance_id=Null, v_thread_id=Null, paired='N' " \ |
| "WHERE instance_id = '%s'" % instance_id |
| self.logger.debug(cmd) |
| self.cur.execute(cmd) |
| |
| # delete all related uuids |
| cmd = "DELETE FROM uuids WHERE root_uuid='%s'" % instance_id |
| self.logger.debug(cmd) |
| self.cur.execute(cmd) |
| |
| # #insert log |
| # cmd = "INSERT INTO logs (related,level,uuid,description) VALUES |
| # ('instances','debug','%s','delete instance %s')" % (instance_id, logcause) |
| # self.logger.debug(cmd) |
| # self.cur.execute(cmd) |
| |
| # delete instance |
| cmd = "DELETE FROM instances WHERE uuid='%s' AND tenant_id='%s'" % (instance_id, tenant_id) |
| self.cur.execute(cmd) |
| return 1, "instance %s from tenant %s DELETED" % (instance_id, tenant_id) |
| |
| except (mdb.Error, AttributeError) as e: |
| r, c = self.format_error(e, "delete_instance", cmd) |
| if r != -HTTP_Request_Timeout or retry_ == 1: |
| return r, c |
| |
| def get_ports(self, WHERE): |
| """ Obtain ports using the WHERE filtering. |
| Attributes: |
| 'where_': dict of key:values, translated to key=value AND ... (Optional) |
| Return: a list with dictionarys at each row |
| """ |
| for retry_ in range(0, 2): |
| cmd = "" |
| try: |
| with self.lock, self.con: |
| |
| self.cur = self.con.cursor(mdb.cursors.DictCursor) |
| select_ = "SELECT uuid,'ACTIVE' as status,admin_state_up,name,net_id,\ |
| tenant_id,type,mac,vlan,switch_port,instance_id,Mbps FROM ports " |
| |
| if WHERE is None or len(WHERE) == 0: |
| where_ = "" |
| else: |
| where_ = "WHERE " + " AND ".join( |
| map(lambda x: str(x) + (" is Null" if WHERE[x] is None else "='" + str(WHERE[x]) + "'"), |
| WHERE.keys())) |
| limit_ = "LIMIT 100" |
| cmd = " ".join((select_, where_, limit_)) |
| # print "SELECT multiple de instance_ifaces, iface_uuid, external_ports" #print cmd |
| self.logger.debug(cmd) |
| self.cur.execute(cmd) |
| ports = self.cur.fetchall() |
| if self.cur.rowcount > 0: |
| af.DeleteNone(ports) |
| return self.cur.rowcount, ports |
| # return self.get_table(FROM=from_, SELECT=select_,WHERE=where_,LIMIT=100) |
| except (mdb.Error, AttributeError) as e: |
| r, c = self.format_error(e, "get_ports", cmd) |
| if r != -HTTP_Request_Timeout or retry_ == 1: |
| return r, c |
| |
| def check_target_net(self, net_id, tenant_id, port_type): |
| """check if valid attachement of a port into a target net |
| Attributes: |
| net_id: target net uuid |
| tenant_id: client where tenant belongs. Not used in this version |
| port_type: string with the option 'instance:bridge', 'instance:data', 'external' |
| Return: |
| (0,net_dict) if ok, where net_dict contain 'uuid','type','vlan', ... |
| (negative,string-error) if error |
| """ |
| for retry_ in range(0, 2): |
| cmd = "" |
| try: |
| with self.lock, self.con: |
| self.cur = self.con.cursor(mdb.cursors.DictCursor) |
| cmd = "SELECT * FROM nets WHERE uuid='%s'" % net_id |
| self.logger.debug(cmd) |
| self.cur.execute(cmd) |
| if self.cur.rowcount == 0: |
| return -1, "network_id %s does not match any net" % net_id |
| net = self.cur.fetchone() |
| break |
| |
| except (mdb.Error, AttributeError) as e: |
| r, c = self.format_error(e, "check_target_net", cmd) |
| if r != -HTTP_Request_Timeout or retry_ == 1: |
| return r, c |
| # check permissions |
| if tenant_id is not None and tenant_id is not "admin": |
| if net['tenant_id'] == tenant_id and net['shared'] == 'false': |
| return -1, "needed admin privileges to attach to the net %s" % net_id |
| # check types |
| if (net['type'] in ('ptp', 'data') and port_type not in ('instance:data', 'external')) or \ |
| (net['type'] in ('bridge_data', 'bridge_man') and port_type not in ('instance:bridge', 'instance:ovs')): |
| return -1, "Cannot attach a port of type %s into a net of type %s" % (port_type, net['type']) |
| if net['type'] == 'ptp': |
| # look how many |
| nb_ports, data = self.get_ports({'net_id': net_id}) |
| if nb_ports < 0: |
| return -1, data |
| else: |
| if net['provider']: |
| nb_ports += 1 |
| if nb_ports >= 2: |
| return -1, "net of type p2p already contain two ports attached. No room for another" |
| |
| return 0, net |
| |
| |
| if __name__ == "__main__": |
| print("Hello World") |