X-Git-Url: https://osm.etsi.org/gitweb/?a=blobdiff_plain;f=osm_openvim%2Fovim.py;h=1177d424e1e7ee483073034e5045d3c688f7a419;hb=refs%2Fchanges%2F38%2F5738%2F1;hp=a0c380d3388421caf1fa9f9c0edfe6f666f77f76;hpb=51068956dc168aa1d99ccbb5e1f8247580b8be2e;p=osm%2Fopenvim.git diff --git a/osm_openvim/ovim.py b/osm_openvim/ovim.py index a0c380d..1177d42 100755 --- a/osm_openvim/ovim.py +++ b/osm_openvim/ovim.py @@ -28,9 +28,11 @@ Two thread will be launched, with normal and administrative permissions. """ import threading +import yaml import vim_db import logging # import imp +import os.path import argparse from netaddr import IPNetwork from jsonschema import validate as js_v, exceptions as js_e @@ -41,9 +43,9 @@ import openflow_conn __author__ = "Alfonso Tierno, Leonardo Mirabal" __date__ = "$06-Feb-2017 12:07:15$" -__version__ = "0.5.11-r527" -version_date = "Apr 2017" -database_version = "0.17" #expected database schema version +__version__ = "0.5.22-r538" +version_date = "Nov 2017" +database_version = 22 #needed database schema version HTTP_Bad_Request = 400 HTTP_Unauthorized = 401 @@ -99,8 +101,6 @@ class ovim(): self.logger = logging.getLogger(self.logger_name) self.db = None self.db = self._create_database_connection() - self.db_lock = None - self.db_of = None self.of_test_mode = False def _create_database_connection(self): @@ -143,7 +143,7 @@ class ovim(): ips = IPNetwork(cidr) if "dhcp_first_ip" not in network: - network["dhcp_first_ip"] = str(ips[2]) + network["dhcp_first_ip"] = str(ips[3]) if "dhcp_last_ip" not in network: network["dhcp_last_ip"] = str(ips[-2]) if "gateway_ip" not in network: @@ -171,21 +171,61 @@ class ovim(): # if self.running_info: # return #TODO service can be checked and rebuild broken threads r = self.db.get_db_version() + db_path = __file__ + db_path = db_path[:db_path.rfind("/")] + if os.path.exists(db_path + "/database_utils/migrate_vim_db.sh"): + db_path += "/database_utils" + else: + db_path += "/../database_utils" + if r[0] < 0: - raise ovimException("DATABASE is not a VIM one or it is a '0.0' version. Try to upgrade to version '{}' with "\ - "'./database_utils/migrate_vim_db.sh'".format(database_version) ) - elif r[1] != database_version: - raise ovimException("DATABASE wrong version '{}'. Try to upgrade/downgrade to version '{}' with "\ - "'./database_utils/migrate_vim_db.sh'".format(r[1], database_version) ) + raise ovimException("DATABASE is not valid. If you think it is corrupted, you can init it with" + " '{db_path}/init_vim_db.sh' script".format(db_path=db_path)) + elif r[0] != database_version: + raise ovimException("DATABASE wrong version '{current}'. Try to upgrade/downgrade to version '{target}'" + " with '{db_path}/migrate_vim_db.sh {target}'".format( + current=r[0], target=database_version, db_path=db_path)) self.logger.critical("Starting ovim server version: '{} {}' database version '{}'".format( self.get_version(), self.get_version_date(), self.get_database_version())) # create database connection for openflow threads - self.db_of = self._create_database_connection() - self.config["db"] = self.db_of - self.db_lock = threading.Lock() - self.config["db_lock"] = self.db_lock + self.config["db"] = self._create_database_connection() + self.config["db_lock"] = threading.Lock() self.of_test_mode = False if self.config['mode'] == 'normal' or self.config['mode'] == "OF only" else True + + # Create one thread for each host + host_test_mode = True if self.config['mode'] == 'test' or self.config['mode'] == "OF only" else False + host_develop_mode = True if self.config['mode'] == 'development' else False + host_develop_bridge_iface = self.config.get('development_bridge', None) + + # get host list from data base before starting threads + r, hosts = self.db.get_table(SELECT=('name', 'ip_name', 'user', 'uuid', 'password', 'keyfile'), + FROM='hosts', WHERE={'status': 'ok'}) + if r < 0: + raise ovimException("Cannot get hosts from database {}".format(hosts)) + + self.config['host_threads'] = {} + + for host in hosts: + thread = ht.host_thread(name=host['name'], user=host['user'], host=host['ip_name'], db=self.config["db"], + password=host['password'], + keyfile=host.get('keyfile', self.config["host_ssh_keyfile"]), + db_lock=self.config["db_lock"], test=host_test_mode, + image_path=self.config['host_image_path'], + version=self.config['version'], host_id=host['uuid'], + develop_mode=host_develop_mode, + develop_bridge_iface=host_develop_bridge_iface, + logger_name=self.logger_name + ".host." + host['name'], + debug=self.config.get('log_level_host')) + + try: + thread.check_connectivity() + except Exception as e: + self.logger.critical('Error detected for compute = {} with ip = {}' + .format(host['name'], host['ip_name'])) + thread.start() + self.config['host_threads'][host['uuid']] = thread + # precreate interfaces; [bridge:, VLAN used at Host, uuid of network camping in this bridge, # speed in Gbit/s @@ -227,32 +267,13 @@ class ovim(): dhcp_params = self.config.get("dhcp_server") if dhcp_params: thread = dt.dhcp_thread(dhcp_params=dhcp_params, test=host_test_mode, dhcp_nets=self.config["dhcp_nets"], - db=self.db_of, db_lock=self.db_lock, logger_name=self.logger_name + ".dhcp", + db=self.config["db"], db_lock=self.config["db_lock"], + logger_name=self.logger_name + ".dhcp", debug=self.config.get('log_level_of')) thread.start() self.config['dhcp_thread'] = thread - # Create one thread for each host - host_test_mode = True if self.config['mode'] == 'test' or self.config['mode'] == "OF only" else False - host_develop_mode = True if self.config['mode'] == 'development' else False - host_develop_bridge_iface = self.config.get('development_bridge', None) - # get host list from data base before starting threads - r, hosts = self.db.get_table(SELECT=('name', 'ip_name', 'user', 'uuid'), FROM='hosts', WHERE={'status': 'ok'}) - if r < 0: - raise ovimException("Cannot get hosts from database {}".format(hosts)) - - self.config['host_threads'] = {} - for host in hosts: - host['image_path'] = '/opt/VNF/images/openvim' - thread = ht.host_thread(name=host['name'], user=host['user'], host=host['ip_name'], db=self.db_of, - db_lock=self.db_lock, test=host_test_mode, image_path=self.config['image_path'], - version=self.config['version'], host_id=host['uuid'], develop_mode=host_develop_mode, - develop_bridge_iface=host_develop_bridge_iface, - logger_name=self.logger_name + ".host." + host['name'], - debug=self.config.get('log_level_host')) - thread.start() - self.config['host_threads'][host['uuid']] = thread # create ovs dhcp thread result, content = self.db.get_table(FROM='nets') @@ -262,13 +283,42 @@ class ovim(): for net in content: net_type = net['type'] - if (net_type == 'bridge_data' or net_type == 'bridge_man') \ - and net["provider"][:4] == 'OVS:' and net["enable_dhcp"] == "true": - self.launch_dhcp_server(net['vlan'], - net['dhcp_first_ip'], - net['dhcp_last_ip'], - net['cidr'], - net['gateway_ip']) + if net['status'] != "INACTIVE" and (net_type == 'bridge_data' or net_type == 'bridge_man') and \ + net["provider"][:4] == 'OVS:' and net["enable_dhcp"] == "true": + try: + config_routes = net.get('routes') + if config_routes: + routes = yaml.safe_load(config_routes) + else: + routes = None + + config_dns = net.get('dns') + if config_dns: + dns = yaml.safe_load(config_dns) + else: + dns = None + + links = net.get('links') + if links: + links = yaml.safe_load(net.get('links')) + if net.get('enable_dhcp'): + self.launch_dhcp_server(net.get('vlan'), + net.get('dhcp_first_ip'), + net.get('dhcp_last_ip'), + net.get('cidr'), + net.get('gateway_ip'), + dns, + routes) + self.launch_link_bridge_to_ovs(net['vlan'], net.get('gateway_ip'), net.get('cidr'), links, routes) + if net["status"] == "ERROR": + self.db.update_rows("nets", UPDATE={"status": "ACTIVE", "last_error": None}, + WHERE={"uuid": net["uuid"]}) + except Exception as e: + self.logger.error("Fail at launching dhcp server for net_id='%s' net_name='%s': %s", + net["uuid"], net["name"], str(e)) + self.db.update_rows("nets", UPDATE={"status": "ERROR", + "last_error": "Fail at launching dhcp server: " + str(e)}, + WHERE={"uuid": net["uuid"]}) def _start_of_db_tasks(self): """ @@ -370,19 +420,12 @@ class ovim(): module = temp_dict['of_controller'] if module not in ovim.of_module: - for base in ("", "osm_openvim.", "lib_osm_openvim."): - try: - pkg = __import__(base + module) - if base: - of_conn_module = getattr(pkg, module) - else: - of_conn_module = pkg - ovim.of_module[module] = of_conn_module - self.logger.debug("Module load from {}".format(base + module)) - break - except Exception as e: - self.logger.warning("Module {} not found {}".format(base + module, e)) - else: + try: + pkg = __import__("osm_openvim." + module) + of_conn_module = getattr(pkg, module) + ovim.of_module[module] = of_conn_module + self.logger.debug("Module load from {}".format("osm_openvim." + module)) + except Exception as e: self.logger.error("Cannot open openflow controller module of type '%s'", module) raise ovimException("Cannot open openflow controller of type module '{}'" "Revise it is installed".format(module), @@ -408,8 +451,11 @@ class ovim(): # ofc_net_same_vlan = False ofc_net_same_vlan = False - thread = oft.openflow_thread(ofc_uuid, of_conn, of_test=self.of_test_mode, db=self.db_of, db_lock=self.db_lock, - pmp_with_same_vlan=ofc_net_same_vlan, debug=self.config['log_level_of']) + thread = oft.openflow_thread(ofc_uuid, of_conn, of_test=self.of_test_mode, db=self.config["db"], + db_lock=self.config["db_lock"], + pmp_with_same_vlan=ofc_net_same_vlan, + logger_name=self.logger_name + ".ofc." + ofc_uuid, + debug=self.config.get('log_level_of')) #r, c = thread.OF_connector.obtain_port_correspondence() #if r < 0: # raise ovimException("Cannot get openflow information %s", c) @@ -428,9 +474,13 @@ class ovim(): if 'dhcp_thread' in self.config: threads['dhcp'] = (self.config['dhcp_thread']) - for thread in threads.values(): + for thread_id, thread in threads.items(): + if thread_id == 'openvim_controller': + continue thread.insert_task("exit") - for thread in threads.values(): + for thread_id, thread in threads.items(): + if thread_id == 'openvim_controller': + continue thread.join() def get_networks(self, columns=None, db_filter={}, limit=None): @@ -498,6 +548,7 @@ class ovim(): net_vlan = network.get("vlan") net_bind_net = network.get("bind_net") net_bind_type = network.get("bind_type") + net_region = network.get("region") name = network["name"] # check if network name ends with : and network exist in order to make and automated bindning @@ -559,7 +610,7 @@ class ovim(): bridge_net_name = net_provider[7:] for brnet in self.config['bridge_nets']: if brnet[0] == bridge_net_name: # free - if not brnet[3]: + if brnet[3]: raise ovimException("invalid 'provider:physical', " "bridge '%s' is already used" % bridge_net_name, HTTP_Conflict) bridge_net = brnet @@ -593,8 +644,13 @@ class ovim(): net_vlan = bridge_net[1] elif net_type == 'bridge_data' or net_type == 'bridge_man' and self.config['network_type'] == 'ovs': net_provider = 'OVS' + if not net_region: + if net_type == "data" or net_type == "ptp": + net_region = "__DATA__" + elif net_provider == "OVS": + net_region = "__OVS__" if not net_vlan and (net_type == "data" or net_type == "ptp" or net_provider == "OVS"): - net_vlan = self.db.get_free_net_vlan() + net_vlan = self.db.get_free_net_vlan(net_region) if net_vlan < 0: raise ovimException("Error getting an available vlan", HTTP_Internal_Server_Error) if net_provider == 'OVS': @@ -603,25 +659,33 @@ class ovim(): network['provider'] = net_provider network['type'] = net_type network['vlan'] = net_vlan + network['region'] = net_region dhcp_integrity = True - if 'enable_dhcp' in network and network['enable_dhcp']: + if network.get('enable_dhcp'): dhcp_integrity = self._check_dhcp_data_integrity(network) + + if network.get('links'): + network['links'] = yaml.safe_dump(network['links'], default_flow_style=True, width=256) + if network.get('dns'): + network['dns'] = yaml.safe_dump(network['dns'], default_flow_style=True, width=256) + if network.get('routes'): + network['routes'] = yaml.safe_dump(network['routes'], default_flow_style=True, width=256) result, content = self.db.new_row('nets', network, True, True) - - if result >= 0 and dhcp_integrity: + if result >= 0: # and dhcp_integrity: if bridge_net: bridge_net[3] = content if self.config.get("dhcp_server") and self.config['network_type'] == 'bridge': if network["name"] in self.config["dhcp_server"].get("nets", ()): self.config["dhcp_nets"].append(content) self.logger.debug("dhcp_server: add new net", content) - elif not bridge_net and bridge_net[0] in self.config["dhcp_server"].get("bridge_ifaces", ()): + elif bridge_net and bridge_net[0] in self.config["dhcp_server"].get("bridge_ifaces", ()): self.config["dhcp_nets"].append(content) self.logger.debug("dhcp_server: add new net", content, content) return content else: - raise ovimException("Error posting network", HTTP_Internal_Server_Error) + raise ovimException("Error creating network: {}".format(content), -result) + # TODO kei change update->edit def edit_network(self, network_id, network): @@ -721,6 +785,7 @@ class ovim(): :param network_id: network id :return: """ + net_data = self.show_network(network_id) # delete from the data base result, content = self.db.delete_row('nets', network_id) @@ -734,9 +799,20 @@ class ovim(): break if self.config.get("dhcp_server") and network_id in self.config["dhcp_nets"]: self.config["dhcp_nets"].remove(network_id) - return content + + if net_data.get('enable_dhcp'): + dhcp_path = self.config['ovs_controller_file_path'] + dhcp_controller = self.get_dhcp_controller() + dhcp_controller.delete_dhcp_server(net_data['vlan'], network_id, dhcp_path) + dhcp_controller.delete_dhcp_port(net_data['vlan'], network_id, dhcp_path) + links = yaml.load(net_data.get('links')) + if links: + links = yaml.load(net_data.get('links')) + self.delete_link_bridge_to_ovs(net_data['vlan'], links) + + return content else: - raise ovimException("Error deleting network %s" % network_id, HTTP_Internal_Server_Error) + raise ovimException("Error deleting network '{}': {}".format(network_id, content), -result) def get_openflow_rules(self, network_id=None): """ @@ -825,7 +901,7 @@ class ovim(): else: raise ovimException("Default Openflow controller not not running", HTTP_Not_Found) - if ofc_id in self.config['ofcs_thread']: + elif ofc_id in self.config['ofcs_thread']: conn = self.config['ofcs_thread'][ofc_id].OF_connector else: raise ovimException("Openflow controller not found with ofc_id={}".format(ofc_id), HTTP_Not_Found) @@ -1286,6 +1362,8 @@ class ovim(): map['switch_dpid'] = switch_dpid if region: map['region'] = region + if map.get("pci"): + map["pci"] = map["pci"].lower() for of_map in of_maps: result, uuid = self.db.new_row('of_port_mappings', of_map, True) @@ -1335,25 +1413,30 @@ class ovim(): bridge_ifaces = [] controller_ip = self.config['ovs_controller_ip'] - ovs_controller_user = self.config['ovs_controller_user'] + ovs_controller_user = self.config.get('ovs_controller_user') host_test_mode = True if self.config['mode'] == 'test' or self.config['mode'] == "OF only" else False host_develop_mode = True if self.config['mode'] == 'development' else False dhcp_host = ht.host_thread(name='openvim_controller', user=ovs_controller_user, host=controller_ip, - db=self.db_of, - db_lock=self.db_lock, test=host_test_mode, - image_path=self.config['image_path'], version=self.config['version'], + password=self.config.get('ovs_controller_password'), + keyfile=self.config.get('ovs_controller_keyfile'), + db=self.config["db"], db_lock=self.config["db_lock"], test=host_test_mode, + image_path=self.config['host_image_path'], version=self.config['version'], host_id='openvim_controller', develop_mode=host_develop_mode, - develop_bridge_iface=bridge_ifaces, logger_name=self.logger_name + ".host.controller", + develop_bridge_iface=bridge_ifaces, + logger_name=self.logger_name + ".host.controller", debug=self.config.get('log_level_host')) - + # dhcp_host.start() self.config['host_threads']['openvim_controller'] = dhcp_host - if not host_test_mode: - dhcp_host.ssh_connect() + try: + dhcp_host.check_connectivity() + except Exception as e: + pass + return dhcp_host - def launch_dhcp_server(self, vlan, first_ip, last_ip, cidr, gateway): + def launch_dhcp_server(self, vlan, first_ip, last_ip, cidr, gateway, dns, routes): """ Launch a dhcpserver base on dnsmasq attached to the net base on vlan id across the the openvim computes :param vlan: vlan identifier @@ -1370,9 +1453,49 @@ class ovim(): dhcp_path = self.config['ovs_controller_file_path'] controller_host = self.get_dhcp_controller() - controller_host.create_linux_bridge(vlan) + + # controller_host.create_linux_bridge(vlan) controller_host.create_dhcp_interfaces(vlan, first_ip, dhcp_netmask) - controller_host.launch_dhcp_server(vlan, ip_range, dhcp_netmask, dhcp_path, gateway) + dhcp_path = self.config['ovs_controller_file_path'] + controller_host.launch_dhcp_server(vlan, ip_range, dhcp_netmask, dhcp_path, gateway, dns, routes) + + def launch_link_bridge_to_ovs(self, vlan, gateway, dhcp_cidr, links=None, routes=None): + """ + Launch creating of connections (veth) between user bridge (link) and OVS + :param vlan: + :param gateway: + :param links: + :return: + """ + + if links: + controller_host = self.get_dhcp_controller() + for link in links: + if 'iface' in link and 'nat' not in link: + controller_host.create_link_bridge_to_ovs(vlan, link['iface']) + elif 'nat' in link: + controller_host.create_qrouter_ovs_connection(vlan, gateway, dhcp_cidr) + controller_host.create_qrouter_br_connection(vlan, dhcp_cidr, link) + + if len(routes): + controller_host.add_ns_routes(vlan, routes) + + def delete_link_bridge_to_ovs(self, vlan, links=None): + """ + Delete connections (veth) between user bridge (link) and OVS + :param vlan: + :param links: + :return: + """ + if links: + controller_host = self.get_dhcp_controller() + + for link in links: + if 'iface' in link and 'nat' not in link: + controller_host.remove_link_bridge_to_ovs(vlan, link['iface']) + elif 'nat' in link: + controller_host.delete_qrouter_connection(vlan, link['iface']) + if __name__ == "__main__":