X-Git-Url: https://osm.etsi.org/gitweb/?a=blobdiff_plain;f=osm_openvim%2Fhttpserver.py;h=3de94094768812e15bfa0a7d7f7a43aec4ff5b1a;hb=refs%2Fchanges%2F50%2F5850%2F4;hp=edf1e8d9183e45ce8360a34ba43900bf091dd529;hpb=9f6571090b203922cabb0382226be0fa48d6e046;p=osm%2Fopenvim.git diff --git a/osm_openvim/httpserver.py b/osm_openvim/httpserver.py index edf1e8d..3de9409 100644 --- a/osm_openvim/httpserver.py +++ b/osm_openvim/httpserver.py @@ -38,6 +38,7 @@ import datetime import hashlib import os import imp +import socket from netaddr import IPNetwork, IPAddress, all_matching_cidrs #import only if needed because not needed in test mode. To allow an easier installation import RADclass from jsonschema import validate as js_v, exceptions as js_e @@ -55,20 +56,20 @@ global my global url_base global config_dic global RADclass_module -RADclass=None #RADclass module is charged only if not in test mode +RADclass_module=None #RADclass module is charged only if not in test mode url_base="/openvim" HTTP_Bad_Request = 400 -HTTP_Unauthorized = 401 -HTTP_Not_Found = 404 +HTTP_Unauthorized = 401 +HTTP_Not_Found = 404 HTTP_Forbidden = 403 -HTTP_Method_Not_Allowed = 405 +HTTP_Method_Not_Allowed = 405 HTTP_Not_Acceptable = 406 HTTP_Request_Timeout = 408 HTTP_Conflict = 409 -HTTP_Service_Unavailable = 503 -HTTP_Internal_Server_Error= 500 +HTTP_Service_Unavailable = 503 +HTTP_Internal_Server_Error= 500 def md5(fname): hash_md5 = hashlib.md5() @@ -150,22 +151,32 @@ def check_extended(extended, allow_net_attach=False): http2db_id={'id':'uuid'} http2db_host={'id':'uuid'} http2db_tenant={'id':'uuid'} -http2db_flavor={'id':'uuid','imageRef':'image_id'} +http2db_flavor={'id':'uuid','imageRef':'image_id', 'size': 'image_size'} http2db_image={'id':'uuid', 'created':'created_at', 'updated':'modified_at', 'public': 'public'} -http2db_server={'id':'uuid','hostId':'host_id','flavorRef':'flavor_id','imageRef':'image_id','created':'created_at'} +http2db_server={'id':'uuid','hostId':'host_id','flavorRef':'flavor_id','osImageType':'os_image_type','imageRef':'image_id','created':'created_at'} #Unikernels extension http2db_network={'id':'uuid','provider:vlan':'vlan', 'provider:physical': 'provider'} http2db_ofc = {'id': 'uuid'} http2db_port={'id':'uuid', 'network_id':'net_id', 'mac_address':'mac', 'device_owner':'type','device_id':'instance_id','binding:switch_port':'switch_port','binding:vlan':'vlan', 'bandwidth':'Mbps'} + def remove_extra_items(data, schema): + import re + deleted=[] if type(data) is tuple or type(data) is list: for d in data: a= remove_extra_items(d, schema['items']) if a is not None: deleted.append(a) elif type(data) is dict: + for k in data.keys(): - if 'properties' not in schema or k not in schema['properties'].keys(): + if 'patternProperties' in schema and k not in schema['properties'].keys(): + reg_ex_list = schema['patternProperties'].keys() + for reg_ex in reg_ex_list: + if not re.match(reg_ex, k): + del data[k] + deleted.append(k) + elif 'properties' not in schema or k not in schema['properties'].keys(): # or k not in schema['patternProperties'].keys(): del data[k] deleted.append(k) else: @@ -174,7 +185,8 @@ def remove_extra_items(data, schema): if len(deleted) == 0: return None elif len(deleted) == 1: return deleted[0] else: return deleted - + + def delete_nulls(var): if type(var) is dict: for k in var.keys(): @@ -501,7 +513,7 @@ def http_get_hosts(): def get_hosts(): select_, where_, limit_ = filter_query_string(bottle.request.query, http2db_host, - ('id', 'name', 'description', 'status', 'admin_state_up', 'ip_name')) + ('id', 'name', 'description', 'status', 'admin_state_up', 'ip_name', 'hypervisors')) #Unikernels extension myself = config_dic['http_threads'][ threading.current_thread().name ] result, content = myself.db.get_table(FROM='hosts', SELECT=select_, WHERE=where_, LIMIT=limit_) @@ -524,6 +536,7 @@ def http_get_host_id(host_id): @bottle.route(url_base + '/hosts', method='POST') def http_post_hosts(): '''insert a host into the database. All resources are got and inserted''' + global RADclass_module my = config_dic['http_threads'][ threading.current_thread().name ] #check permissions if not my.admin: @@ -535,17 +548,17 @@ def http_post_hosts(): if r is not None: print "http_post_host_id: Warning: remove extra items ", r change_keys_http2db(http_content['host'], http2db_host) - host = http_content['host'] - warning_text="" - if 'host-data' in http_content: - host.update(http_content['host-data']) - ip_name=http_content['host-data']['ip_name'] - user=http_content['host-data']['user'] - password=http_content['host-data'].get('password', None) + if 'host' in http_content: + host = http_content['host'] + if 'host-data' in http_content: + host.update(http_content['host-data']) else: - ip_name=host['ip_name'] - user=host['user'] - password=host.get('password', None) + host = http_content['host-data'] + warning_text = "" + ip_name = host['ip_name'] + user = host['user'] + password = host.get('password') + if host.get('autodiscover'): if not RADclass_module: try: RADclass_module = imp.find_module("RADclass") @@ -618,10 +631,18 @@ def http_post_hosts(): memory=node['memory']['node_size'] / (1024*1024*1024) #memory=get_next_2pow(node['memory']['hugepage_nr']) host['numas'].append( {'numa_socket': node['id'], 'hugepages': node['memory']['hugepage_nr'], 'memory':memory, 'interfaces': interfaces, 'cores': cores } ) - print json.dumps(host, indent=4) - #return - # - #insert in data base + # print json.dumps(host, indent=4) + # insert in data base + if "created_at" in host: + del host["created_at"] + for numa in host.get("numas", ()): + if "hugepages_consumed" in numa: + del numa["hugepages_consumed"] + for core in numa.get("cores", ()): + if "instance_id" in core: + del core["instance_id"] + if "v_thread_id" in core: + del core["v_thread_id"] result, content = my.db.new_host(host) if result >= 0: if content['admin_state_up']: @@ -629,21 +650,26 @@ def http_post_hosts(): host_test_mode = True if config_dic['mode']=='test' or config_dic['mode']=="OF only" else False host_develop_mode = True if config_dic['mode']=='development' else False host_develop_bridge_iface = config_dic.get('development_bridge', None) - thread = ht.host_thread(name=host.get('name',ip_name), user=user, host=ip_name, db=config_dic['db'], db_lock=config_dic['db_lock'], - test=host_test_mode, image_path=config_dic['image_path'], - version=config_dic['version'], host_id=content['uuid'], - develop_mode=host_develop_mode, develop_bridge_iface=host_develop_bridge_iface ) + thread = ht.host_thread(name=host.get('name',ip_name), user=user, host=ip_name, + password=host.get('password'), + keyfile=host.get('keyfile', config_dic["host_ssh_keyfile"]), + db=config_dic['db'], db_lock=config_dic['db_lock'], + test=host_test_mode, image_path=config_dic['host_image_path'], + version=config_dic['version'], host_id=content['uuid'], + develop_mode=host_develop_mode, develop_bridge_iface=host_develop_bridge_iface, + hypervisors=host.get('hypervisors', None)) #Unikernels extension + thread.start() - config_dic['host_threads'][ content['uuid'] ] = thread + config_dic['host_threads'][content['uuid']] = thread if config_dic['network_type'] == 'ovs': # create bridge create_dhcp_ovs_bridge() config_dic['host_threads'][content['uuid']].insert_task("new-ovsbridge") - # check if more host exist - create_vxlan_mesh(content['uuid']) + # create vlxan bwt OVS controller and computes + create_vxlan_mesh(content['uuid'], my.logger) - #return host data + # return host data change_keys_http2db(content, http2db_host, reverse=True) if len(warning_text)>0: content["warning"]= warning_text @@ -666,8 +692,8 @@ def delete_dhcp_ovs_bridge(vlan, net_uuid): http_controller = config_dic['http_threads'][threading.current_thread().name] dhcp_controller = http_controller.ovim.get_dhcp_controller() - dhcp_controller.delete_dhcp_port(vlan, net_uuid) dhcp_controller.delete_dhcp_server(vlan, net_uuid, dhcp_path) + dhcp_controller.delete_dhcp_port(vlan, net_uuid, dhcp_path) def create_dhcp_ovs_bridge(): @@ -705,7 +731,7 @@ def set_mac_dhcp(vm_ip, vlan, first_ip, last_ip, cidr, mac): http_controller = config_dic['http_threads'][threading.current_thread().name] dhcp_controller = http_controller.ovim.get_dhcp_controller() - dhcp_controller.set_mac_dhcp_server(vm_ip, mac, vlan, dhcp_netmask, dhcp_path) + dhcp_controller.set_mac_dhcp_server(vm_ip, mac, vlan, dhcp_netmask, first_ip, dhcp_path) def delete_mac_dhcp(vm_ip, vlan, mac): @@ -725,12 +751,12 @@ def delete_mac_dhcp(vm_ip, vlan, mac): dhcp_controller.delete_mac_dhcp_server(vm_ip, mac, vlan, dhcp_path) -def create_vxlan_mesh(host_id): +def create_vxlan_mesh(host_id, logger=None): """ Create vxlan mesh across all openvimc controller and computes. - :param host_id: host identifier - :param host_id: host identifier - :return: + :param host_id: Added compute node id. Anyway vlan is created by all compute nodes + :param logger: To log errors + :return: None """ dhcp_compute_name = get_vxlan_interface("dhcp") existing_hosts = get_hosts() @@ -742,22 +768,27 @@ def create_vxlan_mesh(host_id): dhcp_controller = http_controller.ovim.get_dhcp_controller() for compute in computes_available: + try: + if compute['ip_name'] != 'localhost': + remote_ip = socket.gethostbyname(compute['ip_name']) + else: + remote_ip = 'localhost' + except socket.error as e: + if logger: + logger.error("Cannot get compute node remote ip from '{}'. Skipping: {}".format( + compute['ip_name'], e)) + continue + # vxlan ovs_controller <=> compute node vxlan_interface_name = get_vxlan_interface(compute['id'][:8]) config_dic['host_threads'][compute['id']].insert_task("new-vxlan", dhcp_compute_name, dhcp_controller.host) - dhcp_controller.create_ovs_vxlan_tunnel(vxlan_interface_name, compute['ip_name']) - - # vlxan mesh creation between openvim computes - for count, compute_owner in enumerate(computes_available): - for compute in computes_available: - if compute_owner['id'] == compute['id']: - pass - else: - vxlan_interface_name = get_vxlan_interface(compute_owner['id'][:8]) - dhcp_controller.create_ovs_vxlan_tunnel(vxlan_interface_name, compute_owner['ip_name']) - config_dic['host_threads'][compute['id']].insert_task("new-vxlan", - vxlan_interface_name, - compute_owner['ip_name']) - + dhcp_controller.create_ovs_vxlan_tunnel(vxlan_interface_name, remote_ip) + # vxlan from others compute node to cthis ompute node + for compute_src in computes_available: + if compute_src['id'] == compute['id']: + continue + config_dic['host_threads'][compute_src['id']].insert_task("new-vxlan", + vxlan_interface_name, + remote_ip) def delete_vxlan_mesh(host_id): """ @@ -828,7 +859,7 @@ def http_put_host_id(host_id): if config_dic['network_type'] == 'ovs': # create mesh with new host data config_dic['host_threads'][host_id].insert_task("new-ovsbridge") - create_vxlan_mesh(host_id) + create_vxlan_mesh(host_id, my.logger) #print data return format_out(data) @@ -1606,7 +1637,7 @@ def http_post_server_id(tenant_id): if net['type'] == 'instance:ovs': dhcp_nets_id.append(get_network_id(net['net_id'])) - ports_to_free=[] + ports_to_free = [] new_instance_result, new_instance = my.db.new_instance(content, nets, ports_to_free) if new_instance_result < 0: print "Error http_post_servers() :", new_instance_result, new_instance @@ -1615,6 +1646,8 @@ def http_post_server_id(tenant_id): print print "inserted at DB" print + + for port in ports_to_free: r,c = config_dic['host_threads'][ server['host_id'] ].insert_task( 'restore-iface',*port ) if r < 0: @@ -1637,21 +1670,36 @@ def http_post_server_id(tenant_id): print ':http_post_servers ERROR UPDATING dhcp_server !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!' + c #ensure compute contain the bridge for ovs networks: - server_net = get_network_id(iface['net_id']) - if server_net["network"].get('provider:physical', "")[:3] == 'OVS': - vlan = str(server_net['network']['provider:vlan']) - dhcp_enable = bool(server_net['network']['enable_dhcp']) - if dhcp_enable: - dhcp_firt_ip = str(server_net['network']['dhcp_first_ip']) - dhcp_last_ip = str(server_net['network']['dhcp_last_ip']) - dhcp_cidr = str(server_net['network']['cidr']) - gateway = str(server_net['network']['gateway']) + if iface.get("net_id"): + server_net = get_network_id(iface['net_id']) + if server_net["network"].get('provider:physical', "")[:3] == 'OVS': + vlan = str(server_net['network']['provider:vlan']) + dhcp_enable = bool(server_net['network']['enable_dhcp']) vm_dhcp_ip = c2[0]["ip_address"] config_dic['host_threads'][server['host_id']].insert_task("create-ovs-bridge-port", vlan) + dns = server_net['network'].get("dns") + if dns: + dns = yaml.safe_load(server_net['network'].get("dns")) + routes = server_net['network'].get("routes") + if routes: + routes = yaml.safe_load(server_net['network'].get("routes")) + links = server_net['network'].get("links") + if links: + links = yaml.safe_load(server_net['network'].get("links")) + if dhcp_enable: + dhcp_firt_ip = str(server_net['network']['dhcp_first_ip']) + dhcp_last_ip = str(server_net['network']['dhcp_last_ip']) + dhcp_cidr = str(server_net['network']['cidr']) + gateway = str(server_net['network']['gateway_ip']) + + http_controller = config_dic['http_threads'][threading.current_thread().name] + http_controller.ovim.launch_dhcp_server(vlan, dhcp_firt_ip, dhcp_last_ip, + dhcp_cidr, gateway, dns, routes) + set_mac_dhcp(vm_dhcp_ip, vlan, dhcp_firt_ip, dhcp_last_ip, dhcp_cidr, c2[0]['mac']) + + if links: + http_controller.ovim.launch_link_bridge_to_ovs(vlan, gateway, dhcp_cidr, links, routes) - set_mac_dhcp(vm_dhcp_ip, vlan, dhcp_firt_ip, dhcp_last_ip, dhcp_cidr, c2[0]['mac']) - http_controller = config_dic['http_threads'][threading.current_thread().name] - http_controller.ovim.launch_dhcp_server(vlan, dhcp_firt_ip, dhcp_last_ip, dhcp_cidr, gateway) #Start server server['uuid'] = new_instance @@ -1671,6 +1719,7 @@ def http_post_server_id(tenant_id): bottle.abort(HTTP_Bad_Request, content) return + def http_server_action(server_id, tenant_id, action): '''Perform actions over a server as resume, reboot, terminate, ...''' my = config_dic['http_threads'][ threading.current_thread().name ] @@ -1822,16 +1871,26 @@ def http_server_action(server_id, tenant_id, action): #print "dhcp insert del task" if r < 0: print ':http_post_servers ERROR UPDATING dhcp_server !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!' + c - # delete ovs-port and linux bridge, contains a list of tuple (net_id,vlan) + # delete ovs-port and linux bridge, contains a list of tuple (net_id,vlan, vm_ip, mac) + for net in net_ovs_list: mac = str(net[3]) vm_ip = str(net[2]) vlan = str(net[1]) net_id = net[0] + delete_dhcp_ovs_bridge(vlan, net_id) delete_mac_dhcp(vm_ip, vlan, mac) + + net_data = my.ovim.show_network(net_id) + if net_data.get('links'): + links = yaml.load(net_data.get('links')) + my.ovim.delete_link_bridge_to_ovs(vlan, links) + config_dic['host_threads'][server['host_id']].insert_task('del-ovs-port', vlan, net_id) - return format_out(data + warn_text) + if warn_text: + data["result"] += warn_text + return format_out(data) @@ -1946,7 +2005,7 @@ def http_post_networks(): try: # parse input data - http_content = format_in(network_new_schema ) + http_content = format_in(network_new_schema) r = remove_extra_items(http_content, network_new_schema) if r is not None: print "http_post_networks: Warning: remove extra items ", r