From: tierno Date: Thu, 20 Apr 2017 16:56:07 +0000 (+0200) Subject: Merge branch 'packaging' X-Git-Tag: v2.0.0~22 X-Git-Url: https://osm.etsi.org/gitweb/?a=commitdiff_plain;h=refs%2Fchanges%2F07%2F1607%2F3;hp=4b6216b9d00195bf2e3772bb9278faf18ee9fd46;p=osm%2FRO.git Merge branch 'packaging' Change-Id: I58a236852dab90f025d8300bbbf508d368d799c0 Signed-off-by: tierno --- diff --git a/database_utils/migrate_mano_db.sh b/database_utils/migrate_mano_db.sh index a8ec8f91..ba8a8294 100755 --- a/database_utils/migrate_mano_db.sh +++ b/database_utils/migrate_mano_db.sh @@ -187,6 +187,7 @@ DATABASE_TARGET_VER_NUM=0 [ $OPENMANO_VER_NUM -ge 5003 ] && DATABASE_TARGET_VER_NUM=17 #0.5.3 => 17 [ $OPENMANO_VER_NUM -ge 5004 ] && DATABASE_TARGET_VER_NUM=18 #0.5.4 => 18 [ $OPENMANO_VER_NUM -ge 5005 ] && DATABASE_TARGET_VER_NUM=19 #0.5.5 => 19 +[ $OPENMANO_VER_NUM -ge 5009 ] && DATABASE_TARGET_VER_NUM=20 #0.5.9 => 20 #TODO ... put next versions here @@ -730,6 +731,27 @@ function downgrade_from_19(){ echo "DELETE FROM schema_version WHERE version_int='19';" | $DBCMD || ! echo "ERROR. Aborted!" || exit -1 } +function upgrade_to_20(){ + echo " upgrade database from version 0.19 to version 0.20" + echo " add column 'sdn_net_id' at table 'instance_nets' and columns 'sdn_port_id', 'compute_node', 'pci' and 'vlan' to table 'instance_interfaces'" + echo "ALTER TABLE instance_nets ADD sdn_net_id varchar(36) DEFAULT NULL NULL COMMENT 'Network id in ovim';" | $DBCMD || ! echo "ERROR. Aborted!" || exit -1 + echo "ALTER TABLE instance_interfaces ADD sdn_port_id varchar(36) DEFAULT NULL NULL COMMENT 'Port id in ovim';" | $DBCMD || ! echo "ERROR. Aborted!" || exit -1 + echo "ALTER TABLE instance_interfaces ADD compute_node varchar(100) DEFAULT NULL NULL COMMENT 'Compute node id used to specify the SDN port mapping';" | $DBCMD || ! echo "ERROR. Aborted!" || exit -1 + echo "ALTER TABLE instance_interfaces ADD pci varchar(12) DEFAULT NULL NULL COMMENT 'PCI of the physical port in the host';" | $DBCMD || ! echo "ERROR. Aborted!" || exit -1 + echo "ALTER TABLE instance_interfaces ADD vlan SMALLINT UNSIGNED DEFAULT NULL NULL COMMENT 'VLAN tag used by the port';" | $DBCMD || ! echo "ERROR. Aborted!" || exit -1 + echo "INSERT INTO schema_version (version_int, version, openmano_ver, comments, date) VALUES (20, '0.20', '0.5.9', 'Added columns to store dataplane connectivity info', '2017-03-13');" | $DBCMD || ! echo "ERROR. Aborted!" || exit -1 +} +function downgrade_from_20(){ + echo " downgrade database from version 0.20 to version 0.19" + echo " remove column 'sdn_net_id' at table 'instance_nets' and columns 'sdn_port_id', 'compute_node', 'pci' and 'vlan' to table 'instance_interfaces'" + echo "ALTER TABLE instance_nets DROP COLUMN sdn_net_id;" | $DBCMD || ! echo "ERROR. Aborted!" || exit -1 + echo "ALTER TABLE instance_interfaces DROP COLUMN vlan;" | $DBCMD || ! echo "ERROR. Aborted!" || exit -1 + echo "ALTER TABLE instance_interfaces DROP COLUMN pci;" | $DBCMD || ! echo "ERROR. Aborted!" || exit -1 + echo "ALTER TABLE instance_interfaces DROP COLUMN compute_node;" | $DBCMD || ! echo "ERROR. Aborted!" || exit -1 + echo "ALTER TABLE instance_interfaces DROP COLUMN sdn_port_id;" | $DBCMD || ! echo "ERROR. Aborted!" || exit -1 + echo "DELETE FROM schema_version WHERE version_int='20';" | $DBCMD || ! echo "ERROR. Aborted!" || exit -1 +} + function upgrade_to_X(){ echo " change 'datacenter_nets'" echo "ALTER TABLE datacenter_nets ADD COLUMN vim_tenant_id VARCHAR(36) NOT NULL AFTER datacenter_id, DROP INDEX name_datacenter_id, ADD UNIQUE INDEX name_datacenter_id (name, datacenter_id, vim_tenant_id);" | $DBCMD || ! echo "ERROR. Aborted!" || exit -1 diff --git a/openmano b/openmano index c34d831f..45db3403 100755 --- a/openmano +++ b/openmano @@ -26,10 +26,10 @@ ''' openmano client used to interact with openmano-server (openmanod) ''' -__author__="Alfonso Tierno, Gerardo Garcia" +__author__="Alfonso Tierno, Gerardo Garcia, Pablo Montes" __date__ ="$09-oct-2014 09:09:48$" -__version__="0.4.11-r517" -version_date="Jan 2017" +__version__="0.4.13-r519" +version_date="Mar 2017" from argcomplete.completers import FilesCompleter import os @@ -120,6 +120,7 @@ def _print_verbose(mano_response, verbose_level=0): return result if mano_response.status_code == 200: + uuid = None for content in content_list: if "uuid" in content: uuid = content['uuid'] @@ -931,6 +932,38 @@ def datacenter_attach(args): print "Try to specify a different name with --vim-tenant-name" return result + +def datacenter_edit_vim_tenant(args): + tenant = _get_tenant() + datacenter = _get_datacenter(args.name) + headers_req = {'Accept': 'application/json', 'content-type': 'application/json'} + + if not (args.vim_tenant_id or args.vim_tenant_name or args.user or args.password or args.config): + raise OpenmanoCLIError("Error. At least one parameter must be updated.") + + datacenter_dict = {} + if args.vim_tenant_id != None: + datacenter_dict['vim_tenant'] = args.vim_tenant_id + if args.vim_tenant_name != None: + datacenter_dict['vim_tenant_name'] = args.vim_tenant_name + if args.user != None: + datacenter_dict['vim_username'] = args.user + if args.password != None: + datacenter_dict['vim_password'] = args.password + if args.config != None: + datacenter_dict["config"] = _load_file_or_yaml(args.config) + payload_req = json.dumps({"datacenter": datacenter_dict}) + + # print payload_req + + URLrequest = "http://%s:%s/openmano/%s/datacenters/%s" % (mano_host, mano_port, tenant, datacenter) + logger.debug("openmano request: %s", payload_req) + mano_response = requests.put(URLrequest, headers=headers_req, data=payload_req) + logger.debug("openmano response: %s", mano_response.text) + result = _print_verbose(mano_response, args.verbose) + + return result + def datacenter_detach(args): if args.all: tenant = "any" @@ -960,7 +993,13 @@ def datacenter_create(args): if args.url!=None: datacenter_dict["vim_url_admin"] = args.url_admin if args.config!=None: - datacenter_dict["config"] = _load_file_or_yaml(args.config) + datacenter_dict["config"] = _load_file_or_yaml(args.config) + if args.sdn_controller!=None: + tenant = _get_tenant() + sdn_controller = _get_item_uuid("sdn_controllers", args.sdn_controller, tenant) + if not 'config' in datacenter_dict: + datacenter_dict['config'] = {} + datacenter_dict['config']['sdn-controller'] = sdn_controller payload_req = json.dumps( {"datacenter": datacenter_dict }) #print payload_req @@ -1007,6 +1046,181 @@ def datacenter_list(args): args.verbose += 1 return _print_verbose(mano_response, args.verbose) +def datacenter_sdn_port_mapping_set(args): + tenant = _get_tenant() + datacenter = _get_datacenter(args.name, tenant) + headers_req = {'Accept': 'application/json', 'content-type': 'application/json'} + + if not args.file: + raise OpenmanoCLIError( + "No yaml/json has been provided specifying the SDN port mapping") + + port_mapping = yaml.load(datacenter_sdn_port_mapping_list(args)) + if len(port_mapping["sdn_port_mapping"]["ports_mapping"]) > 0: + if not args.force: + r = raw_input("Datacenter %s already contains a port mapping. Overwrite? (y/N)? " % (datacenter)) + if not (len(r) > 0 and r[0].lower() == "y"): + return 0 + args.force = True + print datacenter_sdn_port_mapping_clear(args) + + sdn_port_mapping = _load_file_or_yaml(args.file) + payload_req = json.dumps({"sdn_port_mapping": sdn_port_mapping}) + + URLrequest = "http://%s:%s/openmano/%s/datacenters/%s/sdn_mapping" % (mano_host, mano_port, tenant, datacenter) + logger.debug("openmano request: %s", payload_req) + mano_response = requests.post(URLrequest, headers=headers_req, data=payload_req) + logger.debug("openmano response: %s", mano_response.text) + + if mano_response.status_code == 200: + return yaml.safe_dump(mano_response.json()) + else: + return mano_response.content + +def datacenter_sdn_port_mapping_list(args): + tenant = _get_tenant() + datacenter = _get_datacenter(args.name, tenant) + + URLrequest = "http://%s:%s/openmano/%s/datacenters/%s/sdn_mapping" % (mano_host, mano_port, tenant, datacenter) + mano_response = requests.get(URLrequest) + logger.debug("openmano response: %s", mano_response.text) + + if mano_response.status_code != 200: + return mano_response.content + + return yaml.safe_dump(mano_response.json()) + +def datacenter_sdn_port_mapping_clear(args): + tenant = _get_tenant() + datacenter = _get_datacenter(args.name, tenant) + + if not args.force: + r = raw_input("Clean SDN port mapping for datacenter %s (y/N)? " %(datacenter)) + if not (len(r)>0 and r[0].lower()=="y"): + return 0 + + URLrequest = "http://%s:%s/openmano/%s/datacenters/%s/sdn_mapping" % (mano_host, mano_port, tenant, datacenter) + mano_response = requests.delete(URLrequest) + logger.debug("openmano response: %s", mano_response.text) + + if mano_response.status_code != 200: + if "No port mapping for datacenter" in mano_response.content: + return "No port mapping for datacenter " + datacenter + " has been found" + return mano_response.content + + return yaml.safe_dump(mano_response.json()) + +def sdn_controller_create(args): + tenant = _get_tenant() + headers_req = {'Accept': 'application/json', 'content-type': 'application/json'} + + if not (args.ip and args.port and args.dpid and args.type): + raise OpenmanoCLIError("The following arguments are required: ip, port, dpid, type") + + controller_dict = {} + controller_dict['name'] = args.name + controller_dict['ip'] = args.ip + controller_dict['port'] = int(args.port) + controller_dict['dpid'] = args.dpid + controller_dict['type'] = args.type + if args.description != None: + controller_dict['description'] = args.description + if args.user != None: + controller_dict['user'] = args.user + if args.password != None: + controller_dict['password'] = args.password + + payload_req = json.dumps({"sdn_controller": controller_dict}) + + # print payload_req + + URLrequest = "http://%s:%s/openmano/%s/sdn_controllers" % (mano_host, mano_port, tenant) + logger.debug("openmano request: %s", payload_req) + mano_response = requests.post(URLrequest, headers=headers_req, data=payload_req) + logger.debug("openmano response: %s", mano_response.text) + result = _print_verbose(mano_response, args.verbose) + + return result + +def sdn_controller_edit(args): + tenant = _get_tenant() + controller_uuid = _get_item_uuid("sdn_controllers", args.name, tenant) + headers_req = {'Accept': 'application/json', 'content-type': 'application/json'} + + if not (args.new_name or args.ip or args.port or args.dpid or args.type): + raise OpenmanoCLIError("At least one parameter must be editd") + + if not args.force: + r = raw_input("Update SDN controller %s (y/N)? " %(args.name)) + if not (len(r)>0 and r[0].lower()=="y"): + return 0 + + controller_dict = {} + if args.new_name != None: + controller_dict['name'] = args.new_name + if args.ip != None: + controller_dict['ip'] = args.ip + if args.port != None: + controller_dict['port'] = int(args.port) + if args.dpid != None: + controller_dict['dpid'] = args.dpid + if args.type != None: + controller_dict['type'] = args.type + if args.description != None: + controller_dict['description'] = args.description + if args.user != None: + controller_dict['user'] = args.user + if args.password != None: + controller_dict['password'] = args.password + + payload_req = json.dumps({"sdn_controller": controller_dict}) + + # print payload_req + + URLrequest = "http://%s:%s/openmano/%s/sdn_controllers/%s" % (mano_host, mano_port, tenant, controller_uuid) + logger.debug("openmano request: %s", payload_req) + mano_response = requests.put(URLrequest, headers=headers_req, data=payload_req) + logger.debug("openmano response: %s", mano_response.text) + result = _print_verbose(mano_response, args.verbose) + + return result + +def sdn_controller_list(args): + tenant = _get_tenant() + headers_req = {'Accept': 'application/json', 'content-type': 'application/json'} + + if args.name: + toshow = _get_item_uuid("sdn_controllers", args.name, tenant) + URLrequest = "http://%s:%s/openmano/%s/sdn_controllers/%s" %(mano_host, mano_port, tenant, toshow) + else: + URLrequest = "http://%s:%s/openmano/%s/sdn_controllers" %(mano_host, mano_port, tenant) + #print URLrequest + mano_response = requests.get(URLrequest) + logger.debug("openmano response: %s", mano_response.text ) + if args.verbose==None: + args.verbose=0 + if args.name!=None: + args.verbose += 1 + + result = json.dumps(mano_response.json(), indent=4) + return result + +def sdn_controller_delete(args): + tenant = _get_tenant() + controller_uuid = _get_item_uuid("sdn_controllers", args.name, tenant) + + if not args.force: + r = raw_input("Delete SDN controller %s (y/N)? " % (args.name)) + if not (len(r) > 0 and r[0].lower() == "y"): + return 0 + + URLrequest = "http://%s:%s/openmano/%s/sdn_controllers/%s" % (mano_host, mano_port, tenant, controller_uuid) + mano_response = requests.delete(URLrequest) + logger.debug("openmano response: %s", mano_response.text) + result = _print_verbose(mano_response, args.verbose) + + return result + def vim_action(args): #print "datacenter-net-action",args tenant = _get_tenant() @@ -1045,11 +1259,11 @@ def vim_action(args): create_dict[args.item]['name'] = args.name #if args.description: # create_dict[args.item]['description'] = args.description - if args.item=="vim-net": + if args.item=="network": if args.bind_net: create_dict[args.item]['bind_net'] = args.bind_net - if args.bind_type: - create_dict[args.item]['bind_type'] = args.bind_type + if args.type: + create_dict[args.item]['type'] = args.type if args.shared: create_dict[args.item]['shared'] = args.shared if "name" not in create_dict[args.item]: @@ -1201,6 +1415,53 @@ def element_edit(args): return _print_verbose(mano_response, args.verbose) +def datacenter_edit(args): + tenant = _get_tenant() + element = _get_item_uuid('datacenters', args.name, tenant) + headers_req = {'Accept': 'application/json', 'content-type': 'application/json'} + URLrequest = "http://%s:%s/openmano/datacenters/%s" % (mano_host, mano_port, element) + + has_arguments = False + if args.file != None: + has_arguments = True + payload = _load_file_or_yaml(args.file) + else: + payload = {} + + if args.sdn_controller != None: + has_arguments = True + if not 'config' in payload: + payload['config'] = {} + if not 'sdn-controller' in payload['config']: + payload['config']['sdn-controller'] = {} + if args.sdn_controller == 'null': + payload['config']['sdn-controller'] = None + else: + payload['config']['sdn-controller'] = _get_item_uuid("sdn_controllers", args.sdn_controller, tenant) + + if not has_arguments: + raise OpenmanoCLIError("At least one argument must be provided to modify the datacenter") + + if 'datacenter' not in payload: + payload = {'datacenter': payload} + payload_req = json.dumps(payload) + + # print payload_req + if not args.force or (args.name == None and args.filer == None): + r = raw_input(" Edit datacenter " + args.name + " (y/N)? ") + if len(r) > 0 and r[0].lower() == "y": + pass + else: + return 0 + logger.debug("openmano request: %s", payload_req) + mano_response = requests.put(URLrequest, headers=headers_req, data=payload_req) + logger.debug("openmano response: %s", mano_response.text) + if args.verbose == None: + args.verbose = 0 + if args.name != None: + args.verbose += 1 + return _print_verbose(mano_response, args.verbose) + global mano_host global mano_port global mano_tenant @@ -1329,7 +1590,7 @@ if __name__=="__main__": tenant_list_parser.add_argument("name", nargs='?', help="name or uuid of the tenant") tenant_list_parser.set_defaults(func=tenant_list) - item_list=('tenant','datacenter') #put tenant before so that help appear in order + item_list=('tenant') #put tenant before so that help appear in order for item in item_list: element_edit_parser = subparsers.add_parser(item+'-edit', parents=[parent_parser], help="edits one "+item) element_edit_parser.add_argument("name", help="name or uuid of the "+item) @@ -1344,6 +1605,7 @@ if __name__=="__main__": datacenter_create_parser.add_argument("--type", action="store", help="datacenter type: openstack or openvim (default)") datacenter_create_parser.add_argument("--config", action="store", help="aditional configuration in json/yaml format") datacenter_create_parser.add_argument("--description", action="store", help="description of the datacenter") + datacenter_create_parser.add_argument("--sdn-controller", action="store", help="Name or uuid of the SDN controller to be used", dest='sdn_controller') datacenter_create_parser.set_defaults(func=datacenter_create) datacenter_delete_parser = subparsers.add_parser('datacenter-delete', parents=[parent_parser], help="deletes a datacenter from the catalogue") @@ -1351,6 +1613,14 @@ if __name__=="__main__": datacenter_delete_parser.add_argument("-f", "--force", action="store_true", help="forces deletion without asking") datacenter_delete_parser.set_defaults(func=datacenter_delete) + datacenter_edit_parser = subparsers.add_parser('datacenter-edit', parents=[parent_parser], help="Edit datacenter") + datacenter_edit_parser.add_argument("name", help="name or uuid of the datacenter") + datacenter_edit_parser.add_argument("--file", help="json/yaml text or file with the changes").completer = FilesCompleter + datacenter_edit_parser.add_argument("--sdn-controller", action="store", + help="Name or uuid of the SDN controller to be used. Specify 'null' to clear entry", dest='sdn_controller') + datacenter_edit_parser.add_argument("-f", "--force", action="store_true", help="do not prompt for confirmation") + datacenter_edit_parser.set_defaults(func=datacenter_edit) + datacenter_list_parser = subparsers.add_parser('datacenter-list', parents=[parent_parser], help="lists information about a datacenter") datacenter_list_parser.add_argument("name", nargs='?', help="name or uuid of the datacenter") datacenter_list_parser.add_argument("-a", "--all", action="store_true", help="shows all datacenters, not only datacenters attached to tenant") @@ -1365,11 +1635,107 @@ if __name__=="__main__": datacenter_attach_parser.add_argument("--config", action="store", help="aditional configuration in json/yaml format") datacenter_attach_parser.set_defaults(func=datacenter_attach) + datacenter_edit_vim_tenant_parser = subparsers.add_parser('datacenter-edit-vim-tenant', parents=[parent_parser], + help="Edit the association of a datacenter to the operating tenant") + datacenter_edit_vim_tenant_parser.add_argument("name", help="name or uuid of the datacenter") + datacenter_edit_vim_tenant_parser.add_argument('--vim-tenant-id', action='store', + help="specify a datacenter tenant to use. A new one is created by default") + datacenter_edit_vim_tenant_parser.add_argument('--vim-tenant-name', action='store', help="specify a datacenter tenant name.") + datacenter_edit_vim_tenant_parser.add_argument("--user", action="store", help="user credentials for the datacenter") + datacenter_edit_vim_tenant_parser.add_argument("--password", action="store", help="password credentials for the datacenter") + datacenter_edit_vim_tenant_parser.add_argument("--config", action="store", + help="aditional configuration in json/yaml format") + datacenter_edit_vim_tenant_parser.set_defaults(func=datacenter_edit_vim_tenant) + datacenter_detach_parser = subparsers.add_parser('datacenter-detach', parents=[parent_parser], help="removes the association between a datacenter and the operating tenant") datacenter_detach_parser.add_argument("name", help="name or uuid of the datacenter") datacenter_detach_parser.add_argument("-a", "--all", action="store_true", help="removes all associations from this datacenter") datacenter_detach_parser.set_defaults(func=datacenter_detach) + #=======================datacenter_sdn_port_mapping_xxx section======================= + #datacenter_sdn_port_mapping_set + datacenter_sdn_port_mapping_set_parser = subparsers.add_parser('datacenter-sdn-port-mapping-set', + parents=[parent_parser], + help="Load a file with the mapping of physical ports " + "and the ports of the dataplaneswitch controlled " + "by a datacenter") + datacenter_sdn_port_mapping_set_parser.add_argument("name", action="store", help="specifies the datacenter") + datacenter_sdn_port_mapping_set_parser.add_argument("file", + help="json/yaml text or file with the port mapping").completer = FilesCompleter + datacenter_sdn_port_mapping_set_parser.add_argument("-f", "--force", action="store_true", + help="forces overwriting without asking") + datacenter_sdn_port_mapping_set_parser.set_defaults(func=datacenter_sdn_port_mapping_set) + + #datacenter_sdn_port_mapping_list + datacenter_sdn_port_mapping_list_parser = subparsers.add_parser('datacenter-sdn-port-mapping-list', + parents=[parent_parser], + help="Show the SDN port mapping in a datacenter") + datacenter_sdn_port_mapping_list_parser.add_argument("name", action="store", help="specifies the datacenter") + datacenter_sdn_port_mapping_list_parser.set_defaults(func=datacenter_sdn_port_mapping_list) + + # datacenter_sdn_port_mapping_clear + datacenter_sdn_port_mapping_clear_parser = subparsers.add_parser('datacenter-sdn-port-mapping-clear', + parents=[parent_parser], + help="Clean the the SDN port mapping in a datacenter") + datacenter_sdn_port_mapping_clear_parser.add_argument("name", action="store", + help="specifies the datacenter") + datacenter_sdn_port_mapping_clear_parser.add_argument("-f", "--force", action="store_true", + help="forces clearing without asking") + datacenter_sdn_port_mapping_clear_parser.set_defaults(func=datacenter_sdn_port_mapping_clear) + # ======================= + + # =======================sdn_controller_xxx section======================= + # sdn_controller_create + sdn_controller_create_parser = subparsers.add_parser('sdn-controller-create', parents=[parent_parser], + help="Creates an SDN controller entity within RO") + sdn_controller_create_parser.add_argument("name", help="name of the SDN controller") + sdn_controller_create_parser.add_argument("--description", action="store", help="description of the SDN controller") + sdn_controller_create_parser.add_argument("--ip", action="store", help="IP of the SDN controller") + sdn_controller_create_parser.add_argument("--port", action="store", help="Port of the SDN controller") + sdn_controller_create_parser.add_argument("--dpid", action="store", + help="DPID of the dataplane switch controlled by this SDN controller") + sdn_controller_create_parser.add_argument("--type", action="store", + help="Specify the SDN controller type. Valid types are 'opendaylight' and 'floodlight'") + sdn_controller_create_parser.add_argument("--user", action="store", help="user credentials for the SDN controller") + sdn_controller_create_parser.add_argument("--passwd", action="store", dest='password', + help="password credentials for the SDN controller") + sdn_controller_create_parser.set_defaults(func=sdn_controller_create) + + # sdn_controller_edit + sdn_controller_edit_parser = subparsers.add_parser('sdn-controller-edit', parents=[parent_parser], + help="Update one or more options of a SDN controller") + sdn_controller_edit_parser.add_argument("name", help="name or uuid of the SDN controller", ) + sdn_controller_edit_parser.add_argument("--name", action="store", help="Update the name of the SDN controller", + dest='new_name') + sdn_controller_edit_parser.add_argument("--description", action="store", help="description of the SDN controller") + sdn_controller_edit_parser.add_argument("--ip", action="store", help="IP of the SDN controller") + sdn_controller_edit_parser.add_argument("--port", action="store", help="Port of the SDN controller") + sdn_controller_edit_parser.add_argument("--dpid", action="store", + help="DPID of the dataplane switch controlled by this SDN controller") + sdn_controller_edit_parser.add_argument("--type", action="store", + help="Specify the SDN controller type. Valid types are 'opendaylight' and 'floodlight'") + sdn_controller_edit_parser.add_argument("--user", action="store", help="user credentials for the SDN controller") + sdn_controller_edit_parser.add_argument("--password", action="store", + help="password credentials for the SDN controller", dest='password') + sdn_controller_edit_parser.add_argument("-f", "--force", action="store_true", help="do not prompt for confirmation") + #TODO: include option --file + sdn_controller_edit_parser.set_defaults(func=sdn_controller_edit) + + #sdn_controller_list + sdn_controller_list_parser = subparsers.add_parser('sdn-controller-list', + parents=[parent_parser], + help="List the SDN controllers") + sdn_controller_list_parser.add_argument("name", nargs='?', help="name or uuid of the SDN controller") + sdn_controller_list_parser.set_defaults(func=sdn_controller_list) + + # sdn_controller_delete + sdn_controller_delete_parser = subparsers.add_parser('sdn-controller-delete', + parents=[parent_parser], + help="Delete the the SDN controller") + sdn_controller_delete_parser.add_argument("name", help="name or uuid of the SDN controller") + sdn_controller_delete_parser.add_argument("-f", "--force", action="store_true", help="forces deletion without asking") + sdn_controller_delete_parser.set_defaults(func=sdn_controller_delete) + # ======================= action_dict={'net-update': 'retrieves external networks from datacenter', 'net-edit': 'edits an external network', diff --git a/openmanod b/openmanod index 0611d4ec..a63a1f07 100755 --- a/openmanod +++ b/openmanod @@ -33,9 +33,9 @@ It loads the configuration file and launches the http_server thread that will li ''' __author__="Alfonso Tierno, Gerardo Garcia, Pablo Montes" __date__ ="$26-aug-2014 11:09:29$" -__version__="0.5.8-r518" -version_date="Jan 2017" -database_version="0.19" #expected database schema version +__version__="0.5.9-r519" +version_date="Mar 2017" +database_version="0.20" #expected database schema version import time import sys @@ -57,14 +57,16 @@ class LoadConfigurationException(Exception): pass def load_configuration(configuration_file): - default_tokens ={'http_port':9090, - 'http_host':'localhost', - 'http_console_proxy': True, - 'http_console_host': None, - 'log_level': 'DEBUG', - 'log_socket_port': 9022, - 'auto_push_VNF_to_VIMs': True - } + default_tokens = {'http_port':9090, + 'http_host':'localhost', + 'http_console_proxy': True, + 'http_console_host': None, + 'log_level': 'DEBUG', + 'log_socket_port': 9022, + 'auto_push_VNF_to_VIMs': True, + 'db_host': 'localhost', + 'db_ovim_host': 'localhost' + } try: #Check config file exists with open(configuration_file, 'r') as f: @@ -241,7 +243,7 @@ if __name__=="__main__": logger.critical("Starting openmano server version: '%s %s' command: '%s'", __version__, version_date, " ".join(sys.argv)) - for log_module in ("nfvo", "http", "vim", "db", "console"): + for log_module in ("nfvo", "http", "vim", "db", "console", "ovim"): log_level_module = "log_level_" + log_module log_file_module = "log_file_" + log_module logger_module = logging.getLogger('openmano.' + log_module) @@ -315,6 +317,4 @@ if __name__=="__main__": nfvo.stop_service() if httpthread: httpthread.join(1) - for thread in global_config["console_thread"]: - thread.terminate = True diff --git a/osm_ro/db_base.py b/osm_ro/db_base.py index 10f94045..4a877213 100644 --- a/osm_ro/db_base.py +++ b/osm_ro/db_base.py @@ -265,6 +265,8 @@ class db_base(): ''' if data==None: return 'Null' + elif isinstance(data[1], str): + return json.dumps(data) else: return json.dumps(str(data)) @@ -277,6 +279,8 @@ class db_base(): ''' if data[1]==None: return str(data[0]) + "=Null" + elif isinstance(data[1], str): + return str(data[0]) + '=' + json.dumps(data[1]) else: return str(data[0]) + '=' + json.dumps(str(data[1])) @@ -289,24 +293,10 @@ class db_base(): ''' if data[1]==None: return str(data[0]) + " is Null" - -# if type(data[1]) is tuple: #this can only happen in a WHERE_OR clause -# text =[] -# for d in data[1]: -# if d==None: -# text.append(str(data[0]) + " is Null") -# continue -# out=str(d) -# if "'" not in out: -# text.append( str(data[0]) + "='" + out + "'" ) -# elif '"' not in out: -# text.append( str(data[0]) + '="' + out + '"' ) -# else: -# text.append( str(data[0]) + '=' + json.dumps(out) ) -# return " OR ".join(text) - - out=str(data[1]) - return str(data[0]) + '=' + json.dumps(out) + elif isinstance(data[1], str): + return str(data[0]) + '=' + json.dumps(data[1]) + else: + return str(data[0]) + '=' + json.dumps(str(data[1])) def __tuple2db_format_where_not(self, data): '''Compose the needed text for a SQL WHERE(not). parameter 'data' is a pair tuple (A,B), @@ -317,8 +307,10 @@ class db_base(): ''' if data[1]==None: return str(data[0]) + " is not Null" - out=str(data[1]) - return str(data[0]) + '<>' + json.dumps(out) + elif isinstance(data[1], str): + return str(data[0]) + '<>' + json.dumps(data[1]) + else: + return str(data[0]) + '<>' + json.dumps(str(data[1])) def __remove_quotes(self, data): '''remove single quotes ' of any string content of data dictionary''' diff --git a/osm_ro/httpserver.py b/osm_ro/httpserver.py index a0216e12..4841a98b 100644 --- a/osm_ro/httpserver.py +++ b/osm_ro/httpserver.py @@ -42,7 +42,9 @@ from openmano_schemas import vnfd_schema_v01, vnfd_schema_v02, \ scenario_action_schema, instance_scenario_action_schema, instance_scenario_create_schema_v01, \ tenant_schema, tenant_edit_schema,\ datacenter_schema, datacenter_edit_schema, datacenter_action_schema, datacenter_associate_schema,\ - object_schema, netmap_new_schema, netmap_edit_schema + object_schema, netmap_new_schema, netmap_edit_schema, sdn_controller_schema, sdn_controller_edit_schema, \ + sdn_port_mapping_schema + import nfvo import utils from db_base import db_base_Exception @@ -506,7 +508,7 @@ def http_get_datacenter_id(tenant_id, datacenter_id): @bottle.route(url_base + '/datacenters', method='POST') def http_post_datacenters(): - '''insert a tenant into the catalogue. ''' + '''insert a datacenter into the catalogue. ''' #parse input data logger.debug('FROM %s %s %s', bottle.request.remote_addr, bottle.request.method, bottle.request.url) http_content,_ = format_in( datacenter_schema ) @@ -544,6 +546,138 @@ def http_edit_datacenter_id(datacenter_id_name): logger.error("Unexpected exception: ", exc_info=True) bottle.abort(HTTP_Internal_Server_Error, type(e).__name__ + ": " + str(e)) +@bottle.route(url_base + '//sdn_controllers', method='POST') +def http_post_sdn_controller(tenant_id): + '''insert a sdn controller into the catalogue. ''' + #parse input data + logger.debug('FROM %s %s %s', bottle.request.remote_addr, bottle.request.method, bottle.request.url) + http_content,_ = format_in( sdn_controller_schema ) + try: + logger.debug("tenant_id: "+tenant_id) + #logger.debug("content: {}".format(http_content['sdn_controller'])) + + data = nfvo.sdn_controller_create(mydb, tenant_id, http_content['sdn_controller']) + return format_out({"sdn_controller": nfvo.sdn_controller_list(mydb, tenant_id, data)}) + except (nfvo.NfvoException, db_base_Exception) as e: + logger.error("http_post_sdn_controller error {}: {}".format(e.http_code, str(e))) + bottle.abort(e.http_code, str(e)) + except Exception as e: + logger.error("Unexpected exception: ", exc_info=True) + bottle.abort(HTTP_Internal_Server_Error, type(e).__name__ + ": " + str(e)) + +@bottle.route(url_base + '//sdn_controllers/', method='PUT') +def http_put_sdn_controller_update(tenant_id, controller_id): + '''Update sdn controller''' + #parse input data + logger.debug('FROM %s %s %s', bottle.request.remote_addr, bottle.request.method, bottle.request.url) + http_content,_ = format_in( sdn_controller_edit_schema ) +# r = utils.remove_extra_items(http_content, datacenter_schema) +# if r: +# logger.debug("Remove received extra items %s", str(r)) + try: + #logger.debug("tenant_id: "+tenant_id) + logger.debug("content: {}".format(http_content['sdn_controller'])) + + data = nfvo.sdn_controller_update(mydb, tenant_id, controller_id, http_content['sdn_controller']) + return format_out({"sdn_controller": nfvo.sdn_controller_list(mydb, tenant_id, controller_id)}) + + except (nfvo.NfvoException, db_base_Exception) as e: + logger.error("http_post_sdn_controller error {}: {}".format(e.http_code, str(e))) + bottle.abort(e.http_code, str(e)) + except Exception as e: + logger.error("Unexpected exception: ", exc_info=True) + bottle.abort(HTTP_Internal_Server_Error, type(e).__name__ + ": " + str(e)) + +@bottle.route(url_base + '//sdn_controllers', method='GET') +def http_get_sdn_controller(tenant_id): + '''get sdn controllers list, can use both uuid or name''' + try: + logger.debug('FROM %s %s %s', bottle.request.remote_addr, bottle.request.method, bottle.request.url) + + data = {'sdn_controllers': nfvo.sdn_controller_list(mydb, tenant_id)} + return format_out(data) + except (nfvo.NfvoException, db_base_Exception) as e: + logger.error("http_get_sdn_controller error {}: {}".format(e.http_code, str(e))) + bottle.abort(e.http_code, str(e)) + except Exception as e: + logger.error("Unexpected exception: ", exc_info=True) + bottle.abort(HTTP_Internal_Server_Error, type(e).__name__ + ": " + str(e)) + +@bottle.route(url_base + '//sdn_controllers/', method='GET') +def http_get_sdn_controller_id(tenant_id, controller_id): + '''get sdn controller details, can use both uuid or name''' + try: + logger.debug('FROM %s %s %s', bottle.request.remote_addr, bottle.request.method, bottle.request.url) + data = nfvo.sdn_controller_list(mydb, tenant_id, controller_id) + return format_out({"sdn_controllers": data}) + except (nfvo.NfvoException, db_base_Exception) as e: + logger.error("http_get_sdn_controller_id error {}: {}".format(e.http_code, str(e))) + bottle.abort(e.http_code, str(e)) + except Exception as e: + logger.error("Unexpected exception: ", exc_info=True) + bottle.abort(HTTP_Internal_Server_Error, type(e).__name__ + ": " + str(e)) + +@bottle.route(url_base + '//sdn_controllers/', method='DELETE') +def http_delete_sdn_controller_id(tenant_id, controller_id): + '''delete sdn controller, can use both uuid or name''' + try: + logger.debug('FROM %s %s %s', bottle.request.remote_addr, bottle.request.method, bottle.request.url) + data = nfvo.sdn_controller_delete(mydb, tenant_id, controller_id) + return format_out(data) + except (nfvo.NfvoException, db_base_Exception) as e: + logger.error("http_delete_sdn_controller_id error {}: {}".format(e.http_code, str(e))) + bottle.abort(e.http_code, str(e)) + except Exception as e: + logger.error("Unexpected exception: ", exc_info=True) + bottle.abort(HTTP_Internal_Server_Error, type(e).__name__ + ": " + str(e)) + +@bottle.route(url_base + '//datacenters//sdn_mapping', method='POST') +def http_post_datacenter_sdn_port_mapping(tenant_id, datacenter_id): + '''Set the sdn port mapping for a datacenter. ''' + #parse input data + logger.debug('FROM %s %s %s', bottle.request.remote_addr, bottle.request.method, bottle.request.url) + http_content, _ = format_in(sdn_port_mapping_schema) +# r = utils.remove_extra_items(http_content, datacenter_schema) +# if r: +# logger.debug("Remove received extra items %s", str(r)) + try: + data = nfvo.datacenter_sdn_port_mapping_set(mydb, tenant_id, datacenter_id, http_content['sdn_port_mapping']) + return format_out({"sdn_port_mapping": data}) + except (nfvo.NfvoException, db_base_Exception) as e: + logger.error("http_post_datacenter_sdn_port_mapping error {}: {}".format(e.http_code, str(e))) + bottle.abort(e.http_code, str(e)) + except Exception as e: + logger.error("Unexpected exception: ", exc_info=True) + bottle.abort(HTTP_Internal_Server_Error, type(e).__name__ + ": " + str(e)) + +@bottle.route(url_base + '//datacenters//sdn_mapping', method='GET') +def http_get_datacenter_sdn_port_mapping(tenant_id, datacenter_id): + '''get datacenter sdn mapping details, can use both uuid or name''' + try: + logger.debug('FROM %s %s %s', bottle.request.remote_addr, bottle.request.method, bottle.request.url) + + data = nfvo.datacenter_sdn_port_mapping_list(mydb, tenant_id, datacenter_id) + return format_out({"sdn_port_mapping": data}) + except (nfvo.NfvoException, db_base_Exception) as e: + logger.error("http_get_datacenter_sdn_port_mapping error {}: {}".format(e.http_code, str(e))) + bottle.abort(e.http_code, str(e)) + except Exception as e: + logger.error("Unexpected exception: ", exc_info=True) + bottle.abort(HTTP_Internal_Server_Error, type(e).__name__ + ": " + str(e)) + +@bottle.route(url_base + '//datacenters//sdn_mapping', method='DELETE') +def http_delete_datacenter_sdn_port_mapping(tenant_id, datacenter_id): + '''clean datacenter sdn mapping, can use both uuid or name''' + try: + logger.debug('FROM %s %s %s', bottle.request.remote_addr, bottle.request.method, bottle.request.url) + data = nfvo.datacenter_sdn_port_mapping_delete(mydb, tenant_id, datacenter_id) + return format_out({"result": data}) + except (nfvo.NfvoException, db_base_Exception) as e: + logger.error("http_delete_datacenter_sdn_port_mapping error {}: {}".format(e.http_code, str(e))) + bottle.abort(e.http_code, str(e)) + except Exception as e: + logger.error("Unexpected exception: ", exc_info=True) + bottle.abort(HTTP_Internal_Server_Error, type(e).__name__ + ": " + str(e)) @bottle.route(url_base + '//datacenters//networks', method='GET') #deprecated @bottle.route(url_base + '//datacenters//netmaps', method='GET') @@ -739,6 +873,30 @@ def http_associate_datacenters(tenant_id, datacenter_id): logger.error("Unexpected exception: ", exc_info=True) bottle.abort(HTTP_Internal_Server_Error, type(e).__name__ + ": " + str(e)) +@bottle.route(url_base + '//datacenters/', method='PUT') +def http_associate_datacenters_edit(tenant_id, datacenter_id): + '''associate an existing datacenter to a this tenant. ''' + logger.debug('FROM %s %s %s', bottle.request.remote_addr, bottle.request.method, bottle.request.url) + #parse input data + http_content,_ = format_in( datacenter_associate_schema ) + r = utils.remove_extra_items(http_content, datacenter_associate_schema) + if r: + logger.debug("Remove received extra items %s", str(r)) + try: + id_ = nfvo.edit_datacenter_to_tenant(mydb, tenant_id, datacenter_id, + http_content['datacenter'].get('vim_tenant'), + http_content['datacenter'].get('vim_tenant_name'), + http_content['datacenter'].get('vim_username'), + http_content['datacenter'].get('vim_password'), + http_content['datacenter'].get('config') + ) + return http_get_datacenter_id(tenant_id, id_) + except (nfvo.NfvoException, db_base_Exception) as e: + logger.error("http_associate_datacenters_edit error {}: {}".format(e.http_code, str(e))) + bottle.abort(e.http_code, str(e)) + except Exception as e: + logger.error("Unexpected exception: ", exc_info=True) + bottle.abort(HTTP_Internal_Server_Error, type(e).__name__ + ": " + str(e)) @bottle.route(url_base + '//datacenters/', method='DELETE') def http_deassociate_datacenters(tenant_id, datacenter_id): diff --git a/osm_ro/nfvo.py b/osm_ro/nfvo.py index 14ec54f1..3ea51077 100644 --- a/osm_ro/nfvo.py +++ b/osm_ro/nfvo.py @@ -39,23 +39,27 @@ import vimconn import logging import collections from db_base import db_base_Exception + import nfvo_db from threading import Lock from time import time +import ovim as ovim_module global global_config global vimconn_imported global logger global default_volume_size default_volume_size = '5' #size in GB - +global ovim +ovim = None +global_config = None vimconn_imported = {} # dictionary with VIM type as key, loaded module as value vim_threads = {"running":{}, "deleting": {}, "names": []} # threads running for attached-VIMs vim_persistent_info = {} logger = logging.getLogger('openmano.nfvo') task_lock = Lock() -task_dict = {} +global_instance_tasks = {} last_task_id = 0.0 db=None db_lock=Lock() @@ -75,13 +79,11 @@ def get_task_id(): return "TASK.{:.6f}".format(task_id) -def new_task(name, params, store=True, depends=None): +def new_task(name, params, depends=None): task_id = get_task_id() task = {"status": "enqueued", "id": task_id, "name": name, "params": params} if depends: task["depends"] = depends - if store: - task_dict[task_id] = task return task @@ -107,6 +109,28 @@ def start_service(mydb): global db, global_config db = nfvo_db.nfvo_db() db.connect(global_config['db_host'], global_config['db_user'], global_config['db_passwd'], global_config['db_name']) + global ovim + + # Initialize openvim for SDN control + # TODO: Avoid static configuration by adding new parameters to openmanod.cfg + # TODO: review ovim.py to delete not needed configuration + ovim_configuration = { + 'logger_name': 'openmano.ovim', + 'network_vlan_range_start': 1000, + 'network_vlan_range_end': 4096, + 'db_name': global_config["db_ovim_name"], + 'db_host': global_config["db_ovim_host"], + 'db_user': global_config["db_ovim_user"], + 'db_passwd': global_config["db_ovim_passwd"], + 'bridge_ifaces': {}, + 'mode': 'normal', + 'network_type': 'bridge', + #TODO: log_level_of should not be needed. To be modified in ovim + 'log_level_of': 'DEBUG' + } + ovim = ovim_module.ovim(ovim_configuration) + ovim.start_service() + from_= 'tenants_datacenters as td join datacenters as d on td.datacenter_id=d.uuid join datacenter_tenants as dt on td.datacenter_tenant_id=dt.uuid' select_ = ('type','d.config as config','d.uuid as datacenter_id', 'vim_url', 'vim_url_admin', 'd.name as datacenter_name', 'dt.uuid as datacenter_tenant_id','dt.vim_tenant_name as vim_tenant_name','dt.vim_tenant_id as vim_tenant_id', @@ -114,7 +138,8 @@ def start_service(mydb): try: vims = mydb.get_rows(FROM=from_, SELECT=select_) for vim in vims: - extra={'datacenter_tenant_id': vim.get('datacenter_tenant_id')} + extra={'datacenter_tenant_id': vim.get('datacenter_tenant_id'), + 'datacenter_id': vim.get('datacenter_id')} if vim["config"]: extra.update(yaml.load(vim["config"])) if vim.get('dt_config'): @@ -132,7 +157,7 @@ def start_service(mydb): raise NfvoException("Unknown vim type '{}'. Can not open file '{}.py'; {}: {}".format( vim["type"], module, type(e).__name__, str(e)), HTTP_Bad_Request) - thread_id = vim["datacenter_id"] + "." + vim['nfvo_tenant_id'] + thread_id = vim['datacenter_tenant_id'] vim_persistent_info[thread_id] = {} try: #if not tenant: @@ -148,7 +173,7 @@ def start_service(mydb): raise NfvoException("Error at VIM {}; {}: {}".format(vim["type"], type(e).__name__, str(e)), HTTP_Internal_Server_Error) thread_name = get_non_used_vim_name(vim['datacenter_name'], vim['vim_tenant_id'], vim['vim_tenant_name'], vim['vim_tenant_id']) new_thread = vim_thread.vim_thread(myvim, task_lock, thread_name, vim['datacenter_name'], - vim.get('datacenter_tenant_id'), db=db, db_lock=db_lock) + vim['datacenter_tenant_id'], db=db, db_lock=db_lock, ovim=ovim) new_thread.start() vim_threads["running"][thread_id] = new_thread except db_base_Exception as e: @@ -156,10 +181,16 @@ def start_service(mydb): def stop_service(): + global ovim, global_config + if ovim: + ovim.stop_service() for thread_id,thread in vim_threads["running"].items(): - thread.insert_task(new_task("exit", None, store=False)) + thread.insert_task(new_task("exit", None)) vim_threads["deleting"][thread_id] = thread vim_threads["running"] = {} + if global_config and global_config.get("console_thread"): + for thread in global_config["console_thread"]: + thread.terminate = True def get_flavorlist(mydb, vnf_id, nfvo_tenant=None): @@ -229,7 +260,8 @@ def get_vim(mydb, nfvo_tenant=None, datacenter_id=None, datacenter_name=None, da vims = mydb.get_rows(FROM=from_, SELECT=select_, WHERE=WHERE_dict ) vim_dict={} for vim in vims: - extra={'datacenter_tenant_id': vim.get('datacenter_tenant_id')} + extra={'datacenter_tenant_id': vim.get('datacenter_tenant_id'), + 'datacenter_id': vim.get('datacenter_id')} if vim["config"]: extra.update(yaml.load(vim["config"])) if vim.get('dt_config'): @@ -248,8 +280,8 @@ def get_vim(mydb, nfvo_tenant=None, datacenter_id=None, datacenter_name=None, da vim["type"], module, type(e).__name__, str(e)), HTTP_Bad_Request) try: - if 'nfvo_tenant_id' in vim: - thread_id = vim["datacenter_id"] + "." + vim['nfvo_tenant_id'] + if 'datacenter_tenant_id' in vim: + thread_id = vim["datacenter_tenant_id"] if thread_id not in vim_persistent_info: vim_persistent_info[thread_id] = {} persistent_info = vim_persistent_info[thread_id] @@ -1834,33 +1866,40 @@ def unify_cloud_config(cloud_config_preserve, cloud_config): return new_cloud_config -def get_vim_thread(tenant_id, datacenter_id_name=None, datacenter_tenant_id=None): +def get_vim_thread(mydb, tenant_id, datacenter_id_name=None, datacenter_tenant_id=None): datacenter_id = None datacenter_name = None thread = None - if datacenter_id_name: - if utils.check_valid_uuid(datacenter_id_name): - datacenter_id = datacenter_id_name + try: + if datacenter_tenant_id: + thread_id = datacenter_tenant_id + thread = vim_threads["running"].get(datacenter_tenant_id) else: - datacenter_name = datacenter_id_name - if datacenter_id: - thread = vim_threads["running"].get(datacenter_id + "." + tenant_id) - else: - for k, v in vim_threads["running"].items(): - datacenter_tenant = k.split(".") - if datacenter_tenant[0] == datacenter_id and datacenter_tenant[1] == tenant_id: - if thread: - raise NfvoException("More than one datacenters found, try to identify with uuid", HTTP_Conflict) - thread = v - elif not datacenter_id and datacenter_tenant[1] == tenant_id: - if thread.datacenter_name == datacenter_name: - if thread: - raise NfvoException("More than one datacenters found, try to identify with uuid", HTTP_Conflict) - thread = v - if not thread: - raise NfvoException("datacenter '{}' not found".format(str(datacenter_id_name)), HTTP_Not_Found) - return thread - + where_={"td.nfvo_tenant_id": tenant_id} + if datacenter_id_name: + if utils.check_valid_uuid(datacenter_id_name): + datacenter_id = datacenter_id_name + where_["dt.datacenter_id"] = datacenter_id + else: + datacenter_name = datacenter_id_name + where_["d.name"] = datacenter_name + if datacenter_tenant_id: + where_["dt.uuid"] = datacenter_tenant_id + datacenters = mydb.get_rows( + SELECT=("dt.uuid as datacenter_tenant_id",), + FROM="datacenter_tenants as dt join tenants_datacenters as td on dt.uuid=td.datacenter_tenant_id " + "join datacenters as d on d.uuid=dt.datacenter_id", + WHERE=where_) + if len(datacenters) > 1: + raise NfvoException("More than one datacenters found, try to identify with uuid", HTTP_Conflict) + elif datacenters: + thread_id = datacenters[0]["datacenter_tenant_id"] + thread = vim_threads["running"].get(thread_id) + if not thread: + raise NfvoException("datacenter '{}' not found".format(str(datacenter_id_name)), HTTP_Not_Found) + return thread_id, thread + except db_base_Exception as e: + raise NfvoException("{} {}".format(type(e).__name__ , str(e)), e.http_code) def get_datacenter_by_name_uuid(mydb, tenant_id, datacenter_id_name=None, **extra_filter): datacenter_id = None @@ -1898,13 +1937,14 @@ def create_instance(mydb, tenant_id, instance_dict): #find main datacenter myvims = {} - myvim_threads = {} - datacenter2tenant = {} + myvim_threads_id = {} + instance_tasks={} + tasks_to_launch={} datacenter = instance_dict.get("datacenter") default_datacenter_id, vim = get_datacenter_by_name_uuid(mydb, tenant_id, datacenter) myvims[default_datacenter_id] = vim - myvim_threads[default_datacenter_id] = get_vim_thread(tenant_id, default_datacenter_id) - datacenter2tenant[default_datacenter_id] = vim['config']['datacenter_tenant_id'] + myvim_threads_id[default_datacenter_id], _ = get_vim_thread(mydb, tenant_id, default_datacenter_id) + tasks_to_launch[myvim_threads_id[default_datacenter_id]] = [] #myvim_tenant = myvim['tenant_id'] # default_datacenter_name = vim['name'] rollbackList=[] @@ -1924,7 +1964,6 @@ def create_instance(mydb, tenant_id, instance_dict): logger.debug("Creating instance from scenario-dict:\n%s", yaml.safe_dump(scenarioDict, indent=4, default_flow_style=False)) #TODO remove instance_name = instance_dict["name"] instance_description = instance_dict.get("description") - instance_tasks={} try: # 0 check correct parameters for net_name, net_instance_desc in instance_dict.get("networks",{}).iteritems(): @@ -1944,8 +1983,8 @@ def create_instance(mydb, tenant_id, instance_dict): #Add this datacenter to myvims d, v = get_datacenter_by_name_uuid(mydb, tenant_id, site["datacenter"]) myvims[d] = v - myvim_threads[d] = get_vim_thread(tenant_id, site["datacenter"]) - datacenter2tenant[d] = v['config']['datacenter_tenant_id'] + myvim_threads_id[d],_ = get_vim_thread(mydb, tenant_id, site["datacenter"]) + tasks_to_launch[myvim_threads_id[d]] = [] site["datacenter"] = d #change name to id else: if site_without_datacenter_field: @@ -1966,8 +2005,8 @@ def create_instance(mydb, tenant_id, instance_dict): if vnf_instance_desc["datacenter"] not in myvims: d, v = get_datacenter_by_name_uuid(mydb, tenant_id, vnf_instance_desc["datacenter"]) myvims[d] = v - myvim_threads[d] = get_vim_thread(tenant_id, vnf_instance_desc["datacenter"]) - datacenter2tenant[d] = v['config']['datacenter_tenant_id'] + myvim_threads_id[d],_ = get_vim_thread(mydb, tenant_id, vnf_instance_desc["datacenter"]) + tasks_to_launch[myvim_threads_id[d]] = [] scenario_vnf["datacenter"] = vnf_instance_desc["datacenter"] #0.1 parse cloud-config parameters @@ -2018,11 +2057,11 @@ def create_instance(mydb, tenant_id, instance_dict): if site.get("datacenter"): vim = myvims[ site["datacenter"] ] datacenter_id = site["datacenter"] - myvim_thread = myvim_threads[ site["datacenter"] ] + myvim_thread_id = myvim_threads_id[ site["datacenter"] ] else: vim = myvims[ default_datacenter_id ] datacenter_id = default_datacenter_id - myvim_thread = myvim_threads[default_datacenter_id] + myvim_thread_id = myvim_threads_id[default_datacenter_id] net_type = sce_net['type'] lookfor_filter = {'admin_state_up': True, 'status': 'ACTIVE'} #'shared': True if sce_net["external"]: @@ -2081,8 +2120,9 @@ def create_instance(mydb, tenant_id, instance_dict): if create_network: #if network is not external task = new_task("new-net", (net_vim_name, net_type, sce_net.get('ip_profile',None))) - task_id = myvim_thread.insert_task(task) + task_id = task["id"] instance_tasks[task_id] = task + tasks_to_launch[myvim_thread_id].append(task) #network_id = vim.new_network(net_vim_name, net_type, sce_net.get('ip_profile',None)) sce_net["vim_id_sites"][datacenter_id] = task_id auxNetDict['scenario'][sce_net['uuid']][datacenter_id] = task_id @@ -2096,11 +2136,11 @@ def create_instance(mydb, tenant_id, instance_dict): if sce_vnf.get("datacenter"): vim = myvims[ sce_vnf["datacenter"] ] datacenter_id = sce_vnf["datacenter"] - myvim_thread = myvim_threads[ sce_vnf["datacenter"]] + myvim_thread_id = myvim_threads_id[ sce_vnf["datacenter"]] else: vim = myvims[ default_datacenter_id ] datacenter_id = default_datacenter_id - myvim_thread = myvim_threads[default_datacenter_id] + myvim_thread_id = myvim_threads_id[default_datacenter_id] descriptor_net = instance_dict.get("vnfs",{}).get(sce_vnf["name"],{}) net_name = descriptor_net.get("name") if not net_name: @@ -2108,8 +2148,9 @@ def create_instance(mydb, tenant_id, instance_dict): net_name = net_name[:255] #limit length net_type = net['type'] task = new_task("new-net", (net_name, net_type, net.get('ip_profile',None))) - task_id = myvim_thread.insert_task(task) + task_id = task["id"] instance_tasks[task_id] = task + tasks_to_launch[myvim_thread_id].append(task) # network_id = vim.new_network(net_name, net_type, net.get('ip_profile',None)) net['vim_id'] = task_id if sce_vnf['uuid'] not in auxNetDict: @@ -2127,11 +2168,11 @@ def create_instance(mydb, tenant_id, instance_dict): for sce_vnf in scenarioDict['vnfs']: if sce_vnf.get("datacenter"): vim = myvims[ sce_vnf["datacenter"] ] - myvim_thread = myvim_threads[ sce_vnf["datacenter"] ] + myvim_thread_id = myvim_threads_id[ sce_vnf["datacenter"] ] datacenter_id = sce_vnf["datacenter"] else: vim = myvims[ default_datacenter_id ] - myvim_thread = myvim_threads[ default_datacenter_id ] + myvim_thread_id = myvim_threads_id[ default_datacenter_id ] datacenter_id = default_datacenter_id sce_vnf["datacenter_id"] = datacenter_id i = 0 @@ -2223,7 +2264,7 @@ def create_instance(mydb, tenant_id, instance_dict): break else: netDict['net_id'] = auxNetDict[ sce_vnf['uuid'] ][ iface['net_id'] ] - if is_task_id(netDict['net_id']): + if netDict.get('net_id') and is_task_id(netDict['net_id']): task_depends[netDict['net_id']] = instance_tasks[netDict['net_id']] #skip bridge ifaces not connected to any net #if 'net_id' not in netDict or netDict['net_id']==None: @@ -2241,9 +2282,9 @@ def create_instance(mydb, tenant_id, instance_dict): task = new_task("new-vm", (myVMDict['name'], myVMDict['description'], myVMDict.get('start', None), myVMDict['imageRef'], myVMDict['flavorRef'], myVMDict['networks'], cloud_config_vm, myVMDict['disks']), depends=task_depends) - vm_id = myvim_thread.insert_task(task) - instance_tasks[vm_id] = task - + instance_tasks[task["id"]] = task + tasks_to_launch[myvim_thread_id].append(task) + vm_id = task["id"] vm['vim_id'] = vm_id rollbackList.append({'what':'vm','where':'vim','vim_id':datacenter_id,'uuid':vm_id}) #put interface uuid back to scenario[vnfs][vms[[interfaces] @@ -2253,19 +2294,24 @@ def create_instance(mydb, tenant_id, instance_dict): if net["name"]==iface["internal_name"]: iface["vim_id"]=net["vim_id"] break - scenarioDict["datacenter2tenant"] = datacenter2tenant + scenarioDict["datacenter2tenant"] = myvim_threads_id logger.debug("create_instance Deployment done scenarioDict: %s", yaml.safe_dump(scenarioDict, indent=4, default_flow_style=False) ) instance_id = mydb.new_instance_scenario_as_a_whole(tenant_id,instance_name, instance_description, scenarioDict) - # Update database with those ended tasks - for task in instance_tasks.values(): - if task["status"] == "ok": - if task["name"] == "new-vm": - mydb.update_rows("instance_vms", UPDATE={"vim_vm_id": task["result"]}, - WHERE={"vim_vm_id": task["id"]}) - elif task["name"] == "new-net": - mydb.update_rows("instance_nets", UPDATE={"vim_net_id": task["result"]}, - WHERE={"vim_net_id": task["id"]}) + for myvim_thread_id,task_list in tasks_to_launch.items(): + for task in task_list: + vim_threads["running"][myvim_thread_id].insert_task(task) + + global_instance_tasks[instance_id] = instance_tasks + # Update database with those ended instance_tasks + # for task in instance_tasks.values(): + # if task["status"] == "ok": + # if task["name"] == "new-vm": + # mydb.update_rows("instance_vms", UPDATE={"vim_vm_id": task["result"]}, + # WHERE={"vim_vm_id": task["id"]}) + # elif task["name"] == "new-net": + # mydb.update_rows("instance_nets", UPDATE={"vim_net_id": task["result"]}, + # WHERE={"vim_net_id": task["id"]}) return mydb.get_instance_scenario(instance_id) except (NfvoException, vimconn.vimconnException,db_base_Exception) as e: message = rollback(mydb, myvims, rollbackList) @@ -2301,7 +2347,7 @@ def delete_instance(mydb, tenant_id, instance_id): datacenter_key = (sce_vnf["datacenter_id"], sce_vnf["datacenter_tenant_id"]) if datacenter_key not in myvims: try: - myvim_thread = get_vim_thread(tenant_id, sce_vnf["datacenter_id"], sce_vnf["datacenter_tenant_id"]) + _,myvim_thread = get_vim_thread(mydb, tenant_id, sce_vnf["datacenter_id"], sce_vnf["datacenter_tenant_id"]) except NfvoException as e: logger.error(str(e)) myvim_thread = None @@ -2324,7 +2370,7 @@ def delete_instance(mydb, tenant_id, instance_id): task=None if is_task_id(vm['vim_vm_id']): task_id = vm['vim_vm_id'] - old_task = task_dict.get(task_id) + old_task = global_instance_tasks[instance_id].get(task_id) if not old_task: error_msg += "\n VM was scheduled for create, but task {} is not found".format(task_id) continue @@ -2334,11 +2380,11 @@ def delete_instance(mydb, tenant_id, instance_id): elif old_task["status"] == "error": continue elif old_task["status"] == "processing": - task = new_task("del-vm", task_id, depends={task_id: old_task}) + task = new_task("del-vm", (task_id, vm["interfaces"]), depends={task_id: old_task}) else: #ok - task = new_task("del-vm", old_task["result"]) + task = new_task("del-vm", (old_task["result"], vm["interfaces"])) else: - task = new_task("del-vm", vm['vim_vm_id'], store=False) + task = new_task("del-vm", (vm['vim_vm_id'], vm["interfaces"]) ) if task: myvim_thread.insert_task(task) except vimconn.vimconnNotFoundException as e: @@ -2358,7 +2404,7 @@ def delete_instance(mydb, tenant_id, instance_id): datacenter_key = (net["datacenter_id"], net["datacenter_tenant_id"]) if datacenter_key not in myvims: try: - myvim_thread = get_vim_thread(tenant_id, sce_vnf["datacenter_id"], sce_vnf["datacenter_tenant_id"]) + _,myvim_thread = get_vim_thread(mydb, tenant_id, sce_vnf["datacenter_id"], sce_vnf["datacenter_tenant_id"]) except NfvoException as e: logger.error(str(e)) myvim_thread = None @@ -2380,7 +2426,7 @@ def delete_instance(mydb, tenant_id, instance_id): task = None if is_task_id(net['vim_net_id']): task_id = net['vim_net_id'] - old_task = task_dict.get(task_id) + old_task = global_instance_tasks[instance_id].get(task_id) if not old_task: error_msg += "\n NET was scheduled for create, but task {} is not found".format(task_id) continue @@ -2394,7 +2440,7 @@ def delete_instance(mydb, tenant_id, instance_id): else: # ok task = new_task("del-net", old_task["result"]) else: - task = new_task("del-net", net['vim_net_id'], store=False) + task = new_task("del-net", (net['vim_net_id'], net['sdn_net_id'])) if task: myvim_thread.insert_task(task) except vimconn.vimconnNotFoundException as e: @@ -2419,161 +2465,161 @@ def refresh_instance(mydb, nfvo_tenant, instanceDict, datacenter=None, vim_tenan - result: <0 if there is any unexpected error, n>=0 if no errors where n is the number of vms and nets that couldn't be updated in the database - error_msg ''' - # Assumption: nfvo_tenant and instance_id were checked before entering into this function - #print "nfvo.refresh_instance begins" - #print json.dumps(instanceDict, indent=4) - - #print "Getting the VIM URL and the VIM tenant_id" - myvims={} - - # 1. Getting VIM vm and net list - vms_updated = [] #List of VM instance uuids in openmano that were updated - vms_notupdated=[] - vm_list = {} - for sce_vnf in instanceDict['vnfs']: - datacenter_key = (sce_vnf["datacenter_id"], sce_vnf["datacenter_tenant_id"]) - if datacenter_key not in vm_list: - vm_list[datacenter_key] = [] - if datacenter_key not in myvims: - vims = get_vim(mydb, nfvo_tenant, datacenter_id=sce_vnf["datacenter_id"], - datacenter_tenant_id=sce_vnf["datacenter_tenant_id"]) - if len(vims) == 0: - logger.error("datacenter '{}' with datacenter_tenant_id '{}' not found".format(sce_vnf["datacenter_id"], sce_vnf["datacenter_tenant_id"])) - myvims[datacenter_key] = None - else: - myvims[datacenter_key] = vims.values()[0] - for vm in sce_vnf['vms']: - vm_list[datacenter_key].append(vm['vim_vm_id']) - vms_notupdated.append(vm["uuid"]) - - nets_updated = [] #List of VM instance uuids in openmano that were updated - nets_notupdated=[] - net_list = {} - for net in instanceDict['nets']: - datacenter_key = (net["datacenter_id"], net["datacenter_tenant_id"]) - if datacenter_key not in net_list: - net_list[datacenter_key] = [] - if datacenter_key not in myvims: - vims = get_vim(mydb, nfvo_tenant, datacenter_id=net["datacenter_id"], - datacenter_tenant_id=net["datacenter_tenant_id"]) - if len(vims) == 0: - logger.error("datacenter '{}' with datacenter_tenant_id '{}' not found".format(net["datacenter_id"], net["datacenter_tenant_id"])) - myvims[datacenter_key] = None - else: - myvims[datacenter_key] = vims.values()[0] - - net_list[datacenter_key].append(net['vim_net_id']) - nets_notupdated.append(net["uuid"]) - - # 1. Getting the status of all VMs - vm_dict={} - for datacenter_key in myvims: - if not vm_list.get(datacenter_key): - continue - failed = True - failed_message="" - if not myvims[datacenter_key]: - failed_message = "datacenter '{}' with datacenter_tenant_id '{}' not found".format(net["datacenter_id"], net["datacenter_tenant_id"]) - else: - try: - vm_dict.update(myvims[datacenter_key].refresh_vms_status(vm_list[datacenter_key]) ) - failed = False - except vimconn.vimconnException as e: - logger.error("VIM exception %s %s", type(e).__name__, str(e)) - failed_message = str(e) - if failed: - for vm in vm_list[datacenter_key]: - vm_dict[vm] = {'status': "VIM_ERROR", 'error_msg': failed_message} - - # 2. Update the status of VMs in the instanceDict, while collects the VMs whose status changed - for sce_vnf in instanceDict['vnfs']: - for vm in sce_vnf['vms']: - vm_id = vm['vim_vm_id'] - interfaces = vm_dict[vm_id].pop('interfaces', []) - #2.0 look if contain manamgement interface, and if not change status from ACTIVE:NoMgmtIP to ACTIVE - has_mgmt_iface = False - for iface in vm["interfaces"]: - if iface["type"]=="mgmt": - has_mgmt_iface = True - if vm_dict[vm_id]['status'] == "ACTIVE:NoMgmtIP" and not has_mgmt_iface: - vm_dict[vm_id]['status'] = "ACTIVE" - if vm_dict[vm_id].get('error_msg') and len(vm_dict[vm_id]['error_msg']) >= 1024: - vm_dict[vm_id]['error_msg'] = vm_dict[vm_id]['error_msg'][:516] + " ... " + vm_dict[vm_id]['error_msg'][-500:] - if vm['status'] != vm_dict[vm_id]['status'] or vm.get('error_msg')!=vm_dict[vm_id].get('error_msg') or vm.get('vim_info')!=vm_dict[vm_id].get('vim_info'): - vm['status'] = vm_dict[vm_id]['status'] - vm['error_msg'] = vm_dict[vm_id].get('error_msg') - vm['vim_info'] = vm_dict[vm_id].get('vim_info') - # 2.1. Update in openmano DB the VMs whose status changed - try: - updates = mydb.update_rows('instance_vms', UPDATE=vm_dict[vm_id], WHERE={'uuid':vm["uuid"]}) - vms_notupdated.remove(vm["uuid"]) - if updates>0: - vms_updated.append(vm["uuid"]) - except db_base_Exception as e: - logger.error("nfvo.refresh_instance error database update: %s", str(e)) - # 2.2. Update in openmano DB the interface VMs - for interface in interfaces: - #translate from vim_net_id to instance_net_id - network_id_list=[] - for net in instanceDict['nets']: - if net["vim_net_id"] == interface["vim_net_id"]: - network_id_list.append(net["uuid"]) - if not network_id_list: - continue - del interface["vim_net_id"] - try: - for network_id in network_id_list: - mydb.update_rows('instance_interfaces', UPDATE=interface, WHERE={'instance_vm_id':vm["uuid"], "instance_net_id":network_id}) - except db_base_Exception as e: - logger.error( "nfvo.refresh_instance error with vm=%s, interface_net_id=%s", vm["uuid"], network_id) - - # 3. Getting the status of all nets - net_dict = {} - for datacenter_key in myvims: - if not net_list.get(datacenter_key): - continue - failed = True - failed_message = "" - if not myvims[datacenter_key]: - failed_message = "datacenter '{}' with datacenter_tenant_id '{}' not found".format(net["datacenter_id"], net["datacenter_tenant_id"]) - else: - try: - net_dict.update(myvims[datacenter_key].refresh_nets_status(net_list[datacenter_key]) ) - failed = False - except vimconn.vimconnException as e: - logger.error("VIM exception %s %s", type(e).__name__, str(e)) - failed_message = str(e) - if failed: - for net in net_list[datacenter_key]: - net_dict[net] = {'status': "VIM_ERROR", 'error_msg': failed_message} - - # 4. Update the status of nets in the instanceDict, while collects the nets whose status changed - # TODO: update nets inside a vnf - for net in instanceDict['nets']: - net_id = net['vim_net_id'] - if net_dict[net_id].get('error_msg') and len(net_dict[net_id]['error_msg']) >= 1024: - net_dict[net_id]['error_msg'] = net_dict[net_id]['error_msg'][:516] + " ... " + net_dict[vm_id]['error_msg'][-500:] - if net['status'] != net_dict[net_id]['status'] or net.get('error_msg')!=net_dict[net_id].get('error_msg') or net.get('vim_info')!=net_dict[net_id].get('vim_info'): - net['status'] = net_dict[net_id]['status'] - net['error_msg'] = net_dict[net_id].get('error_msg') - net['vim_info'] = net_dict[net_id].get('vim_info') - # 5.1. Update in openmano DB the nets whose status changed - try: - updated = mydb.update_rows('instance_nets', UPDATE=net_dict[net_id], WHERE={'uuid':net["uuid"]}) - nets_notupdated.remove(net["uuid"]) - if updated>0: - nets_updated.append(net["uuid"]) - except db_base_Exception as e: - logger.error("nfvo.refresh_instance error database update: %s", str(e)) - - # Returns appropriate output - #print "nfvo.refresh_instance finishes" - logger.debug("VMs updated in the database: %s; nets updated in the database %s; VMs not updated: %s; nets not updated: %s", - str(vms_updated), str(nets_updated), str(vms_notupdated), str(nets_notupdated)) + # # Assumption: nfvo_tenant and instance_id were checked before entering into this function + # #print "nfvo.refresh_instance begins" + # #print json.dumps(instanceDict, indent=4) + # + # #print "Getting the VIM URL and the VIM tenant_id" + # myvims={} + # + # # 1. Getting VIM vm and net list + # vms_updated = [] #List of VM instance uuids in openmano that were updated + # vms_notupdated=[] + # vm_list = {} + # for sce_vnf in instanceDict['vnfs']: + # datacenter_key = (sce_vnf["datacenter_id"], sce_vnf["datacenter_tenant_id"]) + # if datacenter_key not in vm_list: + # vm_list[datacenter_key] = [] + # if datacenter_key not in myvims: + # vims = get_vim(mydb, nfvo_tenant, datacenter_id=sce_vnf["datacenter_id"], + # datacenter_tenant_id=sce_vnf["datacenter_tenant_id"]) + # if len(vims) == 0: + # logger.error("datacenter '{}' with datacenter_tenant_id '{}' not found".format(sce_vnf["datacenter_id"], sce_vnf["datacenter_tenant_id"])) + # myvims[datacenter_key] = None + # else: + # myvims[datacenter_key] = vims.values()[0] + # for vm in sce_vnf['vms']: + # vm_list[datacenter_key].append(vm['vim_vm_id']) + # vms_notupdated.append(vm["uuid"]) + # + # nets_updated = [] #List of VM instance uuids in openmano that were updated + # nets_notupdated=[] + # net_list = {} + # for net in instanceDict['nets']: + # datacenter_key = (net["datacenter_id"], net["datacenter_tenant_id"]) + # if datacenter_key not in net_list: + # net_list[datacenter_key] = [] + # if datacenter_key not in myvims: + # vims = get_vim(mydb, nfvo_tenant, datacenter_id=net["datacenter_id"], + # datacenter_tenant_id=net["datacenter_tenant_id"]) + # if len(vims) == 0: + # logger.error("datacenter '{}' with datacenter_tenant_id '{}' not found".format(net["datacenter_id"], net["datacenter_tenant_id"])) + # myvims[datacenter_key] = None + # else: + # myvims[datacenter_key] = vims.values()[0] + # + # net_list[datacenter_key].append(net['vim_net_id']) + # nets_notupdated.append(net["uuid"]) + # + # # 1. Getting the status of all VMs + # vm_dict={} + # for datacenter_key in myvims: + # if not vm_list.get(datacenter_key): + # continue + # failed = True + # failed_message="" + # if not myvims[datacenter_key]: + # failed_message = "datacenter '{}' with datacenter_tenant_id '{}' not found".format(net["datacenter_id"], net["datacenter_tenant_id"]) + # else: + # try: + # vm_dict.update(myvims[datacenter_key].refresh_vms_status(vm_list[datacenter_key]) ) + # failed = False + # except vimconn.vimconnException as e: + # logger.error("VIM exception %s %s", type(e).__name__, str(e)) + # failed_message = str(e) + # if failed: + # for vm in vm_list[datacenter_key]: + # vm_dict[vm] = {'status': "VIM_ERROR", 'error_msg': failed_message} + # + # # 2. Update the status of VMs in the instanceDict, while collects the VMs whose status changed + # for sce_vnf in instanceDict['vnfs']: + # for vm in sce_vnf['vms']: + # vm_id = vm['vim_vm_id'] + # interfaces = vm_dict[vm_id].pop('interfaces', []) + # #2.0 look if contain manamgement interface, and if not change status from ACTIVE:NoMgmtIP to ACTIVE + # has_mgmt_iface = False + # for iface in vm["interfaces"]: + # if iface["type"]=="mgmt": + # has_mgmt_iface = True + # if vm_dict[vm_id]['status'] == "ACTIVE:NoMgmtIP" and not has_mgmt_iface: + # vm_dict[vm_id]['status'] = "ACTIVE" + # if vm_dict[vm_id].get('error_msg') and len(vm_dict[vm_id]['error_msg']) >= 1024: + # vm_dict[vm_id]['error_msg'] = vm_dict[vm_id]['error_msg'][:516] + " ... " + vm_dict[vm_id]['error_msg'][-500:] + # if vm['status'] != vm_dict[vm_id]['status'] or vm.get('error_msg')!=vm_dict[vm_id].get('error_msg') or vm.get('vim_info')!=vm_dict[vm_id].get('vim_info'): + # vm['status'] = vm_dict[vm_id]['status'] + # vm['error_msg'] = vm_dict[vm_id].get('error_msg') + # vm['vim_info'] = vm_dict[vm_id].get('vim_info') + # # 2.1. Update in openmano DB the VMs whose status changed + # try: + # updates = mydb.update_rows('instance_vms', UPDATE=vm_dict[vm_id], WHERE={'uuid':vm["uuid"]}) + # vms_notupdated.remove(vm["uuid"]) + # if updates>0: + # vms_updated.append(vm["uuid"]) + # except db_base_Exception as e: + # logger.error("nfvo.refresh_instance error database update: %s", str(e)) + # # 2.2. Update in openmano DB the interface VMs + # for interface in interfaces: + # #translate from vim_net_id to instance_net_id + # network_id_list=[] + # for net in instanceDict['nets']: + # if net["vim_net_id"] == interface["vim_net_id"]: + # network_id_list.append(net["uuid"]) + # if not network_id_list: + # continue + # del interface["vim_net_id"] + # try: + # for network_id in network_id_list: + # mydb.update_rows('instance_interfaces', UPDATE=interface, WHERE={'instance_vm_id':vm["uuid"], "instance_net_id":network_id}) + # except db_base_Exception as e: + # logger.error( "nfvo.refresh_instance error with vm=%s, interface_net_id=%s", vm["uuid"], network_id) + # + # # 3. Getting the status of all nets + # net_dict = {} + # for datacenter_key in myvims: + # if not net_list.get(datacenter_key): + # continue + # failed = True + # failed_message = "" + # if not myvims[datacenter_key]: + # failed_message = "datacenter '{}' with datacenter_tenant_id '{}' not found".format(net["datacenter_id"], net["datacenter_tenant_id"]) + # else: + # try: + # net_dict.update(myvims[datacenter_key].refresh_nets_status(net_list[datacenter_key]) ) + # failed = False + # except vimconn.vimconnException as e: + # logger.error("VIM exception %s %s", type(e).__name__, str(e)) + # failed_message = str(e) + # if failed: + # for net in net_list[datacenter_key]: + # net_dict[net] = {'status': "VIM_ERROR", 'error_msg': failed_message} + # + # # 4. Update the status of nets in the instanceDict, while collects the nets whose status changed + # # TODO: update nets inside a vnf + # for net in instanceDict['nets']: + # net_id = net['vim_net_id'] + # if net_dict[net_id].get('error_msg') and len(net_dict[net_id]['error_msg']) >= 1024: + # net_dict[net_id]['error_msg'] = net_dict[net_id]['error_msg'][:516] + " ... " + net_dict[vm_id]['error_msg'][-500:] + # if net['status'] != net_dict[net_id]['status'] or net.get('error_msg')!=net_dict[net_id].get('error_msg') or net.get('vim_info')!=net_dict[net_id].get('vim_info'): + # net['status'] = net_dict[net_id]['status'] + # net['error_msg'] = net_dict[net_id].get('error_msg') + # net['vim_info'] = net_dict[net_id].get('vim_info') + # # 5.1. Update in openmano DB the nets whose status changed + # try: + # updated = mydb.update_rows('instance_nets', UPDATE=net_dict[net_id], WHERE={'uuid':net["uuid"]}) + # nets_notupdated.remove(net["uuid"]) + # if updated>0: + # nets_updated.append(net["uuid"]) + # except db_base_Exception as e: + # logger.error("nfvo.refresh_instance error database update: %s", str(e)) + # + # # Returns appropriate output + # #print "nfvo.refresh_instance finishes" + # logger.debug("VMs updated in the database: %s; nets updated in the database %s; VMs not updated: %s; nets not updated: %s", + # str(vms_updated), str(nets_updated), str(vms_notupdated), str(nets_notupdated)) instance_id = instanceDict['uuid'] - if len(vms_notupdated)+len(nets_notupdated)>0: - error_msg = "VMs not updated: " + str(vms_notupdated) + "; nets not updated: " + str(nets_notupdated) - return len(vms_notupdated)+len(nets_notupdated), 'Scenario instance ' + instance_id + ' refreshed but some elements could not be updated in the database: ' + error_msg + # if len(vms_notupdated)+len(nets_notupdated)>0: + # error_msg = "VMs not updated: " + str(vms_notupdated) + "; nets not updated: " + str(nets_notupdated) + # return len(vms_notupdated)+len(nets_notupdated), 'Scenario instance ' + instance_id + ' refreshed but some elements could not be updated in the database: ' + error_msg return 0, 'Scenario instance ' + instance_id + ' refreshed.' @@ -2731,7 +2777,10 @@ def edit_datacenter(mydb, datacenter_id_name, datacenter_descriptor): if new_config_dict[k]==None: to_delete.append(k) - config_dict = yaml.load(datacenter["config"]) + config_text = datacenter.get("config") + if not config_text: + config_text = '{}' + config_dict = yaml.load(config_text) config_dict.update(new_config_dict) #delete null fields for k in to_delete: @@ -2811,12 +2860,48 @@ def associate_datacenter_to_tenant(mydb, nfvo_tenant, datacenter, vim_tenant_id= # create thread datacenter_id, myvim = get_datacenter_by_name_uuid(mydb, tenant_dict['uuid'], datacenter_id) # reload data thread_name = get_non_used_vim_name(datacenter_name, datacenter_id, tenant_dict['name'], tenant_dict['uuid']) - new_thread = vim_thread.vim_thread(myvim, task_lock, thread_name, datacenter_name, db=db, db_lock=db_lock) + new_thread = vim_thread.vim_thread(myvim, task_lock, thread_name, datacenter_name, db=db, db_lock=db_lock, ovim=ovim) new_thread.start() - thread_id = datacenter_id + "." + tenant_dict['uuid'] + thread_id = datacenter_tenants_dict["uuid"] vim_threads["running"][thread_id] = new_thread return datacenter_id +def edit_datacenter_to_tenant(mydb, nfvo_tenant, datacenter_id, vim_tenant_id=None, vim_tenant_name=None, vim_username=None, vim_password=None, config=None): + #Obtain the data of this datacenter_tenant_id + vim_data = mydb.get_rows( + SELECT=("datacenter_tenants.vim_tenant_name", "datacenter_tenants.vim_tenant_id", "datacenter_tenants.user", + "datacenter_tenants.passwd", "datacenter_tenants.config"), + FROM="datacenter_tenants JOIN tenants_datacenters ON datacenter_tenants.uuid=tenants_datacenters.datacenter_tenant_id", + WHERE={"tenants_datacenters.nfvo_tenant_id": nfvo_tenant, + "tenants_datacenters.datacenter_id": datacenter_id}) + + logger.debug(str(vim_data)) + if len(vim_data) < 1: + raise NfvoException("Datacenter {} is not attached for tenant {}".format(datacenter_id, nfvo_tenant), HTTP_Conflict) + + v = vim_data[0] + if v['config']: + v['config'] = yaml.load(v['config']) + + if vim_tenant_id: + v['vim_tenant_id'] = vim_tenant_id + if vim_tenant_name: + v['vim_tenant_name'] = vim_tenant_name + if vim_username: + v['user'] = vim_username + if vim_password: + v['passwd'] = vim_password + if config: + if not v['config']: + v['config'] = {} + v['config'].update(config) + + logger.debug(str(v)) + deassociate_datacenter_to_tenant(mydb, nfvo_tenant, datacenter_id, vim_tenant_id=v['vim_tenant_id']) + associate_datacenter_to_tenant(mydb, nfvo_tenant, datacenter_id, vim_tenant_id=v['vim_tenant_id'], vim_tenant_name=v['vim_tenant_name'], + vim_username=v['user'], vim_password=v['passwd'], config=v['config']) + + return datacenter_id def deassociate_datacenter_to_tenant(mydb, tenant_id, datacenter, vim_tenant_id=None): #get datacenter info @@ -2857,9 +2942,9 @@ def deassociate_datacenter_to_tenant(mydb, tenant_id, datacenter, vim_tenant_id= except db_base_Exception as e: logger.error("Cannot delete datacenter_tenants " + str(e)) pass # the error will be caused because dependencies, vim_tenant can not be deleted - thread_id = datacenter_id + "." + tenant_datacenter_item["nfvo_tenant_id"] + thread_id = tenant_datacenter_item["datacenter_tenant_id"] thread = vim_threads["running"][thread_id] - thread.insert_task(new_task("exit", None, store=False)) + thread.insert_task(new_task("exit", None)) vim_threads["deleting"][thread_id] = thread return "datacenter {} detached. {}".format(datacenter_id, warning) @@ -3054,3 +3139,122 @@ def vim_action_create(mydb, tenant_id, datacenter, item, descriptor): raise NfvoException("Not possible to create {} at VIM: {}".format(item, str(e)), e.http_code) return vim_action_get(mydb, tenant_id, datacenter, item, content) + +def sdn_controller_create(mydb, tenant_id, sdn_controller): + data = ovim.new_of_controller(sdn_controller) + logger.debug('New SDN controller created with uuid {}'.format(data)) + return data + +def sdn_controller_update(mydb, tenant_id, controller_id, sdn_controller): + data = ovim.edit_of_controller(controller_id, sdn_controller) + msg = 'SDN controller {} updated'.format(data) + logger.debug(msg) + return msg + +def sdn_controller_list(mydb, tenant_id, controller_id=None): + if controller_id == None: + data = ovim.get_of_controllers() + else: + data = ovim.show_of_controller(controller_id) + + msg = 'SDN controller list:\n {}'.format(data) + logger.debug(msg) + return data + +def sdn_controller_delete(mydb, tenant_id, controller_id): + select_ = ('uuid', 'config') + datacenters = mydb.get_rows(FROM='datacenters', SELECT=select_) + for datacenter in datacenters: + if datacenter['config']: + config = yaml.load(datacenter['config']) + if 'sdn-controller' in config and config['sdn-controller'] == controller_id: + raise NfvoException("SDN controller {} is in use by datacenter {}".format(controller_id, datacenter['uuid']), HTTP_Conflict) + + data = ovim.delete_of_controller(controller_id) + msg = 'SDN controller {} deleted'.format(data) + logger.debug(msg) + return msg + +def datacenter_sdn_port_mapping_set(mydb, tenant_id, datacenter_id, sdn_port_mapping): + controller = mydb.get_rows(FROM="datacenters", SELECT=("config",), WHERE={"uuid":datacenter_id}) + if len(controller) < 1: + raise NfvoException("Datacenter {} not present in the database".format(datacenter_id), HTTP_Not_Found) + + try: + sdn_controller_id = yaml.load(controller[0]["config"])["sdn-controller"] + except: + raise NfvoException("The datacenter {} has not an SDN controller associated".format(datacenter_id), HTTP_Bad_Request) + + sdn_controller = ovim.show_of_controller(sdn_controller_id) + switch_dpid = sdn_controller["dpid"] + + maps = list() + for compute_node in sdn_port_mapping: + #element = {"ofc_id": sdn_controller_id, "region": datacenter_id, "switch_dpid": switch_dpid} + element = dict() + element["compute_node"] = compute_node["compute_node"] + for port in compute_node["ports"]: + element["pci"] = port.get("pci") + element["switch_port"] = port.get("switch_port") + element["switch_mac"] = port.get("switch_mac") + if not element["pci"] or not (element["switch_port"] or element["switch_mac"]): + raise NfvoException ("The mapping must contain the 'pci' and at least one of the elements 'switch_port'" + " or 'switch_mac'", HTTP_Bad_Request) + maps.append(dict(element)) + + return ovim.set_of_port_mapping(maps, ofc_id=sdn_controller_id, switch_dpid=switch_dpid, region=datacenter_id) + +def datacenter_sdn_port_mapping_list(mydb, tenant_id, datacenter_id): + maps = ovim.get_of_port_mappings(db_filter={"region": datacenter_id}) + + result = { + "sdn-controller": None, + "datacenter-id": datacenter_id, + "dpid": None, + "ports_mapping": list() + } + + datacenter = mydb.get_table_by_uuid_name('datacenters', datacenter_id) + if datacenter['config']: + config = yaml.load(datacenter['config']) + if 'sdn-controller' in config: + controller_id = config['sdn-controller'] + sdn_controller = sdn_controller_list(mydb, tenant_id, controller_id) + result["sdn-controller"] = controller_id + result["dpid"] = sdn_controller["dpid"] + + if result["sdn-controller"] == None or result["dpid"] == None: + raise NfvoException("Not all SDN controller information for datacenter {} could be found: {}".format(datacenter_id, result), + HTTP_Internal_Server_Error) + + if len(maps) == 0: + return result + + ports_correspondence_dict = dict() + for link in maps: + if result["sdn-controller"] != link["ofc_id"]: + raise NfvoException("The sdn-controller specified for different port mappings differ", HTTP_Internal_Server_Error) + if result["dpid"] != link["switch_dpid"]: + raise NfvoException("The dpid specified for different port mappings differ", HTTP_Internal_Server_Error) + element = dict() + element["pci"] = link["pci"] + if link["switch_port"]: + element["switch_port"] = link["switch_port"] + if link["switch_mac"]: + element["switch_mac"] = link["switch_mac"] + + if not link["compute_node"] in ports_correspondence_dict: + content = dict() + content["compute_node"] = link["compute_node"] + content["ports"] = list() + ports_correspondence_dict[link["compute_node"]] = content + + ports_correspondence_dict[link["compute_node"]]["ports"].append(element) + + for key in sorted(ports_correspondence_dict): + result["ports_mapping"].append(ports_correspondence_dict[key]) + + return result + +def datacenter_sdn_port_mapping_delete(mydb, tenant_id, datacenter_id): + return ovim.clear_of_port_mapping(db_filter={"region":datacenter_id}) diff --git a/osm_ro/nfvo_db.py b/osm_ro/nfvo_db.py index b5ec9a0d..ea6d3397 100644 --- a/osm_ro/nfvo_db.py +++ b/osm_ro/nfvo_db.py @@ -755,6 +755,8 @@ class nfvo_db(db_base.db_base): INSERT_={'vim_net_id': vim_id, 'created': net.get('created', False), 'instance_scenario_id':instance_uuid } #, 'type': net['type'] INSERT_['datacenter_id'] = datacenter_site_id INSERT_['datacenter_tenant_id'] = scenarioDict["datacenter2tenant"][datacenter_site_id] + if not net.get('created', False): + INSERT_['status'] = "ACTIVE" if sce_net_id: INSERT_['sce_net_id'] = sce_net_id created_time += 0.00001 @@ -892,7 +894,7 @@ class nfvo_db(db_base.db_base): vm_manage_iface_list=[] #instance_interfaces cmd = "SELECT vim_interface_id, instance_net_id, internal_name,external_name, mac_address,"\ - " ii.ip_address as ip_address, vim_info, i.type as type"\ + " ii.ip_address as ip_address, vim_info, i.type as type, sdn_port_id"\ " FROM instance_interfaces as ii join interfaces as i on ii.interface_id=i.uuid"\ " WHERE instance_vm_id='{}' ORDER BY created_at".format(vm['uuid']) self.logger.debug(cmd) @@ -912,7 +914,7 @@ class nfvo_db(db_base.db_base): #from_text = "instance_nets join instance_scenarios on instance_nets.instance_scenario_id=instance_scenarios.uuid " + \ # "join sce_nets on instance_scenarios.scenario_id=sce_nets.scenario_id" #where_text = "instance_nets.instance_scenario_id='"+ instance_dict['uuid'] + "'" - cmd = "SELECT uuid,vim_net_id,status,error_msg,vim_info,created, sce_net_id, net_id as vnf_net_id, datacenter_id, datacenter_tenant_id"\ + cmd = "SELECT uuid,vim_net_id,status,error_msg,vim_info,created, sce_net_id, net_id as vnf_net_id, datacenter_id, datacenter_tenant_id, sdn_net_id"\ " FROM instance_nets" \ " WHERE instance_scenario_id='{}' ORDER BY created_at".format(instance_dict['uuid']) self.logger.debug(cmd) diff --git a/osm_ro/openmano_schemas.py b/osm_ro/openmano_schemas.py index 1ea64f63..1dfcffe4 100644 --- a/osm_ro/openmano_schemas.py +++ b/osm_ro/openmano_schemas.py @@ -86,6 +86,10 @@ config_schema = { "db_user": nameshort_schema, "db_passwd": {"type":"string"}, "db_name": nameshort_schema, + "db_ovim_host": nameshort_schema, + "db_ovim_user": nameshort_schema, + "db_ovim_passwd": {"type":"string"}, + "db_ovim_name": nameshort_schema, # Next fields will disappear once the MANO API includes appropriate primitives "vim_url": http_schema, "vim_url_admin": http_schema, @@ -109,16 +113,18 @@ config_schema = { "log_level_nfvo": log_level_schema, "log_level_http": log_level_schema, "log_level_console": log_level_schema, + "log_level_ovim": log_level_schema, "log_file_db": path_schema, "log_file_vim": path_schema, "log_file_nfvo": path_schema, "log_file_http": path_schema, "log_file_console": path_schema, + "log_file_ovim": path_schema, "log_socket_host": nameshort_schema, "log_socket_port": port_schema, "log_file": path_schema, }, - "required": ['db_host', 'db_user', 'db_passwd', 'db_name'], + "required": ['db_user', 'db_passwd', 'db_name'], "additionalProperties": False } @@ -1095,3 +1101,75 @@ instance_scenario_action_schema = { #"maxProperties": 1, "additionalProperties": False } + +sdn_controller_properties={ + "name": name_schema, + "dpid": {"type":"string", "pattern":"^[0-9a-fA-F][02468aceACE](:[0-9a-fA-F]{2}){7}$"}, + "ip": ip_schema, + "port": port_schema, + "type": {"type": "string", "enum": ["opendaylight","floodlight","onos"]}, + "version": {"type" : "string", "minLength":1, "maxLength":12}, + "user": nameshort_schema, + "password": passwd_schema +} +sdn_controller_schema = { + "title":"sdn controller information schema", + "$schema": "http://json-schema.org/draft-04/schema#", + "type":"object", + "properties":{ + "sdn_controller":{ + "type":"object", + "properties":sdn_controller_properties, + "required": ["name", "port", 'ip', 'dpid', 'type'], + "additionalProperties": False + } + }, + "required": ["sdn_controller"], + "additionalProperties": False +} + +sdn_controller_edit_schema = { + "title":"sdn controller update information schema", + "$schema": "http://json-schema.org/draft-04/schema#", + "type":"object", + "properties":{ + "sdn_controller":{ + "type":"object", + "properties":sdn_controller_properties, + "additionalProperties": False + } + }, + "required": ["sdn_controller"], + "additionalProperties": False +} + +sdn_port_mapping_schema = { + "$schema": "http://json-schema.org/draft-04/schema#", + "title":"sdn port mapping information schema", + "type": "object", + "properties": { + "sdn_port_mapping": { + "type": "array", + "items": { + "type": "object", + "properties": { + "compute_node": nameshort_schema, + "ports": { + "type": "array", + "items": { + "type": "object", + "properties": { + "pci": pci_schema, + "switch_port": nameshort_schema, + "switch_mac": mac_schema + }, + "required": ["pci"] + } + } + }, + "required": ["compute_node", "ports"] + } + } + }, + "required": ["sdn_port_mapping"] +} \ No newline at end of file diff --git a/osm_ro/openmanoclient.py b/osm_ro/openmanoclient.py index c11f747e..19a430d4 100644 --- a/osm_ro/openmanoclient.py +++ b/osm_ro/openmanoclient.py @@ -25,10 +25,10 @@ ''' openmano python client used to interact with openmano-server ''' -__author__="Alfonso Tierno" +__author__="Alfonso Tierno, Pablo Montes" __date__ ="$09-Mar-2016 09:09:48$" -__version__="0.0.1-r467" -version_date="Mar 2016" +__version__="0.0.2-r468" +version_date="Feb 2017" import requests import json diff --git a/osm_ro/openmanod.cfg b/osm_ro/openmanod.cfg index ebcee2b6..adc5b623 100644 --- a/osm_ro/openmanod.cfg +++ b/osm_ro/openmanod.cfg @@ -44,6 +44,12 @@ db_host: localhost # by default localhost db_user: mano # DB user db_passwd: manopw # DB password db_name: mano_db # Name of the MANO DB +# Database ovim parameters +db_ovim_host: localhost # by default localhost +db_ovim_user: mano # DB user +db_ovim_passwd: manopw # DB password +db_ovim_name: mano_vim_db # Name of the OVIM MANO DB + #other MANO parameters # Folder where the VNF descriptors will be stored @@ -71,6 +77,8 @@ log_level: DEBUG #general log levels for internal logging #log_file_http: /opt/openmano/logs/openmano_http.log #log_level_console: DEBUG #proxy console log levels #log_file_console: /opt/openmano/logs/openmano_console.log +#log_level_ovim: DEBUG #ovim library log levels +#log_file_ovim: /opt/openmano/logs/openmano_ovim.log #Uncomment to send logs via IP to an external host #log_socket_host: localhost diff --git a/osm_ro/vim_thread.py b/osm_ro/vim_thread.py index 42279a23..3c35d6c9 100644 --- a/osm_ro/vim_thread.py +++ b/osm_ro/vim_thread.py @@ -25,7 +25,7 @@ This is thread that interact with the host and the libvirt to manage VM One thread will be launched per host ''' -__author__ = "Alfonso Tierno" +__author__ = "Alfonso Tierno, Pablo Montes" __date__ = "$10-feb-2017 12:07:15$" import threading @@ -34,6 +34,7 @@ import Queue import logging import vimconn from db_base import db_base_Exception +from ovim import ovimException # from logging import Logger @@ -46,7 +47,7 @@ def is_task_id(id): class vim_thread(threading.Thread): - def __init__(self, vimconn, task_lock, name=None, datacenter_name=None, datacenter_tenant_id=None, db=None, db_lock=None): + def __init__(self, vimconn, task_lock, name=None, datacenter_name=None, datacenter_tenant_id=None, db=None, db_lock=None, ovim=None): """Init a thread. Arguments: 'id' number of thead @@ -64,6 +65,7 @@ class vim_thread(threading.Thread): self.vim = vimconn self.datacenter_name = datacenter_name self.datacenter_tenant_id = datacenter_tenant_id + self.ovim = ovim if not name: self.name = vimconn["id"] + "." + vimconn["config"]["datacenter_tenant_id"] else: @@ -75,6 +77,239 @@ class vim_thread(threading.Thread): self.task_lock = task_lock self.task_queue = Queue.Queue(2000) + self.refresh_list = [] + """Contains time ordered task list for refreshing the status of VIM VMs and nets""" + + def _refres_elements(self): + """Call VIM to get VMs and networks status until 10 elements""" + now = time.time() + vm_to_refresh_list = [] + net_to_refresh_list = [] + vm_to_refresh_dict = {} + net_to_refresh_dict = {} + items_to_refresh = 0 + while self.refresh_list: + task = self.refresh_list[0] + with self.task_lock: + if task['status'] == 'deleted': + self.refresh_list.pop(0) + continue + if task['time'] > now: + break + task["status"] = "processing" + self.refresh_list.pop(0) + if task["name"] == 'get-vm': + vm_to_refresh_list.append(task["vim_id"]) + vm_to_refresh_dict[task["vim_id"]] = task + elif task["name"] == 'get-net': + net_to_refresh_list.append(task["vim_id"]) + net_to_refresh_dict[task["vim_id"]] = task + else: + error_text = "unknown task {}".format(task["name"]) + self.logger.error(error_text) + items_to_refresh += 1 + if items_to_refresh == 10: + break + + if vm_to_refresh_list: + try: + vim_dict = self.vim.refresh_vms_status(vm_to_refresh_list) + for vim_id, vim_info in vim_dict.items(): + #look for task + task = vm_to_refresh_dict[vim_id] + self.logger.debug("get-vm vm_id=%s result=%s", task["vim_id"], str(vim_info)) + + # update database + if vim_info.get("error_msg"): + vim_info["error_msg"] = self._format_vim_error_msg(vim_info["error_msg"]) + if task["vim_info"].get("status") != vim_info["status"] or \ + task["vim_info"].get("error_msg") != vim_info.get("error_msg") or \ + task["vim_info"].get("vim_info") != vim_info["vim_info"]: + with self.db_lock: + temp_dict = {"status": vim_info["status"], + "error_msg": vim_info.get("error_msg"), + "vim_info": vim_info["vim_info"]} + self.db.update_rows('instance_vms', UPDATE=temp_dict, WHERE={"vim_vm_id": vim_id}) + for interface in vim_info["interfaces"]: + for task_interface in task["vim_info"]["interfaces"]: + if task_interface["vim_net_id"] == interface["vim_net_id"]: + break + else: + task_interface = {"vim_net_id": interface["vim_net_id"]} + task["vim_info"]["interfaces"].append(task_interface) + if task_interface != interface: + #delete old port + if task_interface.get("sdn_port_id"): + try: + self.ovim.delete_port(task_interface["sdn_port_id"]) + task_interface["sdn_port_id"] = None + except ovimException as e: + self.logger.error("ovimException deleting external_port={} ".format( + task_interface["sdn_port_id"]) + str(e), exc_info=True) + # TODO Set error_msg at instance_nets + vim_net_id = interface.pop("vim_net_id") + sdn_net_id = None + sdn_port_name = None + with self.db_lock: + where_= {'iv.vim_vm_id': vim_id, "ine.vim_net_id": vim_net_id, + 'ine.datacenter_tenant_id': self.datacenter_tenant_id} + # TODO check why vim_interface_id is not present at database + # if interface.get("vim_interface_id"): + # where_["vim_interface_id"] = interface["vim_interface_id"] + db_ifaces = self.db.get_rows( + FROM="instance_interfaces as ii left join instance_nets as ine on " + "ii.instance_net_id=ine.uuid left join instance_vms as iv on " + "ii.instance_vm_id=iv.uuid", + SELECT=("ii.uuid as iface_id", "ine.uuid as net_id", "iv.uuid as vm_id", "sdn_net_id"), + WHERE=where_) + if len(db_ifaces)>1: + self.logger.critical("Refresing interfaces. " + "Found more than one interface at database for '{}'".format(where_)) + elif len(db_ifaces)==0: + self.logger.critical("Refresing interfaces. " + "Not found any interface at database for '{}'".format(where_)) + continue + else: + db_iface = db_ifaces[0] + if db_iface.get("sdn_net_id") and interface.get("compute_node") and interface.get("pci"): + sdn_net_id = db_iface["sdn_net_id"] + sdn_port_name = sdn_net_id + "." + db_iface["vm_id"] + sdn_port_name = sdn_port_name[:63] + try: + sdn_port_id = self.ovim.new_external_port( + {"compute_node": interface["compute_node"], + "pci": interface["pci"], + "vlan": interface.get("vlan"), + "net_id": sdn_net_id, + "region": self.vim["config"]["datacenter_id"], + "name": sdn_port_name, + "mac": interface.get("mac_address")}) + interface["sdn_port_id"] = sdn_port_id + except (ovimException, Exception) as e: + self.logger.error( + "ovimException creating new_external_port compute_node={} " \ + "pci={} vlan={} ".format( + interface["compute_node"], + interface["pci"], + interface.get("vlan")) + str(e), + exc_info=True) + # TODO Set error_msg at instance_nets + with self.db_lock: + self.db.update_rows('instance_interfaces', UPDATE=interface, + WHERE={'uuid': db_iface["iface_id"]}) + # TODO insert instance_id + interface["vim_net_id"] = vim_net_id + + task["vim_info"] = vim_info + if task["vim_info"]["status"] == "BUILD": + self._insert_refresh(task, now+5) # 5seconds + else: + self._insert_refresh(task, now+300) # 5minutes + except vimconn.vimconnException as e: + self.logger.error("vimconnException Exception when trying to refresh vms " + str(e)) + self._insert_refresh(task, now + 300) # 5minutes + + if net_to_refresh_list: + try: + vim_dict = self.vim.refresh_nets_status(net_to_refresh_list) + for vim_id, vim_info in vim_dict.items(): + #look for task + task = net_to_refresh_dict[vim_id] + self.logger.debug("get-net net_id=%s result=%s", task["vim_id"], str(vim_info)) + + #get database info + where_ = {"vim_net_id": vim_id, 'datacenter_tenant_id': self.datacenter_tenant_id} + with self.db_lock: + db_nets = self.db.get_rows( + FROM="instance_nets", + SELECT=("uuid as net_id", "sdn_net_id"), + WHERE=where_) + if len(db_nets) > 1: + self.logger.critical("Refresing networks. " + "Found more than one instance-networks at database for '{}'".format(where_)) + elif len(db_nets) == 0: + self.logger.critical("Refresing networks. " + "Not found any instance-network at database for '{}'".format(where_)) + continue + else: + db_net = db_nets[0] + if db_net.get("sdn_net_id"): + # get ovim status + try: + sdn_net = self.ovim.show_network(db_net["sdn_net_id"]) + if sdn_net["status"] == "ERROR": + if not vim_info.get("error_msg"): + vim_info["error_msg"] = sdn_net["error_msg"] + else: + vim_info["error_msg"] = "VIM_ERROR: {} && SDN_ERROR: {}".format( + self._format_vim_error_msg(vim_info["error_msg"], 1024//2-14), + self._format_vim_error_msg(sdn_net["error_msg"], 1024//2-14)) + if vim_info["status"] == "VIM_ERROR": + vim_info["status"] = "VIM_SDN_ERROR" + else: + vim_info["status"] = "SDN_ERROR" + + except (ovimException, Exception) as e: + self.logger.error( + "ovimException getting network infor snd_net_id={}".format(db_net["sdn_net_id"]), + exc_info=True) + # TODO Set error_msg at instance_nets + + # update database + if vim_info.get("error_msg"): + vim_info["error_msg"] = self._format_vim_error_msg(vim_info["error_msg"]) + if task["vim_info"].get("status") != vim_info["status"] or \ + task["vim_info"].get("error_msg") != vim_info.get("error_msg") or \ + task["vim_info"].get("vim_info") != vim_info["vim_info"]: + with self.db_lock: + temp_dict = {"status": vim_info["status"], + "error_msg": vim_info.get("error_msg"), + "vim_info": vim_info["vim_info"]} + self.db.update_rows('instance_nets', UPDATE=temp_dict, WHERE={"vim_net_id": vim_id}) + + task["vim_info"] = vim_info + if task["vim_info"]["status"] == "BUILD": + self._insert_refresh(task, now+5) # 5seconds + else: + self._insert_refresh(task, now+300) # 5minutes + except vimconn.vimconnException as e: + self.logger.error("vimconnException Exception when trying to refresh nets " + str(e)) + self._insert_refresh(task, now + 300) # 5minutes + + if not items_to_refresh: + time.sleep(1) + + def _insert_refresh(self, task, threshold_time): + """Insert a task at list of refreshing elements. The refreshing list is ordered by threshold_time (task['time'] + It is assumed that this is called inside this thread + """ + task["time"] = threshold_time + for index in range(0, len(self.refresh_list)): + if self.refresh_list[index]["time"] > threshold_time: + self.refresh_list.insert(index, task) + break + else: + index = len(self.refresh_list) + self.refresh_list.append(task) + self.logger.debug("new refresh task={} name={}, time={} index={}".format( + task["id"], task["name"], task["time"], index)) + + def _remove_refresh(self, task_name, vim_id): + """Remove a task with this name and vim_id from the list of refreshing elements. + It is assumed that this is called inside this thread outside _refres_elements method + Return True if self.refresh_list is modified, task is found + Return False if not found + """ + index_to_delete = None + for index in range(0, len(self.refresh_list)): + if self.refresh_list[index]["name"] == task_name and self.refresh_list[index]["vim_id"] == vim_id: + index_to_delete = index + break + else: + return False + if index_to_delete != None: + del self.refresh_list[index_to_delete] + return True def insert_task(self, task): try: @@ -97,64 +332,113 @@ class vim_thread(threading.Thread): while True: #TODO reload service while True: - if not self.task_queue.empty(): - task = self.task_queue.get() - self.task_lock.acquire() - if task["status"] == "deleted": + try: + if not self.task_queue.empty(): + task = self.task_queue.get() + self.task_lock.acquire() + if task["status"] == "deleted": + self.task_lock.release() + continue + task["status"] = "processing" self.task_lock.release() + else: + self._refres_elements() continue - task["status"] == "processing" - self.task_lock.release() - else: - now=time.time() - time.sleep(1) - continue - self.logger.debug("processing task id={} name={} params={}".format(task["id"], task["name"], - str(task["params"]))) - if task["name"] == 'exit' or task["name"] == 'reload': - result, content = self.terminate(task) - elif task["name"] == 'new-vm': - result, content = self.new_vm(task) - elif task["name"] == 'del-vm': - result, content = self.del_vm(task) - elif task["name"] == 'new-net': - result, content = self.new_net(task) - elif task["name"] == 'del-net': - result, content = self.del_net(task) - else: - error_text = "unknown task {}".format(task["name"]) - self.logger.error(error_text) - result = False - content = error_text + self.logger.debug("processing task id={} name={} params={}".format(task["id"], task["name"], + str(task["params"]))) + if task["name"] == 'exit' or task["name"] == 'reload': + result, content = self.terminate(task) + elif task["name"] == 'new-vm': + result, content = self.new_vm(task) + elif task["name"] == 'del-vm': + result, content = self.del_vm(task) + elif task["name"] == 'new-net': + result, content = self.new_net(task) + elif task["name"] == 'del-net': + result, content = self.del_net(task) + else: + error_text = "unknown task {}".format(task["name"]) + self.logger.error(error_text) + result = False + content = error_text + self.logger.debug("task id={} name={} result={}:{} params={}".format(task["id"], task["name"], + result, content, + str(task["params"]))) - with self.task_lock: - task["status"] = "done" if result else "error" - task["result"] = content - self.task_queue.task_done() + with self.task_lock: + task["status"] = "done" if result else "error" + task["result"] = content + self.task_queue.task_done() - if task["name"] == 'exit': - return 0 - elif task["name"] == 'reload': - break + if task["name"] == 'exit': + return 0 + elif task["name"] == 'reload': + break + except Exception as e: + self.logger.critical("Unexpected exception at run: " + str(e), exc_info=True) self.logger.debug("Finishing") def terminate(self, task): return True, None + def _format_vim_error_msg(self, error_text, max_length=1024): + if error_text and len(error_text) >= max_length: + return error_text[:max_length//2-3] + " ... " + error_text[-max_length//2+3:] + return error_text + def new_net(self, task): try: task_id = task["id"] params = task["params"] net_id = self.vim.new_network(*params) + + net_name = params[0] + net_type = params[1] + + network = None + sdn_net_id = None + sdn_controller = self.vim.config.get('sdn-controller') + if sdn_controller and (net_type == "data" or net_type == "ptp"): + network = {"name": net_name, "type": net_type} + + vim_net = self.vim.get_network(net_id) + if vim_net.get('encapsulation') != 'vlan': + raise vimconn.vimconnException( + "net '{}' defined as type '{}' has not vlan encapsulation '{}'".format( + net_name, net_type, vim_net['encapsulation'])) + network["vlan"] = vim_net.get('segmentation_id') + try: + sdn_net_id = self.ovim.new_network(network) + except (ovimException, Exception) as e: + self.logger.error("task=%s cannot create SDN network vim_net_id=%s input='%s' ovimException='%s'", + str(task_id), net_id, str(network), str(e)) with self.db_lock: - self.db.update_rows("instance_nets", UPDATE={"vim_net_id": net_id}, WHERE={"vim_net_id": task_id}) + self.db.update_rows("instance_nets", UPDATE={"vim_net_id": net_id, "sdn_net_id": sdn_net_id}, + WHERE={"vim_net_id": task_id}) + new_refresh_task = {"status": "enqueued", + "id": task_id, + "name": "get-net", + "vim_id": net_id, + "vim_info": {} } + self._insert_refresh(new_refresh_task, time.time()) return True, net_id except db_base_Exception as e: self.logger.error("Error updating database %s", str(e)) return True, net_id except vimconn.vimconnException as e: + self.logger.error("Error creating NET, task=%s: %s", str(task_id), str(e)) + try: + with self.db_lock: + self.db.update_rows("instance_nets", + UPDATE={"error_msg": self._format_vim_error_msg(str(e)), "status": "VIM_ERROR"}, + WHERE={"vim_net_id": task_id}) + except db_base_Exception as e: + self.logger.error("Error updating database %s", str(e)) return False, str(e) + #except ovimException as e: + # self.logger.error("Error creating NET in ovim, task=%s: %s", str(task_id), str(e)) + # return False, str(e) def new_vm(self, task): try: @@ -162,33 +446,63 @@ class vim_thread(threading.Thread): task_id = task["id"] depends = task.get("depends") net_list = params[5] + error_text = "" for net in net_list: - if is_task_id(net["net_id"]): # change task_id into network_id + if "net_id" in net and is_task_id(net["net_id"]): # change task_id into network_id try: task_net = depends[net["net_id"]] with self.task_lock: if task_net["status"] == "error": - return False, "Cannot create VM because depends on a network that cannot be created: " + \ + error_text = "Cannot create VM because depends on a network that cannot be created: " +\ str(task_net["result"]) + break elif task_net["status"] == "enqueued" or task_net["status"] == "processing": - return False, "Cannot create VM because depends on a network still not created" + error_text = "Cannot create VM because depends on a network still not created" + break network_id = task_net["result"] net["net_id"] = network_id except Exception as e: - return False, "Error trying to map from task_id={} to task result: {}".format(net["net_id"], - str(e)) - vm_id = self.vim.new_vminstance(*params) - with self.db_lock: - self.db.update_rows("instance_vms", UPDATE={"vim_vm_id": vm_id}, WHERE={"vim_vm_id": task_id}) - return True, vm_id - except db_base_Exception as e: - self.logger.error("Error updtaing database %s", str(e)) + error_text = "Error trying to map from task_id={} to task result: {}".format( + net["net_id"],str(e)) + break + if not error_text: + vm_id = self.vim.new_vminstance(*params) + try: + with self.db_lock: + if error_text: + update = self.db.update_rows("instance_vms", + UPDATE={"status": "VIM_ERROR", "error_msg": error_text}, + WHERE={"vim_vm_id": task_id}) + else: + update = self.db.update_rows("instance_vms", UPDATE={"vim_vm_id": vm_id}, WHERE={"vim_vm_id": task_id}) + if not update: + self.logger.error("task id={} name={} database not updated vim_vm_id={}".format( + task["id"], task["name"], vm_id)) + except db_base_Exception as e: + self.logger.error("Error updating database %s", str(e)) + if error_text: + return False, error_text + new_refresh_task = {"status": "enqueued", + "id": task_id, + "name": "get-vm", + "vim_id": vm_id, + "vim_info": {"interfaces":[]} } + self._insert_refresh(new_refresh_task, time.time()) return True, vm_id except vimconn.vimconnException as e: + self.logger.error("Error creating VM, task=%s: %s", str(task_id), str(e)) + try: + with self.db_lock: + self.db.update_rows("instance_vms", + UPDATE={"error_msg": self._format_vim_error_msg(str(e)), "status": "VIM_ERROR"}, + WHERE={"vim_vm_id": task_id}) + except db_base_Exception as edb: + self.logger.error("Error updating database %s", str(edb)) return False, str(e) def del_vm(self, task): - vm_id = task["params"] + vm_id = task["params"][0] + interfaces = task["params"][1] if is_task_id(vm_id): try: task_create = task["depends"][vm_id] @@ -196,17 +510,28 @@ class vim_thread(threading.Thread): if task_create["status"] == "error": return True, "VM was not created. It has error: " + str(task_create["result"]) elif task_create["status"] == "enqueued" or task_create["status"] == "processing": - return False, "Cannot delete VM because still creating" + return False, "Cannot delete VM vim_id={} because still creating".format(vm_id) vm_id = task_create["result"] except Exception as e: return False, "Error trying to get task_id='{}':".format(vm_id, str(e)) try: + self._remove_refresh("get-vm", vm_id) + for iface in interfaces: + if iface.get("sdn_port_id"): + try: + self.ovim.delete_port(iface["sdn_port_id"]) + except ovimException as e: + self.logger.error("ovimException deleting external_port={} at VM vim_id={} deletion ".format( + iface["sdn_port_id"], vm_id) + str(e), exc_info=True) + # TODO Set error_msg at instance_nets + return True, self.vim.delete_vminstance(vm_id) except vimconn.vimconnException as e: return False, str(e) def del_net(self, task): - net_id = task["params"] + net_id = task["params"][0] + sdn_net_id = task["params"][1] if is_task_id(net_id): try: task_create = task["depends"][net_id] @@ -219,8 +544,16 @@ class vim_thread(threading.Thread): except Exception as e: return False, "Error trying to get task_id='{}':".format(net_id, str(e)) try: - return True, self.vim.delete_network(net_id) + self._remove_refresh("get-net", net_id) + result = self.vim.delete_network(net_id) + if sdn_net_id: + with self.db_lock: + self.ovim.delete_network(sdn_net_id) + return True, result except vimconn.vimconnException as e: return False, str(e) + except ovimException as e: + logging.error("Error deleting network from ovim. net_id: {}, sdn_net_id: {}".format(net_id, sdn_net_id)) + return False, str(e) diff --git a/osm_ro/vimconn.py b/osm_ro/vimconn.py index a9bd9be6..18f4334d 100644 --- a/osm_ro/vimconn.py +++ b/osm_ro/vimconn.py @@ -232,6 +232,8 @@ class vimconnector(): 'id': (mandatory) VIM network id 'name': (mandatory) VIM network name 'status': (mandatory) can be 'ACTIVE', 'INACTIVE', 'DOWN', 'BUILD', 'ERROR', 'VIM_ERROR', 'OTHER' + 'network_type': (optional) can be 'vxlan', 'vlan' or 'flat' + 'segmentation_id': (optional) in case network_type is vlan or vxlan this field contains the segmentation id 'error_msg': (optional) text that explains the ERROR status other VIM specific fields: (optional) whenever possible using the same naming of filter_dict param List can be empty if no network map the filter_dict. Raise an exception only upon VIM connectivity, @@ -430,9 +432,9 @@ class vimconnector(): vim_net_id: #network id where this interface is connected, if provided at creation vim_interface_id: #interface/port VIM id ip_address: #null, or text with IPv4, IPv6 address - physical_compute: #identification of compute node where PF,VF interface is allocated - physical_pci: #PCI address of the NIC that hosts the PF,VF - physical_vlan: #physical VLAN used for VF + compute_node: #identification of compute node where PF,VF interface is allocated + pci: #PCI address of the NIC that hosts the PF,VF + vlan: #physical VLAN used for VF """ raise vimconnNotImplemented( "Should have implemented this" ) diff --git a/osm_ro/vimconn_openstack.py b/osm_ro/vimconn_openstack.py index b501d9da..35cffaea 100644 --- a/osm_ro/vimconn_openstack.py +++ b/osm_ro/vimconn_openstack.py @@ -389,6 +389,8 @@ class vimconnector(vimconn.vimconnector): subnet = {"id": subnet_id, "fault": str(e)} subnets.append(subnet) net["subnets"] = subnets + net["encapsulation"] = net.get('provider:network_type') + net["segmentation_id"] = net.get('provider:segmentation_id') return net def delete_network(self, net_id): @@ -550,10 +552,10 @@ class vimconnector(vimconn.vimconnector): elif 'threads' in numa: vcpus = numa['threads'] numa_properties["hw:cpu_policy"] = "isolated" - for interface in numa.get("interfaces",() ): - if interface["dedicated"]=="yes": - raise vimconn.vimconnException("Passthrough interfaces are not supported for the openstack connector", http_code=vimconn.HTTP_Service_Unavailable) - #TODO, add the key 'pci_passthrough:alias"="