From: jomacarpe Date: Wed, 12 Jun 2019 21:20:51 +0000 (+0000) Subject: Migrate to Pyone and refactor X-Git-Tag: v6.0.1~4 X-Git-Url: https://osm.etsi.org/gitweb/?p=osm%2FRO.git;a=commitdiff_plain;h=20c13237430c0f946d4433461ce190129bf0cf8b Migrate to Pyone and refactor Change-Id: I46c92897c394784632b153c7b92cca743f78a67d Signed-off-by: jomacarpe --- diff --git a/Dockerfile b/Dockerfile index c5d85181..a0f45ba9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -38,6 +38,7 @@ RUN apt-get update && \ DEBIAN_FRONTEND=noninteractive apt-get -y install python-argcomplete python-bottle python-cffi python-packaging python-paramiko python-pkgconfig libmysqlclient-dev libssl-dev libffi-dev python-mysqldb && \ DEBIAN_FRONTEND=noninteractive apt-get -y install python-logutils python-openstackclient python-openstacksdk && \ DEBIAN_FRONTEND=noninteractive pip install untangle && \ + DEBIAN_FRONTEND=noninteractive pip install pyone && \ DEBIAN_FRONTEND=noninteractive pip install -e git+https://github.com/python-oca/python-oca#egg=oca diff --git a/docker/Dockerfile-local b/docker/Dockerfile-local index 5d1e02ab..2abdee85 100644 --- a/docker/Dockerfile-local +++ b/docker/Dockerfile-local @@ -19,6 +19,7 @@ RUN apt-get update && \ DEBIAN_FRONTEND=noninteractive apt-get -y install python-networkx && \ DEBIAN_FRONTEND=noninteractive apt-get -y install genisoimage && \ DEBIAN_FRONTEND=noninteractive pip2 install untangle && \ + DEBIAN_FRONTEND=noninteractive pip2 install pyone && \ DEBIAN_FRONTEND=noninteractive pip2 install -e git+https://github.com/python-oca/python-oca#egg=oca && \ DEBIAN_FRONTEND=noninteractive apt-get -y install mysql-client diff --git a/osm_ro/vimconn_opennebula.py b/osm_ro/vimconn_opennebula.py index 8f7fad84..fb0abda6 100644 --- a/osm_ro/vimconn_opennebula.py +++ b/osm_ro/vimconn_opennebula.py @@ -35,29 +35,35 @@ import oca import untangle import math import random - +import pyone class vimconnector(vimconn.vimconnector): def __init__(self, uuid, name, tenant_id, tenant_name, url, url_admin=None, user=None, passwd=None, log_level="DEBUG", config={}, persistent_info={}): + + """Constructor of VIM + Params: + 'uuid': id asigned to this VIM + 'name': name assigned to this VIM, can be used for logging + 'tenant_id', 'tenant_name': (only one of them is mandatory) VIM tenant to be used + 'url_admin': (optional), url used for administrative tasks + 'user', 'passwd': credentials of the VIM user + 'log_level': provider if it should use a different log_level than the general one + 'config': dictionary with extra VIM information. This contains a consolidate version of general VIM config + at creation and particular VIM config at teh attachment + 'persistent_info': dict where the class can store information that will be available among class + destroy/creation cycles. This info is unique per VIM/credential. At first call it will contain an + empty dict. Useful to store login/tokens information for speed up communication + + Returns: Raise an exception is some needed parameter is missing, but it must not do any connectivity + check against the VIM + """ + vimconn.vimconnector.__init__(self, uuid, name, tenant_id, tenant_name, url, url_admin, user, passwd, log_level, config) - self.tenant = None - self.headers_req = {'content-type': 'application/json'} - self.logger = logging.getLogger('openmano.vim.opennebula') - self.persistent_info = persistent_info - if tenant_id: - self.tenant = tenant_id - - def __setitem__(self, index, value): - """Set individuals parameters - Throw TypeError, KeyError - """ - if index == 'tenant_id': - self.tenant = value - elif index == 'tenant_name': - self.tenant = None - vimconn.vimconnector.__setitem__(self, index, value) + + def _new_one_connection(self): + return pyone.OneServer(self.url, session=self.user + ':' + self.passwd) def new_tenant(self, tenant_name, tenant_description): # '''Adds a new tenant to VIM with this name and description, returns the tenant identifier''' @@ -91,25 +97,6 @@ class vimconnector(vimconn.vimconnector): self.logger.error("Create new tenant error: " + str(e)) raise vimconn.vimconnException(e) - def _add_secondarygroup(self, id_user, id_group): - # change secondary_group to primary_group - params = ' \ - \ - one.user.addgroup\ - \ - \ - {}:{}\ - \ - \ - {}\ - \ - \ - {}\ - \ - \ - '.format(self.user, self.passwd, (str(id_user)), (str(id_group))) - requests.post(self.url, params) - def delete_tenant(self, tenant_id): """Delete a tenant from VIM. Returns the old tenant identifier""" try: @@ -130,7 +117,25 @@ class vimconnector(vimconn.vimconnector): self.logger.error("Delete tenant " + str(tenant_id) + " error: " + str(e)) raise vimconn.vimconnException(e) - # to be used in future commits + def _add_secondarygroup(self, id_user, id_group): + # change secondary_group to primary_group + params = ' \ + \ + one.user.addgroup\ + \ + \ + {}:{}\ + \ + \ + {}\ + \ + \ + {}\ + \ + \ + '.format(self.user, self.passwd, (str(id_user)), (str(id_group))) + requests.post(self.url, params) + def _delete_secondarygroup(self, id_user, id_group): params = ' \ \ @@ -149,65 +154,6 @@ class vimconnector(vimconn.vimconnector): '.format(self.user, self.passwd, (str(id_user)), (str(id_group))) requests.post(self.url, params) - # to be used in future commits - # def get_tenant_list(self, filter_dict={}): - # return ["tenant"] - - # to be used in future commits - # def _check_tenant(self): - # try: - # client = oca.Client(self.user + ':' + self.passwd, self.url) - # group_list = oca.GroupPool(client) - # user_list = oca.UserPool(client) - # group_list.info() - # user_list.info() - # for group in group_list: - # if str(group.name) == str(self.tenant_name): - # for user in user_list: - # if str(user.name) == str(self.user): - # self._add_secondarygroup(user.id, group.id) - # user.chgrp(group.id) - # except vimconn.vimconnException as e: - # self.logger.error(e) - - # to be used in future commits, needs refactor to manage networks - # def _create_bridge_host(self, vlan): - # file = open('manage_bridge_OSM', 'w') - # # password_path = self.config["password"]["path"] - # a = "#! /bin/bash\nsudo brctl addbr br_osm_{vlanused}\n" \ - # "sudo ip link add link veth1 name veth1.{vlanused} type vlan id {vlanused}\n" \ - # "sudo brctl addif br_osm_{vlanused} veth1.{vlanused}\n" \ - # "sudo ip link set dev br_osm_{vlanused} up\n" \ - # "sudo ip link set dev veth1.{vlanused} up\n".format(vlanused=vlan) - # # a = "#! /bin/bash\nsudo brctl addbr br_osm\nsudo ip link set dev br_osm up\n" - # file.write(a) - # file.close() - # for host in self.config["cluster"]["ip"]: - # file_scp = "/usr/bin/scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i {} manage_bridge_OSM {}@{}:/home/{}".format( - # self.config["cluster"]["password_path"][host], self.config["cluster"]["login"][host], - # self.config["cluster"]["ip"][host], self.config["cluster"]["login"][host]) - # os.system(file_scp) - # file_permissions = "/usr/bin/ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i {} {}@{} sudo chmod 700 manage_bridge_OSM".format( - # self.config["cluster"]["password_path"][host], self.config["cluster"]["login"][host], - # self.config["cluster"]["ip"][host]) - # os.system(file_permissions) - # exec_script = "/usr/bin/ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i {} {}@{} sudo ./manage_bridge_OSM".format( - # self.config["cluster"]["password_path"][host], self.config["cluster"]["login"][host], - # self.config["cluster"]["ip"][host]) - # os.system(exec_script) - # os.remove("manage_bridge_OSM") - - # to be used to manage networks with vlan - # def delete_bridge_host(self, vlan): - # file = open('manage_bridge_OSM', 'w') - # a = "#! /bin/bash\nsudo ip link set dev veth1.3142 down\nsudo ip link set dev br_3142 down\nsudo brctl delbr br_3142\n" - # file.write(a) - # file.close() - # os.system("/usr/bin/scp -i onlife manage_bridge_OSM sysadmin@10.95.84.12:/home/sysadmin") - # os.system("/usr/bin/ssh -i onlife sysadmin@10.95.84.12 sudo chmod 700 manage_bridge_OSM") - # os.system("/usr/bin/ssh -i onlife sysadmin@10.95.84.12 sudo ./manage_bridge_OSM") - # os.remove("manage_bridge_OSM") - def new_network(self, net_name, net_type, ip_profile=None, shared=False, vlan=None): # , **vim_specific): """Adds a tenant network to VIM Params: @@ -235,15 +181,12 @@ class vimconnector(vimconn.vimconnector): # oca library method cannot be used in this case (problem with cluster parameters) try: - created_items = {} - # vlan = str(random.randint(self.config["vlan"]["start-range"], self.config["vlan"]["finish-range"])) - # self.create_bridge_host(vlan) - bridge_config = self.config["bridge_service"] - ip_version = "IP4" - size = "256" + one = self._new_one_connection() + ip_prefix_type = "IP" + size = "254" if ip_profile is None: - random_number_ipv4 = random.randint(1, 255) - ip_start = "192.168." + str(random_number_ipv4) + ".1" # random value + subnet_rand = random.randint(0, 255) + ip_start = "192.168.{}.1".format(subnet_rand) else: index = ip_profile["subnet_address"].find("/") ip_start = ip_profile["subnet_address"][:index] @@ -255,57 +198,60 @@ class vimconnector(vimconn.vimconnector): if "dhcp_start_address" in ip_profile.keys() and ip_profile["dhcp_start_address"] is not None: ip_start = str(ip_profile["dhcp_start_address"]) if ip_profile["ip_version"] == "IPv6": - ip_version = "IP6" - if ip_version == "IP6": - config = "NAME = {}\ - BRIDGE = {}\ - VN_MAD = dummy\ - AR = [TYPE = {}, GLOBAL_PREFIX = {}, SIZE = {}]".format(net_name, bridge_config, ip_version, - ip_start, size) + ip_prefix_type = "GLOBAL_PREFIX" + + if vlan is not None: + vlan_id = vlan else: - config = 'NAME = "{}"\ - BRIDGE = {}\ - VN_MAD = dummy\ - AR = [TYPE = {}, IP = {}, SIZE = {}]'.format(net_name, bridge_config, ip_version, ip_start, - size) - - params = ' \ - \ - one.vn.allocate\ - \ - \ - {}:{}\ - \ - \ - {}\ - \ - \ - {}\ - \ - \ - '.format(self.user, self.passwd, config, self.config["cluster"]["id"]) - r = requests.post(self.url, params) - obj = untangle.parse(str(r.content)) - return obj.methodResponse.params.param.value.array.data.value[1].i4.cdata.encode('utf-8'), created_items + vlan_id = str(random.randint(100, 4095)) + + net_id = one.vn.allocate({ + 'NAME': net_name, + 'VN_MAD': '802.1Q', + 'PHYDEV': self.config["network"]["phydev"], + 'VLAN_ID': vlan_id + }, self.config["cluster"]["id"]) + + one.vn.add_ar(net_id, { + 'AR_POOL': { + 'AR': { + 'TYPE': 'IP4', + ip_prefix_type: ip_start, + 'SIZE': size + } + } + }) + + return net_id except Exception as e: self.logger.error("Create new network error: " + str(e)) raise vimconn.vimconnException(e) def get_network_list(self, filter_dict={}): """Obtain tenant networks of VIM - Filter_dict can be: - name: network name - id: network uuid - public: boolean - tenant_id: tenant - admin_state_up: boolean - status: 'ACTIVE' - Returns the network list of dictionaries + Params: + 'filter_dict' (optional) contains entries to return only networks that matches ALL entries: + name: string => returns only networks with this name + id: string => returns networks with this VIM id, this imply returns one network at most + shared: boolean >= returns only networks that are (or are not) shared + tenant_id: sting => returns only networks that belong to this tenant/project + ,#(not used yet) admin_state_up: boolean => returns only networks that are (or are not) in admin state active + #(not used yet) status: 'ACTIVE','ERROR',... => filter networks that are on this status + Returns the network list of dictionaries. each dictionary contains: + 'id': (mandatory) VIM network id + 'name': (mandatory) VIM network name + 'status': (mandatory) can be 'ACTIVE', 'INACTIVE', 'DOWN', 'BUILD', 'ERROR', 'VIM_ERROR', 'OTHER' + 'network_type': (optional) can be 'vxlan', 'vlan' or 'flat' + 'segmentation_id': (optional) in case network_type is vlan or vxlan this field contains the segmentation id + 'error_msg': (optional) text that explains the ERROR status + other VIM specific fields: (optional) whenever possible using the same naming of filter_dict param + List can be empty if no network map the filter_dict. Raise an exception only upon VIM connectivity, + authorization, or some other unspecific error """ + try: - client = oca.Client(self.user + ':' + self.passwd, self.url) - networkList = oca.VirtualNetworkPool(client) - networkList.info() + one = self._new_one_connection() + net_pool = one.vnpool.info(-2, -1, -1).VNET response = [] if "name" in filter_dict.keys(): network_name_filter = filter_dict["name"] @@ -315,16 +261,9 @@ class vimconnector(vimconn.vimconnector): network_id_filter = filter_dict["id"] else: network_id_filter = None - for network in networkList: - match = False - if network.name == network_name_filter and str(network.id) == str(network_id_filter): - match = True - if network_name_filter is None and str(network.id) == str(network_id_filter): - match = True - if network_id_filter is None and network.name == network_name_filter: - match = True - if match: - net_dict = {"name": network.name, "id": str(network.id)} + for network in net_pool: + if network.NAME == network_name_filter or str(network.ID) == str(network_id_filter): + net_dict = {"name": network.NAME, "id": str(network.ID), "status": "ACTIVE"} response.append(net_dict) return response except Exception as e: @@ -332,16 +271,23 @@ class vimconnector(vimconn.vimconnector): raise vimconn.vimconnException(e) def get_network(self, net_id): - """Obtain network details of network id""" + """Obtain network details from the 'net_id' VIM network + Return a dict that contains: + 'id': (mandatory) VIM network id, that is, net_id + 'name': (mandatory) VIM network name + 'status': (mandatory) can be 'ACTIVE', 'INACTIVE', 'DOWN', 'BUILD', 'ERROR', 'VIM_ERROR', 'OTHER' + 'error_msg': (optional) text that explains the ERROR status + other VIM specific fields: (optional) whenever possible using the same naming of filter_dict param + Raises an exception upon error or when network is not found + """ try: - client = oca.Client(self.user + ':' + self.passwd, self.url) - networkList = oca.VirtualNetworkPool(client) - networkList.info() + one = self._new_one_connection() + net_pool = one.vnpool.info(-2, -1, -1).VNET net = {} - for network in networkList: - if str(network.id) == str(net_id): - net['id'] = net_id - net['name'] = network.name + for network in net_pool: + if str(network.ID) == str(net_id): + net['id'] = network.ID + net['name'] = network.NAME net['status'] = "ACTIVE" break if net: @@ -349,8 +295,8 @@ class vimconnector(vimconn.vimconnector): else: raise vimconn.vimconnNotFoundException("Network {} not found".format(net_id)) except Exception as e: - self.logger.error("Get network " + str(net_id) + " error): " + str(e)) - raise vimconn.vimconnException(e) + self.logger.error("Get network " + str(net_id) + " error): " + str(e)) + raise vimconn.vimconnException(e) def delete_network(self, net_id, created_items=None): """ @@ -360,32 +306,67 @@ class vimconnector(vimconn.vimconnector): Returns the network identifier or raises an exception upon error or when network is not found """ try: - # self.delete_bridge_host() - client = oca.Client(self.user + ':' + self.passwd, self.url) - networkList = oca.VirtualNetworkPool(client) - networkList.info() - network_deleted = False - for network in networkList: - if str(network.id) == str(net_id): - oca.VirtualNetwork.delete(network) - network_deleted = True - if network_deleted: - return net_id - else: - raise vimconn.vimconnNotFoundException("Network {} not found".format(net_id)) + + one = self._new_one_connection() + one.vn.delete(int(net_id)) + return net_id except Exception as e: - self.logger.error("Delete network " + str(net_id) + "error: " + str(e)) - raise vimconn.vimconnException(e) + self.logger.error("Delete network " + str(net_id) + "error: network not found" + str(e)) + raise vimconn.vimconnException(e) + + def refresh_nets_status(self, net_list): + """Get the status of the networks + Params: + 'net_list': a list with the VIM network id to be get the status + Returns a dictionary with: + 'net_id': #VIM id of this network + status: #Mandatory. Text with one of: + # DELETED (not found at vim) + # VIM_ERROR (Cannot connect to VIM, authentication problems, VIM response error, ...) + # OTHER (Vim reported other status not understood) + # ERROR (VIM indicates an ERROR status) + # ACTIVE, INACTIVE, DOWN (admin down), + # BUILD (on building process) + error_msg: #Text with VIM error message, if any. Or the VIM connection ERROR + vim_info: #Text with plain information obtained from vim (yaml.safe_dump) + 'net_id2': ... + """ + net_dict = {} + try: + for net_id in net_list: + net = {} + try: + net_vim = self.get_network(net_id) + net["status"] = net_vim["status"] + net["vim_info"] = None + except vimconn.vimconnNotFoundException as e: + self.logger.error("Exception getting net status: {}".format(str(e))) + net['status'] = "DELETED" + net['error_msg'] = str(e) + except vimconn.vimconnException as e: + self.logger.error(e) + net["status"] = "VIM_ERROR" + net["error_msg"] = str(e) + net_dict[net_id] = net + return net_dict + except vimconn.vimconnException as e: + self.logger.error(e) + for k in net_dict: + net_dict[k]["status"] = "VIM_ERROR" + net_dict[k]["error_msg"] = str(e) + return net_dict def get_flavor(self, flavor_id): # Esta correcto - """Obtain flavor details from the VIM""" + """Obtain flavor details from the VIM + Returns the flavor dict details {'id':<>, 'name':<>, other vim specific } + Raises an exception upon error or if not found + """ try: - client = oca.Client(self.user + ':' + self.passwd, self.url) - listaTemplate = oca.VmTemplatePool(client) - listaTemplate.info() - for template in listaTemplate: - if str(template.id) == str(flavor_id): - return {'id': template.id, 'name': template.name} + + one = self._new_one_connection() + template = one.template.info(int(flavor_id)) + if template is not None: + return {'id': template.ID, 'name': template.NAME} raise vimconn.vimconnNotFoundException("Flavor {} not found".format(flavor_id)) except Exception as e: self.logger.error("get flavor " + str(flavor_id) + " error: " + str(e)) @@ -393,20 +374,50 @@ class vimconnector(vimconn.vimconnector): def new_flavor(self, flavor_data): """Adds a tenant flavor to VIM - Returns the flavor identifier""" + flavor_data contains a dictionary with information, keys: + name: flavor name + ram: memory (cloud type) in MBytes + vpcus: cpus (cloud type) + extended: EPA parameters + - numas: #items requested in same NUMA + memory: number of 1G huge pages memory + paired-threads|cores|threads: number of paired hyperthreads, complete cores OR individual threads + interfaces: # passthrough(PT) or SRIOV interfaces attached to this numa + - name: interface name + dedicated: yes|no|yes:sriov; for PT, SRIOV or only one SRIOV for the physical NIC + bandwidth: X Gbps; requested guarantee bandwidth + vpci: requested virtual PCI address + disk: disk size + is_public: + #TODO to concrete + Returns the flavor identifier""" + + disk_size = str(int(flavor_data["disk"])*1024) + try: - client = oca.Client(self.user + ':' + self.passwd, self.url) - template_name = flavor_data["name"][:-4] - name = 'NAME = "{}" '.format(template_name) - cpu = 'CPU = "{}" '.format(flavor_data["vcpus"]) - vcpu = 'VCPU = "{}" '.format(flavor_data["vcpus"]) - memory = 'MEMORY = "{}" '.format(flavor_data["ram"]) - context = 'CONTEXT = [NETWORK = "YES",SSH_PUBLIC_KEY = "$USER[SSH_PUBLIC_KEY]" ] ' - graphics = 'GRAPHICS = [ LISTEN = "0.0.0.0", TYPE = "VNC" ] ' - sched_requeriments = 'CLUSTER_ID={}'.format(self.config["cluster"]["id"]) - template = name + cpu + vcpu + memory + context + graphics + sched_requeriments - template_id = oca.VmTemplate.allocate(client, template) + one = self._new_one_connection() + template_id = one.template.allocate({ + 'TEMPLATE': { + 'NAME': flavor_data["name"], + 'CPU': flavor_data["vcpus"], + 'VCPU': flavor_data["vcpus"], + 'MEMORY': flavor_data["ram"], + 'DISK': { + 'SIZE': disk_size + }, + 'CONTEXT': { + 'NETWORK': "YES", + 'SSH_PUBLIC_KEY': '$USER[SSH_PUBLIC_KEY]' + }, + 'GRAPHICS': { + 'LISTEN': '0.0.0.0', + 'TYPE': 'VNC' + }, + 'CLUSTER_ID': self.config["cluster"]["id"] + } + }) return template_id + except Exception as e: self.logger.error("Create new flavor error: " + str(e)) raise vimconn.vimconnException(e) @@ -416,17 +427,11 @@ class vimconnector(vimconn.vimconnector): Returns the old flavor_id """ try: - client = oca.Client(self.user + ':' + self.passwd, self.url) - listaTemplate = oca.VmTemplatePool(client) - listaTemplate.info() - self.logger.info("Deleting VIM flavor DELETE {}".format(self.url)) - for template in listaTemplate: - if str(template.id) == str(flavor_id): - template.delete() - return template.id - raise vimconn.vimconnNotFoundException("Flavor {} not found".format(flavor_id)) + one = self._new_one_connection() + one.template.delete(int(flavor_id), False) + return flavor_id except Exception as e: - self.logger.error("Delete flavor " + str(flavor_id) + " error: " + str(e)) + self.logger.error("Error deleting flavor " + str(flavor_id) + ". Flavor not found") raise vimconn.vimconnException(e) def get_image_list(self, filter_dict={}): @@ -440,12 +445,9 @@ class vimconnector(vimconn.vimconnector): [{}, ...] List can be empty """ - # IMPORTANT!!!!! Modify python oca library path pool.py line 102 - try: - client = oca.Client(self.user + ':' + self.passwd, self.url) - image_pool = oca.ImagePool(client) - image_pool.info() + one = self._new_one_connection() + image_pool = one.imagepool.info(-2, -1, -1).IMAGE images = [] if "name" in filter_dict.keys(): image_name_filter = filter_dict["name"] @@ -456,15 +458,8 @@ class vimconnector(vimconn.vimconnector): else: image_id_filter = None for image in image_pool: - match = False - if str(image_name_filter) == str(image.name) and str(image.id) == str(image_id_filter): - match = True - if image_name_filter is None and str(image.id) == str(image_id_filter): - match = True - if image_id_filter is None and str(image_name_filter) == str(image.name): - match = True - if match: - images_dict = {"name": image.name, "id": str(image.id)} + if str(image_name_filter) == str(image.NAME) or str(image.ID) == str(image_id_filter): + images_dict = {"name": image.NAME, "id": str(image.ID)} images.append(images_dict) return images except Exception as e: @@ -473,157 +468,184 @@ class vimconnector(vimconn.vimconnector): def new_vminstance(self, name, description, start, image_id, flavor_id, net_list, cloud_config=None, disk_list=None, availability_zone_index=None, availability_zone_list=None): + """Adds a VM instance to VIM - Params: - start: indicates if VM must start or boot in pause mode. Ignored - image_id,flavor_id: image and flavor uuid - net_list: list of interfaces, each one is a dictionary with: - name: - net_id: network uuid to connect - vpci: virtual vcpi to assign - model: interface model, virtio, e1000, ... - mac_address: - use: 'data', 'bridge', 'mgmt' - type: 'virtual', 'PF', 'VF', 'VFnotShared' - vim_id: filled/added by this function - #TODO ip, security groups - Returns the instance identifier - """ + Params: + 'start': (boolean) indicates if VM must start or created in pause mode. + 'image_id','flavor_id': image and flavor VIM id to use for the VM + 'net_list': list of interfaces, each one is a dictionary with: + 'name': (optional) name for the interface. + 'net_id': VIM network id where this interface must be connect to. Mandatory for type==virtual + 'vpci': (optional) virtual vPCI address to assign at the VM. Can be ignored depending on VIM capabilities + 'model': (optional and only have sense for type==virtual) interface model: virtio, e1000, ... + 'mac_address': (optional) mac address to assign to this interface + 'ip_address': (optional) IP address to assign to this interface + #TODO: CHECK if an optional 'vlan' parameter is needed for VIMs when type if VF and net_id is not provided, + the VLAN tag to be used. In case net_id is provided, the internal network vlan is used for tagging VF + 'type': (mandatory) can be one of: + 'virtual', in this case always connected to a network of type 'net_type=bridge' + 'PCI-PASSTHROUGH' or 'PF' (passthrough): depending on VIM capabilities it can be connected to a data/ptp network ot it + can created unconnected + 'SR-IOV' or 'VF' (SRIOV with VLAN tag): same as PF for network connectivity. + 'VFnotShared'(SRIOV without VLAN tag) same as PF for network connectivity. VF where no other VFs + are allocated on the same physical NIC + 'bw': (optional) only for PF/VF/VFnotShared. Minimal Bandwidth required for the interface in GBPS + 'port_security': (optional) If False it must avoid any traffic filtering at this interface. If missing + or True, it must apply the default VIM behaviour + After execution the method will add the key: + 'vim_id': must be filled/added by this method with the VIM identifier generated by the VIM for this + interface. 'net_list' is modified + 'cloud_config': (optional) dictionary with: + 'key-pairs': (optional) list of strings with the public key to be inserted to the default user + 'users': (optional) list of users to be inserted, each item is a dict with: + 'name': (mandatory) user name, + 'key-pairs': (optional) list of strings with the public key to be inserted to the user + 'user-data': (optional) can be a string with the text script to be passed directly to cloud-init, + or a list of strings, each one contains a script to be passed, usually with a MIMEmultipart file + 'config-files': (optional). List of files to be transferred. Each item is a dict with: + 'dest': (mandatory) string with the destination absolute path + 'encoding': (optional, by default text). Can be one of: + 'b64', 'base64', 'gz', 'gz+b64', 'gz+base64', 'gzip+b64', 'gzip+base64' + 'content' (mandatory): string with the content of the file + 'permissions': (optional) string with file permissions, typically octal notation '0644' + 'owner': (optional) file owner, string with the format 'owner:group' + 'boot-data-drive': boolean to indicate if user-data must be passed using a boot drive (hard disk) + 'disk_list': (optional) list with additional disks to the VM. Each item is a dict with: + 'image_id': (optional). VIM id of an existing image. If not provided an empty disk must be mounted + 'size': (mandatory) string with the size of the disk in GB + availability_zone_index: Index of availability_zone_list to use for this this VM. None if not AV required + availability_zone_list: list of availability zones given by user in the VNFD descriptor. Ignore if + availability_zone_index is None + Returns a tuple with the instance identifier and created_items or raises an exception on error + created_items can be None or a dictionary where this method can include key-values that will be passed to + the method delete_vminstance and action_vminstance. Can be used to store created ports, volumes, etc. + Format is vimconnector dependent, but do not use nested dictionaries and a value of None should be the same + as not present. + """ self.logger.debug( "new_vminstance input: image='{}' flavor='{}' nics='{}'".format(image_id, flavor_id, str(net_list))) try: - client = oca.Client(self.user + ':' + self.passwd, self.url) - listaTemplate = oca.VmTemplatePool(client) - listaTemplate.info() - for template in listaTemplate: - if str(template.id) == str(flavor_id): - cpu = ' CPU = "{}"'.format(template.template.cpu) - vcpu = ' VCPU = "{}"'.format(template.template.cpu) - memory = ' MEMORY = "{}"'.format(template.template.memory) - context = ' CONTEXT = [NETWORK = "YES",SSH_PUBLIC_KEY = "$USER[SSH_PUBLIC_KEY]" ]' - graphics = ' GRAPHICS = [ LISTEN = "0.0.0.0", TYPE = "VNC" ]' - disk = ' DISK = [ IMAGE_ID = {}]'.format(image_id) - template_updated = cpu + vcpu + memory + context + graphics + disk - networkListVim = oca.VirtualNetworkPool(client) - networkListVim.info() - network = "" - for net in net_list: - network_found = False - for network_existingInVim in networkListVim: - if str(net["net_id"]) == str(network_existingInVim.id): - net["vim_id"] = network_existingInVim["id"] - network = 'NIC = [NETWORK = "{}",NETWORK_UNAME = "{}" ]'.format( - network_existingInVim.name, network_existingInVim.uname) - network_found = True - break - if not network_found: - raise vimconn.vimconnNotFoundException("Network {} not found".format(net["net_id"])) - template_updated += network - if isinstance(cloud_config, dict): - if cloud_config.get("user-data"): - if isinstance(cloud_config["user-data"], str): - template_updated += cloud_config["user-data"] - else: - for u in cloud_config["user-data"]: - template_updated += u - oca.VmTemplate.update(template, template_updated) - self.logger.info( - "Instanciating in OpenNebula a new VM name:{} id:{}".format(template.name, template.id)) - vminstance_id = template.instantiate(name=name) - return str(vminstance_id), None - raise vimconn.vimconnNotFoundException("Flavor {} not found".format(flavor_id)) + one = self._new_one_connection() + template_vim = one.template.info(int(flavor_id), True) + disk_size = str(template_vim.TEMPLATE["DISK"]["SIZE"]) + + one = self._new_one_connection() + template_updated = "" + for net in net_list: + net_in_vim = one.vn.info(int(net["net_id"])) + net["vim_id"] = str(net_in_vim.ID) + network = 'NIC = [NETWORK = "{}",NETWORK_UNAME = "{}" ]'.format( + net_in_vim.NAME, net_in_vim.UNAME) + template_updated += network + + template_updated += "DISK = [ IMAGE_ID = {},\n SIZE = {}]".format(image_id, disk_size) + + if isinstance(cloud_config, dict): + if cloud_config.get("key-pairs"): + context = 'CONTEXT = [\n NETWORK = "YES",\n SSH_PUBLIC_KEY = "' + for key in cloud_config["key-pairs"]: + context += key + '\n' + # if False: + # context += '"\n USERNAME = ' + context += '"]' + template_updated += context + + vm_instance_id = one.template.instantiate(int(flavor_id), name, False, template_updated) + self.logger.info( + "Instanciating in OpenNebula a new VM name:{} id:{}".format(name, flavor_id)) + return str(vm_instance_id), None + except pyone.OneNoExistsException as e: + self.logger.error("Network with id " + str(e) + " not found: " + str(e)) + raise vimconn.vimconnNotFoundException(e) except Exception as e: self.logger.error("Create new vm instance error: " + str(e)) raise vimconn.vimconnException(e) + def get_vminstance(self, vm_id): + """Returns the VM instance information from VIM""" + try: + one = self._new_one_connection() + vm = one.vm.info(int(vm_id)) + return vm + except Exception as e: + self.logger.error("Getting vm instance error: " + str(e) + ": VM Instance not found") + raise vimconn.vimconnException(e) + def delete_vminstance(self, vm_id, created_items=None): - """Removes a VM instance from VIM, returns the deleted vm_id""" + """ + Removes a VM instance from VIM and its associated elements + :param vm_id: VIM identifier of the VM, provided by method new_vminstance + :param created_items: dictionary with extra items to be deleted. provided by method new_vminstance and/or method + action_vminstance + :return: None or the same vm_id. Raises an exception on fail + """ try: - client = oca.Client(self.user + ':' + self.passwd, self.url) - vm_pool = oca.VirtualMachinePool(client) - vm_pool.info() - vm_exist = False - for i in vm_pool: - if str(i.id) == str(vm_id): - vm_exist = True + one = self._new_one_connection() + one.vm.recover(int(vm_id), 3) + vm = None + while True: + if vm is not None and vm.LCM_STATE == 0: break - if not vm_exist: - self.logger.info("The vm " + str(vm_id) + " does not exist or is already deleted") - raise vimconn.vimconnNotFoundException("The vm {} does not exist or is already deleted".format(vm_id)) - params = ' \ - \ - one.vm.recover\ - \ - \ - {}:{}\ - \ - \ - {}\ - \ - \ - {}\ - \ - \ - '.format(self.user, self.passwd, str(vm_id), str(3)) - r = requests.post(self.url, params) - obj = untangle.parse(str(r.content)) - response_success = obj.methodResponse.params.param.value.array.data.value[0].boolean.cdata.encode('utf-8') - response = obj.methodResponse.params.param.value.array.data.value[1].i4.cdata.encode('utf-8') - # response can be the resource ID on success or the error string on failure. - response_error_code = obj.methodResponse.params.param.value.array.data.value[2].i4.cdata.encode('utf-8') - if response_success.lower() == "true": - return response - else: - raise vimconn.vimconnException("vm {} cannot be deleted with error_code {}: {}".format(vm_id, response_error_code, response)) + else: + vm = one.vm.info(int(vm_id)) + + except pyone.OneNoExistsException as e: + self.logger.info("The vm " + str(vm_id) + " does not exist or is already deleted") + raise vimconn.vimconnNotFoundException("The vm {} does not exist or is already deleted".format(vm_id)) except Exception as e: self.logger.error("Delete vm instance " + str(vm_id) + " error: " + str(e)) raise vimconn.vimconnException(e) def refresh_vms_status(self, vm_list): - """Refreshes the status of the virtual machines""" + """Get the status of the virtual machines and their interfaces/ports + Params: the list of VM identifiers + Returns a dictionary with: + vm_id: #VIM id of this Virtual Machine + status: #Mandatory. Text with one of: + # DELETED (not found at vim) + # VIM_ERROR (Cannot connect to VIM, VIM response error, ...) + # OTHER (Vim reported other status not understood) + # ERROR (VIM indicates an ERROR status) + # ACTIVE, PAUSED, SUSPENDED, INACTIVE (not running), + # BUILD (on building process), ERROR + # ACTIVE:NoMgmtIP (Active but any of its interface has an IP address + # + error_msg: #Text with VIM error message, if any. Or the VIM connection ERROR + vim_info: #Text with plain information obtained from vim (yaml.safe_dump) + interfaces: list with interface info. Each item a dictionary with: + vim_info: #Text with plain information obtained from vim (yaml.safe_dump) + mac_address: #Text format XX:XX:XX:XX:XX:XX + vim_net_id: #network id where this interface is connected, if provided at creation + vim_interface_id: #interface/port VIM id + ip_address: #null, or text with IPv4, IPv6 address + compute_node: #identification of compute node where PF,VF interface is allocated + pci: #PCI address of the NIC that hosts the PF,VF + vlan: #physical VLAN used for VF + """ vm_dict = {} try: - client = oca.Client(self.user + ':' + self.passwd, self.url) - vm_pool = oca.VirtualMachinePool(client) - vm_pool.info() for vm_id in vm_list: - vm = {"interfaces": []} - vm_exist = False - vm_element = None - for i in vm_pool: - if str(i.id) == str(vm_id): - vm_exist = True - vm_element = i - break - if not vm_exist: + vm = {} + if self.get_vminstance(vm_id) is not None: + vm_element = self.get_vminstance(vm_id) + else: self.logger.info("The vm " + str(vm_id) + " does not exist.") vm['status'] = "DELETED" vm['error_msg'] = ("The vm " + str(vm_id) + " does not exist.") continue - vm_element.info() vm["vim_info"] = None - VMstatus = vm_element.str_lcm_state - if VMstatus == "RUNNING": + vm_status = vm_element.LCM_STATE + if vm_status == 3: vm['status'] = "ACTIVE" - elif "FAILURE" in VMstatus: + elif vm_status == 36: vm['status'] = "ERROR" vm['error_msg'] = "VM failure" else: vm['status'] = "BUILD" - try: - for red in vm_element.template.nics: - interface = {'vim_info': None, "mac_address": str(red.mac), "vim_net_id": str(red.network_id), - "vim_interface_id": str(red.network_id)} - # maybe it should be 2 different keys for ip_address if an interface has ipv4 and ipv6 - if hasattr(red, 'ip'): - interface["ip_address"] = str(red.ip) - if hasattr(red, 'ip6_global'): - interface["ip_address"] = str(red.ip6_global) - vm["interfaces"].append(interface) - except Exception as e: - self.logger.error("Error getting vm interface_information " + type(e).__name__ + ":" + str(e)) - vm["status"] = "VIM_ERROR" - vm["error_msg"] = "Error getting vm interface_information " + type(e).__name__ + ":" + str(e) + + if vm_element is not None: + interfaces = self._get_networks_vm(vm_element) + vm["interfaces"] = interfaces vm_dict[vm_id] = vm return vm_dict except Exception as e: @@ -633,59 +655,30 @@ class vimconnector(vimconn.vimconnector): vm_dict[k]["error_msg"] = str(e) return vm_dict - def refresh_nets_status(self, net_list): - """Get the status of the networks - Params: the list of network identifiers - Returns a dictionary with: - net_id: #VIM id of this network - status: #Mandatory. Text with one of: - # DELETED (not found at vim) - # VIM_ERROR (Cannot connect to VIM, VIM response error, ...) - # OTHER (Vim reported other status not understood) - # ERROR (VIM indicates an ERROR status) - # ACTIVE, INACTIVE, DOWN (admin down), - # BUILD (on building process) - # - error_msg: #Text with VIM error message, if any. Or the VIM connection ERROR - vim_info: #Text with plain information obtained from vim (yaml.safe_dump) - """ - net_dict = {} + def _get_networks_vm(self, vm_element): + interfaces = [] try: - for net_id in net_list: - net = {} - try: - net_vim = self.get_network(net_id) - net["status"] = net_vim["status"] - net["vim_info"] = None - except vimconn.vimconnNotFoundException as e: - self.logger.error("Exception getting net status: {}".format(str(e))) - net['status'] = "DELETED" - net['error_msg'] = str(e) - except vimconn.vimconnException as e: - self.logger.error(e) - net["status"] = "VIM_ERROR" - net["error_msg"] = str(e) - net_dict[net_id] = net - return net_dict - except vimconn.vimconnException as e: - self.logger.error(e) - for k in net_dict: - net_dict[k]["status"] = "VIM_ERROR" - net_dict[k]["error_msg"] = str(e) - return net_dict - - # to be used and fixed in future commits... not working properly - # def action_vminstance(self, vm_id, action_dict): - # """Send and action over a VM instance from VIM - # Returns the status""" - # try: - # if "console" in action_dict: - # console_dict = {"protocol": "http", - # "server": "10.95.84.42", - # "port": "29876", - # "suffix": "?token=4hsb9cu9utruakon4p3z" - # } - # return console_dict - # except vimconn.vimconnException as e: - # self.logger.error(e) + if isinstance(vm_element.TEMPLATE["NIC"], list): + for net in vm_element.TEMPLATE["NIC"]: + interface = {'vim_info': None, "mac_address": str(net["MAC"]), "vim_net_id": str(net["NETWORK_ID"]), + "vim_interface_id": str(net["NETWORK_ID"])} + # maybe it should be 2 different keys for ip_address if an interface has ipv4 and ipv6 + if u'IP' in net: + interface["ip_address"] = str(net["IP"]) + if u'IP6_GLOBAL' in net: + interface["ip_address"] = str(net["IP6_GLOBAL"]) + interfaces.append(interface) + else: + net = vm_element.TEMPLATE["NIC"] + interface = {'vim_info': None, "mac_address": str(net["MAC"]), "vim_net_id": str(net["NETWORK_ID"]), + "vim_interface_id": str(net["NETWORK_ID"])} + # maybe it should be 2 different keys for ip_address if an interface has ipv4 and ipv6 + if u'IP' in net: + interface["ip_address"] = str(net["IP"]) + if u'IP6_GLOBAL' in net: + interface["ip_address"] = str(net["IP6_GLOBAL"]) + interfaces.append(interface) + return interfaces + except Exception as e: + self.logger.error("Error getting vm interface_information of vm_id: "+str(vm_element.ID)) diff --git a/requirements.txt b/requirements.txt index 113f827e..bda5b8a4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,5 +20,6 @@ prettytable boto genisoimage untangle +pyone oca azure diff --git a/scripts/install-openmano.sh b/scripts/install-openmano.sh index 398317d3..98d09050 100755 --- a/scripts/install-openmano.sh +++ b/scripts/install-openmano.sh @@ -267,6 +267,7 @@ then # required for OpenNebula connector pip2 install untangle || exit 1 + pip2 install pyone || exit 1 pip2 install -e git+https://github.com/python-oca/python-oca#egg=oca || exit 1 # required for AWS connector diff --git a/scripts/python-osm-ro.postinst b/scripts/python-osm-ro.postinst index 32c89034..0f6e5a8f 100755 --- a/scripts/python-osm-ro.postinst +++ b/scripts/python-osm-ro.postinst @@ -29,6 +29,7 @@ pip2 install --upgrade prettytable pip2 install --upgrade pyvmomi pip2 install --upgrade pyang pyangbind pip2 install untangle +pip2 install pyone pip2 install -e git+https://github.com/python-oca/python-oca#egg=oca pip2 install azure