RO-VIM-openstack/osm_rovim_openstack/vimconn_openstack.py

   1 # -*- coding: utf-8 -*-
   2
   3 ##
   4 # Copyright 2015 Telefonica Investigacion y Desarrollo, S.A.U.
   5 # This file is part of openmano
   6 # All Rights Reserved.
   7 #
   8 # Licensed under the Apache License, Version 2.0 (the "License"); you may
   9 # not use this file except in compliance with the License. You may obtain
  10 # a copy of the License at
  11 #
  12 #         http://www.apache.org/licenses/LICENSE-2.0
  13 #
  14 # Unless required by applicable law or agreed to in writing, software
  15 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  16 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  17 # License for the specific language governing permissions and limitations
  18 # under the License.
  19 ##
  20
  21 """
  22 osconnector implements all the methods to interact with openstack using the python-neutronclient.
  23
  24 For the VNF forwarding graph, The OpenStack VIM connector calls the
  25 networking-sfc Neutron extension methods, whose resources are mapped
  26 to the VIM connector's SFC resources as follows:
  27 - Classification (OSM) -> Flow Classifier (Neutron)
  28 - Service Function Instance (OSM) -> Port Pair (Neutron)
  29 - Service Function (OSM) -> Port Pair Group (Neutron)
  30 - Service Function Path (OSM) -> Port Chain (Neutron)
  31 """
  32
  33 from osm_ro_plugin import vimconn
  34 # import json
  35 import logging
  36 import netaddr
  37 import time
  38 import yaml
  39 import random
  40 import re
  41 import copy
  42 from pprint import pformat
  43
  44 from novaclient import client as nClient, exceptions as nvExceptions
  45 from keystoneauth1.identity import v2, v3
  46 from keystoneauth1 import session
  47 import keystoneclient.exceptions as ksExceptions
  48 import keystoneclient.v3.client as ksClient_v3
  49 import keystoneclient.v2_0.client as ksClient_v2
  50 from glanceclient import client as glClient
  51 import glanceclient.exc as gl1Exceptions
  52 from cinderclient import client as cClient
  53 from http.client import HTTPException   # TODO py3 check that this base exception matches python2 httplib.HTTPException
  54 from neutronclient.neutron import client as neClient
  55 from neutronclient.common import exceptions as neExceptions
  56 from requests.exceptions import ConnectionError
  57
  58 __author__ = "Alfonso Tierno, Gerardo Garcia, Pablo Montes, xFlow Research, Igor D.C., Eduardo Sousa"
  59 __date__ = "$22-sep-2017 23:59:59$"
  60
  61 """contain the openstack virtual machine status to openmano status"""
  62 vmStatus2manoFormat = {'ACTIVE': 'ACTIVE',
  63                        'PAUSED': 'PAUSED',
  64                        'SUSPENDED': 'SUSPENDED',
  65                        'SHUTOFF': 'INACTIVE',
  66                        'BUILD': 'BUILD',
  67                        'ERROR': 'ERROR',
  68                        'DELETED': 'DELETED'
  69                        }
  70 netStatus2manoFormat = {'ACTIVE': 'ACTIVE',
  71                         'PAUSED': 'PAUSED',
  72                         'INACTIVE': 'INACTIVE',
  73                         'BUILD': 'BUILD',
  74                         'ERROR': 'ERROR',
  75                         'DELETED': 'DELETED'
  76                         }
  77
  78 supportedClassificationTypes = ['legacy_flow_classifier']
  79
  80 # global var to have a timeout creating and deleting volumes
  81 volume_timeout = 1800
  82 server_timeout = 1800
  83
  84
  85 class SafeDumper(yaml.SafeDumper):
  86     def represent_data(self, data):
  87         # Openstack APIs use custom subclasses of dict and YAML safe dumper
  88         # is designed to not handle that (reference issue 142 of pyyaml)
  89         if isinstance(data, dict) and data.__class__ != dict:
  90             # A simple solution is to convert those items back to dicts
  91             data = dict(data.items())
  92
  93         return super(SafeDumper, self).represent_data(data)
  94
  95
  96 class vimconnector(vimconn.VimConnector):
  97     def __init__(self, uuid, name, tenant_id, tenant_name, url, url_admin=None, user=None, passwd=None,
  98                  log_level=None, config={}, persistent_info={}):
  99         """using common constructor parameters. In this case
 100         'url' is the keystone authorization url,
 101         'url_admin' is not use
 102         """
 103         api_version = config.get('APIversion')
 104         if api_version and api_version not in ('v3.3', 'v2.0', '2', '3'):
 105             raise vimconn.VimConnException("Invalid value '{}' for config:APIversion. "
 106                                            "Allowed values are 'v3.3', 'v2.0', '2' or '3'".format(api_version))
 107         vim_type = config.get('vim_type')
 108         if vim_type and vim_type not in ('vio', 'VIO'):
 109             raise vimconn.VimConnException("Invalid value '{}' for config:vim_type."
 110                                            "Allowed values are 'vio' or 'VIO'".format(vim_type))
 111
 112         if config.get('dataplane_net_vlan_range') is not None:
 113             # validate vlan ranges provided by user
 114             self._validate_vlan_ranges(config.get('dataplane_net_vlan_range'), 'dataplane_net_vlan_range')
 115
 116         if config.get('multisegment_vlan_range') is not None:
 117             # validate vlan ranges provided by user
 118             self._validate_vlan_ranges(config.get('multisegment_vlan_range'), 'multisegment_vlan_range')
 119
 120         vimconn.VimConnector.__init__(self, uuid, name, tenant_id, tenant_name, url, url_admin, user, passwd, log_level,
 121                                       config)
 122
 123         if self.config.get("insecure") and self.config.get("ca_cert"):
 124             raise vimconn.VimConnException("options insecure and ca_cert are mutually exclusive")
 125         self.verify = True
 126         if self.config.get("insecure"):
 127             self.verify = False
 128         if self.config.get("ca_cert"):
 129             self.verify = self.config.get("ca_cert")
 130
 131         if not url:
 132             raise TypeError('url param can not be NoneType')
 133         self.persistent_info = persistent_info
 134         self.availability_zone = persistent_info.get('availability_zone', None)
 135         self.session = persistent_info.get('session', {'reload_client': True})
 136         self.my_tenant_id = self.session.get('my_tenant_id')
 137         self.nova = self.session.get('nova')
 138         self.neutron = self.session.get('neutron')
 139         self.cinder = self.session.get('cinder')
 140         self.glance = self.session.get('glance')
 141         # self.glancev1 = self.session.get('glancev1')
 142         self.keystone = self.session.get('keystone')
 143         self.api_version3 = self.session.get('api_version3')
 144         self.vim_type = self.config.get("vim_type")
 145         if self.vim_type:
 146             self.vim_type = self.vim_type.upper()
 147         if self.config.get("use_internal_endpoint"):
 148             self.endpoint_type = "internalURL"
 149         else:
 150             self.endpoint_type = None
 151
 152         logging.getLogger('urllib3').setLevel(logging.WARNING)
 153         logging.getLogger('keystoneauth').setLevel(logging.WARNING)
 154         logging.getLogger('novaclient').setLevel(logging.WARNING)
 155         self.logger = logging.getLogger('openmano.vim.openstack')
 156
 157         # allow security_groups to be a list or a single string
 158         if isinstance(self.config.get('security_groups'), str):
 159             self.config['security_groups'] = [self.config['security_groups']]
 160         self.security_groups_id = None
 161
 162         # ###### VIO Specific Changes #########
 163         if self.vim_type == "VIO":
 164             self.logger = logging.getLogger('openmano.vim.vio')
 165
 166         if log_level:
 167             self.logger.setLevel(getattr(logging, log_level))
 168
 169     def __getitem__(self, index):
 170         """Get individuals parameters.
 171         Throw KeyError"""
 172         if index == 'project_domain_id':
 173             return self.config.get("project_domain_id")
 174         elif index == 'user_domain_id':
 175             return self.config.get("user_domain_id")
 176         else:
 177             return vimconn.VimConnector.__getitem__(self, index)
 178
 179     def __setitem__(self, index, value):
 180         """Set individuals parameters and it is marked as dirty so to force connection reload.
 181         Throw KeyError"""
 182         if index == 'project_domain_id':
 183             self.config["project_domain_id"] = value
 184         elif index == 'user_domain_id':
 185             self.config["user_domain_id"] = value
 186         else:
 187             vimconn.VimConnector.__setitem__(self, index, value)
 188         self.session['reload_client'] = True
 189
 190     def serialize(self, value):
 191         """Serialization of python basic types.
 192
 193         In the case value is not serializable a message will be logged and a
 194         simple representation of the data that cannot be converted back to
 195         python is returned.
 196         """
 197         if isinstance(value, str):
 198             return value
 199
 200         try:
 201             return yaml.dump(value, Dumper=SafeDumper,
 202                              default_flow_style=True, width=256)
 203         except yaml.representer.RepresenterError:
 204             self.logger.debug('The following entity cannot be serialized in YAML:\n\n%s\n\n', pformat(value),
 205                               exc_info=True)
 206             return str(value)
 207
 208     def _reload_connection(self):
 209         """Called before any operation, it check if credentials has changed
 210         Throw keystoneclient.apiclient.exceptions.AuthorizationFailure
 211         """
 212         # TODO control the timing and possible token timeout, but it seams that python client does this task for us :-)
 213         if self.session['reload_client']:
 214             if self.config.get('APIversion'):
 215                 self.api_version3 = self.config['APIversion'] == 'v3.3' or self.config['APIversion'] == '3'
 216             else:  # get from ending auth_url that end with v3 or with v2.0
 217                 self.api_version3 = self.url.endswith("/v3") or self.url.endswith("/v3/")
 218             self.session['api_version3'] = self.api_version3
 219             if self.api_version3:
 220                 if self.config.get('project_domain_id') or self.config.get('project_domain_name'):
 221                     project_domain_id_default = None
 222                 else:
 223                     project_domain_id_default = 'default'
 224                 if self.config.get('user_domain_id') or self.config.get('user_domain_name'):
 225                     user_domain_id_default = None
 226                 else:
 227                     user_domain_id_default = 'default'
 228                 auth = v3.Password(auth_url=self.url,
 229                                    username=self.user,
 230                                    password=self.passwd,
 231                                    project_name=self.tenant_name,
 232                                    project_id=self.tenant_id,
 233                                    project_domain_id=self.config.get('project_domain_id', project_domain_id_default),
 234                                    user_domain_id=self.config.get('user_domain_id', user_domain_id_default),
 235                                    project_domain_name=self.config.get('project_domain_name'),
 236                                    user_domain_name=self.config.get('user_domain_name'))
 237             else:
 238                 auth = v2.Password(auth_url=self.url,
 239                                    username=self.user,
 240                                    password=self.passwd,
 241                                    tenant_name=self.tenant_name,
 242                                    tenant_id=self.tenant_id)
 243             sess = session.Session(auth=auth, verify=self.verify)
 244             # addedd region_name to keystone, nova, neutron and cinder to support distributed cloud for Wind River
 245             # Titanium cloud and StarlingX
 246             region_name = self.config.get('region_name')
 247             if self.api_version3:
 248                 self.keystone = ksClient_v3.Client(session=sess, endpoint_type=self.endpoint_type,
 249                                                    region_name=region_name)
 250             else:
 251                 self.keystone = ksClient_v2.Client(session=sess, endpoint_type=self.endpoint_type)
 252             self.session['keystone'] = self.keystone
 253             # In order to enable microversion functionality an explicit microversion must be specified in 'config'.
 254             # This implementation approach is due to the warning message in
 255             # https://developer.openstack.org/api-guide/compute/microversions.html
 256             # where it is stated that microversion backwards compatibility is not guaranteed and clients should
 257             # always require an specific microversion.
 258             # To be able to use 'device role tagging' functionality define 'microversion: 2.32' in datacenter config
 259             version = self.config.get("microversion")
 260             if not version:
 261                 version = "2.1"
 262             # addedd region_name to keystone, nova, neutron and cinder to support distributed cloud for Wind River
 263             # Titanium cloud and StarlingX
 264             self.nova = self.session['nova'] = nClient.Client(str(version), session=sess,
 265                                                               endpoint_type=self.endpoint_type, region_name=region_name)
 266             self.neutron = self.session['neutron'] = neClient.Client('2.0', session=sess,
 267                                                                      endpoint_type=self.endpoint_type,
 268                                                                      region_name=region_name)
 269             self.cinder = self.session['cinder'] = cClient.Client(2, session=sess, endpoint_type=self.endpoint_type,
 270                                                                   region_name=region_name)
 271             try:
 272                 self.my_tenant_id = self.session['my_tenant_id'] = sess.get_project_id()
 273             except Exception:
 274                 self.logger.error("Cannot get project_id from session", exc_info=True)
 275             if self.endpoint_type == "internalURL":
 276                 glance_service_id = self.keystone.services.list(name="glance")[0].id
 277                 glance_endpoint = self.keystone.endpoints.list(glance_service_id, interface="internal")[0].url
 278             else:
 279                 glance_endpoint = None
 280             self.glance = self.session['glance'] = glClient.Client(2, session=sess, endpoint=glance_endpoint)
 281             # using version 1 of glance client in new_image()
 282             # self.glancev1 = self.session['glancev1'] = glClient.Client('1', session=sess,
 283             #                                                            endpoint=glance_endpoint)
 284             self.session['reload_client'] = False
 285             self.persistent_info['session'] = self.session
 286             # add availablity zone info inside  self.persistent_info
 287             self._set_availablity_zones()
 288             self.persistent_info['availability_zone'] = self.availability_zone
 289             self.security_groups_id = None  # force to get again security_groups_ids next time they are needed
 290
 291     def __net_os2mano(self, net_list_dict):
 292         """Transform the net openstack format to mano format
 293         net_list_dict can be a list of dict or a single dict"""
 294         if type(net_list_dict) is dict:
 295             net_list_ = (net_list_dict,)
 296         elif type(net_list_dict) is list:
 297             net_list_ = net_list_dict
 298         else:
 299             raise TypeError("param net_list_dict must be a list or a dictionary")
 300         for net in net_list_:
 301             if net.get('provider:network_type') == "vlan":
 302                 net['type'] = 'data'
 303             else:
 304                 net['type'] = 'bridge'
 305
 306     def __classification_os2mano(self, class_list_dict):
 307         """Transform the openstack format (Flow Classifier) to mano format
 308         (Classification) class_list_dict can be a list of dict or a single dict
 309         """
 310         if isinstance(class_list_dict, dict):
 311             class_list_ = [class_list_dict]
 312         elif isinstance(class_list_dict, list):
 313             class_list_ = class_list_dict
 314         else:
 315             raise TypeError("param class_list_dict must be a list or a dictionary")
 316         for classification in class_list_:
 317             id = classification.pop('id')
 318             name = classification.pop('name')
 319             description = classification.pop('description')
 320             project_id = classification.pop('project_id')
 321             tenant_id = classification.pop('tenant_id')
 322             original_classification = copy.deepcopy(classification)
 323             classification.clear()
 324             classification['ctype'] = 'legacy_flow_classifier'
 325             classification['definition'] = original_classification
 326             classification['id'] = id
 327             classification['name'] = name
 328             classification['description'] = description
 329             classification['project_id'] = project_id
 330             classification['tenant_id'] = tenant_id
 331
 332     def __sfi_os2mano(self, sfi_list_dict):
 333         """Transform the openstack format (Port Pair) to mano format (SFI)
 334         sfi_list_dict can be a list of dict or a single dict
 335         """
 336         if isinstance(sfi_list_dict, dict):
 337             sfi_list_ = [sfi_list_dict]
 338         elif isinstance(sfi_list_dict, list):
 339             sfi_list_ = sfi_list_dict
 340         else:
 341             raise TypeError(
 342                 "param sfi_list_dict must be a list or a dictionary")
 343         for sfi in sfi_list_:
 344             sfi['ingress_ports'] = []
 345             sfi['egress_ports'] = []
 346             if sfi.get('ingress'):
 347                 sfi['ingress_ports'].append(sfi['ingress'])
 348             if sfi.get('egress'):
 349                 sfi['egress_ports'].append(sfi['egress'])
 350             del sfi['ingress']
 351             del sfi['egress']
 352             params = sfi.get('service_function_parameters')
 353             sfc_encap = False
 354             if params:
 355                 correlation = params.get('correlation')
 356                 if correlation:
 357                     sfc_encap = True
 358             sfi['sfc_encap'] = sfc_encap
 359             del sfi['service_function_parameters']
 360
 361     def __sf_os2mano(self, sf_list_dict):
 362         """Transform the openstack format (Port Pair Group) to mano format (SF)
 363         sf_list_dict can be a list of dict or a single dict
 364         """
 365         if isinstance(sf_list_dict, dict):
 366             sf_list_ = [sf_list_dict]
 367         elif isinstance(sf_list_dict, list):
 368             sf_list_ = sf_list_dict
 369         else:
 370             raise TypeError(
 371                 "param sf_list_dict must be a list or a dictionary")
 372         for sf in sf_list_:
 373             del sf['port_pair_group_parameters']
 374             sf['sfis'] = sf['port_pairs']
 375             del sf['port_pairs']
 376
 377     def __sfp_os2mano(self, sfp_list_dict):
 378         """Transform the openstack format (Port Chain) to mano format (SFP)
 379         sfp_list_dict can be a list of dict or a single dict
 380         """
 381         if isinstance(sfp_list_dict, dict):
 382             sfp_list_ = [sfp_list_dict]
 383         elif isinstance(sfp_list_dict, list):
 384             sfp_list_ = sfp_list_dict
 385         else:
 386             raise TypeError(
 387                 "param sfp_list_dict must be a list or a dictionary")
 388         for sfp in sfp_list_:
 389             params = sfp.pop('chain_parameters')
 390             sfc_encap = False
 391             if params:
 392                 correlation = params.get('correlation')
 393                 if correlation:
 394                     sfc_encap = True
 395             sfp['sfc_encap'] = sfc_encap
 396             sfp['spi'] = sfp.pop('chain_id')
 397             sfp['classifications'] = sfp.pop('flow_classifiers')
 398             sfp['service_functions'] = sfp.pop('port_pair_groups')
 399
 400     # placeholder for now; read TODO note below
 401     def _validate_classification(self, type, definition):
 402         # only legacy_flow_classifier Type is supported at this point
 403         return True
 404         # TODO(igordcard): this method should be an abstract method of an
 405         # abstract Classification class to be implemented by the specific
 406         # Types. Also, abstract vimconnector should call the validation
 407         # method before the implemented VIM connectors are called.
 408
 409     def _format_exception(self, exception):
 410         """Transform a keystone, nova, neutron  exception into a vimconn exception discovering the cause"""
 411
 412         message_error = str(exception)
 413
 414         if isinstance(exception, (neExceptions.NetworkNotFoundClient, nvExceptions.NotFound, ksExceptions.NotFound,
 415                                   gl1Exceptions.HTTPNotFound)):
 416             raise vimconn.VimConnNotFoundException(type(exception).__name__ + ": " + message_error)
 417         elif isinstance(exception, (HTTPException, gl1Exceptions.HTTPException, gl1Exceptions.CommunicationError,
 418                                     ConnectionError, ksExceptions.ConnectionError, neExceptions.ConnectionFailed)):
 419             raise vimconn.VimConnConnectionException(type(exception).__name__ + ": " + message_error)
 420         elif isinstance(exception, (KeyError, nvExceptions.BadRequest, ksExceptions.BadRequest)):
 421             raise vimconn.VimConnException(type(exception).__name__ + ": " + message_error)
 422         elif isinstance(exception, (nvExceptions.ClientException, ksExceptions.ClientException,
 423                                     neExceptions.NeutronException)):
 424             raise vimconn.VimConnUnexpectedResponse(type(exception).__name__ + ": " + message_error)
 425         elif isinstance(exception, nvExceptions.Conflict):
 426             raise vimconn.VimConnConflictException(type(exception).__name__ + ": " + message_error)
 427         elif isinstance(exception, vimconn.VimConnException):
 428             raise exception
 429         else:  # ()
 430             self.logger.error("General Exception " + message_error, exc_info=True)
 431             raise vimconn.VimConnConnectionException(type(exception).__name__ + ": " + message_error)
 432
 433     def _get_ids_from_name(self):
 434         """
 435          Obtain ids from name of tenant and security_groups. Store at self .security_groups_id"
 436         :return: None
 437         """
 438         # get tenant_id if only tenant_name is supplied
 439         self._reload_connection()
 440         if not self.my_tenant_id:
 441             raise vimconn.VimConnConnectionException("Error getting tenant information from name={} id={}".
 442                                                      format(self.tenant_name, self.tenant_id))
 443         if self.config.get('security_groups') and not self.security_groups_id:
 444             # convert from name to id
 445             neutron_sg_list = self.neutron.list_security_groups(tenant_id=self.my_tenant_id)["security_groups"]
 446
 447             self.security_groups_id = []
 448             for sg in self.config.get('security_groups'):
 449                 for neutron_sg in neutron_sg_list:
 450                     if sg in (neutron_sg["id"], neutron_sg["name"]):
 451                         self.security_groups_id.append(neutron_sg["id"])
 452                         break
 453                 else:
 454                     self.security_groups_id = None
 455                     raise vimconn.VimConnConnectionException("Not found security group {} for this tenant".format(sg))
 456
 457     def check_vim_connectivity(self):
 458         # just get network list to check connectivity and credentials
 459         self.get_network_list(filter_dict={})
 460
 461     def get_tenant_list(self, filter_dict={}):
 462         """Obtain tenants of VIM
 463         filter_dict can contain the following keys:
 464             name: filter by tenant name
 465             id: filter by tenant uuid/id
 466             <other VIM specific>
 467         Returns the tenant list of dictionaries: [{'name':'<name>, 'id':'<id>, ...}, ...]
 468         """
 469         self.logger.debug("Getting tenants from VIM filter: '%s'", str(filter_dict))
 470         try:
 471             self._reload_connection()
 472             if self.api_version3:
 473                 project_class_list = self.keystone.projects.list(name=filter_dict.get("name"))
 474             else:
 475                 project_class_list = self.keystone.tenants.findall(**filter_dict)
 476             project_list = []
 477             for project in project_class_list:
 478                 if filter_dict.get('id') and filter_dict["id"] != project.id:
 479                     continue
 480                 project_list.append(project.to_dict())
 481             return project_list
 482         except (ksExceptions.ConnectionError, ksExceptions.ClientException, ConnectionError) as e:
 483             self._format_exception(e)
 484
 485     def new_tenant(self, tenant_name, tenant_description):
 486         """Adds a new tenant to openstack VIM. Returns the tenant identifier"""
 487         self.logger.debug("Adding a new tenant name: %s", tenant_name)
 488         try:
 489             self._reload_connection()
 490             if self.api_version3:
 491                 project = self.keystone.projects.create(tenant_name, self.config.get("project_domain_id", "default"),
 492                                                         description=tenant_description, is_domain=False)
 493             else:
 494                 project = self.keystone.tenants.create(tenant_name, tenant_description)
 495             return project.id
 496         except (ksExceptions.ConnectionError, ksExceptions.ClientException, ksExceptions.BadRequest, ConnectionError)\
 497                 as e:
 498             self._format_exception(e)
 499
 500     def delete_tenant(self, tenant_id):
 501         """Delete a tenant from openstack VIM. Returns the old tenant identifier"""
 502         self.logger.debug("Deleting tenant %s from VIM", tenant_id)
 503         try:
 504             self._reload_connection()
 505             if self.api_version3:
 506                 self.keystone.projects.delete(tenant_id)
 507             else:
 508                 self.keystone.tenants.delete(tenant_id)
 509             return tenant_id
 510         except (ksExceptions.ConnectionError, ksExceptions.ClientException, ksExceptions.NotFound, ConnectionError)\
 511                 as e:
 512             self._format_exception(e)
 513
 514     def new_network(self, net_name, net_type, ip_profile=None, shared=False, provider_network_profile=None):
 515         """Adds a tenant network to VIM
 516         Params:
 517             'net_name': name of the network
 518             'net_type': one of:
 519                 'bridge': overlay isolated network
 520                 'data':   underlay E-LAN network for Passthrough and SRIOV interfaces
 521                 'ptp':    underlay E-LINE network for Passthrough and SRIOV interfaces.
 522             'ip_profile': is a dict containing the IP parameters of the network
 523                 'ip_version': can be "IPv4" or "IPv6" (Currently only IPv4 is implemented)
 524                 'subnet_address': ip_prefix_schema, that is X.X.X.X/Y
 525                 'gateway_address': (Optional) ip_schema, that is X.X.X.X
 526                 'dns_address': (Optional) comma separated list of ip_schema, e.g. X.X.X.X[,X,X,X,X]
 527                 'dhcp_enabled': True or False
 528                 'dhcp_start_address': ip_schema, first IP to grant
 529                 'dhcp_count': number of IPs to grant.
 530             'shared': if this network can be seen/use by other tenants/organization
 531             'provider_network_profile': (optional) contains {segmentation-id: vlan, network-type: vlan|vxlan,
 532                                                              physical-network: physnet-label}
 533         Returns a tuple with the network identifier and created_items, or raises an exception on error
 534             created_items can be None or a dictionary where this method can include key-values that will be passed to
 535             the method delete_network. Can be used to store created segments, created l2gw connections, etc.
 536             Format is vimconnector dependent, but do not use nested dictionaries and a value of None should be the same
 537             as not present.
 538         """
 539         self.logger.debug("Adding a new network to VIM name '%s', type '%s'", net_name, net_type)
 540         # self.logger.debug(">>>>>>>>>>>>>>>>>> IP profile %s", str(ip_profile))
 541
 542         try:
 543             vlan = None
 544             if provider_network_profile:
 545                 vlan = provider_network_profile.get("segmentation-id")
 546             new_net = None
 547             created_items = {}
 548             self._reload_connection()
 549             network_dict = {'name': net_name, 'admin_state_up': True}
 550             if net_type in ("data", "ptp"):
 551                 provider_physical_network = None
 552                 if provider_network_profile and provider_network_profile.get("physical-network"):
 553                     provider_physical_network = provider_network_profile.get("physical-network")
 554                     # provider-network must be one of the dataplane_physcial_netowrk if this is a list. If it is string
 555                     # or not declared, just ignore the checking
 556                     if isinstance(self.config.get('dataplane_physical_net'), (tuple, list)) and \
 557                             provider_physical_network not in self.config['dataplane_physical_net']:
 558                         raise vimconn.VimConnConflictException(
 559                             "Invalid parameter 'provider-network:physical-network' for network creation. '{}' is not "
 560                             "one of the declared list at VIM_config:dataplane_physical_net".format(
 561                                 provider_physical_network))
 562                 if not provider_physical_network:  # use the default dataplane_physical_net
 563                     provider_physical_network = self.config.get('dataplane_physical_net')
 564                     # if it is non empty list, use the first value. If it is a string use the value directly
 565                     if isinstance(provider_physical_network, (tuple, list)) and provider_physical_network:
 566                         provider_physical_network = provider_physical_network[0]
 567
 568                 if not provider_physical_network:
 569                     raise vimconn.VimConnConflictException("You must provide a 'dataplane_physical_net' at VIM_config "
 570                                                            "for creating underlay networks. or use the NS instantiation"
 571                                                            " parameter provider-network:physical-network for the VLD")
 572
 573                 if not self.config.get('multisegment_support'):
 574                     network_dict["provider:physical_network"] = provider_physical_network
 575                     if provider_network_profile and "network-type" in provider_network_profile:
 576                         network_dict["provider:network_type"] = provider_network_profile["network-type"]
 577                     else:
 578                         network_dict["provider:network_type"] = self.config.get('dataplane_network_type', 'vlan')
 579                     if vlan:
 580                         network_dict["provider:segmentation_id"] = vlan
 581                 else:
 582                     # Multi-segment case
 583                     segment_list = []
 584                     segment1_dict = {
 585                         "provider:physical_network": '',
 586                         "provider:network_type": 'vxlan'
 587                     }
 588                     segment_list.append(segment1_dict)
 589                     segment2_dict = {
 590                         "provider:physical_network": provider_physical_network,
 591                         "provider:network_type": "vlan"
 592                     }
 593                     if vlan:
 594                         segment2_dict["provider:segmentation_id"] = vlan
 595                     elif self.config.get('multisegment_vlan_range'):
 596                         vlanID = self._generate_multisegment_vlanID()
 597                         segment2_dict["provider:segmentation_id"] = vlanID
 598                     # else
 599                     #     raise vimconn.VimConnConflictException(
 600                     #         "You must provide 'multisegment_vlan_range' at config dict before creating a multisegment
 601                     #         network")
 602                     segment_list.append(segment2_dict)
 603                     network_dict["segments"] = segment_list
 604
 605                 # VIO Specific Changes. It needs a concrete VLAN
 606                 if self.vim_type == "VIO" and vlan is None:
 607                     if self.config.get('dataplane_net_vlan_range') is None:
 608                         raise vimconn.VimConnConflictException(
 609                             "You must provide 'dataplane_net_vlan_range' in format [start_ID - end_ID] at VIM_config "
 610                             "for creating underlay networks")
 611                     network_dict["provider:segmentation_id"] = self._generate_vlanID()
 612
 613             network_dict["shared"] = shared
 614             if self.config.get("disable_network_port_security"):
 615                 network_dict["port_security_enabled"] = False
 616             new_net = self.neutron.create_network({'network': network_dict})
 617             # print new_net
 618             # create subnetwork, even if there is no profile
 619             if not ip_profile:
 620                 ip_profile = {}
 621             if not ip_profile.get('subnet_address'):
 622                 # Fake subnet is required
 623                 subnet_rand = random.randint(0, 255)
 624                 ip_profile['subnet_address'] = "192.168.{}.0/24".format(subnet_rand)
 625             if 'ip_version' not in ip_profile:
 626                 ip_profile['ip_version'] = "IPv4"
 627             subnet = {"name": net_name+"-subnet",
 628                       "network_id": new_net["network"]["id"],
 629                       "ip_version": 4 if ip_profile['ip_version'] == "IPv4" else 6,
 630                       "cidr": ip_profile['subnet_address']
 631                       }
 632             # Gateway should be set to None if not needed. Otherwise openstack assigns one by default
 633             if ip_profile.get('gateway_address'):
 634                 subnet['gateway_ip'] = ip_profile['gateway_address']
 635             else:
 636                 subnet['gateway_ip'] = None
 637             if ip_profile.get('dns_address'):
 638                 subnet['dns_nameservers'] = ip_profile['dns_address'].split(";")
 639             if 'dhcp_enabled' in ip_profile:
 640                 subnet['enable_dhcp'] = False if \
 641                     ip_profile['dhcp_enabled'] == "false" or ip_profile['dhcp_enabled'] is False else True
 642             if ip_profile.get('dhcp_start_address'):
 643                 subnet['allocation_pools'] = []
 644                 subnet['allocation_pools'].append(dict())
 645                 subnet['allocation_pools'][0]['start'] = ip_profile['dhcp_start_address']
 646             if ip_profile.get('dhcp_count'):
 647                 # parts = ip_profile['dhcp_start_address'].split('.')
 648                 # ip_int = (int(parts[0]) << 24) + (int(parts[1]) << 16) + (int(parts[2]) << 8) + int(parts[3])
 649                 ip_int = int(netaddr.IPAddress(ip_profile['dhcp_start_address']))
 650                 ip_int += ip_profile['dhcp_count'] - 1
 651                 ip_str = str(netaddr.IPAddress(ip_int))
 652                 subnet['allocation_pools'][0]['end'] = ip_str
 653             # self.logger.debug(">>>>>>>>>>>>>>>>>> Subnet: %s", str(subnet))
 654             self.neutron.create_subnet({"subnet": subnet})
 655
 656             if net_type == "data" and self.config.get('multisegment_support'):
 657                 if self.config.get('l2gw_support'):
 658                     l2gw_list = self.neutron.list_l2_gateways().get("l2_gateways", ())
 659                     for l2gw in l2gw_list:
 660                         l2gw_conn = {
 661                             "l2_gateway_id": l2gw["id"],
 662                             "network_id": new_net["network"]["id"],
 663                             "segmentation_id": str(vlanID),
 664                         }
 665                         new_l2gw_conn = self.neutron.create_l2_gateway_connection({"l2_gateway_connection": l2gw_conn})
 666                         created_items["l2gwconn:" + str(new_l2gw_conn["l2_gateway_connection"]["id"])] = True
 667             return new_net["network"]["id"], created_items
 668         except Exception as e:
 669             # delete l2gw connections (if any) before deleting the network
 670             for k, v in created_items.items():
 671                 if not v:  # skip already deleted
 672                     continue
 673                 try:
 674                     k_item, _, k_id = k.partition(":")
 675                     if k_item == "l2gwconn":
 676                         self.neutron.delete_l2_gateway_connection(k_id)
 677                 except Exception as e2:
 678                     self.logger.error("Error deleting l2 gateway connection: {}: {}".format(type(e2).__name__, e2))
 679             if new_net:
 680                 self.neutron.delete_network(new_net['network']['id'])
 681             self._format_exception(e)
 682
 683     def get_network_list(self, filter_dict={}):
 684         """Obtain tenant networks of VIM
 685         Filter_dict can be:
 686             name: network name
 687             id: network uuid
 688             shared: boolean
 689             tenant_id: tenant
 690             admin_state_up: boolean
 691             status: 'ACTIVE'
 692         Returns the network list of dictionaries
 693         """
 694         self.logger.debug("Getting network from VIM filter: '%s'", str(filter_dict))
 695         try:
 696             self._reload_connection()
 697             filter_dict_os = filter_dict.copy()
 698             if self.api_version3 and "tenant_id" in filter_dict_os:
 699                 filter_dict_os['project_id'] = filter_dict_os.pop('tenant_id')  # TODO check
 700             net_dict = self.neutron.list_networks(**filter_dict_os)
 701             net_list = net_dict["networks"]
 702             self.__net_os2mano(net_list)
 703             return net_list
 704         except (neExceptions.ConnectionFailed, ksExceptions.ClientException, neExceptions.NeutronException,
 705                 ConnectionError) as e:
 706             self._format_exception(e)
 707
 708     def get_network(self, net_id):
 709         """Obtain details of network from VIM
 710         Returns the network information from a network id"""
 711         self.logger.debug(" Getting tenant network %s from VIM", net_id)
 712         filter_dict = {"id": net_id}
 713         net_list = self.get_network_list(filter_dict)
 714         if len(net_list) == 0:
 715             raise vimconn.VimConnNotFoundException("Network '{}' not found".format(net_id))
 716         elif len(net_list) > 1:
 717             raise vimconn.VimConnConflictException("Found more than one network with this criteria")
 718         net = net_list[0]
 719         subnets = []
 720         for subnet_id in net.get("subnets", ()):
 721             try:
 722                 subnet = self.neutron.show_subnet(subnet_id)
 723             except Exception as e:
 724                 self.logger.error("osconnector.get_network(): Error getting subnet %s %s" % (net_id, str(e)))
 725                 subnet = {"id": subnet_id, "fault": str(e)}
 726             subnets.append(subnet)
 727         net["subnets"] = subnets
 728         net["encapsulation"] = net.get('provider:network_type')
 729         net["encapsulation_type"] = net.get('provider:network_type')
 730         net["segmentation_id"] = net.get('provider:segmentation_id')
 731         net["encapsulation_id"] = net.get('provider:segmentation_id')
 732         return net
 733
 734     def delete_network(self, net_id, created_items=None):
 735         """
 736         Removes a tenant network from VIM and its associated elements
 737         :param net_id: VIM identifier of the network, provided by method new_network
 738         :param created_items: dictionary with extra items to be deleted. provided by method new_network
 739         Returns the network identifier or raises an exception upon error or when network is not found
 740         """
 741         self.logger.debug("Deleting network '%s' from VIM", net_id)
 742         if created_items is None:
 743             created_items = {}
 744         try:
 745             self._reload_connection()
 746             # delete l2gw connections (if any) before deleting the network
 747             for k, v in created_items.items():
 748                 if not v:  # skip already deleted
 749                     continue
 750                 try:
 751                     k_item, _, k_id = k.partition(":")
 752                     if k_item == "l2gwconn":
 753                         self.neutron.delete_l2_gateway_connection(k_id)
 754                 except Exception as e:
 755                     self.logger.error("Error deleting l2 gateway connection: {}: {}".format(type(e).__name__, e))
 756             # delete VM ports attached to this networks before the network
 757             ports = self.neutron.list_ports(network_id=net_id)
 758             for p in ports['ports']:
 759                 try:
 760                     self.neutron.delete_port(p["id"])
 761                 except Exception as e:
 762                     self.logger.error("Error deleting port %s: %s", p["id"], str(e))
 763             self.neutron.delete_network(net_id)
 764             return net_id
 765         except (neExceptions.ConnectionFailed, neExceptions.NetworkNotFoundClient, neExceptions.NeutronException,
 766                 ksExceptions.ClientException, neExceptions.NeutronException, ConnectionError) as e:
 767             self._format_exception(e)
 768
 769     def refresh_nets_status(self, net_list):
 770         """Get the status of the networks
 771            Params: the list of network identifiers
 772            Returns a dictionary with:
 773                 net_id:         #VIM id of this network
 774                     status:     #Mandatory. Text with one of:
 775                                 #  DELETED (not found at vim)
 776                                 #  VIM_ERROR (Cannot connect to VIM, VIM response error, ...)
 777                                 #  OTHER (Vim reported other status not understood)
 778                                 #  ERROR (VIM indicates an ERROR status)
 779                                 #  ACTIVE, INACTIVE, DOWN (admin down),
 780                                 #  BUILD (on building process)
 781                                 #
 782                     error_msg:  #Text with VIM error message, if any. Or the VIM connection ERROR
 783                     vim_info:   #Text with plain information obtained from vim (yaml.safe_dump)
 784
 785         """
 786         net_dict = {}
 787         for net_id in net_list:
 788             net = {}
 789             try:
 790                 net_vim = self.get_network(net_id)
 791                 if net_vim['status'] in netStatus2manoFormat:
 792                     net["status"] = netStatus2manoFormat[net_vim['status']]
 793                 else:
 794                     net["status"] = "OTHER"
 795                     net["error_msg"] = "VIM status reported " + net_vim['status']
 796
 797                 if net['status'] == "ACTIVE" and not net_vim['admin_state_up']:
 798                     net['status'] = 'DOWN'
 799
 800                 net['vim_info'] = self.serialize(net_vim)
 801
 802                 if net_vim.get('fault'):  # TODO
 803                     net['error_msg'] = str(net_vim['fault'])
 804             except vimconn.VimConnNotFoundException as e:
 805                 self.logger.error("Exception getting net status: %s", str(e))
 806                 net['status'] = "DELETED"
 807                 net['error_msg'] = str(e)
 808             except vimconn.VimConnException as e:
 809                 self.logger.error("Exception getting net status: %s", str(e))
 810                 net['status'] = "VIM_ERROR"
 811                 net['error_msg'] = str(e)
 812             net_dict[net_id] = net
 813         return net_dict
 814
 815     def get_flavor(self, flavor_id):
 816         """Obtain flavor details from the  VIM. Returns the flavor dict details"""
 817         self.logger.debug("Getting flavor '%s'", flavor_id)
 818         try:
 819             self._reload_connection()
 820             flavor = self.nova.flavors.find(id=flavor_id)
 821             # TODO parse input and translate to VIM format (openmano_schemas.new_vminstance_response_schema)
 822             return flavor.to_dict()
 823         except (nvExceptions.NotFound, nvExceptions.ClientException, ksExceptions.ClientException,
 824                 ConnectionError) as e:
 825             self._format_exception(e)
 826
 827     def get_flavor_id_from_data(self, flavor_dict):
 828         """Obtain flavor id that match the flavor description
 829            Returns the flavor_id or raises a vimconnNotFoundException
 830            flavor_dict: contains the required ram, vcpus, disk
 831            If 'use_existing_flavors' is set to True at config, the closer flavor that provides same or more ram, vcpus
 832                 and disk is returned. Otherwise a flavor with exactly same ram, vcpus and disk is returned or a
 833                 vimconnNotFoundException is raised
 834         """
 835         exact_match = False if self.config.get('use_existing_flavors') else True
 836         try:
 837             self._reload_connection()
 838             flavor_candidate_id = None
 839             flavor_candidate_data = (10000, 10000, 10000)
 840             flavor_target = (flavor_dict["ram"], flavor_dict["vcpus"], flavor_dict["disk"])
 841             # numa=None
 842             extended = flavor_dict.get("extended", {})
 843             if extended:
 844                 # TODO
 845                 raise vimconn.VimConnNotFoundException("Flavor with EPA still not implemented")
 846                 # if len(numas) > 1:
 847                 #     raise vimconn.VimConnNotFoundException("Cannot find any flavor with more than one numa")
 848                 # numa=numas[0]
 849                 # numas = extended.get("numas")
 850             for flavor in self.nova.flavors.list():
 851                 epa = flavor.get_keys()
 852                 if epa:
 853                     continue
 854                     # TODO
 855                 flavor_data = (flavor.ram, flavor.vcpus, flavor.disk)
 856                 if flavor_data == flavor_target:
 857                     return flavor.id
 858                 elif not exact_match and flavor_target < flavor_data < flavor_candidate_data:
 859                     flavor_candidate_id = flavor.id
 860                     flavor_candidate_data = flavor_data
 861             if not exact_match and flavor_candidate_id:
 862                 return flavor_candidate_id
 863             raise vimconn.VimConnNotFoundException("Cannot find any flavor matching '{}'".format(flavor_dict))
 864         except (nvExceptions.NotFound, nvExceptions.ClientException, ksExceptions.ClientException,
 865                 ConnectionError) as e:
 866             self._format_exception(e)
 867
 868     def process_resource_quota(self, quota, prefix, extra_specs):
 869         """
 870         :param prefix:
 871         :param extra_specs:
 872         :return:
 873         """
 874         if 'limit' in quota:
 875             extra_specs["quota:" + prefix + "_limit"] = quota['limit']
 876         if 'reserve' in quota:
 877             extra_specs["quota:" + prefix + "_reservation"] = quota['reserve']
 878         if 'shares' in quota:
 879             extra_specs["quota:" + prefix + "_shares_level"] = "custom"
 880             extra_specs["quota:" + prefix + "_shares_share"] = quota['shares']
 881
 882     def new_flavor(self, flavor_data, change_name_if_used=True):
 883         """Adds a tenant flavor to openstack VIM
 884         if change_name_if_used is True, it will change name in case of conflict, because it is not supported name
 885          repetition
 886         Returns the flavor identifier
 887         """
 888         self.logger.debug("Adding flavor '%s'", str(flavor_data))
 889         retry = 0
 890         max_retries = 3
 891         name_suffix = 0
 892         try:
 893             name = flavor_data['name']
 894             while retry < max_retries:
 895                 retry += 1
 896                 try:
 897                     self._reload_connection()
 898                     if change_name_if_used:
 899                         # get used names
 900                         fl_names = []
 901                         fl = self.nova.flavors.list()
 902                         for f in fl:
 903                             fl_names.append(f.name)
 904                         while name in fl_names:
 905                             name_suffix += 1
 906                             name = flavor_data['name']+"-" + str(name_suffix)
 907
 908                     ram = flavor_data.get('ram', 64)
 909                     vcpus = flavor_data.get('vcpus', 1)
 910                     extra_specs = {}
 911
 912                     extended = flavor_data.get("extended")
 913                     if extended:
 914                         numas = extended.get("numas")
 915                         if numas:
 916                             numa_nodes = len(numas)
 917                             if numa_nodes > 1:
 918                                 return -1, "Can not add flavor with more than one numa"
 919                             extra_specs["hw:numa_nodes"] = str(numa_nodes)
 920                             extra_specs["hw:mem_page_size"] = "large"
 921                             extra_specs["hw:cpu_policy"] = "dedicated"
 922                             extra_specs["hw:numa_mempolicy"] = "strict"
 923                             if self.vim_type == "VIO":
 924                                 extra_specs["vmware:extra_config"] = '{"numa.nodeAffinity":"0"}'
 925                                 extra_specs["vmware:latency_sensitivity_level"] = "high"
 926                             for numa in numas:
 927                                 # overwrite ram and vcpus
 928                                 # check if key 'memory' is present in numa else use ram value at flavor
 929                                 if 'memory' in numa:
 930                                     ram = numa['memory']*1024
 931                                 # See for reference: https://specs.openstack.org/openstack/nova-specs/specs/mitaka/
 932                                 # implemented/virt-driver-cpu-thread-pinning.html
 933                                 extra_specs["hw:cpu_sockets"] = 1
 934                                 if 'paired-threads' in numa:
 935                                     vcpus = numa['paired-threads']*2
 936                                     # cpu_thread_policy "require" implies that the compute node must have an
 937                                     # STM architecture
 938                                     extra_specs["hw:cpu_thread_policy"] = "require"
 939                                     extra_specs["hw:cpu_policy"] = "dedicated"
 940                                 elif 'cores' in numa:
 941                                     vcpus = numa['cores']
 942                                     # cpu_thread_policy "prefer" implies that the host must not have an SMT
 943                                     # architecture, or a non-SMT architecture will be emulated
 944                                     extra_specs["hw:cpu_thread_policy"] = "isolate"
 945                                     extra_specs["hw:cpu_policy"] = "dedicated"
 946                                 elif 'threads' in numa:
 947                                     vcpus = numa['threads']
 948                                     # cpu_thread_policy "prefer" implies that the host may or may not have an SMT
 949                                     # architecture
 950                                     extra_specs["hw:cpu_thread_policy"] = "prefer"
 951                                     extra_specs["hw:cpu_policy"] = "dedicated"
 952                                 # for interface in numa.get("interfaces",() ):
 953                                 #     if interface["dedicated"]=="yes":
 954                                 #         raise vimconn.VimConnException("Passthrough interfaces are not supported
 955                                 #         for the openstack connector", http_code=vimconn.HTTP_Service_Unavailable)
 956                                 #     #TODO, add the key 'pci_passthrough:alias"="<label at config>:<number ifaces>"'
 957                                 #      when a way to connect it is available
 958                         elif extended.get("cpu-quota"):
 959                             self.process_resource_quota(extended.get("cpu-quota"), "cpu", extra_specs)
 960                         if extended.get("mem-quota"):
 961                             self.process_resource_quota(extended.get("mem-quota"), "memory", extra_specs)
 962                         if extended.get("vif-quota"):
 963                             self.process_resource_quota(extended.get("vif-quota"), "vif", extra_specs)
 964                         if extended.get("disk-io-quota"):
 965                             self.process_resource_quota(extended.get("disk-io-quota"), "disk_io", extra_specs)
 966                     # create flavor
 967                     new_flavor = self.nova.flavors.create(name,
 968                                                           ram,
 969                                                           vcpus,
 970                                                           flavor_data.get('disk', 0),
 971                                                           is_public=flavor_data.get('is_public', True)
 972                                                           )
 973                     # add metadata
 974                     if extra_specs:
 975                         new_flavor.set_keys(extra_specs)
 976                     return new_flavor.id
 977                 except nvExceptions.Conflict as e:
 978                     if change_name_if_used and retry < max_retries:
 979                         continue
 980                     self._format_exception(e)
 981         # except nvExceptions.BadRequest as e:
 982         except (ksExceptions.ClientException, nvExceptions.ClientException, ConnectionError, KeyError) as e:
 983             self._format_exception(e)
 984
 985     def delete_flavor(self, flavor_id):
 986         """Deletes a tenant flavor from openstack VIM. Returns the old flavor_id
 987         """
 988         try:
 989             self._reload_connection()
 990             self.nova.flavors.delete(flavor_id)
 991             return flavor_id
 992         # except nvExceptions.BadRequest as e:
 993         except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException,
 994                 ConnectionError) as e:
 995             self._format_exception(e)
 996
 997     def new_image(self, image_dict):
 998         """
 999         Adds a tenant image to VIM. imge_dict is a dictionary with:
1000             name: name
1001             disk_format: qcow2, vhd, vmdk, raw (by default), ...
1002             location: path or URI
1003             public: "yes" or "no"
1004             metadata: metadata of the image
1005         Returns the image_id
1006         """
1007         retry = 0
1008         max_retries = 3
1009         while retry < max_retries:
1010             retry += 1
1011             try:
1012                 self._reload_connection()
1013                 # determine format  http://docs.openstack.org/developer/glance/formats.html
1014                 if "disk_format" in image_dict:
1015                     disk_format = image_dict["disk_format"]
1016                 else:  # autodiscover based on extension
1017                     if image_dict['location'].endswith(".qcow2"):
1018                         disk_format = "qcow2"
1019                     elif image_dict['location'].endswith(".vhd"):
1020                         disk_format = "vhd"
1021                     elif image_dict['location'].endswith(".vmdk"):
1022                         disk_format = "vmdk"
1023                     elif image_dict['location'].endswith(".vdi"):
1024                         disk_format = "vdi"
1025                     elif image_dict['location'].endswith(".iso"):
1026                         disk_format = "iso"
1027                     elif image_dict['location'].endswith(".aki"):
1028                         disk_format = "aki"
1029                     elif image_dict['location'].endswith(".ari"):
1030                         disk_format = "ari"
1031                     elif image_dict['location'].endswith(".ami"):
1032                         disk_format = "ami"
1033                     else:
1034                         disk_format = "raw"
1035                 self.logger.debug("new_image: '%s' loading from '%s'", image_dict['name'], image_dict['location'])
1036                 if self.vim_type == "VIO":
1037                     container_format = "bare"
1038                     if 'container_format' in image_dict:
1039                         container_format = image_dict['container_format']
1040                     new_image = self.glance.images.create(name=image_dict['name'], container_format=container_format,
1041                                                           disk_format=disk_format)
1042                 else:
1043                     new_image = self.glance.images.create(name=image_dict['name'])
1044                 if image_dict['location'].startswith("http"):
1045                     # TODO there is not a method to direct download. It must be downloaded locally with requests
1046                     raise vimconn.VimConnNotImplemented("Cannot create image from URL")
1047                 else:  # local path
1048                     with open(image_dict['location']) as fimage:
1049                         self.glance.images.upload(new_image.id, fimage)
1050                         # new_image = self.glancev1.images.create(name=image_dict['name'], is_public=
1051                         #  image_dict.get('public',"yes")=="yes",
1052                         #    container_format="bare", data=fimage, disk_format=disk_format)
1053                 metadata_to_load = image_dict.get('metadata')
1054                 # TODO location is a reserved word for current openstack versions. fixed for VIO please check
1055                 #  for openstack
1056                 if self.vim_type == "VIO":
1057                     metadata_to_load['upload_location'] = image_dict['location']
1058                 else:
1059                     metadata_to_load['location'] = image_dict['location']
1060                 self.glance.images.update(new_image.id, **metadata_to_load)
1061                 return new_image.id
1062             except (nvExceptions.Conflict, ksExceptions.ClientException, nvExceptions.ClientException) as e:
1063                 self._format_exception(e)
1064             except (HTTPException, gl1Exceptions.HTTPException, gl1Exceptions.CommunicationError, ConnectionError) as e:
1065                 if retry == max_retries:
1066                     continue
1067                 self._format_exception(e)
1068             except IOError as e:  # can not open the file
1069                 raise vimconn.VimConnConnectionException("{}: {} for {}".format(type(e).__name__, e,
1070                                                                                 image_dict['location']),
1071                                                          http_code=vimconn.HTTP_Bad_Request)
1072
1073     def delete_image(self, image_id):
1074         """Deletes a tenant image from openstack VIM. Returns the old id
1075         """
1076         try:
1077             self._reload_connection()
1078             self.glance.images.delete(image_id)
1079             return image_id
1080         except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException,
1081                 gl1Exceptions.CommunicationError, gl1Exceptions.HTTPNotFound, ConnectionError) as e:  # TODO remove
1082             self._format_exception(e)
1083
1084     def get_image_id_from_path(self, path):
1085         """Get the image id from image path in the VIM database. Returns the image_id"""
1086         try:
1087             self._reload_connection()
1088             images = self.glance.images.list()
1089             for image in images:
1090                 if image.metadata.get("location") == path:
1091                     return image.id
1092             raise vimconn.VimConnNotFoundException("image with location '{}' not found".format(path))
1093         except (ksExceptions.ClientException, nvExceptions.ClientException, gl1Exceptions.CommunicationError,
1094                 ConnectionError) as e:
1095             self._format_exception(e)
1096
1097     def get_image_list(self, filter_dict={}):
1098         """Obtain tenant images from VIM
1099         Filter_dict can be:
1100             id: image id
1101             name: image name
1102             checksum: image checksum
1103         Returns the image list of dictionaries:
1104             [{<the fields at Filter_dict plus some VIM specific>}, ...]
1105             List can be empty
1106         """
1107         self.logger.debug("Getting image list from VIM filter: '%s'", str(filter_dict))
1108         try:
1109             self._reload_connection()
1110             # filter_dict_os = filter_dict.copy()
1111             # First we filter by the available filter fields: name, id. The others are removed.
1112             image_list = self.glance.images.list()
1113             filtered_list = []
1114             for image in image_list:
1115                 try:
1116                     if filter_dict.get("name") and image["name"] != filter_dict["name"]:
1117                         continue
1118                     if filter_dict.get("id") and image["id"] != filter_dict["id"]:
1119                         continue
1120                     if filter_dict.get("checksum") and image["checksum"] != filter_dict["checksum"]:
1121                         continue
1122
1123                     filtered_list.append(image.copy())
1124                 except gl1Exceptions.HTTPNotFound:
1125                     pass
1126             return filtered_list
1127         except (ksExceptions.ClientException, nvExceptions.ClientException, gl1Exceptions.CommunicationError,
1128                 ConnectionError) as e:
1129             self._format_exception(e)
1130
1131     def __wait_for_vm(self, vm_id, status):
1132         """wait until vm is in the desired status and return True.
1133         If the VM gets in ERROR status, return false.
1134         If the timeout is reached generate an exception"""
1135         elapsed_time = 0
1136         while elapsed_time < server_timeout:
1137             vm_status = self.nova.servers.get(vm_id).status
1138             if vm_status == status:
1139                 return True
1140             if vm_status == 'ERROR':
1141                 return False
1142             time.sleep(5)
1143             elapsed_time += 5
1144
1145         # if we exceeded the timeout rollback
1146         if elapsed_time >= server_timeout:
1147             raise vimconn.VimConnException('Timeout waiting for instance ' + vm_id + ' to get ' + status,
1148                                            http_code=vimconn.HTTP_Request_Timeout)
1149
1150     def _get_openstack_availablity_zones(self):
1151         """
1152         Get from openstack availability zones available
1153         :return:
1154         """
1155         try:
1156             openstack_availability_zone = self.nova.availability_zones.list()
1157             openstack_availability_zone = [str(zone.zoneName) for zone in openstack_availability_zone
1158                                            if zone.zoneName != 'internal']
1159             return openstack_availability_zone
1160         except Exception:
1161             return None
1162
1163     def _set_availablity_zones(self):
1164         """
1165         Set vim availablity zone
1166         :return:
1167         """
1168
1169         if 'availability_zone' in self.config:
1170             vim_availability_zones = self.config.get('availability_zone')
1171             if isinstance(vim_availability_zones, str):
1172                 self.availability_zone = [vim_availability_zones]
1173             elif isinstance(vim_availability_zones, list):
1174                 self.availability_zone = vim_availability_zones
1175         else:
1176             self.availability_zone = self._get_openstack_availablity_zones()
1177
1178     def _get_vm_availability_zone(self, availability_zone_index, availability_zone_list):
1179         """
1180         Return thge availability zone to be used by the created VM.
1181         :return: The VIM availability zone to be used or None
1182         """
1183         if availability_zone_index is None:
1184             if not self.config.get('availability_zone'):
1185                 return None
1186             elif isinstance(self.config.get('availability_zone'), str):
1187                 return self.config['availability_zone']
1188             else:
1189                 # TODO consider using a different parameter at config for default AV and AV list match
1190                 return self.config['availability_zone'][0]
1191
1192         vim_availability_zones = self.availability_zone
1193         # check if VIM offer enough availability zones describe in the VNFD
1194         if vim_availability_zones and len(availability_zone_list) <= len(vim_availability_zones):
1195             # check if all the names of NFV AV match VIM AV names
1196             match_by_index = False
1197             for av in availability_zone_list:
1198                 if av not in vim_availability_zones:
1199                     match_by_index = True
1200                     break
1201             if match_by_index:
1202                 return vim_availability_zones[availability_zone_index]
1203             else:
1204                 return availability_zone_list[availability_zone_index]
1205         else:
1206             raise vimconn.VimConnConflictException("No enough availability zones at VIM for this deployment")
1207
1208     def new_vminstance(self, name, description, start, image_id, flavor_id, net_list, cloud_config=None, disk_list=None,
1209                        availability_zone_index=None, availability_zone_list=None):
1210         """Adds a VM instance to VIM
1211         Params:
1212             start: indicates if VM must start or boot in pause mode. Ignored
1213             image_id,flavor_id: iamge and flavor uuid
1214             net_list: list of interfaces, each one is a dictionary with:
1215                 name:
1216                 net_id: network uuid to connect
1217                 vpci: virtual vcpi to assign, ignored because openstack lack #TODO
1218                 model: interface model, ignored #TODO
1219                 mac_address: used for  SR-IOV ifaces #TODO for other types
1220                 use: 'data', 'bridge',  'mgmt'
1221                 type: 'virtual', 'PCI-PASSTHROUGH'('PF'), 'SR-IOV'('VF'), 'VFnotShared'
1222                 vim_id: filled/added by this function
1223                 floating_ip: True/False (or it can be None)
1224             'cloud_config': (optional) dictionary with:
1225             'key-pairs': (optional) list of strings with the public key to be inserted to the default user
1226             'users': (optional) list of users to be inserted, each item is a dict with:
1227                 'name': (mandatory) user name,
1228                 'key-pairs': (optional) list of strings with the public key to be inserted to the user
1229             'user-data': (optional) string is a text script to be passed directly to cloud-init
1230             'config-files': (optional). List of files to be transferred. Each item is a dict with:
1231                 'dest': (mandatory) string with the destination absolute path
1232                 'encoding': (optional, by default text). Can be one of:
1233                     'b64', 'base64', 'gz', 'gz+b64', 'gz+base64', 'gzip+b64', 'gzip+base64'
1234                 'content' (mandatory): string with the content of the file
1235                 'permissions': (optional) string with file permissions, typically octal notation '0644'
1236                 'owner': (optional) file owner, string with the format 'owner:group'
1237             'boot-data-drive': boolean to indicate if user-data must be passed using a boot drive (hard disk)
1238             'disk_list': (optional) list with additional disks to the VM. Each item is a dict with:
1239                 'image_id': (optional). VIM id of an existing image. If not provided an empty disk must be mounted
1240                 'size': (mandatory) string with the size of the disk in GB
1241                 'vim_id' (optional) should use this existing volume id
1242             availability_zone_index: Index of availability_zone_list to use for this this VM. None if not AV required
1243             availability_zone_list: list of availability zones given by user in the VNFD descriptor.  Ignore if
1244                 availability_zone_index is None
1245                 #TODO ip, security groups
1246         Returns a tuple with the instance identifier and created_items or raises an exception on error
1247             created_items can be None or a dictionary where this method can include key-values that will be passed to
1248             the method delete_vminstance and action_vminstance. Can be used to store created ports, volumes, etc.
1249             Format is vimconnector dependent, but do not use nested dictionaries and a value of None should be the same
1250             as not present.
1251         """
1252         self.logger.debug("new_vminstance input: image='%s' flavor='%s' nics='%s'", image_id, flavor_id, str(net_list))
1253         try:
1254             server = None
1255             created_items = {}
1256             # metadata = {}
1257             net_list_vim = []
1258             external_network = []
1259             # ^list of external networks to be connected to instance, later on used to create floating_ip
1260             no_secured_ports = []   # List of port-is with port-security disabled
1261             self._reload_connection()
1262             # metadata_vpci = {}   # For a specific neutron plugin
1263             block_device_mapping = None
1264
1265             for net in net_list:
1266                 if not net.get("net_id"):   # skip non connected iface
1267                     continue
1268
1269                 port_dict = {
1270                     "network_id": net["net_id"],
1271                     "name": net.get("name"),
1272                     "admin_state_up": True
1273                 }
1274                 if self.config.get("security_groups") and net.get("port_security") is not False and \
1275                         not self.config.get("no_port_security_extension"):
1276                     if not self.security_groups_id:
1277                         self._get_ids_from_name()
1278                     port_dict["security_groups"] = self.security_groups_id
1279
1280                 if net["type"] == "virtual":
1281                     pass
1282                     # if "vpci" in net:
1283                     #     metadata_vpci[ net["net_id"] ] = [[ net["vpci"], "" ]]
1284                 elif net["type"] == "VF" or net["type"] == "SR-IOV":  # for VF
1285                     # if "vpci" in net:
1286                     #     if "VF" not in metadata_vpci:
1287                     #         metadata_vpci["VF"]=[]
1288                     #     metadata_vpci["VF"].append([ net["vpci"], "" ])
1289                     port_dict["binding:vnic_type"] = "direct"
1290                     # VIO specific Changes
1291                     if self.vim_type == "VIO":
1292                         # Need to create port with port_security_enabled = False and no-security-groups
1293                         port_dict["port_security_enabled"] = False
1294                         port_dict["provider_security_groups"] = []
1295                         port_dict["security_groups"] = []
1296                 else:   # For PT PCI-PASSTHROUGH
1297                     # if "vpci" in net:
1298                     #     if "PF" not in metadata_vpci:
1299                     #         metadata_vpci["PF"]=[]
1300                     #     metadata_vpci["PF"].append([ net["vpci"], "" ])
1301                     port_dict["binding:vnic_type"] = "direct-physical"
1302                 if not port_dict["name"]:
1303                     port_dict["name"] = name
1304                 if net.get("mac_address"):
1305                     port_dict["mac_address"] = net["mac_address"]
1306                 if net.get("ip_address"):
1307                     port_dict["fixed_ips"] = [{'ip_address': net["ip_address"]}]
1308                     # TODO add 'subnet_id': <subnet_id>
1309                 new_port = self.neutron.create_port({"port": port_dict})
1310                 created_items["port:" + str(new_port["port"]["id"])] = True
1311                 net["mac_adress"] = new_port["port"]["mac_address"]
1312                 net["vim_id"] = new_port["port"]["id"]
1313                 # if try to use a network without subnetwork, it will return a emtpy list
1314                 fixed_ips = new_port["port"].get("fixed_ips")
1315                 if fixed_ips:
1316                     net["ip"] = fixed_ips[0].get("ip_address")
1317                 else:
1318                     net["ip"] = None
1319
1320                 port = {"port-id": new_port["port"]["id"]}
1321                 if float(self.nova.api_version.get_string()) >= 2.32:
1322                     port["tag"] = new_port["port"]["name"]
1323                 net_list_vim.append(port)
1324
1325                 if net.get('floating_ip', False):
1326                     net['exit_on_floating_ip_error'] = True
1327                     external_network.append(net)
1328                 elif net['use'] == 'mgmt' and self.config.get('use_floating_ip'):
1329                     net['exit_on_floating_ip_error'] = False
1330                     external_network.append(net)
1331                     net['floating_ip'] = self.config.get('use_floating_ip')
1332
1333                 # If port security is disabled when the port has not yet been attached to the VM, then all vm traffic
1334                 # is dropped.
1335                 # As a workaround we wait until the VM is active and then disable the port-security
1336                 if net.get("port_security") is False and not self.config.get("no_port_security_extension"):
1337                     no_secured_ports.append(new_port["port"]["id"])
1338
1339             # if metadata_vpci:
1340             #     metadata = {"pci_assignement": json.dumps(metadata_vpci)}
1341             #     if len(metadata["pci_assignement"]) >255:
1342             #         #limit the metadata size
1343             #         #metadata["pci_assignement"] = metadata["pci_assignement"][0:255]
1344             #         self.logger.warn("Metadata deleted since it exceeds the expected length (255) ")
1345             #         metadata = {}
1346
1347             self.logger.debug("name '%s' image_id '%s'flavor_id '%s' net_list_vim '%s' description '%s'",
1348                               name, image_id, flavor_id, str(net_list_vim), description)
1349
1350             # cloud config
1351             config_drive, userdata = self._create_user_data(cloud_config)
1352
1353             # Create additional volumes in case these are present in disk_list
1354             base_disk_index = ord('b')
1355             if disk_list:
1356                 block_device_mapping = {}
1357                 for disk in disk_list:
1358                     if disk.get('vim_id'):
1359                         block_device_mapping['_vd' + chr(base_disk_index)] = disk['vim_id']
1360                     else:
1361                         if 'image_id' in disk:
1362                             volume = self.cinder.volumes.create(size=disk['size'], name=name + '_vd' +
1363                                                                 chr(base_disk_index), imageRef=disk['image_id'])
1364                         else:
1365                             volume = self.cinder.volumes.create(size=disk['size'], name=name + '_vd' +
1366                                                                 chr(base_disk_index))
1367                         created_items["volume:" + str(volume.id)] = True
1368                         block_device_mapping['_vd' + chr(base_disk_index)] = volume.id
1369                     base_disk_index += 1
1370
1371                 # Wait until created volumes are with status available
1372                 elapsed_time = 0
1373                 while elapsed_time < volume_timeout:
1374                     for created_item in created_items:
1375                         v, _, volume_id = created_item.partition(":")
1376                         if v == 'volume':
1377                             if self.cinder.volumes.get(volume_id).status != 'available':
1378                                 break
1379                     else:  # all ready: break from while
1380                         break
1381                     time.sleep(5)
1382                     elapsed_time += 5
1383                 # If we exceeded the timeout rollback
1384                 if elapsed_time >= volume_timeout:
1385                     raise vimconn.VimConnException('Timeout creating volumes for instance ' + name,
1386                                                    http_code=vimconn.HTTP_Request_Timeout)
1387             # get availability Zone
1388             vm_av_zone = self._get_vm_availability_zone(availability_zone_index, availability_zone_list)
1389
1390             self.logger.debug("nova.servers.create({}, {}, {}, nics={}, security_groups={}, "
1391                               "availability_zone={}, key_name={}, userdata={}, config_drive={}, "
1392                               "block_device_mapping={})".format(name, image_id, flavor_id, net_list_vim,
1393                                                                 self.config.get("security_groups"), vm_av_zone,
1394                                                                 self.config.get('keypair'), userdata, config_drive,
1395                                                                 block_device_mapping))
1396             server = self.nova.servers.create(name, image_id, flavor_id, nics=net_list_vim,
1397                                               security_groups=self.config.get("security_groups"),
1398                                               # TODO remove security_groups in future versions. Already at neutron port
1399                                               availability_zone=vm_av_zone,
1400                                               key_name=self.config.get('keypair'),
1401                                               userdata=userdata,
1402                                               config_drive=config_drive,
1403                                               block_device_mapping=block_device_mapping
1404                                               )  # , description=description)
1405
1406             vm_start_time = time.time()
1407             # Previously mentioned workaround to wait until the VM is active and then disable the port-security
1408             if no_secured_ports:
1409                 self.__wait_for_vm(server.id, 'ACTIVE')
1410
1411             for port_id in no_secured_ports:
1412                 try:
1413                     self.neutron.update_port(port_id,
1414                                              {"port": {"port_security_enabled": False, "security_groups": None}})
1415                 except Exception:
1416                     raise vimconn.VimConnException("It was not possible to disable port security for port {}".format(
1417                         port_id))
1418             # print "DONE :-)", server
1419
1420             # pool_id = None
1421             if external_network:
1422                 floating_ips = self.neutron.list_floatingips().get("floatingips", ())
1423             for floating_network in external_network:
1424                 try:
1425                     assigned = False
1426                     while not assigned:
1427                         if floating_ips:
1428                             ip = floating_ips.pop(0)
1429                             if ip.get("port_id", False) or ip.get('tenant_id') != server.tenant_id:
1430                                 continue
1431                             if isinstance(floating_network['floating_ip'], str):
1432                                 if ip.get("floating_network_id") != floating_network['floating_ip']:
1433                                     continue
1434                             free_floating_ip = ip["id"]
1435                         else:
1436                             if isinstance(floating_network['floating_ip'], str) and \
1437                                     floating_network['floating_ip'].lower() != "true":
1438                                 pool_id = floating_network['floating_ip']
1439                             else:
1440                                 # Find the external network
1441                                 external_nets = list()
1442                                 for net in self.neutron.list_networks()['networks']:
1443                                     if net['router:external']:
1444                                         external_nets.append(net)
1445
1446                                 if len(external_nets) == 0:
1447                                     raise vimconn.VimConnException(
1448                                         "Cannot create floating_ip automatically since no external network is present",
1449                                         http_code=vimconn.HTTP_Conflict)
1450                                 if len(external_nets) > 1:
1451                                     raise vimconn.VimConnException(
1452                                         "Cannot create floating_ip automatically since multiple external networks are"
1453                                         " present", http_code=vimconn.HTTP_Conflict)
1454
1455                                 pool_id = external_nets[0].get('id')
1456                             param = {'floatingip': {'floating_network_id': pool_id, 'tenant_id': server.tenant_id}}
1457                             try:
1458                                 # self.logger.debug("Creating floating IP")
1459                                 new_floating_ip = self.neutron.create_floatingip(param)
1460                                 free_floating_ip = new_floating_ip['floatingip']['id']
1461                             except Exception as e:
1462                                 raise vimconn.VimConnException(type(e).__name__ + ": Cannot create new floating_ip " +
1463                                                                str(e), http_code=vimconn.HTTP_Conflict)
1464
1465                         while not assigned:
1466                             try:
1467                                 # the vim_id key contains the neutron.port_id
1468                                 self.neutron.update_floatingip(free_floating_ip,
1469                                                                {"floatingip": {"port_id": floating_network["vim_id"]}})
1470                                 # Using nove is deprecated on nova client 10.0
1471                                 assigned = True
1472                             except Exception as e:
1473                                 # openstack need some time after VM creation to asign an IP. So retry if fails
1474                                 vm_status = self.nova.servers.get(server.id).status
1475                                 if vm_status != 'ACTIVE' and vm_status != 'ERROR':
1476                                     if time.time() - vm_start_time < server_timeout:
1477                                         time.sleep(5)
1478                                         continue
1479                                 raise vimconn.VimConnException(
1480                                     "Cannot create floating_ip: {} {}".format(type(e).__name__, e),
1481                                     http_code=vimconn.HTTP_Conflict)
1482
1483                 except Exception as e:
1484                     if not floating_network['exit_on_floating_ip_error']:
1485                         self.logger.warning("Cannot create floating_ip. %s", str(e))
1486                         continue
1487                     raise
1488
1489             return server.id, created_items
1490         # except nvExceptions.NotFound as e:
1491         #     error_value=-vimconn.HTTP_Not_Found
1492         #     error_text= "vm instance %s not found" % vm_id
1493         # except TypeError as e:
1494         #     raise vimconn.VimConnException(type(e).__name__ + ": "+  str(e), http_code=vimconn.HTTP_Bad_Request)
1495
1496         except Exception as e:
1497             server_id = None
1498             if server:
1499                 server_id = server.id
1500             try:
1501                 self.delete_vminstance(server_id, created_items)
1502             except Exception as e2:
1503                 self.logger.error("new_vminstance rollback fail {}".format(e2))
1504
1505             self._format_exception(e)
1506
1507     def get_vminstance(self, vm_id):
1508         """Returns the VM instance information from VIM"""
1509         # self.logger.debug("Getting VM from VIM")
1510         try:
1511             self._reload_connection()
1512             server = self.nova.servers.find(id=vm_id)
1513             # TODO parse input and translate to VIM format (openmano_schemas.new_vminstance_response_schema)
1514             return server.to_dict()
1515         except (ksExceptions.ClientException, nvExceptions.ClientException, nvExceptions.NotFound,
1516                 ConnectionError) as e:
1517             self._format_exception(e)
1518
1519     def get_vminstance_console(self, vm_id, console_type="vnc"):
1520         """
1521         Get a console for the virtual machine
1522         Params:
1523             vm_id: uuid of the VM
1524             console_type, can be:
1525                 "novnc" (by default), "xvpvnc" for VNC types,
1526                 "rdp-html5" for RDP types, "spice-html5" for SPICE types
1527         Returns dict with the console parameters:
1528                 protocol: ssh, ftp, http, https, ...
1529                 server:   usually ip address
1530                 port:     the http, ssh, ... port
1531                 suffix:   extra text, e.g. the http path and query string
1532         """
1533         self.logger.debug("Getting VM CONSOLE from VIM")
1534         try:
1535             self._reload_connection()
1536             server = self.nova.servers.find(id=vm_id)
1537             if console_type is None or console_type == "novnc":
1538                 console_dict = server.get_vnc_console("novnc")
1539             elif console_type == "xvpvnc":
1540                 console_dict = server.get_vnc_console(console_type)
1541             elif console_type == "rdp-html5":
1542                 console_dict = server.get_rdp_console(console_type)
1543             elif console_type == "spice-html5":
1544                 console_dict = server.get_spice_console(console_type)
1545             else:
1546                 raise vimconn.VimConnException("console type '{}' not allowed".format(console_type),
1547                                                http_code=vimconn.HTTP_Bad_Request)
1548
1549             console_dict1 = console_dict.get("console")
1550             if console_dict1:
1551                 console_url = console_dict1.get("url")
1552                 if console_url:
1553                     # parse console_url
1554                     protocol_index = console_url.find("//")
1555                     suffix_index = console_url[protocol_index+2:].find("/") + protocol_index+2
1556                     port_index = console_url[protocol_index+2:suffix_index].find(":") + protocol_index+2
1557                     if protocol_index < 0 or port_index < 0 or suffix_index < 0:
1558                         return -vimconn.HTTP_Internal_Server_Error, "Unexpected response from VIM"
1559                     console_dict = {"protocol": console_url[0:protocol_index],
1560                                     "server": console_url[protocol_index+2:port_index],
1561                                     "port": console_url[port_index:suffix_index],
1562                                     "suffix": console_url[suffix_index+1:]
1563                                     }
1564                     protocol_index += 2
1565                     return console_dict
1566             raise vimconn.VimConnUnexpectedResponse("Unexpected response from VIM")
1567
1568         except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException,
1569                 nvExceptions.BadRequest, ConnectionError) as e:
1570             self._format_exception(e)
1571
1572     def delete_vminstance(self, vm_id, created_items=None):
1573         """Removes a VM instance from VIM. Returns the old identifier
1574         """
1575         # print "osconnector: Getting VM from VIM"
1576         if created_items is None:
1577             created_items = {}
1578         try:
1579             self._reload_connection()
1580             # delete VM ports attached to this networks before the virtual machine
1581             for k, v in created_items.items():
1582                 if not v:  # skip already deleted
1583                     continue
1584                 try:
1585                     k_item, _, k_id = k.partition(":")
1586                     if k_item == "port":
1587                         self.neutron.delete_port(k_id)
1588                 except Exception as e:
1589                     self.logger.error("Error deleting port: {}: {}".format(type(e).__name__, e))
1590
1591             # #commented because detaching the volumes makes the servers.delete not work properly ?!?
1592             # #dettach volumes attached
1593             # server = self.nova.servers.get(vm_id)
1594             # volumes_attached_dict = server._info['os-extended-volumes:volumes_attached']   #volume['id']
1595             # #for volume in volumes_attached_dict:
1596             # #    self.cinder.volumes.detach(volume['id'])
1597
1598             if vm_id:
1599                 self.nova.servers.delete(vm_id)
1600
1601             # delete volumes. Although having detached, they should have in active status before deleting
1602             # we ensure in this loop
1603             keep_waiting = True
1604             elapsed_time = 0
1605             while keep_waiting and elapsed_time < volume_timeout:
1606                 keep_waiting = False
1607                 for k, v in created_items.items():
1608                     if not v:  # skip already deleted
1609                         continue
1610                     try:
1611                         k_item, _, k_id = k.partition(":")
1612                         if k_item == "volume":
1613                             if self.cinder.volumes.get(k_id).status != 'available':
1614                                 keep_waiting = True
1615                             else:
1616                                 self.cinder.volumes.delete(k_id)
1617                     except Exception as e:
1618                         self.logger.error("Error deleting volume: {}: {}".format(type(e).__name__, e))
1619                 if keep_waiting:
1620                     time.sleep(1)
1621                     elapsed_time += 1
1622             return None
1623         except (nvExceptions.NotFound, ksExceptions.ClientException, nvExceptions.ClientException,
1624                 ConnectionError) as e:
1625             self._format_exception(e)
1626
1627     def refresh_vms_status(self, vm_list):
1628         """Get the status of the virtual machines and their interfaces/ports
1629            Params: the list of VM identifiers
1630            Returns a dictionary with:
1631                 vm_id:          #VIM id of this Virtual Machine
1632                     status:     #Mandatory. Text with one of:
1633                                 #  DELETED (not found at vim)
1634                                 #  VIM_ERROR (Cannot connect to VIM, VIM response error, ...)
1635                                 #  OTHER (Vim reported other status not understood)
1636                                 #  ERROR (VIM indicates an ERROR status)
1637                                 #  ACTIVE, PAUSED, SUSPENDED, INACTIVE (not running),
1638                                 #  CREATING (on building process), ERROR
1639                                 #  ACTIVE:NoMgmtIP (Active but any of its interface has an IP address
1640                                 #
1641                     error_msg:  #Text with VIM error message, if any. Or the VIM connection ERROR
1642                     vim_info:   #Text with plain information obtained from vim (yaml.safe_dump)
1643                     interfaces:
1644                      -  vim_info:         #Text with plain information obtained from vim (yaml.safe_dump)
1645                         mac_address:      #Text format XX:XX:XX:XX:XX:XX
1646                         vim_net_id:       #network id where this interface is connected
1647                         vim_interface_id: #interface/port VIM id
1648                         ip_address:       #null, or text with IPv4, IPv6 address
1649                         compute_node:     #identification of compute node where PF,VF interface is allocated
1650                         pci:              #PCI address of the NIC that hosts the PF,VF
1651                         vlan:             #physical VLAN used for VF
1652         """
1653         vm_dict = {}
1654         self.logger.debug("refresh_vms status: Getting tenant VM instance information from VIM")
1655         for vm_id in vm_list:
1656             vm = {}
1657             try:
1658                 vm_vim = self.get_vminstance(vm_id)
1659                 if vm_vim['status'] in vmStatus2manoFormat:
1660                     vm['status'] = vmStatus2manoFormat[vm_vim['status']]
1661                 else:
1662                     vm['status'] = "OTHER"
1663                     vm['error_msg'] = "VIM status reported " + vm_vim['status']
1664
1665                 vm['vim_info'] = self.serialize(vm_vim)
1666
1667                 vm["interfaces"] = []
1668                 if vm_vim.get('fault'):
1669                     vm['error_msg'] = str(vm_vim['fault'])
1670                 # get interfaces
1671                 try:
1672                     self._reload_connection()
1673                     port_dict = self.neutron.list_ports(device_id=vm_id)
1674                     for port in port_dict["ports"]:
1675                         interface = {}
1676                         interface['vim_info'] = self.serialize(port)
1677                         interface["mac_address"] = port.get("mac_address")
1678                         interface["vim_net_id"] = port["network_id"]
1679                         interface["vim_interface_id"] = port["id"]
1680                         # check if OS-EXT-SRV-ATTR:host is there,
1681                         # in case of non-admin credentials, it will be missing
1682                         if vm_vim.get('OS-EXT-SRV-ATTR:host'):
1683                             interface["compute_node"] = vm_vim['OS-EXT-SRV-ATTR:host']
1684                         interface["pci"] = None
1685
1686                         # check if binding:profile is there,
1687                         # in case of non-admin credentials, it will be missing
1688                         if port.get('binding:profile'):
1689                             if port['binding:profile'].get('pci_slot'):
1690                                 # TODO: At the moment sr-iov pci addresses are converted to PF pci addresses by setting
1691                                 #  the slot to 0x00
1692                                 # TODO: This is just a workaround valid for niantinc. Find a better way to do so
1693                                 #   CHANGE DDDD:BB:SS.F to DDDD:BB:00.(F%2)   assuming there are 2 ports per nic
1694                                 pci = port['binding:profile']['pci_slot']
1695                                 # interface["pci"] = pci[:-4] + "00." + str(int(pci[-1]) % 2)
1696                                 interface["pci"] = pci
1697                         interface["vlan"] = None
1698                         if port.get('binding:vif_details'):
1699                             interface["vlan"] = port['binding:vif_details'].get('vlan')
1700                         # Get vlan from network in case not present in port for those old openstacks and cases where
1701                         # it is needed vlan at PT
1702                         if not interface["vlan"]:
1703                             # if network is of type vlan and port is of type direct (sr-iov) then set vlan id
1704                             network = self.neutron.show_network(port["network_id"])
1705                             if network['network'].get('provider:network_type') == 'vlan':
1706                                 # and port.get("binding:vnic_type") in ("direct", "direct-physical"):
1707                                 interface["vlan"] = network['network'].get('provider:segmentation_id')
1708                         ips = []
1709                         # look for floating ip address
1710                         try:
1711                             floating_ip_dict = self.neutron.list_floatingips(port_id=port["id"])
1712                             if floating_ip_dict.get("floatingips"):
1713                                 ips.append(floating_ip_dict["floatingips"][0].get("floating_ip_address"))
1714                         except Exception:
1715                             pass
1716
1717                         for subnet in port["fixed_ips"]:
1718                             ips.append(subnet["ip_address"])
1719                         interface["ip_address"] = ";".join(ips)
1720                         vm["interfaces"].append(interface)
1721                 except Exception as e:
1722                     self.logger.error("Error getting vm interface information {}: {}".format(type(e).__name__, e),
1723                                       exc_info=True)
1724             except vimconn.VimConnNotFoundException as e:
1725                 self.logger.error("Exception getting vm status: %s", str(e))
1726                 vm['status'] = "DELETED"
1727                 vm['error_msg'] = str(e)
1728             except vimconn.VimConnException as e:
1729                 self.logger.error("Exception getting vm status: %s", str(e))
1730                 vm['status'] = "VIM_ERROR"
1731                 vm['error_msg'] = str(e)
1732             vm_dict[vm_id] = vm
1733         return vm_dict
1734
1735     def action_vminstance(self, vm_id, action_dict, created_items={}):
1736         """Send and action over a VM instance from VIM
1737         Returns None or the console dict if the action was successfully sent to the VIM"""
1738         self.logger.debug("Action over VM '%s': %s", vm_id, str(action_dict))
1739         try:
1740             self._reload_connection()
1741             server = self.nova.servers.find(id=vm_id)
1742             if "start" in action_dict:
1743                 if action_dict["start"] == "rebuild":
1744                     server.rebuild()
1745                 else:
1746                     if server.status == "PAUSED":
1747                         server.unpause()
1748                     elif server.status == "SUSPENDED":
1749                         server.resume()
1750                     elif server.status == "SHUTOFF":
1751                         server.start()
1752             elif "pause" in action_dict:
1753                 server.pause()
1754             elif "resume" in action_dict:
1755                 server.resume()
1756             elif "shutoff" in action_dict or "shutdown" in action_dict:
1757                 server.stop()
1758             elif "forceOff" in action_dict:
1759                 server.stop()  # TODO
1760             elif "terminate" in action_dict:
1761                 server.delete()
1762             elif "createImage" in action_dict:
1763                 server.create_image()
1764                 # "path":path_schema,
1765                 # "description":description_schema,
1766                 # "name":name_schema,
1767                 # "metadata":metadata_schema,
1768                 # "imageRef": id_schema,
1769                 # "disk": {"oneOf":[{"type": "null"}, {"type":"string"}] },
1770             elif "rebuild" in action_dict:
1771                 server.rebuild(server.image['id'])
1772             elif "reboot" in action_dict:
1773                 server.reboot()   # reboot_type='SOFT'
1774             elif "console" in action_dict:
1775                 console_type = action_dict["console"]
1776                 if console_type is None or console_type == "novnc":
1777                     console_dict = server.get_vnc_console("novnc")
1778                 elif console_type == "xvpvnc":
1779                     console_dict = server.get_vnc_console(console_type)
1780                 elif console_type == "rdp-html5":
1781                     console_dict = server.get_rdp_console(console_type)
1782                 elif console_type == "spice-html5":
1783                     console_dict = server.get_spice_console(console_type)
1784                 else:
1785                     raise vimconn.VimConnException("console type '{}' not allowed".format(console_type),
1786                                                    http_code=vimconn.HTTP_Bad_Request)
1787                 try:
1788                     console_url = console_dict["console"]["url"]
1789                     # parse console_url
1790                     protocol_index = console_url.find("//")
1791                     suffix_index = console_url[protocol_index+2:].find("/") + protocol_index+2
1792                     port_index = console_url[protocol_index+2:suffix_index].find(":") + protocol_index+2
1793                     if protocol_index < 0 or port_index < 0 or suffix_index < 0:
1794                         raise vimconn.VimConnException("Unexpected response from VIM " + str(console_dict))
1795                     console_dict2 = {"protocol": console_url[0:protocol_index],
1796                                      "server": console_url[protocol_index+2: port_index],
1797                                      "port": int(console_url[port_index+1: suffix_index]),
1798                                      "suffix": console_url[suffix_index+1:]
1799                                      }
1800                     return console_dict2
1801                 except Exception:
1802                     raise vimconn.VimConnException("Unexpected response from VIM " + str(console_dict))
1803
1804             return None
1805         except (ksExceptions.ClientException, nvExceptions.ClientException, nvExceptions.NotFound,
1806                 ConnectionError) as e:
1807             self._format_exception(e)
1808         # TODO insert exception vimconn.HTTP_Unauthorized
1809
1810     # ###### VIO Specific Changes #########
1811     def _generate_vlanID(self):
1812         """
1813          Method to get unused vlanID
1814             Args:
1815                 None
1816             Returns:
1817                 vlanID
1818         """
1819         # Get used VLAN IDs
1820         usedVlanIDs = []
1821         networks = self.get_network_list()
1822         for net in networks:
1823             if net.get('provider:segmentation_id'):
1824                 usedVlanIDs.append(net.get('provider:segmentation_id'))
1825         used_vlanIDs = set(usedVlanIDs)
1826
1827         # find unused VLAN ID
1828         for vlanID_range in self.config.get('dataplane_net_vlan_range'):
1829             try:
1830                 start_vlanid, end_vlanid = map(int, vlanID_range.replace(" ", "").split("-"))
1831                 for vlanID in range(start_vlanid, end_vlanid + 1):
1832                     if vlanID not in used_vlanIDs:
1833                         return vlanID
1834             except Exception as exp:
1835                 raise vimconn.VimConnException("Exception {} occurred while generating VLAN ID.".format(exp))
1836         else:
1837             raise vimconn.VimConnConflictException(
1838                 "Unable to create the SRIOV VLAN network. All given Vlan IDs {} are in use.".format(
1839                     self.config.get('dataplane_net_vlan_range')))
1840
1841     def _generate_multisegment_vlanID(self):
1842         """
1843          Method to get unused vlanID
1844             Args:
1845                 None
1846             Returns:
1847                 vlanID
1848         """
1849         # Get used VLAN IDs
1850         usedVlanIDs = []
1851         networks = self.get_network_list()
1852         for net in networks:
1853             if net.get('provider:network_type') == "vlan" and net.get('provider:segmentation_id'):
1854                 usedVlanIDs.append(net.get('provider:segmentation_id'))
1855             elif net.get('segments'):
1856                 for segment in net.get('segments'):
1857                     if segment.get('provider:network_type') == "vlan" and segment.get('provider:segmentation_id'):
1858                         usedVlanIDs.append(segment.get('provider:segmentation_id'))
1859         used_vlanIDs = set(usedVlanIDs)
1860
1861         # find unused VLAN ID
1862         for vlanID_range in self.config.get('multisegment_vlan_range'):
1863             try:
1864                 start_vlanid, end_vlanid = map(int, vlanID_range.replace(" ", "").split("-"))
1865                 for vlanID in range(start_vlanid, end_vlanid + 1):
1866                     if vlanID not in used_vlanIDs:
1867                         return vlanID
1868             except Exception as exp:
1869                 raise vimconn.VimConnException("Exception {} occurred while generating VLAN ID.".format(exp))
1870         else:
1871             raise vimconn.VimConnConflictException(
1872                 "Unable to create the VLAN segment. All VLAN IDs {} are in use.".format(
1873                     self.config.get('multisegment_vlan_range')))
1874
1875     def _validate_vlan_ranges(self, input_vlan_range, text_vlan_range):
1876         """
1877         Method to validate user given vlanID ranges
1878             Args:  None
1879             Returns: None
1880         """
1881         for vlanID_range in input_vlan_range:
1882             vlan_range = vlanID_range.replace(" ", "")
1883             # validate format
1884             vlanID_pattern = r'(\d)*-(\d)*$'
1885             match_obj = re.match(vlanID_pattern, vlan_range)
1886             if not match_obj:
1887                 raise vimconn.VimConnConflictException(
1888                     "Invalid VLAN range for {}: {}.You must provide '{}' in format [start_ID - end_ID].".format(
1889                         text_vlan_range, vlanID_range, text_vlan_range))
1890
1891             start_vlanid, end_vlanid = map(int, vlan_range.split("-"))
1892             if start_vlanid <= 0:
1893                 raise vimconn.VimConnConflictException(
1894                     "Invalid VLAN range for {}: {}. Start ID can not be zero. For VLAN "
1895                     "networks valid IDs are 1 to 4094 ".format(text_vlan_range, vlanID_range))
1896             if end_vlanid > 4094:
1897                 raise vimconn.VimConnConflictException(
1898                     "Invalid VLAN range for {}: {}. End VLAN ID can not be greater than 4094. For VLAN "
1899                     "networks valid IDs are 1 to 4094 ".format(text_vlan_range, vlanID_range))
1900
1901             if start_vlanid > end_vlanid:
1902                 raise vimconn.VimConnConflictException(
1903                     "Invalid VLAN range for {}: {}. You must provide '{}' in format start_ID - end_ID and "
1904                     "start_ID < end_ID ".format(text_vlan_range, vlanID_range, text_vlan_range))
1905
1906     # NOT USED FUNCTIONS
1907
1908     def new_external_port(self, port_data):
1909         """Adds a external port to VIM
1910            Returns the port identifier"""
1911         # TODO openstack if needed
1912         return -vimconn.HTTP_Internal_Server_Error, "osconnector.new_external_port() not implemented"
1913
1914     def connect_port_network(self, port_id, network_id, admin=False):
1915         """Connects a external port to a network
1916             Returns status code of the VIM response"""
1917         # TODO openstack if needed
1918         return -vimconn.HTTP_Internal_Server_Error, "osconnector.connect_port_network() not implemented"
1919
1920     def new_user(self, user_name, user_passwd, tenant_id=None):
1921         """Adds a new user to openstack VIM
1922            Returns the user identifier"""
1923         self.logger.debug("osconnector: Adding a new user to VIM")
1924         try:
1925             self._reload_connection()
1926             user = self.keystone.users.create(user_name, password=user_passwd, default_project=tenant_id)
1927             # self.keystone.tenants.add_user(self.k_creds["username"], #role)
1928             return user.id
1929         except ksExceptions.ConnectionError as e:
1930             error_value = -vimconn.HTTP_Bad_Request
1931             error_text = type(e).__name__ + ": " + (str(e) if len(e.args) == 0 else str(e.args[0]))
1932         except ksExceptions.ClientException as e:  # TODO remove
1933             error_value = -vimconn.HTTP_Bad_Request
1934             error_text = type(e).__name__ + ": " + (str(e) if len(e.args) == 0 else str(e.args[0]))
1935         # TODO insert exception vimconn.HTTP_Unauthorized
1936         # if reaching here is because an exception
1937         self.logger.debug("new_user " + error_text)
1938         return error_value, error_text
1939
1940     def delete_user(self, user_id):
1941         """Delete a user from openstack VIM
1942            Returns the user identifier"""
1943         if self.debug:
1944             print("osconnector: Deleting  a  user from VIM")
1945         try:
1946             self._reload_connection()
1947             self.keystone.users.delete(user_id)
1948             return 1, user_id
1949         except ksExceptions.ConnectionError as e:
1950             error_value = -vimconn.HTTP_Bad_Request
1951             error_text = type(e).__name__ + ": " + (str(e) if len(e.args) == 0 else str(e.args[0]))
1952         except ksExceptions.NotFound as e:
1953             error_value = -vimconn.HTTP_Not_Found
1954             error_text = type(e).__name__ + ": " + (str(e) if len(e.args) == 0 else str(e.args[0]))
1955         except ksExceptions.ClientException as e:  # TODO remove
1956             error_value = -vimconn.HTTP_Bad_Request
1957             error_text = type(e).__name__ + ": " + (str(e) if len(e.args) == 0 else str(e.args[0]))
1958         # TODO insert exception vimconn.HTTP_Unauthorized
1959         # if reaching here is because an exception
1960         self.logger.debug("delete_tenant " + error_text)
1961         return error_value, error_text
1962
1963     def get_hosts_info(self):
1964         """Get the information of deployed hosts
1965         Returns the hosts content"""
1966         if self.debug:
1967             print("osconnector: Getting Host info from VIM")
1968         try:
1969             h_list = []
1970             self._reload_connection()
1971             hypervisors = self.nova.hypervisors.list()
1972             for hype in hypervisors:
1973                 h_list.append(hype.to_dict())
1974             return 1, {"hosts": h_list}
1975         except nvExceptions.NotFound as e:
1976             error_value = -vimconn.HTTP_Not_Found
1977             error_text = (str(e) if len(e.args) == 0 else str(e.args[0]))
1978         except (ksExceptions.ClientException, nvExceptions.ClientException) as e:
1979             error_value = -vimconn.HTTP_Bad_Request
1980             error_text = type(e).__name__ + ": " + (str(e) if len(e.args) == 0 else str(e.args[0]))
1981         # TODO insert exception vimconn.HTTP_Unauthorized
1982         # if reaching here is because an exception
1983         self.logger.debug("get_hosts_info " + error_text)
1984         return error_value, error_text
1985
1986     def get_hosts(self, vim_tenant):
1987         """Get the hosts and deployed instances
1988         Returns the hosts content"""
1989         r, hype_dict = self.get_hosts_info()
1990         if r < 0:
1991             return r, hype_dict
1992         hypervisors = hype_dict["hosts"]
1993         try:
1994             servers = self.nova.servers.list()
1995             for hype in hypervisors:
1996                 for server in servers:
1997                     if server.to_dict()['OS-EXT-SRV-ATTR:hypervisor_hostname'] == hype['hypervisor_hostname']:
1998                         if 'vm' in hype:
1999                             hype['vm'].append(server.id)
2000                         else:
2001                             hype['vm'] = [server.id]
2002             return 1, hype_dict
2003         except nvExceptions.NotFound as e:
2004             error_value = -vimconn.HTTP_Not_Found
2005             error_text = (str(e) if len(e.args) == 0 else str(e.args[0]))
2006         except (ksExceptions.ClientException, nvExceptions.ClientException) as e:
2007             error_value = -vimconn.HTTP_Bad_Request
2008             error_text = type(e).__name__ + ": " + (str(e) if len(e.args) == 0 else str(e.args[0]))
2009         # TODO insert exception vimconn.HTTP_Unauthorized
2010         # if reaching here is because an exception
2011         self.logger.debug("get_hosts " + error_text)
2012         return error_value, error_text
2013
2014     def new_classification(self, name, ctype, definition):
2015         self.logger.debug('Adding a new (Traffic) Classification to VIM, named %s', name)
2016         try:
2017             new_class = None
2018             self._reload_connection()
2019             if ctype not in supportedClassificationTypes:
2020                 raise vimconn.VimConnNotSupportedException(
2021                     'OpenStack VIM connector does not support provided Classification Type {}, supported ones are: '
2022                     '{}'.format(ctype, supportedClassificationTypes))
2023             if not self._validate_classification(ctype, definition):
2024                 raise vimconn.VimConnException(
2025                     'Incorrect Classification definition '
2026                     'for the type specified.')
2027             classification_dict = definition
2028             classification_dict['name'] = name
2029
2030             new_class = self.neutron.create_sfc_flow_classifier(
2031                 {'flow_classifier': classification_dict})
2032             return new_class['flow_classifier']['id']
2033         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
2034                 neExceptions.NeutronException, ConnectionError) as e:
2035             self.logger.error(
2036                 'Creation of Classification failed.')
2037             self._format_exception(e)
2038
2039     def get_classification(self, class_id):
2040         self.logger.debug(" Getting Classification %s from VIM", class_id)
2041         filter_dict = {"id": class_id}
2042         class_list = self.get_classification_list(filter_dict)
2043         if len(class_list) == 0:
2044             raise vimconn.VimConnNotFoundException(
2045                 "Classification '{}' not found".format(class_id))
2046         elif len(class_list) > 1:
2047             raise vimconn.VimConnConflictException(
2048                 "Found more than one Classification with this criteria")
2049         classification = class_list[0]
2050         return classification
2051
2052     def get_classification_list(self, filter_dict={}):
2053         self.logger.debug("Getting Classifications from VIM filter: '%s'",
2054                           str(filter_dict))
2055         try:
2056             filter_dict_os = filter_dict.copy()
2057             self._reload_connection()
2058             if self.api_version3 and "tenant_id" in filter_dict_os:
2059                 filter_dict_os['project_id'] = filter_dict_os.pop('tenant_id')
2060             classification_dict = self.neutron.list_sfc_flow_classifiers(
2061                 **filter_dict_os)
2062             classification_list = classification_dict["flow_classifiers"]
2063             self.__classification_os2mano(classification_list)
2064             return classification_list
2065         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
2066                 neExceptions.NeutronException, ConnectionError) as e:
2067             self._format_exception(e)
2068
2069     def delete_classification(self, class_id):
2070         self.logger.debug("Deleting Classification '%s' from VIM", class_id)
2071         try:
2072             self._reload_connection()
2073             self.neutron.delete_sfc_flow_classifier(class_id)
2074             return class_id
2075         except (neExceptions.ConnectionFailed, neExceptions.NeutronException,
2076                 ksExceptions.ClientException, neExceptions.NeutronException,
2077                 ConnectionError) as e:
2078             self._format_exception(e)
2079
2080     def new_sfi(self, name, ingress_ports, egress_ports, sfc_encap=True):
2081         self.logger.debug("Adding a new Service Function Instance to VIM, named '%s'", name)
2082         try:
2083             new_sfi = None
2084             self._reload_connection()
2085             correlation = None
2086             if sfc_encap:
2087                 correlation = 'nsh'
2088             if len(ingress_ports) != 1:
2089                 raise vimconn.VimConnNotSupportedException(
2090                     "OpenStack VIM connector can only have "
2091                     "1 ingress port per SFI")
2092             if len(egress_ports) != 1:
2093                 raise vimconn.VimConnNotSupportedException(
2094                     "OpenStack VIM connector can only have "
2095                     "1 egress port per SFI")
2096             sfi_dict = {'name': name,
2097                         'ingress': ingress_ports[0],
2098                         'egress': egress_ports[0],
2099                         'service_function_parameters': {
2100                             'correlation': correlation}}
2101             new_sfi = self.neutron.create_sfc_port_pair({'port_pair': sfi_dict})
2102             return new_sfi['port_pair']['id']
2103         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
2104                 neExceptions.NeutronException, ConnectionError) as e:
2105             if new_sfi:
2106                 try:
2107                     self.neutron.delete_sfc_port_pair(
2108                         new_sfi['port_pair']['id'])
2109                 except Exception:
2110                     self.logger.error(
2111                         'Creation of Service Function Instance failed, with '
2112                         'subsequent deletion failure as well.')
2113             self._format_exception(e)
2114
2115     def get_sfi(self, sfi_id):
2116         self.logger.debug('Getting Service Function Instance %s from VIM', sfi_id)
2117         filter_dict = {"id": sfi_id}
2118         sfi_list = self.get_sfi_list(filter_dict)
2119         if len(sfi_list) == 0:
2120             raise vimconn.VimConnNotFoundException("Service Function Instance '{}' not found".format(sfi_id))
2121         elif len(sfi_list) > 1:
2122             raise vimconn.VimConnConflictException(
2123                 'Found more than one Service Function Instance '
2124                 'with this criteria')
2125         sfi = sfi_list[0]
2126         return sfi
2127
2128     def get_sfi_list(self, filter_dict={}):
2129         self.logger.debug("Getting Service Function Instances from VIM filter: '%s'", str(filter_dict))
2130         try:
2131             self._reload_connection()
2132             filter_dict_os = filter_dict.copy()
2133             if self.api_version3 and "tenant_id" in filter_dict_os:
2134                 filter_dict_os['project_id'] = filter_dict_os.pop('tenant_id')
2135             sfi_dict = self.neutron.list_sfc_port_pairs(**filter_dict_os)
2136             sfi_list = sfi_dict["port_pairs"]
2137             self.__sfi_os2mano(sfi_list)
2138             return sfi_list
2139         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
2140                 neExceptions.NeutronException, ConnectionError) as e:
2141             self._format_exception(e)
2142
2143     def delete_sfi(self, sfi_id):
2144         self.logger.debug("Deleting Service Function Instance '%s' "
2145                           "from VIM", sfi_id)
2146         try:
2147             self._reload_connection()
2148             self.neutron.delete_sfc_port_pair(sfi_id)
2149             return sfi_id
2150         except (neExceptions.ConnectionFailed, neExceptions.NeutronException,
2151                 ksExceptions.ClientException, neExceptions.NeutronException,
2152                 ConnectionError) as e:
2153             self._format_exception(e)
2154
2155     def new_sf(self, name, sfis, sfc_encap=True):
2156         self.logger.debug("Adding a new Service Function to VIM, named '%s'", name)
2157         try:
2158             new_sf = None
2159             self._reload_connection()
2160             # correlation = None
2161             # if sfc_encap:
2162             #     correlation = 'nsh'
2163             for instance in sfis:
2164                 sfi = self.get_sfi(instance)
2165                 if sfi.get('sfc_encap') != sfc_encap:
2166                     raise vimconn.VimConnNotSupportedException(
2167                         "OpenStack VIM connector requires all SFIs of the "
2168                         "same SF to share the same SFC Encapsulation")
2169             sf_dict = {'name': name,
2170                        'port_pairs': sfis}
2171             new_sf = self.neutron.create_sfc_port_pair_group({
2172                 'port_pair_group': sf_dict})
2173             return new_sf['port_pair_group']['id']
2174         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
2175                 neExceptions.NeutronException, ConnectionError) as e:
2176             if new_sf:
2177                 try:
2178                     self.neutron.delete_sfc_port_pair_group(
2179                         new_sf['port_pair_group']['id'])
2180                 except Exception:
2181                     self.logger.error(
2182                         'Creation of Service Function failed, with '
2183                         'subsequent deletion failure as well.')
2184             self._format_exception(e)
2185
2186     def get_sf(self, sf_id):
2187         self.logger.debug("Getting Service Function %s from VIM", sf_id)
2188         filter_dict = {"id": sf_id}
2189         sf_list = self.get_sf_list(filter_dict)
2190         if len(sf_list) == 0:
2191             raise vimconn.VimConnNotFoundException(
2192                 "Service Function '{}' not found".format(sf_id))
2193         elif len(sf_list) > 1:
2194             raise vimconn.VimConnConflictException(
2195                 "Found more than one Service Function with this criteria")
2196         sf = sf_list[0]
2197         return sf
2198
2199     def get_sf_list(self, filter_dict={}):
2200         self.logger.debug("Getting Service Function from VIM filter: '%s'",
2201                           str(filter_dict))
2202         try:
2203             self._reload_connection()
2204             filter_dict_os = filter_dict.copy()
2205             if self.api_version3 and "tenant_id" in filter_dict_os:
2206                 filter_dict_os['project_id'] = filter_dict_os.pop('tenant_id')
2207             sf_dict = self.neutron.list_sfc_port_pair_groups(**filter_dict_os)
2208             sf_list = sf_dict["port_pair_groups"]
2209             self.__sf_os2mano(sf_list)
2210             return sf_list
2211         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
2212                 neExceptions.NeutronException, ConnectionError) as e:
2213             self._format_exception(e)
2214
2215     def delete_sf(self, sf_id):
2216         self.logger.debug("Deleting Service Function '%s' from VIM", sf_id)
2217         try:
2218             self._reload_connection()
2219             self.neutron.delete_sfc_port_pair_group(sf_id)
2220             return sf_id
2221         except (neExceptions.ConnectionFailed, neExceptions.NeutronException,
2222                 ksExceptions.ClientException, neExceptions.NeutronException,
2223                 ConnectionError) as e:
2224             self._format_exception(e)
2225
2226     def new_sfp(self, name, classifications, sfs, sfc_encap=True, spi=None):
2227         self.logger.debug("Adding a new Service Function Path to VIM, named '%s'", name)
2228         try:
2229             new_sfp = None
2230             self._reload_connection()
2231             # In networking-sfc the MPLS encapsulation is legacy
2232             # should be used when no full SFC Encapsulation is intended
2233             correlation = 'mpls'
2234             if sfc_encap:
2235                 correlation = 'nsh'
2236             sfp_dict = {'name': name,
2237                         'flow_classifiers': classifications,
2238                         'port_pair_groups': sfs,
2239                         'chain_parameters': {'correlation': correlation}}
2240             if spi:
2241                 sfp_dict['chain_id'] = spi
2242             new_sfp = self.neutron.create_sfc_port_chain({'port_chain': sfp_dict})
2243             return new_sfp["port_chain"]["id"]
2244         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
2245                 neExceptions.NeutronException, ConnectionError) as e:
2246             if new_sfp:
2247                 try:
2248                     self.neutron.delete_sfc_port_chain(new_sfp['port_chain']['id'])
2249                 except Exception:
2250                     self.logger.error(
2251                         'Creation of Service Function Path failed, with '
2252                         'subsequent deletion failure as well.')
2253             self._format_exception(e)
2254
2255     def get_sfp(self, sfp_id):
2256         self.logger.debug(" Getting Service Function Path %s from VIM", sfp_id)
2257         filter_dict = {"id": sfp_id}
2258         sfp_list = self.get_sfp_list(filter_dict)
2259         if len(sfp_list) == 0:
2260             raise vimconn.VimConnNotFoundException(
2261                 "Service Function Path '{}' not found".format(sfp_id))
2262         elif len(sfp_list) > 1:
2263             raise vimconn.VimConnConflictException(
2264                 "Found more than one Service Function Path with this criteria")
2265         sfp = sfp_list[0]
2266         return sfp
2267
2268     def get_sfp_list(self, filter_dict={}):
2269         self.logger.debug("Getting Service Function Paths from VIM filter: '%s'", str(filter_dict))
2270         try:
2271             self._reload_connection()
2272             filter_dict_os = filter_dict.copy()
2273             if self.api_version3 and "tenant_id" in filter_dict_os:
2274                 filter_dict_os['project_id'] = filter_dict_os.pop('tenant_id')
2275             sfp_dict = self.neutron.list_sfc_port_chains(**filter_dict_os)
2276             sfp_list = sfp_dict["port_chains"]
2277             self.__sfp_os2mano(sfp_list)
2278             return sfp_list
2279         except (neExceptions.ConnectionFailed, ksExceptions.ClientException,
2280                 neExceptions.NeutronException, ConnectionError) as e:
2281             self._format_exception(e)
2282
2283     def delete_sfp(self, sfp_id):
2284         self.logger.debug("Deleting Service Function Path '%s' from VIM", sfp_id)
2285         try:
2286             self._reload_connection()
2287             self.neutron.delete_sfc_port_chain(sfp_id)
2288             return sfp_id
2289         except (neExceptions.ConnectionFailed, neExceptions.NeutronException,
2290                 ksExceptions.ClientException, neExceptions.NeutronException,
2291                 ConnectionError) as e:
2292             self._format_exception(e)
2293
2294     def refresh_sfps_status(self, sfp_list):
2295         """Get the status of the service function path
2296            Params: the list of sfp identifiers
2297            Returns a dictionary with:
2298                 vm_id:          #VIM id of this service function path
2299                     status:     #Mandatory. Text with one of:
2300                                 #  DELETED (not found at vim)
2301                                 #  VIM_ERROR (Cannot connect to VIM, VIM response error, ...)
2302                                 #  OTHER (Vim reported other status not understood)
2303                                 #  ERROR (VIM indicates an ERROR status)
2304                                 #  ACTIVE,
2305                                 #  CREATING (on building process)
2306                     error_msg:  #Text with VIM error message, if any. Or the VIM connection ERROR
2307                     vim_info:   #Text with plain information obtained from vim (yaml.safe_dump)F
2308         """
2309         sfp_dict = {}
2310         self.logger.debug("refresh_sfps status: Getting tenant SFP information from VIM")
2311         for sfp_id in sfp_list:
2312             sfp = {}
2313             try:
2314                 sfp_vim = self.get_sfp(sfp_id)
2315                 if sfp_vim['spi']:
2316                     sfp['status'] = vmStatus2manoFormat['ACTIVE']
2317                 else:
2318                     sfp['status'] = "OTHER"
2319                     sfp['error_msg'] = "VIM status reported " + sfp['status']
2320
2321                 sfp['vim_info'] = self.serialize(sfp_vim)
2322
2323                 if sfp_vim.get('fault'):
2324                     sfp['error_msg'] = str(sfp_vim['fault'])
2325
2326             except vimconn.VimConnNotFoundException as e:
2327                 self.logger.error("Exception getting sfp status: %s", str(e))
2328                 sfp['status'] = "DELETED"
2329                 sfp['error_msg'] = str(e)
2330             except vimconn.VimConnException as e:
2331                 self.logger.error("Exception getting sfp status: %s", str(e))
2332                 sfp['status'] = "VIM_ERROR"
2333                 sfp['error_msg'] = str(e)
2334             sfp_dict[sfp_id] = sfp
2335         return sfp_dict
2336
2337     def refresh_sfis_status(self, sfi_list):
2338         """Get the status of the service function instances
2339            Params: the list of sfi identifiers
2340            Returns a dictionary with:
2341                 vm_id:          #VIM id of this service function instance
2342                     status:     #Mandatory. Text with one of:
2343                                 #  DELETED (not found at vim)
2344                                 #  VIM_ERROR (Cannot connect to VIM, VIM response error, ...)
2345                                 #  OTHER (Vim reported other status not understood)
2346                                 #  ERROR (VIM indicates an ERROR status)
2347                                 #  ACTIVE,
2348                                 #  CREATING (on building process)
2349                     error_msg:  #Text with VIM error message, if any. Or the VIM connection ERROR
2350                     vim_info:   #Text with plain information obtained from vim (yaml.safe_dump)
2351         """
2352         sfi_dict = {}
2353         self.logger.debug("refresh_sfis status: Getting tenant sfi information from VIM")
2354         for sfi_id in sfi_list:
2355             sfi = {}
2356             try:
2357                 sfi_vim = self.get_sfi(sfi_id)
2358                 if sfi_vim:
2359                     sfi['status'] = vmStatus2manoFormat['ACTIVE']
2360                 else:
2361                     sfi['status'] = "OTHER"
2362                     sfi['error_msg'] = "VIM status reported " + sfi['status']
2363
2364                 sfi['vim_info'] = self.serialize(sfi_vim)
2365
2366                 if sfi_vim.get('fault'):
2367                     sfi['error_msg'] = str(sfi_vim['fault'])
2368
2369             except vimconn.VimConnNotFoundException as e:
2370                 self.logger.error("Exception getting sfi status: %s", str(e))
2371                 sfi['status'] = "DELETED"
2372                 sfi['error_msg'] = str(e)
2373             except vimconn.VimConnException as e:
2374                 self.logger.error("Exception getting sfi status: %s", str(e))
2375                 sfi['status'] = "VIM_ERROR"
2376                 sfi['error_msg'] = str(e)
2377             sfi_dict[sfi_id] = sfi
2378         return sfi_dict
2379
2380     def refresh_sfs_status(self, sf_list):
2381         """Get the status of the service functions
2382            Params: the list of sf identifiers
2383            Returns a dictionary with:
2384                 vm_id:          #VIM id of this service function
2385                     status:     #Mandatory. Text with one of:
2386                                 #  DELETED (not found at vim)
2387                                 #  VIM_ERROR (Cannot connect to VIM, VIM response error, ...)
2388                                 #  OTHER (Vim reported other status not understood)
2389                                 #  ERROR (VIM indicates an ERROR status)
2390                                 #  ACTIVE,
2391                                 #  CREATING (on building process)
2392                     error_msg:  #Text with VIM error message, if any. Or the VIM connection ERROR
2393                     vim_info:   #Text with plain information obtained from vim (yaml.safe_dump)
2394         """
2395         sf_dict = {}
2396         self.logger.debug("refresh_sfs status: Getting tenant sf information from VIM")
2397         for sf_id in sf_list:
2398             sf = {}
2399             try:
2400                 sf_vim = self.get_sf(sf_id)
2401                 if sf_vim:
2402                     sf['status'] = vmStatus2manoFormat['ACTIVE']
2403                 else:
2404                     sf['status'] = "OTHER"
2405                     sf['error_msg'] = "VIM status reported " + sf_vim['status']
2406
2407                 sf['vim_info'] = self.serialize(sf_vim)
2408
2409                 if sf_vim.get('fault'):
2410                     sf['error_msg'] = str(sf_vim['fault'])
2411
2412             except vimconn.VimConnNotFoundException as e:
2413                 self.logger.error("Exception getting sf status: %s", str(e))
2414                 sf['status'] = "DELETED"
2415                 sf['error_msg'] = str(e)
2416             except vimconn.VimConnException as e:
2417                 self.logger.error("Exception getting sf status: %s", str(e))
2418                 sf['status'] = "VIM_ERROR"
2419                 sf['error_msg'] = str(e)
2420             sf_dict[sf_id] = sf
2421         return sf_dict
2422
2423     def refresh_classifications_status(self, classification_list):
2424         """Get the status of the classifications
2425            Params: the list of classification identifiers
2426            Returns a dictionary with:
2427                 vm_id:          #VIM id of this classifier
2428                     status:     #Mandatory. Text with one of:
2429                                 #  DELETED (not found at vim)
2430                                 #  VIM_ERROR (Cannot connect to VIM, VIM response error, ...)
2431                                 #  OTHER (Vim reported other status not understood)
2432                                 #  ERROR (VIM indicates an ERROR status)
2433                                 #  ACTIVE,
2434                                 #  CREATING (on building process)
2435                     error_msg:  #Text with VIM error message, if any. Or the VIM connection ERROR
2436                     vim_info:   #Text with plain information obtained from vim (yaml.safe_dump)
2437         """
2438         classification_dict = {}
2439         self.logger.debug("refresh_classifications status: Getting tenant classification information from VIM")
2440         for classification_id in classification_list:
2441             classification = {}
2442             try:
2443                 classification_vim = self.get_classification(classification_id)
2444                 if classification_vim:
2445                     classification['status'] = vmStatus2manoFormat['ACTIVE']
2446                 else:
2447                     classification['status'] = "OTHER"
2448                     classification['error_msg'] = "VIM status reported " + classification['status']
2449
2450                 classification['vim_info'] = self.serialize(classification_vim)
2451
2452                 if classification_vim.get('fault'):
2453                     classification['error_msg'] = str(classification_vim['fault'])
2454
2455             except vimconn.VimConnNotFoundException as e:
2456                 self.logger.error("Exception getting classification status: %s", str(e))
2457                 classification['status'] = "DELETED"
2458                 classification['error_msg'] = str(e)
2459             except vimconn.VimConnException as e:
2460                 self.logger.error("Exception getting classification status: %s", str(e))
2461                 classification['status'] = "VIM_ERROR"
2462                 classification['error_msg'] = str(e)
2463             classification_dict[classification_id] = classification
2464         return classification_dict